diff --git a/scripts/check-dependency-licenses.sh b/scripts/check-dependency-licenses.sh index ed7ea39529..a26deed34f 100755 --- a/scripts/check-dependency-licenses.sh +++ b/scripts/check-dependency-licenses.sh @@ -39,6 +39,7 @@ require_tool() { } require_tool cargo require_tool python3 +require_tool git test -s "${CSV}" @@ -96,8 +97,9 @@ print("\n".join(sorted(names))) # Every first-party workspace root, not just the root one. `cargo metadata` # stops at a `[workspace]` boundary, so one pass per root is the only way to -# reach them all, and the tree has three under version control outside -# `examples/` (the second half below covers those): +# reach them all. The list is no longer a claim: the self-check below compares +# it against every tracked `[workspace]` manifest outside `examples/`, which the +# second half covers. Today that is three, and how each got here matters: # # 1. Cargo.toml -- the root workspace. # 2. crates/rustc-codegen-cuda/Cargo.toml -- its own `[workspace]` for the @@ -115,12 +117,127 @@ print("\n".join(sorted(names))) # for `cargo deny check` -- which judges the license *policy* -- and this # guard covers the other half, that the human-readable inventory does not # fall behind what a workspace declares. Both halves need all three roots. +FIRST_PARTY_WORKSPACE_ROOTS=( + Cargo.toml + crates/rustc-codegen-cuda/Cargo.toml + crates/cuda-macros/tests/device-only/Cargo.toml +) + +# The vendored rustlantis subtree also declares `[workspace]`. It is +# third-party code attributed in THIRD_PARTY_NOTICES, its dependencies are not +# ours to record, and check-spdx-headers.sh excludes it for the same reason. +VENDORED_WORKSPACE_ROOTS=( + crates/fuzzer/rustlantis/Cargo.toml +) + +# Ask Cargo which workspace owns every tracked manifest. This handles valid +# spellings such as `[ workspace ]` without raising the Python requirement. +# The result feeds both checks below: +# +# Cargo.toml -> Cargo workspace root +# |-- named first-party or vendored root +# `-- example root with a tracked Cargo.lock +EXAMPLES_ROOT=crates/rustc-codegen-cuda/examples +all_workspace_roots="$( + git ls-files -z -- '*Cargo.toml' | + while IFS= read -r -d '' manifest; do + root="$( + cargo locate-project --workspace --message-format plain --frozen \ + --manifest-path "${manifest}" + )" + case "${root}" in + "${PWD}/"*) printf '%s\n' "${root#"${PWD}/"}" ;; + *) + echo "error: Cargo returned a workspace root outside this checkout:" >&2 + echo " ${root}" >&2 + exit 1 + ;; + esac + done | LC_ALL=C sort -u +)" +if [[ -z "${all_workspace_roots}" ]]; then + echo "error: Cargo found no workspace roots; the scan broke" >&2 + exit 1 +fi + +on_disk_roots="$( + printf '%s\n' "${all_workspace_roots}" | + grep -v "^${EXAMPLES_ROOT}/" | + LC_ALL=C sort +)" +named_roots="$( + printf '%s\n' "${FIRST_PARTY_WORKSPACE_ROOTS[@]}" "${VENDORED_WORKSPACE_ROOTS[@]}" | + LC_ALL=C sort +)" +if [[ -z "${on_disk_roots}" ]]; then + echo "error: found no tracked [workspace] manifests outside the examples" >&2 + echo " tree; the scan broke, so a clean result means nothing" >&2 + exit 1 +fi +unnamed="$(comm -23 <(printf '%s\n' "${on_disk_roots}") <(printf '%s\n' "${named_roots}"))" +if [[ -n "${unnamed}" ]]; then + echo "error: these manifests declare [workspace] but no pass below reads them," >&2 + echo " so nothing checks that ${CSV} records what they declare:" >&2 + printf '%s\n' "${unnamed}" | sed 's/^/ /' >&2 + echo >&2 + echo "Add each to FIRST_PARTY_WORKSPACE_ROOTS in $0, or to" >&2 + echo "VENDORED_WORKSPACE_ROOTS with the reason it is out of scope." >&2 + exit 1 +fi +stale="$(comm -13 <(printf '%s\n' "${on_disk_roots}") <(printf '%s\n' "${named_roots}"))" +if [[ -n "${stale}" ]]; then + echo "error: these names are listed as workspace roots in $0 but are no longer" >&2 + echo " tracked manifests declaring [workspace]:" >&2 + printf '%s\n' "${stale}" | sed 's/^/ /' >&2 + echo >&2 + echo "A renamed or removed root left behind here is a permanent hole; drop it." >&2 + exit 1 +fi + +# Every example workspace is inventoried from its committed lock file. Compare +# the two sets before reading any package names so a new nested workspace +# cannot disappear merely because it has not committed a lock yet. +mapfile -d '' -t EXAMPLE_LOCKFILES < <( + git ls-files -z -- "${EXAMPLES_ROOT}/**/Cargo.lock" +) +if [[ ${#EXAMPLE_LOCKFILES[@]} -lt 20 ]]; then + echo "error: found only ${#EXAMPLE_LOCKFILES[@]} tracked example lock files;" >&2 + echo " the scan broke, so a clean result means nothing" >&2 + exit 1 +fi +example_roots="$( + printf '%s\n' "${all_workspace_roots}" | + grep "^${EXAMPLES_ROOT}/" | + LC_ALL=C sort +)" +example_lock_roots="$( + for lock in "${EXAMPLE_LOCKFILES[@]}"; do + printf '%s/Cargo.toml\n' "${lock%/Cargo.lock}" + done | LC_ALL=C sort -u +)" +missing_example_locks="$( + comm -23 <(printf '%s\n' "${example_roots}") <(printf '%s\n' "${example_lock_roots}") +)" +if [[ -n "${missing_example_locks}" ]]; then + echo "error: these example workspaces have no adjacent tracked Cargo.lock:" >&2 + printf '%s\n' "${missing_example_locks}" | sed 's/^/ /' >&2 + echo "Commit each lock before trusting the example dependency inventory." >&2 + exit 1 +fi +orphan_example_locks="$( + comm -13 <(printf '%s\n' "${example_roots}") <(printf '%s\n' "${example_lock_roots}") +)" +if [[ -n "${orphan_example_locks}" ]]; then + echo "error: these tracked example locks have no Cargo workspace root:" >&2 + printf '%s\n' "${orphan_example_locks}" | sed 's/^/ /' >&2 + echo "Remove each stale lock or restore its workspace manifest." >&2 + exit 1 +fi + required="$( - { - declared_crates Cargo.toml - declared_crates crates/rustc-codegen-cuda/Cargo.toml - declared_crates crates/cuda-macros/tests/device-only/Cargo.toml - } | LC_ALL=C sort -u + for manifest in "${FIRST_PARTY_WORKSPACE_ROOTS[@]}"; do + declared_crates "${manifest}" + done | LC_ALL=C sort -u )" missing="$(comm -23 <(printf '%s\n' "${required}") <(printf '%s\n' "${recorded}"))" @@ -183,7 +300,7 @@ echo "OK: ${CSV} records all $(printf '%s\n' "${required}" | grep -c .) declared INVENTORY_EXEMPT_EXAMPLES=(cutile_inter_kernel) examples_missing="$(python3 -c ' -import glob, os, re, sys +import os, re, sys def packages(path): """(name, has_source) for every [[package]] in a Cargo.lock.""" @@ -202,17 +319,19 @@ with open("dependency-licenses.csv", newline="") as handle: next(handle, None) covered |= {line.split(",")[0].strip().strip("\r") for line in handle if line.strip()} -examples_root = "crates/rustc-codegen-cuda/examples" -locks = sorted(glob.glob(os.path.join(examples_root, "**", "Cargo.lock"), recursive=True)) +separator = sys.argv.index("--") +exempt = set(sys.argv[1:separator]) +locks = sorted(sys.argv[separator + 1:]) if len(locks) < 20: sys.exit("parse self-test failed: found %d example lock files" % len(locks)) +examples_root = "crates/rustc-codegen-cuda/examples" + def example_of(lock): """Top-level example directory a lockfile belongs to, however deep it sits.""" return os.path.relpath(lock, examples_root).split(os.sep)[0] present = {example_of(lock) for lock in locks} -exempt = set(sys.argv[1:]) unknown = sorted(exempt - present) if unknown: sys.exit("INVENTORY_EXEMPT_EXAMPLES names no such example: " + ", ".join(unknown)) @@ -234,7 +353,7 @@ if seen < 100: for example, extra in findings: print("%s: %s" % (example, " ".join(extra))) -' "${INVENTORY_EXEMPT_EXAMPLES[@]}")" +' "${INVENTORY_EXEMPT_EXAMPLES[@]}" -- "${EXAMPLE_LOCKFILES[@]}")" if [[ -n "${examples_missing}" ]]; then echo "error: ${CSV} is missing rows for third-party crates that example" >&2