From 541c7d4871b1c8d1ebab0d22d5330dc3bf83c79d Mon Sep 17 00:00:00 2001 From: Vyncint Ng <115854244+vyncint@users.noreply.github.com> Date: Sat, 22 Aug 2026 10:06:30 +0700 Subject: [PATCH 1/2] ci(licenses): the guard's own workspace-root list is a claim nothing checks `check-dependency-licenses.sh` reaches each first-party workspace with one `declared_crates` pass per root, because `cargo metadata` stops at a `[workspace]` boundary. Which roots exist was hard-coded, and nothing compared that list against the tree -- which is exactly how the guard has failed twice: * #662 found `crates/rustc-codegen-cuda` unread. The backend, the largest first-party crate in the tree, had no CSV row while every other member did. * #1114 found `crates/cuda-macros/tests/device-only` unread, and its member `device-only-kernels` likewise unrecorded. Both times the guard printed OK. That is the failure mode this repo's guards are built to refuse, and it is the one this guard had: a short list is indistinguishable from a complete one, a count cannot tell them apart, and the comment saying "Both first-party workspaces" was itself the wrong number for months. #1114's review asked for the self-check; this is it. The list becomes two named arrays and is compared against `git ls-files -- '*Cargo.toml'` filtered to manifests declaring `[workspace]`, both directions: * a manifest on disk that no pass reads fails, naming it, and says to add it as first-party or as vendored; * a name in the arrays that is no longer such a manifest fails as stale, since a renamed root left behind is a permanent hole. `crates/fuzzer/rustlantis` is the one vendored root, listed separately with the reason: third-party code attributed in THIRD_PARTY_NOTICES whose dependencies are not ours to record, excluded by `check-spdx-headers.sh` on the same grounds. Roots under `crates/rustc-codegen-cuda/examples` stay out of scope here because the second half of the script already resolves those, including the two nested sub-workspaces, from their committed lock files. Verified, four ways: | mutation | result | |---|---| | drop the device-only root from the list | fails, naming that manifest | | add a name that is not a manifest | fails as stale, naming it | | a new tracked `[workspace]` manifest appears | fails, naming it | | the `[workspace]` scan pattern rots | refuses: "the scan broke, so a clean result means nothing" | Unmutated the guard reports "OK: dependency-licenses.csv records all 59 declared crates" and the example half is unchanged. `bash -n` clean. `git` is added to the `require_tool` list it already uses for `cargo` and `python3`. Signed-off-by: Vyncint Ng <115854244+vyncint@users.noreply.github.com> --- scripts/check-dependency-licenses.sh | 77 +++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 7 deletions(-) diff --git a/scripts/check-dependency-licenses.sh b/scripts/check-dependency-licenses.sh index ed7ea39529..08fb7042cb 100755 --- a/scripts/check-dependency-licenses.sh +++ b/scripts/check-dependency-licenses.sh @@ -39,6 +39,7 @@ require_tool() { } require_tool cargo require_tool python3 +require_tool git test -s "${CSV}" @@ -96,8 +97,9 @@ print("\n".join(sorted(names))) # Every first-party workspace root, not just the root one. `cargo metadata` # stops at a `[workspace]` boundary, so one pass per root is the only way to -# reach them all, and the tree has three under version control outside -# `examples/` (the second half below covers those): +# reach them all. The list is no longer a claim: the self-check below compares +# it against every tracked `[workspace]` manifest outside `examples/`, which the +# second half covers. Today that is three, and how each got here matters: # # 1. Cargo.toml -- the root workspace. # 2. crates/rustc-codegen-cuda/Cargo.toml -- its own `[workspace]` for the @@ -115,12 +117,73 @@ print("\n".join(sorted(names))) # for `cargo deny check` -- which judges the license *policy* -- and this # guard covers the other half, that the human-readable inventory does not # fall behind what a workspace declares. Both halves need all three roots. +FIRST_PARTY_WORKSPACE_ROOTS=( + Cargo.toml + crates/rustc-codegen-cuda/Cargo.toml + crates/cuda-macros/tests/device-only/Cargo.toml +) + +# The vendored rustlantis subtree also declares `[workspace]`. It is +# third-party code attributed in THIRD_PARTY_NOTICES, its dependencies are not +# ours to record, and check-spdx-headers.sh excludes it for the same reason. +VENDORED_WORKSPACE_ROOTS=( + crates/fuzzer/rustlantis/Cargo.toml +) + +# Self-check on the list above, because that list going short is exactly how +# this guard has failed twice. #662 found the backend workspace unread; the +# device-only fixture was unread until the third entry was added, and in both +# cases the guard reported OK while never looking at the workspace. A count +# cannot catch it and neither can a comment, so compare the list against the +# tree: every tracked manifest that declares `[workspace]` must be named here, +# as first-party or as vendored, and every name here must still be such a +# manifest. +# +# Roots under crates/rustc-codegen-cuda/examples are deliberately absent: the +# second half of this script resolves those, including the two nested +# sub-workspaces, from their committed lock files. +on_disk_roots="$( + git ls-files -- '*Cargo.toml' | + grep -v '^crates/rustc-codegen-cuda/examples/' | + while IFS= read -r manifest; do + if grep -q '^\[workspace\]' "${manifest}"; then + printf '%s\n' "${manifest}" + fi + done | LC_ALL=C sort +)" +named_roots="$( + printf '%s\n' "${FIRST_PARTY_WORKSPACE_ROOTS[@]}" "${VENDORED_WORKSPACE_ROOTS[@]}" | + LC_ALL=C sort +)" +if [[ -z "${on_disk_roots}" ]]; then + echo "error: found no tracked [workspace] manifests outside the examples" >&2 + echo " tree; the scan broke, so a clean result means nothing" >&2 + exit 1 +fi +unnamed="$(comm -23 <(printf '%s\n' "${on_disk_roots}") <(printf '%s\n' "${named_roots}"))" +if [[ -n "${unnamed}" ]]; then + echo "error: these manifests declare [workspace] but no pass below reads them," >&2 + echo " so nothing checks that ${CSV} records what they declare:" >&2 + printf '%s\n' "${unnamed}" | sed 's/^/ /' >&2 + echo >&2 + echo "Add each to FIRST_PARTY_WORKSPACE_ROOTS in $0, or to" >&2 + echo "VENDORED_WORKSPACE_ROOTS with the reason it is out of scope." >&2 + exit 1 +fi +stale="$(comm -13 <(printf '%s\n' "${on_disk_roots}") <(printf '%s\n' "${named_roots}"))" +if [[ -n "${stale}" ]]; then + echo "error: these names are listed as workspace roots in $0 but are no longer" >&2 + echo " tracked manifests declaring [workspace]:" >&2 + printf '%s\n' "${stale}" | sed 's/^/ /' >&2 + echo >&2 + echo "A renamed or removed root left behind here is a permanent hole; drop it." >&2 + exit 1 +fi + required="$( - { - declared_crates Cargo.toml - declared_crates crates/rustc-codegen-cuda/Cargo.toml - declared_crates crates/cuda-macros/tests/device-only/Cargo.toml - } | LC_ALL=C sort -u + for manifest in "${FIRST_PARTY_WORKSPACE_ROOTS[@]}"; do + declared_crates "${manifest}" + done | LC_ALL=C sort -u )" missing="$(comm -23 <(printf '%s\n' "${required}") <(printf '%s\n' "${recorded}"))" From 1c460e0701d1671378c8cc57c1bf22c14eb796aa Mon Sep 17 00:00:00 2001 From: nihalp Date: Sun, 23 Aug 2026 10:20:15 +0530 Subject: [PATCH 2/2] ci(licenses): derive workspace roots with Cargo Resolve every tracked manifest through Cargo, compare named non-example roots in both directions, and require each example workspace to have an adjacent tracked lock. Signed-off-by: nihalp --- scripts/check-dependency-licenses.sh | 102 +++++++++++++++++++++------ 1 file changed, 79 insertions(+), 23 deletions(-) diff --git a/scripts/check-dependency-licenses.sh b/scripts/check-dependency-licenses.sh index 08fb7042cb..a26deed34f 100755 --- a/scripts/check-dependency-licenses.sh +++ b/scripts/check-dependency-licenses.sh @@ -130,26 +130,40 @@ VENDORED_WORKSPACE_ROOTS=( crates/fuzzer/rustlantis/Cargo.toml ) -# Self-check on the list above, because that list going short is exactly how -# this guard has failed twice. #662 found the backend workspace unread; the -# device-only fixture was unread until the third entry was added, and in both -# cases the guard reported OK while never looking at the workspace. A count -# cannot catch it and neither can a comment, so compare the list against the -# tree: every tracked manifest that declares `[workspace]` must be named here, -# as first-party or as vendored, and every name here must still be such a -# manifest. +# Ask Cargo which workspace owns every tracked manifest. This handles valid +# spellings such as `[ workspace ]` without raising the Python requirement. +# The result feeds both checks below: # -# Roots under crates/rustc-codegen-cuda/examples are deliberately absent: the -# second half of this script resolves those, including the two nested -# sub-workspaces, from their committed lock files. +# Cargo.toml -> Cargo workspace root +# |-- named first-party or vendored root +# `-- example root with a tracked Cargo.lock +EXAMPLES_ROOT=crates/rustc-codegen-cuda/examples +all_workspace_roots="$( + git ls-files -z -- '*Cargo.toml' | + while IFS= read -r -d '' manifest; do + root="$( + cargo locate-project --workspace --message-format plain --frozen \ + --manifest-path "${manifest}" + )" + case "${root}" in + "${PWD}/"*) printf '%s\n' "${root#"${PWD}/"}" ;; + *) + echo "error: Cargo returned a workspace root outside this checkout:" >&2 + echo " ${root}" >&2 + exit 1 + ;; + esac + done | LC_ALL=C sort -u +)" +if [[ -z "${all_workspace_roots}" ]]; then + echo "error: Cargo found no workspace roots; the scan broke" >&2 + exit 1 +fi + on_disk_roots="$( - git ls-files -- '*Cargo.toml' | - grep -v '^crates/rustc-codegen-cuda/examples/' | - while IFS= read -r manifest; do - if grep -q '^\[workspace\]' "${manifest}"; then - printf '%s\n' "${manifest}" - fi - done | LC_ALL=C sort + printf '%s\n' "${all_workspace_roots}" | + grep -v "^${EXAMPLES_ROOT}/" | + LC_ALL=C sort )" named_roots="$( printf '%s\n' "${FIRST_PARTY_WORKSPACE_ROOTS[@]}" "${VENDORED_WORKSPACE_ROOTS[@]}" | @@ -180,6 +194,46 @@ if [[ -n "${stale}" ]]; then exit 1 fi +# Every example workspace is inventoried from its committed lock file. Compare +# the two sets before reading any package names so a new nested workspace +# cannot disappear merely because it has not committed a lock yet. +mapfile -d '' -t EXAMPLE_LOCKFILES < <( + git ls-files -z -- "${EXAMPLES_ROOT}/**/Cargo.lock" +) +if [[ ${#EXAMPLE_LOCKFILES[@]} -lt 20 ]]; then + echo "error: found only ${#EXAMPLE_LOCKFILES[@]} tracked example lock files;" >&2 + echo " the scan broke, so a clean result means nothing" >&2 + exit 1 +fi +example_roots="$( + printf '%s\n' "${all_workspace_roots}" | + grep "^${EXAMPLES_ROOT}/" | + LC_ALL=C sort +)" +example_lock_roots="$( + for lock in "${EXAMPLE_LOCKFILES[@]}"; do + printf '%s/Cargo.toml\n' "${lock%/Cargo.lock}" + done | LC_ALL=C sort -u +)" +missing_example_locks="$( + comm -23 <(printf '%s\n' "${example_roots}") <(printf '%s\n' "${example_lock_roots}") +)" +if [[ -n "${missing_example_locks}" ]]; then + echo "error: these example workspaces have no adjacent tracked Cargo.lock:" >&2 + printf '%s\n' "${missing_example_locks}" | sed 's/^/ /' >&2 + echo "Commit each lock before trusting the example dependency inventory." >&2 + exit 1 +fi +orphan_example_locks="$( + comm -13 <(printf '%s\n' "${example_roots}") <(printf '%s\n' "${example_lock_roots}") +)" +if [[ -n "${orphan_example_locks}" ]]; then + echo "error: these tracked example locks have no Cargo workspace root:" >&2 + printf '%s\n' "${orphan_example_locks}" | sed 's/^/ /' >&2 + echo "Remove each stale lock or restore its workspace manifest." >&2 + exit 1 +fi + required="$( for manifest in "${FIRST_PARTY_WORKSPACE_ROOTS[@]}"; do declared_crates "${manifest}" @@ -246,7 +300,7 @@ echo "OK: ${CSV} records all $(printf '%s\n' "${required}" | grep -c .) declared INVENTORY_EXEMPT_EXAMPLES=(cutile_inter_kernel) examples_missing="$(python3 -c ' -import glob, os, re, sys +import os, re, sys def packages(path): """(name, has_source) for every [[package]] in a Cargo.lock.""" @@ -265,17 +319,19 @@ with open("dependency-licenses.csv", newline="") as handle: next(handle, None) covered |= {line.split(",")[0].strip().strip("\r") for line in handle if line.strip()} -examples_root = "crates/rustc-codegen-cuda/examples" -locks = sorted(glob.glob(os.path.join(examples_root, "**", "Cargo.lock"), recursive=True)) +separator = sys.argv.index("--") +exempt = set(sys.argv[1:separator]) +locks = sorted(sys.argv[separator + 1:]) if len(locks) < 20: sys.exit("parse self-test failed: found %d example lock files" % len(locks)) +examples_root = "crates/rustc-codegen-cuda/examples" + def example_of(lock): """Top-level example directory a lockfile belongs to, however deep it sits.""" return os.path.relpath(lock, examples_root).split(os.sep)[0] present = {example_of(lock) for lock in locks} -exempt = set(sys.argv[1:]) unknown = sorted(exempt - present) if unknown: sys.exit("INVENTORY_EXEMPT_EXAMPLES names no such example: " + ", ".join(unknown)) @@ -297,7 +353,7 @@ if seen < 100: for example, extra in findings: print("%s: %s" % (example, " ".join(extra))) -' "${INVENTORY_EXEMPT_EXAMPLES[@]}")" +' "${INVENTORY_EXEMPT_EXAMPLES[@]}" -- "${EXAMPLE_LOCKFILES[@]}")" if [[ -n "${examples_missing}" ]]; then echo "error: ${CSV} is missing rows for third-party crates that example" >&2