Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 131 additions & 12 deletions scripts/check-dependency-licenses.sh
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ require_tool() {
}
require_tool cargo
require_tool python3
require_tool git

test -s "${CSV}"

Expand Down Expand Up @@ -96,8 +97,9 @@ print("\n".join(sorted(names)))

# Every first-party workspace root, not just the root one. `cargo metadata`
# stops at a `[workspace]` boundary, so one pass per root is the only way to
# reach them all, and the tree has three under version control outside
# `examples/` (the second half below covers those):
# reach them all. The list is no longer a claim: the self-check below compares
# it against every tracked `[workspace]` manifest outside `examples/`, which the
# second half covers. Today that is three, and how each got here matters:
#
# 1. Cargo.toml -- the root workspace.
# 2. crates/rustc-codegen-cuda/Cargo.toml -- its own `[workspace]` for the
Expand All @@ -115,12 +117,127 @@ print("\n".join(sorted(names)))
# for `cargo deny check` -- which judges the license *policy* -- and this
# guard covers the other half, that the human-readable inventory does not
# fall behind what a workspace declares. Both halves need all three roots.
FIRST_PARTY_WORKSPACE_ROOTS=(
Cargo.toml
crates/rustc-codegen-cuda/Cargo.toml
crates/cuda-macros/tests/device-only/Cargo.toml
)

# The vendored rustlantis subtree also declares `[workspace]`. It is
# third-party code attributed in THIRD_PARTY_NOTICES, its dependencies are not
# ours to record, and check-spdx-headers.sh excludes it for the same reason.
VENDORED_WORKSPACE_ROOTS=(
crates/fuzzer/rustlantis/Cargo.toml
)

# Ask Cargo which workspace owns every tracked manifest. This handles valid
# spellings such as `[ workspace ]` without raising the Python requirement.
# The result feeds both checks below:
#
# Cargo.toml -> Cargo workspace root
# |-- named first-party or vendored root
# `-- example root with a tracked Cargo.lock
EXAMPLES_ROOT=crates/rustc-codegen-cuda/examples
all_workspace_roots="$(
git ls-files -z -- '*Cargo.toml' |
while IFS= read -r -d '' manifest; do
root="$(
cargo locate-project --workspace --message-format plain --frozen \
--manifest-path "${manifest}"
)"
case "${root}" in
"${PWD}/"*) printf '%s\n' "${root#"${PWD}/"}" ;;
*)
echo "error: Cargo returned a workspace root outside this checkout:" >&2
echo " ${root}" >&2
exit 1
;;
esac
done | LC_ALL=C sort -u
)"
if [[ -z "${all_workspace_roots}" ]]; then
echo "error: Cargo found no workspace roots; the scan broke" >&2
exit 1
fi

on_disk_roots="$(
printf '%s\n' "${all_workspace_roots}" |
grep -v "^${EXAMPLES_ROOT}/" |
LC_ALL=C sort
)"
named_roots="$(
printf '%s\n' "${FIRST_PARTY_WORKSPACE_ROOTS[@]}" "${VENDORED_WORKSPACE_ROOTS[@]}" |
LC_ALL=C sort
)"
if [[ -z "${on_disk_roots}" ]]; then
echo "error: found no tracked [workspace] manifests outside the examples" >&2
echo " tree; the scan broke, so a clean result means nothing" >&2
exit 1
fi
unnamed="$(comm -23 <(printf '%s\n' "${on_disk_roots}") <(printf '%s\n' "${named_roots}"))"
if [[ -n "${unnamed}" ]]; then
echo "error: these manifests declare [workspace] but no pass below reads them," >&2
echo " so nothing checks that ${CSV} records what they declare:" >&2
printf '%s\n' "${unnamed}" | sed 's/^/ /' >&2
echo >&2
echo "Add each to FIRST_PARTY_WORKSPACE_ROOTS in $0, or to" >&2
echo "VENDORED_WORKSPACE_ROOTS with the reason it is out of scope." >&2
exit 1
fi
stale="$(comm -13 <(printf '%s\n' "${on_disk_roots}") <(printf '%s\n' "${named_roots}"))"
if [[ -n "${stale}" ]]; then
echo "error: these names are listed as workspace roots in $0 but are no longer" >&2
echo " tracked manifests declaring [workspace]:" >&2
printf '%s\n' "${stale}" | sed 's/^/ /' >&2
echo >&2
echo "A renamed or removed root left behind here is a permanent hole; drop it." >&2
exit 1
fi

# Every example workspace is inventoried from its committed lock file. Compare
# the two sets before reading any package names so a new nested workspace
# cannot disappear merely because it has not committed a lock yet.
mapfile -d '' -t EXAMPLE_LOCKFILES < <(
git ls-files -z -- "${EXAMPLES_ROOT}/**/Cargo.lock"
)
if [[ ${#EXAMPLE_LOCKFILES[@]} -lt 20 ]]; then
echo "error: found only ${#EXAMPLE_LOCKFILES[@]} tracked example lock files;" >&2
echo " the scan broke, so a clean result means nothing" >&2
exit 1
fi
example_roots="$(
printf '%s\n' "${all_workspace_roots}" |
grep "^${EXAMPLES_ROOT}/" |
LC_ALL=C sort
)"
example_lock_roots="$(
for lock in "${EXAMPLE_LOCKFILES[@]}"; do
printf '%s/Cargo.toml\n' "${lock%/Cargo.lock}"
done | LC_ALL=C sort -u
)"
missing_example_locks="$(
comm -23 <(printf '%s\n' "${example_roots}") <(printf '%s\n' "${example_lock_roots}")
)"
if [[ -n "${missing_example_locks}" ]]; then
echo "error: these example workspaces have no adjacent tracked Cargo.lock:" >&2
printf '%s\n' "${missing_example_locks}" | sed 's/^/ /' >&2
echo "Commit each lock before trusting the example dependency inventory." >&2
exit 1
fi
orphan_example_locks="$(
comm -13 <(printf '%s\n' "${example_roots}") <(printf '%s\n' "${example_lock_roots}")
)"
if [[ -n "${orphan_example_locks}" ]]; then
echo "error: these tracked example locks have no Cargo workspace root:" >&2
printf '%s\n' "${orphan_example_locks}" | sed 's/^/ /' >&2
echo "Remove each stale lock or restore its workspace manifest." >&2
exit 1
fi

required="$(
{
declared_crates Cargo.toml
declared_crates crates/rustc-codegen-cuda/Cargo.toml
declared_crates crates/cuda-macros/tests/device-only/Cargo.toml
} | LC_ALL=C sort -u
for manifest in "${FIRST_PARTY_WORKSPACE_ROOTS[@]}"; do
declared_crates "${manifest}"
done | LC_ALL=C sort -u
)"

missing="$(comm -23 <(printf '%s\n' "${required}") <(printf '%s\n' "${recorded}"))"
Expand Down Expand Up @@ -183,7 +300,7 @@ echo "OK: ${CSV} records all $(printf '%s\n' "${required}" | grep -c .) declared
INVENTORY_EXEMPT_EXAMPLES=(cutile_inter_kernel)

examples_missing="$(python3 -c '
import glob, os, re, sys
import os, re, sys

def packages(path):
"""(name, has_source) for every [[package]] in a Cargo.lock."""
Expand All @@ -202,17 +319,19 @@ with open("dependency-licenses.csv", newline="") as handle:
next(handle, None)
covered |= {line.split(",")[0].strip().strip("\r") for line in handle if line.strip()}

examples_root = "crates/rustc-codegen-cuda/examples"
locks = sorted(glob.glob(os.path.join(examples_root, "**", "Cargo.lock"), recursive=True))
separator = sys.argv.index("--")
exempt = set(sys.argv[1:separator])
locks = sorted(sys.argv[separator + 1:])
if len(locks) < 20:
sys.exit("parse self-test failed: found %d example lock files" % len(locks))

examples_root = "crates/rustc-codegen-cuda/examples"

def example_of(lock):
"""Top-level example directory a lockfile belongs to, however deep it sits."""
return os.path.relpath(lock, examples_root).split(os.sep)[0]

present = {example_of(lock) for lock in locks}
exempt = set(sys.argv[1:])
unknown = sorted(exempt - present)
if unknown:
sys.exit("INVENTORY_EXEMPT_EXAMPLES names no such example: " + ", ".join(unknown))
Expand All @@ -234,7 +353,7 @@ if seen < 100:

for example, extra in findings:
print("%s: %s" % (example, " ".join(extra)))
' "${INVENTORY_EXEMPT_EXAMPLES[@]}")"
' "${INVENTORY_EXEMPT_EXAMPLES[@]}" -- "${EXAMPLE_LOCKFILES[@]}")"

if [[ -n "${examples_missing}" ]]; then
echo "error: ${CSV} is missing rows for third-party crates that example" >&2
Expand Down
Loading