From d3c3f3e231f465eb550ae392b45922787ce1d2e2 Mon Sep 17 00:00:00 2001 From: Vyncint Ng <115854244+vyncint@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:33:18 +0700 Subject: [PATCH 1/3] ci: fail on a prose reference to a source file the repository does not track #1190 split large files into module directories and left three prose references pointing at paths it had removed; #1197 repointed those and 14 more. Nothing catches that: a path in prose is compiled by nothing, `check-host-api-paths.sh` checks Rust API paths rather than files, and the book gate only builds. "In the repository" means tracked by git, not present on disk. Those are different questions and only the first is reproducible -- a path that exists as a local build artifact, or an untracked file someone happens to have, resolves for them and nobody else, so a filesystem test makes the guard's answer depend on whose checkout it ran in. Generated outputs are declared, never inferred. An earlier revision skipped any path whose parent directory held no tracked file, reasoning that such a directory must be an output location. That is backwards: deleting or misspelling a directory produces exactly the same signal as generating into one, so the broken reference this guard exists to catch was the case it let through. `GENERATED_OUTPUT_PATHS` is now the whole accommodation -- one entry, `crates/fuzzer/artifacts/summary.jsonl`, which `run_seed.py` writes -- and it is verified rather than trusted: an entry that becomes tracked, or that nothing refers to any more, is an error instead of a silent exemption. Roots are verified too. `docs/` was in the original list and holds no tracked file in this repository, so the pattern anchored there matched nothing and would have gone on matching nothing after a rename. The check that caught it runs before every sweep. Paths containing a traversal segment are reported rather than resolved, since they cannot name a tracked file relative to the repository root. The self-test runs the real scanner over four cases -- a missing file under a tracked directory, a missing file under a directory the repo does not have, a file that exists only in this checkout, and a tracked file -- and requires the first three to be reported and the fourth not. It takes the tracked example from the index rather than naming this script, which had made the self-test depend on the guard already being committed. Crate-relative and bare-basename spellings are still not checked. #1197's follow-up fixed 14 references written that way, so that is a real gap; telling `rvalue.rs` in prose from any other mention of it is guesswork, and a guard that guesses is worse than one that is narrow. Signed-off-by: Vyncint Ng <115854244+vyncint@users.noreply.github.com> --- .github/workflows/status-guard.yml | 17 +++ Justfile | 2 + scripts/check-source-references.sh | 223 +++++++++++++++++++++++++++++ 3 files changed, 242 insertions(+) create mode 100755 scripts/check-source-references.sh diff --git a/.github/workflows/status-guard.yml b/.github/workflows/status-guard.yml index 7e751de6de..eff8d3b6e6 100644 --- a/.github/workflows/status-guard.yml +++ b/.github/workflows/status-guard.yml @@ -134,6 +134,23 @@ jobs: - name: Verify the book's Rust blocks use rustfmt's import order run: bash scripts/check-book-import-order.sh + # #1190 split large files into module directories and left three prose + # references pointing at paths it had removed; #1197 repointed those and 14 + # more. A path in prose is compiled by nothing, so the reference just goes + # quiet and a reader follows it into nothing. This checks the one spelling + # that is unambiguous on its own -- repo-root-anchored, with an extension -- + # in markdown and in `///` / `//!` doc comments. Git only, no toolchain. + source-references: + name: source references + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Clone repository + uses: actions/checkout@v4 + + - name: Verify prose references point at files that exist + run: bash scripts/check-source-references.sh + # The dialect-nvvm book page prints the catalog SHA-256 and states its own # freshness rule: if the stamp no longer matches, every count on the page # predates the catalog being read. Nothing enforced that rule, so the page diff --git a/Justfile b/Justfile index ba879a356e..b00f1a6651 100644 --- a/Justfile +++ b/Justfile @@ -213,7 +213,9 @@ check-guards: bash scripts/check-toolchain-parity.sh bash scripts/check-cli-doc-coverage.sh bash scripts/check-book-api-names.sh + bash scripts/check-book-catalog-stamp.sh bash scripts/check-book-import-order.sh + bash scripts/check-source-references.sh bash scripts/check-reserved-prefixes.sh bash scripts/check-device-only-build.sh bash scripts/check-test-matrix-coverage.sh diff --git a/scripts/check-source-references.sh b/scripts/check-source-references.sh new file mode 100755 index 0000000000..0fbc1bcfe7 --- /dev/null +++ b/scripts/check-source-references.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Reject a prose reference to a source file that is not in the repository. +# +# #1190 split large files into module directories and left three prose +# references pointing at paths it had removed -- a test's module doc at +# `crates/mir-lower/src/convert/types.rs`, two example READMEs at +# `crates/mir-importer/src/translator/rvalue.rs`. #1197 repointed those, and 14 +# more in other spellings. Nothing had noticed: a path in prose is compiled by +# nothing, `check-host-api-paths.sh` checks Rust *API* paths rather than files, +# and the book gate only builds. The reference just goes quiet, and a reader +# follows it into nothing. +# +# "In the repository" means **tracked by git**, not present on disk. Those are +# different questions and only the first one is reproducible: a path that +# exists solely as a local build artifact, or an untracked file someone has in +# their tree, resolves for them and for nobody else. Testing the filesystem +# would make this guard pass or fail depending on whose checkout it ran in. +# +# Scope, deliberately narrow, because this is the kind of guard that goes soft +# the moment it starts inferring: +# +# * A path is only checked when it is anchored at a repo root -- crates/, +# cuda-oxide-book/, scripts/ -- and carries a file extension. That +# is the spelling a new reference normally takes, and the only one that is +# unambiguous on its own. Each root is verified to be a real tracked +# directory before the sweep, so a renamed root fails here rather than +# silently matching nothing. +# * Prose only: every line of a tracked *.md, and `///` or `//!` lines in a +# tracked *.rs. This is what removes the need for a general exemption +# list. The non-existent paths that live in Rust *code* are all deliberate +# -- synthetic fixture names (`intrinsics/probes/removed.ll`, +# `intrinsics/overlay/test.toml`), two paths that cuda-intrinsics-gen +# render tests assert are *absent*, and the mktemp canary in +# check-reserved-prefixes.sh -- and none of them is prose. +# +# Generated outputs are declared, never inferred. An earlier revision skipped a +# path whose parent directory held no tracked file, reasoning that such a +# directory must be an output location. That is exactly backwards: deleting or +# misspelling a directory produces the same signal as generating into one, so +# the broken reference this guard exists to catch was the case it let through. +# The list below is the whole accommodation, and it is verified rather than +# trusted -- an entry nothing refers to any more is an error, so it cannot rot +# into a silent exemption. +# +# Two things stay out of scope on purpose: +# +# * `intrinsics/` is not a root. It is both a real top-level directory and a +# common crate-relative fragment: examples/atomics/README.md names +# `intrinsics/atomic.rs` in a pipeline diagram, meaning +# `mir-importer/src/translator/terminator/intrinsics/atomic.rs`, which is +# correct as shorthand. Rooting there would fail that line. +# * Crate-relative (`mir-lower/src/convert/types.rs`) and bare-basename +# ("the walker in `rvalue.rs`") spellings are not checked. #1197's +# follow-up fixed 14 references written that way, so this is a real gap and +# is stated rather than papered over -- telling `rvalue.rs` in prose from +# any other mention of it is guesswork, and a guard that guesses is worse +# than one that is narrow. +set -euo pipefail + +export LC_ALL=C + +cd "$(dirname "$0")/.." + +ROOTS='crates|cuda-oxide-book|scripts' +EXTS='rs|md|sh|toml|jsonl|json|ll|py|yaml|yml' +# The trailing \b matters: without it `.json` matches inside `.jsonl` and the +# guard reports a path nobody wrote. Longer alternatives lead for the same +# reason. +PATTERN="(${ROOTS})/[A-Za-z0-9._/-]+\.(${EXTS})\b" + +# Paths that prose may name even though the repository does not contain them, +# because something writes them at run time. Each one is an explicit decision, +# not a shape the guard infers. +# +# crates/fuzzer/artifacts/summary.jsonl -- run_seed.py writes it and clears +# the directory on every invocation; crates/fuzzer/README.md documents it. +GENERATED_OUTPUT_PATHS=( + crates/fuzzer/artifacts/summary.jsonl +) + +tracked_list="$(mktemp)" +trap 'rm -f "${tracked_list}"' EXIT +git ls-files >"${tracked_list}" + +is_tracked() { grep -qxF -- "$1" "${tracked_list}"; } + +is_generated_output() { + local declared + for declared in "${GENERATED_OUTPUT_PATHS[@]}"; do + [[ "$1" == "${declared}" ]] && return 0 + done + return 1 +} + +# Roots are verified rather than assumed: if one is renamed, the pattern would +# quietly stop matching anything under it and the guard would report a clean +# tree for the wrong reason. +for root in crates cuda-oxide-book scripts; do + if ! grep -qE "^${root}/" "${tracked_list}"; then + echo "error: source-reference guard: '${root}/' holds no tracked file," >&2 + echo " so the pattern anchored there can no longer match; update" >&2 + echo " ROOTS in $0" >&2 + exit 1 + fi +done + +# `grep -n` keeps the line number; the *.rs arm narrows to doc comments first +# so a path mentioned in code is never read as a claim about the tree. +prose_lines() { + case "$1" in + *.md) grep -nE "${PATTERN}" -- "$1" 2>/dev/null || true ;; + *.rs) grep -nE '^[[:space:]]*(///|//!)' -- "$1" 2>/dev/null | + grep -E "${PATTERN}" || true ;; + esac +} + +# The one scanner. The self-test below runs *this*, not a paraphrase of it, so +# disabling any step -- the pattern, the tracked test, the generated-output +# accommodation -- fails the self-test instead of quietly reporting a clean +# tree. +broken_in_file() { + local file="$1" hit lineno path + while IFS= read -r hit; do + [ -n "${hit}" ] || continue + lineno="${hit%%:*}" + while IFS= read -r path; do + [ -n "${path}" ] || continue + # A traversal segment cannot be resolved against the repository + # root, so it is never a valid claim about a tracked file. + case "${path}" in + */../* | ../* | */..) printf '%s:%s: names %s, which escapes the repository root\n' \ + "${file}" "${lineno}" "${path}"; continue ;; + esac + is_generated_output "${path}" && continue + if ! is_tracked "${path}"; then + printf '%s:%s: names %s, which is not tracked in this repository\n' \ + "${file}" "${lineno}" "${path}" + fi + done < <(printf '%s\n' "${hit#*:}" | grep -oE "${PATTERN}" | sort -u) + done < <(prose_lines "${file}") +} + +# Declared accommodations are verified, not trusted: an entry nothing names any +# more is a silent exemption waiting to hide a real break. +for declared in "${GENERATED_OUTPUT_PATHS[@]}"; do + if is_tracked "${declared}"; then + echo "error: source-reference guard: '${declared}' is listed as a" >&2 + echo " generated output but is tracked; drop it from" >&2 + echo " GENERATED_OUTPUT_PATHS in $0" >&2 + exit 1 + fi + if ! git grep -qF -- "${declared}" -- '*.md' '*.rs' 2>/dev/null; then + echo "error: source-reference guard: nothing refers to '${declared}'" >&2 + echo " any more; drop it from GENERATED_OUTPUT_PATHS in $0" >&2 + exit 1 + fi +done + +# Self-test. The failure mode this guard has to survive is "silently stops +# reporting", so run the real scanner over prose naming, in turn: a missing +# file whose directory the repo does track, a missing file whose directory it +# does *not* (the case an earlier revision let through), an untracked file that +# exists on disk right now, and a tracked file. The first three must be +# reported and the fourth must not. +canary="$(mktemp -d)" +canary_untracked="crates/zz-source-reference-canary-$$.rs" +trap 'rm -f "${tracked_list}" "${canary_untracked}"; rm -rf "${canary}"' EXIT +printf 'prose naming crates/cuda-device/src/no-such-file-%s.rs inline\n' "$$" >"${canary}/dead_in_live_dir.md" +printf 'prose naming crates/no-such-crate-%s/src/lib.rs inline\n' "$$" >"${canary}/dead_dir.md" +printf 'prose naming %s inline\n' "${canary_untracked}" >"${canary}/untracked.md" +# Any tracked path the pattern matches, chosen from the index rather than +# named: hard-coding this script's own path made the self-test depend on the +# guard already being committed, which is false on the branch that adds it and +# in any historical worktree. +canary_live="$(grep -m1 -E "^(${ROOTS})/[A-Za-z0-9._/-]+\.(${EXTS})$" "${tracked_list}")" +if [ -z "${canary_live}" ]; then + echo "error: source-reference guard: the index holds no path the pattern" >&2 + echo " matches, so the self-test cannot prove the tracked case" >&2 + exit 1 +fi +printf 'prose naming %s inline\n' "${canary_live}" >"${canary}/live.md" +: >"${canary_untracked}" + +canary_hits() { broken_in_file "${canary}/$1" | wc -l; } +if [ "$(canary_hits dead_in_live_dir.md)" -ne 1 ] || + [ "$(canary_hits dead_dir.md)" -ne 1 ] || + [ "$(canary_hits untracked.md)" -ne 1 ] || + [ "$(canary_hits live.md)" -ne 0 ]; then + echo "error: source-reference guard self-test failed: the scanner no longer" >&2 + echo " separates a tracked path from a missing one, a missing" >&2 + echo " directory, or a file that exists only in this checkout, so a" >&2 + echo " clean result on the tree means nothing" >&2 + exit 1 +fi +rm -f "${canary_untracked}" + +checked=0 +broken=0 +while IFS= read -r file; do + while IFS= read -r problem; do + [ -n "${problem}" ] || continue + printf '%s\n' "${problem}" >&2 + broken=$((broken + 1)) + done < <(broken_in_file "${file}") + while IFS= read -r _path; do + [ -n "${_path}" ] || continue + checked=$((checked + 1)) + done < <(prose_lines "${file}" | grep -oE "${PATTERN}" | sort -u) +done < <(git ls-files -- '*.md' '*.rs') + +if [ "${broken}" -ne 0 ]; then + echo "" >&2 + echo "error: ${broken} prose reference(s) name a path this repository does" >&2 + echo " not track. Repoint each one at where the code lives now; if" >&2 + echo " the surrounding sentence describes the old shape, it needs" >&2 + echo " rewriting too, not just a new path. A path written at run" >&2 + echo " time belongs in GENERATED_OUTPUT_PATHS in $0, with the reason." >&2 + exit 1 +fi + +echo "source references ok: ${checked} repo-anchored paths in prose, all tracked" From 54eb0b89c6fd581cf3253a7c821f9b0be70351fd Mon Sep 17 00:00:00 2001 From: nihalp Date: Tue, 15 Sep 2026 13:19:48 +0530 Subject: [PATCH 2/3] fix(ci): check complete source references in prose Signed-off-by: nihalp --- .github/workflows/status-guard.yml | 6 +- scripts/check-source-references.sh | 354 ++++++++++++++++------------- 2 files changed, 196 insertions(+), 164 deletions(-) diff --git a/.github/workflows/status-guard.yml b/.github/workflows/status-guard.yml index eff8d3b6e6..fbf1d7f214 100644 --- a/.github/workflows/status-guard.yml +++ b/.github/workflows/status-guard.yml @@ -137,9 +137,9 @@ jobs: # #1190 split large files into module directories and left three prose # references pointing at paths it had removed; #1197 repointed those and 14 # more. A path in prose is compiled by nothing, so the reference just goes - # quiet and a reader follows it into nothing. This checks the one spelling + # quiet and a reader follows it into nothing. This checks the one form # that is unambiguous on its own -- repo-root-anchored, with an extension -- - # in markdown and in `///` / `//!` doc comments. Git only, no toolchain. + # in markdown and in `///` / `//!` doc comments. Git and Python, no toolchain. source-references: name: source references runs-on: ubuntu-latest @@ -148,7 +148,7 @@ jobs: - name: Clone repository uses: actions/checkout@v4 - - name: Verify prose references point at files that exist + - name: Verify prose references name tracked files or declared outputs run: bash scripts/check-source-references.sh # The dialect-nvvm book page prints the catalog SHA-256 and states its own diff --git a/scripts/check-source-references.sh b/scripts/check-source-references.sh index 0fbc1bcfe7..4f0cb4c285 100755 --- a/scripts/check-source-references.sh +++ b/scripts/check-source-references.sh @@ -7,7 +7,7 @@ # references pointing at paths it had removed -- a test's module doc at # `crates/mir-lower/src/convert/types.rs`, two example READMEs at # `crates/mir-importer/src/translator/rvalue.rs`. #1197 repointed those, and 14 -# more in other spellings. Nothing had noticed: a path in prose is compiled by +# more in other forms. Nothing had noticed: a path in prose is compiled by # nothing, `check-host-api-paths.sh` checks Rust *API* paths rather than files, # and the book gate only builds. The reference just goes quiet, and a reader # follows it into nothing. @@ -23,7 +23,7 @@ # # * A path is only checked when it is anchored at a repo root -- crates/, # cuda-oxide-book/, scripts/ -- and carries a file extension. That -# is the spelling a new reference normally takes, and the only one that is +# is the form a new reference normally takes, and the only one that is # unambiguous on its own. Each root is verified to be a real tracked # directory before the sweep, so a renamed root fails here rather than # silently matching nothing. @@ -38,7 +38,7 @@ # Generated outputs are declared, never inferred. An earlier revision skipped a # path whose parent directory held no tracked file, reasoning that such a # directory must be an output location. That is exactly backwards: deleting or -# misspelling a directory produces the same signal as generating into one, so +# mistyping a directory produces the same signal as generating into one, so # the broken reference this guard exists to catch was the case it let through. # The list below is the whole accommodation, and it is verified rather than # trusted -- an entry nothing refers to any more is an error, so it cannot rot @@ -52,172 +52,204 @@ # `mir-importer/src/translator/terminator/intrinsics/atomic.rs`, which is # correct as shorthand. Rooting there would fail that line. # * Crate-relative (`mir-lower/src/convert/types.rs`) and bare-basename -# ("the walker in `rvalue.rs`") spellings are not checked. #1197's +# ("the walker in `rvalue.rs`") forms are not checked. #1197's # follow-up fixed 14 references written that way, so this is a real gap and # is stated rather than papered over -- telling `rvalue.rs` in prose from # any other mention of it is guesswork, and a guard that guesses is worse # than one that is narrow. set -euo pipefail - export LC_ALL=C - cd "$(dirname "$0")/.." -ROOTS='crates|cuda-oxide-book|scripts' -EXTS='rs|md|sh|toml|jsonl|json|ll|py|yaml|yml' -# The trailing \b matters: without it `.json` matches inside `.jsonl` and the -# guard reports a path nobody wrote. Longer alternatives lead for the same -# reason. -PATTERN="(${ROOTS})/[A-Za-z0-9._/-]+\.(${EXTS})\b" - -# Paths that prose may name even though the repository does not contain them, -# because something writes them at run time. Each one is an explicit decision, -# not a shape the guard infers. -# -# crates/fuzzer/artifacts/summary.jsonl -- run_seed.py writes it and clears -# the directory on every invocation; crates/fuzzer/README.md documents it. -GENERATED_OUTPUT_PATHS=( - crates/fuzzer/artifacts/summary.jsonl -) - -tracked_list="$(mktemp)" -trap 'rm -f "${tracked_list}"' EXIT -git ls-files >"${tracked_list}" - -is_tracked() { grep -qxF -- "$1" "${tracked_list}"; } - -is_generated_output() { - local declared - for declared in "${GENERATED_OUTPUT_PATHS[@]}"; do - [[ "$1" == "${declared}" ]] && return 0 - done - return 1 -} - -# Roots are verified rather than assumed: if one is renamed, the pattern would -# quietly stop matching anything under it and the guard would report a clean -# tree for the wrong reason. -for root in crates cuda-oxide-book scripts; do - if ! grep -qE "^${root}/" "${tracked_list}"; then - echo "error: source-reference guard: '${root}/' holds no tracked file," >&2 - echo " so the pattern anchored there can no longer match; update" >&2 - echo " ROOTS in $0" >&2 - exit 1 - fi -done - -# `grep -n` keeps the line number; the *.rs arm narrows to doc comments first -# so a path mentioned in code is never read as a claim about the tree. -prose_lines() { - case "$1" in - *.md) grep -nE "${PATTERN}" -- "$1" 2>/dev/null || true ;; - *.rs) grep -nE '^[[:space:]]*(///|//!)' -- "$1" 2>/dev/null | - grep -E "${PATTERN}" || true ;; - esac -} - -# The one scanner. The self-test below runs *this*, not a paraphrase of it, so -# disabling any step -- the pattern, the tracked test, the generated-output -# accommodation -- fails the self-test instead of quietly reporting a clean -# tree. -broken_in_file() { - local file="$1" hit lineno path - while IFS= read -r hit; do - [ -n "${hit}" ] || continue - lineno="${hit%%:*}" - while IFS= read -r path; do - [ -n "${path}" ] || continue - # A traversal segment cannot be resolved against the repository - # root, so it is never a valid claim about a tracked file. - case "${path}" in - */../* | ../* | */..) printf '%s:%s: names %s, which escapes the repository root\n' \ - "${file}" "${lineno}" "${path}"; continue ;; - esac - is_generated_output "${path}" && continue - if ! is_tracked "${path}"; then - printf '%s:%s: names %s, which is not tracked in this repository\n' \ - "${file}" "${lineno}" "${path}" - fi - done < <(printf '%s\n' "${hit#*:}" | grep -oE "${PATTERN}" | sort -u) - done < <(prose_lines "${file}") -} - -# Declared accommodations are verified, not trusted: an entry nothing names any -# more is a silent exemption waiting to hide a real break. -for declared in "${GENERATED_OUTPUT_PATHS[@]}"; do - if is_tracked "${declared}"; then - echo "error: source-reference guard: '${declared}' is listed as a" >&2 - echo " generated output but is tracked; drop it from" >&2 - echo " GENERATED_OUTPUT_PATHS in $0" >&2 - exit 1 - fi - if ! git grep -qF -- "${declared}" -- '*.md' '*.rs' 2>/dev/null; then - echo "error: source-reference guard: nothing refers to '${declared}'" >&2 - echo " any more; drop it from GENERATED_OUTPUT_PATHS in $0" >&2 - exit 1 - fi -done - -# Self-test. The failure mode this guard has to survive is "silently stops -# reporting", so run the real scanner over prose naming, in turn: a missing -# file whose directory the repo does track, a missing file whose directory it -# does *not* (the case an earlier revision let through), an untracked file that -# exists on disk right now, and a tracked file. The first three must be -# reported and the fourth must not. -canary="$(mktemp -d)" -canary_untracked="crates/zz-source-reference-canary-$$.rs" -trap 'rm -f "${tracked_list}" "${canary_untracked}"; rm -rf "${canary}"' EXIT -printf 'prose naming crates/cuda-device/src/no-such-file-%s.rs inline\n' "$$" >"${canary}/dead_in_live_dir.md" -printf 'prose naming crates/no-such-crate-%s/src/lib.rs inline\n' "$$" >"${canary}/dead_dir.md" -printf 'prose naming %s inline\n' "${canary_untracked}" >"${canary}/untracked.md" -# Any tracked path the pattern matches, chosen from the index rather than -# named: hard-coding this script's own path made the self-test depend on the -# guard already being committed, which is false on the branch that adds it and -# in any historical worktree. -canary_live="$(grep -m1 -E "^(${ROOTS})/[A-Za-z0-9._/-]+\.(${EXTS})$" "${tracked_list}")" -if [ -z "${canary_live}" ]; then - echo "error: source-reference guard: the index holds no path the pattern" >&2 - echo " matches, so the self-test cannot prove the tracked case" >&2 - exit 1 -fi -printf 'prose naming %s inline\n' "${canary_live}" >"${canary}/live.md" -: >"${canary_untracked}" - -canary_hits() { broken_in_file "${canary}/$1" | wc -l; } -if [ "$(canary_hits dead_in_live_dir.md)" -ne 1 ] || - [ "$(canary_hits dead_dir.md)" -ne 1 ] || - [ "$(canary_hits untracked.md)" -ne 1 ] || - [ "$(canary_hits live.md)" -ne 0 ]; then - echo "error: source-reference guard self-test failed: the scanner no longer" >&2 - echo " separates a tracked path from a missing one, a missing" >&2 - echo " directory, or a file that exists only in this checkout, so a" >&2 - echo " clean result on the tree means nothing" >&2 - exit 1 -fi -rm -f "${canary_untracked}" - -checked=0 -broken=0 -while IFS= read -r file; do - while IFS= read -r problem; do - [ -n "${problem}" ] || continue - printf '%s\n' "${problem}" >&2 - broken=$((broken + 1)) - done < <(broken_in_file "${file}") - while IFS= read -r _path; do - [ -n "${_path}" ] || continue - checked=$((checked + 1)) - done < <(prose_lines "${file}" | grep -oE "${PATTERN}" | sort -u) -done < <(git ls-files -- '*.md' '*.rs') - -if [ "${broken}" -ne 0 ]; then - echo "" >&2 - echo "error: ${broken} prose reference(s) name a path this repository does" >&2 - echo " not track. Repoint each one at where the code lives now; if" >&2 - echo " the surrounding sentence describes the old shape, it needs" >&2 - echo " rewriting too, not just a new path. A path written at run" >&2 - echo " time belongs in GENERATED_OUTPUT_PATHS in $0, with the reason." >&2 +if ! command -v python3 >/dev/null 2>&1; then + echo "error: python3 is required to check source references" >&2 exit 1 fi -echo "source references ok: ${checked} repo-anchored paths in prose, all tracked" +python3 - <<'PY' +import pathlib +import re +import subprocess +import sys +import tempfile + +ROOTS = ("crates", "cuda-oxide-book", "scripts") +EXTS = ("rs", "md", "sh", "toml", "jsonl", "json", "ll", "py", "yaml", "yml") +# run_seed.py writes and clears this output; the fuzzer README names it. +GENERATED_OUTPUT_PATHS = {"crates/fuzzer/artifacts/summary.jsonl"} + +# Read complete tokens first, then classify them. Whitespace and ordinary +# Markdown/prose wrappers delimit tokens. Scheme-prefixed and network-path +# URI spans are consumed first, including their internal prose punctuation; +# `https://host/x;crates/foo.rs` is one external reference. Balanced square +# brackets stay within a URI (e.g. an IPv6 host); an unmatched closing bracket +# can still end a Markdown link label. A slash, dot, hyphen, backslash or +# colon stays inside other tokens, so longer paths never donate suffixes. +# This is lexical classification, not a URL resolver or a scheme allowlist. +TOKEN = re.compile( + r'''(?P[*_]*(?:[A-Za-z][A-Za-z0-9+.-]*:|//)(?:\[[^\]\s`'"<>]*\]|[^\s`'"<>\[\]])+)''' + r'''|[^\s`'"<>()\[\],;|]+''' +) +PATH = re.compile( + r"(?:" + "|".join(map(re.escape, ROOTS)) + r")/[A-Za-z0-9._/-]+\." + r"(?:" + "|".join(map(re.escape, EXTS)) + r")" +) +DOC_LINE = re.compile(r"^\s*(?:///(?!/)|//!)") +LINE_SUFFIX = re.compile(r":[0-9]+(?:[-:][0-9]+)*$") + + +def paths_in_line(line): + cursor = 0 + while match := TOKEN.search(line, cursor): + cursor = match.end() + if match.lastgroup == "uri": + # A parenthesized URI ends at its matching closing delimiter, + # not at the end of the next adjacent Markdown link. Balance + # nested parentheses in the URI and preserve escaped delimiters. + # Bare/angle-quoted URIs retain their entire punctuation span. + if match.start() > 0 and line[match.start() - 1] == "(": + depth = 0 + escaped = False + for index in range(match.start(), match.end()): + char = line[index] + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == "(": + depth += 1 + elif char == ")": + if depth == 0: + cursor = index + 1 + break + depth -= 1 + continue + token = match.group().rstrip(".!?:") + # Paired, possibly nested Markdown emphasis is a wrapper, not a glob + # suffix. Punctuation can sit inside or outside the closing marker. + while len(token) > 1 and token[0] in "*_" and token[-1] == token[0]: + token = token[1:-1] + token = token.rstrip(".!?:") + # A link fragment and a numeric source-line/range citation are not + # part of a filename. Other suffixes stay intact: `.rs.backup` must + # not be checked as `.rs`, and `:word` is not a line citation. + token = token.partition("#")[0] + token = LINE_SUFFIX.sub("", token) + # The tree also writes shell commands as ./scripts/foo.sh. Accept + # that explicit current-root form, without resolving parents, + # absolute paths or another project's leading directory. + if token.startswith("./"): + token = token[2:] + if PATH.fullmatch(token): + yield token + + +def references(filename, text): + suffix = pathlib.PurePosixPath(filename).suffix + for lineno, line in enumerate(text.splitlines(), 1): + if suffix == ".rs": + marker = DOC_LINE.match(line) + if marker is None: + continue + line = line[marker.end():] + if suffix in (".md", ".rs"): + for path in sorted(set(paths_in_line(line))): + yield lineno, path + + +def broken_references(refs, tracked, generated): + for lineno, path in refs: + if ".." in path.split("/"): + yield lineno, path, "contains an unsupported traversal segment" + elif path not in generated and path not in tracked: + yield lineno, path, "is not tracked in this repository" + + +def declaration_errors(generated, tracked, referenced): + for path in sorted(generated): + if path in tracked: + yield f"'{path}' is declared generated but is tracked" + elif path not in referenced: + yield f"no scanned prose reference names generated output '{path}'" + + +tracked = set(subprocess.check_output(["git", "ls-files", "-z"]).decode().split("\0")) +tracked.discard("") +for root in ROOTS: + if not any(path.startswith(root + "/") for path in tracked): + sys.exit(f"error: source-reference guard: '{root}/' holds no tracked file; update ROOTS") + +# Exercise the actual extractor, prose filter, and membership decision. The +# untracked control really exists on disk, so replacing index membership with +# filesystem existence fails here. The tracked example comes from this index, +# which also permits running the guard in historical worktrees. +live = next((path for path in sorted(tracked) if PATH.fullmatch(path)), None) +if live is None: + sys.exit("error: source-reference guard: no tracked path matches the declared roots/extensions") +with tempfile.NamedTemporaryFile(prefix="zz-source-reference-", suffix=".rs", dir="crates") as canary: + untracked = pathlib.Path(canary.name).relative_to(pathlib.Path.cwd()).as_posix() + missing = "crates/no-such-source-reference-crate/src/lib.rs" + # Synthetic policy inputs exercise the same decision functions even when + # the repository no longer needs any generated-output declarations. + output = "crates/source-reference-canary/generated.jsonl" + generated = {output} + controls = [ + ("case.md", "crates/cuda-device/src/no-such-source-reference-file.rs", 1), + ("case.md", missing, 1), + ("case.md", untracked, 1), + ("case.md", f"`{live}`, [{live}]({live}#L1); **{live}**; {live}:1-2.", 0), + ("case.md", output, 0), + ("case.md", f"https://example.invalid/{missing}", 0), + ("case.md", f"https://example.invalid/x;{missing} https://example.invalid/x,({missing})", 0), + ("case.md", f"//example.invalid/x;{missing} https://[::1]/x;{missing}", 0), + ("case.md", f"**https://example.invalid/({missing})** *_//example.invalid/x;{missing}_*", 0), + ("case.md", f"old-{missing} vendor/{missing} /opt/{missing}", 0), + ("case.md", f"{missing}.backup {missing}-backup", 0), + ("case.md", f"{missing}.md", 1), + ("case.md", f"./{missing}", 1), + ("case.md", f"`{missing}`,\n[source]({missing}#L1)\n**{missing}**\n{missing}:1-2.", 4), + ("case.md", f"***{missing}***\n**_{missing}_**\n**{missing}:1-2.**", 3), + ("case.md", f"[https://example.invalid]({missing})", 1), + ("case.md", f"[external](https://example.invalid/a)[source]({missing})", 1), + ("case.md", f"[external](https://[::1]/a_(b(c)))[source]({missing})", 1), + ("case.md", "crates/../scripts/no-such-source-reference-file.sh", 1), + ("case.rs", f"/// {missing}\n//! {missing}", 2), + ("case.rs", f"///{missing}\n//!{missing}", 2), + ("case.rs", f'// {missing}\n//// {missing}\nconst P: &str = "{missing}";', 0), + ] + for filename, text, expected in controls: + actual = list(broken_references(references(filename, text), tracked, generated)) + if len(actual) != expected: + sys.exit(f"error: source-reference guard self-test failed for {text!r}: {actual}") + if list(paths_in_line(f"`{live}` {output}")) != [live, output]: + sys.exit("error: source-reference guard self-test failed to extract live/output paths") + for text in (f'const P: &str = "{output}";', f"/// https://example.invalid/{output}"): + seen = {path for _, path in references("case.rs", text)} + if not list(declaration_errors({output}, tracked, seen)): + sys.exit("error: generated-output self-test accepted a reference outside prose scope") + if not list(declaration_errors({output}, tracked | {output}, {output})): + sys.exit("error: generated-output self-test accepted an already tracked output") + +checked = 0 +broken = [] +referenced = set() +for filename in sorted(tracked): + if pathlib.PurePosixPath(filename).suffix not in (".md", ".rs"): + continue + refs = list(references(filename, pathlib.Path(filename).read_text(encoding="utf-8"))) + referenced.update(path for _, path in refs) + checked += len({path for _, path in refs}) + broken.extend( + f"{filename}:{lineno}: names {path}, which {reason}" + for lineno, path, reason in broken_references(refs, tracked, GENERATED_OUTPUT_PATHS) + ) + +errors = list(declaration_errors(GENERATED_OUTPUT_PATHS, tracked, referenced)) +for error in errors: + print(f"error: source-reference guard: {error}; update GENERATED_OUTPUT_PATHS", file=sys.stderr) +for problem in broken: + print(problem, file=sys.stderr) +if errors or broken: + sys.exit("error: source references failed; repoint stale prose and declare generated outputs explicitly") +print(f"source references ok: {checked} repo-anchored paths in prose, all tracked or declared generated outputs") +PY From 603aa8643720a5a7cfe3ba6a7b70fea117e8046e Mon Sep 17 00:00:00 2001 From: nihalp Date: Fri, 18 Sep 2026 09:13:17 +0530 Subject: [PATCH 3/3] fix(ci): check Markdown reference definitions consistently Signed-off-by: nihalp --- scripts/check-source-references.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/check-source-references.sh b/scripts/check-source-references.sh index 4f0cb4c285..c65e77e9ec 100755 --- a/scripts/check-source-references.sh +++ b/scripts/check-source-references.sh @@ -96,10 +96,15 @@ PATH = re.compile( ) DOC_LINE = re.compile(r"^\s*(?:///(?!/)|//!)") LINE_SUFFIX = re.compile(r":[0-9]+(?:[-:][0-9]+)*$") +# A Markdown reference definition owns its colon; it is not part of the +# destination token. Strip only the line's label prefix, before classifying +# the destination, so a URI (including a custom scheme) stays intact. +REFERENCE_DEFINITION = re.compile(r"^ {0,3}\[(?:\\.|[^\[\]\\])+\]:[ \t]*") def paths_in_line(line): - cursor = 0 + definition = REFERENCE_DEFINITION.match(line) + cursor = definition.end() if definition else 0 while match := TOKEN.search(line, cursor): cursor = match.end() if match.lastgroup == "uri": @@ -212,6 +217,9 @@ with tempfile.NamedTemporaryFile(prefix="zz-source-reference-", suffix=".rs", di ("case.md", f"[https://example.invalid]({missing})", 1), ("case.md", f"[external](https://example.invalid/a)[source]({missing})", 1), ("case.md", f"[external](https://[::1]/a_(b(c)))[source]({missing})", 1), + ("case.md", f"[source]:{missing}\n [source]: {missing}\n[source]:<{missing}#L1>", 3), + ("case.md", f"[source]:https://example.invalid/{missing}\n[source]:custom:{missing}", 0), + ("case.md", f"TODO:{missing}\nhttps://example.invalid/[source]:{missing}", 0), ("case.md", "crates/../scripts/no-such-source-reference-file.sh", 1), ("case.rs", f"/// {missing}\n//! {missing}", 2), ("case.rs", f"///{missing}\n//!{missing}", 2), @@ -221,6 +229,11 @@ with tempfile.NamedTemporaryFile(prefix="zz-source-reference-", suffix=".rs", di actual = list(broken_references(references(filename, text), tracked, generated)) if len(actual) != expected: sys.exit(f"error: source-reference guard self-test failed for {text!r}: {actual}") + traversal = "crates/../scripts/no-such-source-reference-file.sh" + if list(broken_references(references("case.md", traversal), tracked, generated)) != [ + (1, traversal, "contains an unsupported traversal segment") + ]: + sys.exit("error: source-reference guard self-test lost the traversal diagnostic") if list(paths_in_line(f"`{live}` {output}")) != [live, output]: sys.exit("error: source-reference guard self-test failed to extract live/output paths") for text in (f'const P: &str = "{output}";', f"/// https://example.invalid/{output}"): @@ -229,6 +242,9 @@ with tempfile.NamedTemporaryFile(prefix="zz-source-reference-", suffix=".rs", di sys.exit("error: generated-output self-test accepted a reference outside prose scope") if not list(declaration_errors({output}, tracked | {output}, {output})): sys.exit("error: generated-output self-test accepted an already tracked output") + seen = {path for _, path in references("case.md", f"[output]:{output}")} + if list(declaration_errors({output}, tracked, seen)): + sys.exit("error: generated-output self-test missed a reference definition") checked = 0 broken = []