diff --git a/.github/workflows/golden-crucible.yml b/.github/workflows/golden-crucible.yml index 38fbd0994..1444532d2 100644 --- a/.github/workflows/golden-crucible.yml +++ b/.github/workflows/golden-crucible.yml @@ -57,7 +57,7 @@ jobs: # no credentials -- the only thing a fork run actually lacks is the ref *string*, # and that string is already committed in-repo as PINNED_TAG. So fall back to it # and give fork contributors real signal. Same escape-hatch shape as - # rosetta-audit.yml's `vars.KEYWORD_ROSETTA_REF || 'main'`, but pinned exactly + # rosetta-audit.yml's unpinned checkout of keyword-rosetta main, but pinned exactly # rather than floating on main. See tests/_crucible_pin.py's docstring for why the # pin lives in two places, and remember to bump BOTH when moving it. ref="${{ vars.LANGUAGE_CRUCIBLE_REF }}" diff --git a/.github/workflows/rosetta-audit.yml b/.github/workflows/rosetta-audit.yml index 53256f83b..73ffb22bf 100644 --- a/.github/workflows/rosetta-audit.yml +++ b/.github/workflows/rosetta-audit.yml @@ -1,16 +1,20 @@ name: rosetta-audit -# Cross-language consistency gate (#2557, Phase 5 of the control-corpus plan): -# checks out keyword-rosetta at the pinned KEYWORD_ROSETTA_REF and runs its -# verifier across ALL 46 language folders against THIS PR's engine build, plus -# the corpus's baseline-gated n/a review audit. An engine change that shifts -# corpus-observed counts, or nulls a rule without a validated ledger entry, -# fails HERE -- in the PR that caused it -- instead of days later in the -# corpus repo (how the #2552 cobol drift was found). +# Cross-language consistency audit (#2557, redesigned in #2682): runs the keyword-rosetta +# control corpus's verifier across ALL language folders against THIS PR's engine build and +# reports which languages the build moves. Modelled on tri-comparison-audit.yml: this job +# only ever MEASURES, never writes, and is baseline-gated -- a language that already fails +# against engine main (the corpus has not caught up with a merged engine change yet) is a +# notice, not a failure, so an unrelated PR stays green while the corpus is behind. # -# Intentional drift is re-baselined via the cross-repo protocol in -# docs/self_scan/BUMPING_THE_ROSETTA_PIN.md (the corpus repo's ENGINE_REF -# escape hatch + a bump of KEYWORD_ROSETTA_REF in the engine PR). +# There is no pin. The corpus is checked out at its main; its own CI verifies against THIS +# repo's main. Both are advisory (neither repo requires a status check to merge), so an +# intentional, corpus-visible engine change simply merges first and the corpus re-blesses +# against engine main afterwards -- see docs/self_scan/ROSETTA_AUDIT.md. Add the +# `rosetta:rebless-owed` label to such a PR and its regressions become warnings. +# +# Every step that can silently do nothing is asserted: rosetta_audit.py exits 2 unless it +# actually verified every language folder, and a verifier crash is "broken", never "fail". on: pull_request: @@ -18,9 +22,13 @@ on: paths: - "gitgalaxy/**" - "pyproject.toml" + - "tests/tools/rosetta_audit.py" - ".github/workflows/rosetta-audit.yml" workflow_dispatch: +permissions: + contents: read + jobs: rosetta-audit: runs-on: ubuntu-latest @@ -31,11 +39,20 @@ jobs: path: gitgalaxy persist-credentials: false - - name: Checkout keyword-rosetta (pinned) + # The baseline build: the branch this PR targets. Failing languages are re-run against + # it so the audit can tell "this PR moved the corpus" from "the corpus is behind main". + - name: Checkout GitGalaxy baseline (PR base branch) + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.ref || github.ref_name }} + path: gitgalaxy-baseline + persist-credentials: false + + - name: Checkout keyword-rosetta (main) uses: actions/checkout@v4 with: repository: squid-protocol/keyword-rosetta - ref: ${{ vars.KEYWORD_ROSETTA_REF || 'main' }} + ref: main path: keyword-rosetta persist-credentials: false @@ -46,33 +63,40 @@ jobs: cache: "pip" # Full-precision deps: without all of them galaxyscope silently drops to - # Zero-Dependency Mode and observed counts drift from what the corpus - # manifests were locked against. - - name: Install GitGalaxy (full precision) + # Zero-Dependency Mode and observed counts drift from what the corpus manifests + # were locked against. The baseline gets its own venv (a snapshot install, not + # editable) so the two builds cannot alias each other. + - name: Install GitGalaxy PR build (full precision) run: | python -m pip install --upgrade pip pip install PyYAML networkx tiktoken numpy pandas xgboost pip install -e ./gitgalaxy - - name: Run all language gates - working-directory: keyword-rosetta + - name: Install GitGalaxy baseline build (full precision, separate venv) + run: | + python -m venv baseline-venv + baseline-venv/bin/pip install --upgrade pip + baseline-venv/bin/pip install PyYAML networkx tiktoken numpy pandas xgboost + baseline-venv/bin/pip install ./gitgalaxy-baseline + + - name: Run all language gates, classify against the baseline env: GITGALAXY_PATH: ${{ github.workspace }}/gitgalaxy run: | - rc=0 - for d in data/*/; do - L=$(basename "$d") - python tools/verify_language.py "$L" || rc=1 - done - exit $rc + allow="" + if [ "${{ contains(github.event.pull_request.labels.*.name, 'rosetta:rebless-owed') }}" = "true" ]; then + allow="--allow-regressions" + echo "::notice::rosetta:rebless-owed label present -- regressions reported as warnings" + fi + python gitgalaxy/tests/tools/rosetta_audit.py \ + --corpus keyword-rosetta \ + --baseline-bin "${{ github.workspace }}/baseline-venv/bin/galaxyscope" \ + $allow + # Baseline-gated: fails only on unreviewed rule-absent (n/a) cells beyond the corpus's + # docs/na_baseline.json -- an engine PR cannot quietly null a rule nobody has ledgered. - name: n/a review audit (baseline-gated) working-directory: keyword-rosetta env: GITGALAXY_PATH: ${{ github.workspace }}/gitgalaxy - run: | - if [ -f tools/na_check.py ]; then - python tools/na_check.py --ci - else - echo "::notice::pinned keyword-rosetta ref predates tools/na_check.py -- bump KEYWORD_ROSETTA_REF to activate the n/a audit" - fi + run: python tools/na_check.py --ci diff --git a/.github/workflows/tree-sitter-accuracy-audit.yml b/.github/workflows/tree-sitter-accuracy-audit.yml index 62dc1dd81..657437589 100644 --- a/.github/workflows/tree-sitter-accuracy-audit.yml +++ b/.github/workflows/tree-sitter-accuracy-audit.yml @@ -59,7 +59,7 @@ jobs: # no credentials -- the only thing a fork run actually lacks is the ref *string*, # and that string is already committed in-repo as PINNED_TAG. So fall back to it # and give fork contributors real signal. Same escape-hatch shape as - # rosetta-audit.yml's `vars.KEYWORD_ROSETTA_REF || 'main'`, but pinned exactly + # rosetta-audit.yml's unpinned checkout of keyword-rosetta main, but pinned exactly # rather than floating on main. See tests/_crucible_pin.py's docstring for why the # pin lives in two places, and remember to bump BOTH when moving it. ref="${{ vars.LANGUAGE_CRUCIBLE_REF }}" diff --git a/.github/workflows/tree-sitter-accuracy-history.yml b/.github/workflows/tree-sitter-accuracy-history.yml index 742007301..b47bad7c4 100644 --- a/.github/workflows/tree-sitter-accuracy-history.yml +++ b/.github/workflows/tree-sitter-accuracy-history.yml @@ -82,7 +82,7 @@ jobs: # no credentials -- the only thing a fork run actually lacks is the ref *string*, # and that string is already committed in-repo as PINNED_TAG. So fall back to it # and give fork contributors real signal. Same escape-hatch shape as - # rosetta-audit.yml's `vars.KEYWORD_ROSETTA_REF || 'main'`, but pinned exactly + # rosetta-audit.yml's unpinned checkout of keyword-rosetta main, but pinned exactly # rather than floating on main. See tests/_crucible_pin.py's docstring for why the # pin lives in two places, and remember to bump BOTH when moving it. ref="${{ vars.LANGUAGE_CRUCIBLE_REF }}" diff --git a/.github/workflows/tri-comparison-audit.yml b/.github/workflows/tri-comparison-audit.yml index 8d8c601c3..d34773411 100644 --- a/.github/workflows/tri-comparison-audit.yml +++ b/.github/workflows/tri-comparison-audit.yml @@ -84,7 +84,7 @@ jobs: # no credentials -- the only thing a fork run actually lacks is the ref *string*, # and that string is already committed in-repo as PINNED_TAG. So fall back to it # and give fork contributors real signal. Same escape-hatch shape as - # rosetta-audit.yml's `vars.KEYWORD_ROSETTA_REF || 'main'`, but pinned exactly + # rosetta-audit.yml's unpinned checkout of keyword-rosetta main, but pinned exactly # rather than floating on main. See tests/_crucible_pin.py's docstring for why the # pin lives in two places, and remember to bump BOTH when moving it. ref="${{ vars.LANGUAGE_CRUCIBLE_REF }}" diff --git a/.github/workflows/tri-comparison-history.yml b/.github/workflows/tri-comparison-history.yml index daf741c32..9440a3e0f 100644 --- a/.github/workflows/tri-comparison-history.yml +++ b/.github/workflows/tri-comparison-history.yml @@ -93,7 +93,7 @@ jobs: # no credentials -- the only thing a fork run actually lacks is the ref *string*, # and that string is already committed in-repo as PINNED_TAG. So fall back to it # and give fork contributors real signal. Same escape-hatch shape as - # rosetta-audit.yml's `vars.KEYWORD_ROSETTA_REF || 'main'`, but pinned exactly + # rosetta-audit.yml's unpinned checkout of keyword-rosetta main, but pinned exactly # rather than floating on main. See tests/_crucible_pin.py's docstring for why the # pin lives in two places, and remember to bump BOTH when moving it. ref="${{ vars.LANGUAGE_CRUCIBLE_REF }}" diff --git a/CLAUDE.md b/CLAUDE.md index b7b94e1f6..0a7d66ddb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -289,20 +289,21 @@ corpus-wide whenever any node's mass changes, so a single-language fix ripples c across every language in the corpus. Those are attributable as a class; filter them out and scope the remainder, which is what actually needs a per-language explanation. -**The two accuracy audits exit 0 while doing nothing if `galaxyscope` is off `PATH`.** -`tests/tools/tree_sitter_accuracy_audit.py` and `tests/tools/tri_comparison_chart.py` report -`galaxyscope not found on PATH` per language, skip every one, and still print -`all OK` with exit code 0 -- a false green that looks exactly like a pass. They need the **main -`.venv`**, which has both `galaxyscope` and `tree_sitter_language_pack`; neither crucible venv -does (`full_precision` has `galaxyscope` but not the tree-sitter pack). Always run them as: +**The corpus-backed audit runners need the main `.venv`, and they refuse to pass on nothing.** +`tests/tools/tree_sitter_accuracy_audit.py`, `tests/tools/tri_comparison_chart.py` and +`tests/tools/rosetta_audit.py` all need `galaxyscope` on `PATH` (the main `.venv` has it plus +`tree_sitter_language_pack`; neither crucible venv has both). Since #2682 a run that checked zero +languages, or a baselined language whose scan could not run, is a hard failure — previously +`tri_comparison_chart.py --ci` skipped every language and printed `all OK` with exit 0 when +`galaxyscope` was missing. Run them as: ```sh PATH="$PWD/.venv/bin:$PATH" .venv/bin/python tests/tools/tree_sitter_accuracy_audit.py --ci --all PATH="$PWD/.venv/bin:$PATH" .venv/bin/python tests/tools/tri_comparison_chart.py --all --ci +PATH="$PWD/.venv/bin:$PATH" .venv/bin/python tests/tools/rosetta_audit.py # needs ../keyword-rosetta ``` -and confirm the summary line names a plausible language count (30 and 3 respectively as of -2026-09-02) -- "0 languages checked, all OK" is a failure wearing a pass. +and still glance at the summary line's language count (30, 3 and 46 respectively as of 2026-09-03). ## Logging cases where GitGalaxy beats tree-sitter/AST ground truth diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f163a1530..cf6741d97 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,17 +80,17 @@ When you modify GitGalaxy's core engine, several CI workflows will rigorously te 4. **Cross-Language Consistency (`rosetta-audit`, the [keyword-rosetta](https://github.com/squid-protocol/keyword-rosetta) corpus)** This runs the control corpus's verifier across all 46 language folders against your engine build, - so a change that shifts corpus-observed counts fails **in the PR that caused it**. Unlike the three - baselines above, the expected values live in *another repository* — you cannot re-bless them here. - - Run one language locally: `GITGALAXY_PATH= python tools/verify_language.py ` from a keyword-rosetta checkout. - - If the drift is an unintentional regression: fix the engine change. **Do not touch the pin.** - - If the drift is an intentional, corpus-visible improvement: it needs a companion re-baseline PR - in keyword-rosetta *before* this one can merge. Follow - [`docs/self_scan/BUMPING_THE_ROSETTA_PIN.md`](docs/self_scan/BUMPING_THE_ROSETTA_PIN.md) — in short, - set that repo's committed `ENGINE_REF` file to `pull//head` so its gates run green - against your unmerged branch, then a maintainer restores it and bumps `KEYWORD_ROSETTA_REF` here. - - Bumping the pin needs repo admin, so **ask a maintainer** rather than trying to do it yourself. - Bumping a pin is never a way to make a red check go away. + then re-runs anything that failed against a build of the branch you target, so it can tell + "this PR moves the corpus" from "the corpus is behind main" (the latter is a notice, not a + failure). Unlike the three baselines above, the expected values live in *another repository* — + you cannot re-bless them here, and you do not have to before merging. + - Run it locally: `python tests/tools/rosetta_audit.py` (needs `../keyword-rosetta` and `galaxyscope` on PATH). + - If the drift is an unintentional regression: fix the engine change. + - If the drift is an intentional, corpus-visible improvement: add the `rosetta:rebless-owed` label + (the check goes green with the languages still listed), merge, then open the re-bless PR in + keyword-rosetta against engine main. Full detail in + [`docs/self_scan/ROSETTA_AUDIT.md`](docs/self_scan/ROSETTA_AUDIT.md). There is no pin on + either side and nothing to reset afterwards. If your PR touches any baseline fixtures, **explain why in the PR description** (e.g. "improved the Rust parser, now correctly detects async trait bounds"). A CI check flags any PR that modifies these files so it's never invisible in a large diff. @@ -123,8 +123,8 @@ If you are moving the pin, bump it in **both** places — `tests/_crucible_pin.p `LANGUAGE_CRUCIBLE_REF` repository variable. Nothing enforces that they match; see that file's docstring for why the pin is deliberately duplicated. -`rosetta-audit` uses the same escape hatch against a different corpus (it falls back to -keyword-rosetta's `main`), and every other check — `full-suite`, the `smoke-test` matrix, +`rosetta-audit` needs no escape hatch: it always checks out keyword-rosetta's `main` (that repo is +public), so it runs for real on a fork PR too. Every other check — `full-suite`, the `smoke-test` matrix, `ruff-audit`, `mypy-audit`, `ast-accuracy-audit` — runs normally on a fork PR too. --- diff --git a/docs/ecosystem.md b/docs/ecosystem.md index 47c739d3f..1787afdbe 100644 --- a/docs/ecosystem.md +++ b/docs/ecosystem.md @@ -16,7 +16,7 @@ Canonical local layout on the dev machine — all siblings under `/srv/storage_1 |---|---|---| | **gitgalaxy** | `gitgalaxy/v6` | **The engine** (this repo). AST-free, LLM-free static analysis: bounded-regex structural signatures → knowledge graph → risk scoring / SBOM / 3D map. Everything else in this table exists to feed, verify, or showcase it. | | **language-crucible** | `all_language_repo` | Zero-execution structural-parser **benchmark corpus** (`data///`, per-category `SOURCES.md`, machine-readable `PROVENANCE.json`). gitgalaxy CI pins it to a release tag (`LANGUAGE_CRUCIBLE_REF` GH Actions var + `tests/_crucible_pin.py`) and diffs golden masters against it. Releases per its `RELEASING.md`. | -| **keyword-rosetta** | `keyword-rosetta` | **Control corpus**: one identical 12-probe program shell in all 46 signature-bearing languages, exact planted keyword counts — measures whether the engine treats identical intent identically across languages (cross-language bias). Gates via `tools/verify_language.py`; deviations live in `deviation_ledger.json` per its `docs/GATING.md`. Its CI checks out gitgalaxy **main** (see choreography below). | +| **keyword-rosetta** | `keyword-rosetta` | **Control corpus**: one identical 12-probe program shell in all 46 signature-bearing languages, exact planted keyword counts — measures whether the engine treats identical intent identically across languages (cross-language bias). Gates via `tools/verify_language.py`; deviations live in `deviation_ledger.json` per its `docs/GATING.md`. Its CI checks out gitgalaxy **main**, and gitgalaxy's `rosetta-audit` checks out its **main** — no pins in either direction (#2682). Its `bias-history.yml` regenerates the bias chart after every corpus push and daily. | | **gitgalaxy-raw-output** | `gitgalaxy-raw-output` | Real, unedited **scan outputs** on independently-chosen production repos (`v//_galaxy_llm.md` + gzipped audit/SBOM) plus speed charts. Evidence source for README claims and `docs/language_status/` §8 sections. | | **squid-telemetry** | `squid-telemetry` | **Distribution/adoption analytics** (the engine itself is air-gapped and phones nothing home; this pipeline scrapes public GitHub/GitLab/PyPI fetch metrics daily via Actions and commits regenerated chart PNGs). | | **gitgalaxy-population-analyses** | `gitgalaxy-population-analyses` | Offline **statistical analyses** over scan populations (risk-distribution ridgeplots, archetype clustering, threat-prediction distribution studies). Reads raw inputs from gitgalaxy-raw-output; never on any CI path. | @@ -80,7 +80,7 @@ rule, sync all three. | Workflow | Repos touched (in merge order) | Documented in | |---|---|---| | **Crucible corpus growth → release → pin bump** | language-crucible (data PRs, tag per `RELEASING.md`) → gitgalaxy (`docs/self_scan/BUMPING_THE_CRUCIBLE_PIN.md`: regen golden masters + tri-comparison + tree-sitter artifacts, bump `LANGUAGE_CRUCIBLE_REF` + `PINNED_TAG`) | crucible `RELEASING.md`; gitgalaxy `BUMPING_THE_CRUCIBLE_PIN.md` | -| **Rosetta sweep** (work one language's bias issue) | gitgalaxy engine PR first (its `rosetta-audit` check fails at the source — expected) → keyword-rosetta corpus re-baseline PR with `ENGINE_REF=pull//head` (green immediately, no draft limbo) → restore `ENGINE_REF=main`, merge corpus → engine PR bumps `KEYWORD_ROSETTA_REF` var, merges green. Capstone lands back in gitgalaxy `docs/language_status/.md` §10. | keyword-rosetta `rosetta-language-sweep` skill; gitgalaxy `docs/self_scan/BUMPING_THE_ROSETTA_PIN.md` | +| **Rosetta sweep** (work one language's bias issue) | gitgalaxy engine PR first (its `rosetta-audit` lists the languages it moves; add the `rosetta:rebless-owed` label if that is intended) → merge → keyword-rosetta corpus re-bless PR against engine main (green by construction; manifests + ledger) → merge; its `bias-history.yml` regenerates the chart. Capstone lands back in gitgalaxy `docs/language_status/.md` §10. | keyword-rosetta `rosetta-language-sweep` skill; gitgalaxy `docs/self_scan/ROSETTA_AUDIT.md` | | **Adding a language to the engine** | gitgalaxy (`standards/how_to_add_a_language.md`, includes authoring the rosetta control folder) → keyword-rosetta (`SPEC.md` shell + manifest) → optionally language-crucible (`expand-language-coverage`) | those three docs | | **Tri-comparison / accuracy verification** | gitgalaxy only (ledger, chart, `manual_verification.json`), but reads the pinned crucible corpus | gitgalaxy `docs/self_scan/tri_comparison_README.md` | | **README / evidence claims** | gitgalaxy README cites gitgalaxy-raw-output artifacts and the keyword-rosetta chart (embedded from that repo's raw main URL — it self-updates when rosetta main moves) | gitgalaxy `readme-maintenance` skill | diff --git a/docs/self_scan/BUMPING_THE_ROSETTA_PIN.md b/docs/self_scan/BUMPING_THE_ROSETTA_PIN.md deleted file mode 100644 index 44dde6c86..000000000 --- a/docs/self_scan/BUMPING_THE_ROSETTA_PIN.md +++ /dev/null @@ -1,84 +0,0 @@ -# Bumping the keyword-rosetta pin (KEYWORD_ROSETTA_REF) - -The `rosetta-audit` CI check (`.github/workflows/rosetta-audit.yml`, issue #2557) runs the -[keyword-rosetta](https://github.com/squid-protocol/keyword-rosetta) control corpus's verifier — -all 46 language folders plus its baseline-gated n/a review audit — against every PR that touches -engine code, with the corpus checked out at the **`KEYWORD_ROSETTA_REF`** GitHub Actions -variable (falls back to `main` if unset). This is the cross-language -*consistency* twin of the crucible pin (`docs/self_scan/BUMPING_THE_CRUCIBLE_PIN.md`), and the -same principle applies: the pin makes the gate deterministic, and bumping it is a deliberate, -reviewed act — never a way to make a red check go away. - -## The value MUST be a full 40-character SHA - -`actions/checkout` only treats a **40-character** hex string as a commit SHA. Anything -shorter -- an abbreviated `357345f`, a branch name, a tag -- it treats as a ref *name* and -resolves by fetching `refs/heads/*`. An abbreviated SHA matches no ref, so the job -dies in **Checkout keyword-rosetta (pinned)** before running a single verification: - -``` -# a full SHA -- fetched directly, works -git fetch --depth=1 origin 6b1efab63f70bc9f4bb315915e58935e88bd427d - -# an abbreviated SHA -- treated as a ref name, matches nothing, retries twice, fails -git fetch --depth=1 origin +refs/heads/357345f*:refs/remotes/origin/357345f* -``` - -This is not hypothetical: on 2026-09-02 the pin was bumped to the 7-character `357345f` -after keyword-rosetta#32 merged, and `rosetta-audit` failed on `main` and on every engine PR -until it was repaired -- silently, because a checkout failure and a real gate failure look -identical in the checks list. Always pass `git rev-parse origin/main`, never `--short`: - -```sh -cd && git fetch origin && git rev-parse origin/main # 40 chars -``` - -## rosetta-audit is red -- did it actually run? - -Check this **before** assuming corpus drift. A red check has two very different causes: - -1. **The job never ran the verifier** -- it failed in `Checkout keyword-rosetta (pinned)`. - Almost always a malformed pin (see above). Nothing about your PR is implicated. -2. **The verifier ran and disagreed** -- that is the gate working; go to the next section. - -```sh -gh run view --log-failed | grep -m1 'Checkout keyword-rosetta' # empty => it ran -``` - -## When rosetta-audit fails on your PR - -That is the gate working: your change shifts what the engine observes on the planted corpus -(or removes a signal rule nobody has ledgered). Decide which of these you are in: - -1. **Unintentional regression** — fix the engine change. Do not touch the pin. -2. **Intentional, corpus-visible improvement** (a rule fix/addition, a stripper change): the - expected values live in the corpus repo, so re-baseline there, then bump the pin here: - - 1. In keyword-rosetta, open the re-baseline PR per its `docs/GATING.md`: manifest edits + - a validated `deviation_ledger.json` entry (or `still_reproduces` flip) justifying every - changed number — with the committed **`ENGINE_REF`** file set to `pull//head` for - YOUR engine PR `N`, so its gates run against your unmerged build and go green now. - 2. Restore `ENGINE_REF` to `main` and merge the corpus PR (once your engine PR is approved). - 3. In your engine PR: update `KEYWORD_ROSETTA_REF` to the corpus repo's new main SHA — - `gh api -X PATCH repos/squid-protocol/gitgalaxy/actions/variables/KEYWORD_ROSETTA_REF -f value=$(git -C rev-parse origin/main)` - -- the **full 40-character** SHA; an abbreviated one breaks the checkout (see above) - (needs repo admin; in a review, request the maintainer do it) — and note the bump + the - companion corpus PR in the PR body's **Cross-repo** section. - 4. `rosetta-audit` reruns green → merge. - -3. **A new rule absence** (you nulled/removed a rule): `na_check --ci` fails until the corpus - PR ships a validated ledger entry naming the language and signal — absence is either real - morphology (ledger it) or a gap (don't ship it). Never regenerate `docs/na_baseline.json` - to absorb an unreviewed cell. - -## Invariants - -- The variable is always a **full 40-character SHA** -- never abbreviated, never a branch name. -- The variable always points at a keyword-rosetta commit whose gates pass against the engine - `main` that existed when it was set (the brief corpus-ahead-of-engine window during step 2 - is covered by this gate's own pin). -- Never bump the pin and engine behavior in *unrelated* PRs simultaneously; the bump belongs - to the PR whose behavior change it validates. -- The corpus repo's independence is the point: expected values and their audit trail - (`deviation_ledger.json`) live there, under its gating rules — this repo only pins which - snapshot it holds itself accountable to. diff --git a/docs/self_scan/ROSETTA_AUDIT.md b/docs/self_scan/ROSETTA_AUDIT.md new file mode 100644 index 000000000..5dfa7f34f --- /dev/null +++ b/docs/self_scan/ROSETTA_AUDIT.md @@ -0,0 +1,89 @@ +# The rosetta-audit check (keyword-rosetta, no pins) + +`rosetta-audit` (`.github/workflows/rosetta-audit.yml`, #2557, redesigned in #2682) runs the +[keyword-rosetta](https://github.com/squid-protocol/keyword-rosetta) control corpus's verifier +across every language folder against your PR's engine build. It is the cross-language +*consistency* twin of the tri-comparison check, and it is built the same way: **it only ever +measures, it is baseline-gated, and it is advisory.** Neither repo requires a status check to +merge, and there is no pin to bump on either side. + +## What the corpus is + +One identical 12-probe program written in all 46 signature-bearing languages, with the exact +planted count of every signal keyword recorded in `data//expected_signals.json`. Identical +intent everywhere means any divergence in what the engine extracts is measured language bias. +Those manifests are the corpus's human-blessed baselines: every number in them has a +`status: "validated"` entry in its `deviation_ledger.json` (its `docs/GATING.md`), the same bar +`tests/tri_comparison_baseline_*.json` and the tri-comparison ledger hold here. + +## The three outcomes + +`tests/tools/rosetta_audit.py` runs every language against your build, then re-runs anything +that failed against a second build of the branch you are targeting, and classifies each +language: + +| verdict | meaning | job result | +|---|---|---| +| **clean** | manifest matches your build | pass | +| **pre-existing** | fails against your build **and** against main: the corpus has not caught up with an engine change that already merged. Not your PR's doing. | notice, pass | +| **regression** | fails against your build, passes against main: **your change moves the corpus** | fail, unless labelled (below) | +| **broken** | a verifier run crashed instead of reporting PASS/FAIL (no `galaxyscope`, scan error) | fail, exit 2 | + +The step summary lists the moved languages with the verifier's own diff lines. A run that +checked zero languages is exit 2, never "all OK" (#2682 — that false green happened). + +## When your PR moves the corpus + +Decide which of these you are in: + +1. **Unintentional regression.** Fix the engine change. +2. **Intentional, corpus-visible improvement** (a rule fix or addition, a stripper change). + The expected values live in the corpus repo, so: + - add the **`rosetta:rebless-owed`** label to this PR — the audit reports the regressions + as warnings and goes green, with the languages still listed in the summary; + - merge; + - open the re-bless PR in keyword-rosetta **against engine main** (manifests + ledger + per its `docs/GATING.md`). Its `verify.yml` checks out engine main, so it is green by + construction once the numbers are right. Nothing to set beforehand, nothing to reset + after. + - Note the companion PR in this PR's **Cross-repo** section (`docs/ecosystem.md`). +3. **A new rule absence** (you nulled/removed a rule): the separate `n/a review audit` step + fails until the corpus ships a validated ledger entry naming the language and signal. + Absence is either real morphology (ledger it) or a gap (don't ship it). Never regenerate + the corpus's `docs/na_baseline.json` to absorb an unreviewed cell. + +Adding a rule that was `None` is corpus-visible even though every manifest still says 0 — +see the corpus's `AGENTS.md` rule 8: pair it with a corpus plant or it manufactures a red cell +in the bias report. + +## What happens after you merge + +keyword-rosetta's `bias-history.yml` (the twin of `tri-comparison-history.yml` here) runs on +every corpus push and daily: it verifies the corpus against engine main, regenerates the bias +report and chart, commits them, and keeps one issue — *"corpus owes a re-bless against engine +main"* — open with the failing languages until the re-bless lands. That issue, not a red check +on someone else's PR, is where drift shows up. + +## Reproducing locally + +```sh +# from this repo, with the main .venv active (it has galaxyscope) and ../keyword-rosetta checked out +python tests/tools/rosetta_audit.py +# classify against a second build, e.g. the full-precision crucible venv built from main +python tests/tools/rosetta_audit.py --baseline-bin .crucible_venvs/full_precision/bin/galaxyscope +# one language, with the corpus's own verifier +GITGALAXY_PATH=$PWD python ../keyword-rosetta/tools/verify_language.py go --report +``` + +`GALAXYSCOPE_BIN` selects the engine build under test; `--baseline-bin` the one to classify +against. Exit codes: 0 clean or pre-existing only, 1 regression, 2 the audit did not run. + +## Invariants + +- The audit checks out keyword-rosetta at `main`; keyword-rosetta's CI checks out this repo + at `main`. Both are advisory. There is no `KEYWORD_ROSETTA_REF` and no `ENGINE_REF`. +- Merge order for a count-changing engine change is engine first, corpus second. The corpus + never needs to verify against an unmerged engine PR in CI; if you want that anyway, + dispatch its `verify.yml` with `engine_ref=pull//head` — a run parameter, not a file. +- The corpus repo's independence is the point: expected values and their audit trail live + there, under its gating rules. This repo only reports what its build does to them. diff --git a/tests/tools/rosetta_audit.py b/tests/tools/rosetta_audit.py new file mode 100644 index 000000000..c82bc0532 --- /dev/null +++ b/tests/tools/rosetta_audit.py @@ -0,0 +1,263 @@ +"""Run the keyword-rosetta control corpus against the CURRENT engine build. + +The corpus (squid-protocol/keyword-rosetta) plants an identical 12-probe program in every +signature-bearing language with exact expected signal counts. Its own `tools/verify_language.py` +is the per-language gate; this script runs that gate for every language folder and answers the +one question an engine PR actually needs answered: + + which languages does THIS build move, and were they already moving on engine main? + +A language whose manifest fails against this build but PASSES against a baseline build (engine +main) is a **regression** this change introduces -- or an intentional, corpus-visible improvement +the corpus now owes a re-bless for (docs/self_scan/ROSETTA_AUDIT.md). A language that fails +against BOTH is **pre-existing drift**: engine main already moved past the corpus and the corpus +has not caught up yet. That is not this PR's problem and is reported as a notice, never a failure, +so an unrelated PR opened while the corpus is behind stays green. + +Usage: + python tests/tools/rosetta_audit.py # sibling ../keyword-rosetta, current galaxyscope + python tests/tools/rosetta_audit.py --corpus --baseline-bin + python tests/tools/rosetta_audit.py --allow-regressions # report regressions as warnings, exit 0 + +Environment: + GALAXYSCOPE_BIN the CURRENT build's galaxyscope (default: `galaxyscope` on PATH) + GITHUB_STEP_SUMMARY / GITHUB_ACTIONS honoured when present (summary table + annotations) + +Exit codes: + 0 every language passes, or the only failures are pre-existing drift (or --allow-regressions) + 1 at least one regression this build introduces + 2 the audit did not actually run: no galaxyscope, no corpus, zero languages, or a verifier + that crashed instead of reporting PASS/FAIL. "0 languages checked, all OK" is a failure + wearing a pass (gitgalaxy#2682) and this script refuses to print it. +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +DEFAULT_CORPUS = REPO_ROOT.parent / "keyword-rosetta" + +# Per-language outcome of one verifier run. +PASS = "pass" +FAIL = "fail" +ERROR = "error" # the verifier crashed / never reported -- infrastructure, not drift + +# Classification of a language after the (optional) baseline re-run. +CLEAN = "clean" +REGRESSION = "regression" # fails here, passes on the baseline build +PRE_EXISTING = "pre-existing" # fails on both builds: corpus is behind engine main +MOVED = "moved" # fails here, no baseline given to classify against +BROKEN = "broken" # a verifier run errored + + +@dataclass +class Outcome: + language: str + status: str + tail: str = "" + baseline_status: str | None = None + verdict: str = field(default=CLEAN) + + +def _corpus_languages(corpus: Path) -> list[str]: + data = corpus / "data" + if not data.is_dir(): + return [] + return sorted(p.parent.name for p in data.glob("*/expected_signals.json")) + + +def _run_verifier(corpus: Path, language: str, galaxyscope_bin: str) -> tuple[str, str]: + """Runs keyword-rosetta's verify_language.py once. Returns (status, output tail). + + PASS/FAIL come from the verifier's own final line, not from its exit code alone: a crash + (no galaxyscope, a schema import error, a scan timeout) also exits nonzero, and treating + that as "FAIL" would report an infrastructure problem as corpus drift. + """ + env = {**os.environ, "GALAXYSCOPE_BIN": galaxyscope_bin, "GITGALAXY_PATH": str(REPO_ROOT)} + try: + result = subprocess.run( + [sys.executable, str(corpus / "tools" / "verify_language.py"), language], + cwd=corpus, + env=env, + capture_output=True, + text=True, + timeout=900, + ) + except subprocess.TimeoutExpired: + return ERROR, "verify_language.py timed out after 900s" + out = (result.stdout + result.stderr).strip() + lines = [ln for ln in out.splitlines() if ln.strip()] + last = lines[-1] if lines else "" + for ln in reversed(lines): + if ln.startswith(f"PASS {language}"): + return PASS, ln + if ln.startswith(f"FAIL {language}"): + return FAIL, "\n".join(lines[-12:]) + return ERROR, "\n".join(lines[-12:]) or f"no output (rc={result.returncode}); last line: {last!r}" + + +def classify(current: dict[str, str], baseline: dict[str, str] | None) -> dict[str, str]: + """Pure classification of per-language statuses -- the part worth unit-testing. + + `current` maps language -> PASS/FAIL/ERROR for this build. `baseline` maps the languages + that failed here -> their status on the baseline build (None when no baseline was run). + """ + verdicts: dict[str, str] = {} + for lang, status in current.items(): + if status == PASS: + verdicts[lang] = CLEAN + elif status == ERROR: + verdicts[lang] = BROKEN + elif baseline is None or lang not in baseline: + verdicts[lang] = MOVED + elif baseline[lang] == PASS: + verdicts[lang] = REGRESSION + elif baseline[lang] == FAIL: + verdicts[lang] = PRE_EXISTING + else: + verdicts[lang] = BROKEN + return verdicts + + +def _annotate(kind: str, message: str) -> None: + if os.environ.get("GITHUB_ACTIONS"): + print(f"::{kind}::{message}") + + +def _write_summary(outcomes: list[Outcome], baseline_used: bool, allow_regressions: bool) -> None: + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + counts = {v: sum(1 for o in outcomes if o.verdict == v) for v in (CLEAN, REGRESSION, PRE_EXISTING, MOVED, BROKEN)} + lines = [ + "## rosetta-audit", + "", + f"{len(outcomes)} languages checked against this build" + + (" and against engine main for anything that failed." if baseline_used else "."), + "", + f"clean **{counts[CLEAN]}** · regression **{counts[REGRESSION]}** · pre-existing drift " + f"**{counts[PRE_EXISTING]}** · moved (unclassified) **{counts[MOVED]}** · broken **{counts[BROKEN]}**", + "", + ] + moved = [o for o in outcomes if o.verdict != CLEAN] + if moved: + lines += ["| language | this build | engine main | verdict |", "|---|---|---|---|"] + for o in moved: + lines.append(f"| {o.language} | {o.status} | {o.baseline_status or '-'} | {o.verdict} |") + lines.append("") + if counts[REGRESSION] or counts[MOVED]: + lines.append( + ("Regressions are allowed on this PR (`rosetta:rebless-owed`). " if allow_regressions else "") + + "Intentional? Merge, then re-bless those languages in keyword-rosetta against engine main " + "(its `docs/GATING.md`). Unintentional? Fix the engine change. " + "See `docs/self_scan/ROSETTA_AUDIT.md`." + ) + if counts[PRE_EXISTING]: + lines.append( + "Pre-existing drift is engine main already ahead of the corpus; not this PR's doing. " + "keyword-rosetta's `bias-history` workflow tracks it as an open issue there." + ) + lines.append("") + lines.append("
verifier output\n") + for o in moved: + lines.append(f"**{o.language}**\n\n```\n{o.tail}\n```\n") + lines.append("
") + Path(path).open("a", encoding="utf-8").write("\n".join(lines) + "\n") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS, help="keyword-rosetta checkout") + parser.add_argument( + "--baseline-bin", + help="galaxyscope of the baseline build (engine main); failing languages are re-run against it", + ) + parser.add_argument( + "--allow-regressions", + action="store_true", + help="report regressions as warnings and exit 0 (the PR carries the rosetta:rebless-owed label)", + ) + parser.add_argument("--jobs", type=int, default=max(1, min(4, os.cpu_count() or 1))) + args = parser.parse_args(argv) + + corpus: Path = args.corpus.resolve() + if not (corpus / "tools" / "verify_language.py").is_file(): + print(f"rosetta_audit: no keyword-rosetta checkout at {corpus} (need tools/verify_language.py)") + return 2 + languages = _corpus_languages(corpus) + if not languages: + print(f"rosetta_audit: zero language folders with expected_signals.json under {corpus / 'data'}") + return 2 + + current_bin = os.environ.get("GALAXYSCOPE_BIN") or shutil.which("galaxyscope") or "" + if not current_bin or not Path(current_bin).exists(): + print("rosetta_audit: galaxyscope not found -- set GALAXYSCOPE_BIN or activate the venv (pip install -e .)") + return 2 + baseline_bin: str | None = args.baseline_bin + if baseline_bin and not Path(baseline_bin).exists(): + print(f"rosetta_audit: --baseline-bin {baseline_bin} does not exist") + return 2 + + print(f"rosetta_audit: {len(languages)} languages, corpus {corpus}, engine {current_bin}") + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + results = list(pool.map(lambda lang: _run_verifier(corpus, lang, current_bin), languages)) + outcomes = [Outcome(lang, status, tail) for lang, (status, tail) in zip(languages, results)] + current = {o.language: o.status for o in outcomes} + + failed = [o for o in outcomes if o.status == FAIL] + baseline: dict[str, str] | None = None + if baseline_bin and failed: + print(f"rosetta_audit: {len(failed)} failed here; re-running those against the baseline build {baseline_bin}") + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + base_results = list(pool.map(lambda o: _run_verifier(corpus, o.language, baseline_bin), failed)) + baseline = {} + for o, (status, _tail) in zip(failed, base_results): + o.baseline_status = status + baseline[o.language] = status + + verdicts = classify(current, baseline) + for o in outcomes: + o.verdict = verdicts[o.language] + + for o in outcomes: + marker = {CLEAN: "ok ", REGRESSION: "REG ", PRE_EXISTING: "pre ", MOVED: "MOV ", BROKEN: "ERR "}[o.verdict] + print(f" {marker} {o.language:18s} {o.tail.splitlines()[0] if o.tail else ''}") + if o.verdict != CLEAN and o.tail.count("\n"): + for ln in o.tail.splitlines()[1:]: + print(f" {ln}") + + n_reg = sum(1 for o in outcomes if o.verdict in (REGRESSION, MOVED)) + n_pre = sum(1 for o in outcomes if o.verdict == PRE_EXISTING) + n_broken = sum(1 for o in outcomes if o.verdict == BROKEN) + _write_summary(outcomes, baseline is not None, args.allow_regressions) + + print( + f"\nrosetta_audit: {len(outcomes)} language(s) checked -- " + f"{n_reg} regression(s), {n_pre} pre-existing, {n_broken} broken." + ) + if n_broken: + _annotate("error", f"rosetta-audit did not run cleanly: {n_broken} verifier run(s) crashed -- see the job log") + return 2 + for o in outcomes: + if o.verdict == PRE_EXISTING: + _annotate("notice", f"{o.language}: corpus is behind engine main (pre-existing drift, not this PR)") + elif o.verdict in (REGRESSION, MOVED): + _annotate( + "warning" if args.allow_regressions else "error", + f"{o.language}: this build moves the control corpus -- {o.tail.splitlines()[0]}", + ) + if n_reg and not args.allow_regressions: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/tools/test_audit_runners_refuse_empty.py b/tests/tools/test_audit_runners_refuse_empty.py new file mode 100644 index 000000000..3c6761343 --- /dev/null +++ b/tests/tools/test_audit_runners_refuse_empty.py @@ -0,0 +1,164 @@ +"""#2682: every corpus-backed audit runner must refuse to report success when it checked nothing. + +Three runners used to (or could) print "all OK" with exit 0 after skipping every language -- +tri_comparison_chart.py did exactly that when galaxyscope was off PATH. These tests pin the +refuse-to-no-op behaviour of each, plus rosetta_audit.py's pure classification. + +Sibling-module import pattern per CLAUDE.md "Testing conventions" (no tests/__init__.py). +""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +TOOLS = Path(__file__).resolve().parent +if str(TOOLS) not in sys.path: + sys.path.insert(0, str(TOOLS)) + +import rosetta_audit # noqa: E402 + + +# --- rosetta_audit.classify ----------------------------------------------------------------- + + +def test_classify_pass_is_clean_and_error_is_broken(): + verdicts = rosetta_audit.classify({"go": rosetta_audit.PASS, "lua": rosetta_audit.ERROR}, None) + assert verdicts == {"go": rosetta_audit.CLEAN, "lua": rosetta_audit.BROKEN} + + +def test_classify_without_baseline_reports_failures_as_moved(): + verdicts = rosetta_audit.classify({"go": rosetta_audit.FAIL}, None) + assert verdicts == {"go": rosetta_audit.MOVED} + + +def test_classify_against_baseline_splits_regression_from_pre_existing_drift(): + current = {"go": rosetta_audit.FAIL, "yaml": rosetta_audit.FAIL, "tcl": rosetta_audit.FAIL, "c": rosetta_audit.PASS} + baseline = {"go": rosetta_audit.PASS, "yaml": rosetta_audit.FAIL, "tcl": rosetta_audit.ERROR} + verdicts = rosetta_audit.classify(current, baseline) + assert verdicts["go"] == rosetta_audit.REGRESSION # fails here, passed on main: this build moved it + assert verdicts["yaml"] == rosetta_audit.PRE_EXISTING # fails on both: corpus is behind main + assert verdicts["tcl"] == rosetta_audit.BROKEN # the baseline run itself crashed + assert verdicts["c"] == rosetta_audit.CLEAN + + +# --- rosetta_audit.main refuses to run on nothing -------------------------------------------- + + +def test_rosetta_audit_exits_2_without_a_corpus(tmp_path): + assert rosetta_audit.main(["--corpus", str(tmp_path)]) == 2 + + +def test_rosetta_audit_exits_2_with_zero_language_folders(tmp_path, monkeypatch): + (tmp_path / "tools").mkdir() + (tmp_path / "tools" / "verify_language.py").write_text("raise SystemExit(0)\n") + (tmp_path / "data").mkdir() + monkeypatch.setenv("GALAXYSCOPE_BIN", sys.executable) # any existing file; never reached + assert rosetta_audit.main(["--corpus", str(tmp_path)]) == 2 + + +def test_rosetta_audit_exits_2_without_galaxyscope(tmp_path, monkeypatch): + (tmp_path / "tools").mkdir() + (tmp_path / "tools" / "verify_language.py").write_text("raise SystemExit(0)\n") + (tmp_path / "data" / "go").mkdir(parents=True) + (tmp_path / "data" / "go" / "expected_signals.json").write_text("{}") + monkeypatch.delenv("GALAXYSCOPE_BIN", raising=False) + monkeypatch.setenv("PATH", str(tmp_path)) # nothing on it + assert rosetta_audit.main(["--corpus", str(tmp_path)]) == 2 + + +def _fake_corpus(tmp_path: Path, script: str, languages=("go",)) -> Path: + (tmp_path / "tools").mkdir() + (tmp_path / "tools" / "verify_language.py").write_text(script) + for lang in languages: + (tmp_path / "data" / lang).mkdir(parents=True) + (tmp_path / "data" / lang / "expected_signals.json").write_text("{}") + return tmp_path + + +def test_rosetta_audit_verifier_crash_is_broken_not_fail(tmp_path, monkeypatch): + # Nonzero exit with no PASS/FAIL line = infrastructure, never "corpus drift". + corpus = _fake_corpus(tmp_path, "import sys; print('Traceback: boom'); sys.exit(1)\n") + monkeypatch.setenv("GALAXYSCOPE_BIN", sys.executable) + monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) + assert rosetta_audit.main(["--corpus", str(corpus)]) == 2 + + +def test_rosetta_audit_fail_without_baseline_exits_1_and_allow_flag_exits_0(tmp_path, monkeypatch): + corpus = _fake_corpus(tmp_path, "import sys; print(f'FAIL {sys.argv[1]}: 1 mismatch(es)'); sys.exit(1)\n") + monkeypatch.setenv("GALAXYSCOPE_BIN", sys.executable) + monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) + assert rosetta_audit.main(["--corpus", str(corpus)]) == 1 + assert rosetta_audit.main(["--corpus", str(corpus), "--allow-regressions"]) == 0 + + +def test_rosetta_audit_pre_existing_drift_exits_0(tmp_path, monkeypatch): + # The fake verifier fails under the "current" binary and ALSO under the baseline binary: + # the corpus is behind engine main, which is not this build's doing. + script = "import os, sys; print(f'FAIL {sys.argv[1]}: 1 mismatch(es)'); sys.exit(1)\n" + corpus = _fake_corpus(tmp_path, script) + monkeypatch.setenv("GALAXYSCOPE_BIN", sys.executable) + monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) + assert rosetta_audit.main(["--corpus", str(corpus), "--baseline-bin", sys.executable]) == 0 + + +def test_rosetta_audit_regression_against_baseline_exits_1(tmp_path, monkeypatch): + # Passes when GALAXYSCOPE_BIN is the baseline marker, fails otherwise. + baseline_marker = tmp_path / "baseline-galaxyscope" + baseline_marker.write_text("") + script = ( + "import os, sys\n" + "lang = sys.argv[1]\n" + f"if os.environ['GALAXYSCOPE_BIN'] == {str(baseline_marker)!r}:\n" + " print(f'PASS {lang}: 1 assertions across 1 files'); sys.exit(0)\n" + "print(f'FAIL {lang}: 1 mismatch(es)'); sys.exit(1)\n" + ) + corpus = _fake_corpus(tmp_path, script) + monkeypatch.setenv("GALAXYSCOPE_BIN", sys.executable) + monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) + assert rosetta_audit.main(["--corpus", str(corpus), "--baseline-bin", str(baseline_marker)]) == 1 + + +def test_rosetta_audit_writes_step_summary(tmp_path, monkeypatch): + corpus = _fake_corpus(tmp_path, "import sys; print(f'PASS {sys.argv[1]}: 1 assertions across 1 files')\n") + summary = tmp_path / "summary.md" + monkeypatch.setenv("GALAXYSCOPE_BIN", sys.executable) + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + assert rosetta_audit.main(["--corpus", str(corpus)]) == 0 + assert "1 languages checked" in summary.read_text() + + +# --- the two baseline-gated accuracy runners ------------------------------------------------ + + +def test_tri_comparison_all_ci_refuses_zero_languages(): + import tri_comparison_chart + + assert tri_comparison_chart.run_all_baseline_mode([], lambda lang, verbose: 0) == 1 + + +def test_tri_comparison_ci_fails_closed_when_a_baselined_language_has_no_data(monkeypatch): + import tri_comparison_chart + + class NoData: + has_data = False + awaiting_note = "gather failed: galaxyscope not found on PATH" + + monkeypatch.setattr(tri_comparison_chart, "run_pipeline", lambda langs, verbose=True: {langs[0]: NoData()}) + assert tri_comparison_chart.run_ci_check("python", verbose=False) == 1 + + +@pytest.mark.skipif( + subprocess.run( + [sys.executable, "-c", "import tree_sitter_language_pack"], capture_output=True, env=os.environ + ).returncode + != 0, + reason="tree_sitter_language_pack not installed in this interpreter", +) +def test_tree_sitter_all_refuses_zero_languages(monkeypatch): + import tree_sitter_accuracy_audit + + monkeypatch.setattr(tree_sitter_accuracy_audit, "_all_baseline_langs", lambda: []) + assert tree_sitter_accuracy_audit.run_all(lambda lang: 0) == 1 diff --git a/tests/tools/tree_sitter_accuracy_audit.py b/tests/tools/tree_sitter_accuracy_audit.py index 22e2a0d01..fcfecbf28 100755 --- a/tests/tools/tree_sitter_accuracy_audit.py +++ b/tests/tools/tree_sitter_accuracy_audit.py @@ -2928,6 +2928,10 @@ def run_all(mode_fn) -> int: """Runs a single-language mode function (run_ci_check/run_regenerate/run_full_report) across every baselined language, in one process. See `_all_baseline_langs`.""" langs = _all_baseline_langs() + if not langs: + # "0 languages checked, all OK" is a failure wearing a pass (#2682). + print("tree_sitter_accuracy_audit --all: no committed baselines found -- refusing to report success.") + return 1 failed = [] for lang in langs: print(f"\n=== {lang} ===") diff --git a/tests/tools/tri_comparison_chart.py b/tests/tools/tri_comparison_chart.py index 74d6940bb..9f36388fb 100644 --- a/tests/tools/tri_comparison_chart.py +++ b/tests/tools/tri_comparison_chart.py @@ -331,6 +331,10 @@ def run_pipeline(languages: list[str], verbose: bool = True) -> dict[str, Langua try: results = gather_language(lang) except SystemExit as e: + # Remember WHY, not just that it failed: run_ci_check must be able to tell a + # missing corpus from a scan that could not run at all (#2682 -- with galaxyscope + # off PATH this used to skip every language and still print "all OK"). + data.awaiting_note = f"gather failed: {e}" if verbose: print(f"tri_comparison_chart: {lang} -- gather failed ({e}), skipping") data.has_data = False @@ -958,8 +962,15 @@ def _regressions(current: dict, baseline: dict) -> list[str]: def run_ci_check(lang: str, verbose: bool = True) -> int: data = run_pipeline([lang], verbose=verbose)[lang] if not data.has_data: - print(f"tri_comparison_chart: {lang} -- no corpus data available, skipping.") - return 0 + # A baselined language with nothing to measure is a FAILURE in --ci, never a skip: + # the baseline proves corpus data existed when it was written, so "no data" here + # means the corpus is missing or galaxyscope could not run (#2682). Skipping used + # to turn either into a green "all OK" with zero languages actually checked. + print( + f"tri_comparison_chart: {lang} -- no corpus data available ({data.awaiting_note}); " + "a baselined language must be measurable in --ci, failing closed." + ) + return 1 current = _extract_precision(data) baseline = load_baseline(lang) @@ -1007,6 +1018,10 @@ def _all_baseline_langs() -> list[str]: def run_all_baseline_mode(languages: list[str], mode_fn, verbose: bool = True) -> int: + if not languages: + # "0 languages checked, all OK" is a failure wearing a pass (#2682). + print("tri_comparison_chart --ci: no languages to check -- refusing to report success.") + return 1 failed = [] for lang in languages: print(f"\n=== {lang} ===")