diff --git a/.github/scripts/test_floor.py b/.github/scripts/test_floor.py new file mode 100644 index 00000000..a148d326 --- /dev/null +++ b/.github/scripts/test_floor.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Fail CI when the test suite quietly shrinks or a file's tests all skip. + +sable-d2x2 detection rule C ("assert non-emptiness"). Two ways a green run can +be vacuous that the runner's own exit code does not catch: + + 1. A whole file's tests are skipped. sable-cazq: 40 parity tests were + `describe.skip`'d because Python was missing, and the release path exited + 0. A total-count floor does NOT catch this (2112 - 40 is still a big + number); a per-file "every test skipped" rule does. + 2. The suite shrinks sharply: a config change, a renamed directory, a broken + glob, and the runner cheerfully runs the 30 tests it found. + +Reads a vitest JSON report (--vitest) or a pytest JUnit XML written with +`-o junit_family=xunit1` (--junit; xunit1 is what carries the per-test `file` +attribute). Exits 1 when: + + * executed tests (passed + failed) < --min-executed, or + * any file has >= 1 test and every one of them was skipped, unless that + file is listed in --allow-all-skipped (a visible, reviewed exception). + +Always writes the executed / skipped counts and every skipped test's name to +$GITHUB_STEP_SUMMARY when set, so a skip is never invisible even when it is +allowed. Stdlib only: this runs before any project dependency is guaranteed. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import xml.etree.ElementTree as ET +from collections import defaultdict + +SKIPPED_STATES = {"skipped", "pending", "todo", "disabled"} + + +def read_vitest(path: str) -> dict[str, dict[str, list[str]]]: + """{file: {"executed": [names], "skipped": [names]}} from a vitest JSON report.""" + with open(path, encoding="utf-8") as fh: + report = json.load(fh) + files: dict[str, dict[str, list[str]]] = {} + cwd = os.getcwd() + os.sep + for result in report.get("testResults", []): + name = result.get("name", "") + rel = name[len(cwd):] if name.startswith(cwd) else name + bucket = files.setdefault(rel, {"executed": [], "skipped": []}) + for case in result.get("assertionResults", []): + title = case.get("fullName") or case.get("title") or "" + state = case.get("status", "") + (bucket["skipped"] if state in SKIPPED_STATES else bucket["executed"]).append(title) + return files + + +def read_junit(path: str) -> dict[str, dict[str, list[str]]]: + """Same shape from a pytest JUnit XML (xunit1 family, which carries `file`).""" + root = ET.parse(path).getroot() + files: dict[str, dict[str, list[str]]] = defaultdict(lambda: {"executed": [], "skipped": []}) + missing_file_attr = 0 + for case in root.iter("testcase"): + file_attr = case.get("file") + if not file_attr: + missing_file_attr += 1 + # xunit2 drops `file`; fall back to the module part of classname so + # the per-file rule still has something to group by. + classname = case.get("classname", "") + parts = [p for p in classname.split(".") if p and not p[:1].isupper()] + file_attr = "/".join(parts) + ".py" if parts else "" + title = f'{case.get("classname", "")}::{case.get("name", "")}' + skipped = case.find("skipped") is not None + (files[file_attr]["skipped"] if skipped else files[file_attr]["executed"]).append(title) + if missing_file_attr: + print( + f"::warning::{missing_file_attr} testcase(s) had no `file` attribute; " + "run pytest with `-o junit_family=xunit1` for exact per-file grouping.", + flush=True, + ) + return dict(files) + + +def summarize(label: str, files: dict[str, dict[str, list[str]]], executed: int, + skipped: int, min_executed: int, all_skipped: list[str], + allowed: set[str]) -> str: + lines = [f"### Test floor — {label}", ""] + lines.append("| Executed | Skipped | Floor | Files |") + lines.append("|---------:|--------:|------:|------:|") + lines.append(f"| {executed} | {skipped} | {min_executed} | {len(files)} |") + lines.append("") + if all_skipped: + lines.append("**Files with every test skipped:**") + for f in all_skipped: + tag = " (allowed by --allow-all-skipped)" if f in allowed else " **← FAIL**" + lines.append(f"- `{f}`{tag}") + lines.append("") + if skipped: + lines.append("
Skipped tests") + lines.append("") + for f, bucket in sorted(files.items()): + for name in bucket["skipped"]: + lines.append(f"- `{f}` — {name}") + lines.append("") + lines.append("
") + lines.append("") + return "\n".join(lines) + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("--vitest", help="vitest JSON report (--reporter=json)") + src.add_argument("--junit", help="pytest JUnit XML (-o junit_family=xunit1 --junitxml=...)") + ap.add_argument("--min-executed", type=int, required=True, + help="fail if fewer than this many tests actually ran (passed + failed)") + ap.add_argument("--allow-all-skipped", default="", + help="comma-separated files allowed to have every test skipped") + ap.add_argument("--label", default=None, help="label for the step summary") + args = ap.parse_args(argv) + + if args.vitest: + files = read_vitest(args.vitest) + label = args.label or "vitest" + else: + files = read_junit(args.junit) + label = args.label or "pytest" + + allowed = {f.strip() for f in args.allow_all_skipped.split(",") if f.strip()} + executed = sum(len(b["executed"]) for b in files.values()) + skipped = sum(len(b["skipped"]) for b in files.values()) + all_skipped = sorted(f for f, b in files.items() if b["skipped"] and not b["executed"]) + offending = [f for f in all_skipped if f not in allowed] + + summary = summarize(label, files, executed, skipped, args.min_executed, all_skipped, allowed) + print(summary, flush=True) + step_summary = os.environ.get("GITHUB_STEP_SUMMARY") + if step_summary: + with open(step_summary, "a", encoding="utf-8") as fh: + fh.write(summary + "\n") + + failed = False + if not files: + print(f"::error::{label}: the report lists no test files at all — nothing ran.", flush=True) + failed = True + if executed < args.min_executed: + print( + f"::error::{label}: only {executed} tests executed, floor is {args.min_executed}. " + "If tests were deliberately removed, lower the floor in the workflow in the same PR " + "so the shrink is a reviewed decision, not a silent one.", + flush=True, + ) + failed = True + for f in offending: + print( + f"::error::{label}: every test in {f} was skipped ({len(files[f]['skipped'])} tests). " + "A file that runs nothing is a broken prerequisite, not a passing file. Fix the " + "prerequisite, or list the file in --allow-all-skipped with a reason in the workflow.", + flush=True, + ) + failed = True + if not failed: + allowed_note = ( + f", {len(all_skipped)} fully-skipped file(s) on the allow-list" if all_skipped + else ", no file fully skipped" + ) + print(f"OK: {label}: {executed} executed (floor {args.min_executed}), " + f"{skipped} skipped{allowed_note}.", flush=True) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 86e67dfb..bd8ed3da 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -49,7 +49,13 @@ jobs: pip install -e ".[dev]" 2>/dev/null || pip install -e . - name: Run tests - run: pnpm test + run: pnpm exec vitest run --reporter=default --reporter=json --outputFile.json=vitest-report.json + + # sable-d2x2 rule C — the release path is exactly where sable-cazq's 40 + # silently-skipped tests reported green. Same floor as + # test-comprehensive.yml; keep the two in step. + - name: Assert the suite actually ran + run: python3 ../.github/scripts/test_floor.py --vitest vitest-report.json --min-executed 1990 --label "release test-node (vitest)" # sable-cazq — python/tests/ ran in exactly one place in this repo # (test-comprehensive.yml) and it was not the release path. publish.yaml @@ -80,10 +86,14 @@ jobs: pip install pytest pytest-mock pytest-asyncio - name: Run all tests - run: python -m pytest tests/ -v + run: python -m pytest tests/ -v -o junit_family=xunit1 --junitxml=pytest-report.xml env: RAFTER_API_KEY: ${{ secrets.RAFTER_API_KEY }} + # sable-d2x2 rule C — same floor as test-comprehensive.yml; keep in step. + - name: Assert the suite actually ran + run: python3 ../.github/scripts/test_floor.py --junit pytest-report.xml --min-executed 1530 --label "release test-python (pytest)" + test-package: runs-on: ubuntu-latest defaults: diff --git a/.github/workflows/test-comprehensive.yml b/.github/workflows/test-comprehensive.yml index 5fd6c4ba..aa30ddac 100644 --- a/.github/workflows/test-comprehensive.yml +++ b/.github/workflows/test-comprehensive.yml @@ -26,6 +26,14 @@ jobs: run: ${{ steps.decide.outputs.run }} run_core: ${{ steps.decide.outputs.run_core }} steps: + # Checked out only so the skip notice below can read the list of gated + # jobs from this file instead of carrying a copy of it. The first + # hand-typed copy omitted e2e-node within the hour it was written. + - uses: actions/checkout@v4 + with: + sparse-checkout: .github/workflows/test-comprehensive.yml + sparse-checkout-cone-mode: false + - id: decide # Values go through env rather than direct ${{ }} interpolation into # the script, so nothing from the PR can be shell-injected. @@ -53,7 +61,31 @@ jobs: echo "run=true" >> "$GITHUB_OUTPUT" elif [ "$HEAD_OWNER" = "Raftersecurity" ] || [ "$AUTHOR" = "Rome-1" ]; then echo "run=false" >> "$GITHUB_OUTPUT" - echo "Internal PR into main (author=$AUTHOR, head repo owner=$HEAD_OWNER) — unit tests still run; extended matrix skipped." >> "$GITHUB_STEP_SUMMARY" + # sable-d2x2 rule B — a skipped job renders as a grey check and + # satisfies a required status check, so the skip has to be said + # out loud, by name, where a reviewer looks: the checks annotation + # and the run summary. The list is READ FROM THIS FILE (every job + # whose `if:` is gated on needs.gate.outputs.run), never typed by + # hand, so it cannot drift from the jobs it describes. + SKIPPED_JOBS=$(awk ' + /^ [A-Za-z0-9_-]+:[[:space:]]*$/ { job=$1; sub(":", "", job) } + /^[[:space:]]*if:.*needs\.gate\.outputs\.run == .true./ { if (job != "" && !seen[job]++) print job } + ' .github/workflows/test-comprehensive.yml) + SKIPPED_CSV=$(printf '%s' "$SKIPPED_JOBS" | paste -sd ',' - | sed 's/,/, /g') + if [ -z "$SKIPPED_JOBS" ]; then + echo "::error::gate: found no jobs gated on needs.gate.outputs.run — the skip notice would be empty. Either the gate is now pointless or this awk no longer matches the file." + exit 1 + fi + echo "::warning::Internal PR into main: extended matrix NOT run — ${SKIPPED_CSV}. Unit tests (test-node, test-python) still run. To run everything, push to a branch and open the PR from a fork, or use workflow_dispatch." + { + echo "### Extended matrix skipped on this run" + echo "" + echo "Internal PR into main (author=\`$AUTHOR\`, head repo owner=\`$HEAD_OWNER\`). These jobs did **not** run and their grey checks mean *skipped*, not *passed*:" + echo "" + printf '%s\n' "$SKIPPED_JOBS" | sed 's/.*/- `&`/' + echo "" + echo "\`test-node\` and \`test-python\` ran. Trigger the full matrix with **workflow_dispatch** if this PR touches packaging, SARIF output, secret patterns, or platform-specific code." + } >> "$GITHUB_STEP_SUMMARY" else echo "run=true" >> "$GITHUB_OUTPUT" fi @@ -100,10 +132,20 @@ jobs: run: node ./dist/index.js --version - name: Run all tests - run: pnpm test + # The JSON report feeds the floor check below; the default reporter + # keeps the log readable. + run: pnpm exec vitest run --reporter=default --reporter=json --outputFile.json=vitest-report.json env: RAFTER_API_KEY: ${{ secrets.RAFTER_API_KEY }} + # sable-d2x2 rule C — a green run must have RUN something. Fails if the + # executed count falls below the floor or any file's tests all skipped + # (sable-cazq: 40 parity tests describe.skip'd, exit 0). The floor is + # ~95% of the count on 2026-09-02 (2099 executed); lower it in the same + # PR that removes tests, so a shrink is a reviewed decision. + - name: Assert the suite actually ran + run: python3 ../.github/scripts/test_floor.py --vitest vitest-report.json --min-executed 1990 --label "test-node (vitest)" + test-python: needs: gate if: needs.gate.outputs.run_core == 'true' @@ -124,10 +166,17 @@ jobs: pip install pytest pytest-mock pytest-asyncio - name: Run all tests - run: python -m pytest tests/ -v + # xunit1 is the JUnit family that records `file` per test case, which + # the floor check below groups by. + run: python -m pytest tests/ -v -o junit_family=xunit1 --junitxml=pytest-report.xml env: RAFTER_API_KEY: ${{ secrets.RAFTER_API_KEY }} + # sable-d2x2 rule C — see test-node. Floor is ~95% of 1614 executed on + # 2026-09-02. + - name: Assert the suite actually ran + run: python3 ../.github/scripts/test_floor.py --junit pytest-report.xml --min-executed 1530 --label "test-python (pytest)" + # ── E2E CLI tests ───────────────────────────────────────────────── e2e-node: needs: gate