ci: fail when the suite shrinks or a file's tests all skip; name skipped jobs (sable-5d3q, sable-wl3p)

.github/scripts/test_floor.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""Fail CI when the test suite quietly shrinks or a file's tests all skip.
+
+sable-d2x2 detection rule C ("assert non-emptiness"). Two ways a green run can
+be vacuous that the runner's own exit code does not catch:
+
+  1. A whole file's tests are skipped. sable-cazq: 40 parity tests were
+     `describe.skip`'d because Python was missing, and the release path exited
+     0. A total-count floor does NOT catch this (2112 - 40 is still a big
+     number); a per-file "every test skipped" rule does.
+  2. The suite shrinks sharply: a config change, a renamed directory, a broken
+     glob, and the runner cheerfully runs the 30 tests it found.
+
+Reads a vitest JSON report (--vitest) or a pytest JUnit XML written with
+`-o junit_family=xunit1` (--junit; xunit1 is what carries the per-test `file`
+attribute). Exits 1 when:
+
+  * executed tests (passed + failed) < --min-executed, or
+  * any file has >= 1 test and every one of them was skipped, unless that
+    file is listed in --allow-all-skipped (a visible, reviewed exception).
+
+Always writes the executed / skipped counts and every skipped test's name to
+$GITHUB_STEP_SUMMARY when set, so a skip is never invisible even when it is
+allowed. Stdlib only: this runs before any project dependency is guaranteed.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+
+SKIPPED_STATES = {"skipped", "pending", "todo", "disabled"}
+
+
+def read_vitest(path: str) -> dict[str, dict[str, list[str]]]:
+    """{file: {"executed": [names], "skipped": [names]}} from a vitest JSON report."""
+    with open(path, encoding="utf-8") as fh:
+        report = json.load(fh)
+    files: dict[str, dict[str, list[str]]] = {}
+    cwd = os.getcwd() + os.sep
+    for result in report.get("testResults", []):
+        name = result.get("name", "")
+        rel = name[len(cwd):] if name.startswith(cwd) else name
+        bucket = files.setdefault(rel, {"executed": [], "skipped": []})
+        for case in result.get("assertionResults", []):
+            title = case.get("fullName") or case.get("title") or "<unnamed>"
+            state = case.get("status", "")
+            (bucket["skipped"] if state in SKIPPED_STATES else bucket["executed"]).append(title)
+    return files
+
+
+def read_junit(path: str) -> dict[str, dict[str, list[str]]]:
+    """Same shape from a pytest JUnit XML (xunit1 family, which carries `file`)."""
+    root = ET.parse(path).getroot()
+    files: dict[str, dict[str, list[str]]] = defaultdict(lambda: {"executed": [], "skipped": []})
+    missing_file_attr = 0
+    for case in root.iter("testcase"):
+        file_attr = case.get("file")
+        if not file_attr:
+            missing_file_attr += 1
+            # xunit2 drops `file`; fall back to the module part of classname so
+            # the per-file rule still has something to group by.
+            classname = case.get("classname", "")
+            parts = [p for p in classname.split(".") if p and not p[:1].isupper()]
+            file_attr = "/".join(parts) + ".py" if parts else "<unknown>"
+        title = f'{case.get("classname", "")}::{case.get("name", "")}'
+        skipped = case.find("skipped") is not None
+        (files[file_attr]["skipped"] if skipped else files[file_attr]["executed"]).append(title)
+    if missing_file_attr:
+        print(
+            f"::warning::{missing_file_attr} testcase(s) had no `file` attribute; "
+            "run pytest with `-o junit_family=xunit1` for exact per-file grouping.",
+            flush=True,
+        )
+    return dict(files)
+
+
+def summarize(label: str, files: dict[str, dict[str, list[str]]], executed: int,
+              skipped: int, min_executed: int, all_skipped: list[str],
+              allowed: set[str]) -> str:
+    lines = [f"### Test floor — {label}", ""]
+    lines.append("| Executed | Skipped | Floor | Files |")
+    lines.append("|---------:|--------:|------:|------:|")
+    lines.append(f"| {executed} | {skipped} | {min_executed} | {len(files)} |")
+    lines.append("")
+    if all_skipped:
+        lines.append("**Files with every test skipped:**")
+        for f in all_skipped:
+            tag = " (allowed by --allow-all-skipped)" if f in allowed else " **← FAIL**"
+            lines.append(f"- `{f}`{tag}")
+        lines.append("")
+    if skipped:
+        lines.append("<details><summary>Skipped tests</summary>")
+        lines.append("")
+        for f, bucket in sorted(files.items()):
+            for name in bucket["skipped"]:
+                lines.append(f"- `{f}` — {name}")
+        lines.append("")
+        lines.append("</details>")
+        lines.append("")
+    return "\n".join(lines)
+
+
+def main(argv: list[str]) -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    src = ap.add_mutually_exclusive_group(required=True)
+    src.add_argument("--vitest", help="vitest JSON report (--reporter=json)")
+    src.add_argument("--junit", help="pytest JUnit XML (-o junit_family=xunit1 --junitxml=...)")
+    ap.add_argument("--min-executed", type=int, required=True,
+                    help="fail if fewer than this many tests actually ran (passed + failed)")
+    ap.add_argument("--allow-all-skipped", default="",
+                    help="comma-separated files allowed to have every test skipped")
+    ap.add_argument("--label", default=None, help="label for the step summary")
+    args = ap.parse_args(argv)
+
+    if args.vitest:
+        files = read_vitest(args.vitest)
+        label = args.label or "vitest"
+    else:
+        files = read_junit(args.junit)
+        label = args.label or "pytest"
+
+    allowed = {f.strip() for f in args.allow_all_skipped.split(",") if f.strip()}
+    executed = sum(len(b["executed"]) for b in files.values())
+    skipped = sum(len(b["skipped"]) for b in files.values())
+    all_skipped = sorted(f for f, b in files.items() if b["skipped"] and not b["executed"])
+    offending = [f for f in all_skipped if f not in allowed]
+
+    summary = summarize(label, files, executed, skipped, args.min_executed, all_skipped, allowed)
+    print(summary, flush=True)
+    step_summary = os.environ.get("GITHUB_STEP_SUMMARY")
+    if step_summary:
+        with open(step_summary, "a", encoding="utf-8") as fh:
+            fh.write(summary + "\n")
+
+    failed = False
+    if not files:
+        print(f"::error::{label}: the report lists no test files at all — nothing ran.", flush=True)
+        failed = True
+    if executed < args.min_executed:
+        print(
+            f"::error::{label}: only {executed} tests executed, floor is {args.min_executed}. "
+            "If tests were deliberately removed, lower the floor in the workflow in the same PR "
+            "so the shrink is a reviewed decision, not a silent one.",
+            flush=True,
+        )
+        failed = True
+    for f in offending:
+        print(
+            f"::error::{label}: every test in {f} was skipped ({len(files[f]['skipped'])} tests). "
+            "A file that runs nothing is a broken prerequisite, not a passing file. Fix the "
+            "prerequisite, or list the file in --allow-all-skipped with a reason in the workflow.",
+            flush=True,
+        )
+        failed = True
+    if not failed:
+        allowed_note = (
+            f", {len(all_skipped)} fully-skipped file(s) on the allow-list" if all_skipped
+            else ", no file fully skipped"
+        )
+        print(f"OK: {label}: {executed} executed (floor {args.min_executed}), "
+              f"{skipped} skipped{allowed_note}.", flush=True)
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

.github/workflows/publish.yaml

@@ -49,7 +49,13 @@ jobs:
           pip install -e ".[dev]" 2>/dev/null || pip install -e .
 
       - name: Run tests
-        run: pnpm test
+        run: pnpm exec vitest run --reporter=default --reporter=json --outputFile.json=vitest-report.json
+
+      # sable-d2x2 rule C — the release path is exactly where sable-cazq's 40
+      # silently-skipped tests reported green. Same floor as
+      # test-comprehensive.yml; keep the two in step.
+      - name: Assert the suite actually ran
+        run: python3 ../.github/scripts/test_floor.py --vitest vitest-report.json --min-executed 1990 --label "release test-node (vitest)"
 
   # sable-cazq — python/tests/ ran in exactly one place in this repo
   # (test-comprehensive.yml) and it was not the release path. publish.yaml
@@ -80,10 +86,14 @@ jobs:
           pip install pytest pytest-mock pytest-asyncio
 
       - name: Run all tests
-        run: python -m pytest tests/ -v
+        run: python -m pytest tests/ -v -o junit_family=xunit1 --junitxml=pytest-report.xml
         env:
           RAFTER_API_KEY: ${{ secrets.RAFTER_API_KEY }}
 
+      # sable-d2x2 rule C — same floor as test-comprehensive.yml; keep in step.
+      - name: Assert the suite actually ran
+        run: python3 ../.github/scripts/test_floor.py --junit pytest-report.xml --min-executed 1530 --label "release test-python (pytest)"
+
   test-package:
     runs-on: ubuntu-latest
     defaults:

.github/workflows/test-comprehensive.yml

@@ -26,6 +26,14 @@ jobs:
       run: ${{ steps.decide.outputs.run }}
       run_core: ${{ steps.decide.outputs.run_core }}
     steps:
+      # Checked out only so the skip notice below can read the list of gated
+      # jobs from this file instead of carrying a copy of it. The first
+      # hand-typed copy omitted e2e-node within the hour it was written.
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/workflows/test-comprehensive.yml
+          sparse-checkout-cone-mode: false
+
       - id: decide
         # Values go through env rather than direct ${{ }} interpolation into
         # the script, so nothing from the PR can be shell-injected.
@@ -53,7 +61,31 @@ jobs:
             echo "run=true" >> "$GITHUB_OUTPUT"
           elif [ "$HEAD_OWNER" = "Raftersecurity" ] || [ "$AUTHOR" = "Rome-1" ]; then
             echo "run=false" >> "$GITHUB_OUTPUT"
-            echo "Internal PR into main (author=$AUTHOR, head repo owner=$HEAD_OWNER) — unit tests still run; extended matrix skipped." >> "$GITHUB_STEP_SUMMARY"
+            # sable-d2x2 rule B — a skipped job renders as a grey check and
+            # satisfies a required status check, so the skip has to be said
+            # out loud, by name, where a reviewer looks: the checks annotation
+            # and the run summary. The list is READ FROM THIS FILE (every job
+            # whose `if:` is gated on needs.gate.outputs.run), never typed by
+            # hand, so it cannot drift from the jobs it describes.
+            SKIPPED_JOBS=$(awk '
+              /^  [A-Za-z0-9_-]+:[[:space:]]*$/ { job=$1; sub(":", "", job) }
+              /^[[:space:]]*if:.*needs\.gate\.outputs\.run == .true./ { if (job != "" && !seen[job]++) print job }
+            ' .github/workflows/test-comprehensive.yml)
+            SKIPPED_CSV=$(printf '%s' "$SKIPPED_JOBS" | paste -sd ',' - | sed 's/,/, /g')
+            if [ -z "$SKIPPED_JOBS" ]; then
+              echo "::error::gate: found no jobs gated on needs.gate.outputs.run — the skip notice would be empty. Either the gate is now pointless or this awk no longer matches the file."
+              exit 1
+            fi
+            echo "::warning::Internal PR into main: extended matrix NOT run — ${SKIPPED_CSV}. Unit tests (test-node, test-python) still run. To run everything, push to a branch and open the PR from a fork, or use workflow_dispatch."
+            {
+              echo "### Extended matrix skipped on this run"
+              echo ""
+              echo "Internal PR into main (author=\`$AUTHOR\`, head repo owner=\`$HEAD_OWNER\`). These jobs did **not** run and their grey checks mean *skipped*, not *passed*:"
+              echo ""
+              printf '%s\n' "$SKIPPED_JOBS" | sed 's/.*/- `&`/'
+              echo ""
+              echo "\`test-node\` and \`test-python\` ran. Trigger the full matrix with **workflow_dispatch** if this PR touches packaging, SARIF output, secret patterns, or platform-specific code."
+            } >> "$GITHUB_STEP_SUMMARY"
           else
             echo "run=true" >> "$GITHUB_OUTPUT"
           fi
@@ -100,10 +132,20 @@ jobs:
         run: node ./dist/index.js --version
 
       - name: Run all tests
-        run: pnpm test
+        # The JSON report feeds the floor check below; the default reporter
+        # keeps the log readable.
+        run: pnpm exec vitest run --reporter=default --reporter=json --outputFile.json=vitest-report.json
         env:
           RAFTER_API_KEY: ${{ secrets.RAFTER_API_KEY }}
 
+      # sable-d2x2 rule C — a green run must have RUN something. Fails if the
+      # executed count falls below the floor or any file's tests all skipped
+      # (sable-cazq: 40 parity tests describe.skip'd, exit 0). The floor is
+      # ~95% of the count on 2026-09-02 (2099 executed); lower it in the same
+      # PR that removes tests, so a shrink is a reviewed decision.
+      - name: Assert the suite actually ran
+        run: python3 ../.github/scripts/test_floor.py --vitest vitest-report.json --min-executed 1990 --label "test-node (vitest)"
+
   test-python:
     needs: gate
     if: needs.gate.outputs.run_core == 'true'
@@ -124,10 +166,17 @@ jobs:
           pip install pytest pytest-mock pytest-asyncio
 
       - name: Run all tests
-        run: python -m pytest tests/ -v
+        # xunit1 is the JUnit family that records `file` per test case, which
+        # the floor check below groups by.
+        run: python -m pytest tests/ -v -o junit_family=xunit1 --junitxml=pytest-report.xml
         env:
           RAFTER_API_KEY: ${{ secrets.RAFTER_API_KEY }}
 
+      # sable-d2x2 rule C — see test-node. Floor is ~95% of 1614 executed on
+      # 2026-09-02.
+      - name: Assert the suite actually ran
+        run: python3 ../.github/scripts/test_floor.py --junit pytest-report.xml --min-executed 1530 --label "test-python (pytest)"
+
   # ── E2E CLI tests ─────────────────────────────────────────────────
   e2e-node:
     needs: gate