diff --git a/benchmaxxing/degeneracy.py b/benchmaxxing/degeneracy.py index 367470b..37e43b3 100644 --- a/benchmaxxing/degeneracy.py +++ b/benchmaxxing/degeneracy.py @@ -6,8 +6,8 @@ structurally zero, hardcoded significance verdicts. Every one of them reached the paper because nothing looked for them. -This module is what looks for them. It reads the committed artifacts, not the prose about them, so -a comment cannot satisfy it. Three screens: +This module is what looks for them. It reads the on-disk artifacts under the screened paths, not the +prose about them, so a comment cannot satisfy it. Three screens: `constant_columns` Every committed per-case file under `experiments/*/results/**` (`.jsonl`, and the de-identified @@ -42,8 +42,9 @@ `referee_deployable.py`, is character-for-character equal to `adopted` on all 40 rows, and both vary. Column-wise constancy cannot see that. Neither can it see a predicate that reduces to the label through an intermediate variable. -- **Uncommitted artifacts.** The screen reads what is in git. The eight SUPPORT2 summary p-values - named in #374 are not in the tree, so nothing here reports them. +- **Artifacts that are not on disk under the screened paths.** The eight SUPPORT2 summary p-values + named in #374 were never written under `experiments/*/results/`, so nothing here reports them. + Untracked files that *are* on disk are screened (#422); only absence from the filesystem hides them. - **Non-Python report generators**, and verdicts assembled by concatenating variables rather than by literal text. @@ -120,8 +121,20 @@ def __str__(self) -> str: # ----------------------------------------------------------------------------- file discovery +def _glob(root: Path, pattern: str) -> list[Path]: + """Filesystem match for a git-style ``**.ext`` pathspec under ``root``.""" + return sorted(root.glob(pattern.replace("**.", "**/*."))) + + def _tracked(root: Path, pattern: str) -> list[Path]: - """Committed files matching a git pathspec, or a glob when `root` is not a repo (tests).""" + """Files matching ``pattern``: git-tracked when available, unioned with a filesystem glob. + + ``git ls-files`` alone is blind to untracked results (#422): a new arm can pass the guard while + unstaged and fail the moment it is added. The glob covers those files and non-repo roots + (pytest ``tmp_path``). Union, not replace-on-empty, so a repo that already has tracked matches + still sees newly written siblings beside them. + """ + globbed = _glob(root, pattern) try: out = subprocess.run( ["git", "-C", str(root), "ls-files", "-z", pattern], @@ -129,16 +142,18 @@ def _tracked(root: Path, pattern: str) -> list[Path]: text=True, check=True, ).stdout - return [root / p for p in out.split("\0") if p] + tracked = [root / p for p in out.split("\0") if p] except (subprocess.CalledProcessError, FileNotFoundError): - return sorted(root.glob(pattern.replace("**.", "**/*."))) + return globbed + return sorted(set(tracked) | set(globbed)) def per_case_files(root: Path) -> list[Path]: - """Committed per-case artifacts: every results jsonl, plus the de-identified csv exports. + """Per-case artifacts on disk: every results jsonl, plus the de-identified csv exports. Only `deid/` csv is included. The other committed csv under results/ are cohort manifests, where - a constant label column is the point rather than a defect. + a constant label column is the point rather than a defect. Untracked files under the same + paths are included so an unstaged arm cannot sneak past the guard (#422). """ files = list(_tracked(root, "experiments/*/results/**.jsonl")) files += [p for p in _tracked(root, "experiments/*/results/**.csv") if "deid" in p.parts] diff --git a/tests/test_degeneracy_guard.py b/tests/test_degeneracy_guard.py index d41d687..f5ca53c 100644 --- a/tests/test_degeneracy_guard.py +++ b/tests/test_degeneracy_guard.py @@ -297,6 +297,38 @@ def test_a_synthetic_constant_scored_column_turns_the_screen_red(tmp_path): assert "constant at False" in found["control_adopt"].detail +def test_untracked_results_in_a_git_repo_are_still_screened(tmp_path): + """#422: ``git ls-files`` alone misses unstaged results; the guard must still see them. + + Non-repo ``tmp_path`` roots already fell back to a glob via exit 128. The live hole is a real + git tree where ``ls-files`` succeeds with an empty (or incomplete) list for the pathspec. + """ + import subprocess + + subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True) + subprocess.run(["git", "config", "user.email", "t@t"], cwd=tmp_path, check=True, capture_output=True) + subprocess.run(["git", "config", "user.name", "t"], cwd=tmp_path, check=True, capture_output=True) + (tmp_path / "README").write_text("init\n") + subprocess.run(["git", "add", "README"], cwd=tmp_path, check=True, capture_output=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=tmp_path, check=True, capture_output=True) + + rows = [{"case_id": f"c{i}", "control_adopt": 0} for i in range(MIN_ROWS + 4)] + planted = _write_cases(tmp_path, "untracked.jsonl", rows) + # Deliberately not ``git add``ed — this is the #422 failure mode. + status = subprocess.run( + ["git", "-C", str(tmp_path), "ls-files", "-z", "experiments/*/results/**.jsonl"], + capture_output=True, + text=True, + check=True, + ).stdout + assert not status.strip("\0"), "precondition: planted file must be untracked" + + found = {f.locus: f for f in constant_columns(tmp_path)} + assert "control_adopt" in found, ( + f"untracked {planted.name} was invisible to the guard, got {list(found)}" + ) + + def test_a_synthetic_constant_column_in_a_deid_csv_turns_the_screen_red(tmp_path): out = tmp_path / "experiments" / "synth" / "results" / "deid" out.mkdir(parents=True)