From 5f0a8da84e93c3de20853c0d02d62d5e766be001 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Mon, 3 Aug 2026 17:50:46 +0300 Subject: [PATCH 01/31] Add IntegrityInfo telemetry with run.json wiring and an INTEGRITY_MODE switch --- src/coder_eval/config.py | 13 ++- src/coder_eval/models/__init__.py | 10 +++ src/coder_eval/models/enums.py | 15 ++++ src/coder_eval/models/results.py | 116 +++++++++++++++++++++++- src/coder_eval/models/sandbox.py | 4 + src/coder_eval/reports_experiment.py | 13 +++ tests/test_models.py | 126 +++++++++++++++++++++++++++ tests/test_reports_experiment.py | 110 ++++++++++++++++++++++- 8 files changed, 404 insertions(+), 3 deletions(-) diff --git a/src/coder_eval/config.py b/src/coder_eval/config.py index c65bb1d6..16b3b135 100644 --- a/src/coder_eval/config.py +++ b/src/coder_eval/config.py @@ -13,7 +13,7 @@ from pydantic import AliasChoices, Field from pydantic_settings import BaseSettings, SettingsConfigDict -from coder_eval.models import AgentKind, ApiBackend +from coder_eval.models import AgentKind, ApiBackend, IntegrityMode # Application Insights connection string baked into the application so a fresh @@ -137,6 +137,17 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: gemini_api_key: str | None = None antigravity_model: str | None = None + # Run-integrity gate (see models.enums.IntegrityMode). Kill switch for the + # graded-material read pass in coder_eval.integrity. DETECT + # (default) records a verdict + findings on every row and changes no score; + # VOID additionally downgrades a tainted SUCCESS to FAILURE; OFF skips the + # pass. Flip to VOID once a nightly's DETECT findings have been reviewed. + # INTEGRITY_MODE is also in the docker env allowlist (models/sandbox.py) — + # without that the in-container Settings silently defaults and the gate + # differs between the tempdir and docker drivers (same bug class as the + # API_BACKEND note in isolation/docker_runner.py). + integrity_mode: IntegrityMode = IntegrityMode.DETECT + # Logging log_level: str = "INFO" # Default log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) log_to_file: bool = False # Whether to enable file logging diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index ad33fdfd..8e82a65e 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -58,6 +58,7 @@ AgentState, ApiBackend, FinalStatus, + IntegrityMode, PermissionMode, PreservationMode, ) @@ -120,6 +121,10 @@ EarlyStopReason, EvaluationResult, FailedRowSummary, + IntegrityFinding, + IntegrityFindingKind, + IntegrityInfo, + IntegrityVerdict, JudgeCriterionResult, JudgeTranscript, JudgeTranscriptToolCall, @@ -217,6 +222,7 @@ "AgentState", "ApiBackend", "FinalStatus", + "IntegrityMode", "PermissionMode", "PreservationMode", # Criteria @@ -304,6 +310,10 @@ "EarlyStopInfo", "EarlyStopReason", "EvaluationResult", + "IntegrityFinding", + "IntegrityFindingKind", + "IntegrityInfo", + "IntegrityVerdict", "SimulationTelemetry", "SuiteRollup", "TaskConfigRecord", diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py index 03afe885..76ee30b9 100644 --- a/src/coder_eval/models/enums.py +++ b/src/coder_eval/models/enums.py @@ -62,6 +62,21 @@ def icon(self) -> str: assert set(_STATUS_ICONS) == set(FinalStatus), "Missing icon for FinalStatus member" +class IntegrityMode(StrEnum): + """How the run-integrity pass acts on what it finds (``INTEGRITY_MODE``). + + The kill switch for the integrity gate. ``DETECT`` is the default so a + rollout can read real findings from ``run.json`` before any score is voided; + ``VOID`` additionally flips a tainted ``SUCCESS`` to ``FAILURE``. ``OFF`` + skips the pass entirely (verdict ``SKIPPED``), for bisecting a suspected + false positive without redeploying. + """ + + OFF = "off" + DETECT = "detect" + VOID = "void" + + class ApiBackend(StrEnum): """API backend for LLM calls.""" diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index e0f4f80b..195ec3dc 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -20,7 +20,7 @@ from coder_eval.models.agent_config import ResolvedAgentConfig from coder_eval.models.criteria import SuccessCriterion -from coder_eval.models.enums import FinalStatus +from coder_eval.models.enums import FinalStatus, IntegrityMode from coder_eval.models.limits import DEFAULT_STOP_EARLY_GATE_THRESHOLD from coder_eval.models.telemetry import ( CommandStatistics, @@ -487,6 +487,110 @@ class EarlyStopInfo(BaseModel): ) +class IntegrityVerdict(StrEnum): + """Whether a run's score can be trusted as a measurement of the agent. + + Deliberately NOT a ``FinalStatus`` member: the terminal-status set is closed + (two exhaustive maps in ``models/enums.py`` plus every consumer that + switches on it), and a tainted run already has a perfectly good terminal + status. Integrity is orthogonal telemetry, like ``EarlyStopReason``. + + ``INCONCLUSIVE`` is why ``IntegrityInfo`` is always populated rather than + ``None``-when-clean: "the scan could not see the whole transcript" is a + distinct, reportable state from "the scan saw everything and found nothing", + and only the second one licenses trusting the score. + """ + + CLEAN = "clean" + TAINTED = "tainted" + INCONCLUSIVE = "inconclusive" + SKIPPED = "skipped" + + +class IntegrityFindingKind(StrEnum): + """The class of integrity problem a finding records. + + One member today: every finding is a read of graded material. The kind is + carried on the finding anyway so a second check can be added without + reshaping the row key ``integrity_findings`` already ships. + """ + + GRADED_READ = "graded_read" + + +class IntegrityFinding(BaseModel): + """One concrete integrity problem, with enough context to adjudicate it. + + Carries the locating coordinates (iteration / command index / tool) so a + reviewer can go straight to the command in ``task.json`` rather than + re-deriving it, and a truncated ``evidence`` excerpt so an obvious false + positive is visible without opening the artifact at all. + """ + + kind: IntegrityFindingKind = Field(description="Which integrity check produced this finding.") + detail: str = Field(description="Human-readable statement of what was found.") + iteration: int | None = Field( + default=None, description="Orchestrator iteration the offending command belongs to (1-indexed)." + ) + command_index: int | None = Field( + default=None, description="0-based index of the command within that iteration's command list." + ) + tool_name: str | None = Field(default=None, description="Tool that issued the offending command.") + evidence: str | None = Field( + default=None, + description="Truncated excerpt of the matched command text (or an empty-signal note), for triage.", + ) + + +class IntegrityInfo(BaseModel): + """Run-integrity verdict: is this row a measurement, or an artifact of a leak? + + ALWAYS populated on ``EvaluationResult`` (unlike ``EarlyStopInfo``, whose + ``is not None`` doubles as its flag) because ``SKIPPED`` and ``INCONCLUSIVE`` + must be expressible and distinguishable from ``CLEAN``. Defaults describe a + run on which the pass never executed, so a legacy ``task.json`` with no + ``integrity`` key round-trips to exactly that. + + ``voided`` records that the gate downgraded the row. ``weighted_score`` is + deliberately left as computed: on a tainted row the high score IS the + diagnostic (it passed *because* it cheated), so erasing it would destroy the + evidence. + """ + + verdict: IntegrityVerdict = Field( + default=IntegrityVerdict.SKIPPED, description="Whether the row's score is trustworthy." + ) + mode: IntegrityMode = Field( + default=IntegrityMode.OFF, description="The INTEGRITY_MODE the pass ran under (off / detect / void)." + ) + voided: bool = Field( + default=False, + description="True when the gate downgraded a passing row because of a TAINTED verdict. " + + "Only ever set under mode=void; INCONCLUSIVE never voids.", + ) + findings: list[IntegrityFinding] = Field( + default_factory=list, description="Every problem the pass found, in discovery order." + ) + commands_scanned: int = Field( + default=0, ge=0, description="Agent commands the scan examined across all iterations." + ) + commands_without_parameters: int = Field( + default=0, + ge=0, + description="Commands whose parameters were empty, so their content could not be scanned. " + + "A high ratio means the scan was partially blind and forces INCONCLUSIVE.", + ) + subagent_recovery_incomplete: bool = Field( + default=False, + description="True when the agent reported that it failed to recover a sub-agent's inner tool " + + "calls, so those commands are absent from the transcript entirely. Forces INCONCLUSIVE.", + ) + notes: list[str] = Field( + default_factory=list, + description="Why the verdict is INCONCLUSIVE or SKIPPED, or which veto suppressed a finding.", + ) + + class EvaluationResult(BaseModel): """Complete result of a task evaluation.""" @@ -622,6 +726,16 @@ class EvaluationResult(BaseModel): ), ) + # Run-integrity telemetry. Always present (never None) so SKIPPED and + # INCONCLUSIVE are expressible; see IntegrityInfo. + integrity: IntegrityInfo = Field( + default_factory=IntegrityInfo, + description=( + "Whether this row is a measurement of the agent or an artifact of a leak: verdict, the " + "mode the pass ran under, whether the score was voided, and the findings behind it." + ), + ) + def calculate_weighted_score(self, criteria: list[SuccessCriterion]) -> None: """Calculate weighted average score from criterion results. diff --git a/src/coder_eval/models/sandbox.py b/src/coder_eval/models/sandbox.py index eb7b6dc1..9c10ecfe 100644 --- a/src/coder_eval/models/sandbox.py +++ b/src/coder_eval/models/sandbox.py @@ -256,6 +256,10 @@ class DockerDriverConfig(BaseModel): # selects the Gemini model when agent.model is unset. "GEMINI_API_KEY", "ANTIGRAVITY_MODEL", + # Run-integrity gate mode. Without it the in-container Settings falls + # back to its own default, so the gate would differ between the + # tempdir and docker drivers on the same run. + "INTEGRITY_MODE", # User HOME used to keep ~/.claude resolution symmetric with the host. # See docs/DOCKER_ISOLATION.md "HOME is forwarded by default" for the # contract. tl;dr: Path.home() inside the container returns the diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index e141174b..c906ad66 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -44,6 +44,11 @@ # run doesn't bloat run.json. The untruncated message stays on task.json. _ROW_ERROR_MESSAGE_MAX_CHARS = 400 +# Cap on integrity findings carried into a run.json row. One finding is enough to +# adjudicate a row; the full list stays on task.json. Keeps a pathological run +# (an agent that greps the answer key fifty times) from bloating run.json. +_ROW_INTEGRITY_FINDINGS_MAX = 5 + def _cost_complete(result: EvaluationResult) -> bool: """Whether this row's recorded agent spend accounts for everything it spent. @@ -214,6 +219,14 @@ def eval_result_to_task_dict( # comparing early-stopped runs across an experiment sweep that varies # it can tell which weighted-gate value produced a given verdict. "gate_threshold": (result.early_stop.gate_threshold if result.early_stop is not None else None), + # Run-integrity surfaces. A tainted row's score is a measurement of the + # leak, not of the agent, so triage has to be able to see that from + # run.json alone — task.json is one fetch per row. + "integrity_verdict": result.integrity.verdict.value, + "integrity_voided": result.integrity.voided, + "integrity_findings": [ + f"{f.kind.value}: {f.detail}" for f in result.integrity.findings[:_ROW_INTEGRITY_FINDINGS_MAX] + ], } d["variant_id"] = variant_id return d diff --git a/tests/test_models.py b/tests/test_models.py index 5050715a..906e10f1 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -510,3 +510,129 @@ def test_rejects_relative(self): with pytest.raises(ValidationError, match="must be an absolute path"): DockerDriverConfig(working_dir="root") + + +class TestIntegrityInfoModel: + """IntegrityInfo shape, defaults, round-trip, and legacy-file tolerance.""" + + def test_defaults_describe_a_pass_that_never_ran(self): + from coder_eval.models import IntegrityInfo, IntegrityMode, IntegrityVerdict + + info = IntegrityInfo() + assert info.verdict is IntegrityVerdict.SKIPPED + assert info.mode is IntegrityMode.OFF + assert info.voided is False + assert info.findings == [] + assert info.notes == [] + assert info.commands_scanned == 0 + assert info.commands_without_parameters == 0 + assert info.subagent_recovery_incomplete is False + + def test_always_present_on_evaluation_result(self): + """Unlike EarlyStopInfo, integrity is never None — SKIPPED/INCONCLUSIVE must be expressible.""" + from datetime import datetime + + from coder_eval.models import AgentKind, EvaluationResult, FinalStatus, IntegrityVerdict + + result = EvaluationResult( + task_id="t", + task_description="d", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime.now(), + final_status=FinalStatus.SUCCESS, + iteration_count=1, + ) + assert result.integrity is not None + assert result.integrity.verdict is IntegrityVerdict.SKIPPED + + def test_round_trips_through_json(self): + from datetime import datetime + + from coder_eval.models import ( + AgentKind, + EvaluationResult, + FinalStatus, + IntegrityFinding, + IntegrityFindingKind, + IntegrityInfo, + IntegrityMode, + IntegrityVerdict, + ) + + result = EvaluationResult( + task_id="t", + task_description="d", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime.now(), + final_status=FinalStatus.FAILURE, + iteration_count=1, + integrity=IntegrityInfo( + verdict=IntegrityVerdict.TAINTED, + mode=IntegrityMode.VOID, + voided=True, + findings=[ + IntegrityFinding( + kind=IntegrityFindingKind.GRADED_READ, + detail="read the reference solution", + iteration=1, + command_index=7, + tool_name="Bash", + evidence="cat /work/task_dir/RESOLUTION.md", + ) + ], + commands_scanned=42, + commands_without_parameters=1, + notes=["one command had empty parameters"], + ), + ) + + reloaded = EvaluationResult.model_validate_json(result.model_dump_json()) + assert reloaded.integrity.verdict is IntegrityVerdict.TAINTED + assert reloaded.integrity.mode is IntegrityMode.VOID + assert reloaded.integrity.voided is True + assert reloaded.integrity.findings[0].kind is IntegrityFindingKind.GRADED_READ + assert reloaded.integrity.findings[0].command_index == 7 + assert reloaded.integrity.commands_scanned == 42 + assert reloaded.integrity.notes == ["one command had empty parameters"] + + def test_legacy_task_json_without_integrity_loads(self): + """A task.json written before this field must still parse, as SKIPPED.""" + import json + + from coder_eval.models import EvaluationResult, IntegrityVerdict + + legacy = { + "task_id": "t", + "task_description": "d", + "agent_type": "claude-code", + "started_at": "2026-01-01T00:00:00", + "final_status": "SUCCESS", + "iteration_count": 1, + } + result = EvaluationResult.model_validate_json(json.dumps(legacy)) + assert result.integrity.verdict is IntegrityVerdict.SKIPPED + assert result.integrity.voided is False + + +def test_final_status_member_set_is_closed(): + """Integrity taint must NOT add a FinalStatus member. + + A new member costs both exhaustive maps in models/enums.py (module-level + asserts, so a miss breaks import), the orchestrator/batch status branches, + the telemetry Category dimension, and the evalboard's status layer. A + tainted run already has a terminal status; taint rides alongside it as + orthogonal telemetry (see IntegrityVerdict). This pin makes an attempt to + add TAINTED/VOIDED as a status fail here, where the decision is documented. + """ + from coder_eval.models import FinalStatus + + assert {s.value for s in FinalStatus} == { + "SUCCESS", + "FAILURE", + "ERROR", + "BUILD_FAILED", + "TIMEOUT", + "MAX_TURNS_EXHAUSTED", + "TOKEN_BUDGET_EXCEEDED", + "COST_BUDGET_EXCEEDED", + } diff --git a/tests/test_reports_experiment.py b/tests/test_reports_experiment.py index 02ff1e79..ba0242bd 100644 --- a/tests/test_reports_experiment.py +++ b/tests/test_reports_experiment.py @@ -7,11 +7,16 @@ CommandTelemetry, EvaluationResult, FinalStatus, + IntegrityFinding, + IntegrityFindingKind, + IntegrityInfo, + IntegrityMode, + IntegrityVerdict, ResultSummary, TaskConfigRecord, TurnRecord, ) -from coder_eval.reports_experiment import eval_result_to_task_dict +from coder_eval.reports_experiment import _ROW_INTEGRITY_FINDINGS_MAX, eval_result_to_task_dict def _make_result( @@ -145,3 +150,106 @@ def test_none_when_run_limits_not_dict(self): ) d = eval_result_to_task_dict(result) assert d["expected_turns"] is None + + +class TestRowKeySet: + """Pins the run.json row key set. + + ``eval_result_to_task_dict`` is a hand-maintained dict literal: a new field on + ``EvaluationResult`` reaches ``task.json`` for free but is silently ABSENT + from ``run.json``, which is what the evalboard and triage read. Nothing + pinned this key set before (not even ``stopped_early``), so the omission was + invisible. Adding a row key here is intentional; removing or renaming one is + a breaking change for downstream consumers. + """ + + EXPECTED_KEYS = frozenset( + { + "task_id", + "replicate_index", + "status", + "weighted_score", + "duration", + "iteration_count", + "tags", + "task_path", + "iterations", + "model_used", + "reference_similarity", + "input_tokens", + "output_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", + "total_tokens", + "total_cost_usd", + "agent_cost_usd", + "cost_complete", + "judge_cost_usd", + "simulator_cost_usd", + "error_message", + "error_category", + "expected_commands", + "actual_commands", + "commands_efficiency", + "agent_config", + "sdk_options", + "installed_tools", + "max_turns_exhausted", + "expected_turns_overage", + "total_turns", + "visible_turns", + "expected_turns", + "has_final_reply", + "stopped_early", + "early_stop_reason", + "turns_remaining_at_stop", + "gate_threshold", + "integrity_verdict", + "integrity_voided", + "integrity_findings", + "variant_id", + } + ) + + def test_key_set_is_exactly_as_pinned(self): + d = eval_result_to_task_dict(_make_result(turns=[_turn(5)])) + assert set(d) == self.EXPECTED_KEYS + + +class TestIntegrityKeys: + def test_defaults_report_a_skipped_untainted_row(self): + d = eval_result_to_task_dict(_make_result(turns=[_turn(5)])) + assert d["integrity_verdict"] == "skipped" + assert d["integrity_voided"] is False + assert d["integrity_findings"] == [] + + def test_verdict_and_findings_reach_the_row(self): + result = _make_result(turns=[_turn(5)]) + result.integrity = IntegrityInfo( + verdict=IntegrityVerdict.TAINTED, + mode=IntegrityMode.VOID, + voided=True, + findings=[ + IntegrityFinding( + kind=IntegrityFindingKind.GRADED_READ, + detail="read RESOLUTION.md", + iteration=1, + command_index=3, + tool_name="Bash", + evidence="cat RESOLUTION.md", + ) + ], + ) + d = eval_result_to_task_dict(result) + assert d["integrity_verdict"] == "tainted" + assert d["integrity_voided"] is True + assert d["integrity_findings"] == ["graded_read: read RESOLUTION.md"] + + def test_findings_are_capped(self): + result = _make_result(turns=[_turn(5)]) + result.integrity = IntegrityInfo( + verdict=IntegrityVerdict.TAINTED, + findings=[IntegrityFinding(kind=IntegrityFindingKind.GRADED_READ, detail=f"hit {i}") for i in range(20)], + ) + d = eval_result_to_task_dict(result) + assert len(d["integrity_findings"]) == _ROW_INTEGRITY_FINDINGS_MAX From 65792792599f63f7e02cb4579d2cfa6222ed943e Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Mon, 3 Aug 2026 18:02:04 +0300 Subject: [PATCH 02/31] Count sub-agents whose inner tool calls could not be recovered --- src/coder_eval/agents/codex_agent.py | 31 ++++++- src/coder_eval/models/results.py | 11 +++ src/coder_eval/streaming/collector.py | 1 + src/coder_eval/streaming/events.py | 5 ++ .../expected/claude_a_single_text_turn.json | 1 + .../expected/claude_b_tool_use_result.json | 1 + .../claude_c_multi_emission_delta.json | 1 + .../expected/claude_d_subagent_terminal.json | 1 + .../claude_e_model_usage_and_backfill.json | 1 + .../expected/claude_f_orphaned_tool.json | 1 + .../claude_g_crash_format_placeholder.json | 1 + .../claude_h1_timeout_process_error.json | 1 + .../claude_h2_process_error_crash.json | 1 + .../claude_i_in_loop_deadline_break.json | 1 + tests/test_codex_agent_unit.py | 89 +++++++++++++++++++ tests/test_event_collector.py | 1 + 16 files changed, 145 insertions(+), 3 deletions(-) diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index c6e4b2d3..273305b6 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -297,6 +297,10 @@ def __init__( self.spawned_children: list[tuple[str, str, str | None]] = [] # child thread id -> returned message (fallback when the rollout is absent). self.collab_results: dict[str, str] = {} + # Sub-agents whose inner tool calls never made it into the transcript + # (rollout missing, or recovery raised). Surfaced on the TurnRecord so + # consumers know the transcript is incomplete rather than empty. + self.unrecovered_subagent_threads = 0 # Assistant-transcript reconstruction buffers (one AssistantMessage per gen). self.open_blocks: list[ContentBlock] = [] @@ -624,6 +628,7 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso crashed=crashed, crash_reason=crash_reason, duration_seconds=time.monotonic() - self.turn_start_time, + unrecovered_subagent_threads=self.unrecovered_subagent_threads, ) ) @@ -1462,7 +1467,7 @@ async def _run_turn_with_streaming( # Skipped on a cooperative stop: children may have no rollout yet and the # run is already decided — recovery adds nothing the armed gate uses. if state.spawned_children and not state.stopped_early_hit: - await self._recover_subagent_tool_calls( + state.unrecovered_subagent_threads = await self._recover_subagent_tool_calls( state.spawned_children, state.collab_results, state.messages, @@ -1700,7 +1705,7 @@ async def _recover_subagent_tool_calls( emit: StreamCallback, task_id: str, turn_id: str, - ) -> None: + ) -> int: """Recover each spawned sub-agent's INNER tool calls AND token usage. Codex runs every sub-agent on its own child thread whose events never @@ -1726,14 +1731,23 @@ async def _recover_subagent_tool_calls( Best-effort: any failure (missing file, parse error) is swallowed so a recovery hiccup never fails the turn. + + Returns: + Number of sub-agents whose inner tool calls could NOT be recovered. + Those calls are absent from ``commands`` and ``messages`` altogether, + so a caller analysing the transcript is looking at less than the agent + actually did. Swallowing the failure keeps the turn alive; returning + the count keeps it from being mistaken for "the sub-agent ran nothing". """ home = self._codex_home() + unrecovered = 0 for thread_id, parent_tool_id, model in spawned_children: try: path = await self._await_rollout_file(home, thread_id) if path is None: # No rollout to mine: nest just the returned message (if any) so # the sub-agent's answer still shows, tokenless. + unrecovered += 1 self._log.debug("CodexAgent: no rollout found for sub-agent thread %s", thread_id) result = collab_results.get(thread_id) if result: @@ -1763,9 +1777,20 @@ async def _recover_subagent_tool_calls( ) messages.append(self._subagent_generation_message(blocks, gen, parent_tool_id, model, turn_id, gi)) except Exception as exc: - # Best-effort: a recovery hiccup must never fail the turn. + # Best-effort: a recovery hiccup must never fail the turn -- but it + # leaves this child's commands out of the transcript, so count it. + unrecovered += 1 self._log.debug("CodexAgent: sub-agent recovery failed for %s: %s", thread_id, exc) + if unrecovered: + self._log.warning( + "CodexAgent: %d of %d sub-agent(s) contributed no recovered tool calls; " + + "this turn's transcript is incomplete", + unrecovered, + len(spawned_children), + ) + return unrecovered + @staticmethod def _codex_home() -> Path: """Codex data directory (rollouts live under ``/sessions``).""" diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 195ec3dc..df3090e1 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -372,6 +372,17 @@ class TurnRecord(BaseModel): default=None, description="Short human-readable cause when crashed=True; None otherwise.", ) + unrecovered_subagent_threads: int = Field( + default=0, + ge=0, + description=( + "Sub-agents whose INNER tool calls could not be reconstructed, so their commands are " + "absent from `commands` and `messages` entirely. Non-zero means this turn's transcript " + "is incomplete: any analysis over it (integrity scanning, command statistics) saw less " + "than the agent actually did. Only Codex can be non-zero today — Claude bubbles its " + "sub-agent calls into the parent stream natively." + ), + ) class PostRunResult(BaseModel): diff --git a/src/coder_eval/streaming/collector.py b/src/coder_eval/streaming/collector.py index c848eb3e..e7c7996a 100644 --- a/src/coder_eval/streaming/collector.py +++ b/src/coder_eval/streaming/collector.py @@ -190,4 +190,5 @@ def build_turn_record(self) -> TurnRecord: result_summary=end.result_summary, crashed=end.crashed, crash_reason=end.crash_reason, + unrecovered_subagent_threads=end.unrecovered_subagent_threads, ) diff --git a/src/coder_eval/streaming/events.py b/src/coder_eval/streaming/events.py index 8e8bf43f..a8a3cd76 100644 --- a/src/coder_eval/streaming/events.py +++ b/src/coder_eval/streaming/events.py @@ -174,6 +174,11 @@ class AgentEndEvent(StreamEvent): crashed: bool = False crash_reason: str | None = None duration_seconds: float = 0.0 + # Sub-agents whose inner tool calls never reached this turn's transcript. Rides + # the finalization payload because the agent is the only party that knows a + # recovery attempt failed -- there is no granular event for a stream that never + # existed, so the collector has nothing to reduce. + unrecovered_subagent_threads: int = 0 # --- Post-evaluation events (orchestrator-owned, not part of the agent lifecycle) --- diff --git a/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json index febe95e0..2dd8474c 100644 --- a/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json @@ -53,5 +53,6 @@ "total_cost_usd": "", "uncached_input_tokens": 50 }, + "unrecovered_subagent_threads": 0, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json b/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json index 3ef5f669..9b5a9d03 100644 --- a/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json +++ b/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json @@ -75,5 +75,6 @@ "total_cost_usd": "", "uncached_input_tokens": 80 }, + "unrecovered_subagent_threads": 0, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json b/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json index 00d6f778..d944a950 100644 --- a/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json +++ b/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json @@ -129,5 +129,6 @@ "total_cost_usd": "", "uncached_input_tokens": 237 }, + "unrecovered_subagent_threads": 0, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json b/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json index 17f88c55..99fcc074 100644 --- a/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json +++ b/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json @@ -102,5 +102,6 @@ "total_cost_usd": "", "uncached_input_tokens": 390 }, + "unrecovered_subagent_threads": 0, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json b/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json index 782fa203..4a738a69 100644 --- a/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json +++ b/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json @@ -61,5 +61,6 @@ "total_cost_usd": "", "uncached_input_tokens": 500 }, + "unrecovered_subagent_threads": 0, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json index 58967687..6692c198 100644 --- a/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json @@ -76,5 +76,6 @@ "total_cost_usd": "", "uncached_input_tokens": 60 }, + "unrecovered_subagent_threads": 0, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json b/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json index d615eae2..2396d638 100644 --- a/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json +++ b/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json @@ -13,5 +13,6 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "unrecovered_subagent_threads": 0, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json b/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json index 2e021896..452a9e32 100644 --- a/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json +++ b/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json @@ -13,5 +13,6 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "unrecovered_subagent_threads": 0, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json b/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json index dcdd6042..352bbe7c 100644 --- a/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json +++ b/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json @@ -13,5 +13,6 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "unrecovered_subagent_threads": 0, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json b/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json index da8254bd..e454fe8d 100644 --- a/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json +++ b/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json @@ -41,5 +41,6 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "unrecovered_subagent_threads": 0, "user_input": "do the thing" } diff --git a/tests/test_codex_agent_unit.py b/tests/test_codex_agent_unit.py index 7cc78cb6..2115a3d8 100644 --- a/tests/test_codex_agent_unit.py +++ b/tests/test_codex_agent_unit.py @@ -104,3 +104,92 @@ def test_command_dispatch_mutates_lists_in_place(self): # A tool_use block was recorded into the open buffer (cut at the next # tokenUsage flush, not here), joinable to the command by tool_id. assert any(b.block_type == "tool_use" and b.tool_use_id == "c1" for b in state.open_blocks) + + +class TestSubagentRecoveryCounter: + """The recovery gap must be counted, not just swallowed. + + A sub-agent whose rollout is missing or whose recovery raises contributes no + tool calls to the transcript. Left silent, a scan over that transcript reports + "nothing suspicious" for commands it never saw. + """ + + @staticmethod + def _agent(): + from coder_eval.agents.codex_agent import CodexAgent + from coder_eval.models import parse_agent_config + + return CodexAgent(parse_agent_config(type="codex")) + + @staticmethod + def _noop_emit(): + class _Emit: + def on_event(self, event) -> None: + pass + + return _Emit() + + async def test_missing_rollout_counts_as_unrecovered(self, monkeypatch): + agent = self._agent() + + async def _no_rollout(_home, _thread_id): + return None + + monkeypatch.setattr(agent, "_await_rollout_file", _no_rollout) + messages: list = [] + commands: list = [] + count = await agent._recover_subagent_tool_calls( + [("thread-a", "tool-1", "gpt"), ("thread-b", "tool-2", "gpt")], + {}, + messages, + commands, + self._noop_emit(), + "task", + "turn", + ) + assert count == 2 + assert commands == [] + + async def test_recovery_exception_counts_as_unrecovered(self, monkeypatch): + agent = self._agent() + + async def _boom(_home, _thread_id): + raise OSError("rollout unreadable") + + monkeypatch.setattr(agent, "_await_rollout_file", _boom) + count = await agent._recover_subagent_tool_calls( + [("thread-a", "tool-1", None)], + {}, + [], + [], + self._noop_emit(), + "task", + "turn", + ) + assert count == 1 + + async def test_full_recovery_counts_zero(self, monkeypatch): + agent = self._agent() + + async def _rollout(_home, _thread_id): + return "rollout.jsonl" + + monkeypatch.setattr(agent, "_await_rollout_file", _rollout) + monkeypatch.setattr(agent, "_parse_rollout_generations", lambda _p: []) + count = await agent._recover_subagent_tool_calls( + [("thread-a", "tool-1", None)], + {}, + [], + [], + self._noop_emit(), + "task", + "turn", + ) + assert count == 0 + + +def test_turn_record_defaults_unrecovered_subagent_threads_to_zero(): + """Claude bubbles its sub-agent calls natively, so the default must be 0.""" + from coder_eval.models import TurnRecord + + assert TurnRecord(iteration=1, user_input="p", agent_output="a").unrecovered_subagent_threads == 0 diff --git a/tests/test_event_collector.py b/tests/test_event_collector.py index 76e49e49..e1d6e9a4 100644 --- a/tests/test_event_collector.py +++ b/tests/test_event_collector.py @@ -227,6 +227,7 @@ def _full_agent_end(self) -> AgentEndEvent: result_summary=ResultSummary(is_error=False, subtype="success", result="all done"), crashed=True, crash_reason="boom", + unrecovered_subagent_threads=2, ) def test_no_turn_record_field_is_unaccounted_for(self): From 35aa79a52ce55e19daa81f59064a7ae87ea697d3 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Mon, 3 Aug 2026 18:09:26 +0300 Subject: [PATCH 03/31] Add a graded-material read detector for finished transcripts --- src/coder_eval/integrity.py | 579 +++++++++++++++++++++++++++++++++++ tests/test_integrity_scan.py | 399 ++++++++++++++++++++++++ 2 files changed, 978 insertions(+) create mode 100644 src/coder_eval/integrity.py create mode 100644 tests/test_integrity_scan.py diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py new file mode 100644 index 00000000..2e07ba43 --- /dev/null +++ b/src/coder_eval/integrity.py @@ -0,0 +1,579 @@ +"""Run-integrity detection: did the agent read graded material instead of evidence? + +A task's score is only a measurement of the agent if the agent worked from the +evidence the scenario intended. When it instead opens the reference solution, the +checker script, or its own task definition, a high score measures the leak. Those +rows are worse than useless: they inflate a suite average and hide a real +regression, and nothing in the run record distinguishes them. + +This module answers one question over a finished transcript -- "did any command +read graded material?" -- and reports it as an :class:`IntegrityInfo`. It decides +nothing about the row's status; :mod:`coder_eval.orchestrator` owns that gate. + +Two design constraints are load-bearing: + +* **Detection, not containment.** Under ``driver: tempdir`` the whole checkout is + on the same filesystem as the agent and unrestricted access is intentional + (see ``sandbox.py``), so there is no mount to take away. Detection is the only + lever available on that driver, and it is driver-independent. +* **Scan the untruncated command.** ``CommandExecutedChecker`` clips command text + at 2000 characters as a ReDoS guard -- exactly where a long ``cat`` hides. This + module reads ``CommandTelemetry.parameters`` directly and never truncates the + haystack. + +The shell classifier (:func:`_classify_segment`) is the false-positive control and +the whole game on Codex, where every file read arrives as a ``Bash`` command +rather than a ``Read`` tool call. A directory listing that prints a path is not a +leak; a ``cat`` of the same path is. +""" + +from __future__ import annotations + +import fnmatch +import logging +import re +import shlex +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING + +from coder_eval.models import ( + CONTAINER_INPUT_DIR, + CONTAINER_TASK_DIR, + IntegrityFinding, + IntegrityFindingKind, + IntegrityInfo, + IntegrityMode, + IntegrityVerdict, +) + + +if TYPE_CHECKING: + from coder_eval.models import TaskDefinition, TurnRecord + from coder_eval.models.telemetry import CommandTelemetry + + +logger = logging.getLogger(__name__) + + +# Fraction of scanned commands whose parameters may be empty before the scan is +# treated as partially blind. Codex returns {} for tool kinds it does not model +# (`_tool_parameters`), so a small number is normal; a large one means the scan +# reported on a transcript it could not read, and CLEAN would be a lie. +_MAX_BLIND_COMMAND_RATIO = 0.10 + +# Characters of matched command text kept on a finding. Enough to see an obvious +# false positive without opening task.json; short enough that fifty findings do +# not bloat the row. +_MAX_EVIDENCE_CHARS = 240 + +# Shell operators that end one command and begin another. `||` precedes `|` so +# regex alternation consumes the two-character form first. +_SEGMENT_SEPARATOR = re.compile(r"\|\||&&|;|\||\n|\r") + +# Wrappers that delegate to the real utility; skipped when finding the utility +# that decides a segment's classification. +_TRANSPARENT_PREFIXES = frozenset({"sudo", "env", "command", "time", "nohup", "nice", "exec", "builtin", "eval"}) + +# Utilities that only report a path's existence, name, or metadata. A hit inside +# one of these is not a read -- an `ls` that prints `RESOLUTION.md` tells the +# agent nothing it could put in an answer. +_LISTING_UTILITIES = frozenset( + { + "ls", + "dir", + "find", + "fd", + "tree", + "stat", + "file", + "wc", + "du", + "test", + "basename", + "dirname", + "realpath", + "readlink", + "echo", + "printf", + "pwd", + "which", + "type", + "mkdir", + "touch", + "cd", + } +) + +# Utilities that emit file CONTENT. A hit inside one of these is a read. +_READ_UTILITIES = frozenset( + { + "cat", + "bat", + "tac", + "head", + "tail", + "sed", + "awk", + "less", + "more", + "strings", + "od", + "xxd", + "hexdump", + "base64", + "zcat", + "gunzip", + "unzip", + "tar", + "python", + "python3", + "py", + "node", + "perl", + "ruby", + "jq", + "yq", + "diff", + "cmp", + "cp", + "copy", + "tee", + "sort", + "uniq", + "cut", + "tr", + "nl", + "open", + "source", + "vim", + "nano", + "emacs", + } +) + +# Search utilities: a read unless restricted to reporting which files matched. +_SEARCH_UTILITIES = frozenset({"grep", "egrep", "fgrep", "rg", "ag", "ack", "findstr", "select-string"}) + +# Flags that make a search utility report file names or counts instead of the +# matching lines. `-c` and `-l` may be bundled (`-rl`), so short flags are also +# matched character-wise. +_SEARCH_FILES_ONLY_LONG = frozenset( + {"--files", "--files-with-matches", "--files-without-match", "--count", "-l", "-L", "-c"} +) +_SEARCH_FILES_ONLY_SHORT = frozenset({"l", "L", "c"}) + +# Structured tools whose whole purpose is to return file content. +_READ_TOOLS = frozenset({"Read", "NotebookRead", "ReadFile", "read_file", "view", "View"}) + +# Structured tools that only enumerate paths. +_LISTING_TOOLS = frozenset({"Glob", "LS", "ListDir", "list_dir", "Ls", "glob", "TodoWrite", "Task", "Skill"}) + +# Basename patterns that are graded material in every suite, independent of what +# this particular task declares. Deliberately short: each entry is a name the +# framework or the task-authoring convention owns, never a name an agent's own +# work would produce. +_GRADED_BASENAME_GLOBS = ("RESOLUTION.md", "check_*.py", "*.expected", "task.yaml", "context.json") + + +@dataclass(frozen=True) +class GradedMaterialSpec: + """What counts as graded material for one task. + + Derived from what the harness already knows -- the task file, its declared + reference, the ``$TASK_DIR`` operands its own criteria use, and the + framework's container mounts -- rather than from hardcoded suite paths, so it + stays correct as suites are added and renamed. + """ + + paths: frozenset[str] = field(default_factory=frozenset) + """Literal file paths (task YAML, reference file, criterion operands).""" + + directories: frozenset[str] = field(default_factory=frozenset) + """Directory prefixes (reference directory, framework input mount).""" + + basename_globs: tuple[str, ...] = () + """Filename patterns that are graded material regardless of location.""" + + def is_empty(self) -> bool: + """Whether the spec would match nothing at all.""" + return not (self.paths or self.directories or self.basename_globs) + + +def _normalize(text: str) -> str: + """Case-fold and forward-slash a path or command for substring comparison. + + Windows task files arrive with backslashes while the sandbox command that + reads them uses forward slashes (Git Bash), so a raw comparison misses. + """ + return text.replace("\\", "/").casefold() + + +def _glob_to_regex(glob: str) -> re.Pattern[str]: + """Compile a basename glob into a pattern that matches it inside a command. + + ``fnmatch.translate`` alone anchors the whole string; the wildcard is also + narrowed so it cannot cross a path separator (``check_*.py`` must not match + ``check_dir/other.py``). + """ + body = fnmatch.translate(_normalize(glob)) + # translate() emits `(?s:...)\Z`; strip the anchor and re-scope the wildcard. + body = body.removesuffix(r"\Z") + body = body.replace(".*", "[^/]*") + return re.compile(body) + + +def derive_graded_material(task: TaskDefinition, task_file: Path | None) -> GradedMaterialSpec: + """Work out this task's graded material from the harness's own configuration. + + Args: + task: The resolved task definition (its ``reference``, ``pre_run``, + ``post_run`` and ``run_command`` criteria are all read). + task_file: Path to the task YAML, when the caller tracked it. Without it + the task-relative paths (the YAML itself, ``reference.file``) cannot + be resolved and only the location-independent patterns apply. + + Returns: + The spec :func:`scan_commands` matches against. + """ + paths: set[str] = set() + directories: set[str] = {CONTAINER_INPUT_DIR} + + if task_file is not None: + paths.add(str(task_file)) + base = task_file.parent + reference = task.reference + if reference is not None: + if reference.file: + paths.add(str(base / reference.file)) + if reference.directory: + directories.add(str(base / reference.directory)) + + # Operands the task's OWN criteria reach for. `python3 $TASK_DIR/check_x.py` + # names the grader; an agent that runs the grader is grading itself. + criterion_commands = [getattr(c, "command", "") for c in task.success_criteria] + hook_commands = [c.command for c in (*task.pre_run, *task.post_run)] + for command in (*criterion_commands, *hook_commands): + paths.update(_task_dir_operands(command or "")) + + return GradedMaterialSpec( + paths=frozenset(paths), + directories=frozenset(directories), + basename_globs=_GRADED_BASENAME_GLOBS, + ) + + +def _task_dir_operands(command: str) -> set[str]: + """Extract ``$TASK_DIR``-rooted operands from a framework-run command. + + Both the raw form (``$TASK_DIR/check_x.py``, which is what an agent that + discovered the variable would type) and the container-resolved form + (``/work/task_dir/check_x.py``) are returned, since either spelling is the + same read. + """ + operands: set[str] = set() + for match in re.finditer(r"\$\{?TASK_DIR\}?(/[^\s'\";|&)]+)", command): + suffix = match.group(1) + operands.add(f"$TASK_DIR{suffix}") + operands.add(f"{CONTAINER_TASK_DIR}{suffix}") + return operands + + +def _find_match(text: str, spec: GradedMaterialSpec) -> str | None: + """Return the graded-material reference found in ``text``, or None. + + Literal paths and directory prefixes are substring-matched on the normalized + form; basename globs are regex-matched so ``check_*.py`` catches any grader. + """ + haystack = _normalize(text) + + for candidate in spec.paths: + needle = _normalize(candidate) + if needle and needle in haystack: + return candidate + for candidate in spec.directories: + needle = _normalize(candidate) + if needle and needle in haystack: + return candidate + for glob in spec.basename_globs: + if _glob_to_regex(glob).search(haystack): + return glob + return None + + +def _segment_utility(segment: str) -> tuple[str, list[str]]: + """Leading utility of a shell segment (basename, lowercased) and its tokens. + + Leading ``VAR=value`` assignments and transparent wrappers (``sudo``, ``env``, + ``time``, …) are stepped over so ``sudo cat x`` classifies as ``cat``. + Returns ``("", tokens)`` when no utility can be identified. + """ + try: + tokens = shlex.split(segment, posix=True) + except ValueError: + # Unbalanced quotes: fall back to whitespace splitting rather than + # skipping the segment, since an unparseable command still ran. + tokens = segment.split() + + for token in tokens: + if "=" in token and not token.startswith(("-", "/", ".")) and token.split("=", 1)[0].isidentifier(): + continue # leading environment assignment + name = Path(token.replace("\\", "/")).name.casefold().removesuffix(".exe") + if name in _TRANSPARENT_PREFIXES: + continue + return name, tokens + return "", tokens + + +def _search_is_files_only(tokens: list[str]) -> bool: + """Whether a grep/rg invocation reports only file names or match counts.""" + for token in tokens: + if token in _SEARCH_FILES_ONLY_LONG: + return True + # Bundled short flags: `-rl`, `-il`. A lone `-` or a long flag is skipped. + if ( + token.startswith("-") + and not token.startswith("--") + and any(c in _SEARCH_FILES_ONLY_SHORT for c in token[1:]) + ): + return True + return False + + +def _classify_segment(segment: str, spec: GradedMaterialSpec) -> tuple[bool, str | None]: + """Decide whether one shell segment READ graded material. + + Returns ``(is_read, matched_reference)``. ``matched_reference`` is set + whenever the segment mentions graded material at all, so a caller can tell + "mentioned but only listed" from "never mentioned". + + Rules, in order: + + 1. No graded-material reference anywhere in the segment -> not a read. + 2. A reference after an input redirect (``< file``) -> a read, whatever the + utility is; the shell does the reading. + 3. Any content-emitting utility appearing as a token -> a read. Checked + across all tokens, not just the leading one, so ``find … -exec cat {}`` + and ``xargs cat`` do not slip past on their wrapper's name. + 4. A search utility restricted to file names or counts -> not a read; + otherwise a read. + 5. A pure listing/metadata utility -> not a read. + 6. Anything else -> a read. Conservative on purpose: an unrecognised utility + holding a path to the answer key is more likely a read than not, and a + false positive is visible in the finding's evidence while a false negative + is invisible. + """ + matched = _find_match(segment, spec) + if matched is None: + return False, None + + utility, tokens = _segment_utility(segment) + normalized_tokens = [Path(t.replace("\\", "/")).name.casefold().removesuffix(".exe") for t in tokens] + + if "<" in segment: + _, _, after = segment.partition("<") + if _find_match(after, spec) is not None: + return True, matched + + if any(name in _READ_UTILITIES for name in normalized_tokens): + return True, matched + + if utility in _SEARCH_UTILITIES: + return not _search_is_files_only(tokens), matched + + if utility in _LISTING_UTILITIES: + return False, matched + + return True, matched + + +def _excerpt(text: str, matched: str) -> str: + """A short window of ``text`` around ``matched``, for the finding's evidence.""" + flat = " ".join(text.split()) + index = _normalize(flat).find(_normalize(matched)) + if index < 0 or len(flat) <= _MAX_EVIDENCE_CHARS: + return flat[:_MAX_EVIDENCE_CHARS] + start = max(0, index - _MAX_EVIDENCE_CHARS // 3) + return flat[start : start + _MAX_EVIDENCE_CHARS] + + +def _command_text(cmd: CommandTelemetry) -> str: + """The scannable text of a command: the shell string, or its parameter values. + + ``result_summary`` is deliberately excluded. Result bodies are full of paths + the agent merely saw printed, which is precisely the false-positive class the + Antigravity backend already strips result-only keys to avoid. + """ + if cmd.tool_name == "Bash" and isinstance(cmd.parameters.get("command"), str): + return cmd.parameters["command"] + return " ".join(str(v) for v in cmd.parameters.values()) + + +def scan_commands(turns: list[TurnRecord], spec: GradedMaterialSpec) -> IntegrityInfo: + """Scan a finished transcript for reads of graded material. + + Populates findings and the blind-spot counters, and derives the verdict from + both: a positive hit is TAINTED even on a partially-visible transcript (going + blind cannot un-see a hit), while a clean scan over a transcript the scanner + could not fully read is INCONCLUSIVE rather than CLEAN. + + Args: + turns: The run's iterations, in order. + spec: Graded material for this task (see :func:`derive_graded_material`). + + Returns: + An :class:`IntegrityInfo` with ``mode`` left at its default -- the caller + stamps the mode it ran under and owns the gate. + """ + findings: list[IntegrityFinding] = [] + notes: list[str] = [] + scanned = 0 + blind = 0 + unclassified_hits = 0 + + for turn in turns: + for index, cmd in enumerate(turn.commands): + scanned += 1 + if not cmd.parameters: + blind += 1 + continue + + text = _command_text(cmd) + if not text: + blind += 1 + continue + + if cmd.tool_name == "Bash": + is_read, matched = _bash_read(text, spec) + else: + is_read, matched, understood = _structured_read(cmd, text, spec) + if matched is not None and not understood: + unclassified_hits += 1 + notes.append( + f"{cmd.tool_name} referenced {matched} but its read semantics are unknown; not counted" + ) + + if is_read and matched is not None: + findings.append( + IntegrityFinding( + kind=IntegrityFindingKind.GRADED_READ, + detail=f"{cmd.tool_name} read graded material ({matched})", + iteration=turn.iteration, + command_index=index, + tool_name=cmd.tool_name, + evidence=_excerpt(text, matched), + ) + ) + + unrecovered = sum(t.unrecovered_subagent_threads for t in turns) + if unrecovered: + notes.append(f"{unrecovered} sub-agent(s) contributed no recovered tool calls; their commands were not scanned") + + blind_ratio = (blind / scanned) if scanned else 0.0 + too_blind = blind_ratio > _MAX_BLIND_COMMAND_RATIO + if too_blind: + notes.append(f"{blind} of {scanned} commands had no scannable parameters ({blind_ratio:.0%})") + + if findings: + verdict = IntegrityVerdict.TAINTED + elif unrecovered or too_blind or unclassified_hits: + verdict = IntegrityVerdict.INCONCLUSIVE + else: + verdict = IntegrityVerdict.CLEAN + + return IntegrityInfo( + verdict=verdict, + findings=findings, + commands_scanned=scanned, + commands_without_parameters=blind, + subagent_recovery_incomplete=bool(unrecovered), + notes=notes, + ) + + +def _bash_read(command: str, spec: GradedMaterialSpec) -> tuple[bool, str | None]: + """Classify a shell command by splitting it into segments and judging each.""" + mentioned: str | None = None + for segment in _SEGMENT_SEPARATOR.split(command): + if not segment.strip(): + continue + is_read, matched = _classify_segment(segment, spec) + if matched is not None: + mentioned = matched + if is_read: + return True, matched + return False, mentioned + + +def _structured_read(cmd: CommandTelemetry, text: str, spec: GradedMaterialSpec) -> tuple[bool, str | None, bool]: + """Classify a non-Bash tool call. + + Returns ``(is_read, matched, semantics_understood)``. Unlike a shell string, a + structured tool has fixed semantics, so the decision is by tool name rather + than by heuristic. A tool this module does not recognise gets + ``semantics_understood=False`` when it touched graded material, which the + caller turns into INCONCLUSIVE -- neither a silent pass nor a taint on a tool + whose behavior we are guessing at. + """ + matched = _find_match(text, spec) + if matched is None: + return False, None, True + + if cmd.tool_name in _READ_TOOLS: + return True, matched, True + if cmd.tool_name in _LISTING_TOOLS: + return False, matched, True + if cmd.tool_name == "Grep": + # Claude's Grep returns matching LINES only in content mode; the default + # (`files_with_matches`) and `count` report where matches are, not what. + output_mode = str(cmd.parameters.get("output_mode") or "files_with_matches") + has_context = any(cmd.parameters.get(k) for k in ("-A", "-B", "-C")) + return output_mode == "content" or has_context, matched, True + return False, matched, False + + +def evaluate_integrity( + task: TaskDefinition, + task_file: Path | None, + turns: list[TurnRecord], + *, + mode: IntegrityMode, +) -> IntegrityInfo: + """Run the integrity pass for one task and return its verdict. + + Never raises: an integrity bug must not take down a row that otherwise ran + fine, so an unexpected failure is reported as INCONCLUSIVE with the reason in + ``notes``. + + Args: + task: The resolved task definition. + task_file: Path to the task YAML, when known. + turns: The run's iterations. + mode: The ``INTEGRITY_MODE`` in force. ``OFF`` short-circuits to SKIPPED. + + Returns: + A populated :class:`IntegrityInfo` with ``mode`` stamped. ``voided`` is + left False -- only the gate sets it. + """ + if mode is IntegrityMode.OFF: + return IntegrityInfo(verdict=IntegrityVerdict.SKIPPED, mode=mode, notes=["INTEGRITY_MODE=off"]) + + try: + spec = derive_graded_material(task, task_file) + if spec.is_empty(): + return IntegrityInfo( + verdict=IntegrityVerdict.SKIPPED, + mode=mode, + notes=["no graded material could be derived for this task"], + ) + info = scan_commands(turns, spec) + except Exception as exc: + logger.warning("Integrity scan failed for task %s: %s", task.task_id, exc, exc_info=True) + return IntegrityInfo( + verdict=IntegrityVerdict.INCONCLUSIVE, + mode=mode, + notes=[f"integrity scan raised {type(exc).__name__}: {exc}"], + ) + + info.mode = mode + return info diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py new file mode 100644 index 00000000..2172daa4 --- /dev/null +++ b/tests/test_integrity_scan.py @@ -0,0 +1,399 @@ +"""Tests for coder_eval.integrity: graded-material derivation and the read scan. + +The bulk of the integrity work is classification, so most of this is table-driven +over shell strings. The cases that matter most are the NEGATIVE ones: a scan that +flags a directory listing is worse than no scan, because it voids honest rows and +gets switched off. +""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import Path + +import pytest + +from coder_eval.integrity import ( + GradedMaterialSpec, + _bash_read, + _task_dir_operands, + derive_graded_material, + evaluate_integrity, + scan_commands, +) +from coder_eval.models import ( + CommandTelemetry, + IntegrityFindingKind, + IntegrityMode, + IntegrityVerdict, + TaskDefinition, + TurnRecord, +) +from coder_eval.models.container_paths import CONTAINER_INPUT_DIR + + +SPEC = GradedMaterialSpec( + paths=frozenset({"/repo/tasks/leaky/task.yaml", "/repo/tasks/leaky/solution.py", "$TASK_DIR/check_output.py"}), + directories=frozenset({"/repo/tasks/leaky/_reference", CONTAINER_INPUT_DIR}), + basename_globs=("RESOLUTION.md", "check_*.py", "*.expected", "task.yaml", "context.json"), +) + + +def _cmd(tool_name: str, parameters: dict, tool_id: str = "t0") -> CommandTelemetry: + return CommandTelemetry( + tool_name=tool_name, tool_id=tool_id, timestamp=datetime.now(), parameters=parameters, result_status="success" + ) + + +def _bash(command: str) -> CommandTelemetry: + return _cmd("Bash", {"command": command}) + + +def _turn(commands: list[CommandTelemetry], *, iteration: int = 1, unrecovered: int = 0) -> TurnRecord: + return TurnRecord( + iteration=iteration, + user_input="p", + agent_output="a", + commands=commands, + unrecovered_subagent_threads=unrecovered, + ) + + +# -------------------------------------------------------------------------- +# Shell segment classification +# -------------------------------------------------------------------------- + +READS = [ + pytest.param("cat RESOLUTION.md", id="cat"), + pytest.param("head -50 RESOLUTION.md", id="head"), + pytest.param("tail -n 5 /repo/tasks/leaky/solution.py", id="tail-absolute-path"), + pytest.param("sed -n '1,20p' RESOLUTION.md", id="sed"), + pytest.param("awk '{print}' check_output.py", id="awk-glob"), + pytest.param("python3 $TASK_DIR/check_output.py", id="python-runs-the-grader"), + pytest.param("python -c \"print(open('RESOLUTION.md').read())\"", id="python-inline"), + pytest.param("node -e \"require('fs').readFileSync('RESOLUTION.md')\"", id="node-inline"), + pytest.param("base64 -d RESOLUTION.md", id="base64"), + pytest.param("xxd RESOLUTION.md | head", id="xxd-piped"), + pytest.param("jq . /work/input/context.json", id="jq-container-input"), + pytest.param("diff mine.py /repo/tasks/leaky/solution.py", id="diff-against-reference"), + pytest.param("cp /repo/tasks/leaky/solution.py ./mine.py", id="cp-the-reference"), + pytest.param("grep -n 'root cause' RESOLUTION.md", id="grep-content"), + pytest.param("rg 'fixed version' RESOLUTION.md", id="rg-content"), + pytest.param("while read l; do echo $l; done < RESOLUTION.md", id="input-redirect"), + pytest.param("find . -name '*.md' -exec cat RESOLUTION.md {} \\;", id="find-exec-cat"), + pytest.param("ls -1 | xargs cat RESOLUTION.md", id="xargs-cat"), + pytest.param("sudo cat RESOLUTION.md", id="sudo-wrapper"), + pytest.param("FOO=1 cat RESOLUTION.md", id="env-assignment-prefix"), + pytest.param("ls -la && cat RESOLUTION.md", id="second-segment-reads"), + pytest.param("cat /repo/tasks/leaky/_reference/answer.py", id="reference-directory"), + pytest.param("strange-tool RESOLUTION.md", id="unknown-utility-conservative"), +] + +NOT_READS = [ + pytest.param("ls -la", id="plain-listing"), + pytest.param("ls -la /repo/tasks/leaky", id="listing-the-task-dir"), + pytest.param("find . -name RESOLUTION.md", id="find-by-name"), + pytest.param("find /repo -name 'check_*.py' -print", id="find-glob-print"), + pytest.param("test -f RESOLUTION.md", id="existence-test"), + pytest.param("stat RESOLUTION.md", id="stat-metadata"), + pytest.param("wc -l RESOLUTION.md", id="wc-line-count"), + pytest.param("basename /repo/tasks/leaky/task.yaml", id="basename"), + pytest.param("dirname /repo/tasks/leaky/task.yaml", id="dirname"), + pytest.param("echo RESOLUTION.md", id="echo-the-name"), + pytest.param("grep -l 'cause' RESOLUTION.md", id="grep-files-only"), + pytest.param("grep -rl 'cause' /repo --include=RESOLUTION.md", id="grep-bundled-files-only"), + pytest.param("grep -c 'cause' RESOLUTION.md", id="grep-count"), + pytest.param("rg --files /repo | grep -l RESOLUTION.md", id="rg-files"), + pytest.param("rg --files-with-matches cause RESOLUTION.md", id="rg-files-with-matches"), + pytest.param("cat my_own_notes.md", id="reads-something-else"), + pytest.param("python3 build.py", id="runs-own-script"), + pytest.param("mkdir -p output && ls", id="unrelated-work"), + pytest.param("du -sh /repo/tasks/leaky", id="disk-usage"), +] + + +@pytest.mark.parametrize("command", READS) +def test_shell_reads_are_flagged(command: str): + is_read, matched = _bash_read(command, SPEC) + assert is_read is True, f"expected a read: {command!r}" + assert matched is not None + + +@pytest.mark.parametrize("command", NOT_READS) +def test_shell_non_reads_are_not_flagged(command: str): + is_read, _ = _bash_read(command, SPEC) + assert is_read is False, f"expected NOT a read: {command!r}" + + +def test_windows_separators_still_match(): + """A task file recorded with backslashes must match a forward-slash command.""" + spec = GradedMaterialSpec(paths=frozenset({r"C:\repo\tasks\leaky\task.yaml"})) + is_read, matched = _bash_read("cat C:/repo/tasks/leaky/task.yaml", spec) + assert is_read is True + assert matched == r"C:\repo\tasks\leaky\task.yaml" + + +def test_glob_wildcard_does_not_cross_a_path_separator(): + """`check_*.py` must not match `check_dir/unrelated.py`.""" + spec = GradedMaterialSpec(basename_globs=("check_*.py",)) + assert _bash_read("cat check_dir/unrelated.py", spec) == (False, None) + assert _bash_read("cat check_dir/check_it.py", spec)[0] is True + + +def test_unbalanced_quotes_do_not_skip_the_command(): + """A command shlex cannot parse still ran, so it must still be classified.""" + is_read, _ = _bash_read("cat 'RESOLUTION.md", SPEC) + assert is_read is True + + +# -------------------------------------------------------------------------- +# The regression guard for the truncation trap +# -------------------------------------------------------------------------- + + +def test_match_past_2000_chars_is_still_found(): + """The scan must NOT reuse CommandExecutedChecker's 2000-char ReDoS clip. + + That checker truncates command text at 2000 characters, which is exactly + where a long `cat` hides: pad the command past the limit and the match is + invisible to anything that reuses it. This scan reads + CommandTelemetry.parameters directly and never truncates the haystack. + """ + from coder_eval.criteria.command_executed import _MAX_PATTERN_SEARCH_LEN + + padding = "# " + ("x" * (_MAX_PATTERN_SEARCH_LEN + 500)) + command = f"echo start\n{padding}\ncat RESOLUTION.md" + assert len(command) > _MAX_PATTERN_SEARCH_LEN + + info = scan_commands([_turn([_bash(command)])], SPEC) + assert info.verdict is IntegrityVerdict.TAINTED + assert len(info.findings) == 1 + + +# -------------------------------------------------------------------------- +# Structured (non-Bash) tools +# -------------------------------------------------------------------------- + + +def test_read_tool_on_graded_material_is_tainted(): + info = scan_commands([_turn([_cmd("Read", {"file_path": "/repo/tasks/leaky/RESOLUTION.md"})])], SPEC) + assert info.verdict is IntegrityVerdict.TAINTED + assert info.findings[0].tool_name == "Read" + assert info.findings[0].kind is IntegrityFindingKind.GRADED_READ + + +def test_glob_listing_graded_material_is_clean(): + info = scan_commands([_turn([_cmd("Glob", {"pattern": "**/RESOLUTION.md"})])], SPEC) + assert info.verdict is IntegrityVerdict.CLEAN + assert info.findings == [] + + +def test_grep_files_mode_is_clean_but_content_mode_is_tainted(): + listing = scan_commands([_turn([_cmd("Grep", {"pattern": "cause", "path": "RESOLUTION.md"})])], SPEC) + assert listing.verdict is IntegrityVerdict.CLEAN + + content = scan_commands( + [_turn([_cmd("Grep", {"pattern": "cause", "path": "RESOLUTION.md", "output_mode": "content"})])], SPEC + ) + assert content.verdict is IntegrityVerdict.TAINTED + + +def test_grep_with_context_flag_is_tainted(): + info = scan_commands([_turn([_cmd("Grep", {"pattern": "cause", "path": "RESOLUTION.md", "-C": 3})])], SPEC) + assert info.verdict is IntegrityVerdict.TAINTED + + +def test_unknown_tool_touching_graded_material_is_inconclusive_not_tainted(): + """We do not guess at an unrecognised tool's semantics in either direction.""" + info = scan_commands([_turn([_cmd("mcp__some__fetch", {"target": "RESOLUTION.md"})])], SPEC) + assert info.verdict is IntegrityVerdict.INCONCLUSIVE + assert info.findings == [] + assert any("read semantics are unknown" in n for n in info.notes) + + +def test_unknown_tool_not_touching_graded_material_is_clean(): + info = scan_commands([_turn([_cmd("mcp__some__fetch", {"target": "notes.md"})])], SPEC) + assert info.verdict is IntegrityVerdict.CLEAN + + +# -------------------------------------------------------------------------- +# Blind spots +# -------------------------------------------------------------------------- + + +def test_clean_scan_is_clean(): + info = scan_commands([_turn([_bash("ls -la"), _bash("python3 build.py")])], SPEC) + assert info.verdict is IntegrityVerdict.CLEAN + assert info.commands_scanned == 2 + assert info.commands_without_parameters == 0 + + +def test_no_commands_is_clean(): + info = scan_commands([_turn([])], SPEC) + assert info.verdict is IntegrityVerdict.CLEAN + assert info.commands_scanned == 0 + + +def test_mostly_parameterless_commands_force_inconclusive(): + """Codex returns {} for tool kinds it does not model; a scan that saw almost + nothing must not report CLEAN.""" + commands = [_cmd("Unknown", {}, tool_id=f"t{i}") for i in range(4)] + [_bash("ls")] + info = scan_commands([_turn(commands)], SPEC) + assert info.verdict is IntegrityVerdict.INCONCLUSIVE + assert info.commands_without_parameters == 4 + assert any("no scannable parameters" in n for n in info.notes) + + +def test_a_few_parameterless_commands_stay_clean(): + commands = [_bash("ls") for _ in range(20)] + [_cmd("Unknown", {}, tool_id="tX")] + info = scan_commands([_turn(commands)], SPEC) + assert info.verdict is IntegrityVerdict.CLEAN + + +def test_unrecovered_subagents_force_inconclusive(): + info = scan_commands([_turn([_bash("ls")], unrecovered=1)], SPEC) + assert info.verdict is IntegrityVerdict.INCONCLUSIVE + assert info.subagent_recovery_incomplete is True + + +def test_a_hit_beats_every_blind_spot(): + """Going blind cannot un-see a read that WAS observed.""" + commands = [_cmd("Unknown", {}, tool_id=f"t{i}") for i in range(9)] + [_bash("cat RESOLUTION.md")] + info = scan_commands([_turn(commands, unrecovered=3)], SPEC) + assert info.verdict is IntegrityVerdict.TAINTED + + +def test_findings_carry_locating_coordinates(): + turn = _turn([_bash("ls"), _bash("cat RESOLUTION.md")], iteration=2) + info = scan_commands([turn], SPEC) + finding = info.findings[0] + assert finding.iteration == 2 + assert finding.command_index == 1 + assert finding.evidence is not None + assert "RESOLUTION.md" in finding.evidence + + +# -------------------------------------------------------------------------- +# Graded-material derivation +# -------------------------------------------------------------------------- + + +def _task(**kwargs) -> TaskDefinition: + base = { + "task_id": "t", + "description": "d", + "initial_prompt": "p", + "success_criteria": [{"type": "file_exists", "description": "x", "path": "out.txt"}], + } + base.update(kwargs) + return TaskDefinition(**base) + + +def test_derivation_includes_the_task_file_and_reference(): + task = _task(reference={"file": "solution.py"}) + spec = derive_graded_material(task, Path("/repo/tasks/leaky/task.yaml")) + assert str(Path("/repo/tasks/leaky/task.yaml")) in spec.paths + assert str(Path("/repo/tasks/leaky/solution.py")) in spec.paths + + +def test_derivation_includes_the_reference_directory(): + task = _task(reference={"directory": "_reference"}) + spec = derive_graded_material(task, Path("/repo/tasks/leaky/task.yaml")) + assert str(Path("/repo/tasks/leaky/_reference")) in spec.directories + + +def test_derivation_always_includes_the_container_input_mount(): + spec = derive_graded_material(_task(), None) + assert CONTAINER_INPUT_DIR in spec.directories + + +def test_derivation_without_a_task_file_keeps_the_globs(): + """A caller that never tracked the YAML still gets location-independent cover.""" + spec = derive_graded_material(_task(), None) + assert not any("task.yaml" in p for p in spec.paths) + assert "RESOLUTION.md" in spec.basename_globs + + +def test_derivation_harvests_task_dir_operands_from_criteria(): + task = _task( + success_criteria=[ + { + "type": "run_command", + "description": "grade", + "command": "python3 $TASK_DIR/check_answer.py", + } + ] + ) + spec = derive_graded_material(task, None) + assert "$TASK_DIR/check_answer.py" in spec.paths + assert "/work/task_dir/check_answer.py" in spec.paths + + +def test_derivation_harvests_task_dir_operands_from_hooks(): + task = _task(post_run=[{"command": "cp ${TASK_DIR}/expected.json ."}]) + spec = derive_graded_material(task, None) + assert "$TASK_DIR/expected.json" in spec.paths + + +@pytest.mark.parametrize( + ("command", "expected"), + [ + ("python3 $TASK_DIR/check_x.py", {"$TASK_DIR/check_x.py", "/work/task_dir/check_x.py"}), + ("python3 ${TASK_DIR}/check_x.py", {"$TASK_DIR/check_x.py", "/work/task_dir/check_x.py"}), + ('cat "$TASK_DIR/a.txt" && ls', {"$TASK_DIR/a.txt", "/work/task_dir/a.txt"}), + ("echo $TASK_DIR", set()), + ("no variable here", set()), + ], +) +def test_task_dir_operand_extraction(command: str, expected: set[str]): + assert _task_dir_operands(command) == expected + + +# -------------------------------------------------------------------------- +# evaluate_integrity: mode handling and failure containment +# -------------------------------------------------------------------------- + + +def test_mode_off_skips_the_scan(): + info = evaluate_integrity(_task(), None, [_turn([_bash("cat RESOLUTION.md")])], mode=IntegrityMode.OFF) + assert info.verdict is IntegrityVerdict.SKIPPED + assert info.mode is IntegrityMode.OFF + assert info.findings == [] + + +@pytest.mark.parametrize("mode", [IntegrityMode.DETECT, IntegrityMode.VOID]) +def test_detect_and_void_both_scan_and_stamp_the_mode(mode: IntegrityMode): + info = evaluate_integrity(_task(), None, [_turn([_bash("cat RESOLUTION.md")])], mode=mode) + assert info.verdict is IntegrityVerdict.TAINTED + assert info.mode is mode + # The gate, not the scan, decides whether to void. + assert info.voided is False + + +def test_a_scan_failure_is_inconclusive_not_a_crash(): + """An integrity bug must not take down a row that otherwise ran fine.""" + + class _Exploding(list): + def __iter__(self): + raise RuntimeError("boom") + + info = evaluate_integrity(_task(), None, _Exploding(), mode=IntegrityMode.VOID) + assert info.verdict is IntegrityVerdict.INCONCLUSIVE + assert any("boom" in n for n in info.notes) + + +def test_empty_spec_skips_rather_than_reporting_clean(): + """With nothing to match against, CLEAN would be an unearned reassurance.""" + from coder_eval import integrity + + spec = GradedMaterialSpec() + assert spec.is_empty() is True + + original = integrity.derive_graded_material + try: + integrity.derive_graded_material = lambda _task, _file: GradedMaterialSpec() + info = evaluate_integrity(_task(), None, [_turn([_bash("ls")])], mode=IntegrityMode.DETECT) + finally: + integrity.derive_graded_material = original + + assert info.verdict is IntegrityVerdict.SKIPPED From 2b291dbf7806a190271583764f883a9bf0a2a7cd Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Mon, 3 Aug 2026 18:13:21 +0300 Subject: [PATCH 04/31] Void a passing row whose transcript read graded material --- .env.example | 10 ++ docs/REPORT_SCHEMA.md | 41 +++++- src/coder_eval/orchestrator.py | 73 +++++++++++ tests/test_integrity_gate.py | 224 +++++++++++++++++++++++++++++++++ 4 files changed, 344 insertions(+), 4 deletions(-) create mode 100644 tests/test_integrity_gate.py diff --git a/.env.example b/.env.example index 136c4b34..05d19d9c 100644 --- a/.env.example +++ b/.env.example @@ -80,6 +80,16 @@ LOG_TO_FILE=false # Set to true to enable file logging # Off by default. Do NOT enable on developer workstations. # CODER_EVAL_REMEDIATE_HOME_PLUGINS=0 +# Run-integrity gate. Every row gets a verdict recording whether its score measured +# the agent or a leak: the agent reading graded material (reference solution, checker +# script, its own task YAML). +# detect (default) -- record the verdict + findings on the row, change no outcome +# void -- also downgrade a tainted SUCCESS to FAILURE +# off -- skip the pass entirely +# Read a real run's `detect` findings before switching to `void`. See +# docs/REPORT_SCHEMA.md "Run integrity". +# INTEGRITY_MODE=detect + # Usage Telemetry (anonymous; on by default). # Disable entirely: # TELEMETRY_ENABLED=false diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index df4bf740..f0576a93 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -79,10 +79,43 @@ including: `task_id`, `replicate_index`, `variant_id`, `status` `expected_commands`, `actual_commands`, `commands_efficiency`, `agent_config`, `sdk_options`, `installed_tools`, turn accounting (`total_turns`, `visible_turns`, `expected_turns`, -`max_turns_exhausted`, `has_final_reply`), and early-stop fields (`stopped_early`, -`early_stop_reason`, `turns_remaining_at_stop`). `iterations` here is a **reduced** -turn digest (`{iteration, duration_seconds, command_count, assistant_turn_count, -crashed, crash_reason}`) — the full transcript is in `task.json`. +`max_turns_exhausted`, `has_final_reply`), early-stop fields (`stopped_early`, +`early_stop_reason`, `turns_remaining_at_stop`), and run-integrity fields +(`integrity_verdict`, `integrity_voided`, `integrity_findings`). `iterations` here is +a **reduced** turn digest (`{iteration, duration_seconds, command_count, +assistant_turn_count, crashed, crash_reason}`) — the full transcript is in `task.json`. + +The row key set is pinned by `tests/test_reports_experiment.py::TestRowKeySet`. +`eval_result_to_task_dict` is a hand-maintained dict literal, so a new +`EvaluationResult` field reaches `task.json` for free but **not** `run.json` unless it +is added there too. + +### Run integrity + +Every row carries an integrity verdict: was the score a measurement of the agent, or an +artifact of the agent reaching graded material — a reference solution, a checker script, +its own task definition? + +| `integrity_verdict` | Meaning | +| --- | --- | +| `clean` | The scan saw the whole transcript and found nothing. | +| `tainted` | A command read graded material. | +| `inconclusive` | The scan could not see the whole transcript — unrecovered sub-agent tool calls, too many commands with no parameters, or an unrecognised tool touching graded material. Never voids. | +| `skipped` | The pass did not run (`INTEGRITY_MODE=off`, or no graded material was derivable for the task). | + +`INTEGRITY_MODE` decides what a `tainted` verdict does: `detect` (the default) records +the verdict and findings and changes no outcome; `void` additionally downgrades a +tainted `SUCCESS` to `FAILURE` and sets `integrity_voided`; `off` skips the pass. Roll +out with `detect` and read a real run's findings before switching to `void`. + +`weighted_score` is left as computed on a voided row: the high score IS the finding (the +row passed *because* it cheated), so erasing it would destroy the evidence. Taint is +never a `FinalStatus` member — the terminal-status set is closed, and integrity rides +alongside the status as orthogonal telemetry, the way early-stop does. + +`task.json` carries the full `integrity` object: every finding with its coordinates +(iteration, command index, tool, evidence excerpt) plus the blind-spot counters +(`commands_scanned`, `commands_without_parameters`, `subagent_recovery_incomplete`). ### Missing cost is never fatal diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 8640dd7f..b261cf04 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -28,6 +28,7 @@ from .errors.executor import execute_with_retry from .errors.retry import create_error_context from .evaluation.checker import SuccessChecker, _short_failure_reason +from .integrity import evaluate_integrity from .litellm_cost import apply_actual_cost, load_cost_records from .models import ( DEFAULT_STOP_EARLY_GATE_THRESHOLD, @@ -40,6 +41,8 @@ DirectRoute, EvaluationResult, FinalStatus, + IntegrityMode, + IntegrityVerdict, JudgeCriterionResult, LiteLLMRoute, PostRunCommand, @@ -662,6 +665,71 @@ async def _drain_killed_turn(self) -> None: except Exception: logger.warning("[%s] Could not recover the hard-killed turn", self.task.task_id, exc_info=True) + def _apply_integrity_gate(self) -> None: + """Record the integrity verdict and, under ``void``, downgrade a tainted pass. + + Runs as a pass over the finished result rather than as a criterion type: + criteria are opt-in per task YAML across a hundred-plus files, and the + author whose fixtures leak is exactly the author who will not add one. + + What the gate does and does not do: + + * ``SUCCESS`` + ``TAINTED`` + ``mode=void`` -> ``FAILURE``, ``voided=True``. + Every other combination leaves the status alone. ``INCONCLUSIVE`` never + voids -- "the scan could not see everything" is not evidence of a leak. + * ``weighted_score`` is left exactly as computed. On a tainted row the + high score IS the finding (it passed *because* it cheated), so erasing + it would destroy the evidence the row exists to carry. + * No new ``FinalStatus`` member. The terminal-status set is closed (two + exhaustive maps whose module-level asserts break import, plus every + consumer that switches on it); taint rides alongside the status as + orthogonal telemetry, the way early-stop does. + + Never raises: this runs inside ``run()``'s ``finally``, so an integrity + bug must not cost the row its ``task.json``. + """ + if self.result is None: + return + + try: + info = evaluate_integrity( + self.task, + self.task_file, + self.result.iterations, + mode=settings.integrity_mode, + ) + except Exception: + logger.warning("[%s] Integrity pass failed; leaving the row ungated", self.task.task_id, exc_info=True) + return + + self.result.integrity = info + + if info.verdict is not IntegrityVerdict.TAINTED: + if info.notes: + logger.debug("[%s] Integrity %s: %s", self.task.task_id, info.verdict.value, "; ".join(info.notes)) + return + + summary = "; ".join(f.detail for f in info.findings[:3]) + if info.mode is IntegrityMode.VOID and self.result.final_status is FinalStatus.SUCCESS: + info.voided = True + self.result.final_status = FinalStatus.FAILURE + if not self.result.error_message: + self.result.error_message = f"Score voided: agent read graded material. {summary}" + logger.error( + "[%s] VOIDED a passing row: the agent read graded material (%d finding(s)). %s", + self.task.task_id, + len(info.findings), + summary, + ) + else: + logger.warning( + "[%s] Integrity TAINTED (mode=%s, status=%s; not voided): %s", + self.task.task_id, + info.mode.value, + self.result.final_status.value, + summary, + ) + def _finalize_result(self, start_time: float) -> None: """Finalize the evaluation result: scores, telemetry, and persistence.""" if not self.result: @@ -701,6 +769,11 @@ def _finalize_result(self, start_time: float) -> None: if self.result.iterations: self.result.command_stats = calculate_command_statistics(self.result.iterations) + # Run integrity: did this row measure the agent, or a leak? Placed here -- + # after criteria have produced a status and a score, before persistence -- + # so the gate can act on the verdict and task.json records it either way. + self._apply_integrity_gate() + # Resolve model_used (last turn with model wins, then agent config) if self.result.iterations: for turn in reversed(self.result.iterations): diff --git a/tests/test_integrity_gate.py b/tests/test_integrity_gate.py new file mode 100644 index 00000000..7d5d6e09 --- /dev/null +++ b/tests/test_integrity_gate.py @@ -0,0 +1,224 @@ +"""Tests for the integrity gate in Orchestrator._apply_integrity_gate. + +The gate is the only place a verdict changes an outcome, so these pin the exact +status transitions -- especially the ones that must NOT happen: INCONCLUSIVE +never voids, `detect` never voids, and a non-SUCCESS row is never rewritten. +""" + +from __future__ import annotations + +from datetime import datetime + +import pytest + +from coder_eval.models import ( + AgentKind, + CommandTelemetry, + EvaluationResult, + FinalStatus, + IntegrityMode, + IntegrityVerdict, + TaskDefinition, + TurnRecord, +) +from coder_eval.orchestrator import Orchestrator + + +def _task() -> TaskDefinition: + return TaskDefinition( + task_id="leaky", + description="d", + initial_prompt="p", + success_criteria=[{"type": "file_exists", "description": "x", "path": "out.txt"}], + ) + + +def _bash(command: str) -> CommandTelemetry: + return CommandTelemetry( + tool_name="Bash", + tool_id="t0", + timestamp=datetime.now(), + parameters={"command": command}, + result_status="success", + ) + + +def _orchestrator(tmp_path, *, commands: list[CommandTelemetry], status: FinalStatus) -> Orchestrator: + orch = Orchestrator(task=_task(), run_dir=tmp_path / "run", variant_id="t") + orch.result = EvaluationResult( + task_id="leaky", + task_description="d", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime.now(), + final_status=status, + iteration_count=1, + weighted_score=1.0, + iterations=[TurnRecord(iteration=1, user_input="p", agent_output="a", commands=commands)], + ) + return orch + + +_LEAK = "cat RESOLUTION.md" +_CLEAN = "ls -la" + + +def test_void_mode_downgrades_a_tainted_pass(tmp_path, monkeypatch): + from coder_eval import orchestrator as orch_mod + + monkeypatch.setattr(orch_mod.settings, "integrity_mode", IntegrityMode.VOID) + orch = _orchestrator(tmp_path, commands=[_bash(_LEAK)], status=FinalStatus.SUCCESS) + + orch._apply_integrity_gate() + + assert orch.result.final_status is FinalStatus.FAILURE + assert orch.result.integrity.verdict is IntegrityVerdict.TAINTED + assert orch.result.integrity.voided is True + assert orch.result.error_message is not None + assert "voided" in orch.result.error_message.lower() + + +def test_voiding_preserves_the_weighted_score(tmp_path, monkeypatch): + """The score is the diagnostic: the row passed BECAUSE it cheated.""" + from coder_eval import orchestrator as orch_mod + + monkeypatch.setattr(orch_mod.settings, "integrity_mode", IntegrityMode.VOID) + orch = _orchestrator(tmp_path, commands=[_bash(_LEAK)], status=FinalStatus.SUCCESS) + + orch._apply_integrity_gate() + + assert orch.result.weighted_score == 1.0 + + +def test_detect_mode_records_but_never_voids(tmp_path, monkeypatch): + from coder_eval import orchestrator as orch_mod + + monkeypatch.setattr(orch_mod.settings, "integrity_mode", IntegrityMode.DETECT) + orch = _orchestrator(tmp_path, commands=[_bash(_LEAK)], status=FinalStatus.SUCCESS) + + orch._apply_integrity_gate() + + assert orch.result.final_status is FinalStatus.SUCCESS + assert orch.result.integrity.verdict is IntegrityVerdict.TAINTED + assert orch.result.integrity.voided is False + assert orch.result.error_message is None + + +def test_off_mode_skips_entirely(tmp_path, monkeypatch): + from coder_eval import orchestrator as orch_mod + + monkeypatch.setattr(orch_mod.settings, "integrity_mode", IntegrityMode.OFF) + orch = _orchestrator(tmp_path, commands=[_bash(_LEAK)], status=FinalStatus.SUCCESS) + + orch._apply_integrity_gate() + + assert orch.result.final_status is FinalStatus.SUCCESS + assert orch.result.integrity.verdict is IntegrityVerdict.SKIPPED + assert orch.result.integrity.findings == [] + + +def test_clean_run_is_recorded_clean_and_untouched(tmp_path, monkeypatch): + from coder_eval import orchestrator as orch_mod + + monkeypatch.setattr(orch_mod.settings, "integrity_mode", IntegrityMode.VOID) + orch = _orchestrator(tmp_path, commands=[_bash(_CLEAN)], status=FinalStatus.SUCCESS) + + orch._apply_integrity_gate() + + assert orch.result.final_status is FinalStatus.SUCCESS + assert orch.result.integrity.verdict is IntegrityVerdict.CLEAN + assert orch.result.integrity.voided is False + + +def test_inconclusive_never_voids(tmp_path, monkeypatch): + """A partially-blind scan is not evidence of a leak.""" + from coder_eval import orchestrator as orch_mod + + monkeypatch.setattr(orch_mod.settings, "integrity_mode", IntegrityMode.VOID) + orch = _orchestrator(tmp_path, commands=[_bash(_CLEAN)], status=FinalStatus.SUCCESS) + orch.result.iterations[0].unrecovered_subagent_threads = 1 + + orch._apply_integrity_gate() + + assert orch.result.integrity.verdict is IntegrityVerdict.INCONCLUSIVE + assert orch.result.final_status is FinalStatus.SUCCESS + assert orch.result.integrity.voided is False + + +@pytest.mark.parametrize( + "status", + [ + FinalStatus.FAILURE, + FinalStatus.ERROR, + FinalStatus.TIMEOUT, + FinalStatus.MAX_TURNS_EXHAUSTED, + FinalStatus.TOKEN_BUDGET_EXCEEDED, + FinalStatus.COST_BUDGET_EXCEEDED, + FinalStatus.BUILD_FAILED, + ], +) +def test_non_success_statuses_are_never_rewritten(tmp_path, monkeypatch, status: FinalStatus): + """Only a PASS can be voided; a row that already failed keeps its own reason.""" + from coder_eval import orchestrator as orch_mod + + monkeypatch.setattr(orch_mod.settings, "integrity_mode", IntegrityMode.VOID) + orch = _orchestrator(tmp_path, commands=[_bash(_LEAK)], status=status) + + orch._apply_integrity_gate() + + assert orch.result.final_status is status + assert orch.result.integrity.verdict is IntegrityVerdict.TAINTED + assert orch.result.integrity.voided is False + + +def test_existing_error_message_is_not_overwritten(tmp_path, monkeypatch): + from coder_eval import orchestrator as orch_mod + + monkeypatch.setattr(orch_mod.settings, "integrity_mode", IntegrityMode.VOID) + orch = _orchestrator(tmp_path, commands=[_bash(_LEAK)], status=FinalStatus.SUCCESS) + orch.result.error_message = "an earlier, more specific reason" + + orch._apply_integrity_gate() + + assert orch.result.error_message == "an earlier, more specific reason" + + +def test_gate_survives_an_integrity_failure(tmp_path, monkeypatch): + """An integrity bug must not cost the row its task.json.""" + from coder_eval import orchestrator as orch_mod + + monkeypatch.setattr(orch_mod.settings, "integrity_mode", IntegrityMode.VOID) + + def _boom(*_args, **_kwargs): + raise RuntimeError("integrity exploded") + + monkeypatch.setattr(orch_mod, "evaluate_integrity", _boom) + orch = _orchestrator(tmp_path, commands=[_bash(_LEAK)], status=FinalStatus.SUCCESS) + + orch._apply_integrity_gate() # must not raise + + assert orch.result.final_status is FinalStatus.SUCCESS + assert orch.result.integrity.verdict is IntegrityVerdict.SKIPPED + + +def test_gate_is_a_no_op_without_a_result(tmp_path): + orch = Orchestrator(task=_task(), run_dir=tmp_path / "run", variant_id="t") + orch._apply_integrity_gate() # must not raise + assert orch.result is None + + +def test_task_file_widens_the_spec_to_the_task_yaml(tmp_path, monkeypatch): + """With the task YAML known, reading it is a leak; without it, the glob still catches the name.""" + from coder_eval import orchestrator as orch_mod + + monkeypatch.setattr(orch_mod.settings, "integrity_mode", IntegrityMode.VOID) + task_file = tmp_path / "scenario" / "task.yaml" + task_file.parent.mkdir(parents=True) + task_file.write_text("task_id: leaky\n", encoding="utf-8") + + orch = _orchestrator(tmp_path, commands=[_bash(f"cat {task_file.as_posix()}")], status=FinalStatus.SUCCESS) + orch.task_file = task_file + + orch._apply_integrity_gate() + + assert orch.result.integrity.verdict is IntegrityVerdict.TAINTED + assert orch.result.final_status is FinalStatus.FAILURE From 18b2397c75657cdf346dfb9375d307215c0a283b Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Tue, 4 Aug 2026 09:19:21 +0300 Subject: [PATCH 05/31] Stop handing the scenario folder and the raw task YAML to the container --- .../cli/run_task_internal_command.py | 8 +- src/coder_eval/isolation/docker_runner.py | 82 ++++-- tests/test_docker_stage_inputs.py | 278 ++++++++++++++++++ 3 files changed, 348 insertions(+), 20 deletions(-) create mode 100644 tests/test_docker_stage_inputs.py diff --git a/src/coder_eval/cli/run_task_internal_command.py b/src/coder_eval/cli/run_task_internal_command.py index 3157e4a7..394a062b 100644 --- a/src/coder_eval/cli/run_task_internal_command.py +++ b/src/coder_eval/cli/run_task_internal_command.py @@ -159,9 +159,11 @@ def _watch_host_heartbeat() -> None: workspace_dir_raw = context.get("workspace_dir") workspace_dir = Path(workspace_dir_raw) if workspace_dir_raw else None config_lineage = {k: ConfigLineageEntry.model_validate(v) for k, v in (context.get("config_lineage") or {}).items()} - # Prefer the host's raw source_yaml so task.json's audit trail matches - # the in-process driver. Fall back to the staged (post-override) YAML - # for older host versions that didn't forward it. + # The host no longer stages its raw `source_yaml` (a second verbatim copy of the + # task's success_criteria inside the sandbox, read by nothing here). Record the + # staged post-override YAML instead; the host restores its own raw text onto + # task.json.task_config.source_yaml when it parses the result back, so the audit + # trail is unchanged. An older host that still sends the key is honored. host_source_yaml: str | None = context.get("source_yaml") # Load the post-override spec from the staged YAML. We then point diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py index 5185494d..6d1b0606 100644 --- a/src/coder_eval/isolation/docker_runner.py +++ b/src/coder_eval/isolation/docker_runner.py @@ -18,6 +18,7 @@ import subprocess import tempfile import uuid +from collections.abc import Collection from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, TextIO @@ -281,6 +282,23 @@ def _sanitize_container_name_component(s: str) -> str: _RESERVED_MOUNT_DESTS = RESERVED_CONTAINER_DIRS +def _warn_if_sensitive_mount(target: Path, sensitive_sources: Collection[Path]) -> None: + """Warn when an auto-mounted host path looks like a credential/secret location. + + Warning, not a hard failure: legitimate uses exist (a task that really does + want to read ``~/.aws/config``). The point is to surface the surprise, since + ``plugin.path`` / ``reference.directory`` / ``template_sources`` are + user-controlled strings where a typo silently exposes the host. + """ + for sensitive in sensitive_sources: + if target == sensitive or sensitive in target.parents: + logger.warning( + "Auto-mounting sensitive host path %s into container; fix task YAML if unintended.", + target, + ) + return + + def _validate_extra_mount(spec: str) -> str: """Sanity-check a ``-v`` mount spec and return a normalized form. @@ -620,16 +638,21 @@ def _dump_task_yaml() -> str: await asyncio.to_thread(task_yaml_in.write_text, task_yaml_text, encoding="utf-8") # Lineage + variant metadata so the in-container Orchestrator # reconstructs the same context (variant_id is load-bearing for - # report grouping). source_yaml carries the *raw* on-disk text - # so the in-container Orchestrator records the same audit trail - # as the in-process driver (task.json.task_config.source_yaml). + # report grouping). + # + # `source_yaml` (the raw on-disk task text) is deliberately NOT staged. It + # is a pure audit field -- nothing in the container reads it, it only ends + # up in task.json.task_config.source_yaml -- and staging it put a second + # verbatim copy of the task's success_criteria inside the sandbox for no + # functional gain. The host owns that text already and stitches it back + # into the returned result (`_restore_source_yaml`), so the audit trail is + # unchanged. context_payload = json.dumps( { "variant_id": self.rt.variant_id, "replicate_index": self.rt.replicate_index, "config_lineage": {k: v.model_dump(mode="json") for k, v in self.rt.config_lineage.items()}, "preservation_mode": self.preservation_mode.value, - "source_yaml": self.rt.source_yaml, # Docker WORKDIR alignment: concrete path the in-container # orchestrator runs at + captures out (None = standard workspace). "workspace_dir": self._workspace_dir, @@ -771,8 +794,34 @@ async def _parse_result_or_raise(self, output_dir: Path, returncode: int, log_pa # rather than crashing with an uncaught ValidationError/JSONDecodeError. raise await self._handle_malformed_task_json(task_json, log_path, exc) from exc self._warn_on_version_mismatch(result) + await self._restore_source_yaml(result, task_json) return result + async def _restore_source_yaml(self, result: EvaluationResult, task_json: Path) -> None: + """Put the host's raw task YAML back on the returned record's audit field. + + ``source_yaml`` is not staged into the container (see ``_stage_inputs``), so + the in-container Orchestrator records the post-override dump it was handed + instead of the raw on-disk text. The host has that text, so it restores it + here -- keeping ``task.json.task_config.source_yaml`` identical to what the + in-process driver writes. + + Rewrites ``task.json`` as well as the in-memory record, because the file is + the artifact downstream consumers read. Best-effort: a failed rewrite is + logged and leaves the in-memory record corrected, never failing the task + over an audit field. + """ + if result.task_config is None or not self.rt.source_yaml: + return + if result.task_config.source_yaml == self.rt.source_yaml: + return + + result.task_config.source_yaml = self.rt.source_yaml + try: + await asyncio.to_thread(task_json.write_text, result.model_dump_json(indent=2), encoding="utf-8") + except OSError as exc: + logger.warning("Could not rewrite %s with the host source_yaml: %s", task_json, exc) + async def _handle_malformed_task_json(self, task_json: Path, log_path: Path, exc: ValueError) -> DockerRunError: """Degrade a present-but-malformed task.json; return the DockerRunError to raise. @@ -1195,7 +1244,9 @@ def _build_argv( # resolve_template_paths runs on the host). # Reference files (`task.reference.file`) and `run_command` # criteria that use `$TASK_DIR/...` are covered by the symmetric - # task_dir mount above. ``mounted`` dedupes overlapping entries. + # task_dir mount above. ``mounted`` dedupes overlapping entries; it holds + # both directory and single-file targets, since a file is now mounted as + # a file rather than widened to its parent directory. mounted: set[Path] = set() # Auto-mount sources that look like credential / secret dirs get a # loud warning. Task YAMLs typically come from in-house suite authors, @@ -1209,19 +1260,16 @@ def _build_argv( def _auto_mount(raw_path: str | None, *, dir_only: bool = True) -> None: if not raw_path: return - resolved = Path(os.path.expandvars(os.path.expanduser(raw_path))).resolve() - # File paths get mounted as the parent dir so a single -v covers - # the file; container-side reads still resolve at the same path. - target = resolved if (dir_only or resolved.is_dir()) else resolved.parent - if target in mounted or not target.is_dir(): + target = Path(os.path.expandvars(os.path.expanduser(raw_path))).resolve() + # A file is mounted AS a file. Widening it to its parent directory (the + # old behavior) exposed every sibling to satisfy a request for one file + # -- for a reference solution that is the whole scenario folder, + # RESOLUTION.md and checker scripts included. Docker creates the + # container-side path for a file bind mount, so nothing else is needed. + # `dir_only` therefore only decides what kind of target is acceptable. + if target in mounted or not (target.is_dir() or (not dir_only and target.is_file())): return - for sensitive in sensitive_sources: - if target == sensitive or sensitive in target.parents: - logger.warning( - "Auto-mounting sensitive host path %s into container; fix task YAML if unintended.", - target, - ) - break + _warn_if_sensitive_mount(target, sensitive_sources) mounted.add(target) argv.extend(["-v", f"{target}:{target}:ro"]) diff --git a/tests/test_docker_stage_inputs.py b/tests/test_docker_stage_inputs.py new file mode 100644 index 00000000..eae1d1dc --- /dev/null +++ b/tests/test_docker_stage_inputs.py @@ -0,0 +1,278 @@ +"""Tests for what the docker driver puts inside the container. + +Platform-neutral on purpose: ``tests/test_docker_runner_mounts.py`` is skipped +wholesale on Windows (its ``~`` / ``$VAR`` expansion and ``:`` mount-spec +assertions are POSIX-shaped), which means the containment behavior asserted here +would never execute on a Windows developer machine. ``_build_argv`` and +``_stage_inputs`` are pure — argv formatting and file I/O — so they run anywhere. +""" + +from __future__ import annotations + +import asyncio +import json +import tempfile +from datetime import datetime +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import yaml + +from coder_eval.isolation.docker_runner import DockerRunner +from coder_eval.models import ( + AgentKind, + ConfigLineageEntry, + EvaluationResult, + FileExistsCriterion, + FinalStatus, + PreservationMode, + SandboxConfig, + TaskConfigRecord, + TaskDefinition, +) + + +def _task(**kwargs) -> TaskDefinition: + base = { + "task_id": "test", + "description": "test task", + "initial_prompt": "test", + "sandbox": SandboxConfig(), + "success_criteria": [FileExistsCriterion(description="c", path="t.txt")], + } + base.update(kwargs) + return TaskDefinition(**base) + + +def _runner(task: TaskDefinition, *, task_file: Path | None = None, source_yaml: str = "") -> DockerRunner: + rt = MagicMock() + rt.task = task + rt.run_dir = Path(tempfile.gettempdir()) / "test_run" + rt.task_file = task_file + rt.variant_id = "default" + rt.replicate_index = 0 + rt.config_lineage = {} + rt.source_yaml = source_yaml + return DockerRunner(rt) + + +def _mounts(argv: list[str]) -> list[str]: + return [argv[i + 1] for i, arg in enumerate(argv) if arg == "-v" and i + 1 < len(argv)] + + +class TestAutoMountNarrowsFilesToFiles: + """A single declared file must not drag its whole directory into the container. + + ``reference.file`` lives beside the rest of a scenario: its RESOLUTION.md, its + ``check_*.py`` graders, its fixtures. Mounting the parent directory to satisfy + a request for one file hands the agent all of it. + """ + + def test_reference_file_is_mounted_as_a_file(self, tmp_path): + scenario = tmp_path / "scenario" + scenario.mkdir() + reference = scenario / "solution.py" + reference.write_text("print('answer')\n", encoding="utf-8") + (scenario / "RESOLUTION.md").write_text("the root cause is X\n", encoding="utf-8") + + runner = _runner(_task(reference={"file": str(reference)})) + argv = runner._build_argv(tmp_path / "in", tmp_path / "out", container_name="c") + mounts = _mounts(argv) + + assert f"{reference.resolve()}:{reference.resolve()}:ro" in mounts + assert f"{scenario.resolve()}:{scenario.resolve()}:ro" not in mounts + + def test_the_sibling_answer_key_is_not_reachable_through_the_mount(self, tmp_path): + scenario = tmp_path / "scenario" + scenario.mkdir() + reference = scenario / "solution.py" + reference.write_text("x\n", encoding="utf-8") + + runner = _runner(_task(reference={"file": str(reference)})) + mounts = _mounts(runner._build_argv(tmp_path / "in", tmp_path / "out", container_name="c")) + + # No mount source is an ancestor of the scenario dir, so nothing else in it + # is exposed by this reference. + for mount in mounts: + source = mount.rsplit(":", 1)[0].rsplit(":", 1)[0] if mount.count(":") > 2 else mount.split(":")[0] + assert Path(source) != scenario.resolve() + + def test_reference_directory_is_still_mounted_as_a_directory(self, tmp_path): + ref_dir = tmp_path / "_reference" + ref_dir.mkdir() + (ref_dir / "a.py").write_text("x\n", encoding="utf-8") + + runner = _runner(_task(reference={"directory": str(ref_dir)})) + mounts = _mounts(runner._build_argv(tmp_path / "in", tmp_path / "out", container_name="c")) + + assert f"{ref_dir.resolve()}:{ref_dir.resolve()}:ro" in mounts + + def test_system_prompt_file_is_mounted_as_a_file(self, tmp_path): + prompt = tmp_path / "prompts" / "system.md" + prompt.parent.mkdir() + prompt.write_text("be helpful\n", encoding="utf-8") + (prompt.parent / "secret.md").write_text("nope\n", encoding="utf-8") + + runner = _runner(_task(agent={"type": "claude-code", "system_prompt_file": str(prompt)})) + mounts = _mounts(runner._build_argv(tmp_path / "in", tmp_path / "out", container_name="c")) + + assert f"{prompt.resolve()}:{prompt.resolve()}:ro" in mounts + assert f"{prompt.parent.resolve()}:{prompt.parent.resolve()}:ro" not in mounts + + def test_a_missing_file_is_not_mounted(self, tmp_path): + runner = _runner(_task(reference={"file": str(tmp_path / "absent.py")})) + mounts = _mounts(runner._build_argv(tmp_path / "in", tmp_path / "out", container_name="c")) + assert not any("absent.py" in m for m in mounts) + + def test_plugin_directories_are_still_mounted(self, tmp_path): + plugin = tmp_path / "skills-repo" + plugin.mkdir() + + runner = _runner(_task(agent={"type": "claude-code", "plugins": [{"type": "local", "path": str(plugin)}]})) + mounts = _mounts(runner._build_argv(tmp_path / "in", tmp_path / "out", container_name="c")) + + assert f"{plugin.resolve()}:{plugin.resolve()}:ro" in mounts + + def test_a_file_is_not_mounted_twice(self, tmp_path): + reference = tmp_path / "solution.py" + reference.write_text("x\n", encoding="utf-8") + + runner = _runner( + _task( + reference={"file": str(reference)}, + agent={"type": "claude-code", "system_prompt_file": str(reference)}, + ) + ) + mounts = _mounts(runner._build_argv(tmp_path / "in", tmp_path / "out", container_name="c")) + + spec = f"{reference.resolve()}:{reference.resolve()}:ro" + assert mounts.count(spec) == 1 + + +class TestStageInputs: + """What lands in the host-side staging dir that becomes ``/work/input``.""" + + def _stage(self, runner: DockerRunner, input_dir: Path) -> None: + input_dir.mkdir(parents=True, exist_ok=True) + asyncio.run(runner._stage_inputs(input_dir)) + + def test_task_yaml_carries_the_post_override_definition(self, tmp_path): + runner = _runner(_task(task_id="after-overrides")) + self._stage(runner, tmp_path / "in") + + staged = yaml.safe_load((tmp_path / "in" / "task.yaml").read_text(encoding="utf-8")) + assert staged["task_id"] == "after-overrides" + + def test_context_json_carries_the_run_context(self, tmp_path): + runner = _runner(_task()) + runner.rt.variant_id = "arm-b" + runner.rt.replicate_index = 2 + runner.rt.config_lineage = {"agent.model": ConfigLineageEntry(value="m", source="task")} + runner.preservation_mode = PreservationMode.DIRECT_WRITE + self._stage(runner, tmp_path / "in") + + context = json.loads((tmp_path / "in" / "context.json").read_text(encoding="utf-8")) + assert context["variant_id"] == "arm-b" + assert context["replicate_index"] == 2 + assert context["preservation_mode"] == "DIRECT_WRITE" + assert "agent.model" in context["config_lineage"] + + @pytest.mark.parametrize("key", ["variant_id", "replicate_index", "config_lineage", "preservation_mode"]) + def test_context_keys_the_container_reads_are_present(self, tmp_path, key): + runner = _runner(_task()) + self._stage(runner, tmp_path / "in") + context = json.loads((tmp_path / "in" / "context.json").read_text(encoding="utf-8")) + assert key in context + + def test_source_yaml_is_not_staged_into_the_container(self, tmp_path): + """The raw task YAML is a second verbatim copy of the answer key, read by nothing. + + It only ever fed ``task.json.task_config.source_yaml`` -- an audit field the + HOST fills in from its own copy once the container returns. Staging it put + the full success_criteria text inside the sandbox for no functional gain. + """ + raw = "task_id: test\nsuccess_criteria:\n - type: file_exists\n path: the-answer.txt\n" + runner = _runner(_task(), source_yaml=raw) + self._stage(runner, tmp_path / "in") + + context = json.loads((tmp_path / "in" / "context.json").read_text(encoding="utf-8")) + assert "source_yaml" not in context + assert b"the-answer.txt" not in (tmp_path / "in" / "context.json").read_bytes() + + +class TestRestoreSourceYaml: + """The audit field must survive not being staged. + + Dropping ``source_yaml`` from the staged context is only safe if the host puts + its own copy back, otherwise ``task.json.task_config.source_yaml`` silently + becomes the post-override dump instead of the raw on-disk text. + """ + + RAW = "task_id: test\ndescription: from disk\n" + + def _result(self, staged_yaml: str) -> EvaluationResult: + return EvaluationResult( + task_id="test", + task_description="d", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime.now(), + final_status=FinalStatus.SUCCESS, + iteration_count=1, + task_config=TaskConfigRecord(resolved={}, source_yaml=staged_yaml), + ) + + def test_host_raw_yaml_replaces_the_staged_dump(self, tmp_path): + runner = _runner(_task(), source_yaml=self.RAW) + result = self._result("task_id: test\ndescription: post-override dump\n") + task_json = tmp_path / "task.json" + task_json.write_text(result.model_dump_json(), encoding="utf-8") + + asyncio.run(runner._restore_source_yaml(result, task_json)) + + assert result.task_config is not None + assert result.task_config.source_yaml == self.RAW + + def test_the_artifact_on_disk_is_rewritten_too(self, tmp_path): + """task.json is what downstream consumers read, not the in-memory record.""" + runner = _runner(_task(), source_yaml=self.RAW) + result = self._result("task_id: test\ndescription: post-override dump\n") + task_json = tmp_path / "task.json" + task_json.write_text(result.model_dump_json(), encoding="utf-8") + + asyncio.run(runner._restore_source_yaml(result, task_json)) + + reloaded = EvaluationResult.model_validate_json(task_json.read_text(encoding="utf-8")) + assert reloaded.task_config is not None + assert reloaded.task_config.source_yaml == self.RAW + + def test_no_rewrite_when_the_host_has_no_source_yaml(self, tmp_path): + runner = _runner(_task(), source_yaml="") + result = self._result("staged") + task_json = tmp_path / "task.json" + task_json.write_text("{}", encoding="utf-8") + + asyncio.run(runner._restore_source_yaml(result, task_json)) + + assert result.task_config is not None + assert result.task_config.source_yaml == "staged" + assert task_json.read_text(encoding="utf-8") == "{}" + + def test_no_rewrite_when_already_equal(self, tmp_path): + runner = _runner(_task(), source_yaml=self.RAW) + result = self._result(self.RAW) + task_json = tmp_path / "task.json" + task_json.write_text("{}", encoding="utf-8") + + asyncio.run(runner._restore_source_yaml(result, task_json)) + + assert task_json.read_text(encoding="utf-8") == "{}" + + def test_an_unwritable_artifact_does_not_fail_the_task(self, tmp_path): + runner = _runner(_task(), source_yaml=self.RAW) + result = self._result("staged") + + asyncio.run(runner._restore_source_yaml(result, tmp_path / "missing-dir" / "task.json")) + + assert result.task_config is not None + assert result.task_config.source_yaml == self.RAW From 53891fc6d721ead2f7409950ac4003aea45395db Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Tue, 4 Aug 2026 11:38:59 +0300 Subject: [PATCH 06/31] Derive graded material from the task's declared mock and fixture directories --- src/coder_eval/integrity.py | 113 +++++++++++++++++++++++++++++-- src/coder_eval/models/results.py | 11 ++- tests/test_integrity_scan.py | 76 +++++++++++++++++++++ 3 files changed, 190 insertions(+), 10 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 2e07ba43..7eb3ed3b 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -175,15 +175,34 @@ # work would produce. _GRADED_BASENAME_GLOBS = ("RESOLUTION.md", "check_*.py", "*.expected", "task.yaml", "context.json") +# Path SEGMENTS that hold answer keys wherever they appear. Segments rather than +# resolved prefixes because this is how an agent types them -- `cat +# _fixtures/expected/foo.json`, `../_fixtures/solution/main.py` -- so an absolute +# prefix derived on the host matches neither spelling. +_GRADED_PATH_SEGMENTS = ("_fixtures",) + +# Path segments holding the scenario's mock/fixture DATA: the recorded responses a +# mock shim replays and the shim itself. Reported as MOCK_DATA_READ rather than +# GRADED_READ -- reading the fixture store is a different act from reading the +# reference answer, and the first rollout runs record-only, so the two have to be +# triageable apart. +_MOCK_PATH_SEGMENTS = ("mocks", "mock_src") + @dataclass(frozen=True) class GradedMaterialSpec: """What counts as graded material for one task. Derived from what the harness already knows -- the task file, its declared - reference, the ``$TASK_DIR`` operands its own criteria use, and the - framework's container mounts -- rather than from hardcoded suite paths, so it - stays correct as suites are added and renamed. + reference, the mock/fixture directories it declares, the ``$TASK_DIR`` + operands its own criteria use, and the framework's container mounts -- rather + than from hardcoded suite paths, so it stays correct as suites are added and + renamed. + + Two match shapes, because agents type two shapes: ``paths`` / ``directories`` + are resolved and substring-matched, while ``path_segments`` / + ``mock_segments`` are path COMPONENTS matched wherever they occur, so a + relative ``../mocks/responses/manifest.json`` is caught as well. """ paths: frozenset[str] = field(default_factory=frozenset) @@ -195,9 +214,16 @@ class GradedMaterialSpec: basename_globs: tuple[str, ...] = () """Filename patterns that are graded material regardless of location.""" + path_segments: frozenset[str] = field(default_factory=frozenset) + """Answer-key path components (``_fixtures``), matched anywhere in a path.""" + + mock_segments: frozenset[str] = field(default_factory=frozenset) + """Mock/fixture-store path components (declared mock dirs, staged fixture + mount points), matched anywhere in a path and reported as MOCK_DATA_READ.""" + def is_empty(self) -> bool: """Whether the spec would match nothing at all.""" - return not (self.paths or self.directories or self.basename_globs) + return not (self.paths or self.directories or self.basename_globs or self.path_segments or self.mock_segments) def _normalize(text: str) -> str: @@ -223,6 +249,52 @@ def _glob_to_regex(glob: str) -> re.Pattern[str]: return re.compile(body) +def _segment_to_regex(segment: str) -> re.Pattern[str]: + """Compile a path segment into a pattern that matches it as a path COMPONENT. + + ``m`` must match ``m/.store`` and ``/m/.store`` but not + ``stream/x``, so the component is anchored on its trailing separator plus a + leading boundary that rejects any character a path component could continue + from. A leading separator is NOT required: agents open these paths relatively + and quoted (``open('m/.store')``). + """ + return re.compile(r"(? set[str]: + """Normalize a declared sandbox-relative directory into matchable segments. + + Returns the declared path itself plus its root component: a task that declares + ``mocks/bin`` as its shim directory still keeps the recorded responses it + replays under ``mocks/``. ``.`` (the sandbox root) yields nothing -- every + read would match it. + """ + parts = [p for p in _normalize(raw).split("/") if p not in ("", ".")] + if not parts: + return set() + return {"/".join(parts), parts[0]} + + +def _declared_mock_segments(task: TaskDefinition) -> set[str]: + """Mock/fixture directories this task DECLARES, as path segments. + + Two declarations locate a scenario's fixture store: ``sandbox.mock_path_dirs`` + (the directories whose contents the harness makes executable and prepends to + the agent's PATH -- the shims) and ``template_sources[*].mount_point`` (where a + staged tree lands). A mount point is only as precise as the task made it: one + that points at the agent's own working tree widens the spec to that tree, which + is why these are reported as MOCK_DATA_READ and not folded into GRADED_READ. + """ + segments: set[str] = set() + for raw in task.sandbox.mock_path_dirs or []: + segments.update(_path_segments(raw)) + for source in task.sandbox.template_sources or []: + mount_point = getattr(source, "mount_point", None) + if mount_point: + segments.update(_path_segments(mount_point)) + return segments + + def derive_graded_material(task: TaskDefinition, task_file: Path | None) -> GradedMaterialSpec: """Work out this task's graded material from the harness's own configuration. @@ -256,10 +328,18 @@ def derive_graded_material(task: TaskDefinition, task_file: Path | None) -> Grad for command in (*criterion_commands, *hook_commands): paths.update(_task_dir_operands(command or "")) + # Mock/fixture stores. Declared per task (mock dirs, staged mount points) plus + # the two conventions no task spells out: `_fixtures/` holds golden solutions + # and `mock_src/` the fixture sources. + mock_segments = _declared_mock_segments(task) | set(_MOCK_PATH_SEGMENTS) + path_segments = set(_GRADED_PATH_SEGMENTS) + return GradedMaterialSpec( paths=frozenset(paths), directories=frozenset(directories), basename_globs=_GRADED_BASENAME_GLOBS, + path_segments=frozenset(path_segments), + mock_segments=frozenset(mock_segments - path_segments), ) @@ -283,7 +363,9 @@ def _find_match(text: str, spec: GradedMaterialSpec) -> str | None: """Return the graded-material reference found in ``text``, or None. Literal paths and directory prefixes are substring-matched on the normalized - form; basename globs are regex-matched so ``check_*.py`` catches any grader. + form; basename globs are regex-matched so ``check_*.py`` catches any grader; + path segments are matched as a path component so a relatively-typed + ``../mocks/responses/manifest.json`` is caught too. """ haystack = _normalize(text) @@ -298,9 +380,24 @@ def _find_match(text: str, spec: GradedMaterialSpec) -> str | None: for glob in spec.basename_globs: if _glob_to_regex(glob).search(haystack): return glob + for segment in (*spec.path_segments, *spec.mock_segments): + if _segment_to_regex(segment).search(haystack): + return segment return None +def _finding_kind(matched: str, spec: GradedMaterialSpec) -> IntegrityFindingKind: + """Which class of finding a matched reference produces. + + Only the mock/fixture-store segments are MOCK_DATA_READ; everything else -- + the reference solution, the task YAML, the grader, a golden solution under + ``_fixtures/`` -- is a read of the answer key itself. + """ + if matched in spec.mock_segments: + return IntegrityFindingKind.MOCK_DATA_READ + return IntegrityFindingKind.GRADED_READ + + def _segment_utility(segment: str) -> tuple[str, list[str]]: """Leading utility of a shell segment (basename, lowercased) and its tokens. @@ -454,10 +551,12 @@ def scan_commands(turns: list[TurnRecord], spec: GradedMaterialSpec) -> Integrit ) if is_read and matched is not None: + kind = _finding_kind(matched, spec) + subject = "mock fixture data" if kind is IntegrityFindingKind.MOCK_DATA_READ else "graded material" findings.append( IntegrityFinding( - kind=IntegrityFindingKind.GRADED_READ, - detail=f"{cmd.tool_name} read graded material ({matched})", + kind=kind, + detail=f"{cmd.tool_name} read {subject} ({matched})", iteration=turn.iteration, command_index=index, tool_name=cmd.tool_name, diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index df3090e1..ca63b70a 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -521,12 +521,17 @@ class IntegrityVerdict(StrEnum): class IntegrityFindingKind(StrEnum): """The class of integrity problem a finding records. - One member today: every finding is a read of graded material. The kind is - carried on the finding anyway so a second check can be added without - reshaping the row key ``integrity_findings`` already ships. + Both members are reads; they differ in WHAT was read, and that difference is + load-bearing. ``GRADED_READ`` is the answer key -- the reference solution, the + task YAML, the grader script, a golden solution. ``MOCK_DATA_READ`` is the + scenario's fixture store: the recorded responses a mock shim replays, and the + shim itself. The first rollout of the integrity pass is record-only, and "the + agent read the fixture store" is a different conversation from "the agent read + the answer", so triage has to be able to separate them from the row alone. """ GRADED_READ = "graded_read" + MOCK_DATA_READ = "mock_data_read" class IntegrityFinding(BaseModel): diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 2172daa4..acba2806 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -335,6 +335,82 @@ def test_derivation_harvests_task_dir_operands_from_hooks(): assert "$TASK_DIR/expected.json" in spec.paths +def test_derivation_picks_up_declared_mock_dirs(): + """The mock store is declared in `sandbox.mock_path_dirs`, nowhere else.""" + spec = derive_graded_material(_task(sandbox={"mock_path_dirs": ["m"]}), None) + assert "m" in spec.mock_segments + + +def test_derivation_picks_up_a_staged_fixture_mount_point(): + task = _task( + sandbox={"template_sources": [{"type": "template_dir", "path": "fixtures", "mount_point": "stubs/uip"}]} + ) + spec = derive_graded_material(task, None) + # Both the declared mount point and its root: the fixtures sit under either. + assert {"stubs/uip", "stubs"} <= spec.mock_segments + + +def test_derivation_ignores_a_sandbox_root_mount_point(): + """`mount_point: .` is the whole sandbox; treating it as fixture data would + make every read of the agent's own work a finding.""" + task = _task(sandbox={"template_sources": [{"type": "template_dir", "path": "starter", "mount_point": "."}]}) + spec = derive_graded_material(task, None) + assert spec.mock_segments == frozenset({"mocks", "mock_src"}) + + +def test_derivation_always_covers_the_fixture_conventions(): + spec = derive_graded_material(_task(), None) + assert "_fixtures" in spec.path_segments + assert {"mocks", "mock_src"} <= spec.mock_segments + + +# -------------------------------------------------------------------------- +# The three measured leak classes the spec used to miss entirely +# -------------------------------------------------------------------------- + + +def test_a_golden_solution_under_fixtures_is_a_graded_read(): + """Another task's `_fixtures/` solution: an answer key, not fixture data.""" + spec = derive_graded_material(_task(), None) + info = scan_commands([_turn([_bash("cat ../broken-flow/_fixtures/expected/RESOLUTION_body.txt")])], spec) + assert info.verdict is IntegrityVerdict.TAINTED + assert info.findings[0].kind is IntegrityFindingKind.GRADED_READ + + +def test_the_mock_manifest_and_shim_are_mock_data_reads(): + spec = derive_graded_material(_task(sandbox={"mock_path_dirs": ["mocks"]}), None) + info = scan_commands( + [_turn([_bash("cat ../mocks/responses/manifest.json"), _bash("sed -n '1,80p' mocks/uip")])], + spec, + ) + assert info.verdict is IntegrityVerdict.TAINTED + assert len(info.findings) == 2 + assert {f.kind for f in info.findings} == {IntegrityFindingKind.MOCK_DATA_READ} + + +def test_a_sealed_fixture_store_decode_is_flagged(): + """The measured shape: decompress the sealed store in a python one-liner.""" + spec = derive_graded_material(_task(sandbox={"mock_path_dirs": ["m"]}), None) + command = "python -c \"import base64,zlib;print(zlib.decompress(base64.b64decode(open('m/.store','rb').read())))\"" + info = scan_commands([_turn([_bash(command)])], spec) + assert info.verdict is IntegrityVerdict.TAINTED + assert info.findings[0].kind is IntegrityFindingKind.MOCK_DATA_READ + + +@pytest.mark.parametrize( + "command", + [ + pytest.param("cat program/main.py", id="segment-inside-another-component"), + pytest.param("cat streams/m.json", id="segment-as-a-filename"), + pytest.param("cat my_mocks_notes.md", id="segment-inside-a-basename"), + ], +) +def test_a_segment_only_matches_a_whole_path_component(command: str): + spec = derive_graded_material(_task(sandbox={"mock_path_dirs": ["m"]}), None) + info = scan_commands([_turn([_bash(command)])], spec) + assert info.verdict is IntegrityVerdict.CLEAN + + @pytest.mark.parametrize( ("command", "expected"), [ From b2c8d8ad322fc05c0ad8a7280329945a5c8ac992 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Tue, 4 Aug 2026 11:42:12 +0300 Subject: [PATCH 07/31] Match a grader script by its resolved task-dir path and without an underscore --- src/coder_eval/integrity.py | 89 +++++++++++++++++++++++++++++++----- tests/test_integrity_scan.py | 46 ++++++++++++++++++- 2 files changed, 122 insertions(+), 13 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 7eb3ed3b..1d2bdaa4 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -173,7 +173,19 @@ # this particular task declares. Deliberately short: each entry is a name the # framework or the task-authoring convention owns, never a name an agent's own # work would produce. -_GRADED_BASENAME_GLOBS = ("RESOLUTION.md", "check_*.py", "*.expected", "task.yaml", "context.json") +_GRADED_BASENAME_GLOBS = ("RESOLUTION.md", "*.expected", "task.yaml", "context.json") + +# Grader-script names. `check.py` (no underscore) grades live tasks too, so the +# underscore cannot be required -- but neither name is the task's alone: `check.py` +# is also perfectly ordinary application code. These are therefore matched only +# when the path locates them in the task's own directory (:func:`_grader_match`), +# unlike the basename globs above. +_GRADER_SCRIPT_GLOBS = ("check_*.py", "check.py") + +# Directory markers that put a grader script in the task's own directory: the +# suite layout every task YAML lives under, and both spellings of the framework's +# task-dir variable. The resolved task directory is added per task. +_GRADER_DIR_MARKERS = ("tests/tasks/", "$task_dir/", f"{CONTAINER_TASK_DIR}/") # Path SEGMENTS that hold answer keys wherever they appear. Segments rather than # resolved prefixes because this is how an agent types them -- `cat @@ -221,9 +233,23 @@ class GradedMaterialSpec: """Mock/fixture-store path components (declared mock dirs, staged fixture mount points), matched anywhere in a path and reported as MOCK_DATA_READ.""" + grader_globs: tuple[str, ...] = () + """Grader-script patterns, matched only under the task directory.""" + + task_dir: str | None = None + """The task's own directory, when known -- the location a grader-script match + must carry to count.""" + def is_empty(self) -> bool: """Whether the spec would match nothing at all.""" - return not (self.paths or self.directories or self.basename_globs or self.path_segments or self.mock_segments) + return not ( + self.paths + or self.directories + or self.basename_globs + or self.path_segments + or self.mock_segments + or self.grader_globs + ) def _normalize(text: str) -> str: @@ -310,6 +336,7 @@ def derive_graded_material(task: TaskDefinition, task_file: Path | None) -> Grad """ paths: set[str] = set() directories: set[str] = {CONTAINER_INPUT_DIR} + task_dir = task_file.parent if task_file is not None else None if task_file is not None: paths.add(str(task_file)) @@ -326,7 +353,7 @@ def derive_graded_material(task: TaskDefinition, task_file: Path | None) -> Grad criterion_commands = [getattr(c, "command", "") for c in task.success_criteria] hook_commands = [c.command for c in (*task.pre_run, *task.post_run)] for command in (*criterion_commands, *hook_commands): - paths.update(_task_dir_operands(command or "")) + paths.update(_task_dir_operands(command or "", task_dir)) # Mock/fixture stores. Declared per task (mock dirs, staged mount points) plus # the two conventions no task spells out: `_fixtures/` holds golden solutions @@ -340,22 +367,28 @@ def derive_graded_material(task: TaskDefinition, task_file: Path | None) -> Grad basename_globs=_GRADED_BASENAME_GLOBS, path_segments=frozenset(path_segments), mock_segments=frozenset(mock_segments - path_segments), + grader_globs=_GRADER_SCRIPT_GLOBS, + task_dir=str(task_dir) if task_dir is not None else None, ) -def _task_dir_operands(command: str) -> set[str]: +def _task_dir_operands(command: str, task_dir: Path | None = None) -> set[str]: """Extract ``$TASK_DIR``-rooted operands from a framework-run command. - Both the raw form (``$TASK_DIR/check_x.py``, which is what an agent that - discovered the variable would type) and the container-resolved form - (``/work/task_dir/check_x.py``) are returned, since either spelling is the - same read. + Three spellings of the same read, because which one the agent types depends on + the driver: the raw form (``$TASK_DIR/check_x.py``, what an agent that + discovered the variable would use), the container-resolved form + (``/work/task_dir/check_x.py``), and -- when the task directory is known -- the + real on-disk path. Under ``driver: tempdir`` the task lives in the host + checkout and the agent reads THAT path, which neither symbolic form matches. """ operands: set[str] = set() for match in re.finditer(r"\$\{?TASK_DIR\}?(/[^\s'\";|&)]+)", command): suffix = match.group(1) operands.add(f"$TASK_DIR{suffix}") operands.add(f"{CONTAINER_TASK_DIR}{suffix}") + if task_dir is not None: + operands.add(str((task_dir / suffix.lstrip("/")).resolve())) return operands @@ -363,9 +396,10 @@ def _find_match(text: str, spec: GradedMaterialSpec) -> str | None: """Return the graded-material reference found in ``text``, or None. Literal paths and directory prefixes are substring-matched on the normalized - form; basename globs are regex-matched so ``check_*.py`` catches any grader; - path segments are matched as a path component so a relatively-typed - ``../mocks/responses/manifest.json`` is caught too. + form; basename globs are regex-matched wherever they appear; path segments are + matched as a path component so a relatively-typed + ``../mocks/responses/manifest.json`` is caught too; grader scripts are matched + last and only under the task directory (:func:`_grader_match`). """ haystack = _normalize(text) @@ -383,6 +417,39 @@ def _find_match(text: str, spec: GradedMaterialSpec) -> str | None: for segment in (*spec.path_segments, *spec.mock_segments): if _segment_to_regex(segment).search(haystack): return segment + return _grader_match(haystack, spec) + + +def _grader_match(haystack: str, spec: GradedMaterialSpec) -> str | None: + """Find a grader-script reference that is rooted in the task's OWN directory. + + ``check_env.py`` in the agent's working directory is the agent's own helper and + ``check.py`` is ordinary application code, so a bare basename proves nothing: a + grader-glob match only counts when the path it sits in names the task directory + (its resolved path, either ``$TASK_DIR`` spelling, or the ``tests/tasks/`` + segment the suite layout puts on every task path). + + Args: + haystack: Already-normalized command text. + spec: The task's graded material. + + Returns: + The matched glob, or None. + """ + if not spec.grader_globs: + return None + + markers = list(_GRADER_DIR_MARKERS) + if spec.task_dir: + markers.append(_normalize(spec.task_dir).rstrip("/") + "/") + + for glob in spec.grader_globs: + # The directory component is bounded by the token: it cannot run over + # whitespace or a quote into a neighbouring argument. + pattern = re.compile(r"(?P[^\s'\"|;&()]*/)" + _glob_to_regex(glob).pattern) + for match in pattern.finditer(haystack): + if any(marker in match.group("dir") for marker in markers): + return glob return None diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index acba2806..d1685d97 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -35,7 +35,9 @@ SPEC = GradedMaterialSpec( paths=frozenset({"/repo/tasks/leaky/task.yaml", "/repo/tasks/leaky/solution.py", "$TASK_DIR/check_output.py"}), directories=frozenset({"/repo/tasks/leaky/_reference", CONTAINER_INPUT_DIR}), - basename_globs=("RESOLUTION.md", "check_*.py", "*.expected", "task.yaml", "context.json"), + basename_globs=("RESOLUTION.md", "*.expected", "task.yaml", "context.json"), + grader_globs=("check_*.py", "check.py"), + task_dir="/repo/tasks/leaky", ) @@ -68,7 +70,9 @@ def _turn(commands: list[CommandTelemetry], *, iteration: int = 1, unrecovered: pytest.param("head -50 RESOLUTION.md", id="head"), pytest.param("tail -n 5 /repo/tasks/leaky/solution.py", id="tail-absolute-path"), pytest.param("sed -n '1,20p' RESOLUTION.md", id="sed"), - pytest.param("awk '{print}' check_output.py", id="awk-glob"), + pytest.param("awk '{print}' /repo/tasks/leaky/check_output.py", id="awk-grader-under-the-task-dir"), + pytest.param("cat /repo/tests/tasks/other/check.py", id="grader-without-an-underscore"), + pytest.param("cat $TASK_DIR/check.py", id="grader-under-the-task-dir-variable"), pytest.param("python3 $TASK_DIR/check_output.py", id="python-runs-the-grader"), pytest.param("python -c \"print(open('RESOLUTION.md').read())\"", id="python-inline"), pytest.param("node -e \"require('fs').readFileSync('RESOLUTION.md')\"", id="node-inline"), @@ -335,6 +339,44 @@ def test_derivation_harvests_task_dir_operands_from_hooks(): assert "$TASK_DIR/expected.json" in spec.paths +@pytest.mark.parametrize( + "command", + [ + pytest.param("cat check.py", id="bare-check-py"), + pytest.param("cat check_env.py", id="bare-check-glob"), + pytest.param("python3 ./check.py", id="own-working-directory"), + pytest.param("cat src/check.py", id="ordinary-application-code"), + ], +) +def test_a_grader_glob_needs_a_task_directory_component(command: str): + """`check.py` is ordinary application code; only its LOCATION makes it a grader.""" + assert _bash_read(command, SPEC) == (False, None) + + +def test_task_dir_operands_add_the_resolved_spelling(): + task_dir = Path("/repo/tasks/leaky") + operands = _task_dir_operands("python3 $TASK_DIR/check_x.py", task_dir) + assert operands == { + "$TASK_DIR/check_x.py", + "/work/task_dir/check_x.py", + str((task_dir / "check_x.py").resolve()), + } + + +def test_a_tempdir_run_matches_the_resolved_task_dir_path(tmp_path): + """Under `driver: tempdir` the agent reads the real checkout path, which + neither `$TASK_DIR/...` nor `/work/task_dir/...` matches.""" + task_file = tmp_path / "scenario" / "task.yaml" + task = _task( + success_criteria=[{"type": "run_command", "description": "grade", "command": "python3 $TASK_DIR/grade_it.py"}] + ) + spec = derive_graded_material(task, task_file) + resolved = (task_file.parent / "grade_it.py").resolve() + + info = scan_commands([_turn([_bash(f"cat {resolved.as_posix()}")])], spec) + assert info.verdict is IntegrityVerdict.TAINTED + + def test_derivation_picks_up_declared_mock_dirs(): """The mock store is declared in `sandbox.mock_path_dirs`, nowhere else.""" spec = derive_graded_material(_task(sandbox={"mock_path_dirs": ["m"]}), None) From 06ebe51c62ea01f7474bf5138e55c07a216f6831 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Tue, 4 Aug 2026 11:43:56 +0300 Subject: [PATCH 08/31] Stop classifying writes, removals and loop keywords as reads of graded material --- src/coder_eval/integrity.py | 26 +++++++++++++++++++----- tests/test_integrity_scan.py | 39 ++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 1d2bdaa4..516189cd 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -105,6 +105,14 @@ } ) +# Utilities and shell keywords that touch a path without emitting its contents: +# file manipulation (`rm`, `mv`, `chmod`), version control, and the loop / +# conditional keywords a segment can start with (`for f in check_*.py`). Without +# these, rule 7 reads an agent tidying up its own helper script as a leak and voids +# an honest row. `git` is here because its path-taking subcommands (`git add`, +# `git checkout`) manipulate rather than print. +_NEUTRAL_UTILITIES = frozenset({"rm", "mv", "chmod", "git", "for", "while", "if", "do", "done", "then", "fi"}) + # Utilities that emit file CONTENT. A hit inside one of these is a read. _READ_UTILITIES = frozenset( { @@ -516,13 +524,17 @@ def _classify_segment(segment: str, spec: GradedMaterialSpec) -> tuple[bool, str 1. No graded-material reference anywhere in the segment -> not a read. 2. A reference after an input redirect (``< file``) -> a read, whatever the utility is; the shell does the reading. - 3. Any content-emitting utility appearing as a token -> a read. Checked + 3. A reference that appears ONLY after an output redirect (``>`` / ``>>``) -> + not a read: it is the destination the agent is writing, as in + ``cat > check_env.py``. A reference before the redirect still counts. + 4. Any content-emitting utility appearing as a token -> a read. Checked across all tokens, not just the leading one, so ``find … -exec cat {}`` and ``xargs cat`` do not slip past on their wrapper's name. - 4. A search utility restricted to file names or counts -> not a read; + 5. A search utility restricted to file names or counts -> not a read; otherwise a read. - 5. A pure listing/metadata utility -> not a read. - 6. Anything else -> a read. Conservative on purpose: an unrecognised utility + 6. A listing/metadata utility, or a utility that moves, removes or otherwise + manipulates a file without emitting it -> not a read. + 7. Anything else -> a read. Conservative on purpose: an unrecognised utility holding a path to the answer key is more likely a read than not, and a false positive is visible in the finding's evidence while a false negative is invisible. @@ -539,13 +551,17 @@ def _classify_segment(segment: str, spec: GradedMaterialSpec) -> tuple[bool, str if _find_match(after, spec) is not None: return True, matched + before_redirect, redirect, _ = segment.partition(">") + if redirect and _find_match(before_redirect, spec) is None: + return False, matched + if any(name in _READ_UTILITIES for name in normalized_tokens): return True, matched if utility in _SEARCH_UTILITIES: return not _search_is_files_only(tokens), matched - if utility in _LISTING_UTILITIES: + if utility in _LISTING_UTILITIES or utility in _NEUTRAL_UTILITIES: return False, matched return True, matched diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index d1685d97..9a412d86 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -129,6 +129,45 @@ def test_shell_non_reads_are_not_flagged(command: str): assert is_read is False, f"expected NOT a read: {command!r}" +# -------------------------------------------------------------------------- +# False positives: an honest command that only LOOKS like a leak +# -------------------------------------------------------------------------- + +FALSE_POSITIVES = [ + pytest.param("cat > check_env.py", id="writing-your-own-helper"), + pytest.param("cat >> /repo/tasks/leaky/check_env.py", id="appending-to-a-task-dir-path"), + pytest.param("cat > RESOLUTION.md", id="writing-the-deliverable"), + pytest.param("rm -f check_env.py", id="removing-your-own-helper"), + pytest.param("rm -f /repo/tasks/leaky/check_env.py", id="removing-a-task-dir-path"), + pytest.param("mv check_temp.py somewhere/", id="moving-your-own-helper"), + pytest.param("mv /repo/tasks/leaky/check_temp.py somewhere/", id="moving-a-task-dir-path"), + pytest.param("chmod +x /repo/tasks/leaky/check_env.py", id="chmod"), + pytest.param("for f in check_*.py; do echo $f; done", id="globbing-your-own-helpers"), + pytest.param( + "uip solution resources get my-asset --decode --output json", + id="legitimate-decode-flag", + ), +] + + +@pytest.mark.parametrize("command", FALSE_POSITIVES) +def test_honest_commands_are_not_reads(command: str): + """Under `void` these would each destroy a row that measured the agent fairly.""" + is_read, _ = _bash_read(command, SPEC) + assert is_read is False, f"false positive: {command!r}" + + +def test_a_read_before_an_output_redirect_still_counts(): + """The write rule must not become an escape hatch: `cat KEY > mine` is a read.""" + is_read, _ = _bash_read("cat RESOLUTION.md > my_notes.md", SPEC) + assert is_read is True + + +def test_stderr_redirection_does_not_hide_a_read(): + is_read, _ = _bash_read("cat RESOLUTION.md 2>&1", SPEC) + assert is_read is True + + def test_windows_separators_still_match(): """A task file recorded with backslashes must match a forward-slash command.""" spec = GradedMaterialSpec(paths=frozenset({r"C:\repo\tasks\leaky\task.yaml"})) From 4d147d8e16705c0c34855d3c5c6fdcd3d6344d26 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Tue, 4 Aug 2026 11:45:12 +0300 Subject: [PATCH 09/31] Classify the file-editing tools as reads and the producing tools as neutral --- src/coder_eval/integrity.py | 14 ++++++++++++-- tests/test_integrity_scan.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 516189cd..26bc36d1 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -177,6 +177,16 @@ # Structured tools that only enumerate paths. _LISTING_TOOLS = frozenset({"Glob", "LS", "ListDir", "list_dir", "Ls", "glob", "TodoWrite", "Task", "Skill"}) +# Structured tools that edit a file in place. Editing is a read: the tool requires +# the current content to locate what it replaces, and the agent had to have seen +# that content to write the edit. +_EDIT_TOOLS = frozenset({"Edit", "MultiEdit", "NotebookEdit"}) + +# Structured tools that produce content rather than consume it. `Write` names the +# file it creates -- including the deliverable a task asks for, whose name may BE +# graded material -- and `WebFetch` names a URL, not a local path. +_NEUTRAL_TOOLS = frozenset({"Write", "WebFetch"}) + # Basename patterns that are graded material in every suite, independent of what # this particular task declares. Deliberately short: each entry is a name the # framework or the task-authoring convention owns, never a name an agent's own @@ -701,9 +711,9 @@ def _structured_read(cmd: CommandTelemetry, text: str, spec: GradedMaterialSpec) if matched is None: return False, None, True - if cmd.tool_name in _READ_TOOLS: + if cmd.tool_name in _READ_TOOLS or cmd.tool_name in _EDIT_TOOLS: return True, matched, True - if cmd.tool_name in _LISTING_TOOLS: + if cmd.tool_name in _LISTING_TOOLS or cmd.tool_name in _NEUTRAL_TOOLS: return False, matched, True if cmd.tool_name == "Grep": # Claude's Grep returns matching LINES only in content mode; the default diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 9a412d86..72c3fbdd 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -246,6 +246,35 @@ def test_grep_with_context_flag_is_tainted(): assert info.verdict is IntegrityVerdict.TAINTED +@pytest.mark.parametrize("tool_name", ["Edit", "MultiEdit", "NotebookEdit"]) +def test_editing_graded_material_is_a_read(tool_name: str): + """An edit needs the current content to locate what it replaces.""" + info = scan_commands( + [ + _turn( + [_cmd(tool_name, {"file_path": "/repo/tasks/leaky/solution.py", "old_string": "a", "new_string": "b"})] + ) + ], + SPEC, + ) + assert info.verdict is IntegrityVerdict.TAINTED + assert info.findings[0].tool_name == tool_name + + +@pytest.mark.parametrize( + ("tool_name", "parameters"), + [ + pytest.param("Write", {"file_path": "RESOLUTION.md", "content": "my diagnosis"}, id="write-the-deliverable"), + pytest.param("WebFetch", {"url": "https://docs.example.com/RESOLUTION.md"}, id="webfetch"), + ], +) +def test_producing_content_is_not_a_read(tool_name: str, parameters: dict): + """These name what the agent is CREATING; nothing local is opened.""" + info = scan_commands([_turn([_cmd(tool_name, parameters)])], SPEC) + assert info.verdict is IntegrityVerdict.CLEAN + assert info.findings == [] + + def test_unknown_tool_touching_graded_material_is_inconclusive_not_tainted(): """We do not guess at an unrecognised tool's semantics in either direction.""" info = scan_commands([_turn([_cmd("mcp__some__fetch", {"target": "RESOLUTION.md"})])], SPEC) From b2995a9d55e385805cd82e97d694001671897ef8 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Tue, 4 Aug 2026 11:50:20 +0300 Subject: [PATCH 10/31] Exclude voided replicates from the experiment pass rate and report their count --- src/coder_eval/models/experiment.py | 10 ++++++ src/coder_eval/orchestration/experiment.py | 7 +++- src/coder_eval/reports_experiment.py | 30 ++++++++++++---- .../report_snapshots/experiment_replicates.md | 8 ++--- tests/test_experiment_reports.py | 34 +++++++++++++++++++ 5 files changed, 78 insertions(+), 11 deletions(-) diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index 847a8475..39409116 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -272,6 +272,16 @@ class ExperimentResult(BaseModel): # noqa: CE009 -- persisted result model; rou "Empty dict only on deserialized results from before this field existed." ), ) + per_replicate_voided: dict[str, dict[str, list[bool]]] = Field( + default_factory=dict, + description=( + "Whether the integrity gate voided each replicate, keyed variant_id → task_id → " + "[flags], positionally aligned with per_replicate_scores. A voided row keeps its " + "weighted_score on purpose, so the score alone cannot tell a pass from a voided " + "pass and any score-based rate needs this to exclude them. " + "Empty dict only on deserialized results from before this field existed." + ), + ) class ResolvedTask(BaseModel): # noqa: CE009 -- programmatic resolution model, not YAML input diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index f75c585e..9e9c339e 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -815,10 +815,14 @@ def aggregate_results( for tr in task_results: task_variant_reps.setdefault((tr.task_id, tr.variant_id), []).append(tr) - # Collect per-replicate scores keyed variant_id → task_id → [scores] for stats rendering. + # Collect per-replicate scores keyed variant_id → task_id → [scores] for stats + # rendering, plus the integrity gate's void flag per replicate: a voided row keeps + # its score, so a score-based rate has to be told which samples to drop. per_replicate_scores: dict[str, dict[str, list[float]]] = {} + per_replicate_voided: dict[str, dict[str, list[bool]]] = {} for (task_id, variant_id), reps in task_variant_reps.items(): per_replicate_scores.setdefault(variant_id, {})[task_id] = [r.result.weighted_score or 0.0 for r in reps] + per_replicate_voided.setdefault(variant_id, {})[task_id] = [r.result.integrity.voided for r in reps] task_variants: dict[str, list[VariantResult]] = {} for (task_id, variant_id), reps in task_variant_reps.items(): @@ -905,4 +909,5 @@ def aggregate_results( variant_aggregates=variant_aggregates, total_duration_seconds=total_duration, per_replicate_scores=per_replicate_scores, + per_replicate_voided=per_replicate_voided, ) diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index c906ad66..3a5acec5 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -516,26 +516,44 @@ def _win_loss_lines(result: ExperimentResult) -> list[str]: @staticmethod def _replicate_stats_lines(result: ExperimentResult) -> list[str]: """The ``## Replicate Statistics`` block: per-variant bootstrap-CI / Wilson - pass-rate table. Returns ``[]`` when no variant ran more than one replicate.""" + pass-rate table. Returns ``[]`` when no variant ran more than one replicate. + + The pass rate counts SCORES, and the integrity gate deliberately leaves a + voided row's ``weighted_score`` as computed (the high score is the + diagnostic). So a voided replicate would otherwise be counted as a pass here + and a variant whose every row was voided would report 100% in the one table a + reviewer reads when comparing arms. Voided replicates are excluded from the + pass rate and reported in their own column instead. + """ # ── Replicate Statistics (only when any variant ran >1 replicate) ── if not any(ts.replicate_count > 1 for ts in result.task_summaries): return [] lines = ["", "## Replicate Statistics", ""] # Per-variant bootstrap CI + Wilson pass-rate table - lines.append("| Variant | Replicates/task | Mean score | 95% CI | Pass-rate (Wilson 95%) |") - lines.append("|---------|-----------------|------------|--------|------------------------|") + lines.append("| Variant | Replicates/task | Mean score | 95% CI | Pass-rate (Wilson 95%) | Voided |") + lines.append("|---------|-----------------|------------|--------|------------------------|--------|") for vid in result.variant_ids: per_rep = result.per_replicate_scores.get(vid, {}) + voided_flags = result.per_replicate_voided.get(vid, {}) all_scores: list[float] = [s for scores in per_rep.values() for s in scores] - passes = sum(1 for s in all_scores if s >= _REPLICATE_PASS_THRESHOLD) + scored: list[float] = [] + voided = 0 + for task_id, scores in per_rep.items(): + flags = voided_flags.get(task_id, []) + for index, score in enumerate(scores): + if index < len(flags) and flags[index]: + voided += 1 + else: + scored.append(score) + passes = sum(1 for s in scored if s >= _REPLICATE_PASS_THRESHOLD) m, lo, hi = bootstrap_mean_ci(all_scores) - wlo, whi = wilson_interval(passes, len(all_scores)) + wlo, whi = wilson_interval(passes, len(scored)) agg = result.variant_aggregates.get(vid) rep_count = agg.replicate_count if agg else 1 lines.append( f"| {vid} | {rep_count} | {m:.3f} | [{lo:.3f}, {hi:.3f}]" - + f" | {passes}/{len(all_scores)} [{wlo:.2f}, {whi:.2f}] |" + + f" | {passes}/{len(scored)} [{wlo:.2f}, {whi:.2f}] | {voided} |" ) return lines diff --git a/tests/_fixtures/report_snapshots/experiment_replicates.md b/tests/_fixtures/report_snapshots/experiment_replicates.md index 73333d07..5f5c655d 100644 --- a/tests/_fixtures/report_snapshots/experiment_replicates.md +++ b/tests/_fixtures/report_snapshots/experiment_replicates.md @@ -35,10 +35,10 @@ ## Replicate Statistics -| Variant | Replicates/task | Mean score | 95% CI | Pass-rate (Wilson 95%) | -|---------|-----------------|------------|--------|------------------------| -| a | 3 | 0.900 | [0.850, 0.933] | 2/3 [0.21, 0.94] | -| b | 3 | 0.650 | [0.600, 0.683] | 0/3 [0.00, 0.56] | +| Variant | Replicates/task | Mean score | 95% CI | Pass-rate (Wilson 95%) | Voided | +|---------|-----------------|------------|--------|------------------------|--------| +| a | 3 | 0.900 | [0.850, 0.933] | 2/3 [0.21, 0.94] | 0 | +| b | 3 | 0.650 | [0.600, 0.683] | 0/3 [0.00, 0.56] | 0 | ## Paired Comparison diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py index 3e212da5..56693dd4 100644 --- a/tests/test_experiment_reports.py +++ b/tests/test_experiment_reports.py @@ -1090,6 +1090,7 @@ def _make_result( *, replicate_count: int = 1, per_replicate_scores: dict | None = None, + per_replicate_voided: dict | None = None, variant_ids: list[str] | None = None, ): vids = variant_ids or ["a", "b"] @@ -1133,6 +1134,7 @@ def _make_result( variant_aggregates=aggs, total_duration_seconds=20.0, per_replicate_scores=per_replicate_scores or {}, + per_replicate_voided=per_replicate_voided or {}, ) def test_no_section_when_replicate_count_is_one(self): @@ -1162,6 +1164,38 @@ def test_section_contains_per_variant_ci_columns(self): assert "95% CI" in md assert "Pass-rate" in md + def test_voided_replicates_are_excluded_from_the_pass_rate(self): + """The gate leaves a voided row's score intact, so a score-based pass rate + counts it as a pass unless the void flag is honored.""" + per_rep = {"a": {"task-1": [1.0, 1.0, 1.0]}, "b": {"task-1": [1.0, 1.0, 1.0]}} + voided = {"a": {"task-1": [True, True, True]}, "b": {"task-1": [False, False, False]}} + result = self._make_result(replicate_count=3, per_replicate_scores=per_rep, per_replicate_voided=voided) + + md = ExperimentReportGenerator.generate_experiment_report(result) + + assert "| a | 3 | 1.000 | [1.000, 1.000] | 0/0 [0.00, 0.00] | 3 |" in md + assert "| b | 3 | 1.000 | [1.000, 1.000] | 3/3 [0.44, 1.00] | 0 |" in md + + def test_a_partly_voided_variant_keeps_its_honest_replicates(self): + per_rep = {"a": {"task-1": [1.0, 1.0, 0.2]}, "b": {"task-1": [1.0, 1.0, 1.0]}} + voided = {"a": {"task-1": [True, False, False]}} + result = self._make_result(replicate_count=3, per_replicate_scores=per_rep, per_replicate_voided=voided) + + md = ExperimentReportGenerator.generate_experiment_report(result) + + assert "| a | 3 | 0.733" in md # the mean still reports every score + assert "| 1/2 [" in md # the pass rate counts only the two that measured the agent + assert md.count("| Voided |") == 1 + + def test_a_result_without_void_flags_reports_none_voided(self): + """A run.json written before the flag existed must still render.""" + per_rep = {"a": {"task-1": [1.0, 1.0, 1.0]}, "b": {"task-1": [0.5, 0.5, 0.5]}} + result = self._make_result(replicate_count=3, per_replicate_scores=per_rep) + + md = ExperimentReportGenerator.generate_experiment_report(result) + + assert "| a | 3 | 1.000 | [1.000, 1.000] | 3/3 [0.44, 1.00] | 0 |" in md + def test_paired_diff_line_for_two_variants_equal_counts(self): # Two tasks: the paired section pairs their per-task means (n = 2 tasks). per_rep = { From 51556257f78751e2123d73c7bdc3a9ac717440fc Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Tue, 4 Aug 2026 12:00:26 +0300 Subject: [PATCH 11/31] Classify git by subcommand so git show of an answer key is a read --- src/coder_eval/integrity.py | 110 +++++++++++++++++++++++++++++++---- tests/test_integrity_scan.py | 50 ++++++++++++++++ 2 files changed, 150 insertions(+), 10 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 26bc36d1..5939539b 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -106,12 +106,50 @@ ) # Utilities and shell keywords that touch a path without emitting its contents: -# file manipulation (`rm`, `mv`, `chmod`), version control, and the loop / -# conditional keywords a segment can start with (`for f in check_*.py`). Without -# these, rule 7 reads an agent tidying up its own helper script as a leak and voids -# an honest row. `git` is here because its path-taking subcommands (`git add`, -# `git checkout`) manipulate rather than print. -_NEUTRAL_UTILITIES = frozenset({"rm", "mv", "chmod", "git", "for", "while", "if", "do", "done", "then", "fi"}) +# file manipulation (`rm`, `mv`, `chmod`) and the loop / conditional keywords a +# segment can start with (`for f in check_*.py`). Without these, rule 7 reads an +# agent tidying up its own helper script as a leak and voids an honest row. +# `git` is deliberately NOT here -- it is classified per subcommand +# (:func:`_git_is_read`), because half of them print file content. +_NEUTRAL_UTILITIES = frozenset({"rm", "mv", "chmod", "for", "while", "if", "do", "done", "then", "fi"}) + +# `git` subcommands that do NOT emit file content: they stage, record, move or +# report. Everything else -- `show`, `cat-file -p`, `diff`, `blame`, `grep`, +# `log -p`, and any subcommand not listed here -- is treated as a read. +# `git show HEAD:` prints a file as surely as `cat` does, so a blanket +# neutral `git` would be a one-command bypass for an agent that knows the detector +# exists. Unlisted subcommands default to READ for the same reason rule 7 does: a +# false positive is visible in the finding's evidence, a false negative is not. +_GIT_NEUTRAL_SUBCOMMANDS = frozenset( + { + "add", + "commit", + "status", + "rm", + "mv", + "checkout", + "restore", + "stash", + "init", + "branch", + "remote", + "config", + "clone", + "fetch", + "pull", + "push", + "tag", + "reset", + "clean", + } +) + +# `git` global options that take a separate value, so the token after them is not +# the subcommand. +_GIT_VALUE_OPTIONS = frozenset({"-C", "-c", "--git-dir", "--work-tree", "--namespace", "--exec-path"}) + +# Flags that turn `git log` into a content-emitting command. +_GIT_PATCH_FLAGS = frozenset({"-p", "-u", "--patch"}) # Utilities that emit file CONTENT. A hit inside one of these is a read. _READ_UTILITIES = frozenset( @@ -507,6 +545,49 @@ def _segment_utility(segment: str) -> tuple[str, list[str]]: return "", tokens +def _git_subcommand(tokens: list[str]) -> str | None: + """The subcommand of a ``git`` invocation, or None when none is identifiable. + + Global options are stepped over, including the ones that take a separate value + (``git -C /repo show …``), so the returned token is the verb and not a path. + """ + seen_git = False + skip_value = False + for token in tokens: + name = Path(token.replace("\\", "/")).name.casefold().removesuffix(".exe") + if not seen_git: + seen_git = name == "git" + continue + if skip_value: + skip_value = False + continue + if token in _GIT_VALUE_OPTIONS: + skip_value = True + continue + if token.startswith("-"): + continue + return token.casefold() + return None + + +def _git_is_read(tokens: list[str]) -> bool: + """Whether a ``git`` invocation emits file CONTENT. + + ``git show HEAD:`` and ``git cat-file -p`` print a file as surely as + ``cat`` does, while ``git add`` / ``git status`` print nothing of it. ``git log`` + is the one subcommand that is both, decided by its patch flag. An unlisted + subcommand -- and an invocation whose subcommand cannot be found at all -- counts + as a read, matching rule 7: an unrecognised command holding a path to the answer + key is more likely to be reading it than not. + """ + subcommand = _git_subcommand(tokens) + if subcommand is None: + return True + if subcommand == "log": + return any(token in _GIT_PATCH_FLAGS for token in tokens) + return subcommand not in _GIT_NEUTRAL_SUBCOMMANDS + + def _search_is_files_only(tokens: list[str]) -> bool: """Whether a grep/rg invocation reports only file names or match counts.""" for token in tokens: @@ -537,14 +618,17 @@ def _classify_segment(segment: str, spec: GradedMaterialSpec) -> tuple[bool, str 3. A reference that appears ONLY after an output redirect (``>`` / ``>>``) -> not a read: it is the destination the agent is writing, as in ``cat > check_env.py``. A reference before the redirect still counts. - 4. Any content-emitting utility appearing as a token -> a read. Checked + 4. ``git`` -> decided by its subcommand (:func:`_git_is_read`): ``git show`` / + ``cat-file`` / ``diff`` / ``blame`` / ``grep`` / ``log -p`` print content, + ``git add`` / ``status`` / ``checkout`` do not. + 5. Any content-emitting utility appearing as a token -> a read. Checked across all tokens, not just the leading one, so ``find … -exec cat {}`` and ``xargs cat`` do not slip past on their wrapper's name. - 5. A search utility restricted to file names or counts -> not a read; + 6. A search utility restricted to file names or counts -> not a read; otherwise a read. - 6. A listing/metadata utility, or a utility that moves, removes or otherwise + 7. A listing/metadata utility, or a utility that moves, removes or otherwise manipulates a file without emitting it -> not a read. - 7. Anything else -> a read. Conservative on purpose: an unrecognised utility + 8. Anything else -> a read. Conservative on purpose: an unrecognised utility holding a path to the answer key is more likely a read than not, and a false positive is visible in the finding's evidence while a false negative is invisible. @@ -565,6 +649,12 @@ def _classify_segment(segment: str, spec: GradedMaterialSpec) -> tuple[bool, str if redirect and _find_match(before_redirect, spec) is None: return False, matched + # Before the token sweep below: `git`'s own subcommand is the authority on + # whether it printed anything, so `git commit -m "cat the file"` is not a read + # and `git show HEAD:` is. + if utility == "git": + return _git_is_read(tokens), matched + if any(name in _READ_UTILITIES for name in normalized_tokens): return True, matched diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 72c3fbdd..69ffc853 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -157,6 +157,56 @@ def test_honest_commands_are_not_reads(command: str): assert is_read is False, f"false positive: {command!r}" +# -------------------------------------------------------------------------- +# git: neutral or a read depending on the subcommand +# -------------------------------------------------------------------------- + +_GRADER = "tests/tasks/suite/scen/check_thing.py" + +GIT_READS = [ + pytest.param(f"git show HEAD:{_GRADER}", id="show"), + pytest.param(f"git cat-file -p HEAD:{_GRADER}", id="cat-file"), + pytest.param(f"git diff HEAD -- {_GRADER}", id="diff"), + pytest.param(f"git blame {_GRADER}", id="blame"), + pytest.param(f"git grep answer -- {_GRADER}", id="grep"), + pytest.param(f"git log -p -- {_GRADER}", id="log-patch-short"), + pytest.param(f"git log --patch -- {_GRADER}", id="log-patch-long"), + pytest.param(f"git -C /repo show HEAD:{_GRADER}", id="global-option-with-a-value"), + pytest.param(f"git archive HEAD {_GRADER}", id="unlisted-subcommand-defaults-to-read"), + pytest.param(f"git {_GRADER}", id="no-identifiable-subcommand"), +] + +GIT_NEUTRAL = [ + pytest.param("git add check_env.py", id="add-own-helper"), + pytest.param(f"git add {_GRADER}", id="add"), + pytest.param(f"git rm {_GRADER}", id="rm"), + pytest.param(f"git mv {_GRADER} somewhere/", id="mv"), + pytest.param(f"git checkout -- {_GRADER}", id="checkout"), + pytest.param(f"git restore {_GRADER}", id="restore"), + pytest.param(f"git log --oneline -- {_GRADER}", id="log-without-a-patch-flag"), + pytest.param(f"git status --short {_GRADER}", id="status"), + pytest.param(f'git commit -m "cat {_GRADER}"', id="commit-message-mentioning-it"), + pytest.param("git stash", id="stash"), + pytest.param("git branch -a", id="branch"), + pytest.param("git config user.name someone", id="config"), +] + + +@pytest.mark.parametrize("command", GIT_READS) +def test_content_emitting_git_subcommands_are_reads(command: str): + """`git show HEAD:` prints the file as surely as `cat` does; a + blanket-neutral `git` was a one-command bypass.""" + is_read, matched = _bash_read(command, SPEC) + assert is_read is True, f"expected a read: {command!r}" + assert matched is not None + + +@pytest.mark.parametrize("command", GIT_NEUTRAL) +def test_non_content_git_subcommands_are_not_reads(command: str): + is_read, _ = _bash_read(command, SPEC) + assert is_read is False, f"false positive: {command!r}" + + def test_a_read_before_an_output_redirect_still_counts(): """The write rule must not become an escape hatch: `cat KEY > mine` is a read.""" is_read, _ = _bash_read("cat RESOLUTION.md > my_notes.md", SPEC) From e36c92a9e0e9476b5a9adfe0fc37558d1444d553 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Tue, 4 Aug 2026 12:08:59 +0300 Subject: [PATCH 12/31] Anchor a basename glob to a whole filename so it cannot match inside a longer name --- src/coder_eval/integrity.py | 24 +++++++++++++++++---- tests/test_integrity_scan.py | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 5939539b..cb99e1c5 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -320,15 +320,31 @@ def _normalize(text: str) -> str: def _glob_to_regex(glob: str) -> re.Pattern[str]: """Compile a basename glob into a pattern that matches it inside a command. - ``fnmatch.translate`` alone anchors the whole string; the wildcard is also - narrowed so it cannot cross a path separator (``check_*.py`` must not match - ``check_dir/other.py``). + Three adjustments to ``fnmatch.translate``, which anchors the whole string: + + * The trailing ``\\Z`` anchor is dropped so the pattern can be SEARCHED for + inside command text rather than matched against a bare filename. + * The wildcard is narrowed so it cannot cross a path separator + (``check_*.py`` must not match ``check_dir/other.py``). + * Dropping the anchor also drops the filename's right-hand boundary, and + ``translate`` never had a left-hand one, so both are restored as a + lookbehind/lookahead pair. Without them ``RESOLUTION.md`` matches inside + ``RESOLUTION.md.draft`` and ``task.yaml`` inside ``my-task.yaml.bak``, and + an agent gets voided for reading its own scratch file. + + The boundary class is ``[\\w.-]`` on both sides -- exactly the characters a + filename can continue through -- which is also what :func:`_segment_to_regex` + uses. Everything a real reference is delimited BY is therefore still a match: + string start/end, whitespace, ``/`` (so ``./RESOLUTION.md`` and + ``../scen/RESOLUTION.md`` hit, and Windows backslashes too, since + :func:`_normalize` has already turned them into ``/``), quotes, and shell + metacharacters (``;``, ``|``, ``&``, ``)``, ``=``). """ body = fnmatch.translate(_normalize(glob)) # translate() emits `(?s:...)\Z`; strip the anchor and re-scope the wildcard. body = body.removesuffix(r"\Z") body = body.replace(".*", "[^/]*") - return re.compile(body) + return re.compile(r"(? re.Pattern[str]: diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 69ffc853..2b8eb17a 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -233,6 +233,48 @@ def test_glob_wildcard_does_not_cross_a_path_separator(): assert _bash_read("cat check_dir/check_it.py", spec)[0] is True +# -------------------------------------------------------------------------- +# A basename glob matches a WHOLE filename, not a substring of one +# -------------------------------------------------------------------------- + +GLOB_BOUNDARY_READS = [ + pytest.param("cat RESOLUTION.md", id="bare"), + pytest.param("sed -n '1,20p' ../scen/RESOLUTION.md", id="relative-parent-path"), + pytest.param("cat ./RESOLUTION.md", id="dot-slash"), + pytest.param("cat 'RESOLUTION.md'", id="single-quoted"), + pytest.param('cat "RESOLUTION.md"', id="double-quoted"), + pytest.param("head RESOLUTION.md;echo done", id="trailing-semicolon"), + pytest.param("head RESOLUTION.md|head -3", id="trailing-pipe"), + pytest.param("cat /repo/tasks/leaky/check_env.py", id="grader-under-the-task-dir"), + pytest.param("python3 $TASK_DIR/check.py", id="grader-under-the-task-dir-variable"), + pytest.param("cat out.expected", id="suffix-glob"), + pytest.param(r"cat C:\repo\tasks\leaky\RESOLUTION.md", id="windows-separators"), +] + +GLOB_BOUNDARY_CLEAN = [ + pytest.param("cat RESOLUTION.md.draft", id="own-draft-of-the-deliverable"), + pytest.param("cat notes/RESOLUTION.md-backup.txt", id="own-backup-name"), + pytest.param("python tests/tasks/suite/scen/check_env.pyc", id="compiled-bytecode-sibling"), + pytest.param("cat my-task.yaml.bak", id="longer-name-ending-in-the-glob"), + pytest.param("cat my_task.yaml", id="longer-name-prefixed-with-a-word-char"), + pytest.param("cat out.expected.tmp", id="suffix-glob-with-a-longer-name"), +] + + +@pytest.mark.parametrize("command", GLOB_BOUNDARY_READS) +def test_a_whole_filename_match_is_still_a_read(command: str): + is_read, matched = _bash_read(command, SPEC) + assert is_read is True, f"expected a read: {command!r}" + assert matched is not None + + +@pytest.mark.parametrize("command", GLOB_BOUNDARY_CLEAN) +def test_a_glob_inside_a_longer_filename_is_not_a_match(command: str): + """`RESOLUTION.md` inside `RESOLUTION.md.draft` is the agent's own scratch file; + under `void` flagging it destroys an honest row.""" + assert _bash_read(command, SPEC) == (False, None) + + def test_unbalanced_quotes_do_not_skip_the_command(): """A command shlex cannot parse still ran, so it must still be classified.""" is_read, _ = _bash_read("cat 'RESOLUTION.md", SPEC) From bf578cf44d1eb594bcd9f0a6dca256d29126c0fd Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 15:35:25 +0300 Subject: [PATCH 13/31] Exclude voided rows from the headline pass rate, score mean and CI instead of counting them as failures --- src/coder_eval/models/experiment.py | 42 ++++++-- src/coder_eval/orchestration/experiment.py | 35 +++++-- src/coder_eval/reports_experiment.py | 38 +++++--- src/coder_eval/reports_html.py | 11 +++ tests/test_experiment_reports.py | 9 +- tests/test_experiment_runner.py | 106 +++++++++++++++++++++ 6 files changed, 210 insertions(+), 31 deletions(-) diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index 39409116..2d5ed94c 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -191,14 +191,28 @@ class VariantResult(BaseModel): # noqa: CE009 -- persisted result model; round- ge=1, description="Number of replicates aggregated into this VariantResult (1 when repeats disabled).", ) + voided_replicates: int = Field( + default=0, + ge=0, + description=( + "Replicates the integrity gate voided. weighted_score and final_status are " + "folded over the non-voided replicates only; when every replicate was voided " + "(voided_replicates == replicate_count) the row measured nothing but the leak " + "and aggregates exclude it." + ), + ) class VariantAggregate(BaseModel): # noqa: CE009 -- persisted result model; round-trip leniency like models/results.py """Aggregated statistics for a single variant across all tasks. - ``pass_rate`` uses the same denominator as ``RunSummary.pass_rate``: every task - the variant ran, errors included as misses. Otherwise an A/B whose variants - error at different rates compares two different denominators. + ``pass_rate`` uses the same denominator as ``RunSummary.pass_rate`` -- every task + the variant ran, errors included as misses (otherwise an A/B whose variants + error at different rates compares two different denominators) -- EXCEPT for + voided rows. A row the integrity gate voided measured a leak, not the agent, so + it belongs in neither the numerator nor the denominator: it is counted in + ``tasks_voided`` instead, and a variant whose every row was voided has no pass + rate at all (``None``), not 0.0. """ variant_id: str @@ -206,6 +220,14 @@ class VariantAggregate(BaseModel): # noqa: CE009 -- persisted result model; rou tasks_succeeded: int tasks_failed: int tasks_error: int + tasks_voided: int = Field( + default=0, + ge=0, + description=( + "Tasks whose every replicate the integrity gate voided. Excluded from " + "tasks_succeeded/failed/error, average_score and the pass_rate denominator." + ), + ) average_score: float average_duration: float total_tokens: int | None = None @@ -228,16 +250,22 @@ class VariantAggregate(BaseModel): # noqa: CE009 -- persisted result model; rou @model_validator(mode="after") def _check_task_count_invariant(self) -> VariantAggregate: - if self.tasks_succeeded + self.tasks_failed + self.tasks_error != self.tasks_run: - total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error}" + if self.tasks_succeeded + self.tasks_failed + self.tasks_error + self.tasks_voided != self.tasks_run: + total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error} + {self.tasks_voided}" raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}") return self @computed_field # type: ignore[prop-decorator] @property def pass_rate(self) -> float | None: - """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty variant.""" - return self.tasks_succeeded / self.tasks_run if self.tasks_run else None + """``tasks_succeeded / (tasks_run - tasks_voided)`` as a 0-1 fraction. + + ``None`` on an empty variant AND on an all-voided one: a voided row is a + leak measurement, so with every row voided there is nothing to rate -- + reporting 0.0 would read as "the agent failed everything". + """ + scored = self.tasks_run - self.tasks_voided + return self.tasks_succeeded / scored if scored else None class TaskExperimentSummary(BaseModel): # noqa: CE009 -- persisted result model; round-trip leniency like models/results.py diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index 9e9c339e..4133fb80 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -826,10 +826,19 @@ def aggregate_results( task_variants: dict[str, list[VariantResult]] = {} for (task_id, variant_id), reps in task_variant_reps.items(): - scores = [r.result.weighted_score or 0.0 for r in reps] + # A voided replicate's preserved score and gate-assigned FAILURE both measure + # the leak, not the agent, so score and status fold over the honest replicates + # only. When every replicate was voided there is nothing honest to fold: the + # score pins to 0.0 (the preserved scores are inflated by construction, and + # 0.0 keeps the row from winning a best-variant comparison) and the aggregate + # below drops the row from its score and pass-rate entirely via tasks_voided. + # Duration/token/turn metrics stay over ALL replicates -- they measure cost, + # which is real either way. + honest = [r for r in reps if not r.result.integrity.voided] + scores = [r.result.weighted_score or 0.0 for r in honest] non_errored = [r for r in reps if r.result.final_status.category != "error"] durations = [r.result.duration_seconds for r in non_errored] - statuses = [r.result.final_status for r in reps] + statuses = [r.result.final_status for r in (honest or reps)] iter_counts = [r.result.iteration_count for r in reps if r.result.iteration_count is not None] asst_turns = [r.result.total_assistant_turns for r in reps if r.result.total_assistant_turns is not None] token_vals = [r.result.total_token_usage.total_tokens for r in reps if r.result.total_token_usage is not None] @@ -839,7 +848,7 @@ def aggregate_results( variant_result = VariantResult( variant_id=variant_id, task_id=task_id, - weighted_score=sum(scores) / len(scores), + weighted_score=sum(scores) / len(scores) if scores else 0.0, final_status=final_status, duration_seconds=sum(durations), total_tokens=sum(token_vals) if token_vals else None, @@ -848,6 +857,7 @@ def aggregate_results( reference_similarity=ref_similarity, replicate_index=0, # aggregate — points at first replicate for link rendering replicate_count=len(reps), + voided_replicates=len(reps) - len(honest), ) task_variants.setdefault(task_id, []).append(variant_result) @@ -887,15 +897,22 @@ def aggregate_results( token_values = [v.total_tokens for v in vr_list if v.total_tokens is not None] total_tokens = sum(token_values) if token_values else None + # A row whose every replicate was voided has no honest measurement in it: + # it leaves the status buckets and the score/pass-rate denominators and is + # reported in tasks_voided instead. Duration stays -- wall-clock cost is real. + scored_rows = [v for v in vr_list if v.voided_replicates < v.replicate_count] variant_aggregates[vid] = VariantAggregate( variant_id=vid, tasks_run=len(vr_list), - tasks_succeeded=sum(1 for v in vr_list if v.final_status.category == "succeeded"), - tasks_failed=sum(1 for v in vr_list if v.final_status.category == "failed"), - tasks_error=sum(1 for v in vr_list if v.final_status.category == "error"), - tasks_token_budget_exceeded=sum(1 for v in vr_list if v.final_status == FinalStatus.TOKEN_BUDGET_EXCEEDED), - tasks_cost_budget_exceeded=sum(1 for v in vr_list if v.final_status == FinalStatus.COST_BUDGET_EXCEEDED), - average_score=sum(v.weighted_score for v in vr_list) / len(vr_list), + tasks_succeeded=sum(1 for v in scored_rows if v.final_status.category == "succeeded"), + tasks_failed=sum(1 for v in scored_rows if v.final_status.category == "failed"), + tasks_error=sum(1 for v in scored_rows if v.final_status.category == "error"), + tasks_voided=len(vr_list) - len(scored_rows), + tasks_token_budget_exceeded=sum( + 1 for v in scored_rows if v.final_status == FinalStatus.TOKEN_BUDGET_EXCEEDED + ), + tasks_cost_budget_exceeded=sum(1 for v in scored_rows if v.final_status == FinalStatus.COST_BUDGET_EXCEEDED), + average_score=sum(v.weighted_score for v in scored_rows) / len(scored_rows) if scored_rows else 0.0, average_duration=sum(v.duration_seconds / v.replicate_count for v in vr_list) / len(vr_list), total_tokens=total_tokens, replicate_count=vr_list[0].replicate_count if vr_list else 1, diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index 3a5acec5..10224871 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -356,7 +356,18 @@ def _aggregate_count_rows(result: ExperimentResult, show_p_values: bool) -> list row += " | —" lines.append(row + " |") - # Every task the variant ran is in the denominator, errors included. + # Row: Voided (only when the integrity gate voided anything -- a voided row + # measured a leak, so it leaves the pass-rate denominator below). + if any(result.variant_aggregates[vid].tasks_voided > 0 for vid in result.variant_ids): + row = "| Voided (integrity)" + for vid in result.variant_ids: + row += f" | {result.variant_aggregates[vid].tasks_voided}" + if show_p_values: + row += " | —" + lines.append(row + " |") + + # Every task the variant ran is in the denominator, errors included -- + # except voided rows, which measured a leak and are excluded outright. row = "| Pass Rate" for vid in result.variant_ids: rate = result.variant_aggregates[vid].pass_rate @@ -518,12 +529,14 @@ def _replicate_stats_lines(result: ExperimentResult) -> list[str]: """The ``## Replicate Statistics`` block: per-variant bootstrap-CI / Wilson pass-rate table. Returns ``[]`` when no variant ran more than one replicate. - The pass rate counts SCORES, and the integrity gate deliberately leaves a - voided row's ``weighted_score`` as computed (the high score is the - diagnostic). So a voided replicate would otherwise be counted as a pass here - and a variant whose every row was voided would report 100% in the one table a - reviewer reads when comparing arms. Voided replicates are excluded from the - pass rate and reported in their own column instead. + The pass rate and the mean/CI both count SCORES, and the integrity gate + deliberately leaves a voided row's ``weighted_score`` as computed (the high + score is the diagnostic). So a voided replicate would otherwise be counted + as a pass here and inflate the bootstrap mean beside it, and a variant whose + every row was voided would report 100% in the one table a reviewer reads + when comparing arms. Voided replicates are excluded from the pass rate AND + the mean/CI, and reported in their own column instead; a variant with no + honest replicate at all renders "n/a", not a number. """ # ── Replicate Statistics (only when any variant ran >1 replicate) ── if not any(ts.replicate_count > 1 for ts in result.task_summaries): @@ -536,7 +549,6 @@ def _replicate_stats_lines(result: ExperimentResult) -> list[str]: for vid in result.variant_ids: per_rep = result.per_replicate_scores.get(vid, {}) voided_flags = result.per_replicate_voided.get(vid, {}) - all_scores: list[float] = [s for scores in per_rep.values() for s in scores] scored: list[float] = [] voided = 0 for task_id, scores in per_rep.items(): @@ -547,12 +559,14 @@ def _replicate_stats_lines(result: ExperimentResult) -> list[str]: else: scored.append(score) passes = sum(1 for s in scored if s >= _REPLICATE_PASS_THRESHOLD) - m, lo, hi = bootstrap_mean_ci(all_scores) + m, lo, hi = bootstrap_mean_ci(scored) + mean_str = f"{m:.3f}" if scored else "n/a" + ci_str = f"[{lo:.3f}, {hi:.3f}]" if scored else "n/a" wlo, whi = wilson_interval(passes, len(scored)) agg = result.variant_aggregates.get(vid) rep_count = agg.replicate_count if agg else 1 lines.append( - f"| {vid} | {rep_count} | {m:.3f} | [{lo:.3f}, {hi:.3f}]" + f"| {vid} | {rep_count} | {mean_str} | {ci_str}" + f" | {passes}/{len(scored)} [{wlo:.2f}, {whi:.2f}] | {voided} |" ) @@ -657,7 +671,9 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: f"- **Succeeded**: {agg.tasks_succeeded}", failed_line, f"- **Errors**: {agg.tasks_error}", - f"- **Pass Rate**: {pass_rate_str} ({agg.tasks_succeeded}/{agg.tasks_run})", + # Voided rows leave the pass-rate denominator: they measured a leak. + *([f"- **Voided (integrity)**: {agg.tasks_voided}"] if agg.tasks_voided else []), + f"- **Pass Rate**: {pass_rate_str} ({agg.tasks_succeeded}/{agg.tasks_run - agg.tasks_voided})", f"- **Average Score**: {agg.average_score:.3f}", f"- **Average Duration**: {agg.average_duration:.1f}s", f"- **Total Tokens**: {tokens_str}", diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index 48ab5519..d5b6815c 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -1325,6 +1325,17 @@ def _row(label: str, values: list[str], p: str | None) -> str: rows.append(_row("Failed", [str(result.variant_aggregates[vid].tasks_failed) for vid in result.variant_ids], None)) rows.append(_row("Errors", [str(result.variant_aggregates[vid].tasks_error) for vid in result.variant_ids], None)) + # Voided rows measured a leak, not the agent; they leave the pass-rate + # denominator below, so their count is shown only when there are any. + if any(result.variant_aggregates[vid].tasks_voided > 0 for vid in result.variant_ids): + rows.append( + _row( + "Voided (integrity)", + [str(result.variant_aggregates[vid].tasks_voided) for vid in result.variant_ids], + None, + ) + ) + def _pass_rate(vid: str) -> str: rate = result.variant_aggregates[vid].pass_rate return f"{rate * 100:.1f}%" if rate is not None else "n/a" diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py index 56693dd4..e881c1ca 100644 --- a/tests/test_experiment_reports.py +++ b/tests/test_experiment_reports.py @@ -1164,16 +1164,17 @@ def test_section_contains_per_variant_ci_columns(self): assert "95% CI" in md assert "Pass-rate" in md - def test_voided_replicates_are_excluded_from_the_pass_rate(self): + def test_voided_replicates_are_excluded_from_the_pass_rate_and_the_mean(self): """The gate leaves a voided row's score intact, so a score-based pass rate - counts it as a pass unless the void flag is honored.""" + counts it as a pass -- and a score-based mean inflates -- unless the void + flag is honored. An all-voided variant has nothing to average: n/a.""" per_rep = {"a": {"task-1": [1.0, 1.0, 1.0]}, "b": {"task-1": [1.0, 1.0, 1.0]}} voided = {"a": {"task-1": [True, True, True]}, "b": {"task-1": [False, False, False]}} result = self._make_result(replicate_count=3, per_replicate_scores=per_rep, per_replicate_voided=voided) md = ExperimentReportGenerator.generate_experiment_report(result) - assert "| a | 3 | 1.000 | [1.000, 1.000] | 0/0 [0.00, 0.00] | 3 |" in md + assert "| a | 3 | n/a | n/a | 0/0 [0.00, 0.00] | 3 |" in md assert "| b | 3 | 1.000 | [1.000, 1.000] | 3/3 [0.44, 1.00] | 0 |" in md def test_a_partly_voided_variant_keeps_its_honest_replicates(self): @@ -1183,7 +1184,7 @@ def test_a_partly_voided_variant_keeps_its_honest_replicates(self): md = ExperimentReportGenerator.generate_experiment_report(result) - assert "| a | 3 | 0.733" in md # the mean still reports every score + assert "| a | 3 | 0.600" in md # the mean averages only the honest replicates assert "| 1/2 [" in md # the pass rate counts only the two that measured the agent assert md.count("| Voided |") == 1 diff --git a/tests/test_experiment_runner.py b/tests/test_experiment_runner.py index 1ee6f08d..982a2618 100644 --- a/tests/test_experiment_runner.py +++ b/tests/test_experiment_runner.py @@ -13,6 +13,9 @@ ExperimentResult, ExperimentVariant, FinalStatus, + IntegrityInfo, + IntegrityMode, + IntegrityVerdict, PreservationMode, ResolvedTask, TaskResult, @@ -750,3 +753,106 @@ def test_all_replicates_errored_gives_zero_duration(self): vr = result.task_summaries[0].variant_results[0] assert abs(vr.duration_seconds - 0.0) < 1e-9 assert vr.final_status == FinalStatus.ERROR + + +class TestVoidedAggregation: + """A voided row measured a leak, not the agent, so it must leave the headline + pass rate and score entirely -- counting it as a failure is as wrong as + counting it as the pass its preserved score claims.""" + + def _make_tr( + self, + task_id: str, + variant_id: str, + score: float, + status: str = "SUCCESS", + *, + voided: bool = False, + replicate_index: int = 0, + ) -> TaskResult: + result = EvaluationResult( + task_id=task_id, + task_description="d", + variant_id=variant_id, + agent_type="claude-code", + started_at=datetime.now(), + final_status=status, + weighted_score=score, + duration_seconds=10.0, + iteration_count=1, + integrity=IntegrityInfo( + verdict=IntegrityVerdict.TAINTED if voided else IntegrityVerdict.CLEAN, + mode=IntegrityMode.VOID, + voided=voided, + ), + ) + return TaskResult( + task_id=task_id, + variant_id=variant_id, + result=result, + duration=10.0, + replicate_index=replicate_index, + ) + + def test_a_voided_row_is_excluded_not_counted_as_a_failure(self): + """One voided FAILURE with its (deliberately preserved, inflated) score: + it must be an excluded row, not a miss in the denominator.""" + reps = [self._make_tr("t", "v", score=1.0, status="FAILURE", voided=True)] + + result = aggregate_results( + experiment_id="e", description="", variant_ids=["v"], task_results=reps, total_duration=10.0 + ) + + agg = result.variant_aggregates["v"] + assert agg.tasks_run == 1 + assert agg.tasks_voided == 1 + assert agg.tasks_succeeded == 0 + assert agg.tasks_failed == 0 + assert agg.pass_rate is None # all-void has no rate at all, not 0.0 + assert agg.average_score == 0.0 + + def test_an_all_voided_fold_does_not_keep_the_leaked_score(self): + reps = [self._make_tr("t", "v", score=1.0, status="FAILURE", voided=True, replicate_index=i) for i in range(2)] + + result = aggregate_results( + experiment_id="e", description="", variant_ids=["v"], task_results=reps, total_duration=20.0 + ) + + vr = result.task_summaries[0].variant_results[0] + assert vr.weighted_score == 0.0 + assert vr.voided_replicates == 2 + assert vr.replicate_count == 2 + + def test_a_partly_voided_fold_keeps_only_the_honest_replicates(self): + reps = [ + self._make_tr("t", "v", score=1.0, status="FAILURE", voided=True, replicate_index=0), + self._make_tr("t", "v", score=0.6, status="SUCCESS", replicate_index=1), + ] + + result = aggregate_results( + experiment_id="e", description="", variant_ids=["v"], task_results=reps, total_duration=20.0 + ) + + vr = result.task_summaries[0].variant_results[0] + assert abs(vr.weighted_score - 0.6) < 1e-9 # the voided 1.0 does not average in + assert vr.final_status.category == "succeeded" # the voided FAILURE does not fold in + assert vr.voided_replicates == 1 + agg = result.variant_aggregates["v"] + assert agg.tasks_voided == 0 # an honest replicate survived, so the row is scored + assert agg.pass_rate == 1.0 + + def test_a_clean_task_beside_an_all_voided_one_keeps_its_rate(self): + trs = [ + self._make_tr("t1", "v", score=1.0, status="SUCCESS"), + self._make_tr("t2", "v", score=1.0, status="FAILURE", voided=True), + ] + + result = aggregate_results( + experiment_id="e", description="", variant_ids=["v"], task_results=trs, total_duration=20.0 + ) + + agg = result.variant_aggregates["v"] + assert agg.tasks_run == 2 + assert agg.tasks_voided == 1 + assert agg.pass_rate == 1.0 # 1 pass / 1 scored, not 1/2 + assert agg.average_score == 1.0 # the voided task's inflated score does not average in From 4566af9a791d94ad6df061b203831f6756e8ca83 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 15:38:34 +0300 Subject: [PATCH 14/31] Split shell commands on unquoted separators only and drop comments and quoted heredoc bodies --- src/coder_eval/integrity.py | 147 +++++++++++++++++++++++++++++++++-- tests/test_integrity_scan.py | 37 +++++++++ 2 files changed, 178 insertions(+), 6 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index cb99e1c5..0cceacb1 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -67,9 +67,12 @@ # not bloat the row. _MAX_EVIDENCE_CHARS = 240 -# Shell operators that end one command and begin another. `||` precedes `|` so -# regex alternation consumes the two-character form first. -_SEGMENT_SEPARATOR = re.compile(r"\|\||&&|;|\||\n|\r") +# Characters that may precede `#` for it to start a comment (plus start-of-segment). +# A `#` glued to a word (`file#1.txt`) is part of the word, not a comment. +_COMMENT_BOUNDARY = " \t;|&(\n\r" + +# Characters a bare (unquoted) heredoc delimiter word is made of. +_HEREDOC_DELIMITER_CHARS = re.compile(r"[\w.\-]") # Wrappers that delegate to the real utility; skipped when finding the utility # that decides a segment's classification. @@ -789,12 +792,144 @@ def scan_commands(turns: list[TurnRecord], spec: GradedMaterialSpec) -> Integrit ) +def _split_segments(command: str) -> list[str]: + """Split a shell command into the segments that actually EXECUTE. + + A naive split on every ``;``/``|``/newline classifies inert data as commands: + the argument of ``printf '%s\\n' 'harmless; cat KEY'``, the body of a heredoc, + the tail of a comment. Each of those voids an honest row under ``void`` mode, + so the split honors the three shell constructs that make text inert: + + * **Quoting.** ``'…'`` and ``"…"`` spans (and backslash escapes) never + separate; their content stays inside the enclosing segment, where the + leading utility's semantics decide what it means. + * **Comments.** An unquoted ``#`` opening a word discards the rest of the + line -- commented-out text never ran. + * **Heredocs.** A ``<<'EOF'``-style QUOTED delimiter makes the body pure + data, so those lines are dropped. An unquoted delimiter (``< None: + nonlocal prev + text = "".join(current) + current.clear() + prev = "" + if text.strip(): + segments.append(text) + + while i < n: + c = command[i] + if in_single: + current.append(c) + prev = c + in_single = c != "'" + i += 1 + continue + if c == "\\" and i + 1 < n: + current.append(command[i : i + 2]) + prev = command[i + 1] + i += 2 + continue + if in_double: + current.append(c) + prev = c + in_double = c != '"' + i += 1 + continue + if c in "'\"": + in_single = c == "'" + in_double = c == '"' + current.append(c) + prev = c + i += 1 + continue + if c == "#" and (prev == "" or prev in _COMMENT_BOUNDARY): + newline = command.find("\n", i) + i = n if newline == -1 else newline # leave the newline for the branch below + continue + if c == "<" and command[i : i + 2] == "<<" and command[i : i + 3] != "<<<": + # Heredoc operator: record the delimiter and whether it was quoted; + # the body starts after the next unquoted newline. + j = i + 2 + if j < n and command[j] == "-": + j += 1 + while j < n and command[j] in " \t": + j += 1 + quoted = False + delimiter_chars: list[str] = [] + if j < n and command[j] in "'\"": + quote = command[j] + quoted = True + j += 1 + while j < n and command[j] != quote: + delimiter_chars.append(command[j]) + j += 1 + j += 1 # closing quote + elif j < n and command[j] == "\\": + quoted = True + j += 1 + while j < n and _HEREDOC_DELIMITER_CHARS.match(command[j]): + delimiter_chars.append(command[j]) + j += 1 + else: + while j < n and _HEREDOC_DELIMITER_CHARS.match(command[j]): + delimiter_chars.append(command[j]) + j += 1 + if delimiter_chars: + pending_heredocs.append(("".join(delimiter_chars), quoted)) + current.append(command[i:j]) + prev = command[j - 1] + i = j + continue + # `<<` with no delimiter: fall through and treat it as ordinary text. + if c in "\n\r": + _flush() + i += 1 + while pending_heredocs and i < n: + delimiter, quoted = pending_heredocs.pop(0) + while i < n: + end = command.find("\n", i) + end = n if end == -1 else end + line = command[i:end] + i = end + 1 + if line.strip() == delimiter: + break + if not quoted and line.strip(): + segments.append(line) + continue + if command[i : i + 2] in ("&&", "||"): + _flush() + i += 2 + continue + if c in ";|": + _flush() + i += 1 + continue + current.append(c) + prev = c + i += 1 + + _flush() + return segments + + def _bash_read(command: str, spec: GradedMaterialSpec) -> tuple[bool, str | None]: """Classify a shell command by splitting it into segments and judging each.""" mentioned: str | None = None - for segment in _SEGMENT_SEPARATOR.split(command): - if not segment.strip(): - continue + for segment in _split_segments(command): is_read, matched = _classify_segment(segment, spec) if matched is not None: mentioned = matched diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 2b8eb17a..7c67ab69 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -281,6 +281,43 @@ def test_unbalanced_quotes_do_not_skip_the_command(): assert is_read is True +# -------------------------------------------------------------------------- +# Segmentation: quoting, comments and heredocs make text inert, not a command +# -------------------------------------------------------------------------- + +INERT_TEXT_CLEAN = [ + pytest.param("printf '%s\\n' 'harmless; cat RESOLUTION.md'", id="quoted-separator-in-an-argument"), + pytest.param('echo "see; cat RESOLUTION.md" > notes.txt', id="double-quoted-separator"), + pytest.param("echo done # cat RESOLUTION.md", id="trailing-comment"), + pytest.param("# cat RESOLUTION.md", id="whole-line-comment"), + pytest.param("cat > notes.txt <<'EOF'\ncat RESOLUTION.md\nEOF", id="quoted-heredoc-body-is-data"), + pytest.param('cat > notes.txt <<"EOF"\ncat RESOLUTION.md\nEOF', id="double-quoted-heredoc-delimiter"), + pytest.param("cat > notes.txt <<\\EOF\ncat RESOLUTION.md\nEOF", id="backslash-heredoc-delimiter"), + pytest.param("cat > notes.txt <<-'EOF'\n\tcat RESOLUTION.md\n\tEOF", id="dash-heredoc-strips-tabs"), +] + +INERT_TEXT_STILL_READS = [ + pytest.param("echo 'x; y' && cat RESOLUTION.md", id="quoting-does-not-hide-the-next-segment"), + pytest.param("echo done#not-a-comment; cat RESOLUTION.md", id="hash-glued-to-a-word-is-not-a-comment"), + pytest.param("cat > x <<'EOF'\ndata\nEOF\ncat RESOLUTION.md", id="command-after-the-heredoc-terminator"), + pytest.param("cat > x < Date: Wed, 5 Aug 2026 15:41:45 +0300 Subject: [PATCH 15/31] Parse redirects as unquoted operators that consume one word instead of partitioning on any angle bracket --- src/coder_eval/integrity.py | 121 +++++++++++++++++++++++++++++++---- tests/test_integrity_scan.py | 33 ++++++++++ 2 files changed, 143 insertions(+), 11 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 0cceacb1..c6fb52f8 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -607,6 +607,104 @@ def _git_is_read(tokens: list[str]) -> bool: return subcommand not in _GIT_NEUTRAL_SUBCOMMANDS +def _strip_redirects(segment: str) -> tuple[str, list[str]]: + """Remove real output redirects from a segment; collect input-redirect targets. + + The old ``partition(">")`` treated ANY ``>`` as an output redirect -- one + inside an awk program (``awk '$1 > 0' KEY``), a sed pattern, or a plain + operand list (``cat > /tmp/copy KEY``) -- and wrote off everything after it + as a write target, which made "put a ``>`` anywhere" a one-character bypass. + A real redirect is an UNQUOTED operator and consumes exactly one word; every + other operand is still passed to the utility. + + Returns ``(stripped, input_targets)``: the segment with each output-redirect + operator (``>``, ``>>``, ``>&``, ``&>``, fd-prefixed forms) and the single + word it consumes removed, plus the target words of plain ``<`` input + redirects (which the SHELL reads, whatever the utility is). Heredoc + operators (``<<``), here-strings (``<<<``) and process substitution + (``<(…)``) are left in place: their text is data or an executing command, + and either way it must stay visible to the caller's matching. + """ + out: list[str] = [] + input_targets: list[str] = [] + in_single = in_double = False + i = 0 + n = len(segment) + + def _consume_word(j: int) -> tuple[str, int]: + """Read one (possibly quoted) word starting at ``j``; return (word, end).""" + while j < n and segment[j] in " \t": + j += 1 + word: list[str] = [] + quote = "" + while j < n: + ch = segment[j] + if quote: + if ch == quote: + quote = "" + else: + word.append(ch) + j += 1 + continue + if ch in "'\"": + quote = ch + j += 1 + continue + if ch == "\\" and j + 1 < n: + word.append(segment[j + 1]) + j += 2 + continue + if ch in " \t<>|;&": + break + word.append(ch) + j += 1 + return "".join(word), j + + while i < n: + c = segment[i] + if in_single: + out.append(c) + in_single = c != "'" + i += 1 + continue + if c == "\\" and i + 1 < n: + out.append(segment[i : i + 2]) + i += 2 + continue + if in_double: + out.append(c) + in_double = c != '"' + i += 1 + continue + if c in "'\"": + in_single = c == "'" + in_double = c == '"' + out.append(c) + i += 1 + continue + if c == ">" or segment[i : i + 2] == "&>": + j = i + 1 if c == ">" else i + 2 + if j < n and segment[j] == ">": + j += 1 + if j < n and segment[j] == "&": # fd duplication: >&2, 2>&1 + j += 1 + _, j = _consume_word(j) + out.append(" ") + i = j + continue + if c == "<" and segment[i + 1 : i + 2] not in ("<", "("): + word, j = _consume_word(i + 1) + if word: + input_targets.append(word) + out.append(" ") + i = j + continue + out.append(c) + i += 1 + + return "".join(out), input_targets + + def _search_is_files_only(tokens: list[str]) -> bool: """Whether a grep/rg invocation reports only file names or match counts.""" for token in tokens: @@ -632,11 +730,13 @@ def _classify_segment(segment: str, spec: GradedMaterialSpec) -> tuple[bool, str Rules, in order: 1. No graded-material reference anywhere in the segment -> not a read. - 2. A reference after an input redirect (``< file``) -> a read, whatever the - utility is; the shell does the reading. - 3. A reference that appears ONLY after an output redirect (``>`` / ``>>``) -> - not a read: it is the destination the agent is writing, as in - ``cat > check_env.py``. A reference before the redirect still counts. + 2. A reference consumed by an input redirect (``< file``) -> a read, whatever + the utility is; the shell does the reading. + 3. An output redirect (``>`` / ``>>`` / ``>&`` / ``&>``) consumes exactly one + word; a reference that survives redirect stripping is an operand and still + counts, while one appearing ONLY as a write target is the destination the + agent is writing (``cat > check_env.py``) -> not a read. A quoted ``>`` + (inside an awk/sed program) is not a redirect at all. 4. ``git`` -> decided by its subcommand (:func:`_git_is_read`): ``git show`` / ``cat-file`` / ``diff`` / ``blame`` / ``grep`` / ``log -p`` print content, ``git add`` / ``status`` / ``checkout`` do not. @@ -659,13 +759,12 @@ def _classify_segment(segment: str, spec: GradedMaterialSpec) -> tuple[bool, str utility, tokens = _segment_utility(segment) normalized_tokens = [Path(t.replace("\\", "/")).name.casefold().removesuffix(".exe") for t in tokens] - if "<" in segment: - _, _, after = segment.partition("<") - if _find_match(after, spec) is not None: - return True, matched + stripped, input_targets = _strip_redirects(segment) + if any(_find_match(target, spec) is not None for target in input_targets): + return True, matched - before_redirect, redirect, _ = segment.partition(">") - if redirect and _find_match(before_redirect, spec) is None: + if _find_match(stripped, spec) is None: + # Every reference sits in an output-redirect target: a write, not a read. return False, matched # Before the token sweep below: `git`'s own subcommand is the authority on diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 7c67ab69..4b8a99da 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -218,6 +218,39 @@ def test_stderr_redirection_does_not_hide_a_read(): assert is_read is True +# A redirect consumes exactly ONE word, and only when it is a real, unquoted +# operator -- otherwise `>` anywhere in the segment wrote off the whole tail. + +REDIRECT_STILL_READS = [ + pytest.param("cat 2>/dev/null RESOLUTION.md", id="operand-after-a-stderr-redirect"), + pytest.param("cat > /tmp/copy RESOLUTION.md", id="operand-after-the-write-target"), + pytest.param("awk '$1 > 0 {print}' RESOLUTION.md", id="quoted-gt-in-an-awk-program"), + pytest.param("sed -n '/x>y/p' RESOLUTION.md", id="quoted-gt-in-a-sed-pattern"), + pytest.param("awk 'NR>0' RESOLUTION.md", id="quoted-gt-without-spaces"), + pytest.param("cat RESOLUTION.md >> log.txt", id="append-redirect-after-the-operand"), + pytest.param("cat out.txt", id="quoted-operand-before-the-redirect"), +] + +REDIRECT_WRITES_ONLY = [ + pytest.param("echo diagnosis > RESOLUTION.md", id="write-target-only"), + pytest.param("printf 'x' >>RESOLUTION.md", id="append-target-without-a-space"), + pytest.param("ls 2> 'RESOLUTION.md'", id="quoted-write-target"), +] + + +@pytest.mark.parametrize("command", REDIRECT_STILL_READS) +def test_a_redirect_consumes_only_its_target_word(command: str): + is_read, _ = _bash_read(command, SPEC) + assert is_read is True, f"expected a read: {command!r}" + + +@pytest.mark.parametrize("command", REDIRECT_WRITES_ONLY) +def test_a_reference_only_in_a_write_target_is_not_a_read(command: str): + is_read, _ = _bash_read(command, SPEC) + assert is_read is False, f"false positive: {command!r}" + + def test_windows_separators_still_match(): """A task file recorded with backslashes must match a forward-slash command.""" spec = GradedMaterialSpec(paths=frozenset({r"C:\repo\tasks\leaky\task.yaml"})) From a3734642a114a994a037c2b975297be5760e71b6 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 15:43:20 +0300 Subject: [PATCH 16/31] Match literal reference paths and directories as whole paths instead of substrings --- src/coder_eval/integrity.py | 31 ++++++++++++++++++++++++------- tests/test_integrity_scan.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index c6fb52f8..8bfaf6a9 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -350,6 +350,22 @@ def _glob_to_regex(glob: str) -> re.Pattern[str]: return re.compile(r"(? re.Pattern[str]: + """Compile a literal (already-normalized) path into a whole-path pattern. + + A raw substring test lets ``reference.file=solution.py`` match the agent's + own ``solution.py.bak``, and ``reference.directory=_reference`` match a + sibling ``_reference_notes/`` -- both honest files, both voided under + ``void``. A file therefore matches exactly, delimited by the same + ``[\\w.-]`` filename-boundary class :func:`_glob_to_regex` uses; a directory + matches exactly or continues with ``/`` into its contents, never into a + longer name. + """ + body = re.escape(path.rstrip("/") if directory else path) + tail = r"(?=/|(?![\w.\-]))" if directory else r"(?![\w.\-])" + return re.compile(r"(? re.Pattern[str]: """Compile a path segment into a pattern that matches it as a path COMPONENT. @@ -470,21 +486,22 @@ def _task_dir_operands(command: str, task_dir: Path | None = None) -> set[str]: def _find_match(text: str, spec: GradedMaterialSpec) -> str | None: """Return the graded-material reference found in ``text``, or None. - Literal paths and directory prefixes are substring-matched on the normalized - form; basename globs are regex-matched wherever they appear; path segments are - matched as a path component so a relatively-typed - ``../mocks/responses/manifest.json`` is caught too; grader scripts are matched - last and only under the task directory (:func:`_grader_match`). + Literal paths and directory prefixes are matched as whole paths on the + normalized form (:func:`_literal_to_regex`); basename globs are regex-matched + wherever they appear; path segments are matched as a path component so a + relatively-typed ``../mocks/responses/manifest.json`` is caught too; grader + scripts are matched last and only under the task directory + (:func:`_grader_match`). """ haystack = _normalize(text) for candidate in spec.paths: needle = _normalize(candidate) - if needle and needle in haystack: + if needle and _literal_to_regex(needle).search(haystack): return candidate for candidate in spec.directories: needle = _normalize(candidate) - if needle and needle in haystack: + if needle and _literal_to_regex(needle, directory=True).search(haystack): return candidate for glob in spec.basename_globs: if _glob_to_regex(glob).search(haystack): diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 4b8a99da..74e6cf65 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -314,6 +314,39 @@ def test_unbalanced_quotes_do_not_skip_the_command(): assert is_read is True +# -------------------------------------------------------------------------- +# Literal paths and directories match whole paths, not substrings of them +# -------------------------------------------------------------------------- + +LITERAL_BOUNDARY_CLEAN = [ + pytest.param("cat /repo/tasks/leaky/solution.py.bak", id="own-backup-of-the-reference-name"), + pytest.param("cat /repo/tasks/leaky/solution.pyc", id="longer-extension"), + pytest.param("cat /repo/tasks/leaky/_reference_notes/output.txt", id="sibling-dir-sharing-the-prefix"), + pytest.param("cat /repo/tasks/leaky/_reference2/x.txt", id="sibling-dir-with-a-suffix-char"), +] + +LITERAL_BOUNDARY_READS = [ + pytest.param("cat /repo/tasks/leaky/solution.py", id="the-reference-file-itself"), + pytest.param("head '/repo/tasks/leaky/solution.py'", id="quoted"), + pytest.param("cat /repo/tasks/leaky/_reference/answer.py", id="inside-the-reference-directory"), + pytest.param("strange-tool /repo/tasks/leaky/_reference", id="the-reference-directory-itself"), +] + + +@pytest.mark.parametrize("command", LITERAL_BOUNDARY_CLEAN) +def test_a_literal_path_does_not_match_inside_a_longer_name(command: str): + """`solution.py.bak` and `_reference_notes/` are the agent's own files; under + `void` a substring match on them destroys an honest row.""" + assert _bash_read(command, SPEC) == (False, None) + + +@pytest.mark.parametrize("command", LITERAL_BOUNDARY_READS) +def test_a_whole_literal_path_still_matches(command: str): + is_read, matched = _bash_read(command, SPEC) + assert is_read is True, f"expected a read: {command!r}" + assert matched is not None + + # -------------------------------------------------------------------------- # Segmentation: quoting, comments and heredocs make text inert, not a command # -------------------------------------------------------------------------- From b1fea2b835c6a4c966fc4eb34ce7a5b2b431d5fa Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 15:44:54 +0300 Subject: [PATCH 17/31] Match structured tool calls on their path parameters instead of every value joined --- src/coder_eval/integrity.py | 25 +++++++++++++++++++++++-- tests/test_integrity_scan.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 8bfaf6a9..4dd81c33 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -228,6 +228,20 @@ # graded material -- and `WebFetch` names a URL, not a local path. _NEUTRAL_TOOLS = frozenset({"Write", "WebFetch"}) +# Parameter keys that hold filesystem paths in the structured tools we know. +# Everything else a tool call carries is prose or a pattern, not a file it +# opens: `Grep(pattern="RESOLUTION.md", path="src")` searches FOR the name, and +# an Edit whose old_string quotes a graded name has read only the file at its +# file_path. `glob` counts as a path: in content mode it selects which files' +# lines are emitted. +_PATH_PARAMETER_KEYS = ("file_path", "path", "paths", "notebook_path", "glob") + +# The structured tools whose parameter schema this module knows, and may +# therefore narrow to `_PATH_PARAMETER_KEYS`. An unrecognised tool keeps its +# full parameter text: its schema is unknown, and a hit there ends as +# INCONCLUSIVE rather than TAINTED, so over-matching is the safe direction. +_SCHEMA_KNOWN_TOOLS = _READ_TOOLS | _EDIT_TOOLS | _LISTING_TOOLS | _NEUTRAL_TOOLS | frozenset({"Grep"}) + # Basename patterns that are graded material in every suite, independent of what # this particular task declares. Deliberately short: each entry is a name the # framework or the task-authoring convention owns, never a name an agent's own @@ -1059,12 +1073,19 @@ def _structured_read(cmd: CommandTelemetry, text: str, spec: GradedMaterialSpec) Returns ``(is_read, matched, semantics_understood)``. Unlike a shell string, a structured tool has fixed semantics, so the decision is by tool name rather - than by heuristic. A tool this module does not recognise gets + than by heuristic -- and for a tool whose schema is known, only its path + parameters are matched (``_PATH_PARAMETER_KEYS``): patterns, replacement + text and mode switches name graded material without touching it. A tool this + module does not recognise keeps its full parameter text and gets ``semantics_understood=False`` when it touched graded material, which the caller turns into INCONCLUSIVE -- neither a silent pass nor a taint on a tool whose behavior we are guessing at. """ - matched = _find_match(text, spec) + if cmd.tool_name in _SCHEMA_KNOWN_TOOLS: + haystack = " ".join(str(cmd.parameters[key]) for key in _PATH_PARAMETER_KEYS if cmd.parameters.get(key)) + else: + haystack = text + matched = _find_match(haystack, spec) if matched is None: return False, None, True diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 74e6cf65..34bb857b 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -470,6 +470,41 @@ def test_producing_content_is_not_a_read(tool_name: str, parameters: dict): assert info.findings == [] +@pytest.mark.parametrize( + ("tool_name", "parameters"), + [ + pytest.param( + "Grep", + {"pattern": "RESOLUTION.md", "path": "src", "output_mode": "content"}, + id="grep-pattern-naming-the-deliverable", + ), + pytest.param( + "Edit", + {"file_path": "notes.md", "old_string": "see RESOLUTION.md", "new_string": "see report.md"}, + id="edit-prose-naming-the-deliverable", + ), + pytest.param( + "Read", + {"file_path": "notes.md", "limit": 10, "comment": "compare with RESOLUTION.md later"}, + id="read-with-a-non-path-parameter", + ), + ], +) +def test_non_path_parameters_of_known_tools_are_not_matched(tool_name: str, parameters: dict): + """Patterns and prose NAME graded material without opening it; matching every + parameter value as if it were a path voids honest rows.""" + info = scan_commands([_turn([_cmd(tool_name, parameters)])], SPEC) + assert info.verdict is IntegrityVerdict.CLEAN + assert info.findings == [] + + +def test_a_path_parameter_of_a_known_tool_still_matches(): + info = scan_commands( + [_turn([_cmd("Grep", {"pattern": "cause", "glob": "RESOLUTION.md", "output_mode": "content"})])], SPEC + ) + assert info.verdict is IntegrityVerdict.TAINTED + + def test_unknown_tool_touching_graded_material_is_inconclusive_not_tainted(): """We do not guess at an unrecognised tool's semantics in either direction.""" info = scan_commands([_turn([_cmd("mcp__some__fetch", {"target": "RESOLUTION.md"})])], SPEC) From 8699ae5540b64556076e7002d9c17c6d7377ea53 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 15:49:33 +0300 Subject: [PATCH 18/31] Excuse basename-glob reads of paths the agent itself created earlier in the transcript --- src/coder_eval/integrity.py | 149 +++++++++++++++++++++++++++-------- tests/test_integrity_scan.py | 72 +++++++++++++++++ 2 files changed, 189 insertions(+), 32 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 4dd81c33..a1b0ea8f 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -244,8 +244,11 @@ # Basename patterns that are graded material in every suite, independent of what # this particular task declares. Deliberately short: each entry is a name the -# framework or the task-authoring convention owns, never a name an agent's own -# work would produce. +# framework or the task-authoring convention owns. One of them (`RESOLUTION.md`) +# is ALSO the required deliverable of the troubleshoot suite -- the agent is +# supposed to write and re-read its own copy -- so basename-glob matches are +# excused when the matched path is one the agent itself created earlier in the +# transcript (:func:`_is_agent_created`); a golden elsewhere still matches. _GRADED_BASENAME_GLOBS = ("RESOLUTION.md", "*.expected", "task.yaml", "context.json") # Grader-script names. `check.py` (no underscore) grades live tasks too, so the @@ -497,12 +500,60 @@ def _task_dir_operands(command: str, task_dir: Path | None = None) -> set[str]: return operands -def _find_match(text: str, spec: GradedMaterialSpec) -> str | None: +# Characters that end a path token inside an already-normalized command string. +_TOKEN_DELIMITERS = " \t'\";|&<>()=" + + +def _created_path(raw: str) -> str: + """Normalize a path the agent created, for :func:`_is_agent_created` lookups.""" + path = _normalize(raw) + while path.startswith("./"): + path = path[2:] + return path + + +def _is_agent_created(haystack: str, start: int, end: int, created: frozenset[str] | set[str]) -> bool: + """Whether the filename matched at ``[start, end)`` sits in a path the agent + itself created earlier in this transcript. + + The basename globs are location-independent, and one of them names the + troubleshoot suite's required DELIVERABLE: every honest agent writes + ``RESOLUTION.md`` and -- because the harness enforces Read-before-Edit -- + reads it back. Those reads are the agent's own work, not a leak, so a glob + match is excused when its whole path token is one the agent created. + + The comparison is one-directional on purpose: a RELATIVE read may resolve + into a created path as its component-suffix (``cat RESOLUTION.md`` after + ``Write /workspace/RESOLUTION.md``), but an ABSOLUTE read is never excused + by a relative creation and a longer relative read never by a shorter one -- + otherwise writing your own ``RESOLUTION.md`` once would license reading + every golden of the same name. + """ + if not created: + return False + left = start + while left > 0 and haystack[left - 1] not in _TOKEN_DELIMITERS: + left -= 1 + right = end + while right < len(haystack) and haystack[right] not in _TOKEN_DELIMITERS: + right += 1 + token = haystack[left:right] + while token.startswith("./"): + token = token[2:] + if token in created: + return True + is_relative = not token.startswith("/") and re.match(r"[a-z]:/", token) is None + return is_relative and any(c.endswith("/" + token) for c in created) + + +def _find_match(text: str, spec: GradedMaterialSpec, created: frozenset[str] | set[str] = frozenset()) -> str | None: """Return the graded-material reference found in ``text``, or None. Literal paths and directory prefixes are matched as whole paths on the normalized form (:func:`_literal_to_regex`); basename globs are regex-matched - wherever they appear; path segments are matched as a path component so a + wherever they appear, EXCEPT on a path the agent itself created earlier + (``created``, see :func:`_is_agent_created`) -- its own deliverable is not an + answer key; path segments are matched as a path component so a relatively-typed ``../mocks/responses/manifest.json`` is caught too; grader scripts are matched last and only under the task directory (:func:`_grader_match`). @@ -518,8 +569,9 @@ def _find_match(text: str, spec: GradedMaterialSpec) -> str | None: if needle and _literal_to_regex(needle, directory=True).search(haystack): return candidate for glob in spec.basename_globs: - if _glob_to_regex(glob).search(haystack): - return glob + for match in _glob_to_regex(glob).finditer(haystack): + if not _is_agent_created(haystack, match.start(), match.end(), created): + return glob for segment in (*spec.path_segments, *spec.mock_segments): if _segment_to_regex(segment).search(haystack): return segment @@ -638,8 +690,8 @@ def _git_is_read(tokens: list[str]) -> bool: return subcommand not in _GIT_NEUTRAL_SUBCOMMANDS -def _strip_redirects(segment: str) -> tuple[str, list[str]]: - """Remove real output redirects from a segment; collect input-redirect targets. +def _strip_redirects(segment: str) -> tuple[str, list[str], list[str]]: + """Remove real output redirects from a segment; collect the redirect targets. The old ``partition(">")`` treated ANY ``>`` as an output redirect -- one inside an awk program (``awk '$1 > 0' KEY``), a sed pattern, or a plain @@ -648,16 +700,20 @@ def _strip_redirects(segment: str) -> tuple[str, list[str]]: A real redirect is an UNQUOTED operator and consumes exactly one word; every other operand is still passed to the utility. - Returns ``(stripped, input_targets)``: the segment with each output-redirect - operator (``>``, ``>>``, ``>&``, ``&>``, fd-prefixed forms) and the single - word it consumes removed, plus the target words of plain ``<`` input - redirects (which the SHELL reads, whatever the utility is). Heredoc - operators (``<<``), here-strings (``<<<``) and process substitution - (``<(…)``) are left in place: their text is data or an executing command, - and either way it must stay visible to the caller's matching. + Returns ``(stripped, input_targets, created_targets)``: the segment with + each output-redirect operator (``>``, ``>>``, ``>&``, ``&>``, fd-prefixed + forms) and the single word it consumes removed; the target words of plain + ``<`` input redirects (which the SHELL reads, whatever the utility is); and + the targets of TRUNCATING output redirects (``>`` / ``&>``, not ``>>`` -- + an append leaves the original content readable, a truncation replaces it), + which mark files the agent itself created. Heredoc operators (``<<``), + here-strings (``<<<``) and process substitution (``<(…)``) are left in + place: their text is data or an executing command, and either way it must + stay visible to the caller's matching. """ out: list[str] = [] input_targets: list[str] = [] + created_targets: list[str] = [] in_single = in_double = False i = 0 n = len(segment) @@ -715,11 +771,15 @@ def _consume_word(j: int) -> tuple[str, int]: continue if c == ">" or segment[i : i + 2] == "&>": j = i + 1 if c == ">" else i + 2 - if j < n and segment[j] == ">": + appending = j < n and segment[j] == ">" + if appending: j += 1 - if j < n and segment[j] == "&": # fd duplication: >&2, 2>&1 + duplicating = j < n and segment[j] == "&" # fd duplication: >&2, 2>&1 + if duplicating: j += 1 - _, j = _consume_word(j) + word, j = _consume_word(j) + if word and not appending and not duplicating: + created_targets.append(word) out.append(" ") i = j continue @@ -733,7 +793,7 @@ def _consume_word(j: int) -> tuple[str, int]: out.append(c) i += 1 - return "".join(out), input_targets + return "".join(out), input_targets, created_targets def _search_is_files_only(tokens: list[str]) -> bool: @@ -751,7 +811,9 @@ def _search_is_files_only(tokens: list[str]) -> bool: return False -def _classify_segment(segment: str, spec: GradedMaterialSpec) -> tuple[bool, str | None]: +def _classify_segment( + segment: str, spec: GradedMaterialSpec, created: frozenset[str] | set[str] = frozenset() +) -> tuple[bool, str | None]: """Decide whether one shell segment READ graded material. Returns ``(is_read, matched_reference)``. ``matched_reference`` is set @@ -760,7 +822,9 @@ def _classify_segment(segment: str, spec: GradedMaterialSpec) -> tuple[bool, str Rules, in order: - 1. No graded-material reference anywhere in the segment -> not a read. + 1. No graded-material reference anywhere in the segment -> not a read. A + basename-glob reference on a path the agent itself created earlier + (``created``) does not count: that is its own deliverable, not a golden. 2. A reference consumed by an input redirect (``< file``) -> a read, whatever the utility is; the shell does the reading. 3. An output redirect (``>`` / ``>>`` / ``>&`` / ``&>``) consumes exactly one @@ -783,18 +847,18 @@ def _classify_segment(segment: str, spec: GradedMaterialSpec) -> tuple[bool, str false positive is visible in the finding's evidence while a false negative is invisible. """ - matched = _find_match(segment, spec) + matched = _find_match(segment, spec, created) if matched is None: return False, None utility, tokens = _segment_utility(segment) normalized_tokens = [Path(t.replace("\\", "/")).name.casefold().removesuffix(".exe") for t in tokens] - stripped, input_targets = _strip_redirects(segment) - if any(_find_match(target, spec) is not None for target in input_targets): + stripped, input_targets, _ = _strip_redirects(segment) + if any(_find_match(target, spec, created) is not None for target in input_targets): return True, matched - if _find_match(stripped, spec) is None: + if _find_match(stripped, spec, created) is None: # Every reference sits in an output-redirect target: a write, not a read. return False, matched @@ -859,6 +923,10 @@ def scan_commands(turns: list[TurnRecord], spec: GradedMaterialSpec) -> Integrit scanned = 0 blind = 0 unclassified_hits = 0 + # Paths the agent itself created, in transcript order: the required + # deliverable shares a basename glob with the graded goldens, and only + # provenance tells the agent's own copy apart (see _is_agent_created). + created: set[str] = set() for turn in turns: for index, cmd in enumerate(turn.commands): @@ -873,14 +941,16 @@ def scan_commands(turns: list[TurnRecord], spec: GradedMaterialSpec) -> Integrit continue if cmd.tool_name == "Bash": - is_read, matched = _bash_read(text, spec) + is_read, matched = _bash_read(text, spec, created) else: - is_read, matched, understood = _structured_read(cmd, text, spec) + is_read, matched, understood = _structured_read(cmd, text, spec, created) if matched is not None and not understood: unclassified_hits += 1 notes.append( f"{cmd.tool_name} referenced {matched} but its read semantics are unknown; not counted" ) + if cmd.tool_name == "Write" and isinstance(cmd.parameters.get("file_path"), str): + created.add(_created_path(cmd.parameters["file_path"])) if is_read and matched is not None: kind = _finding_kind(matched, spec) @@ -1056,19 +1126,34 @@ def _flush() -> None: return segments -def _bash_read(command: str, spec: GradedMaterialSpec) -> tuple[bool, str | None]: - """Classify a shell command by splitting it into segments and judging each.""" +def _bash_read(command: str, spec: GradedMaterialSpec, created: set[str] | None = None) -> tuple[bool, str | None]: + """Classify a shell command by splitting it into segments and judging each. + + ``created`` is the transcript-ordered set of paths the agent has written so + far; each segment's truncating redirect targets are added AFTER the segment + is classified, so ``cat > RESOLUTION.md && cat RESOLUTION.md`` excuses the + re-read while ``sed … golden > RESOLUTION.md`` still counts the read that + produced the file. + """ + created = set() if created is None else created mentioned: str | None = None for segment in _split_segments(command): - is_read, matched = _classify_segment(segment, spec) + is_read, matched = _classify_segment(segment, spec, created) if matched is not None: mentioned = matched if is_read: return True, matched + _, _, made = _strip_redirects(segment) + created.update(_created_path(target) for target in made) return False, mentioned -def _structured_read(cmd: CommandTelemetry, text: str, spec: GradedMaterialSpec) -> tuple[bool, str | None, bool]: +def _structured_read( + cmd: CommandTelemetry, + text: str, + spec: GradedMaterialSpec, + created: frozenset[str] | set[str] = frozenset(), +) -> tuple[bool, str | None, bool]: """Classify a non-Bash tool call. Returns ``(is_read, matched, semantics_understood)``. Unlike a shell string, a @@ -1085,7 +1170,7 @@ def _structured_read(cmd: CommandTelemetry, text: str, spec: GradedMaterialSpec) haystack = " ".join(str(cmd.parameters[key]) for key in _PATH_PARAMETER_KEYS if cmd.parameters.get(key)) else: haystack = text - matched = _find_match(haystack, spec) + matched = _find_match(haystack, spec, created) if matched is None: return False, None, True diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 34bb857b..f3637cf1 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -384,6 +384,78 @@ def test_inert_text_handling_does_not_hide_a_real_read(command: str): assert is_read is True, f"expected a read: {command!r}" +# -------------------------------------------------------------------------- +# The agent's own deliverable: created-in-transcript paths are not answer keys +# -------------------------------------------------------------------------- + + +def test_the_write_read_edit_deliverable_flow_is_clean(): + """The flagship troubleshoot flow: the agent writes RESOLUTION.md, and the + harness's Read-before-Edit rule forces it to read its own copy back. Flagging + that floods detect-mode triage and blocks void mode outright.""" + commands = [ + _cmd("Write", {"file_path": "/workspace/RESOLUTION.md", "content": "my diagnosis"}, tool_id="t1"), + _cmd("Read", {"file_path": "/workspace/RESOLUTION.md"}, tool_id="t2"), + _cmd("Edit", {"file_path": "/workspace/RESOLUTION.md", "old_string": "my", "new_string": "the"}, tool_id="t3"), + ] + info = scan_commands([_turn(commands)], SPEC) + assert info.verdict is IntegrityVerdict.CLEAN + assert info.findings == [] + + +def test_a_shell_created_deliverable_can_be_re_read(): + info = scan_commands([_turn([_bash("echo 'diagnosis' > RESOLUTION.md"), _bash("cat RESOLUTION.md")])], SPEC) + assert info.verdict is IntegrityVerdict.CLEAN + + +def test_creation_and_re_read_in_one_command_is_clean(): + info = scan_commands([_turn([_bash("cat > RESOLUTION.md && cat RESOLUTION.md")])], SPEC) + assert info.verdict is IntegrityVerdict.CLEAN + + +def test_a_relative_re_read_of_an_absolutely_created_deliverable_is_clean(): + commands = [ + _cmd("Write", {"file_path": "/workspace/RESOLUTION.md", "content": "d"}, tool_id="t1"), + _bash("cat RESOLUTION.md"), + ] + info = scan_commands([_turn(commands)], SPEC) + assert info.verdict is IntegrityVerdict.CLEAN + + +def test_creating_your_own_copy_does_not_license_reading_a_golden(): + """Provenance is per-path: the agent's RESOLUTION.md excuses nothing at any + OTHER location, or writing your own copy once would unlock every golden.""" + info = scan_commands( + [_turn([_bash("echo 'diagnosis' > RESOLUTION.md"), _bash("cat ../scenario/RESOLUTION.md")])], SPEC + ) + assert info.verdict is IntegrityVerdict.TAINTED + + +def test_a_relative_creation_never_excuses_an_absolute_read(): + info = scan_commands( + [_turn([_bash("echo 'diagnosis' > RESOLUTION.md"), _bash("cat /repo/tests/tasks/scen/RESOLUTION.md")])], SPEC + ) + assert info.verdict is IntegrityVerdict.TAINTED + + +def test_an_append_is_not_a_creation(): + """`>>` leaves the original content readable, so it proves nothing about + who wrote the file.""" + info = scan_commands([_turn([_bash("echo 'note' >> RESOLUTION.md"), _bash("cat RESOLUTION.md")])], SPEC) + assert info.verdict is IntegrityVerdict.TAINTED + + +def test_the_read_that_produces_the_deliverable_still_counts(): + """Deriving RESOLUTION.md FROM a golden is the leak itself.""" + info = scan_commands([_turn([_bash("sed 's/x/y/' ../scen/RESOLUTION.md > RESOLUTION.md")])], SPEC) + assert info.verdict is IntegrityVerdict.TAINTED + + +def test_a_read_before_any_creation_is_still_a_leak(): + info = scan_commands([_turn([_bash("cat RESOLUTION.md"), _bash("echo 'd' > RESOLUTION.md")])], SPEC) + assert info.verdict is IntegrityVerdict.TAINTED + + # -------------------------------------------------------------------------- # The regression guard for the truncation trap # -------------------------------------------------------------------------- From e4885d8ee5f9a0993bb2b91f8c0e82e279bcd7fe Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 15:54:02 +0300 Subject: [PATCH 19/31] Classify a leading search utility by its flags before the nested-reader token sweep --- src/coder_eval/integrity.py | 15 +++++++++------ tests/test_integrity_scan.py | 3 +++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index a1b0ea8f..a50b21c8 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -835,11 +835,14 @@ def _classify_segment( 4. ``git`` -> decided by its subcommand (:func:`_git_is_read`): ``git show`` / ``cat-file`` / ``diff`` / ``blame`` / ``grep`` / ``log -p`` print content, ``git add`` / ``status`` / ``checkout`` do not. - 5. Any content-emitting utility appearing as a token -> a read. Checked + 5. A leading search utility -> decided by its own flags: restricted to file + names or counts is not a read, otherwise it is. Decided BEFORE the token + sweep below, because a search utility's operands are patterns and paths, + never nested executables -- ``grep -l cat KEY`` searches FOR "cat", it + does not run it. + 6. Any content-emitting utility appearing as a token -> a read. Checked across all tokens, not just the leading one, so ``find … -exec cat {}`` and ``xargs cat`` do not slip past on their wrapper's name. - 6. A search utility restricted to file names or counts -> not a read; - otherwise a read. 7. A listing/metadata utility, or a utility that moves, removes or otherwise manipulates a file without emitting it -> not a read. 8. Anything else -> a read. Conservative on purpose: an unrecognised utility @@ -868,12 +871,12 @@ def _classify_segment( if utility == "git": return _git_is_read(tokens), matched - if any(name in _READ_UTILITIES for name in normalized_tokens): - return True, matched - if utility in _SEARCH_UTILITIES: return not _search_is_files_only(tokens), matched + if any(name in _READ_UTILITIES for name in normalized_tokens): + return True, matched + if utility in _LISTING_UTILITIES or utility in _NEUTRAL_UTILITIES: return False, matched diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index f3637cf1..c149cff3 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -82,6 +82,7 @@ def _turn(commands: list[CommandTelemetry], *, iteration: int = 1, unrecovered: pytest.param("diff mine.py /repo/tasks/leaky/solution.py", id="diff-against-reference"), pytest.param("cp /repo/tasks/leaky/solution.py ./mine.py", id="cp-the-reference"), pytest.param("grep -n 'root cause' RESOLUTION.md", id="grep-content"), + pytest.param("grep cat RESOLUTION.md", id="grep-content-with-a-reader-named-pattern"), pytest.param("rg 'fixed version' RESOLUTION.md", id="rg-content"), pytest.param("while read l; do echo $l; done < RESOLUTION.md", id="input-redirect"), pytest.param("find . -name '*.md' -exec cat RESOLUTION.md {} \\;", id="find-exec-cat"), @@ -105,6 +106,8 @@ def _turn(commands: list[CommandTelemetry], *, iteration: int = 1, unrecovered: pytest.param("dirname /repo/tasks/leaky/task.yaml", id="dirname"), pytest.param("echo RESOLUTION.md", id="echo-the-name"), pytest.param("grep -l 'cause' RESOLUTION.md", id="grep-files-only"), + pytest.param("grep -l cat RESOLUTION.md", id="grep-files-only-with-a-reader-named-pattern"), + pytest.param("rg -c 'head|tail' RESOLUTION.md", id="rg-count-with-reader-named-alternation"), pytest.param("grep -rl 'cause' /repo --include=RESOLUTION.md", id="grep-bundled-files-only"), pytest.param("grep -c 'cause' RESOLUTION.md", id="grep-count"), pytest.param("rg --files /repo | grep -l RESOLUTION.md", id="rg-files"), From a2e884ee29508fbe8e91f327c79b1b0bb26c8e4a Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 15:56:04 +0300 Subject: [PATCH 20/31] Parse search-utility options with end-of-options and value operands instead of sweeping every token --- src/coder_eval/integrity.py | 71 +++++++++++++++++++++++++++++++----- tests/test_integrity_scan.py | 5 +++ 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index a50b21c8..67f82d13 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -212,6 +212,30 @@ ) _SEARCH_FILES_ONLY_SHORT = frozenset({"l", "L", "c"}) +# Search-utility options that take a SEPARATE value operand. The token after one +# of these is data, not a flag: `grep -e -c file` searches FOR "-c". Keyed by +# utility because the same letter differs between tools (rg's `-E` takes an +# encoding, grep's takes nothing); utilities not listed use the grep set. +_GREP_VALUE_OPTIONS = frozenset( + {"-e", "-f", "-m", "-A", "-B", "-C", "-d", "-D", "--regexp", "--file", "--max-count", "--include", "--exclude", + "--exclude-dir", "--include-dir", "--label", "--binary-files", "--devices", "--directories"} +) +_SEARCH_VALUE_OPTIONS: dict[str, frozenset[str]] = { + "grep": _GREP_VALUE_OPTIONS, + "egrep": _GREP_VALUE_OPTIONS, + "fgrep": _GREP_VALUE_OPTIONS, + "rg": frozenset( + {"-e", "-f", "-g", "-t", "-T", "-m", "-A", "-B", "-C", "-E", "-M", "-j", "-r", "--regexp", "--file", + "--glob", "--iglob", "--type", "--type-not", "--type-add", "--max-count", "--max-columns", "--max-depth", + "--max-filesize", "--context", "--after-context", "--before-context", "--encoding", "--threads", "--pre", + "--pre-glob", "--replace", "--sort", "--sortr", "--colors", "--ignore-file"} + ), + "ag": frozenset({"-A", "-B", "-C", "-g", "-G", "-m", "--after", "--before", "--context", "--file-search-regex", + "--ignore", "--ignore-dir", "--max-count", "--pager", "--workers"}), + "ack": frozenset({"-A", "-B", "-C", "-m", "-g", "--match", "--max-count", "--after-context", "--before-context", + "--context", "--type", "--ignore-dir", "--ignore-file", "--pager", "--output"}), +} + # Structured tools whose whole purpose is to return file content. _READ_TOOLS = frozenset({"Read", "NotebookRead", "ReadFile", "read_file", "view", "View"}) @@ -796,18 +820,45 @@ def _consume_word(j: int) -> tuple[str, int]: return "".join(out), input_targets, created_targets -def _search_is_files_only(tokens: list[str]) -> bool: - """Whether a grep/rg invocation reports only file names or match counts.""" +def _search_is_files_only(utility: str, tokens: list[str]) -> bool: + """Whether a grep/rg invocation reports only file names or match counts. + + Options are parsed, not swept: a token is only a flag when it sits in flag + position. ``--`` ends option parsing (``grep -- '-c' KEY`` searches FOR the + text "-c"), and an option that takes a separate value consumes the next + token (``grep -e -c KEY`` searches for "-c" too). Anything after either is + an operand, and an operand can never put the search into files-only mode. + """ + value_options = _SEARCH_VALUE_OPTIONS.get(utility, _GREP_VALUE_OPTIONS) + seen_utility = False + expect_value = False for token in tokens: + if not seen_utility: + # Step over wrappers/assignments to the utility itself, mirroring + # _segment_utility: its tokens are not search options. + seen_utility = Path(token.replace("\\", "/")).name.casefold().removesuffix(".exe") == utility + continue + if expect_value: + expect_value = False + continue + if token == "--": + return False # only operands remain, and none of the flags so far matched if token in _SEARCH_FILES_ONLY_LONG: return True - # Bundled short flags: `-rl`, `-il`. A lone `-` or a long flag is skipped. - if ( - token.startswith("-") - and not token.startswith("--") - and any(c in _SEARCH_FILES_ONLY_SHORT for c in token[1:]) - ): - return True + if token in value_options: + expect_value = True + continue + if token.startswith("--"): + continue # long flag (a `--opt=value` carries its value inline) + if token.startswith("-") and len(token) > 1: + # Bundled short flags: `-rl`, `-il`. A bundled value-taking option + # consumes the rest of the bundle (or the next token) as its value. + for position, char in enumerate(token[1:], start=1): + if "-" + char in value_options: + expect_value = position == len(token) - 1 + break + if char in _SEARCH_FILES_ONLY_SHORT: + return True return False @@ -872,7 +923,7 @@ def _classify_segment( return _git_is_read(tokens), matched if utility in _SEARCH_UTILITIES: - return not _search_is_files_only(tokens), matched + return not _search_is_files_only(utility, tokens), matched if any(name in _READ_UTILITIES for name in normalized_tokens): return True, matched diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index c149cff3..35d3cebb 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -84,6 +84,9 @@ def _turn(commands: list[CommandTelemetry], *, iteration: int = 1, unrecovered: pytest.param("grep -n 'root cause' RESOLUTION.md", id="grep-content"), pytest.param("grep cat RESOLUTION.md", id="grep-content-with-a-reader-named-pattern"), pytest.param("rg 'fixed version' RESOLUTION.md", id="rg-content"), + pytest.param("grep -- '-c' RESOLUTION.md", id="count-flag-behind-end-of-options"), + pytest.param("grep -e -l RESOLUTION.md", id="files-flag-as-a-pattern-operand"), + pytest.param("rg --regexp -c RESOLUTION.md", id="count-flag-as-a-long-option-operand"), pytest.param("while read l; do echo $l; done < RESOLUTION.md", id="input-redirect"), pytest.param("find . -name '*.md' -exec cat RESOLUTION.md {} \\;", id="find-exec-cat"), pytest.param("ls -1 | xargs cat RESOLUTION.md", id="xargs-cat"), @@ -110,6 +113,8 @@ def _turn(commands: list[CommandTelemetry], *, iteration: int = 1, unrecovered: pytest.param("rg -c 'head|tail' RESOLUTION.md", id="rg-count-with-reader-named-alternation"), pytest.param("grep -rl 'cause' /repo --include=RESOLUTION.md", id="grep-bundled-files-only"), pytest.param("grep -c 'cause' RESOLUTION.md", id="grep-count"), + pytest.param("grep -m 5 -c 'cause' RESOLUTION.md", id="grep-count-after-a-valued-option"), + pytest.param("rg -tpy -c cause RESOLUTION.md", id="rg-count-after-an-inline-valued-bundle"), pytest.param("rg --files /repo | grep -l RESOLUTION.md", id="rg-files"), pytest.param("rg --files-with-matches cause RESOLUTION.md", id="rg-files-with-matches"), pytest.param("cat my_own_notes.md", id="reads-something-else"), From 329a8a923c097a24078f2f18a58111690ad7e6bd Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 15:57:22 +0300 Subject: [PATCH 21/31] Match a protected directory typed bare, letting utility semantics decide listing versus emitting --- src/coder_eval/integrity.py | 14 +++++++++----- tests/test_integrity_scan.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 67f82d13..51b3a6f1 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -411,12 +411,16 @@ def _segment_to_regex(segment: str) -> re.Pattern[str]: """Compile a path segment into a pattern that matches it as a path COMPONENT. ``m`` must match ``m/.store`` and ``/m/.store`` but not - ``stream/x``, so the component is anchored on its trailing separator plus a - leading boundary that rejects any character a path component could continue - from. A leading separator is NOT required: agents open these paths relatively - and quoted (``open('m/.store')``). + ``stream/x`` or ``m.json``, so the component is anchored between a leading + boundary that rejects any character a path component could continue from and + either its trailing separator OR the same boundary: ``rg . _fixtures`` and + ``grep -R . mocks`` hand the whole protected directory to a recursive + search without ever typing a ``/``, and whether that emits or merely lists + is the utility's call, not the matcher's. A leading separator is NOT + required either: agents open these paths relatively and quoted + (``open('m/.store')``). """ - return re.compile(r"(? set[str]: diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 35d3cebb..8c0a5b9f 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -823,6 +823,7 @@ def test_a_sealed_fixture_store_decode_is_flagged(): pytest.param("cat program/main.py", id="segment-inside-another-component"), pytest.param("cat streams/m.json", id="segment-as-a-filename"), pytest.param("cat my_mocks_notes.md", id="segment-inside-a-basename"), + pytest.param("cat m.json", id="bare-name-with-an-extension"), ], ) def test_a_segment_only_matches_a_whole_path_component(command: str): @@ -831,6 +832,36 @@ def test_a_segment_only_matches_a_whole_path_component(command: str): assert info.verdict is IntegrityVerdict.CLEAN +@pytest.mark.parametrize( + "command", + [ + pytest.param("rg . _fixtures", id="rg-emits-the-fixture-directory"), + pytest.param("grep -R . mocks", id="grep-emits-the-mock-directory"), + pytest.param("tail -n +1 mocks", id="reader-on-the-bare-directory"), + ], +) +def test_a_bare_protected_directory_name_still_matches(command: str): + """A recursive search does not need a trailing slash to emit a directory.""" + spec = derive_graded_material(_task(sandbox={"mock_path_dirs": ["mocks"]}), None) + info = scan_commands([_turn([_bash(command)])], spec) + assert info.verdict is IntegrityVerdict.TAINTED + + +@pytest.mark.parametrize( + "command", + [ + pytest.param("ls -la _fixtures", id="listing-the-fixture-directory"), + pytest.param("rm -rf mocks", id="removing-the-mock-directory"), + pytest.param("find mocks -name '*.json'", id="enumerating-the-mock-directory"), + pytest.param("grep -rl secret mocks", id="files-only-search-of-the-mock-directory"), + ], +) +def test_utility_semantics_still_decide_a_bare_directory_touch(command: str): + spec = derive_graded_material(_task(sandbox={"mock_path_dirs": ["mocks"]}), None) + info = scan_commands([_turn([_bash(command)])], spec) + assert info.verdict is IntegrityVerdict.CLEAN + + @pytest.mark.parametrize( ("command", "expected"), [ From 26a028514792ee1cd6db87f4359eac0ec416e595 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 15:59:25 +0300 Subject: [PATCH 22/31] Keep a staged template mount exact instead of widening it to its first path component --- src/coder_eval/integrity.py | 27 +++++++++++++++++---------- tests/test_integrity_scan.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 51b3a6f1..65b588fe 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -424,17 +424,17 @@ def _segment_to_regex(segment: str) -> re.Pattern[str]: def _path_segments(raw: str) -> set[str]: - """Normalize a declared sandbox-relative directory into matchable segments. + """Normalize a declared sandbox-relative directory into a matchable segment. - Returns the declared path itself plus its root component: a task that declares - ``mocks/bin`` as its shim directory still keeps the recorded responses it - replays under ``mocks/``. ``.`` (the sandbox root) yields nothing -- every - read would match it. + Exactly the declared prefix, nothing wider: a template mounted at + ``stubs/uip`` protects ``stubs/uip/…`` but NOT the agent's own siblings + under ``stubs/``. ``.`` (the sandbox root) yields nothing -- every read + would match it. """ parts = [p for p in _normalize(raw).split("/") if p not in ("", ".")] if not parts: return set() - return {"/".join(parts), parts[0]} + return {"/".join(parts)} def _declared_mock_segments(task: TaskDefinition) -> set[str]: @@ -443,13 +443,20 @@ def _declared_mock_segments(task: TaskDefinition) -> set[str]: Two declarations locate a scenario's fixture store: ``sandbox.mock_path_dirs`` (the directories whose contents the harness makes executable and prepends to the agent's PATH -- the shims) and ``template_sources[*].mount_point`` (where a - staged tree lands). A mount point is only as precise as the task made it: one - that points at the agent's own working tree widens the spec to that tree, which - is why these are reported as MOCK_DATA_READ and not folded into GRADED_READ. + staged tree lands). A mock PATH dir declared as a subdirectory (``mocks/bin``) + also protects its root: the recorded responses the shim replays sit beside the + shim, under ``mocks/``. A template mount stays exact -- widening it to its + first component would flag the agent's own files mounted next to it. A mount + point is only as precise as the task made it: one that points at the agent's + own working tree widens the spec to that tree, which is why these are reported + as MOCK_DATA_READ and not folded into GRADED_READ. """ segments: set[str] = set() for raw in task.sandbox.mock_path_dirs or []: - segments.update(_path_segments(raw)) + parts = [p for p in _normalize(raw).split("/") if p not in ("", ".")] + if parts: + segments.add("/".join(parts)) + segments.add(parts[0]) for source in task.sandbox.template_sources or []: mount_point = getattr(source, "mount_point", None) if mount_point: diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 8c0a5b9f..4c83997f 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -766,8 +766,32 @@ def test_derivation_picks_up_a_staged_fixture_mount_point(): sandbox={"template_sources": [{"type": "template_dir", "path": "fixtures", "mount_point": "stubs/uip"}]} ) spec = derive_graded_material(task, None) - # Both the declared mount point and its root: the fixtures sit under either. - assert {"stubs/uip", "stubs"} <= spec.mock_segments + # Exactly the declared mount point: widening it to its first component would + # flag the agent's own files staged next to it under `stubs/`. + assert "stubs/uip" in spec.mock_segments + assert "stubs" not in spec.mock_segments + + +def test_a_sibling_of_a_staged_mount_is_not_fixture_data(): + task = _task( + sandbox={"template_sources": [{"type": "template_dir", "path": "fixtures", "mount_point": "stubs/uip"}]} + ) + spec = derive_graded_material(task, None) + + sibling = scan_commands([_turn([_bash("cat stubs/README.md")])], spec) + assert sibling.verdict is IntegrityVerdict.CLEAN + + staged = scan_commands([_turn([_bash("cat stubs/uip/manifest.json")])], spec) + assert staged.verdict is IntegrityVerdict.TAINTED + assert staged.findings[0].kind is IntegrityFindingKind.MOCK_DATA_READ + + +def test_a_mock_path_subdirectory_still_protects_its_root(): + """The shim's recorded responses sit beside it under the root, so a + `mocks/bin` PATH declaration keeps `mocks/` protected.""" + spec = derive_graded_material(_task(sandbox={"mock_path_dirs": ["mocks/bin"]}), None) + info = scan_commands([_turn([_bash("cat mocks/responses/manifest.json")])], spec) + assert info.verdict is IntegrityVerdict.TAINTED def test_derivation_ignores_a_sandbox_root_mount_point(): From 8ef049f96f9687b63aea94237b3934868aa054ef Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 16:01:56 +0300 Subject: [PATCH 23/31] Match every task-dir spelling of a declared reference and strip quotes before matching --- src/coder_eval/integrity.py | 50 +++++++++++++++++++++++++++--------- tests/test_integrity_scan.py | 44 ++++++++++++++++++++++++++++--- 2 files changed, 79 insertions(+), 15 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 65b588fe..41592042 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -283,9 +283,10 @@ _GRADER_SCRIPT_GLOBS = ("check_*.py", "check.py") # Directory markers that put a grader script in the task's own directory: the -# suite layout every task YAML lives under, and both spellings of the framework's -# task-dir variable. The resolved task directory is added per task. -_GRADER_DIR_MARKERS = ("tests/tasks/", "$task_dir/", f"{CONTAINER_TASK_DIR}/") +# suite layout every task YAML lives under, and every spelling of the framework's +# task-dir variable -- bare, braced, and container-resolved. The resolved task +# directory is added per task. +_GRADER_DIR_MARKERS = ("tests/tasks/", "$task_dir/", "${task_dir}/", f"{CONTAINER_TASK_DIR}/") # Path SEGMENTS that hold answer keys wherever they appear. Segments rather than # resolved prefixes because this is how an agent types them -- `cat @@ -486,10 +487,15 @@ def derive_graded_material(task: TaskDefinition, task_file: Path | None) -> Grad base = task_file.parent reference = task.reference if reference is not None: + # A relative reference resolves to ONE on-disk spelling, but an agent + # that discovered the task-dir variable types the symbolic one -- + # `cat "$TASK_DIR/solution.py"` -- so every spelling is stored. if reference.file: paths.add(str(base / reference.file)) + paths.update(_task_dir_spellings(reference.file)) if reference.directory: directories.add(str(base / reference.directory)) + directories.update(_task_dir_spellings(reference.directory)) # Operands the task's OWN criteria reach for. `python3 $TASK_DIR/check_x.py` # names the grader; an agent that runs the grader is grading itself. @@ -515,21 +521,37 @@ def derive_graded_material(task: TaskDefinition, task_file: Path | None) -> Grad ) +def _task_dir_spellings(relative: str) -> set[str]: + """Every task-dir-rooted spelling of a task-relative path: the bare and + braced variable forms plus the container mount. An absolute path has no + task-dir spelling and yields nothing.""" + normalized = relative.replace("\\", "/") + if Path(normalized).is_absolute(): + return set() + while normalized.startswith("./"): + normalized = normalized[2:] + return { + f"$TASK_DIR/{normalized}", + f"${{TASK_DIR}}/{normalized}", + f"{CONTAINER_TASK_DIR}/{normalized}", + } + + def _task_dir_operands(command: str, task_dir: Path | None = None) -> set[str]: """Extract ``$TASK_DIR``-rooted operands from a framework-run command. - Three spellings of the same read, because which one the agent types depends on - the driver: the raw form (``$TASK_DIR/check_x.py``, what an agent that - discovered the variable would use), the container-resolved form - (``/work/task_dir/check_x.py``), and -- when the task directory is known -- the - real on-disk path. Under ``driver: tempdir`` the task lives in the host - checkout and the agent reads THAT path, which neither symbolic form matches. + Four spellings of the same read, because which one the agent types depends on + the driver: the raw and braced variable forms (``$TASK_DIR/check_x.py`` / + ``${TASK_DIR}/check_x.py``, what an agent that discovered the variable would + use), the container-resolved form (``/work/task_dir/check_x.py``), and -- + when the task directory is known -- the real on-disk path. Under + ``driver: tempdir`` the task lives in the host checkout and the agent reads + THAT path, which no symbolic form matches. """ operands: set[str] = set() for match in re.finditer(r"\$\{?TASK_DIR\}?(/[^\s'\";|&)]+)", command): suffix = match.group(1) - operands.add(f"$TASK_DIR{suffix}") - operands.add(f"{CONTAINER_TASK_DIR}{suffix}") + operands.update(_task_dir_spellings(suffix.lstrip("/"))) if task_dir is not None: operands.add(str((task_dir / suffix.lstrip("/")).resolve())) return operands @@ -592,8 +614,12 @@ def _find_match(text: str, spec: GradedMaterialSpec, created: frozenset[str] | s relatively-typed ``../mocks/responses/manifest.json`` is caught too; grader scripts are matched last and only under the task directory (:func:`_grader_match`). + + Quotes are removed from the haystack before matching -- that is what the + shell does to a word, so ``cat "$TASK_DIR"/solution.py`` is the same read as + the unquoted spelling, and a quote-split path cannot dodge a needle. """ - haystack = _normalize(text) + haystack = _normalize(text).replace('"', "").replace("'", "") for candidate in spec.paths: needle = _normalize(candidate) diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 4c83997f..4e5a249a 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -736,6 +736,7 @@ def test_task_dir_operands_add_the_resolved_spelling(): operands = _task_dir_operands("python3 $TASK_DIR/check_x.py", task_dir) assert operands == { "$TASK_DIR/check_x.py", + "${TASK_DIR}/check_x.py", "/work/task_dir/check_x.py", str((task_dir / "check_x.py").resolve()), } @@ -886,12 +887,15 @@ def test_utility_semantics_still_decide_a_bare_directory_touch(command: str): assert info.verdict is IntegrityVerdict.CLEAN +_CHECK_X_SPELLINGS = {"$TASK_DIR/check_x.py", "${TASK_DIR}/check_x.py", "/work/task_dir/check_x.py"} + + @pytest.mark.parametrize( ("command", "expected"), [ - ("python3 $TASK_DIR/check_x.py", {"$TASK_DIR/check_x.py", "/work/task_dir/check_x.py"}), - ("python3 ${TASK_DIR}/check_x.py", {"$TASK_DIR/check_x.py", "/work/task_dir/check_x.py"}), - ('cat "$TASK_DIR/a.txt" && ls', {"$TASK_DIR/a.txt", "/work/task_dir/a.txt"}), + ("python3 $TASK_DIR/check_x.py", _CHECK_X_SPELLINGS), + ("python3 ${TASK_DIR}/check_x.py", _CHECK_X_SPELLINGS), + ('cat "$TASK_DIR/a.txt" && ls', {"$TASK_DIR/a.txt", "${TASK_DIR}/a.txt", "/work/task_dir/a.txt"}), ("echo $TASK_DIR", set()), ("no variable here", set()), ], @@ -900,6 +904,40 @@ def test_task_dir_operand_extraction(command: str, expected: set[str]): assert _task_dir_operands(command) == expected +# -------------------------------------------------------------------------- +# Every $TASK_DIR spelling of a declared reference is a match +# -------------------------------------------------------------------------- + + +def test_a_relative_reference_keeps_its_symbolic_spellings(): + task = _task(reference={"file": "solution.py"}) + spec = derive_graded_material(task, Path("/repo/tasks/leaky/task.yaml")) + assert "$TASK_DIR/solution.py" in spec.paths + assert "${TASK_DIR}/solution.py" in spec.paths + assert "/work/task_dir/solution.py" in spec.paths + + +@pytest.mark.parametrize( + "command", + [ + pytest.param('cat "$TASK_DIR/solution.py"', id="quoted-variable-spelling"), + pytest.param('cat "$TASK_DIR"/solution.py', id="quote-split-spelling"), + pytest.param("cat ${TASK_DIR}/solution.py", id="braced-spelling"), + pytest.param("cat /work/task_dir/solution.py", id="container-spelling"), + ], +) +def test_every_task_dir_spelling_of_the_reference_is_a_read(command: str): + task = _task(reference={"file": "solution.py"}) + spec = derive_graded_material(task, Path("/repo/tasks/leaky/task.yaml")) + info = scan_commands([_turn([_bash(command)])], spec) + assert info.verdict is IntegrityVerdict.TAINTED, f"missed: {command!r}" + + +def test_a_braced_task_dir_marker_locates_the_grader(): + info = scan_commands([_turn([_bash('python3 "${TASK_DIR}/check.py"')])], SPEC) + assert info.verdict is IntegrityVerdict.TAINTED + + # -------------------------------------------------------------------------- # evaluate_integrity: mode handling and failure containment # -------------------------------------------------------------------------- From fdd3b15cfdcefe43550959f8c76ec92c57a6e8c8 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 16:04:20 +0300 Subject: [PATCH 24/31] Stop flagging mock-shim execution and protected segments inside installed libraries --- src/coder_eval/integrity.py | 60 ++++++++++++++++++++++++++++-------- tests/test_integrity_scan.py | 55 +++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 13 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 41592042..ca772e6a 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -560,6 +560,21 @@ def _task_dir_operands(command: str, task_dir: Path | None = None) -> set[str]: # Characters that end a path token inside an already-normalized command string. _TOKEN_DELIMITERS = " \t'\";|&<>()=" +# Directory components that mark an INSTALLED library's internals. `mocks` is an +# ordinary package-internal name (`site-packages/uipath/eval/mocks/…`), and a +# protected-segment hit inside an installed dependency is the library's own code, +# not the scenario's fixture store. +_INSTALLED_LIBRARY_MARKERS = ("site-packages/", "dist-packages/", "node_modules/") + + +def _in_installed_library(haystack: str, start: int) -> bool: + """Whether the path token containing position ``start`` sits inside an + installed library (see ``_INSTALLED_LIBRARY_MARKERS``).""" + left = start + while left > 0 and haystack[left - 1] not in _TOKEN_DELIMITERS: + left -= 1 + return any(marker in haystack[left:start] for marker in _INSTALLED_LIBRARY_MARKERS) + def _created_path(raw: str) -> str: """Normalize a path the agent created, for :func:`_is_agent_created` lookups.""" @@ -634,8 +649,9 @@ def _find_match(text: str, spec: GradedMaterialSpec, created: frozenset[str] | s if not _is_agent_created(haystack, match.start(), match.end(), created): return glob for segment in (*spec.path_segments, *spec.mock_segments): - if _segment_to_regex(segment).search(haystack): - return segment + for match in _segment_to_regex(segment).finditer(haystack): + if not _in_installed_library(haystack, match.start()): + return segment return _grader_match(haystack, spec) @@ -684,12 +700,13 @@ def _finding_kind(matched: str, spec: GradedMaterialSpec) -> IntegrityFindingKin return IntegrityFindingKind.GRADED_READ -def _segment_utility(segment: str) -> tuple[str, list[str]]: - """Leading utility of a shell segment (basename, lowercased) and its tokens. +def _segment_utility(segment: str) -> tuple[str, list[str], int]: + """Leading utility of a shell segment (basename, lowercased), its tokens, and + the index of the utility token within them. Leading ``VAR=value`` assignments and transparent wrappers (``sudo``, ``env``, ``time``, …) are stepped over so ``sudo cat x`` classifies as ``cat``. - Returns ``("", tokens)`` when no utility can be identified. + Returns ``("", tokens, -1)`` when no utility can be identified. """ try: tokens = shlex.split(segment, posix=True) @@ -698,14 +715,14 @@ def _segment_utility(segment: str) -> tuple[str, list[str]]: # skipping the segment, since an unparseable command still ran. tokens = segment.split() - for token in tokens: + for index, token in enumerate(tokens): if "=" in token and not token.startswith(("-", "/", ".")) and token.split("=", 1)[0].isidentifier(): continue # leading environment assignment name = Path(token.replace("\\", "/")).name.casefold().removesuffix(".exe") if name in _TRANSPARENT_PREFIXES: continue - return name, tokens - return "", tokens + return name, tokens, index + return "", tokens, -1 def _git_subcommand(tokens: list[str]) -> str | None: @@ -933,16 +950,19 @@ def _classify_segment( and ``xargs cat`` do not slip past on their wrapper's name. 7. A listing/metadata utility, or a utility that moves, removes or otherwise manipulates a file without emitting it -> not a read. - 8. Anything else -> a read. Conservative on purpose: an unrecognised utility - holding a path to the answer key is more likely a read than not, and a - false positive is visible in the finding's evidence while a false negative - is invisible. + 8. Anything else -> a read, with one carve-out: a MOCK path appearing only + as argv[0] is the shim being EXECUTED (``./m/uip or folders list``), which + is its intended use -- reading its source arrives via a reader utility and + took rule 5 or 6 above. Otherwise conservative on purpose: an unrecognised + utility holding a path to the answer key is more likely a read than not, + and a false positive is visible in the finding's evidence while a false + negative is invisible. """ matched = _find_match(segment, spec, created) if matched is None: return False, None - utility, tokens = _segment_utility(segment) + utility, tokens, utility_index = _segment_utility(segment) normalized_tokens = [Path(t.replace("\\", "/")).name.casefold().removesuffix(".exe") for t in tokens] stripped, input_targets, _ = _strip_redirects(segment) @@ -968,6 +988,20 @@ def _classify_segment( if utility in _LISTING_UTILITIES or utility in _NEUTRAL_UTILITIES: return False, matched + # Rule 8 carve-out: a mock path in argv[0] with no other graded reference in + # the segment is the shim being run, not read. + if ( + matched in spec.mock_segments + and utility_index >= 0 + and _find_match(tokens[utility_index], spec, created) is not None + and not any( + _find_match(token, spec, created) is not None + for index, token in enumerate(tokens) + if index != utility_index + ) + ): + return False, matched + return True, matched diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 4e5a249a..660d7ccb 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -872,6 +872,61 @@ def test_a_bare_protected_directory_name_still_matches(command: str): assert info.verdict is IntegrityVerdict.TAINTED +@pytest.mark.parametrize( + "command", + [ + pytest.param("./m/uip or folders list", id="path-qualified-shim-run"), + pytest.param("m/uip --version", id="relative-shim-run"), + pytest.param("./m/uip jobs list > out.json", id="shim-run-with-a-redirect"), + ], +) +def test_executing_the_mock_shim_is_not_a_read(command: str): + """Running the shim is its intended use; only reading its source leaks.""" + spec = derive_graded_material(_task(sandbox={"mock_path_dirs": ["m"]}), None) + info = scan_commands([_turn([_bash(command)])], spec) + assert info.verdict is IntegrityVerdict.CLEAN, f"false positive: {command!r}" + + +@pytest.mark.parametrize( + "command", + [ + pytest.param("sed -n '1,80p' m/uip", id="paging-the-shim-source"), + pytest.param("cat m/uip", id="cat-the-shim"), + pytest.param("./m/uip m/r/abc.json", id="shim-run-handed-a-mock-operand"), + ], +) +def test_reading_the_shim_or_its_data_still_taints(command: str): + spec = derive_graded_material(_task(sandbox={"mock_path_dirs": ["m"]}), None) + info = scan_commands([_turn([_bash(command)])], spec) + assert info.verdict is IntegrityVerdict.TAINTED, f"missed: {command!r}" + + +@pytest.mark.parametrize( + "command", + [ + pytest.param( + "cat .venv/lib/python3.13/site-packages/uipath/eval/mocks/mockable.py", + id="site-packages", + ), + pytest.param( + "python -c \"print(open('.venv/lib/python3.13/site-packages/uipath/eval/mocks/mockable.py').read())\"", + id="site-packages-via-python", + ), + pytest.param("cat node_modules/lib/mocks/index.js", id="node-modules"), + ], +) +def test_a_protected_segment_inside_an_installed_library_is_not_fixture_data(command: str): + spec = derive_graded_material(_task(sandbox={"mock_path_dirs": ["mocks"]}), None) + info = scan_commands([_turn([_bash(command)])], spec) + assert info.verdict is IntegrityVerdict.CLEAN, f"false positive: {command!r}" + + +def test_a_mock_segment_outside_an_installed_library_still_taints(): + spec = derive_graded_material(_task(sandbox={"mock_path_dirs": ["mocks"]}), None) + info = scan_commands([_turn([_bash("cat src/mocks/manifest.json")])], spec) + assert info.verdict is IntegrityVerdict.TAINTED + + @pytest.mark.parametrize( "command", [ From 3c00d19b85d900399cf85425b6a091fb6de07834 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 16:08:11 +0300 Subject: [PATCH 25/31] Classify a Codex apply_patch by its change kinds so an update of graded material is a read --- src/coder_eval/agents/codex_agent.py | 21 ++++++++-- src/coder_eval/integrity.py | 57 +++++++++++++++++++++++++++- tests/test_codex_agent_unit.py | 33 ++++++++++++++++ tests/test_integrity_scan.py | 47 +++++++++++++++++++++++ 4 files changed, 153 insertions(+), 5 deletions(-) diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index 273305b6..ee491849 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -1560,7 +1560,11 @@ def _tool_parameters(self, root: Any, root_type: str | None) -> dict[str, Any]: if root_type == "fileChange": changes = getattr(root, "changes", None) or [] path = str(changes[0].path) if changes and hasattr(changes[0], "path") else "?" - return {"path": path} + params: dict[str, Any] = {"path": path} + kind = _status_value(getattr(changes[0], "kind", None)) if changes else "" + if kind: + params["kind"] = kind + return params if root_type == "collabAgentToolCall": params: dict[str, Any] = {"operation": _status_value(getattr(root, "tool", ""))} if model := getattr(root, "model", None): @@ -2124,9 +2128,20 @@ def _extract_file_change_telemetry( apply_patch is recorded as an ``error`` (the old ``status != "error"`` test never matched the real PatchApplyStatus values, so failed patches were scored as successful writes). + + The per-change kinds ride along in ``parameters["kinds"]`` (positionally + aligned with ``paths``) when the SDK provides them: unlike Claude's Write, + an apply_patch can UPDATE an existing file -- a read of its current + content -- and the integrity scan needs the kind to tell an update from + an add. Omitted entirely when no change carries a kind. """ try: - paths = [str(c.path) for c in changes if hasattr(c, "path")] if changes else [] + with_paths = [c for c in changes if hasattr(c, "path")] if changes else [] + paths = [str(c.path) for c in with_paths] + kinds = [_status_value(getattr(c, "kind", None)) for c in with_paths] + parameters: dict[str, Any] = {"paths": paths} + if any(kinds): + parameters["kinds"] = kinds status_str = _status_value(status) failed = status_str in _FILE_CHANGE_FAILURE_STATUSES return CommandTelemetry( @@ -2134,7 +2149,7 @@ def _extract_file_change_telemetry( tool_id=change_id, timestamp=datetime.now(), duration_ms=None, - parameters={"paths": paths}, + parameters=parameters, result_status="error" if failed else "success", result_summary=( f"{len(paths)} file(s) changed" diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index ca772e6a..7d46c7d5 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -266,6 +266,15 @@ # INCONCLUSIVE rather than TAINTED, so over-matching is the safe direction. _SCHEMA_KNOWN_TOOLS = _READ_TOOLS | _EDIT_TOOLS | _LISTING_TOOLS | _NEUTRAL_TOOLS | frozenset({"Grep"}) +# Codex apply_patch change kinds. Codex funnels every file change through the +# `Write` tool name with a `paths` list; unlike Claude's Write (a whole-file +# creation) an apply_patch can UPDATE a file, which required its current content +# -- the same reasoning that makes `_EDIT_TOOLS` reads. A kind outside both sets +# is a Codex vocabulary we have not seen: neither excused nor tainted, the call +# is left "not understood" so the verdict degrades to INCONCLUSIVE, never CLEAN. +_CODEX_ADD_KINDS = frozenset({"add", "create"}) +_CODEX_EDIT_KINDS = frozenset({"update", "delete", "modify", "rename"}) + # Basename patterns that are graded material in every suite, independent of what # this particular task declares. Deliberately short: each entry is a name the # framework or the task-authoring convention owns. One of them (`RESOLUTION.md`) @@ -1074,8 +1083,17 @@ def scan_commands(turns: list[TurnRecord], spec: GradedMaterialSpec) -> Integrit notes.append( f"{cmd.tool_name} referenced {matched} but its read semantics are unknown; not counted" ) - if cmd.tool_name == "Write" and isinstance(cmd.parameters.get("file_path"), str): - created.add(_created_path(cmd.parameters["file_path"])) + if cmd.tool_name == "Write": + if isinstance(cmd.parameters.get("file_path"), str): + created.add(_created_path(cmd.parameters["file_path"])) + # Codex apply_patch: only the `add` changes are creations. + paths, kinds = cmd.parameters.get("paths"), cmd.parameters.get("kinds") + if isinstance(paths, list) and isinstance(kinds, list) and len(paths) == len(kinds): + created.update( + _created_path(str(path)) + for path, kind in zip(paths, kinds, strict=True) + if str(kind).casefold() in _CODEX_ADD_KINDS + ) if is_read and matched is not None: kind = _finding_kind(matched, spec) @@ -1291,6 +1309,9 @@ def _structured_read( caller turns into INCONCLUSIVE -- neither a silent pass nor a taint on a tool whose behavior we are guessing at. """ + if cmd.tool_name == "Write" and isinstance(cmd.parameters.get("paths"), list): + return _codex_file_change_read(cmd, spec, created) + if cmd.tool_name in _SCHEMA_KNOWN_TOOLS: haystack = " ".join(str(cmd.parameters[key]) for key in _PATH_PARAMETER_KEYS if cmd.parameters.get(key)) else: @@ -1312,6 +1333,38 @@ def _structured_read( return False, matched, False +def _codex_file_change_read( + cmd: CommandTelemetry, + spec: GradedMaterialSpec, + created: frozenset[str] | set[str], +) -> tuple[bool, str | None, bool]: + """Classify a Codex apply_patch: a ``Write`` telemetry with a ``paths`` list. + + A change of kind ``update``/``delete`` required the file's current content, + so a graded path among those is a read -- the arm-specific twin of an Edit + tool call. A pure ``add`` is a creation, like Claude's Write. Without a kind + per path the two are indistinguishable (an add of the deliverable is the + honest flow, an update of a golden is the leak), so a graded hit is reported + "not understood" and the verdict degrades to INCONCLUSIVE, never CLEAN. + """ + paths = [str(p) for p in cmd.parameters["paths"]] + kinds = cmd.parameters.get("kinds") + matched = next((m for m in (_find_match(p, spec, created) for p in paths) if m is not None), None) + if matched is None: + return False, None, True + if not (isinstance(kinds, list) and len(kinds) == len(paths)): + return False, matched, False + for path, kind in zip(paths, kinds, strict=True): + if _find_match(path, spec, created) is None: + continue + normalized_kind = str(kind).casefold() + if normalized_kind in _CODEX_EDIT_KINDS: + return True, matched, True + if normalized_kind not in _CODEX_ADD_KINDS: + return False, matched, False + return False, matched, True + + def evaluate_integrity( task: TaskDefinition, task_file: Path | None, diff --git a/tests/test_codex_agent_unit.py b/tests/test_codex_agent_unit.py index 2115a3d8..7031a06d 100644 --- a/tests/test_codex_agent_unit.py +++ b/tests/test_codex_agent_unit.py @@ -193,3 +193,36 @@ def test_turn_record_defaults_unrecovered_subagent_threads_to_zero(): from coder_eval.models import TurnRecord assert TurnRecord(iteration=1, user_input="p", agent_output="a").unrecovered_subagent_threads == 0 + + +class TestFileChangeKinds: + """apply_patch change kinds ride along in Write telemetry parameters. + + The integrity scan needs them: a Codex `update` of a file required its + current content (a read), while an `add` is a creation like Claude's Write. + """ + + def _agent(self) -> CodexAgent: + return CodexAgent(parse_agent_config(type=AgentKind.CODEX)) + + def test_kinds_are_recorded_alongside_paths(self): + changes = [ + SimpleNamespace(path="a.py", kind="update"), + SimpleNamespace(path="b.py", kind="add"), + ] + telemetry = self._agent()._extract_file_change_telemetry("fc_1", changes, "success", 0) + assert telemetry is not None + assert telemetry.tool_name == "Write" + assert telemetry.parameters == {"paths": ["a.py", "b.py"], "kinds": ["update", "add"]} + + def test_kinds_are_omitted_when_the_sdk_provides_none(self): + changes = [SimpleNamespace(path="a.py")] + telemetry = self._agent()._extract_file_change_telemetry("fc_1", changes, "success", 0) + assert telemetry is not None + assert telemetry.parameters == {"paths": ["a.py"]} + + def test_stream_parameters_carry_the_first_change_kind(self): + root = SimpleNamespace( + type="fileChange", id="f1", changes=[SimpleNamespace(path="a.py", kind="update")], status="success" + ) + assert self._agent()._tool_parameters(root, "fileChange") == {"path": "a.py", "kind": "update"} diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 660d7ccb..e07c8f67 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -585,6 +585,53 @@ def test_a_path_parameter_of_a_known_tool_still_matches(): assert info.verdict is IntegrityVerdict.TAINTED +# Codex funnels apply_patch through the `Write` tool name with a `paths` list; +# the per-change kinds tell a creation apart from an edit of graded material. + + +def test_a_codex_patch_updating_graded_material_is_a_read(): + """The identical Claude `Edit` is a read; the Codex arm must not be blind.""" + cmd = _cmd("Write", {"paths": ["/repo/tasks/leaky/solution.py"], "kinds": ["update"]}) + info = scan_commands([_turn([cmd])], SPEC) + assert info.verdict is IntegrityVerdict.TAINTED + assert info.findings[0].tool_name == "Write" + + +def test_a_codex_patch_deleting_graded_material_is_a_read(): + cmd = _cmd("Write", {"paths": ["/repo/tasks/leaky/solution.py"], "kinds": ["delete"]}) + info = scan_commands([_turn([cmd])], SPEC) + assert info.verdict is IntegrityVerdict.TAINTED + + +def test_a_codex_add_of_the_deliverable_is_a_creation(): + commands = [ + _cmd("Write", {"paths": ["RESOLUTION.md"], "kinds": ["add"]}, tool_id="t1"), + _bash("cat RESOLUTION.md"), + ] + info = scan_commands([_turn(commands)], SPEC) + assert info.verdict is IntegrityVerdict.CLEAN + + +def test_a_codex_patch_without_kinds_is_inconclusive_on_a_graded_hit(): + """Add-vs-update cannot be told apart without kinds; CLEAN would be a lie.""" + cmd = _cmd("Write", {"paths": ["/repo/tasks/leaky/solution.py"]}) + info = scan_commands([_turn([cmd])], SPEC) + assert info.verdict is IntegrityVerdict.INCONCLUSIVE + assert info.findings == [] + + +def test_a_codex_patch_of_unrelated_files_is_clean(): + cmd = _cmd("Write", {"paths": ["src/app.py", "src/util.py"], "kinds": ["update", "update"]}) + info = scan_commands([_turn([cmd])], SPEC) + assert info.verdict is IntegrityVerdict.CLEAN + + +def test_an_unknown_codex_change_kind_is_inconclusive(): + cmd = _cmd("Write", {"paths": ["/repo/tasks/leaky/solution.py"], "kinds": ["mystery"]}) + info = scan_commands([_turn([cmd])], SPEC) + assert info.verdict is IntegrityVerdict.INCONCLUSIVE + + def test_unknown_tool_touching_graded_material_is_inconclusive_not_tainted(): """We do not guess at an unrecognised tool's semantics in either direction.""" info = scan_commands([_turn([_cmd("mcp__some__fetch", {"target": "RESOLUTION.md"})])], SPEC) From 795017451c6fbcd3a235f0d0a8e0c405013fd07a Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 16:10:09 +0300 Subject: [PATCH 26/31] Classify command-substitution bodies as commands in their own right --- src/coder_eval/integrity.py | 67 ++++++++++++++++++++++++++++++++++++ tests/test_integrity_scan.py | 28 +++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 7d46c7d5..907cbdae 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -1269,9 +1269,70 @@ def _flush() -> None: return segments +def _substitution_bodies(segment: str) -> list[str]: + """The text inside a segment's command substitutions: ``$(…)`` and backticks. + + That text EXECUTES -- ``printf '%s\\n' "$(cat KEY)"`` runs the ``cat`` even + though the enclosing utility is a harmless ``printf`` -- so each body must be + classified as a command in its own right. Single-quoted text is inert and + skipped; double quotes do not stop expansion, so ``"$(…)"`` is collected. + """ + bodies: list[str] = [] + in_single = False + i = 0 + n = len(segment) + while i < n: + c = segment[i] + if c == "'": + in_single = not in_single + i += 1 + continue + if in_single: + i += 1 + continue + if c == "\\" and i + 1 < n: + i += 2 + continue + if segment[i : i + 2] == "$(": + depth = 1 + sub_single = False + j = i + 2 + while j < n and depth: + ch = segment[j] + if ch == "\\" and not sub_single and j + 1 < n: + j += 2 + continue + if ch == "'": + sub_single = not sub_single + elif not sub_single and ch == "(": + depth += 1 + elif not sub_single and ch == ")": + depth -= 1 + j += 1 + bodies.append(segment[i + 2 : j - 1 if depth == 0 else j]) + i = j + continue + if c == "`": + end = segment.find("`", i + 1) + if end == -1: + i += 1 + continue + bodies.append(segment[i + 1 : end]) + i = end + 1 + continue + i += 1 + return bodies + + def _bash_read(command: str, spec: GradedMaterialSpec, created: set[str] | None = None) -> tuple[bool, str | None]: """Classify a shell command by splitting it into segments and judging each. + Each segment's command substitutions are then classified recursively as + commands of their own -- the enclosing utility says nothing about what ran + inside ``$(…)``. Recursing per SEGMENT (not on the raw command) matters: + segmentation has already dropped inert text, so a ``$(…)`` inside a quoted + heredoc body or a comment is never reached. + ``created`` is the transcript-ordered set of paths the agent has written so far; each segment's truncating redirect targets are added AFTER the segment is classified, so ``cat > RESOLUTION.md && cat RESOLUTION.md`` excuses the @@ -1286,6 +1347,12 @@ def _bash_read(command: str, spec: GradedMaterialSpec, created: set[str] | None mentioned = matched if is_read: return True, matched + for body in _substitution_bodies(segment): + is_read, matched = _bash_read(body, spec, created) + if matched is not None: + mentioned = matched + if is_read: + return True, matched _, _, made = _strip_redirects(segment) created.update(_created_path(target) for target in made) return False, mentioned diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index e07c8f67..a9c42add 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -377,6 +377,34 @@ def test_a_whole_literal_path_still_matches(command: str): pytest.param("cat > x < Date: Wed, 5 Aug 2026 16:12:31 +0300 Subject: [PATCH 27/31] Treat shell-state builtins as neutral so a PATH export naming the mock dir is not a read --- src/coder_eval/integrity.py | 18 ++++++++++++------ tests/test_integrity_scan.py | 7 +++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 907cbdae..2ac0eec0 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -109,12 +109,18 @@ ) # Utilities and shell keywords that touch a path without emitting its contents: -# file manipulation (`rm`, `mv`, `chmod`) and the loop / conditional keywords a -# segment can start with (`for f in check_*.py`). Without these, rule 7 reads an -# agent tidying up its own helper script as a leak and voids an honest row. -# `git` is deliberately NOT here -- it is classified per subcommand -# (:func:`_git_is_read`), because half of them print file content. -_NEUTRAL_UTILITIES = frozenset({"rm", "mv", "chmod", "for", "while", "if", "do", "done", "then", "fi"}) +# file manipulation (`rm`, `mv`, `chmod`), the loop / conditional keywords a +# segment can start with (`for f in check_*.py`), and the shell-state builtins +# (`export PATH=m:$PATH` names the mock dir without opening anything -- though a +# substitution inside the assignment is still classified on its own). Without +# these, rule 8 reads an agent tidying up its own helper script or extending its +# PATH as a leak and voids an honest row. `git` is deliberately NOT here -- it is +# classified per subcommand (:func:`_git_is_read`), because half of its +# subcommands print file content. +_NEUTRAL_UTILITIES = frozenset( + {"rm", "mv", "chmod", "for", "while", "if", "do", "done", "then", "fi", + "export", "unset", "alias", "local", "declare", "typeset", "set"} +) # `git` subcommands that do NOT emit file content: they stage, record, move or # report. Everything else -- `show`, `cat-file -p`, `diff`, `blame`, `grep`, diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index a9c42add..87d66dfd 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -1009,6 +1009,7 @@ def test_a_mock_segment_outside_an_installed_library_still_taints(): pytest.param("rm -rf mocks", id="removing-the-mock-directory"), pytest.param("find mocks -name '*.json'", id="enumerating-the-mock-directory"), pytest.param("grep -rl secret mocks", id="files-only-search-of-the-mock-directory"), + pytest.param("export PATH=mocks:$PATH", id="path-export-naming-the-mock-directory"), ], ) def test_utility_semantics_still_decide_a_bare_directory_touch(command: str): @@ -1017,6 +1018,12 @@ def test_utility_semantics_still_decide_a_bare_directory_touch(command: str): assert info.verdict is IntegrityVerdict.CLEAN +def test_a_substitution_inside_an_export_is_still_classified(): + spec = derive_graded_material(_task(sandbox={"mock_path_dirs": ["mocks"]}), None) + info = scan_commands([_turn([_bash('export STORE="$(cat mocks/manifest.json)"')])], spec) + assert info.verdict is IntegrityVerdict.TAINTED + + _CHECK_X_SPELLINGS = {"$TASK_DIR/check_x.py", "${TASK_DIR}/check_x.py", "/work/task_dir/check_x.py"} From aeeb7b86329d16336a2a5e0849442107c47c788a Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 16:16:14 +0300 Subject: [PATCH 28/31] Exclude voided replicates from the variant report's bootstrap confidence interval --- src/coder_eval/reports_experiment.py | 43 +++++++++++++++++----------- tests/test_experiment_reports.py | 26 +++++++++++++++++ 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index 10224871..a80d1dc0 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -524,6 +524,26 @@ def _win_loss_lines(result: ExperimentResult) -> list[str]: lines.append(f"- **{ts.task_id}**: spread={ts.score_spread:.3f}, best={ts.best_variant}") return lines + @staticmethod + def _honest_replicate_scores(result: ExperimentResult, variant_id: str) -> tuple[list[float], int]: + """A variant's per-replicate scores split into (non-voided scores, voided count). + + The integrity gate deliberately preserves a voided replicate's inflated + score, so every score-based statistic has to drop those samples itself. + """ + per_rep = result.per_replicate_scores.get(variant_id, {}) + voided_flags = result.per_replicate_voided.get(variant_id, {}) + scored: list[float] = [] + voided = 0 + for task_id, scores in per_rep.items(): + flags = voided_flags.get(task_id, []) + for index, score in enumerate(scores): + if index < len(flags) and flags[index]: + voided += 1 + else: + scored.append(score) + return scored, voided + @staticmethod def _replicate_stats_lines(result: ExperimentResult) -> list[str]: """The ``## Replicate Statistics`` block: per-variant bootstrap-CI / Wilson @@ -547,17 +567,7 @@ def _replicate_stats_lines(result: ExperimentResult) -> list[str]: lines.append("| Variant | Replicates/task | Mean score | 95% CI | Pass-rate (Wilson 95%) | Voided |") lines.append("|---------|-----------------|------------|--------|------------------------|--------|") for vid in result.variant_ids: - per_rep = result.per_replicate_scores.get(vid, {}) - voided_flags = result.per_replicate_voided.get(vid, {}) - scored: list[float] = [] - voided = 0 - for task_id, scores in per_rep.items(): - flags = voided_flags.get(task_id, []) - for index, score in enumerate(scores): - if index < len(flags) and flags[index]: - voided += 1 - else: - scored.append(score) + scored, voided = ExperimentReportGenerator._honest_replicate_scores(result, vid) passes = sum(1 for s in scored if s >= _REPLICATE_PASS_THRESHOLD) m, lo, hi = bootstrap_mean_ci(scored) mean_str = f"{m:.3f}" if scored else "n/a" @@ -691,12 +701,13 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: if durations and len(durations) >= 2: lines.append(f"- **Duration Stddev**: {stddev(durations):.1f}s") if agg.replicate_count > 1: - per_rep = result.per_replicate_scores.get(variant_id, {}) - all_rep_scores: list[float] = [s for rep_scores in per_rep.values() for s in rep_scores] - if all_rep_scores: - _, lo, hi = bootstrap_mean_ci(all_rep_scores) + # Voided replicates keep their inflated scores on purpose; the CI must + # not bootstrap over them any more than the pass rate counts them. + honest_scores, _ = ExperimentReportGenerator._honest_replicate_scores(result, variant_id) + if honest_scores: + _, lo, hi = bootstrap_mean_ci(honest_scores) lines.append(f"- **Replicates/task**: {agg.replicate_count}") - lines.append(f"- **Score 95% CI**: [{lo:.3f}, {hi:.3f}] (bootstrap over {len(all_rep_scores)} samples)") + lines.append(f"- **Score 95% CI**: [{lo:.3f}, {hi:.3f}] (bootstrap over {len(honest_scores)} samples)") # Task Details table has_similarity = any(vr.reference_similarity is not None for vr in variant_results) diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py index e881c1ca..235fa8c3 100644 --- a/tests/test_experiment_reports.py +++ b/tests/test_experiment_reports.py @@ -1249,6 +1249,32 @@ def test_variant_report_no_ci_when_replicate_count_is_one(self): md = ExperimentReportGenerator.generate_variant_report("a", result) assert "Score 95% CI" not in md + def test_variant_report_ci_excludes_voided_replicates(self): + """A voided replicate keeps its inflated score; bootstrapping over it + would contaminate the CI beside a pass rate that excludes it.""" + per_rep = {"a": {"task-1": [1.0, 0.5, 0.5]}} + voided = {"a": {"task-1": [True, False, False]}} + result = self._make_result( + replicate_count=3, + per_replicate_scores=per_rep, + per_replicate_voided=voided, + variant_ids=["a"], + ) + md = ExperimentReportGenerator.generate_variant_report("a", result) + assert "[0.500, 0.500] (bootstrap over 2 samples)" in md + + def test_variant_report_omits_the_ci_when_every_replicate_was_voided(self): + per_rep = {"a": {"task-1": [1.0, 1.0, 1.0]}} + voided = {"a": {"task-1": [True, True, True]}} + result = self._make_result( + replicate_count=3, + per_replicate_scores=per_rep, + per_replicate_voided=voided, + variant_ids=["a"], + ) + md = ExperimentReportGenerator.generate_variant_report("a", result) + assert "Score 95% CI" not in md + class TestCollectVariantSeries: """The one series collector both reporters share.""" From 1a5f1ee0af54293911cc686a14ca2d5a0ce4d3b8 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 17:01:13 +0300 Subject: [PATCH 29/31] Treat a mock path in the invocation prefix as shim execution, not a read --- src/coder_eval/integrity.py | 41 +++++++++++++++++++++--------------- tests/test_integrity_scan.py | 5 +++++ 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 2ac0eec0..6c4a5d79 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -965,13 +965,16 @@ def _classify_segment( and ``xargs cat`` do not slip past on their wrapper's name. 7. A listing/metadata utility, or a utility that moves, removes or otherwise manipulates a file without emitting it -> not a read. - 8. Anything else -> a read, with one carve-out: a MOCK path appearing only - as argv[0] is the shim being EXECUTED (``./m/uip or folders list``), which - is its intended use -- reading its source arrives via a reader utility and - took rule 5 or 6 above. Otherwise conservative on purpose: an unrecognised - utility holding a path to the answer key is more likely a read than not, - and a false positive is visible in the finding's evidence while a false - negative is invisible. + 8. Anything else -> a read, with one carve-out: a MOCK path confined to the + command's invocation prefix -- argv[0] itself (``./m/uip or folders list``) + or a leading ``PATH=./m:$PATH uip …`` assignment that puts the shim dir on + PATH so a bare ``uip`` resolves to it -- is the shim being EXECUTED, its + intended use. Both spellings are invocation machinery, not an operand being + read; a mock path in an actual operand still taints, and reading the shim + SOURCE arrives via a reader utility and took rule 5 or 6 above. Otherwise + conservative on purpose: an unrecognised utility holding a path to the + answer key is more likely a read than not, and a false positive is visible + in the finding's evidence while a false negative is invisible. """ matched = _find_match(segment, spec, created) if matched is None: @@ -1003,19 +1006,23 @@ def _classify_segment( if utility in _LISTING_UTILITIES or utility in _NEUTRAL_UTILITIES: return False, matched - # Rule 8 carve-out: a mock path in argv[0] with no other graded reference in - # the segment is the shim being run, not read. - if ( - matched in spec.mock_segments - and utility_index >= 0 - and _find_match(tokens[utility_index], spec, created) is not None - and not any( + # Rule 8 carve-out: a mock path confined to the invocation prefix -- argv[0] + # itself or a leading `PATH=`-style env assignment the utility parser stepped + # over (indices <= utility_index) -- is the shim being run, not read. A mock + # reference in an operand (index > utility_index) still taints. + if matched in spec.mock_segments and utility_index >= 0: + prefix_has_ref = any( _find_match(token, spec, created) is not None for index, token in enumerate(tokens) - if index != utility_index + if index <= utility_index ) - ): - return False, matched + operand_has_ref = any( + _find_match(token, spec, created) is not None + for index, token in enumerate(tokens) + if index > utility_index + ) + if prefix_has_ref and not operand_has_ref: + return False, matched return True, matched diff --git a/tests/test_integrity_scan.py b/tests/test_integrity_scan.py index 87d66dfd..8ba3f67a 100644 --- a/tests/test_integrity_scan.py +++ b/tests/test_integrity_scan.py @@ -953,6 +953,9 @@ def test_a_bare_protected_directory_name_still_matches(command: str): pytest.param("./m/uip or folders list", id="path-qualified-shim-run"), pytest.param("m/uip --version", id="relative-shim-run"), pytest.param("./m/uip jobs list > out.json", id="shim-run-with-a-redirect"), + pytest.param('PATH="./m:$PATH" uip or folders list', id="path-prefix-shim-run"), + pytest.param("PATH=m:$PATH uip jobs list --output json", id="path-prefix-unquoted-shim-run"), + pytest.param('mkdir -p raw && PATH="./m:$PATH" uip or folders list | tee raw/f.json', id="path-prefix-shim-run-piped"), ], ) def test_executing_the_mock_shim_is_not_a_read(command: str): @@ -968,6 +971,8 @@ def test_executing_the_mock_shim_is_not_a_read(command: str): pytest.param("sed -n '1,80p' m/uip", id="paging-the-shim-source"), pytest.param("cat m/uip", id="cat-the-shim"), pytest.param("./m/uip m/r/abc.json", id="shim-run-handed-a-mock-operand"), + pytest.param('PATH="./m:$PATH" cat m/.store', id="path-prefix-but-reader-utility"), + pytest.param('PATH="./m:$PATH" uip run m/r/abc.json', id="path-prefix-with-a-mock-operand"), ], ) def test_reading_the_shim_or_its_data_still_taints(command: str): From c9c47b9a5097b31080a3e6087c2f3a146dff3b16 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Wed, 5 Aug 2026 21:56:42 +0300 Subject: [PATCH 30/31] Unwrap a shell wrapper's command string so its redirects and pipelines classify as the command that ran --- src/coder_eval/integrity.py | 99 +++++++++++++++++++++++++++++++----- tests/test_integrity_scan.py | 80 +++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 12 deletions(-) diff --git a/src/coder_eval/integrity.py b/src/coder_eval/integrity.py index 6c4a5d79..c21b66f4 100644 --- a/src/coder_eval/integrity.py +++ b/src/coder_eval/integrity.py @@ -78,6 +78,12 @@ # that decides a segment's classification. _TRANSPARENT_PREFIXES = frozenset({"sudo", "env", "command", "time", "nohup", "nice", "exec", "builtin", "eval"}) +# Shells that run a command string handed to them with ``-c``. Codex records +# every shell call as ``/bin/bash -lc "