From 1fb962d46bc788b7e77b209c7678881b7191ce22 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 13 Nov 2026 00:04:23 -0800 Subject: [PATCH 01/10] fix(harness): a verifier that exits 0 without reward.txt errors the cell The exit-code fallback scored such a verifier 1.0. A missing reward file is now no verdict: the cell is errored. Non-zero exit without a reward file stays 0.0. Signed-off-by: Nikolay Petrov --- ...0261113_120500_napetrov_fsm687-verifier.md | 8 +++ dokimi/dokimi/harnesses/command.py | 11 ++++ dokimi/dokimi/harnesses/docker_solver.py | 5 +- dokimi/scripts/growth_baseline.json | 2 +- dokimi/tests/test_docker_task_md.py | 7 ++- dokimi/tests/test_docker_verify_reward.py | 61 +++++++++++++++++++ 6 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md create mode 100644 dokimi/tests/test_docker_verify_reward.py diff --git a/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md b/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md new file mode 100644 index 000000000..458b9ea98 --- /dev/null +++ b/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **A verifier that exits 0 without writing `reward.txt` is an error, not a pass** — a Docker task whose verifier wrote no `/logs/verifier/reward.txt` used to score 1.0 on exit code 0. The cell is now errored, with no reward, and `run.log` names the missing file. A verifier that exits non-zero without a reward file still scores 0.0. diff --git a/dokimi/dokimi/harnesses/command.py b/dokimi/dokimi/harnesses/command.py index a2da8ace2..e939c9433 100644 --- a/dokimi/dokimi/harnesses/command.py +++ b/dokimi/dokimi/harnesses/command.py @@ -443,6 +443,17 @@ def _find_reward(output_dir: Path | None, stdout: str, stderr: str) -> float | N return parsed if parsed is not None else _parse_reward_text(stderr) +def _exit_code_verdict(returncode: int) -> float: + """The verdict of a reward-file verifier that wrote no ``reward.txt``. + + Exit 0 is not a verdict: a verifier that wrote no reward file passed nothing, + so the cell is errored, as a verifier that did not run is. Non-zero stays 0.0. + """ + if returncode == 0: + raise RuntimeError("verifier exited 0 but wrote no /logs/verifier/reward.txt; see run.log") + return 0.0 + + def _parse_reward_text(text: str, *, reward_file: bool = False) -> float | None: """A reward from a ``reward.txt`` (``reward_file``) or from command output. diff --git a/dokimi/dokimi/harnesses/docker_solver.py b/dokimi/dokimi/harnesses/docker_solver.py index c91a3d571..5d5b10917 100644 --- a/dokimi/dokimi/harnesses/docker_solver.py +++ b/dokimi/dokimi/harnesses/docker_solver.py @@ -71,7 +71,7 @@ solver_cli_flags, ) from .claude_json import _last_json_object, _parse_claude_json # noqa: F401 -from .command import _parse_reward_text +from .command import _exit_code_verdict, _parse_reward_text from .container_agent import (AGENT_BIN, DOCKER_STEP_TIMEOUT, FILES_DIR, TASK_CONTAINER_HARDENING, ContainerSandbox, Docker) from .models import HarnessResult, OperationRecord @@ -657,8 +657,7 @@ def _verify(self, task: TaskSpec, container: str, output_dir: Path, log, raise RuntimeError(f"task container {container} state inspect timed out at verify time; see run.log") if (getattr(running, "stdout", "") or "").strip() != "true": raise RuntimeError(f"task container {container} is not running at verify time; see run.log") - # No reward file written: fall back to verifier exit code. - return 1.0 if proc.returncode == 0 else 0.0 + return _exit_code_verdict(proc.returncode) def _plan(self, task: TaskSpec, record: Optional[Path] = None) -> NetworkPlan: return resolve_recorded(task, self.network, agent=self.mode != "oracle", record=record) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index fcbde4ba2..ae7873b4d 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -2,7 +2,7 @@ "files": { "dokimi/commands/novelty.py": {"lines": 818, "rationale": "#403 --out check and the review controls (--context-framing, no_artifact_repeat) on top of the routing, --all-vs-all and estimate-only output from main; routing output in commands/novelty_routing_report.py; role clients, spend cap, USD estimate in novelty/cost.py; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, - "dokimi/harnesses/docker_solver.py": {"lines": 897, "rationale": "pre-existing when the gate was introduced"}, + "dokimi/harnesses/docker_solver.py": {"lines": 896, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1031, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, "dokimi/novelty/routing.py": {"lines": 822, "rationale": "grew with the #172 routing field, the keep-copies and dropped-query rules, the #405 over-limit warnings and the #403 one-repeat noise and measured reply tokens; file split tracked separately"}, "dokimi/report/model_compare.py": {"lines": 1161, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, diff --git a/dokimi/tests/test_docker_task_md.py b/dokimi/tests/test_docker_task_md.py index 2fcf4f0d2..a791a365b 100644 --- a/dokimi/tests/test_docker_task_md.py +++ b/dokimi/tests/test_docker_task_md.py @@ -103,8 +103,9 @@ def _hermetic(tmp_path, monkeypatch): def _capture(stdout="") -> DockerCLI: - """A docker CLI where every call exits 0 and prints ``stdout`` (no reward file).""" - return DockerCLI(answer=lambda call: reply(stdout)) + """A docker CLI where every call exits 0 and prints ``stdout``. The verifier's + ``docker cp`` falls through to the fake's reward file (1.0), so a verify is a verdict.""" + return DockerCLI(answer=lambda call: None if call.argv[1:2] == ("cp",) else reply(stdout)) def _docker_via(cli: DockerCLI): @@ -375,6 +376,8 @@ def respond(call): got = answer(call) if answer else None if got is not None: return got + if call.argv[1:2] == ("cp",): # the verifier's reward file comes from the fake's own cp + return None return reply(image_workdir) if call.argv[0] == "docker" else None return DockerCLI(answer=respond, solver='{"total_cost_usd": 0.42}') diff --git a/dokimi/tests/test_docker_verify_reward.py b/dokimi/tests/test_docker_verify_reward.py new file mode 100644 index 000000000..25407a4bf --- /dev/null +++ b/dokimi/tests/test_docker_verify_reward.py @@ -0,0 +1,61 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A docker verifier's exit code is a verdict only with a reward file behind it.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from dokimi.harnesses import load_task + + +def _harbor(root: Path, name: str = "hb-task") -> Path: + task = root / name + for sub in ("environment", "solution", "tests"): + (task / sub).mkdir(parents=True) + (task / "solution" / "solve.sh").write_text("#!/bin/bash\n", encoding="utf-8") + (task / "tests" / "test.sh").write_text("#!/bin/bash\n", encoding="utf-8") + (task / "instruction.md").write_text("Fix it.", encoding="utf-8") + (task / "task.toml").write_text( + '[metadata]\ntags = ["onedal"]\n\n[environment]\nworkdir = "/app"\n', encoding="utf-8" + ) + return task + + +def test_docker_verify_exit_zero_without_a_reward_file_is_an_error_not_a_pass(tmp_path): + import io + + from docker_cli_fake import DockerCLI + from dokimi.harnesses.docker_solver import DockerSolverHarness + + out = tmp_path / "o" + out.mkdir() + task = load_task(_harbor(tmp_path)) + harness = DockerSolverHarness(harness_id="docker-oracle", mode="oracle", + runner=DockerCLI(reward=None)) + # The verifier exited 0 but never wrote reward.txt: no verdict, so the cell is errored. + with pytest.raises(RuntimeError, match="reward.txt"): + harness._verify(task, "c1", out, log=io.StringIO()) + + +def test_docker_verify_nonzero_exit_without_a_reward_file_stays_a_fail(tmp_path): + import io + + from docker_cli_fake import DockerCLI, reply + from dokimi.harnesses.docker_solver import DockerSolverHarness + + out = tmp_path / "o" + out.mkdir() + task = load_task(_harbor(tmp_path)) + + def failing_verifier(call): + if "-lc" in call.argv: + return reply(returncode=1) + return None + + harness = DockerSolverHarness(harness_id="docker-oracle", mode="oracle", + runner=DockerCLI(reward=None, answer=failing_verifier)) + assert harness._verify(task, "c1", out, log=io.StringIO()) == 0.0 From 81c4164877b1aa5f7d66aa7cf10543d88efd11b0 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sat, 14 Nov 2026 01:08:26 -0800 Subject: [PATCH 02/10] fix(harness): a docker reward.txt with no number in it errors the cell PASS or an empty file parsed to reward None with no error, so the summary scored the cell as a fail. It is no verdict, as in a command harness's reward file (command.py _exit_code_reward): the cell is errored whatever the exit code. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/harnesses/command.py | 11 +++++++++++ dokimi/dokimi/harnesses/docker_solver.py | 6 ++---- dokimi/scripts/growth_baseline.json | 2 +- dokimi/tests/test_docker_verify_reward.py | 19 +++++++++++++++++++ 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/harnesses/command.py b/dokimi/dokimi/harnesses/command.py index e939c9433..2c5bd0ce5 100644 --- a/dokimi/dokimi/harnesses/command.py +++ b/dokimi/dokimi/harnesses/command.py @@ -454,6 +454,17 @@ def _exit_code_verdict(returncode: int) -> float: return 0.0 +def _reward_file_verdict(text: str) -> float: + """The reward in a verifier's ``reward.txt``; one with no number in it is no verdict. + + As in :func:`_exit_code_reward`, whatever the exit code: the cell is errored, not failed. + """ + reward = _parse_reward_text(text, reward_file=True) + if reward is None: + raise RuntimeError("/logs/verifier/reward.txt has no reward in it; see run.log") + return reward + + def _parse_reward_text(text: str, *, reward_file: bool = False) -> float | None: """A reward from a ``reward.txt`` (``reward_file``) or from command output. diff --git a/dokimi/dokimi/harnesses/docker_solver.py b/dokimi/dokimi/harnesses/docker_solver.py index 5d5b10917..c5960a99c 100644 --- a/dokimi/dokimi/harnesses/docker_solver.py +++ b/dokimi/dokimi/harnesses/docker_solver.py @@ -71,7 +71,7 @@ solver_cli_flags, ) from .claude_json import _last_json_object, _parse_claude_json # noqa: F401 -from .command import _exit_code_verdict, _parse_reward_text +from .command import _exit_code_verdict, _reward_file_verdict from .container_agent import (AGENT_BIN, DOCKER_STEP_TIMEOUT, FILES_DIR, TASK_CONTAINER_HARDENING, ContainerSandbox, Docker) from .models import HarnessResult, OperationRecord @@ -648,9 +648,7 @@ def _verify(self, task: TaskSpec, container: str, output_dir: Path, log, if pulled.timed_out: # not "no reward file": the exit code would pass a failing verifier raise subprocess.TimeoutExpired(pulled.argv, DOCKER_STEP_TIMEOUT) if pulled.returncode == 0 and reward_file.is_file(): - return _parse_reward_text( - reward_file.read_text(encoding="utf-8", errors="replace"), reward_file=True - ) + return _reward_file_verdict(reward_file.read_text(encoding="utf-8", errors="replace")) if proc.returncode != 0: # no verdict at all if the container is gone, not a 0 running = self._docker(["inspect", "-f", "{{.State.Running}}", container], log, check=False) if running.timed_out: # no answer is not "stopped": say so, the cell is errored either way diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index ae7873b4d..92845ba17 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -2,7 +2,7 @@ "files": { "dokimi/commands/novelty.py": {"lines": 818, "rationale": "#403 --out check and the review controls (--context-framing, no_artifact_repeat) on top of the routing, --all-vs-all and estimate-only output from main; routing output in commands/novelty_routing_report.py; role clients, spend cap, USD estimate in novelty/cost.py; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, - "dokimi/harnesses/docker_solver.py": {"lines": 896, "rationale": "pre-existing when the gate was introduced"}, + "dokimi/harnesses/docker_solver.py": {"lines": 894, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1031, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, "dokimi/novelty/routing.py": {"lines": 822, "rationale": "grew with the #172 routing field, the keep-copies and dropped-query rules, the #405 over-limit warnings and the #403 one-repeat noise and measured reply tokens; file split tracked separately"}, "dokimi/report/model_compare.py": {"lines": 1161, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, diff --git a/dokimi/tests/test_docker_verify_reward.py b/dokimi/tests/test_docker_verify_reward.py index 25407a4bf..71de2772a 100644 --- a/dokimi/tests/test_docker_verify_reward.py +++ b/dokimi/tests/test_docker_verify_reward.py @@ -59,3 +59,22 @@ def failing_verifier(call): harness = DockerSolverHarness(harness_id="docker-oracle", mode="oracle", runner=DockerCLI(reward=None, answer=failing_verifier)) assert harness._verify(task, "c1", out, log=io.StringIO()) == 0.0 + + +@pytest.mark.parametrize("exit_code", [0, 1]) +@pytest.mark.parametrize("text", ["PASS\n", ""]) +def test_docker_verify_a_reward_file_with_no_number_is_an_error(tmp_path, text, exit_code): + import io + + from docker_cli_fake import DockerCLI, reply + from dokimi.harnesses.docker_solver import DockerSolverHarness + + out = tmp_path / "o" + out.mkdir() + task = load_task(_harbor(tmp_path)) + verifier = (lambda call: reply(returncode=exit_code) if "-lc" in call.argv else None) + harness = DockerSolverHarness(harness_id="docker-oracle", mode="oracle", + runner=DockerCLI(reward=text, answer=verifier)) + # A reward file with no reward is no verdict, whatever the exit code (as in a command harness). + with pytest.raises(RuntimeError, match="reward.txt has no reward"): + harness._verify(task, "c1", out, log=io.StringIO()) From e5f41504a3af242737a3ac4e9535e7409dd9f085 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sat, 14 Nov 2026 01:09:01 -0800 Subject: [PATCH 03/10] fix(harness): a failed or hung docker cp of reward.txt errors the cell A cp that failed for a daemon error was reported as 'wrote no reward.txt' (or scored 0.0 on a non-zero exit), and a cp timeout raised TimeoutExpired, which run() scores 0.0. Both lost the verdict: they now error the cell with their own message, the cp exit code in it. Only docker's own no-such-file stays no reward. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/harnesses/command.py | 17 +++++++++++ dokimi/dokimi/harnesses/docker_solver.py | 5 ++-- dokimi/scripts/growth_baseline.json | 2 +- dokimi/tests/test_cleanup_timeouts.py | 30 ++++--------------- dokimi/tests/test_docker_verify_reward.py | 36 +++++++++++++++++++++++ 5 files changed, 62 insertions(+), 28 deletions(-) diff --git a/dokimi/dokimi/harnesses/command.py b/dokimi/dokimi/harnesses/command.py index 2c5bd0ce5..0522de488 100644 --- a/dokimi/dokimi/harnesses/command.py +++ b/dokimi/dokimi/harnesses/command.py @@ -465,6 +465,23 @@ def _reward_file_verdict(text: str) -> float: return reward +# What docker says when the file is not there (v20+, older CLIs, podman); a daemon error says none. +_CP_NO_FILE_RE = re.compile(r"could not find the file|no such container:path|no such file", re.I) + + +def _check_reward_copy(pulled: process.Completed) -> None: + """Raise unless ``docker cp`` of ``reward.txt`` worked or found no file there. + + A copy that failed or hung says nothing about the verifier, so it is not "no reward file". + """ + if pulled.timed_out: + raise RuntimeError("docker cp of /logs/verifier/reward.txt timed out; see run.log") + if pulled.returncode != 0 and not _CP_NO_FILE_RE.search(pulled.stderr or ""): + raise RuntimeError( + f"docker cp of /logs/verifier/reward.txt failed (exit {pulled.returncode}); see run.log" + ) + + def _parse_reward_text(text: str, *, reward_file: bool = False) -> float | None: """A reward from a ``reward.txt`` (``reward_file``) or from command output. diff --git a/dokimi/dokimi/harnesses/docker_solver.py b/dokimi/dokimi/harnesses/docker_solver.py index c5960a99c..8d9fba8d0 100644 --- a/dokimi/dokimi/harnesses/docker_solver.py +++ b/dokimi/dokimi/harnesses/docker_solver.py @@ -71,7 +71,7 @@ solver_cli_flags, ) from .claude_json import _last_json_object, _parse_claude_json # noqa: F401 -from .command import _exit_code_verdict, _reward_file_verdict +from .command import _check_reward_copy, _exit_code_verdict, _reward_file_verdict from .container_agent import (AGENT_BIN, DOCKER_STEP_TIMEOUT, FILES_DIR, TASK_CONTAINER_HARDENING, ContainerSandbox, Docker) from .models import HarnessResult, OperationRecord @@ -645,8 +645,6 @@ def _verify(self, task: TaskSpec, container: str, output_dir: Path, log, ["cp", f"{container}:/logs/verifier/reward.txt", str(reward_file)], log, check=False, ) - if pulled.timed_out: # not "no reward file": the exit code would pass a failing verifier - raise subprocess.TimeoutExpired(pulled.argv, DOCKER_STEP_TIMEOUT) if pulled.returncode == 0 and reward_file.is_file(): return _reward_file_verdict(reward_file.read_text(encoding="utf-8", errors="replace")) if proc.returncode != 0: # no verdict at all if the container is gone, not a 0 @@ -655,6 +653,7 @@ def _verify(self, task: TaskSpec, container: str, output_dir: Path, log, raise RuntimeError(f"task container {container} state inspect timed out at verify time; see run.log") if (getattr(running, "stdout", "") or "").strip() != "true": raise RuntimeError(f"task container {container} is not running at verify time; see run.log") + _check_reward_copy(pulled) # a failed or hung copy errors the cell: it is not "no reward file" return _exit_code_verdict(proc.returncode) def _plan(self, task: TaskSpec, record: Optional[Path] = None) -> NetworkPlan: diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 92845ba17..93a50630c 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -2,7 +2,7 @@ "files": { "dokimi/commands/novelty.py": {"lines": 818, "rationale": "#403 --out check and the review controls (--context-framing, no_artifact_repeat) on top of the routing, --all-vs-all and estimate-only output from main; routing output in commands/novelty_routing_report.py; role clients, spend cap, USD estimate in novelty/cost.py; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, - "dokimi/harnesses/docker_solver.py": {"lines": 894, "rationale": "pre-existing when the gate was introduced"}, + "dokimi/harnesses/docker_solver.py": {"lines": 893, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1031, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, "dokimi/novelty/routing.py": {"lines": 822, "rationale": "grew with the #172 routing field, the keep-copies and dropped-query rules, the #405 over-limit warnings and the #403 one-repeat noise and measured reply tokens; file split tracked separately"}, "dokimi/report/model_compare.py": {"lines": 1161, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, diff --git a/dokimi/tests/test_cleanup_timeouts.py b/dokimi/tests/test_cleanup_timeouts.py index 1296eb8fd..3b47aee57 100644 --- a/dokimi/tests/test_cleanup_timeouts.py +++ b/dokimi/tests/test_cleanup_timeouts.py @@ -253,24 +253,17 @@ def test_credential_file_removal_timeout_aborts_cell(tmp_path): assert len(test_calls) == 0, "Verification should not run after credential removal timeout" -def test_reward_file_copy_timeout_fails_task(tmp_path): - """Docker cp of reward.txt timeout fails the task (reward 0.0, rc 124). - - If the copy of reward.txt from the container times out, the verifier's - decision is lost. The task must fail as reward 0.0 rather than proceeding - with an empty reward file or an incorrect assumption. - """ +@pytest.mark.parametrize("reward", [1.0, 0.0]) +def test_reward_file_copy_timeout_errors_the_cell(tmp_path, reward): + """A hung ``docker cp`` of reward.txt lost the verdict: the cell is errored, not passed or failed.""" task = load_task(_fixture_task(tmp_path)) - - # Mock docker to timeout on cp of reward.txt from container - cli = _pipeline(reward=1.0, answer=lambda call: reply(timed_out=True) if ( + cli = _pipeline(reward=reward, answer=lambda call: reply(timed_out=True) if ( call.argv[1] == "cp" and "reward.txt" in " ".join(call.argv)) else None) result = create_harness("docker-claude", runner=cli).run(task, output_dir=tmp_path / "out") - # The timeout should result in returncode 124 and reward 0.0 - assert result.returncode == 124, f"Should timeout when reward.txt copy times out; got {result.returncode}" - assert result.reward == 0.0, "Should get reward 0.0 when reward.txt copy times out" + assert (result.reward, result.passed) == (None, False) + assert "docker cp of /logs/verifier/reward.txt timed out" in result.metrics["error"] def test_plugin_removal_timeout_aborts_cell(tmp_path): @@ -286,17 +279,6 @@ def test_plugin_removal_timeout_aborts_cell(tmp_path): assert not [c for c in cli.calls if c.argv[1] == "exec" and "test.sh" in " ".join(c.argv)] -def test_reward_copy_timeout_does_not_fall_back_to_the_exit_code(tmp_path): - """A verifier that exits 0 must not pass because the reward.txt copy timed out.""" - task = load_task(_fixture_task(tmp_path)) - cli = _pipeline(reward=0.0, answer=lambda call: reply(timed_out=True) if ( - call.argv[1] == "cp" and "reward.txt" in " ".join(call.argv)) else None) - - result = create_harness("docker-claude", runner=cli).run(task, output_dir=tmp_path / "out") - - assert (result.returncode, result.reward, result.passed) == (124, 0.0, False) - - def test_no_egress_file_when_sidecar_log_unknown(tmp_path): """An unreadable sidecar log writes no egress.jsonl: an empty file reads as a clean run.""" def docker(args, log=None, *, check=True, timeout=None): diff --git a/dokimi/tests/test_docker_verify_reward.py b/dokimi/tests/test_docker_verify_reward.py index 71de2772a..d28e7174c 100644 --- a/dokimi/tests/test_docker_verify_reward.py +++ b/dokimi/tests/test_docker_verify_reward.py @@ -78,3 +78,39 @@ def test_docker_verify_a_reward_file_with_no_number_is_an_error(tmp_path, text, # A reward file with no reward is no verdict, whatever the exit code (as in a command harness). with pytest.raises(RuntimeError, match="reward.txt has no reward"): harness._verify(task, "c1", out, log=io.StringIO()) + + +_NO_FILE = "Error response from daemon: Could not find the file /logs/verifier/reward.txt in container c1" + + +@pytest.mark.parametrize(("stderr", "exit_code", "outcome"), [ + (_NO_FILE, 1, 0.0), # docker's own "no such file": still a verifier that wrote none + (_NO_FILE, 0, "wrote no /logs/verifier/reward.txt"), + ("Error response from daemon: connection refused", 0, r"docker cp .* failed \(exit 1\)"), + ("Error response from daemon: connection refused", 1, r"docker cp .* failed \(exit 1\)"), +]) +def test_docker_verify_a_failed_reward_copy_is_not_a_missing_reward_file( + tmp_path, stderr, exit_code, outcome): + import io + + from docker_cli_fake import DockerCLI, reply + from dokimi.harnesses.docker_solver import DockerSolverHarness + + out = tmp_path / "o" + out.mkdir() + task = load_task(_harbor(tmp_path)) + + def answer(call): + if "-lc" in call.argv: + return reply(returncode=exit_code) + if call.argv[1] == "cp" and call.argv[2].endswith(":/logs/verifier/reward.txt"): + return reply(stderr=stderr, returncode=1) + return None + + harness = DockerSolverHarness(harness_id="docker-oracle", mode="oracle", + runner=DockerCLI(reward="1.0", answer=answer)) + if isinstance(outcome, float): + assert harness._verify(task, "c1", out, log=io.StringIO()) == outcome + else: + with pytest.raises(RuntimeError, match=outcome): + harness._verify(task, "c1", out, log=io.StringIO()) From bfdf48b1621a8244d0f3c3c19f963fbaebdefadf Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sat, 14 Nov 2026 01:09:08 -0800 Subject: [PATCH 04/10] docs(harness): file the reward.txt rule under Changed and fix the fallback docs A docker-* verifier's exit code alone is no longer a pass, which moves scores for the same tasks: the fragment moves to Changed and says what a task author does. The task contract, the artifact docs and the self-test reference stop describing the exit-code fallback for docker-* cells. Signed-off-by: Alexander Rybkin --- .../changelog.d/20261113_120500_napetrov_fsm687-verifier.md | 4 ++-- .../dokimi-design-evals/references/verifier-self-test.md | 4 ++-- dokimi/docs/coding-harnesses.md | 5 ++++- dokimi/docs/contributing-terminal-bench-task.md | 6 +++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md b/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md index 458b9ea98..a284e8e3f 100644 --- a/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md +++ b/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md @@ -3,6 +3,6 @@ Copyright (C) 2026 Intel Corporation SPDX-License-Identifier: Apache-2.0 --> -### Fixed +### Changed -- **A verifier that exits 0 without writing `reward.txt` is an error, not a pass** — a Docker task whose verifier wrote no `/logs/verifier/reward.txt` used to score 1.0 on exit code 0. The cell is now errored, with no reward, and `run.log` names the missing file. A verifier that exits non-zero without a reward file still scores 0.0. +- **A `docker-*` verifier must write `reward.txt`: exit 0 alone is no longer a pass** — a Docker task whose verifier wrote no `/logs/verifier/reward.txt` used to score 1.0 on exit code 0. The cell is now errored, with no reward, and `run.log` names the missing file. A verifier that exits non-zero without a reward file still scores 0.0. A `reward.txt` with no number in it (`PASS`, empty) now errors the cell too, instead of scoring a fail, and so does a `docker cp` of it that fails or times out (the timeout used to score 0.0). Make such a verifier write `1` or `0` to `/logs/verifier/reward.txt`; pass rates of tasks whose verifier only set its exit code are not comparable with earlier runs. diff --git a/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md index a1290a9e9..19e60ee9b 100644 --- a/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md +++ b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md @@ -130,8 +130,8 @@ reference state and on the state before it, and expect pass then fail. To run a `tests/test.sh` by hand, put that step's `tests/` at `/tests` and at `/verifier` in the task image and run `mkdir -p /logs/verifier && chmod +x /tests/test.sh && /tests/test.sh`, as the harness does. -Read `/logs/verifier/reward.txt`, as the harness does; the exit code counts only if the -file is missing. +Read `/logs/verifier/reward.txt`, as the harness does; if the file is missing, exit 0 +errors the cell and a non-zero exit is a fail. The task needs a top-level `instruction.md`; without it the task does not load. ## What a passing self-test does not show diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index 9540e2d27..4e0e5fa25 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -1100,7 +1100,10 @@ python cli.py tasks run --tasks \ partial credit such as `0.75`, is kept as the reward and counts as a fail. If no reward is found, the adapter's exit code is used as the fallback, except for a Harbor-backed cell, which is then errored (see the Harbor grading - table above). + table above), and a `docker-*` cell, where exit 0 without + `/logs/verifier/reward.txt` is errored and a non-zero exit scores 0.0. A + `docker-*` reward file with no number in it, or a `docker cp` of it that + fails or times out, errors the cell. - `results[].metrics`: for LLM harnesses (`docker-claude*`), the per-run `UsageRecord` fields — `cost_usd`, `prompt_tokens`, `completion_tokens`, `total_tokens`, `cache_read_tokens`, `cache_write_tokens`, `latency_sec`, diff --git a/dokimi/docs/contributing-terminal-bench-task.md b/dokimi/docs/contributing-terminal-bench-task.md index d5b517736..9067f5f7f 100644 --- a/dokimi/docs/contributing-terminal-bench-task.md +++ b/dokimi/docs/contributing-terminal-bench-task.md @@ -152,9 +152,9 @@ The contract: - Write the outcome to `/logs/verifier/reward.txt` — `1` (or `1.0`) for pass, `0` (or `0.0`) for fail. A fraction such as `0.75` (partial credit) is kept - as the cell's reward and counts as a fail. The harness copies that file out; - if it is missing it falls back to the verifier's exit code, so also exit - non-zero on failure. + as the cell's reward and counts as a fail. The harness copies that file out. + Without it the exit code is no pass: exit 0 errors the cell and a non-zero + exit scores 0. A file with no number in it (`PASS`, empty) errors the cell. - Do not let a failure abort before the reward file is written (capture the pytest exit code, write the file, then exit with it). From 62da1329c373ef4a951398b59252858c9688b866 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 11:20:48 -0800 Subject: [PATCH 05/10] fix(harness): a missing Docker socket is a failed docker cp, not a missing reward The third alternative of the docker cp no-file pattern was a bare "no such file", which matches a missing Docker socket. A failed cp was then read as "verifier wrote no reward.txt", scoring a verifier that exited 1 as 0.0 and blaming a verifier that exited 0. Keep Docker's own no-file wordings only. Thread: PRRT_kwDOVBg7g863cHuu Thread: PRRT_kwDOVBg7g863finG Signed-off-by: Nikolay Petrov --- dokimi/dokimi/harnesses/command.py | 2 +- dokimi/tests/test_docker_verify_reward.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/harnesses/command.py b/dokimi/dokimi/harnesses/command.py index 0522de488..6b9ca266d 100644 --- a/dokimi/dokimi/harnesses/command.py +++ b/dokimi/dokimi/harnesses/command.py @@ -466,7 +466,7 @@ def _reward_file_verdict(text: str) -> float: # What docker says when the file is not there (v20+, older CLIs, podman); a daemon error says none. -_CP_NO_FILE_RE = re.compile(r"could not find the file|no such container:path|no such file", re.I) +_CP_NO_FILE_RE = re.compile(r"could not find the file|no such container:path", re.I) def _check_reward_copy(pulled: process.Completed) -> None: diff --git a/dokimi/tests/test_docker_verify_reward.py b/dokimi/tests/test_docker_verify_reward.py index d28e7174c..9fd362953 100644 --- a/dokimi/tests/test_docker_verify_reward.py +++ b/dokimi/tests/test_docker_verify_reward.py @@ -84,10 +84,15 @@ def test_docker_verify_a_reward_file_with_no_number_is_an_error(tmp_path, text, @pytest.mark.parametrize(("stderr", "exit_code", "outcome"), [ - (_NO_FILE, 1, 0.0), # docker's own "no such file": still a verifier that wrote none + (_NO_FILE, 1, 0.0), # docker's own "could not find the file": still a verifier that wrote none (_NO_FILE, 0, "wrote no /logs/verifier/reward.txt"), ("Error response from daemon: connection refused", 0, r"docker cp .* failed \(exit 1\)"), ("Error response from daemon: connection refused", 1, r"docker cp .* failed \(exit 1\)"), + # A missing Docker socket says "no such file or directory" but names no reward file. + ("dial unix /var/run/docker.sock: connect: no such file or directory", 1, + r"docker cp .* failed \(exit 1\)"), + ("dial unix /var/run/docker.sock: connect: no such file or directory", 0, + r"docker cp .* failed \(exit 1\)"), ]) def test_docker_verify_a_failed_reward_copy_is_not_a_missing_reward_file( tmp_path, stderr, exit_code, outcome): From 3c143e668d8bec4bb0d6cf2839bc3d16567e22ba Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 11:21:14 -0800 Subject: [PATCH 06/10] Address review: four threads already fixed on this branch Thread: PRRT_kwDOVBg7g863EIdg: already fixed in 81c416487 Thread: PRRT_kwDOVBg7g863EIdn: already fixed in bfdf48b16 Thread: PRRT_kwDOVBg7g863EIdt: already fixed in bfdf48b16 Thread: PRRT_kwDOVBg7g863EId0: already fixed in e5f41504a Signed-off-by: Nikolay Petrov From 23c11d2e2c121556f2bd8bae2a8ec55885005018 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 11:24:19 -0800 Subject: [PATCH 07/10] fix(harness): podman's missing-file wording is still no reward file The regex lost its bare "no such file" alternative in the previous commit, which the comment above it says covered podman. Match podman's own "could not be found on container" wording instead, so a missing file still scores 0.0 or the no-reward error, and a socket error stays a failed cp. The wording is from memory: podman is not installed here. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/harnesses/command.py | 3 ++- dokimi/tests/test_docker_verify_reward.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/harnesses/command.py b/dokimi/dokimi/harnesses/command.py index 6b9ca266d..53b2696cf 100644 --- a/dokimi/dokimi/harnesses/command.py +++ b/dokimi/dokimi/harnesses/command.py @@ -466,7 +466,8 @@ def _reward_file_verdict(text: str) -> float: # What docker says when the file is not there (v20+, older CLIs, podman); a daemon error says none. -_CP_NO_FILE_RE = re.compile(r"could not find the file|no such container:path", re.I) +_CP_NO_FILE_RE = re.compile( + r"could not find the file|no such container:path|could not be found on container", re.I) def _check_reward_copy(pulled: process.Completed) -> None: diff --git a/dokimi/tests/test_docker_verify_reward.py b/dokimi/tests/test_docker_verify_reward.py index 9fd362953..bfc7fa5b1 100644 --- a/dokimi/tests/test_docker_verify_reward.py +++ b/dokimi/tests/test_docker_verify_reward.py @@ -88,6 +88,11 @@ def test_docker_verify_a_reward_file_with_no_number_is_an_error(tmp_path, text, (_NO_FILE, 0, "wrote no /logs/verifier/reward.txt"), ("Error response from daemon: connection refused", 0, r"docker cp .* failed \(exit 1\)"), ("Error response from daemon: connection refused", 1, r"docker cp .* failed \(exit 1\)"), + # podman's wording for a missing file (from memory; no podman on this box): still no reward file. + ('"/logs/verifier/reward.txt" could not be found on container c1: no such file or directory', 1, + 0.0), + ('"/logs/verifier/reward.txt" could not be found on container c1: no such file or directory', 0, + "wrote no /logs/verifier/reward.txt"), # A missing Docker socket says "no such file or directory" but names no reward file. ("dial unix /var/run/docker.sock: connect: no such file or directory", 1, r"docker cp .* failed \(exit 1\)"), From 87f8d38f616a28d2c2facb7465f296f36e97297a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 11:25:50 -0800 Subject: [PATCH 08/10] Address review: review-level claims already fixed on this branch Thread: PRR_kwDOVBg7g88AAAABXxKgQw: already fixed in 3c143e668 Thread: PRR_kwDOVBg7g88AAAABX67UcQ: already fixed in 3c143e668 Thread: PRR_kwDOVBg7g88AAAABX_tPZg: already fixed in 62da1329c Thread: IC_kwDOVBg7g88AAAABmaYJNA: already fixed in 3c143e668 Thread: IC_kwDOVBg7g88AAAABmcuxrQ: already fixed in 3c143e668 Signed-off-by: Nikolay Petrov From 74e61c630577ba617d962eddc032451497eed51c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 12:01:31 -0800 Subject: [PATCH 09/10] docs(harness): the reward-file rule covers reward-file verifiers, not terminal-bench v1 A terminal-bench v1 task (task.yaml) is graded by its pytest summary before the reward file is read, so exit 0 without reward.txt is not an error there. The coding-harnesses doc, the task contract and the changelog said it for every docker-* verifier. Thread: PRRT_kwDOVBg7g863iPHE Signed-off-by: Nikolay Petrov --- .../20261113_120500_napetrov_fsm687-verifier.md | 2 +- dokimi/docs/coding-harnesses.md | 9 +++++---- dokimi/docs/contributing-terminal-bench-task.md | 4 +++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md b/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md index a284e8e3f..2d7c75ddc 100644 --- a/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md +++ b/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A `docker-*` verifier must write `reward.txt`: exit 0 alone is no longer a pass** — a Docker task whose verifier wrote no `/logs/verifier/reward.txt` used to score 1.0 on exit code 0. The cell is now errored, with no reward, and `run.log` names the missing file. A verifier that exits non-zero without a reward file still scores 0.0. A `reward.txt` with no number in it (`PASS`, empty) now errors the cell too, instead of scoring a fail, and so does a `docker cp` of it that fails or times out (the timeout used to score 0.0). Make such a verifier write `1` or `0` to `/logs/verifier/reward.txt`; pass rates of tasks whose verifier only set its exit code are not comparable with earlier runs. +- **A `docker-*` reward-file verifier must write `reward.txt`: exit 0 alone is no longer a pass** — a Docker task (`task.toml` or `task.md`) whose verifier wrote no `/logs/verifier/reward.txt` used to score 1.0 on exit code 0. The cell is now errored, with no reward, and `run.log` names the missing file. A verifier that exits non-zero without a reward file still scores 0.0. A terminal-bench v1 task keeps its pytest summary as the verdict. A `reward.txt` with no number in it (`PASS`, empty) now errors the cell too, instead of scoring a fail, and so does a `docker cp` of it that fails or times out (the timeout used to score 0.0). Make such a verifier write `1` or `0` to `/logs/verifier/reward.txt`; pass rates of tasks whose verifier only set its exit code are not comparable with earlier runs. diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index f111bb472..ba6dea5bb 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -1100,10 +1100,11 @@ python cli.py tasks run --tasks \ partial credit such as `0.75`, is kept as the reward and counts as a fail. If no reward is found, the adapter's exit code is used as the fallback, except for a Harbor-backed cell, which is then errored (see the Harbor grading - table above), and a `docker-*` cell, where exit 0 without - `/logs/verifier/reward.txt` is errored and a non-zero exit scores 0.0. A - `docker-*` reward file with no number in it, or a `docker cp` of it that - fails or times out, errors the cell. + table above), and a `docker-*` cell with a reward-file verifier (`task.toml` + or `task.md`), where exit 0 without `/logs/verifier/reward.txt` is errored + and a non-zero exit scores 0.0. A terminal-bench v1 verifier keeps its pytest + summary as the verdict. A `docker-*` reward file with no number in it, or a + `docker cp` of it that fails or times out, errors the cell. - `results[].metrics`: for LLM harnesses (`docker-claude*`), the per-run `UsageRecord` fields — `cost_usd`, `prompt_tokens`, `completion_tokens`, `total_tokens`, `cache_read_tokens`, `cache_write_tokens`, `latency_sec`, diff --git a/dokimi/docs/contributing-terminal-bench-task.md b/dokimi/docs/contributing-terminal-bench-task.md index 9067f5f7f..7db69eae4 100644 --- a/dokimi/docs/contributing-terminal-bench-task.md +++ b/dokimi/docs/contributing-terminal-bench-task.md @@ -154,7 +154,9 @@ The contract: `0` (or `0.0`) for fail. A fraction such as `0.75` (partial credit) is kept as the cell's reward and counts as a fail. The harness copies that file out. Without it the exit code is no pass: exit 0 errors the cell and a non-zero - exit scores 0. A file with no number in it (`PASS`, empty) errors the cell. + exit scores 0. This is the rule for this format; a terminal-bench v1 task + (`task.yaml`) is still graded by its pytest summary. A file with no number + in it (`PASS`, empty) errors the cell. - Do not let a failure abort before the reward file is written (capture the pytest exit code, write the file, then exit with it). From 97f70ef9bdaabea1a4e5a0497765d231ebac2393 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sat, 14 Nov 2026 21:32:21 +0000 Subject: [PATCH 10/10] Word the reward.txt error for the 0 to 1 range Signed-off-by: Anton Volkov --- .../changelog.d/20261113_120500_napetrov_fsm687-verifier.md | 2 +- dokimi/docs/coding-harnesses.md | 5 +++-- dokimi/docs/contributing-terminal-bench-task.md | 2 +- dokimi/dokimi/harnesses/command.py | 4 +++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md b/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md index 2d7c75ddc..b476f0864 100644 --- a/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md +++ b/dokimi/changelog.d/20261113_120500_napetrov_fsm687-verifier.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A `docker-*` reward-file verifier must write `reward.txt`: exit 0 alone is no longer a pass** — a Docker task (`task.toml` or `task.md`) whose verifier wrote no `/logs/verifier/reward.txt` used to score 1.0 on exit code 0. The cell is now errored, with no reward, and `run.log` names the missing file. A verifier that exits non-zero without a reward file still scores 0.0. A terminal-bench v1 task keeps its pytest summary as the verdict. A `reward.txt` with no number in it (`PASS`, empty) now errors the cell too, instead of scoring a fail, and so does a `docker cp` of it that fails or times out (the timeout used to score 0.0). Make such a verifier write `1` or `0` to `/logs/verifier/reward.txt`; pass rates of tasks whose verifier only set its exit code are not comparable with earlier runs. +- **A `docker-*` reward-file verifier must write `reward.txt`: exit 0 alone is no longer a pass** — a Docker task (`task.toml` or `task.md`) whose verifier wrote no `/logs/verifier/reward.txt` used to score 1.0 on exit code 0. The cell is now errored, with no reward, and `run.log` names the missing file. A verifier that exits non-zero without a reward file still scores 0.0. A terminal-bench v1 task keeps its pytest summary as the verdict. A `reward.txt` with no number between 0 and 1 in it (`PASS`, empty, `75`) now errors the cell too, instead of scoring a fail, and so does a `docker cp` of it that fails or times out (the timeout used to score 0.0). Make such a verifier write `1` or `0` to `/logs/verifier/reward.txt`; pass rates of tasks whose verifier only set its exit code are not comparable with earlier runs. diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index ba6dea5bb..fcff41d0d 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -1103,8 +1103,9 @@ python cli.py tasks run --tasks \ table above), and a `docker-*` cell with a reward-file verifier (`task.toml` or `task.md`), where exit 0 without `/logs/verifier/reward.txt` is errored and a non-zero exit scores 0.0. A terminal-bench v1 verifier keeps its pytest - summary as the verdict. A `docker-*` reward file with no number in it, or a - `docker cp` of it that fails or times out, errors the cell. + summary as the verdict. A `docker-*` reward file with no number between 0 + and 1 in it (`PASS`, empty, `75`), or a `docker cp` of it that fails or + times out, errors the cell. - `results[].metrics`: for LLM harnesses (`docker-claude*`), the per-run `UsageRecord` fields — `cost_usd`, `prompt_tokens`, `completion_tokens`, `total_tokens`, `cache_read_tokens`, `cache_write_tokens`, `latency_sec`, diff --git a/dokimi/docs/contributing-terminal-bench-task.md b/dokimi/docs/contributing-terminal-bench-task.md index 7db69eae4..46105f72f 100644 --- a/dokimi/docs/contributing-terminal-bench-task.md +++ b/dokimi/docs/contributing-terminal-bench-task.md @@ -156,7 +156,7 @@ The contract: Without it the exit code is no pass: exit 0 errors the cell and a non-zero exit scores 0. This is the rule for this format; a terminal-bench v1 task (`task.yaml`) is still graded by its pytest summary. A file with no number - in it (`PASS`, empty) errors the cell. + between 0 and 1 in it (`PASS`, empty, `75`) errors the cell. - Do not let a failure abort before the reward file is written (capture the pytest exit code, write the file, then exit with it). diff --git a/dokimi/dokimi/harnesses/command.py b/dokimi/dokimi/harnesses/command.py index 53b2696cf..ca7fbc562 100644 --- a/dokimi/dokimi/harnesses/command.py +++ b/dokimi/dokimi/harnesses/command.py @@ -461,7 +461,9 @@ def _reward_file_verdict(text: str) -> float: """ reward = _parse_reward_text(text, reward_file=True) if reward is None: - raise RuntimeError("/logs/verifier/reward.txt has no reward in it; see run.log") + raise RuntimeError( + "/logs/verifier/reward.txt has no reward between 0 and 1 in it; see run.log" + ) return reward