From 28c898ebb47f396cd7f3f5e01dec6338d3289992 Mon Sep 17 00:00:00 2001 From: song Date: Fri, 16 Oct 2026 07:54:49 +0800 Subject: [PATCH 1/5] benchmark(runtime): opt-in owner-locked acceptance gate for planned entry Four LHTB trials derived terminal no-follow-up at 43-60 min of a 90 min budget because every Todo was marked done while the deliverable still failed the verifier's own interface (CLI non-zero on the hidden wrapper manifest, model unable to predict in the sandbox). The closure rule itself is correct: it only sees Todo completeness. Add `acceptance_contract: planned` (default `off`) to the shared benchmark runtime. During the planning checkpoint the model also writes a Goal acceptance document whose criteria exercise the deliverable through the task's declared verifier interface on public inputs. The trial host, as the trusted local owner inside the container, configures it through the existing `goal-acceptance` contract scoped to the Todos planning returned, so a selected Todo cannot complete until its criteria pass. On each wake the host re-confirms associations that the model's own Todo edits made stale without touching the criteria. A missing or invalid document fails the planning phase. `off` keeps byte-identical prompts and behavior. No scoring, hidden-verifier feedback or new control-plane vocabulary is introduced; `ACCEPTANCE_CONTRACTS` is a runner-local closed set. Co-Authored-By: Claude Fable 5.1 Signed-off-by: song --- benchmark/LHTB/run.sh | 6 +- benchmark/LHTB/scripts/render_config.py | 6 +- benchmark/runtime/acceptance.py | 173 +++++++++++++ benchmark/runtime/codex.py | 8 + benchmark/runtime/harbor.py | 14 + benchmark/runtime/planning.py | 7 + benchmark/runtime/worker.py | 13 + .../test_benchmark_planned_acceptance.py | 242 ++++++++++++++++++ 8 files changed, 467 insertions(+), 2 deletions(-) create mode 100644 benchmark/runtime/acceptance.py create mode 100644 tests/control_plane/test_benchmark_planned_acceptance.py diff --git a/benchmark/LHTB/run.sh b/benchmark/LHTB/run.sh index e816f65739..3811ddf11f 100755 --- a/benchmark/LHTB/run.sh +++ b/benchmark/LHTB/run.sh @@ -51,6 +51,7 @@ LOOPX_TASK_ENTRY="${LOOPX_TASK_ENTRY:-}" [[ -z "${LOOPX_PLANNING_TIMEOUT_SEC:-}" ]] || die "Remove LOOPX_PLANNING_TIMEOUT_SEC; planning shares LOOPX_SCHEDULER_TIMEOUT_SEC with execution" LOOPX_ITERATION_CONTEXT="${LOOPX_ITERATION_CONTEXT:-fresh}" LOOPX_VALIDATION_COMMAND_JSON="${LOOPX_VALIDATION_COMMAND_JSON:-[]}" +LOOPX_ACCEPTANCE_CONTRACT="${LOOPX_ACCEPTANCE_CONTRACT:-off}" SHARED_CODEX_AGENT_DIR="$LOOPX_SRC_DIR/benchmark/runtime" [[ -f "$SHARED_CODEX_AGENT_DIR/codex_offline.py" ]] || \ die "Shared offline Codex adapter not found: $SHARED_CODEX_AGENT_DIR/codex_offline.py" @@ -117,7 +118,9 @@ if [[ "$MODE" == smoke ]]; then expected_task_count=1 job_suffix="smoke-${SMOKE_TASK}" fi -job_name="lhtb-${LOOPX_EXECUTION_MODE}-${LOOPX_TASK_ENTRY:-default}-${LOOPX_ITERATION_CONTEXT}-${job_suffix}-${run_stamp}" +acceptance_suffix="" +[[ "$LOOPX_ACCEPTANCE_CONTRACT" == off ]] || acceptance_suffix="-acceptance-${LOOPX_ACCEPTANCE_CONTRACT}" +job_name="lhtb-${LOOPX_EXECUTION_MODE}-${LOOPX_TASK_ENTRY:-default}${acceptance_suffix}-${LOOPX_ITERATION_CONTEXT}-${job_suffix}-${run_stamp}" generated_config="$CODE_DIR/.generated/${job_name}.yaml" jobs_dir="$CODE_DIR/runs" @@ -142,6 +145,7 @@ fi "${task_entry_args[@]}" \ --iteration-context "$LOOPX_ITERATION_CONTEXT" \ --validation-command-json "$LOOPX_VALIDATION_COMMAND_JSON" \ + --acceptance-contract "$LOOPX_ACCEPTANCE_CONTRACT" \ "${turn_timeout_args[@]}" \ --scheduler-timeout "$LOOPX_SCHEDULER_TIMEOUT_SEC" \ "${task_args[@]}" diff --git a/benchmark/LHTB/scripts/render_config.py b/benchmark/LHTB/scripts/render_config.py index f154632e5c..29d9f9332b 100755 --- a/benchmark/LHTB/scripts/render_config.py +++ b/benchmark/LHTB/scripts/render_config.py @@ -9,7 +9,7 @@ from pathlib import Path import yaml -from benchmark.runtime.codex import CONTEXTS, MODES, TASK_ENTRIES, Execution +from benchmark.runtime.codex import ACCEPTANCE_CONTRACTS, CONTEXTS, MODES, TASK_ENTRIES, Execution def main() -> int: @@ -28,6 +28,8 @@ def main() -> int: parser.add_argument("--task-entry", choices=TASK_ENTRIES, help="LoopX modes default to loopx-planned; explicit seeded-todo retains legacy entry") parser.add_argument("--validation-command-json", default="[]") + parser.add_argument("--acceptance-contract", choices=ACCEPTANCE_CONTRACTS, default="off", + help="planned: planning also writes an owner-confirmed Goal acceptance document") parser.add_argument("--turn-timeout", type=float, default=None) parser.add_argument("--scheduler-timeout", type=int, default=5080) args = parser.parse_args() @@ -62,6 +64,7 @@ def main() -> int: timeout_seconds=args.turn_timeout, validation_command=json.loads(args.validation_command_json), task_entry=args.task_entry, + acceptance_contract=args.acceptance_contract, ) agent["kwargs"].update( execution_mode=execution.mode, @@ -70,6 +73,7 @@ def main() -> int: turn_timeout_sec=execution.timeout_seconds, scheduler_timeout_sec=args.scheduler_timeout, task_entry=execution.task_entry, + acceptance_contract=execution.acceptance_contract, ) if agent["kwargs"].get("planning_timeout_sec") is not None: parser.error("remove planning_timeout_sec from the template; planning shares scheduler-timeout") diff --git a/benchmark/runtime/acceptance.py b/benchmark/runtime/acceptance.py new file mode 100644 index 0000000000..5a11b60e6c --- /dev/null +++ b/benchmark/runtime/acceptance.py @@ -0,0 +1,173 @@ +"""Owner-confirmed Goal acceptance for the planned benchmark entry. + +The planning checkpoint asks the model to write a Goal acceptance document; the +benchmark host, acting as the trusted local owner inside the trial, configures +it through the existing ``goal-acceptance`` contract. The product owner then +holds the selected Todos open until their criteria pass, so "all Todos done" +can no longer derive terminal no-follow-up before the deliverable was exercised. +No scoring, hidden-verifier feedback or new state vocabulary is introduced. +""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from pathlib import Path +from typing import Any + +ACCEPTANCE_DOCUMENT_ENV = "LOOPX_ACCEPTANCE_DOCUMENT" + + +def acceptance_planning_guidance(document_path: str) -> str: + """Host instructions appended to the planning prompt for the planned contract.""" + return ( + "Host acceptance checkpoint:\n" + f"Before returning, write a Goal acceptance document to {document_path}. " + "It is a JSON object with objective, non_goals (list), scope " + '{"kind": "selected_work", "todo_ids": [...]} naming the Todo(s) whose ' + "completion delivers the task result, criteria (list of {id, description, " + "validation_argv, validation_timeout_seconds, optional validation_files}) and " + "bindings (one {todo_id, criterion_ids} per selected Todo). Use the exact " + "Todo ids you created. Each criterion is an argv command run from the task " + "workspace without a shell, 1-25 seconds each and 25 seconds in total. " + "Criteria must exercise the deliverable through the interface the task says " + "its verifier uses (for example run the documented command on the public " + "inputs into a fresh output directory, then require exit 0 and the required " + "artifacts), must fail until the deliverable exists, and must not read " + "held-out or hidden data or fixture answers. For a check longer than its " + "timeout, run it during the work and let the criterion verify its receipt " + "artifact. The host configures this document as the owner-confirmed " + "acceptance contract: a selected Todo cannot complete until its criteria " + "pass, and the Goal cannot close while it stays open. A missing or invalid " + "document fails this planning phase.\n\n" + ) + + +def _run_cli(env: dict[str, str], cli: list[str], args: list[str]) -> dict[str, Any]: + response = subprocess.run( + cli + args, cwd=env["LOOPX_PROJECT"], env=env, text=True, + capture_output=True, check=False, timeout=120, + ) + try: + payload = json.loads(response.stdout or "") + except json.JSONDecodeError as exc: + raise ValueError( + f"goal-acceptance {args[1]} returned invalid JSON (exit {response.returncode})" + ) from exc + if response.returncode != 0 or payload.get("ok") is False: + raise ValueError(f"goal-acceptance {args[1]} failed: {payload.get('error')}") + return payload + + +def _load_document(env: dict[str, str]) -> tuple[Path, dict[str, Any]]: + path = Path(env[ACCEPTANCE_DOCUMENT_ENV]) + try: + document = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise ValueError("planning did not write the Goal acceptance document") from exc + except (OSError, ValueError) as exc: + raise ValueError("Goal acceptance document is unreadable or not JSON") from exc + if not isinstance(document, dict): + raise ValueError("Goal acceptance document must be a JSON object") + return path, document + + +def _selected_and_bound(document: dict[str, Any]) -> tuple[list[str], list[dict[str, Any]]]: + scope = document.get("scope") + selected = scope.get("todo_ids") if isinstance(scope, dict) else None + if ( + not isinstance(scope, dict) or scope.get("kind") != "selected_work" + or not isinstance(selected, list) or not selected + or any(not isinstance(item, str) or not item for item in selected) + or len(set(selected)) != len(selected) + ): + raise ValueError("Goal acceptance scope must select the delivering Todo ids") + bindings = document.get("bindings") + if not isinstance(bindings, list) or not bindings or any( + not isinstance(row, dict) or row.get("todo_id") not in selected for row in bindings + ): + raise ValueError("Goal acceptance bindings must cover only selected Todos") + if set(selected) - {row["todo_id"] for row in bindings}: + raise ValueError("every selected Todo needs an acceptance binding") + criteria = document.get("criteria") + if not isinstance(criteria, list) or not criteria or any( + not isinstance(row, dict) or not isinstance(row.get("id"), str) for row in criteria + ): + raise ValueError("Goal acceptance requires identified criteria") + return list(selected), bindings + + +def _configure(env, cli, document_path: Path, *, operation_id: str, + expected_todo_ids: list[str]) -> dict[str, Any]: + goal = ["--goal-id", env["LOOPX_GOAL_ID"]] + basis = _run_cli(env, cli, ["goal-acceptance", "inspect", *goal]) + applied = _run_cli(env, cli, [ + "goal-acceptance", "configure", *goal, "--document", str(document_path), + "--expected-provider-revision", str(basis["provider_revision"]), + "--operation-id", operation_id, "--execute", + ]) + contract = applied.get("goal_acceptance_contract") or {} + if applied.get("status") != "applied" or contract.get("enabled") is not True: + raise ValueError("Goal acceptance configuration was not applied") + states = {task.get("todo_id"): task.get("state") for task in contract.get("tasks", []) + if isinstance(task, dict)} + not_ready = [todo_id for todo_id in expected_todo_ids if states.get(todo_id) != "ready"] + if not_ready: + raise ValueError(f"acceptance association is not ready for {', '.join(not_ready)}") + return { + "status": "applied", + "revision": contract["revision"], + "digest": contract["digest"], + "operation_id": operation_id, + "selected_todo_ids": expected_todo_ids, + "criterion_ids": [row["id"] for row in contract.get("criteria", [])], + } + + +def apply_planned_acceptance(env: dict[str, str], planning: dict[str, Any], + cli: list[str]) -> dict[str, Any]: + """Configure the model-written document for the Todos planning returned.""" + document_path, document = _load_document(env) + selected, _ = _selected_and_bound(document) + unknown = sorted(set(selected) - set(planning["todo_ids"])) + if unknown: + raise ValueError(f"acceptance selects Todos outside the planning result: {', '.join(unknown)}") + digest = hashlib.sha256( + document_path.read_bytes() + planning["input_digest"].encode("utf-8") + ).hexdigest() + return _configure(env, cli, document_path, + operation_id=f"planned-acceptance:{digest[:32]}", + expected_todo_ids=selected) + + +def reconfirm_planned_acceptance(env: dict[str, str], cli: list[str]) -> dict[str, Any]: + """Re-confirm associations the model's own Todo edits made stale. + + The criteria never change here: owner confirmation of the current Todo text + is what the product requires after a selected Todo was edited. Completed or + cancelled Todos leave the selection; their checks already ran at completion. + """ + document_path, document = _load_document(env) + selected, bindings = _selected_and_bound(document) + goal = ["--goal-id", env["LOOPX_GOAL_ID"]] + contract = _run_cli(env, cli, ["goal-acceptance", "inspect", *goal]).get( + "goal_acceptance_contract") or {} + if contract.get("enabled") is not True: + raise ValueError("planned acceptance contract is no longer enabled") + tasks = {task["todo_id"]: task for task in contract.get("tasks", []) + if isinstance(task, dict) and isinstance(task.get("todo_id"), str)} + applicable = [todo_id for todo_id in selected + if tasks.get(todo_id, {}).get("applicable") is True] + stale = [todo_id for todo_id in applicable if tasks[todo_id].get("state") == "stale"] + if not stale: + return {"reconfirmed": [], "revision": contract["revision"]} + revised = {**document, + "scope": {"kind": "selected_work", "todo_ids": applicable}, + "bindings": [row for row in bindings if row["todo_id"] in applicable]} + document_path.write_text(json.dumps(revised, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8") + receipt = _configure(env, cli, document_path, + operation_id=f"planned-acceptance-reconfirm:{contract['revision']}", + expected_todo_ids=applicable) + return {"reconfirmed": stale, "revision": receipt["revision"]} diff --git a/benchmark/runtime/codex.py b/benchmark/runtime/codex.py index 1c7dff6296..1f9705da0f 100644 --- a/benchmark/runtime/codex.py +++ b/benchmark/runtime/codex.py @@ -16,6 +16,9 @@ MODES = ("plain", "native-goal", "heartbeat", "turn", "loopx-goal") CONTEXTS = ("fresh", "resume") TASK_ENTRIES = ("seeded-todo", "loopx-planned") +# "planned": the planning checkpoint also writes an owner-confirmed Goal +# acceptance document; "off" keeps the existing closure semantics. +ACCEPTANCE_CONTRACTS = ("off", "planned") SANDBOXES = ("read-only", "workspace-write", "danger-full-access") @@ -28,6 +31,7 @@ class Execution: validation_command: tuple[str, ...] = () task_entry: str | None = None turn_envelope: bool = False + acceptance_contract: str = "off" def __post_init__(self) -> None: if not isinstance(self.turn_envelope, bool) or (self.turn_envelope and self.mode != "heartbeat"): @@ -40,6 +44,10 @@ def __post_init__(self) -> None: raise ValueError("unsupported task entry") if self.task_entry == "loopx-planned" and not self.uses_loopx: raise ValueError("loopx-planned requires a LoopX execution mode") + if self.acceptance_contract not in ACCEPTANCE_CONTRACTS: + raise ValueError("unsupported acceptance contract") + if self.acceptance_contract == "planned" and self.task_entry != "loopx-planned": + raise ValueError("acceptance_contract=planned requires loopx-planned task entry") if self.context != "fresh" and self.mode not in {"turn", "heartbeat"}: raise ValueError("resume requires mode=turn or mode=heartbeat") if self.sandbox not in SANDBOXES: diff --git a/benchmark/runtime/harbor.py b/benchmark/runtime/harbor.py index bd1bc60ed7..860e186d44 100644 --- a/benchmark/runtime/harbor.py +++ b/benchmark/runtime/harbor.py @@ -60,6 +60,7 @@ def __init__( task_entry=None, planning_timeout_sec=None, turn_envelope=False, + acceptance_contract="off", task_objective=None, goal_id=_GOAL_ID, agent_id=_AGENT_ID, @@ -76,6 +77,7 @@ def __init__( validation_command if validation_command is not None else (), task_entry, turn_envelope, + acceptance_contract=acceptance_contract, ) if planning_timeout_sec is not None: raise ValueError( @@ -334,6 +336,10 @@ async def _write_task_document( finally: Path(name).unlink(missing_ok=True) + @property + def _acceptance_document(self) -> str: + return f"{_CONTROL}/acceptance-phase-{self._phase_number:03d}.json" + @property def _task_document(self) -> str: # Existing Todos keep their original input when Harbor supplies a new phase. @@ -543,6 +549,9 @@ def _worker_env(self, *, cwd: str) -> dict[str, str]: **({"LOOPX_CHECKPOINT_REQUEST": self.checkpoint_request_path} if self.checkpoint_request_path is not None else {}), **({"LOOPX_TURN_ENVELOPE": "1"} if self.execution.turn_envelope else {}), + "LOOPX_ACCEPTANCE_CONTRACT": self.execution.acceptance_contract, + **({"LOOPX_ACCEPTANCE_DOCUMENT": self._acceptance_document} + if self.execution.acceptance_contract == "planned" else {}), "LOOPX_ITERATION_CONTEXT": self.execution.context, "LOOPX_CODEX_SANDBOX": self.execution.sandbox, "LOOPX_VALIDATION_COMMAND_JSON": json.dumps( @@ -655,6 +664,8 @@ def _populate_context( "iteration_context": self.execution.context, "task_entry": self.execution.task_entry, **({"turn_envelope": True} if self.execution.turn_envelope else {}), + **({"acceptance_contract": self.execution.acceptance_contract} + if self.execution.acceptance_contract != "off" else {}), "home_scope": "trial", **self._replan_receipt(), "benchmark_phase": self._phase_number, @@ -715,6 +726,9 @@ async def run( raise RuntimeError("Planning did not return verified Todo readback") if entry["status"] == "blocked": return + if (self.execution.acceptance_contract == "planned" + and (entry.get("acceptance") or {}).get("status") != "applied"): + raise RuntimeError("Planning did not configure the Goal acceptance contract") remaining = int(deadline - time.monotonic()) if remaining <= 160: raise TimeoutError("Task budget exhausted before execution handoff") diff --git a/benchmark/runtime/planning.py b/benchmark/runtime/planning.py index 5e4ffb2ca3..cfb6161609 100644 --- a/benchmark/runtime/planning.py +++ b/benchmark/runtime/planning.py @@ -8,6 +8,11 @@ import time from pathlib import Path +from benchmark.runtime.acceptance import ( + ACCEPTANCE_DOCUMENT_ENV, + acceptance_planning_guidance, +) + def render_planning_prompt(packet: dict, env: dict[str, str], *, execution_reserve_seconds: int) -> str: """Project caller-owned clocks without changing the product planning packet.""" @@ -31,6 +36,8 @@ def render_planning_prompt(packet: dict, env: dict[str, str], *, execution_reser "runtime startup and settlement reserves also consume it.", ]) prefix = "$loopx\n\n" + if env.get(ACCEPTANCE_DOCUMENT_ENV): + prefix += acceptance_planning_guidance(env[ACCEPTANCE_DOCUMENT_ENV]) if window: prefix += "Host execution window:\n" + "\n".join(window) + "\n" prefix += ( diff --git a/benchmark/runtime/worker.py b/benchmark/runtime/worker.py index e33d7e6181..80fa344def 100644 --- a/benchmark/runtime/worker.py +++ b/benchmark/runtime/worker.py @@ -281,6 +281,7 @@ def run_once(env: dict[str, str]) -> dict: validation_command=json.loads(env.get("LOOPX_VALIDATION_COMMAND_JSON", "[]")), task_entry=env.get("LOOPX_TASK_ENTRY"), turn_envelope=env.get("LOOPX_TURN_ENVELOPE", "0") == "1", + acceptance_contract=env.get("LOOPX_ACCEPTANCE_CONTRACT", "off"), ) stage = env.get("LOOPX_TASK_STAGE", "execute") if stage not in {"plan", "execute"} or (stage == "plan" and execution.task_entry != "loopx-planned"): @@ -300,6 +301,8 @@ def run_once(env: dict[str, str]) -> dict: **({"turn_envelope": True} if execution.turn_envelope else {}), "stage": stage, "home_scope": "trial", + **({"acceptance_contract": execution.acceptance_contract} + if execution.acceptance_contract != "off" else {}), "ok": False, "timed_out": False, } @@ -341,6 +344,11 @@ def run_once(env: dict[str, str]) -> dict: from benchmark.runtime.checkpoint_wake import CheckpointWake checkpoint_wake = checkpoint_context.enter_context(CheckpointWake(env, receipt)) + if stage == "execute" and execution.acceptance_contract == "planned": + from benchmark.runtime.acceptance import reconfirm_planned_acceptance + + # Owner re-confirmation of edited selected Todos; criteria never change here. + receipt["acceptance"] = reconfirm_planned_acceptance(env, loopx_command(env)) body = "Finish the task." if stage == "plan": from benchmark.runtime.planning import render_planning_prompt, task_plan_packet @@ -410,6 +418,11 @@ def run_once(env: dict[str, str]) -> dict: receipt["planning"] = validate_plan_readback( result, planning_before, task_plan_packet(env, loopx_command(env)), ) + if execution.acceptance_contract == "planned" and receipt["planning"]["status"] == "ready": + from benchmark.runtime.acceptance import apply_planned_acceptance + + receipt["planning"]["acceptance"] = apply_planned_acceptance( + env, receipt["planning"], loopx_command(env)) target = Path(env["LOOPX_PLANNING_RESULT"]) temporary = target.with_suffix(".tmp") temporary.write_text(json.dumps(receipt["planning"])) diff --git a/tests/control_plane/test_benchmark_planned_acceptance.py b/tests/control_plane/test_benchmark_planned_acceptance.py new file mode 100644 index 0000000000..cadd5abf84 --- /dev/null +++ b/tests/control_plane/test_benchmark_planned_acceptance.py @@ -0,0 +1,242 @@ +"""Planned benchmark entry can lock an owner-confirmed acceptance gate. + +The real bootstrap, Todo, lease and ``goal-acceptance`` owners are exercised; +only the model is replaced. Feature-off parity keeps the previous closure rules. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest +from canonical_authority_fixture import isolate_sqlite_runtime + +from benchmark.runtime.acceptance import ( + ACCEPTANCE_DOCUMENT_ENV, + apply_planned_acceptance, + reconfirm_planned_acceptance, +) +from benchmark.runtime.codex import Execution +from benchmark.runtime.planning import render_planning_prompt +from benchmark.runtime.worker import loopx_command + +REPO_ROOT = Path(__file__).resolve().parents[2] +GOAL_ID, AGENT_ID = "benchmark-goal", "benchmark-agent" + + +def test_acceptance_contract_is_an_explicit_planned_entry_opt_in(): + assert Execution().acceptance_contract == "off" + assert Execution(mode="heartbeat", acceptance_contract="planned").task_entry == "loopx-planned" + for overrides in (dict(mode="heartbeat", task_entry="seeded-todo"), dict(mode="plain"), + dict(mode="native-goal")): + with pytest.raises(ValueError, match="loopx-planned"): + Execution(**overrides, acceptance_contract="planned") + with pytest.raises(ValueError, match="acceptance contract"): + Execution(acceptance_contract="required") + + +def test_planning_prompt_feature_off_parity_and_opt_in_guidance(): + packet = {"schema_version": "loopx_task_planning_v0", "input_digest": "d"} + off = render_planning_prompt(packet, {"UNRELATED": "1"}, execution_reserve_seconds=150) + assert off == render_planning_prompt(packet, {}, execution_reserve_seconds=150) + assert "acceptance document" not in off + on = render_planning_prompt(packet, {ACCEPTANCE_DOCUMENT_ENV: "/control/acceptance.json"}, + execution_reserve_seconds=150) + assert on.endswith(off.removeprefix("$loopx\n\n")) and on.startswith("$loopx\n\n") + for required in ("/control/acceptance.json", '"selected_work"', "must not read", + "fails this planning phase", "25 seconds in total"): + assert required in on + + +def _bootstrap_goal(tmp_path: Path) -> tuple[dict[str, str], list[str]]: + """Mirror the Harbor adapter's bootstrap, registration and seeded Todo.""" + project, control, runtime = tmp_path / "project", tmp_path / "control", tmp_path / "runtime" + project.mkdir(); control.mkdir() + (control / "task.md").write_text("# Task\nProduce outputs/result.txt containing ok.\n") + env = {**os.environ, "PYTHONPATH": str(REPO_ROOT), "LOOPX_PROJECT": str(project), + "LOOPX_REGISTRY": str(control / "registry.json"), "LOOPX_RUNTIME_ROOT": str(runtime), + "LOOPX_GOAL_ID": GOAL_ID, "LOOPX_AGENT_ID": AGENT_ID, + "LOOPX_TASK_DOC": str(control / "task.md"), + "LOOPX_CLI": str(Path(sys.executable).parent / "loopx"), + ACCEPTANCE_DOCUMENT_ENV: str(control / "acceptance-phase-001.json")} + cli = loopx_command(env) + + def run(*args, ok=True): + proc = subprocess.run(cli + list(args), cwd=project, env=env, text=True, capture_output=True) + payload = json.loads(proc.stdout) + assert (proc.returncode == 0) is ok, payload + return payload + + run("bootstrap", "--project", ".", "--goal-id", GOAL_ID, "--objective", "Complete the task", + "--goal-doc", str(control / "task.md"), "--adapter-kind", "read_only_project_map_v0", + "--adapter-status", "connected-read-only", "--write-scope", "**", "--no-global-sync") + run("configure-goal", "--goal-id", GOAL_ID, "--registered-agent", AGENT_ID, + "--boundary-authority-scope", "**", "--boundary-authority-source", "harbor-task-workspace", + "--boundary-authority-decision-id", "trial-workspace", "--execution-replan-after-turns", "6", + "--agent-work-mode", f"{AGENT_ID}=active", "--execute") + todo_id = run("todo", "add", "--goal-id", GOAL_ID, "--role", "agent", "--text", + "[P0] Deliver outputs/result.txt", "--task-class", "advancement_task", + "--action-kind", "benchmark_task", "--claimed-by", AGENT_ID, "--status", "open", + "--execute")["todo_id"] + env["TEST_TODO_ID"] = todo_id + return env, cli + + +def _document(todo_id: str) -> dict: + return { + "objective": "Deliver outputs/result.txt", "non_goals": ["Reading hidden data"], + "scope": {"kind": "selected_work", "todo_ids": [todo_id]}, + "criteria": [{"id": "result-present", "description": "result file says ok", + "validation_argv": [sys.executable, "-c", + "from pathlib import Path; assert Path('outputs/result.txt').read_text().strip() == 'ok'"], + "validation_timeout_seconds": 5}], + "bindings": [{"todo_id": todo_id, "criterion_ids": ["result-present"]}], + } + + +def test_real_goal_holds_selected_todo_until_criteria_pass_and_reconfirms_edits(tmp_path, monkeypatch): + isolate_sqlite_runtime(tmp_path, monkeypatch) + env, cli = _bootstrap_goal(tmp_path) + todo_id, project = env["TEST_TODO_ID"], Path(env["LOOPX_PROJECT"]) + document_path = Path(env[ACCEPTANCE_DOCUMENT_ENV]) + goal = ["--goal-id", GOAL_ID] + + def run(*args): + proc = subprocess.run(cli + list(args), cwd=project, env=env, text=True, capture_output=True) + return proc.returncode, json.loads(proc.stdout) + + def tasks(): + return [(row["state"], row["applicable"]) + for row in run("goal-acceptance", "inspect", *goal)[1]["goal_acceptance_contract"]["tasks"]] + + planning = {"todo_ids": [todo_id], "input_digest": "digest-1", "status": "ready"} + with pytest.raises(ValueError, match="did not write"): + apply_planned_acceptance(env, planning, cli) + document_path.write_text(json.dumps(_document("todo_elsewhere"))) + with pytest.raises(ValueError, match="outside the planning result"): + apply_planned_acceptance(env, planning, cli) + unbound = _document(todo_id); unbound["bindings"] = [] + document_path.write_text(json.dumps(unbound)) + with pytest.raises(ValueError, match="binding"): + apply_planned_acceptance(env, planning, cli) + assert run("goal-acceptance", "inspect", *goal)[1]["goal_acceptance_contract"] == {"enabled": False} + + document_path.write_text(json.dumps(_document(todo_id))) + applied = apply_planned_acceptance(env, planning, cli) + assert applied["status"] == "applied" and applied["selected_todo_ids"] == [todo_id] + assert applied["criterion_ids"] == ["result-present"] and "validation_argv" not in json.dumps(applied) + assert tasks() == [("ready", True)] + assert reconfirm_planned_acceptance(env, cli) == {"reconfirmed": [], "revision": 1} + + # The model edits its own selected Todo under a real lease: the association + # goes stale, and the host re-confirms it without changing any criterion. + code, lease = run("task-lease", "acquire", *goal, "--todo-id", todo_id, "--owner", AGENT_ID, + "--idempotency-key", "exec-1", "--expected-version", "0", "--ttl-seconds", "600") + assert code == 0 and lease["lease"]["version"] == 1 + binding = ["--todo-id", todo_id, "--agent-id", AGENT_ID, + "--task-lease-idempotency-key", "exec-1", "--task-lease-expected-version", "1"] + code, updated = run("todo", "update", *goal, *binding, "--text", "[P0] Deliver outputs/result.txt (ok)", "--execute") + assert code == 0 and updated["ok"], updated + assert tasks() == [("stale", True)] + assert reconfirm_planned_acceptance(env, cli) == {"reconfirmed": [todo_id], "revision": 2} + assert tasks() == [("ready", True)] + assert json.loads(document_path.read_text())["criteria"] == _document(todo_id)["criteria"] + + code, rejected = run("todo", "complete", *goal, *binding, "--evidence", "local run", "--no-follow-up") + assert code == 1 and rejected["error_code"] == "goal_acceptance_validation_rejected", rejected + assert rejected["goal_acceptance_validation_failure"]["criterion_id"] == "result-present" + assert run("todo", "list", *goal, "--role", "agent")[1]["todos"][0]["status"] == "open" + (project / "outputs").mkdir(); (project / "outputs" / "result.txt").write_text("ok\n") + code, completed = run("todo", "complete", *goal, *binding, "--evidence", "local run", "--no-follow-up") + assert code == 0 and completed["changed"] is True, completed + assert run("todo", "list", *goal, "--role", "agent")[1]["todos"][0]["status"] == "done" + assert reconfirm_planned_acceptance(env, cli)["reconfirmed"] == [] + + +_SYNTHETIC_CODEX = """#!{python} +import json, os, pathlib, sys +argv = sys.argv +body = sys.stdin.read() +packet = json.loads(body.split("Host-supplied planning checkpoint:\\n", 1)[1]) +result = {{"input_digest": packet["input_digest"], "status": "ready", + "todo_ids": packet["runnable_todo_ids"]}} +pathlib.Path(argv[argv.index("--output-last-message") + 1]).write_text(json.dumps(result)) +pathlib.Path("planning-body.txt").write_text(body) +if os.environ.get("LOOPX_ACCEPTANCE_DOCUMENT") and os.environ.get("SYNTHETIC_WRITE_DOCUMENT") == "1": + todo_id = packet["runnable_todo_ids"][0] + pathlib.Path(os.environ["LOOPX_ACCEPTANCE_DOCUMENT"]).write_text(json.dumps({{ + "objective": "Deliver outputs/result.txt", "non_goals": [], + "scope": {{"kind": "selected_work", "todo_ids": [todo_id]}}, + "criteria": [{{"id": "result-present", "description": "result file says ok", + "validation_argv": [sys.executable, "-c", "from pathlib import Path; assert Path('outputs/result.txt').read_text().strip() == 'ok'"], + "validation_timeout_seconds": 5}}], + "bindings": [{{"todo_id": todo_id, "criterion_ids": ["result-present"]}}]}})) +print(json.dumps({{"type": "thread.started", "thread_id": "synthetic-planning-session"}})) +""" + + +@pytest.mark.parametrize("contract", ["off", "planned", "planned-missing-document"]) +def test_worker_planning_stage_configures_or_requires_the_document(tmp_path, monkeypatch, contract): + from benchmark.runtime.worker import run_once + + isolate_sqlite_runtime(tmp_path, monkeypatch) + env, cli = _bootstrap_goal(tmp_path) + binary = tmp_path / "codex" + binary.write_text(_SYNTHETIC_CODEX.format(python=sys.executable)); binary.chmod(0o755) + (tmp_path / "skills").mkdir() + env.update({"CODEX_BIN": str(binary), "LOOPX_CODEX_HOME": str(tmp_path / "home"), + "LOOPX_WAKE_LOG_DIR": str(tmp_path / "wakes"), "LOOPX_SHARED_SKILLS": str(tmp_path / "skills"), + "MODEL_NAME": "fixture", "REASONING_EFFORT": "high", "OPENAI_BASE_URL": "http://localhost:8123/v1", + "OPENAI_API_KEY": "fixture-key", "LOOPX_TASK_ENTRY": "loopx-planned", "LOOPX_TASK_STAGE": "plan", + "LOOPX_PLANNING_RESULT": str(tmp_path / "control" / "planning-phase-001.json"), + "LOOPX_PHASE_DEADLINE_EPOCH": str(time.time() + 600), + "LOOPX_ACCEPTANCE_CONTRACT": "off" if contract == "off" else "planned", + "SYNTHETIC_WRITE_DOCUMENT": "1" if contract == "planned" else "0"}) + if contract == "off": + env.pop(ACCEPTANCE_DOCUMENT_ENV) + document_path = tmp_path / "control" / "acceptance-phase-001.json" + if contract == "planned-missing-document": + with pytest.raises(ValueError, match="did not write"): + run_once(env) + assert not Path(env["LOOPX_PLANNING_RESULT"]).exists() + return + receipt = run_once(env) + assert receipt["ok"] and receipt["stage"] == "plan", receipt + planning = json.loads(Path(env["LOOPX_PLANNING_RESULT"]).read_text()) + assert planning["status"] == "ready" and planning["todo_ids"] == [env["TEST_TODO_ID"]] + body = (Path(env["LOOPX_PROJECT"]) / "planning-body.txt").read_text() + inspect = subprocess.run(cli + ["goal-acceptance", "inspect", "--goal-id", GOAL_ID], cwd=env["LOOPX_PROJECT"], + env=env, text=True, capture_output=True) + contract_state = json.loads(inspect.stdout)["goal_acceptance_contract"] + if contract == "off": + assert "acceptance" not in planning and "acceptance_contract" not in receipt + assert "acceptance document" not in body and not document_path.exists() + assert contract_state == {"enabled": False} + else: + assert receipt["acceptance_contract"] == "planned" and str(document_path) in body + assert planning["acceptance"]["status"] == "applied" + assert planning["acceptance"]["selected_todo_ids"] == [env["TEST_TODO_ID"]] + assert contract_state["enabled"] and contract_state["tasks"] == [ + {**contract_state["tasks"][0], "state": "ready", "applicable": True}] + + +def test_render_config_discloses_the_default_and_forwards_the_opt_in(tmp_path): + pytest.importorskip("yaml") + base = [sys.executable, str(REPO_ROOT / "benchmark/LHTB/scripts/render_config.py"), "--template", + str(REPO_ROOT / "benchmark/LHTB/configs/heartbeat-generic-cli.yaml"), "--job-name", "smoke", + "--jobs-dir", str(tmp_path), "--concurrency", "1", "--model", "openai/fixture", "--effort", "max", + "--timeout", "5400", "--task", "tabular-data-feature-covshift"] + import yaml + for extra, expected in (([], "off"), (["--acceptance-contract", "planned"], "planned")): + output = tmp_path / f"{expected}.yaml" + subprocess.run(base + ["--output", str(output), *extra], check=True, env={**os.environ, "PYTHONPATH": str(REPO_ROOT)}) + kwargs = yaml.safe_load(output.read_text())["agents"][0]["kwargs"] + assert kwargs["acceptance_contract"] == expected and kwargs["task_entry"] == "loopx-planned" + failed = subprocess.run(base + ["--output", str(tmp_path / "bad.yaml"), "--task-entry", "seeded-todo", + "--acceptance-contract", "planned"], capture_output=True, text=True, + env={**os.environ, "PYTHONPATH": str(REPO_ROOT)}) + assert failed.returncode != 0 and "loopx-planned" in failed.stderr From 0a8735f0e247511f81429373d25b5eefe3217973 Mon Sep 17 00:00:00 2001 From: song Date: Fri, 16 Oct 2026 07:54:49 +0800 Subject: [PATCH 2/5] docs(benchmark): document the planned acceptance-contract ablation Co-Authored-By: Claude Fable 5.1 Signed-off-by: song --- benchmark/LHTB/README.md | 14 ++++++++++++++ benchmark/runtime/SETTINGS.md | 2 ++ 2 files changed, 16 insertions(+) diff --git a/benchmark/LHTB/README.md b/benchmark/LHTB/README.md index c839376108..c71d32eef9 100644 --- a/benchmark/LHTB/README.md +++ b/benchmark/LHTB/README.md @@ -130,6 +130,20 @@ settings. The worker uses `LOOPX_PHASE_DEADLINE_EPOCH`; legacy existing waits when new phases arrive. See the shared runtime for session and planning-readback semantics. +`LOOPX_ACCEPTANCE_CONTRACT=planned` (default `off`; requires `loopx-planned`) +adds a host acceptance checkpoint to planning: the model also writes a Goal +acceptance document whose criteria exercise the deliverable through the +interface the task declares for its verifier, on public inputs only. The +trial host configures that document through the existing owner-confirmed +`goal-acceptance` contract, scoped to the Todos planning returned. The +selected Todos cannot complete until their criteria pass, so the Goal cannot +derive terminal no-follow-up from Todo completion alone. On each wake the host +re-confirms associations that the model's own Todo edits made stale; criteria +never change after planning. A missing or invalid document fails the planning +phase. Criteria run as bounded argv commands inside the task container and +read no hidden verifier material; they are the model's own closure check, +not benchmark feedback. `off` keeps the previous behavior unchanged. + Model and effort defaults remain unchanged but may be selected explicitly. `run.sh prepare` performs networking/Harbor preparation. `preflight` now checks existing preparation without patching Harbor or creating a network. Smoke/full diff --git a/benchmark/runtime/SETTINGS.md b/benchmark/runtime/SETTINGS.md index f54c257749..f27a60da41 100644 --- a/benchmark/runtime/SETTINGS.md +++ b/benchmark/runtime/SETTINGS.md @@ -21,6 +21,7 @@ use matched repetitions to test its effect. | Task entry | LoopX modes: loopx-planned | Record planned or seeded for every arm | | Explore | Off in heartbeat-resume; on in heartbeat-explore | Keep resume as the reference | | Turn envelope | Off | Enable only in its ablation | +| Acceptance contract | Off | Harbor `acceptance_contract: planned` is an explicit closure-gate ablation | | Replan cadence | 6 settled effective work Turns | Use explicit completed-Todo cadence or another Turn count for ablation | | Iteration context | Harbor: fresh; EdgeBench heartbeat: resume | Freeze the provider and context within a comparison | | Evaluator feedback | EdgeBench: best-only (strict new-best snapshot notifications) | Native and blind remain explicit controls; freeze feedback mode for every comparison | @@ -50,6 +51,7 @@ These are recommendations, not automatically launched experiments. | Short envelope | Add `--turn-envelope` | Does progressive context loading reduce overhead without losing decisions? | | Todo cadence | Replace `--replan-after-turns 6` with `--replan-after-todos 3` | How do units and thresholds change interruptions? This is a combined ablation. | | No LoopX | `--worker official`; omit LoopX-specific flags | What is the net effect of the whole LoopX treatment? | +| Planned acceptance (Harbor) | Add `LOOPX_ACCEPTANCE_CONTRACT=planned` to the LHTB reference | Does an owner-locked, verifier-shaped acceptance gate stop premature closure before the budget is used? | The no-LoopX comparison changes several mechanisms; do not attribute its delta to one component. Native, blind and best-only feedback are a separate factor: repeat From a10f988c4daf9d8f36651319987e1bd59709cd73 Mon Sep 17 00:00:00 2001 From: song Date: Fri, 16 Oct 2026 09:01:06 +0800 Subject: [PATCH 3/5] benchmark(runtime): reject bare validation_files pins and keep acceptance failures in the wake receipt The first live planning run wrote `validation_files` as a list of paths; the owner contract needs `{path, sha256}` pins, and the TypeScript rejection only reached the host as a truncated traceback. Tell the planner to omit the field, fail with the criterion id named, and record the public-safe reason in the wake receipt so a failed plan stage is diagnosable from downloaded logs. Co-Authored-By: Claude Fable 5.1 Signed-off-by: song --- benchmark/runtime/acceptance.py | 12 +++++++++++- benchmark/runtime/worker.py | 10 ++++++++-- .../test_benchmark_planned_acceptance.py | 10 +++++++++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/benchmark/runtime/acceptance.py b/benchmark/runtime/acceptance.py index 5a11b60e6c..4052bf25ec 100644 --- a/benchmark/runtime/acceptance.py +++ b/benchmark/runtime/acceptance.py @@ -27,7 +27,8 @@ def acceptance_planning_guidance(document_path: str) -> str: "It is a JSON object with objective, non_goals (list), scope " '{"kind": "selected_work", "todo_ids": [...]} naming the Todo(s) whose ' "completion delivers the task result, criteria (list of {id, description, " - "validation_argv, validation_timeout_seconds, optional validation_files}) and " + "validation_argv, validation_timeout_seconds}; do not add validation_files or " + "other keys) and " "bindings (one {todo_id, criterion_ids} per selected Todo). Use the exact " "Todo ids you created. Each criterion is an argv command run from the task " "workspace without a shell, 1-25 seconds each and 25 seconds in total. " @@ -95,6 +96,15 @@ def _selected_and_bound(document: dict[str, Any]) -> tuple[list[str], list[dict[ not isinstance(row, dict) or not isinstance(row.get("id"), str) for row in criteria ): raise ValueError("Goal acceptance requires identified criteria") + # The owner contract pins validator files as {path, sha256}; planning cannot + # know digests of files that do not exist yet, so a bare path list is a + # misunderstanding of the schema rather than a usable pin. + for row in criteria: + pins = row.get("validation_files", []) + if pins and any(not isinstance(pin, dict) or set(pin) != {"path", "sha256"} for pin in pins): + raise ValueError( + f"criterion {row['id']}: validation_files must be [{{path, sha256}}] pins or omitted" + ) return list(selected), bindings diff --git a/benchmark/runtime/worker.py b/benchmark/runtime/worker.py index 80fa344def..81b56b6bad 100644 --- a/benchmark/runtime/worker.py +++ b/benchmark/runtime/worker.py @@ -421,8 +421,14 @@ def run_once(env: dict[str, str]) -> dict: if execution.acceptance_contract == "planned" and receipt["planning"]["status"] == "ready": from benchmark.runtime.acceptance import apply_planned_acceptance - receipt["planning"]["acceptance"] = apply_planned_acceptance( - env, receipt["planning"], loopx_command(env)) + try: + receipt["planning"]["acceptance"] = apply_planned_acceptance( + env, receipt["planning"], loopx_command(env)) + except ValueError as exc: + # Keep the public-safe reason in the wake receipt; host + # transports truncate the traceback. + receipt["planning"]["acceptance"] = {"status": "failed", "error": str(exc)} + raise target = Path(env["LOOPX_PLANNING_RESULT"]) temporary = target.with_suffix(".tmp") temporary.write_text(json.dumps(receipt["planning"])) diff --git a/tests/control_plane/test_benchmark_planned_acceptance.py b/tests/control_plane/test_benchmark_planned_acceptance.py index cadd5abf84..2e570791c1 100644 --- a/tests/control_plane/test_benchmark_planned_acceptance.py +++ b/tests/control_plane/test_benchmark_planned_acceptance.py @@ -48,7 +48,7 @@ def test_planning_prompt_feature_off_parity_and_opt_in_guidance(): execution_reserve_seconds=150) assert on.endswith(off.removeprefix("$loopx\n\n")) and on.startswith("$loopx\n\n") for required in ("/control/acceptance.json", '"selected_work"', "must not read", - "fails this planning phase", "25 seconds in total"): + "fails this planning phase", "25 seconds in total", "do not add validation_files"): assert required in on @@ -123,6 +123,10 @@ def tasks(): document_path.write_text(json.dumps(unbound)) with pytest.raises(ValueError, match="binding"): apply_planned_acceptance(env, planning, cli) + bare_pins = _document(todo_id); bare_pins["criteria"][0]["validation_files"] = ["outputs/result.txt"] + document_path.write_text(json.dumps(bare_pins)) + with pytest.raises(ValueError, match="validation_files must be"): + apply_planned_acceptance(env, planning, cli) assert run("goal-acceptance", "inspect", *goal)[1]["goal_acceptance_contract"] == {"enabled": False} document_path.write_text(json.dumps(_document(todo_id))) @@ -203,6 +207,10 @@ def test_worker_planning_stage_configures_or_requires_the_document(tmp_path, mon with pytest.raises(ValueError, match="did not write"): run_once(env) assert not Path(env["LOOPX_PLANNING_RESULT"]).exists() + (receipt_path,) = (tmp_path / "wakes").glob("*/receipt.json") + saved = json.loads(receipt_path.read_text()) + assert saved["ok"] is False and saved["planning"]["acceptance"]["status"] == "failed" + assert "did not write" in saved["planning"]["acceptance"]["error"] return receipt = run_once(env) assert receipt["ok"] and receipt["stage"] == "plan", receipt From 1f8b68d2937a60f52ea78e38ef35d9d3fc886f14 Mon Sep 17 00:00:00 2001 From: song Date: Fri, 16 Oct 2026 11:13:13 +0800 Subject: [PATCH 4/5] benchmark(runtime): re-confirm only the host-accepted acceptance document Re-confirmation re-applied the model-writable planning draft, so a Todo edit followed by a rewritten criterion let a selected Todo complete without its artifact. The host now keeps the configured document in its runtime root, re-applies only that copy, and refuses when the owner contract digest moved outside the host. The planning prompt uses the same explicit LOOPX_ACCEPTANCE_CONTRACT switch, so off with a leftover locator keeps the original prompt. Also fixes the E702 lint failures. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: song --- benchmark/runtime/acceptance.py | 48 ++++++++++++-- benchmark/runtime/planning.py | 7 +- .../test_benchmark_planned_acceptance.py | 65 +++++++++++++++---- 3 files changed, 101 insertions(+), 19 deletions(-) diff --git a/benchmark/runtime/acceptance.py b/benchmark/runtime/acceptance.py index 4052bf25ec..4bef8ed027 100644 --- a/benchmark/runtime/acceptance.py +++ b/benchmark/runtime/acceptance.py @@ -17,6 +17,19 @@ from typing import Any ACCEPTANCE_DOCUMENT_ENV = "LOOPX_ACCEPTANCE_DOCUMENT" +ACCEPTANCE_CONTRACT_ENV = "LOOPX_ACCEPTANCE_CONTRACT" +# Host-held copy of the last configured document. The model may keep editing its +# planning draft; re-confirmation only ever re-applies what the host accepted. +ACCEPTED_RECORD_NAME = "benchmark-accepted-acceptance.json" + + +def planned_acceptance_document(env: dict[str, str]) -> str | None: + """Return the document locator only when the explicit planned switch is on.""" + if env.get(ACCEPTANCE_CONTRACT_ENV, "off") != "planned": + return None + if not env.get(ACCEPTANCE_DOCUMENT_ENV): + raise ValueError(f"{ACCEPTANCE_CONTRACT_ENV}=planned requires {ACCEPTANCE_DOCUMENT_ENV}") + return env[ACCEPTANCE_DOCUMENT_ENV] def acceptance_planning_guidance(document_path: str) -> str: @@ -74,6 +87,17 @@ def _load_document(env: dict[str, str]) -> tuple[Path, dict[str, Any]]: return path, document +def _accepted_record_path(env: dict[str, str]) -> Path: + return Path(env.get("LOOPX_RUNTIME_ROOT") or env["LOOPX_CODEX_HOME"]) / ACCEPTED_RECORD_NAME + + +def _write_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + def _selected_and_bound(document: dict[str, Any]) -> tuple[list[str], list[dict[str, Any]]]: scope = document.get("scope") selected = scope.get("todo_ids") if isinstance(scope, dict) else None @@ -125,6 +149,10 @@ def _configure(env, cli, document_path: Path, *, operation_id: str, not_ready = [todo_id for todo_id in expected_todo_ids if states.get(todo_id) != "ready"] if not_ready: raise ValueError(f"acceptance association is not ready for {', '.join(not_ready)}") + _write_json(_accepted_record_path(env), { + "revision": contract["revision"], "digest": contract["digest"], + "document": json.loads(document_path.read_text(encoding="utf-8")), + }) return { "status": "applied", "revision": contract["revision"], @@ -155,16 +183,26 @@ def reconfirm_planned_acceptance(env: dict[str, str], cli: list[str]) -> dict[st """Re-confirm associations the model's own Todo edits made stale. The criteria never change here: owner confirmation of the current Todo text - is what the product requires after a selected Todo was edited. Completed or + is what the product requires after a selected Todo was edited. The document + re-applied is the host's accepted copy, never the model-writable draft, and + it must still match the owner's current contract digest. Completed or cancelled Todos leave the selection; their checks already ran at completion. """ - document_path, document = _load_document(env) + record_path = _accepted_record_path(env) + try: + record = json.loads(record_path.read_text(encoding="utf-8")) + document = record["document"] + except (OSError, ValueError, KeyError, TypeError) as exc: + raise ValueError("no host-accepted Goal acceptance document to re-confirm") from exc selected, bindings = _selected_and_bound(document) goal = ["--goal-id", env["LOOPX_GOAL_ID"]] contract = _run_cli(env, cli, ["goal-acceptance", "inspect", *goal]).get( "goal_acceptance_contract") or {} if contract.get("enabled") is not True: raise ValueError("planned acceptance contract is no longer enabled") + if contract.get("digest") != record.get("digest"): + raise ValueError("Goal acceptance contract changed outside the planned host; " + "refusing to re-confirm it from the host copy") tasks = {task["todo_id"]: task for task in contract.get("tasks", []) if isinstance(task, dict) and isinstance(task.get("todo_id"), str)} applicable = [todo_id for todo_id in selected @@ -175,9 +213,9 @@ def reconfirm_planned_acceptance(env: dict[str, str], cli: list[str]) -> dict[st revised = {**document, "scope": {"kind": "selected_work", "todo_ids": applicable}, "bindings": [row for row in bindings if row["todo_id"] in applicable]} - document_path.write_text(json.dumps(revised, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8") - receipt = _configure(env, cli, document_path, + revised_path = record_path.with_name("benchmark-acceptance-reconfirm.json") + _write_json(revised_path, revised) + receipt = _configure(env, cli, revised_path, operation_id=f"planned-acceptance-reconfirm:{contract['revision']}", expected_todo_ids=applicable) return {"reconfirmed": stale, "revision": receipt["revision"]} diff --git a/benchmark/runtime/planning.py b/benchmark/runtime/planning.py index cfb6161609..950bbd4f4e 100644 --- a/benchmark/runtime/planning.py +++ b/benchmark/runtime/planning.py @@ -9,8 +9,8 @@ from pathlib import Path from benchmark.runtime.acceptance import ( - ACCEPTANCE_DOCUMENT_ENV, acceptance_planning_guidance, + planned_acceptance_document, ) @@ -36,8 +36,9 @@ def render_planning_prompt(packet: dict, env: dict[str, str], *, execution_reser "runtime startup and settlement reserves also consume it.", ]) prefix = "$loopx\n\n" - if env.get(ACCEPTANCE_DOCUMENT_ENV): - prefix += acceptance_planning_guidance(env[ACCEPTANCE_DOCUMENT_ENV]) + document = planned_acceptance_document(env) + if document: + prefix += acceptance_planning_guidance(document) if window: prefix += "Host execution window:\n" + "\n".join(window) + "\n" prefix += ( diff --git a/tests/control_plane/test_benchmark_planned_acceptance.py b/tests/control_plane/test_benchmark_planned_acceptance.py index 2e570791c1..a6d1a6e642 100644 --- a/tests/control_plane/test_benchmark_planned_acceptance.py +++ b/tests/control_plane/test_benchmark_planned_acceptance.py @@ -44,8 +44,16 @@ def test_planning_prompt_feature_off_parity_and_opt_in_guidance(): off = render_planning_prompt(packet, {"UNRELATED": "1"}, execution_reserve_seconds=150) assert off == render_planning_prompt(packet, {}, execution_reserve_seconds=150) assert "acceptance document" not in off - on = render_planning_prompt(packet, {ACCEPTANCE_DOCUMENT_ENV: "/control/acceptance.json"}, - execution_reserve_seconds=150) + # An explicit off keeps the original prompt even when a stale locator leaks in. + legacy = render_planning_prompt( + packet, {"LOOPX_ACCEPTANCE_CONTRACT": "off", ACCEPTANCE_DOCUMENT_ENV: "/control/acceptance.json"}, + execution_reserve_seconds=150) + assert legacy == off + with pytest.raises(ValueError, match="requires LOOPX_ACCEPTANCE_DOCUMENT"): + render_planning_prompt(packet, {"LOOPX_ACCEPTANCE_CONTRACT": "planned"}, execution_reserve_seconds=150) + on = render_planning_prompt( + packet, {"LOOPX_ACCEPTANCE_CONTRACT": "planned", ACCEPTANCE_DOCUMENT_ENV: "/control/acceptance.json"}, + execution_reserve_seconds=150) assert on.endswith(off.removeprefix("$loopx\n\n")) and on.startswith("$loopx\n\n") for required in ("/control/acceptance.json", '"selected_work"', "must not read", "fails this planning phase", "25 seconds in total", "do not add validation_files"): @@ -55,7 +63,8 @@ def test_planning_prompt_feature_off_parity_and_opt_in_guidance(): def _bootstrap_goal(tmp_path: Path) -> tuple[dict[str, str], list[str]]: """Mirror the Harbor adapter's bootstrap, registration and seeded Todo.""" project, control, runtime = tmp_path / "project", tmp_path / "control", tmp_path / "runtime" - project.mkdir(); control.mkdir() + project.mkdir() + control.mkdir() (control / "task.md").write_text("# Task\nProduce outputs/result.txt containing ok.\n") env = {**os.environ, "PYTHONPATH": str(REPO_ROOT), "LOOPX_PROJECT": str(project), "LOOPX_REGISTRY": str(control / "registry.json"), "LOOPX_RUNTIME_ROOT": str(runtime), @@ -119,11 +128,13 @@ def tasks(): document_path.write_text(json.dumps(_document("todo_elsewhere"))) with pytest.raises(ValueError, match="outside the planning result"): apply_planned_acceptance(env, planning, cli) - unbound = _document(todo_id); unbound["bindings"] = [] + unbound = _document(todo_id) + unbound["bindings"] = [] document_path.write_text(json.dumps(unbound)) with pytest.raises(ValueError, match="binding"): apply_planned_acceptance(env, planning, cli) - bare_pins = _document(todo_id); bare_pins["criteria"][0]["validation_files"] = ["outputs/result.txt"] + bare_pins = _document(todo_id) + bare_pins["criteria"][0]["validation_files"] = ["outputs/result.txt"] document_path.write_text(json.dumps(bare_pins)) with pytest.raises(ValueError, match="validation_files must be"): apply_planned_acceptance(env, planning, cli) @@ -146,21 +157,52 @@ def tasks(): code, updated = run("todo", "update", *goal, *binding, "--text", "[P0] Deliver outputs/result.txt (ok)", "--execute") assert code == 0 and updated["ok"], updated assert tasks() == [("stale", True)] + # The model rewrites its draft to a vacuous check; re-confirmation must keep + # the host-accepted criterion instead of adopting the draft. + weakened = _document(todo_id) + weakened["criteria"][0]["validation_argv"] = [sys.executable, "-c", "pass"] + document_path.write_text(json.dumps(weakened)) assert reconfirm_planned_acceptance(env, cli) == {"reconfirmed": [todo_id], "revision": 2} assert tasks() == [("ready", True)] - assert json.loads(document_path.read_text())["criteria"] == _document(todo_id)["criteria"] + assert json.loads(document_path.read_text()) == weakened code, rejected = run("todo", "complete", *goal, *binding, "--evidence", "local run", "--no-follow-up") assert code == 1 and rejected["error_code"] == "goal_acceptance_validation_rejected", rejected assert rejected["goal_acceptance_validation_failure"]["criterion_id"] == "result-present" assert run("todo", "list", *goal, "--role", "agent")[1]["todos"][0]["status"] == "open" - (project / "outputs").mkdir(); (project / "outputs" / "result.txt").write_text("ok\n") + (project / "outputs").mkdir() + (project / "outputs" / "result.txt").write_text("ok\n") code, completed = run("todo", "complete", *goal, *binding, "--evidence", "local run", "--no-follow-up") assert code == 0 and completed["changed"] is True, completed assert run("todo", "list", *goal, "--role", "agent")[1]["todos"][0]["status"] == "done" assert reconfirm_planned_acceptance(env, cli)["reconfirmed"] == [] +def test_reconfirm_refuses_a_contract_changed_outside_the_host(tmp_path, monkeypatch): + isolate_sqlite_runtime(tmp_path, monkeypatch) + env, cli = _bootstrap_goal(tmp_path) + todo_id, document_path = env["TEST_TODO_ID"], Path(env[ACCEPTANCE_DOCUMENT_ENV]) + planning = {"todo_ids": [todo_id], "input_digest": "digest-1", "status": "ready"} + with pytest.raises(ValueError, match="no host-accepted"): + reconfirm_planned_acceptance(env, cli) + document_path.write_text(json.dumps(_document(todo_id))) + apply_planned_acceptance(env, planning, cli) + weakened = _document(todo_id) + weakened["criteria"][0]["validation_argv"] = [sys.executable, "-c", "pass"] + other = Path(env["LOOPX_PROJECT"]).parent / "control" / "out-of-band.json" + other.write_text(json.dumps(weakened)) + goal = ["--goal-id", GOAL_ID] + basis = json.loads(subprocess.run(cli + ["goal-acceptance", "inspect", *goal], cwd=env["LOOPX_PROJECT"], + env=env, text=True, capture_output=True).stdout) + configured = subprocess.run(cli + ["goal-acceptance", "configure", *goal, "--document", str(other), + "--expected-provider-revision", basis["provider_revision"], + "--operation-id", "out-of-band", "--execute"], + cwd=env["LOOPX_PROJECT"], env=env, text=True, capture_output=True) + assert configured.returncode == 0, configured.stdout + with pytest.raises(ValueError, match="changed outside the planned host"): + reconfirm_planned_acceptance(env, cli) + + _SYNTHETIC_CODEX = """#!{python} import json, os, pathlib, sys argv = sys.argv @@ -183,14 +225,15 @@ def tasks(): """ -@pytest.mark.parametrize("contract", ["off", "planned", "planned-missing-document"]) +@pytest.mark.parametrize("contract", ["off", "off-legacy-locator", "planned", "planned-missing-document"]) def test_worker_planning_stage_configures_or_requires_the_document(tmp_path, monkeypatch, contract): from benchmark.runtime.worker import run_once isolate_sqlite_runtime(tmp_path, monkeypatch) env, cli = _bootstrap_goal(tmp_path) binary = tmp_path / "codex" - binary.write_text(_SYNTHETIC_CODEX.format(python=sys.executable)); binary.chmod(0o755) + binary.write_text(_SYNTHETIC_CODEX.format(python=sys.executable)) + binary.chmod(0o755) (tmp_path / "skills").mkdir() env.update({"CODEX_BIN": str(binary), "LOOPX_CODEX_HOME": str(tmp_path / "home"), "LOOPX_WAKE_LOG_DIR": str(tmp_path / "wakes"), "LOOPX_SHARED_SKILLS": str(tmp_path / "skills"), @@ -198,7 +241,7 @@ def test_worker_planning_stage_configures_or_requires_the_document(tmp_path, mon "OPENAI_API_KEY": "fixture-key", "LOOPX_TASK_ENTRY": "loopx-planned", "LOOPX_TASK_STAGE": "plan", "LOOPX_PLANNING_RESULT": str(tmp_path / "control" / "planning-phase-001.json"), "LOOPX_PHASE_DEADLINE_EPOCH": str(time.time() + 600), - "LOOPX_ACCEPTANCE_CONTRACT": "off" if contract == "off" else "planned", + "LOOPX_ACCEPTANCE_CONTRACT": "off" if contract.startswith("off") else "planned", "SYNTHETIC_WRITE_DOCUMENT": "1" if contract == "planned" else "0"}) if contract == "off": env.pop(ACCEPTANCE_DOCUMENT_ENV) @@ -220,7 +263,7 @@ def test_worker_planning_stage_configures_or_requires_the_document(tmp_path, mon inspect = subprocess.run(cli + ["goal-acceptance", "inspect", "--goal-id", GOAL_ID], cwd=env["LOOPX_PROJECT"], env=env, text=True, capture_output=True) contract_state = json.loads(inspect.stdout)["goal_acceptance_contract"] - if contract == "off": + if contract.startswith("off"): assert "acceptance" not in planning and "acceptance_contract" not in receipt assert "acceptance document" not in body and not document_path.exists() assert contract_state == {"enabled": False} From 6c60727af26256e0ba1eb4c49b7e5048a632420a Mon Sep 17 00:00:00 2001 From: song Date: Fri, 16 Oct 2026 12:47:24 +0800 Subject: [PATCH 5/5] fix(benchmark): configure acceptance from one host snapshot Signed-off-by: song --- benchmark/runtime/acceptance.py | 39 +++++++++++-------- .../test_benchmark_planned_acceptance.py | 28 ++++++++++++- 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/benchmark/runtime/acceptance.py b/benchmark/runtime/acceptance.py index 4bef8ed027..b52d9690a1 100644 --- a/benchmark/runtime/acceptance.py +++ b/benchmark/runtime/acceptance.py @@ -13,6 +13,7 @@ import hashlib import json import subprocess +import tempfile from pathlib import Path from typing import Any @@ -74,7 +75,7 @@ def _run_cli(env: dict[str, str], cli: list[str], args: list[str]) -> dict[str, return payload -def _load_document(env: dict[str, str]) -> tuple[Path, dict[str, Any]]: +def _load_document(env: dict[str, str]) -> dict[str, Any]: path = Path(env[ACCEPTANCE_DOCUMENT_ENV]) try: document = json.loads(path.read_text(encoding="utf-8")) @@ -84,7 +85,7 @@ def _load_document(env: dict[str, str]) -> tuple[Path, dict[str, Any]]: raise ValueError("Goal acceptance document is unreadable or not JSON") from exc if not isinstance(document, dict): raise ValueError("Goal acceptance document must be a JSON object") - return path, document + return document def _accepted_record_path(env: dict[str, str]) -> Path: @@ -132,15 +133,22 @@ def _selected_and_bound(document: dict[str, Any]) -> tuple[list[str], list[dict[ return list(selected), bindings -def _configure(env, cli, document_path: Path, *, operation_id: str, +def _configure(env, cli, document: dict[str, Any], *, operation_id: str, expected_todo_ids: list[str]) -> dict[str, Any]: goal = ["--goal-id", env["LOOPX_GOAL_ID"]] basis = _run_cli(env, cli, ["goal-acceptance", "inspect", *goal]) - applied = _run_cli(env, cli, [ - "goal-acceptance", "configure", *goal, "--document", str(document_path), - "--expected-provider-revision", str(basis["provider_revision"]), - "--operation-id", operation_id, "--execute", - ]) + record_path = _accepted_record_path(env) + record_path.parent.mkdir(parents=True, exist_ok=True) + # Configure exactly the document already read and validated by the host. + # Neither side of this IO may re-read the model's mutable planning draft. + with tempfile.TemporaryDirectory(prefix="acceptance-", dir=record_path.parent) as directory: + snapshot_path = Path(directory) / "document.json" + _write_json(snapshot_path, document) + applied = _run_cli(env, cli, [ + "goal-acceptance", "configure", *goal, "--document", str(snapshot_path), + "--expected-provider-revision", str(basis["provider_revision"]), + "--operation-id", operation_id, "--execute", + ]) contract = applied.get("goal_acceptance_contract") or {} if applied.get("status") != "applied" or contract.get("enabled") is not True: raise ValueError("Goal acceptance configuration was not applied") @@ -149,9 +157,9 @@ def _configure(env, cli, document_path: Path, *, operation_id: str, not_ready = [todo_id for todo_id in expected_todo_ids if states.get(todo_id) != "ready"] if not_ready: raise ValueError(f"acceptance association is not ready for {', '.join(not_ready)}") - _write_json(_accepted_record_path(env), { + _write_json(record_path, { "revision": contract["revision"], "digest": contract["digest"], - "document": json.loads(document_path.read_text(encoding="utf-8")), + "document": document, }) return { "status": "applied", @@ -166,15 +174,16 @@ def _configure(env, cli, document_path: Path, *, operation_id: str, def apply_planned_acceptance(env: dict[str, str], planning: dict[str, Any], cli: list[str]) -> dict[str, Any]: """Configure the model-written document for the Todos planning returned.""" - document_path, document = _load_document(env) + document = _load_document(env) selected, _ = _selected_and_bound(document) unknown = sorted(set(selected) - set(planning["todo_ids"])) if unknown: raise ValueError(f"acceptance selects Todos outside the planning result: {', '.join(unknown)}") digest = hashlib.sha256( - document_path.read_bytes() + planning["input_digest"].encode("utf-8") + json.dumps(document, sort_keys=True, ensure_ascii=False).encode("utf-8") + + planning["input_digest"].encode("utf-8") ).hexdigest() - return _configure(env, cli, document_path, + return _configure(env, cli, document, operation_id=f"planned-acceptance:{digest[:32]}", expected_todo_ids=selected) @@ -213,9 +222,7 @@ def reconfirm_planned_acceptance(env: dict[str, str], cli: list[str]) -> dict[st revised = {**document, "scope": {"kind": "selected_work", "todo_ids": applicable}, "bindings": [row for row in bindings if row["todo_id"] in applicable]} - revised_path = record_path.with_name("benchmark-acceptance-reconfirm.json") - _write_json(revised_path, revised) - receipt = _configure(env, cli, revised_path, + receipt = _configure(env, cli, revised, operation_id=f"planned-acceptance-reconfirm:{contract['revision']}", expected_todo_ids=applicable) return {"reconfirmed": stale, "revision": receipt["revision"]} diff --git a/tests/control_plane/test_benchmark_planned_acceptance.py b/tests/control_plane/test_benchmark_planned_acceptance.py index a6d1a6e642..6cc62c8197 100644 --- a/tests/control_plane/test_benchmark_planned_acceptance.py +++ b/tests/control_plane/test_benchmark_planned_acceptance.py @@ -107,7 +107,10 @@ def _document(todo_id: str) -> dict: } -def test_real_goal_holds_selected_todo_until_criteria_pass_and_reconfirms_edits(tmp_path, monkeypatch): +@pytest.mark.parametrize("draft_edit", [None, "before-configure", "after-configure"]) +def test_real_goal_holds_selected_todo_until_criteria_pass_and_reconfirms_edits( + tmp_path, monkeypatch, draft_edit, +): isolate_sqlite_runtime(tmp_path, monkeypatch) env, cli = _bootstrap_goal(tmp_path) todo_id, project = env["TEST_TODO_ID"], Path(env["LOOPX_PROJECT"]) @@ -141,6 +144,29 @@ def tasks(): assert run("goal-acceptance", "inspect", *goal)[1]["goal_acceptance_contract"] == {"enabled": False} document_path.write_text(json.dumps(_document(todo_id))) + # Change only the model draft at either edge of the real configuration IO. + # Native configuration, lease/edit, reconfirm and completion stay real. + from benchmark.runtime import acceptance + + original_run_cli = acceptance._run_cli + changed_draft = False + + def configure_barrier(env, cli, args): + nonlocal changed_draft + mutate = draft_edit and not changed_draft and args[:2] == ["goal-acceptance", "configure"] + weakened = _document(todo_id) + weakened["criteria"][0]["validation_argv"] = [sys.executable, "-c", "pass"] + if mutate and draft_edit == "before-configure": + document_path.write_text(json.dumps(weakened)) + payload = original_run_cli(env, cli, args) + if mutate: + assert payload["status"] == "applied" + if draft_edit == "after-configure": + document_path.write_text(json.dumps(weakened)) + changed_draft = True + return payload + + monkeypatch.setattr(acceptance, "_run_cli", configure_barrier) applied = apply_planned_acceptance(env, planning, cli) assert applied["status"] == "applied" and applied["selected_todo_ids"] == [todo_id] assert applied["criterion_ids"] == ["result-present"] and "validation_argv" not in json.dumps(applied)