diff --git a/benchmark/LHTB/README.md b/benchmark/LHTB/README.md index c839376108..c71d32eef9 100644 --- a/benchmark/LHTB/README.md +++ b/benchmark/LHTB/README.md @@ -130,6 +130,20 @@ settings. The worker uses `LOOPX_PHASE_DEADLINE_EPOCH`; legacy existing waits when new phases arrive. See the shared runtime for session and planning-readback semantics. +`LOOPX_ACCEPTANCE_CONTRACT=planned` (default `off`; requires `loopx-planned`) +adds a host acceptance checkpoint to planning: the model also writes a Goal +acceptance document whose criteria exercise the deliverable through the +interface the task declares for its verifier, on public inputs only. The +trial host configures that document through the existing owner-confirmed +`goal-acceptance` contract, scoped to the Todos planning returned. The +selected Todos cannot complete until their criteria pass, so the Goal cannot +derive terminal no-follow-up from Todo completion alone. On each wake the host +re-confirms associations that the model's own Todo edits made stale; criteria +never change after planning. A missing or invalid document fails the planning +phase. Criteria run as bounded argv commands inside the task container and +read no hidden verifier material; they are the model's own closure check, +not benchmark feedback. `off` keeps the previous behavior unchanged. + Model and effort defaults remain unchanged but may be selected explicitly. `run.sh prepare` performs networking/Harbor preparation. `preflight` now checks existing preparation without patching Harbor or creating a network. Smoke/full diff --git a/benchmark/LHTB/run.sh b/benchmark/LHTB/run.sh index e816f65739..3811ddf11f 100755 --- a/benchmark/LHTB/run.sh +++ b/benchmark/LHTB/run.sh @@ -51,6 +51,7 @@ LOOPX_TASK_ENTRY="${LOOPX_TASK_ENTRY:-}" [[ -z "${LOOPX_PLANNING_TIMEOUT_SEC:-}" ]] || die "Remove LOOPX_PLANNING_TIMEOUT_SEC; planning shares LOOPX_SCHEDULER_TIMEOUT_SEC with execution" LOOPX_ITERATION_CONTEXT="${LOOPX_ITERATION_CONTEXT:-fresh}" LOOPX_VALIDATION_COMMAND_JSON="${LOOPX_VALIDATION_COMMAND_JSON:-[]}" +LOOPX_ACCEPTANCE_CONTRACT="${LOOPX_ACCEPTANCE_CONTRACT:-off}" SHARED_CODEX_AGENT_DIR="$LOOPX_SRC_DIR/benchmark/runtime" [[ -f "$SHARED_CODEX_AGENT_DIR/codex_offline.py" ]] || \ die "Shared offline Codex adapter not found: $SHARED_CODEX_AGENT_DIR/codex_offline.py" @@ -117,7 +118,9 @@ if [[ "$MODE" == smoke ]]; then expected_task_count=1 job_suffix="smoke-${SMOKE_TASK}" fi -job_name="lhtb-${LOOPX_EXECUTION_MODE}-${LOOPX_TASK_ENTRY:-default}-${LOOPX_ITERATION_CONTEXT}-${job_suffix}-${run_stamp}" +acceptance_suffix="" +[[ "$LOOPX_ACCEPTANCE_CONTRACT" == off ]] || acceptance_suffix="-acceptance-${LOOPX_ACCEPTANCE_CONTRACT}" +job_name="lhtb-${LOOPX_EXECUTION_MODE}-${LOOPX_TASK_ENTRY:-default}${acceptance_suffix}-${LOOPX_ITERATION_CONTEXT}-${job_suffix}-${run_stamp}" generated_config="$CODE_DIR/.generated/${job_name}.yaml" jobs_dir="$CODE_DIR/runs" @@ -142,6 +145,7 @@ fi "${task_entry_args[@]}" \ --iteration-context "$LOOPX_ITERATION_CONTEXT" \ --validation-command-json "$LOOPX_VALIDATION_COMMAND_JSON" \ + --acceptance-contract "$LOOPX_ACCEPTANCE_CONTRACT" \ "${turn_timeout_args[@]}" \ --scheduler-timeout "$LOOPX_SCHEDULER_TIMEOUT_SEC" \ "${task_args[@]}" diff --git a/benchmark/LHTB/scripts/render_config.py b/benchmark/LHTB/scripts/render_config.py index f154632e5c..29d9f9332b 100755 --- a/benchmark/LHTB/scripts/render_config.py +++ b/benchmark/LHTB/scripts/render_config.py @@ -9,7 +9,7 @@ from pathlib import Path import yaml -from benchmark.runtime.codex import CONTEXTS, MODES, TASK_ENTRIES, Execution +from benchmark.runtime.codex import ACCEPTANCE_CONTRACTS, CONTEXTS, MODES, TASK_ENTRIES, Execution def main() -> int: @@ -28,6 +28,8 @@ def main() -> int: parser.add_argument("--task-entry", choices=TASK_ENTRIES, help="LoopX modes default to loopx-planned; explicit seeded-todo retains legacy entry") parser.add_argument("--validation-command-json", default="[]") + parser.add_argument("--acceptance-contract", choices=ACCEPTANCE_CONTRACTS, default="off", + help="planned: planning also writes an owner-confirmed Goal acceptance document") parser.add_argument("--turn-timeout", type=float, default=None) parser.add_argument("--scheduler-timeout", type=int, default=5080) args = parser.parse_args() @@ -62,6 +64,7 @@ def main() -> int: timeout_seconds=args.turn_timeout, validation_command=json.loads(args.validation_command_json), task_entry=args.task_entry, + acceptance_contract=args.acceptance_contract, ) agent["kwargs"].update( execution_mode=execution.mode, @@ -70,6 +73,7 @@ def main() -> int: turn_timeout_sec=execution.timeout_seconds, scheduler_timeout_sec=args.scheduler_timeout, task_entry=execution.task_entry, + acceptance_contract=execution.acceptance_contract, ) if agent["kwargs"].get("planning_timeout_sec") is not None: parser.error("remove planning_timeout_sec from the template; planning shares scheduler-timeout") diff --git a/benchmark/runtime/SETTINGS.md b/benchmark/runtime/SETTINGS.md index f54c257749..f27a60da41 100644 --- a/benchmark/runtime/SETTINGS.md +++ b/benchmark/runtime/SETTINGS.md @@ -21,6 +21,7 @@ use matched repetitions to test its effect. | Task entry | LoopX modes: loopx-planned | Record planned or seeded for every arm | | Explore | Off in heartbeat-resume; on in heartbeat-explore | Keep resume as the reference | | Turn envelope | Off | Enable only in its ablation | +| Acceptance contract | Off | Harbor `acceptance_contract: planned` is an explicit closure-gate ablation | | Replan cadence | 6 settled effective work Turns | Use explicit completed-Todo cadence or another Turn count for ablation | | Iteration context | Harbor: fresh; EdgeBench heartbeat: resume | Freeze the provider and context within a comparison | | Evaluator feedback | EdgeBench: best-only (strict new-best snapshot notifications) | Native and blind remain explicit controls; freeze feedback mode for every comparison | @@ -50,6 +51,7 @@ These are recommendations, not automatically launched experiments. | Short envelope | Add `--turn-envelope` | Does progressive context loading reduce overhead without losing decisions? | | Todo cadence | Replace `--replan-after-turns 6` with `--replan-after-todos 3` | How do units and thresholds change interruptions? This is a combined ablation. | | No LoopX | `--worker official`; omit LoopX-specific flags | What is the net effect of the whole LoopX treatment? | +| Planned acceptance (Harbor) | Add `LOOPX_ACCEPTANCE_CONTRACT=planned` to the LHTB reference | Does an owner-locked, verifier-shaped acceptance gate stop premature closure before the budget is used? | The no-LoopX comparison changes several mechanisms; do not attribute its delta to one component. Native, blind and best-only feedback are a separate factor: repeat diff --git a/benchmark/runtime/acceptance.py b/benchmark/runtime/acceptance.py new file mode 100644 index 0000000000..b52d9690a1 --- /dev/null +++ b/benchmark/runtime/acceptance.py @@ -0,0 +1,228 @@ +"""Owner-confirmed Goal acceptance for the planned benchmark entry. + +The planning checkpoint asks the model to write a Goal acceptance document; the +benchmark host, acting as the trusted local owner inside the trial, configures +it through the existing ``goal-acceptance`` contract. The product owner then +holds the selected Todos open until their criteria pass, so "all Todos done" +can no longer derive terminal no-follow-up before the deliverable was exercised. +No scoring, hidden-verifier feedback or new state vocabulary is introduced. +""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import tempfile +from pathlib import Path +from typing import Any + +ACCEPTANCE_DOCUMENT_ENV = "LOOPX_ACCEPTANCE_DOCUMENT" +ACCEPTANCE_CONTRACT_ENV = "LOOPX_ACCEPTANCE_CONTRACT" +# Host-held copy of the last configured document. The model may keep editing its +# planning draft; re-confirmation only ever re-applies what the host accepted. +ACCEPTED_RECORD_NAME = "benchmark-accepted-acceptance.json" + + +def planned_acceptance_document(env: dict[str, str]) -> str | None: + """Return the document locator only when the explicit planned switch is on.""" + if env.get(ACCEPTANCE_CONTRACT_ENV, "off") != "planned": + return None + if not env.get(ACCEPTANCE_DOCUMENT_ENV): + raise ValueError(f"{ACCEPTANCE_CONTRACT_ENV}=planned requires {ACCEPTANCE_DOCUMENT_ENV}") + return env[ACCEPTANCE_DOCUMENT_ENV] + + +def acceptance_planning_guidance(document_path: str) -> str: + """Host instructions appended to the planning prompt for the planned contract.""" + return ( + "Host acceptance checkpoint:\n" + f"Before returning, write a Goal acceptance document to {document_path}. " + "It is a JSON object with objective, non_goals (list), scope " + '{"kind": "selected_work", "todo_ids": [...]} naming the Todo(s) whose ' + "completion delivers the task result, criteria (list of {id, description, " + "validation_argv, validation_timeout_seconds}; do not add validation_files or " + "other keys) and " + "bindings (one {todo_id, criterion_ids} per selected Todo). Use the exact " + "Todo ids you created. Each criterion is an argv command run from the task " + "workspace without a shell, 1-25 seconds each and 25 seconds in total. " + "Criteria must exercise the deliverable through the interface the task says " + "its verifier uses (for example run the documented command on the public " + "inputs into a fresh output directory, then require exit 0 and the required " + "artifacts), must fail until the deliverable exists, and must not read " + "held-out or hidden data or fixture answers. For a check longer than its " + "timeout, run it during the work and let the criterion verify its receipt " + "artifact. The host configures this document as the owner-confirmed " + "acceptance contract: a selected Todo cannot complete until its criteria " + "pass, and the Goal cannot close while it stays open. A missing or invalid " + "document fails this planning phase.\n\n" + ) + + +def _run_cli(env: dict[str, str], cli: list[str], args: list[str]) -> dict[str, Any]: + response = subprocess.run( + cli + args, cwd=env["LOOPX_PROJECT"], env=env, text=True, + capture_output=True, check=False, timeout=120, + ) + try: + payload = json.loads(response.stdout or "") + except json.JSONDecodeError as exc: + raise ValueError( + f"goal-acceptance {args[1]} returned invalid JSON (exit {response.returncode})" + ) from exc + if response.returncode != 0 or payload.get("ok") is False: + raise ValueError(f"goal-acceptance {args[1]} failed: {payload.get('error')}") + return payload + + +def _load_document(env: dict[str, str]) -> dict[str, Any]: + path = Path(env[ACCEPTANCE_DOCUMENT_ENV]) + try: + document = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise ValueError("planning did not write the Goal acceptance document") from exc + except (OSError, ValueError) as exc: + raise ValueError("Goal acceptance document is unreadable or not JSON") from exc + if not isinstance(document, dict): + raise ValueError("Goal acceptance document must be a JSON object") + return document + + +def _accepted_record_path(env: dict[str, str]) -> Path: + return Path(env.get("LOOPX_RUNTIME_ROOT") or env["LOOPX_CODEX_HOME"]) / ACCEPTED_RECORD_NAME + + +def _write_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def _selected_and_bound(document: dict[str, Any]) -> tuple[list[str], list[dict[str, Any]]]: + scope = document.get("scope") + selected = scope.get("todo_ids") if isinstance(scope, dict) else None + if ( + not isinstance(scope, dict) or scope.get("kind") != "selected_work" + or not isinstance(selected, list) or not selected + or any(not isinstance(item, str) or not item for item in selected) + or len(set(selected)) != len(selected) + ): + raise ValueError("Goal acceptance scope must select the delivering Todo ids") + bindings = document.get("bindings") + if not isinstance(bindings, list) or not bindings or any( + not isinstance(row, dict) or row.get("todo_id") not in selected for row in bindings + ): + raise ValueError("Goal acceptance bindings must cover only selected Todos") + if set(selected) - {row["todo_id"] for row in bindings}: + raise ValueError("every selected Todo needs an acceptance binding") + criteria = document.get("criteria") + if not isinstance(criteria, list) or not criteria or any( + not isinstance(row, dict) or not isinstance(row.get("id"), str) for row in criteria + ): + raise ValueError("Goal acceptance requires identified criteria") + # The owner contract pins validator files as {path, sha256}; planning cannot + # know digests of files that do not exist yet, so a bare path list is a + # misunderstanding of the schema rather than a usable pin. + for row in criteria: + pins = row.get("validation_files", []) + if pins and any(not isinstance(pin, dict) or set(pin) != {"path", "sha256"} for pin in pins): + raise ValueError( + f"criterion {row['id']}: validation_files must be [{{path, sha256}}] pins or omitted" + ) + return list(selected), bindings + + +def _configure(env, cli, document: dict[str, Any], *, operation_id: str, + expected_todo_ids: list[str]) -> dict[str, Any]: + goal = ["--goal-id", env["LOOPX_GOAL_ID"]] + basis = _run_cli(env, cli, ["goal-acceptance", "inspect", *goal]) + record_path = _accepted_record_path(env) + record_path.parent.mkdir(parents=True, exist_ok=True) + # Configure exactly the document already read and validated by the host. + # Neither side of this IO may re-read the model's mutable planning draft. + with tempfile.TemporaryDirectory(prefix="acceptance-", dir=record_path.parent) as directory: + snapshot_path = Path(directory) / "document.json" + _write_json(snapshot_path, document) + applied = _run_cli(env, cli, [ + "goal-acceptance", "configure", *goal, "--document", str(snapshot_path), + "--expected-provider-revision", str(basis["provider_revision"]), + "--operation-id", operation_id, "--execute", + ]) + contract = applied.get("goal_acceptance_contract") or {} + if applied.get("status") != "applied" or contract.get("enabled") is not True: + raise ValueError("Goal acceptance configuration was not applied") + states = {task.get("todo_id"): task.get("state") for task in contract.get("tasks", []) + if isinstance(task, dict)} + not_ready = [todo_id for todo_id in expected_todo_ids if states.get(todo_id) != "ready"] + if not_ready: + raise ValueError(f"acceptance association is not ready for {', '.join(not_ready)}") + _write_json(record_path, { + "revision": contract["revision"], "digest": contract["digest"], + "document": document, + }) + return { + "status": "applied", + "revision": contract["revision"], + "digest": contract["digest"], + "operation_id": operation_id, + "selected_todo_ids": expected_todo_ids, + "criterion_ids": [row["id"] for row in contract.get("criteria", [])], + } + + +def apply_planned_acceptance(env: dict[str, str], planning: dict[str, Any], + cli: list[str]) -> dict[str, Any]: + """Configure the model-written document for the Todos planning returned.""" + document = _load_document(env) + selected, _ = _selected_and_bound(document) + unknown = sorted(set(selected) - set(planning["todo_ids"])) + if unknown: + raise ValueError(f"acceptance selects Todos outside the planning result: {', '.join(unknown)}") + digest = hashlib.sha256( + json.dumps(document, sort_keys=True, ensure_ascii=False).encode("utf-8") + + planning["input_digest"].encode("utf-8") + ).hexdigest() + return _configure(env, cli, document, + operation_id=f"planned-acceptance:{digest[:32]}", + expected_todo_ids=selected) + + +def reconfirm_planned_acceptance(env: dict[str, str], cli: list[str]) -> dict[str, Any]: + """Re-confirm associations the model's own Todo edits made stale. + + The criteria never change here: owner confirmation of the current Todo text + is what the product requires after a selected Todo was edited. The document + re-applied is the host's accepted copy, never the model-writable draft, and + it must still match the owner's current contract digest. Completed or + cancelled Todos leave the selection; their checks already ran at completion. + """ + record_path = _accepted_record_path(env) + try: + record = json.loads(record_path.read_text(encoding="utf-8")) + document = record["document"] + except (OSError, ValueError, KeyError, TypeError) as exc: + raise ValueError("no host-accepted Goal acceptance document to re-confirm") from exc + selected, bindings = _selected_and_bound(document) + goal = ["--goal-id", env["LOOPX_GOAL_ID"]] + contract = _run_cli(env, cli, ["goal-acceptance", "inspect", *goal]).get( + "goal_acceptance_contract") or {} + if contract.get("enabled") is not True: + raise ValueError("planned acceptance contract is no longer enabled") + if contract.get("digest") != record.get("digest"): + raise ValueError("Goal acceptance contract changed outside the planned host; " + "refusing to re-confirm it from the host copy") + tasks = {task["todo_id"]: task for task in contract.get("tasks", []) + if isinstance(task, dict) and isinstance(task.get("todo_id"), str)} + applicable = [todo_id for todo_id in selected + if tasks.get(todo_id, {}).get("applicable") is True] + stale = [todo_id for todo_id in applicable if tasks[todo_id].get("state") == "stale"] + if not stale: + return {"reconfirmed": [], "revision": contract["revision"]} + revised = {**document, + "scope": {"kind": "selected_work", "todo_ids": applicable}, + "bindings": [row for row in bindings if row["todo_id"] in applicable]} + receipt = _configure(env, cli, revised, + operation_id=f"planned-acceptance-reconfirm:{contract['revision']}", + expected_todo_ids=applicable) + return {"reconfirmed": stale, "revision": receipt["revision"]} diff --git a/benchmark/runtime/codex.py b/benchmark/runtime/codex.py index 1c7dff6296..1f9705da0f 100644 --- a/benchmark/runtime/codex.py +++ b/benchmark/runtime/codex.py @@ -16,6 +16,9 @@ MODES = ("plain", "native-goal", "heartbeat", "turn", "loopx-goal") CONTEXTS = ("fresh", "resume") TASK_ENTRIES = ("seeded-todo", "loopx-planned") +# "planned": the planning checkpoint also writes an owner-confirmed Goal +# acceptance document; "off" keeps the existing closure semantics. +ACCEPTANCE_CONTRACTS = ("off", "planned") SANDBOXES = ("read-only", "workspace-write", "danger-full-access") @@ -28,6 +31,7 @@ class Execution: validation_command: tuple[str, ...] = () task_entry: str | None = None turn_envelope: bool = False + acceptance_contract: str = "off" def __post_init__(self) -> None: if not isinstance(self.turn_envelope, bool) or (self.turn_envelope and self.mode != "heartbeat"): @@ -40,6 +44,10 @@ def __post_init__(self) -> None: raise ValueError("unsupported task entry") if self.task_entry == "loopx-planned" and not self.uses_loopx: raise ValueError("loopx-planned requires a LoopX execution mode") + if self.acceptance_contract not in ACCEPTANCE_CONTRACTS: + raise ValueError("unsupported acceptance contract") + if self.acceptance_contract == "planned" and self.task_entry != "loopx-planned": + raise ValueError("acceptance_contract=planned requires loopx-planned task entry") if self.context != "fresh" and self.mode not in {"turn", "heartbeat"}: raise ValueError("resume requires mode=turn or mode=heartbeat") if self.sandbox not in SANDBOXES: diff --git a/benchmark/runtime/harbor.py b/benchmark/runtime/harbor.py index bd1bc60ed7..860e186d44 100644 --- a/benchmark/runtime/harbor.py +++ b/benchmark/runtime/harbor.py @@ -60,6 +60,7 @@ def __init__( task_entry=None, planning_timeout_sec=None, turn_envelope=False, + acceptance_contract="off", task_objective=None, goal_id=_GOAL_ID, agent_id=_AGENT_ID, @@ -76,6 +77,7 @@ def __init__( validation_command if validation_command is not None else (), task_entry, turn_envelope, + acceptance_contract=acceptance_contract, ) if planning_timeout_sec is not None: raise ValueError( @@ -334,6 +336,10 @@ async def _write_task_document( finally: Path(name).unlink(missing_ok=True) + @property + def _acceptance_document(self) -> str: + return f"{_CONTROL}/acceptance-phase-{self._phase_number:03d}.json" + @property def _task_document(self) -> str: # Existing Todos keep their original input when Harbor supplies a new phase. @@ -543,6 +549,9 @@ def _worker_env(self, *, cwd: str) -> dict[str, str]: **({"LOOPX_CHECKPOINT_REQUEST": self.checkpoint_request_path} if self.checkpoint_request_path is not None else {}), **({"LOOPX_TURN_ENVELOPE": "1"} if self.execution.turn_envelope else {}), + "LOOPX_ACCEPTANCE_CONTRACT": self.execution.acceptance_contract, + **({"LOOPX_ACCEPTANCE_DOCUMENT": self._acceptance_document} + if self.execution.acceptance_contract == "planned" else {}), "LOOPX_ITERATION_CONTEXT": self.execution.context, "LOOPX_CODEX_SANDBOX": self.execution.sandbox, "LOOPX_VALIDATION_COMMAND_JSON": json.dumps( @@ -655,6 +664,8 @@ def _populate_context( "iteration_context": self.execution.context, "task_entry": self.execution.task_entry, **({"turn_envelope": True} if self.execution.turn_envelope else {}), + **({"acceptance_contract": self.execution.acceptance_contract} + if self.execution.acceptance_contract != "off" else {}), "home_scope": "trial", **self._replan_receipt(), "benchmark_phase": self._phase_number, @@ -715,6 +726,9 @@ async def run( raise RuntimeError("Planning did not return verified Todo readback") if entry["status"] == "blocked": return + if (self.execution.acceptance_contract == "planned" + and (entry.get("acceptance") or {}).get("status") != "applied"): + raise RuntimeError("Planning did not configure the Goal acceptance contract") remaining = int(deadline - time.monotonic()) if remaining <= 160: raise TimeoutError("Task budget exhausted before execution handoff") diff --git a/benchmark/runtime/planning.py b/benchmark/runtime/planning.py index 5e4ffb2ca3..950bbd4f4e 100644 --- a/benchmark/runtime/planning.py +++ b/benchmark/runtime/planning.py @@ -8,6 +8,11 @@ import time from pathlib import Path +from benchmark.runtime.acceptance import ( + acceptance_planning_guidance, + planned_acceptance_document, +) + def render_planning_prompt(packet: dict, env: dict[str, str], *, execution_reserve_seconds: int) -> str: """Project caller-owned clocks without changing the product planning packet.""" @@ -31,6 +36,9 @@ def render_planning_prompt(packet: dict, env: dict[str, str], *, execution_reser "runtime startup and settlement reserves also consume it.", ]) prefix = "$loopx\n\n" + document = planned_acceptance_document(env) + if document: + prefix += acceptance_planning_guidance(document) if window: prefix += "Host execution window:\n" + "\n".join(window) + "\n" prefix += ( diff --git a/benchmark/runtime/worker.py b/benchmark/runtime/worker.py index e33d7e6181..81b56b6bad 100644 --- a/benchmark/runtime/worker.py +++ b/benchmark/runtime/worker.py @@ -281,6 +281,7 @@ def run_once(env: dict[str, str]) -> dict: validation_command=json.loads(env.get("LOOPX_VALIDATION_COMMAND_JSON", "[]")), task_entry=env.get("LOOPX_TASK_ENTRY"), turn_envelope=env.get("LOOPX_TURN_ENVELOPE", "0") == "1", + acceptance_contract=env.get("LOOPX_ACCEPTANCE_CONTRACT", "off"), ) stage = env.get("LOOPX_TASK_STAGE", "execute") if stage not in {"plan", "execute"} or (stage == "plan" and execution.task_entry != "loopx-planned"): @@ -300,6 +301,8 @@ def run_once(env: dict[str, str]) -> dict: **({"turn_envelope": True} if execution.turn_envelope else {}), "stage": stage, "home_scope": "trial", + **({"acceptance_contract": execution.acceptance_contract} + if execution.acceptance_contract != "off" else {}), "ok": False, "timed_out": False, } @@ -341,6 +344,11 @@ def run_once(env: dict[str, str]) -> dict: from benchmark.runtime.checkpoint_wake import CheckpointWake checkpoint_wake = checkpoint_context.enter_context(CheckpointWake(env, receipt)) + if stage == "execute" and execution.acceptance_contract == "planned": + from benchmark.runtime.acceptance import reconfirm_planned_acceptance + + # Owner re-confirmation of edited selected Todos; criteria never change here. + receipt["acceptance"] = reconfirm_planned_acceptance(env, loopx_command(env)) body = "Finish the task." if stage == "plan": from benchmark.runtime.planning import render_planning_prompt, task_plan_packet @@ -410,6 +418,17 @@ def run_once(env: dict[str, str]) -> dict: receipt["planning"] = validate_plan_readback( result, planning_before, task_plan_packet(env, loopx_command(env)), ) + if execution.acceptance_contract == "planned" and receipt["planning"]["status"] == "ready": + from benchmark.runtime.acceptance import apply_planned_acceptance + + try: + receipt["planning"]["acceptance"] = apply_planned_acceptance( + env, receipt["planning"], loopx_command(env)) + except ValueError as exc: + # Keep the public-safe reason in the wake receipt; host + # transports truncate the traceback. + receipt["planning"]["acceptance"] = {"status": "failed", "error": str(exc)} + raise target = Path(env["LOOPX_PLANNING_RESULT"]) temporary = target.with_suffix(".tmp") temporary.write_text(json.dumps(receipt["planning"])) diff --git a/tests/control_plane/test_benchmark_planned_acceptance.py b/tests/control_plane/test_benchmark_planned_acceptance.py new file mode 100644 index 0000000000..6cc62c8197 --- /dev/null +++ b/tests/control_plane/test_benchmark_planned_acceptance.py @@ -0,0 +1,319 @@ +"""Planned benchmark entry can lock an owner-confirmed acceptance gate. + +The real bootstrap, Todo, lease and ``goal-acceptance`` owners are exercised; +only the model is replaced. Feature-off parity keeps the previous closure rules. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest +from canonical_authority_fixture import isolate_sqlite_runtime + +from benchmark.runtime.acceptance import ( + ACCEPTANCE_DOCUMENT_ENV, + apply_planned_acceptance, + reconfirm_planned_acceptance, +) +from benchmark.runtime.codex import Execution +from benchmark.runtime.planning import render_planning_prompt +from benchmark.runtime.worker import loopx_command + +REPO_ROOT = Path(__file__).resolve().parents[2] +GOAL_ID, AGENT_ID = "benchmark-goal", "benchmark-agent" + + +def test_acceptance_contract_is_an_explicit_planned_entry_opt_in(): + assert Execution().acceptance_contract == "off" + assert Execution(mode="heartbeat", acceptance_contract="planned").task_entry == "loopx-planned" + for overrides in (dict(mode="heartbeat", task_entry="seeded-todo"), dict(mode="plain"), + dict(mode="native-goal")): + with pytest.raises(ValueError, match="loopx-planned"): + Execution(**overrides, acceptance_contract="planned") + with pytest.raises(ValueError, match="acceptance contract"): + Execution(acceptance_contract="required") + + +def test_planning_prompt_feature_off_parity_and_opt_in_guidance(): + packet = {"schema_version": "loopx_task_planning_v0", "input_digest": "d"} + off = render_planning_prompt(packet, {"UNRELATED": "1"}, execution_reserve_seconds=150) + assert off == render_planning_prompt(packet, {}, execution_reserve_seconds=150) + assert "acceptance document" not in off + # An explicit off keeps the original prompt even when a stale locator leaks in. + legacy = render_planning_prompt( + packet, {"LOOPX_ACCEPTANCE_CONTRACT": "off", ACCEPTANCE_DOCUMENT_ENV: "/control/acceptance.json"}, + execution_reserve_seconds=150) + assert legacy == off + with pytest.raises(ValueError, match="requires LOOPX_ACCEPTANCE_DOCUMENT"): + render_planning_prompt(packet, {"LOOPX_ACCEPTANCE_CONTRACT": "planned"}, execution_reserve_seconds=150) + on = render_planning_prompt( + packet, {"LOOPX_ACCEPTANCE_CONTRACT": "planned", ACCEPTANCE_DOCUMENT_ENV: "/control/acceptance.json"}, + execution_reserve_seconds=150) + assert on.endswith(off.removeprefix("$loopx\n\n")) and on.startswith("$loopx\n\n") + for required in ("/control/acceptance.json", '"selected_work"', "must not read", + "fails this planning phase", "25 seconds in total", "do not add validation_files"): + assert required in on + + +def _bootstrap_goal(tmp_path: Path) -> tuple[dict[str, str], list[str]]: + """Mirror the Harbor adapter's bootstrap, registration and seeded Todo.""" + project, control, runtime = tmp_path / "project", tmp_path / "control", tmp_path / "runtime" + project.mkdir() + control.mkdir() + (control / "task.md").write_text("# Task\nProduce outputs/result.txt containing ok.\n") + env = {**os.environ, "PYTHONPATH": str(REPO_ROOT), "LOOPX_PROJECT": str(project), + "LOOPX_REGISTRY": str(control / "registry.json"), "LOOPX_RUNTIME_ROOT": str(runtime), + "LOOPX_GOAL_ID": GOAL_ID, "LOOPX_AGENT_ID": AGENT_ID, + "LOOPX_TASK_DOC": str(control / "task.md"), + "LOOPX_CLI": str(Path(sys.executable).parent / "loopx"), + ACCEPTANCE_DOCUMENT_ENV: str(control / "acceptance-phase-001.json")} + cli = loopx_command(env) + + def run(*args, ok=True): + proc = subprocess.run(cli + list(args), cwd=project, env=env, text=True, capture_output=True) + payload = json.loads(proc.stdout) + assert (proc.returncode == 0) is ok, payload + return payload + + run("bootstrap", "--project", ".", "--goal-id", GOAL_ID, "--objective", "Complete the task", + "--goal-doc", str(control / "task.md"), "--adapter-kind", "read_only_project_map_v0", + "--adapter-status", "connected-read-only", "--write-scope", "**", "--no-global-sync") + run("configure-goal", "--goal-id", GOAL_ID, "--registered-agent", AGENT_ID, + "--boundary-authority-scope", "**", "--boundary-authority-source", "harbor-task-workspace", + "--boundary-authority-decision-id", "trial-workspace", "--execution-replan-after-turns", "6", + "--agent-work-mode", f"{AGENT_ID}=active", "--execute") + todo_id = run("todo", "add", "--goal-id", GOAL_ID, "--role", "agent", "--text", + "[P0] Deliver outputs/result.txt", "--task-class", "advancement_task", + "--action-kind", "benchmark_task", "--claimed-by", AGENT_ID, "--status", "open", + "--execute")["todo_id"] + env["TEST_TODO_ID"] = todo_id + return env, cli + + +def _document(todo_id: str) -> dict: + return { + "objective": "Deliver outputs/result.txt", "non_goals": ["Reading hidden data"], + "scope": {"kind": "selected_work", "todo_ids": [todo_id]}, + "criteria": [{"id": "result-present", "description": "result file says ok", + "validation_argv": [sys.executable, "-c", + "from pathlib import Path; assert Path('outputs/result.txt').read_text().strip() == 'ok'"], + "validation_timeout_seconds": 5}], + "bindings": [{"todo_id": todo_id, "criterion_ids": ["result-present"]}], + } + + +@pytest.mark.parametrize("draft_edit", [None, "before-configure", "after-configure"]) +def test_real_goal_holds_selected_todo_until_criteria_pass_and_reconfirms_edits( + tmp_path, monkeypatch, draft_edit, +): + isolate_sqlite_runtime(tmp_path, monkeypatch) + env, cli = _bootstrap_goal(tmp_path) + todo_id, project = env["TEST_TODO_ID"], Path(env["LOOPX_PROJECT"]) + document_path = Path(env[ACCEPTANCE_DOCUMENT_ENV]) + goal = ["--goal-id", GOAL_ID] + + def run(*args): + proc = subprocess.run(cli + list(args), cwd=project, env=env, text=True, capture_output=True) + return proc.returncode, json.loads(proc.stdout) + + def tasks(): + return [(row["state"], row["applicable"]) + for row in run("goal-acceptance", "inspect", *goal)[1]["goal_acceptance_contract"]["tasks"]] + + planning = {"todo_ids": [todo_id], "input_digest": "digest-1", "status": "ready"} + with pytest.raises(ValueError, match="did not write"): + apply_planned_acceptance(env, planning, cli) + document_path.write_text(json.dumps(_document("todo_elsewhere"))) + with pytest.raises(ValueError, match="outside the planning result"): + apply_planned_acceptance(env, planning, cli) + unbound = _document(todo_id) + unbound["bindings"] = [] + document_path.write_text(json.dumps(unbound)) + with pytest.raises(ValueError, match="binding"): + apply_planned_acceptance(env, planning, cli) + bare_pins = _document(todo_id) + bare_pins["criteria"][0]["validation_files"] = ["outputs/result.txt"] + document_path.write_text(json.dumps(bare_pins)) + with pytest.raises(ValueError, match="validation_files must be"): + apply_planned_acceptance(env, planning, cli) + assert run("goal-acceptance", "inspect", *goal)[1]["goal_acceptance_contract"] == {"enabled": False} + + document_path.write_text(json.dumps(_document(todo_id))) + # Change only the model draft at either edge of the real configuration IO. + # Native configuration, lease/edit, reconfirm and completion stay real. + from benchmark.runtime import acceptance + + original_run_cli = acceptance._run_cli + changed_draft = False + + def configure_barrier(env, cli, args): + nonlocal changed_draft + mutate = draft_edit and not changed_draft and args[:2] == ["goal-acceptance", "configure"] + weakened = _document(todo_id) + weakened["criteria"][0]["validation_argv"] = [sys.executable, "-c", "pass"] + if mutate and draft_edit == "before-configure": + document_path.write_text(json.dumps(weakened)) + payload = original_run_cli(env, cli, args) + if mutate: + assert payload["status"] == "applied" + if draft_edit == "after-configure": + document_path.write_text(json.dumps(weakened)) + changed_draft = True + return payload + + monkeypatch.setattr(acceptance, "_run_cli", configure_barrier) + applied = apply_planned_acceptance(env, planning, cli) + assert applied["status"] == "applied" and applied["selected_todo_ids"] == [todo_id] + assert applied["criterion_ids"] == ["result-present"] and "validation_argv" not in json.dumps(applied) + assert tasks() == [("ready", True)] + assert reconfirm_planned_acceptance(env, cli) == {"reconfirmed": [], "revision": 1} + + # The model edits its own selected Todo under a real lease: the association + # goes stale, and the host re-confirms it without changing any criterion. + code, lease = run("task-lease", "acquire", *goal, "--todo-id", todo_id, "--owner", AGENT_ID, + "--idempotency-key", "exec-1", "--expected-version", "0", "--ttl-seconds", "600") + assert code == 0 and lease["lease"]["version"] == 1 + binding = ["--todo-id", todo_id, "--agent-id", AGENT_ID, + "--task-lease-idempotency-key", "exec-1", "--task-lease-expected-version", "1"] + code, updated = run("todo", "update", *goal, *binding, "--text", "[P0] Deliver outputs/result.txt (ok)", "--execute") + assert code == 0 and updated["ok"], updated + assert tasks() == [("stale", True)] + # The model rewrites its draft to a vacuous check; re-confirmation must keep + # the host-accepted criterion instead of adopting the draft. + weakened = _document(todo_id) + weakened["criteria"][0]["validation_argv"] = [sys.executable, "-c", "pass"] + document_path.write_text(json.dumps(weakened)) + assert reconfirm_planned_acceptance(env, cli) == {"reconfirmed": [todo_id], "revision": 2} + assert tasks() == [("ready", True)] + assert json.loads(document_path.read_text()) == weakened + + code, rejected = run("todo", "complete", *goal, *binding, "--evidence", "local run", "--no-follow-up") + assert code == 1 and rejected["error_code"] == "goal_acceptance_validation_rejected", rejected + assert rejected["goal_acceptance_validation_failure"]["criterion_id"] == "result-present" + assert run("todo", "list", *goal, "--role", "agent")[1]["todos"][0]["status"] == "open" + (project / "outputs").mkdir() + (project / "outputs" / "result.txt").write_text("ok\n") + code, completed = run("todo", "complete", *goal, *binding, "--evidence", "local run", "--no-follow-up") + assert code == 0 and completed["changed"] is True, completed + assert run("todo", "list", *goal, "--role", "agent")[1]["todos"][0]["status"] == "done" + assert reconfirm_planned_acceptance(env, cli)["reconfirmed"] == [] + + +def test_reconfirm_refuses_a_contract_changed_outside_the_host(tmp_path, monkeypatch): + isolate_sqlite_runtime(tmp_path, monkeypatch) + env, cli = _bootstrap_goal(tmp_path) + todo_id, document_path = env["TEST_TODO_ID"], Path(env[ACCEPTANCE_DOCUMENT_ENV]) + planning = {"todo_ids": [todo_id], "input_digest": "digest-1", "status": "ready"} + with pytest.raises(ValueError, match="no host-accepted"): + reconfirm_planned_acceptance(env, cli) + document_path.write_text(json.dumps(_document(todo_id))) + apply_planned_acceptance(env, planning, cli) + weakened = _document(todo_id) + weakened["criteria"][0]["validation_argv"] = [sys.executable, "-c", "pass"] + other = Path(env["LOOPX_PROJECT"]).parent / "control" / "out-of-band.json" + other.write_text(json.dumps(weakened)) + goal = ["--goal-id", GOAL_ID] + basis = json.loads(subprocess.run(cli + ["goal-acceptance", "inspect", *goal], cwd=env["LOOPX_PROJECT"], + env=env, text=True, capture_output=True).stdout) + configured = subprocess.run(cli + ["goal-acceptance", "configure", *goal, "--document", str(other), + "--expected-provider-revision", basis["provider_revision"], + "--operation-id", "out-of-band", "--execute"], + cwd=env["LOOPX_PROJECT"], env=env, text=True, capture_output=True) + assert configured.returncode == 0, configured.stdout + with pytest.raises(ValueError, match="changed outside the planned host"): + reconfirm_planned_acceptance(env, cli) + + +_SYNTHETIC_CODEX = """#!{python} +import json, os, pathlib, sys +argv = sys.argv +body = sys.stdin.read() +packet = json.loads(body.split("Host-supplied planning checkpoint:\\n", 1)[1]) +result = {{"input_digest": packet["input_digest"], "status": "ready", + "todo_ids": packet["runnable_todo_ids"]}} +pathlib.Path(argv[argv.index("--output-last-message") + 1]).write_text(json.dumps(result)) +pathlib.Path("planning-body.txt").write_text(body) +if os.environ.get("LOOPX_ACCEPTANCE_DOCUMENT") and os.environ.get("SYNTHETIC_WRITE_DOCUMENT") == "1": + todo_id = packet["runnable_todo_ids"][0] + pathlib.Path(os.environ["LOOPX_ACCEPTANCE_DOCUMENT"]).write_text(json.dumps({{ + "objective": "Deliver outputs/result.txt", "non_goals": [], + "scope": {{"kind": "selected_work", "todo_ids": [todo_id]}}, + "criteria": [{{"id": "result-present", "description": "result file says ok", + "validation_argv": [sys.executable, "-c", "from pathlib import Path; assert Path('outputs/result.txt').read_text().strip() == 'ok'"], + "validation_timeout_seconds": 5}}], + "bindings": [{{"todo_id": todo_id, "criterion_ids": ["result-present"]}}]}})) +print(json.dumps({{"type": "thread.started", "thread_id": "synthetic-planning-session"}})) +""" + + +@pytest.mark.parametrize("contract", ["off", "off-legacy-locator", "planned", "planned-missing-document"]) +def test_worker_planning_stage_configures_or_requires_the_document(tmp_path, monkeypatch, contract): + from benchmark.runtime.worker import run_once + + isolate_sqlite_runtime(tmp_path, monkeypatch) + env, cli = _bootstrap_goal(tmp_path) + binary = tmp_path / "codex" + binary.write_text(_SYNTHETIC_CODEX.format(python=sys.executable)) + binary.chmod(0o755) + (tmp_path / "skills").mkdir() + env.update({"CODEX_BIN": str(binary), "LOOPX_CODEX_HOME": str(tmp_path / "home"), + "LOOPX_WAKE_LOG_DIR": str(tmp_path / "wakes"), "LOOPX_SHARED_SKILLS": str(tmp_path / "skills"), + "MODEL_NAME": "fixture", "REASONING_EFFORT": "high", "OPENAI_BASE_URL": "http://localhost:8123/v1", + "OPENAI_API_KEY": "fixture-key", "LOOPX_TASK_ENTRY": "loopx-planned", "LOOPX_TASK_STAGE": "plan", + "LOOPX_PLANNING_RESULT": str(tmp_path / "control" / "planning-phase-001.json"), + "LOOPX_PHASE_DEADLINE_EPOCH": str(time.time() + 600), + "LOOPX_ACCEPTANCE_CONTRACT": "off" if contract.startswith("off") else "planned", + "SYNTHETIC_WRITE_DOCUMENT": "1" if contract == "planned" else "0"}) + if contract == "off": + env.pop(ACCEPTANCE_DOCUMENT_ENV) + document_path = tmp_path / "control" / "acceptance-phase-001.json" + if contract == "planned-missing-document": + with pytest.raises(ValueError, match="did not write"): + run_once(env) + assert not Path(env["LOOPX_PLANNING_RESULT"]).exists() + (receipt_path,) = (tmp_path / "wakes").glob("*/receipt.json") + saved = json.loads(receipt_path.read_text()) + assert saved["ok"] is False and saved["planning"]["acceptance"]["status"] == "failed" + assert "did not write" in saved["planning"]["acceptance"]["error"] + return + receipt = run_once(env) + assert receipt["ok"] and receipt["stage"] == "plan", receipt + planning = json.loads(Path(env["LOOPX_PLANNING_RESULT"]).read_text()) + assert planning["status"] == "ready" and planning["todo_ids"] == [env["TEST_TODO_ID"]] + body = (Path(env["LOOPX_PROJECT"]) / "planning-body.txt").read_text() + inspect = subprocess.run(cli + ["goal-acceptance", "inspect", "--goal-id", GOAL_ID], cwd=env["LOOPX_PROJECT"], + env=env, text=True, capture_output=True) + contract_state = json.loads(inspect.stdout)["goal_acceptance_contract"] + if contract.startswith("off"): + assert "acceptance" not in planning and "acceptance_contract" not in receipt + assert "acceptance document" not in body and not document_path.exists() + assert contract_state == {"enabled": False} + else: + assert receipt["acceptance_contract"] == "planned" and str(document_path) in body + assert planning["acceptance"]["status"] == "applied" + assert planning["acceptance"]["selected_todo_ids"] == [env["TEST_TODO_ID"]] + assert contract_state["enabled"] and contract_state["tasks"] == [ + {**contract_state["tasks"][0], "state": "ready", "applicable": True}] + + +def test_render_config_discloses_the_default_and_forwards_the_opt_in(tmp_path): + pytest.importorskip("yaml") + base = [sys.executable, str(REPO_ROOT / "benchmark/LHTB/scripts/render_config.py"), "--template", + str(REPO_ROOT / "benchmark/LHTB/configs/heartbeat-generic-cli.yaml"), "--job-name", "smoke", + "--jobs-dir", str(tmp_path), "--concurrency", "1", "--model", "openai/fixture", "--effort", "max", + "--timeout", "5400", "--task", "tabular-data-feature-covshift"] + import yaml + for extra, expected in (([], "off"), (["--acceptance-contract", "planned"], "planned")): + output = tmp_path / f"{expected}.yaml" + subprocess.run(base + ["--output", str(output), *extra], check=True, env={**os.environ, "PYTHONPATH": str(REPO_ROOT)}) + kwargs = yaml.safe_load(output.read_text())["agents"][0]["kwargs"] + assert kwargs["acceptance_contract"] == expected and kwargs["task_entry"] == "loopx-planned" + failed = subprocess.run(base + ["--output", str(tmp_path / "bad.yaml"), "--task-entry", "seeded-todo", + "--acceptance-contract", "planned"], capture_output=True, text=True, + env={**os.environ, "PYTHONPATH": str(REPO_ROOT)}) + assert failed.returncode != 0 and "loopx-planned" in failed.stderr