From dd757aa4625bdfb636d673422fc223b8933cb051 Mon Sep 17 00:00:00 2001 From: song Date: Tue, 20 Oct 2026 14:23:57 +0800 Subject: [PATCH 1/2] feat(benchmark): preserve feedback recovery as its own dependent slice Rebuild PR #6902 as only the unchanged C3 delta on C2, whose newly introduced files this patch modifies. Remove unrelated historical ancestry. Retarget to main after C2 lands, and retain C1 request admission as an explicit additional merge prerequisite rather than copying it. The current C2+C3 tree lacks ResearchRequestGuard from C1: full feedback/wait/continuity collection remains blocked. Preserve that gate and report optional-dependency skips separately from passing tests. Original source: a083bbf70880f09975e5f44fee67a60251ffbab3 Original author: song Original patch-id: 10824ec803e744fee91bc52232d633080e1f2a2a Local source: 78c9c71190993a218dd656ba91cc787e0944af30 Tree: 0dde069ea8cb9a3a448e1964fbfb53ae6cfc192f Original history: codex/research-split-archive-20261020 Signed-off-by: song Signed-off-by: Codex --- benchmark/edgebench/README.md | 5 + benchmark/edgebench/export_report.py | 18 + benchmark/edgebench/research_cost.py | 188 +++++++ benchmark/edgebench/research_feedback.py | 347 +++++++++++++ benchmark/edgebench/run.py | 31 ++ benchmark/runtime/RESEARCH_LOOP.md | 277 ++++++++-- benchmark/runtime/harbor.py | 27 +- benchmark/runtime/research_feedback_sync.py | 204 ++++++++ benchmark/runtime/research_feedback_wait.py | 236 +++++++++ benchmark/runtime/research_loop.py | 34 +- benchmark/runtime/research_probe.py | 7 + benchmark/runtime/research_scheduler.py | 67 +++ benchmark/runtime/research_submission.py | 394 ++++++++++++++ benchmark/runtime/scheduler.py | 5 +- benchmark/runtime/sforge.py | 73 +++ benchmark/runtime/sforge_backend.py | 10 +- benchmark/tests/test_edgebench_report.py | 57 +++ .../tests/test_native_research_feedback.py | 482 ++++++++++++++++++ benchmark/tests/test_research_cost.py | 188 +++++++ benchmark/tests/test_sforge_runtime.py | 113 ++++ .../capabilities/explore/result_writeback.py | 66 +++ scripts/external_scheduler_worker.py | 4 +- .../test_native_research_continuity.py | 237 +++++++++ .../test_research_execution_loop.py | 59 +++ .../test_research_feedback_wait.py | 215 ++++++++ .../control_plane/test_research_scheduler.py | 248 +++++++++ 26 files changed, 3524 insertions(+), 68 deletions(-) create mode 100644 benchmark/edgebench/research_cost.py create mode 100644 benchmark/edgebench/research_feedback.py create mode 100644 benchmark/runtime/research_feedback_sync.py create mode 100644 benchmark/runtime/research_feedback_wait.py create mode 100644 benchmark/runtime/research_scheduler.py create mode 100644 benchmark/runtime/research_submission.py create mode 100644 benchmark/tests/test_native_research_feedback.py create mode 100644 benchmark/tests/test_research_cost.py create mode 100644 tests/control_plane/test_native_research_continuity.py create mode 100644 tests/control_plane/test_research_feedback_wait.py create mode 100644 tests/control_plane/test_research_scheduler.py diff --git a/benchmark/edgebench/README.md b/benchmark/edgebench/README.md index e1fcb4d356..eb6b535ae0 100644 --- a/benchmark/edgebench/README.md +++ b/benchmark/edgebench/README.md @@ -1,5 +1,10 @@ # Native EdgeBench worker trials +The explicit fresh-run [`--research-loop` treatment](../runtime/RESEARCH_LOOP.md) +connects candidate-stage separation, per-request admission, actual local probe +diagnostics and source-bound Explore feedback. It is not enabled for existing +attempts or baselines and does not change scoring or extend budget. + This research adapter calls SForge's native `run_agent`. Task images, prompts, submission cooldowns, judge isolation and scoring remain owned by the pinned EdgeBench checkout. Best-only adds host capture/admission scheduling; native and diff --git a/benchmark/edgebench/export_report.py b/benchmark/edgebench/export_report.py index ad54a6e11c..66ae815b1d 100644 --- a/benchmark/edgebench/export_report.py +++ b/benchmark/edgebench/export_report.py @@ -29,6 +29,19 @@ def _validate_feedback_payload(runner, profile): raise ValueError("Unsupported feedback payload for this mode") +def _validate_research_loop(runner, profile): + # Compare the original sources before the allowlisted settings projection + # can discard a one-sided flag. Historical absence remains distinct from + # an explicitly disabled treatment, and bool cannot be supplied as 0/1. + if ("research_loop" in runner) != ("research_loop" in profile): + raise ValueError("Settings profile disagrees with runtime: research_loop") + for source in (runner, profile): + if "research_loop" in source and type(source["research_loop"]) is not bool: + raise ValueError("research_loop must be a boolean") + if runner.get("research_loop") != profile.get("research_loop"): + raise ValueError("Settings profile disagrees with runtime: research_loop") + + def digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -129,6 +142,7 @@ def _validate_run_data(row, config, summary, points): raise ValueError(f"Settings identity mismatch: {key}") runner, profile = config["runner"], config["worker_profile"] _validate_feedback_payload(runner, profile) + _validate_research_loop(runner, profile) if ( runner["runner_commit"] != row["runner_revision"] or runner["model"] != row["model_id"] @@ -254,6 +268,7 @@ def export(runs_root: Path, selection: list, output: Path, *, observed_at: str): if row.get("runner_revision") != receipt["runner_commit"]: raise ValueError("Board and runner revision disagree") _validate_feedback_payload(receipt, profile) + _validate_research_loop(receipt, profile) if receipt.get("planning_horizon_goal") != profile.get("planning_horizon_goal"): raise ValueError("Profile disagrees with runtime: planning_horizon_goal") for source in (receipt, profile): @@ -337,6 +352,9 @@ def export(runs_root: Path, selection: list, output: Path, *, observed_at: str): if "planning_horizon_goal" in receipt: config["runner"]["planning_horizon_goal"] = receipt["planning_horizon_goal"] config["worker_profile"]["planning_horizon_goal"] = profile["planning_horizon_goal"] + if "research_loop" in receipt: + config["runner"]["research_loop"] = receipt["research_loop"] + config["worker_profile"]["research_loop"] = profile["research_loop"] if "stride_strategy" in receipt: config["runner"]["stride_strategy"] = receipt["stride_strategy"] config["worker_profile"]["stride_strategy"] = profile["stride_strategy"] diff --git a/benchmark/edgebench/research_cost.py b/benchmark/edgebench/research_cost.py new file mode 100644 index 0000000000..34c7c6bba3 --- /dev/null +++ b/benchmark/edgebench/research_cost.py @@ -0,0 +1,188 @@ +"""Passive whole-run cost readback over existing private research artifacts. + +The TypeScript trajectory owner interprets Codex counters. This adapter only +selects non-overlapping complete captures and reports observed counts; it does +not price tokens, divide costs between stages, or infer missing consumption. +""" +from __future__ import annotations + +import hashlib +import json +import math +from pathlib import Path +import re + + +def _json(path): + try: + return json.loads(path.read_bytes()) + except (OSError, ValueError): + return None + + +def _coverage(known, missing): + return "partial" if known and missing else "complete" if known else "unknown" + + +def _collection(trial): + rows = _json(trial / "collected" / "artifact-collection.json") + return {row["artifact"] for row in rows if isinstance(row, dict) + and isinstance(row.get("artifact"), str) and row.get("collected") is True} \ + if isinstance(rows, list) else set() + + +def _requests(trial, collected): + paths = sorted((trial / "collected" / "runtime").rglob("provider-requests.jsonl")) + grouped, malformed, readable = {}, 0, 0 + for path in paths: + try: + if path.resolve() != path.absolute(): + raise ValueError("nonphysical capture") + lines = path.read_text(encoding="utf-8").splitlines() + readable += 1 + for line in lines: + try: + row = json.loads(line) + if (not isinstance(row, dict) or not isinstance(row.get("request_id"), str) + or not row["request_id"] or row.get("event") not in {"dispatch", "finished"}): + raise ValueError("invalid request record") + grouped.setdefault(row["request_id"], []).append(row) + except (ValueError, TypeError): + malformed += 1 + except (OSError, UnicodeError, ValueError): + malformed += 1 + counts = {"forwarded": 0, "not_forwarded": 0, "send_unknown": 0} + for rows in grouped.values(): + finished = {json.dumps(row, sort_keys=True, separators=(",", ":")): + row for row in rows if row["event"] == "finished"} + value = next(iter(finished.values())) if len(finished) == 1 else {} + forwarded = value.get("forwarded") + if forwarded is True: + counts["forwarded"] += 1 + elif forwarded is False and not any(row["event"] == "dispatch" for row in rows): + counts["not_forwarded"] += 1 + else: + counts["send_unknown"] += 1 + return {"capture_coverage": _coverage(readable, malformed or "runtime" not in collected), + "captured_distinct_requests": len(grouped) if readable else None, + **(counts if readable else {key: None for key in counts}), "invalid_records": malformed, + "scope": "observed_gateway_requests_only"} + + +def _prefix(shorter, longer): + with shorter.open("rb") as left, longer.open("rb") as right: + while chunk := left.read(1024 * 1024): + if chunk != right.read(len(chunk)): + return False + return True + + +def _tokens(trial, collected): + from loopx.capabilities.benchmark_toolkit.trajectory_assessment_store import _usage_capture + + root = trial / "collected" + paths = {path for name in ("native-sessions", "worker-sessions") for path in (root / name).rglob("*.jsonl")} + paths.update(path for path in (root / "runtime").rglob("*.jsonl") + if "sessions" in path.relative_to(root / "runtime").parts) + grouped, invalid = {}, 0 + for path in sorted(paths): + try: + if path.resolve() != path.absolute(): + raise ValueError("nonphysical session capture") + with path.open("rb") as stream: + header = json.loads(stream.readline()) + payload = header.get("payload") or {} + session = payload.get("id") + if header.get("type") != "session_meta" or not isinstance(session, str) or not session: + raise ValueError("missing native session identity") + stream.seek(0) + digest = hashlib.file_digest(stream, "sha256").hexdigest() + grouped.setdefault(session, []).append((path, path.stat().st_size, digest)) + except (OSError, ValueError, AttributeError): + invalid += 1 + fields = ("input_tokens", "cached_input_tokens", "uncached_input_tokens", "output_tokens") + totals, qualified, unknown, duplicates = dict.fromkeys(fields, 0), 0, invalid, 0 + for session, captures in grouped.items(): + longest, _, digest = max(captures, key=lambda row: row[1]) + try: + # Same identity alone does not make divergent/forked snapshots one + # charge. Only byte-identical or strict-prefix captures are deduped. + if not all(row[2] == digest or _prefix(row[0], longest) for row in captures): + raise ValueError("divergent session captures") + result = _usage_capture(root, longest.relative_to(root).as_posix(), session) + if result["artifact_sha256"] != digest: + raise ValueError("session moved during readback") + projection = result["projection"] + for field in fields: + totals[field] += projection[field] + qualified += 1 + duplicates += len(captures) - 1 + except (OSError, ValueError, RuntimeError): + # Missing prefixes, forks, gaps and unfinished Turns are rejected + # by the existing typed owner. Never substitute the maximum count. + unknown += 1 + complete_collection = {"runtime", "native-sessions", "worker-sessions"}.issubset(collected) + return {"capture_coverage": _coverage(qualified, unknown or not complete_collection), + "whole_run_coverage": "unknown", "qualified_sessions": qualified, + "unknown_sessions_or_files": unknown, "duplicate_or_prefix_captures": duplicates, + "known_captured_totals": ({**totals, "input_plus_output_tokens": + totals["input_tokens"] + totals["output_tokens"]} if qualified else None), + "method": "codex_complete_session_counters_v1", + "limitations": "Known complete captured sessions only. Input includes cached input; output includes reasoning. Missing/provider-side consumption and billing remain unknown."} + + +def _submissions(trial): + from .research_feedback import validate_packet + + packet = _json(trial / "native-research-feedback" / "native.json") + final = _json(trial / "final_result.json") + native_total = final.get("total_rounds") if isinstance(final, dict) else None + if type(native_total) is not int or native_total < 0: + native_total = None + try: + validate_packet(packet) + except (ValueError, TypeError): + return {"capture_coverage": "unknown", "manual_accepted_count": None, + "first_accepted": None, "native_total_rounds": native_total} + grouped, unknown = {}, 0 + for row in packet["records"]: + if (not isinstance(row, dict) or row.get("accepted_record") is not True + or not isinstance(row.get("submission_id"), str) or not row["submission_id"] + or not isinstance(row.get("round_id"), str) + or not re.fullmatch(r"agent-[1-9][0-9]*", row["round_id"])): + unknown += 1 + continue + grouped.setdefault(row["submission_id"], []).append(row) + accepted = [] + for rows in grouped.values(): + if len({json.dumps(row, sort_keys=True) for row in rows}) != 1: + unknown += 1 + else: + accepted.append(rows[0]) + rounds = [row["round_id"] for row in accepted] + unique = [row for row in accepted if rounds.count(row["round_id"]) == 1] + unknown += len(accepted) - len(unique) + ordered = sorted(unique, key=lambda row: int(row["round_id"].split("-")[1])) + first = ordered[0] if ordered and ordered[0]["round_id"] == "agent-1" else None + released = first and first.get("attribution_status") == "verified" and first.get("result_ref") + summary = first.get("result_summary") if released else None + safe_summary = {key: summary[key] for key in ("score", "pass_rate", "valid", "passed", "failed", "total_tests") + if isinstance(summary, dict) and key in summary and (summary[key] is None + or type(summary[key]) is bool or type(summary[key]) in (int, float) and math.isfinite(summary[key]))} + return {"capture_coverage": "partial" if unknown else "complete", "manual_accepted_count": len(unique), + "first_accepted": ({"acceptance_ordinal": 1, "released": bool(released), + "result_summary": safe_summary if released else None} + if first else None), "native_total_rounds": native_total} + + +def summarize_research_cost(trial: Path, runner_elapsed_seconds): + """No new I/O to models/judges; report only already captured local evidence.""" + trial = Path(trial) + elapsed = runner_elapsed_seconds if type(runner_elapsed_seconds) in (int, float) \ + and math.isfinite(runner_elapsed_seconds) and runner_elapsed_seconds >= 0 else None + collected = _collection(trial) + return {"schema_version": "edgebench_research_cost_v0", "runner_wall_seconds": elapsed, + "wall_scope": "measured_runner_entry_through_native_cleanup_and_final_result", + "stage_seconds": None, "stage_allocation_coverage": "unknown", "billing_usd": None, + "provider_requests": _requests(trial, collected), "tokens": _tokens(trial, collected), + "evaluations": _submissions(trial)} diff --git a/benchmark/edgebench/research_feedback.py b/benchmark/edgebench/research_feedback.py new file mode 100644 index 0000000000..d2c83e9bf6 --- /dev/null +++ b/benchmark/edgebench/research_feedback.py @@ -0,0 +1,347 @@ +"""Read native accepted snapshots and released feedback for an explicit research run. + +SForge retains admission, evaluation, scoring and opportunity ownership. This +adapter uses only the token-visible history/results and the native submitted +source archive; it never reads judge artifacts or initiates an evaluation. +""" +from __future__ import annotations + +from datetime import datetime, timezone +import gzip +import hashlib +import io +import json +import os +from pathlib import Path, PurePosixPath +import re +import stat +import tarfile +import threading +import uuid +import zlib + +SCHEMA = "edgebench_native_feedback_v1" +MAX_ARCHIVE_BYTES = 256 * 1024 * 1024 +MAX_UNPACKED_BYTES = 512 * 1024 * 1024 +_ID = re.compile(r"[A-Za-z0-9_-]{1,128}\Z") +_ROUND = re.compile(r"agent-[1-9][0-9]*\Z") + + +def canonical_json(value) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=False, allow_nan=False).encode() + + +def write_bytes_atomic(path: Path, raw: bytes, *, create_only=False): + """Commit one private provider receipt before reporting the associated fact.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp") + try: + with temporary.open("xb") as stream: + stream.write(raw) + stream.flush() + os.fsync(stream.fileno()) + if create_only: + os.link(temporary, path) + else: + temporary.replace(path) + descriptor = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + finally: + temporary.unlink(missing_ok=True) + + +def read_archive(path: Path, *, root: Path) -> bytes: + """Read one physical native source file; detect movement during capture.""" + root = root.resolve(strict=True) + path = path.absolute() + if not path.is_relative_to(root) or path.resolve(strict=True) != path: + raise ValueError("native_archive_not_physical") + with os.fdopen(os.open(path, os.O_RDONLY | os.O_NOFOLLOW), "rb") as stream: + before = os.fstat(stream.fileno()) + if not stat.S_ISREG(before.st_mode) or before.st_size > MAX_ARCHIVE_BYTES: + raise ValueError("native_archive_size_or_type") + raw = stream.read(MAX_ARCHIVE_BYTES + 1) + after = os.fstat(stream.fileno()) + current = path.stat(follow_symlinks=False) + def signature(row): + return row.st_dev, row.st_ino, row.st_size, row.st_mtime_ns, row.st_ctime_ns + if (signature(before) != signature(after) or signature(after) != signature(current) + or len(raw) != before.st_size or path.resolve(strict=True) != path): + raise ValueError("native_archive_moved") + return raw + + +def archive_manifest(raw: bytes, selected_paths: list[str] | None = None): + """Use exactly research_probe's sorted selected-file manifest, not tar bytes. + + Nothing is extracted. Ambiguous member names, links and oversized sources + remain unqualified, including when the current workspace looks plausible. + """ + if len(raw) > MAX_ARCHIVE_BYTES: + raise ValueError("native_archive_too_large") + requested = None if selected_paths is None else set(selected_paths) + if requested is not None and (not 1 <= len(requested) <= 32 or len(requested) != len(selected_paths)): + raise ValueError("invalid_selected_files") + members, files, total = set(), [], 0 + with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as archive: + for member in archive: + name = member.name + while name.startswith("./"): + name = name[2:] + if member.isdir() and name in ("", "."): + continue + path = PurePosixPath(name) + if (path.is_absolute() or ".." in path.parts or not name or "\\" in name + or path.as_posix() != name.rstrip("/") or name in members): + raise ValueError("ambiguous_archive_member") + members.add(name) + if member.isdir(): + continue + if not member.isfile(): + raise ValueError("non_regular_archive_member") + total += member.size + if len(members) > 100000 or total > MAX_UNPACKED_BYTES: + raise ValueError("native_archive_expansion_limit") + if requested is not None and name not in requested: + continue + stream = archive.extractfile(member) + if stream is None: + raise ValueError("unreadable_archive_member") + with stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() + files.append({"path": name, "sha256": digest}) + if requested is not None and {row["path"] for row in files} != requested: + raise ValueError("selected_files_not_in_accepted_archive") + if not 1 <= len(files) <= 32: + raise ValueError("select_one_to_32_files") + files.sort(key=lambda row: row["path"]) + # research_probe uses json.dumps' default ensure_ascii=True. + encoded = json.dumps(files, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest(), files + + +def validate_packet(packet): + if (not isinstance(packet, dict) or packet.get("schema_version") != SCHEMA + or not _ID.fullmatch(packet.get("run_id") or "") + or not _ID.fullmatch(packet.get("task_id") or "") + or not isinstance(packet.get("records"), list)): + raise ValueError("invalid_native_feedback_packet") + return packet + + +class NativeResearchFeedback: + """One low-cost publisher scoped to the registration, surviving native wakes. + + ``after_publish(backend, handle, packet)`` invokes the installed normal + Explore/Todo consumer. Publication is retried until that consumer succeeds; + neither a copied file nor an unchanged poll is a control-plane settlement. + """ + def __init__(self, *, trial: Path, run_id: str, task_id: str, judge_url: str, + remote_root: str, logger=None, session=None, poll_seconds: float = 5): + if not _ID.fullmatch(run_id) or not _ID.fullmatch(task_id) or poll_seconds <= 0: + raise ValueError("invalid_native_feedback_scope") + self.trial, self.run_id, self.task_id = Path(trial), run_id, task_id + self.directory = self.trial / "native-research-feedback" + self.judge_url, self.remote_root = judge_url.rstrip("/"), PurePosixPath(remote_root) + self.logger, self.poll_seconds = logger, poll_seconds + self.token, self.thread, self.after_publish = None, None, None + self._stop, self._active = threading.Event(), threading.Event() + self._lock = threading.RLock() + self._published, self._packet = None, None + self._observed_at = {} + previous = self.directory / "native.json" + if previous.exists(): + packet = validate_packet(json.loads(previous.read_bytes())) + if (packet["run_id"], packet["task_id"]) != (run_id, task_id): + raise ValueError("native_feedback_saved_scope_mismatch") + self._observed_at = {(row["submission_id"], row.get("result_sha256")): row["result_observed_at"] + for row in packet["records"] if row.get("result_observed_at")} + self._artifacts = {} + if session is None: + import requests + session = requests.Session() + session.trust_env = False # Host path to its declared native judge. + self.session = session + + def bind(self, environment): + token = (environment or {}).get("SFORGE_TOKEN") + if not token or (self.token is not None and self.token != token): + raise ValueError("native_feedback_registration_mismatch") + self.token = token + + def _get(self, path, **kwargs): + response = self.session.get(self.judge_url + path, timeout=(3, 5), **kwargs) + response.raise_for_status() + return response.json() + + def poll(self): + if self.token is None: + raise ValueError("native_feedback_registration_missing") + history = self._get("/api/v1/history", params={"token": self.token}) + if history.get("run_id") != self.run_id or not isinstance(history.get("entries"), list): + raise ValueError("native_feedback_history_scope_mismatch") + entries = [row for row in history["entries"] if row.get("type") == "submission" + and isinstance(row.get("round"), str) and row["round"].startswith("agent-")] + ids = [row.get("submission_id") for row in entries] + rounds = [row.get("round") for row in entries] + if any(row.get("task_id") != self.task_id for row in entries): + raise ValueError("native_feedback_task_scope_mismatch") + records, artifacts = [], {} + for entry in entries: + sid, round_id, status = entry.get("submission_id"), entry.get("round"), entry.get("status") + if not _ID.fullmatch(sid or "") or not _ROUND.fullmatch(round_id or ""): + raise ValueError("native_feedback_identity_invalid") + row = {"accepted_record": True, "submission_id": sid, "round_id": round_id, + "status": status, "attribution_status": "unknown"} + records.append(row) + if ids.count(sid) != 1 or rounds.count(round_id) != 1: + row["accepted_record"] = False + row["reason"] = "ambiguous_native_identity" + continue + if status not in {"queued", "running", "completed", "error"}: + row["accepted_record"] = False + row["reason"] = "unknown_native_status" + continue + try: + source = self.trial / "submissions" / round_id / "submission.tar.gz" + raw = read_archive(source, root=self.trial) + digest = hashlib.sha256(raw).hexdigest() + self._bind_archive(sid, round_id, digest, len(raw)) + archive_name = f"{sid}-{digest}.tar.gz" + row.update(archive_sha256=digest, archive_bytes=len(raw), + archive_path=str(self.remote_root / "archives" / archive_name)) + artifacts["archives/" + archive_name] = raw + # Large/multi-file submissions remain usable for selected probe + # manifests. Do not confuse an absent full-manifest digest with + # an absent exact native archive binding. + try: + row["candidate_sha256"], _ = archive_manifest(raw) + except (EOFError, ValueError, gzip.BadGzipFile, tarfile.TarError, zlib.error): + pass + row["attribution_status"] = "verified" + if status not in {"completed", "error"}: + continue + result = self._get("/api/v1/result/" + sid) + self._validate_result(result, entry) + result_bytes = canonical_json(result) + result_sha = hashlib.sha256(result_bytes).hexdigest() + identity = dict(run_id=self.run_id, task_id=self.task_id, submission_id=sid, + round_id=round_id, archive_sha256=digest, result_sha256=result_sha) + ref = "native-result:" + hashlib.sha256(canonical_json(identity)).hexdigest() + result_name = f"{sid}-{result_sha}.json" + artifacts["results/" + result_name] = result_bytes + report = result.get("report") or {} + observed_at = self._observed_at.setdefault((sid, result_sha), + datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")) + row.update(result_ref=ref, result_sha256=result_sha, result_observed_at=observed_at, + result_path=str(self.remote_root / "results" / result_name), + result_summary={key: report.get(key) for key in ( + "score", "pass_rate", "valid", "passed", "failed", "total_tests")}) + except (OSError, ValueError, tarfile.TarError) as error: + row["attribution_status"] = "unknown" + row["reason"] = str(error) if isinstance(error, ValueError) else type(error).__name__ + row.pop("result_ref", None) + row.pop("result_sha256", None) + packet = {"schema_version": SCHEMA, "run_id": self.run_id, "task_id": self.task_id, + "records": records} + with self._lock: + self._artifacts, self._packet = artifacts, packet + return packet + + def _bind_archive(self, sid, round_id, digest, size): + """A native submission can never silently move to a different archive.""" + binding = dict(run_id=self.run_id, task_id=self.task_id, submission_id=sid, + round_id=round_id, archive_sha256=digest, archive_bytes=size) + path = self.directory / "bindings" / (sid + ".json") + try: + write_bytes_atomic(path, canonical_json(binding), create_only=True) + except FileExistsError: + if json.loads(path.read_bytes()) != binding: + raise ValueError("native_accepted_archive_changed") + + def _validate_result(self, result, entry): + if (not isinstance(result, dict) or result.get("submission_id") != entry["submission_id"] + or result.get("status") != entry["status"]): + raise ValueError("released_result_identity_mismatch") + report = result.get("report") + if entry["status"] == "error": + if report is not None: + raise ValueError("native_error_report_ambiguous") + return + if (not isinstance(report, dict) or result.get("error") is not None + or report.get("submission_id") != entry["submission_id"] + or report.get("task_id") != self.task_id): + raise ValueError("released_result_scope_mismatch") + if any(report.get(key) != entry.get(key) for key in ("score", "pass_rate", "valid")): + raise ValueError("released_result_history_mismatch") + + def update(self, backend, handle): + packet = self.poll() + encoded = canonical_json(packet) + revision = hashlib.sha256(encoded).hexdigest() + if revision == self._published: + return packet + self.directory.mkdir(parents=True, exist_ok=True) + for name, raw in {**self._artifacts, "native.json": encoded}.items(): + local = self.directory / name + local.parent.mkdir(parents=True, exist_ok=True) + write_bytes_atomic(local, raw) + # Copy to a fresh sibling, then atomically publish the complete file. + remote = self.remote_root / name + temporary = str(remote) + "." + uuid.uuid4().hex + ".tmp" + result = backend.exec_run(handle, ["mkdir", "-p", str(remote.parent)], user="root") + if result.exit_code: + raise RuntimeError("native_feedback_directory_failed") + backend.copy_to_container(handle, local, PurePosixPath(temporary)) + result = backend.exec_run(handle, ["chmod", "0644", temporary], user="root") + if result.exit_code: + raise RuntimeError("native_feedback_permissions_failed") + result = backend.exec_run(handle, ["mv", "-f", temporary, str(remote)], user="root") + if result.exit_code: + raise RuntimeError("native_feedback_publish_failed") + if self.after_publish is not None: + self.after_publish(backend, handle, packet) + self._published = revision + return packet + + def snapshot(self): + with self._lock: + return json.loads(canonical_json(self._packet)) if self._packet is not None else None + + def start(self, backend, handle): + if self._stop.is_set(): + raise RuntimeError("native_feedback_already_closed") + # Establish the scope packet before any worker submission can happen. + self.update(backend, handle) + self._active.set() + if self.thread is None: + self.thread = threading.Thread(target=self._loop, args=(backend, handle), daemon=True) + self.thread.start() + + def _loop(self, backend, handle): + while not self._stop.wait(self.poll_seconds): + if not self._active.is_set(): + continue + try: + self.update(backend, handle) + except Exception as error: + # Never log a native URL containing a token or raw feedback. + if self.logger: + self.logger.warning("Native research observation failed: %s", type(error).__name__) + + def pause(self): + self._active.clear() + + def close(self): + self._stop.set() + if self.thread is not None: + # Let a bounded consumer finish or kill its process group before + # the native runner removes the container and collects artifacts. + self.thread.join(timeout=90) + if self.thread.is_alive(): + raise RuntimeError("native_feedback_publisher_did_not_stop") + self.session.close() diff --git a/benchmark/edgebench/run.py b/benchmark/edgebench/run.py index f8ef59bf85..28f015013c 100644 --- a/benchmark/edgebench/run.py +++ b/benchmark/edgebench/run.py @@ -98,6 +98,7 @@ def _task_default(task_id: str, key: str, explicit: int | None, fallback: int) - def main(argv=None): + runner_started = time.monotonic() parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--task", required=True) parser.add_argument("--tasks-dir", required=True, type=Path) @@ -108,6 +109,8 @@ def main(argv=None): help="New-run default: native public feedback; blind/best-only are explicit controls") parser.add_argument("--turn-envelope", action="store_true", help="Opt-in short heartbeat context with same-invocation full captures") + parser.add_argument("--research-loop", action="store_true", + help="Fresh heartbeat-explore treatment: separate candidate/research work, request admission and execution evidence") parser.add_argument("--native-goal-continue", action="store_true", help="Opt-in: when a native Goal declares completion with budget left, start " "continuation Goals until the budget edge (native-goal worker only)") @@ -205,6 +208,15 @@ def main(argv=None): parser.error("Warm continuation requires full native checkpoint capture, no periodic evaluation and effective-Turn cadence") # One directory is one attempt: never reuse native registration or overwrite # source/profile evidence after an ambiguous launch. + if args.research_loop and (args.worker != "heartbeat-explore" or args.feedback != "native" + or args.restore_checkpoint is not None or not api_base_url): + parser.error("--research-loop requires a fresh heartbeat-explore native-feedback attempt and a custom model endpoint") + from benchmark.runtime.research_loop import research_stride_strategy + + try: + args.stride_strategy = research_stride_strategy(args.research_loop, args.stride_strategy) + except ValueError as error: + parser.error(str(error)) trial = args.log_dir / "runs" / args.run_id / args.task if args.restore_checkpoint is None: trial.mkdir(parents=True, exist_ok=False) @@ -290,6 +302,7 @@ def main(argv=None): timeout_seconds=args.timeout, feedback_prompt=feedback_prompt, feedback=args.feedback, workspace_instructions=workspace_instructions, task_objective=task_objective, + research_loop=args.research_loop, task_entry=args.task_entry, turn_envelope=args.turn_envelope, native_goal_continue=args.native_goal_continue, @@ -317,6 +330,14 @@ def main(argv=None): judge_url=args.judge_url.replace("host.docker.internal", "127.0.0.1"), admin_secret=get_admin_secret(args.log_dir), logger=logger, sampler=sampler, ) if args.feedback == "best-only" else None) + research_feedback = None + if args.research_loop: + from benchmark.edgebench.research_feedback import NativeResearchFeedback + from benchmark.runtime.research_feedback_sync import FEEDBACK_ROOT + + research_feedback = NativeResearchFeedback(trial=trial, run_id=args.run_id, task_id=args.task, + judge_url=args.judge_url.replace("host.docker.internal", "127.0.0.1"), + remote_root=FEEDBACK_ROOT, logger=logger) checkpoint = None if args.checkpoint_controller_file is not None: from benchmark.edgebench.checkpoint_supervisor import CheckpointSupervisor @@ -326,6 +347,7 @@ def main(argv=None): backend = RecordingDockerBackend(log_dir=trial / "collected", logger=logger, oauth_proxy=bool(args.api_proxy_url or api_base_url), blind_api_endpoint=blind_endpoint, feedback=feedback, + **({"research_feedback": research_feedback} if research_feedback is not None else {}), **({"checkpoint": checkpoint} if checkpoint is not None else {})) for image in (task.work_image_key, task.judge_image_key): if not backend.image_exists(image): @@ -372,6 +394,7 @@ def main(argv=None): if agent.replan_after_turns is not None else {}), **({"replan_after_completed_todos": agent.replan_after_todos} if agent.replan_after_todos is not None else {}), + **({"research_loop": True} if args.research_loop else {}), **({"planning_horizon_goal": args.planning_horizon_goal} if args.planning_horizon_goal is not None else {}), **({"stride_strategy": args.stride_strategy} @@ -399,6 +422,8 @@ def main(argv=None): finally: if feedback is not None: feedback.close() + if research_feedback is not None: + research_feedback.close() # Native cancellation and swallowed Docker failures return runtime=0. # Observe the signal independently; never infer cancellation from output prose. execution = backend.execution_receipt or {} @@ -415,6 +440,12 @@ def main(argv=None): _write_native_final_result(trial, result, status=status, agent=agent.name, task=task.task_id, run_id=args.run_id, model=args.model, effort=args.effort, feedback=args.feedback) + if args.research_loop: + from benchmark.edgebench.research_cost import summarize_research_cost + from benchmark.edgebench.research_feedback import canonical_json, write_bytes_atomic + + cost = summarize_research_cost(trial, time.monotonic() - runner_started) + write_bytes_atomic(trial / "research-cost.json", canonical_json(cost)) receipt_path.write_text(json.dumps(receipt, indent=2)) print(json.dumps(receipt)) return 0 if status == "terminal" else 130 if status == "cancelled" else 1 diff --git a/benchmark/runtime/RESEARCH_LOOP.md b/benchmark/runtime/RESEARCH_LOOP.md index dcb6cab911..f1d0e21141 100644 --- a/benchmark/runtime/RESEARCH_LOOP.md +++ b/benchmark/runtime/RESEARCH_LOOP.md @@ -1,24 +1,58 @@ -# Research stages and execution evidence - -The heartbeat runtime's explicit `research_loop=True` option retains the -original research objective while separating an initial-candidate Todo from -its deferred research successor. Omission preserves the previous treatment. -The runtime selects the existing `evidence_conditioned_v1` replan strategy and -rejects an explicitly conflicting strategy. The current request guard checks -ordinary Host admission at each actual provider send. A configured -Responses-compatible endpoint is required by that request boundary. - -The entry creates the two stages using existing `todo_done` and successor -semantics. Finishing the candidate releases real optimization work; it cannot -by itself close that work. Research retains the best candidate, meaningful -negative results and reasoned closeout against the original objective. User -Stop, the original budget and genuine blockers remain authoritative. -Preparation records scope through the ordinary Agent Vision writer as +# Coordinated research-loop treatment + +For a **fresh** EdgeBench `heartbeat-explore` attempt with native public feedback +and a custom Responses-compatible endpoint, `--research-loop` enables one +coordinated treatment. Omission preserves the previous treatment. The runner +rejects adding it to an older checkpoint. Runtime, worker and export receipts +retain the flag; do not compare mixed treatments as identical configurations. +Roll back by omitting the flag on a new attempt. Retained findings containing +the new optional observation require a supporting reader; do not rewrite old +history or claim that an older strict event reader understands the new field. +The treatment selects the existing `evidence_conditioned_v1` replan strategy; +an explicitly conflicting strategy is rejected. This is a combined treatment, +not an isolated estimate of the contribution of any one component. + +The entry creates an initial-candidate Todo and a separate deferred research +Todo using existing `todo_done`/successor semantics. Finishing the candidate +releases real optimization work; it cannot by itself close that work. The +original task remains the Goal objective. Research retains the best candidate, +meaningful negative results and reasoned closeout against the original objective. +User Stop, the original budget and genuine blockers remain authoritative. There +is no requirement to fill time or change the native scoring/submission rules. +Preparation records this scope through the ordinary Agent Vision writer as `surface_only`, without claiming research progress or spending a work Turn. -Existing research work is not reopened or its later Vision overwritten. +Existing prepared research work is not reopened or its later Vision overwritten. + +The worker reuses the live Host launch owner before **each** provider request, +including requests inside one long native wake. Current wait/Stop/refusal means +no upstream send and interruption of the owned native process; the original +scheduler handles wait/recovery. Checkpoint continuations retain the held Turn. +Unreadable authority fails closed. Already-forwarded requests cannot be +retroactively cancelled, and absent canonical waits are not inferred from prose. +Connection establishment (including a forward proxy's CONNECT and TLS) happens +outside the authority lock. The original owner/process is pinned before that +connection; its binding, liveness, current control and phase deadline are checked +again immediately before sending. A delayed old request cannot borrow a newer +process or Turn, and a slow connection does not prevent a concurrent Stop write. +For guarded requests, headers and body share a one-second send deadline (or +the caller's shorter timeout); response reads retain their original timeout. +An incomplete send releases the authority lock and remains ambiguous. Refusing +an obsolete binding interrupts only its captured process; it cannot mark a +valid replacement binding refused. A current binding's refusal still requires +the existing recovery path. + +Private `provider-requests.jsonl` correlates wake, LoopX Turn, native +session/thread/Turn when supplied, client request, gateway request and returned +upstream request identities. A dispatch without a finished record is unknown; +`forwarded=null` preserves ambiguity during a failed send. Missing identifiers +remain null. The journal excludes bodies, credentials and upstream endpoints. +Qualify the pinned client at its request boundary. Admission adds control cost; +include it in subsequent matched comparisons instead of promising token savings. +The adapter retains the declared HTTP(S) forward-proxy route and `NO_PROXY` +exclusions; it does not bypass an isolated task's model egress boundary. -The installed `loopx-research-probe` uses the staged runner/Python. The same -producer can be called as `python -m benchmark.runtime.research_probe`: +The initial work context requests a local probe before first submission. The +installed `loopx-research-probe` uses the staged runner/Python: ```sh loopx-research-probe --candidate-file candidate.py --output probe-001.json \ @@ -31,39 +65,180 @@ loopx-research-probe --candidate-file candidate.py --output probe-001.json \ The solver may write `{"search_iterations":2,"stop_reason":"search_cap"}` to `LOOPX_RESEARCH_DIAGNOSTICS`. The host measures elapsed time, exit/timeout and the selected candidate-file digest before/after. A changed candidate rejects -attribution. Solver counters and reasons remain reports; missing counters -remain null. Low resource use does not impose a mandatory time floor. -Malformed optional diagnostics preserve reliable host measurements and retain -a bounded diagnostic-error receipt. Evidence paths must be fresh; raw output -stays local. The probe neither submits nor accesses hidden evaluation. - -The adjacent `.candidate.json` preserves selected relative paths and hashes. -`candidate_sha256` hashes the sorted JSON file manifest, not native archive -container bytes. Pins cover only declared files at the probe boundaries, not -transitive imports or independent writers. Result attribution must still prove -that the evaluated snapshot matches the selected candidate and input revision. - -Attach the emitted `execution_observation` through normal Explore result -writeback. Its input revision and evidence refs must belong to the enclosing -finding; released results must identify the candidate actually evaluated. -Different planned/executed method ids require a change reason. Findings, -replan and research-history details retain this same source. They do not -certify method benefit or Goal completion. Missing observations preserve old -events; a reader for the optional field is required for new retained findings. - -Result followthrough uses existing material delivery classifications. -`surface_only` and unknown historical outcomes remain readable but cannot -consume a cited work result or create a new unresolved experiment. Useful -negative results and concrete blockers remain eligible evidence. +attribution. Solver counters/reasons remain reports, missing counters remain +null, and low resource use is a diagnostic rather than a mandatory time floor. +Pins cover only declared files at those boundaries, not transitive imports or +independent writers. Evidence paths must be fresh; raw output stays local. +The probe neither submits nor accesses hidden evaluation. +Malformed optional solver diagnostics leave host elapsed/exit evidence intact; +their counters stay unknown and a bounded diagnostic-error receipt explains why. + +The adjacent `.candidate.json` preserves the selected relative paths and file +hashes. `candidate_sha256` hashes that sorted JSON file manifest (sorted keys, +compact separators), not a native archive's container bytes. Compare the same +manifest against the submitted/released snapshot before attaching feedback; +an archive SHA-256 is not interchangeable. The native adapter performs this +comparison automatically for registered probes, including selected files within +an archive containing more than 32 files. A missing match remains unknown. + +Attach the emitted object as `execution_observation` in the existing Explore +result writeback for the model's own local findings. Preserve its input revision +and evidence refs. On native EdgeBench, the installed probe also registers its +manifest with the native result producer. Released native feedback is ingested +automatically with `result_ref` and, only when an accepted snapshot matches, +`result_candidate_sha256`. Late feedback belongs to that older candidate. +Automatic binding covers the registered probes available when that native +observation completes. Later probes use ordinary local Explore writeback; a +completed observation is not reopened to ingest revised results or new probes. +Different planned/executed method ids require a change reason. Replan and the +research-history page read the same Explore source; neither certifies method +benefit or Goal completion. + +## Native acceptance and recovery + +The research adapter instruments the installed native `sforge-submit` at its +actual packaging and HTTP-send boundaries. It preserves the native tar command, +submission limits, evaluation and scoring. An unsupported native script layout +or a pinned product lacking the matching Explore observer capability fails +installation before model entry. If product and runner revisions differ, pin +both compatible versions; the runner overlay does not replace product owners. +The host observes only public token-visible +history/results and the native accepted source archive. It does not initiate an +evaluation or read hidden judge artifacts. + +Each submit invocation journals an operation ID and immutable packaged bytes +before its single POST. A successful native response binds the returned +submission ID and round; the host then verifies their accepted archive digest. +The archive SHA-256, selected-file manifest and result digest remain distinct +linked facts. A completed/error native result is usable only after its public +identity, task, status and history agree. Duplicate identities, replaced archives +or ambiguous recovery retain unknown attribution without blocking unrelated +verified submissions. The first submitted candidate is ordered by native +acceptance (`agent-N`), not by when evaluations finish. + +If the submission response disappears, rerunning `sforge-submit` first recovers +the unresolved operation from its journal and native history. Once it uniquely +matches the accepted archive, the helper reads the original submission ID even +if the workspace has changed. It never posts that unresolved operation again. +An unresolved acceptance stops that retry with an explicit unknown reason; +independent research may continue. Receiving external feedback does not pretend +that the interrupted native client observed its result or completed its operation. +The supported native provider's two exact capacity-full 503 responses are +pre-admission refusals: they consume neither a round nor a submission. Those +operations retain their snapshots but are marked rejected, allowing a later +explicit submit invocation. Other 5xx, unrecognized 503 responses and lost +responses remain unknown; the response hook never automatically resubmits. +If the retained archive and result identities are verified but the archive +cannot be parsed, its native error remains a generic finding without a bound +probe manifest. That record does not prevent unrelated verified results from +being ingested. + +## Explicit feedback dependencies and wakeup + +When the current work truly needs the external result, use its canonical Todo ID: + +```sh +sforge-submit --wait-todo TODO_ID +# Or attach a dependency after an already accepted submission: +loopx-research-feedback wait --todo-id TODO_ID --submission-id SUBMISSION_ID +``` + +Pending evaluation by itself does not defer a model Todo. The explicit option +registers exactly that work's dependency as soon as the native accepted receipt +is available. Existing different waits, completed work, claims and original +operation identities are preserved. The finite observation Todo belongs to the +runner's `edgebench-feedback-observer`; model agents cannot claim it as work. +This is real low-cost external observation through the existing Todo owner. +An independent model Todo remains eligible while one evaluation is pending. + +The runner polls public native history without invoking a model. Released facts +go through the existing Explore attachment codec, append-only event log and +canonical Todo linking API. This external observer entry does not create a model +Turn, spend a slot, acknowledge replan or claim research progress. Only durable +finding/link readback permits completion of that observation dependency. Partial +write/copy failures retry with the same source identity. Negative scores and +native error results remain evidence; neither automatically marks research blocked. +Consumption runs under a 60-second process-group timeout with bounded cleanup; +durable results can release independent work even if a later item needs retry. + +Consumers serialize ingestion with a nonblocking local lock; a busy consumer +leaves retry to the existing runner. The revision signal is derived from the +current canonical completed observation Todos, rather than the caller's possibly +older feedback packet. Each successful release publishes it immediately, and +replay reconstructs it if completion persisted before notification failed. +An old packet cannot replace a newer completion signal, and unchanged completed +observations do not create fresh wakeups. + +The opt-in scheduler wrapper +observes that small file during its ordinary sleep and promptly returns to the +same scheduler's quota check. Repeated or invalid signals do not busy-loop, and a +signal alone grants no model admission. When all model work is waiting, the +per-request guard prevents upstream sends. Resumption uses current evidence and +the existing owner of the original operation and budget. The wrapper does not +change Turn IDs, extend phase deadlines or resubmit candidates. Ordinary newly +admitted successor work can still have its own normal work Turn. + +## Measurement and checkpoint boundary + +The passive `research-cost.json` receipt records runner wall time, observed +provider requests and native evaluation counts, plus captured Codex token +coverage. Token accounting uses the existing complete-session counter owner; +identical/prefix copies of one session are deduplicated and incompatible, +forked, incomplete or missing captures remain unknown. Cached input is included +in input, and token counts do not imply an invoice. Request and session coverage +are separate: a missing journal or unfinished request cannot be interpreted as +zero cost. Known subtotals and unknown coverage are explicit. + +Runner elapsed time includes its preparation, planning, control checks, waiting, +probes and checkpoint activity inside that measured interval. These overlapping +durations are not added together. Detailed phase allocation and costs incurred +before runner entry are unknown. Compare candidate quality against complete +observed costs; do not infer benefit from more Turns, Todos or reports. + +This change does not qualify checkpoint restoration of the new native journals. +Cold runs remain the supported entry; applying `--research-loop` to an older +checkpoint is rejected. Before restoring or forking a research attempt, capture +the native submission/probe/ingestion journals with the existing checkpoint +writer barrier and verify original submission identity, available evidence, +budget and the first resumed experiment. Decisions or feedback created after +capture must not be indexed as capture-time knowledge; preserve the existing +`research_point_qualified=false` until that qualification is actually performed. + +This treatment does not automatically instrument arbitrary solvers or guarantee +a stronger first score. The model may omit a probe or misreport a method; actual +adoption and benefit still require qualification. Stage separation does not +replace the default whole-Goal acceptance owner. Do not extend budget or suppress +legal closeout to manufacture a successful long run. ## 中文交付边界 -显式研究入口保留原 Goal，把初始候选与后续研究表示成既有 Todo 和后继依赖。 -候选完成只释放后续研究，准备阶段使用 `surface_only`，不会声称已经取得收益。 -Stop、原预算和真实阻塞保持有效，不要求填满时长或改变原生评测规则。 +这一显式处理将初始候选、持续研究、等待准入、执行证据与复盘接在既有 +Goal/Todo/Host/Explore 上。候选完成只释放后续研究；本次没有包含默认全 Goal +验收工作线，也没有重新启动历史运行或长测。 + +原生真正打包后保存操作 ID 和不可变快照；接受提交后读取真实提交 ID、轮次和 +原生归档。归档哈希与所选文件清单哈希分别保存并验证关系。结果晚到时始终指向 +原候选，不改写新工作区或自动采用、回滚。提交响应丢失时恢复原操作，无法唯一 +匹配时保留未知，不重复消耗评测机会。首次候选按实际接受顺序评价。 +支持的原生 provider 在受理前返回的两种明确容量 503 会记为拒绝，保留快照并允许 +稍后显式提交；其他 5xx、无法识别的 503 与丢失响应仍为未知，不自动重发。 +原始归档与结果身份可靠但归档无法解析时，保留未绑定探针的通用错误 finding， +不阻断独立正常结果，也不声称坏归档具有已验证的候选文件清单。 + +只有显式 `--wait-todo` 或反馈依赖命令才延后指定 Todo。运行器负责真实外部观察， +先写入 Explore 并读回，再完成观察依赖；其自身不占模型 Turn 或预算。 +全部可执行工作等待时停止模型请求；有独立工作时保持准入。新反馈信号打断 +调度休眠后仍需通过当前准入，不通过换 Turn、重发操作或重置预算逃出等待。 +准入锁前后的原进程、绑定和截止时间都校验，慢连接也不会占住 Stop 的写锁。 +带准入保护的请求头与请求体共用一秒发送期限（调用者设置更短时沿用更短值）， +响应读取仍用原超时；发送不完整会释放锁并保留未知转发状态。旧绑定的拒绝只影响 +旧进程，不能污染合法替换进程；当前绑定的真实拒绝仍须走原恢复路径。 +反馈消费者非阻塞互斥，繁忙时由原运行器稍后重试。唤醒版本来自当前 canonical +已完成观察 Todo，独立结果完成后立即发布；完成已落盘但通知失败时可重建补发， +旧包不能覆盖新完成信号，重复消费也不会持续制造唤醒。 -本地探针固定所选文件，记录真实耗时、退出与超时状态；可选 solver 报告与宿主 -测量分开，坏诊断不能抹去可靠测量。所选文件清单哈希不是原生归档哈希。 -证据通过既有 Explore 写回并进入 replan 和研究历程；缺失测量保持未知，负结果 -继续可用。该切片交付本地执行证据与阶段接续，原生提交、自动反馈依赖和成本 -归约由后续组合切片接入。工程回归不证明首次分数或长期收益已经改善。 +探针保留实际耗时、退出状态及可选 solver 报告；坏诊断不能抹掉可靠测量。 +完整成本先提供已知总时长、去重计数和明确的未知覆盖，阶段分摊不伪造为零。 +检查点恢复仍需在准备恢复／分叉时补齐并验收，不能把保存成功当作恢复资格。 +有界 CLI、HTTP、真实本地进程验证只证明工程链路；模型是否采用诊断、首次质量 +是否不低于 native、能否超越平台期，仍需冻结新版本后的真实实验与合适对照。 diff --git a/benchmark/runtime/harbor.py b/benchmark/runtime/harbor.py index c17c7afd62..8859bf8ddf 100644 --- a/benchmark/runtime/harbor.py +++ b/benchmark/runtime/harbor.py @@ -74,6 +74,7 @@ def __init__( native_goal_continue=False, task_objective=None, research_loop=False, + research_feedback_root=None, goal_id=_GOAL_ID, agent_id=_AGENT_ID, defer_codex_home=False, @@ -129,6 +130,9 @@ def __init__( if not isinstance(research_loop, bool) or (research_loop and self.execution.mode != "heartbeat"): raise ValueError("research_loop requires heartbeat execution") self.research_loop = research_loop + if research_feedback_root is not None and (not research_loop or not str(research_feedback_root).startswith("/")): + raise ValueError("native research feedback requires an enabled treatment and absolute staged root") + self.research_feedback_root = research_feedback_root self.checkpoint_request_path = None self._checkpoint_declaration = None if checkpoint_request is not None: @@ -205,7 +209,12 @@ async def _stage_source(self, environment: BaseEnvironment, source: Path) -> str if runner != source: # Only research runtime code is overlaid; LoopX product code # and its installer remain exactly at the product revision. - archive_source(runner, runner_head, archive, ("benchmark/runtime",)) + overlay = ("benchmark/runtime",) + if self.research_feedback_root is not None: + # The native submission producer is a pinned EdgeBench + # adapter, not product scoring or control-plane policy. + overlay += ("benchmark/edgebench/research_feedback.py",) + archive_source(runner, runner_head, archive, overlay) await environment.upload_file(archive, f"{_ROOT}/runner.tar") await self.exec_as_root(environment, command=( f"tar -xf {_ROOT}/runner.tar -C {_SRC} && rm {_ROOT}/runner.tar" @@ -335,11 +344,19 @@ async def install(self, environment: BaseEnvironment) -> None: if doctor_payload.get("ok") is not True: raise RuntimeError(f"LoopX doctor failed: {doctor_payload}") if self.research_loop: - wrapper = (f"#!/bin/sh\nexec env PYTHONPATH={shlex.quote(_SRC)} " + feedback_env = (f"LOOPX_RESEARCH_FEEDBACK_ROOT={shlex.quote(self.research_feedback_root)} " + if self.research_feedback_root is not None else "") + wrapper = (f"#!/bin/sh\nexec env PYTHONPATH={shlex.quote(_SRC)} {feedback_env}" f"{_PYTHON}/bin/python3 -m benchmark.runtime.research_probe \"$@\"\n") await self.exec_as_agent(environment, command=f"printf %s {shlex.quote(wrapper)} > {_PROFILE}/bin/loopx-research-probe; " f"chmod 0755 {_PROFILE}/bin/loopx-research-probe", env=self._profile_env()) + if self.research_feedback_root is not None: + wrapper = (f"#!/bin/sh\nexec env PYTHONPATH={shlex.quote(_SRC)} {feedback_env}" + f"{_PYTHON}/bin/python3 -m benchmark.runtime.research_feedback_wait \"$@\"\n") + await self.exec_as_agent(environment, + command=f"printf %s {shlex.quote(wrapper)} > {_PROFILE}/bin/loopx-research-feedback; " + f"chmod 0755 {_PROFILE}/bin/loopx-research-feedback", env=self._profile_env()) receipt = { "loopx_commit": actual_commit, @@ -477,6 +494,9 @@ async def _prepare_phase( self.goal_id, "--registered-agent", self.agent_id, + *(["--registered-agent", "edgebench-feedback-observer", + "--agent-work-mode", "edgebench-feedback-observer=active"] + if self.research_feedback_root is not None else []), "--boundary-authority-scope", "**", "--boundary-authority-source", @@ -608,6 +628,9 @@ def _worker_env(self, *, cwd: str) -> dict[str, str]: "LOOPX_EXECUTION_MODE": self.execution.mode, "LOOPX_TASK_ENTRY": self.execution.task_entry, **({"LOOPX_RESEARCH_LOOP": "1"} if self.research_loop else {}), + **({"LOOPX_RESEARCH_FEEDBACK_ROOT": self.research_feedback_root, + "LOOPX_RESEARCH_FEEDBACK_AGENT": "edgebench-feedback-observer"} + if self.research_feedback_root is not None else {}), **({"LOOPX_CHECKPOINT_REQUEST": self.checkpoint_request_path} if self.checkpoint_request_path is not None else {}), **({"LOOPX_TURN_ENVELOPE": "1"} if self.execution.turn_envelope else {}), diff --git a/benchmark/runtime/research_feedback_sync.py b/benchmark/runtime/research_feedback_sync.py new file mode 100644 index 0000000000..f260729837 --- /dev/null +++ b/benchmark/runtime/research_feedback_sync.py @@ -0,0 +1,204 @@ +"""Deliver native feedback through ordinary Explore writeback before Todo resume. + +This is the EdgeBench producer adapter. Explore owns its attachment codec and +history, Todo owns waiting, and the current Host owns subsequent admission. +No score comparison, candidate adoption or research completion is inferred here. +""" +from __future__ import annotations + +import argparse +from contextlib import contextmanager +import fcntl +import hashlib +import json +import os +from pathlib import Path + +from .layout import _CONTROL + + +FEEDBACK_ROOT = f"{_CONTROL}/native-feedback" +_OWNER_ENV_KEYS = frozenset({"LOOPX_CLI", "LOOPX_REGISTRY", "LOOPX_RUNTIME_ROOT", "LOOPX_GOAL_ID", + "LOOPX_AGENT_ID", "LOOPX_PROJECT", "LOOPX_RESEARCH_FEEDBACK_ROOT", + "LOOPX_RESEARCH_FEEDBACK_AGENT", "LOOPX_PYTHON", "LOOPX_USAGE_PING", "PATH", "PYTHONPATH"}) + + +def feedback_owner_environment(env): + """Only control coordinates and staged tooling enter the installed config.""" + return {key: value for key, value in env.items() if key in _OWNER_ENV_KEYS} + + +def feedback_environment(root: Path, base=None): + """Use the installed, credential-free owner when a tool filters its env.""" + env = dict(os.environ if base is None else base) + owner = json.loads((root / "owner.json").read_text(encoding="utf-8")) + if (not isinstance(owner, dict) or set(owner) - _OWNER_ENV_KEYS + or any(not isinstance(value, str) for value in owner.values()) + or owner.get("LOOPX_RESEARCH_FEEDBACK_ROOT") != str(root)): + raise ValueError("native feedback owner configuration mismatch") + env.update(owner) + return env + + +def _attachment(packet, record, *, observation=None, probe_ref=None): + # Stable native identity survives delayed feedback and a lost writeback + # response. Identity includes the exact released result digest, never the + # current workspace. A closed native observation is not silently revised. + source = [packet["run_id"], packet["task_id"], record["submission_id"], + record["round_id"], record["archive_sha256"], record["result_sha256"], probe_ref] + identity = hashlib.sha256(json.dumps(source, separators=(",", ":")).encode()).hexdigest() + evidence = [record["result_ref"], "native-archive:" + record["archive_sha256"]] + summary = {"status": record["status"], **record.get("result_summary", {})} + attachment = { + "schema_version": "explore_result_attachment_v0", + "node_id": "native-feedback-" + identity[:32], + "question": "What did the released native evaluation report for this accepted snapshot?", + "applicability": "One native accepted snapshot; local probe measurements cover only their declared files and input.", + "input_revision": "native-submission:" + identity[:32], + "observation": json.dumps(summary, sort_keys=True, allow_nan=False), + "interpretation": "The result belongs to the archived submission. It does not describe newer workspace files. " + "Candidate adoption, rollback and the next experiment require the research owner's separate decision; " + "negative or error results remain evidence, not automatic research failure.", + "status": "tentative", + "evidence_refs": evidence, + } + if observation is not None: + attachment["execution_observation"] = observation + attachment["input_revision"] = observation["input_revision"] + attachment["evidence_refs"] = sorted(set([*evidence, *observation["evidence_refs"]])) + return identity, attachment + + +def write_feedback_finding(env, packet, record, todo_id): + """Write an actual observer result; never settle the model's original Turn. + +The caller completes the finite observation Todo only after this readback. Each +attachment has a deterministic ordinary writeback identity, so a partial copy, +lost response or repeated native history read can finish without a second result. +""" + from .research_feedback_wait import FEEDBACK_OBSERVER_AGENT + from .research_submission import bind_execution_observations + from benchmark.edgebench.research_feedback import write_bytes_atomic + from loopx.capabilities.explore.result_writeback import record_observer_result_attachment + + root = Path(env["LOOPX_RESEARCH_FEEDBACK_ROOT"]) + attached = [row for row in bind_execution_observations(root, packet) + if row["submission_id"] == record["submission_id"]] + variants = [(None, None), *[(row["execution_observation"], row["probe_ref"]) for row in attached]] + receipts = [] + for observation, probe_ref in variants: + identity, attachment = _attachment(packet, record, observation=observation, probe_ref=probe_ref) + directory = root / "ingestion" + directory.mkdir(parents=True, exist_ok=True) + path = directory / (identity + ".json") + encoded = json.dumps(attachment, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n" + try: + write_bytes_atomic(path, encoded.encode(), create_only=True) + except FileExistsError: + if path.read_text(encoding="utf-8") != encoded: + raise ValueError("native feedback ingestion identity changed") + # Native observation is external evidence, not a model Turn. The + # capability's ordinary log/link owner records it without a synthetic + # quota receipt, stride progress or acceptance declaration. + result = record_observer_result_attachment(attachment, + registry_path=Path(env["LOOPX_REGISTRY"]), runtime_root=Path(env["LOOPX_RUNTIME_ROOT"]), + goal_id=env["LOOPX_GOAL_ID"], + agent_id=env.get("LOOPX_RESEARCH_FEEDBACK_AGENT", FEEDBACK_OBSERVER_AGENT), + todo_id=todo_id, source_id="native-feedback:" + identity, + recorded_at=record["result_observed_at"]) + if result.get("ok") is not True: + raise RuntimeError("native feedback Explore writeback is incomplete") + receipts.append({"observation_id": identity, "written": True}) + return {"submission_id": record["submission_id"], "findings": receipts} + + +@contextmanager +def _consumer_lock(root): + directory = root / "ingestion" + directory.mkdir(parents=True, exist_ok=True) + with (directory / "consumer.lock").open("a") as lock: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + # The runner retries incomplete consumption. Never wait behind a + # second consumer while holding any Goal/provider authority lock. + raise RuntimeError("native_feedback_consumer_busy") from None + yield + + +def _publish_feedback_wake(env, root): + from .research_feedback_wait import _is_feedback_todo, _loopx, _scope + from benchmark.edgebench.research_feedback import canonical_json, write_bytes_atomic + + goal, agent, observer = _scope(env) + listed = _loopx(env, "todo", "list", "--goal-id", goal, "--role", "agent") + completed = sorted((todo["target_key"], todo["todo_id"]) for todo in listed["todos"] + if todo.get("status") == "done" and _is_feedback_todo(todo, observer, agent)) + # Current canonical completions, including ones absent from an older + # packet, own this hint. No publication journal or new admission authority + # is needed to recover a crash between Todo completion and notification. + revision = hashlib.sha256(canonical_json(completed)).hexdigest() + encoded = canonical_json({"revision": revision}) + path = root / "ingestion" / "wake.json" + try: + with path.open("rb") as existing: + if existing.read(len(encoded) + 1) == encoded: + return revision + except FileNotFoundError: + pass + write_bytes_atomic(path, encoded) + return revision + + +def synchronize_native_feedback(env, *, packet=None): + """Consume runner/manual feedback once, then hint a fresh scheduler read.""" + from .research_feedback_wait import synchronize_feedback_waits, wait_for_feedback + from .research_submission import declared_wait_packet, read_packet, synchronize_intents + + root = Path(env["LOOPX_RESEARCH_FEEDBACK_ROOT"]) + # Lock order: consumer, the short submission journal transaction, then + # individual ordinary Todo/Explore calls. No provider lock encloses this. + with _consumer_lock(root): + packet = read_packet(root) if packet is None else packet + intents = synchronize_intents(root, packet) + + def released(_observed): + # Publish each durable success before another result can fail. + _publish_feedback_wake(env, root) + + try: + result = synchronize_feedback_waits(env, packet, after_release=released) + waiting = [] + for intent in intents: + if (intent.get("wait_todo_id") and intent.get("acceptance") == "accepted" + and intent.get("submission_id") and not intent.get("result_observed")): + declaration = declared_wait_packet(root, intent) + current = [row for row in declaration["records"] + if row.get("submission_id") == intent["submission_id"]] + if (len(current) == 1 and current[0].get("status") in {"completed", "error"} + and current[0].get("attribution_status") == "verified" + and current[0].get("result_ref")): + # Released evidence must not re-defer completed or + # repurposed work after a lost native result receipt. + continue + waiting.append(wait_for_feedback(env, todo_id=intent["wait_todo_id"], + submission_id=intent["submission_id"], packet=declaration)) + finally: + # Repair a lost notification even when its Todo was already done + # before this retry, or a later independent result still fails. + revision = _publish_feedback_wake(env, root) + return {**result, "waits": waiting, "revision": revision} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, + default=os.environ.get("LOOPX_RESEARCH_FEEDBACK_ROOT", FEEDBACK_ROOT)) + args = parser.parse_args(argv) + env = feedback_environment(args.root) + print(json.dumps(synchronize_native_feedback(env), allow_nan=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/runtime/research_feedback_wait.py b/benchmark/runtime/research_feedback_wait.py new file mode 100644 index 0000000000..643db04d00 --- /dev/null +++ b/benchmark/runtime/research_feedback_wait.py @@ -0,0 +1,236 @@ +"""Adapt an explicit native-feedback dependency to the existing Todo owner. + +The native observer owns accepted submission/feedback facts. This transport +only creates its finite observation Todo and defers the caller-selected work. +It never infers a wait from prose, an unchanged score, or another pending job. +The regular quota and Host owners retain all admission and resume decisions. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess + +from .layout import _CONTROL + + +FEEDBACK_OBSERVER_AGENT = "edgebench-feedback-observer" + + +def _loopx(env, *args): + from .worker import loopx_command + + result = subprocess.run(loopx_command(env) + list(args), cwd=env["LOOPX_PROJECT"], + env=env, capture_output=True, text=True, timeout=120, check=False) + try: + payload = json.loads(result.stdout) + except (ValueError, TypeError): + raise RuntimeError("native feedback Todo command returned no structured result") from None + if result.returncode or not isinstance(payload, dict) or payload.get("ok") is False: + # Native results, credentials and host-local paths do not enter errors. + code = payload.get("error_code", "todo_command_failed") if isinstance(payload, dict) else "todo_command_failed" + raise RuntimeError("native feedback Todo command rejected: " + str(code)) + return payload + + +def _identity(value, label): + if not isinstance(value, str) or not 1 <= len(value) <= 256 or any(ord(char) < 32 for char in value): + raise ValueError("native feedback requires a bounded " + label) + return value + + +def _records(packet): + if not isinstance(packet, dict) or packet.get("schema_version") != "edgebench_native_feedback_v1": + raise ValueError("native feedback packet schema mismatch") + run, task = (_identity(packet.get(key), key) for key in ("run_id", "task_id")) + if not isinstance(packet.get("records"), list): + raise ValueError("native feedback records are unavailable") + accepted = [record for record in packet["records"] + if isinstance(record, dict) and record.get("accepted_record") is True] + identities = [record.get("submission_id") for record in accepted] + records = {} + for record in accepted: + submission = _identity(record.get("submission_id"), "submission_id") + round_id = _identity(record.get("round_id"), "round_id") + if identities.count(submission) != 1 or record.get("status") not in {"queued", "running", "completed", "error"}: + continue # One unresolved native identity cannot suppress independent feedback. + key = hashlib.sha256(json.dumps([run, task, submission, round_id], separators=(",", ":")).encode()).hexdigest() + records[submission] = ("native-feedback:" + key[:32], record) + return records + + +def _released(record): + if record.get("status") not in {"completed", "error"} or record.get("attribution_status") != "verified": + return False + if not record.get("result_ref"): + return False + _identity(record["result_ref"], "result_ref") + # Native archive/result identity proves released feedback even when no + # optional selected-file probe manifest covers that submitted archive. + for field in ("archive_sha256", "result_sha256"): + if not isinstance(record.get(field), str) or not re.fullmatch(r"[0-9a-f]{64}", record[field]): + raise ValueError("released native feedback is missing its verified " + field) + return True + + +def _scope(env): + goal = _identity(env.get("LOOPX_GOAL_ID"), "goal_id") + agent = _identity(env.get("LOOPX_AGENT_ID"), "agent_id") + observer = _identity(env.get("LOOPX_RESEARCH_FEEDBACK_AGENT", FEEDBACK_OBSERVER_AGENT), "observer_agent_id") + if agent == observer: + raise ValueError("native feedback observation requires its separate runner owner") + return goal, agent, observer + + +def _is_feedback_todo(todo, observer, agent): + key = todo.get("target_key") + return (isinstance(key, str) and re.fullmatch(r"native-feedback:[0-9a-f]{32}", key) is not None + and todo.get("role") == "agent" and todo.get("claimed_by") == observer + and todo.get("task_class") == "advancement_task" and todo.get("action_kind") == "observe" + and agent in (todo.get("excluded_agents") or [])) + + +def _dependency(todos, key, observer, agent): + matches = [todo for todo in todos if todo.get("target_key") == key] + if len(matches) > 1: + raise ValueError("native feedback Todo identity is ambiguous") + if not matches: + return None + todo = matches[0] + if not _is_feedback_todo(todo, observer, agent): + raise ValueError("native feedback Todo belongs to another operation") + return todo + + +def _create_feedback_todo(env, *, goal, agent, observer, key, record): + created = _loopx(env, "todo", "add", "--goal-id", goal, "--role", "agent", + "--claimed-by", observer, "--excluded-agent", agent, "--task-class", "advancement_task", + "--action-kind", "observe", "--target-key", key, "--operation-id", key + ":observe", + "--text", "Receive released feedback for native submission " + record["submission_id"] + + " (" + key + ").", + "--execute") + return _loopx(env, "todo", "list", "--goal-id", goal, "--todo-id", created["todo_id"])["todo"] + + +def wait_for_feedback(env, *, todo_id, submission_id, packet): + """Declare one exact dependency, preserving unrelated work and its owner. + + Submission/recovery must have returned an accepted native identity before + this call. A missing or ambiguous response is not authority to guess one. + Reentry reads canonical state; it neither resubmits nor changes Turn budget. + """ + goal, agent, observer = _scope(env) + selected = _records(packet).get(submission_id) + if selected is None: + raise ValueError("native feedback wait requires one accepted submission; its identity is absent or ambiguous") + key, record = selected + binding = ("--goal-id", goal) + listed = _loopx(env, "todo", "list", *binding, "--role", "agent") + source = _dependency(listed["todos"], key, observer, agent) + waiting = next((todo for todo in listed["todos"] if todo.get("todo_id") == todo_id), None) + if (waiting is None or waiting.get("role") != "agent" or waiting.get("claimed_by") != agent + or waiting.get("task_class") != "advancement_task"): + raise ValueError("native feedback wait requires the caller's advancement Todo") + if _released(record): + # A delayed/replayed availability notice never reopens a completed Todo + # or replaces a newer wait; the current control owner still decides it. + return {"waiting": False, "todo_id": todo_id, "submission_id": submission_id, + "result_ref": record["result_ref"]} + if waiting.get("status") not in {"open", "deferred"}: + raise ValueError("native feedback wait cannot reopen completed or blocked work") + resume = "todo_done:" + source["todo_id"] if source else None + if waiting.get("resume_when") and waiting["resume_when"] != resume: + raise ValueError("native feedback wait cannot replace another resume condition") + if waiting.get("status") == "deferred" and (source is None or waiting.get("resume_when") != resume): + raise ValueError("native feedback wait cannot replace an existing deferral") + if source is None: + source = _create_feedback_todo(env, goal=goal, agent=agent, observer=observer, key=key, record=record) + resume = "todo_done:" + source["todo_id"] + # Observe again after source creation: feedback or an owner edit may have + # committed meanwhile. The public CAS binds deferral to this fresh source. + current = _loopx(env, "todo", "list", *binding, "--role", "agent") + source = _dependency(current["todos"], key, observer, agent) + waiting = next(todo for todo in current["todos"] if todo.get("todo_id") == todo_id) + if source["status"] == "done": + return {"waiting": False, "todo_id": todo_id, "submission_id": submission_id, + "dependency_todo_id": source["todo_id"]} + if source["status"] != "open": + raise ValueError("native feedback observation was paused or cancelled") + if waiting.get("status") == "deferred" and waiting.get("resume_when") == resume: + return {"waiting": True, "todo_id": todo_id, "submission_id": submission_id, + "dependency_todo_id": source["todo_id"], "resume_when": resume} + if waiting.get("status") != "open" or waiting.get("resume_when") or waiting.get("claimed_by") != agent: + raise ValueError("native feedback work changed before wait registration") + revision = current["authority_read"]["provider_revision"] + operation = key + ":wait:" + hashlib.sha256((todo_id + "\0" + revision).encode()).hexdigest()[:24] + _loopx(env, "todo", "update", *binding, "--agent-id", agent, "--todo-id", todo_id, + "--update-operation-id", operation, "--update-expected-provider-revision", revision, + "--status", "deferred", "--resume-when", resume, + "--reason", "This work requires released feedback from its accepted native submission.", "--execute") + return {"waiting": True, "todo_id": todo_id, "submission_id": submission_id, + "dependency_todo_id": source["todo_id"], "resume_when": resume} + + +def synchronize_feedback_waits(env, packet, *, after_release=None): + """A runner observation releases only its exact existing dependencies. + + A native error or negative scored result can still be usable evidence. + Unknown attribution stays unknown; receiving a record does not authorize + adopting a candidate, rolling back files, completing research or resubmitting. + """ + goal, agent, observer = _scope(env) + records = _records(packet) + if not records: + return {"released": []} + listed = _loopx(env, "todo", "list", "--goal-id", goal, "--role", "agent") + completed = [] + for key, record in records.values(): + source = _dependency(listed["todos"], key, observer, agent) + if source is None: + source = _create_feedback_todo(env, goal=goal, agent=agent, observer=observer, key=key, record=record) + if source.get("status") == "done" or not _released(record): + continue # An unchanged replay creates no work; unknown is not failure. + if source.get("status") != "open": + continue # A runner does not override an owner pause or cancellation. + from .research_feedback_sync import write_feedback_finding + + # Publish the existing Explore finding before changing runnable state; + # otherwise the request guard could admit a replan without its evidence. + write_feedback_finding(env, packet, record, source["todo_id"]) + result = _loopx(env, "todo", "complete", "--goal-id", goal, "--agent-id", observer, + "--todo-id", source["todo_id"], "--no-follow-up", "--evidence", record["result_ref"], + "--note", "Released native feedback is available for this exact submitted snapshot.", "--execute") + completed.append({"submission_id": record["submission_id"], "dependency_todo_id": source["todo_id"], + "result_ref": record["result_ref"], "completed": result.get("ok") is not False}) + if after_release is not None: + after_release(completed[-1]) + return {"released": completed} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + wait = commands.add_parser("wait", help="Defer one Todo until its accepted native submission has released feedback.") + wait.add_argument("--todo-id", required=True) + wait.add_argument("--submission-id", required=True) + commands.add_parser("sync", help="Apply released native feedback through the runner-owned Todo observer.") + args = parser.parse_args() + from .research_feedback_sync import feedback_environment, synchronize_native_feedback + + root = Path(os.environ.get("LOOPX_RESEARCH_FEEDBACK_ROOT", _CONTROL + "/native-feedback")) + env = feedback_environment(root) + if args.command == "wait": + packet = json.loads((root / "native.json").read_text(encoding="utf-8")) + result = wait_for_feedback(env, todo_id=args.todo_id, submission_id=args.submission_id, packet=packet) + else: + result = synchronize_native_feedback(env) + print(json.dumps(result, allow_nan=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/runtime/research_loop.py b/benchmark/runtime/research_loop.py index 3f18edad20..5182b514a7 100644 --- a/benchmark/runtime/research_loop.py +++ b/benchmark/runtime/research_loop.py @@ -59,15 +59,27 @@ async def prepare_research_continuation(runtime, environment, *, cwd): def research_execution_guidance() -> str: return ( - '\nResearch-loop execution evidence:\nThe initial candidate and ongoing research are separate canonical Todos. ' - 'Do not close the research successor merely because the candidate passes tests. Preserve the original ' - 'objective and existing Stop/budget/blocked conditions. Before the first native submission, run a relevant ' - 'local candidate check and retain actual resource observations, including unknown measurements. Use ' - '`loopx-research-probe --help` for the local probe wrapper: it pins explicit candidate files, measures ' - 'runtime and enforces the selected local limit. A solver can write search_iterations and stop_reason as JSON ' - 'to LOOPX_RESEARCH_DIAGNOSTICS; absent values remain unknown, and solver reports are not independently ' - 'verified. This is not another evaluation or a requirement to use every available second. Attach the emitted ' - 'execution_observation in the normal Explore result writeback. Preserve its input revision and evidence ' - 'refs. Record changed method ids and the reason when actual implementation differs from the proposal. ' - 'Resolve this evidence at the next replan.\n' + "\nResearch-loop execution evidence:\n" + "The initial candidate and ongoing research are separate canonical Todos. Do not close the " + "research successor merely because the candidate passes tests. Preserve the original objective " + "and existing Stop/budget/blocked conditions. Before the first native submission, run a relevant " + "local candidate check and retain actual resource observations, including unknown measurements. " + "Use `loopx-research-probe --help` for the local probe wrapper: it pins " + "explicit candidate files, measures runtime and enforces the selected local limit. A solver can " + "write search_iterations and stop_reason as JSON to LOOPX_RESEARCH_DIAGNOSTICS; absent values " + "remain unknown, and solver reports are not independently verified. This is not another " + "evaluation or a requirement to use every available second. Attach the emitted " + "execution_observation in the normal Explore result writeback. Preserve its input revision " + "and evidence refs. On the native EdgeBench adapter, probes register automatically and released " + "results are matched to the actual accepted archive; do not fill result attribution from the " + "current workspace. An unknown match cannot justify adoption or rollback. " + "Record changed method ids and the reason when " + "actual implementation differs from the proposal. Resolve this evidence at the next replan. " + "If the current Todo requires that external result, use `sforge-submit --wait-todo TODO_ID` " + "or `loopx-research-feedback wait --todo-id TODO_ID --submission-id SUBMISSION_ID`; " + "the native accepted receipt creates the specific dependency and the runner observes it. " + "Independent useful work remains runnable. When all useful work is waiting, return to the " + "existing scheduler; do not repeatedly call a model to inspect unchanged feedback. " + "If a submission response is lost, repeat the same submit entrypoint to recover its original " + "operation and native ID; unresolved acceptance stays unknown without another evaluation.\n" ) diff --git a/benchmark/runtime/research_probe.py b/benchmark/runtime/research_probe.py index a8a913e0dc..55e6382c95 100644 --- a/benchmark/runtime/research_probe.py +++ b/benchmark/runtime/research_probe.py @@ -129,6 +129,13 @@ def run_probe(*, command: list[str], candidate_files: list[Path], output: Path, record.write(json.dumps(result, ensure_ascii=False, allow_nan=False) + "\n") record.flush() os.fsync(record.fileno()) + feedback_root = os.environ.get("LOOPX_RESEARCH_FEEDBACK_ROOT") + if feedback_root: + from .research_submission import register_probe + + # The observation must be fully persisted before it becomes a source + # for matching a later native accepted archive/released result. + register_probe(Path(feedback_root), result, {"candidate_sha256": before, "files": files}) return result diff --git a/benchmark/runtime/research_scheduler.py b/benchmark/runtime/research_scheduler.py new file mode 100644 index 0000000000..49b43a9e39 --- /dev/null +++ b/benchmark/runtime/research_scheduler.py @@ -0,0 +1,67 @@ +"""Let durable research feedback interrupt only the outer scheduler's sleep. + +The event is a hint to read current quota again. It grants no model admission, +changes no Turn or native operation identity, and cannot reset a phase budget. +The ordinary scheduler still owns its state route, dispatch and stop handling. +""" +from __future__ import annotations + +import json +import math +import os +from pathlib import Path +import re +import signal +import time + +from .layout import _CONTROL + + +class FeedbackEventSleep: + """Bounded local file observation while the ordinary worker is asleep.""" + + def __init__(self, path: Path, *, clock=time.monotonic, pause=time.sleep): + self.path, self.clock, self.pause = path, clock, pause + # Capture before the first quota read, not on entry into sleep: a + # consumer may publish between the decision and its eventual sleep. + self.revision = self._revision() + + def _revision(self): + try: + with self.path.open("r", encoding="utf-8") as source: + value = json.loads(source.read(4097)) + revision = value.get("revision") if isinstance(value, dict) else None + return revision if isinstance(revision, str) and re.fullmatch(r"[0-9a-f]{64}", revision) else None + except (OSError, ValueError): + return None # Unavailable/partial signals never substitute for state. + + def __call__(self, seconds): + if not math.isfinite(seconds) or seconds < 0: + raise ValueError("scheduler sleep requires a finite nonnegative duration") + deadline = self.clock() + seconds + while True: + current = self._revision() + if current is not None and current != self.revision: + self.revision = current + return + remaining = deadline - self.clock() + if remaining <= 0: + return + self.pause(min(1.0, remaining)) + + +def main(argv=None): + if os.environ.get("LOOPX_RESEARCH_LOOP") != "1": + raise ValueError("research feedback scheduler requires the explicit research-loop treatment") + from scripts.external_scheduler_worker import main as run_scheduler + + root = Path(os.environ.get("LOOPX_RESEARCH_FEEDBACK_ROOT", _CONTROL + "/native-feedback")) + return run_scheduler(argv, sleep=FeedbackEventSleep(root / "ingestion" / "wake.json")) + + +if __name__ == "__main__": + def cancelled(signum, frame): + raise KeyboardInterrupt("scheduler cancelled") + + signal.signal(signal.SIGTERM, cancelled) + raise SystemExit(main()) diff --git a/benchmark/runtime/research_submission.py b/benchmark/runtime/research_submission.py new file mode 100644 index 0000000000..eb98fee8ee --- /dev/null +++ b/benchmark/runtime/research_submission.py @@ -0,0 +1,394 @@ +"""Exact native archive journaling and feedback binding for research-loop workers. + +The generated SForge script still packages, submits once and prints its native +result. These hooks retain its actual archive before that send. Recovery reads +native history; it never posts an archive or invents an operation identity. +""" +from __future__ import annotations + +import argparse +from contextlib import contextmanager +import fcntl +import gzip +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import sys +import tarfile +import uuid +import urllib.parse +import urllib.request +import zlib + +from benchmark.edgebench.research_feedback import ( + archive_manifest, canonical_json, read_archive, validate_packet, write_bytes_atomic, +) + +_OPERATION = re.compile(r"[0-9a-f]{32}\Z") +# Native _reserve rejects before allocating a round; this protocol exposes the +# refusal through exact detail strings, without a separate admission error code. +_NATIVE_CAPACITY_REJECTIONS = frozenset({ + "Judge capacity full; retry this snapshot later", + "Run/task evaluation capacity full; retry this snapshot later", +}) + + +def _atomic(path, value): + write_bytes_atomic(path, canonical_json(value)) + +@contextmanager +def _locked(root): + root.mkdir(parents=True, exist_ok=True) + with (root / "submission.lock").open("a") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + yield + + +def read_packet(root: Path): + return validate_packet(json.loads((root / "native.json").read_bytes())) + + +def register_probe(root: Path, observation: dict, manifest: dict): + """Register a completed local probe; no result attribution is added here.""" + files = manifest.get("files") + if not isinstance(files, list) or not 1 <= len(files) <= 32: + raise ValueError("invalid_probe_manifest") + encoded = json.dumps(sorted(files, key=lambda row: row["path"]), sort_keys=True, + separators=(",", ":")).encode() + digest = hashlib.sha256(encoded).hexdigest() + if observation.get("candidate_sha256") != digest or manifest.get("candidate_sha256") != digest: + raise ValueError("probe_manifest_digest_mismatch") + if observation.get("result_ref") is not None: + raise ValueError("probe_registration_must_precede_result_binding") + value = dict(observation=observation, manifest=manifest) + identifier = hashlib.sha256(canonical_json(value)).hexdigest() + with _locked(root): + _atomic(root / "probes" / (identifier + ".json"), value) + return "native-probe:" + identifier + + +def bind_execution_observations(root: Path, packet: dict): + """Automatically bind matching probe manifests to the *accepted* snapshot. + + Multiple probes of the same source remain independent observations. A probe + with different selected source, no exact native archive, or an unresolved + result stays unbound; a changed workspace is never read by this function. + """ + validate_packet(packet) + probes = [(path.stem, json.loads(path.read_bytes())) + for path in sorted((root / "probes").glob("*.json"))] + bound = [] + for row in packet["records"]: + if (row.get("attribution_status") != "verified" or not row.get("result_ref") + or row.get("status") not in {"completed", "error"}): + continue + try: + source = Path(row["archive_path"]) + raw = read_archive(source, root=root) + if hashlib.sha256(raw).hexdigest() != row["archive_sha256"]: + continue + for identifier, probe in probes: + observation = probe["observation"] + manifest = probe["manifest"] + try: + digest, files = archive_manifest(raw, [entry["path"] for entry in manifest["files"]]) + except (EOFError, ValueError, gzip.BadGzipFile, tarfile.TarError, zlib.error): + # A retained native error archive can be invalid tar. Keep + # its generic result usable without attributing a probe. + continue + if (digest != observation["candidate_sha256"] or digest != manifest["candidate_sha256"] + or files != manifest["files"]): + continue + observation = dict(observation, result_ref=row["result_ref"], result_candidate_sha256=digest) + observation["evidence_refs"] = sorted(set([ + *observation["evidence_refs"], row["result_ref"], + "native-archive:" + row["archive_sha256"], + ])) + bound.append({**row, "probe_ref": "native-probe:" + identifier, + "execution_observation": observation}) + except (KeyError, OSError, ValueError): + continue + return bound + + +def _intents(root): + return [json.loads(path.read_bytes()) for path in sorted((root / "intents").glob("*.json"))] + + +def _path(root, operation_id): + if not _OPERATION.fullmatch(operation_id): + raise ValueError("invalid_submission_operation") + return root / "intents" / (operation_id + ".json") + + +def _reconcile(root, packet): + validate_packet(packet) + records = packet["records"] + intents = _intents(root) + claimed = {row["submission_id"] for row in intents if row.get("submission_id")} + for intent in intents: + if any(intent.get(key) != packet[key] for key in ("run_id", "task_id")): + raise ValueError("submission_operation_scope_mismatch") + if intent.get("acceptance") == "rejected" or not intent.get("dispatch_started"): + continue + sid = intent.get("submission_id") + candidates = [row for row in records if row.get("accepted_record") is True + and row.get("submission_id") not in intent["before_submission_ids"] + and row.get("archive_sha256") == intent["archive_sha256"] + and (row.get("submission_id") == sid if sid else row.get("submission_id") not in claimed)] + if len(candidates) == 1: + matched = candidates[0] + if sid and intent.get("round_id") != matched["round_id"]: + intent["recovery_reason"] = "native_round_identity_changed" + else: + intent.update(acceptance="accepted", submission_id=matched["submission_id"], + round_id=matched["round_id"], recovered=not bool(sid) or intent.get("recovered", False)) + if matched.get("attribution_status") == "verified" and matched.get("result_ref"): + intent["result_ref"] = matched["result_ref"] + claimed.add(matched["submission_id"]) + intent.pop("recovery_reason", None) + else: + intent["recovery_reason"] = "ambiguous_native_archive_match" if candidates else "native_archive_match_unavailable" + _atomic(_path(root, intent["operation_id"]), intent) + return intents + + +def synchronize_intents(root: Path, packet: dict): + with _locked(root): + return _reconcile(root, packet) + + +def _fresh_history(packet, *, history_reader=None): + if history_reader is not None: + history = history_reader() + else: + endpoint = os.environ.get("SFORGE_JUDGE_URL", "").rstrip("/") + token = os.environ.get("SFORGE_TOKEN") + if not endpoint or not token: + raise ValueError("native_registration_environment_missing") + query = urllib.parse.urlencode({"token": token}) + # Read only the native public history. urllib retains the environment's + # declared proxy/no_proxy route and never retries a submission. + with urllib.request.urlopen(endpoint + "/api/v1/history?" + query, timeout=15) as response: + history = json.load(response) + if history.get("run_id") != packet["run_id"] or not isinstance(history.get("entries"), list): + raise ValueError("native_history_scope_mismatch") + entries = [row for row in history["entries"] if row.get("type") == "submission"] + if any(row.get("task_id") != packet["task_id"] for row in entries): + raise ValueError("native_history_task_mismatch") + ids = [row.get("submission_id") for row in entries] + if any(not isinstance(sid, str) or not sid for sid in ids) or len(ids) != len(set(ids)): + raise ValueError("native_history_identity_ambiguous") + return ids + + +def prepare_submission(root: Path, archive: Path, *, history_reader=None, wait_todo_id=None): + """Retain native packaged bytes before an effect; recover an unresolved call.""" + with _locked(root): + packet = read_packet(root) + intents = _reconcile(root, packet) + unresolved = [row for row in intents if row.get("acceptance") != "rejected" + and not row.get("result_observed")] + if len(unresolved) > 1: + raise ValueError("multiple_unresolved_native_operations") + if unresolved: + intent = unresolved[0] + if intent.get("submission_id") and intent.get("acceptance") == "accepted": + return dict(action="recover", operation_id=intent["operation_id"], + submission_id=intent["submission_id"]) + if intent.get("dispatch_started"): + raise ValueError("native_acceptance_unknown_no_resubmit") + return dict(action="submit", operation_id=intent["operation_id"], archive_path=intent["archive_path"]) + before = _fresh_history(packet, history_reader=history_reader) + raw = read_archive(archive.absolute(), root=archive.parent.resolve()) + operation_id = uuid.uuid4().hex + snapshot = root / "snapshots" / (operation_id + ".tar.gz") + write_bytes_atomic(snapshot, raw, create_only=True) + snapshot.chmod(0o400) + intent = dict(operation_id=operation_id, run_id=packet["run_id"], task_id=packet["task_id"], + archive_path=str(snapshot), archive_sha256=hashlib.sha256(raw).hexdigest(), + archive_bytes=len(raw), before_submission_ids=before, dispatch_started=False, + acceptance="unknown", result_observed=False) + if wait_todo_id: + intent["wait_todo_id"] = wait_todo_id + _atomic(_path(root, operation_id), intent) + return dict(action="submit", operation_id=operation_id, archive_path=str(snapshot)) + + +def mark_dispatched(root: Path, operation_id: str): + with _locked(root): + path = _path(root, operation_id) + intent = json.loads(path.read_bytes()) + if intent["dispatch_started"]: + raise ValueError("native_operation_already_dispatched") + raw = read_archive(Path(intent["archive_path"]), root=root) + if hashlib.sha256(raw).hexdigest() != intent["archive_sha256"]: + raise ValueError("prepared_native_archive_changed") + intent["dispatch_started"] = True + _atomic(path, intent) + return intent + + +def record_response(root: Path, operation_id: str, http_status: int, response: dict): + with _locked(root): + path = _path(root, operation_id) + intent = json.loads(path.read_bytes()) + if not intent.get("dispatch_started"): + raise ValueError("native_operation_not_dispatched") + sid, round_id = response.get("submission_id"), response.get("round_id") + detail = response.get("detail") + if (http_status == 200 and isinstance(sid, str) and re.fullmatch(r"[A-Za-z0-9_-]{1,128}", sid) + and isinstance(round_id, str) and re.fullmatch(r"agent-[1-9][0-9]*", round_id) + and sid not in intent["before_submission_ids"]): + if intent.get("submission_id") not in (None, sid) or intent.get("round_id") not in (None, round_id): + raise ValueError("native_acceptance_receipt_changed") + intent.update(acceptance="accepted", submission_id=sid, round_id=round_id) + elif isinstance(detail, str) and (http_status in {400, 401, 403, 404, 413, 429} + or (http_status == 503 and detail in _NATIVE_CAPACITY_REJECTIONS)): + # Only proven native refusals precede admission. Other 5xx, unknown + # 503 details, parse errors and lost replies remain unresolved. + intent["acceptance"] = "rejected" + _atomic(path, intent) + return intent + + +def declared_wait_packet(root: Path, intent: dict): + """An actual accepted response can enter wait before the next host poll. + + This says only that the named operation was accepted. It makes no archive + or released-result attribution claim; that requires the native observer. + """ + if intent.get("acceptance") != "accepted" or not intent.get("submission_id"): + raise ValueError("native_acceptance_unconfirmed") + packet = read_packet(root) + if any(intent.get(key) != packet[key] for key in ("run_id", "task_id")): + raise ValueError("native_acceptance_scope_mismatch") + records = [row for row in packet["records"] if row.get("submission_id") == intent["submission_id"]] + if len(records) > 1 or (records and records[0].get("round_id") != intent["round_id"]): + raise ValueError("native_acceptance_identity_ambiguous") + if not records: + packet["records"].append(dict(accepted_record=True, submission_id=intent["submission_id"], + round_id=intent["round_id"], status="queued", attribution_status="unknown")) + return packet + + +def _synchronize_declared_wait(root, intent): + if intent.get("wait_todo_id") and intent.get("acceptance") == "accepted": + from .research_feedback_wait import wait_for_feedback + from .research_feedback_sync import feedback_environment + wait_for_feedback(feedback_environment(root), todo_id=intent["wait_todo_id"], + submission_id=intent["submission_id"], packet=declared_wait_packet(root, intent)) + + +def record_result(root: Path, submission_id: str, response: dict): + if response.get("submission_id") != submission_id: + raise ValueError("native_result_id_mismatch") + if response.get("status") not in {"completed", "error"}: + return + with _locked(root): + for intent in _intents(root): + if intent.get("submission_id") == submission_id and intent.get("acceptance") == "accepted": + report = response.get("report") + if response["status"] == "completed" and (not isinstance(report, dict) + or report.get("submission_id") != submission_id + or report.get("task_id") != intent["task_id"]): + raise ValueError("native_completed_result_scope_mismatch") + if response["status"] == "error" and report is not None: + raise ValueError("native_error_result_ambiguous") + intent["result_observed"] = True + _atomic(_path(root, intent["operation_id"]), intent) + + +def instrument_submit_script(script: str, helper_command: list[str], root: str) -> str: + """Fail closed on an unqualified native client, without changing its HTTP API.""" + command = shlex.join([*helper_command, "--root", root]) + replacements = { + ' echo " --list, -l List all previous submissions and scores for this run"': + ' echo " --wait-todo ID Declare that this owned work depends on the accepted feedback"\n' + ' echo " --list, -l List all previous submissions and scores for this run"', + ' --details|-d) DETAILS_MODE=1 ;;': + ' --wait-todo)\n [ "$#" -gt 0 ] && [ -n "$1" ] || exit 1\n' + ' RESEARCH_WAIT_TODO="$1"; shift ;;\n --details|-d) DETAILS_MODE=1 ;;', + 'ARCHIVE_SIZE=$(wc -c < "$ARCHIVE_FILE")': + f'RESEARCH_PREPARED=$({command} prepare --archive "$ARCHIVE_FILE" --wait-todo "${{RESEARCH_WAIT_TODO:-}}")\n' + 'if [ "$(echo "$RESEARCH_PREPARED" | jq -r .action)" = "recover" ]; then\n' + ' RECOVERED_ID=$(echo "$RESEARCH_PREPARED" | jq -r .submission_id)\n' + ' if [ "$DETAILS_MODE" = "1" ]; then exec "$0" --result "$RECOVERED_ID" --details; fi\n' + ' exec "$0" --result "$RECOVERED_ID"\nfi\n' + 'RESEARCH_OPERATION=$(echo "$RESEARCH_PREPARED" | jq -r .operation_id)\n' + 'ARCHIVE_FILE=$(echo "$RESEARCH_PREPARED" | jq -r .archive_path)\n' + 'ARCHIVE_SIZE=$(wc -c < "$ARCHIVE_FILE")', + 'if ! HTTP_CODE=$(curl -sS -o "$REQUEST_TMP/submit.json"': + f'{command} dispatched --operation-id "$RESEARCH_OPERATION" >/dev/null\n' + 'if ! HTTP_CODE=$(curl -sS -o "$REQUEST_TMP/submit.json"', + 'SUBMIT_RESP=$(cat "$REQUEST_TMP/submit.json")': + 'SUBMIT_RESP=$(cat "$REQUEST_TMP/submit.json")\n' + f'{command} response --operation-id "$RESEARCH_OPERATION" --http-status "$HTTP_CODE" ' + '--response-file "$REQUEST_TMP/submit.json" >/dev/null', + ' RESULT=$(get_json "$JUDGE_URL/api/v1/result/$SUBMISSION_ID")': + ' RESULT=$(get_json "$JUDGE_URL/api/v1/result/$SUBMISSION_ID")\n' + f' {command} result --submission-id "$SUBMISSION_ID" --response-file "$REQUEST_TMP/response.json"', + } + for anchor, replacement in replacements.items(): + if script.count(anchor) != 1: + raise ValueError("unsupported_native_submit_script_boundary") + script = script.replace(anchor, replacement, 1) + return script + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=os.environ.get("LOOPX_RESEARCH_FEEDBACK_ROOT")) + sub = parser.add_subparsers(dest="action", required=True) + prepare = sub.add_parser("prepare") + prepare.add_argument("--archive", type=Path, required=True) + prepare.add_argument("--wait-todo") + dispatched = sub.add_parser("dispatched") + dispatched.add_argument("--operation-id", required=True) + response = sub.add_parser("response") + response.add_argument("--operation-id", required=True) + response.add_argument("--http-status", type=int, required=True) + response.add_argument("--response-file", type=Path, required=True) + result = sub.add_parser("result") + result.add_argument("--submission-id", required=True) + result.add_argument("--response-file", type=Path, required=True) + sub.add_parser("sync") + args = parser.parse_args(argv) + if args.root is None: + parser.error("native research feedback root is required") + try: + if args.action == "prepare": + value = prepare_submission(args.root, args.archive, wait_todo_id=args.wait_todo) + if value["action"] == "recover": + _synchronize_declared_wait(args.root, json.loads(_path(args.root, value["operation_id"]).read_bytes())) + elif args.action == "dispatched": + value = mark_dispatched(args.root, args.operation_id) + elif args.action == "response": + try: + payload = json.loads(args.response_file.read_bytes()) + except (ValueError, OSError): + payload = {} + value = record_response(args.root, args.operation_id, args.http_status, payload) + _synchronize_declared_wait(args.root, value) + elif args.action == "result": + record_result(args.root, args.submission_id, json.loads(args.response_file.read_bytes())) + return 0 + else: + packet = read_packet(args.root) + value = dict(packet=packet, intents=synchronize_intents(args.root, packet), + observations=bind_execution_observations(args.root, packet)) + print(json.dumps(value, allow_nan=False)) + return 0 + except Exception as error: + # Transport exceptions may contain credential-bearing URLs. Expose only + # controlled validation codes or their type, never the raw exception. + reason = str(error) if type(error) is ValueError else type(error).__name__ + print("Research submission halted: " + reason + "; no automatic resubmission.", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/runtime/scheduler.py b/benchmark/runtime/scheduler.py index 40eea1f383..99031adda8 100644 --- a/benchmark/runtime/scheduler.py +++ b/benchmark/runtime/scheduler.py @@ -12,8 +12,11 @@ def worker_command(env: dict[str, str], *, python: str, source: str, wake = [python, "-m", "benchmark.runtime.worker"] if env["LOOPX_EXECUTION_MODE"] not in {"heartbeat", "turn"}: return wake + scheduler = ([python, "-m", "benchmark.runtime.research_scheduler"] + if env.get("LOOPX_RESEARCH_LOOP") == "1" and env.get("LOOPX_RESEARCH_FEEDBACK_ROOT") + else [python, f"{source}/scripts/external_scheduler_worker.py"]) return [ - python, f"{source}/scripts/external_scheduler_worker.py", + *scheduler, "--cli-bin", env["LOOPX_CLI"], "--registry", env["LOOPX_REGISTRY"], "--runtime-root", env["LOOPX_RUNTIME_ROOT"], "--runtime-profile", "generic_cli", "--goal-id", env["LOOPX_GOAL_ID"], diff --git a/benchmark/runtime/sforge.py b/benchmark/runtime/sforge.py index 9a16b1b4c5..e1797e3d54 100644 --- a/benchmark/runtime/sforge.py +++ b/benchmark/runtime/sforge.py @@ -89,6 +89,7 @@ def __init__(self, config, *, profile: str, cwd: str, feedback: str = "native", workspace_instructions: str | None = None, task_objective: str | None = None, + research_loop: bool = False, task_entry: str | None = None, turn_envelope: bool = False, native_goal_continue: bool = False, @@ -107,6 +108,12 @@ def __init__(self, config, *, profile: str, cwd: str, ): raise ValueError("turn_envelope requires a heartbeat worker") self.turn_envelope = turn_envelope + if not isinstance(research_loop, bool) or (research_loop and ( + profile != "heartbeat-explore" or feedback != "native" or not config.agent_api_base_url)): + raise ValueError("research_loop requires heartbeat-explore, native feedback and a custom model endpoint") + if research_loop and warm_start is not None: + raise ValueError("research_loop must be admitted as a fresh treatment, not added to an old checkpoint") + self.research_loop = research_loop if timeout_seconds <= PHASE_WORK_RESERVE_SECONDS: raise ValueError("Worker budget must exceed the 160s startup/settlement reserve") if not config.agent_model or not config.agent_effort: @@ -139,6 +146,9 @@ def __init__(self, config, *, profile: str, cwd: str, replan_after_turns = DEFAULT_REPLAN_AFTER_TURNS self.replan_after_turns = replan_after_turns self.replan_after_todos = replan_after_todos + from .research_loop import research_stride_strategy + + stride_strategy = research_stride_strategy(research_loop, stride_strategy) if stride_strategy is not None and not profile.startswith("heartbeat-"): raise ValueError("stride_strategy requires a heartbeat profile") if stride_strategy is not None: @@ -280,6 +290,8 @@ def _install_worker(self, backend, handle, log_dir, logger): checkpoint_request_path=self.checkpoint_request_path, checkpoint_request=self.checkpoint_request, task_objective=self.task_objective, + research_loop=self.research_loop, + **({"research_feedback_root": f"{_CONTROL}/native-feedback"} if self.research_loop else {}), **({"goal_id": self.warm_start.goal_id, "agent_id": self.warm_start.agent_id, "defer_codex_home": True} if self.warm_start is not None else {}), ) @@ -332,8 +344,11 @@ def _install_worker(self, backend, handle, log_dir, logger): f"Path({str(hook)!r}).read_bytes(); Path('/etc/codex/hooks.json').read_bytes()"], user="agent") if result.exit_code: raise RuntimeError("Best-only Codex delivery hooks are unreadable by worker") + if self.research_loop: + self._install_native_research_feedback(backend, handle) (log_dir / "worker-profile.json").write_text(json.dumps({ "profile": self.profile, "model": self._config.agent_model, + **({"research_loop": True} if self.research_loop else {}), "task_entry": self.task_entry, "reasoning_effort": effort, "timeout_seconds": self.timeout_seconds, "loopx_usage_ping_enabled": False, @@ -354,6 +369,60 @@ def _install_worker(self, backend, handle, log_dir, logger): self.profile.startswith("heartbeat-") else {}), }, indent=2)) + def _install_native_research_feedback(self, backend, handle): + from .research_submission import instrument_submit_script + from .research_feedback_sync import FEEDBACK_ROOT + + if getattr(backend, "research_feedback", None) is None: + raise ValueError("research_loop requires the native accepted-snapshot observer on RecordingDockerBackend") + capability = backend.exec_run(handle, [f"{_PYTHON}/bin/python3", "-c", + "from loopx.capabilities.explore.result_writeback import record_observer_result_attachment; " + "from benchmark.edgebench.research_feedback import archive_manifest"], + environment={"PYTHONPATH": _SRC, "LOOPX_USAGE_PING": "0"}, user="agent") + if capability.exit_code: + raise RuntimeError("Native research feedback requires a matching Explore observer capability in the pinned product source") + native = backend.exec_run(handle, ["cat", "/usr/local/bin/sforge-submit"], user="root") + if native.exit_code: + raise RuntimeError("The native submission entrypoint is unavailable") + script = instrument_submit_script(native.output, ["env", f"PYTHONPATH={_SRC}", + f"{_PYTHON}/bin/python3", "-m", "benchmark.runtime.research_submission"], FEEDBACK_ROOT) + directories = [f"{FEEDBACK_ROOT}/{name}" for name in ("intents", "snapshots", "probes", "ingestion")] + for command in (["mkdir", "-p", *directories], + ["chown", "agent:agent", *directories], + ["chmod", "0700", *directories], + ["touch", f"{FEEDBACK_ROOT}/submission.lock"], + ["chown", "agent:agent", f"{FEEDBACK_ROOT}/submission.lock"]): + if backend.exec_run(handle, command, user="root").exit_code: + raise RuntimeError("Could not prepare the native research evidence owner") + with tempfile.TemporaryDirectory(prefix="native-research-client-") as directory: + path = Path(directory) / "sforge-submit" + path.write_text(script, encoding="utf-8") + path.chmod(0o755) + backend.copy_to_container(handle, path, PurePosixPath("/usr/local/bin/sforge-submit")) + if backend.exec_run(handle, ["chmod", "0755", "/usr/local/bin/sforge-submit"], user="root").exit_code: + raise RuntimeError("The research submission entrypoint is not executable") + + def _prepare_native_research_consumer(self, env): + from .research_feedback_sync import FEEDBACK_ROOT, feedback_owner_environment + + owner = feedback_owner_environment(env) + with tempfile.TemporaryDirectory(prefix="native-research-owner-") as directory: + path = Path(directory) / "owner.json" + path.write_text(json.dumps(owner, sort_keys=True), encoding="utf-8") + path.chmod(0o644) + self.backend.copy_to_container(self.handle, path, PurePosixPath(f"{FEEDBACK_ROOT}/owner.json")) + + def consume(backend, handle, _packet): + # The runner keeps observing while the scheduler is sleeping. A + # failed writeback is retried before any dependency is released. + result = backend.exec_run_with_timeout(handle, + ["timeout", "--signal=TERM", "--kill-after=5", "60s", + f"{_PYTHON}/bin/python3", "-m", "benchmark.runtime.research_feedback_sync", + "--root", FEEDBACK_ROOT], timeout=70, environment=owner, user="agent") + if result.exit_code or result.timed_out: + raise RuntimeError("Native feedback consumption is incomplete") + self.backend.research_feedback.after_publish = consume + def format_run_cmd(self, prompt_path, *, model=None, cwd="", internet=True, resume=False): if self.workspace_instructions is not None and not self.instructions_installed and self.warm_start is None: self._install_workspace_instructions() @@ -409,6 +478,8 @@ def format_run_cmd(self, prompt_path, *, model=None, cwd="", internet=True, resu raise RuntimeError("Could not protect fresh receiver credentials") self.prepared = True env = self.runtime._worker_env(cwd=self.cwd) + if self.research_loop: + self._prepare_native_research_consumer(env) if self.warm_start is not None: env["LOOPX_WARM_START"] = f"{_CONTROL}/warm-start.json" command = worker_command(env, python=f"{_PYTHON}/bin/python3", source=_SRC, @@ -492,4 +563,6 @@ def _execution_command(self, command, *, execution_start_marker=None): self.backend.execution_start_marker = execution_start_marker if self.feedback == "best-only": self.backend.start_feedback(self.handle) + if self.research_loop: + self.backend.research_feedback.start(self.backend, self.handle) return command diff --git a/benchmark/runtime/sforge_backend.py b/benchmark/runtime/sforge_backend.py index aa301db308..191d57171a 100644 --- a/benchmark/runtime/sforge_backend.py +++ b/benchmark/runtime/sforge_backend.py @@ -15,11 +15,13 @@ class RecordingDockerBackend(DockerBackend): def __init__(self, *, log_dir: Path, logger, oauth_proxy: bool = False, - blind_api_endpoint: tuple[str, int] | None = None, feedback=None, checkpoint=None, client=None): + blind_api_endpoint: tuple[str, int] | None = None, feedback=None, checkpoint=None, + research_feedback=None, client=None): super().__init__(client) self.log_dir, self.logger = log_dir, logger self.blind_api_endpoint = blind_api_endpoint self.feedback = feedback + self.research_feedback = research_feedback self.checkpoint = checkpoint self.execution_command = None self.execution_start_marker = None @@ -35,6 +37,8 @@ def create_container(self, image, name, **kwargs): kwargs["extra_hosts"] = hosts if self.feedback is not None: self.feedback.bind(kwargs.get("environment")) + if self.research_feedback is not None: + self.research_feedback.bind(kwargs.get("environment")) if self.checkpoint is not None: self.checkpoint.bind(kwargs.get("environment")) kwargs["environment"] = self._agent_environment(kwargs.get("environment")) @@ -73,6 +77,8 @@ def exec_run_with_timeout(self, handle, cmd, timeout=60, **kwargs): self.checkpoint.close() if is_solver and self.feedback is not None: self.feedback.pause() + if is_solver and self.research_feedback is not None: + self.research_feedback.pause() def create_network_isolation(self, handle, allowed_endpoints, logger): # ChatGPT login may refresh during an 18h run. Preserve the native @@ -103,6 +109,8 @@ def cleanup_container(self, handle, logger=None): self.checkpoint.close() if self.feedback is not None: self.feedback.close() + if self.research_feedback is not None: + self.research_feedback.close() if handle is not None: self.log_dir.mkdir(parents=True, exist_ok=True) observations = [] diff --git a/benchmark/tests/test_edgebench_report.py b/benchmark/tests/test_edgebench_report.py index 31e39e5a7f..19dbeae9ac 100644 --- a/benchmark/tests/test_edgebench_report.py +++ b/benchmark/tests/test_edgebench_report.py @@ -654,3 +654,60 @@ def test_report_rejects_unbound_horizon_setting(trial, receipt, profile): write(path, value) with pytest.raises(ValueError, match="planning_horizon_goal"): export(runs, selection, output, observed_at="2026-01-02T00:00:00+00:00") + + +@pytest.mark.parametrize("enabled", [None, False, True]) +def test_report_preserves_research_treatment_and_historical_absence(trial, enabled): + for name in ("runtime-receipt", "worker-profile"): + path = trial[3] / (name + ".json") + value = json.loads(path.read_text()) + if enabled is not None: + value["research_loop"] = enabled + if enabled: + value["worker" if name == "runtime-receipt" else "profile"] = "heartbeat-explore" + value["feedback"] = "native" + write(path, value) + out = build(trial) + config = json.loads((out / "settings/run.json").read_text()) + for owner in ("runner", "worker_profile"): + assert config[owner].get("research_loop") is enabled + assert ("research_loop" in config[owner]) is (enabled is not None) + assert verify(out)["verified"] is True + + +@pytest.mark.parametrize("receipt,profile", [ + ({}, {"research_loop": True}), + ({"research_loop": True}, {}), + ({}, {"research_loop": False}), + ({"research_loop": True}, {"research_loop": False}), + ({"research_loop": True}, {"research_loop": 1}), + ({"research_loop": "true"}, {"research_loop": "true"}), + ({"research_loop": None}, {"research_loop": None}), +]) +def test_report_rejects_one_sided_or_invalid_raw_research_treatment_before_projection(trial, receipt, profile): + for name, fields in (("runtime-receipt", receipt), ("worker-profile", profile)): + path = trial[3] / (name + ".json") + value = json.loads(path.read_text()) + value.update(fields) + write(path, value) + with pytest.raises(ValueError, match="research_loop"): + build(trial) + assert not trial[2].exists(), "a discarded source flag must not become a verified portable report" + + +@pytest.mark.parametrize("runner,profile", [ + ({}, {"research_loop": True}), + ({"research_loop": True}, {}), + ({"research_loop": True}, {"research_loop": 1}), + ({"research_loop": None}, {"research_loop": None}), +]) +def test_rehashed_portable_research_treatment_still_requires_matching_booleans(trial, runner, profile): + out = build(trial) + path = out / "settings/run.json" + config = json.loads(path.read_text()) + config["runner"].update(runner) + config["worker_profile"].update(profile) + write(path, config) + reseal(out, "settings/run.json") + with pytest.raises(ValueError, match="research_loop"): + verify(out) diff --git a/benchmark/tests/test_native_research_feedback.py b/benchmark/tests/test_native_research_feedback.py new file mode 100644 index 0000000000..c31b33554c --- /dev/null +++ b/benchmark/tests/test_native_research_feedback.py @@ -0,0 +1,482 @@ +"""Native public HTTP + exact tar snapshots, with bounded synthetic evaluations.""" +from __future__ import annotations + +from contextlib import contextmanager +from email.parser import BytesParser +from email.policy import default +import gzip +import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import io +import json +import os +from pathlib import Path +import shutil +import socket +import subprocess +import sys +import tarfile +import threading +from types import SimpleNamespace +from urllib.parse import parse_qs, urlsplit + +import pytest + +from benchmark.edgebench.research_feedback import SCHEMA, NativeResearchFeedback, archive_manifest, canonical_json +from benchmark.runtime.research_submission import ( + bind_execution_observations, declared_wait_packet, instrument_submit_script, + mark_dispatched, prepare_submission, record_response, record_result, + register_probe, synchronize_intents, +) + + +def archive_bytes(files): + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode="w:gz") as archive: + for name, data in files.items(): + entry = tarfile.TarInfo(name) + entry.size = len(data) + archive.addfile(entry, io.BytesIO(data)) + return stream.getvalue() + + +def probe_record(source=b"original"): + files = [{"path": "candidate.py", "sha256": hashlib.sha256(source).hexdigest()}] + digest = hashlib.sha256(json.dumps(files, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + observation = dict(schema_version="explore_execution_observation_v0", candidate_sha256=digest, + input_revision="synthetic-public-v1", planned_method_id="local-search", executed_method_id="local-search", + change_reason=None, elapsed_seconds=0.4, limit_seconds=30.0, search_iterations=2, + stop_reason="process_exit:0; solver_report:search_cap", exit_code=0, + evidence_refs=["probe:synthetic", "source:" + digest]) + return observation, {"candidate_sha256": digest, "files": files} + + +class LocalCopyBackend: + """Exercise the publisher's actual copy/rename protocol in a disposable tree.""" + def exec_run(self, _handle, command, **_kwargs): + result = subprocess.run(command, capture_output=True) + return SimpleNamespace(exit_code=result.returncode) + + def copy_to_container(self, _handle, source, destination): + shutil.copyfile(source, destination) + + +@contextmanager +def native_http(tmp_path, *, immediately_complete=False, lose_reply=False): + state = dict(entries=[], results={}, posts=0, result_reads=0, immediate=immediately_complete, + lose_reply=lose_reply, run_id="synthetic-run", task_id="synthetic-task") + trial = tmp_path / "trial" + trial.mkdir() + + def release(sid, *, score=0, status="completed"): + entry = next(row for row in state["entries"] if row["submission_id"] == sid) + report = (dict(submission_id=sid, task_id=state["task_id"], score=score, pass_rate=0.0, + valid=True, passed=0, failed=2, total_tests=2, summary="Synthetic negative result") + if status == "completed" else None) + entry.update(status=status, score=score if report else None, + pass_rate=0.0 if report else None, valid=True if report else None) + state["results"][sid] = dict(submission_id=sid, status=status, report=report, + error=None if report else "Synthetic evaluator failure") + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def send(self, payload, status=200): + raw = canonical_json(payload) + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + def do_GET(self): + url = urlsplit(self.path) + if url.path == "/api/v1/history": + assert parse_qs(url.query) == {"token": ["synthetic-token"]} + return self.send(dict(run_id=state["run_id"], entries=state["entries"], + best_score=0, best_pass_rate=0.0, best_round=None)) + if url.path.startswith("/api/v1/result/"): + sid = url.path.rsplit("/", 1)[-1] + state["result_reads"] += 1 + return self.send(state["results"][sid]) + self.send({}, 404) + + def do_POST(self): + assert self.path == "/api/v1/submit" + state["posts"] += 1 + raw = self.rfile.read(int(self.headers["Content-Length"])) + message = BytesParser(policy=default).parsebytes( + b"Content-Type: " + self.headers["Content-Type"].encode() + b"\r\n\r\n" + raw) + parts = {part.get_param("name", header="content-disposition"): part.get_payload(decode=True) + for part in message.iter_parts()} + assert parts["token"] == b"synthetic-token" + number = state["posts"] + sid, round_id = f"submission-{number}", f"agent-{number}" + folder = trial / "submissions" / round_id + folder.mkdir(parents=True) + (folder / "submission.tar.gz").write_bytes(parts["archive"]) + state["entries"].append(dict(type="submission", submission_id=sid, round=round_id, + task_id=state["task_id"], status="queued", valid=None, score=None, pass_rate=None)) + state["results"][sid] = dict(submission_id=sid, status="queued", report=None, error=None) + if state["immediate"]: + release(sid) + if state["lose_reply"]: + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + return + self.send(dict(submission_id=sid, round_id=round_id, status="queued", remaining_submissions=8)) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + root = tmp_path / "feedback" + observer = NativeResearchFeedback(trial=trial, run_id=state["run_id"], task_id=state["task_id"], + judge_url=f"http://127.0.0.1:{server.server_port}", remote_root=str(root)) + observer.bind({"SFORGE_TOKEN": "synthetic-token"}) + observer.update(LocalCopyBackend(), None) + try: + yield state, release, observer, root + finally: + observer.close() + server.shutdown() + thread.join(timeout=5) + server.server_close() + + +def submit_fixture(observer, raw): + response = observer.session.post(observer.judge_url + "/api/v1/submit", + data={"token": "synthetic-token"}, files={"archive": ("submission.tar.gz", raw)}, timeout=5) + response.raise_for_status() + return response.json() + + +def test_late_negative_result_binds_original_selected_manifest_not_current_files(tmp_path): + with native_http(tmp_path) as (state, release, observer, root): + raw = archive_bytes({"candidate.py": b"original", **{f"data/{i}.txt": b"x" for i in range(35)}}) + accepted = submit_fixture(observer, raw) + old, manifest = probe_record() + register_probe(root, old, manifest) + new, new_manifest = probe_record(b"new workspace") + register_probe(root, new, new_manifest) + pending = observer.update(LocalCopyBackend(), None) + assert state["result_reads"] == 0, "pending public history must not read a result" + assert "result_ref" not in pending["records"][0] + assert "candidate_sha256" not in pending["records"][0], "a large full archive is not a 32-file probe" + (tmp_path / "candidate.py").write_text("new workspace") + release(accepted["submission_id"], score=0) + completed = observer.update(LocalCopyBackend(), None) + [bound] = bind_execution_observations(root, completed) + assert bound["execution_observation"]["result_candidate_sha256"] == old["candidate_sha256"] + assert bound["archive_sha256"] == hashlib.sha256(raw).hexdigest() + assert bound["archive_sha256"] != old["candidate_sha256"] + assert bound["result_summary"]["score"] == 0 + assert completed == observer.poll(), "unchanged results retain their first observation identity" + assert (tmp_path / "candidate.py").read_text() == "new workspace" + + +@pytest.mark.parametrize("corruption", ["invalid-gzip", "truncated-gzip", "invalid-deflate", "bad-crc"]) +def test_malformed_native_error_archive_does_not_block_independent_probe_binding(tmp_path, corruption): + with native_http(tmp_path) as (state, release, observer, root): + observation, manifest = probe_record() + register_probe(root, observation, manifest) + # Native admission accepts opaque bytes; failed archive listing releases + # an error while retaining those exact bytes, without a source manifest. + archive = archive_bytes({"candidate.py": b"original"}) + if corruption == "invalid-gzip": + invalid = b"not-a-tar-gzip" + elif corruption == "truncated-gzip": + invalid = archive[:len(archive) // 2] + else: + # Keep the tar header in one gzip member so decompression/CRC + # errors surface during member reads, after tarfile.open returns. + unpacked = gzip.decompress(archive) + first = bytearray(gzip.compress(unpacked[:512])) + if corruption == "bad-crc": + first[-8] ^= 1 + following = gzip.compress(unpacked[512:]) + else: + following = archive[:10] + b"\x07" + archive[-8:] # Reserved deflate block type. + invalid = bytes(first) + following + failed = submit_fixture(observer, invalid) + release(failed["submission_id"], status="error") + good = submit_fixture(observer, archive) + release(good["submission_id"]) + packet = observer.update(LocalCopyBackend(), None) + failed_record = next(row for row in packet["records"] if row["submission_id"] == failed["submission_id"]) + assert failed_record["status"] == "error" and failed_record["result_ref"] + assert failed_record["attribution_status"] == "verified", "the native archive/result identity remains evidence" + assert "candidate_sha256" not in failed_record, "invalid tar bytes cannot qualify a source manifest" + [bound] = bind_execution_observations(root, packet) + assert bound["submission_id"] == good["submission_id"] + assert bound["execution_observation"]["result_candidate_sha256"] == observation["candidate_sha256"] + + +@pytest.mark.parametrize("per_run_limit", [False, True], ids=["aggregate", "run-task"]) +def test_native_capacity_refusal_allows_an_explicit_later_submission(tmp_path, monkeypatch, per_run_limit): + native = pytest.importorskip("sforge.harness.judge_server") + pytest.importorskip("httpx") + from fastapi.testclient import TestClient + from sforge.harness.config import SForgeConfig + + run_id, task_id = "synthetic-run", "synthetic-task" + task = SimpleNamespace(judge=SimpleNamespace(selection="pass_rate_first", score_direction="maximize")) + monkeypatch.setattr(native, "create_backend_from_config", lambda _config: SimpleNamespace(backend_name="synthetic")) + monkeypatch.setattr(native.JudgeState, "load_tasks", lambda state: setattr(state, "tasks", {task_id: task})) + config = SForgeConfig(log_dir=tmp_path / "judge", tasks_dir=tmp_path / "tasks", + judge_max_concurrent=1, judge_max_pending=0, + judge_max_outstanding_per_run=1 if per_run_limit else None) + with TestClient(native.create_app(config)) as client: + state = client.app.state.judge + token = state.register_session(task_id, run_id) + def history(): + response = client.get("/api/v1/history", params={"token": token}) + response.raise_for_status() + return response.json() + root = tmp_path / "feedback" + root.mkdir() + (root / "native.json").write_bytes(canonical_json(dict( + schema_version=SCHEMA, run_id=run_id, task_id=task_id, records=[]))) + archive = tmp_path / "packaged.tar.gz" + archive.write_bytes(archive_bytes({"candidate.py": b"original"})) + prepared = prepare_submission(root, archive, history_reader=history) + mark_dispatched(root, prepared["operation_id"]) + # Occupy real native admission capacity without starting an evaluation. + state._reserve(run_id, task_id) + try: + response = client.post("/api/v1/submit", data={"token": token}, + files={"archive": ("submission.tar.gz", Path(prepared["archive_path"]).read_bytes())}) + assert response.status_code == 503 and response.headers["retry-after"] == "5" + intent = record_response(root, prepared["operation_id"], response.status_code, response.json()) + assert intent["acceptance"] == "rejected" + finally: + state._release(run_id, task_id) + assert history()["entries"] == [] + assert state.resolve_token(token)["next_agent"] == 1 + state._reserve(run_id, task_id) + state._release(run_id, task_id) + later = prepare_submission(root, archive, history_reader=history) + assert later["action"] == "submit" and later["operation_id"] != prepared["operation_id"] + assert not state.submissions and history()["entries"] == [], "neither response handling nor prepare resubmits" + + +@pytest.mark.parametrize("http_status,response", [ + (503, {"detail": "Service Unavailable"}), + (503, {"detail": "Judge capacity full; retry this snapshot later; acceptance unknown"}), + (500, {"detail": "Judge capacity full; retry this snapshot later"}), + (502, {"detail": "Bad Gateway"}), + (503, {"detail": {"message": "Judge capacity full; retry this snapshot later"}}), +]) +def test_unproven_server_failure_cannot_allow_resubmission(tmp_path, http_status, response): + root = tmp_path / "feedback" + root.mkdir() + (root / "native.json").write_bytes(canonical_json(dict(schema_version=SCHEMA, + run_id="synthetic-run", task_id="synthetic-task", records=[]))) + archive = tmp_path / "packaged.tar.gz" + archive.write_bytes(archive_bytes({"candidate.py": b"original"})) + prepared = prepare_submission(root, archive, + history_reader=lambda: {"run_id": "synthetic-run", "entries": []}) + mark_dispatched(root, prepared["operation_id"]) + intent = record_response(root, prepared["operation_id"], http_status, response) + assert intent["acceptance"] == "unknown" + with pytest.raises(ValueError, match="native_acceptance_unknown_no_resubmit"): + prepare_submission(root, archive) + assert len(list((root / "intents").glob("*.json"))) == 1 + + +def test_lost_reply_recovers_same_operation_and_never_resubmits(tmp_path): + with native_http(tmp_path) as (state, release, observer, root): + archive = tmp_path / "packaged.tar.gz" + archive.write_bytes(archive_bytes({"candidate.py": b"original"})) + initial = prepare_submission(root, archive, + history_reader=lambda: {"run_id": state["run_id"], "entries": []}) + operation = initial["operation_id"] + mark_dispatched(root, operation) + accepted = submit_fixture(observer, Path(initial["archive_path"]).read_bytes()) + # Simulate response loss: there is no record_response call. + with pytest.raises(ValueError, match="unknown_no_resubmit"): + prepare_submission(root, archive) + packet = observer.update(LocalCopyBackend(), None) + [intent] = synchronize_intents(root, packet) + assert intent["operation_id"] == operation + assert intent["submission_id"] == accepted["submission_id"] + archive.write_bytes(archive_bytes({"candidate.py": b"newer"})) + recovered = prepare_submission(root, archive) + assert recovered == dict(action="recover", operation_id=operation, submission_id=accepted["submission_id"]) + assert state["posts"] == 1 + with pytest.raises(ValueError, match="already_dispatched"): + mark_dispatched(root, operation) + + +def test_ambiguous_archives_remain_unknown_and_duplicate_native_ids_cannot_release(tmp_path): + with native_http(tmp_path) as (state, release, observer, root): + source = tmp_path / "packaged.tar.gz" + source.write_bytes(archive_bytes({"candidate.py": b"original"})) + prepared = prepare_submission(root, source, + history_reader=lambda: {"run_id": state["run_id"], "entries": []}) + mark_dispatched(root, prepared["operation_id"]) + for _ in range(2): + submit_fixture(observer, source.read_bytes()) + packet = observer.update(LocalCopyBackend(), None) + [intent] = synchronize_intents(root, packet) + assert intent["acceptance"] == "unknown" + assert intent["recovery_reason"] == "ambiguous_native_archive_match" + with pytest.raises(ValueError, match="unknown_no_resubmit"): + prepare_submission(root, source) + state["entries"].append(dict(state["entries"][0])) + duplicate = observer.poll() + rows = [row for row in duplicate["records"] if row["submission_id"] == "submission-1"] + assert all(row["attribution_status"] == "unknown" and "result_ref" not in row for row in rows) + + +def test_pending_acceptance_can_register_wait_before_host_poll_without_result_attribution(tmp_path): + with native_http(tmp_path) as (state, release, observer, root): + archive = tmp_path / "packaged.tar.gz" + archive.write_bytes(archive_bytes({"candidate.py": b"original"})) + prepared = prepare_submission(root, archive, wait_todo_id="requested-work", + history_reader=lambda: {"run_id": state["run_id"], "entries": []}) + mark_dispatched(root, prepared["operation_id"]) + receipt = submit_fixture(observer, archive.read_bytes()) + intent = record_response(root, prepared["operation_id"], 200, receipt) + [row] = declared_wait_packet(root, intent)["records"] + assert row["accepted_record"] and row["status"] == "queued" + assert row["submission_id"] == receipt["submission_id"] + assert row["attribution_status"] == "unknown" and "result_ref" not in row + assert observer.snapshot()["records"] == [], "the receipt fills only this explicitly requested wait" + + +def test_failure_to_publish_control_effect_retries_and_restart_preserves_release_time(tmp_path): + with native_http(tmp_path, immediately_complete=True) as (state, release, observer, root): + submit_fixture(observer, archive_bytes({"candidate.py": b"original"})) + attempts = [] + def after_publish(*_args): + attempts.append(True) + if len(attempts) == 1: + raise RuntimeError("synthetic control writeback interruption") + observer.after_publish = after_publish + with pytest.raises(RuntimeError, match="control writeback"): + observer.update(LocalCopyBackend(), None) + completed = observer.update(LocalCopyBackend(), None) + assert len(attempts) == 2 + duplicate = observer.update(LocalCopyBackend(), None) + assert duplicate == completed and len(attempts) == 2 + restarted = NativeResearchFeedback(trial=observer.trial, run_id=state["run_id"], + task_id=state["task_id"], judge_url=observer.judge_url, remote_root=str(root)) + try: + restarted.bind({"SFORGE_TOKEN": "synthetic-token"}) + assert restarted.poll() == completed + finally: + restarted.close() + + +def test_mismatched_released_details_and_native_symlink_stay_unattributed(tmp_path): + with native_http(tmp_path, immediately_complete=True) as (state, release, observer, root): + receipt = submit_fixture(observer, archive_bytes({"candidate.py": b"original"})) + state["results"][receipt["submission_id"]]["report"]["task_id"] = "unrelated-task" + [row] = observer.poll()["records"] + assert row["attribution_status"] == "unknown" and "result_ref" not in row + archive = observer.trial / "submissions" / "agent-1" / "submission.tar.gz" + outside = tmp_path / "untrusted.tar.gz" + archive.rename(outside) + archive.symlink_to(outside) + [row] = observer.poll()["records"] + assert row["reason"] == "native_archive_not_physical" and "archive_sha256" not in row + + +def test_archive_manifest_rejects_ambiguous_paths_links_and_matches_probe_hash(): + observation, _ = probe_record() + digest, _ = archive_manifest(archive_bytes({"./candidate.py": b"original"}), ["candidate.py"]) + assert digest == observation["candidate_sha256"] + for files in ({"candidate.py": b"x", "./candidate.py": b"y"}, {"../candidate.py": b"x"}): + with pytest.raises(ValueError, match="ambiguous_archive_member"): + archive_manifest(archive_bytes(files)) + + +@pytest.mark.parametrize("lose_reply", [False, True]) +def test_actual_native_submit_script_archives_once_and_recovers_lost_reply(tmp_path, lose_reply): + native = pytest.importorskip("sforge.harness.evolve_scripts") + with native_http(tmp_path, immediately_complete=True, lose_reply=lose_reply) as (state, release, observer, root): + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "candidate.py").write_bytes(b"original") + generated = native.generate_submit_script() + # Give this real client a private display cache rather than the fixed + # global /tmp display cache; admission/packaging/HTTP paths stay native. + generated = generated.replace('STATE_FILE="/tmp/sforge_state.json"', + 'STATE_FILE="' + str(tmp_path / "display.json") + '"') + script = tmp_path / "sforge-submit" + script.write_text(instrument_submit_script(generated, + [sys.executable, "-m", "benchmark.runtime.research_submission"], str(root))) + script.chmod(0o755) + env = dict(os.environ, SFORGE_JUDGE_URL=observer.judge_url, SFORGE_TOKEN="synthetic-token", + SFORGE_PATCH_DIR=str(workspace), SFORGE_SUBMIT_PATHS="candidate.py", + PYTHONPATH=str(Path(__file__).resolve().parents[2]), NO_PROXY="127.0.0.1") + first = subprocess.run([str(script)], cwd=workspace, env=env, capture_output=True, text=True, timeout=30) + assert (first.returncode != 0) == lose_reply, first.stderr + assert state["posts"] == 1 + [intent_file] = list((root / "intents").glob("*.json")) + intent = json.loads(intent_file.read_bytes()) + archive = observer.trial / "submissions" / "agent-1" / "submission.tar.gz" + assert archive.read_bytes() == Path(intent["archive_path"]).read_bytes() + if lose_reply: + observer.update(LocalCopyBackend(), None) + (workspace / "candidate.py").write_bytes(b"newer workspace") + recovered = subprocess.run([str(script)], cwd=workspace, env=env, + capture_output=True, text=True, timeout=30) + assert recovered.returncode == 0, recovered.stderr + assert "Reading existing submission: submission-1" in recovered.stdout + assert state["posts"] == 1, "recovery never creates a second native opportunity" + assert json.loads(intent_file.read_bytes())["operation_id"] == intent["operation_id"] + + +def test_native_client_hook_requires_the_observed_packaging_boundaries(): + with pytest.raises(ValueError, match="unsupported_native_submit_script"): + instrument_submit_script("#!/bin/sh\nexit 0\n", [sys.executable, "-m", "fixture"], "/synthetic") + + +def test_ambiguous_record_does_not_prevent_an_independent_verified_result(tmp_path): + from benchmark.runtime.research_feedback_wait import _records + with native_http(tmp_path, immediately_complete=True) as (state, release, observer, root): + submit_fixture(observer, archive_bytes({"candidate.py": b"first"})) + independent = submit_fixture(observer, archive_bytes({"candidate.py": b"second"})) + state["entries"].append(dict(state["entries"][0])) + packet = observer.poll() + accepted = _records(packet) + assert set(accepted) == {independent["submission_id"]} + assert accepted[independent["submission_id"]][1]["result_ref"] + + +def test_malformed_terminal_feedback_cannot_make_a_retry_submit_again(tmp_path): + with native_http(tmp_path) as (state, release, observer, root): + source = tmp_path / "packaged.tar.gz" + source.write_bytes(archive_bytes({"candidate.py": b"original"})) + prepared = prepare_submission(root, source, + history_reader=lambda: {"run_id": state["run_id"], "entries": []}) + mark_dispatched(root, prepared["operation_id"]) + receipt = submit_fixture(observer, source.read_bytes()) + record_response(root, prepared["operation_id"], 200, receipt) + with pytest.raises(ValueError, match="completed_result_scope"): + record_result(root, receipt["submission_id"], + dict(submission_id=receipt["submission_id"], status="completed", report=None)) + assert prepare_submission(root, source)["action"] == "recover" + assert state["posts"] == 1 + + +def test_accepted_archive_identity_cannot_be_replaced_after_first_observation(tmp_path): + with native_http(tmp_path, immediately_complete=True) as (state, release, observer, root): + submit_fixture(observer, archive_bytes({"candidate.py": b"original"})) + original = observer.update(LocalCopyBackend(), None) + native_archive = observer.trial / "submissions" / "agent-1" / "submission.tar.gz" + native_archive.write_bytes(archive_bytes({"candidate.py": b"changed after acceptance"})) + [changed] = observer.update(LocalCopyBackend(), None)["records"] + assert changed["attribution_status"] == "unknown" + assert changed["reason"] == "native_accepted_archive_changed" + assert "result_ref" not in changed + restarted = NativeResearchFeedback(trial=observer.trial, run_id=state["run_id"], + task_id=state["task_id"], judge_url=observer.judge_url, remote_root=str(root)) + try: + restarted.bind({"SFORGE_TOKEN": "synthetic-token"}) + assert restarted.poll()["records"][0]["reason"] == "native_accepted_archive_changed" + finally: + restarted.close() + assert Path(original["records"][0]["archive_path"]).read_bytes() != native_archive.read_bytes() diff --git a/benchmark/tests/test_research_cost.py b/benchmark/tests/test_research_cost.py new file mode 100644 index 0000000000..54d8575312 --- /dev/null +++ b/benchmark/tests/test_research_cost.py @@ -0,0 +1,188 @@ +"""Passive cost readback uses captured transport facts and the real TS owner.""" +from __future__ import annotations + +import copy +import json + +import pytest + +from benchmark.edgebench.research_cost import summarize_research_cost +from benchmark.edgebench.research_feedback import SCHEMA + + +def _write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value), encoding="utf-8") + + +def _lines(path, events): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(event) + "\n" for event in events), encoding="utf-8") + + +def _collected(trial, names=("runtime", "native-sessions", "worker-sessions")): + _write(trial / "collected/artifact-collection.json", + [{"artifact": name, "collected": True} for name in names]) + + +def _session(session_id): + def usage(i, c, o): + return {"input_tokens": i, "cached_input_tokens": c, "output_tokens": o} + + rows = [ + {"type": "session_meta", "payload": {"id": session_id}}, + {"type": "event_msg", "payload": {"type": "task_started", "turn_id": "a"}}, + {"type": "event_msg", "payload": {"type": "token_count", "info": { + "total_token_usage": usage(100, 60, 10), "last_token_usage": usage(100, 60, 10)}}}, + {"type": "event_msg", "payload": {"type": "task_complete", "turn_id": "a"}}, + {"type": "event_msg", "payload": {"type": "task_started", "turn_id": "b"}}, + {"type": "event_msg", "payload": {"type": "token_count", "info": { + "total_token_usage": usage(140, 80, 15), "last_token_usage": usage(40, 20, 5)}}}, + {"type": "event_msg", "payload": {"type": "task_complete", "turn_id": "b"}}, + ] + for index, row in enumerate(rows): + row["timestamp"] = f"2026-01-01T00:00:0{index}Z" + return rows + + +def test_missing_captures_are_unknown_and_wall_is_not_a_sum_of_stages(tmp_path): + report = summarize_research_cost(tmp_path, 123.5) + assert report["runner_wall_seconds"] == 123.5 + assert report["stage_seconds"] is None and report["billing_usd"] is None + assert report["provider_requests"]["capture_coverage"] == "unknown" + assert report["provider_requests"]["captured_distinct_requests"] is None + assert report["provider_requests"]["forwarded"] is None + assert report["tokens"]["known_captured_totals"] is None + assert report["tokens"]["capture_coverage"] == "unknown" + assert report["evaluations"]["manual_accepted_count"] is None + assert report["evaluations"]["native_total_rounds"] is None + assert summarize_research_cost(tmp_path, float("nan"))["runner_wall_seconds"] is None + + +def test_gateway_replicas_dedup_and_interrupted_dispatch_remains_unknown(tmp_path): + _collected(tmp_path) + events = [ + {"request_id": "private-forwarded", "event": "dispatch", "forwarded": None}, + {"request_id": "private-forwarded", "event": "finished", "forwarded": True}, + {"request_id": "private-refused", "event": "finished", "forwarded": False}, + {"request_id": "private-lost", "event": "dispatch", "forwarded": None}, + {"request_id": "private-maybe", "event": "dispatch", "forwarded": None}, + {"request_id": "private-maybe", "event": "finished", "forwarded": None}, + {"request_id": "private-conflict", "event": "finished", "forwarded": True}, + {"request_id": "private-conflict", "event": "finished", "forwarded": False}, + ] + _lines(tmp_path / "collected/runtime/agent/first/provider-requests.jsonl", events) + _lines(tmp_path / "collected/runtime/agent/replica/provider-requests.jsonl", events) + result = summarize_research_cost(tmp_path, 1) + counts = result["provider_requests"] + assert counts["captured_distinct_requests"] == 5 + assert (counts["forwarded"], counts["not_forwarded"], counts["send_unknown"]) == (1, 1, 3) + assert "private-" not in json.dumps(result) + + +def test_unreadable_capture_is_not_zero_but_an_observed_empty_log_can_be(tmp_path): + _collected(tmp_path) + source = tmp_path / "outside.jsonl" + _lines(source, [{"request_id": "outside", "event": "finished", "forwarded": True}]) + path = tmp_path / "collected/runtime/agent/provider-requests.jsonl" + path.parent.mkdir(parents=True) + path.symlink_to(source) + counts = summarize_research_cost(tmp_path, 1)["provider_requests"] + assert counts["capture_coverage"] == "unknown" + assert counts["forwarded"] is None + path.unlink() + _lines(path, []) + counts = summarize_research_cost(tmp_path, 1)["provider_requests"] + assert counts["capture_coverage"] == "complete" and counts["forwarded"] == 0 + + +def test_real_usage_owner_dedups_same_and_prefix_captures_across_runtime_homes(tmp_path): + _collected(tmp_path) + session = _session("private-native-session") + # A retained prefix, worker home and copied runtime are the same session. + _lines(tmp_path / "collected/native-sessions/sessions/old.jsonl", session[:4]) + _lines(tmp_path / "collected/worker-sessions/sessions/full.jsonl", session) + _lines(tmp_path / "collected/runtime/agent/loopx/sessions/replica.jsonl", session) + _lines(tmp_path / "collected/runtime/agent/other/sessions/independent.jsonl", + _session("private-independent-session")[:4]) + # Non-session runtime JSONL is not interpreted as a token capture. + _lines(tmp_path / "collected/runtime/agent/decisions.jsonl", [{"not": "a rollout"}]) + report = summarize_research_cost(tmp_path, 420) + tokens = report["tokens"] + assert tokens["capture_coverage"] == "complete" + assert tokens["whole_run_coverage"] == "unknown" + assert tokens["qualified_sessions"] == 2 and tokens["duplicate_or_prefix_captures"] == 2 + assert tokens["known_captured_totals"] == { + "input_tokens": 240, "cached_input_tokens": 140, "uncached_input_tokens": 100, + "output_tokens": 25, "input_plus_output_tokens": 265, + } + assert "private-" not in json.dumps(report) and str(tmp_path) not in json.dumps(report) + _collected(tmp_path, ("runtime",)) + assert summarize_research_cost(tmp_path, 420)["tokens"]["capture_coverage"] == "partial" + + +@pytest.mark.parametrize("case", ["divergent", "forked", "unfinished", "counter_gap"]) +def test_ambiguous_or_incomplete_session_never_adds_an_unproven_subtotal(tmp_path, case): + _collected(tmp_path) + rows = _session("ambiguous-session") + if case == "divergent": + other = copy.deepcopy(rows) + other[-1]["timestamp"] = "2026-01-01T00:00:07Z" + _lines(tmp_path / "collected/worker-sessions/sessions/divergence.jsonl", other) + elif case == "forked": + rows[0]["payload"]["forked_from_id"] = "parent-session" + elif case == "unfinished": + # The longest capture is unfinished; do not fall back to an older + # completed prefix and call the missing second Turn free. + _lines(tmp_path / "collected/worker-sessions/sessions/older.jsonl", rows[:4]) + rows = rows[:-1] + else: + rows[5]["payload"]["info"]["total_token_usage"]["input_tokens"] = 999 + _lines(tmp_path / "collected/native-sessions/sessions/unknown.jsonl", rows) + _lines(tmp_path / "collected/runtime/agent/sessions/known.jsonl", _session("known")[:4]) + tokens = summarize_research_cost(tmp_path, 1)["tokens"] + assert tokens["capture_coverage"] == "partial" and tokens["qualified_sessions"] == 1 + assert tokens["unknown_sessions_or_files"] == 1 + assert tokens["known_captured_totals"]["input_tokens"] == 100 + assert tokens["known_captured_totals"]["input_plus_output_tokens"] == 110 + + +def test_native_acceptance_order_precedes_release_order_and_total_uses_native_report(tmp_path): + first = {"submission_id": "private-first", "round_id": "agent-1", "accepted_record": True, + "status": "running", "attribution_status": "verified"} + second = {"submission_id": "private-second", "round_id": "agent-2", "accepted_record": True, + "status": "completed", "attribution_status": "verified", "result_ref": "native-result:abc", + "result_summary": {"score": 4}} + packet = {"schema_version": SCHEMA, "run_id": "private-run", "task_id": "private-task", + "records": [second, first, second]} + target = tmp_path / "native-research-feedback/native.json" + _write(target, packet) + _write(tmp_path / "final_result.json", {"total_rounds": 5, "private": "not part of cost"}) + evaluations = summarize_research_cost(tmp_path, 1)["evaluations"] + assert evaluations["manual_accepted_count"] == 2 and evaluations["native_total_rounds"] == 5 + assert evaluations["first_accepted"] == {"acceptance_ordinal": 1, "released": False, "result_summary": None} + first.update(status="completed", result_ref="native-result:def", result_summary={ + "score": 0, "valid": False, "feedback": "private model feedback", "passed": 0}) + _write(target, packet) + report = summarize_research_cost(tmp_path, 1) + assert report["evaluations"]["first_accepted"]["result_summary"] == {"score": 0, "valid": False, "passed": 0} + assert "private" not in json.dumps(report) + # A disagreement about an accepted identity must not double count it or + # select the earlier-released second candidate as the first submission. + packet["records"].append({**first, "round_id": "agent-3"}) + _write(target, packet) + evaluations = summarize_research_cost(tmp_path, 1)["evaluations"] + assert evaluations["capture_coverage"] == "partial" and evaluations["manual_accepted_count"] == 1 + assert evaluations["first_accepted"] is None + + +def test_bad_native_scopes_and_round_collisions_do_not_invent_counts(tmp_path): + target = tmp_path / "native-research-feedback/native.json" + _write(target, {"schema_version": SCHEMA, "run_id": 123, "task_id": "t", "records": []}) + assert summarize_research_cost(tmp_path, 1)["evaluations"]["manual_accepted_count"] is None + _write(target, {"schema_version": SCHEMA, "run_id": "r", "task_id": "t", "records": [ + {"accepted_record": True, "submission_id": "a", "round_id": "agent-1"}, + {"accepted_record": True, "submission_id": "b", "round_id": "agent-1"}, + ]}) + result = summarize_research_cost(tmp_path, 1)["evaluations"] + assert result["capture_coverage"] == "partial" and result["first_accepted"] is None diff --git a/benchmark/tests/test_sforge_runtime.py b/benchmark/tests/test_sforge_runtime.py index e0cce406b9..aa57c3bf8e 100644 --- a/benchmark/tests/test_sforge_runtime.py +++ b/benchmark/tests/test_sforge_runtime.py @@ -94,6 +94,23 @@ def test_only_official_delegates_continuation_to_sforge(profile, resumes, monkey assert 'web_search="disabled"' in command +def test_research_loop_uses_one_explicit_profile_and_rejects_conflicting_controls(): + pytest.importorskip("sforge") + pytest.importorskip("harbor") + from sforge.harness.config import SForgeConfig + from benchmark.runtime.sforge import SForgeWorker + + config = SForgeConfig(agent_model="fixture", agent_effort="high", + agent_api_base_url="https://model.example.invalid/v1", agent_api_key="fixture-credential") + worker = SForgeWorker(config, profile="heartbeat-explore", cwd="/task", research_loop=True) + assert worker.research_loop and worker.stride_strategy == "evidence_conditioned_v1" + for options in ({"stride_strategy": "off"}, {"profile": "official"}, {"warm_start": object()}): + with pytest.raises(ValueError): + SForgeWorker(config, **{"profile": "heartbeat-explore", "cwd": "/task", "research_loop": True, **options}) + ordinary = SForgeWorker(config, profile="heartbeat-explore", cwd="/task") + assert ordinary.research_loop is False and ordinary.stride_strategy is None + + def test_invalid_worker_inputs_fail_before_install(monkeypatch): pytest.importorskip("sforge") pytest.importorskip("harbor") @@ -401,6 +418,7 @@ def test_artifact_collection_preserves_both_session_homes_without_auth(tmp_path, pytest.importorskip("harbor") from benchmark.runtime.sforge_backend import RecordingDockerBackend, DockerBackend backend = object.__new__(RecordingDockerBackend) + backend.research_feedback = None backend.checkpoint = None backend.log_dir = tmp_path / "artifacts" backend.feedback = None @@ -455,6 +473,7 @@ def test_blind_policy_removes_judge_route_and_credentials_native_is_unchanged(mo pytest.importorskip("harbor") from benchmark.runtime.sforge_backend import RecordingDockerBackend, DockerBackend, AllowedEndpoint backend = object.__new__(RecordingDockerBackend) + backend.research_feedback = None backend.checkpoint = None backend.auth_ips = [] backend.blind_api_endpoint = None @@ -973,6 +992,7 @@ def test_recorded_solver_exit_does_not_take_status_from_output(tmp_path, monkeyp from types import SimpleNamespace from benchmark.runtime.sforge_backend import RecordingDockerBackend, DockerBackend backend = object.__new__(RecordingDockerBackend) + backend.research_feedback = None backend.checkpoint = None backend.log_dir = tmp_path / "collected" backend.blind_api_endpoint = None @@ -1195,3 +1215,96 @@ def before_solver(**kwargs): args += ["--checkpoint-request", str(declaration)] with pytest.raises(RuntimeError, match="validated before model execution"): run.main(args) + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_native_research_install_is_explicit_and_consumer_owner_excludes_credentials(tmp_path, monkeypatch, enabled): + pytest.importorskip("sforge") + pytest.importorskip("harbor") + from types import SimpleNamespace + from sforge.harness.config import SForgeConfig + from sforge.harness.evolve_scripts import generate_submit_script + from benchmark.runtime.sforge import BenchmarkCodex, SForgeWorker + from benchmark.runtime.research_feedback_sync import FEEDBACK_ROOT, feedback_environment + + async def installed(*_args): + pass # Bound packaging/owner install; no network setup or model call. + monkeypatch.setattr(BenchmarkCodex, "install", installed) + native = generate_submit_script() + copied, calls, starts = {}, [], [] + observer = SimpleNamespace(after_publish=None, start=lambda *args: starts.append(args)) + def execute(_handle, command, **kwargs): + calls.append((command, kwargs)) + return SimpleNamespace(exit_code=0, output=native if command == ["cat", "/usr/local/bin/sforge-submit"] else "") + def copy(_handle, path, target): + copied[str(target)] = path.read_bytes() + backend = SimpleNamespace(research_feedback=observer, exec_run=execute, copy_to_container=copy) + worker = SForgeWorker(SForgeConfig(agent_model="synthetic-model", agent_effort="high", + agent_api_base_url="https://model.example.invalid/v1", agent_api_key="synthetic-provider-secret"), + profile="heartbeat-explore", cwd="/task", research_loop=enabled) + worker.install_stop_hook(backend, "synthetic-container", tmp_path, None) + env = worker.runtime._worker_env(cwd="/task") + script = copied.get("/usr/local/bin/sforge-submit") + if not enabled: + assert script is None and not calls + assert "LOOPX_RESEARCH_FEEDBACK_ROOT" not in env + worker._execution_command("synthetic-execution") + assert not starts and observer.after_publish is None + return + assert script and script.decode() != native + parsed = subprocess.run(["bash", "-n"], input=script, capture_output=True) + assert parsed.returncode == 0, parsed.stderr + assert env["LOOPX_RESEARCH_FEEDBACK_ROOT"] == FEEDBACK_ROOT + assert b"--wait-todo" in script and b"benchmark.runtime.research_submission" in script + assert b'sforge-submit --result' in script, "native read-only recovery remains available" + env.update(SFORGE_TOKEN="synthetic-native-secret", OPENAI_API_KEY="synthetic-provider-secret", + CODEX_AUTH_JSON_PATH="/synthetic-unused-auth.json") + consumer_calls = [] + def consume(_handle, command, **kwargs): + consumer_calls.append((command, kwargs)) + return SimpleNamespace(exit_code=0, timed_out=False) + backend.exec_run_with_timeout = consume + worker._prepare_native_research_consumer(env) + owner = json.loads(copied[FEEDBACK_ROOT + "/owner.json"]) + assert not {"SFORGE_TOKEN", "OPENAI_API_KEY", "CODEX_AUTH_JSON_PATH"} & owner.keys() + assert "synthetic-provider-secret" not in json.dumps(owner) + root = tmp_path / "feedback-owner" + root.mkdir() + (root / "owner.json").write_text(json.dumps({**owner, "LOOPX_RESEARCH_FEEDBACK_ROOT": str(root)})) + restored = feedback_environment(root, base={"UNRELATED": "kept"}) + assert restored["LOOPX_GOAL_ID"] == env["LOOPX_GOAL_ID"] + assert restored["LOOPX_AGENT_ID"] == env["LOOPX_AGENT_ID"] + assert restored["UNRELATED"] == "kept" and "OPENAI_API_KEY" not in restored + worker._execution_command("synthetic-execution") + assert starts == [(backend, "synthetic-container")] + observer.after_publish(backend, "synthetic-container", {}) + [command, options] = consumer_calls[0] + assert "benchmark.runtime.research_feedback_sync" in command + assert command[:4] == ["timeout", "--signal=TERM", "--kill-after=5", "60s"] + assert options["timeout"] == 70 + assert options["user"] == "agent" and options["environment"] == owner + backend.exec_run_with_timeout = lambda *_args, **_kwargs: SimpleNamespace(exit_code=1, timed_out=False) + with pytest.raises(RuntimeError, match="consumption is incomplete"): + observer.after_publish(backend, "synthetic-container", {}) + + +def test_native_research_installer_rejects_unobserved_or_changed_provider_before_execution(tmp_path, monkeypatch): + pytest.importorskip("sforge") + pytest.importorskip("harbor") + from types import SimpleNamespace + from sforge.harness.config import SForgeConfig + from benchmark.runtime.sforge import SForgeWorker + + worker = SForgeWorker(SForgeConfig(agent_model="synthetic", agent_effort="high", + agent_api_base_url="https://model.example.invalid/v1", agent_api_key="synthetic-secret"), + profile="heartbeat-explore", cwd="/task", research_loop=True) + with pytest.raises(ValueError, match="accepted-snapshot observer"): + worker._install_native_research_feedback(SimpleNamespace(research_feedback=None), None) + missing_capability = SimpleNamespace(research_feedback=object(), + exec_run=lambda *_args, **_kwargs: SimpleNamespace(exit_code=1, output="")) + with pytest.raises(RuntimeError, match="matching Explore observer capability"): + worker._install_native_research_feedback(missing_capability, None) + unsupported = SimpleNamespace(research_feedback=object(), + exec_run=lambda *_args, **_kwargs: SimpleNamespace(exit_code=0, output="#!/bin/bash\nexit 0\n")) + with pytest.raises(ValueError, match="unsupported_native_submit_script_boundary"): + worker._install_native_research_feedback(unsupported, None) diff --git a/loopx/capabilities/explore/result_writeback.py b/loopx/capabilities/explore/result_writeback.py index 8497e13fdb..59c8f96254 100644 --- a/loopx/capabilities/explore/result_writeback.py +++ b/loopx/capabilities/explore/result_writeback.py @@ -254,6 +254,72 @@ def prepare_result_attachment( return result +def record_observer_result_attachment( + attachment, *, registry_path, runtime_root, goal_id, agent_id, todo_id, + source_id, recorded_at, +): + """Record external facts without settling a model work Turn. + + A runner or Monitor may finish real observation work while every model + Todo is waiting. It uses the same attachment codec, event log and Todo + link owner as ordinary writeback. It cannot spend a slot, acknowledge a + replan, evolve a research question or declare a Goal outcome. The caller + may complete its observation dependency only after this durable readback. + """ + attachment = normalize_result_attachment(attachment) + if not isinstance(source_id, str) or not source_id or not isinstance(recorded_at, str) or not recorded_at: + raise ValueError("External observation requires stable source and observation-time identity") + if _evolves_question(attachment): + raise ValueError("External observation cannot decide question evolution") + _, graph, gate = _policy(registry_path, goal_id) + if not (graph or gate["enabled"]): + raise ValueError("External result observation requires enabled Explore evidence") + detail = list_goal_todos(registry_path=registry_path, runtime_root_arg=str(runtime_root), + goal_id=goal_id, todo_id=todo_id) + todo = detail.get("todo") + if (not detail.get("matched") or detail.get("ambiguous") or not isinstance(todo, dict) + or todo.get("claimed_by") != agent_id or todo.get("task_class") != "advancement_task" + or todo.get("action_kind") != "observe" or todo.get("status") not in {"open", "done"}): + raise ValueError("External result must belong to the caller's finite observation Todo") + expected = _events(attachment, goal_id=goal_id, agent_id=agent_id, + source_id=source_id, recorded_at=recorded_at) + path = explore_result_log_path(runtime_root, goal_id) + existing = load_explore_result_events_strict(path, goal_id=goal_id) + projection = build_explore_result_projection(existing, goal_id=goal_id) + for node in projection.get("nodes", []): + if node.get("node_id") == attachment["node_id"] and any( + node.get(key) != expected[0].get(key) for key in ("node_kind", "title", "summary") + ): + raise ValueError("External observation question conflicts with existing scope") + if expected[1] not in existing or attachment["node_id"] not in (todo.get("explore_result_node_refs") or []): + if todo.get("status") != "open": + raise ValueError("An already completed observation cannot acquire new result evidence") + # Reject a stable source identity being reused for different facts. + collisions = [row for row in existing if row.get("event_kind") == "finding" + and row.get("result_id") == expected[1]["result_id"]] + if collisions and expected[1] not in collisions: + raise ValueError("External result source identity changed") + proof = _link_arguments(registry_path=registry_path, runtime_root=runtime_root, + goal_id=goal_id, agent_id=agent_id, todo_id=todo_id, node_id=attachment["node_id"]) + if not update_goal_todo(**proof, dry_run=True).get("ok"): + raise ValueError("External observation Todo link was not admitted") + append_explore_result_events(path, expected, expected_goal_id=goal_id, + create_only_node_ids=[attachment["node_id"]], + expected_event_ids=[row["event_id"] for row in existing]) + # The normal mutation rechecks claim/lease after append. A concurrent + # change leaves a replayable fact, never a falsely released dependency. + linked = update_goal_todo(**proof, dry_run=False) + if not linked.get("ok"): + raise ValueError("External observation Todo link is incomplete") + stored = load_explore_result_events_strict(path, goal_id=goal_id) + current = list_goal_todos(registry_path=registry_path, runtime_root_arg=str(runtime_root), + goal_id=goal_id, todo_id=todo_id).get("todo") + if (expected[1] not in stored or not isinstance(current, dict) + or attachment["node_id"] not in (current.get("explore_result_node_refs") or [])): + raise ValueError("External observation readback is incomplete") + return {"ok": True, "finding_id": expected[1]["result_id"], "node_id": attachment["node_id"]} + + def deliver_result_attachment( *, payload, diff --git a/scripts/external_scheduler_worker.py b/scripts/external_scheduler_worker.py index 480bbf1d2f..def7035afb 100644 --- a/scripts/external_scheduler_worker.py +++ b/scripts/external_scheduler_worker.py @@ -523,7 +523,7 @@ def run_worker( sleep(max(5, interval_minutes * 60)) -def main(argv: Sequence[str] | None = None) -> int: +def main(argv: Sequence[str] | None = None, *, sleep: Callable[[float], None] | None = None) -> int: parser = argparse.ArgumentParser( description="Scheduler-hint-aware external shell_worker for generic CLI loops.", ) @@ -588,7 +588,7 @@ def main(argv: Sequence[str] | None = None) -> int: args.registry = str(global_registry_path()) except ValueError as exc: parser.error(str(exc)) - return run_worker(args) + return run_worker(args) if sleep is None else run_worker(args, sleep=sleep) if __name__ == "__main__": diff --git a/tests/control_plane/test_native_research_continuity.py b/tests/control_plane/test_native_research_continuity.py new file mode 100644 index 0000000000..c863e0ee53 --- /dev/null +++ b/tests/control_plane/test_native_research_continuity.py @@ -0,0 +1,237 @@ +"""A real probe, native HTTP acceptance, canonical wait and next experiment.""" +from __future__ import annotations + +import json +from pathlib import Path +import sys + +import pytest + +from benchmark.runtime.research_probe import run_probe +from benchmark.runtime.research_feedback_sync import synchronize_native_feedback +from benchmark.runtime.research_submission import ( + mark_dispatched, prepare_submission, record_response, register_probe, synchronize_intents, +) +from benchmark.tests.test_native_research_feedback import ( + LocalCopyBackend, archive_bytes, native_http, probe_record, submit_fixture, +) +from test_research_feedback_wait import runtime as runtime # real File/SQLite fixture +import test_quota_settlement_cli as fixture + + +def test_observer_replays_after_result_and_todo_link_commit_but_response_is_lost(runtime, monkeypatch): + from benchmark.runtime.research_feedback_wait import synchronize_feedback_waits + from test_research_feedback_wait import packet, waiting, todos + from loopx.capabilities.explore import result_writeback + + dependency = waiting(runtime)["dependency_todo_id"] + released = packet(status="completed", attribution="verified") + released["records"][0]["result_observed_at"] = "2026-01-01T00:01:00Z" + original = result_writeback.update_goal_todo + interrupted = [] + + def lose_committed_response(**args): + result = original(**args) + if args.get("dry_run") is False and not interrupted: + interrupted.append(True) + raise RuntimeError("synthetic lost Todo-link response") + return result + + monkeypatch.setattr(result_writeback, "update_goal_todo", lose_committed_response) + with pytest.raises(RuntimeError, match="lost Todo-link response"): + synchronize_feedback_waits(runtime.env, released) + assert next(row for row in todos(runtime)["todos"] if row["todo_id"] == dependency)["status"] == "open" + assert not runtime.quota()["should_run"] + observed = synchronize_feedback_waits(runtime.env, released) + assert len(observed["released"]) == 1 + context = runtime.call("explore", "turn-context", *runtime.binding) + assert len(context["graph"]["writeback_results"]) == 1 + assert runtime.quota()["should_run"] + assert synchronize_feedback_waits(runtime.env, released) == {"released": []} + + +def test_released_independent_work_gets_a_wake_even_when_a_later_result_needs_retry(runtime, monkeypatch): + from benchmark.runtime import research_feedback_sync + from test_research_feedback_wait import packet, waiting, todos + + dependency = waiting(runtime)["dependency_todo_id"] + released = packet(status="completed", attribution="verified") + released["records"][0]["result_observed_at"] = "2026-01-01T00:01:00Z" + released["records"].append({**released["records"][0], "submission_id": "submission-2", + "round_id": "agent-2", "result_ref": "native-result:submission-2"}) + original = research_feedback_sync.write_feedback_finding + signal = Path(runtime.env["LOOPX_RESEARCH_FEEDBACK_ROOT"]) / "ingestion" / "wake.json" + published = [] + + def unavailable(env, packet, record, todo_id): + if record["submission_id"] == "submission-2": + # The first release must be visible while this consumer still + # owns the lock, before a later result has returned or failed. + published.append(json.loads(signal.read_text())["revision"]) + assert runtime.quota()["should_run"] + raise RuntimeError("synthetic independent result needs retry") + return original(env, packet, record, todo_id) + + monkeypatch.setattr(research_feedback_sync, "write_feedback_finding", unavailable) + with pytest.raises(RuntimeError, match="independent result needs retry"): + synchronize_native_feedback(runtime.env, packet=released) + assert len(published) == 1 and len(published[0]) == 64 + assert json.loads(signal.read_text())["revision"] == published[0] + assert next(row for row in todos(runtime)["todos"] if row["todo_id"] == dependency)["status"] == "done" + assert runtime.quota()["should_run"], "one failing result cannot suppress already released work" + + +def test_error_archive_and_independent_probe_result_reach_real_writeback(runtime): + from benchmark.runtime.research_feedback_wait import wait_for_feedback + + native = runtime.temp / "native" + native.mkdir() + with native_http(native) as (_state, release, observer, root): + env = dict(runtime.env, LOOPX_RESEARCH_FEEDBACK_ROOT=str(root)) + register_probe(root, *probe_record()) + invalid = submit_fixture(observer, b"not a gzip archive") + valid = submit_fixture(observer, archive_bytes({"candidate.py": b"original"})) + pending = observer.update(LocalCopyBackend(), None) + wait_for_feedback(env, todo_id=fixture.TODO_ID, + submission_id=valid["submission_id"], packet=pending) + assert not runtime.quota()["should_run"] + observer.after_publish = lambda _backend, _handle, packet: synchronize_native_feedback(env, packet=packet) + release(invalid["submission_id"], status="error") + release(valid["submission_id"], score=0) + completed = observer.update(LocalCopyBackend(), None) + records = {row["submission_id"]: row for row in completed["records"]} + error_ref = records[invalid["submission_id"]]["result_ref"] + valid_ref = records[valid["submission_id"]]["result_ref"] + assert "candidate_sha256" not in records[invalid["submission_id"]] + context = runtime.call("explore", "turn-context", *runtime.binding) + findings = context["graph"]["writeback_results"] + [error] = [row for row in findings if error_ref in row["evidence_refs"]] + assert '"status": "error"' in error["summary"] and not error.get("execution_observation") + valid_findings = [row for row in findings if valid_ref in row["evidence_refs"]] + assert len(valid_findings) == 2 + [bound] = [row for row in valid_findings if row.get("execution_observation")] + assert bound["execution_observation"]["candidate_sha256"] == probe_record()[0]["candidate_sha256"] + assert runtime.quota()["should_run"], "an unrelated invalid archive must not strand released work" + assert synchronize_native_feedback(env, packet=completed)["released"] == [] + assert runtime.call("explore", "turn-context", *runtime.binding)["graph"]["writeback_results"] == findings + + +def test_native_result_reaches_replan_and_a_changed_real_experiment(runtime, monkeypatch): + monkeypatch.chdir(runtime.project) + native_root = runtime.temp / "native" + native_root.mkdir() + runtime.call("configure-goal", "--goal-id", fixture.GOAL_ID, + "--execution-stride-strategy", "evidence_conditioned_v1", "--execute") + original = runtime.quota("--turn-instance-id", runtime.env["LOOPX_TURN"]) + original_budget = original["quota"]["spent_slots"] + with native_http(native_root) as (state, release, observer, root): + env = dict(runtime.env, LOOPX_RESEARCH_FEEDBACK_ROOT=str(root)) + monkeypatch.setenv("LOOPX_RESEARCH_FEEDBACK_ROOT", str(root)) + observer.after_publish = lambda _backend, _handle, packet: synchronize_native_feedback(env, packet=packet) + candidate = runtime.project / "candidate.py" + candidate.write_text("import json,os; from pathlib import Path; " + "Path(os.environ['LOOPX_RESEARCH_DIAGNOSTICS']).write_text(json.dumps(" + "{'search_iterations':2,'stop_reason':'search_cap'})); print(2)") + measured = run_probe(command=[sys.executable, str(candidate)], candidate_files=[candidate], + output=runtime.project / "first-probe.json", input_revision="public-local-v1", + planned_method_id="scheduler", executed_method_id="mask-selection", + change_reason="A local control isolated mask selection before changing the scheduler.", + evidence_ref="probe:first", limit_seconds=10, diagnostics=runtime.project / "first-diagnostics.json") + archive = runtime.project / "packaged.tar.gz" + archive.write_bytes(archive_bytes({"candidate.py": candidate.read_bytes()})) + intent = prepare_submission(root, archive, wait_todo_id=fixture.TODO_ID, + history_reader=lambda: {"run_id": state["run_id"], "entries": []}) + mark_dispatched(root, intent["operation_id"]) + accepted = submit_fixture(observer, Path(intent["archive_path"]).read_bytes()) + record_response(root, intent["operation_id"], 200, accepted) + # The accepted response can declare the exact dependency before the + # runner's next history poll. It does not assert result attribution. + synchronized = synchronize_native_feedback(env) + assert synchronized["waits"][0]["waiting"] + pending = observer.update(LocalCopyBackend(), None) + from loopx.cli_commands.turn_decision import current_heartbeat_host_launch_decision + quiet = current_heartbeat_host_launch_decision(registry_path=runtime.registry, + runtime_root=runtime.root, project=runtime.project, goal_id=fixture.GOAL_ID, + agent_id=fixture.AGENT_ID, turn_instance_id=env["LOOPX_TURN"], original_quota=original)["current"] + assert not quiet["should_run"] + assert quiet["quota"]["spent_slots"] == original_budget + assert not runtime.quota("--turn-instance-id", "another-wake-while-pending")["should_run"] + assert state["posts"] == 1 and state["result_reads"] == 0 + + # Interrupted native operation, newer workspace, then late negative + # feedback: the journal and accepted archive remain the sole binding. + candidate.write_text("print('unsubmitted newer version')") + observer.after_publish = None + release(accepted["submission_id"], score=0) + completed = observer.update(LocalCopyBackend(), None) + assert pending["records"][0]["archive_sha256"] == completed["records"][0]["archive_sha256"] + first_write = synchronize_native_feedback(env, packet=completed) + assert first_write["released"] and not first_write["waits"] + assert json.loads((root / "ingestion" / "wake.json").read_text())["revision"] == first_write["revision"] + assert candidate.read_text() == "print('unsubmitted newer version')" + recovered = prepare_submission(root, archive) + assert recovered == {"action": "recover", "operation_id": intent["operation_id"], + "submission_id": accepted["submission_id"]} + assert state["posts"] == 1 + [current_intent] = synchronize_intents(root, completed) + assert not current_intent.get("result_observed"), "observer must not forge native operation completion" + + # Observe readiness without minting another work Turn. The original + # interrupted Turn still needs the current owner's normal recovery. + ready = runtime.quota() + assert ready["should_run"] and ready["effective_action"] == "successor_replan_required", { + key: ready.get(key) for key in ("should_run", "effective_action", "reason", "blocker")} + assert ready["quota"]["spent_slots"] == original_budget + context = runtime.call("explore", "turn-context", *runtime.binding) + finding = next(row for row in context["graph"]["writeback_results"] if row.get("execution_observation")) + bound = finding["execution_observation"] + assert bound["candidate_sha256"] == bound["result_candidate_sha256"] == measured["candidate_sha256"] + assert bound["search_iterations"] == 2 + assert '"score": 0' in finding["summary"] + assert bound["result_ref"] == completed["records"][0]["result_ref"] + + runtime.call("todo", "update", *runtime.binding, "--todo-id", fixture.TODO_ID, + "--status", "open", "--clear-resume-when", "--reason", "Use the released source-bound result in the resumed experiment.") + decision = runtime.project / "resume-decision.json" + decision.write_text(json.dumps({"schema_version": "goal_vision_replan_contract_v0", "state": "active", + "vision_patch": {"advancement_policy": "as_needed", "vision_summary": "Vary the measured search cap while retaining the incumbent.", + "acceptance_summary": "Execute the declared local control before attributing any scheduler benefit."}, + "path_delta": {"schema_version": "goal_path_delta_v0", "outcome": "continue", + "prior_assumption": "The scheduler proposal was expected to explain improvement.", + "observed_reality": "The archived mask candidate stopped at two search steps and its native result was zero.", + "retained": ["Original objective, best validated candidate and original budget."], + "changed": ["Test five search steps before changing the scheduler."], + "evidence_refs": [finding["finding_id"], bound["result_ref"]]}})) + next_turn = env["LOOPX_TURN"] + admitted = runtime.quota("--turn-instance-id", next_turn) + assert admitted["should_run"] and admitted["effective_action"] == "normal_run", { + key: admitted.get(key) for key in ("should_run", "effective_action", "unsettled_host_turn_recovery")} + assert admitted["heartbeat_receipt"]["settlement_identity"] == original["heartbeat_receipt"]["settlement_identity"] + candidate.write_text("import json,os; from pathlib import Path; " + "Path(os.environ['LOOPX_RESEARCH_DIAGNOSTICS']).write_text(json.dumps(" + "{'search_iterations':5,'stop_reason':'local_cases_complete'})); print(5)") + next_probe = run_probe(command=[sys.executable, str(candidate)], candidate_files=[candidate], + output=runtime.project / "next-probe.json", input_revision="public-local-v1", + planned_method_id="mask-selection", executed_method_id="mask-selection", + evidence_ref="probe:next", limit_seconds=10, diagnostics=runtime.project / "next-diagnostics.json") + assert next_probe["search_iterations"] == 5 and next_probe["candidate_sha256"] != measured["candidate_sha256"] + assert (runtime.project / "next-probe.json.stdout").read_text().strip() == "5" + followthrough = runtime.project / "followthrough.json" + followthrough.write_text(json.dumps({"schema_version": "explore_result_attachment_v0", + "question": "Did the declared search-cap control actually execute?", + "applicability": "This synthetic local case and the declared selected candidate files.", + "input_revision": next_probe["input_revision"], "observation": "The modified process executed five search steps and exited successfully.", + "interpretation": "The feedback-led next action ran as declared; improvement on native evaluation remains unknown.", + "status": "tentative", "evidence_refs": next_probe["evidence_refs"], "execution_observation": next_probe})) + runtime.call("refresh-state", *runtime.binding, "--todo-id", fixture.TODO_ID, + "--turn-instance-id", next_turn, "--classification", "native_feedback_replan_executed", + "--delivery-batch-scale", "implementation", "--delivery-outcome", "outcome_progress", + "--agent-vision-json", str(decision), "--explore-result-json", str(followthrough), + "--no-global-sync", "--suppress-external-sinks") + runtime.call("quota", "spend-slot", *runtime.binding, "--todo-id", fixture.TODO_ID, + "--turn-instance-id", next_turn, "--slots", "1", "--source", "heartbeat", "--execute") + assert synchronize_native_feedback(env, packet=completed)["released"] == [] + after = runtime.quota() + assert after["quota"]["spent_slots"] == original_budget + 1 + assert after["effective_action"] != "unsettled_host_turn_recovery" + assert state["posts"] == 1, "a new local experiment does not consume a native evaluation" diff --git a/tests/control_plane/test_research_execution_loop.py b/tests/control_plane/test_research_execution_loop.py index c46a97c444..05454549a6 100644 --- a/tests/control_plane/test_research_execution_loop.py +++ b/tests/control_plane/test_research_execution_loop.py @@ -94,6 +94,65 @@ def test_invalid_solver_report_cannot_hide_a_host_timeout(tmp_path, monkeypatch) assert (tmp_path / "observation.json.diagnostics-error.json").is_file() +def test_persisted_probe_auto_registers_and_binds_late_native_feedback(tmp_path, monkeypatch): + from benchmark.runtime import research_submission + from benchmark.tests.test_native_research_feedback import ( + LocalCopyBackend, archive_bytes, native_http, submit_fixture, + ) + + register = research_submission.register_probe + registrations = [] + + def register_after_persist(root, observed, manifest): + assert json.loads((tmp_path / "observation.json").read_text()) == observed + assert json.loads((tmp_path / "observation.json.candidate.json").read_text()) == manifest + registrations.append(root) + return register(root, observed, manifest) + + monkeypatch.setattr(research_submission, "register_probe", register_after_persist) + with native_http(tmp_path) as (_state, release, observer, root): + monkeypatch.setenv("LOOPX_RESEARCH_FEEDBACK_ROOT", str(root)) + observed = probe(tmp_path, monkeypatch) + assert registrations == [root] + [registered] = list((root / "probes").glob("*.json")) + assert json.loads(registered.read_text())["observation"] == observed + assert "result_ref" not in observed, "a completed local probe does not invent a native result" + source = tmp_path / "candidate.py" + accepted = submit_fixture(observer, archive_bytes({"candidate.py": source.read_bytes()})) + pending = observer.update(LocalCopyBackend(), None) + assert research_submission.bind_execution_observations(root, pending) == [] + source.write_text("print('newer candidate')") + release(accepted["submission_id"], score=0) + packet = observer.update(LocalCopyBackend(), None) + [bound] = research_submission.bind_execution_observations(root, packet) + assert bound["submission_id"] == accepted["submission_id"] + assert bound["execution_observation"]["result_candidate_sha256"] == observed["candidate_sha256"] + assert source.read_text() == "print('newer candidate')" + + +def test_probe_without_feedback_root_does_not_register(tmp_path, monkeypatch): + from benchmark.runtime import research_submission + + monkeypatch.delenv("LOOPX_RESEARCH_FEEDBACK_ROOT", raising=False) + + def unexpected(*_args): + raise AssertionError("feature-off probe must not create a native registration") + + monkeypatch.setattr(research_submission, "register_probe", unexpected) + assert probe(tmp_path, monkeypatch)["exit_code"] == 0 + + +def test_registration_failure_preserves_the_completed_observation(tmp_path, monkeypatch): + feedback_root = tmp_path / "unavailable-feedback" + feedback_root.write_text("an existing non-directory") + monkeypatch.setenv("LOOPX_RESEARCH_FEEDBACK_ROOT", str(feedback_root)) + with pytest.raises(FileExistsError): + probe(tmp_path, monkeypatch) + observed = json.loads((tmp_path / "observation.json").read_text()) + assert observed["exit_code"] == 0 and observed["elapsed_seconds"] > 0 + assert observed["search_iterations"] == 2 + + def test_probe_cli_preserves_a_failed_candidate_exit_and_its_observation(tmp_path): candidate = tmp_path / "candidate.py" candidate.write_text("raise SystemExit(3)") diff --git a/tests/control_plane/test_research_feedback_wait.py b/tests/control_plane/test_research_feedback_wait.py new file mode 100644 index 0000000000..1447b3cc79 --- /dev/null +++ b/tests/control_plane/test_research_feedback_wait.py @@ -0,0 +1,215 @@ +"""Accepted native feedback reaches canonical waits and real request admission.""" +from __future__ import annotations + +import copy +from contextlib import redirect_stdout +import io +import json +import os +import subprocess +import sys +import time +from types import SimpleNamespace +import urllib.error +import urllib.request + +import pytest + +from benchmark.runtime import research_feedback_wait as feedback +from benchmark.runtime.research_request_guard import ResearchRequestGuard +from canonical_authority_fixture import isolate_sqlite_runtime +from loopx.control_plane.goals.first_party_host_admission import FirstPartyHostGoalAdmission +from loopx.cli import main as cli_main +import test_quota_settlement_cli as fixture +from tests.capabilities.test_provider_gateway import _recording_upstream + + +@pytest.fixture(params=["file", "sqlite"]) +def runtime(request, tmp_path, monkeypatch): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, root, registry = fixture._write_fixture(tmp_path) + fixture._initialize_fixture_authority(project, root, provider=request.param) + + def call(*args): + output = io.StringIO() + with monkeypatch.context() as scope, redirect_stdout(output): + scope.chdir(project) + code = cli_main(["--registry", str(registry), "--runtime-root", str(root), "--format", "json", *args]) + payload = json.loads(output.getvalue()) + assert code == 0, {key: payload.get(key) for key in ( + "ok", "error", "errors", "error_code", "reason", "blocker", "diagnostics")} + return payload + + call("configure-goal", "--goal-id", fixture.GOAL_ID, "--registered-agent", fixture.AGENT_ID, + "--registered-agent", feedback.FEEDBACK_OBSERVER_AGENT, "--explore-graph-enabled", "--execute") + env = dict(os.environ, LOOPX_REGISTRY=str(registry), LOOPX_RUNTIME_ROOT=str(root), + LOOPX_PROJECT=str(project), LOOPX_GOAL_ID=fixture.GOAL_ID, LOOPX_AGENT_ID=fixture.AGENT_ID, + LOOPX_PHASE_DEADLINE_EPOCH=str(time.time() + 300), LOOPX_TURN="original-feedback-turn", + LOOPX_RESEARCH_FEEDBACK_ROOT=str(tmp_path / "native-feedback")) + # Keep product state, CLI validation/CAS, provider transactions and readback + # real; only share the fixture's public command construction. + monkeypatch.setattr(feedback, "_loopx", lambda _env, *args: call(*args)) + binding = ("--goal-id", fixture.GOAL_ID, "--agent-id", fixture.AGENT_ID) + + def quota(*args): + return call("quota", "should-run", *binding, "--runtime-profile", "generic_cli", + "--include-detail", "scheduler", "--scan-path", str(project), *args) + + return SimpleNamespace(env=env, call=call, quota=quota, project=project, root=root, registry=registry, + binding=binding, temp=tmp_path) + + +def packet(*, status="running", attribution="unknown", submission="submission-1", run="native-attempt-1"): + record = dict(accepted_record=True, submission_id=submission, round_id="agent-1", + status=status, attribution_status=attribution) + if attribution == "verified": + record.update(archive_sha256="a" * 64, result_sha256="b" * 64, + result_ref="native-result:" + submission, + result_observed_at="2026-10-19T00:00:00Z") + return dict(schema_version="edgebench_native_feedback_v1", run_id=run, + task_id="synthetic-task", records=[record]) + + +def waiting(runtime, value=None): + return feedback.wait_for_feedback(runtime.env, todo_id=fixture.TODO_ID, + submission_id="submission-1", packet=value or packet()) + + +def todos(runtime): + return runtime.call("todo", "list", "--goal-id", fixture.GOAL_ID) + + +def test_pending_native_submission_requires_explicit_dependency_and_preserves_independent_work(runtime): + pending = packet() + feedback.synchronize_feedback_waits(runtime.env, pending) + initial = runtime.quota() + assert initial["should_run"] and initial["selected_todo"]["todo_id"] == fixture.TODO_ID + result = waiting(runtime, pending) + source = next(todo for todo in todos(runtime)["todos"] if todo["todo_id"] == result["dependency_todo_id"]) + assert source["claimed_by"] == feedback.FEEDBACK_OBSERVER_AGENT + assert fixture.AGENT_ID in source["excluded_agents"] + code, denied = fixture._run_cli(runtime.registry, runtime.root, "quota", "should-run", *runtime.binding, + "--runtime-profile", "generic_cli", "--todo-id", source["todo_id"], cwd=runtime.project) + assert code == 1 and denied.get("should_run") is not True + quiet = runtime.quota() + assert quiet["should_run"] is False + assert quiet["effective_action"] == "agent_scope_wait" + current_work = next(todo for todo in todos(runtime)["todos"] if todo["todo_id"] == fixture.TODO_ID) + assert current_work["status"] == "deferred" and current_work["resume_when"] == result["resume_when"] + before = todos(runtime)["authority_read"]["provider_revision"] + assert waiting(runtime, pending) == result + assert feedback.synchronize_feedback_waits(runtime.env, pending) == {"released": []} + assert todos(runtime)["authority_read"]["provider_revision"] == before + + independent = runtime.call("todo", "add", "--goal-id", fixture.GOAL_ID, "--role", "agent", + "--claimed-by", fixture.AGENT_ID, "--task-class", "advancement_task", "--action-kind", "validate", + "--text", "Validate an independent local mechanism without external feedback.", "--execute") + ready = runtime.quota() + assert ready["should_run"] and ready["selected_todo"]["todo_id"] == independent["todo_id"] + assert not ready.get("vision_wait_state") or ready["vision_wait_state"].get("applied") is not True + assert next(todo for todo in todos(runtime)["todos"] if todo["todo_id"] == fixture.TODO_ID)["resume_when"] == result["resume_when"] + + +def test_native_wait_stops_actual_upstream_send_under_original_turn(runtime): + original = runtime.quota("--todo-id", fixture.TODO_ID, "--turn-instance-id", runtime.env["LOOPX_TURN"]) + assert original["should_run"] + session = SimpleNamespace(session_id="native-feedback-session", goal_ref=None, + admission=FirstPartyHostGoalAdmission.for_plan(registry_path=runtime.registry, + goal_id=fixture.GOAL_ID, planned_goal_ref=None)) + wake = runtime.temp / "wake" + wake.mkdir() + receipt = {"turn_id": "native-feedback-wake"} + with _recording_upstream() as (url, upstream): + env = dict(runtime.env, OPENAI_BASE_URL=url, OPENAI_API_KEY="synthetic-credential", NO_PROXY="127.0.0.1") + guard = ResearchRequestGuard(env, receipt, wake) + process = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(180)"], start_new_session=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + with guard.serve(): + guard.bind(process, env, session, original_quota=original) + + def send(): + return urllib.request.urlopen(urllib.request.Request(guard.base_url + "/responses", + data=b'{"input":"synthetic input"}', headers={"Content-Type": "application/json"}), timeout=45) + + with send() as response: + assert json.load(response) == {"ok": True} + upstream.clear() + waiting(runtime) + with pytest.raises(urllib.error.HTTPError) as refused: + send() + assert refused.value.code == 409 and upstream == {} + process.wait(timeout=10) + assert guard.refused + finally: + if process.poll() is None: + process.kill() + process.wait() + rows = [json.loads(line) for line in (wake / "provider-requests.jsonl").read_text().splitlines()] + finished = [row for row in rows if row["event"] == "finished"] + assert [row["forwarded"] for row in finished] == [True, False] + assert all(row["loopx_turn_id"] == runtime.env["LOOPX_TURN"] for row in finished) + assert runtime.env["LOOPX_PHASE_DEADLINE_EPOCH"] == env["LOOPX_PHASE_DEADLINE_EPOCH"] + + +def test_ambiguous_or_other_attempt_feedback_never_releases_original_wait(runtime, monkeypatch): + result = waiting(runtime) + ambiguous = packet() + ambiguous["records"].append(copy.deepcopy(ambiguous["records"][0])) + with pytest.raises(ValueError, match="ambiguous"): + waiting(runtime, ambiguous) + with pytest.raises(ValueError, match="accepted submission"): + waiting(runtime, packet(submission="submission-2")) + current = next(todo for todo in todos(runtime)["todos"] if todo["todo_id"] == result["dependency_todo_id"]) + assert current["status"] == "open" + before = todos(runtime)["authority_read"]["provider_revision"] + assert feedback.synchronize_feedback_waits(runtime.env, packet(status="completed")) == {"released": []} + assert todos(runtime)["authority_read"]["provider_revision"] == before + from benchmark.runtime import research_feedback_sync + written = [] + monkeypatch.setattr(research_feedback_sync, "write_feedback_finding", lambda *args: written.append(args[2]["submission_id"])) + other = packet(status="completed", attribution="verified", run="native-attempt-2") + feedback.synchronize_feedback_waits(runtime.env, other) + assert written == ["submission-1"] + assert next(todo for todo in todos(runtime)["todos"] if todo["todo_id"] == current["todo_id"])["status"] == "open" + assert runtime.quota()["should_run"] is False + + +@pytest.mark.parametrize("native_status", ["completed", "error"]) +def test_result_writeback_precedes_wait_release_and_negative_feedback_is_not_blocked(runtime, monkeypatch, native_status): + result = waiting(runtime) + released = packet(status=native_status, attribution="verified") + from benchmark.runtime import research_feedback_sync + real_writer = research_feedback_sync.write_feedback_finding + + def unavailable(*_args): + raise RuntimeError("synthetic finding write failure") + + monkeypatch.setattr(research_feedback_sync, "write_feedback_finding", unavailable) + with pytest.raises(RuntimeError, match="finding write failure"): + feedback.synchronize_feedback_waits(runtime.env, released) + assert next(todo for todo in todos(runtime)["todos"] if todo["todo_id"] == result["dependency_todo_id"])["status"] == "open" + assert runtime.quota()["should_run"] is False + + def persisted(*args): + assert args[3] == result["dependency_todo_id"] + assert next(todo for todo in todos(runtime)["todos"] if todo["todo_id"] == args[3])["status"] == "open" + return real_writer(*args) + + monkeypatch.setattr(research_feedback_sync, "write_feedback_finding", persisted) + observed = feedback.synchronize_feedback_waits(runtime.env, released) + assert observed["released"][0]["dependency_todo_id"] == result["dependency_todo_id"] + after = todos(runtime) + assert next(todo for todo in after["todos"] if todo["todo_id"] == result["dependency_todo_id"])["status"] == "done" + resume = runtime.quota() + assert resume["should_run"] and resume["selected_todo"]["todo_id"] == fixture.TODO_ID + assert resume["effective_action"] == "successor_replan_required" + context = runtime.call("explore", "turn-context", *runtime.binding) + [finding] = context["graph"]["writeback_results"] + assert released["records"][0]["result_ref"] in finding["evidence_refs"] + assert f'"status": "{native_status}"' in finding["summary"] + runs = runtime.root / f"goals/{fixture.GOAL_ID}/runs/index.jsonl" + run_events = map(json.loads, runs.read_text().splitlines()) if runs.exists() else [] + assert not any(row.get("classification") == "quota_slot_spent" for row in run_events) + assert feedback.synchronize_feedback_waits(runtime.env, released) == {"released": []} + assert todos(runtime)["authority_read"]["provider_revision"] == after["authority_read"]["provider_revision"] diff --git a/tests/control_plane/test_research_scheduler.py b/tests/control_plane/test_research_scheduler.py new file mode 100644 index 0000000000..ecb10400c2 --- /dev/null +++ b/tests/control_plane/test_research_scheduler.py @@ -0,0 +1,248 @@ +"""A durable feedback hint wakes the existing worker, never grants dispatch.""" +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +import copy +import json +from pathlib import Path +import subprocess +import sys +import threading +import time +from types import SimpleNamespace + +import pytest + +from benchmark.runtime.research_scheduler import FeedbackEventSleep, main +from benchmark.runtime.research_feedback_wait import synchronize_feedback_waits +from scripts import external_scheduler_worker as worker +import test_research_feedback_wait as wait_fixture + +runtime = wait_fixture.runtime + + +def publish(path, revision): + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps({"revision": revision})) + temporary.replace(path) + + +def test_event_between_quota_and_sleep_is_not_lost_or_replayed_as_endless_wake(tmp_path): + path = tmp_path / "ingestion/wake.json" + publish(path, "a" * 64) + now, pauses = [0.0], [] + + def pause(seconds): + pauses.append(seconds) + now[0] += seconds + + sleeper = FeedbackEventSleep(path, clock=lambda: now[0], pause=pause) + publish(path, "b" * 64) # Publication races the last quota read, before sleep. + sleeper(3600) + assert now[0] == 0 and pauses == [] + sleeper(3) + assert now[0] == 3 and sum(pauses) == 3 + path.write_text("{partial") + sleeper(2) + assert now[0] == 5 # Missing/partial signals preserve the requested cadence. + publish(path, "c" * 64) + sleeper(3600) + assert now[0] == 5 + + +def test_research_scheduler_entry_is_explicit_and_reuses_ordinary_arguments(tmp_path, monkeypatch): + monkeypatch.delenv("LOOPX_RESEARCH_LOOP", raising=False) + with pytest.raises(ValueError, match="explicit research-loop"): + main([]) + monkeypatch.setenv("LOOPX_RESEARCH_LOOP", "1") + monkeypatch.setenv("LOOPX_RESEARCH_FEEDBACK_ROOT", str(tmp_path)) + called = [] + monkeypatch.setattr(worker, "main", lambda argv, **kwargs: called.append((argv, kwargs)) or 0) + argv = ["--goal-id", "synthetic-goal", "--agent-id", "synthetic-agent", "--once"] + assert main(argv) == 0 + assert called[0][0] is argv + assert called[0][1]["sleep"].path == tmp_path / "ingestion/wake.json" + + +def test_feedback_sync_cli_refuses_overlap_before_reading_native_packet(tmp_path, monkeypatch): + from benchmark.runtime.research_feedback_sync import _consumer_lock + from benchmark.runtime.research_feedback_wait import main as feedback_main + + monkeypatch.setenv("LOOPX_RESEARCH_FEEDBACK_ROOT", str(tmp_path)) + monkeypatch.setattr(sys, "argv", ["loopx-research-feedback", "sync"]) + (tmp_path / "owner.json").write_text(json.dumps({"LOOPX_RESEARCH_FEEDBACK_ROOT": str(tmp_path)})) + # There is deliberately no native packet: the entry point must acquire + # consumer ownership before reading a snapshot that could become stale. + with _consumer_lock(tmp_path): + with pytest.raises(RuntimeError, match="native_feedback_consumer_busy"): + feedback_main() + with pytest.raises(FileNotFoundError, match="native.json"): + feedback_main() # The failed contender did not retain the lock. + + +def test_event_rechecks_real_quota_and_only_released_dependency_dispatches(runtime, monkeypatch): + original = wait_fixture.waiting(runtime) + event = runtime.temp / "native-feedback/ingestion/wake.json" + publish(event, "a" * 64) + seen, pauses, now, dispatches = [], [], [0.0], [] + original_turn = runtime.env["LOOPX_TURN"] + original_deadline = runtime.env["LOOPX_PHASE_DEADLINE_EPOCH"] + + def quota(_command, **_kwargs): + value = runtime.quota() + seen.append(value) + return value + + def observe(seconds): + pauses.append(seconds) + now[0] += seconds + if len(pauses) == 1: + publish(event, "b" * 64) # A signal alone cannot satisfy the dependency. + elif len(pauses) == 2: + value = wait_fixture.packet(status="completed", attribution="verified") + receipt = synchronize_feedback_waits(runtime.env, value) + assert receipt["released"][0]["dependency_todo_id"] == original["dependency_todo_id"] + publish(event, "c" * 64) # Consumer publication follows durable evidence. + else: + raise AssertionError("released evidence did not interrupt the long scheduler sleep") + + def wake(_command, **_kwargs): + assert len(seen) == 3 and [packet["should_run"] for packet in seen] == [False, False, True] + assert seen[-1]["selected_todo"]["todo_id"] == wait_fixture.fixture.TODO_ID + assert runtime.env["LOOPX_TURN"] == original_turn + assert runtime.env["LOOPX_PHASE_DEADLINE_EPOCH"] == original_deadline + dispatches.append(original_turn) + # Transport-only bounded synthetic Host; no model or scoring request. + return SimpleNamespace(returncode=75, failure_kind=None) + + monkeypatch.setattr(worker, "run_quota_should_run", quota) + monkeypatch.setattr(worker, "_run_wake", wake) + args = SimpleNamespace(cli_bin="unused", registry=str(runtime.registry), runtime_root=str(runtime.root), + runtime_profile="generic_cli", goal_id=runtime.env["LOOPX_GOAL_ID"], agent_id=runtime.env["LOOPX_AGENT_ID"], + state_file=str(runtime.temp / "scheduler.json"), once=False, wake_cmd="synthetic-host", + wake_stop_exit_code=75, error_backoff_seconds=5, quota_timeout_seconds=60, wake_timeout_seconds=30) + sleeper = FeedbackEventSleep(event, clock=lambda: now[0], pause=observe) + assert worker.run_worker(args, sleep=sleeper) == 0 + assert len(dispatches) == 1 and now[0] == 2 + assert json.loads((runtime.temp / "scheduler.json").read_text())["unchanged_count"] == 0 + + +def test_overlapping_consumers_and_old_packets_preserve_newly_runnable_work(runtime, monkeypatch, record_property): + from benchmark.edgebench.research_feedback import canonical_json, write_bytes_atomic + from benchmark.runtime import research_feedback_sync as consumer + from benchmark.runtime.research_feedback_wait import wait_for_feedback + + root = Path(runtime.env["LOOPX_RESEARCH_FEEDBACK_ROOT"]) + old = wait_fixture.packet(status="completed", attribution="verified") + pending = wait_fixture.packet(submission="submission-2")["records"][0] + pending["round_id"] = "agent-2" + old["records"].append(pending) + write_bytes_atomic(root / "native.json", canonical_json(old)) + wait_for_feedback(runtime.env, todo_id=wait_fixture.fixture.TODO_ID, + submission_id="submission-2", packet=old) + consumer.synchronize_native_feedback(runtime.env) + assert not runtime.quota()["should_run"] + event = root / "ingestion/wake.json" + before = event.read_bytes() + now, pauses = [0.0], [] + + def pause(seconds): + pauses.append(seconds) + now[0] += seconds + + sleeper = FeedbackEventSleep(event, clock=lambda: now[0], pause=pause) + entered, finish = threading.Event(), threading.Event() + original_publish = consumer._publish_feedback_wake + + def delayed_publication(env, feedback_root): + entered.set() + assert finish.wait(timeout=45), "the test must release its paused consumer" + return original_publish(env, feedback_root) + + new = copy.deepcopy(old) + new["records"][1] = wait_fixture.packet(status="completed", attribution="verified", + submission="submission-2")["records"][0] + new["records"][1]["round_id"] = "agent-2" + with monkeypatch.context() as patch, ThreadPoolExecutor(max_workers=1) as executor: + patch.setattr(consumer, "_publish_feedback_wake", delayed_publication) + older = executor.submit(consumer.synchronize_native_feedback, runtime.env) + try: + assert entered.wait(timeout=45), "the old snapshot did not reach publication" + write_bytes_atomic(root / "native.json", canonical_json(new)) + # The real second process must be refused before reading the new + # snapshot or touching any Goal/provider state. There is no polling + # sleep or elapsed-time threshold deciding the race's outcome. + contender = subprocess.run([sys.executable, "-c", """ +import os +from benchmark.runtime.research_feedback_sync import synchronize_native_feedback +try: + synchronize_native_feedback(dict(os.environ)) +except RuntimeError as error: + assert str(error) == "native_feedback_consumer_busy", str(error) + print("busy") +else: + raise AssertionError("a second consumer entered while publication was held") +"""], env=runtime.env, cwd=Path(__file__).resolve().parents[2], + text=True, capture_output=True, timeout=15, check=True) + assert contender.stdout.strip() == "busy" + assert event.read_bytes() == before + assert not runtime.quota()["should_run"], "contention cannot grant model admission" + finally: + finish.set() + assert older.result(timeout=45)["released"] == [] + + started = time.monotonic() + released = consumer.synchronize_native_feedback(runtime.env) + record_property("feedback_consumer_seconds", time.monotonic() - started) + assert len(released["released"]) == 1 and runtime.quota()["should_run"] + after = event.read_bytes() + assert after != before + # A delayed caller may still hold the older in-memory packet. Its readback + # must include the newer canonical completion, not restore the old marker. + assert consumer.synchronize_native_feedback(runtime.env, packet=old)["released"] == [] + assert event.read_bytes() == after + sleeper(600) + assert now[0] == 0 and pauses == [] + assert consumer.synchronize_native_feedback(runtime.env, packet=new)["released"] == [] + sleeper(3) + assert now[0] == 3, "unchanged feedback must not manufacture another wake" + + +def test_retry_rebuilds_notification_after_todo_completion_was_durable(runtime, monkeypatch): + from benchmark.edgebench import research_feedback as producer + from benchmark.runtime import research_feedback_sync as consumer + + wait_fixture.waiting(runtime) + pending = wait_fixture.packet() + consumer.synchronize_native_feedback(runtime.env, packet=pending) + event = Path(runtime.env["LOOPX_RESEARCH_FEEDBACK_ROOT"]) / "ingestion/wake.json" + before = event.read_bytes() + completed = wait_fixture.packet(status="completed", attribution="verified") + writer = producer.write_bytes_atomic + + def unavailable(path, raw, **kwargs): + if path == event: + raise OSError("synthetic wake publication unavailable") + return writer(path, raw, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(producer, "write_bytes_atomic", unavailable) + with pytest.raises(OSError, match="wake publication unavailable"): + consumer.synchronize_native_feedback(runtime.env, packet=completed) + assert event.read_bytes() == before + assert runtime.quota()["should_run"], "notification failure must not undo committed evidence" + context = runtime.call("explore", "turn-context", *runtime.binding) + assert len(context["graph"]["writeback_results"]) == 1 + now = [0.0] + sleeper = FeedbackEventSleep(event, clock=lambda: now[0], + pause=lambda seconds: now.__setitem__(0, now[0] + seconds)) + # Even an older packet whose row still says running must recover the wake + # from the already completed canonical observation, without another result. + recovered = consumer.synchronize_native_feedback(runtime.env, packet=pending) + assert recovered["released"] == [] and event.read_bytes() != before + sleeper(600) + assert now[0] == 0 + consumer.synchronize_native_feedback(runtime.env, packet=completed) + sleeper(3) + assert now[0] == 3 From c678476fd3dfc90f83c7085cc586f60e870ee10b Mon Sep 17 00:00:00 2001 From: song Date: Tue, 20 Oct 2026 19:11:20 +0800 Subject: [PATCH 2/2] fix(benchmark): accept native temporary parent aliases Signed-off-by: song --- benchmark/runtime/research_submission.py | 5 +- .../tests/test_native_research_feedback.py | 52 ++++++++++++++++++- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/benchmark/runtime/research_submission.py b/benchmark/runtime/research_submission.py index eb98fee8ee..152883d8a1 100644 --- a/benchmark/runtime/research_submission.py +++ b/benchmark/runtime/research_submission.py @@ -203,7 +203,10 @@ def prepare_submission(root: Path, archive: Path, *, history_reader=None, wait_t raise ValueError("native_acceptance_unknown_no_resubmit") return dict(action="submit", operation_id=intent["operation_id"], archive_path=intent["archive_path"]) before = _fresh_history(packet, history_reader=history_reader) - raw = read_archive(archive.absolute(), root=archive.parent.resolve()) + # Native temporary directories can have a platform alias. Canonicalize + # only that parent; resolving the leaf would accept archive symlinks. + parent = archive.parent.resolve(strict=True) + raw = read_archive(parent / archive.name, root=parent) operation_id = uuid.uuid4().hex snapshot = root / "snapshots" / (operation_id + ".tar.gz") write_bytes_atomic(snapshot, raw, create_only=True) diff --git a/benchmark/tests/test_native_research_feedback.py b/benchmark/tests/test_native_research_feedback.py index c31b33554c..57e1f32352 100644 --- a/benchmark/tests/test_native_research_feedback.py +++ b/benchmark/tests/test_native_research_feedback.py @@ -393,7 +393,8 @@ def test_archive_manifest_rejects_ambiguous_paths_links_and_matches_probe_hash() @pytest.mark.parametrize("lose_reply", [False, True]) -def test_actual_native_submit_script_archives_once_and_recovers_lost_reply(tmp_path, lose_reply): +@pytest.mark.parametrize("aliased_parent", [False, True]) +def test_actual_native_submit_script_archives_once_and_recovers_lost_reply(tmp_path, lose_reply, aliased_parent): native = pytest.importorskip("sforge.harness.evolve_scripts") with native_http(tmp_path, immediately_complete=True, lose_reply=lose_reply) as (state, release, observer, root): workspace = tmp_path / "workspace" @@ -408,9 +409,15 @@ def test_actual_native_submit_script_archives_once_and_recovers_lost_reply(tmp_p script.write_text(instrument_submit_script(generated, [sys.executable, "-m", "benchmark.runtime.research_submission"], str(root))) script.chmod(0o755) + native_tmp = tmp_path / "native-tmp" + native_tmp.mkdir() + if aliased_parent: + alias = tmp_path / "native-tmp-alias" + alias.symlink_to(native_tmp, target_is_directory=True) + native_tmp = alias env = dict(os.environ, SFORGE_JUDGE_URL=observer.judge_url, SFORGE_TOKEN="synthetic-token", SFORGE_PATCH_DIR=str(workspace), SFORGE_SUBMIT_PATHS="candidate.py", - PYTHONPATH=str(Path(__file__).resolve().parents[2]), NO_PROXY="127.0.0.1") + PYTHONPATH=str(Path(__file__).resolve().parents[2]), NO_PROXY="127.0.0.1", TMPDIR=str(native_tmp)) first = subprocess.run([str(script)], cwd=workspace, env=env, capture_output=True, text=True, timeout=30) assert (first.returncode != 0) == lose_reply, first.stderr assert state["posts"] == 1 @@ -429,6 +436,47 @@ def test_actual_native_submit_script_archives_once_and_recovers_lost_reply(tmp_p assert json.loads(intent_file.read_bytes())["operation_id"] == intent["operation_id"] +@pytest.mark.parametrize("invalid_archive", ["leaf_symlink", "oversized", "moved"]) +def test_submission_parent_alias_preserves_archive_integrity_checks(tmp_path, monkeypatch, invalid_archive): + from benchmark.edgebench.research_feedback import MAX_ARCHIVE_BYTES + + with native_http(tmp_path) as (state, _release, _observer, root): + physical = tmp_path / "physical" + physical.mkdir() + alias = tmp_path / "alias" + alias.symlink_to(physical, target_is_directory=True) + archive = physical / "packaged.tar.gz" + archive.write_bytes(archive_bytes({"candidate.py": b"original"})) + expected = "native_archive_size_or_type" + if invalid_archive == "leaf_symlink": + target = physical / "target.tar.gz" + archive.rename(target) + archive.symlink_to(target) + expected = "native_archive_not_physical" + elif invalid_archive == "oversized": + with archive.open("r+b") as stream: + stream.truncate(MAX_ARCHIVE_BYTES + 1) + else: + original_open = os.open + + def move_after_open(path, flags, *args, **kwargs): + descriptor = original_open(path, flags, *args, **kwargs) + if Path(path) == archive: + replacement = physical / "replacement.tar.gz" + replacement.write_bytes(archive_bytes({"candidate.py": b"replacement"})) + replacement.replace(archive) + return descriptor + + monkeypatch.setattr(os, "open", move_after_open) + expected = "native_archive_moved" + with pytest.raises(ValueError, match=expected): + prepare_submission(root, alias / archive.name, + history_reader=lambda: {"run_id": state["run_id"], "entries": []}) + assert not (root / "intents").exists() + assert not (root / "snapshots").exists() + assert state["posts"] == 0 + + def test_native_client_hook_requires_the_observed_packaging_boundaries(): with pytest.raises(ValueError, match="unsupported_native_submit_script"): instrument_submit_script("#!/bin/sh\nexit 0\n", [sys.executable, "-m", "fixture"], "/synthetic")