From 51a7722f05de440474693a7578a5bcd7195716d8 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 18:48:04 +0100 Subject: [PATCH 01/19] Add bounded operator coding audits with HF Sandboxes --- examples/coding_audit/README.md | 64 + examples/coding_audit/__init__.py | 1 + examples/coding_audit/__main__.py | 63 + examples/coding_audit/config.json | 83 ++ examples/coding_audit/hf.py | 1261 ++++++++++++++++++ examples/coding_audit/runner.py | 736 +++++++++++ examples/coding_audit/scoring.py | 506 ++++++++ examples/coding_audit/test_hf.py | 1698 +++++++++++++++++++++++++ examples/coding_audit/test_runner.py | 757 +++++++++++ examples/coding_audit/test_scoring.py | 575 +++++++++ 10 files changed, 5744 insertions(+) create mode 100644 examples/coding_audit/README.md create mode 100644 examples/coding_audit/__init__.py create mode 100644 examples/coding_audit/__main__.py create mode 100644 examples/coding_audit/config.json create mode 100644 examples/coding_audit/hf.py create mode 100644 examples/coding_audit/runner.py create mode 100644 examples/coding_audit/scoring.py create mode 100644 examples/coding_audit/test_hf.py create mode 100644 examples/coding_audit/test_runner.py create mode 100644 examples/coding_audit/test_scoring.py diff --git a/examples/coding_audit/README.md b/examples/coding_audit/README.md new file mode 100644 index 000000000..1e0cd52c8 --- /dev/null +++ b/examples/coding_audit/README.md @@ -0,0 +1,64 @@ +# Operator coding audit + +This example collects and scores bounded coding-task evidence. It belongs to the +operator workflow, separate from the installed OpenEnv validator and its RFC 008 +checks. Acceptance here does not establish full RFC 008 conformance, containment, +served-model revision attestation, or training benefit. + +From a repository checkout, install the tested SDK and run the example: + +```bash +uv sync --group dev +uv pip install 'huggingface_hub==1.33.0' +PYTHONPATH=src:envs uv run --no-sync python -m examples.coding_audit \ + --task /path/to/task \ + --config examples/coding_audit/config.json \ + --output /path/to/new-audit-directory \ + --json +``` + +The output directory is required and must be new. It retains `report.json`, +`evidence.json`, `config.json`, raw evidence artifacts, and `SHA256SUMS`. Exit codes +are `0` for accepted evidence, `1` for rejected or incomplete evidence, and `2` for +configuration or input failure. `--json` prints the report as well. + +`scoring.py` provides pure `assess_coding_audit` and `validate_audit_config` +functions. `runner.py` provides `normalize_coding_config`, `coding_budget`, +`validate_coding_task`, and `run_coding_validation`. `hf.py` contains +`HFTaskBackend`, the bounded Sandbox and Inference Providers implementation. +The schemas bind task, verifier, immutable image references and independent execution identities. + +Models may declare an explicit boolean `enable_thinking`, forwarded through the +HF client's `extra_body.chat_template_kwargs`. It is omitted by default. A +candidate-only override leaves the target protocol unchanged and is recorded +alongside request parameters. Support depends on the selected provider/model. +Candidate generation streams bounded content and reasoning; only a complete +`stop` response with validated usage can produce an executable candidate. A +`length` response remains rejected even when its terminal usage is retained. + +Candidate generation can opt into public task API context by adding +`"context_paths": ["environment/source/package/api.py"]` to the `generator` +model configuration. Only those declared regular UTF-8 files are read; paths must +be unique, normalized, and below `environment/source/`. Up to 32 files and 65,536 +combined bytes are allowed. Linked files or parents, hidden or private paths, +NUL bytes, invalid UTF-8 and oversized context are rejected before inference. +The existing message and token budgets still apply. No source files are added by +default, and `target.context_paths` is rejected. + +The candidate prompt labels each source path and its runtime filename relative to +the configured workdir, with `environment/source/` removed. Each generation record +retains `candidate_context` entries containing `path`, `runtime_path`, `sha256`, +and `size`, bound to the frozen task and checksummed evidence. This option does +not enumerate task directories or include verifier, reference or specification +files. + +The HF backend launches paid services when invoked without a supplied test +backend. Use reviewed tasks and distinct execution and inference credentials. +The Hub pipeline's cloud worker packages these production example modules with +its frozen source and task bundle; its tests stay outside the bundle. + +Run the offline suite with: + +```bash +PYTHONPATH=src:envs uv run pytest examples/coding_audit -q +``` diff --git a/examples/coding_audit/__init__.py b/examples/coding_audit/__init__.py new file mode 100644 index 000000000..1169e0fb8 --- /dev/null +++ b/examples/coding_audit/__init__.py @@ -0,0 +1 @@ +"""Operator-owned coding audits, separate from core RFC 008 validation.""" diff --git a/examples/coding_audit/__main__.py b/examples/coding_audit/__main__.py new file mode 100644 index 000000000..5ace417df --- /dev/null +++ b/examples/coding_audit/__main__.py @@ -0,0 +1,63 @@ +"""Run a bounded operator coding audit and retain its evidence bundle.""" + +import argparse +import signal +import sys +import threading +from contextlib import contextmanager +from pathlib import Path + +from openenv.validation.runtime.json_values import dumps, loads + +from .runner import run_coding_validation + + +@contextmanager +def _sigterm_as_interrupt(): + if threading.current_thread() is not threading.main_thread(): + yield + return + + def interrupt(signum, frame): + raise KeyboardInterrupt + + previous = signal.signal(signal.SIGTERM, interrupt) + try: + yield + finally: + signal.signal(signal.SIGTERM, previous) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--task", type=Path, required=True) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument( + "--output", + type=Path, + required=True, + help="New directory for the report, evidence and checksummed artifacts", + ) + parser.add_argument("--json", action="store_true", dest="json_output") + args = parser.parse_args(argv) + try: + config = loads(args.config.read_text()) + with _sigterm_as_interrupt(): + report = run_coding_validation(args.task, config, args.output) + except (OSError, ValueError) as error: + print( + f"Coding audit configuration/input failure ({type(error).__name__})", + file=sys.stderr, + ) + return 2 + if args.json_output: + print(dumps(report)) + else: + print(f"Coding audit: {report['verdict'].upper()} ({report['profile']})") + for name, check in report["checks"].items(): + print(f" {check['status'].upper():5s} {name}") + return 0 if report["accepted"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/coding_audit/config.json b/examples/coding_audit/config.json new file mode 100644 index 000000000..bfb7b3912 --- /dev/null +++ b/examples/coding_audit/config.json @@ -0,0 +1,83 @@ +{ + "schema_version": 1, + "profile": "harbor-coding-audit-v1", + "target": { + "model_id": "Qwen/Qwen3.8-27B", + "revision": "37561816e66b5b9153b1b64c10968c18c1e6f4a0", + "provider": "deepinfra", + "supports_seed": false, + "supports_tools": true, + "temperature": 0.7, + "max_steps": 16, + "max_tokens": 2048, + "max_total_tokens": 16384 + }, + "execution": { + "agent_image": "docker.io/library/python@sha256:48d884813cf44e927cf64c5529dd630dd3b18338c6dc727991eef9b71e934a20", + "verifier_image": "docker.io/library/python@sha256:48d884813cf44e927cf64c5529dd630dd3b18338c6dc727991eef9b71e934a20", + "flavor": "cpu-basic", + "timeout_seconds": 120, + "workdir": "/app", + "submission_paths": [ + "/app/answer.txt" + ], + "private_paths": [ + "/tests", + "/solution" + ], + "verifier_command": [ + "/bin/bash", + "/tests/test.sh" + ], + "reward_path": "/logs/verifier/reward.txt", + "reward_key": "reward" + }, + "validation": { + "reference_repeats": 3, + "target_attempts": 8, + "reward_min": 0, + "reward_max": 1, + "oracle_tolerance": 0, + "floor_margin": 0.1, + "wrong_answer_max": 0, + "required_negative_cases": [ + "wrong-value", + "empty-answer", + "near-miss" + ] + }, + "generation": { + "candidate_count": 2 + }, + "negative_cases": [ + { + "id": "wrong-value", + "script": "mkdir -p /app; printf '0\\n' > /app/answer.txt" + }, + { + "id": "empty-answer", + "script": "mkdir -p /app; : > /app/answer.txt" + }, + { + "id": "near-miss", + "script": "mkdir -p /app; printf '41\\n' > /app/answer.txt" + } + ], + "seeds": [ + 1001, + 1002, + 1003, + 1004, + 1005, + 1006, + 1007, + 1008 + ], + "cloud": { + "results_repo": "example-org/private-audit-results", + "flavor": "cpu-basic", + "timeout_seconds": 1800, + "max_sandboxes": 64, + "max_inference_calls": 160 + } +} diff --git a/examples/coding_audit/hf.py b/examples/coding_audit/hf.py new file mode 100644 index 000000000..fa8614090 --- /dev/null +++ b/examples/coding_audit/hf.py @@ -0,0 +1,1261 @@ +"""Bounded HF Jobs coding trials for reviewed tasks, with controller-side inference. + +Image receipts attest only the requested immutable control-plane reference. The +Sandbox RPC bearer reaches a workload-controlled server; this is not a hostile +code security boundary or a served-model weight revision guarantee. +""" + +import hashlib +import json +import math +import os +import re +import signal +import stat +import threading +import time +import uuid +from contextlib import closing, contextmanager +from decimal import Decimal +from importlib.metadata import version +from pathlib import Path, PurePosixPath + +from huggingface_hub import HfApi, InferenceClient, Sandbox +from huggingface_hub.errors import SandboxError + +from .runner import _candidate_context_paths + +_IMAGE = re.compile(r"[^\s@]+@sha256:[0-9a-f]{64}\Z") +_REVISION = re.compile(r"[0-9a-f]{40}\Z") +_TERMINAL = {"COMPLETED", "CANCELED", "ERROR", "DELETED"} +_LIMITATIONS = [ + "Immutable images are control-plane references, not measured runtime digests.", + "Sandbox RPC exposes the execution bearer to the workload server; hostile-code credential isolation is not qualified.", + "The verifier executes submitted code as the same user; verifier integrity against hostile submissions is not qualified.", + "Inference Providers do not pin served weights to the recorded model source revision.", +] +_TOOL = { + "type": "function", + "function": { + "name": "terminal", + "description": "Run a shell command in the task workspace.", + "parameters": { + "type": "object", + "properties": {"command": {"type": "string"}}, + "required": ["command"], + "additionalProperties": False, + }, + }, +} + + +class HFBackendError(ValueError): + """A credential-free controller diagnostic.""" + + +class _DeadlineExceeded(HFBackendError): + pass + + +@contextmanager +def _deadline(seconds): + # SDK control-plane calls have no timeout argument. A controller must run on + # the main Unix thread so a blocked synchronous SDK call can be interrupted. + if threading.current_thread() is not threading.main_thread(): + raise HFBackendError("HF coding controller requires the main thread") + previous_handler = signal.getsignal(signal.SIGALRM) + previous_timer = signal.getitimer(signal.ITIMER_REAL) + started = time.monotonic() + + def expired(signum, frame): + raise _DeadlineExceeded("controller deadline exceeded") + + signal.signal(signal.SIGALRM, expired) + signal.setitimer( + signal.ITIMER_REAL, + min(seconds, previous_timer[0]) if previous_timer[0] else seconds, + 0.1, + ) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_handler) + if previous_timer[0]: + signal.setitimer( + signal.ITIMER_REAL, + max(0.000001, previous_timer[0] - (time.monotonic() - started)), + previous_timer[1], + ) + + +def _path(value): + if not isinstance(value, str) or "\x00" in value: + raise HFBackendError("paths must be absolute normalized file paths") + path = PurePosixPath(value) + if ( + not path.is_absolute() + or str(path) != value + or ".." in path.parts + or value == "/" + or value.startswith("//") + ): + raise HFBackendError("paths must be absolute normalized file paths") + return value + + +def _under(path, parent): + return path == parent or path.startswith(parent + "/") + + +def submission_digest(files): + """Hash sorted relative paths, NUL, content, NUL as source_digest does.""" + digest = hashlib.sha256() + for name, content in sorted(files.items()): + digest.update(name.lstrip("/").encode()) + digest.update(b"\0") + digest.update(content) + digest.update(b"\0") + return digest.hexdigest() + + +def _local_file(path, limit): + # Walk descriptors so neither a final link nor a linked parent is followed. + path = Path(path).absolute() + fd = os.open("/", os.O_RDONLY | os.O_DIRECTORY) + try: + for component in path.parts[1:-1]: + next_fd = os.open( + component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd + ) + os.close(fd) + fd = next_fd + file_fd = os.open( + path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd + ) + with os.fdopen(file_fd, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise HFBackendError("task artifacts must be regular unlinked files") + if info.st_size > limit: + raise HFBackendError("task artifact exceeds byte limit") + content = stream.read(limit + 1) + if len(content) > limit: + raise HFBackendError("task artifact exceeds byte limit") + return content, format(stat.S_IMODE(info.st_mode), "o") + finally: + os.close(fd) + + +def _plain(value): + # HF generated output types derive from dict and preserve extra response fields. + return json.loads(json.dumps(value)) + + +def _unique_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise HFBackendError("duplicate JSON key") + result[key] = value + return result + + +class HFTaskBackend: + """Execute each solve and grade in a fresh, dedicated HF Jobs Sandbox.""" + + def __init__( + self, + task_root: Path, + execution: dict, + *, + token=None, + api=None, + sandbox_factory=None, + inference_factory=None, + canary: str | None = None, + inference_token=None, + ): + if version("huggingface_hub") != "1.33.0": + raise HFBackendError( + "this execution adapter requires huggingface_hub==1.33.0" + ) + self.task_root = Path(task_root) + self.execution = dict(execution) + self.token = token if token is not None else os.environ.get("HF_SANDBOX_TOKEN") + self.inference_token = ( + inference_token + if inference_token is not None + else os.environ.get("HF_INFERENCE_TOKEN") + ) + credentials = [ + value + for value in (self.token, self.inference_token, os.environ.get("HF_TOKEN")) + if value + ] + if len(set(credentials)) != len(credentials): + raise HFBackendError( + "execution, inference and persistence credentials must differ" + ) + if execution.get("strong_isolation") or execution.get( + "require_strong_isolation" + ): + raise HFBackendError("strong verifier isolation is unsupported") + for key in ("agent_image", "verifier_image"): + if not isinstance(execution.get(key), str) or not _IMAGE.fullmatch( + execution[key] + ): + raise HFBackendError( + f"{key} must be an immutable OCI name@sha256 reference" + ) + self.workdir = _path(execution.get("workdir", "/app")) + self.private_paths = list( + dict.fromkeys( + _path(item) + for item in ["/tests", "/solution", *execution.get("private_paths", [])] + ) + ) + self.reward_path = _path( + execution.get("reward_path", "/logs/verifier/reward.txt") + ) + paths = execution.get("submission_paths") + if not isinstance(paths, list) or not paths: + raise HFBackendError("submission_paths must be a nonempty file allowlist") + self.submission_paths = [_path(item) for item in paths] + for index, path in enumerate(self.submission_paths): + if any( + _under(path, protected) or _under(protected, path) + for protected in [ + "/tests", + "/solution", + self.reward_path, + *self.private_paths, + ] + ): + raise HFBackendError("submission path overlaps private or reward paths") + if any( + _under(path, other) or _under(other, path) + for other in self.submission_paths[:index] + ): + raise HFBackendError("submission paths overlap or repeat") + self.timeout = self._bound(execution, "timeout_seconds", 120, 3600) + self.max_bytes = self._bound( + execution, "max_submission_bytes", 1048576, 16 * 1048576 + ) + self.max_script = self._bound(execution, "max_script_bytes", 65536, 1048576) + self.max_output = self._bound( + execution, "max_tool_output_bytes", 16384, 1048576 + ) + self.cleanup_attempts = self._bound(execution, "cleanup_attempts", 3, 10) + self.cleanup_timeout = self._bound(execution, "cleanup_timeout_seconds", 10, 60) + self.api = api if api is not None else HfApi(token=self.token) + self.sandbox_factory = sandbox_factory or Sandbox.create + self.inference_factory = inference_factory or InferenceClient + self.canary = canary if canary is not None else uuid.uuid4().hex + if not isinstance(self.canary, str) or not self.canary: + raise HFBackendError("canary must be nonempty text") + self.canary_digest = hashlib.sha256(self.canary.encode()).hexdigest() + self.marker_path = str( + PurePosixPath(self.workdir) / (".openenv-state-" + uuid.uuid4().hex) + ) + self._audit = [] + self._transcripts = [] + self._owned_ids = set() + self._credentials = tuple(credentials) + + def _redact(self, value): + if isinstance(value, str): + for secret in self._credentials: + value = value.replace(secret, "[REDACTED]") + elif isinstance(value, bytes): + for secret in self._credentials: + value = value.replace(secret.encode(), b"[REDACTED]") + elif isinstance(value, dict): + value = { + self._redact(key): self._redact(item) for key, item in value.items() + } + elif isinstance(value, list): + value = [self._redact(item) for item in value] + return value + + def _has_canary(self, value): + if isinstance(value, str): + return self.canary in value + if isinstance(value, dict): + return any(self._has_canary(item) for item in value.values()) + if isinstance(value, list): + return any(self._has_canary(item) for item in value) + return False + + @staticmethod + def _bound(config, key, default, maximum): + value = config.get(key, default) + if type(value) is not int or not 0 < value <= maximum: + raise HFBackendError(f"{key} must be an integer from 1 to {maximum}") + return value + + def _record(self): + return { + "files": {}, + "submission_digest": submission_digest({}), + "sandbox_id": None, + "image_ref": self.execution["agent_image"], + "image_reference_kind": "control_plane_immutable_reference", + "cleanup_verified": False, + "error": None, + "transcript": [], + "usage": {"input_tokens": 0, "output_tokens": 0}, + "input_tokens": 0, + "output_tokens": 0, + "latency_seconds": 0.0, + "model_metadata": {}, + "limitations": list(_LIMITATIONS), + } + + @staticmethod + def _error(exc): + return ( + str(exc) + if isinstance(exc, HFBackendError) + else f"HF backend operation failed ({type(exc).__name__})" + ) + + def _inspect(self, sandbox_id): + job = self.api.inspect_job( + job_id=sandbox_id, + namespace=self.execution.get("namespace"), + token=self.token, + ) + if job.id != sandbox_id or ( + self.execution.get("namespace") + and job.owner.name != self.execution["namespace"] + ): + raise HFBackendError("control-plane job identity mismatch") + return job + + def _create(self, image, record, *, grader=False): + if not self.token: + raise HFBackendError("HF_SANDBOX_TOKEN execution credential is required") + record["creation_label"] = uuid.uuid4().hex + sandbox = self.sandbox_factory( + image=image, + flavor=self.execution.get("flavor", "cpu-basic"), + idle_timeout=self.timeout, + namespace=self.execution.get("namespace"), + forward_hf_token=False, + token=self.token, + start_timeout=self.timeout, + labels={ + "name": "openenv-coding-" + record["creation_label"], + "openenv-coding-attempt": record["creation_label"], + }, + ) + sandbox_id = sandbox.id + # Register before inspection so image/identity failures still clean up. + record["grade_sandbox_id" if grader else "sandbox_id"] = sandbox_id + if sandbox_id in self._owned_ids: + raise HFBackendError("sandbox identity was reused") + self._owned_ids.add(sandbox_id) + # SDK readiness checks the RPC server; the Jobs status can still lag. + # Reconcile only SCHEDULING within the caller's existing deadline, and + # recheck identity and the immutable image on every observation. + for attempt in range(20): + job = self._inspect(sandbox_id) + if job.docker_image != image: + raise HFBackendError("sandbox control-plane image mismatch") + if job.status.stage == "RUNNING": + break + if job.status.stage != "SCHEDULING": + raise HFBackendError("sandbox control-plane state is not runnable") + if attempt < 19: + time.sleep(0.5) + else: + raise HFBackendError("sandbox control-plane readiness did not converge") + # The SDK stat/read pair otherwise permits up to 512 MiB per read. + sandbox.files.MAX_READ_BYTES = self.max_bytes + sandbox.files.mkdir(self.workdir) + return sandbox + + def _recover_startup_jobs(self, record, pending): + # create() can allocate a job before failing during readiness. The label + # is unique to this attempt and is filtered again before any cancellation. + jobs = self.api.list_jobs( + labels={"openenv-coding-attempt": record["creation_label"]}, + namespace=self.execution.get("namespace"), + token=self.token, + timeout=self.cleanup_timeout, + ) + for index, job in enumerate(jobs): + if ( + index >= 4 + or (job.labels or {}).get("openenv-coding-attempt") + != record["creation_label"] + ): + raise HFBackendError("startup cleanup ownership could not be verified") + if ( + self.execution.get("namespace") + and job.owner.name != self.execution["namespace"] + ): + raise HFBackendError("startup cleanup namespace mismatch") + pending.add(job.id) + self._owned_ids.add(job.id) + record["cleanup_job_ids"] = sorted(pending) + record[ + "grade_sandbox_id" if "grade_sandbox_id" in record else "sandbox_id" + ] = sorted(pending)[0] + + def _cleanup(self, record, sandbox): + sandbox_id = record.get("grade_sandbox_id") or record.get("sandbox_id") + if sandbox_id is None and not record.get("creation_label"): + return + pending = {sandbox_id} if sandbox_id is not None else set() + recovery_complete = sandbox_id is not None + try: + with _deadline(self.cleanup_timeout): + for _ in range(self.cleanup_attempts): + if not recovery_complete: + try: + self._recover_startup_jobs(record, pending) + recovery_complete = bool(pending) + except _DeadlineExceeded: + raise + except Exception: + pass + if not pending: + continue + for owned_id in list(pending): + try: + self.api.cancel_job( + job_id=owned_id, + namespace=self.execution.get("namespace"), + token=self.token, + ) + except _DeadlineExceeded: + raise + except Exception: + if sandbox is not None and sandbox.id == owned_id: + try: + sandbox.kill() + except _DeadlineExceeded: + raise + except Exception: + pass + try: + if self._inspect(owned_id).status.stage in _TERMINAL: + pending.remove(owned_id) + except _DeadlineExceeded: + raise + except Exception: + continue + if not pending and recovery_complete: + record["cleanup_verified"] = True + break + except Exception: + pass + if not record["cleanup_verified"]: + record["error"] = record["error"] or "sandbox cleanup could not be verified" + if "reward" in record: + record["reward"] = None + + def _audit_solver(self, sandbox): + evidence = { + "sandbox_id": sandbox.id, + "private_paths_absent": False, + "fresh_state_absent": False, + } + self._audit.append(evidence) + for path in self.private_paths: + if sandbox.files.exists(path): + raise HFBackendError("private path exists in solver sandbox") + evidence["private_paths_absent"] = True + if sandbox.files.exists(self.marker_path): + raise HFBackendError("fresh sandbox contains prior-state marker") + evidence["fresh_state_absent"] = True + sandbox.files.write(self.marker_path, self.canary_digest.encode()) + + def _run(self, sandbox, argv, end, *, cwd=None, trace=None): + output = [] + size = 0 + + def collect(chunk): + nonlocal size + raw = chunk.encode() + if size + len(raw) > self.max_output: + if trace is not None: + trace["output_truncated"] = True + raise HFBackendError("terminal output exceeded byte limit") + size += len(raw) + output.append(chunk) + + try: + remaining = end - time.monotonic() + if remaining <= 0: + raise HFBackendError("controller deadline exceeded") + result = sandbox.run( + argv, + shell=False, + cwd=cwd or self.workdir, + timeout=remaining, + check=False, + capture_output=False, + on_stdout=collect, + on_stderr=collect, + ) + if trace is not None: + trace["exit_code"] = result.exit_code + if result.timed_out: + raise HFBackendError("sandbox command timed out") + return result.exit_code, "".join(output) + finally: + if trace is not None: + trace["output"] = "" if trace["output_truncated"] else "".join(output) + + def _collect(self, sandbox): + files = {} + remaining = self.max_bytes + for path in self.submission_paths: + try: + entry = sandbox.files.stat(path) + except SandboxError as exc: + if exc.status_code == 404: + continue + raise + if ( + entry.type != "file" + or type(entry.size) is not int + or not 0 <= entry.size <= remaining + ): + raise HFBackendError("submission is not a bounded regular file") + sandbox.files.MAX_READ_BYTES = remaining + data = sandbox.files.read(path) + if ( + not isinstance(data, bytes) + or len(data) != entry.size + or len(data) > remaining + ): + raise HFBackendError("submission changed size or exceeded byte limit") + files[path] = data + if any(secret.encode() in data for secret in self._credentials): + raise HFBackendError("credential detected in submission artifact") + remaining -= len(data) + return files + + def execute_script(self, script: str, role: str) -> dict: + """Execute a reference/candidate script without exposing controller credentials.""" + record = self._record() + record["role"] = role + sandbox = None + started = time.monotonic() + try: + with _deadline(self.timeout): + if ( + not isinstance(script, str) + or len(script.encode()) > self.max_script + ): + raise HFBackendError("script exceeds byte limit or is not text") + sandbox = self._create(self.execution["agent_image"], record) + self._audit_solver(sandbox) + exit_code, output = self._run( + sandbox, ["/bin/bash", "-c", script], started + self.timeout + ) + record["transcript"].append( + { + "role": role, + "script": script, + "output": output, + "exit_code": exit_code, + } + ) + if exit_code != 0: + raise HFBackendError("task script exited nonzero") + record["files"] = self._collect(sandbox) + record["submission_digest"] = submission_digest(record["files"]) + except Exception as exc: + record["error"] = self._error(exc) + finally: + self._cleanup(record, sandbox) + record["latency_seconds"] = time.monotonic() - started + self._transcripts.append(record["transcript"]) + return self._redact(record) + + def _model(self, model, seed, *, tools=False): + model = dict(model) + if tools and "context_paths" in model: + raise HFBackendError("target.context_paths is unsupported") + if ( + not isinstance(model.get("model_id"), str) + or not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", model["model_id"]) + or not isinstance(model.get("revision"), str) + or not _REVISION.fullmatch(model["revision"]) + or not isinstance(model.get("provider"), str) + or not model["provider"] + or model["provider"] in {"auto", "cheapest", "fastest", "preferred"} + ): + raise HFBackendError( + "model_id, pinned source revision and explicit provider are required" + ) + if type(seed) is not int or seed < 0: + raise HFBackendError("seed must be a nonnegative integer") + if tools and model.get("supports_tools") is False: + raise HFBackendError( + "configured model does not support terminal function calling" + ) + if "enable_thinking" in model and type(model["enable_thinking"]) is not bool: + raise HFBackendError("enable_thinking must be a boolean") + if type(model.get("supports_seed", False)) is not bool: + raise HFBackendError("supports_seed must be an explicit boolean") + aliases = model.get("response_model_aliases", []) + if not isinstance(aliases, list) or any( + not isinstance(item, str) or not item for item in aliases + ): + raise HFBackendError( + "response_model_aliases must be exact nonempty strings" + ) + allowed = {model["model_id"], *aliases} + if model.get("provider_model_id"): + if not isinstance(model["provider_model_id"], str): + raise HFBackendError("provider_model_id must be text") + allowed.add(model["provider_model_id"]) + limits = { + key: self._bound(model, key, default, maximum) + for key, default, maximum in ( + ("max_steps", 8, 128), + ("max_tokens", 2048, 32768), + ("max_total_tokens", 16384, 1048576), + ("max_output_bytes", 16384, 1048576), + ) + } + if not self.inference_token: + raise HFBackendError("controller HF_INFERENCE_TOKEN is required") + client = self.inference_factory( + provider=model["provider"], + api_key=self.inference_token, + timeout=self.timeout, + ) + return model, allowed, limits, client + + def _chat(self, client, model, allowed, limits, messages, record, seed, *, tools): + used = record["input_tokens"] + record["output_tokens"] + # Bound the request body as well as provider-reported usage. Providers do + # not expose a preflight tokenizer through this API. + message_bytes = len(json.dumps(messages).encode()) + if message_bytes > limits["max_total_tokens"] * 4: + raise HFBackendError("chat context exceeds byte budget") + remaining = limits["max_total_tokens"] - used + if remaining <= 0: + raise HFBackendError("model total token budget exhausted") + request = { + "messages": self._redact(messages), + "model": model["model_id"], + "max_tokens": min(limits["max_tokens"], remaining), + "temperature": model.get("temperature", 0), + } + if "top_p" in model: + value = model["top_p"] + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or not 0 < value <= 1 + ): + raise HFBackendError("top_p must be a finite number in (0, 1]") + request["top_p"] = value + if "enable_thinking" in model: + request["extra_body"] = { + "chat_template_kwargs": {"enable_thinking": model["enable_thinking"]} + } + if model.get("supports_seed", False): + request["seed"] = seed + if tools: + request.update(tools=[_TOOL], tool_choice="auto") + if not tools: + request.update(stream=True, stream_options={"include_usage": True}) + metadata = record["model_metadata"] + metadata.setdefault("requests", []).append( + {key: value for key, value in request.items() if key != "messages"} + ) + if tools: + response = _plain(client.chat_completion(**request)) + metadata.setdefault("responses", []).append( + { + "model": response.get("model"), + "system_fingerprint": response.get("system_fingerprint"), + "id": response.get("id"), + "seed_sent": request.get("seed"), + } + ) + else: + response = self._candidate_response( + client, request, allowed, limits, record + ) + if response.get("model") not in allowed: + raise HFBackendError( + "provider response model does not match configured exact identities" + ) + usage = response.get("usage") + if not isinstance(usage, dict): + raise HFBackendError("provider response omitted token usage") + for source, target in ( + ("prompt_tokens", "input_tokens"), + ("completion_tokens", "output_tokens"), + ): + value = usage.get(source) + if type(value) is not int or value < 0: + raise HFBackendError("provider returned invalid token usage") + if tools: + record[target] += value + record["usage"][target] += value + if ( + usage["completion_tokens"] > request["max_tokens"] + or record["input_tokens"] + record["output_tokens"] + > limits["max_total_tokens"] + ): + raise HFBackendError("provider exceeded token budget") + choices = response.get("choices", []) + if len(choices) != 1 or not isinstance(choices[0].get("message"), dict): + raise HFBackendError("provider must return exactly one assistant message") + message = choices[0]["message"] + record["transcript"].append(message) + if len(json.dumps(message).encode()) > limits["max_output_bytes"]: + raise HFBackendError("model output exceeded byte limit") + if choices[0].get("finish_reason") not in {"stop", "tool_calls"}: + raise HFBackendError("model response was incomplete") + return message + + def _candidate_response(self, client, request, allowed, limits, record): + # The pinned SDK yields parsed deltas but owns HTTP contexts on the client. + response = {} + parts = {"content": []} + size = script_size = 0 + stopped = False + finish_reason = None + usage = None + try: + with closing(client.chat_completion(**request)) as stream: + for parsed in stream: + event = _plain(parsed) + if not isinstance(event, dict) or event.get("error") is not None: + raise HFBackendError("provider returned a stream error") + # Outer SDK dicts omit extra fields on nested dataclasses. + if isinstance(event.get("choices"), list): + for index, choice in enumerate(parsed.choices): + event["choices"][index]["delta"] = _plain(choice.delta) + if not response: + response = { + key: event.get(key) + for key in ("model", "id", "system_fingerprint") + } + record["model_metadata"]["responses"].append( + {**response, "seed_sent": request.get("seed")} + ) + if event.get("model") not in allowed: + raise HFBackendError( + "provider response model does not match configured exact identities" + ) + if ( + not isinstance(event.get("id"), str) + or not event["id"] + or any( + event.get(key) != value for key, value in response.items() + ) + ): + raise HFBackendError( + "provider stream response identity changed" + ) + if usage is not None: + raise HFBackendError("provider returned data after final usage") + choices = event.get("choices") + if not isinstance(choices, list) or len(choices) > 1: + raise HFBackendError( + "provider must stream exactly one assistant choice" + ) + if choices: + if stopped: + raise HFBackendError( + "provider returned a choice after stop" + ) + choice = choices[0] + if ( + not isinstance(choice, dict) + or type(choice.get("index")) is not int + or choice["index"] != 0 + or not isinstance(choice.get("delta"), dict) + ): + raise HFBackendError( + "provider must stream exactly one assistant choice at index 0" + ) + delta = choice["delta"] + if delta.get("role") not in {None, "assistant"} or ( + not parts["content"] and delta.get("role") != "assistant" + ): + raise HFBackendError( + "provider must stream an assistant message" + ) + for key, value in delta.items(): + if key == "role" or value is None: + continue + if key == "tool_calls" and value == []: + continue + if key not in {"content", "reasoning", "reasoning_content"}: + raise HFBackendError( + "candidate stream contains tool calls or unsupported deltas" + ) + if not isinstance(value, str): + raise HFBackendError( + "provider stream delta must be text" + ) + length = len(value.encode()) + size += length + if key == "content": + script_size += length + if ( + size > limits["max_output_bytes"] + or script_size > self.max_script + ): + raise HFBackendError("model output exceeded byte limit") + if value: + parts.setdefault(key, []).append(value) + # Empty assistant role events still establish the message. + if not parts["content"]: + parts["content"].append("") + finish = choice.get("finish_reason") + if finish is not None: + finish_reason = finish + record["model_metadata"]["responses"][-1][ + "finish_reason" + ] = finish + if finish not in {"stop", "length"}: + raise HFBackendError("model response was incomplete") + stopped = True + elif not stopped: + raise HFBackendError( + "model response was incomplete before usage" + ) + event_usage = event.get("usage") + if event_usage is not None: + if not stopped: + raise HFBackendError("provider returned usage before stop") + if not isinstance(event_usage, dict) or any( + type(event_usage.get(key)) is not int + or event_usage[key] < 0 + for key in ( + "prompt_tokens", + "completion_tokens", + "total_tokens", + ) + ): + raise HFBackendError( + "provider returned invalid token usage" + ) + if event_usage["total_tokens"] != ( + event_usage["prompt_tokens"] + + event_usage["completion_tokens"] + ): + raise HFBackendError( + "provider returned inconsistent token usage" + ) + usage = event_usage + # Usage already observed remains evidence if the stream + # fails later. The caller accounts only non-streamed calls. + for source, target in ( + ("prompt_tokens", "input_tokens"), + ("completion_tokens", "output_tokens"), + ): + record[target] += usage[source] + record["usage"][target] += usage[source] + if ( + usage["completion_tokens"] > request["max_tokens"] + or record["input_tokens"] + record["output_tokens"] + > limits["max_total_tokens"] + ): + raise HFBackendError("provider exceeded token budget") + elif not choices: + raise HFBackendError("provider response omitted token usage") + if not stopped: + raise HFBackendError("model response was incomplete") + if usage is None: + raise HFBackendError("provider response omitted token usage") + if finish_reason != "stop": + raise HFBackendError("model response was incomplete") + response.update( + usage=usage, + choices=[ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + **{key: "".join(value) for key, value in parts.items()}, + }, + } + ], + ) + return response + except Exception: + if parts["content"]: + record["transcript"].append( + { + "role": "assistant", + **{key: "".join(value) for key, value in parts.items()}, + } + ) + raise + finally: + client.close() + + def _model_metadata(self, model, seed): + return { + "requested_model": model["model_id"], + "source_revision": model["revision"], + "provider": model["provider"], + "provider_model_id": model.get("provider_model_id"), + "response_model_aliases": model.get("response_model_aliases", []), + "seed": seed, + "seed_supported": model.get("supports_seed", False), + "served_weight_revision_pinned": False, + "responses": [], + } + + def _instruction(self): + raw, _ = _local_file(self.task_root / "instruction.md", self.max_script) + return raw.decode("utf-8") + + def _candidate_instruction(self, model, record): + instruction = self._instruction() + if "context_paths" not in model: + return instruction + paths = _candidate_context_paths(model["context_paths"]) + record["candidate_context"] = [] + parts = [instruction] + remaining = 65536 + for path in paths: + raw, _ = _local_file(self.task_root / path, remaining) + remaining -= len(raw) + if b"\0" in raw: + raise HFBackendError("candidate context contains NUL") + text = raw.decode("utf-8") + runtime_path = ( + PurePosixPath(path).relative_to("environment/source").as_posix() + ) + record["candidate_context"].append( + { + "path": path, + "runtime_path": runtime_path, + "size": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + } + ) + parts.append( + f"Public source path: {path}\n" + f"Runtime filename relative to {self.workdir}: {runtime_path}\n" + f"---\n{text}\n---" + ) + return "\n\n".join(parts) + + def generate_candidate(self, model: dict, seed: int) -> dict: + """Generate a shell script from the instruction and explicit public context.""" + record = self._record() + record["script"] = "" + started = time.monotonic() + try: + with _deadline(self.timeout): + instruction = self._candidate_instruction(model, record) + model, allowed, limits, client = self._model(model, seed) + record["model_metadata"] = self._model_metadata(model, seed) + messages = [ + { + "role": "system", + "content": "Return only a bash script that attempts the task. Do not use Markdown fences.", + }, + {"role": "user", "content": instruction}, + ] + record["transcript"] = messages + message = self._chat( + client, model, allowed, limits, messages, record, seed, tools=False + ) + script = message.get("content") + if ( + not isinstance(script, str) + or not script.strip() + or message.get("tool_calls") + ): + raise HFBackendError( + "candidate response must contain a shell script" + ) + if len(script.encode()) > self.max_script: + raise HFBackendError("candidate script exceeds byte limit") + record["script"] = script + except Exception as exc: + record["error"] = self._error(exc) + finally: + record["latency_seconds"] = time.monotonic() - started + self._transcripts.append(record["transcript"]) + return self._redact(record) + + def solve(self, model: dict, seed: int, role="target") -> dict: + """Run a bounded terminal-tool agent with inference kept in the controller.""" + record = self._record() + record["role"] = role + record.update(truncated=False, termination_reason="completed") + sandbox = None + started = time.monotonic() + try: + with _deadline(self.timeout): + model, allowed, limits, client = self._model(model, seed, tools=True) + record["model_metadata"] = self._model_metadata(model, seed) + messages = [ + { + "role": "system", + "content": "Complete the task using the terminal function. Commands run in the task workspace. Finish with a short message after saving your solution files.", + }, + {"role": "user", "content": self._instruction()}, + ] + record["transcript"] = messages + sandbox = self._create(self.execution["agent_image"], record) + self._audit_solver(sandbox) + calls_used = 0 + for _ in range(limits["max_steps"]): + message = self._chat( + client, + model, + allowed, + limits, + messages, + record, + seed, + tools=True, + ) + calls = message.get("tool_calls") or [] + if not calls: + break + if not isinstance(calls, list): + raise HFBackendError("invalid terminal function calls") + if calls_used + len(calls) > limits["max_steps"]: + record.update( + truncated=True, termination_reason="tool_call_limit" + ) + break + ids = set() + for call in calls: + if ( + call.get("type") != "function" + or call.get("function", {}).get("name") != "terminal" + or not isinstance(call.get("id"), str) + or not call["id"] + or call["id"] in ids + ): + raise HFBackendError("invalid terminal function call") + ids.add(call["id"]) + arguments = json.loads( + call["function"]["arguments"], + object_pairs_hook=_unique_pairs, + ) + if ( + not isinstance(arguments, dict) + or set(arguments) != {"command"} + or not isinstance(arguments["command"], str) + or len(arguments["command"].encode()) > self.max_script + ): + raise HFBackendError("invalid terminal function arguments") + calls_used += 1 + exit_code, output = self._run( + sandbox, + ["/bin/bash", "-c", arguments["command"]], + started + self.timeout, + ) + messages.append( + { + "role": "tool", + "tool_call_id": call["id"], + "content": json.dumps( + {"exit_code": exit_code, "output": output} + ), + } + ) + else: + record.update(truncated=True, termination_reason="step_limit") + record["files"] = self._collect(sandbox) + record["submission_digest"] = submission_digest(record["files"]) + except Exception as exc: + record["error"] = self._error(exc) + finally: + self._cleanup(record, sandbox) + record["latency_seconds"] = time.monotonic() - started + self._transcripts.append(record["transcript"]) + return self._redact(record) + + def _test_files(self): + root = self.task_root / "tests" + if root.is_symlink() or not root.is_dir(): + raise HFBackendError("task tests must be a regular directory") + files = [] + remaining = 16 * 1048576 + for index, path in enumerate(root.rglob("*")): + if index >= 4096: + raise HFBackendError("too many verifier filesystem entries") + info = path.lstat() + if stat.S_ISDIR(info.st_mode): + continue + relative = path.relative_to(root).as_posix() + raw, mode = _local_file(path, remaining) + remaining -= len(raw) + files.append(("/tests/" + relative, raw, mode)) + return sorted(files) + + def grade(self, submission: dict) -> dict: + """Grade exact allowlisted submission bytes in a separate verifier Sandbox.""" + record = { + "grade_sandbox_id": None, + "verifier_image_ref": self.execution["verifier_image"], + "image_reference_kind": "control_plane_immutable_reference", + "reward": None, + "rawreward": b"", + "transcript": [], + "cleanup_verified": False, + "error": None, + "limitations": list(_LIMITATIONS), + } + sandbox = None + started = time.monotonic() + try: + with _deadline(self.timeout): + files = submission["files"] + if ( + submission.get("error") + or not isinstance(files, dict) + or any( + path not in self.submission_paths or not isinstance(raw, bytes) + for path, raw in files.items() + ) + or sum(map(len, files.values())) > self.max_bytes + ): + raise HFBackendError("invalid submission transfer") + if submission.get("submission_digest") != submission_digest(files): + raise HFBackendError("submission digest mismatch") + files = dict(files) + tests = self._test_files() + command = self.execution.get( + "verifier_command", ["/bin/bash", "/tests/test.sh"] + ) + if ( + not isinstance(command, list) + or not command + or any(not isinstance(arg, str) or "\x00" in arg for arg in command) + ): + raise HFBackendError("verifier_command must be an argv list") + sandbox = self._create( + self.execution["verifier_image"], record, grader=True + ) + if sandbox.files.exists("/tests") or sandbox.files.exists("/solution"): + raise HFBackendError( + "verifier image contains preexisting private paths" + ) + for path in self.submission_paths: + if sandbox.files.exists(path): + if sandbox.files.stat(path).type != "file": + raise HFBackendError( + "verifier submission destination is not a regular file" + ) + sandbox.files.delete(path) + for path, raw, mode in tests: + sandbox.files.write(path, raw, mode=mode) + sandbox.files.write( + "/tests/.openenv-canary-" + uuid.uuid4().hex, self.canary.encode() + ) + for path, raw in files.items(): + sandbox.files.write(path, raw) + # Harbor normally mounts this directory. Fresh Sandbox graders + # need it before a native verifier can write its reward file. + sandbox.files.mkdir(str(PurePosixPath(self.reward_path).parent)) + # A stale image-provided reward must not become new evidence. + if sandbox.files.exists(self.reward_path): + sandbox.files.delete(self.reward_path) + trace = {"exit_code": None, "output": "", "output_truncated": False} + record["transcript"].append(trace) + exit_code, _ = self._run( + sandbox, command, started + self.timeout, trace=trace + ) + if exit_code != 0: + raise HFBackendError("verifier command exited nonzero") + entry = sandbox.files.stat(self.reward_path) + if ( + entry.type != "file" + or type(entry.size) is not int + or not 0 < entry.size <= min(self.max_bytes, 65536) + ): + raise HFBackendError("reward is not a bounded regular file") + sandbox.files.MAX_READ_BYTES = min(self.max_bytes, 65536) + raw = sandbox.files.read(self.reward_path) + if len(raw) != entry.size: + raise HFBackendError("reward changed during collection") + if any(secret.encode() in raw for secret in self._credentials): + raise HFBackendError("credential detected in reward artifact") + record["rawreward"] = raw + text = raw.decode("utf-8").strip() + if text.startswith("{"): + value = json.loads( + text, parse_float=Decimal, object_pairs_hook=_unique_pairs + )[self.execution.get("reward_key", "reward")] + else: + value = Decimal(text) + if ( + isinstance(value, bool) + or not isinstance(value, (Decimal, int)) + or not Decimal(value).is_finite() + or not math.isfinite(float(value)) + ): + raise HFBackendError("reward must be finite numeric data") + record["reward"] = float(value) + except Exception as exc: + record["error"] = self._error(exc) + record["reward"] = None + finally: + self._cleanup(record, sandbox) + record["latency_seconds"] = time.monotonic() - started + self._transcripts.append(record.get("transcript", [])) + record = self._redact(record) + for trace in record["transcript"]: + if len(trace["output"].encode()) > self.max_output: + trace["output"] = "" + trace["output_truncated"] = True + return record + + def leakage_probes(self) -> list: + """Audit private paths, two fresh solver instances, and accumulated transcripts.""" + probes = [] + while len(self._audit) < 2: + record = self.execute_script(":", "leakage_probe") + if record["error"]: + probes.append( + { + "name": "fresh_state_absent", + "passed": False, + "error": record["error"], + "evidence": list(self._audit), + } + ) + break + for name in ("private_paths_absent", "fresh_state_absent"): + if any(probe["name"] == name for probe in probes): + continue + probes.append( + { + "name": name, + "error": None, + "passed": len(self._audit) >= 2 + and all(item[name] for item in self._audit), + "evidence": list(self._audit), + } + ) + probes.append( + { + "name": "canary_not_in_transcript", + "error": None, + "passed": not self._has_canary(self._transcripts), + "evidence": { + "canary_digest": self.canary_digest, + "transcript_count": len(self._transcripts), + }, + } + ) + return probes diff --git a/examples/coding_audit/runner.py b/examples/coding_audit/runner.py new file mode 100644 index 000000000..2ea17cc51 --- /dev/null +++ b/examples/coding_audit/runner.py @@ -0,0 +1,736 @@ +"""Collect a bounded coding audit using fresh, credential-scoped HF Sandboxes. + +This supplemental profile does not replace RFC 008 admission or demonstrate RL +training benefit. Inference runs in the controller; task/verifier data cross only +the explicit file boundary in ``hf``. +""" + +import hashlib +import json +import os +import re +import shutil +import tempfile +import tomllib +from copy import deepcopy +from fractions import Fraction +from pathlib import Path, PurePosixPath + +from openenv.validation.runner import _DIGEST_EXCLUDED_DIRS, source_digest +from openenv.validation.runtime.json_values import dumps, loads + +from .scoring import assess_coding_audit, validate_audit_config + +PROFILE = "harbor-coding-audit-v1" +TARGET_MODEL = "Qwen/Qwen3.8-27B" +TARGET_REVISION = "37561816e66b5b9153b1b64c10968c18c1e6f4a0" +_IMAGE = re.compile(r"[^\s@]+@sha256:[0-9a-f]{64}\Z") + + +def _integer(value, name, low, high): + if type(value) is not int or not low <= value <= high: + raise ValueError(f"{name} must be an integer in [{low}, {high}]") + return value + + +def _candidate_context_paths(paths): + message = ( + "generator.context_paths must list at most 32 unique normalized public " + "files under environment/source/" + ) + if not isinstance(paths, list) or len(paths) > 32: + raise ValueError(message) + result = [] + private = { + "tests", + "test", + "solution", + "validation", + "verifier", + "reference", + "references", + "spec", + "specs", + "secret", + "secrets", + "credential", + "credentials", + "private", + } + for value in paths: + if ( + not isinstance(value, str) + or "\\" in value + or ":" in value + or any(ord(char) < 32 or ord(char) == 127 for char in value) + ): + raise ValueError(message) + path = PurePosixPath(value) + if ( + path.as_posix() != value + or path.parts[:2] != ("environment", "source") + or len(path.parts) < 3 + or any( + part.startswith(".") or part.lower() in private for part in path.parts + ) + or value in result + ): + raise ValueError(message) + result.append(value) + return result + + +def normalize_coding_config(config): + """Validate operator inputs without launching Jobs, Sandboxes or inference.""" + if not isinstance(config, dict): + raise ValueError("coding configuration must be an object") + quality = deepcopy(config.get("validation", {})) + config = json.loads(dumps(config)) + allowed = { + "schema_version", + "profile", + "target", + "generator", + "execution", + "validation", + "generation", + "negative_cases", + "seeds", + "cloud", + } + if ( + set(config) - allowed + or type(config.get("schema_version")) is not int + or config["schema_version"] != 1 + ): + raise ValueError("unsupported coding audit configuration") + if config.get("profile", PROFILE) != PROFILE: + raise ValueError("unsupported coding audit profile") + config["profile"] = PROFILE + config["validation"] = validate_audit_config(quality) + _integer(config["validation"]["reference_repeats"], "reference_repeats", 2, 16) + _integer(config["validation"]["target_attempts"], "target_attempts", 2, 128) + for name in ("target", "generator"): + model = config.get(name, config.get("target")) + if not isinstance(model, dict): + raise ValueError(f"{name} model configuration is required") + model = dict(model) + model_keys = { + "model_id", + "revision", + "provider", + "provider_model_id", + "response_model_aliases", + "max_steps", + "max_tokens", + "max_total_tokens", + "max_output_bytes", + "temperature", + "supports_seed", + "supports_tools", + "top_p", + "enable_thinking", + "context_paths", + } + if name == "target" and "context_paths" in model: + raise ValueError("target.context_paths is unsupported") + if set(model) - model_keys: + raise ValueError( + f"unknown {name} model option; credentials belong in environment secrets" + ) + if not re.fullmatch(r"[^\s/@]+/[^\s/@]+", model.get("model_id", "")): + raise ValueError(f"{name}.model_id must be a canonical Hub repository") + if not re.fullmatch(r"[0-9a-f]{40}", model.get("revision", "")): + raise ValueError(f"{name}.revision must be a full source commit") + if ( + not isinstance(model.get("provider"), str) + or not re.fullmatch(r"[a-z0-9-]+", model["provider"]) + or model["provider"] == "auto" + ): + raise ValueError(f"{name}.provider must be selected explicitly") + if "enable_thinking" in model and type(model["enable_thinking"]) is not bool: + raise ValueError(f"{name}.enable_thinking must be a boolean") + if "context_paths" in model: + model["context_paths"] = _candidate_context_paths(model["context_paths"]) + model.setdefault("max_steps", 8) + model.setdefault("max_tokens", 1024) + model.setdefault("max_total_tokens", 16384) + model.setdefault("supports_seed", False) + model.setdefault("supports_tools", True) + for key, high in ( + ("max_steps", 128), + ("max_tokens", 32768), + ("max_total_tokens", 262144), + ): + _integer(model[key], f"{name}.{key}", 1, high) + config[name] = model + execution = config.get("execution", {}) + if not isinstance(execution, dict): + raise ValueError("execution must be an object") + execution_keys = { + "agent_image", + "verifier_image", + "submission_paths", + "private_paths", + "workdir", + "flavor", + "namespace", + "timeout_seconds", + "verifier_command", + "reward_path", + "reward_key", + "max_submission_bytes", + "cleanup_attempts", + "cleanup_timeout_seconds", + "max_tool_output_bytes", + "max_script_bytes", + "strong_isolation", + "require_strong_isolation", + } + if set(execution) - execution_keys: + raise ValueError( + "unknown execution option; task credentials cannot be embedded" + ) + for key in ("agent_image", "verifier_image"): + if not _IMAGE.fullmatch(execution.get(key, "")): + raise ValueError(f"execution.{key} must be an immutable OCI reference") + if not execution.get("submission_paths"): + raise ValueError( + "execution.submission_paths must declare the artifact boundary" + ) + generation = config.setdefault("generation", {}) + if not isinstance(generation, dict): + raise ValueError("generation must be an object") + if set(generation) - {"candidate_count"}: + raise ValueError("unknown generation option") + generation.setdefault("candidate_count", 2) + _integer(generation["candidate_count"], "candidate_count", 1, 16) + negatives = config.get("negative_cases", []) + expected = config["validation"]["required_negative_cases"] + if ( + not isinstance(negatives, list) + or any( + not isinstance(case, dict) or set(case) != {"id", "script"} + for case in negatives + ) + or [case["id"] for case in negatives] != expected + or any( + not isinstance(case["script"], str) or not case["script"].strip() + for case in negatives + ) + ): + raise ValueError( + "negative_cases must supply the trusted required IDs and scripts in order" + ) + attempts = config["validation"]["target_attempts"] + seeds = config.setdefault("seeds", list(range(attempts))) + if ( + not isinstance(seeds, list) + or len(seeds) != attempts + or any(type(seed) is not int for seed in seeds) + or len(set(seeds)) != len(seeds) + ): + raise ValueError( + "one distinct seed/attempt identifier per target attempt is required" + ) + for seed in seeds: + _integer(seed, "seed", 0, 2**32 - 1) + cloud = config.setdefault("cloud", {}) + if not isinstance(cloud, dict): + raise ValueError("cloud must be an object") + if set(cloud) - { + "results_repo", + "namespace", + "flavor", + "timeout_seconds", + "max_sandboxes", + "max_inference_calls", + }: + raise ValueError("unknown cloud option") + cloud.setdefault("max_sandboxes", 100) + cloud.setdefault("max_inference_calls", 256) + _integer(cloud["max_sandboxes"], "max_sandboxes", 1, 10000) + _integer(cloud["max_inference_calls"], "max_inference_calls", 1, 10000) + return config + + +def coding_budget(config, *, has_reference=True): + """A conservative, finite launch ceiling, including leakage probes.""" + repeats = config["validation"]["reference_repeats"] + candidates = config["generation"]["candidate_count"] + references = candidates + int(has_reference) + executions = ( + 1 + references * repeats + len(config["negative_cases"]) + len(config["seeds"]) + ) + return { + "sandbox_jobs_max": 2 * executions + 6, + "inference_calls_max": candidates + + len(config["seeds"]) * config["target"]["max_steps"], + "generated_candidates": candidates, + "target_attempts": len(config["seeds"]), + } + + +def _credentials(): + names = ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN") + values = {name: os.environ.get(name) for name in names} + if not all(values.values()): + raise ValueError( + "HF_TOKEN, HF_INFERENCE_TOKEN and HF_SANDBOX_TOKEN are required" + ) + if len(set(values.values())) != len(values): + raise ValueError( + "artifact, inference and sandbox credentials must be distinct and separately scoped" + ) + return values + + +def _write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(dumps(value) + "\n") + + +def live_provider_id(info, provider): + """Read both Hub SDK provider-map representations without selecting an alias.""" + mappings = info.inference_provider_mapping + if isinstance(mappings, dict): + candidates = [mappings[provider]] if provider in mappings else [] + elif isinstance(mappings, list): + candidates = [ + entry + for entry in mappings + if ( + entry.get("provider") + if isinstance(entry, dict) + else getattr(entry, "provider", None) + ) + == provider + ] + else: + candidates = [] + if len(candidates) != 1: + raise ValueError("selected model/provider mapping is missing or ambiguous") + entry = candidates[0] + status = ( + entry.get("status") + if isinstance(entry, dict) + else getattr(entry, "status", None) + ) + identity = ( + entry.get("provider_id") + if isinstance(entry, dict) + else getattr(entry, "provider_id", None) + ) + if status != "live" or not isinstance(identity, str) or not identity: + raise ValueError("selected model/provider is not live") + return identity + + +def validate_coding_task(task_root, config): + """Check supported native task contracts locally, before controller spending. + + Returns normalized audit configuration. Passing this check is preparation, + not evidence that the task, image, or grader passes runtime validation. + """ + config = normalize_coding_config(config) + task_root = Path(task_root).resolve() + task_config = tomllib.loads((task_root / "task.toml").read_text()) + if any((task_root / "steps").glob("*")): + raise ValueError("multi-step Harbor tasks are not supported by this profile") + for section in ("environment", "agent", "verifier", "solution"): + section_config = task_config.get(section, {}) + if not isinstance(section_config, dict): + raise ValueError("task phase configuration must be an object") + if section_config.get("environment"): + raise ValueError("separate phase environment contracts are unsupported") + if section_config.get("allow_internet") is False: + raise ValueError("non-public phase network contracts are unsupported") + if section_config.get("allowed_hosts"): + raise ValueError("network allowlist contracts are unsupported") + if section_config.get("env"): + raise ValueError( + "task environment/credential substitutions are unsupported" + ) + if section_config.get("network_mode", "public") != "public": + raise ValueError("non-public phase network contracts are unsupported") + environment = task_config.get("environment", {}) + if task_config.get("verifier", {}).get("collect"): + raise ValueError("verifier collection commands are unsupported") + if environment.get("workdir") not in ( + None, + config["execution"].get("workdir", "/app"), + ): + raise ValueError("source and audit workdirs disagree") + if any( + environment.get(key) + for key in ("gpus", "tpu", "mcp_servers", "healthcheck", "skills_dir") + ): + raise ValueError( + "accelerator, healthcheck, skills and external MCP requirements are unsupported" + ) + if any( + (task_root / "environment" / name).exists() + for name in ( + "compose.yaml", + "compose.yml", + "docker-compose.yaml", + "docker-compose.yml", + ) + ): + raise ValueError("multi-service Compose tasks are unsupported") + if ( + not (task_root / "instruction.md").is_file() + or not (task_root / "tests/test.sh").is_file() + ): + raise ValueError("a native instruction and verifier are required") + budget = coding_budget( + config, has_reference=(task_root / "solution/solve.sh").is_file() + ) + if ( + budget["sandbox_jobs_max"] > config["cloud"]["max_sandboxes"] + or budget["inference_calls_max"] > config["cloud"]["max_inference_calls"] + ): + raise ValueError( + "configured launch budget is too small for the requested audit" + ) + return config + + +def run_coding_validation(task_root, config, output_dir, *, backend=None): + """Generate, challenge, replay and measure a single prepared native task. + + Every result is bound to frozen task/verifier bytes and immutable requested + image references. Provider source revisions are recorded without pretending + they pin the provider's served weights. Outputs must be a new directory. + """ + config = validate_coding_task(task_root, config) + task_root, output_dir = Path(task_root).resolve(), Path(output_dir).resolve() + if output_dir.is_relative_to(task_root) and not any( + part in _DIGEST_EXCLUDED_DIRS + for part in output_dir.relative_to(task_root).parts + ): + raise ValueError("audit output must be outside the digested task tree") + secrets = [ + value + for name in ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN") + if (value := os.environ.get(name)) + ] + if any(secret in dumps(config) for secret in secrets): + raise ValueError("credentials may not be embedded in audit metadata") + original_digest = source_digest(task_root) + budget = coding_budget( + config, has_reference=(task_root / "solution/solve.sh").is_file() + ) + output_dir.mkdir(parents=True, exist_ok=False) + evidence = { + "task_digest": original_digest, + "verifier_digest": None, + "agent_image": config["execution"]["agent_image"], + "verifier_image": config["execution"]["verifier_image"], + "target": { + "model": config["target"]["model_id"], + "revision": config["target"]["revision"], + "provider": config["target"]["provider"], + }, + "references": [], + "trials": [], + "leakage_probes": [], + "generation": [], + "budget": budget, + "configuration_digest": hashlib.sha256(dumps(config).encode()).hexdigest(), + "requested_configuration_digest": hashlib.sha256( + dumps(config).encode() + ).hexdigest(), + "limitations": [ + "Supplemental coding audit; full RFC 008 qualification is not established.", + "Provider model source SHA does not attest served weights or deterministic sampling.", + "Image identities are control-plane immutable references, not runtime image attestations.", + "Sandbox RPC exposes its execution credential to the workload server; only reviewed workloads are supported.", + "Separate verifier Jobs do not prove submitted code cannot read or tamper with its grader.", + ], + } + + def sanitize(value): + if isinstance(value, str): + for secret in secrets: + value = value.replace(secret, "[REDACTED]") + return value + if isinstance(value, dict): + return {sanitize(key): sanitize(child) for key, child in value.items()} + if isinstance(value, list): + return [sanitize(child) for child in value] + return value + + def artifact(relative, data): + if isinstance(data, str): + data = data.encode() + digest = hashlib.sha256(data).hexdigest() + if any(secret.encode() in data for secret in secrets): + evidence["leakage_probes"].append( + {"name": "credential_in_artifact", "passed": False, "error": None} + ) + return {"sha256": digest, "omitted": "credential detected"} + path = output_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return {"path": relative, "sha256": digest} + + def persist(): + evidence["configuration_digest"] = hashlib.sha256( + dumps(config).encode() + ).hexdigest() + if any(secret in dumps(evidence) for secret in secrets) and not any( + probe.get("name") == "credential_in_metadata" + for probe in evidence["leakage_probes"] + ): + evidence["leakage_probes"].append( + {"name": "credential_in_metadata", "passed": False, "error": None} + ) + _write(output_dir / "evidence.json", sanitize(evidence)) + + try: + with tempfile.TemporaryDirectory(prefix="openenv-coding-frozen-") as temporary: + frozen = (Path(temporary) / "task").resolve() + shutil.copytree( + task_root, + frozen, + symlinks=True, + ignore=shutil.ignore_patterns(*_DIGEST_EXCLUDED_DIRS), + ) + if source_digest(frozen) != original_digest: + raise ValueError("task changed during snapshot") + evidence["verifier_digest"] = source_digest(frozen / "tests") + if backend is None: + from .hf import HFTaskBackend + + credentials = _credentials() + # Provider preflight is read-only and precedes every paid sandbox. + from huggingface_hub import HfApi + + api = HfApi(token=credentials["HF_INFERENCE_TOKEN"]) + for model in (config["target"], config["generator"]): + info = api.model_info(model["model_id"], revision=model["revision"]) + if info.sha != model["revision"]: + raise ValueError("model source revision could not be verified") + current = api.model_info( + model["model_id"], expand=["inferenceProviderMapping"] + ) + model.setdefault( + "provider_model_id", + live_provider_id(current, model["provider"]), + ) + backend = HFTaskBackend( + frozen, config["execution"], token=credentials["HF_SANDBOX_TOKEN"] + ) + + def trial(role, case_id, script=None, seed=None): + if source_digest(frozen) != original_digest: + raise ValueError("frozen task source changed") + number = len(evidence["trials"]) + submission = ( + backend.solve(config["target"], seed, role="target") + if role == "target" + else backend.execute_script(script, role) + ) + row = { + "role": role, + "case_id": case_id, + "task_digest": original_digest, + "verifier_digest": evidence["verifier_digest"], + "image_ref": submission.get("image_ref"), + "submission_digest": submission.get("submission_digest"), + "sandbox_id": submission.get("sandbox_id"), + "error": submission["error"] + if "error" in submission + else "execution error status missing", + "cleanup_verified": submission.get("cleanup_verified") is True, + "latency_seconds": submission.get("latency_seconds"), + "usage": submission.get("usage"), + "seed": seed, + "model_metadata": submission.get("model_metadata"), + "truncated": submission.get("truncated", False), + "termination_reason": submission.get("termination_reason"), + } + row["transcript"] = artifact( + f"trials/{number:04d}/transcript.json", + dumps(submission.get("transcript", [])), + ) + for index, (name, data) in enumerate( + sorted(submission.get("files", {}).items()) + ): + item = artifact(f"trials/{number:04d}/submission/{index:04d}", data) + row.setdefault("submission_files", []).append( + {"destination": name, **item} + ) + if row["error"] is None and row["cleanup_verified"]: + grade = backend.grade(submission) + row["grader_transcript"] = artifact( + f"trials/{number:04d}/grader-transcript.json", + dumps(grade.get("transcript", [])), + ) + row.update( + { + "grade_sandbox_id": grade.get("grade_sandbox_id"), + "verifier_image_ref": grade.get("verifier_image_ref"), + "reward": grade.get("reward"), + "error": grade["error"] + if "error" in grade + else "grader error status missing", + "cleanup_verified": grade.get("cleanup_verified") is True, + } + ) + if grade.get("rawreward"): + row["raw_reward"] = artifact( + f"trials/{number:04d}/reward.raw", grade["rawreward"] + ) + try: + raw_value = loads(grade["rawreward"]) + if isinstance(raw_value, dict): + raw_value = raw_value[ + config["execution"].get("reward_key", "reward") + ] + if isinstance(raw_value, (bool, str)) or isinstance( + row["reward"], bool + ): + raise ValueError("reward is not numeric") + if Fraction(str(raw_value)) != Fraction(str(row["reward"])): + raise ValueError( + "raw reward and parsed reward disagree" + ) + except (ValueError, TypeError, KeyError, ZeroDivisionError): + row.update( + error=row["error"] + if row["error"] is not None + else "raw reward fidelity check failed", + reward=None, + ) + elif row["error"] is None: + row.update(error="raw reward evidence missing", reward=None) + if role == "target": + row.update( + { + "model": config["target"]["model_id"], + "revision": config["target"]["revision"], + "provider": config["target"]["provider"], + } + ) + evidence["trials"].append(row) + persist() + if not row["cleanup_verified"]: + raise ValueError("stop collection after unverified cleanup") + return row + + trial("nop", "nop", "true") + candidates = [] + provided = frozen / "solution/solve.sh" + if provided.is_file(): + candidates.append(("provided", "provided", provided.read_text())) + for index in range(config["generation"]["candidate_count"]): + generated = backend.generate_candidate(config["generator"], index) + script = generated.get("script", "") + entry = { + key: value for key, value in generated.items() if key != "script" + } + entry["artifact"] = artifact(f"candidates/generated-{index}.sh", script) + evidence["generation"].append(entry) + if generated.get("error") or not script.strip(): + evidence["generation_error"] = True + continue + candidates.append((f"generated-{index}", "generated", script)) + for case_id, origin, script in candidates: + records = [ + trial("reference", case_id, script) + for _ in range(config["validation"]["reference_repeats"]) + ] + evidence["references"].append( + { + "case_id": case_id, + "origin": origin, + "submission_digest": records[0].get("submission_digest"), + "candidate_script_digest": hashlib.sha256( + script.encode() + ).hexdigest(), + } + ) + for case in config["negative_cases"]: + trial("negative", case["id"], case["script"]) + evidence["leakage_probes"].extend(backend.leakage_probes()) + if evidence.get("generation_error"): + evidence["leakage_probes"].append( + { + "name": "generation_complete", + "passed": False, + "error": "candidate generation incomplete", + } + ) + persist() + screening = assess_coding_audit(config["validation"], evidence) + if all( + check["status"] == "pass" + for name, check in screening["checks"].items() + if name != "target_difficulty" + ): + for index, seed in enumerate(config["seeds"]): + trial("target", f"target-{index}", seed=seed) + # Include target transcripts in the final leakage measurement. + names = { + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", + } + evidence["leakage_probes"] = [ + probe + for probe in evidence["leakage_probes"] + if probe.get("name") not in names + ] + evidence["leakage_probes"].extend(backend.leakage_probes()) + else: + evidence["target_not_run_reason"] = ( + "reference/verifier/reliability/leakage screening did not pass" + ) + if ( + source_digest(task_root) != original_digest + or source_digest(frozen) != original_digest + ): + raise ValueError("task source changed during audit") + except (Exception, KeyboardInterrupt) as exc: + # Provider exceptions may include request URLs or credentials. + evidence["collection_error"] = type(exc).__name__ + evidence["leakage_probes"].append( + { + "name": "collection_complete", + "passed": False, + "error": type(exc).__name__, + } + ) + persist() + report = assess_coding_audit(config["validation"], evidence) + report.update( + { + "task_digest": original_digest, + "verifier_digest": evidence["verifier_digest"], + "target": evidence["target"], + "budget": budget, + "limitations": evidence["limitations"], + "configuration_digest": evidence["configuration_digest"], + } + ) + report["checks"]["target_difficulty"]["truncated_attempts"] = sum( + trial.get("truncated") is True + for trial in evidence["trials"] + if trial["role"] == "target" + ) + report = sanitize(report) + _write(output_dir / "config.json", sanitize(config)) + _write(output_dir / "report.json", report) + files = sorted(path for path in output_dir.rglob("*") if path.is_file()) + (output_dir / "SHA256SUMS").write_text( + "".join( + hashlib.sha256(path.read_bytes()).hexdigest() + + " " + + path.relative_to(output_dir).as_posix() + + "\n" + for path in files + ) + ) + return report diff --git a/examples/coding_audit/scoring.py b/examples/coding_audit/scoring.py new file mode 100644 index 000000000..ff04abeb4 --- /dev/null +++ b/examples/coding_audit/scoring.py @@ -0,0 +1,506 @@ +"""Pure scoring of bounded coding-audit evidence collected by a trusted runner. + +This profile measures reference replay, known-negative rejection, explicit leakage +probes, and target difficulty. It does not establish containment or trainability. +""" + +import math +import re +from collections import Counter, defaultdict +from decimal import Decimal +from fractions import Fraction +from statistics import pvariance +from typing import Any + +PROFILE = "harbor-coding-audit-v1" +_CHECKS = ( + "references", + "verifier_rejection", + "reliability", + "leakage", + "target_difficulty", +) +_PROBES = ( + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", +) +_RANK = {"pass": 0, "skip": 1, "fail": 2, "error": 3} +_SHA256 = re.compile(r"(?:sha256:)?[0-9a-f]{64}\Z") +_IMAGE = re.compile(r"(?:[^\s@]+@)?sha256:[0-9a-f]{64}\Z") +_REVISION = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z") + + +def _number(value: Any) -> bool: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + try: + return math.isfinite(value) + except OverflowError: + return False + + +def _text(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _matches(pattern: re.Pattern[str], value: Any) -> bool: + return isinstance(value, str) and pattern.fullmatch(value) is not None + + +def _sha(value: str) -> str: + return value.removeprefix("sha256:") + + +def _exact(value: int | float) -> Fraction: + return Fraction(str(value)) + + +def validate_audit_config(config: dict[str, Any]) -> dict[str, Any]: + """Return quality thresholds, raising `ValueError` for invalid configuration. + + `required_negative_cases` must come from a trusted reviewed configuration; + generated labels in evidence cannot supply this rejection coverage. + """ + defaults = { + "reference_repeats": 2, + "target_attempts": 2, + "reward_min": 0, + "reward_max": 1, + "oracle_tolerance": 0, + "floor_margin": 0.1, + "wrong_answer_max": 0, + "required_negative_cases": None, + } + if not isinstance(config, dict) or set(config) - defaults.keys(): + raise ValueError("config must contain only documented quality thresholds") + result = defaults | config + for name in ("reference_repeats", "target_attempts"): + if type(result[name]) is not int or result[name] < 2: + raise ValueError(f"{name} must be an integer >= 2") + for name in ( + "reward_min", + "reward_max", + "oracle_tolerance", + "floor_margin", + "wrong_answer_max", + ): + if not ( + _number(result[name]) + or isinstance(result[name], Decimal) + and result[name].is_finite() + ): + raise ValueError(f"{name} must be a finite nonbool number") + low, high = (_exact(result[name]) for name in ("reward_min", "reward_max")) + tolerance, margin = ( + _exact(result[name]) for name in ("oracle_tolerance", "floor_margin") + ) + if not ( + low < high + and 0 <= tolerance < high - low + and 0 < margin <= high - low + and low <= _exact(result["wrong_answer_max"]) < high - tolerance + and low + tolerance < high - tolerance + ): + raise ValueError("reward bounds, tolerance, margin or negative limit conflict") + cases = result["required_negative_cases"] + if ( + not isinstance(cases, list) + or not cases + or any(not _text(case) for case in cases) + or len(set(cases)) != len(cases) + ): + raise ValueError("required_negative_cases must be nonempty unique case IDs") + result["required_negative_cases"] = list(cases) + return result + + +def _issue(check: dict[str, Any], status: str, reason: str) -> None: + if _RANK[status] > _RANK[check["status"]]: + check["status"] = status + check["reasons"].append(reason) + + +def _finish(checks: dict[str, Any]) -> dict[str, Any]: + verdict = max((check["status"] for check in checks.values()), key=_RANK.get) + return { + "profile": PROFILE, + "accepted": verdict == "pass", + "verdict": verdict, + "checks": checks, + "rfc008_complete": False, + "trainability_established": False, + } + + +def _wilson(solved: int, count: int) -> list[float] | None: + if not count: + return None + z = 1.959963984540054 + rate = solved / count + denominator = 1 + z * z / count + center = (rate + z * z / (2 * count)) / denominator + half = z * math.sqrt(rate * (1 - rate) / count + z * z / (4 * count**2)) + return [ + max(0.0, center - half / denominator), + min(1.0, center + half / denominator), + ] + + +def _target_metrics( + raw: list[dict[str, Any]], trials: list[dict[str, Any]], config: dict[str, Any] +) -> dict[str, Any]: + rewards = [trial["reward"] for trial in trials] + count = len(rewards) + solved = sum( + _exact(reward) + >= _exact(config["reward_max"]) - _exact(config["oracle_tolerance"]) + for reward in rewards + ) + telemetry_missing = {"latency_seconds": 0, "input_tokens": 0, "output_tokens": 0} + telemetry_invalid = dict(telemetry_missing) + telemetry_errors = [] + latency: list[Fraction] = [] + usage: dict[str, list[int]] = {"input_tokens": [], "output_tokens": []} + # Failed executions can still consume tokens/time. Account for all records. + for trial in raw: + seconds = trial.get("latency_seconds") + if "latency_seconds" not in trial: + telemetry_missing["latency_seconds"] += 1 + elif _number(seconds) and seconds >= 0: + latency.append(_exact(seconds)) + else: + telemetry_invalid["latency_seconds"] += 1 + telemetry_errors.append("invalid target latency_seconds") + tokens = trial.get("usage") + if "usage" in trial and not isinstance(tokens, dict): + telemetry_errors.append("invalid target usage object") + for name in usage: + value = tokens.get(name) if isinstance(tokens, dict) else None + if "usage" not in trial or isinstance(tokens, dict) and name not in tokens: + telemetry_missing[name] += 1 + elif type(value) is int and value >= 0: + usage[name].append(value) + else: + telemetry_invalid[name] += 1 + telemetry_errors.append(f"invalid target {name}") + latency_total = None + if latency: + try: + latency_total = float(sum(latency)) + except OverflowError: + telemetry_errors.append("latency total exceeds finite report precision") + return { + "attempted": len(raw), + "completed": count, + "failed": len(raw) - count, + "infrastructure_errors": sum(trial.get("error") is not None for trial in raw), + "missing": max(0, config["target_attempts"] - count), + "solve_count": solved, + "solve_rate": solved / count if count else None, + "solve_rate_wilson95": _wilson(solved, count), + "reward_variance": _variance(rewards), + "mixed_success": 0 < solved < count if count else None, + "difficulty": ( + "unmeasured" + if not count + else "mixed" + if 0 < solved < count + else "all_solved" + if solved == count + else "none_solved" + ), + "latency_seconds_total": latency_total, + "usage_totals": { + name: sum(values) if values else None for name, values in usage.items() + }, + "telemetry_missing": telemetry_missing, + "telemetry_invalid": telemetry_invalid, + "telemetry_errors": telemetry_errors, + } + + +def _variance(rewards: list[float]) -> float | None: + if not rewards: + return None + try: + value = pvariance(rewards) + except OverflowError: + return None + return value if _number(value) else None + + +def assess_coding_audit( + config: dict[str, Any], evidence: dict[str, Any] +) -> dict[str, Any]: + """Score a trusted runner's evidence without executing tasks or making IO calls. + + Trials bind to the common task, verifier, images and pinned target identity. + Reference inventories bind replay groups by `(case_id, submission_digest)`. + Missing coverage is `skip`, contradictions are `fail`, and malformed or + infrastructure evidence is `error`; only five passing checks admit an audit. + """ + checks = {name: {"status": "pass", "reasons": []} for name in _CHECKS} + try: + config = validate_audit_config(config) + except ValueError as exc: + for check in checks.values(): + _issue(check, "error", str(exc)) + return _finish(checks) + if not isinstance(evidence, dict): + for check in checks.values(): + _issue(check, "error", "evidence must be an object") + return _finish(checks) + + reliability = checks["reliability"] + references = checks["references"] + rejection = checks["verifier_rejection"] + target = checks["target_difficulty"] + low, high, tolerance, margin, negative_max = ( + _exact(config[name]) + for name in ( + "reward_min", + "reward_max", + "oracle_tolerance", + "floor_margin", + "wrong_answer_max", + ) + ) + common_ok = True + for name, pattern in ( + ("task_digest", _SHA256), + ("verifier_digest", _SHA256), + ("agent_image", _IMAGE), + ("verifier_image", _IMAGE), + ): + if not _matches(pattern, evidence.get(name)): + _issue(reliability, "error", f"invalid or missing common {name}") + common_ok = False + identity = evidence.get("target") + if not ( + isinstance(identity, dict) + and all(_text(identity.get(name)) for name in ("model", "provider")) + and _matches(_REVISION, identity.get("revision")) + ): + _issue( + target, "error", "target requires model, provider and immutable revision" + ) + identity = None + + inventory = evidence.get("references", []) + candidates: dict[tuple[str, str], dict[str, Any]] = {} + if not isinstance(inventory, list): + _issue(references, "error", "references must be a list") + inventory = [] + for candidate in inventory: + if not ( + isinstance(candidate, dict) + and _text(candidate.get("case_id")) + and _matches(_SHA256, candidate.get("submission_digest")) + and candidate.get("origin") in ("provided", "generated") + and ( + candidate["origin"] != "generated" + or _matches(_SHA256, candidate.get("candidate_script_digest")) + ) + ): + _issue(references, "error", "invalid reference candidate metadata") + continue + key = (candidate["case_id"], _sha(candidate["submission_digest"])) + if key in candidates: + _issue(references, "error", f"duplicate reference candidate {key[0]}") + candidates[key] = candidate + if not candidates: + _issue(references, "skip", "no declared reference candidates") + + raw_trials = evidence.get("trials", []) + if not isinstance(raw_trials, list): + _issue(reliability, "error", "trials must be a list") + raw_trials = [] + sandbox_counts = Counter( + trial[name] + for trial in raw_trials + if isinstance(trial, dict) + for name in ("sandbox_id", "grade_sandbox_id") + if _text(trial.get(name)) + ) + target_cases = Counter( + trial["case_id"] + for trial in raw_trials + if isinstance(trial, dict) + and trial.get("role") == "target" + and _text(trial.get("case_id")) + ) + by_role: dict[str, list[dict[str, Any]]] = defaultdict(list) + raw_target = [] + for index, trial in enumerate(raw_trials): + if not isinstance(trial, dict) or trial.get("role") not in ( + "nop", + "reference", + "negative", + "target", + ): + _issue(reliability, "error", f"trial {index}: invalid role or record") + continue + role = trial["role"] + role_check = checks[ + { + "nop": "verifier_rejection", + "negative": "verifier_rejection", + "reference": "references", + "target": "target_difficulty", + }[role] + ] + if role == "target": + raw_target.append(trial) + errors = [] + if not common_ok: + errors.append("common binding unavailable") + for name, expected in ( + ("task_digest", evidence.get("task_digest")), + ("verifier_digest", evidence.get("verifier_digest")), + ("image_ref", evidence.get("agent_image")), + ("verifier_image_ref", evidence.get("verifier_image")), + ): + actual = trial.get(name) + if name in ("task_digest", "verifier_digest"): + matches = ( + _matches(_SHA256, actual) + and _matches(_SHA256, expected) + and _sha(actual) == _sha(expected) + ) + else: + matches = actual == expected + if not matches: + errors.append(f"{name} binding mismatch") + submission_ok = _text(trial.get("case_id")) and _matches( + _SHA256, trial.get("submission_digest") + ) + if not submission_ok: + errors.append("missing case ID or submission digest") + for name in ("sandbox_id", "grade_sandbox_id"): + value = trial.get(name) + if not _text(value) or sandbox_counts[value] != 1: + errors.append(f"{name} must be fresh and distinct") + if trial.get("cleanup_verified") is not True: + errors.append("cleanup not verified") + if "error" not in trial or trial["error"] is not None: + errors.append("execution error or missing error status") + reward = trial.get("reward") + if not _number(reward) or not low <= _exact(reward) <= high: + errors.append("reward must be finite nonbool and in bounds") + if role == "target": + if identity is None or any( + trial.get(name) != identity[name] + for name in ("model", "revision", "provider") + ): + errors.append("target identity mismatch") + if _text(trial.get("case_id")) and target_cases[trial["case_id"]] != 1: + errors.append("duplicate target attempt case ID") + if ( + role == "reference" + and submission_ok + and (trial["case_id"], _sha(trial["submission_digest"])) not in candidates + ): + errors.append("undeclared reference submission") + if errors: + message = f"trial {index}: " + "; ".join(errors) + _issue(role_check, "error", message) + _issue(reliability, "error", message) + else: + by_role[role].append(trial) + if not raw_trials: + _issue(reliability, "skip", "no trial evidence") + + floor_rewards = [_exact(trial["reward"]) for trial in by_role["nop"]] + if not floor_rewards: + _issue(rejection, "skip", "NOP floor evidence missing") + _issue(references, "skip", "NOP floor evidence missing") + elif max(floor_rewards) > low + tolerance: + _issue(rejection, "fail", "NOP reward exceeds configured floor") + + groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for trial in by_role["reference"]: + groups[(trial["case_id"], _sha(trial["submission_digest"]))].append(trial) + validated = [] + excluded = [] + for key, candidate in candidates.items(): + trials = groups[key] + rewards = [_exact(trial["reward"]) for trial in trials] + if not rewards: + _issue(references, "skip", f"candidate {key[0]} execution missing") + continue + stable = max(rewards) - min(rewards) <= tolerance + if not stable: + _issue(reliability, "fail", f"reference {key[0]} replay reward drift") + correct = min(rewards) >= high - tolerance + if not correct: + if candidate["origin"] == "generated": + excluded.append({"case_id": key[0], "submission_digest": key[1]}) + else: + _issue( + references, "fail", f"provided reference {key[0]} reward too low" + ) + continue + if len(trials) < config["reference_repeats"]: + _issue(references, "skip", f"reference {key[0]} replay coverage incomplete") + _issue( + reliability, "skip", f"reference {key[0]} replay coverage incomplete" + ) + continue + if floor_rewards and min(rewards) - max(floor_rewards) < margin: + _issue(references, "fail", f"reference {key[0]} lacks floor separation") + continue + if stable and floor_rewards: + validated.append({"case_id": key[0], "submission_digest": key[1]}) + references.update(validated=validated, excluded=excluded) + if not validated: + _issue(references, "skip", "no validated reference") + _issue(reliability, "skip", "no validated reference replay group") + + negatives = defaultdict(list) + for trial in by_role["negative"]: + negatives[trial["case_id"]].append(_exact(trial["reward"])) + covered = [] + for case in config["required_negative_cases"]: + if not negatives[case]: + _issue(rejection, "skip", f"trusted negative {case} execution missing") + elif max(negatives[case]) > negative_max: + _issue(rejection, "fail", f"trusted negative {case} was rewarded") + else: + covered.append(case) + rejection["covered_negative_cases"] = covered + + leakage = checks["leakage"] + leakage["scope"] = "bounded probes; universal containment is not established" + probes = evidence.get("leakage_probes", []) + if not isinstance(probes, list): + _issue(leakage, "error", "leakage_probes must be a list") + probes = [] + seen = set() + for probe in probes: + if not isinstance(probe, dict) or not _text(probe.get("name")): + _issue(leakage, "error", "invalid leakage probe record") + continue + name = probe["name"] + if name in seen: + _issue(leakage, "error", f"duplicate leakage probe {name}") + seen.add(name) + if type(probe.get("passed")) is not bool or "error" not in probe: + _issue(leakage, "error", f"invalid leakage probe result {name}") + elif probe["error"] is not None: + _issue(leakage, "error", f"leakage probe {name} execution error") + elif not probe["passed"]: + _issue(leakage, "fail", f"leakage probe {name} failed") + for name in _PROBES: + if name not in seen: + _issue(leakage, "skip", f"required leakage probe {name} missing") + leakage["observed_probes"] = sorted(seen) + + target.update(_target_metrics(raw_target, by_role["target"], config)) + if target["completed"] and target["reward_variance"] is None: + _issue(target, "error", "reward variance exceeds finite report precision") + for message in target["telemetry_errors"]: + _issue(target, "error", message) + if len(by_role["target"]) < config["target_attempts"]: + _issue(target, "skip", "target attempt coverage incomplete") + return _finish(checks) diff --git a/examples/coding_audit/test_hf.py b/examples/coding_audit/test_hf.py new file mode 100644 index 000000000..7aaa0ab6b --- /dev/null +++ b/examples/coding_audit/test_hf.py @@ -0,0 +1,1698 @@ +"""Offline SDK-shaped tests for the HF coding task execution boundary.""" + +import hashlib +import json +import os +import signal +import time +from pathlib import PurePosixPath +from types import SimpleNamespace + +import httpx +import pytest +from examples.coding_audit.hf import ( + _deadline, + HFBackendError, + HFTaskBackend, + submission_digest, +) +from huggingface_hub import hf_api, InferenceClient, Sandbox +from huggingface_hub.errors import SandboxError +from huggingface_hub.inference._generated.types.chat_completion import ( + ChatCompletionOutput, + ChatCompletionStreamOutput, +) +from huggingface_hub.inference._providers.deepinfra import DeepInfraConversationalTask +from openenv.validation.runner import source_digest + +AGENT = "python@sha256:" + "a" * 64 +VERIFIER = "python@sha256:" + "b" * 64 +MODEL = { + "model_id": "Qwen/Qwen3.8-27B", + "revision": "37561816e66b5b9153b1b64c10968c18c1e6f4a0", + "provider": "deepinfra", + "provider_model_id": "Qwen/Qwen3.8-27B", +} + + +def test_real_sandbox_sdk_serializes_short_name_for_long_image(harness, monkeypatch): + captured = {} + image = ( + "registry.hf.space/example-organization-a-long-starter-image-name" + "@sha256:" + "a" * 64 + ) + + def capture(request): + assert request.method == "POST" and request.url.host == "huggingface.co" + captured.update(json.loads(request.content)) + return httpx.Response(400, json={"error": "offline request capture"}) + + harness.backend.execution["agent_image"] = image + harness.backend.sandbox_factory = Sandbox.create + with httpx.Client(transport=httpx.MockTransport(capture)) as client: + monkeypatch.setattr(hf_api, "get_session", lambda: client) + record = harness.backend.execute_script(":", "nop") + assert "BadRequestError" in record["error"] + assert captured["dockerImage"] == image + assert captured["labels"]["name"] == ( + "openenv-coding-" + captured["labels"]["openenv-coding-attempt"] + ) + assert 0 < len(captured["labels"]["name"]) <= 100 + + +def response( + content="done", *, calls=None, model=None, fingerprint="fp-123", usage=None +): + return ChatCompletionOutput.parse_obj_as_instance( + { + "id": "chatcmpl-1", + "created": 1, + "object": "chat.completion", + "model": model or MODEL["model_id"], + "system_fingerprint": fingerprint, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls" if calls else "stop", + "message": { + "role": "assistant", + "content": content, + "tool_calls": calls, + }, + } + ], + "usage": usage + or {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + ) + + +def terminal(command="printf answer > /app/answer.txt", identifier="call-1"): + return { + "id": identifier, + "type": "function", + "function": {"name": "terminal", "arguments": json.dumps({"command": command})}, + } + + +def stream_event(*, delta=None, finish=None, choices=None, usage=None, **identity): + return ChatCompletionStreamOutput.parse_obj_as_instance( + { + "id": "chatcmpl-1", + "created": 1, + "model": MODEL["model_id"], + "system_fingerprint": "fp-123", + **identity, + "choices": choices + if choices is not None + else [{"index": 0, "delta": delta or {}, "finish_reason": finish}], + "usage": usage, + } + ) + + +def stream_response(reply): + yield stream_event( + delta=reply["choices"][0]["message"], + finish=reply["choices"][0]["finish_reason"], + model=reply["model"], + system_fingerprint=reply["system_fingerprint"], + ) + yield stream_event( + choices=[], + usage=reply["usage"], + model=reply["model"], + system_fingerprint=reply["system_fingerprint"], + ) + + +class Files: + MAX_READ_BYTES = 512 * 1048576 + + def __init__(self, initial=None): + self.data = dict(initial or {}) + self.types = {} + self.directories = {"/", "/tmp"} + self.writes = [] + self.reads = [] + self.grow_on_read = False + + def stat(self, path): + if path in self.directories: + return SimpleNamespace(type="dir", size=0) + if path not in self.data: + raise SandboxError("missing", status_code=404) + return SimpleNamespace( + type=self.types.get(path, "file"), size=len(self.data[path]) + ) + + def exists(self, path): + try: + self.stat(path) + return True + except SandboxError as exc: + if exc.status_code == 404: + return False + raise + + def read(self, path): + self.reads.append(path) + data = self.data[path] + if self.grow_on_read: + data = data + b"x" * self.MAX_READ_BYTES + if len(data) > self.MAX_READ_BYTES: + raise SandboxError("too large") + return data + + def write(self, path, data, mode=None): + assert isinstance(data, (str, bytes)) + data = data.encode() if isinstance(data, str) else data + self.data[path] = data + self.writes.append((path, data, mode)) + + def mkdir(self, path): + self.directories.add(path) + + def delete(self, path): + del self.data[path] + + +class FakeSandbox: + def __init__(self, identity, image, api, factory): + self.id = identity + self.image = image + self.files = Files(factory.initial) + self.api = api + self.factory = factory + self.commands = [] + self.killed = False + + def run(self, argv, **kwargs): + self.commands.append((argv, kwargs)) + assert isinstance(argv, list) + assert kwargs["shell"] is False + assert kwargs["check"] is False + assert kwargs["capture_output"] is False + assert kwargs["cwd"] in self.files.directories + assert 0 < kwargs["timeout"] <= 120 + assert "env" not in kwargs + if self.factory.output: + kwargs["on_stdout"](self.factory.output) + if self.image == VERIFIER: + self.files.data["/logs/verifier/reward.txt"] = self.factory.reward + elif self.factory.submitted is not None: + self.files.data["/app/answer.txt"] = self.factory.submitted + return SimpleNamespace( + exit_code=self.factory.exit_code, + stdout="", + stderr="", + timed_out=self.factory.timed_out, + ) + + def kill(self): + self.killed = True + if not self.api.ignore_cancel: + self.api.stages[self.id] = "CANCELED" + + +class API: + def __init__(self): + self.images = {} + self.stages = {} + self.cancelled = [] + self.inspections = [] + self.ignore_cancel = False + self.cancel_error = False + self.inspect_error_after_cancel = False + self.owner = "owner" + self.image_override = None + self.labels = {} + + def inspect_job(self, *, job_id, namespace=None, token=None): + self.inspections.append((job_id, namespace, token)) + if self.inspect_error_after_cancel and job_id in self.cancelled: + raise RuntimeError("inspection contains an execution-token-secret") + return SimpleNamespace( + id=job_id, + docker_image=self.image_override or self.images[job_id], + status=SimpleNamespace(stage=self.stages[job_id]), + owner=SimpleNamespace(name=self.owner), + labels=self.labels.get(job_id, {}), + ) + + def list_jobs(self, *, labels=None, timeout=None, namespace=None, token=None): + return ( + self.inspect_job(job_id=job_id, namespace=namespace, token=token) + for job_id in self.images + if all( + self.labels.get(job_id, {}).get(key) == value + for key, value in (labels or {}).items() + ) + ) + + def cancel_job(self, *, job_id, namespace=None, token=None): + self.cancelled.append(job_id) + if self.cancel_error: + raise RuntimeError("cancel failed") + if not self.ignore_cancel: + self.stages[job_id] = "CANCELED" + + +class Factory: + def __init__(self, api): + self.api = api + self.created = [] + self.kwargs = [] + self.initial = {} + self.output = "" + self.submitted = b"answer" + self.reward = b"1.0\n" + self.exit_code = 0 + self.timed_out = False + self.fail_after_create = False + + def __call__(self, **kwargs): + identity = f"job-{len(self.created)}" + self.kwargs.append(kwargs) + self.api.images[identity] = kwargs["image"] + self.api.stages[identity] = "RUNNING" + self.api.labels[identity] = kwargs.get("labels", {}) + sandbox = FakeSandbox(identity, kwargs["image"], self.api, self) + self.created.append(sandbox) + if self.fail_after_create: + raise RuntimeError("readiness failed after allocation") + return sandbox + + +class InferenceFactory: + def __init__(self): + self.kwargs = [] + self.requests = [] + self.responses = [] + self.closed = 0 + + def __call__(self, **kwargs): + self.kwargs.append(kwargs) + return self + + def chat_completion(self, **kwargs): + self.requests.append(json.loads(json.dumps(kwargs))) + reply = self.responses.pop(0) + if kwargs.get("stream") and isinstance(reply, ChatCompletionOutput): + return stream_response(reply) + return reply + + def close(self): + self.closed += 1 + + +@pytest.fixture +def harness(tmp_path, monkeypatch): + for name in ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN"): + monkeypatch.delenv(name, raising=False) + root = tmp_path.resolve() + (root / "instruction.md").write_text("Write /app/answer.txt containing answer.") + (root / "tests").mkdir() + (root / "tests/test.sh").write_bytes( + b"#!/bin/bash\nprintf 1 > /logs/verifier/reward.txt\n" + ) + (root / "solution").mkdir() + (root / "solution/solve.sh").write_text("PRIVATE_SOLUTION_SENTINEL") + api = API() + factory = Factory(api) + inference = InferenceFactory() + execution = { + "agent_image": AGENT, + "verifier_image": VERIFIER, + "submission_paths": ["/app/answer.txt"], + "namespace": "owner", + } + backend = HFTaskBackend( + root, + execution, + token="execution-token-secret", + inference_token="inference-token-secret", + api=api, + sandbox_factory=factory, + inference_factory=inference, + canary="PRIVATE_CANARY_SENTINEL", + ) + return SimpleNamespace( + root=root, + backend=backend, + api=api, + factory=factory, + inference=inference, + execution=execution, + ) + + +def test_execution_grade_are_fresh_and_credentials_stay_in_controller(harness): + h = harness + submission = h.backend.execute_script( + "printf answer > /app/answer.txt", "reference" + ) + grade = h.backend.grade(submission) + assert submission["error"] is None and submission["cleanup_verified"] + assert grade["error"] is None and grade["cleanup_verified"] + assert grade["reward"] == 1 and grade["rawreward"] == b"1.0\n" + assert submission["sandbox_id"] != grade["grade_sandbox_id"] + assert [item["image"] for item in h.factory.kwargs] == [AGENT, VERIFIER] + for kwargs in h.factory.kwargs: + assert kwargs["forward_hf_token"] is False + assert kwargs["token"] == "execution-token-secret" + assert ( + "env" not in kwargs and "secrets" not in kwargs and "volumes" not in kwargs + ) + solver, verifier = h.factory.created + assert not any( + path.startswith(("/tests", "/solution")) for path in solver.files.data + ) + assert ( + verifier.files.data["/tests/test.sh"] == (h.root / "tests/test.sh").read_bytes() + ) + assert verifier.files.data["/app/answer.txt"] == b"answer" + assert not any("solution" in path for path in verifier.files.data) + assert b"PRIVATE_CANARY_SENTINEL" in verifier.files.data.values() + assert submission["image_reference_kind"] == "control_plane_immutable_reference" + assert len(submission["limitations"]) >= 4 + assert "token-secret" not in repr(submission) + repr(grade) + assert h.api.cancelled == ["job-0", "job-1"] + + +def test_absent_submission_is_empty_and_can_legitimately_grade_zero(harness): + h = harness + h.factory.submitted = None + submission = h.backend.execute_script(":", "nop") + assert submission["files"] == {} and submission["error"] is None + h.factory.reward = b"0" + grade = h.backend.grade(submission) + assert grade["error"] is None and grade["reward"] == 0 + + +def test_native_verifier_reward_directory_exists_before_execution(harness, monkeypatch): + h = harness + submission = h.backend.execute_script(":", "nop") + original_run = FakeSandbox.run + + def native_run(sandbox, argv, **kwargs): + if sandbox.image == VERIFIER: + assert "/logs/verifier" in sandbox.files.directories + return original_run(sandbox, argv, **kwargs) + + monkeypatch.setattr(FakeSandbox, "run", native_run) + grade = h.backend.grade(submission) + assert grade["error"] is None + assert grade["cleanup_verified"] + + +@pytest.mark.parametrize("phase", ["script", "grade"]) +def test_nonzero_is_error_not_synthesized_reward(harness, phase): + h = harness + submission = h.backend.execute_script(":", "reference") + h.factory.exit_code = 3 + record = ( + h.backend.grade(submission) + if phase == "grade" + else h.backend.execute_script("false", "target") + ) + assert "nonzero" in record["error"] + assert record.get("reward") is None and record["cleanup_verified"] + + +@pytest.mark.parametrize("failure", ["overflow", "timeout", "nonzero", "exception"]) +def test_failed_grader_keeps_logs_only_without_overflow(harness, monkeypatch, failure): + h = harness + submission = h.backend.execute_script(":", "reference") + h.backend.max_output = 17 + output = "setup\n雪💥雪💥" if failure == "overflow" else "setup\nfailed\n" + + def failed_run(sandbox, argv, **kwargs): + kwargs["on_stdout"]("setup\n") + kwargs["on_stderr"](output.removeprefix("setup\n")) + if failure == "exception": + raise RuntimeError("RPC failed after output") + return SimpleNamespace( + exit_code=3 if failure == "nonzero" else 0, + timed_out=failure == "timeout", + ) + + monkeypatch.setattr(FakeSandbox, "run", failed_run) + grade = h.backend.grade(submission) + assert grade["error"] and grade["cleanup_verified"] + assert grade["reward"] is None and grade["rawreward"] == b"" + trace = grade["transcript"][0] + assert trace["output"] == ("" if failure == "overflow" else output) + assert len(trace["output"].encode()) <= 17 + assert trace["output_truncated"] is (failure == "overflow") + assert trace["exit_code"] == ( + 3 if failure == "nonzero" else 0 if failure == "timeout" else None + ) + + +@pytest.mark.parametrize("cap", [17, 18]) +def test_grader_output_stays_bounded_after_redaction(harness, cap): + h = harness + submission = h.backend.execute_script(":", "nop") + backend = HFTaskBackend( + h.root, + dict(h.execution, max_tool_output_bytes=cap), + token="secret", + api=h.api, + sandbox_factory=h.factory, + ) + h.factory.output = "雪secret💥\n" + h.factory.reward = b"0" + grade = backend.grade(submission) + assert grade["error"] is None and grade["reward"] == 0 + assert grade["transcript"] == [ + { + "exit_code": 0, + "output": "" if cap == 17 else "雪[REDACTED]💥\n", + "output_truncated": cap == 17, + } + ] + assert len(grade["transcript"][0]["output"].encode()) <= cap + assert grade["cleanup_verified"] + + +@pytest.mark.parametrize("callbacks", ["single", "separate", "split_secret"]) +@pytest.mark.parametrize( + "credential", ["execution", "inference", "overlapping", "repeated", "intersecting"] +) +@pytest.mark.parametrize("retained", [1, 28]) +def test_grader_overflow_discards_output( + harness, monkeypatch, callbacks, credential, retained +): + h = harness + submission = h.backend.execute_script(":", "reference") + credentials = { + "execution": "hf_SYNTHETIC_EXECUTION_TOKEN_FOR_REVIEW_ONLY", + "inference": "hf_SYNTHETIC_INFERENCE_TOKEN_FOR_REVIEW_ONLY", + } + if credential == "overlapping": + credentials["execution"] = "SYNTHETIC_EXECUTION_TOKEN" + credentials["inference"] = "hf_SYNTHETIC_EXECUTION_TOKEN_FOR_REVIEW_ONLY" + elif credential == "repeated": + credentials["execution"] = "hf_" + "X" * 22 + "hf_" + credential = "execution" + elif credential == "intersecting": + credentials["execution"] = "hf_SYNTHETIC_EXECUTION_TOKEN_REVIEW_ONLYh" + credentials["inference"] = "hf_SYNTHETIC_INFERENCE_TOKEN_REVIEW_ONLY" + cap = 1048576 + backend = HFTaskBackend( + h.root, + dict(h.execution, max_tool_output_bytes=cap), + token=credentials["execution"], + inference_token=credentials["inference"], + api=h.api, + sandbox_factory=h.factory, + ) + secret = credentials[ + "inference" if credential in ("overlapping", "intersecting") else credential + ] + prefix = "雪" + "A" * (cap - len("雪".encode()) - retained) + if credential == "intersecting": + complete = credentials["execution"] + prefix = prefix[: -len(complete[:-1])] + complete[:-1] + assert complete in prefix + secret + + def overflow(sandbox, argv, **kwargs): + if callbacks == "single": + kwargs["on_stdout"](prefix + secret + "\n") + else: + split = min(7, retained) if callbacks == "split_secret" else 0 + kwargs["on_stdout"](prefix + secret[:split]) + kwargs["on_stderr"](secret[split:] + "\n") + pytest.fail("grader output overflow was not rejected") + + monkeypatch.setattr(FakeSandbox, "run", overflow) + grade = backend.grade(submission) + assert grade["error"] == "terminal output exceeded byte limit" + assert grade["reward"] is None and grade["rawreward"] == b"" + assert grade["cleanup_verified"] + trace = grade["transcript"][0] + assert trace == { + "exit_code": None, + "output": "", + "output_truncated": True, + } + assert len(trace["output"].encode()) <= cap + assert secret[:retained] not in trace["output"] + assert h.api.stages and all(stage == "CANCELED" for stage in h.api.stages.values()) + + +def test_image_mismatch_cancels_before_any_workload_command(harness): + h = harness + h.api.image_override = VERIFIER + record = h.backend.execute_script("dangerous", "reference") + assert "image" in record["error"] and record["cleanup_verified"] + assert not h.factory.created[0].commands + assert h.api.cancelled == ["job-0"] + + +def test_rpc_ready_sandbox_waits_for_running_control_plane(harness, monkeypatch): + h = harness + inspect = h.api.inspect_job + observed = [] + + def lagged_inspect(**kwargs): + job = inspect(**kwargs) + if not h.api.cancelled: + job.status.stage = "SCHEDULING" if not observed else "RUNNING" + observed.append(job.status.stage) + return job + + monkeypatch.setattr(h.api, "inspect_job", lagged_inspect) + monkeypatch.setattr("examples.coding_audit.hf.time.sleep", lambda _: None) + record = h.backend.execute_script(":", "reference") + assert record["error"] is None + assert observed == ["SCHEDULING", "RUNNING"] + assert record["cleanup_verified"] + assert h.factory.created[0].commands + + +@pytest.mark.parametrize("stage", ["SCHEDULING", "ERROR", "COMPLETED", "UNKNOWN"]) +def test_nonrunning_control_plane_never_executes_and_is_cleaned_up( + harness, monkeypatch, stage +): + h = harness + inspect = h.api.inspect_job + attempts = [] + + def never_running(**kwargs): + job = inspect(**kwargs) + if not h.api.cancelled: + attempts.append(job.id) + job.status.stage = stage + return job + + monkeypatch.setattr(h.api, "inspect_job", never_running) + monkeypatch.setattr("examples.coding_audit.hf.time.sleep", lambda _: None) + record = h.backend.execute_script("dangerous", "reference") + assert record["error"] is not None and record["cleanup_verified"] + assert not h.factory.created[0].commands + assert len(attempts) <= 20 + + +def test_startup_reconciliation_rechecks_image_before_execution(harness, monkeypatch): + h = harness + inspect = h.api.inspect_job + attempts = [] + + def changed_image(**kwargs): + job = inspect(**kwargs) + if not h.api.cancelled: + if attempts: + job.docker_image = VERIFIER + else: + job.status.stage = "SCHEDULING" + attempts.append(job.id) + return job + + monkeypatch.setattr(h.api, "inspect_job", changed_image) + monkeypatch.setattr("examples.coding_audit.hf.time.sleep", lambda _: None) + record = h.backend.execute_script("dangerous", "reference") + assert "image" in record["error"] and record["cleanup_verified"] + assert not h.factory.created[0].commands + assert len(attempts) == 2 + + +@pytest.mark.parametrize("failure", ["nonterminal", "inspection"]) +def test_cleanup_cannot_pass_from_cancel_return_alone(harness, failure): + h = harness + submission = h.backend.execute_script(":", "reference") + h.api.ignore_cancel = failure == "nonterminal" + h.api.inspect_error_after_cancel = failure == "inspection" + grade = h.backend.grade(submission) + assert grade["cleanup_verified"] is False + assert grade["reward"] is None and "cleanup" in grade["error"] + assert h.api.cancelled.count("job-1") == 3 + + +def test_exact_kill_fallback_requires_independent_terminal_inspection(harness): + h = harness + h.api.cancel_error = True + record = h.backend.execute_script(":", "reference") + assert record["cleanup_verified"] and h.factory.created[0].killed + assert h.api.inspections[-1][0] == record["sandbox_id"] + + +@pytest.mark.parametrize( + "path", + [ + "relative", + "/app/../tests/x", + "/app//x", + "/app/./x", + "/tests/x", + "/solution/x", + "/logs/verifier/reward.txt", + "/", + "//app/x", + ], +) +def test_submission_path_preflight(harness, path): + h = harness + with pytest.raises(HFBackendError): + HFTaskBackend(h.root, {**h.execution, "submission_paths": [path]}) + + +@pytest.mark.parametrize("paths", [["/app/a", "/app/a/b"], ["/app/a", "/app/a"]]) +def test_overlapping_paths_rejected(harness, paths): + with pytest.raises(HFBackendError, match="overlap|repeat"): + HFTaskBackend(harness.root, {**harness.execution, "submission_paths": paths}) + + +@pytest.mark.parametrize( + "mutation", ["extra_path", "tampered_digest", "not_bytes", "oversize"] +) +def test_grade_rejects_unsafe_transfer_before_launch(harness, mutation): + h = harness + files = {"/app/answer.txt": b"answer"} + submission = {"files": files, "submission_digest": submission_digest(files)} + if mutation == "extra_path": + files["/tests/test.sh"] = b"fake reward" + elif mutation == "tampered_digest": + files["/app/answer.txt"] = b"changed" + elif mutation == "not_bytes": + files["/app/answer.txt"] = "answer" + else: + files["/app/answer.txt"] = b"x" * (h.backend.max_bytes + 1) + grade = h.backend.grade(submission) + assert grade["error"] and grade["reward"] is None + assert not h.factory.created + + +@pytest.mark.parametrize("kind", ["symlink", "hardlink", "fifo", "symlink_parent"]) +def test_local_verifier_artifacts_reject_nonregular_or_linked_paths(harness, kind): + h = harness + path = h.root / "tests/extra" + if kind == "symlink": + path.symlink_to(h.root / "instruction.md") + elif kind == "hardlink": + os.link(h.root / "instruction.md", path) + elif kind == "fifo": + os.mkfifo(path) + else: + path.symlink_to(h.root / "solution", target_is_directory=True) + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert grade["error"] and not h.factory.created + + +@pytest.mark.parametrize( + "reward", + [ + b"NaN", + b"Infinity", + b"1e9999", + b'{"reward": true}', + b'{"reward": "1"}', + b'{"reward":1,"reward":0}', + b"garbage", + ], +) +def test_reward_requires_finite_trusted_numeric_bytes(harness, reward): + h = harness + h.factory.reward = reward + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert grade["error"] and grade["reward"] is None and grade["cleanup_verified"] + + +def test_json_reward_collects_original_bytes(harness): + h = harness + h.factory.reward = b'{"reward":0.5,"detail":"ok"}\n' + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert grade["reward"] == 0.5 and grade["rawreward"] == h.factory.reward + + +def test_provider_requests_base_id_with_distinct_token_and_records_source_limit( + harness, +): + h = harness + h.inference.responses = [response("printf answer > /app/answer.txt")] + generated = h.backend.generate_candidate(MODEL, 42) + assert generated["error"] is None + assert not h.factory.created + request = h.inference.requests[0] + assert request["model"] == MODEL["model_id"] and "@" not in request["model"] + assert "seed" not in request and "tools" not in request + assert h.inference.kwargs == [ + {"provider": "deepinfra", "api_key": "inference-token-secret", "timeout": 120} + ] + assert "PRIVATE_SOLUTION_SENTINEL" not in repr(request) + assert "PRIVATE_CANARY_SENTINEL" not in repr(request) + assert "test.sh" not in repr(request) + metadata = generated["model_metadata"] + assert metadata["source_revision"] == MODEL["revision"] + assert metadata["served_weight_revision_pinned"] is False + assert metadata["seed_supported"] is False + assert metadata["responses"][0]["system_fingerprint"] == "fp-123" + assert generated["usage"] == {"input_tokens": 10, "output_tokens": 5} + + +def test_seed_and_exact_provider_alias_are_explicit(harness): + h = harness + h.inference.responses = [response("true", model="provider/exact")] + record = h.backend.generate_candidate( + {**MODEL, "supports_seed": True, "provider_model_id": "provider/exact"}, 7 + ) + assert record["error"] is None + assert h.inference.requests[0]["seed"] == 7 + h.inference.responses = [response("true", model="provider/unlisted")] + record = h.backend.generate_candidate(MODEL, 7) + assert "response model" in record["error"] + assert record["model_metadata"]["responses"][0]["model"] == "provider/unlisted" + + +def test_top_p_is_forwarded_and_recorded(harness): + h = harness + h.inference.responses = [response("true")] + record = h.backend.generate_candidate({**MODEL, "top_p": 0.1}, 7) + assert record["error"] is None + assert h.inference.requests[0]["top_p"] == 0.1 + assert record["model_metadata"]["requests"][0]["top_p"] == 0.1 + + +def test_candidate_stream_assembles_text_and_final_usage_without_changing_recipe( + harness, +): + h = harness + events = [ + stream_event(delta={"role": "assistant", "content": "printf "}), + stream_event(delta={"reasoning_content": "Think 雪"}), + stream_event(delta={"content": "answer > /app/answer.txt"}, finish="stop"), + stream_event( + choices=[], + usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + ), + ] + h.inference.responses = [(event for event in events)] + record = h.backend.generate_candidate( + {**MODEL, "max_tokens": 8192, "max_total_tokens": 32768, "top_p": 0.1}, 7 + ) + assert record["error"] is None + assert record["script"] == "printf answer > /app/answer.txt" + assert record["transcript"][-1]["reasoning_content"] == "Think 雪" + assert record["usage"] == {"input_tokens": 10, "output_tokens": 5} + request = h.inference.requests[0] + assert request["stream"] is True + assert request["stream_options"] == {"include_usage": True} + assert request["max_tokens"] == 8192 and request["top_p"] == 0.1 + assert request["model"] == MODEL["model_id"] and request["temperature"] == 0 + assert "tools" not in request and "seed" not in request + assert record["model_metadata"]["requests"] == [ + {key: value for key, value in request.items() if key != "messages"} + ] + assert h.inference.closed == 1 and not h.factory.created + + +@pytest.mark.parametrize( + "outcome", ["complete", "early_close", "provider_error", "bad_json"] +) +def test_real_sdk_stream_serializes_request_and_closes_http_response( + harness, monkeypatch, outcome +): + h = harness + captured = {} + closed = [] + events = list(stream_response(response("printf answer > /app/answer.txt"))) + + class Body(httpx.SyncByteStream): + def __iter__(self): + for event in events[:1] if outcome == "early_close" else events: + yield ("data: " + json.dumps(event) + "\n\n").encode() + if outcome == "provider_error": + yield b'data: {"error": "provider failed"}\n\n' + elif outcome == "bad_json": + yield b"data: {broken\n\n" + elif outcome == "complete": + yield b"data: [DONE]\n\n" + + def close(self): + closed.append(True) + + def capture(request): + assert request.url.host == "api.deepinfra.com" + captured.update(json.loads(request.content)) + return httpx.Response(200, stream=Body()) + + monkeypatch.setattr( + DeepInfraConversationalTask, + "_prepare_mapping_info", + lambda self, model: hf_api.InferenceProviderMapping( + provider="deepinfra", + hf_model_id=model, + providerId=model, + task="conversational", + status="live", + ), + ) + h.backend.inference_factory = InferenceClient + with httpx.Client(transport=httpx.MockTransport(capture)) as client: + monkeypatch.setattr( + "huggingface_hub.inference._client.get_session", lambda: client + ) + record = h.backend.generate_candidate({**MODEL, "enable_thinking": False}, 0) + assert closed == [True] + assert captured["chat_template_kwargs"] == {"enable_thinking": False} + assert captured["stream"] is True + assert captured["stream_options"] == {"include_usage": True} + assert captured["model"] == MODEL["model_id"] + if outcome == "complete": + assert record["error"] is None and record["script"].startswith("printf answer") + assert record["usage"] == {"input_tokens": 10, "output_tokens": 5} + else: + assert record["error"] and record["script"] == "" + + +@pytest.mark.parametrize( + "mutation,diagnostic", + [ + ("model_drift", "response model"), + ("allowed_model_drift", "identity"), + ("id_drift", "identity"), + ("fingerprint_drift", "identity"), + ("two_choices", "one assistant"), + ("wrong_index", "one assistant"), + ("boolean_index", "one assistant"), + ("wrong_role", "assistant"), + ("missing_role", "assistant"), + ("nontext", "text"), + ("nontext_list", "text"), + ("tool_call", "tool"), + ("no_stop", "incomplete"), + ("length_finish", "incomplete"), + ("no_usage", "usage"), + ("missing_usage_field", "usage"), + ("boolean_usage", "usage"), + ("negative_usage", "usage"), + ("inconsistent_usage", "usage"), + ("early_usage", "usage"), + ("extra_content", "stop"), + ("duplicate_usage", "usage"), + ("provider_error", "provider"), + ("completion_budget", "token budget"), + ("total_budget", "token budget"), + ], +) +def test_candidate_stream_rejects_invalid_or_incomplete_sdk_events( + harness, mutation, diagnostic +): + h = harness + events = [ + stream_event(delta={"role": "assistant", "content": "true"}), + stream_event(finish="stop"), + stream_event( + choices=[], + usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + ), + ] + events = json.loads(json.dumps(events)) + if mutation == "model_drift": + events[1]["model"] = "unlisted/model" + elif mutation == "allowed_model_drift": + events[1]["model"] = "configured/alias" + elif mutation == "id_drift": + events[1]["id"] = "chatcmpl-other" + elif mutation == "fingerprint_drift": + events[1]["system_fingerprint"] = "fp-other" + elif mutation == "two_choices": + events[0]["choices"] *= 2 + elif mutation in {"wrong_index", "boolean_index"}: + events[0]["choices"][0]["index"] = True if mutation == "boolean_index" else 1 + elif mutation in {"wrong_role", "missing_role"}: + events[0]["choices"][0]["delta"]["role"] = ( + None if mutation == "missing_role" else "user" + ) + elif mutation == "nontext": + events[0]["choices"][0]["delta"]["content"] = 7 + elif mutation == "nontext_list": + events[0]["choices"][0]["delta"]["content"] = [] + elif mutation == "tool_call": + events[0]["choices"][0]["delta"]["tool_calls"] = [terminal()] + elif mutation == "no_stop": + events = events[:1] + elif mutation == "length_finish": + events[1]["choices"][0]["finish_reason"] = "length" + elif mutation == "no_usage": + events = events[:2] + elif mutation == "missing_usage_field": + del events[-1]["usage"]["total_tokens"] + elif mutation == "boolean_usage": + events[-1]["usage"]["prompt_tokens"] = True + elif mutation == "negative_usage": + events[-1]["usage"].update(prompt_tokens=-1, total_tokens=4) + elif mutation == "inconsistent_usage": + events[-1]["usage"]["total_tokens"] = 16 + elif mutation == "early_usage": + events[0]["usage"] = events[-1]["usage"] + elif mutation == "extra_content": + events.insert(2, stream_event(delta={"content": "false"})) + elif mutation == "duplicate_usage": + events.append(events[-1]) + elif mutation == "provider_error": + events[-1]["error"] = "provider failed after partial output" + elif mutation == "completion_budget": + events[-1]["usage"].update(completion_tokens=2049, total_tokens=2059) + elif mutation == "total_budget": + events[-1]["usage"].update(prompt_tokens=16384, total_tokens=16389) + closed = [] + + def streaming(): + try: + for event in events: + yield ChatCompletionStreamOutput.parse_obj_as_instance(event) + finally: + closed.append(True) + + h.inference.responses = [streaming()] + record = h.backend.generate_candidate( + {**MODEL, "response_model_aliases": ["configured/alias"]}, 0 + ) + assert diagnostic in record["error"], record + assert record["script"] == "" and not h.factory.created + assert closed == [True] and h.inference.closed == 1 + assert record["model_metadata"]["requests"][0]["stream"] is True + + +@pytest.mark.parametrize("stage", ["before_first", "partial", "after_stop"]) +def test_candidate_stream_provider_failures_preserve_request_and_partial_error( + harness, stage +): + h = harness + closed = [] + + def streaming(): + try: + if stage != "before_first": + yield stream_event(delta={"role": "assistant", "content": "true"}) + if stage == "after_stop": + yield stream_event(finish="stop") + raise RuntimeError("provider error with inference-token-secret") + finally: + closed.append(True) + + h.inference.responses = [streaming()] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] == "HF backend operation failed (RuntimeError)" + assert record["script"] == "" and not h.factory.created + assert "inference-token-secret" not in repr(record) + assert record["model_metadata"]["requests"][0]["stream"] is True + assert closed == [True] and h.inference.closed == 1 + + +@pytest.mark.parametrize("ending", ["provider_error", "extra_event", "over_budget"]) +def test_failed_candidate_stream_retains_observed_final_usage(harness, ending): + h = harness + output_tokens = 2049 if ending == "over_budget" else 5 + + def streaming(): + yield stream_event(delta={"role": "assistant", "content": "true"}) + yield stream_event(finish="stop") + yield stream_event( + choices=[], + usage={ + "prompt_tokens": 10, + "completion_tokens": output_tokens, + "total_tokens": 10 + output_tokens, + }, + ) + if ending == "provider_error": + raise RuntimeError("provider interrupted after usage") + if ending == "extra_event": + yield stream_event(choices=[]) + + h.inference.responses = [streaming()] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] and record["script"] == "" + assert record["usage"] == {"input_tokens": 10, "output_tokens": output_tokens} + assert record["input_tokens"] == 10 and record["output_tokens"] == output_tokens + assert h.inference.closed == 1 and not h.factory.created + + +def test_failed_candidate_request_retains_metadata_and_closes_client(harness): + h = harness + + def fail(**kwargs): + raise RuntimeError("provider unavailable") + + h.inference.chat_completion = fail + record = h.backend.generate_candidate({**MODEL, "top_p": 0.1}, 0) + assert record["error"] == "HF backend operation failed (RuntimeError)" + assert record["model_metadata"]["requests"][0]["top_p"] == 0.1 + assert record["model_metadata"]["requests"][0]["stream"] is True + assert h.inference.closed == 1 and record["script"] == "" + + +@pytest.mark.parametrize("field", ["content", "reasoning", "reasoning_content"]) +def test_candidate_stream_byte_bounds_stop_consuming_and_close(harness, field): + h = harness + h.backend.max_script = 16 if field == "content" else 65536 + (h.root / "instruction.md").write_text("write answer") + continued = [] + + def streaming(): + yield stream_event(delta={"role": "assistant", field: "雪" * 6}) + continued.append(True) + pytest.fail("oversized stream was consumed after its byte limit") + + h.inference.responses = [streaming()] + record = h.backend.generate_candidate( + {**MODEL, "max_output_bytes": 512 if field == "content" else 16}, 0 + ) + assert "byte limit" in record["error"] and record["script"] == "" + assert not continued and h.inference.closed == 1 + + +def test_candidate_stream_keeps_serialized_output_byte_gate(harness): + h = harness + h.inference.responses = [response("雪" * 10)] + record = h.backend.generate_candidate({**MODEL, "max_output_bytes": 64}, 0) + assert "output exceeded byte limit" in record["error"] + assert record["script"] == "" and h.inference.closed == 1 + + +def test_candidate_stream_accepts_usage_on_final_stop(harness): + h = harness + h.inference.responses = [ + ( + event + for event in [ + stream_event( + delta={"role": "assistant", "content": "true"}, + finish="stop", + usage={ + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + }, + ) + ] + ) + ] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] is None and record["script"] == "true" + assert record["usage"] == {"input_tokens": 10, "output_tokens": 5} + + +def test_candidate_stream_is_interrupted_by_root_deadline_and_closes(harness): + h = harness + h.backend.timeout = 0.02 + closed = [] + + def streaming(): + try: + yield stream_event(delta={"role": "assistant", "content": "true"}) + time.sleep(1) + finally: + closed.append(True) + + h.inference.responses = [streaming()] + started = time.monotonic() + record = h.backend.generate_candidate(MODEL, 0) + assert time.monotonic() - started < 0.5 + assert "deadline" in record["error"] and record["script"] == "" + assert closed == [True] and h.inference.closed == 1 + + +def test_unqualified_sdk_version_refused_before_launch(harness, monkeypatch): + monkeypatch.setattr("examples.coding_audit.hf.version", lambda name: "0.0.0") + with pytest.raises(HFBackendError, match="requires huggingface_hub"): + HFTaskBackend( + harness.backend.task_root, + harness.backend.execution, + sandbox_factory=harness.factory, + ) + assert not harness.factory.created + + +@pytest.mark.parametrize( + "model", + [ + {}, + {**MODEL, "revision": "main"}, + {**MODEL, "model_id": MODEL["model_id"] + "@" + MODEL["revision"]}, + {**MODEL, "provider": "auto"}, + {**MODEL, "supports_tools": False}, + ], +) +def test_invalid_model_preflight_never_launches_or_calls_provider(harness, model): + record = harness.backend.solve(model, 0) + assert record["error"] + assert not harness.factory.created and not harness.inference.requests + + +def test_target_tool_loop_is_bounded_and_uses_instruction_only(harness): + h = harness + h.inference.responses = [response(None, calls=[terminal()]), response("saved")] + record = h.backend.solve(MODEL, 1) + assert record["error"] is None and record["cleanup_verified"] + assert record["files"] == {"/app/answer.txt": b"answer"} + assert len(h.factory.created) == 1 + assert h.factory.created[0].commands[0][0] == [ + "/bin/bash", + "-c", + "printf answer > /app/answer.txt", + ] + assert h.inference.requests[0]["tools"][0]["function"]["name"] == "terminal" + assert all("stream" not in request for request in h.inference.requests) + assert h.inference.closed == 0 + assert "PRIVATE_SOLUTION_SENTINEL" not in repr(h.inference.requests) + assert "PRIVATE_CANARY_SENTINEL" not in repr(h.inference.requests) + assert record["input_tokens"] == 20 and record["output_tokens"] == 10 + + +def test_candidate_context_binds_only_declared_public_bytes_to_prompt( + harness, monkeypatch +): + from examples.coding_audit import hf + + h = harness + path = "environment/source/pkg/api.py" + source = h.root / path + source.parent.mkdir(parents=True) + raw = 'def answer():\n return "雪"\n'.encode() + source.write_bytes(raw) + (source.parent / "unused.py").write_text("UNDECLARED_PUBLIC_SENTINEL") + (source.parent / ".hidden").mkdir() + (source.parent / ".hidden/private.txt").write_text("HIDDEN_SENTINEL") + reads = [] + original = hf._local_file + + def read(path, limit): + reads.append(path.relative_to(h.root).as_posix()) + return original(path, limit) + + monkeypatch.setattr(hf, "_local_file", read) + h.backend.workdir = "/workspace/runtime" + h.inference.responses = [response("true")] + record = h.backend.generate_candidate({**MODEL, "context_paths": [path]}, 0) + assert record["error"] is None + assert reads == ["instruction.md", path] + prompt = h.inference.requests[0]["messages"][1]["content"] + assert prompt.startswith((h.root / "instruction.md").read_text()) + assert path in prompt and "pkg/api.py" in prompt and "/workspace/runtime" in prompt + assert raw.decode() in prompt + assert ( + "UNDECLARED_PUBLIC_SENTINEL" not in prompt and "HIDDEN_SENTINEL" not in prompt + ) + assert "PRIVATE_SOLUTION_SENTINEL" not in prompt + assert record["candidate_context"] == [ + { + "path": path, + "runtime_path": "pkg/api.py", + "size": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + } + ] + assert "context_paths" not in h.inference.requests[0] + + +def test_default_candidate_prompt_and_target_protocol_are_unchanged(harness): + h = harness + h.inference.responses = [response("true"), response("done")] + generated = h.backend.generate_candidate(MODEL, 0) + solved = h.backend.solve(MODEL, 1) + assert generated["error"] is None and solved["error"] is None + assert "candidate_context" not in generated + assert all( + request["messages"][1]["content"] == (h.root / "instruction.md").read_text() + for request in h.inference.requests + ) + + +@pytest.mark.parametrize( + "kind", ["symlink", "symlink_parent", "hardlink", "directory", "fifo"] +) +def test_candidate_context_rejects_linked_or_nonregular_files_before_inference( + harness, kind +): + h = harness + parent = h.root / "environment/source" + parent.mkdir(parents=True) + source = parent / "api.py" + if kind == "symlink": + source.symlink_to(h.root / "solution/solve.sh") + elif kind == "symlink_parent": + (h.root / "solution/api.py").write_text("PRIVATE_API_SENTINEL") + (parent / "linked").symlink_to(h.root / "solution", target_is_directory=True) + source = parent / "linked/api.py" + elif kind == "hardlink": + os.link(h.root / "solution/solve.sh", source) + elif kind == "directory": + source.mkdir() + else: + os.mkfifo(source) + record = h.backend.generate_candidate( + {**MODEL, "context_paths": [source.relative_to(h.root).as_posix()]}, 0 + ) + assert record["error"] and record["script"] == "" + assert not h.inference.requests and not h.inference.kwargs + assert not h.factory.created + + +@pytest.mark.parametrize( + "contents", + [ + [b"public\x00hidden"], + [b"\xff"], + [b"x" * 65537], + [b"x" * 32768, b"y" * 32769], + ], +) +def test_candidate_context_requires_bounded_utf8_without_nul(harness, contents): + h = harness + parent = h.root / "environment/source" + parent.mkdir(parents=True) + paths = [] + for index, raw in enumerate(contents): + path = parent / f"public-{index}.txt" + path.write_bytes(raw) + paths.append(path.relative_to(h.root).as_posix()) + record = h.backend.generate_candidate({**MODEL, "context_paths": paths}, 0) + assert record["error"] and record["script"] == "" + assert not h.inference.requests and not h.inference.kwargs + + +@pytest.mark.parametrize( + "size,budget,accepted", [(65536, 32768, True), (1024, 64, False)] +) +def test_candidate_context_uses_combined_byte_and_existing_message_budget( + harness, size, budget, accepted +): + h = harness + path = h.root / "environment/source/api.py" + path.parent.mkdir(parents=True) + path.write_bytes(b"x" * size) + h.inference.responses = [response("true")] + record = h.backend.generate_candidate( + { + **MODEL, + "context_paths": ["environment/source/api.py"], + "max_total_tokens": budget, + }, + 0, + ) + assert (record["error"] is None) is accepted + assert bool(h.inference.requests) is accepted + if not accepted: + assert "context exceeds byte budget" in record["error"] + + +def test_target_context_is_rejected_before_sandbox_or_inference(harness): + h = harness + record = h.backend.solve( + {**MODEL, "context_paths": ["environment/source/api.py"]}, 0 + ) + assert record["error"] and not h.factory.created + assert not h.inference.requests and not h.inference.kwargs + + +@pytest.mark.parametrize( + "path", + [ + "solution/solve.sh", + "environment/source/../../solution/solve.sh", + "environment/source/tests/hidden.py", + "environment/source/validation/spec.json", + "environment/source/.hidden/secret.py", + "environment/source/secrets/token", + "environment/source/api\x00.py", + ], +) +def test_invalid_candidate_context_is_rejected_without_reading_private_files( + harness, monkeypatch, path +): + from examples.coding_audit import hf + + h = harness + reads = [] + original = hf._local_file + + def read(filename, limit): + assert filename == h.root / "instruction.md" + reads.append(filename) + return original(filename, limit) + + monkeypatch.setattr(hf, "_local_file", read) + record = h.backend.generate_candidate({**MODEL, "context_paths": [path]}, 0) + assert record["error"] and record["script"] == "" + assert reads == [h.root / "instruction.md"] + assert not h.inference.requests and not h.inference.kwargs + + +def test_excess_parallel_tool_calls_execute_nothing_and_cleanup(harness): + h = harness + h.inference.responses = [ + response(None, calls=[terminal(identifier="a"), terminal(identifier="b")]) + ] + record = h.backend.solve({**MODEL, "max_steps": 1}, 0) + assert record["error"] is None and record["cleanup_verified"] + assert record["truncated"] and record["termination_reason"] == "tool_call_limit" + assert not h.factory.created[0].commands + + +def test_step_exhaustion_hands_actual_submission_to_grader(harness): + h = harness + h.inference.responses = [response(None, calls=[terminal()])] + record = h.backend.solve({**MODEL, "max_steps": 1}, 0) + assert record["error"] is None and record["files"] == {"/app/answer.txt": b"answer"} + assert record["truncated"] and record["termination_reason"] == "step_limit" + assert len(h.inference.requests) == 1 and record["cleanup_verified"] + grade = h.backend.grade(record) + assert grade["error"] is None and grade["reward"] == 1 + + +def test_invalid_tool_name_is_not_executed(harness): + h = harness + call = terminal() + call["function"]["name"] = "provider_request" + h.inference.responses = [response(None, calls=[call])] + record = h.backend.solve(MODEL, 0) + assert "invalid terminal" in record["error"] and not h.factory.created[0].commands + + +def test_output_and_token_limits_are_fail_closed(harness): + h = harness + h.factory.output = "💥" * 4097 + record = h.backend.execute_script(":", "reference") + assert "output exceeded" in record["error"] and record["cleanup_verified"] + h.factory.output = "" + h.inference.responses = [ + response( + "true", + usage={ + "prompt_tokens": 16000, + "completion_tokens": 400, + "total_tokens": 16400, + }, + ) + ] + record = h.backend.generate_candidate(MODEL, 0) + assert "token budget" in record["error"] + + +def test_private_path_audit_blocks_before_execution(harness): + h = harness + h.factory.initial["/tests"] = b"private" + record = h.backend.execute_script("true", "reference") + assert "private path" in record["error"] and record["cleanup_verified"] + assert not h.factory.created[0].commands + + +def test_leakage_probes_recheck_transcripts_without_extra_jobs(harness): + h = harness + h.factory.submitted = None + probes = h.backend.leakage_probes() + assert {row["name"] for row in probes} == { + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", + } + assert all(row["passed"] and row["error"] is None for row in probes) + assert len(h.factory.created) == 2 + h.inference.responses = [response("# PRIVATE_CANARY_SENTINEL\ntrue")] + h.backend.generate_candidate(MODEL, 0) + probes = h.backend.leakage_probes() + assert len(h.factory.created) == 2 + assert not next(row for row in probes if row["name"] == "canary_not_in_transcript")[ + "passed" + ] + + +def test_fresh_state_marker_blocks_reused_filesystem(harness): + h = harness + first = h.backend.execute_script(":", "reference") + h.factory.initial.update(h.factory.created[0].files.data) + second = h.backend.execute_script(":", "target") + assert first["error"] is None and "prior-state" in second["error"] + assert not h.factory.created[1].commands + assert not next( + row for row in h.backend.leakage_probes() if row["name"] == "fresh_state_absent" + )["passed"] + + +def test_state_marker_uses_configured_workdir(harness): + h = harness + h.execution["workdir"] = "/workspace/task" + backend = HFTaskBackend( + h.root, + h.execution, + token="execution-token-secret", + inference_token="inference-token-secret", + api=h.api, + sandbox_factory=h.factory, + inference_factory=h.inference, + ) + assert PurePosixPath(backend.marker_path).parent == PurePosixPath("/workspace/task") + assert PurePosixPath(backend.marker_path).name.startswith(".openenv-state-") + first = backend.execute_script(":", "reference") + h.factory.initial.update(h.factory.created[0].files.data) + second = backend.execute_script(":", "target") + assert first["error"] is None and "prior-state" in second["error"] + + +def test_submission_tree_digest_matches_canonical_algorithm(tmp_path): + (tmp_path / "app").mkdir() + (tmp_path / "app/a").write_bytes(b"alpha") + (tmp_path / "app/z").write_bytes(b"omega") + assert submission_digest({"/app/z": b"omega", "/app/a": b"alpha"}) == source_digest( + tmp_path + ) + assert submission_digest({}) == hashlib.sha256(b"").hexdigest() + + +def test_deadline_interrupts_blocking_controller_and_restores_handler(): + handler = signal.getsignal(signal.SIGALRM) + started = time.monotonic() + with pytest.raises(HFBackendError, match="deadline"): + with _deadline(0.02): + time.sleep(1) + assert time.monotonic() - started < 0.5 + assert signal.getsignal(signal.SIGALRM) == handler + assert signal.getitimer(signal.ITIMER_REAL)[0] == 0 + + +def test_strong_isolation_and_equal_credentials_rejected(harness): + h = harness + with pytest.raises(HFBackendError, match="strong"): + HFTaskBackend(h.root, {**h.execution, "require_strong_isolation": True}) + with pytest.raises(HFBackendError, match="credentials must differ"): + HFTaskBackend(h.root, h.execution, token="same", inference_token="same") + + +def test_runtime_tokens_are_redacted_and_credential_files_rejected(harness): + h = harness + h.factory.output = "server echoed execution-token-secret" + record = h.backend.execute_script(":", "reference") + assert "execution-token-secret" not in repr(record) + assert "[REDACTED]" in repr(record["transcript"]) + h.factory.output = "" + h.factory.submitted = b"inference-token-secret" + record = h.backend.execute_script(":", "reference") + assert "credential detected" in record["error"] and record["files"] == {} + assert "inference-token-secret" not in repr(record) + + +def test_empty_submission_does_not_inherit_verifier_image_answer(harness): + h = harness + h.factory.initial["/app/answer.txt"] = b"image-provided-answer" + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert grade["error"] is None + assert "/app/answer.txt" not in h.factory.created[0].files.data + + +def test_verifier_private_image_paths_are_not_overlaid(harness): + h = harness + h.factory.initial["/tests"] = b"stale-tests" + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert "preexisting private" in grade["error"] and grade["cleanup_verified"] + assert not h.factory.created[0].commands + + +def test_remote_regular_file_and_read_growth_checks(harness): + h = harness + sandbox = h.factory(image=AGENT) + sandbox.files.data["/app/answer.txt"] = b"answer" + sandbox.files.types["/app/answer.txt"] = "symlink" + with pytest.raises(HFBackendError, match="regular"): + h.backend._collect(sandbox) + sandbox.files.types.clear() + sandbox.files.grow_on_read = True + with pytest.raises(SandboxError): + h.backend._collect(sandbox) + assert sandbox.files.MAX_READ_BYTES == h.backend.max_bytes + + +def test_missing_execution_or_inference_credentials_never_fall_back_to_hf_token( + harness, monkeypatch +): + h = harness + monkeypatch.setenv("HF_TOKEN", "persistence-token") + backend = HFTaskBackend( + h.root, + h.execution, + api=h.api, + sandbox_factory=h.factory, + inference_factory=h.inference, + ) + assert "HF_SANDBOX_TOKEN" in backend.execute_script(":", "nop")["error"] + assert "HF_INFERENCE_TOKEN" in backend.solve(MODEL, 0)["error"] + assert not h.factory.created and not h.inference.requests + + +def test_timed_out_command_with_zero_exit_is_error(harness): + harness.factory.timed_out = True + record = harness.backend.execute_script(":", "reference") + assert "timed out" in record["error"] and record["cleanup_verified"] + + +def test_cleanup_deadline_is_not_swallowed_by_retry_loop(harness): + h = harness + original = h.api.cancel_job + + def blocked_cancel(**kwargs): + time.sleep(2) + return original(**kwargs) + + h.api.cancel_job = blocked_cancel + h.backend.cleanup_timeout = 0.02 + started = time.monotonic() + record = h.backend.execute_script(":", "reference") + assert time.monotonic() - started < 0.5 + assert record["error"] and not record["cleanup_verified"] + + +def test_canary_probe_checks_actual_unicode_message_text(harness): + h = harness + h.backend.canary = "private\n雪" + h.backend._transcripts = [[{"content": "private\n雪"}]] + assert not h.backend.leakage_probes()[-1]["passed"] + + +def test_failed_shell_command_is_observed_then_agent_repairs(harness, monkeypatch): + h = harness + original = FakeSandbox.run + calls = [] + + def run_with_failure(sandbox, argv, **kwargs): + calls.append(argv) + result = original(sandbox, argv, **kwargs) + result.exit_code = 1 if len(calls) == 1 else 0 + return result + + monkeypatch.setattr(FakeSandbox, "run", run_with_failure) + h.inference.responses = [ + response(None, calls=[terminal("grep missing absent", "a")]), + response(None, calls=[terminal("printf answer > /app/answer.txt", "b")]), + response("saved"), + ] + record = h.backend.solve(MODEL, 0) + assert record["error"] is None and not record["truncated"] + assert record["files"] == {"/app/answer.txt": b"answer"} + observation = h.inference.requests[1]["messages"][-1] + assert json.loads(observation["content"])["exit_code"] == 1 + assert h.backend.grade(record)["reward"] == 1 + + +def test_execution_token_in_terminal_output_never_reaches_provider(harness): + h = harness + h.factory.output = "server echoed execution-token-secret" + h.inference.responses = [response(None, calls=[terminal()]), response("done")] + record = h.backend.solve(MODEL, 0) + assert record["error"] is None + assert "execution-token-secret" not in repr(h.inference.requests) + assert "[REDACTED]" in repr(h.inference.requests[1]) + + +def test_failed_startup_recovers_exact_labeled_job_and_independently_cleans(harness): + h = harness + h.factory.fail_after_create = True + original = h.api.cancel_job + cancelled = [] + + def transient_cancel(**kwargs): + cancelled.append(kwargs["job_id"]) + if len(cancelled) == 1: + raise RuntimeError("transient controller failure") + return original(**kwargs) + + h.api.cancel_job = transient_cancel + record = h.backend.execute_script(":", "reference") + assert record["error"] and record["cleanup_verified"] + assert record["sandbox_id"] == "job-0" and cancelled == ["job-0", "job-0"] + assert record["cleanup_job_ids"] == ["job-0"] + assert h.api.stages["job-0"] == "CANCELED" + + +def test_transient_kill_exception_does_not_abort_cleanup_retries(harness, monkeypatch): + h = harness + h.api.cancel_error = True + original = FakeSandbox.kill + killed = [] + + def transient_kill(sandbox): + killed.append(sandbox.id) + if len(killed) == 1: + raise RuntimeError("transient kill failure") + original(sandbox) + + monkeypatch.setattr(FakeSandbox, "kill", transient_kill) + record = h.backend.execute_script(":", "reference") + assert record["error"] is None and record["cleanup_verified"] + assert killed == ["job-0", "job-0"] + + +def test_startup_recovery_retries_transient_listing_error(harness): + h = harness + h.factory.fail_after_create = True + original = h.api.list_jobs + calls = [] + + def transient_listing(**kwargs): + calls.append(kwargs) + if len(calls) == 1: + raise RuntimeError("transient lookup failure") + return original(**kwargs) + + h.api.list_jobs = transient_listing + record = h.backend.execute_script(":", "reference") + assert record["error"] and record["cleanup_verified"] + assert len(calls) == 2 and h.api.cancelled == ["job-0"] + + +def test_rejected_length_stream_keeps_finish_and_terminal_usage(harness): + h = harness + h.inference.responses = [ + ( + ChatCompletionStreamOutput.parse_obj_as_instance(event) + for event in ( + [ + stream_event(delta={"role": "assistant", "content": "partial"}), + stream_event(finish="length"), + stream_event( + choices=[], + usage={ + "prompt_tokens": 10, + "completion_tokens": 2048, + "total_tokens": 2058, + }, + ), + ] + ) + ) + ] + record = h.backend.generate_candidate(MODEL, 0) + assert "incomplete" in record["error"] and record["script"] == "" + assert record["model_metadata"]["responses"][0]["finish_reason"] == "length" + assert record["usage"] == {"input_tokens": 10, "output_tokens": 2048} + assert h.inference.closed == 1 and not h.factory.created + + +@pytest.mark.parametrize("value", [0, 1, "false", None]) +def test_thinking_flag_requires_an_explicit_boolean(harness, value): + h = harness + record = h.backend.generate_candidate({**MODEL, "enable_thinking": value}, 0) + assert "enable_thinking must be a boolean" in record["error"] + assert not h.inference.requests and record["script"] == "" diff --git a/examples/coding_audit/test_runner.py b/examples/coding_audit/test_runner.py new file mode 100644 index 000000000..f6bc7dbf6 --- /dev/null +++ b/examples/coding_audit/test_runner.py @@ -0,0 +1,757 @@ +"""Exercise collection, gating and artifact binding without paid services.""" + +import hashlib +import json +import runpy +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +from examples.coding_audit.runner import ( + coding_budget, + live_provider_id, + normalize_coding_config, + run_coding_validation, +) + + +@pytest.mark.parametrize("representation", ["dict", "list", "json-list"]) +def test_live_hub_provider_mapping_shapes(representation): + entry = SimpleNamespace( + provider="deepinfra", provider_id="Qwen/Qwen3.8-27B", status="live" + ) + mappings = ( + {"deepinfra": entry} + if representation == "dict" + else [vars(entry)] + if representation == "json-list" + else [entry] + ) + assert ( + live_provider_id( + SimpleNamespace(inference_provider_mapping=mappings), "deepinfra" + ) + == "Qwen/Qwen3.8-27B" + ) + + +@pytest.mark.parametrize( + "mappings", + [ + None, + [], + [SimpleNamespace(provider="deepinfra", provider_id="model", status="staging")], + [SimpleNamespace(provider="deepinfra", provider_id="model", status="live")] * 2, + ], +) +def test_provider_mapping_must_be_unique_and_live(mappings): + with pytest.raises(ValueError): + live_provider_id( + SimpleNamespace(inference_provider_mapping=mappings), "deepinfra" + ) + + +def configuration(): + config = json.loads(Path(__file__).with_name("config.json").read_text()) + config["generation"]["candidate_count"] = 1 + config["validation"].update(reference_repeats=2, target_attempts=2) + config["seeds"] = [1, 2] + return config + + +class Backend: + def __init__(self): + self.number = 0 + self.target_calls = 0 + self.generated = "mkdir -p /app; echo 42 > /app/answer.txt" + self.bad_verifier = False + self.bad_raw = False + self.target_error = False + self.mutate = None + self.transcript = [] + + def execute_script(self, script, role): + self.number += 1 + value = b"42\n" if "42" in script else b"41\n" + files = {} if script == "true" else {"/app/answer.txt": value} + digest = hashlib.sha256( + json.dumps({k: v.hex() for k, v in files.items()}, sort_keys=True).encode() + ).hexdigest() + image = configuration()["execution"]["agent_image"] + return { + "files": files, + "submission_digest": digest, + "sandbox_id": f"solver-{self.number}", + "image_ref": image, + "cleanup_verified": True, + "error": None, + "transcript": list(self.transcript), + "usage": {"input_tokens": 10, "output_tokens": 5}, + "latency_seconds": 1, + } + + def solve(self, model, seed, role): + self.target_calls += 1 + result = self.execute_script("42" if seed == 2 else "41", role) + if self.target_error: + result["error"] = "provider failure" + return result + + def generate_candidate(self, model, seed): + if self.mutate: + self.mutate() + return {"script": self.generated, "error": None} + + def grade(self, submission): + self.number += 1 + reward = int( + submission["files"].get("/app/answer.txt") == b"42\n" or self.bad_verifier + ) + return { + "grade_sandbox_id": f"grader-{self.number}", + "verifier_image_ref": configuration()["execution"]["verifier_image"], + "reward": reward, + "rawreward": b"0.999999999999999999999999" + if self.bad_raw + else str(reward).encode(), + "cleanup_verified": True, + "error": None, + "transcript": [ + { + "exit_code": 0, + "output": f"Native reward: {reward}\n", + "output_truncated": False, + } + ], + } + + def leakage_probes(self): + return [ + {"name": name, "passed": True, "error": None} + for name in ( + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", + ) + ] + + +@pytest.fixture +def task(tmp_path): + root = tmp_path / "task" + for name in ("tests", "solution", "environment"): + (root / name).mkdir(parents=True) + (root / "task.toml").write_text('version = "1.0"\n') + (root / "instruction.md").write_text("Write exactly 42 into /app/answer.txt.\n") + (root / "tests/test.sh").write_text("#!/bin/bash\nprintf 1\n") + (root / "solution/solve.sh").write_text("printf 42 > /app/answer.txt\n") + (root / "environment/Dockerfile").write_text( + "FROM python:3.12-slim\nWORKDIR /app\n" + ) + return root + + +def test_complete_collection_is_bound_and_keeps_model_difficulty(task, tmp_path): + backend = Backend() + output = tmp_path / "audit" + report = run_coding_validation(task, configuration(), output, backend=backend) + assert report["accepted"] + assert not report["rfc008_complete"] + assert not report["trainability_established"] + assert backend.target_calls == 2 + assert report["checks"]["target_difficulty"]["solve_rate"] == 0.5 + evidence = json.loads((output / "evidence.json").read_text()) + assert {r["origin"] for r in evidence["references"]} == {"provided", "generated"} + assert len(evidence["generation"]) == 1 + for row in evidence["trials"]: + artifact = row["grader_transcript"] + assert artifact["path"].endswith("/grader-transcript.json") + data = (output / artifact["path"]).read_bytes() + assert hashlib.sha256(data).hexdigest() == artifact["sha256"] + assert json.loads(data)[0]["output"] == f"Native reward: {row['reward']}\n" + assert evidence["trials"][0]["reward"] == 0 + for line in (output / "SHA256SUMS").read_text().splitlines(): + digest, name = line.split(" ", 1) + assert hashlib.sha256((output / name).read_bytes()).hexdigest() == digest + + +def test_bad_verifier_blocks_target_inference(task, tmp_path): + backend = Backend() + backend.bad_verifier = True + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + assert report["checks"]["verifier_rejection"]["status"] == "fail" + assert backend.target_calls == 0 + + +def test_failed_clean_candidate_excluded_without_overwriting_frozen_verifier( + task, tmp_path +): + backend = Backend() + backend.generated = "echo wrong" + before = (task / "tests/test.sh").read_bytes() + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert report["accepted"] + assert len(report["checks"]["references"]["excluded"]) == 1 + assert (task / "tests/test.sh").read_bytes() == before + + +def test_raw_reward_precision_disagreement_cannot_pass(task, tmp_path): + backend = Backend() + backend.bad_raw = True + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + assert backend.target_calls == 0 + assert report["checks"]["reliability"]["status"] == "error" + + +def test_target_infra_failure_preserves_missing_not_zero(task, tmp_path): + backend = Backend() + backend.target_error = True + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + check = report["checks"]["target_difficulty"] + assert not report["accepted"] + assert check["completed"] == 0 + assert check["solve_rate"] is None + + +def test_changed_source_cannot_be_admitted(task, tmp_path): + backend = Backend() + backend.mutate = lambda: (task / "instruction.md").write_text("Changed task") + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + evidence = json.loads((tmp_path / "audit/evidence.json").read_text()) + assert evidence["collection_error"] == "ValueError" + + +def test_credentials_in_transcript_are_omitted_and_block_acceptance( + task, tmp_path, monkeypatch +): + token = "test-secret-not-a-real-token" + monkeypatch.setenv("HF_SANDBOX_TOKEN", token) + backend = Backend() + backend.transcript = [{"content": token}] + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + for path in (tmp_path / "audit").rglob("*"): + if path.is_file(): + assert token.encode() not in path.read_bytes() + + +@pytest.mark.parametrize( + "path,value", [("agent_image", "python:latest"), ("api_key", "secret")] +) +def test_mutable_images_and_embedded_credentials_rejected(path, value): + config = configuration() + config["execution"][path] = value + with pytest.raises(ValueError): + normalize_coding_config(config) + + +def test_budget_is_bounded_before_any_remote_execution(task, tmp_path): + config = configuration() + config["cloud"]["max_sandboxes"] = 1 + with pytest.raises(ValueError, match="budget"): + run_coding_validation(task, config, tmp_path / "audit", backend=Backend()) + assert not (tmp_path / "audit").exists() + assert ( + coding_budget(normalize_coding_config(configuration()))["inference_calls_max"] + == 33 + ) + + +@pytest.mark.parametrize("accepted", [True, False]) +def test_cli_dispatch_uses_example_runner_and_retains_evidence( + task, tmp_path, monkeypatch, capsys, accepted +): + config = tmp_path / "config.json" + config.write_text(json.dumps(configuration())) + original = run_coding_validation + backend = Backend() + backend.bad_verifier = not accepted + monkeypatch.setattr( + "examples.coding_audit.runner.run_coding_validation", + lambda task, config, output: original(task, config, output, backend=backend), + ) + output = tmp_path / "audit" + monkeypatch.setattr( + sys, + "argv", + [ + "examples.coding_audit", + "--task", + str(task), + "--config", + str(config), + "--output", + str(output), + "--json", + ], + ) + with pytest.raises(SystemExit) as result: + runpy.run_module("examples.coding_audit", run_name="__main__") + assert result.value.code == (0 if accepted else 1) + report = json.loads((output / "report.json").read_text()) + assert report["accepted"] is accepted + assert json.loads(capsys.readouterr().out) == report + assert (output / "evidence.json").is_file() + for line in (output / "SHA256SUMS").read_text().splitlines(): + digest, name = line.split(" ", 1) + assert hashlib.sha256((output / name).read_bytes()).hexdigest() == digest + + +def test_cli_requires_persisted_evidence(task, tmp_path, monkeypatch, capsys): + monkeypatch.setattr( + sys, + "argv", + [ + "examples.coding_audit", + "--task", + str(task), + "--config", + str(tmp_path / "missing.json"), + ], + ) + with pytest.raises(SystemExit) as result: + runpy.run_module("examples.coding_audit", run_name="__main__") + assert result.value.code == 2 + assert "--output" in capsys.readouterr().err + + +def test_cli_input_failure_exits_two_without_starting_backend( + task, tmp_path, monkeypatch, capsys +): + from examples.coding_audit.__main__ import main + + monkeypatch.setattr( + "examples.coding_audit.__main__.run_coding_validation", + lambda *args: pytest.fail("input failure reached backend"), + ) + assert ( + main( + [ + "--task", + str(task), + "--config", + str(tmp_path / "missing.json"), + "--output", + str(tmp_path / "audit"), + ] + ) + == 2 + ) + assert "input failure (FileNotFoundError)" in capsys.readouterr().err + + +def test_exact_json_threshold_cannot_round_into_a_pass(task, tmp_path): + from decimal import Decimal + + from openenv.validation.runtime.json_values import loads + + config = loads( + json.dumps(configuration()).replace( + '"reward_max": 1,', '"reward_max": 1.00000000000000000001,' + ) + ) + normalized = normalize_coding_config(config) + assert normalized["validation"]["reward_max"] == Decimal("1.00000000000000000001") + report = run_coding_validation(task, config, tmp_path / "audit", backend=Backend()) + assert not report["accepted"] + saved = loads((tmp_path / "audit/config.json").read_text()) + assert saved["validation"]["reward_max"] == normalized["validation"]["reward_max"] + + +@pytest.mark.parametrize("value", ["", "missing"]) +def test_execution_error_status_cannot_be_overwritten(task, tmp_path, value): + class BrokenStatus(Backend): + def execute_script(self, script, role): + row = super().execute_script(script, role) + if value == "missing": + del row["error"] + else: + row["error"] = value + return row + + backend = BrokenStatus() + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + assert backend.target_calls == 0 + rows = json.loads((tmp_path / "audit/evidence.json").read_text())["trials"] + assert all(row["error"] is not None for row in rows) + + +def test_verifier_identity_comes_from_verified_snapshot(task, tmp_path, monkeypatch): + from examples.coding_audit import runner as coding_runner + + original = coding_runner.source_digest + expected = original(task / "tests") + + def raced(path): + if path == task / "tests": + raise AssertionError( + "mutable original verifier must not be hashed independently" + ) + return original(path) + + monkeypatch.setattr(coding_runner, "source_digest", raced) + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=Backend() + ) + assert report["accepted"] + assert report["verifier_digest"] == expected + + +def test_credential_metadata_is_failed_and_report_is_sanitized( + task, tmp_path, monkeypatch +): + secret = "dummy-execution-credential-for-regression" + monkeypatch.setenv("HF_SANDBOX_TOKEN", secret) + + class DiagnosticBackend(Backend): + def leakage_probes(self): + return super().leakage_probes() + [ + { + "name": "diagnostic:" + secret, + "passed": True, + "error": None, + secret: "key", + } + ] + + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=DiagnosticBackend() + ) + assert not report["accepted"] + assert secret not in json.dumps(report) + for path in (tmp_path / "audit").rglob("*"): + if path.is_file(): + assert secret.encode() not in path.read_bytes() + + +def test_credential_in_config_is_rejected_before_execution(task, tmp_path, monkeypatch): + secret = "dummy-config-credential-for-regression" + monkeypatch.setenv("HF_SANDBOX_TOKEN", secret) + config = configuration() + config["negative_cases"][0]["id"] = secret + config["validation"]["required_negative_cases"][0] = secret + with pytest.raises(ValueError, match="credentials"): + run_coding_validation(task, config, tmp_path / "audit", backend=Backend()) + + +def test_in_task_output_rejected_before_inference(task): + backend = Backend() + with pytest.raises(ValueError, match="outside"): + run_coding_validation( + task, configuration(), task / "audit.artifacts", backend=backend + ) + assert backend.target_calls == 0 + assert not (task / "audit.artifacts").exists() + + +@pytest.mark.parametrize("with_context", [False, True]) +def test_real_backend_contract_reaches_difficulty_with_sdk_fakes( + task, tmp_path, monkeypatch, with_context +): + """Join the actual collector/backend/scorer, not just each seam in isolation.""" + from examples.coding_audit.hf import HFTaskBackend + + m = SimpleNamespace(**runpy.run_path(str(Path(__file__).with_name("test_hf.py")))) + api, inference = m.API(), m.InferenceFactory() + factory = m.Factory(api) + original = m.FakeSandbox.run + + def run(sandbox, argv, **kwargs): + if sandbox.image == m.VERIFIER: + sandbox.factory.reward = ( + b"1" if sandbox.files.data.get("/app/answer.txt") == b"42\n" else b"0" + ) + else: + command = " ".join(argv) + sandbox.factory.submitted = ( + b"42\n" if "42" in command else b"41\n" if "41" in command else None + ) + return original(sandbox, argv, **kwargs) + + monkeypatch.setattr(m.FakeSandbox, "run", run) + config = configuration() + context_path = "environment/source/pkg/api.py" + context = b"def public_answer():\n return 42\n" + if with_context: + source = task / context_path + source.parent.mkdir(parents=True) + source.write_bytes(context) + config.setdefault("generator", dict(config["target"]))["context_paths"] = [ + context_path + ] + config["execution"].update( + agent_image=m.AGENT, verifier_image=m.VERIFIER, namespace="owner" + ) + inference.responses = [ + m.response('printf "42\\n" > /app/answer.txt'), + m.response(None, calls=[m.terminal('printf "41\\n" > /app/answer.txt')]), + m.response("done"), + m.response(None, calls=[m.terminal('printf "42\\n" > /app/answer.txt')]), + m.response("done"), + ] + backend = HFTaskBackend( + task.resolve(), + config["execution"], + token="execution-only-test-token", + inference_token="inference-only-test-token", + api=api, + sandbox_factory=factory, + inference_factory=inference, + ) + output = tmp_path / "audit" + report = run_coding_validation(task, config, output, backend=backend) + assert report["accepted"], report + assert report["checks"]["target_difficulty"]["solve_rate"] == 0.5 + assert len(inference.requests) == 5 + evidence_path = output / "evidence.json" + generated = json.loads(evidence_path.read_text())["generation"][0] + if with_context: + assert generated["candidate_context"] == [ + { + "path": context_path, + "runtime_path": "pkg/api.py", + "size": len(context), + "sha256": hashlib.sha256(context).hexdigest(), + } + ] + assert context.decode() in inference.requests[0]["messages"][1]["content"] + else: + assert "candidate_context" not in generated + assert all( + request["messages"][1]["content"] == (task / "instruction.md").read_text() + for request in inference.requests[1:] + ) + checksums = dict( + line.split(" ", 1)[::-1] + for line in (output / "SHA256SUMS").read_text().splitlines() + ) + assert ( + checksums["evidence.json"] + == hashlib.sha256(evidence_path.read_bytes()).hexdigest() + ) + assert factory.created + assert all(stage == "CANCELED" for stage in api.stages.values()) + + +def test_grader_exception_keeps_sanitized_sha_bound_logs(task, tmp_path, monkeypatch): + from examples.coding_audit.hf import HFTaskBackend + + m = SimpleNamespace(**runpy.run_path(str(Path(__file__).with_name("test_hf.py")))) + api, inference = m.API(), m.InferenceFactory() + factory = m.Factory(api) + original = m.FakeSandbox.run + secret = "execution-only-test-token" + + def run(sandbox, argv, **kwargs): + if sandbox.image == m.VERIFIER: + kwargs["on_stdout"]("APT: partial output 雪 " + secret) + raise RuntimeError("grader RPC failed: " + secret) + return original(sandbox, argv, **kwargs) + + monkeypatch.setattr(m.FakeSandbox, "run", run) + config = configuration() + config["execution"].update( + agent_image=m.AGENT, verifier_image=m.VERIFIER, namespace="owner" + ) + inference.responses = [m.response('printf "42\\n" > /app/answer.txt')] + backend = HFTaskBackend( + task.resolve(), + config["execution"], + token=secret, + inference_token="inference-only-test-token", + api=api, + sandbox_factory=factory, + inference_factory=inference, + ) + output = tmp_path / "audit" + report = run_coding_validation(task, config, output, backend=backend) + assert not report["accepted"] + evidence = json.loads((output / "evidence.json").read_text()) + assert evidence["target_not_run_reason"] + for row in evidence["trials"]: + assert row["error"] == "HF backend operation failed (RuntimeError)" + assert row["reward"] is None and row["cleanup_verified"] + artifact = row["grader_transcript"] + data = (output / artifact["path"]).read_bytes() + assert hashlib.sha256(data).hexdigest() == artifact["sha256"] + trace = json.loads(data)[0] + assert trace["output"] == "APT: partial output 雪 [REDACTED]" + assert trace["exit_code"] is None and not trace["output_truncated"] + for path in output.rglob("*"): + if path.is_file(): + assert secret.encode() not in path.read_bytes() + assert all(stage == "CANCELED" for stage in api.stages.values()) + + +@pytest.mark.parametrize("callbacks", ["single", "separate", "split_secret"]) +@pytest.mark.parametrize("intersecting", [False, True]) +def test_grader_overflow_discards_output_in_sha_bound_artifact( + task, tmp_path, monkeypatch, callbacks, intersecting +): + from examples.coding_audit.hf import HFTaskBackend + + m = SimpleNamespace(**runpy.run_path(str(Path(__file__).with_name("test_hf.py")))) + api, inference = m.API(), m.InferenceFactory() + factory = m.Factory(api) + original = m.FakeSandbox.run + execution_secret = ( + "hf_SYNTHETIC_EXECUTION_TOKEN_REVIEW_ONLYh" + if intersecting + else "hf_SYNTHETIC_EXECUTION_TOKEN_FOR_REVIEW_ONLY" + ) + inference_secret = "hf_SYNTHETIC_INFERENCE_TOKEN_REVIEW_ONLY" + secret = inference_secret if intersecting else execution_secret + cap, retained = 1048576, 28 + prefix = "雪" + "A" * (cap - len("雪".encode()) - retained) + if intersecting: + prefix = prefix[: -len(execution_secret[:-1])] + execution_secret[:-1] + assert execution_secret in prefix + secret + + def run(sandbox, argv, **kwargs): + if sandbox.image == m.VERIFIER: + if callbacks == "single": + kwargs["on_stdout"](prefix + secret) + else: + split = 7 if callbacks == "split_secret" else 0 + kwargs["on_stdout"](prefix + secret[:split]) + kwargs["on_stderr"](secret[split:]) + pytest.fail("grader output overflow was not rejected") + return original(sandbox, argv, **kwargs) + + monkeypatch.setattr(m.FakeSandbox, "run", run) + config = configuration() + config["execution"].update( + agent_image=m.AGENT, + verifier_image=m.VERIFIER, + namespace="owner", + max_tool_output_bytes=cap, + ) + inference.responses = [m.response('printf "42\\\\n" > /app/answer.txt')] + backend = HFTaskBackend( + task.resolve(), + config["execution"], + token=execution_secret, + inference_token=inference_secret, + api=api, + sandbox_factory=factory, + inference_factory=inference, + ) + output = tmp_path / "audit" + report = run_coding_validation(task, config, output, backend=backend) + assert not report["accepted"] + evidence = json.loads((output / "evidence.json").read_text()) + assert evidence["target_not_run_reason"] and evidence["trials"] + for row in evidence["trials"]: + assert row["error"] == "terminal output exceeded byte limit" + assert row["reward"] is None and row["cleanup_verified"] + assert "raw_reward" not in row + artifact = row["grader_transcript"] + data = (output / artifact["path"]).read_bytes() + assert hashlib.sha256(data).hexdigest() == artifact["sha256"] + assert json.loads(data) == [ + { + "exit_code": None, + "output": "", + "output_truncated": True, + } + ] + assert len(json.loads(data)[0]["output"].encode()) <= cap + for line in (output / "SHA256SUMS").read_text().splitlines(): + digest, name = line.split(" ", 1) + data = (output / name).read_bytes() + assert hashlib.sha256(data).hexdigest() == digest + assert secret[:retained].encode() not in data + assert execution_secret[:-1].encode() not in data + assert api.stages and all(stage == "CANCELED" for stage in api.stages.values()) + + +@pytest.mark.parametrize("value", [False, True]) +def test_explicit_candidate_thinking_control_preserves_target(value): + raw = configuration() + raw["generator"] = {**raw["target"], "enable_thinking": value} + normalized = normalize_coding_config(raw) + assert normalized["generator"]["enable_thinking"] is value + assert "enable_thinking" not in normalized["target"] + + +@pytest.mark.parametrize("value", [0, 1, "false", None]) +def test_thinking_configuration_rejects_non_boolean(value): + raw = configuration() + raw["generator"] = {**raw["target"], "enable_thinking": value} + with pytest.raises(ValueError, match="enable_thinking"): + normalize_coding_config(raw) + + +@pytest.mark.parametrize("count", [0, 1, 32]) +def test_generator_context_is_optional_explicit_and_preserves_target(count): + raw = configuration() + paths = [f"environment/source/pkg/api-{index}.py" for index in range(count)] + raw["generator"] = {**raw["target"], "context_paths": paths} + normalized = normalize_coding_config(raw) + assert normalized["generator"]["context_paths"] == paths + assert "context_paths" not in normalized["target"] + assert "context_paths" not in normalize_coding_config(configuration())["generator"] + + +@pytest.mark.parametrize( + "paths", + [ + "environment/source/api.py", + None, + [1], + ["environment/source/api.py"] * 2, + [f"environment/source/{index}.py" for index in range(33)], + ["environment/source"], + ["environment/source/api.py/"], + ["/environment/source/api.py"], + ["environment/source/../tests/test.sh"], + ["environment/source//api.py"], + ["environment/source/./api.py"], + ["environment/source/api\x00.py"], + ["environment/source/a\nb.py"], + ["environment/source/a\\b.py"], + ["environment/source/a:b.py"], + ["tests/test.sh"], + ["solution/solve.sh"], + ["validation/spec.json"], + ["environment/source/tests/test.py"], + ["environment/source/solution/solve.sh"], + ["environment/source/validation/spec.json"], + ["environment/source/.hidden/api.py"], + ["environment/source/.env"], + ["environment/source/secrets/token"], + ["environment/source/reference/solve.py"], + ["environment/source/spec/task.md"], + ], +) +def test_generator_context_rejects_unsafe_paths(paths): + raw = configuration() + raw["generator"] = {**raw["target"], "context_paths": paths} + with pytest.raises(ValueError, match="context_paths"): + normalize_coding_config(raw) + + +def test_target_cannot_declare_context_paths(): + raw = configuration() + raw["target"]["context_paths"] = [] + with pytest.raises(ValueError, match="target.context_paths"): + normalize_coding_config(raw) diff --git a/examples/coding_audit/test_scoring.py b/examples/coding_audit/test_scoring.py new file mode 100644 index 000000000..a30cd8746 --- /dev/null +++ b/examples/coding_audit/test_scoring.py @@ -0,0 +1,575 @@ +"""Admission tests for pure coding-audit evidence scoring; no execution backends.""" + +import copy +import json + +import pytest +from examples.coding_audit.scoring import assess_coding_audit, validate_audit_config + +TASK = "a" * 64 +VERIFIER = "b" * 64 +IMAGE = "registry/agent@sha256:" + "c" * 64 +VERIFIER_IMAGE = "registry/verifier@sha256:" + "d" * 64 +TARGET = {"model": "Qwen/Qwen3.8-27B", "revision": "e" * 40, "provider": "hf-inference"} +CONFIG = {"required_negative_cases": ["wrong_answer", "empty_answer"]} + + +def trial(role, case_id, reward, number, submission_digest=None): + result = { + "role": role, + "case_id": case_id, + "task_digest": TASK, + "verifier_digest": VERIFIER, + "image_ref": IMAGE, + "verifier_image_ref": VERIFIER_IMAGE, + "submission_digest": submission_digest or "f" * 64, + "sandbox_id": f"agent-{number}", + "grade_sandbox_id": f"grade-{number}", + "reward": reward, + "cleanup_verified": True, + "error": None, + } + if role == "target": + result.update(TARGET) + result.update( + latency_seconds=2.5, usage={"input_tokens": 100, "output_tokens": 20} + ) + return result + + +@pytest.fixture +def evidence(): + return { + "task_digest": TASK, + "verifier_digest": VERIFIER, + "agent_image": IMAGE, + "verifier_image": VERIFIER_IMAGE, + "target": dict(TARGET), + "references": [ + { + "case_id": "oracle", + "submission_digest": "f" * 64, + "origin": "provided", + } + ], + "trials": [ + trial("nop", "nop", 0, 0), + trial("reference", "oracle", 1, 1), + trial("reference", "oracle", 1, 2), + trial("negative", "wrong_answer", 0, 3), + trial("negative", "empty_answer", 0, 4), + trial("target", "attempt-1", 0, 5), + trial("target", "attempt-2", 1, 6), + ], + "leakage_probes": [ + {"name": name, "passed": True, "error": None} + for name in ( + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", + ) + ], + } + + +def test_complete_audit_reports_bounded_measurements_without_mutating_inputs(evidence): + original = copy.deepcopy(evidence) + report = assess_coding_audit(CONFIG, evidence) + assert report["profile"] == "harbor-coding-audit-v1" + assert report["accepted"] is True + assert report["verdict"] == "pass" + assert not report["rfc008_complete"] + assert not report["trainability_established"] + assert {check["status"] for check in report["checks"].values()} == {"pass"} + target = report["checks"]["target_difficulty"] + assert target["completed"] == 2 + assert target["failed"] == target["missing"] == 0 + assert target["solve_rate"] == 0.5 + assert target["solve_rate_wilson95"] == pytest.approx([0.0945312057, 0.9054687943]) + assert target["reward_variance"] == 0.25 + assert target["mixed_success"] is True + assert target["difficulty"] == "mixed" + assert target["latency_seconds_total"] == 5 + assert target["usage_totals"] == {"input_tokens": 200, "output_tokens": 40} + assert all(value == 0 for value in target["telemetry_missing"].values()) + assert "bounded probes" in report["checks"]["leakage"]["scope"] + assert evidence == original + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize("reward,difficulty", [(0, "none_solved"), (1, "all_solved")]) +def test_uniform_target_rewards_are_valid_difficulty_measurements( + evidence, reward, difficulty +): + for record in evidence["trials"][-2:]: + record["reward"] = reward + report = assess_coding_audit(CONFIG, evidence) + assert report["accepted"] + target = report["checks"]["target_difficulty"] + assert target["difficulty"] == difficulty + assert target["mixed_success"] is False + assert target["reward_variance"] == 0 + assert target["solve_rate"] == reward + + +def test_target_infrastructure_error_is_missing_evidence_not_a_zero_reward(evidence): + failed = evidence["trials"][-2] + failed.update(error="provider timeout", reward=None) + report = assess_coding_audit(CONFIG, evidence) + assert not report["accepted"] + target = report["checks"]["target_difficulty"] + assert target["status"] == "error" + assert target["attempted"] == 2 + assert target["completed"] == target["failed"] == target["missing"] == 1 + assert target["infrastructure_errors"] == 1 + assert target["solve_rate"] == 1 + assert target["reward_variance"] == 0 + assert target["usage_totals"]["input_tokens"] == 200 + + +def test_missing_target_attempt_is_skip(evidence): + evidence["trials"].pop() + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "skip" + assert report["checks"]["target_difficulty"]["missing"] == 1 + + +def test_configured_repeat_and_attempt_counts_are_honored(evidence): + config = CONFIG | {"reference_repeats": 3, "target_attempts": 3} + report = assess_coding_audit(config, evidence) + assert report["checks"]["references"]["status"] == "skip" + assert report["checks"]["target_difficulty"]["missing"] == 1 + evidence["trials"].extend( + [trial("reference", "oracle", 1, 10), trial("target", "attempt-3", 0, 11)] + ) + assert assess_coding_audit(config, evidence)["accepted"] + + +@pytest.mark.parametrize( + "duplicate", ["record", "case_id", "sandbox_id", "grade_sandbox_id", "cross_role"] +) +def test_duplicate_target_attempts_never_count_as_coverage(evidence, duplicate): + first, second = evidence["trials"][-2:] + if duplicate == "record": + evidence["trials"][-1] = dict(first) + elif duplicate == "cross_role": + second["sandbox_id"] = evidence["trials"][1]["grade_sandbox_id"] + else: + second[duplicate] = first[duplicate] + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "error" + assert report["checks"]["target_difficulty"]["completed"] < 2 + + +def test_no_target_data_reports_null_metrics(evidence): + evidence["trials"] = [ + record for record in evidence["trials"] if record["role"] != "target" + ] + report = assess_coding_audit(CONFIG, evidence) + target = report["checks"]["target_difficulty"] + assert target["status"] == "skip" + for name in ( + "solve_rate", + "solve_rate_wilson95", + "reward_variance", + "mixed_success", + ): + assert target[name] is None + assert target["usage_totals"] == {"input_tokens": None, "output_tokens": None} + assert target["difficulty"] == "unmeasured" + + +def test_missing_telemetry_is_explicit_and_does_not_invalidate_reward(evidence): + for record in evidence["trials"][-2:]: + del record["usage"] + del record["latency_seconds"] + report = assess_coding_audit(CONFIG, evidence) + assert report["accepted"] + target = report["checks"]["target_difficulty"] + assert target["telemetry_missing"] == { + "latency_seconds": 2, + "input_tokens": 2, + "output_tokens": 2, + } + assert target["latency_seconds_total"] is None + + +@pytest.mark.parametrize( + "field,value", + [ + ("latency_seconds", -1), + ("latency_seconds", None), + ("latency_seconds", True), + ("latency_seconds", float("inf")), + ("usage", None), + ("usage", {"input_tokens": True, "output_tokens": -5}), + ], +) +def test_supplied_malformed_telemetry_is_error_not_missing(evidence, field, value): + evidence["trials"][-1][field] = value + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "error" + target = report["checks"]["target_difficulty"] + assert target["status"] == "error" + assert target["telemetry_errors"] + assert all(value == 0 for value in target["telemetry_missing"].values()) + + +def test_latency_total_overflow_returns_error_report(evidence): + evidence["trials"].append(trial("target", "attempt-3", 1, 10)) + for record, seconds in zip(evidence["trials"][-3:], [10**308, 10**308, 0.0]): + record["latency_seconds"] = seconds + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "error" + assert report["checks"]["target_difficulty"]["latency_seconds_total"] is None + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize( + "reward", + [True, False, float("nan"), float("inf"), -float("inf"), None, "1", -0.01, 1.01], +) +@pytest.mark.parametrize("index", [0, 1, 3, 5]) +def test_invalid_rewards_block_all_roles(evidence, reward, index): + evidence["trials"][index]["reward"] = reward + report = assess_coding_audit(CONFIG, evidence) + assert not report["accepted"] + assert report["checks"]["reliability"]["status"] == "error" + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize("cleanup", [False, 1, "true", None]) +def test_cleanup_requires_literal_true(evidence, cleanup): + evidence["trials"][1]["cleanup_verified"] = cleanup + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +@pytest.mark.parametrize( + "field", + [ + "error", + "cleanup_verified", + "sandbox_id", + "grade_sandbox_id", + "submission_digest", + ], +) +def test_missing_trial_provenance_never_passes(evidence, field): + del evidence["trials"][1][field] + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +@pytest.mark.parametrize("field", ["case_id", "submission_digest"]) +@pytest.mark.parametrize("value", [[], {}, True]) +def test_malformed_reference_group_keys_return_error_without_crashing( + evidence, field, value +): + evidence["trials"][1][field] = value + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +def test_sha256_prefix_does_not_split_identical_reference_replays(evidence): + evidence["references"][0]["submission_digest"] = "sha256:" + "f" * 64 + evidence["trials"][1]["submission_digest"] = "sha256:" + "f" * 64 + evidence["trials"][0]["task_digest"] = "sha256:" + TASK + evidence["trials"][3]["verifier_digest"] = "sha256:" + VERIFIER + assert assess_coding_audit(CONFIG, evidence)["accepted"] + + +def test_unrepresentable_reward_variance_blocks_acceptance_without_nonfinite_json( + evidence, +): + config = CONFIG | {"reward_max": 1e308} + for record in evidence["trials"]: + if record["reward"] == 1: + record["reward"] = 1e308 + report = assess_coding_audit(config, evidence) + assert report["verdict"] == "error" + assert report["checks"]["target_difficulty"]["reward_variance"] is None + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize("index", [0, 1, 3, 5]) +@pytest.mark.parametrize( + "field,value", + [ + ("task_digest", "0" * 64), + ("verifier_digest", "1" * 64), + ("image_ref", "other@sha256:" + "2" * 64), + ("verifier_image_ref", "other@sha256:" + "3" * 64), + ], +) +def test_every_trial_binds_to_expected_task_verifier_and_images( + evidence, index, field, value +): + evidence["trials"][index][field] = value + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "error" + assert any( + "binding mismatch" in reason + for reason in report["checks"]["reliability"]["reasons"] + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("task_digest", "a" * 63), + ("verifier_digest", "b" * 65), + ("agent_image", "agent:latest"), + ("verifier_image", "sha256:abc"), + ], +) +def test_common_binding_requires_full_digests_and_immutable_images( + evidence, field, value +): + evidence[field] = value + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +@pytest.mark.parametrize( + "field,value", + [("model", "other/model"), ("revision", "a" * 40), ("provider", "other-provider")], +) +def test_target_identity_is_bound(evidence, field, value): + evidence["trials"][-1][field] = value + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["target_difficulty"]["status"] == "error" + assert report["checks"]["target_difficulty"]["completed"] == 1 + + +@pytest.mark.parametrize("revision", ["main", "abc", "a" * 39, None]) +def test_common_target_revision_must_be_immutable(evidence, revision): + evidence["target"]["revision"] = revision + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +def test_reference_replay_drift_blocks_acceptance(evidence): + evidence["trials"][2]["reward"] = 0 + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["references"]["status"] == "fail" + assert report["checks"]["reliability"]["status"] == "fail" + + +def test_separate_generated_submissions_are_not_deterministic_replays(evidence): + evidence["references"] = [ + { + "case_id": "oracle", + "submission_digest": digest * 64, + "origin": "generated", + "candidate_script_digest": digest * 64, + } + for digest in ("0", "1") + ] + evidence["trials"][1]["submission_digest"] = "0" * 64 + evidence["trials"][2]["submission_digest"] = "1" * 64 + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "skip" + assert report["checks"]["references"]["validated"] == [] + assert report["checks"]["reliability"]["status"] == "skip" + + +def test_clean_failed_generated_candidate_is_excluded_but_must_execute(evidence): + evidence["references"].append( + { + "case_id": "generated", + "submission_digest": "0" * 64, + "origin": "generated", + "candidate_script_digest": "1" * 64, + } + ) + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["references"]["status"] == "skip" + evidence["trials"].append(trial("reference", "generated", 0, 10, "0" * 64)) + report = assess_coding_audit(CONFIG, evidence) + assert report["accepted"] + assert report["checks"]["references"]["excluded"] == [ + {"case_id": "generated", "submission_digest": "0" * 64} + ] + evidence["trials"][-1]["error"] = "execution failed" + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +def test_generated_reference_can_be_validated_without_provided_oracle(evidence): + evidence["references"][0].update( + origin="generated", candidate_script_digest="1" * 64 + ) + assert assess_coding_audit(CONFIG, evidence)["accepted"] + del evidence["references"][0]["candidate_script_digest"] + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +@pytest.mark.parametrize("kind", ["nop", "wrong_answer", "contradictory_negative"]) +def test_verifier_must_reject_nop_and_every_observed_known_negative(evidence, kind): + if kind == "nop": + evidence["trials"][0]["reward"] = 1 + elif kind == "wrong_answer": + evidence["trials"][3]["reward"] = 1 + else: + evidence["trials"].append(trial("negative", "wrong_answer", 1, 10)) + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["verifier_rejection"]["status"] == "fail" + + +def test_generated_negative_labels_do_not_establish_trusted_coverage(evidence): + evidence["trials"][3]["case_id"] = "model_says_wrong" + evidence["required_negative_cases"] = ["model_says_wrong", "empty_answer"] + evidence["trials"][3]["reviewed"] = True + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["verifier_rejection"]["status"] == "skip" + assert report["checks"]["verifier_rejection"]["covered_negative_cases"] == [ + "empty_answer" + ] + + +@pytest.mark.parametrize("name", ["nop", "reference", "negative"]) +def test_missing_role_is_incomplete(evidence, name): + evidence["trials"] = [ + record for record in evidence["trials"] if record["role"] != name + ] + report = assess_coding_audit(CONFIG, evidence) + assert not report["accepted"] + assert report["verdict"] == "skip" + + +def test_floor_margin_is_required(evidence): + config = CONFIG | {"floor_margin": 0.95, "oracle_tolerance": 0.1} + evidence["trials"][0]["reward"] = 0.1 + report = assess_coding_audit(config, evidence) + assert report["checks"]["references"]["status"] == "fail" + + +def test_floor_gap_does_not_round_away_tiny_nop_reward(evidence): + config = CONFIG | {"oracle_tolerance": 1e-300, "floor_margin": 1} + evidence["trials"][0]["reward"] = 1e-300 + report = assess_coding_audit(config, evidence) + assert report["checks"]["verifier_rejection"]["status"] == "pass" + assert report["checks"]["references"]["status"] == "fail" + assert not report["accepted"] + + +def test_margin_must_fit_exact_reward_range(): + config = CONFIG | { + "reward_min": 1e-300, + "wrong_answer_max": 1e-300, + "floor_margin": 1, + } + with pytest.raises(ValueError): + validate_audit_config(config) + + +def test_zero_float_tolerance_preserves_large_integer_reference_and_solve_threshold( + evidence, +): + config = CONFIG | {"reward_max": 2**53 + 1, "oracle_tolerance": 0.0} + for record in evidence["trials"]: + if record["role"] in ("reference", "target"): + record["reward"] = 2**53 + report = assess_coding_audit(config, evidence) + assert report["checks"]["references"]["status"] == "fail" + target = report["checks"]["target_difficulty"] + assert target["solve_count"] == 0 + assert target["difficulty"] == "none_solved" + assert not report["accepted"] + + +def test_floor_separation_uses_decimal_values_without_float_subtraction_error(evidence): + config = CONFIG | { + "reward_min": 0.7, + "reward_max": 1, + "oracle_tolerance": 0.1, + "wrong_answer_max": 0.8, + "floor_margin": 0.1, + } + for record in evidence["trials"]: + record["reward"] = 0.9 if record["role"] == "reference" else 0.8 + assert assess_coding_audit(config, evidence)["accepted"] + + +@pytest.mark.parametrize( + "change,status", + [ + ("missing", "skip"), + ("unknown", "skip"), + ("duplicate", "error"), + ("false", "fail"), + ("truthy", "error"), + ("error", "error"), + ("missing_error", "error"), + ], +) +def test_leakage_requires_explicit_named_successful_probes(evidence, change, status): + if change == "missing": + evidence["leakage_probes"].pop() + elif change == "unknown": + evidence["leakage_probes"][-1]["name"] = "unknown_probe" + elif change == "duplicate": + evidence["leakage_probes"].append(dict(evidence["leakage_probes"][0])) + elif change == "false": + evidence["leakage_probes"][0]["passed"] = False + elif change == "truthy": + evidence["leakage_probes"][0]["passed"] = 1 + elif change == "error": + evidence["leakage_probes"][0]["error"] = "probe failed to execute" + else: + del evidence["leakage_probes"][0]["error"] + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["leakage"]["status"] == status + assert not report["accepted"] + + +def test_unknown_probe_cannot_replace_required_names_but_can_be_diagnostic(evidence): + evidence["leakage_probes"].append({"name": "extra", "passed": True, "error": None}) + report = assess_coding_audit(CONFIG, evidence) + assert report["accepted"] + assert "extra" in report["checks"]["leakage"]["observed_probes"] + + +@pytest.mark.parametrize( + "config", + [ + {}, + {"reference_repeats": 1}, + {"reference_repeats": True}, + {"target_attempts": 1}, + {"reward_min": True}, + {"reward_max": float("nan")}, + {"oracle_tolerance": 1}, + {"floor_margin": 0}, + {"floor_margin": float("inf")}, + {"wrong_answer_max": 1}, + {"required_negative_cases": []}, + {"required_negative_cases": ["a", "a"]}, + {"required_negative_cases": [1]}, + {"unknown": True}, + ], +) +def test_invalid_config_raises_and_assessment_returns_error(evidence, config): + config = config if not config else CONFIG | config + with pytest.raises(ValueError): + validate_audit_config(config) + report = assess_coding_audit(config, evidence) + assert report["verdict"] == "error" + assert not report["accepted"] + + +def test_empty_evidence_cannot_be_accepted(): + report = assess_coding_audit(CONFIG, {}) + assert not report["accepted"] + assert report["checks"]["leakage"]["status"] == "skip" + assert report["checks"]["references"]["status"] == "skip" + + +@pytest.mark.parametrize( + "field,value", + [ + ("trials", {}), + ("references", {}), + ("leakage_probes", {}), + ("trials", [None]), + ("references", [None]), + ], +) +def test_malformed_containers_return_error(evidence, field, value): + evidence[field] = value + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" From 85692d7818e3e4c7b08fcdfb583005416ab0383a Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 18:48:04 +0100 Subject: [PATCH 02/19] Add reproducible Hub environment validation pipelines --- .claude/docs/REPO_WALKTHROUGH.md | 5 + ...hub-environment-trainability-validation.md | 291 +++++ .../hub-environment-validation-pipeline.md | 189 +++ .../hub_validation_pipeline/CODING_AUDIT.md | 257 ++++ examples/hub_validation_pipeline/README.md | 197 +++ examples/hub_validation_pipeline/cloud.py | 1064 +++++++++++++++++ .../hub_validation_pipeline/contract.json | 18 + examples/hub_validation_pipeline/demo.py | 94 ++ examples/hub_validation_pipeline/full_run.py | 354 ++++++ examples/hub_validation_pipeline/pipeline.py | 832 +++++++++++++ .../hub_validation_pipeline/prepared_batch.py | 266 +++++ .../hub_validation_pipeline/real_tasks.py | 309 +++++ examples/hub_validation_pipeline/sources.py | 343 ++++++ examples/hub_validation_pipeline/spend.py | 288 +++++ .../hub_validation_pipeline/test_cloud.py | 754 ++++++++++++ .../hub_validation_pipeline/test_full_run.py | 818 +++++++++++++ .../hub_validation_pipeline/test_pipeline.py | 566 +++++++++ .../test_prepared_batch.py | 689 +++++++++++ .../test_real_tasks.py | 577 +++++++++ .../hub_validation_pipeline/test_sources.py | 411 +++++++ .../hub_validation_pipeline/test_spend.py | 714 +++++++++++ 21 files changed, 9036 insertions(+) create mode 100644 docs/plans/hub-environment-trainability-validation.md create mode 100644 docs/plans/hub-environment-validation-pipeline.md create mode 100644 examples/hub_validation_pipeline/CODING_AUDIT.md create mode 100644 examples/hub_validation_pipeline/README.md create mode 100644 examples/hub_validation_pipeline/cloud.py create mode 100644 examples/hub_validation_pipeline/contract.json create mode 100644 examples/hub_validation_pipeline/demo.py create mode 100644 examples/hub_validation_pipeline/full_run.py create mode 100644 examples/hub_validation_pipeline/pipeline.py create mode 100644 examples/hub_validation_pipeline/prepared_batch.py create mode 100644 examples/hub_validation_pipeline/real_tasks.py create mode 100644 examples/hub_validation_pipeline/sources.py create mode 100644 examples/hub_validation_pipeline/spend.py create mode 100644 examples/hub_validation_pipeline/test_cloud.py create mode 100644 examples/hub_validation_pipeline/test_full_run.py create mode 100644 examples/hub_validation_pipeline/test_pipeline.py create mode 100644 examples/hub_validation_pipeline/test_prepared_batch.py create mode 100644 examples/hub_validation_pipeline/test_real_tasks.py create mode 100644 examples/hub_validation_pipeline/test_sources.py create mode 100644 examples/hub_validation_pipeline/test_spend.py diff --git a/.claude/docs/REPO_WALKTHROUGH.md b/.claude/docs/REPO_WALKTHROUGH.md index 63548b51b..2f0067e7d 100644 --- a/.claude/docs/REPO_WALKTHROUGH.md +++ b/.claude/docs/REPO_WALKTHROUGH.md @@ -284,6 +284,11 @@ ThinkingBox examples: `example_usage.py` is a public-client smoke test, while `e ## Harbor integration +`examples/coding_audit/` owns the supplemental operator coding audit, its standalone +`python -m examples.coding_audit` CLI, and offline tests. It is separate from core +RFC 008 validation. `examples/hub_validation_pipeline/` prepares and submits its +frozen cloud bundles. + `src/openenv/core/harness/capture/` records inference calls independently of a trainer or tokenizer. `src/openenv/harbor/` runs Harbor tasks, reconciles their trajectories, and exposes the shared training contract through clients and the Gradio playground. `envs/harbor_env/` is the installable environment wrapper. `examples/harbor/nemo_shell_profile/` provides the explicitly qualified NeMo shell workflow. See `docs/source/guides/harbor-provider-qualification.md` for provider and adapter evidence, `rfcs/012-harbor-capture-providers.md` for explicit evaluation/training and session ownership, and `tests/envs/test_harbor*.py` / `test_capture*.py` for deterministic capture regressions. diff --git a/docs/plans/hub-environment-trainability-validation.md b/docs/plans/hub-environment-trainability-validation.md new file mode 100644 index 000000000..a2dfe5976 --- /dev/null +++ b/docs/plans/hub-environment-trainability-validation.md @@ -0,0 +1,291 @@ +# Selecting OpenEnv/Harbor environments for training + +This research plan separates native validation, supplemental audit evidence and +controlled learning experiments. Its scientific recommendations do not qualify +a private corpus, a provider or an OpenEnv package. + +Choose environments for **trustworthy rewards, useful policy-dependent difficulty, +diversity, and measured held-out learning per unit cost**. Generate missing solutions, +tests, and scaffolding as candidates, then validate them independently. A generated +solution passing a generated grader establishes agreement between those components; +it does not establish task correctness or trainability. + +## Current evidence boundary + +The [example README](../../examples/hub_validation_pipeline/README.md), +[`pipeline.py`](../../examples/hub_validation_pipeline/pipeline.py), and +[`harbor.py`](../../src/openenv/validation/harbor.py) define the native boundary: + +- `harbor-semantic-v1` requires `static.manifest`, `runtime.startup`, + `runtime.reward_well_formed`, `runtime.episode_determinism`, + `semantic.oracle_max` and `semantic.floor_gap` to PASS. Declared canaries also + require `semantic.canary_floor` to PASS. Any FAIL/ERROR, required SKIP or missing + required evidence blocks admission; other standard checks remain named SKIPs. +- Schema 2 native evidence binds fresh NOP, oracle, oracle-replay and declared + canary Trials to original/effective task digests, the same retained image and + inspected configuration, raw verifier rewards, successful oracle exits and + verified cleanup. Replay checks concern the same exact submission. +- The maximum, floor gap and canary outcomes are checked against an explicit + operator contract. They measure the frozen controls, not general specification + correctness. The complete `full` admission inventory remains unqualified. +- `evaluate()` requests a configured model attempt per admitted task. It does not + perform optimizer updates or a controlled multi-model training experiment. +- The native adapter rejects separate verifier environments, declared artifact + extensions and non-public/differing phase network policies. Upstream capability + alone does not qualify those contracts in this adapter. + +The supplemental `harbor-coding-audit-v1` implementation lives in +[`examples/coding_audit`](../../examples/coding_audit/) and is invoked with +`python -m examples.coding_audit --task TASK --config CONFIG --output REPORT`. +Cloud and discovery orchestration import this example package. It generates bounded +candidate references, tests operator-reviewed negatives, repeats verifier runs and +measures target attempts after screening. Its reports always retain +`rfc008_complete: false` and `trainability_established: false`. + +Real-task recipes require an explicit `--panel PATH`, including pinned source +identity, execution boundaries, full audit configuration and privately reviewed +counterexamples. The exact panel byte snapshot is hashed into preparation/bundle +provenance; native task/verifier files stay unchanged. Keep evaluation-only tasks, +private controls and populated results outside public source and training data. + +Keep semantic checks in the validator and corpus/experiment selection in examples. +[The #1591 contract](https://github.com/openenv/OpenEnv/pull/1591) versions the +native evidence; new trial roles or guarantees require a versioned extension. +[RFC 008](../../rfcs/008-environment-auto-validation.md) also names containment, +verifier determinism, leakage, canaries and replayability. Implemented IDs or a +supplemental PASS cannot substitute for complete evidence of those contracts. + +The example pins Harbor at `9c7a39b991d7e2205019f6be0598cbd22d84bfc3`. +Its Oracle agent executes a supplied solution script; its verifier executes +provided tests and reads their rewards. Those roles do not independently certify +the solution or verifier. [Pinned Oracle source][harbor-oracle], +[pinned verifier source][harbor-source] + +## What the research supports + +- **Verifier quality needs both acceptance and rejection controls.** SWE-bench's + original verification process reviews task clarity and tests that reject valid + implementations; its evaluation uses hidden issue-fixing and regression tests. + Human screening improves reliability without proving complete correctness. + [Original SWE-bench Verified methodology][verified] +- **Generated tests can find false positives, but are fallible.** UTBoost reports + 345 previously accepted erroneous patch evaluations exposed through test + augmentation **and parser corrections**. Its behavioral comparisons depend on + reference patches, so they do not independently prove specification completeness. + R2E-Gym separately documents low discrimination of generated tests and biases in + learned verifiers. [UTBoost][utboost], [R2E-Gym][r2e] +- **Public benchmark success needs contamination checks.** OpenAI's February 2026 + audit reports residual test-design defects and elicitable task/solution knowledge. + Its 59.4% defect figure concerns a selected set of 138 difficult instances, + not the full benchmark or arbitrary Harbor tasks. [2026 SWE-bench audit][audit] +- **Synthetic tasks have demonstrated training value in particular settings.** + SWE-smith generates bugs that break previously passing repository tests and + reports improved external benchmark performance after supervised fine-tuning. + SWE-Gym trains agents and outcome verifiers; its repeated-attempt selection gains + are distinct from single-attempt training gains. Neither result establishes + online RL benefit for this corpus. [SWE-smith][smith], [SWE-Gym][gym] +- **RL and self-evolving tasks have direct, bounded evidence.** DAPO reports math + RL results and filters uniform-reward groups to retain useful relative-advantage + signal; this does not establish coding transfer or universal cost savings. + Self-play SWE-RL reports coding RL gains on external benchmarks, validates + injected bugs through execution/inverse mutation checks, and restores oracle + tests before grading. It also discusses degenerate challengers and narrow + curricula: solve-rate targeting alone can be gamed. These are results for the + papers' recipes, not qualification of OpenEnv packages. [DAPO][dapo], + [Self-play SWE-RL, June 2026 revision][ssr] + +## Proposed stages, in priority order + +### 1. Establish the specification and a frozen verifier + +Select a small, relevant, training-permitted corpus. Record provenance, licenses, +repository/release, task lineage, bug/template family, and intended skills. Split +by repository **and task-generation/bug family before generating variants**; +near-duplicate prompts, related patches, and descendants of one source task stay +together. Keep an evaluation-only benchmark outside the training candidate pool. + +Review each task's behavioral requirements before using its reward. Assemble: + +- A known-bad starting state/NOP; known-good repairs checked against the requirements; + and an independently implemented valid alternative where feasible. Reject + implementation-specific assertions that the instructions do not justify. +- Issue-specific `FAIL_TO_PASS` checks plus `PASS_TO_PASS` regression coverage. + Validate required test discovery/counts and individual results; a zero-test run, + skipped required tests, swallowed collection error, or stale reward cannot pass. +- Hidden edge cases, property/metamorphic checks, and counterexamples justified by + the specification. Differential comparison with a reference implementation is a + useful signal; agreement with one potentially wrong reference is insufficient. + +Freeze the task specification, verifier source/image/dependencies, test inventory, +and reward aggregation before comparative rollouts. Repeat grading of the **same +submission** in fresh environments to measure verifier flakiness separately from +policy sampling variance. Test ordering, filesystem state, clock/seed dependence, +and parser behavior need controls. These recommendations extend the failure modes +documented in [Verified][verified] and [UTBoost][utboost]. + +### 2. Separate public scaffolding from private evaluation + +Use three explicit artifact classes: + +- **Solver-visible:** instructions, starter repository, dependencies, public examples + and development tests, generated API stubs/documentation, and tool scaffolding. +- **Training evaluator-private:** reference repairs, hidden reward tests, expected + outputs, counterexamples, grader configuration, and candidate-generation traces. + The policy may receive the declared reward/limited feedback, not these artifacts. +- **Sealed evaluation:** held-out task families and their independent verifier/tests; + inaccessible to training, task generators, candidate selection, and tuning. + +Harbor uploads shared-mode tests after the agent runs and has a separate-verifier +mode with explicit output artifact transfer. A clean verifier image can also avoid +runtime dependency installation. Qualify this adapter's support for that mode with +both agent and verifier image/configuration identities and run-owned cleanup. +Artifact contents remain untrusted. [Harbor verifier][harbor-verifier], +[separate verifier and artifact boundary][harbor-separate] + +For code execution, a separate container alone is insufficient: submitted code +must not run with the grader's authority to read answer keys or write rewards. +Use a restricted execution worker and a protected result collector, with only +declared submission files crossing the boundary. Probe copied answers, forged +reward files, test-runner/PATH/import shadowing, disabled assertions, symlink/path +escapes, background-process interference, and grader-file modification. Inspect +the solver image, mounts, history/caches, logs, network access, and captured +observations for solutions, generation prompts, upstream fixed commits, or private +test material. Public-network v1 cannot establish an offline leakage boundary. +Runtime isolation and pretraining contamination are separate concerns; hidden +files do not remove public-source memorization. [Harbor boundary][harbor-separate], +[contamination evidence][audit] + +### 3. Generate candidates and challenge them + +Prefer existing reviewed solutions. When one is missing, generate a bounded number +of candidate repairs from solver-visible inputs, and record generator/model revision, +prompt/input digests, visibility, seeds, dependencies, cost, and patch digest. Replay +each from the same clean start; inspect privileged oracle scripts so writing reward +files or modifying tests cannot masquerade as solving the task. Independently +check a passing repair before promoting it to a reference candidate. +Generated components create a derived task version: retain the parent source digest +and new content/component digests, then revalidate its bound artifacts. + +Generate counterexamples and task-relevant mutants: boundary-condition changes, +removed error handling, partial fixes, constant/hard-coded outputs, regressions, +and exploit submissions. Record which known-invalid behaviors the verifier rejects +and which valid alternatives it accepts. Review surviving mutants: equivalent or +out-of-scope mutations are not verifier failures. A mutation score is relative to +that panel, not a correctness probability. Search especially for submissions that +pass training rewards but fail the independent verifier. [UTBoost][utboost], +[SWE-smith][smith], [R2E-Gym][r2e] + +**Prevent circular validation:** a component author cannot certify their own +solution/grader pair. Generate challenger tests from requirements and development +failures, independently review their assertions, then freeze them. Do not rewrite +the grader until a generated solution passes or obtain expected outputs solely +from that solution. A different generator model is useful diversity, not sufficient +independence. Disagreements trigger specification review/quarantine and a new +version; never silently repair the evaluator during a comparison. Final held-out +feedback must not enter this generation loop. + +Start with generated wrappers/public scaffolds and candidate repairs; expand next +to validated mutants of working repositories. Try adaptive/self-evolving task +generation only after the static pipeline shows useful transfer. Keep generator +changes on training families, and audit diversity and exploitability as well as +difficulty. [SWE-smith][smith], [Self-play SWE-RL][ssr] + +### 4. Measure rollout difficulty, variance, and efficiency + +Run repeated attempts with the intended trainable policy and at least one other +model family. Pin model/checkpoint, harness, prompt/tool schemas, sampling settings, +seed where supported, task/verifier/image digests, token/step/time limits, hardware, +and concurrency. Randomize task/run ordering. Retain every attempt, including +failures and retries; provider seeds do not guarantee reproducibility. + +Report per task/model: single-attempt solve rate with uncertainty, bounded-budget +pass@k separately, reward distribution, mixed-success group frequency, solution +diversity, truncation, environment/verifier error rates, and disagreement with the +independent evaluator. Regrade identical submissions to isolate verifier noise. +Infrastructure errors stay explicit rather than becoming model reward zero. + +For binary rewards, `p(1-p)` describes Bernoulli reward variance; high variance +could reflect useful policy uncertainty **or a broken verifier**. Uniform-reward +groups have no relative reward advantage in GRPO-style updates, motivating DAPO's +dynamic sampling. Neither a tiny all-zero sample proves impossibility nor an +all-one sample proves the task useless to every policy. Select a mixture of +reliable easy/moderate/hard tasks; refresh difficulty as the policy changes. +[DAPO][dapo] + +Measure build/cold-start, reset, agent, and verification time separately, with +median/tail latency, tokens, sandbox/GPU time, cache/storage, retries, and human +review. Rank reliable, diverse tasks by useful rollout yield and total cost; +include discarded generations and failed builds. Cache immutable builds and +preinstall verifier dependencies, while keeping fresh episode state. Timeouts and +long-horizon skills deserve explicit strata rather than deletion for cost alone. +[Harbor separate verifier][harbor-separate] + +### 5. Demonstrate trainability in a controlled RL pilot + +Pre-register the split, metric, budgets, and comparisons. Use identical initialization, +harness, optimizer recipe, and inference budgets. Compare the unchanged base model, +RL on a reviewed reference curriculum, and RL on a cost-matched curriculum with +the candidate subset replacing part of that reference. A matched-data SFT control +helps distinguish RL benefit from imitation of generated solutions. Repeat training +seeds and use development families for checkpoint selection. + +Evaluate on the sealed family/repository split using the frozen independent +verifier. Report baseline-to-trained **pass@1 transfer**, uncertainty across seeds +and families, regressions, reward-versus-independent-score divergence, and total +cost/time. Cluster uncertainty by task family; repeated attempts on one task are +not independent new tasks. Report best-of-k selection separately so additional +inference spending cannot appear as a learning gain. These controls are proposed +experimental design, not a protocol established wholesale by the cited papers. + +Advance only with positive held-out transfer against the base and useful improvement +against the matched reference, within a declared cost budget. Report incremental +cost per additional held-out solve and performance at equal total spend, including +generation/qualification/training/evaluation. If uncertainty includes no gain, +report inconclusive; if only training reward rises, trainability remains unproven. +Evidence qualifies the tested model/task distribution/recipe, not an environment +universally. [SWE-Gym][gym], [Self-play SWE-RL][ssr] + +## Suggested starting budgets and implementation seam + +These are **engineering suggestions, not scientific acceptance thresholds**: + +- Screen about 20 tasks across several families. Try up to 2–3 repair candidates + for missing oracles, 3 fresh replays per control, and 10–20 relevant faulty mutants + per task. Expand targeted panels when survivors expose gaps. +- Begin with 2 models × 4 attempts × 20 tasks = 160 rollouts. On roughly 10 + finalists, target 3 models × 8 attempts = 240 observations, reusing compatible + screening runs. Small samples provide coarse estimates; increase repeats/tasks + according to observed uncertainty and cost. +- For an RL feasibility pilot, reserve disjoint training, development, and held-out + families; for example, 60–100 training tasks, 20 development tasks, and 40 held-out + tasks across multiple repositories. Start with 256–512 attempted training episodes + per RL arm/seed and 2 training seeds; expand seeds and held-out families before + making a broad claim. Fix an attempted-rollout/token/wall-clock ceiling so dynamic + sampling cannot create an unbounded budget. Fewer tasks/seeds justify weaker claims. + +Implement incrementally: first native determinism/containment/replay evidence and +separate-verifier support; then private artifact, mutation, and leakage audits; +then repeated `evaluate()` orchestration; finally an external training experiment +consuming the frozen corpus. Keep training-framework-specific machinery in examples. +Suggested sidecars are `verifier-audit.json`, `generated-candidates.jsonl`, +`rollout-matrix.jsonl`, and `trainability.json`, bound to corpus/profile/verifier +digests. Keep explicit exclusions and incomplete outcomes. + +The resulting evidence labels should distinguish **runtime sanity**, **audited +verifier**, **rollout-characterized**, and **training benefit demonstrated for a +specified experiment**. These are proposed research labels, not new RFC PASS +statuses. No combined quality score should let a cheap rollout or oracle pass +compensate for leakage, an unreliable verifier, or missing required validation. + +[verified]: https://openai.com/index/introducing-swe-bench-verified/ +[audit]: https://openai.com/index/why-we-no-longer-evaluate-swe-bench-verified/ +[utboost]: https://aclanthology.org/2025.acl-long.189.pdf +[r2e]: https://arxiv.org/html/2504.07164v1 +[smith]: https://arxiv.org/html/2504.21798v1 +[gym]: https://arxiv.org/html/2412.21139v2 +[dapo]: https://arxiv.org/html/2503.14476v1 +[ssr]: https://arxiv.org/html/2512.18552v3 +[harbor-verifier]: https://docs.harborframework.com/tasks/verifier +[harbor-separate]: https://docs.harborframework.com/tasks/separate-verifier +[harbor-oracle]: https://github.com/harbor-framework/harbor/blob/9c7a39b991d7e2205019f6be0598cbd22d84bfc3/src/harbor/agents/oracle.py +[harbor-source]: https://github.com/harbor-framework/harbor/blob/9c7a39b991d7e2205019f6be0598cbd22d84bfc3/src/harbor/verifier/verifier.py diff --git a/docs/plans/hub-environment-validation-pipeline.md b/docs/plans/hub-environment-validation-pipeline.md new file mode 100644 index 000000000..5c829fe2b --- /dev/null +++ b/docs/plans/hub-environment-validation-pipeline.md @@ -0,0 +1,189 @@ +# Hub discovery, native validation and model execution + +The runnable implementation is [the Hub validation example](../../examples/hub_validation_pipeline/README.md). +It acquires immutable Hugging Face or Harbor Hub task sources, invokes OpenEnv's +native validator, admits tasks against an explicit profile, and optionally runs a +configured model on a new corpus containing only admitted tasks. Source packages +and exclusions are retained. This document describes the reusable implementation +and its qualification boundary, without operational corpus or run records. + +Native semantic validation uses the `harbor-semantic-v1` contract and schema 2 +evidence introduced by [#1591](https://github.com/openenv/OpenEnv/pull/1591). +The [supplemental coding audit](../../examples/hub_validation_pipeline/CODING_AUDIT.md) +is example code in `examples/coding_audit`, with orchestration in +`examples/hub_validation_pipeline`. Neither a bounded native admission nor a +supplemental audit establishes complete RFC 008 coverage or training benefit. + +## Pipeline and ownership + +| Stage | Responsibility | Retained output | +| --- | --- | --- | +| Discover | Query both hubs, exhaust pagination and freeze candidate identities. | Candidate metadata and source aliases | +| Acquire | Resolve immutable identities and bound selected downloads. | Original task files, inventories and hashes | +| Enumerate | Identify task roots and preserve ordered canonical identities. | Per-task source records | +| Validate | Use OpenEnv's parser, native collector and standard graders. | JSON reports, raw evidence and checksums | +| Admit | Verify required PASS results and provenance for the selected profile. | Accepted/excluded records with reasons | +| Materialize | Copy admitted tasks with explicit image/resource overlays. | Frozen corpus and index-to-task mapping | +| Evaluate | Use the existing Harbor harness and capture runner. | Every attempted outcome and cleanup evidence | + +Core native validation remains in OpenEnv. Discovery, admission policy, cloud +launching, model selection, ratebooks and research experiments remain in examples. +A sibling sandbox or outer `harbor_env` server cannot establish properties of the +actual task sandbox. Every execution copy must bind to its acquired source and +validated image/configuration. + +## Acquisition and HF conventions + +HF discovery uses both `library:harbor` and `rl-environment`, checks those tags in +returned metadata, and consumes listing pagination before applying a candidate +limit. Trending rank selects discovery priority; it is not a stable task identity. +HF sources require full commit revisions and Harbor packages require full `sha256:` +references. Resolve a mutable HF alias once and record the resulting commit before +acquisition. Never infer byte identity from matching dataset names or task counts. + +The SDK's `snapshot_download`/`hf_hub_download` arguments correspond to +`hf download --repo-type dataset --revision ... --local-dir ...`. The example +adds anonymous public-source reads (`token=False`), explicit selected-file/task +bounds and immutable revision checks. It materializes regular files for Harbor +rather than relying on cache links. Gated, disabled, ambiguous or unsupported +sources are exclusions; they do not silently fall back to another registry. +Harbor's Git source pin is checked because a matching package version alone does +not establish that the required listing API exists. + +Check Git blob/content hashes and declared file sizes, reject escaping paths, +symlinks, special files and secret-file inputs, and retain the original acquired +inventory. Parsing/acquisition never executes downloaded task scripts on the +controller host. Source revisions do not pin container bases, package downloads +or external services; those need separate image/runtime evidence. + +HF publication uses explicit SDK add operations and retained immutable commit IDs. +It does not synchronize a local folder by deleting remote files. This follows the +`hf upload` convention that deletion is explicit. The cloud workflow additionally +requires an existing private result dataset and a fresh content-derived run claim; +ordinary upload success does not authorize duplicate paid execution. + +## Native semantic admission + +The default `full` profile requires the complete pinned static/runtime/semantic +inventory. Missing or skipped required coverage prevents full admission. +The explicitly selected `harbor-semantic-v1` profile requires: + +- `static.manifest`, `runtime.startup`, `runtime.reward_well_formed` and + `runtime.episode_determinism` to PASS. +- `semantic.oracle_max` and `semantic.floor_gap` to PASS, plus + `semantic.canary_floor` when the operator declares canaries. +- No FAIL or ERROR anywhere in the report; no duplicate, unknown or missing + required result, and execution of all required validation levels. +- Schema 2 native evidence for fresh NOP, oracle and oracle-replay Trials, + followed by the declared canaries, all with the same retained image and inspected + configuration, unchanged task identities and verified cleanup. +- Bound raw reward files and successful oracle exits. Recompute replay equality, + the declared oracle maximum, the NOP gap and canary floors from those files. + +Other standard checks remain named SKIPs and can leave the underlying report WARN. +The admission envelope records these limits beside the unchanged report; it does +not turn the report into full RFC qualification. Missing required evidence is +incomplete, not PASS, and any confirmed failure keeps FAIL precedence. + +The contract lives at `validation/openenv.json` in the validation subject and +explicitly declares the reward key/range, tolerance, floor margin, resource +allocation and optional canary directory. A pipeline-supplied overlay records +both the original source digest and the effective validation digest. The current +native adapter supports self-contained, single-step Linux tasks with one +Dockerfile, shared verifier execution and public networking. Unsupported resource, +credential, multi-service and extension contracts stay named SKIPs. + +```bash +uv run --no-sync openenv validate inputs/task --level semantic --policy v2 \ + --json --output outputs/task-report.json + +uv run --no-sync python -m examples.hub_validation_pipeline.pipeline \ + --contract inputs/native-contract.json --profile harbor-semantic-v1 \ + --hf-dataset 'org/repo@FULL_COMMIT_SHA' --tasks-per-dataset 1 \ + --output outputs/native-selection +``` + +Use [the README](../../examples/hub_validation_pipeline/README.md) for installation, +Docker requirements and the synthetic demo. The generic example contract is not +a substitute for reviewing another task's reward/resource/canary declarations. + +## Safe model handoff and evidence + +Build a new run-owned corpus from admitted tasks only. Record its ordered mapping +from transport index to canonical task ID; filtering must not change identities +or expose excluded siblings. Materialize only the validated immutable image and +explicit resource overlay. A trusted pre-execution Trial hook checks each task +copy, source digest, actual image and inspected container configuration. + +Pin the model/harness/backend and sampling, token, step and time limits. Hosted +model IDs and returned fingerprints describe available provider evidence; a +public source revision does not attest the weights served remotely. Keep inference +credentials outside validation workers and task arguments. + +Record reward maps, the selected headline metric, execution status, errors, +traces, timing and cleanup. A valid reward of zero is a model outcome. Missing +rewards, failed infrastructure and partial result sets remain ungraded/incomplete. +Reconcile expected and returned task IDs. Do not replace failures with zero or +delete their source/evidence. Clean up exact run-owned containers, networks and +volumes; retained image tags are recorded for explicit later removal. + +## Supplemental coding audit + +`examples/coding_audit/{scoring.py,hf.py,runner.py,__main__.py}` contains the +operator-run `harbor-coding-audit-v1` workflow. It generates bounded candidate +references, repeats exact submissions against fresh verifier instances, tests +reviewed known-wrong cases, probes limited leakage/stale-state behavior and then +measures repeated target attempts. Cloud and pipeline orchestration import this +example package. The controller command is: + +```bash +uv run --no-sync python -m examples.coding_audit \ + --task inputs/task --config inputs/audit.json --output outputs/audit-report.json +``` + +Reference solutions, verifier-private files, known-wrong submissions and their +counterexamples remain operator-controlled evaluation inputs. Real-task recipes +require `--panel PATH`; they contain no built-in benchmark selection or negative +panel. The operator panel declares a pinned HF source, task paths, immutable +execution boundaries and complete audit configuration. Its exact byte snapshot +is read once and SHA256-bound to preparation and bundle provenance. Acquisition +preserves every native task/verifier byte. + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.real_tasks \ + --panel "$PANEL_PATH" --output outputs/native-recipes --bundle + +uv run --no-sync python -m examples.hub_validation_pipeline.full_run \ + --panel "$PANEL_PATH" --hf-census "$HF_CENSUS_PATH" \ + --harbor-census "$HARBOR_CENSUS_PATH" --budget-usd "$BUDGET_USD" \ + --output outputs/native-plans +``` + +These commands prepare inputs without paid execution. Operator-selected +`--task NAME` values are validated against the supplied panel. A metadata census +is an inventory, not a deduplicated or runtime-qualified corpus. Evaluation-only +panels never become training data. + +Paid native-cohort submission additionally requires a shared initialized ledger, +a reservation identifier and `--submit`. The aggregate estimate reserves the +configured request/token ceilings, controller allowance and each child's remote +maximum lifetime before the first submission. The fixed ratebook is a declared +modeling assumption; it is neither a current-price guarantee nor an account-enforced +billing cap. Every initial budget-bearing plan parse and final submission checks +the same expected plan-byte SHA256. Partial or uncertain receipts and unresolved +reservations remain durable for manual reconciliation; launchers never blindly +resume or retry. + +## Qualification and research boundary + +Keep source-level tests, Linux Docker evidence and live-provider qualification +separate. Framework fakes exercise binding, parsing, incomplete outcomes, budget +races and cleanup accounting; they do not establish a provider's execution or +billing behavior. Supplemental reports always retain `rfc008_complete: false` +and `trainability_established: false`. + +Verifier acceptance/rejection controls measure the reviewed panel, not complete +specification correctness. Repeated attempts measure policy-dependent difficulty; +they do not demonstrate learning. Training value requires a separate controlled +experiment with training-permitted data, sealed held-out task families, matched +budgets and independent evaluation. See [the trainability research plan](hub-environment-trainability-validation.md). diff --git a/examples/hub_validation_pipeline/CODING_AUDIT.md b/examples/hub_validation_pipeline/CODING_AUDIT.md new file mode 100644 index 000000000..e18291ee8 --- /dev/null +++ b/examples/hub_validation_pipeline/CODING_AUDIT.md @@ -0,0 +1,257 @@ +# Reproducible supplemental coding audits + +The operator implementation lives in `examples/coding_audit`: `scoring.py` defines +reports/checks, `hf.py` adapts fresh HF Sandboxes, `runner.py` orchestrates attempts, +and `__main__.py` exposes the controller command. Cloud and discovery orchestration +in this directory import that example package. This workflow does not add a coding +mode to the installed `openenv validate` CLI. + +`cloud.py` freezes a Harbor task, operator audit configuration, required source and +`uv.lock` into a content-addressed bundle. Submission publishes it to an existing +private results dataset, records the immutable commit, and launches an HF Job +controller. Each solution execution and verifier run uses a fresh dedicated HF +Sandbox. Docker inside the controller Job is unnecessary. + +The supplemental profile is `harbor-coding-audit-v1`. Its PASS covers the measured +reference, rejection, leakage/reliability and rollout checks. Reports retain +`rfc008_complete: false` and `trainability_established: false`. OpenEnv's native +`harbor-semantic-v1` contract and the complete `full` admission requirements keep +their own meanings. + +## What runs + +1. Freeze/hash the task, verifier and operator configuration. Reject source drift, + mutable image references, unsafe artifact paths and excessive workload bounds. +2. Run NOP in a clean solver Sandbox and transfer only allowlisted submission file + bytes to a separate clean verifier Sandbox. +3. Generate bounded candidate shell solutions from solver-visible instructions. + Execute provided/generated candidates and repeat exact submissions in fresh + sandboxes. Clean incorrect generated candidates are excluded; execution and + cleanup errors remain errors. Never rewrite a verifier to make a candidate pass. +4. Execute the operator's reviewed known-wrong cases and require the frozen + verifier to reject them. The checked-in toy cases apply only to the synthetic + demo task. A generator's claim that its own solution is wrong does not establish + trusted negative coverage for another task. +5. Probe private-path visibility, stale solver state and a verifier-private canary + in collected transcripts. Check source/image binding and terminal Job status + after cleanup. +6. Only after screening passes, measure repeated target attempts. Retain successes, + failures, infrastructure errors, rewards, variance, mixed-success groups, + uncertainty, token usage and latency. Uniform success/failure is a measured + difficulty result, not proof of task invalidity or universal usefulness. + +A candidate passing a weak verifier does not establish intended correctness. +Demonstrating training benefit requires an independent controlled learning +experiment with held-out task families; see [the research plan](../../docs/plans/hub-environment-trainability-validation.md). + +## Model and protocol + +The synthetic configuration pins public source provenance to +[`Qwen/Qwen3.8-27B`](https://huggingface.co/Qwen/Qwen3.8-27B) at +`37561816e66b5b9153b1b64c10968c18c1e6f4a0`, with `deepinfra` explicitly selected. +Its attempt/candidate/replay and token/step limits are declared in +[config.json](../coding_audit/config.json). Real-task protocols come entirely from the +operator panel and must be reviewed independently. + +Model/provider availability is checked before task Sandbox creation. A source +commit does not force a provider to serve that revision. Evidence retains requested +and returned model identities and supplied fingerprints. Provider-specific seed +behavior is not assumed; recorded attempt seeds may be identifiers only. Frozen +inputs and the same declared protocol do not imply identical hosted generations. + +## Prepare a task and configuration + +Use a reviewed, self-contained Harbor task with `task.toml`, `instruction.md` and +`tests/test.sh`. A supplied `solution/solve.sh` is optional for the supplemental +audit because it can generate candidate references. This differs from native +semantic oracle admission. Multi-service, multi-step, credential-dependent or +unsupported resource/network contracts require a qualified adapter. + +Solver/verifier images must already exist under immutable OCI registry digests; +HF Sandbox has no Dockerfile build-context API. Build required images on a trusted +builder. Keep private tests, reference solutions and answer keys out of solver +images and public layers. The generic pinned Python image in the toy configuration +is not a qualified image for arbitrary downloaded tasks. + +Declare exact `submission_paths`, `private_paths`, verifier command, reward file/key, +reward range, trusted negatives and resource/time budgets. Submission paths name +individual files; directory/glob exports are unsupported. Only those bytes cross +into the fresh verifier. Set `cloud.results_repo` to an existing private dataset +under your control. + +Create the trusted synthetic controls without executing them: + +```bash +uv run --no-sync python -c \ + 'from pathlib import Path; from examples.hub_validation_pipeline.demo import create_tasks; create_tasks(Path("outputs/coding-controls"))' +``` + +Prepare a local bundle and reviewable launch plan; this performs no paid operations +or remote writes: + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.cloud prepare \ + --task outputs/coding-controls/good \ + --config examples/coding_audit/config.json \ + --output outputs/audit-plan +``` + +The plan records hashes, source provenance, dependency lock and conservative +Sandbox/inference workload bounds. Review `plan.json`, the configured protocol and +applicable prices before submission. Child command/start/idle deadlines do not +replace the SDK's 24-hour remote Job maximum lifetime. An interrupted controller +may leave an active child billable until shutdown; inspect/cancel exact run-owned +Jobs and retain cleanup evidence. + +To prepare discovered tasks, supply a configuration appropriate for the selected +family: + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.pipeline \ + --source both --dataset-limit 1 --tasks-per-dataset 1 \ + --coding-config inputs/reviewed-audit.json --output outputs/hub-coding-plans +``` + +This pipeline option selects supplemental cloud preparation; it is not an option +on `openenv validate`. The driver writes bundles and `planned.jsonl`; preparation +or a queued Job never marks a task accepted. Access/preparation failures remain +explicit exclusions. Explicit pinned dataset inputs or `--local-dataset` reproduce +the selection without relying on a later discovery ranking. + +Acquisition uses ordinary HF SDK revision/local-directory semantics, corresponding +to `hf download --repo-type dataset --revision ... --local-dir ...`, with anonymous +public reads and additional selection/size/hash checks. Publication uses explicit +SDK additions and commit identities. Like `hf upload`, it does not delete remote +files unless deletion is explicitly requested. + +## Operator-supplied native evaluation panels + +`real_tasks.py` and `full_run.py` require `--panel PATH`. Their Python entry points +likewise require the keyword `panel_path`. There is no embedded real-task list, +reference implementation, known-wrong submission or fallback private input path. +Keep panels, private task material and populated reports/ledgers in protected +operational storage; do not include their payloads in public source or PR prose. + +A panel has schema `openenv-native-evaluation-panel-v1`, `evaluation_only: true`, +a pinned HF `source` (`kind`, `repo_id`, full `revision`) and a nonempty `tasks` list. +Each task declares: + +- A unique safe `name` and package-relative `task_path`. +- An immutable `image` and exact `submission_path`, matching its configuration. +- Complete audit `config`, including reviewed negative cases and required IDs. +- Operator-reviewed `negative_counterexamples` metadata. +- Optional `annotations` for task-specific caveats or provenance. + +The panel's exact bytes are read once. Manifests record its path, byte length and +SHA256; each prepared bundle's source provenance includes the same panel SHA256. +Native task/verifier bytes remain unchanged and bind to the original acquired +file inventory. Configuration and annotations come from the operator input. + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.real_tasks \ + --panel "$PANEL_PATH" --output outputs/native-recipes --bundle + +uv run --no-sync python -m examples.hub_validation_pipeline.full_run \ + --panel "$PANEL_PATH" --hf-census "$HF_CENSUS_PATH" \ + --harbor-census "$HARBOR_CENSUS_PATH" --budget-usd "$BUDGET_USD" \ + --output outputs/native-cohort-plan +``` + +Set the variables to reviewed operator inputs. `full_run` freezes small census +summaries and hashes referenced inventories; census membership does not qualify +a task. Repeat `--task NAME` to select distinct names from the panel in that order; +omitting it selects all panel tasks. Empty, unknown and duplicate selections fail +before acquisition/spending. Native panels remain evaluation-only. + +## Shared spending and one-shot submission + +Initialize one shared ledger using your global allowance, without publishing its +contents. Set `SPEND_LEDGER` and `GLOBAL_BUDGET_USD` before this command: + +```bash +uv run --no-sync python -c \ + 'import os; from examples.hub_validation_pipeline import spend; spend.init(os.environ["SPEND_LEDGER"], os.environ["GLOBAL_BUDGET_USD"])' +``` + +Account for prior/concurrent spending and unresolved reservations in that ledger. +Use the same ledger across all cooperating launchers. For a native cohort, +`full_run(..., submit=True)` requires nonempty `spend_ledger` and `reservation_id` +before preparation; the CLI requires both flags: + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.full_run \ + --panel "$PANEL_PATH" --hf-census "$HF_CENSUS_PATH" \ + --harbor-census "$HARBOR_CENSUS_PATH" --budget-usd "$BUDGET_USD" \ + --spend-ledger "$SPEND_LEDGER" --reservation-id "$RESERVATION_ID" \ + --output outputs/native-cohort-submit --submit +``` + +The aggregate reservation covers configured request/token ceilings, controller +allowances and each child's full remote lifetime plus startup allowance. The +ratebook is a declared model, not a current-price guarantee or account billing cap; +unsupported model/provider/flavor changes require a reviewed ratebook. Both the +per-run cap and remaining shared allowance must cover the selected cohort. + +Every budget-bearing initial plan parse and final submission checks the expected +SHA256 against the same bytes it parses. A new directory or reservation identifier +does not bypass immutable run-ID duplicate protection. Partial/uncertain receipts +and unresolved reservations remain durable for manual reconciliation; never +release them merely because a controller stopped. Jobs are never automatically +resumed or blindly retried. + +`prepared_batch` provides shared reservation and bounded controller concurrency +for an already reviewed preparation manifest. A direct single-plan submission is: + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.cloud submit \ + --plan outputs/audit-plan/plan.json +``` + +Before using that lower-level command, reserve its reviewed cost in the shared +ledger. Its per-plan workload checks do not replace aggregate spending control. +The submitter verifies private dataset access, stages immutable inputs, claims the +run identity and persists the returned Job identifier and receipt. `prepare` never +makes model calls or launches paid Jobs. + +## Credentials and execution boundary + +Supply three distinct scoped credentials at runtime: `HF_TOKEN` for the private +input/result dataset, `HF_INFERENCE_TOKEN` for inference and `HF_SANDBOX_TOKEN` for +execution-only Jobs/Sandbox access. Keep their values out of configs, arguments, +bundles and Git. Submission uses HF Job secrets; they are not deliberately passed +into task environment variables or tool arguments. + +The current Sandbox RPC sends its execution bearer to the workload server. +Hostile root code could replace that server, so this backend is for reviewed +workloads; separate credentials limit authority without proving hostile-code +containment. A separate verifier Sandbox also does not establish that arbitrary +submitted code cannot read its grader or forge same-user output. Strong verifier +isolation requests are refused; these limits remain in reports. + +The worker retains reports, config, raw rewards, generated candidates, submission +hashes, sanitized transcripts and checksums in the private result dataset even +when an audit fails normally. Infrastructure termination can prevent final upload; +absent results remain incomplete. Keep original source and prior evidence intact. + +Run the same controller locally, with task execution still in HF Sandboxes: + +```bash +uv run --no-sync python -m examples.coding_audit \ + --task inputs/task --config inputs/audit.json --output outputs/audit-report.json +``` + +Missing evidence, malformed/ambiguous rewards, source/image drift, incorrect +provided references, accepted wrong answers, failed probes and unverified cleanup +block acceptance. Provider/Job failures never become synthetic zero-reward model +attempts. Checks retain PASS/FAIL/SKIP/ERROR and qualification limits. + +## Verification + +```bash +PYTHONPATH=.:src:envs uv run --no-sync python -m pytest \ + examples/hub_validation_pipeline/ -q +``` + +Injected SDK checks consume no cloud/inference credits and establish source-level +behavior. They do not establish Linux Docker, live-provider, billing or learning +qualification. A live audit needs separately authorized compute and scoped access. diff --git a/examples/hub_validation_pipeline/README.md b/examples/hub_validation_pipeline/README.md new file mode 100644 index 000000000..aa1571154 --- /dev/null +++ b/examples/hub_validation_pipeline/README.md @@ -0,0 +1,197 @@ +# Discover, validate and evaluate Harbor tasks + +This example discovers Hugging Face datasets tagged with both `library:harbor` +and `rl-environment`, discovers Harbor Hub packages, acquires pinned task files, +and calls OpenEnv's native validator for each task. It builds an admitted corpus +before optionally running a configured model through the Harbor capture runner. +The supplemental coding audit lives in [`examples/coding_audit`](../coding_audit/); +its configuration, cloud orchestration and selection policy remain example code. + +The native contract is `harbor-semantic-v1`, with schema 2 evidence from OpenEnv's +Harbor semantic validation support. See [RFC 008](../../rfcs/008-environment-auto-validation.md) +and the [pipeline plan](../../docs/plans/hub-environment-validation-pipeline.md) +for its boundaries. Neither corpus admission nor a supplemental coding-audit PASS +establishes full RFC 008 qualification or training benefit. + +## Install + +From the repository root, with Python 3.12+ and a Linux Docker daemon/Compose: + +```bash +uv sync --python 3.13 --group dev +uv pip install 'huggingface_hub==1.33.0' +uv pip install 'harbor @ git+https://github.com/harbor-framework/harbor.git@9c7a39b991d7e2205019f6be0598cbd22d84bfc3' +uv pip check +``` + +Use `uv run --no-sync` below to preserve that exact Harbor source installation. +The PyPI 0.25.0 wheel lacks the listing API used here; the example checks the +installed Git revision. Its cloud adapter targets `huggingface_hub==1.33.0`. + +## Run the synthetic Docker demo + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.demo \ + --output outputs/hub-pipeline-demo +``` + +The output directory must be new. The demo creates three trusted toy tasks and +runs native Harbor validation in Docker. The expected result is one admitted task +and two exclusions: an always-rewarding verifier and a missing reference solution. +The valid task has NOP reward 0 and oracle reward 1; its declared wrong-answer +canaries must remain at the measured floor. This command makes no model calls. +These controls are synthetic framework examples. + +## Choose the qualification profile + +`--profile full` is the default. It requires the complete versioned static, +runtime and semantic inventory to pass. Native Harbor coverage remains incomplete, +so missing or skipped required checks block full admission. + +Select `--profile harbor-semantic-v1` explicitly for the bounded native path. +Its required results are `static.manifest`, `runtime.startup`, +`runtime.reward_well_formed`, `runtime.episode_determinism`, `semantic.oracle_max` +and `semantic.floor_gap`, plus `semantic.canary_floor` when canaries are declared. +Every required check must PASS. Any FAIL or ERROR, duplicate or missing result, +or required SKIP excludes the task. Other standard checks remain named SKIPs; +the unchanged validator report may therefore retain a WARN verdict. + +The driver verifies schema 2 native evidence for fresh NOP, oracle and oracle-replay +Trials, followed by one Trial per declared canary. It checks exact task, retained +image and inspected configuration identities, raw verifier rewards, replay equality, +zero oracle exit codes and independently verified cleanup. It recomputes the +oracle maximum, floor gap and canary floors from the bound raw evidence. + +This establishes bounded executable/verifier sanity and exact-submission replay. +It does not establish complete semantic coverage, hostile-code containment, +resource enforcement, network conformance or general verifier correctness. + +Supply a trusted reward/resource contract with `--contract`; see +[contract.json](contract.json). Its binary reward range and canary directory are +appropriate for the demo. For another task, review the range/resources and provide +the declared canary scripts, or omit that optional declaration. The contract is an +operator input with a recorded digest. Direct CLI use reads it from +`validation/openenv.json` inside the task: + +```bash +uv run --no-sync openenv validate inputs/task --level semantic --policy v2 \ + --json --output outputs/task-report.json +``` + +The native adapter supports self-contained single-step Linux tasks with one +Dockerfile, shared verifier execution, public networking and no GPU, task-defined +environment variables, healthchecks, external MCP/credential requirements or +task-authored Compose. Unsupported shapes remain explicit SKIPs. + +## Search both hubs and validate a small selection + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.pipeline \ + --output outputs/hub-pipeline-search \ + --contract examples/hub_validation_pipeline/contract.json \ + --profile harbor-semantic-v1 \ + --source both --dataset-limit 1 --tasks-per-dataset 1 +``` + +Use a contract appropriate for the selected tasks. HF listing exhausts pagination +before limiting the recorded candidates and checks the exact tag conjunction. +HF repositories are pinned to full commits; Harbor dataset/task packages use full +`sha256:` references. Gated or disabled sources become access failures. Acquisition +preserves native task/verifier bytes and downloads only selected trees and required +shared assets; the runnable profile requires self-contained tasks. + +The example uses the HF SDK's `repo_type`, `revision`, `local_dir` and selection +arguments, corresponding to `hf download --repo-type dataset --revision ... +--local-dir ...`. It deliberately resolves mutable source aliases before execution, +uses anonymous public reads (`token=False`) and applies explicit byte/task limits. +Uploads use ordinary SDK commits with explicit additions; they do not mirror-delete +remote files. As with `hf upload`, deletion requires a separate explicit operation. + +Explicit inputs replace discovery. An HF repository without `@SHA` is resolved +once to its current commit; explicit Harbor input requires an immutable digest. + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.pipeline \ + --output outputs/hub-pipeline-selected \ + --contract inputs/native-contract.json \ + --profile harbor-semantic-v1 \ + --hf-dataset 'org/repo@FULL_COMMIT_SHA' \ + --harbor-dataset 'org/dataset@sha256:FULL_PACKAGE_DIGEST' \ + --tasks-per-dataset 2 +``` + +`--local-dataset inputs/dataset` accepts already acquired files without Hub access. +The driver copies tasks into run-owned subjects, writes the trusted contract there +and retains both original and effective digests. Failures are excluded per task. + +## Evaluate a model on the admitted corpus + +Add these options to a search or explicit/local-input invocation: + +```bash + --llm-url "$LLM_URL" --model "$MODEL_ID" \ + --harness opencode --model-timeout 120 --model-step-limit 32 +``` + +Supply the inference credential through `OPENENV_LLM_API_KEY`. Validation workers +receive Docker access and a small environment whitelist; inference and Hub +credentials are not forwarded. Model execution uses the existing capture proxy and +native harness. Default forwarding is `gradio`; `--expose direct` requires sandbox +access to the host capture endpoint. + +Admitted tasks are copied into a new corpus with an explicit image/resource overlay. +Before execution, a trusted Trial hook checks the selected task, source digest, +actual image and inspected container configuration. The driver rebuilds indices +and reconciles canonical task IDs; the model runner cannot enumerate excluded +siblings. Reward 0 is a graded model outcome. Model, capture or cleanup failures +remain unsuccessful/incomplete outcomes and never become fabricated zero rewards. + +## Outputs and cleanup + +Runs retain discovery/task/admission JSONL files, per-task JSON reports and sibling +artifact bundles, run-owned subjects, an approved corpus and `summary.json`. +Requested model runs additionally retain rollouts and native trial/capture evidence. +`harbor-evidence.json`, raw reward files and checksums bind native admission to +its task, image, contract and execution copies. + +Cleanup targets exact run-owned Docker containers, networks and volumes and is +independently rechecked. Validated images are retained for model handoff; their +exact tags are recorded in native evidence. Remove only those recorded tags with +`docker image rm ` when no longer needed. Sources and excluded +reports remain available; reruns use fresh output directories. + +## Supplemental coding audit and operator panels + +[CODING_AUDIT.md](CODING_AUDIT.md) describes generated candidate references, +operator-reviewed wrong-answer controls, fresh verifier replays, bounded leakage +probes and repeated model attempts. The local controller command is: + +```bash +uv run --no-sync python -m examples.coding_audit \ + --task inputs/task --config inputs/audit.json --output outputs/coding-report.json +``` + +This executes tasks in fresh HF Sandboxes. The `cloud` module prepares immutable +bundles and submits HF Job controllers. `real_tasks` and `full_run` additionally +require `--panel PATH`: no real corpus, reference implementation or negative panel +is embedded in their source. Keep operator panels and populated reports/ledgers +in private operational storage. + +The default model configuration records a public source revision and explicit +Inference Provider. Hosted inference does not attest an immutable served-weight +revision. Supplemental reports retain `rfc008_complete: false` and +`trainability_established: false`. + +## Focused verification + +```bash +PYTHONPATH=.:src:envs uv run --no-sync python -m pytest \ + examples/hub_validation_pipeline/ \ + tests/test_validation/test_harbor_validation.py \ + tests/envs/test_harbor_validated_trial_hook.py -q +``` + +Injected SDK tests consume no cloud/inference credits. They check source and +artifact binding, incomplete admission, raw rewards, source-mutation races, +model guards and shared-spend gates. Linux Docker execution and live provider +qualification remain separate verification steps. diff --git a/examples/hub_validation_pipeline/cloud.py b/examples/hub_validation_pipeline/cloud.py new file mode 100644 index 000000000..ddab8c0a0 --- /dev/null +++ b/examples/hub_validation_pipeline/cloud.py @@ -0,0 +1,1064 @@ +"""Prepare immutable coding audits locally, then explicitly submit one HF Job. + +Only ``submit`` writes to the Hub or launches compute. Jobs use three distinct +runtime secrets. Native Sandbox RPC exposes its execution bearer to root task +code; use reviewed tasks. A provider's served weights are not revision-attested. +""" + +import argparse +import hashlib +import inspect +import io +import json +import os +import re +import stat +import subprocess +import sys +import tempfile +import urllib.parse +import urllib.request +import uuid +import zipfile +from pathlib import Path, PurePosixPath + +WORKER_IMAGE = "docker.io/library/python@sha256:48d884813cf44e927cf64c5529dd630dd3b18338c6dc727991eef9b71e934a20" +UV_VERSION = "0.9.3" +SECRET_NAMES = ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN") +MAX_FILES = 5000 +MAX_FILE_BYTES = 16 * 1024 * 1024 +MAX_TOTAL_BYTES = 128 * 1024 * 1024 +MAX_ARCHIVE_BYTES = 160 * 1024 * 1024 +MANIFEST = "bundle-manifest.json" +LIMITATIONS = [ + "Reviewed tasks only: Native Sandbox root code can observe its execution RPC token.", + "Target revision records Hub source provenance; provider-served weights are not attested.", + "Worker image digest is a launch reference, not runtime attestation.", + "Hard termination or unavailable Hub can prevent final evidence upload.", + "This supplemental coding audit does not establish full RFC008 compliance or trainability.", +] +_EXCLUDED = {".git", "__pycache__", ".venv", ".worktrees", "outputs"} +_FORBIDDEN = { + ".env", + "env", + "secrets", + "credentials", + "logs", + "outputs", + "output", + "runs", +} + + +def _json_bytes(value): + from openenv.validation.runtime.json_values import dumps + + return (dumps(value) + "\n").encode() + + +def _loads(data): + from decimal import Decimal + + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("Duplicate JSON key") + result[key] = value + return result + + def invalid(value): + raise ValueError("Nonfinite JSON number") + + return json.loads( + data, object_pairs_hook=pairs, parse_constant=invalid, parse_float=Decimal + ) + + +def _safe_name(name): + if not isinstance(name, str): + raise ValueError("Unsafe bundle path") + path = PurePosixPath(name) + if ( + not path.parts + or path.is_absolute() + or path.as_posix() != name + or any(part in (".", "..") for part in path.parts) + or any(ord(char) < 32 for char in name) + or "\\" in name + or ":" in name + ): + raise ValueError("Unsafe bundle path") + return path + + +def _checked_archive(data, expected_sha=None): + """Check every archive member before writing or executing any payload.""" + if len(data) > MAX_ARCHIVE_BYTES: + raise ValueError("Archive size limit exceeded") + if expected_sha is not None and hashlib.sha256(data).hexdigest() != expected_sha: + raise ValueError("Bundle SHA256 mismatch") + with zipfile.ZipFile(io.BytesIO(data)) as archive: + members = archive.infolist() + if not 1 < len(members) <= MAX_FILES + 1: + raise ValueError("Bundle file count limit exceeded") + names = set() + folded = set() + total = 0 + for member in members: + name = str(_safe_name(member.filename)) + mode = member.external_attr >> 16 + if stat.S_IFMT(mode) != stat.S_IFREG or member.flag_bits & 1: + raise ValueError("Bundle members must be unencrypted regular files") + if name in names or name.casefold() in folded: + raise ValueError("Bundle path collision") + names.add(name) + folded.add(name.casefold()) + total += member.file_size + if member.file_size > MAX_FILE_BYTES or total > MAX_TOTAL_BYTES: + raise ValueError("Bundle size limit exceeded") + for name in names: + if any( + str(parent).casefold() in folded + for parent in PurePosixPath(name).parents + if str(parent) != "." + ): + raise ValueError("Bundle file/directory collision") + if MANIFEST not in names: + raise ValueError("Bundle inventory missing") + manifest = _loads(archive.read(MANIFEST)) + inventory = manifest["files"] + if not isinstance(inventory, dict) or set(inventory) != names - {MANIFEST}: + raise ValueError("Bundle inventory is incomplete") + files = {} + for member in members: + if member.filename == MANIFEST: + continue + content = archive.read(member) + record = inventory[member.filename] + if ( + not isinstance(record, dict) + or record.get("size") != len(content) + or record.get("sha256") != hashlib.sha256(content).hexdigest() + or record.get("mode") not in (0o644, 0o755) + or stat.S_IMODE(member.external_attr >> 16) != record["mode"] + ): + raise ValueError("Bundle inventory hash, size or mode mismatch") + files[member.filename] = content + files[MANIFEST] = archive.read(MANIFEST) + return manifest, files + + +def extract_bundle(data, destination, expected_sha): + """Extract a fully verified bundle into a fresh directory, without extractall.""" + manifest, files = _checked_archive(data, expected_sha) + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=False) + for name, content in files.items(): + path = destination.joinpath(*_safe_name(name).parts) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("xb") as stream: + stream.write(content) + path.chmod(manifest["files"].get(name, {}).get("mode", 0o644)) + return manifest + + +def _text_file(path): + mode = path.lstat().st_mode + if not stat.S_ISREG(mode): + raise ValueError(f"Only regular source files are allowed: {path.name}") + if path.stat().st_size > MAX_FILE_BYTES: + raise ValueError("Source file size limit exceeded") + with path.open("rb") as stream: + content = stream.read(MAX_FILE_BYTES + 1) + if len(content) > MAX_FILE_BYTES or b"\0" in content: + raise ValueError("Binary or oversized source file") + content.decode("utf-8") + _no_secrets(content) + return content, 0o755 if mode & 0o111 else 0o644 + + +def _no_secrets(content, secrets=()): + if re.search(rb"hf_[A-Za-z0-9]{16,}", content) or re.search( + rb"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----", content + ): + raise ValueError("Credential material is not allowed in persisted artifacts") + if any(secret and secret.encode() in content for secret in secrets): + raise ValueError("Runtime credential is not allowed in persisted artifacts") + for name in SECRET_NAMES: + value = os.environ.get(name) + if value and value.encode() in content: + raise ValueError("Runtime credential is not allowed in persisted artifacts") + + +def _tree(root, *, source=False): + if root.is_symlink() or not root.is_dir(): + raise ValueError("Source root must be a real directory") + result = {} + for directory, dirs, files in os.walk(root, followlinks=False): + parent = Path(directory) + excluded = _EXCLUDED | ({".pytest_cache", ".ruff_cache"} if source else set()) + dirs[:] = sorted(name for name in dirs if name not in excluded) + files = sorted(name for name in files if name not in excluded) + for name in dirs + files: + if (parent / name).is_symlink(): + raise ValueError("Symlinks are not allowed in bundles") + if any(name.lower() in _FORBIDDEN for name in dirs + files): + raise ValueError( + "Environment, secret, log or output files are not bundle inputs" + ) + for name in sorted(files): + path = parent / name + if not stat.S_ISREG(path.lstat().st_mode): + raise ValueError("Nonregular bundle input") + if source and name.endswith((".pyc", ".pyo")): + continue + relative = path.relative_to(root).as_posix() + if name.startswith(".env") or path.suffix.lower() in ( + ".pem", + ".key", + ".log", + ".zip", + ".tar", + ".gz", + ): + raise ValueError( + "Environment, secret, binary or log file is not a bundle input" + ) + if source and not ( + path.suffix == ".py" + or ( + path.suffix == ".json" + and any(part in ("schemas", "policies") for part in path.parts) + ) + ): + continue + result[relative] = _text_file(path) + if ( + len(result) > MAX_FILES + or sum(len(item[0]) for item in result.values()) > MAX_TOTAL_BYTES + ): + raise ValueError("Source tree size limit exceeded") + return result + + +def _positive(value, name, *, maximum=10000, minimum=1): + if type(value) is not int or not minimum <= value <= maximum: + raise ValueError(f"{name} must be an integer in [{minimum}, {maximum}]") + return value + + +def _check_acquisition_files(source, task_files): + """Bind a captured task to optional hashes verified during acquisition.""" + if "task_file_hashes" not in source: + return + expected = source["task_file_hashes"] + if ( + not isinstance(expected, dict) + or set(expected) != set(task_files) + or any( + not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value) + for value in expected.values() + ) + or any( + hashlib.sha256(data).hexdigest() != expected[name] + for name, data in task_files.items() + ) + ): + raise ValueError("Captured task files disagree with verified acquisition") + + +def _config(config): + from examples.coding_audit.runner import coding_budget, normalize_coding_config + + _no_secrets(_json_bytes(config)) + config = _loads(_json_bytes(config)) + cloud = config.setdefault("cloud", {}) + for key, value in { + "flavor": "cpu-basic", + "timeout_seconds": 1800, + "max_sandboxes": 40, + }.items(): + cloud.setdefault(key, value) + config = normalize_coding_config(config) + cloud = config["cloud"] + if "results_repo" not in cloud: + raise ValueError("Supply cloud.results_repo explicitly") + if not re.fullmatch(r"[\w.-]+/[\w.-]+", cloud["results_repo"]): + raise ValueError("cloud.results_repo must be org/repo") + if cloud["flavor"] != "cpu-basic": + raise ValueError("This bounded audit worker requires flavor cpu-basic") + if cloud.get("namespace") is not None and not re.fullmatch( + r"[\w.-]+", cloud["namespace"] + ): + raise ValueError("Invalid Job namespace") + _positive(cloud["timeout_seconds"], "timeout_seconds", maximum=86400) + budget = coding_budget(config, has_reference=True) + if ( + budget["sandbox_jobs_max"] > cloud["max_sandboxes"] + or budget["inference_calls_max"] > cloud["max_inference_calls"] + ): + raise ValueError( + "Audit upper bound exceeds max_sandboxes or max_inference_calls" + ) + budget.update( + job_timeout_seconds=cloud["timeout_seconds"], + max_sandboxes=cloud["max_sandboxes"], + max_inference_calls=cloud["max_inference_calls"], + output_token_upper_bound=config["generation"]["candidate_count"] + * config["generator"]["max_tokens"] + + len(config["seeds"]) + * config["target"]["max_steps"] + * config["target"]["max_tokens"], + cost_usd=None, + sandbox_remote_job_lifetime_seconds=86400, + concurrent_sandbox_jobs_max=1, + orphan_job_lifetime_note="SDK 1.33.0 hardcodes a 24-hour child Job lifetime; command/start/idle deadlines are shorter but cannot replace this server-side orphan cap. Collection stops after unverified cleanup.", + cost_note="Call/token counts bound requests, not current provider prices or input-token billing.", + ) + return config, budget + + +def _git_provenance(root): + def git(*args): + return ( + subprocess.check_output( + ["git", "-C", str(root), *args], stderr=subprocess.DEVNULL + ) + .decode() + .strip() + ) + + return { + "commit": git("rev-parse", "HEAD"), + "dirty": bool(git("status", "--porcelain", "--untracked-files=normal")), + } + + +class _HubRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + if urllib.parse.urlsplit(newurl).scheme != "https": + raise ValueError("Hub download redirect requires HTTPS") + redirected = super().redirect_request(req, fp, code, msg, headers, newurl) + if ( + urllib.parse.urlsplit(req.full_url).netloc + != urllib.parse.urlsplit(newurl).netloc + ): + redirected.remove_header("Authorization") + return redirected + + +def _bootstrap_main(uv_version): + import base64 + import tempfile + + revision = os.environ["OPENENV_BUNDLE_REVISION"] + sha = os.environ["OPENENV_BUNDLE_SHA256"] + repo = os.environ["OPENENV_RESULTS_REPO"] + token = os.environ["HF_TOKEN"] + if ( + not re.fullmatch(r"[0-9a-f]{40}", revision) + or not re.fullmatch(r"[0-9a-f]{64}", sha) + or not re.fullmatch(r"[\w.-]+/[\w.-]+", repo) + ): + raise ValueError("Invalid immutable input identity") + opener = urllib.request.build_opener(_HubRedirect()) + headers = {"Authorization": "Bearer " + token} + stage = "download" + try: + url = ( + "https://huggingface.co/datasets/" + + repo + + "/resolve/" + + revision + + "/inputs/" + + sha + + "/bundle.zip" + ) + with opener.open( + urllib.request.Request(url, headers=headers), timeout=120 + ) as response: + data = response.read(MAX_ARCHIVE_BYTES + 1) + root = Path(tempfile.mkdtemp(prefix="openenv-coding-")) / "bundle" + stage = "integrity" + extract_bundle(data, root, sha) + stage = "install_uv" + subprocess.run( + [sys.executable, "-m", "pip", "install", "uv==" + uv_version], check=True + ) + stage = "sync" + subprocess.run(["uv", "sync", "--frozen", "--no-dev"], cwd=root, check=True) + stage = "worker" + subprocess.run( + [ + str(root / ".venv/bin/python"), + "-m", + "examples.hub_validation_pipeline.cloud", + "worker", + "--root", + str(root), + "--bundle-sha", + sha, + ], + cwd=root, + check=True, + ) + except BaseException as error: + # Stdlib fallback also covers dependency failures before HfApi is installed. + report = { + "status": "error", + "accepted": False, + "stage": stage, + "error_type": type(error).__name__, + "bundle_sha256": sha, + } + try: + info_url = "https://huggingface.co/api/datasets/" + repo + with opener.open( + urllib.request.Request(info_url, headers=headers), timeout=30 + ) as response: + info = _loads(response.read(1024 * 1024)) + if info.get("private") is not True: + raise ValueError("Results repository is not private") + records = [ + { + "key": "header", + "value": { + "summary": "Persist coding bootstrap error", + "description": "", + }, + }, + { + "key": "file", + "value": { + "path": "runs/coding-" + sha + "/bootstrap-error.json", + "encoding": "base64", + "content": base64.b64encode( + json.dumps(report).encode() + ).decode(), + }, + }, + ] + payload = "".join(json.dumps(row) + "\n" for row in records).encode() + request = urllib.request.Request( + info_url + "/commit/main", + data=payload, + headers={**headers, "Content-Type": "application/x-ndjson"}, + method="POST", + ) + with opener.open(request, timeout=60) as response: + response.read(1024 * 1024) + except Exception: + print("Bootstrap error evidence upload failed", file=sys.stderr) + raise RuntimeError( + "Coding bootstrap failed during " + stage + "; inspect runs/coding-" + sha + ) from None + + +def _bootstrap_code(): + # Reuse the exact stdlib-only archive validator in the remote bootstrap. + header = "import hashlib, io, json, os, re, stat, subprocess, sys, urllib.parse, urllib.request, zipfile\nfrom pathlib import Path, PurePosixPath\n" + for name in ( + "MAX_FILES", + "MAX_FILE_BYTES", + "MAX_TOTAL_BYTES", + "MAX_ARCHIVE_BYTES", + "MANIFEST", + ): + header += f"{name} = {globals()[name]!r}\n" + header += "\n".join( + inspect.getsource(function) + for function in ( + _loads, + _safe_name, + _checked_archive, + extract_bundle, + _HubRedirect, + _bootstrap_main, + ) + ) + return header + f"\n_bootstrap_main({UV_VERSION!r})\n" + + +def _launch_spec(config, digest, revision=None): + cloud = config["cloud"] + env = { + "OPENENV_RESULTS_REPO": cloud["results_repo"], + "OPENENV_BUNDLE_SHA256": digest, + "PYTHONUNBUFFERED": "1", + } + if revision is not None: + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ValueError("Input upload did not return a full commit SHA") + env["OPENENV_BUNDLE_REVISION"] = revision + spec = { + "image": WORKER_IMAGE, + "command": ["python", "-c", _bootstrap_code()], + "env": env, + "flavor": cloud["flavor"], + "timeout": f"{cloud['timeout_seconds']}s", + "labels": {"openenv_bundle": digest, "openenv_profile": "coding-audit-v1"}, + } + if cloud.get("namespace"): + spec["namespace"] = cloud["namespace"] + return spec + + +def prepare(task_root, config, output_dir, *, repo_root=None, source=None): + """Write a deterministic bundle and reviewable launch plan without network IO.""" + repo_root = Path(repo_root or Path(__file__).resolve().parents[2]) + task_root, output_dir = Path(task_root), Path(output_dir) + if output_dir.exists() or output_dir.is_symlink(): + raise FileExistsError("Prepare requires a fresh output directory") + if output_dir.resolve().is_relative_to(task_root.resolve()): + raise ValueError("Output directory must be outside the task") + config, budget = _config(config) + from examples.coding_audit.runner import validate_coding_task + + validate_coding_task(task_root, config) + files = {} + for name in ("pyproject.toml", "README.md", "LICENSE", "uv.lock"): + files[name] = _text_file(repo_root / name) + files.update( + { + "src/openenv/" + name: value + for name, value in _tree(repo_root / "src/openenv", source=True).items() + } + ) + audit_files = { + name: value + for name, value in _tree( + repo_root / "examples/coding_audit", source=True + ).items() + if not PurePosixPath(name).name.startswith("test_") + } + if ( + not {"__init__.py", "__main__.py", "scoring.py", "hf.py", "runner.py"} + <= audit_files.keys() + ): + raise ValueError("Operator coding-audit production package is incomplete") + files.update( + {"examples/coding_audit/" + name: value for name, value in audit_files.items()} + ) + worker_path = "examples/hub_validation_pipeline/cloud.py" + files[worker_path] = _text_file(repo_root / worker_path) + for name in ( + "full_run.py", + "real_tasks.py", + "sources.py", + "spend.py", + ): + relative = "examples/hub_validation_pipeline/" + name + if (repo_root / relative).exists(): + files[relative] = _text_file(repo_root / relative) + task_files = _tree(task_root) + if not task_files or "task.toml" not in task_files: + raise ValueError("Local task must contain task.toml") + # Check the exact captured bytes: a live source can change and be restored + # between its initial preflight and inventory collection. + with tempfile.TemporaryDirectory(prefix="openenv-cloud-preflight-") as temporary: + frozen = Path(temporary) + for name, (content, mode) in task_files.items(): + path = frozen / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + path.chmod(mode) + validate_coding_task(frozen, config) + files.update({"task/" + name: value for name, value in task_files.items()}) + files["coding-config.json"] = (_json_bytes(config), 0o644) + if source is None: + source = { + "kind": "local", + "identity": "sha256:" + + hashlib.sha256( + _json_bytes( + { + name: hashlib.sha256(value[0]).hexdigest() + for name, value in task_files.items() + } + ) + ).hexdigest(), + } + else: + source = dict(source) + kind = source.get("kind", source.get("source")) + if ( + source.get("kind") + and source.get("source") + and source["kind"] != source["source"] + ): + raise ValueError("Source kind declarations disagree") + revision = source.get("revision", "") + patterns = { + "hf": r"[0-9a-f]{40}", + "harbor": r"sha256:[0-9a-f]{64}", + "local": r"[0-9a-f]{64}", + } + if ( + kind not in patterns + or not isinstance(revision, str) + or not re.fullmatch(patterns[kind], revision) + ): + raise ValueError("Dataset source requires its full immutable revision") + if source.get("task_path") != ".": + _safe_name(source["task_path"]) + if kind == "local": + from openenv.validation.runner import source_digest + + if source_digest(task_root) != revision: + raise ValueError("Local source revision differs from the prepared task") + source["kind"] = kind + _check_acquisition_files( + source, {name: value[0] for name, value in task_files.items()} + ) + manifest = { + "schema": "openenv-coding-bundle-v1", + "git": _git_provenance(repo_root), + "task_source": source, + "target_source": { + name: config["target"][name] + for name in ("model_id", "revision", "provider") + }, + "files": { + name: { + "size": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + "mode": mode, + } + for name, (content, mode) in sorted(files.items()) + }, + "inventory_note": "Every payload file is inventoried; the outer ZIP digest also binds this manifest.", + } + files[MANIFEST] = (_json_bytes(manifest), 0o644) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_STORED) as archive: + for name, (content, mode) in sorted(files.items()): + member = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + member.create_system = 3 + member.external_attr = (stat.S_IFREG | mode) << 16 + archive.writestr(member, content) + data = buffer.getvalue() + digest = hashlib.sha256(data).hexdigest() + _checked_archive(data, digest) + plan = { + "schema": "openenv-coding-cloud-v1", + "bundle": "bundle.zip", + "bundle_sha256": digest, + "run_id": "coding-" + digest, + "results_repo": config["cloud"]["results_repo"], + "manifest_sha256": hashlib.sha256(files[MANIFEST][0]).hexdigest(), + "git": manifest["git"], + "task_source": source, + "target_source": manifest["target_source"], + "budget": budget, + "secret_names": list(SECRET_NAMES), + "mcp_request": { + "operation": "run", + "args": _launch_spec(config, digest), + "required_env_after_upload": ["OPENENV_BUNDLE_REVISION"], + "secret_names": list(SECRET_NAMES), + }, + "limitations": LIMITATIONS, + } + output_dir.mkdir(parents=True, exist_ok=False) + (output_dir / "bundle.zip").write_bytes(data) + (output_dir / "plan.json").write_bytes(_json_bytes(plan)) + return plan + + +def _read_plan(plan_path, *, expected_sha256=None): + plan_path = Path(plan_path) + plan_bytes = plan_path.read_bytes() + if expected_sha256 is not None and ( + not isinstance(expected_sha256, str) + or not re.fullmatch(r"[0-9a-f]{64}", expected_sha256) + or hashlib.sha256(plan_bytes).hexdigest() != expected_sha256 + ): + raise ValueError("Prepared plan changed before submission") + plan = _loads(plan_bytes) + if ( + plan.get("schema") != "openenv-coding-cloud-v1" + or plan.get("bundle") != "bundle.zip" + ): + raise ValueError("Unsupported cloud plan") + data = _read_bundle(plan_path.parent / "bundle.zip") + manifest, files = _checked_archive(data, plan["bundle_sha256"]) + _check_acquisition_files( + manifest["task_source"], + { + name.removeprefix("task/"): value + for name, value in files.items() + if name.startswith("task/") + }, + ) + config, budget = _config(_loads(files["coding-config.json"])) + if ( + plan["run_id"] != "coding-" + plan["bundle_sha256"] + or plan["results_repo"] != config["cloud"]["results_repo"] + or plan["budget"] != budget + or plan["manifest_sha256"] != hashlib.sha256(files[MANIFEST]).hexdigest() + or any( + plan.get(name) != manifest.get(name) + for name in ("git", "task_source", "target_source") + ) + or manifest["target_source"] + != { + name: config["target"][name] + for name in ("model_id", "revision", "provider") + } + or plan["mcp_request"]["args"] != _launch_spec(config, plan["bundle_sha256"]) + ): + raise ValueError("Plan does not match its immutable bundle") + return plan, config, data, manifest + + +def _read_bundle(path): + if ( + not stat.S_ISREG(path.lstat().st_mode) + or path.stat().st_size > MAX_ARCHIVE_BYTES + ): + raise ValueError("Bundle must be a bounded regular file") + with path.open("rb") as stream: + return stream.read(MAX_ARCHIVE_BYTES + 1) + + +def _runtime_secrets(supplied=None): + from huggingface_hub import get_token + + values = ( + dict(supplied) + if supplied is not None + else { + "HF_TOKEN": get_token(), + **{name: os.environ.get(name) for name in SECRET_NAMES[1:]}, + } + ) + if set(values) != set(SECRET_NAMES) or any( + not isinstance(values.get(name), str) + or not values[name] + or values[name].startswith("$") + for name in SECRET_NAMES + ): + raise ValueError( + "Provide HF_TOKEN, HF_INFERENCE_TOKEN and HF_SANDBOX_TOKEN as runtime secrets" + ) + if len(set(values.values())) != len(SECRET_NAMES): + raise ValueError("All three runtime secret values must be distinct") + return values + + +def _private_repo(api, repo, token): + try: + info = api.dataset_info(repo, token=token) + except Exception: + raise RuntimeError( + "Results dataset must already exist and be accessible with HF_TOKEN; create a private dataset before submission" + ) from None + if ( + info.private is not True + or not isinstance(info.sha, str) + or not re.fullmatch(r"[0-9a-f]{40}", info.sha) + ): + raise ValueError( + "Results dataset must already be private with a known commit SHA" + ) + return info + + +def _write_receipt(path, value): + temporary = path.with_suffix(".tmp") + temporary.write_bytes(_json_bytes(value)) + temporary.replace(path) + + +def stage_inputs(plan_path, *, api=None, secrets=None, expected_plan_sha256=None): + """Claim a run and upload inputs; return exact launch kwargs without secrets. + + This writes to the private Hub, but does not launch compute. A stale/uncertain + receipt requires manual reconciliation. Never automatically retry a launch. + An expected plan SHA binds the parsed bytes to the caller's reviewed budget. + """ + from huggingface_hub import CommitOperationAdd, HfApi + + plan_path = Path(plan_path) + plan, config, data, manifest = _read_plan( + plan_path, expected_sha256=expected_plan_sha256 + ) + secrets = _runtime_secrets(secrets) + _no_secrets(data, secrets.values()) + api = api or HfApi(token=secrets["HF_TOKEN"]) + info = _private_repo(api, plan["results_repo"], secrets["HF_TOKEN"]) + receipt_path = plan_path.parent / "submission.json" + receipt = { + "run_id": plan["run_id"], + "status": "reserved", + "claim_nonce": uuid.uuid4().hex, + } + try: + with receipt_path.open("xb") as stream: + stream.write(_json_bytes(receipt)) + except FileExistsError: + raise FileExistsError( + "Submission receipt exists; reuse/review existing evidence explicitly instead of rerunning" + ) from None + prefix = "runs/" + plan["run_id"] + "/" + try: + names = api.list_repo_files( + plan["results_repo"], + repo_type="dataset", + revision=info.sha, + token=secrets["HF_TOKEN"], + ) + if any(name.startswith(prefix) for name in names): + raise FileExistsError( + "Run already claimed on Hub; reuse existing evidence explicitly" + ) + commit = api.create_commit( + repo_id=plan["results_repo"], + repo_type="dataset", + revision="main", + create_pr=False, + parent_commit=info.sha, + token=secrets["HF_TOKEN"], + commit_message="Claim immutable coding audit " + plan["run_id"], + operations=[ + CommitOperationAdd( + path_in_repo="inputs/" + plan["bundle_sha256"] + "/bundle.zip", + path_or_fileobj=data, + ), + CommitOperationAdd( + path_in_repo="inputs/" + plan["bundle_sha256"] + "/manifest.json", + path_or_fileobj=_json_bytes(manifest), + ), + CommitOperationAdd( + path_in_repo=prefix + "claim.json", + path_or_fileobj=_json_bytes(receipt), + ), + ], + ) + spec = _launch_spec(config, plan["bundle_sha256"], commit.oid) + receipt.update( + status="inputs_uploaded", + input_revision=commit.oid, + launch_spec=spec, + secret_names=list(SECRET_NAMES), + ) + _write_receipt(receipt_path, receipt) + (plan_path.parent / "mcp-request.json").write_bytes( + _json_bytes( + {"operation": "run", "args": spec, "secret_names": list(SECRET_NAMES)} + ) + ) + except Exception as error: + receipt.update(status="staging_failed", error_type=type(error).__name__) + _write_receipt(receipt_path, receipt) + raise + return receipt + + +def submit(plan_path, *, api=None, secrets=None, expected_plan_sha256=None): + """Upload/claim immutable inputs and launch once, resolving actual secrets.""" + from huggingface_hub import HfApi + + secrets = _runtime_secrets(secrets) + api = api or HfApi(token=secrets["HF_TOKEN"]) + receipt = stage_inputs( + plan_path, + api=api, + secrets=secrets, + expected_plan_sha256=expected_plan_sha256, + ) + receipt_path = Path(plan_path).parent / "submission.json" + receipt["status"] = "launch_pending" + _write_receipt(receipt_path, receipt) + try: + job = api.run_job( + **receipt["launch_spec"], secrets=secrets, token=secrets["HF_SANDBOX_TOKEN"] + ) + except Exception as error: + receipt.update(status="launch_unknown", error_type=type(error).__name__) + _write_receipt(receipt_path, receipt) + raise RuntimeError( + "Job launch outcome unknown; inspect the submission receipt and HF Jobs before any manual retry" + ) from None + receipt.update(status="submitted", job_id=job.id, job_url=job.url) + _write_receipt(receipt_path, receipt) + try: + api.upload_file( + repo_id=receipt["launch_spec"]["env"]["OPENENV_RESULTS_REPO"], + repo_type="dataset", + path_in_repo="runs/" + receipt["run_id"] + "/submission.json", + path_or_fileobj=_json_bytes(receipt), + token=secrets["HF_TOKEN"], + commit_message="Record coding Job " + job.id, + ) + except Exception as error: + receipt.update( + receipt_sync_status="error", receipt_sync_error_type=type(error).__name__ + ) + _write_receipt(receipt_path, receipt) + return receipt + + +def worker(root, *, bundle_sha=None, api=None, runner=None): + """Run the bundled audit and persist report/evidence even after runner errors.""" + from huggingface_hub import HfApi + + root = Path(root) + secrets = _runtime_secrets() + api = api or HfApi(token=secrets["HF_TOKEN"]) + digest = bundle_sha or os.environ.get("OPENENV_BUNDLE_SHA256", "") + if not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ValueError("Worker requires bundle SHA256") + output = root / "worker-output" + output.mkdir(exist_ok=False) + run_id = "coding-" + digest + # Repo identity comes from bootstrap env so invalid config still has a sink. + repo = os.environ.get("OPENENV_RESULTS_REPO") + report = { + "status": "error", + "accepted": False, + "run_id": run_id, + "bundle_sha256": digest, + "limitations": LIMITATIONS, + } + failure = None + try: + manifest = _loads((root / MANIFEST).read_bytes()) + for name, record in manifest["files"].items(): + path = root.joinpath(*_safe_name(name).parts) + if any( + parent.is_symlink() + for parent in [ + path, + *(root / parent for parent in path.relative_to(root).parents), + ] + ): + raise ValueError("Worker payload contains a symlink") + data, mode = _text_file(path) + if ( + hashlib.sha256(data).hexdigest() != record["sha256"] + or len(data) != record["size"] + or mode != record["mode"] + ): + raise ValueError("Worker inventory mismatch") + config, budget = _config(_loads((root / "coding-config.json").read_bytes())) + if repo != config["cloud"]["results_repo"]: + raise ValueError("Worker results repository disagrees with launch") + _private_repo(api, repo, secrets["HF_TOKEN"]) + if runner is None: + from examples.coding_audit.runner import run_coding_validation + + runner = run_coding_validation + result = runner(root / "task", config, output / "audit") + if not isinstance(result, dict): + raise TypeError("Coding runner must return a report object") + report.update( + status="completed", + report=result, + budget=budget, + git=manifest["git"], + task_source=manifest["task_source"], + target_source=manifest["target_source"], + ) + report["accepted"] = result.get("accepted") is True + except BaseException as error: + failure = error + report.update(status="error", accepted=False, error_type=type(error).__name__) + # SDK exceptions can contain credentials; persist type, never raw repr. + finally: + (output / "cloud-report.json").write_bytes(_json_bytes(report)) + try: + _private_repo(api, repo, secrets["HF_TOKEN"]) + # A malformed artifact must not prevent persistence of the error report. + try: + evidence = _tree(output) + for data, _ in evidence.values(): + _no_secrets(data, secrets.values()) + except Exception as error: + failure = error + report.pop("report", None) + report.update( + status="error", + accepted=False, + evidence_collection_error_type=type(error).__name__, + evidence_omitted=True, + ) + report_data = _json_bytes(report) + (output / "cloud-report.json").write_bytes(report_data) + evidence = {"cloud-report.json": (report_data, 0o644)} + from huggingface_hub import CommitOperationAdd + + api.create_commit( + repo_id=repo, + repo_type="dataset", + commit_message="Persist coding audit " + run_id, + operations=[ + CommitOperationAdd( + path_in_repo="runs/" + run_id + "/" + name, path_or_fileobj=data + ) + for name, (data, _) in sorted(evidence.items()) + ], + token=secrets["HF_TOKEN"], + ) + except Exception as error: + report.update( + status="error", + accepted=False, + persistence_status="error", + persistence_error_type=type(error).__name__, + ) + (output / "cloud-report.json").write_bytes(_json_bytes(report)) + raise RuntimeError( + "Evidence upload failed; local worker-output/cloud-report.json retained" + ) from None + if failure is not None: + raise RuntimeError( + "Coding worker failed; error report and available evidence persisted" + ) from None + return report + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + local = commands.add_parser( + "prepare", help="Build local immutable inputs; no network or paid compute" + ) + local.add_argument("--task", type=Path, required=True) + local.add_argument("--config", type=Path, required=True) + local.add_argument("--output", type=Path, required=True) + launch = commands.add_parser( + "submit", help="Upload private inputs and launch one paid HF Job" + ) + launch.add_argument("--plan", type=Path, required=True) + internal = commands.add_parser("worker", help=argparse.SUPPRESS) + internal.add_argument("--root", type=Path, required=True) + internal.add_argument("--bundle-sha", required=True) + args = parser.parse_args(argv) + if args.command == "prepare": + result = prepare(args.task, _loads(args.config.read_bytes()), args.output) + elif args.command == "submit": + result = submit(args.plan) + else: + result = worker(args.root, bundle_sha=args.bundle_sha) + print(_json_bytes(result).decode(), end="") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as error: + from huggingface_hub import get_token + + message = str(error) + for value in [get_token(), *(os.environ.get(name) for name in SECRET_NAMES)]: + if value: + message = message.replace(value, "[REDACTED]") + message = re.sub(r"hf_[A-Za-z0-9]{16,}", "[REDACTED]", message) + print( + "Cloud operation failed (" + type(error).__name__ + "): " + message[:1200], + file=sys.stderr, + ) + sys.exit(1) diff --git a/examples/hub_validation_pipeline/contract.json b/examples/hub_validation_pipeline/contract.json new file mode 100644 index 000000000..6bf3b848f --- /dev/null +++ b/examples/hub_validation_pipeline/contract.json @@ -0,0 +1,18 @@ +{ + "profile": "harbor-semantic-v1", + "reward_key": "reward", + "reward": { + "range": [0, 1], + "oracle_tolerance": 0, + "floor_margin": 0.1 + }, + "resources": { + "cpu": 1, + "memory_mb": 512, + "disk_mb": 1024, + "episode_timeout_s": 120 + }, + "types": {"tags": ["swe"]}, + "requires_credentials": false, + "canaries": "validation/canaries" +} diff --git a/examples/hub_validation_pipeline/demo.py b/examples/hub_validation_pipeline/demo.py new file mode 100644 index 000000000..c116298cf --- /dev/null +++ b/examples/hub_validation_pipeline/demo.py @@ -0,0 +1,94 @@ +"""Create three trusted tiny tasks and exercise validation/filtering with local Docker.""" + +import argparse +from pathlib import Path + +from examples.hub_validation_pipeline.pipeline import PILOT, run + + +def create_tasks(root): + """One valid task, one broken verifier, and one missing reference solution.""" + config = """version = "1.0" +[agent] +timeout_sec = 30.0 +[verifier] +timeout_sec = 10.0 +environment_mode = "shared" +[environment] +cpus = 1 +memory_mb = 512 +storage_mb = 1024 +""" + verifier = """#!/bin/bash +mkdir -p /logs/verifier +if [ "$(cat /app/answer.txt 2>/dev/null)" = "42" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi +""" + for name in ("good", "always-rewarding", "missing-oracle"): + task = root / name + (task / "environment").mkdir(parents=True) + (task / "tests").mkdir() + canaries = task / "validation/canaries" + canaries.mkdir(parents=True) + (canaries / "empty.sh").write_text("#!/bin/bash\n:\n") + (canaries / "wrong-answer.sh").write_text( + "#!/bin/bash\nprintf '41\\n' > /app/answer.txt\n" + ) + (task / "task.toml").write_text(config) + (task / "instruction.md").write_text("Write exactly 42 into /app/answer.txt.\n") + (task / "environment/Dockerfile").write_text( + "FROM python:3.12-slim\nWORKDIR /app\n" + ) + (task / "tests/test.sh").write_text( + "#!/bin/bash\nmkdir -p /logs/verifier\necho 1 > /logs/verifier/reward.txt\n" + if name == "always-rewarding" + else verifier + ) + if name != "missing-oracle": + (task / "solution").mkdir() + (task / "solution/solve.sh").write_text( + "#!/bin/bash\nprintf '42\\n' > /app/answer.txt\n" + ) + return root + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True, help="New demo directory") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + dataset = create_tasks(args.output.resolve() / "dataset") + options = argparse.Namespace( + output=args.output / "run", + contract=Path(__file__).with_name("contract.json"), + profile=PILOT, + source="both", + dataset_limit=0, + tasks_per_dataset=3, + hf_dataset=[], + harbor_dataset=[], + local_dataset=[dataset], + validation_timeout=300, + llm_url="", + model="", + harness="opencode", + expose="gradio", + model_timeout=120, + model_step_limit=32, + ) + status = run(options) + import json + + summary = json.loads((options.output / "summary.json").read_text()) + if status != 0 or summary["accepted"] != 1 or summary["excluded"] != 2: + raise SystemExit("demo failed: expected one admitted task and two exclusions") + print( + "Demo passed: good task admitted; broken verifier and missing oracle excluded." + ) + + +if __name__ == "__main__": + main() diff --git a/examples/hub_validation_pipeline/full_run.py b/examples/hub_validation_pipeline/full_run.py new file mode 100644 index 000000000..3e16bd98f --- /dev/null +++ b/examples/hub_validation_pipeline/full_run.py @@ -0,0 +1,354 @@ +"""Freeze census metadata and prepare the reviewed cohort; --submit launches once. + +Preparation acquires selected pinned public task trees, never paid compute. Only +cloud.submit publishes existing inputs/results. No catalog-wide execution claim. +""" + +import argparse +import hashlib +import os +from decimal import Decimal, InvalidOperation +from fractions import Fraction +from math import ceil +from pathlib import Path + +from . import cloud, real_tasks, spend + +MODEL_SOURCE = ( + "Qwen/Qwen3.8-27B", + "37561816e66b5b9153b1b64c10968c18c1e6f4a0", + "deepinfra", +) +RATEBOOK = { + "as_of": "2026-11-10", + "input_usd_per_million": "0.15", + "output_usd_per_million": "1.875", + "cpu_basic_usd_per_hour": "0.01", + "sandbox_port_usd_per_hour": "0.01", + "child_lifetime_seconds": 86400, + "startup_allowance_seconds": 300, +} +ASSUMPTIONS = [ + "Modeled launch reservation at recorded prices; not an account-enforced billing limit.", + "Reservation covers the selected cohort, without builds; the shared ledger must account for prior/concurrent spending and retries.", + "Every child reserves the SDK hard 86400-second lifetime plus 300-second startup; controllers reserve configured timeout plus 300 seconds, without an exposed port.", + "Prior billed tokens stay below max_total_tokens before the last request; message bytes <= 4*total+8192, including tool/framing allowance; request context ceiling 262144.", + "Source revision does not attest provider-served weights; rate/model/flavor changes require a different reviewed ratebook.", + "Only the configured cohort has reviewed execution contracts; all remaining catalog tasks are metadata only. Native tasks are evaluation-only, never training data.", + "An uncertain submission requires manual reconciliation; never automatically resume or retry.", +] + + +def _budget(value): + try: + number = Decimal(value) if type(value) is str else Decimal("NaN") + if number.is_finite() and number > 0: + return Fraction(number) + except InvalidOperation: + pass + raise ValueError("budget_usd must be a finite positive decimal string") + + +def estimate(configs): + """Return exact aggregate launch reservation and request/token ceilings.""" + if not configs: + raise ValueError("A configured cohort is required") + children = calls = inputs = outputs = controller_seconds = 0 + for raw in configs: + config, limits = cloud._config(raw) + if ( + limits["sandbox_remote_job_lifetime_seconds"] + != RATEBOOK["child_lifetime_seconds"] + ): + raise ValueError("A different child lifetime requires a different ratebook") + if any(config[key]["flavor"] != "cpu-basic" for key in ("cloud", "execution")): + raise ValueError("A different flavor requires a different ratebook") + for key in ("target", "generator"): + model = config[key] + if tuple( + model[k] for k in ("model_id", "revision", "provider") + ) != MODEL_SOURCE or model.get("provider_model_id"): + raise ValueError( + "A different model/source/provider requires a different ratebook" + ) + total, maximum = model["max_total_tokens"], model["max_tokens"] + count = ( + limits["target_attempts"] + if key == "target" + else limits["generated_candidates"] + ) + inputs += count * ( + min(262144, 4 * total + 8192) + (total if key == "target" else 0) + ) + outputs += count * min( + total, maximum * (model["max_steps"] if key == "target" else 1) + ) + children += limits["sandbox_jobs_max"] + calls += limits["inference_calls_max"] + controller_seconds += ( + config["cloud"]["timeout_seconds"] + RATEBOOK["startup_allowance_seconds"] + ) + inference = ( + inputs * Fraction(RATEBOOK["input_usd_per_million"]) + + outputs * Fraction(RATEBOOK["output_usd_per_million"]) + ) / 1000000 + cpu, port = ( + Fraction(RATEBOOK[key]) + for key in ("cpu_basic_usd_per_hour", "sandbox_port_usd_per_hour") + ) + child_seconds = ( + RATEBOOK["child_lifetime_seconds"] + RATEBOOK["startup_allowance_seconds"] + ) + compute = ( + children * child_seconds * (cpu + port) + controller_seconds * cpu + ) / 3600 + cost = inference + compute + micros = ceil(cost * 1000000) + return { + "sandbox_jobs_max": children, + "controller_jobs_max": len(configs), + "inference_calls_max": calls, + "input_tokens_max": inputs, + "output_tokens_max": outputs, + "inference_usd_exact": str(inference), + "compute_usd_exact": str(compute), + "reservation_usd_exact": str(cost), + "reservation_usd_rounded_up": f"{micros // 1000000}.{micros % 1000000:06d}", + } + + +def _write(path, value): + temporary = path.with_suffix(".tmp") + with temporary.open("wb") as stream: + stream.write(cloud._json_bytes(value)) + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def _hash(path): + digest, size = hashlib.sha256(), 0 + with path.open("rb") as stream: + while block := stream.read(1024 * 1024): + digest.update(block) + size += len(block) + return {"path": str(path), "sha256": digest.hexdigest(), "bytes": size} + + +def _catalog(path, destination, kind): + path = Path(path).absolute() + with path.open("rb") as stream: + data = stream.read(1024 * 1024 + 1) + if len(data) > 1024 * 1024: + raise ValueError("Supply a small census summary, not a raw census") + summary = cloud._loads(data) + destination.write_bytes(data) + references = ( + { + key: summary[key] + for key in ("compact_repo_inventory", "compact_identity_inventory") + } + if kind == "hf" + else { + key: summary["artifacts"][key]["path"] + for key in ("task_identity_inventory", "dataset_version_membership_index") + } + ) + inventories = {} + for name, reference in references.items(): + inventory = Path(reference) + if not inventory.is_absolute(): + inventory = path.parent / inventory + inventories[name] = ( + _hash(inventory) + if inventory.is_file() + else {"path": str(inventory), "status": "unavailable"} + ) + return { + **_hash(destination), + "scope": summary["scope"], + "counts": summary["counts"], + "inventories": inventories, + "runtime_qualified": False, + } + + +def run( + output, + hf_census, + harbor_census, + budget_usd, + *, + panel_path, + task_names=None, + submit=False, + spend_ledger=None, + reservation_id=None, +): + if (submit or spend_ledger is not None or reservation_id is not None) and ( + spend_ledger is None + or (isinstance(spend_ledger, str) and not spend_ledger.strip()) + or not isinstance(reservation_id, str) + or not reservation_id.strip() + ): + raise ValueError("Supply both a shared ledger and a reservation ID") + cap = _budget(budget_usd) + panel, panel_provenance = real_tasks._load_panel(panel_path) + names = [task["name"] for task in real_tasks._select_tasks(panel, task_names)] + output = Path(output).absolute() + output.mkdir(parents=True, exist_ok=False) + manifest_path = output / "manifest.json" + manifest = { + "schema": "openenv-full-run-v1", + "status": "preparing", + "stage": "catalogs", + "accepted": False, + "rfc008_qualified": False, + "trainable": False, + "evaluation_only": True, + "full_catalog_executed": False, + "budget_usd": budget_usd, + "ratebook": RATEBOOK, + "assumptions": ASSUMPTIONS, + "evaluation_panel": panel_provenance, + "catalogs": {}, + "tasks": [], + } + _write(manifest_path, manifest) + try: + for kind, source in (("hf", hf_census), ("harbor", harbor_census)): + manifest["catalogs"][kind] = _catalog( + source, output / f"{kind}-census-summary.json", kind + ) + _write(manifest_path, manifest) + manifest["stage"] = "prepare" + recipes = real_tasks._prepare( + output / "recipes", + panel=panel, + panel_provenance=panel_provenance, + task_names=names, + bundle=True, + ) + configs = [] + manifest["stage"] = "plans" + for row in recipes["tasks"]: + plan_sha256 = _hash(Path(row["bundle_plan"]))["sha256"] + plan, config, _, _ = cloud._read_plan( + row["bundle_plan"], expected_sha256=plan_sha256 + ) + configs.append(config) + manifest["tasks"].append( + { + "task_id": row["task_id"], + "plan": row["bundle_plan"], + "plan_sha256": plan_sha256, + "run_id": plan["run_id"], + "bundle_sha256": plan["bundle_sha256"], + "budget": plan["budget"], + "accepted": False, + "trainable": False, + "rfc008_qualified": False, + "submission": {"status": "not_submitted"}, + } + ) + _write(manifest_path, manifest) + manifest["stage"] = "budget" + manifest["estimate"] = estimate(configs) + manifest["within_budget"] = ( + Fraction(manifest["estimate"]["reservation_usd_exact"]) <= cap + ) + manifest["status"] = "prepared" + _write(manifest_path, manifest) + if submit: + if not manifest["within_budget"]: + raise ValueError("Reservation exceeds budget") + manifest["stage"] = "reserve" + fingerprint = hashlib.sha256( + "\0".join(task["bundle_sha256"] for task in manifest["tasks"]).encode() + ).hexdigest()[:16] + identifier = f"{reservation_id}:{fingerprint}" + spend.reserve( + spend_ledger, + identifier, + manifest["estimate"]["reservation_usd_rounded_up"], + "Reviewed native Hub audit cohort", + evidence=str(manifest_path), + ) + manifest["shared_spend"] = { + "ledger": str(Path(spend_ledger).absolute()), + "reservation_id": identifier, + "reserved_usd": manifest["estimate"]["reservation_usd_rounded_up"], + } + _write(manifest_path, manifest) + manifest.update(stage="submit", status="submitting") + for task in manifest["tasks"]: + if _hash(Path(task["plan"]))["sha256"] != task["plan_sha256"]: + raise ValueError("Prepared plan changed before submission") + task["submission"] = { + "status": "pending_or_unknown", + "receipt": str(Path(task["plan"]).parent / "submission.json"), + } + _write(manifest_path, manifest) + receipt = cloud.submit( + task["plan"], expected_plan_sha256=task["plan_sha256"] + ) + task["submission"].update( + { + key: receipt[key] + for key in ( + "status", + "job_id", + "job_url", + "input_revision", + "receipt_sync_status", + "receipt_sync_error_type", + ) + if key in receipt + } + ) + _write(manifest_path, manifest) + manifest["status"] = "submitted" + manifest["stage"] = "done" + _write(manifest_path, manifest) + except BaseException as error: + manifest.update(status="failed", error_type=type(error).__name__) + _write(manifest_path, manifest) + raise RuntimeError( + f"Full run stopped during {manifest['stage']}; inspect manifest.json and per-plan submission receipts" + ) from None + return manifest + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--panel", dest="panel_path", type=Path, required=True) + parser.add_argument("--hf-census", type=Path, required=True) + parser.add_argument("--harbor-census", type=Path, required=True) + parser.add_argument("--budget-usd", required=True) + parser.add_argument("--task", action="append", dest="task_names") + parser.add_argument("--submit", action="store_true") + parser.add_argument("--spend-ledger", type=Path) + parser.add_argument("--reservation-id") + args = parser.parse_args(argv) + if args.submit and (args.spend_ledger is None or not args.reservation_id): + parser.error("Paid submission requires --spend-ledger and --reservation-id") + manifest = run(**vars(args)) + print( + cloud._json_bytes( + { + "manifest": str(args.output.absolute() / "manifest.json"), + "status": manifest["status"], + } + ).decode(), + end="", + ) + + +if __name__ == "__main__": + main() diff --git a/examples/hub_validation_pipeline/pipeline.py b/examples/hub_validation_pipeline/pipeline.py new file mode 100644 index 000000000..3574bb865 --- /dev/null +++ b/examples/hub_validation_pipeline/pipeline.py @@ -0,0 +1,832 @@ +"""Discover pinned tasks, validate them, then evaluate only an admitted local corpus.""" + +import argparse +import asyncio +import hashlib +import math +import os +import re +import shutil +import subprocess +import sys +import time +import tomllib +from decimal import Decimal +from fractions import Fraction +from pathlib import Path + +import tomli_w +from examples.hub_validation_pipeline import sources +from openenv.validation.parsers.harbor_toml import ( + _unique_object, + canary_scripts, + read_harbor_contract, +) +from openenv.validation.policy import load_policy +from openenv.validation.report import ValidationReportV2 +from openenv.validation.runner import _DIGEST_EXCLUDED_DIRS, source_digest +from openenv.validation.runtime.json_values import dumps, loads +from openenv.validation.types import Lane + +PILOT = "harbor-semantic-v1" +PILOT_CHECKS = { + "static.manifest", + "runtime.startup", + "runtime.reward_well_formed", + "runtime.episode_determinism", + "semantic.oracle_max", + "semantic.floor_gap", +} +REPO = Path(__file__).resolve().parents[2] + + +def write_json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(dumps(value) + "\n") + temporary.replace(path) + + +def write_rows(path, rows): + path.write_text("".join(dumps(row) + "\n" for row in rows)) + + +def copy_subject(source, destination, expected_digest): + shutil.copytree( + source, + destination, + symlinks=True, + ignore=shutil.ignore_patterns(*_DIGEST_EXCLUDED_DIRS), + ) + if source_digest(destination) != expected_digest: + raise ValueError("source changed while copying the task") + + +def validation_environment(): + """Give the validation worker Docker access without inference or Hub secrets.""" + names = { + "PATH", + "HOME", + "TMPDIR", + "LANG", + "LC_ALL", + "DOCKER_HOST", + "DOCKER_CONTEXT", + "DOCKER_CONFIG", + "DOCKER_CERT_PATH", + "DOCKER_TLS_VERIFY", + "SSH_AUTH_SOCK", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + } + env = {key: value for key, value in os.environ.items() if key in names} + env["PYTHONPATH"] = os.pathsep.join((str(REPO / "src"), str(REPO / "envs"))) + env["PYTHONDONTWRITEBYTECODE"] = "1" + return env + + +def validate_task(subject, report_path, profile, timeout_s): + if profile not in {PILOT, "full"}: + raise ValueError("unsupported admission profile") + command = [ + sys.executable, + "-m", + "openenv.cli", + "validate", + str(subject), + "--level", + "semantic", + "--policy", + "v2", + "--json", + "--output", + str(report_path), + ] + report_path.parent.mkdir(parents=True, exist_ok=True) + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=validation_environment(), + cwd=REPO, + ) + try: + _, stderr = process.communicate(timeout=timeout_s) + except subprocess.TimeoutExpired: + # SIGTERM reaches validate's cleanup handler before a final hard stop. + process.terminate() + try: + process.communicate(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.communicate() + raise TimeoutError("validation deadline exceeded; task excluded") from None + if not report_path.is_file(): + raise RuntimeError( + f"validator exited {process.returncode} without a report: {stderr[-500:]}" + ) + return process.returncode, loads(report_path.read_text()) + + +def verified_bundle(directory): + """Verify every retained artifact before using its admission evidence.""" + entries = {} + for row in (directory / "SHA256SUMS").read_text().splitlines(): + digest, name = row.split(" ", 1) + relative = Path(name) + if ( + not re.fullmatch(r"[0-9a-f]{64}", digest) + or relative.is_absolute() + or ".." in relative.parts + or name in entries + ): + raise ValueError("invalid artifact checksum inventory") + path = directory / relative + if path.is_symlink() or not path.resolve().is_relative_to(directory.resolve()): + raise ValueError("artifact path escapes evidence directory") + if hashlib.sha256(path.read_bytes()).hexdigest() != digest: + raise ValueError(f"artifact checksum mismatch: {name}") + entries[name] = digest + if not {"report.json", "harbor-evidence.json"}.issubset(entries): + raise ValueError( + "native validation evidence is missing from checksum inventory" + ) + return entries + + +def reward_fraction(value): + if ( + isinstance(value, bool) + or not isinstance(value, (int, Decimal)) + or not Decimal(value).is_finite() + ): + raise ValueError("native reward must be finite and numeric") + return Fraction(value) + + +def trial_rewards(trial, artifact_dir, checksums, key, low, high): + """Bind every raw output, including an optional nonzero oracle exit marker.""" + raw = trial["raw_rewards"] + allowed = {"reward.json", "reward.txt", "oracle-exit-code.txt"} + if not isinstance(raw, dict) or set(raw) - allowed: + raise ValueError("unexpected native raw output inventory") + prefix = f"harbor-trials/{trial['role']}" + paths = {f"{prefix}/{name}" for name in raw} + actual = { + path.relative_to(artifact_dir).as_posix() + for path in (artifact_dir / prefix).iterdir() + if path.name in allowed + } + if paths != actual: + raise ValueError("native raw output inventory is incomplete") + outputs = {} + for name, reference in raw.items(): + if reference.get("path") != f"{prefix}/{name}" or checksums.get( + reference["path"] + ) != reference.get("sha256"): + raise ValueError( + "raw native output is not in the verified artifact inventory" + ) + outputs[name] = (artifact_dir / reference["path"]).read_text() + if "reward.json" in outputs: + values = loads(outputs["reward.json"], object_pairs_hook=_unique_object) + elif "reward.txt" in outputs: + try: + values = {"reward": Decimal(outputs["reward.txt"].strip())} + except ArithmeticError as error: + raise ValueError("invalid native reward text") from error + else: + raise ValueError("native trial has no raw rewards") + if not isinstance(values, dict) or not values or key not in values: + raise ValueError("native reward map lacks the declared key") + rewards = {name: reward_fraction(value) for name, value in values.items()} + if any(not low <= value <= high for value in rewards.values()): + raise ValueError("native reward is outside the declared range") + observed = { + name: reward_fraction(value) for name, value in trial["rewards"].items() + } + if rewards != observed: + raise ValueError("raw native rewards differ from the recorded reward map") + if trial["role"] != "nop": + code = trial.get("oracle_exit_code") + if ( + trial.get("oracle_exit_observed_at_agent_end") is not True + or isinstance(code, bool) + or not isinstance(code, int) + or code != int(outputs.get("oracle-exit-code.txt", "0")) + ): + raise ValueError("native oracle exit evidence is inconsistent") + return rewards + + +def admission(report, exit_code, subject, artifact_dir, profile, original_digest): + """Return a task/image-bound admission envelope, or reject the task.""" + if profile not in {PILOT, "full"}: + raise ValueError("unsupported admission profile") + ValidationReportV2.model_validate(report) + if exit_code != 0 or report["verdict"] not in {"pass", "warn"}: + raise ValueError("validation failed") + if ( + report["signature"] != "task.toml" + or report["policy_version"] != "v2" + or report["lane"] != "local" + ): + raise ValueError("unexpected validation signature or policy") + effective_digest = source_digest(subject) + if report["source_digest"] != effective_digest: + raise ValueError("task changed after validation") + parsed_contract = read_harbor_contract(subject) + contract = loads( + (subject / "validation/openenv.json").read_text(), + object_pairs_hook=_unique_object, + ) + canaries = ( + canary_scripts(subject, parsed_contract.canaries) + if parsed_contract.canaries is not None + else [] + ) + results = {row["check_id"]: row for row in report["results"]} + if len(results) != len(report["results"]): + raise ValueError("duplicate validation check IDs") + if any(row["status"] in {"fail", "error"} for row in results.values()): + raise ValueError("a validation check failed or errored") + known_checks = set(load_policy("v2").entries_for_lane(Lane.LOCAL)) + if set(results) - known_checks: + raise ValueError("unknown validation check ID") + required = ( + PILOT_CHECKS | ({"semantic.canary_floor"} if canaries else set()) + if profile == PILOT + else set(load_policy("v2").entries_for_lane(Lane.LOCAL)) + ) + if not required.issubset(results) or any( + results[key]["status"] != "pass" for key in required + ): + raise ValueError("required validation coverage is incomplete") + if not {1, 2, 3}.issubset(report["levels_run"]): + raise ValueError("semantic validation was not executed") + if profile != PILOT and ( + report["verdict"] != "pass" or 3 not in report["levels_run"] + ): + raise ValueError("full validation is incomplete") + checksums = verified_bundle(artifact_dir) + if dumps(loads((artifact_dir / "report.json").read_text())) != dumps(report): + raise ValueError("bundled report differs from the validator result") + native = loads((artifact_dir / "harbor-evidence.json").read_text()) + if ( + native.get("schema_version") != "2" + or native["profile"] != PILOT + or native["task_effective_digest"] != effective_digest + or native["original_source_digest"] != original_digest + or contract.get("original_source_digest") != original_digest + or native["reward_key"] != contract["reward_key"] + or report["manifest"]["execution"]["profile"] != PILOT + or report["manifest"]["capabilities"]["canaries"] != parsed_contract.canaries + or not re.fullmatch(r"sha256:[0-9a-f]{64}", native.get("image_ref") or "") + or not re.fullmatch( + r"openenv-validation:[a-f0-9]{32}", native.get("retained_image_tag") or "" + ) + or native["cleanup"]["verified"] is not True + or "failure" in native + ): + raise ValueError( + "native oracle/floor, identity or cleanup evidence is incomplete" + ) + trials = native["trials"] + roles = ["nop", "oracle", "oracle-replay"] + [ + f"canary-{index}" for index in range(1, len(canaries) + 1) + ] + if ( + [row["role"] for row in trials] != roles + or len({row["container_id"] for row in trials}) != len(trials) + or any( + not re.fullmatch(r"[0-9a-f]{64}", row["container_id"] or "") + or row["image_ref"] != native["image_ref"] + or row["retained_image_tag"] != native["retained_image_tag"] + or row["unmodified_task_digest"] != report["source_digest"] + or row["configuration"] != trials[0]["configuration"] + or row["cleanup"]["verified"] is not True + or "failure" in row + or "reward_error" in row + for row in trials + ) + ): + raise ValueError( + "native trials do not establish fresh identical-image execution" + ) + reward_contract = contract["reward"] + low, high = map(reward_fraction, reward_contract["range"]) + margin = reward_fraction(reward_contract["floor_margin"]) + tolerance = reward_fraction(reward_contract["oracle_tolerance"]) + rewards = [] + for index, trial in enumerate(trials): + overlay = ( + {} + if index == 0 + else { + "environment.docker_image": native["retained_image_tag"], + } + ) + if index >= 3: + canary = canaries[index - 3] + overlay["solution/solve.sh"] = { + "canary": canary, + "sha256": hashlib.sha256((subject / canary).read_bytes()).hexdigest(), + } + if trial["task_overlay"] != overlay: + raise ValueError( + "native task overlay differs from the declared source bytes" + ) + rewards.append( + trial_rewards( + trial, artifact_dir, checksums, native["reward_key"], low, high + ) + ) + if ( + rewards[1] != rewards[2] + or trials[1]["oracle_exit_code"] != trials[2]["oracle_exit_code"] + or trials[1]["oracle_exit_code"] != 0 + ): + raise ValueError("raw native oracle replay or exit code is inconsistent") + floor, oracle = (row[native["reward_key"]] for row in rewards[:2]) + if ( + floor != reward_fraction(native.get("floor_reward")) + or oracle != reward_fraction(native.get("oracle_reward")) + or oracle < high - tolerance + or oracle - floor < margin + or high - floor < margin + or any(row[native["reward_key"]] > floor for row in rewards[3:]) + ): + raise ValueError("raw native rewards do not satisfy the oracle/floor contract") + return { + "profile": profile, + "rfc008_complete": profile == "full", + "original_source_digest": original_digest, + "task_effective_digest": effective_digest, + "image_ref": native["image_ref"], + "reward_key": native["reward_key"], + "floor_reward": native["floor_reward"], + "oracle_reward": native["oracle_reward"], + "resources": report["manifest"]["resources"], + "container_configuration": native["trials"][0]["configuration"], + "report_sha256": checksums["report.json"], + "evidence_sha256": checksums["harbor-evidence.json"], + "out_of_profile_skips": sorted( + key for key, row in results.items() if row["status"] == "skip" + ), + } + + +def materialize(record, destination): + """Pin the execution image while retaining the validated task and overlay hashes.""" + subject = Path(record["subject"]) + if source_digest(subject) != record["task_effective_digest"]: + raise ValueError("validated subject changed before materialization") + copy_subject(subject, destination, record["task_effective_digest"]) + config_path = destination / "task.toml" + config = tomllib.loads(config_path.read_text()) + config.setdefault("environment", {})["docker_image"] = record["image_ref"] + for key, value in ( + ("cpus", record["resources"]["cpu"]), + ("memory_mb", record["resources"]["memory_mb"]), + ("storage_mb", record["resources"]["disk_mb"]), + ): + config["environment"][key] = float(value) if key == "cpus" else value + config_path.write_text(tomli_w.dumps(config)) + return { + **record, + "execution_path": str(destination), + "execution_digest": source_digest(destination), + "execution_overlay": { + "environment": config["environment"], + }, + } + + +def docker(*arguments): + return subprocess.run( + ["docker", *arguments], check=True, text=True, capture_output=True, timeout=30 + ).stdout.strip() + + +def model_guard(records, projects): + """Attach an actual-container image check before the agent can call its model.""" + by_path = {str(Path(row["execution_path"]).resolve()): row for row in records} + + def created(trial): + from harbor.trial.hooks import TrialEvent + + path = str(trial.config.task.path.resolve()) + if path not in by_path: + raise ValueError("model runner selected an unapproved task") + record = by_path[path] + if source_digest(Path(path)) != record["execution_digest"]: + raise ValueError("approved task changed before execution") + project = trial.agent_environment.session_id + if not re.fullmatch(r"[a-z0-9_-]+", project): + raise ValueError("unexpected Harbor project identity") + projects.add(project) + + async def inspect_image(event): + ids = docker( + "ps", "-aq", "--filter", f"label=com.docker.compose.project={project}" + ).splitlines() + if len(ids) != 1: + raise ValueError("expected one approved task container") + actual = loads(docker("inspect", ids[0]))[0] + if actual["Image"] != record["image_ref"]: + raise ValueError("model task image differs from validated image") + expected = record["container_configuration"] + mounts = sorted( + [ + { + "type": item["Type"], + "destination": item["Destination"], + "rw": item["RW"], + } + for item in actual["Mounts"] + ], + key=lambda item: item["destination"], + ) + network = actual["HostConfig"]["NetworkMode"] + if network == project + "_default": + network = "compose-default" + if ( + actual["Config"]["User"] != expected["user"] + or actual["Config"]["WorkingDir"] != expected["workdir"] + or {key: actual["HostConfig"].get(key) for key in expected["host"]} + != expected["host"] + or mounts != expected["mounts"] + or network != expected["network_mode"] + ): + raise ValueError("model sandbox configuration differs from validation") + + trial.add_hook(TrialEvent.AGENT_START, inspect_image) + + return created + + +def cleanup_projects(projects): + """Remove and recheck only resources belonging to this run's model trials.""" + for project in projects: + selector = f"label=com.docker.compose.project={project}" + containers = docker("ps", "-aq", "--filter", selector).splitlines() + owned_volumes = set() + if containers: + for container in loads(docker("inspect", *containers)): + for mount in container["Mounts"]: + if mount["Type"] == "volume": + name = mount["Name"] + if not re.fullmatch(r"[a-f0-9]{64}", name): + raise ValueError( + "model trial mounted a volume of unproven ownership" + ) + owned_volumes.add(name) + for kind, listing, removal in ( + ("container", ("ps", "-aq"), ("rm", "-f", "-v")), + ("network", ("network", "ls", "-q"), ("network", "rm")), + ("volume", ("volume", "ls", "-q"), ("volume", "rm")), + ): + ids = docker(*listing, "--filter", selector).splitlines() + if ids: + docker(*removal, *ids) + if docker(*listing, "--filter", selector): + raise RuntimeError(f"model {kind} cleanup could not be verified") + if owned_volumes.intersection(docker("volume", "ls", "-q").splitlines()): + raise RuntimeError("model anonymous-volume cleanup could not be verified") + + +def evaluate( + records, output, *, llm_url, model, harness, expose, timeout_s, step_limit +): + """Run existing OpenEnv Harbor capture on the frozen approved corpus.""" + from openenv.harbor.runner import run_batch + from openenv.harbor.tasks import resolve_task_dirs + + if not records: + return [] + approved = output / "approved" + expected = {str(Path(row["execution_path"]).resolve()): row for row in records} + paths = resolve_task_dirs(str(approved)) + if {str(path.resolve()) for path in paths} != set(expected): + raise ValueError("OpenEnv loader and approved corpus disagree") + results = [] + # Different tasks can choose different headline reward keys. + for index, path in enumerate(paths): + record = expected[str(path.resolve())] + projects = set() + cleanup_ok = True + try: + batch = asyncio.run( + run_batch( + llm_url=llm_url, + model=model, + dataset=str(approved), + task_indices=[index], + harness=harness, + sandbox="docker", + expose=expose, + trials_dir=output / "model-trials", + reward_key=record["reward_key"], + agent_timeout_sec=timeout_s, + agent_step_limit=step_limit, + on_trial_created=model_guard(records, projects), + keep_sandbox=True, + environment_kwargs={ + "auto_mounting": False, + "keep_containers": True, + }, + ) + ) + if len(batch) != 1 or Path(batch[0].task_id).resolve() != path.resolve(): + raise ValueError("model result accounting is incomplete") + results.append( + { + "task_id": record["task_id"], + "admission": record, + "result": batch[0].model_dump(), + } + ) + except Exception as error: + results.append( + {"task_id": record["task_id"], "error": str(error), "ok": False} + ) + finally: + try: + cleanup_projects(projects) + except Exception as error: + cleanup_ok = False + results[-1]["cleanup_error"] = str(error) + write_rows(output / "rollouts.jsonl", results) + if not cleanup_ok: + break + return results + + +def local_tasks(root, limit): + root = root.resolve(strict=True) + manifests = [ + root / path / "task.toml" + for path in sources.task_roots( + manifest.relative_to(root).as_posix() + for manifest in root.rglob("task.toml") + ) + ] + return [ + { + "task_id": f"local:{root.name}@{source_digest(manifest.parent)}:{manifest.parent.relative_to(root)}", + "path": str(manifest.parent), + "source": "local", + "source_id": str(root), + "revision": source_digest(manifest.parent), + "task_path": manifest.parent.relative_to(root).as_posix(), + } + for manifest in manifests[:limit] + ] + + +def run(args): + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + coding_config = getattr(args, "coding_config", None) + coding_config = loads(coding_config.read_text()) if coding_config else None + contract = loads(args.contract.read_text()) if coding_config is None else None + if contract is not None and contract.get("profile") != PILOT: + raise ValueError(f"contract must declare {PILOT}") + candidates, tasks, excluded, accepted = [], [], [], [] + plans = [] + explicit = bool(args.hf_dataset or args.harbor_dataset or args.local_dataset) + if not explicit: + for source, discover in ( + ("hf", sources.discover_hf), + ("harbor", sources.discover_harbor), + ): + if args.source in {source, "both"}: + try: + candidates.extend(discover(args.dataset_limit)) + except Exception as error: + excluded.append( + {"source": source, "stage": "discovery", "reason": str(error)} + ) + for spec in args.hf_dataset: + repo, separator, revision = spec.partition("@") + if not separator: + revision = sources.HfApi(token=False).dataset_info(repo).sha + candidates.append({"source": "hf", "id": repo, "revision": revision}) + for spec in args.harbor_dataset: + name, separator, revision = spec.partition("@") + if not separator: + raise ValueError("explicit Harbor datasets require org/name@sha256:digest") + candidates.append({"source": "harbor", "id": name, "revision": revision}) + write_rows(output / "candidates.jsonl", candidates) + for candidate in candidates: + try: + tasks.extend( + sources.acquire(candidate, output / "downloads", args.tasks_per_dataset) + ) + except Exception as error: + excluded.append( + {"candidate": candidate, "stage": "acquisition", "reason": str(error)} + ) + for root in args.local_dataset: + tasks.extend(local_tasks(root, args.tasks_per_dataset)) + write_rows(output / "tasks.jsonl", tasks) + for task in tasks: + key = hashlib.sha256(task["task_id"].encode()).hexdigest()[:24] + subject = output / "subjects" / key + try: + original_digest = source_digest(Path(task["path"])) + copy_subject(task["path"], subject, original_digest) + from openenv.harbor.tasks import resolve_task_dirs + + if subject not in resolve_task_dirs(str(subject.parent), refresh=True): + raise ValueError("OpenEnv did not load the acquired task") + if coding_config is not None: + from examples.hub_validation_pipeline.cloud import prepare, submit + + plan_dir = output / "plans" / key + plan = prepare( + subject, + coding_config, + plan_dir, + source={ + name: task[name] + for name in ( + "source", + "source_id", + "revision", + "task_path", + "task_id", + ) + }, + ) + row = { + "task_id": task["task_id"], + "run_id": plan["run_id"], + "plan": str(plan_dir / "plan.json"), + "state": "prepared", + "accepted": False, + } + if getattr(args, "submit_hf_jobs", False): + receipt = submit(plan_dir / "plan.json") + row.update( + state=receipt["status"], + job_id=receipt.get("job_id"), + job_url=receipt.get("job_url"), + ) + plans.append(row) + write_rows(output / "planned.jsonl", plans) + continue + write_json( + subject / "validation" / "openenv.json", + { + **contract, + "original_source_digest": original_digest, + }, + ) + report_path = output / "reports" / f"{key}.json" + exit_code, report = validate_task( + subject, report_path, args.profile, args.validation_timeout + ) + envelope = admission( + report, + exit_code, + subject, + report_path.with_suffix(".artifacts"), + args.profile, + original_digest, + ) + accepted.append( + materialize( + {**task, **envelope, "subject": str(subject)}, + output / "approved" / "tasks" / key, + ) + ) + except Exception as error: + stage = ( + "cloud_submission" + if coding_config is not None and getattr(args, "submit_hf_jobs", False) + else "cloud_preparation" + if coding_config is not None + else "validation" + ) + excluded.append({**task, "stage": stage, "reason": str(error)}) + write_rows(output / "accepted.jsonl", accepted) + write_rows(output / "excluded.jsonl", excluded) + write_rows(output / "excluded.jsonl", excluded) + write_rows(output / "accepted.jsonl", accepted) + if coding_config is not None: + write_rows(output / "planned.jsonl", plans) + summary = { + "profile": "harbor-coding-audit-v1", + "phase": "submitted" + if getattr(args, "submit_hf_jobs", False) + else "prepared", + "candidates": len(candidates), + "tasks": len(tasks), + "plans": len(plans), + "excluded": len(excluded), + "accepted": 0, + "rfc008_complete": False, + "note": "Plans/submitted Jobs are not qualification results; inspect each immutable run report.", + "finished_at": time.time(), + } + write_json(output / "summary.json", summary) + print(dumps(summary)) + return 0 if plans else 1 + results = ( + evaluate( + accepted, + output, + llm_url=args.llm_url, + model=args.model, + harness=args.harness, + expose=args.expose, + timeout_s=args.model_timeout, + step_limit=args.model_step_limit, + ) + if args.llm_url and accepted + else [] + ) + summary = { + "profile": args.profile, + "rfc008_complete": args.profile == "full" and bool(accepted), + "candidates": len(candidates), + "tasks": len(tasks), + "accepted": len(accepted), + "excluded": len(excluded), + "model_trials": len(results), + "model_requested": bool(args.llm_url), + "finished_at": time.time(), + } + write_json(output / "summary.json", summary) + print(dumps(summary)) + model_ok = len(results) == len(accepted) and all( + row.get("result", {}).get("ok") is True and "cleanup_error" not in row + for row in results + ) + return 0 if accepted and (not args.llm_url or model_ok) else 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output", type=Path, required=True, help="New run directory; must not exist" + ) + parser.add_argument( + "--contract", + type=Path, + help="Trusted reward/resource contract for local native Harbor validation", + ) + parser.add_argument( + "--coding-config", + type=Path, + help="Prepare the supplemental coding audit on HF Jobs instead of local Docker validation", + ) + parser.add_argument( + "--submit-hf-jobs", + action="store_true", + help="Submit prepared coding audits; incurs cloud/inference charges", + ) + parser.add_argument("--profile", choices=["full", PILOT], default="full") + parser.add_argument("--source", choices=["hf", "harbor", "both"], default="both") + parser.add_argument("--dataset-limit", type=int, default=2) + parser.add_argument("--tasks-per-dataset", type=int, default=2) + parser.add_argument("--hf-dataset", action="append", default=[]) + parser.add_argument("--harbor-dataset", action="append", default=[]) + parser.add_argument("--local-dataset", action="append", type=Path, default=[]) + parser.add_argument("--validation-timeout", type=float, default=900) + parser.add_argument("--llm-url", default="") + parser.add_argument("--model", default="") + parser.add_argument("--harness", default="opencode") + parser.add_argument( + "--expose", choices=["direct", "gradio", "cloudflare"], default="gradio" + ) + parser.add_argument("--model-timeout", type=float, default=120) + parser.add_argument("--model-step-limit", type=int, default=32) + args = parser.parse_args() + if args.coding_config is None and args.contract is None: + parser.error("--contract or --coding-config is required") + if args.submit_hf_jobs and args.coding_config is None: + parser.error("--submit-hf-jobs requires --coding-config") + if args.coding_config is not None and (args.contract is not None or args.llm_url): + parser.error( + "--coding-config selects its own contract/model and cannot combine --contract or --llm-url" + ) + if args.dataset_limit < 0 or args.tasks_per_dataset < 0: + parser.error("discovery/task limits must be nonnegative") + if ( + not all( + math.isfinite(value) and value > 0 + for value in (args.validation_timeout, args.model_timeout) + ) + or args.model_step_limit < 1 + ): + parser.error("timeouts and model step limit must be positive") + if args.llm_url and not args.model: + parser.error("--model is required with --llm-url") + raise SystemExit(run(args)) + + +if __name__ == "__main__": + main() diff --git a/examples/hub_validation_pipeline/prepared_batch.py b/examples/hub_validation_pipeline/prepared_batch.py new file mode 100644 index 000000000..0574734aa --- /dev/null +++ b/examples/hub_validation_pipeline/prepared_batch.py @@ -0,0 +1,266 @@ +"""Validate a frozen cohort offline; --submit reserves and launches it once. + +Only owned controllers are polled. Completed controllers remain unmeasured; +result collection and acceptance require separate evidence review. +""" + +import argparse +import hashlib +import re +import time +from fractions import Fraction +from pathlib import Path +from urllib.parse import urlsplit + +from examples.coding_audit.hf import _deadline, _DeadlineExceeded + +from . import cloud, full_run, spend + +STAGES = "SCHEDULING RUNNING COMPLETED ERROR CANCELED DELETED".split() + + +def _plan(root, task): + path = (root / cloud._safe_name(task["plan_path"])).resolve(strict=True) + if not path.is_relative_to(root) or not path.is_file(): + raise ValueError("Plan must resolve inside the preparation directory") + receipt = path.parent / "submission.json" + if receipt.exists() or receipt.is_symlink(): + raise FileExistsError("Submission receipt exists; manual review required") + return path, cloud._read_plan(path, expected_sha256=task["plan_sha256"]) + + +def _record(task, receipt): + for key in ("job_id", "job_url", "status"): + if isinstance(receipt.get(key), str): + task["submission_status" if key == "status" else key] = receipt[key] + + +def _status(task, spec, owner, deadline, api, secrets, clock): + if clock() >= deadline: + return "TIMEOUT" + try: + with _deadline(deadline - clock()): + job = api.inspect_job( + job_id=task["job_id"], + namespace=spec.get("namespace"), + token=secrets["HF_SANDBOX_TOKEN"], + ) + except _DeadlineExceeded: + return "TIMEOUT" + if ( + job.id != task["job_id"] + or job.docker_image != spec["image"] + or not owner + or job.owner.name != owner + ): + raise ValueError("Controller identity mismatch") + if clock() >= deadline: + return "TIMEOUT" + return job.status.stage if job.status.stage in STAGES else "UNKNOWN" + + +def run( + prepared_manifest, + output, + spend_ledger, + reservation_id, + budget_usd, + *, + submit=False, + api=None, + secrets=None, + sleep=time.sleep, + clock=time.monotonic, +): + cap = full_run._budget(budget_usd) + spend._text(reservation_id) + if not str(spend_ledger or "").strip(): + raise ValueError("Supply a shared ledger and a reservation ID") + output = Path(output).absolute() + output.mkdir(parents=True, exist_ok=False) + source = Path(prepared_manifest).resolve(strict=True) + root, destination = source.parent, output / "manifest.json" + source_bytes = source.read_bytes() + binding = { + "path": str(source), + "sha256": hashlib.sha256(source_bytes).hexdigest(), + "bytes": len(source_bytes), + } + manifest = dict( + schema="openenv-prepared-batch-v1", + status="preparing", + stage="plans", + prepared_manifest=binding, + accepted=False, + results_collected=False, + budget_usd=budget_usd, + tasks=[], + ) + + def save(**state): + manifest.update(**state) + full_run._write(destination, manifest) + + save() + try: + rows = cloud._loads(source_bytes)["tasks"] + if not isinstance(rows, list) or not rows: + raise ValueError("A nonempty prepared cohort is required") + configs, task_ids, run_ids = [], set(), set() + for row in rows: + if not isinstance(row["task_id"], str) or not row["task_id"].strip(): + raise ValueError("Task IDs must be nonempty strings") + path, (plan, config, _, _) = _plan(root, row) + if row["task_id"] in task_ids or plan["run_id"] in run_ids: + raise ValueError("Task IDs, run IDs and plan paths must be distinct") + task_ids.add(row["task_id"]) + run_ids.add(plan["run_id"]) + configs.append(config) + task = {key: row[key] for key in ("task_id", "plan_path", "plan_sha256")} + task.update( + run_id=plan["run_id"], + bundle_sha256=plan["bundle_sha256"], + status="not_submitted", + accepted=False, + ) + manifest["tasks"].append(task) + manifest["stage"] = "budget" + manifest["estimate"] = full_run.estimate(configs) + amount = manifest["estimate"]["reservation_usd_rounded_up"] + manifest["within_budget"] = Fraction(amount) <= cap + fingerprint = hashlib.sha256( + "\0".join(task["bundle_sha256"] for task in manifest["tasks"]).encode() + ).hexdigest()[:16] + manifest["shared_spend"] = dict( + ledger=str(Path(spend_ledger).absolute()), + reservation_id=f"{reservation_id}:{fingerprint}", + reserved_usd=amount, + status="not_reserved", + ) + manifest["status"] = "prepared" + save() + if not submit: + save(stage="done") + return manifest + with _deadline(1): + pass + if not manifest["within_budget"]: + raise ValueError("Aggregate reservation exceeds budget") + if full_run._hash(source) != binding: + raise ValueError("Preparation manifest changed") + for task in manifest["tasks"]: + _plan(root, task) + manifest["stage"] = "reserve" + manifest["shared_spend"]["status"] = "pending_or_unknown" + save() + spend.reserve( + spend_ledger, + manifest["shared_spend"]["reservation_id"], + amount, + "Frozen prepared cohort", + evidence=str(destination), + ) + manifest["shared_spend"]["status"] = "reserved" + save() + secrets = cloud._runtime_secrets(secrets) + if api is None: + from huggingface_hub import HfApi + + api = HfApi(token=secrets["HF_TOKEN"]) + job_ids = set() + for offset in range(0, len(rows), 2): + active = [] + for task in manifest["tasks"][offset : offset + 2]: + if any(clock() >= entry[3] for entry in active): + raise TimeoutError("Controller deadline exceeded") + if full_run._hash(source) != binding: + raise ValueError("Preparation manifest changed") + path, (plan, config, _, _) = _plan(root, task) + spec = plan["mcp_request"]["args"] + task["status"] = "pending_or_unknown" + save(stage="submit", status="submitting") + deadline = ( + clock() + + config["cloud"]["timeout_seconds"] + + full_run.RATEBOOK["startup_allowance_seconds"] + ) + try: + receipt = cloud.submit( + path, + api=api, + secrets=secrets, + expected_plan_sha256=task["plan_sha256"], + ) + except BaseException: + local = path.parent / "submission.json" + if local.is_file(): + _record(task, cloud._loads(local.read_bytes())) + raise + _record(task, receipt) + save() + if ( + receipt["status"] != "submitted" + or receipt["run_id"] != task["run_id"] + or not task.get("job_id") + ): + raise ValueError("Submission identity or outcome unknown") + if task["job_id"] in job_ids: + raise ValueError("Duplicate controller Job ID") + job_ids.add(task["job_id"]) + url = urlsplit(receipt["job_url"]) + parts = url.path.split("/") + if ( + url.scheme != "https" + or url.netloc != "huggingface.co" + or len(parts) != 4 + or parts[:2] != ["", "jobs"] + or parts[-1] != task["job_id"] + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,95}", parts[2]) + ): + raise ValueError("Controller owner URL is invalid") + owner = spec.get("namespace") or parts[2] + if owner != parts[2]: + raise ValueError( + "Controller owner differs from the launch namespace" + ) + task["status"] = "accepted_unmeasured" + active.append((task, spec, owner, deadline)) + save() + save(stage="poll", status="waiting") + while active: + for entry in active[:]: + task, spec, owner, deadline = entry + task["controller_status"] = "UNKNOWN" + stage = task["controller_status"] = _status( + task, spec, owner, deadline, api, secrets, clock + ) + save() + if stage == "COMPLETED": + active.remove(entry) + elif stage not in {"SCHEDULING", "RUNNING"}: + raise RuntimeError("Controller failed or status unknown") + if active: + sleep(max(0, min(30, min(entry[3] - clock() for entry in active)))) + manifest["status"] = "controllers_completed_unmeasured" + save(stage="done") + except BaseException as error: + save(status="failed", error_type=type(error).__name__) + raise RuntimeError( + f"Prepared batch stopped during {manifest['stage']}; inspect manifest.json and per-plan receipts" + ) from None + return manifest + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + names = "prepared-manifest output spend-ledger reservation-id budget-usd" + for name in names.split(): + parser.add_argument("--" + name, required=True) + parser.add_argument("--submit", action="store_true") + args = parser.parse_args(argv) + manifest = run(**vars(args)) + print(f"{manifest['status']}: {Path(args.output).absolute() / 'manifest.json'}") + + +if __name__ == "__main__": + main() diff --git a/examples/hub_validation_pipeline/real_tasks.py b/examples/hub_validation_pipeline/real_tasks.py new file mode 100644 index 000000000..37bace582 --- /dev/null +++ b/examples/hub_validation_pipeline/real_tasks.py @@ -0,0 +1,309 @@ +"""Prepare operator-supplied evaluation-only native tasks without paid work. + +Run ``python -m examples.hub_validation_pipeline.real_tasks --panel PANEL --output DIR``. +The panel supplies pinned source identity, task boundaries and audit controls. +``--bundle`` also calls the local, network-free ``cloud.prepare`` preparer. +Only the selected immutable HF task trees are downloaded anonymously. +""" + +import argparse +import hashlib +import json +import re +import stat +import tempfile +import tomllib +from pathlib import Path + +from huggingface_hub import hf_hub_download, HfApi, RepoFile, RepoFolder +from openenv.validation.runner import source_digest + +from . import cloud, sources + +MAX_FILES = 128 +MAX_FILE_BYTES = 2 * 1024 * 1024 +MAX_TOTAL_BYTES = 8 * 1024 * 1024 +_REQUIRED = {"task.toml", "instruction.md", "solution/solve.sh", "tests/test.sh"} + + +def _load_panel(panel_path): + """Validate one exact byte snapshot and return its provenance.""" + path = Path(panel_path).absolute() + data, _ = cloud._text_file(path) + panel = cloud._loads(data) + if ( + not isinstance(panel, dict) + or panel.get("schema") != "openenv-native-evaluation-panel-v1" + or panel.get("evaluation_only") is not True + ): + raise ValueError("Supply an evaluation-only native panel v1") + source = panel.get("source", {}) + if ( + not isinstance(source, dict) + or source.get("kind") != "hf" + or not isinstance(source.get("repo_id"), str) + or not re.fullmatch(r"[\w.-]+/[\w.-]+", source["repo_id"]) + ): + raise ValueError("Panel source must identify an HF dataset") + sources._pin(source.get("revision"), "hf") + tasks = panel.get("tasks") + if not isinstance(tasks, list) or not tasks: + raise ValueError("Panel requires a nonempty task list") + names, paths = set(), set() + for task in tasks: + if not isinstance(task, dict): + raise ValueError("Panel tasks must be objects") + name = cloud._safe_name(task.get("name", "")) + task_path = cloud._safe_name(task.get("task_path", "")) + if ( + len(name.parts) != 1 + or name.as_posix().casefold() in names + or task_path.as_posix().casefold() in paths + or name.as_posix().lower().startswith(".env") + or any(part.lower().startswith(".env") for part in task_path.parts) + ): + raise ValueError("Panel requires distinct safe task names and paths") + names.add(name.as_posix().casefold()) + paths.add(task_path.as_posix().casefold()) + config, _ = cloud._config(task.get("config", {})) + execution = config["execution"] + if ( + execution["agent_image"] != task.get("image") + or execution["verifier_image"] != task.get("image") + or execution["submission_paths"] != [task.get("submission_path")] + ): + raise ValueError("Panel task and execution boundaries disagree") + if not isinstance(task.get("negative_counterexamples"), list) or not all( + isinstance(item, dict) for item in task["negative_counterexamples"] + ): + raise ValueError("Panel counterexamples must be a list of objects") + if not isinstance(task.get("annotations", {}), dict): + raise ValueError("Panel annotations must be an object") + task["config"] = config + return panel, { + "path": str(path), + "sha256": hashlib.sha256(data).hexdigest(), + "bytes": len(data), + } + + +def _select_tasks(panel, task_names): + tasks = {task["name"]: task for task in panel["tasks"]} + names = list(tasks if task_names is None else task_names) + if ( + not names + or len(set(names)) != len(names) + or any(name not in tasks for name in names) + ): + raise ValueError("Select distinct configured panel task names") + return [tasks[name] for name in names] + + +def acquire_task(task, destination, *, source, api=None, download=None): + """Acquire only one pinned native tree, with size and Git-blob checks.""" + revision = sources._pin(source["revision"], "hf") + repo_id, task_path = source["repo_id"], task["task_path"] + cloud._safe_name(task_path) + destination = Path(destination) + if destination.exists() or destination.is_symlink(): + raise FileExistsError("Acquisition requires a fresh task directory") + destination = destination.absolute() + api = api if api is not None else HfApi(token=False) + download = download if download is not None else hf_hub_download + info = api.dataset_info(repo_id, revision=revision) + if info.sha != revision or info.gated or info.disabled: + raise ValueError("Public source immutable revision could not be verified") + files = {} + total = 0 + for item in api.list_repo_tree( + repo_id, + path_in_repo=task_path, + repo_type="dataset", + revision=revision, + recursive=True, + token=False, + ): + path = cloud._safe_name(item.path) + if not path.is_relative_to(task_path): + raise ValueError("Source listing escaped the selected task tree") + if any(part.lower().startswith(".env") for part in path.parts): + raise ValueError("Environment files are not recipe inputs") + if isinstance(item, RepoFolder): + continue + if not isinstance(item, RepoFile) or item.lfs: + raise ValueError("Only native regular Git source files are supported") + relative = path.relative_to(task_path).as_posix() + if relative.casefold() in {key.casefold() for key in files}: + raise ValueError("Source path collision") + if type(item.size) is not int or not 0 <= item.size <= MAX_FILE_BYTES: + raise ValueError("Source per-file byte cap exceeded") + total += item.size + files[relative] = item + if len(files) > MAX_FILES or total > MAX_TOTAL_BYTES: + raise ValueError("Source file count or total byte cap exceeded") + if not _REQUIRED.issubset(files): + raise ValueError("Native task files are missing") + if sources.task_roots(files) != ["."]: + raise ValueError("Selected native task root is ambiguous") + inventory, payloads = {}, {} + with tempfile.TemporaryDirectory(prefix="openenv-real-task-") as temporary: + staging = Path(temporary).resolve() + for relative, item in sorted(files.items()): + path = Path( + download( + repo_id, + item.path, + repo_type="dataset", + revision=revision, + local_dir=staging, + token=False, + force_download=True, + ) + ) + if path.is_symlink() or path.resolve() != staging / item.path: + raise ValueError("Download must materialize a contained regular file") + if ( + not stat.S_ISREG(path.lstat().st_mode) + or path.stat().st_size != item.size + ): + raise ValueError("Source file type or size disagrees with pinned tree") + data, _ = cloud._text_file(path) + blob = hashlib.sha1(f"blob {len(data)}\0".encode() + data).hexdigest() + if blob != item.blob_id: + raise ValueError("Source Git blob hash mismatch") + inventory[relative] = { + "size": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + "git_blob": blob, + } + payloads[relative] = data + destination.mkdir(parents=True) + for relative, data in payloads.items(): + path = destination / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return inventory + + +def prepare( + output_dir, + *, + panel_path, + task_names=None, + bundle=False, + api=None, + download=None, +): + """Load an explicit operator panel and prepare recipes, never Jobs.""" + panel, provenance = _load_panel(panel_path) + return _prepare( + output_dir, + panel=panel, + panel_provenance=provenance, + task_names=task_names, + bundle=bundle, + api=api, + download=download, + ) + + +def _prepare( + output_dir, + *, + panel, + panel_provenance, + task_names=None, + bundle=False, + api=None, + download=None, +): + tasks = _select_tasks(panel, task_names) + output = Path(output_dir).absolute() + output.mkdir(parents=True, exist_ok=False) + rows = [] + for task in tasks: + name, config = task["name"], task["config"] + task_root = output / "sources" / name + inventory = acquire_task( + task, task_root, source=panel["source"], api=api, download=download + ) + config_path = output / "configs" / f"{name}.json" + config_path.parent.mkdir(exist_ok=True) + config_bytes = cloud._json_bytes(config) + config_path.write_bytes(config_bytes) + native = tomllib.loads((task_root / "task.toml").read_text()) + source = { + **panel["source"], + "task_path": task["task_path"], + "evaluation_panel_sha256": panel_provenance["sha256"], + "task_file_hashes": { + path: record["sha256"] for path, record in inventory.items() + }, + } + row = { + "name": name, + "task_id": f"hf:{source['repo_id']}@{source['revision']}:{source['task_path']}", + "task_root": str(task_root), + "config_path": str(config_path), + "config_sha256": hashlib.sha256(config_bytes).hexdigest(), + "source": source, + "source_digest": source_digest(task_root), + "files": inventory, + "evaluation_only": True, + "negative_counterexamples": task["negative_counterexamples"], + "annotations": task.get("annotations", {}), + "source_image": native["environment"]["docker_image"], + "execution_image": task["image"], + "budget": cloud._config(config)[1], + } + if bundle: + directory = output / "bundles" / name + cloud.prepare(task_root, config, directory, source=source) + row["bundle_plan"] = str(directory / "plan.json") + rows.append(row) + manifest = { + "schema": "openenv-real-task-recipes-v1", + "status": "prepared", + "evaluation_only": True, + "evaluation_panel": panel_provenance, + "scope": "Operator-selected tasks only; no whole-catalog qualification or admission claim.", + "limitations": [ + "No Jobs, inference, containers, builds or pushes are performed by this recipe.", + "Hosted model source revision does not attest provider-served weights.", + "Native task and verifier bytes are preserved; dependency and resource reproducibility require separate qualification.", + ], + "tasks": rows, + } + (output / "manifest.json").write_bytes(cloud._json_bytes(manifest)) + return manifest + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--panel", dest="panel_path", type=Path, required=True) + parser.add_argument("--task", action="append", dest="task_names") + parser.add_argument( + "--bundle", + action="store_true", + help="Also prepare local bundles; does not submit", + ) + args = parser.parse_args(argv) + prepare( + args.output, + panel_path=args.panel_path, + task_names=args.task_names, + bundle=args.bundle, + ) + print( + json.dumps( + { + "manifest": str(args.output.absolute() / "manifest.json"), + "status": "prepared", + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/hub_validation_pipeline/sources.py b/examples/hub_validation_pipeline/sources.py new file mode 100644 index 000000000..0648c5c36 --- /dev/null +++ b/examples/hub_validation_pipeline/sources.py @@ -0,0 +1,343 @@ +"""Discover public Hub bundles and acquire pinned task files without executing them.""" + +import asyncio +import hashlib +import json +import re +import shutil +import tempfile +from importlib.metadata import distribution, PackageNotFoundError +from pathlib import Path, PurePosixPath + +from huggingface_hub import HfApi, snapshot_download + +HARBOR_GIT_REVISION = "9c7a39b991d7e2205019f6be0598cbd22d84bfc3" +HARBOR_INSTALL = ( + "uv pip install --python .venv/bin/python " + f"'harbor @ git+https://github.com/harbor-framework/harbor.git@{HARBOR_GIT_REVISION}'" +) + + +def discover_hf(limit: int) -> list[dict]: + """List the exact Harbor-library AND RL-environment view, in trending order.""" + if limit < 0: + raise ValueError("limit must be nonnegative") + if limit == 0: + return [] + required = {"library:harbor", "rl-environment"} + found = {} + # Exhaust the SDK iterator: its limit is a total cap, not a page size. + for rank, item in enumerate( + HfApi(token=False).list_datasets( + filter=("library:harbor", "rl-environment"), + sort="trending_score", + expand=["sha", "tags", "gated", "disabled"], + limit=None, + ) + ): + if not required.issubset(item.tags or []) or item.id in found: + continue + found[item.id] = { + "source": "hf", + "id": item.id, + "revision": _pin(item.sha, "hf"), + "metadata": { + "tags": item.tags, + "gated": item.gated, + "disabled": item.disabled, + "rank": rank, + }, + } + return list(found.values())[:limit] + + +def _harbor_api(): + # The PyPI 0.25.0 wheel lacks listing; the official Git pin provides it. + try: + installed = distribution("harbor") + origin = json.loads(installed.read_text("direct_url.json") or "{}") + if ( + installed.version != "0.25.0" + or origin.get("vcs_info", {}).get("commit_id") != HARBOR_GIT_REVISION + ): + raise RuntimeError( + f"Harbor source capability mismatch (installed {installed.version}); " + f"native Hub listing requires official Git {HARBOR_GIT_REVISION}. " + f"Install: {HARBOR_INSTALL}" + ) + from harbor.models.task.id import PackageTaskId + from harbor.registry.client.package import PackageDatasetClient + from harbor.tasks.client import TaskClient + except (ImportError, PackageNotFoundError) as error: + raise RuntimeError( + f"Pinned Harbor Hub package APIs unavailable. Install: {HARBOR_INSTALL}" + ) from error + return PackageDatasetClient, PackageTaskId, TaskClient + + +def discover_harbor(limit: int) -> list[dict]: + """List current Harbor Hub dataset versions, retaining their package digests.""" + if limit < 0: + raise ValueError("limit must be nonnegative") + if limit == 0: + return [] + registry, _, _ = _harbor_api() + # This Git-pinned API exhausts pages, as does `harbor dataset list --json`. + try: + rows = asyncio.run(registry().list_datasets()) + except NotImplementedError as error: + raise RuntimeError( + f"Harbor native Hub listing is unsupported. Install: {HARBOR_INSTALL}" + ) from error + found = {} + for row in sorted(rows, key=lambda row: (row.name, row.version or "")): + revision = _pin(row.version, "harbor") + _package_name(row.name) + found.setdefault( + (row.name, revision), + { + "source": "harbor", + "id": row.name, + "revision": revision, + "metadata": { + "kind": "dataset", + "description": row.description, + "task_count": row.task_count, + "harbor_source_revision": HARBOR_GIT_REVISION, + }, + }, + ) + return list(found.values())[:limit] + + +def acquire(candidate: dict, cache_dir: Path, max_tasks: int) -> list[dict]: + """Download at most max_tasks into a fresh revision-addressed directory. + + Errors propagate to the driver for exclusion. Zero selects no tasks. Harbor + single-task candidates use metadata.kind="task"; discovery returns datasets. + """ + if max_tasks < 0: + raise ValueError("max_tasks must be nonnegative") + if max_tasks == 0: + return [] + source = candidate["source"] + revision = _pin(candidate["revision"], source) + if source == "hf": + return _acquire_hf(candidate, cache_dir, revision, max_tasks) + if source == "harbor": + return asyncio.run(_acquire_harbor(candidate, cache_dir, revision, max_tasks)) + raise ValueError(f"Unknown source: {source}") + + +def _pin(value: str, source: str) -> str: + pattern = {"hf": r"[0-9a-f]{40}", "harbor": r"sha256:[0-9a-f]{64}"} + if source not in pattern or not isinstance(value, str): + raise ValueError(f"Invalid source or immutable revision: {source}, {value}") + if not re.fullmatch(pattern[source], value): + raise ValueError(f"Expected immutable {source} revision, got {value!r}") + return value + + +def _relative(value: str) -> PurePosixPath: + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts or "\\" in value or not path.parts: + raise ValueError(f"Unsafe source path: {value!r}") + return path + + +def _package_name(name: str) -> None: + path = _relative(name) + if len(path.parts) != 2 or path.as_posix() != name or re.search(r"[@:\s]", name): + raise ValueError(f"Expected Harbor org/name, got {name!r}") + + +def _fresh(cache_dir: Path, source: str, identity: str, revision: str) -> Path: + identity_hash = hashlib.sha256(identity.encode()).hexdigest()[:16] + parent = cache_dir / source / identity_hash / revision.replace(":", "-") + parent.mkdir(parents=True, exist_ok=True) + return Path(tempfile.mkdtemp(prefix="selection-", dir=parent)).resolve() + + +def _real_files(root: Path) -> None: + """Reject escaping links and materialize contained links as real files.""" + has_links = False + for path in root.rglob("*"): + if path.is_symlink(): + target = path.resolve(strict=True) + if not target.is_relative_to(root): + raise ValueError(f"Unsafe task symlink: {path}") + has_links = True + elif not path.is_dir() and not path.is_file(): + raise ValueError(f"Unsupported task file: {path}") + if has_links: + # copytree also rejects dangling links and directory-link cycles. + with tempfile.TemporaryDirectory(dir=root.parent) as temporary: + materialized = Path(temporary) / "files" + shutil.copytree(root, materialized, symlinks=False) + shutil.rmtree(root) + materialized.rename(root) + + +def _record(source, identity, revision, root, task_path, metadata, **extra): + return { + "task_id": f"{source}:{identity}@{revision}:{task_path}", + "path": str((root / task_path).resolve(strict=True)), + "source": source, + "source_id": identity, + "revision": revision, + "task_path": task_path, + "acquisition_ref": f"{source}:{identity}@{revision}", + "metadata": metadata, + **extra, + } + + +def task_roots(files): + """A nested task.toml inside an existing task is an asset, not another task.""" + roots = { + PurePosixPath(name).parent + for name in files + if PurePosixPath(name).name == "task.toml" + } + return sorted( + str(root) + for root in roots + if not any(parent in roots for parent in root.parents) + ) + + +def _acquire_hf(candidate, cache_dir, revision, max_tasks): + metadata = candidate.get("metadata", {}) + if metadata.get("gated") or metadata.get("disabled"): + raise PermissionError(f"HF source is gated or disabled: {candidate['id']}") + api = HfApi(token=False) + info = api.dataset_info(candidate["id"], revision=revision) + if info.sha != revision: + raise ValueError("HF revision resolution disagrees with the selected commit") + if info.gated or info.disabled: + raise PermissionError(f"HF source is gated or disabled: {candidate['id']}") + files = sorted( + { + _relative(name).as_posix() + for name in api.list_repo_files( + candidate["id"], repo_type="dataset", revision=revision + ) + } + ) + roots = set(task_roots(files)) + selected = sorted(roots)[:max_tasks] + if not selected: + raise ValueError(f"No task.toml files in {candidate['id']}@{revision}") + # Retain shared repository assets while excluding every unselected task tree. + selected_files = [] + for name in files: + owner = next( + ( + str(parent) + for parent in PurePosixPath(name).parents + if str(parent) in roots + ), + None, + ) + if owner is None or owner in selected: + selected_files.append(name) + # Literal filename patterns prevent [brackets], ? and * from widening a subset. + patterns = [ + "".join({"[": "[[]", "?": "[?]", "*": "[*]"}.get(char, char) for char in name) + for name in selected_files + ] + root = _fresh(cache_dir, "hf", candidate["id"], revision) + snapshot_download( + candidate["id"], + repo_type="dataset", + revision=revision, + local_dir=root, + allow_patterns=patterns, + token=False, + force_download=True, + ) + _real_files(root) + for name in selected_files: + if not (root / name).is_file(): + raise FileNotFoundError(f"HF download did not materialize {name}") + return [ + _record( + "hf", + candidate["id"], + revision, + root, + task_path, + metadata, + download_patterns=patterns, + ) + for task_path in selected + ] + + +async def _acquire_harbor(candidate, cache_dir, revision, max_tasks): + registry_type, task_id_type, task_client_type = _harbor_api() + _package_name(candidate["id"]) + registry = registry_type() + metadata = candidate.get("metadata", {}) + kind = metadata.get("kind", "dataset") + bundle = None + if kind == "dataset": + bundle = await registry.get_dataset_metadata(f"{candidate['id']}@{revision}") + if bundle.version != revision: + raise ValueError("Harbor dataset resolution disagrees with its digest") + tasks = sorted( + bundle.task_ids, key=lambda task: (task.org, task.name, task.ref) + )[:max_tasks] + elif kind == "task": + org, name = candidate["id"].split("/") + tasks = [task_id_type(org=org, name=name, ref=revision)] + else: + raise ValueError(f"Unknown Harbor package kind: {kind}") + if not tasks: + raise ValueError(f"Harbor dataset contains no tasks: {candidate['id']}") + for task in tasks: + if not isinstance(task, task_id_type): + raise ValueError("Current Harbor Hub acquisition requires package task IDs") + _package_name(f"{task.org}/{task.name}") + _pin(task.ref, "harbor") + root = _fresh(cache_dir, "harbor", candidate["id"], revision) + dataset_files = {} + if bundle is not None: + for file in bundle.files: + _relative(file.path) + dataset_files = await registry.download_dataset_files( + bundle, output_dir=root / "dataset", overwrite=True + ) + downloaded = await task_client_type().download_tasks( + task_ids=tasks, output_dir=root / "tasks", overwrite=True, export=False + ) + _real_files(root) + records = [] + for task, result in zip(tasks, downloaded.results, strict=True): + if result.content_hash != task.ref.removeprefix("sha256:"): + raise ValueError("Harbor task resolution disagrees with its digest") + task_root = result.path.resolve(strict=True) + if not task_root.is_relative_to(root): + raise ValueError("Harbor downloaded a task outside the selected cache") + if not (task_root / "task.toml").is_file(): + raise ValueError( + f"Harbor package contains no root task.toml: {task.org}/{task.name}" + ) + records.append( + _record( + "harbor", + f"{task.org}/{task.name}", + task.ref, + task_root, + ".", + { + **metadata, + "candidate_id": candidate["id"], + "candidate_revision": revision, + "dataset_files": { + name: str(path) for name, path in dataset_files.items() + }, + }, + ) + ) + return records[:max_tasks] diff --git a/examples/hub_validation_pipeline/spend.py b/examples/hub_validation_pipeline/spend.py new file mode 100644 index 000000000..65a98a3ed --- /dev/null +++ b/examples/hub_validation_pipeline/spend.py @@ -0,0 +1,288 @@ +"""Offline shared spend ledger; amounts are decimal strings, Decimal, or integers.""" + +import argparse +import fcntl +import json +import os +import re +import tempfile +from contextlib import contextmanager +from copy import deepcopy +from datetime import datetime, timezone +from decimal import Decimal +from fractions import Fraction +from pathlib import Path + + +def _amount(value): + if type(value) not in (str, Decimal, int) or not re.fullmatch( + r"\+?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?", str(value) + ): + raise ValueError("Amount must be a nonnegative finite decimal") + return Fraction(Decimal(value)) + + +def _text(value): + if not isinstance(value, str) or not value.strip(): + raise ValueError("ID, purpose and evidence must be nonempty strings") + return value + + +def _pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("Duplicate ledger key") + result[key] = value + return result + + +def _report(data): + if not isinstance(data, dict) or data.get("schema") != "openenv-shared-spend-v1": + raise ValueError("Invalid ledger schema") + limit, reserved, spent = _amount(data["limit_usd"]), Fraction(0), Fraction(0) + changes = data.get("limit_changes", []) + if not isinstance(changes, list): + raise ValueError("Invalid limit change history") + previous = None + for change in changes: + if ( + not isinstance(change, dict) + or not {"previous_limit_usd", "limit_usd", "evidence", "changed_utc"} + <= change.keys() + ): + raise ValueError("Invalid limit change history") + old, new = _amount(change["previous_limit_usd"]), _amount(change["limit_usd"]) + _text(change["evidence"]) + changed = datetime.fromisoformat(_text(change["changed_utc"])) + if ( + changed.utcoffset() != timezone.utc.utcoffset(None) + or old == new + or (previous is not None and previous != old) + ): + raise ValueError("Invalid limit change history") + previous = new + if changes and previous != limit: + raise ValueError("Limit change history differs from current limit") + over = data.get("over_budget", False) + statuses = {} + for identifier, row in data["reservations"].items(): + amount = _amount(row["reserved_usd"]) + statuses[identifier] = row["status"] + if row["status"] == "reserved" and row["spent_usd"] is None: + reserved += amount + elif row["status"] == "settled": + _text(row.get("evidence")) + actual = _amount(row["spent_usd"]) + spent += actual + over = over or actual > amount + else: + raise ValueError("Invalid reservation status") + totals = dict(limit=limit, reserved=reserved, spent=spent, charged=reserved + spent) + totals["remaining"] = limit - totals["charged"] + result = { + "over_budget": bool(over or totals["charged"] > limit), + "statuses": statuses, + } + for key, value in totals.items(): + micros = -((-value.numerator * 1000000) // value.denominator) + sign, whole = ("-" if micros < 0 else ""), abs(micros) + result[f"{key}_usd_exact"] = str(value) + result[f"{key}_usd_rounded_up"] = ( + f"{sign}{whole // 1000000}.{whole % 1000000:06d}" + ) + return result + + +@contextmanager +def _ledger(path, limit=None): + path = Path(path).resolve() + with path.with_name(path.name + ".lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + source = None + if path.exists(): + source = path.read_bytes().decode("utf-8") + data = json.loads(source, object_pairs_hook=_pairs) + elif limit is not None: + data = dict( + schema="openenv-shared-spend-v1", + limit_usd=str(limit), + reservations={}, + created_utc=datetime.now(timezone.utc).isoformat(), + ) + else: + raise ValueError("Ledger must be initialized first") + _report(data) + before = deepcopy(data) + yield data + if data != before or not path.exists(): + data["updated_utc"] = datetime.now(timezone.utc).isoformat() + with tempfile.NamedTemporaryFile( + mode="w", dir=path.parent, delete=False + ) as out: + temporary = Path(out.name) + try: + rendered = json.dumps(data, indent=2, allow_nan=False) + if ( + source is not None + and data["reservations"] == before["reservations"] + ): + # Decode top-level values to retain the original reservation bytes. + decoder = json.JSONDecoder() + index = source.index("{") + 1 + for _ in before: + index += re.match(r"\s*", source[index:]).end() + key, index = decoder.raw_decode(source, index) + index = source.index(":", index) + 1 + index += re.match(r"\s*", source[index:]).end() + _, end = decoder.raw_decode(source, index) + if key == "reservations": + field = json.dumps( + {"reservations": data["reservations"]}, indent=2 + )[1:-2] + rendered = rendered.replace( + field, '\n "reservations": ' + source[index:end], 1 + ) + break + index = source.index(",", end) + 1 + out.write(rendered) + out.write("\n") + out.flush() + os.fsync(out.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + # A prior replace may have succeeded before directory syncing failed. + # Idempotent retries must confirm that durability before acknowledging. + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def init(path, limit_usd): + """Initialize once; an existing ledger must have the same limit.""" + limit = _amount(limit_usd) + with _ledger(path, str(limit_usd)) as data: + if _amount(data["limit_usd"]) != limit: + raise ValueError("Existing ledger limit differs") + return _report(data) + + +def set_limit(path, limit_usd, evidence): + """Record an evidenced limit decision, retaining all holds and overrun blocks.""" + limit = _amount(limit_usd) + _text(evidence) + with _ledger(path) as data: + report = _report(data) + if _amount(data["limit_usd"]) == limit: + return report + data.setdefault("limit_changes", []).append( + dict( + previous_limit_usd=data["limit_usd"], + limit_usd=str(limit_usd), + evidence=evidence, + changed_utc=datetime.now(timezone.utc).isoformat(), + ) + ) + data["limit_usd"] = str(limit_usd) + if report["over_budget"] or _report(data)["over_budget"]: + data["over_budget"] = True + return _report(data) + + +def reserve(path, id, amount_usd, purpose, evidence=None): + """Reserve before paid work; settled IDs cannot be reused.""" + amount = _amount(amount_usd) + _text(id), _text(purpose) + if evidence is not None: + _text(evidence) + with _ledger(path) as data: + report = _report(data) + rows = data["reservations"] + if id in rows: + if ( + rows[id]["status"] == "reserved" + and _amount(rows[id]["reserved_usd"]) == amount + ): + return report + raise ValueError("Reservation ID already used; use a fresh ID") + if report["over_budget"] or amount > Fraction(report["remaining_usd_exact"]): + raise ValueError("Shared budget exceeded or blocked") + rows[id] = dict( + reserved_usd=str(amount_usd), + spent_usd=None, + status="reserved", + purpose=purpose, + evidence=evidence, + ) + return _report(data) + + +def settle(path, id, spent_usd, evidence): + """Record actual cost, freezing new reservations on any reservation/cap overrun.""" + actual = _amount(spent_usd) + _text(id), _text(evidence) + with _ledger(path) as data: + rows = data["reservations"] + if id not in rows: + raise ValueError("Unknown reservation ID") + row = rows[id] + if row["status"] == "settled": + if _amount(row["spent_usd"]) == actual: + return _report(data) + raise ValueError("Different settlement requires manual reconciliation") + row.update( + status="settled", + spent_usd=str(spent_usd), + evidence=evidence, + settled_utc=datetime.now(timezone.utc).isoformat(), + ) + report = _report(data) + data["over_budget"] = report["over_budget"] + return report + + +def summary(path): + """Return exact fraction totals, upward-rounded six-place displays and ID statuses.""" + with _ledger(path) as data: + return _report(data) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ledger", required=True) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("summary") + sub = commands.add_parser("set-limit") + sub.add_argument("--usd", required=True) + sub.add_argument("--evidence", required=True) + for command in ("reserve", "settle"): + sub = commands.add_parser(command) + sub.add_argument("--id", required=True) + sub.add_argument("--usd", required=True) + sub.add_argument("--evidence", required=command == "settle") + if command == "reserve": + sub.add_argument("--purpose", required=True) + args = parser.parse_args(argv) + try: + if args.command == "set-limit": + result = set_limit(args.ledger, args.usd, args.evidence) + elif args.command == "reserve": + result = reserve( + args.ledger, args.id, args.usd, args.purpose, args.evidence + ) + elif args.command == "settle": + result = settle(args.ledger, args.id, args.usd, args.evidence) + else: + result = summary(args.ledger) + print(json.dumps(result, sort_keys=True)) + except (ValueError, TypeError, KeyError, AttributeError, OSError): + parser.exit( + 2, "Spend operation rejected; inspect ledger and arguments locally.\n" + ) + + +if __name__ == "__main__": + main() diff --git a/examples/hub_validation_pipeline/test_cloud.py b/examples/hub_validation_pipeline/test_cloud.py new file mode 100644 index 000000000..8949bbfcc --- /dev/null +++ b/examples/hub_validation_pipeline/test_cloud.py @@ -0,0 +1,754 @@ +"""Offline cloud planning, input integrity, one-shot launch and failure evidence.""" + +import hashlib +import io +import json +import shutil +import stat +import subprocess +import tempfile +import urllib.request +import zipfile +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from examples.hub_validation_pipeline import cloud + +SHA = "a" * 40 +UPLOAD_SHA = "b" * 40 +SECRETS = { + "HF_TOKEN": "results-write-credential-123", + "HF_INFERENCE_TOKEN": "inference-only-credential-456", + "HF_SANDBOX_TOKEN": "execution-only-credential-789", +} + + +@pytest.fixture +def inputs(tmp_path, monkeypatch): + repo = tmp_path / "repo" + (repo / "src/openenv").mkdir(parents=True) + (repo / "src/openenv/__init__.py").write_text("# source\n") + (repo / "examples/hub_validation_pipeline").mkdir(parents=True) + (repo / "examples/hub_validation_pipeline/cloud.py").write_bytes( + Path(cloud.__file__).read_bytes() + ) + shutil.copytree( + Path(cloud.__file__).parents[1] / "coding_audit", + repo / "examples/coding_audit", + ignore=shutil.ignore_patterns("test_*", "__pycache__", ".pytest_cache"), + ) + (repo / "examples/coding_audit/test_excluded.py").write_text( + "raise AssertionError\n" + ) + for name in ("pyproject.toml", "README.md", "LICENSE", "uv.lock"): + (repo / name).write_text(name + "\n") + task = tmp_path / "task" + (task / "tests").mkdir(parents=True) + (task / "solution").mkdir() + (task / "task.toml").write_text("version = '1.0'\n") + (task / "instruction.md").write_text("Write an answer\n") + (task / "tests/test.sh").write_text("printf 1\n") + (task / "solution/solve.sh").write_text("printf answer > /app/answer\n") + (task / "solution/solve.sh").chmod(0o755) + config = { + "schema_version": 1, + "target": { + "model_id": "org/model", + "revision": SHA, + "provider": "example-provider", + "max_steps": 3, + }, + "execution": { + "agent_image": "registry/agent@sha256:" + "c" * 64, + "verifier_image": "registry/verifier@sha256:" + "d" * 64, + "submission_paths": ["/app/answer"], + }, + "validation": { + "reference_repeats": 2, + "target_attempts": 2, + "required_negative_cases": ["wrong", "empty"], + }, + "generation": {"candidate_count": 2}, + "negative_cases": [ + {"id": "wrong", "script": "printf wrong > /app/answer"}, + {"id": "empty", "script": "true"}, + ], + "seeds": [7, 13], + "cloud": { + "results_repo": "org/private-results", + "namespace": "org", + "flavor": "cpu-basic", + "timeout_seconds": 1800, + "max_sandboxes": 40, + }, + } + monkeypatch.setattr( + cloud, "_git_provenance", lambda root: {"commit": SHA, "dirty": True} + ) + return repo, task, config + + +def prepared(inputs, tmp_path, name="prepared"): + repo, task, config = inputs + output = tmp_path / name + plan = cloud.prepare(task, config, output, repo_root=repo) + return output, plan + + +def fake_api(): + api = Mock() + api.dataset_info.return_value = SimpleNamespace(private=True, sha=SHA) + api.list_repo_files.return_value = ["corpus/task.toml"] + api.create_commit.return_value = SimpleNamespace(oid=UPLOAD_SHA) + api.run_job.return_value = SimpleNamespace( + id="job-123", url="https://huggingface.co/jobs/org/job-123" + ) + return api + + +def worker_root(inputs, tmp_path, monkeypatch): + output, plan = prepared(inputs, tmp_path) + root = tmp_path / "extracted" + cloud.extract_bundle( + (output / "bundle.zip").read_bytes(), root, plan["bundle_sha256"] + ) + for name, value in SECRETS.items(): + monkeypatch.setenv(name, value) + monkeypatch.setenv("OPENENV_RESULTS_REPO", plan["results_repo"]) + return root, plan + + +def rewrite_archive(data, *, changes=None, additions=None, remove=()): + output = io.BytesIO() + with zipfile.ZipFile(io.BytesIO(data)) as old, zipfile.ZipFile(output, "w") as new: + for member in old.infolist(): + if member.filename not in remove: + new.writestr( + member, (changes or {}).get(member.filename, old.read(member)) + ) + for name, content, mode in additions or []: + member = zipfile.ZipInfo(name) + member.create_system = 3 + member.external_attr = mode << 16 + new.writestr(member, content) + return output.getvalue() + + +def test_prepare_is_deterministic_offline_and_complete(inputs, tmp_path, monkeypatch): + network = Mock(side_effect=AssertionError("No network")) + monkeypatch.setattr(urllib.request, "urlopen", network) + repo, task, config = inputs + (repo / "src/openenv/__pycache__").mkdir() + (repo / "src/openenv/__pycache__/cached.pyc").write_bytes(b"\x00binary") + (repo / ".env").write_text("never bundle me") + (repo / ".venv").mkdir() + output, plan = prepared(inputs, tmp_path) + second, again = prepared(inputs, tmp_path, "second") + assert plan == again + data = (output / "bundle.zip").read_bytes() + assert data == (second / "bundle.zip").read_bytes() + assert plan["bundle_sha256"] == hashlib.sha256(data).hexdigest() + manifest, files = cloud._checked_archive(data, plan["bundle_sha256"]) + assert set(files) == set(manifest["files"]) | {cloud.MANIFEST} + assert files["uv.lock"] == b"uv.lock\n" + assert { + "examples/coding_audit/" + name + for name in ("__init__.py", "__main__.py", "scoring.py", "hf.py", "runner.py") + } <= files.keys() + assert "examples/coding_audit/test_excluded.py" not in files + assert manifest["git"] == {"commit": SHA, "dirty": True} + assert manifest["target_source"]["revision"] == SHA + assert manifest["task_source"]["identity"].startswith("sha256:") + assert manifest["files"]["task/solution/solve.sh"]["mode"] == 0o755 + assert not any( + ".env" in name or ".venv" in name or "__pycache__" in name for name in files + ) + assert plan["budget"]["sandbox_jobs_max"] == 28 + assert plan["budget"]["sandbox_remote_job_lifetime_seconds"] == 86400 + assert plan["budget"]["concurrent_sandbox_jobs_max"] == 1 + assert plan["budget"]["inference_calls_max"] == 8 + assert plan["mcp_request"]["args"]["image"] == cloud.WORKER_IMAGE + assert plan["mcp_request"]["required_env_after_upload"] == [ + "OPENENV_BUNDLE_REVISION" + ] + assert "secrets" not in plan["mcp_request"]["args"] + network.assert_not_called() + (task / "instruction.md").write_text("different task\n") + _, changed = prepared(inputs, tmp_path, "changed") + assert changed["bundle_sha256"] != plan["bundle_sha256"] + + +def test_prepare_fresh_directory_and_locked_inputs(inputs, tmp_path): + repo, task, config = inputs + output, _ = prepared(inputs, tmp_path) + with pytest.raises(FileExistsError, match="fresh"): + cloud.prepare(task, config, output, repo_root=repo) + with pytest.raises(ValueError, match="outside"): + cloud.prepare(task, config, task / "output", repo_root=repo) + (repo / "uv.lock").unlink() + with pytest.raises(FileNotFoundError): + prepared(inputs, tmp_path, "missing-lock") + + +@pytest.mark.parametrize("bad", ["link", "dirlink", "binary", "secret", "env", "fifo"]) +def test_prepare_rejects_unsafe_local_inputs(inputs, tmp_path, bad): + _, task, _ = inputs + if bad == "link": + (task / "link").symlink_to(task / "instruction.md") + elif bad == "dirlink": + (task / "link").symlink_to(task / "tests", target_is_directory=True) + elif bad == "binary": + (task / "binary").write_bytes(b"\x00binary") + elif bad == "secret": + (task / "secret.txt").write_text("hf_" + "z" * 30) + elif bad == "env": + (task / ".env.local").write_text("password=123") + else: + import os + + os.mkfifo(task / "fifo") + with pytest.raises((ValueError, UnicodeDecodeError)): + prepared(inputs, tmp_path) + assert not (tmp_path / "prepared").exists() + + +@pytest.mark.parametrize( + "kind", + ["timeout", "sandboxes", "calls", "tag", "provider", "revision", "credentials"], +) +def test_preflight_rejects_unbounded_or_mutable_config(inputs, tmp_path, kind): + config = inputs[2] + if kind == "timeout": + config["cloud"]["timeout_seconds"] = 0 + elif kind == "sandboxes": + config["cloud"]["max_sandboxes"] = 1 + elif kind == "calls": + config["cloud"]["max_inference_calls"] = 1 + elif kind == "tag": + config["execution"]["agent_image"] = "python:latest" + elif kind == "provider": + config["target"]["provider"] = "auto" + elif kind == "revision": + config["target"]["revision"] = "main" + else: + config["execution"]["token"] = "do-not-persist" + with pytest.raises(ValueError): + prepared(inputs, tmp_path) + + +def test_archive_digest_inventory_and_size_limits(inputs, tmp_path, monkeypatch): + output, plan = prepared(inputs, tmp_path) + data = (output / "bundle.zip").read_bytes() + with pytest.raises(ValueError, match="SHA256"): + cloud.extract_bundle(data, tmp_path / "bad", "0" * 64) + for changed in ( + rewrite_archive(data, changes={"task/instruction.md": b"tampered"}), + rewrite_archive(data, remove=["task/instruction.md"]), + rewrite_archive(data, additions=[("extra", b"extra", stat.S_IFREG | 0o644)]), + ): + with pytest.raises(ValueError, match="inventory"): + cloud.extract_bundle( + changed, tmp_path / "bad", hashlib.sha256(changed).hexdigest() + ) + assert not (tmp_path / "bad").exists() + monkeypatch.setattr(cloud, "MAX_TOTAL_BYTES", 1) + with pytest.raises(ValueError, match="size limit"): + cloud.extract_bundle(data, tmp_path / "bad", plan["bundle_sha256"]) + + +@pytest.mark.parametrize( + "name,mode", + [ + ("../escape", stat.S_IFREG | 0o644), + ("/absolute", stat.S_IFREG | 0o644), + ("task\\escape", stat.S_IFREG | 0o644), + ("task//escape", stat.S_IFREG | 0o644), + ("C:escape", stat.S_IFREG | 0o644), + ("task/instruction.md/child", stat.S_IFREG | 0o644), + ("TASK/INSTRUCTION.MD", stat.S_IFREG | 0o644), + ("link", stat.S_IFLNK | 0o777), + ("directory", stat.S_IFDIR | 0o755), + ], +) +def test_archive_rejects_path_collisions_and_nonregular_files( + inputs, tmp_path, name, mode +): + output, _ = prepared(inputs, tmp_path) + data = rewrite_archive( + (output / "bundle.zip").read_bytes(), additions=[(name, b"payload", mode)] + ) + with pytest.raises(ValueError): + cloud.extract_bundle(data, tmp_path / "bad", hashlib.sha256(data).hexdigest()) + assert not (tmp_path / "bad").exists() + + +def test_bootstrap_uses_pinned_download_and_frozen_sync(inputs, tmp_path, monkeypatch): + output, plan = prepared(inputs, tmp_path) + data = (output / "bundle.zip").read_bytes() + request = Mock(return_value=io.BytesIO(data)) + monkeypatch.setattr( + urllib.request, "build_opener", lambda *args: SimpleNamespace(open=request) + ) + destination = tmp_path / "bootstrap" + destination.mkdir() + monkeypatch.setattr(tempfile, "mkdtemp", lambda **kwargs: str(destination)) + run = Mock() + monkeypatch.setattr(subprocess, "run", run) + spec = cloud._launch_spec( + cloud._config(inputs[2])[0], plan["bundle_sha256"], UPLOAD_SHA + ) + for name, value in {**spec["env"], **SECRETS}.items(): + monkeypatch.setenv(name, value) + exec(spec["command"][2], {}) + assert "/resolve/" + UPLOAD_SHA + "/inputs/" in request.call_args.args[0].full_url + assert ( + request.call_args.args[0].get_header("Authorization") + == "Bearer " + SECRETS["HF_TOKEN"] + ) + calls = [call.args[0] for call in run.call_args_list] + assert calls[0][-2:] == ["install", "uv==0.9.3"] + assert calls[1] == ["uv", "sync", "--frozen", "--no-dev"] + assert calls[2][1:4] == ["-m", "examples.hub_validation_pipeline.cloud", "worker"] + assert ( + destination / "bundle/task/instruction.md" + ).read_bytes() == b"Write an answer\n" + + +def test_submit_real_runtime_secrets_pins_and_one_shot_receipt(inputs, tmp_path): + output, plan = prepared(inputs, tmp_path) + api = fake_api() + receipt = cloud.submit( + output / "plan.json", + api=api, + secrets=SECRETS, + expected_plan_sha256=hashlib.sha256( + (output / "plan.json").read_bytes() + ).hexdigest(), + ) + assert receipt["job_id"] == "job-123" + launch = api.run_job.call_args.kwargs + assert launch["secrets"] == SECRETS + assert launch["token"] == SECRETS["HF_SANDBOX_TOKEN"] + assert launch["env"]["OPENENV_BUNDLE_REVISION"] == UPLOAD_SHA + assert launch["timeout"] == "1800s" + assert launch["image"] == cloud.WORKER_IMAGE + commit = api.create_commit.call_args.kwargs + assert commit["parent_commit"] == SHA + assert commit["create_pr"] is False + assert [op.path_in_repo for op in commit["operations"]] == [ + "inputs/" + plan["bundle_sha256"] + "/bundle.zip", + "inputs/" + plan["bundle_sha256"] + "/manifest.json", + "runs/" + plan["run_id"] + "/claim.json", + ] + for path in output.iterdir(): + for value in SECRETS.values(): + assert value.encode() not in path.read_bytes() + with pytest.raises(FileExistsError, match="receipt"): + cloud.submit(output / "plan.json", api=api, secrets=SECRETS) + assert api.run_job.call_count == 1 + + +@pytest.mark.parametrize("case", ["public", "missing", "duplicate", "conflict"]) +def test_no_launch_until_private_repo_and_exclusive_claim(inputs, tmp_path, case): + output, plan = prepared(inputs, tmp_path) + api = fake_api() + if case == "public": + api.dataset_info.return_value.private = False + elif case == "missing": + api.dataset_info.side_effect = RuntimeError("missing") + elif case == "duplicate": + api.list_repo_files.return_value = ["runs/" + plan["run_id"] + "/claim.json"] + else: + api.create_commit.side_effect = RuntimeError("parent conflict") + with pytest.raises((ValueError, RuntimeError, FileExistsError)): + cloud.submit(output / "plan.json", api=api, secrets=SECRETS) + api.run_job.assert_not_called() + if case in ("public", "missing", "duplicate"): + api.create_commit.assert_not_called() + + +def test_uncertain_launch_never_retries_or_leaks_exception(inputs, tmp_path): + output, _ = prepared(inputs, tmp_path) + api = fake_api() + api.run_job.side_effect = RuntimeError(SECRETS["HF_TOKEN"]) + with pytest.raises(RuntimeError, match="unknown") as caught: + cloud.submit(output / "plan.json", api=api, secrets=SECRETS) + assert SECRETS["HF_TOKEN"] not in str(caught.value) + receipt = json.loads((output / "submission.json").read_text()) + assert receipt["status"] == "launch_unknown" + with pytest.raises(FileExistsError): + cloud.submit(output / "plan.json", api=api, secrets=SECRETS) + assert api.run_job.call_count == 1 + + +@pytest.mark.parametrize( + "values", + [ + {}, + {**SECRETS, "HF_INFERENCE_TOKEN": "$HF_INFERENCE_TOKEN"}, + {**SECRETS, "HF_SANDBOX_TOKEN": SECRETS["HF_TOKEN"]}, + ], +) +def test_secret_values_required_and_distinct(inputs, tmp_path, values): + output, _ = prepared(inputs, tmp_path) + api = fake_api() + with pytest.raises(ValueError): + cloud.submit(output / "plan.json", api=api, secrets=values) + api.dataset_info.assert_not_called() + api.run_job.assert_not_called() + + +@pytest.mark.parametrize("section", ["launch", "git", "task_source", "target_source"]) +def test_plan_tampering_stops_before_hub(inputs, tmp_path, section): + output, _ = prepared(inputs, tmp_path) + path = output / "plan.json" + plan = json.loads(path.read_text()) + if section == "launch": + plan["mcp_request"]["args"]["image"] = "python:latest" + else: + plan[section] = { + "tampered": "reviewed provenance no longer matches bundled input" + } + path.write_text(json.dumps(plan)) + api = fake_api() + with pytest.raises(ValueError, match="immutable"): + cloud.submit(path, api=api, secrets=SECRETS) + api.dataset_info.assert_not_called() + + +@pytest.mark.parametrize("fail", [False, True]) +def test_worker_persists_report_and_partial_evidence_on_failure( + inputs, tmp_path, monkeypatch, fail +): + root, plan = worker_root(inputs, tmp_path, monkeypatch) + api = fake_api() + + def runner(task, config, output): + assert task == root / "task" + assert not output.exists() + output.mkdir() + (output / "evidence.json").write_text('{"attempted":1}\n') + if fail: + raise RuntimeError(SECRETS["HF_TOKEN"]) + return {"accepted": False, "verdict": "skip"} + + if fail: + with pytest.raises(RuntimeError, match="persisted"): + cloud.worker(root, bundle_sha=plan["bundle_sha256"], api=api, runner=runner) + else: + report = cloud.worker( + root, bundle_sha=plan["bundle_sha256"], api=api, runner=runner + ) + assert report["status"] == "completed" + assert report["accepted"] is False + commit = api.create_commit.call_args.kwargs + operations = {op.path_in_repo: op.path_or_fileobj for op in commit["operations"]} + prefix = "runs/" + plan["run_id"] + "/" + assert prefix + "audit/evidence.json" in operations + report = json.loads(operations[prefix + "cloud-report.json"]) + assert report["status"] == ("error" if fail else "completed") + assert report["accepted"] is False + if fail: + assert report["error_type"] == "RuntimeError" + assert all(name.startswith(prefix) for name in operations) + assert all(SECRETS["HF_TOKEN"].encode() not in data for data in operations.values()) + + +def test_worker_inventory_failure_still_persists_error(inputs, tmp_path, monkeypatch): + root, plan = worker_root(inputs, tmp_path, monkeypatch) + (root / "task/instruction.md").write_text("modified") + api, runner = fake_api(), Mock() + with pytest.raises(RuntimeError, match="persisted"): + cloud.worker(root, bundle_sha=plan["bundle_sha256"], api=api, runner=runner) + runner.assert_not_called() + assert ( + json.loads((root / "worker-output/cloud-report.json").read_text())["status"] + == "error" + ) + api.create_commit.assert_called_once() + + +def test_worker_upload_failure_retains_local_report(inputs, tmp_path, monkeypatch): + root, plan = worker_root(inputs, tmp_path, monkeypatch) + api = fake_api() + api.create_commit.side_effect = RuntimeError("Hub unavailable") + with pytest.raises(RuntimeError, match="upload failed"): + cloud.worker( + root, + bundle_sha=plan["bundle_sha256"], + api=api, + runner=lambda *args: {"accepted": True}, + ) + report = json.loads((root / "worker-output/cloud-report.json").read_text()) + assert report["persistence_status"] == "error" + + +def test_admission_thresholds_survive_bundle_roundtrip_exactly(inputs, tmp_path): + from decimal import Decimal + + repo, task, config = inputs + config["validation"]["reward_max"] = Decimal("1.00000000000000000001") + cloud.prepare(task, config, tmp_path / "precise", repo_root=repo) + _, restored, _, _ = cloud._read_plan(tmp_path / "precise/plan.json") + assert restored["validation"]["reward_max"] == Decimal("1.00000000000000000001") + + +@pytest.mark.parametrize("kind", ["hf", "harbor", "local"]) +def test_pinned_discovery_sources_bind_into_cloud_bundle(inputs, tmp_path, kind): + from openenv.validation.runner import source_digest + + repo, task, config = inputs + revision = { + "hf": "a" * 40, + "harbor": "sha256:" + "b" * 64, + "local": source_digest(task), + }[kind] + source = { + "source": kind, + "source_id": "org/repo", + "revision": revision, + "task_path": ".", + } + plan = cloud.prepare(task, config, tmp_path / kind, repo_root=repo, source=source) + assert plan["task_source"]["kind"] == kind + assert plan["task_source"]["revision"] == revision + + +def test_duplicate_json_keys_rejected(): + with pytest.raises(ValueError, match="Duplicate"): + cloud._loads('{"cloud":{},"cloud":{}}') + + +def test_bootstrap_persists_dependency_failure_without_sdk( + inputs, tmp_path, monkeypatch +): + import base64 + + output, plan = prepared(inputs, tmp_path) + data = (output / "bundle.zip").read_bytes() + requests = [] + + def open_request(request, **kwargs): + requests.append(request) + if request.get_method() == "POST": + return io.BytesIO(b'{"commitOid":"done"}') + if "/resolve/" in request.full_url: + return io.BytesIO(data) + return io.BytesIO(b'{"private":true}') + + monkeypatch.setattr( + urllib.request, "build_opener", lambda *args: SimpleNamespace(open=open_request) + ) + destination = tmp_path / "bootstrap" + destination.mkdir() + monkeypatch.setattr(tempfile, "mkdtemp", lambda **kwargs: str(destination)) + monkeypatch.setattr( + subprocess, "run", Mock(side_effect=RuntimeError(SECRETS["HF_TOKEN"])) + ) + spec = cloud._launch_spec( + cloud._config(inputs[2])[0], plan["bundle_sha256"], UPLOAD_SHA + ) + for name, value in {**spec["env"], **SECRETS}.items(): + monkeypatch.setenv(name, value) + with pytest.raises(RuntimeError, match="install_uv") as caught: + exec(spec["command"][2], {}) + assert SECRETS["HF_TOKEN"] not in str(caught.value) + commit = requests[-1] + assert ( + commit.full_url + == "https://huggingface.co/api/datasets/org/private-results/commit/main" + ) + rows = [json.loads(line) for line in commit.data.splitlines()] + assert ( + rows[1]["value"]["path"] == "runs/" + plan["run_id"] + "/bootstrap-error.json" + ) + report = json.loads(base64.b64decode(rows[1]["value"]["content"])) + assert report["stage"] == "install_uv" + assert report["accepted"] is False + assert report["error_type"] == "RuntimeError" + + +def test_cross_host_redirect_drops_results_token(): + redirect = cloud._HubRedirect() + request = urllib.request.Request( + "https://huggingface.co/resolve/blob", + headers={"Authorization": "Bearer private", "Range": "bytes=0-"}, + ) + target = redirect.redirect_request( + request, None, 302, "Found", {}, "https://cdn.example/blob" + ) + assert target.get_header("Authorization") is None + assert target.get_header("Range") == "bytes=0-" + same = redirect.redirect_request( + request, None, 302, "Found", {}, "https://huggingface.co/blob" + ) + assert same.get_header("Authorization") == "Bearer private" + with pytest.raises(ValueError, match="HTTPS"): + redirect.redirect_request( + request, None, 302, "Found", {}, "http://cdn.example/blob" + ) + + +@pytest.mark.parametrize("kind", ["symlink", "binary", "credential"]) +def test_worker_unsafe_evidence_still_uploads_error_only( + inputs, tmp_path, monkeypatch, kind +): + root, plan = worker_root(inputs, tmp_path, monkeypatch) + api = fake_api() + + def runner(task, config, output): + output.mkdir() + if kind == "symlink": + (output / "link").symlink_to(root / "coding-config.json") + elif kind == "binary": + (output / "binary").write_bytes(b"\x00binary") + else: + (output / "unsafe.json").write_text(json.dumps(SECRETS)) + return {"accepted": True} + + with pytest.raises(RuntimeError, match="persisted"): + cloud.worker(root, bundle_sha=plan["bundle_sha256"], api=api, runner=runner) + operations = api.create_commit.call_args.kwargs["operations"] + assert len(operations) == 1 + assert operations[0].path_in_repo.endswith("/cloud-report.json") + report = json.loads(operations[0].path_or_fileobj) + assert report["status"] == "error" + assert report["accepted"] is False + assert report["evidence_omitted"] is True + + +def test_task_exclusions_match_runner_source_identity(inputs, tmp_path): + from openenv.validation.runner import source_digest + + task = inputs[1] + for name in (".git", ".worktrees", ".venv", "__pycache__", "outputs"): + (task / name).mkdir() + (task / name / "binary-secret").write_bytes(b"\x00ignored") + before = source_digest(task) + output, plan = prepared(inputs, tmp_path) + destination = tmp_path / "snapshot" + cloud.extract_bundle( + (output / "bundle.zip").read_bytes(), destination, plan["bundle_sha256"] + ) + assert source_digest(destination / "task") == before + assert not (destination / "task/.worktrees").exists() + assert not (destination / "task/outputs").exists() + + +@pytest.mark.parametrize( + "fault", + ["compose", "steps", "private-network", "phase-environment", "missing-verifier"], +) +def test_prepare_rejects_unsupported_task_before_controller_spending( + inputs, tmp_path, fault +): + repo, task, config = inputs + if fault == "compose": + (task / "environment").mkdir() + (task / "environment/compose.yaml").write_text("services: {}\n") + elif fault == "steps": + (task / "steps").mkdir() + (task / "steps/step.toml").write_text("name='step'\n") + elif fault == "private-network": + (task / "task.toml").write_text("[environment]\nallow_internet=false\n") + elif fault == "phase-environment": + (task / "task.toml").write_text("[verifier.environment]\nnetwork_mode='none'\n") + else: + (task / "tests/test.sh").unlink() + output = tmp_path / "plan" + with pytest.raises(ValueError): + cloud.prepare(task, config, output, repo_root=repo) + assert not output.exists() + + +def test_prepare_checks_the_frozen_task_contract_not_a_restored_source( + inputs, tmp_path, monkeypatch +): + repo, task, config = inputs + original_tree = cloud._tree + original = (task / "task.toml").read_bytes() + + def changed_during_snapshot(root, **kwargs): + if Path(root) != task: + return original_tree(root, **kwargs) + (task / "task.toml").write_text("[environment]\nallow_internet=false\n") + try: + return original_tree(root, **kwargs) + finally: + (task / "task.toml").write_bytes(original) + + monkeypatch.setattr(cloud, "_tree", changed_during_snapshot) + output = tmp_path / "plan" + with pytest.raises(ValueError, match="network"): + cloud.prepare(task, config, output, repo_root=repo) + assert (task / "task.toml").read_bytes() == original + assert not output.exists() + + +def test_bundle_retains_available_full_run_reproduction_helpers(inputs, tmp_path): + repo, _, _ = inputs + names = ( + "full_run.py", + "real_tasks.py", + "sources.py", + "spend.py", + ) + for name in names: + path = repo / "examples/hub_validation_pipeline" / name + path.write_text("{}\n" if name.endswith(".json") else "# helper\n") + output, plan = prepared(inputs, tmp_path) + _, files = cloud._checked_archive( + (output / "bundle.zip").read_bytes(), plan["bundle_sha256"] + ) + for name in names: + relative = "examples/hub_validation_pipeline/" + name + assert files[relative] == (repo / relative).read_bytes() + + +def test_hub_driver_prepares_coding_audits_without_admitting_or_running_docker( + inputs, tmp_path, monkeypatch +): + from argparse import Namespace + + from examples.hub_validation_pipeline import cloud, pipeline + + _, task, raw_config = inputs + config = tmp_path / "coding.json" + config.write_text(json.dumps(raw_config)) + calls = [] + real_prepare = cloud.prepare + + def prepare(subject, config, output, *, source): + calls.append((subject, source)) + assert (subject / "task.toml").is_file() + return real_prepare(subject, config, output, source=source) + + monkeypatch.setattr(cloud, "prepare", prepare) + monkeypatch.setattr( + pipeline, + "validate_task", + lambda *args: pytest.fail("cloud mode must not invoke Docker validation"), + ) + args = Namespace( + output=tmp_path / "run", + contract=None, + coding_config=config, + submit_hf_jobs=False, + hf_dataset=[], + harbor_dataset=[], + local_dataset=[task], + source="both", + dataset_limit=0, + tasks_per_dataset=1, + ) + assert pipeline.run(args) == 0 + summary = json.loads((args.output / "summary.json").read_text()) + assert summary["phase"] == "prepared" + assert summary["plans"] == 1 and summary["accepted"] == 0 + assert not summary["rfc008_complete"] + planned = json.loads((args.output / "planned.jsonl").read_text()) + assert planned["accepted"] is False and planned["state"] == "prepared" + assert (args.output / "accepted.jsonl").read_text() == "" + assert calls[0][1]["revision"] diff --git a/examples/hub_validation_pipeline/test_full_run.py b/examples/hub_validation_pipeline/test_full_run.py new file mode 100644 index 000000000..1edd93367 --- /dev/null +++ b/examples/hub_validation_pipeline/test_full_run.py @@ -0,0 +1,818 @@ +"""Local-only checks for modeled reservation and durable one-shot orchestration.""" + +import hashlib +import json +import socket +from copy import deepcopy +from fractions import Fraction +from pathlib import Path +from unittest.mock import Mock + +import pytest +from examples.hub_validation_pipeline import cloud, full_run, real_tasks, spend +from examples.hub_validation_pipeline.test_real_tasks import ( + toy_config, + TOY_NAMES, + TOY_SOURCE, + write_panel, +) + + +@pytest.fixture +def local_run(tmp_path, monkeypatch): + forbidden = Mock(side_effect=AssertionError("Live operations forbidden")) + monkeypatch.setattr(socket.socket, "connect", forbidden) + monkeypatch.setattr(real_tasks, "HfApi", forbidden) + monkeypatch.setattr(cloud, "_runtime_secrets", forbidden) + panel_path = write_panel(tmp_path) + panel_sha256 = full_run._hash(panel_path)["sha256"] + plans = {} + + def prepare(output, *, panel, panel_provenance, task_names, bundle): + assert panel_provenance["sha256"] == panel_sha256 + tasks = {task["name"]: task for task in panel["tasks"]} + assert bundle is True + output.mkdir() + rows = [] + for index, name in enumerate(task_names): + config, budget = cloud._config(tasks[name]["config"]) + path = output / name / "plan.json" + path.parent.mkdir() + plan = { + "run_id": f"coding-{index}", + "bundle_sha256": str(index) * 64, + "budget": budget, + } + path.write_bytes(cloud._json_bytes(plan)) + plans[str(path)] = (plan, config, b"", {}) + rows.append( + { + "task_id": f"hf:{TOY_SOURCE['repo_id']}@{TOY_SOURCE['revision']}:fixtures/{name}", + "bundle_plan": str(path), + } + ) + result = {"tasks": rows} + (output / "manifest.json").write_bytes(cloud._json_bytes(result)) + return result + + prepare = Mock(side_effect=prepare) + monkeypatch.setattr(real_tasks, "_prepare", prepare) + + def read_plan(path, *, expected_sha256=None): + if expected_sha256 is not None: + if hashlib.sha256(Path(path).read_bytes()).hexdigest() != expected_sha256: + raise ValueError("Prepared plan changed before submission") + return plans[str(path)] + + monkeypatch.setattr(cloud, "_read_plan", read_plan) + submit = Mock(return_value={"status": "submitted", "job_id": "queued-job"}) + monkeypatch.setattr(cloud, "submit", submit) + hf, harbor = tmp_path / "hf.json", tmp_path / "harbor.json" + inventory = tmp_path / "inventory.json.gz" + # Deliberately not JSON/gzip: inventory bytes must only be streamed/hashed. + inventory.write_bytes(b"unparsed inventory" * 100000) + missing = tmp_path / "missing.json" + hf.write_text( + json.dumps( + { + "scope": "Frozen loose native identities; no runtime qualification", + "counts": {"repositories": 3, "native_tasks": 7}, + "compact_repo_inventory": str(inventory), + "compact_identity_inventory": str(missing), + "raw_census": str(tmp_path / "do-not-read-raw.json"), + } + ) + ) + harbor.write_text( + json.dumps( + { + "scope": "Anonymous native registry metadata only", + "counts": { + "dataset_versions_listed": 9, + "dataset_names": 5, + "unique_task_digests": 11, + }, + "artifacts": { + key: {"path": str(inventory)} + for key in ( + "task_identity_inventory", + "dataset_version_membership_index", + ) + }, + } + ) + ) + return tmp_path / "run", hf, harbor, prepare, submit, plans + + +def assert_prepared(prepare, output, names): + prepare.assert_called_once() + assert prepare.call_args.args == (output / "recipes",) + assert prepare.call_args.kwargs["task_names"] == names + assert prepare.call_args.kwargs["bundle"] is True + assert ( + prepare.call_args.kwargs["panel_provenance"]["sha256"] + == full_run._hash(output.parent / "panel.json")["sha256"] + ) + + +@pytest.fixture +def paid_reservation(tmp_path): + ledger = tmp_path / "spend.json" + spend.init(ledger, "500") + return {"spend_ledger": ledger, "reservation_id": "test-cohort"} + + +@pytest.mark.parametrize("supplied", [False, True], ids=["omitted", "exhausted"]) +def test_paid_api_cannot_bypass_fully_reserved_global_ledger(local_run, supplied): + output, hf, harbor, prepare, submit, _ = local_run + ledger = output.parent / "spend.json" + spend.init(ledger, "500") + spend.reserve(ledger, "prior-unresolved", "500", "Prior unresolved work") + arguments = ( + {"spend_ledger": ledger, "reservation_id": "toy-selection-v3"} + if supplied + else {} + ) + error, message = ( + (RuntimeError, "during reserve") + if supplied + else (ValueError, "shared ledger and a reservation ID") + ) + with pytest.raises(error, match=message): + full_run.run( + output, + hf, + harbor, + "500", + task_names=["toy-alpha"], + submit=True, + **arguments, + panel_path=output.parent / "panel.json", + ) + submit.assert_not_called() + if not supplied: + prepare.assert_not_called() + assert not output.exists() + assert spend.summary(ledger)["reserved_usd_exact"] == "500" + assert spend.summary(ledger)["remaining_usd_exact"] == "0" + + +@pytest.mark.parametrize( + "ledger_value,identifier", + [ + (None, None), + (None, "cohort"), + ("", "cohort"), + (" ", "cohort"), + ("initialized", None), + ("initialized", ""), + ("initialized", " "), + ], +) +def test_paid_api_requires_nonempty_shared_reservation_before_preparation( + local_run, paid_reservation, monkeypatch, ledger_value, identifier +): + output, hf, harbor, prepare, submit, _ = local_run + monkeypatch.chdir(output.parent) + ledger = paid_reservation["spend_ledger"] + with pytest.raises(ValueError, match="shared ledger and a reservation ID"): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + spend_ledger=ledger if ledger_value == "initialized" else ledger_value, + reservation_id=identifier, + panel_path=output.parent / "panel.json", + ) + assert not output.exists() + prepare.assert_not_called() + submit.assert_not_called() + assert spend.summary(ledger)["reserved_usd_exact"] == "0" + + +@pytest.mark.parametrize("empty_file", [False, True], ids=["missing", "empty"]) +def test_paid_api_requires_initialized_ledger_before_submission(local_run, empty_file): + output, hf, harbor, _, submit, _ = local_run + ledger = output.parent / "spend.json" + if empty_file: + ledger.write_bytes(b"") + with pytest.raises(RuntimeError, match="during reserve"): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + spend_ledger=ledger, + reservation_id="test-cohort", + panel_path=output.parent / "panel.json", + ) + submit.assert_not_called() + + +def test_exact_aggregate_reservation_and_normalized_generator(): + configs = [toy_config(name) for name in TOY_NAMES] + for config in configs: + config.pop("generator") + result = full_run.estimate(configs) + # Missing generator inherits the target's request limits. + inputs_per_task = 2 * (4 * 256 + 8192 + 256) + (4 * 256 + 8192) + outputs_per_task = 2 * 3 * 64 + 64 + assert result["input_tokens_max"] == 2 * inputs_per_task + assert result["output_tokens_max"] == 2 * outputs_per_task + assert result["controller_jobs_max"] == 2 + budgets = [cloud._config(config)[1] for config in configs] + children = sum(budget["sandbox_jobs_max"] for budget in budgets) + assert result["sandbox_jobs_max"] == children + inference = ( + 2 * inputs_per_task * Fraction("0.15") + + 2 * outputs_per_task * Fraction("1.875") + ) / 1000000 + compute = (children * 86700 * Fraction("0.02") + 2 * 360 * Fraction("0.01")) / 3600 + assert Fraction(result["inference_usd_exact"]) == inference + assert Fraction(result["compute_usd_exact"]) == compute + assert Fraction(result["reservation_usd_exact"]) == inference + compute + assert all(isinstance(value, (str, int)) for value in result.values()) + + +def test_operator_runtime_limits_drive_reservation(): + configs = [toy_config(name) for name in TOY_NAMES] + base = full_run.estimate(configs) + configs[0]["target"]["max_total_tokens"] *= 2 + changed = full_run.estimate(configs) + assert changed["input_tokens_max"] > base["input_tokens_max"] + assert Fraction(changed["reservation_usd_exact"]) > Fraction( + base["reservation_usd_exact"] + ) + + +@pytest.mark.parametrize( + "names", [[], ["unknown"], ["toy-alpha", "unknown"], ["toy-beta"] * 2] +) +def test_invalid_task_selection_is_rejected_before_preparation(local_run, names): + output, hf, harbor, prepare, submit, _ = local_run + ledger = output.parent / "spend.json" + spend.init(ledger, "500") + with pytest.raises(ValueError, match="distinct configured panel task names"): + full_run.run( + output, + hf, + harbor, + "500", + task_names=names, + submit=True, + spend_ledger=ledger, + reservation_id="invalid-cohort", + panel_path=output.parent / "panel.json", + ) + assert not output.exists() + prepare.assert_not_called() + submit.assert_not_called() + assert spend.summary(ledger)["reserved_usd_exact"] == "0" + + +def test_selection_budgets_and_submits_only_selected_task(local_run): + output, hf, harbor, prepare, submit, _ = local_run + ledger = output.parent / "spend.json" + spend.init(ledger, "30") + spend.reserve(ledger, "prior-launch", "5", "Unreconciled prior launch") + manifest = full_run.run( + output, + hf, + harbor, + "15", + task_names=["toy-alpha"], + submit=True, + spend_ledger=ledger, + reservation_id="toy-selection-v2", + panel_path=output.parent / "panel.json", + ) + assert_prepared(prepare, output, ["toy-alpha"]) + assert manifest["status"] == "submitted" and manifest["within_budget"] is True + assert manifest == cloud._loads((output / "manifest.json").read_bytes()) + assert len(manifest["tasks"]) == 1 + task = manifest["tasks"][0] + assert task["task_id"] == ( + f"hf:{TOY_SOURCE['repo_id']}@{TOY_SOURCE['revision']}:fixtures/toy-alpha" + ) + assert task["submission"]["status"] == "submitted" + submit.assert_called_once_with( + task["plan"], expected_plan_sha256=task["plan_sha256"] + ) + assert not (output / "recipes/toy-beta").exists() + assert "toy-beta" not in (output / "manifest.json").read_text() + estimate = manifest["estimate"] + assert estimate == full_run.estimate([toy_config("toy-alpha")]) + assert estimate["controller_jobs_max"] == 1 + assert Fraction(spend.summary(ledger)["reserved_usd_exact"]) == ( + 5 + Fraction(estimate["reservation_usd_rounded_up"]) + ) + assert manifest["shared_spend"]["reservation_id"].startswith("toy-selection-v2:") + with pytest.raises(FileExistsError): + full_run.run( + output, + hf, + harbor, + "15", + task_names=["toy-alpha"], + submit=True, + spend_ledger=ledger, + reservation_id="toy-selection-v2", + panel_path=output.parent / "panel.json", + ) + assert submit.call_count == prepare.call_count == 1 + + +def test_selected_task_still_obeys_global_spend_cap(local_run): + output, hf, harbor, _, submit, _ = local_run + ledger = output.parent / "spend.json" + spend.init(ledger, "15") + spend.reserve(ledger, "prior-launch", "10", "Unreconciled prior launch") + with pytest.raises(RuntimeError, match="during reserve"): + full_run.run( + output, + hf, + harbor, + "15", + task_names=["toy-alpha"], + submit=True, + spend_ledger=ledger, + reservation_id="toy-selection-v2", + panel_path=output.parent / "panel.json", + ) + submit.assert_not_called() + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert manifest["within_budget"] is True + assert len(manifest["tasks"]) == 1 + assert manifest["tasks"][0]["submission"]["status"] == "not_submitted" + assert spend.summary(ledger)["reserved_usd_exact"] == "10" + + +@pytest.mark.parametrize( + "budget", ["0", "-1", "NaN", "Infinity", "1/2", "invalid", 50.0, True, None] +) +def test_invalid_cap_is_rejected_before_preparation( + local_run, paid_reservation, budget +): + output, hf, harbor, prepare, submit, _ = local_run + with pytest.raises(ValueError, match="decimal string"): + full_run.run( + output, + hf, + harbor, + budget, + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + assert not output.exists() + prepare.assert_not_called() + submit.assert_not_called() + + +def test_prepare_only_freezes_small_summaries_and_hashes_inventories(local_run): + output, hf, harbor, prepare, submit, _ = local_run + manifest = full_run.run( + output, hf, harbor, "500", panel_path=output.parent / "panel.json" + ) + assert_prepared(prepare, output, list(TOY_NAMES)) + submit.assert_not_called() + assert manifest == cloud._loads((output / "manifest.json").read_bytes()) + assert manifest["status"] == "prepared" and manifest["within_budget"] is True + assert manifest["full_catalog_executed"] is False + assert ( + manifest["accepted"] + is manifest["rfc008_qualified"] + is manifest["trainable"] + is False + ) + assert len(manifest["tasks"]) == 2 + for kind, source in (("hf", hf), ("harbor", harbor)): + record = manifest["catalogs"][kind] + assert Path(record["path"]).read_bytes() == source.read_bytes() + assert record["sha256"] == hashlib.sha256(source.read_bytes()).hexdigest() + assert record["runtime_qualified"] is False + inventories = manifest["catalogs"]["hf"]["inventories"] + record = inventories["compact_repo_inventory"] + assert ( + record["sha256"] + == hashlib.sha256(Path(record["path"]).read_bytes()).hexdigest() + ) + assert inventories["compact_identity_inventory"]["status"] == "unavailable" + assert manifest["catalogs"]["hf"]["counts"]["native_tasks"] == 7 + assert not (output.parent / "spend.json").exists() + with pytest.raises(FileExistsError): + full_run.run(output, hf, harbor, "500", panel_path=output.parent / "panel.json") + prepare.assert_called_once() + + +@pytest.mark.parametrize( + "cap", ["1", "0.0000000000000000000000000000000000000000000000000000000001"] +) +def test_aggregate_cap_stops_before_any_submit_and_preserves_prepared_manifest( + local_run, paid_reservation, cap +): + output, hf, harbor, _, submit, _ = local_run + with pytest.raises(RuntimeError, match="during budget"): + full_run.run( + output, + hf, + harbor, + cap, + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + submit.assert_not_called() + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert manifest["within_budget"] is False and manifest["stage"] == "budget" + assert len(manifest["tasks"]) == 2 and (output / "recipes/manifest.json").exists() + assert all( + task["submission"]["status"] == "not_submitted" for task in manifest["tasks"] + ) + + +@pytest.mark.parametrize( + "section,field,value", + [ + ("target", "model_id", "other/model"), + ("generator", "provider", "other"), + ("target", "revision", "a" * 40), + ("generator", "provider_model_id", "different-model"), + ("execution", "flavor", "cpu-upgrade"), + ("cloud", "flavor", "cpu-upgrade"), + ], +) +def test_unquoted_profiles_are_rejected(section, field, value): + config = deepcopy(toy_config("toy-beta")) + config[section][field] = value + with pytest.raises(ValueError): + full_run.estimate([config]) + + +def test_partial_ambiguous_submission_is_durable_safe_and_never_resumed( + local_run, paid_reservation +): + output, hf, harbor, prepare, submit, _ = local_run + observed = [] + + def launch(path, *, expected_plan_sha256): + assert full_run._hash(Path(path))["sha256"] == expected_plan_sha256 + durable = cloud._loads((output / "manifest.json").read_bytes()) + assert durable["shared_spend"]["ledger"] == str( + paid_reservation["spend_ledger"] + ) + assert Fraction( + spend.summary(paid_reservation["spend_ledger"])["reserved_usd_exact"] + ) == Fraction(durable["estimate"]["reservation_usd_rounded_up"]) + index = len(observed) + observed.append(path) + assert durable["tasks"][index]["submission"]["status"] == "pending_or_unknown" + if index: + assert durable["tasks"][0]["submission"]["job_id"] == "job-first" + (Path(path).parent / "submission.json").write_text( + '{"status":"launch_unknown"}' + ) + raise RuntimeError("sensitive-error-value-must-not-escape") + return { + "status": "submitted", + "job_id": "job-first", + "launch_spec": {"command": ["omitted"]}, + } + + submit.side_effect = launch + with pytest.raises(RuntimeError, match="during submit") as failure: + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert manifest["status"] == "failed" and manifest["accepted"] is False + assert manifest["tasks"][0]["submission"]["status"] == "submitted" + assert manifest["tasks"][1]["submission"]["status"] == "pending_or_unknown" + receipt = Path(manifest["tasks"][1]["submission"]["receipt"]) + assert json.loads(receipt.read_text())["status"] == "launch_unknown" + assert ( + "sensitive-error-value" + not in str(failure.value) + (output / "manifest.json").read_text() + ) + assert "launch_spec" not in manifest["tasks"][0]["submission"] + assert Fraction( + spend.summary(paid_reservation["spend_ledger"])["reserved_usd_exact"] + ) == Fraction(manifest["estimate"]["reservation_usd_rounded_up"]) + with pytest.raises(FileExistsError): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + assert submit.call_count == 2 and prepare.call_count == 1 + + +def test_submitted_jobs_remain_unaccepted(local_run, paid_reservation): + output, hf, harbor, _, submit, _ = local_run + manifest = full_run.run( + output, + hf, + harbor, + "500", + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + assert submit.call_count == 2 and manifest["status"] == "submitted" + assert manifest["shared_spend"]["ledger"] == str(paid_reservation["spend_ledger"]) + assert ( + manifest["accepted"] + is manifest["trainable"] + is manifest["rfc008_qualified"] + is False + ) + assert all( + task["accepted"] is task["trainable"] is task["rfc008_qualified"] is False + for task in manifest["tasks"] + ) + + +def test_prepare_failure_preserves_own_manifest_and_partial_recipes( + local_run, paid_reservation +): + output, hf, harbor, prepare, submit, _ = local_run + + def fail(path, **kwargs): + path.mkdir() + (path / "partial.json").write_text("{}") + raise ValueError("untrusted exception text") + + prepare.side_effect = fail + with pytest.raises(RuntimeError, match="during prepare"): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert manifest["stage"] == "prepare" and manifest["error_type"] == "ValueError" + assert (output / "recipes/partial.json").exists() + assert "untrusted exception text" not in (output / "manifest.json").read_text() + submit.assert_not_called() + + +def test_second_plan_failure_preserves_first_plan_and_stops_all_submits( + local_run, paid_reservation, monkeypatch +): + output, hf, harbor, _, submit, plans = local_run + + def read(path, *, expected_sha256=None): + if Path(path).parent.name == "toy-beta": + raise ValueError("bad plan") + return plans[str(path)] + + monkeypatch.setattr(cloud, "_read_plan", read) + with pytest.raises(RuntimeError, match="during plans"): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert len(manifest["tasks"]) == 1 and manifest["accepted"] is False + assert (output / "recipes/toy-beta/plan.json").exists() + submit.assert_not_called() + + +def test_changed_plan_cannot_reuse_reviewed_estimate( + local_run, paid_reservation, monkeypatch +): + output, hf, harbor, _, submit, plans = local_run + estimate = full_run.estimate + + def change(configs): + result = estimate(configs) + Path(next(iter(plans))).write_text("changed") + return result + + monkeypatch.setattr(full_run, "estimate", change) + with pytest.raises(RuntimeError, match="during submit"): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + submit.assert_not_called() + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert Fraction( + spend.summary(paid_reservation["spend_ledger"])["reserved_usd_exact"] + ) == Fraction(manifest["estimate"]["reservation_usd_rounded_up"]) + + +def test_raw_census_is_rejected_without_preparation(local_run): + output, hf, harbor, prepare, _, _ = local_run + hf.write_bytes(b" " * (1024 * 1024 + 1)) + with pytest.raises(RuntimeError, match="during catalogs"): + full_run.run(output, hf, harbor, "500", panel_path=output.parent / "panel.json") + prepare.assert_not_called() + + +def test_cli_requires_explicit_budget_and_defaults_to_preparation(local_run, capsys): + output, hf, harbor, _, submit, _ = local_run + arguments = [ + "--panel", + str(output.parent / "panel.json"), + "--output", + str(output), + "--hf-census", + str(hf), + "--harbor-census", + str(harbor), + ] + with pytest.raises(SystemExit): + full_run.main(arguments) + assert not output.exists() + full_run.main(arguments + ["--budget-usd", "500"]) + assert json.loads(capsys.readouterr().out)["status"] == "prepared" + submit.assert_not_called() + + +@pytest.mark.parametrize("names", [["toy-alpha"], ["toy-beta", "toy-alpha"]]) +def test_cli_passes_repeatable_task_selection(local_run, capsys, names): + output, hf, harbor, prepare, submit, _ = local_run + arguments = [ + "--panel", + str(output.parent / "panel.json"), + "--output", + str(output), + "--hf-census", + str(hf), + "--harbor-census", + str(harbor), + "--budget-usd", + "500", + ] + for name in names: + arguments.extend(["--task", name]) + full_run.main(arguments) + assert_prepared(prepare, output, names) + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert [Path(task["plan"]).parent.name for task in manifest["tasks"]] == names + assert json.loads(capsys.readouterr().out)["status"] == "prepared" + submit.assert_not_called() + + +@pytest.mark.parametrize( + "selection", [["--task", "unknown"], ["--task", ""], ["--task"]] +) +def test_cli_rejects_unknown_or_empty_task_before_preparation(local_run, selection): + output, hf, harbor, prepare, submit, _ = local_run + with pytest.raises((SystemExit, ValueError)): + full_run.main( + [ + "--panel", + str(output.parent / "panel.json"), + "--output", + str(output), + "--hf-census", + str(hf), + "--harbor-census", + str(harbor), + "--budget-usd", + "500", + *selection, + ] + ) + assert not output.exists() + prepare.assert_not_called() + submit.assert_not_called() + + +def test_shared_ledger_blocks_before_any_paid_submission(local_run): + output, hf, harbor, _, submit, _ = local_run + ledger = output.parent / "spend.json" + spend.init(ledger, "1") + with pytest.raises(RuntimeError, match="during reserve"): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + spend_ledger=ledger, + reservation_id="cohort-v2", + panel_path=output.parent / "panel.json", + ) + submit.assert_not_called() + assert spend.summary(ledger)["reserved_usd_exact"] == "0" + + +def test_shared_reservation_is_bound_to_the_immutable_cohort(local_run): + output, hf, harbor, _, submit, _ = local_run + ledger = output.parent / "spend.json" + spend.init(ledger, "500") + manifest = full_run.run( + output, + hf, + harbor, + "500", + submit=True, + spend_ledger=ledger, + reservation_id="cohort-v2", + panel_path=output.parent / "panel.json", + ) + assert submit.call_count == 2 + assert manifest["shared_spend"]["reservation_id"].startswith("cohort-v2:") + assert Fraction(spend.summary(ledger)["reserved_usd_exact"]) == Fraction( + manifest["estimate"]["reservation_usd_rounded_up"] + ) + + +def test_paid_cli_requires_shared_reservation(local_run): + output, hf, harbor, _, submit, _ = local_run + with pytest.raises((SystemExit, ValueError)): + full_run.main( + [ + "--panel", + str(output.parent / "panel.json"), + "--output", + str(output), + "--hf-census", + str(hf), + "--harbor-census", + str(harbor), + "--budget-usd", + "500", + "--submit", + ] + ) + assert not output.exists() + submit.assert_not_called() + + +def test_full_run_reads_and_binds_one_panel_snapshot(local_run, monkeypatch): + output, hf, harbor, prepare, _, _ = local_run + panel_path = output.parent / "panel.json" + original_bytes = panel_path.read_bytes() + read_file = cloud._text_file + reads = [] + + def replace_after_read(path): + result = read_file(path) + if path == panel_path: + reads.append(path) + panel_path.write_text('{"replacement": true}') + return result + + monkeypatch.setattr(cloud, "_text_file", replace_after_read) + manifest = full_run.run(output, hf, harbor, "500", panel_path=panel_path) + assert reads == [panel_path] + assert ( + manifest["evaluation_panel"]["sha256"] + == hashlib.sha256(original_bytes).hexdigest() + ) + assert prepare.call_args.kwargs["panel_provenance"] == manifest["evaluation_panel"] + assert prepare.call_args.kwargs["panel"]["tasks"][0]["config"] == toy_config() + + +def test_full_run_requires_explicit_panel_before_preparation(local_run): + output, hf, harbor, prepare, submit, _ = local_run + with pytest.raises(TypeError, match="panel_path"): + full_run.run(output, hf, harbor, "500") + with pytest.raises(SystemExit): + full_run.main( + [ + "--output", + str(output), + "--hf-census", + str(hf), + "--harbor-census", + str(harbor), + "--budget-usd", + "500", + ] + ) + assert not output.exists() + prepare.assert_not_called() + submit.assert_not_called() diff --git a/examples/hub_validation_pipeline/test_pipeline.py b/examples/hub_validation_pipeline/test_pipeline.py new file mode 100644 index 000000000..c81404683 --- /dev/null +++ b/examples/hub_validation_pipeline/test_pipeline.py @@ -0,0 +1,566 @@ +"""Admission and execution regressions at the report/runner boundaries.""" + +import asyncio +import hashlib +import json +import shutil +from copy import deepcopy +from decimal import Decimal +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from examples.hub_validation_pipeline import demo, pipeline +from openenv.harbor.models import HarborRolloutResult +from openenv.validation.parsers.harbor_toml import HarborTaskParser +from openenv.validation.runner import source_digest + + +@pytest.fixture +def evidence(tmp_path): + return make_evidence(tmp_path) + + +def make_evidence(tmp_path, canary_count=2): + dataset = demo.create_tasks(tmp_path / "input") + subject = tmp_path / "subject" + shutil.copytree(dataset / "good", subject) + canaries = subject / "validation/canaries" + if canaries.exists(): + shutil.rmtree(canaries) + canaries.mkdir(parents=True) + for index in range(canary_count): + (canaries / f"wrong-{index}.sh").write_text(f"#!/bin/bash\necho {index}\n") + original = source_digest(subject) + contract = json.loads( + Path(pipeline.__file__).with_name("contract.json").read_text() + ) + if canary_count: + contract["canaries"] = "validation/canaries" + else: + contract.pop("canaries", None) + pipeline.write_json( + subject / "validation/openenv.json", + {**contract, "original_source_digest": original}, + ) + manifest = HarborTaskParser().parse(subject).model_dump(mode="json") + report = { + "report_schema_version": "2", + "target": str(subject), + "source_digest": source_digest(subject), + "signature": "task.toml", + "manifest": manifest, + "policy_version": "v2", + "lane": "local", + "levels_run": [1, 2, 3], + "verdict": "warn", + "results": [ + {"check_id": key, "status": "pass", "duration_s": 0} + for key in sorted(pipeline.PILOT_CHECKS) + ] + + ( + [{"check_id": "semantic.canary_floor", "status": "pass", "duration_s": 0}] + if canary_count + else [] + ) + + [ + { + "check_id": "runtime.host_containment", + "status": "skip", + "duration_s": 0, + "evidence": ["out of profile: native process boundary"], + }, + ], + } + native = { + "schema_version": "2", + "profile": pipeline.PILOT, + "task_effective_digest": report["source_digest"], + "original_source_digest": original, + "image_ref": "sha256:" + "a" * 64, + "retained_image_tag": "openenv-validation:" + "a" * 32, + "reward_key": "reward", + "floor_reward": 0, + "oracle_reward": 1, + "cleanup": {"verified": True}, + "trials": [], + } + artifacts = tmp_path / "evidence" + roles = ["nop", "oracle", "oracle-replay"] + [ + f"canary-{index}" for index in range(1, canary_count + 1) + ] + for index, role in enumerate(roles): + rewards = {"reward": int(role in {"oracle", "oracle-replay"}), "aux": 0} + trial = { + "role": role, + "image_ref": native["image_ref"], + "retained_image_tag": native["retained_image_tag"], + "container_id": f"{index + 1:064x}", + "unmodified_task_digest": report["source_digest"], + "task_overlay": {} + if role == "nop" + else { + "environment.docker_image": native["retained_image_tag"], + }, + "cleanup": {"verified": True}, + "configuration": { + "user": "", + "workdir": "/app", + "host": {"Memory": 512 * 1024**2}, + "network_mode": "compose-default", + "mounts": [], + }, + "rewards": rewards, + "raw_rewards": {}, + } + if role != "nop": + trial.update(oracle_exit_observed_at_agent_end=True, oracle_exit_code=0) + if role.startswith("canary-"): + canary = sorted(canaries.iterdir())[index - 3] + trial["task_overlay"]["solution/solve.sh"] = { + "canary": canary.relative_to(subject).as_posix(), + "sha256": hashlib.sha256(canary.read_bytes()).hexdigest(), + } + native["trials"].append(trial) + set_raw(artifacts, trial, "reward.json", pipeline.dumps(rewards) + "\n") + save_bundle(artifacts, report, native) + return subject, report, artifacts, original, native + + +def set_raw(directory, trial, filename, text): + name = f"harbor-trials/{trial['role']}/{filename}" + path = directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + trial["raw_rewards"][filename] = { + "path": name, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + + +def save_bundle(directory, report, native): + pipeline.write_json(directory / "report.json", report) + pipeline.write_json(directory / "harbor-evidence.json", native) + (directory / "SHA256SUMS").write_text( + "".join( + f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.relative_to(directory)}\n" + for path in sorted(directory.rglob("*")) + if path.is_file() and path.name != "SHA256SUMS" + ) + ) + + +def admit(evidence, profile=pipeline.PILOT): + subject, report, artifacts, original, _ = evidence + return pipeline.admission(report, 0, subject, artifacts, profile, original) + + +def test_explicit_pilot_admits_but_full_gate_stays_incomplete(evidence): + admitted = admit(evidence) + assert admitted["out_of_profile_skips"] == ["runtime.host_containment"] + assert admitted["rfc008_complete"] is False + with pytest.raises(ValueError, match="coverage"): + admit(evidence, "full") + + +def test_full_profile_requires_every_policy_check_to_pass(evidence): + _, report, artifacts, _, native = evidence + report["verdict"] = "pass" + report["results"] = [ + {"check_id": key, "status": "pass", "duration_s": 0} + for key in pipeline.load_policy("v2").entries_for_lane(pipeline.Lane.LOCAL) + ] + save_bundle(artifacts, report, native) + assert admit(evidence, "full")["rfc008_complete"] is True + report["results"][-1]["status"] = "skip" + with pytest.raises(ValueError, match="coverage"): + admit(evidence, "full") + + +@pytest.mark.parametrize("profile", [pipeline.PILOT, "full"]) +def test_local_validator_requests_semantic_level(tmp_path, monkeypatch, profile): + calls = [] + report = tmp_path / "report.json" + + class Process: + returncode = 0 + + def communicate(self, timeout): + report.write_text('{"validator":"result"}') + return "", "" + + def start(command, **kwargs): + calls.append((command, kwargs)) + return Process() + + monkeypatch.setattr(pipeline.subprocess, "Popen", start) + assert pipeline.validate_task(tmp_path, report, profile, 10) == ( + 0, + {"validator": "result"}, + ) + command = calls[0][0] + assert command[command.index("--level") + 1] == "semantic" + + +@pytest.mark.parametrize("fault", ["missing", "duplicate", "skip", "fail", "unknown"]) +def test_exit_zero_cannot_hide_bad_coverage(evidence, fault): + report = evidence[1] + if fault == "missing": + report["results"].pop(0) + elif fault == "duplicate": + report["results"].append(deepcopy(report["results"][0])) + elif fault == "skip": + report["results"][0]["status"] = "skip" + elif fault == "fail": + report["results"][-1]["status"] = "fail" + else: + report["results"].append( + {"check_id": "runtime.invented", "status": "pass", "duration_s": 0} + ) + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize( + "check", + [ + "runtime.episode_determinism", + "semantic.oracle_max", + "semantic.floor_gap", + "semantic.canary_floor", + ], +) +@pytest.mark.parametrize("status", [None, "skip", "fail", "error"]) +def test_semantic_coverage_is_required(evidence, check, status): + report = evidence[1] + row = next(row for row in report["results"] if row["check_id"] == check) + if status is None: + report["results"].remove(row) + else: + row["status"] = status + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize("canary_count", [0, 1, 8]) +def test_declared_canary_count_sets_the_trial_plan(tmp_path, canary_count): + sample = make_evidence(tmp_path, canary_count) + assert len(sample[-1]["trials"]) == 3 + canary_count + assert admit(sample)["profile"] == "harbor-semantic-v1" + + +@pytest.mark.parametrize( + "fault", + [ + "missing", + "reordered", + "duplicate-container", + "image", + "source", + "failed", + "cleanup", + "top-cleanup", + "configuration", + "missing-floor", + "missing-tag", + ], +) +def test_every_trial_must_complete_with_bound_identity(evidence, fault): + _, report, artifacts, _, native = evidence + trials = native["trials"] + if fault == "missing": + trials.pop() + elif fault == "reordered": + trials[1], trials[2] = trials[2], trials[1] + elif fault == "duplicate-container": + trials[-1]["container_id"] = trials[0]["container_id"] + elif fault == "image": + trials[-1]["image_ref"] = "sha256:" + "b" * 64 + elif fault == "source": + trials[-1]["unmodified_task_digest"] = "b" * 64 + elif fault == "failed": + trials[-1]["failure"] = "RewardArtifactDrift" + elif fault == "cleanup": + trials[-1]["cleanup"]["verified"] = False + elif fault == "top-cleanup": + native["cleanup"]["verified"] = False + elif fault == "configuration": + trials[-1]["configuration"]["workdir"] = "/different" + elif fault == "missing-floor": + native.pop("floor_reward") + else: + native.pop("retained_image_tag") + save_bundle(artifacts, report, native) + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize( + "fault", ["reward-map", "exit", "unobserved", "marker", "unbound-marker"] +) +def test_oracle_replay_checks_every_reward_and_trusted_exit(evidence, fault): + _, report, artifacts, _, native = evidence + replay = native["trials"][2] + if fault == "reward-map": + replay["rewards"]["aux"] = 1 + set_raw(artifacts, replay, "reward.json", pipeline.dumps(replay["rewards"])) + elif fault == "exit": + replay["oracle_exit_code"] = 1 + set_raw(artifacts, replay, "oracle-exit-code.txt", "1\n") + elif fault == "unobserved": + replay["oracle_exit_observed_at_agent_end"] = False + else: + set_raw(artifacts, replay, "oracle-exit-code.txt", "1\n") + save_bundle(artifacts, report, native) + if fault == "unbound-marker": + inventory = artifacts / "SHA256SUMS" + inventory.write_text( + "".join( + row + for row in inventory.read_text().splitlines(keepends=True) + if not row.rstrip().endswith("oracle-exit-code.txt") + ) + ) + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize("fault", ["reward", "path", "digest", "missing", "extra"]) +def test_canary_reward_and_overlay_match_declared_bytes(evidence, fault): + _, report, artifacts, _, native = evidence + canary = native["trials"][-1] + overlay = canary["task_overlay"] + if fault == "reward": + canary["rewards"]["reward"] = 1 + set_raw(artifacts, canary, "reward.json", pipeline.dumps(canary["rewards"])) + elif fault == "path": + overlay["solution/solve.sh"]["canary"] = "solution/solve.sh" + elif fault == "digest": + overlay["solution/solve.sh"]["sha256"] = "b" * 64 + elif fault == "missing": + overlay.pop("solution/solve.sh") + else: + overlay["tests/test.sh"] = "changed" + save_bundle(artifacts, report, native) + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize( + "raw", + [ + '{"reward":true}', + '{"reward":"1"}', + '{"reward":NaN}', + '{"reward":1,"aux":2}', + '{"reward":1,"reward":0}', + "{}", + ], +) +def test_raw_reward_maps_fail_closed(evidence, raw): + _, report, artifacts, _, native = evidence + set_raw(artifacts, native["trials"][1], "reward.json", raw) + save_bundle(artifacts, report, native) + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize( + "fault", + ["old-profile", "old-schema", "missing-schema", "operator-profile", "runtime-only"], +) +def test_stale_native_contract_is_rejected(evidence, fault): + _, report, artifacts, _, native = evidence + if fault == "old-profile": + native["profile"] = "harbor-oracle-floor-v1" + elif fault == "old-schema": + native["schema_version"] = "1" + elif fault == "missing-schema": + native.pop("schema_version") + elif fault == "runtime-only": + report["levels_run"] = [1, 2] + save_bundle(artifacts, report, native) + with pytest.raises(ValueError): + admit( + evidence, + "harbor-oracle-floor-v1" if fault == "operator-profile" else pipeline.PILOT, + ) + + +def test_admission_recomputes_oracle_floor_from_raw_rewards(evidence): + _, report, artifacts, _, native = evidence + trial = native["trials"][0] + trial["rewards"]["reward"] = 1 + set_raw(artifacts, trial, "reward.json", pipeline.dumps(trial["rewards"])) + native["floor_reward"] = 1 + # Even a self-consistent bundle with stale semantic PASS results is rejected. + save_bundle(artifacts, report, native) + with pytest.raises(ValueError, match="oracle/floor contract"): + admit(evidence) + + +@pytest.mark.parametrize( + "floor,oracle,margin,tolerance", + [ + (Decimal("0.9"), 1, Decimal("0.100000000000000001"), 0), + (0, Decimal("0.95"), 1, Decimal("0.1")), + ], +) +def test_exact_operator_margin_overrides_rounded_semantic_pass( + evidence, monkeypatch, floor, oracle, margin, tolerance +): + subject, report, artifacts, _, native = evidence + path = subject / "validation/openenv.json" + contract = pipeline.loads(path.read_text()) + contract["reward"].update(floor_margin=margin, oracle_tolerance=tolerance) + pipeline.write_json(path, contract) + report["manifest"]["reward"].update( + floor_margin=float(margin), oracle_tolerance=float(tolerance) + ) + # Exercise admission independently of the parser's representability guard. + monkeypatch.setattr( + pipeline, + "read_harbor_contract", + lambda root: SimpleNamespace(canaries="validation/canaries"), + ) + report["source_digest"] = source_digest(subject) + native["task_effective_digest"] = report["source_digest"] + for trial in native["trials"]: + trial["unmodified_task_digest"] = report["source_digest"] + trial["rewards"]["reward"] = ( + oracle if trial["role"] in {"oracle", "oracle-replay"} else floor + ) + set_raw(artifacts, trial, "reward.json", pipeline.dumps(trial["rewards"])) + native.update(floor_reward=floor, oracle_reward=oracle) + save_bundle(artifacts, report, native) + with pytest.raises(ValueError, match="oracle/floor contract"): + admit(evidence) + + +def test_bundled_report_must_equal_the_cli_result(evidence): + _, report, artifacts, _, native = evidence + bundled = deepcopy(report) + bundled["target"] = "different-subject" + save_bundle(artifacts, bundled, native) + with pytest.raises(ValueError, match="bundled report differs"): + admit(evidence) + + +def test_raw_output_cannot_redirect_to_another_trials_file(evidence): + _, report, artifacts, _, native = evidence + native["trials"][2]["raw_rewards"] = deepcopy(native["trials"][1]["raw_rewards"]) + save_bundle(artifacts, report, native) + with pytest.raises(ValueError, match="verified artifact inventory"): + admit(evidence) + + +def test_tampered_artifact_or_subject_cannot_be_reused(evidence): + subject, _, artifacts, _, _ = evidence + (artifacts / "harbor-evidence.json").write_text("{}") + with pytest.raises(ValueError, match="checksum"): + admit(evidence) + (subject / "instruction.md").write_text("changed") + with pytest.raises(ValueError, match="changed after validation"): + admit(evidence) + + +def approved_task(evidence, output): + record = {**admit(evidence), "subject": str(evidence[0]), "task_id": "good-task"} + return pipeline.materialize(record, output / "approved/tasks/good") + + +def test_execution_pins_only_declared_overlay_and_preserves_validated_source( + evidence, tmp_path +): + subject = evidence[0] + original = (subject / "task.toml").read_bytes() + record = approved_task(evidence, tmp_path / "output") + assert (subject / "task.toml").read_bytes() == original + assert source_digest(Path(record["execution_path"])) == record["execution_digest"] + assert ( + record["execution_overlay"]["environment"]["docker_image"] + == record["image_ref"] + ) + + +def test_copy_omits_unhashed_files_and_never_dereferences_links(evidence, tmp_path): + source = evidence[0] + hidden = source / ".worktrees" + hidden.mkdir() + (hidden / "host-file").symlink_to("/etc/hosts") + expected = source_digest(source) + destination = tmp_path / "copy" + pipeline.copy_subject(source, destination, expected) + assert not (destination / ".worktrees").exists() + (source / "escaped").symlink_to("/etc/hosts") + with pytest.raises(ValueError, match="symbolic links"): + pipeline.copy_subject(source, tmp_path / "unsafe", expected) + + +def test_only_approved_tasks_reach_existing_model_runner( + evidence, tmp_path, monkeypatch +): + from openenv.harbor import runner + + output = tmp_path / "output" + record = approved_task(evidence, output) + call = AsyncMock( + return_value=[ + HarborRolloutResult(task_id=record["execution_path"], reward=0, ok=True) + ] + ) + monkeypatch.setattr(runner, "run_batch", call) + rows = pipeline.evaluate( + [record], + output, + llm_url="http://model/v1", + model="test", + harness="opencode", + expose="direct", + timeout_s=10, + step_limit=4, + ) + assert len(rows) == 1 and rows[0]["result"]["reward"] == 0 + assert call.call_args.kwargs["task_indices"] == [0] + assert call.call_args.kwargs["agent_step_limit"] == 4 + # An unapproved sibling must stop the runner before another model call. + shutil.copytree(record["execution_path"], output / "approved/tasks/unapproved") + from openenv.harbor.tasks import resolve_task_dirs + + resolve_task_dirs(str(output / "approved"), refresh=True) + with pytest.raises(ValueError, match="corpus disagree"): + pipeline.evaluate( + [record], + output, + llm_url="http://model/v1", + model="test", + harness="opencode", + expose="direct", + timeout_s=10, + step_limit=4, + ) + assert call.await_count == 1 + + +def test_model_image_guard_runs_before_agent_execution(evidence, tmp_path, monkeypatch): + record = approved_task(evidence, tmp_path / "output") + hooks = [] + trial = SimpleNamespace( + config=SimpleNamespace( + task=SimpleNamespace(path=Path(record["execution_path"])) + ), + agent_environment=SimpleNamespace(session_id="oe-good__env"), + add_hook=lambda event, callback: hooks.append(callback), + ) + actual = {"Image": "sha256:" + "b" * 64} + monkeypatch.setattr( + pipeline, + "docker", + lambda *args: "container" if args[0] == "ps" else json.dumps([actual]), + ) + projects = set() + pipeline.model_guard([record], projects)(trial) + assert projects == {"oe-good__env"} + with pytest.raises(ValueError, match="image differs"): + asyncio.run(hooks[0](None)) diff --git a/examples/hub_validation_pipeline/test_prepared_batch.py b/examples/hub_validation_pipeline/test_prepared_batch.py new file mode 100644 index 000000000..83175e7fd --- /dev/null +++ b/examples/hub_validation_pipeline/test_prepared_batch.py @@ -0,0 +1,689 @@ +"""Offline cohort binding, spend gates and one-shot controller polling.""" + +import hashlib +import json +import shutil +import socket +import time +from collections import Counter +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from examples.hub_validation_pipeline import cloud, full_run, prepared_batch, spend +from examples.hub_validation_pipeline.test_real_tasks import write_panel + +SECRETS = dict( + HF_TOKEN="local-results", + HF_INFERENCE_TOKEN="local-inference", + HF_SANDBOX_TOKEN="local-execution", +) + + +@pytest.fixture +def cohort(tmp_path, monkeypatch): + forbidden = Mock(side_effect=AssertionError("Live operations forbidden")) + monkeypatch.setattr(socket.socket, "connect", forbidden) + import huggingface_hub + + monkeypatch.setattr(huggingface_hub, "HfApi", forbidden) + monkeypatch.setattr(huggingface_hub, "get_token", forbidden) + repo = tmp_path / "repo" + (repo / "src/openenv").mkdir(parents=True) + (repo / "src/openenv/__init__.py").write_text("# source\n") + (repo / "examples/hub_validation_pipeline").mkdir(parents=True) + (repo / "examples/hub_validation_pipeline/cloud.py").write_bytes( + Path(cloud.__file__).read_bytes() + ) + shutil.copytree( + Path(cloud.__file__).parents[1] / "coding_audit", + repo / "examples/coding_audit", + ignore=shutil.ignore_patterns("test_*", "__pycache__", ".pytest_cache"), + ) + for name in ("pyproject.toml", "README.md", "LICENSE", "uv.lock"): + (repo / name).write_text(name + "\n") + monkeypatch.setattr( + cloud, "_git_provenance", lambda root: {"commit": "a" * 40, "dirty": False} + ) + task = tmp_path / "task" + (task / "tests").mkdir(parents=True) + (task / "solution").mkdir() + (task / "task.toml").write_text("version = '1.0'\n") + (task / "tests/test.sh").write_text("printf 1\n") + (task / "solution/solve.sh").write_text("true\n") + prepared = tmp_path / "prepared" + configs, plans, rows = [], [], [] + for index in range(4): + (task / "instruction.md").write_text(f"Distinct task {index}\n") + config = cloud._loads( + (Path(__file__).parents[1] / "coding_audit/config.json").read_bytes() + ) + config["cloud"].update(namespace="owner", timeout_seconds=60) + path = prepared / f"task-{index}" / "plan.json" + cloud.prepare(task, config, path.parent, repo_root=repo) + configs.append(config) + plans.append(path) + rows.append( + dict( + task_id=f"task-{index}", + plan_path=str(path.relative_to(prepared)), + plan_sha256=full_run._hash(path)["sha256"], + ) + ) + source = prepared / "preparation-manifest.json" + source.write_bytes( + cloud._json_bytes({"tasks": rows, "evaluation_protocol": "test"}) + ) + ledger = tmp_path / "local-spend.json" + spend.init(ledger, "500") + output = tmp_path / "batch" + api = Mock() + now, sleeps, events = [0], [], [] + counts, launched, active = Counter(), [], set() + + def sleep(seconds): + assert 0 <= seconds <= 30 + sleeps.append(seconds) + now[0] += seconds + + actual_submit = cloud.submit + + def submit(path, *, api, secrets, expected_plan_sha256): + assert secrets == SECRETS + assert full_run._hash(path)["sha256"] == expected_plan_sha256 + durable = cloud._loads((output / "manifest.json").read_bytes()) + index = plans.index(path) + assert durable["tasks"][index]["status"] == "pending_or_unknown" + if index: + assert durable["tasks"][index - 1]["job_id"] == f"job-{index - 1}" + assert durable["tasks"][index - 1]["status"] == "accepted_unmeasured" + assert durable["shared_spend"]["status"] == "reserved" + assert ( + spend.summary(ledger)["reserved_usd_rounded_up"] + == durable["estimate"]["reservation_usd_rounded_up"] + ) + job_id = f"job-{index}" + launched.append(job_id) + active.add(job_id) + assert len(active) <= 2 + events.append(("submit", job_id, now[0])) + plan = cloud._loads(path.read_bytes()) + receipt = dict( + status="submitted", + run_id=plan["run_id"], + job_id=job_id, + job_url=f"https://huggingface.co/jobs/owner/{job_id}", + launch_spec=plan["mcp_request"]["args"], + ) + (path.parent / "submission.json").write_bytes(cloud._json_bytes(receipt)) + return receipt + + def inspect_job(*, job_id, namespace, token): + assert job_id in launched + assert namespace == "owner" and token == SECRETS["HF_SANDBOX_TOKEN"] + counts[job_id] += 1 + stage = "RUNNING" if counts[job_id] == 1 else "COMPLETED" + if stage == "COMPLETED": + active.remove(job_id) + events.append(("inspect", job_id, now[0])) + return SimpleNamespace( + id=job_id, + docker_image=cloud.WORKER_IMAGE, + owner=SimpleNamespace(name="owner"), + status=SimpleNamespace(stage=stage), + ) + + launch = Mock(side_effect=submit) + api.inspect_job.side_effect = inspect_job + monkeypatch.setattr(cloud, "submit", launch) + return SimpleNamespace( + source=source, + output=output, + ledger=ledger, + configs=configs, + plans=plans, + rows=rows, + api=api, + actual_submit=actual_submit, + launch=launch, + now=now, + sleeps=sleeps, + events=events, + arguments=dict( + prepared_manifest=source, + output=output, + spend_ledger=ledger, + reservation_id="cohort-v1", + budget_usd="500", + api=api, + secrets=SECRETS, + sleep=sleep, + clock=lambda: now[0], + ), + ) + + +def saved(cohort): + return cloud._loads((cohort.output / "manifest.json").read_bytes()) + + +def rewrite_manifest(cohort): + cohort.source.write_bytes(cloud._json_bytes({"tasks": cohort.rows})) + + +def test_prepare_only_has_no_credential_network_or_ledger_io(cohort, monkeypatch): + forbidden = Mock(side_effect=AssertionError("No credentials/ledger IO")) + monkeypatch.setattr(cloud, "_runtime_secrets", forbidden) + monkeypatch.setattr(spend, "reserve", forbidden) + cohort.ledger.unlink() + result = prepared_batch.run(**cohort.arguments) + assert result == saved(cohort) + assert result["status"] == "prepared" + assert result["accepted"] is result["results_collected"] is False + assert result["estimate"] == full_run.estimate(cohort.configs) + assert result["prepared_manifest"] == full_run._hash(cohort.source) + assert result["shared_spend"]["status"] == "not_reserved" + assert not cohort.ledger.exists() + cohort.launch.assert_not_called() + cohort.api.inspect_job.assert_not_called() + forbidden.assert_not_called() + + +def test_exact_aggregate_is_reserved_first_and_only_two_controllers_run(cohort): + result = prepared_batch.run(**cohort.arguments, submit=True) + assert result == saved(cohort) + assert result["status"] == "controllers_completed_unmeasured" + assert result["accepted"] is result["results_collected"] is False + assert all( + task["status"] == "accepted_unmeasured" + and task["controller_status"] == "COMPLETED" + and task["accepted"] is False + for task in result["tasks"] + ) + assert cohort.sleeps == [30, 30] + assert cohort.events == [ + ("submit", "job-0", 0), + ("submit", "job-1", 0), + ("inspect", "job-0", 0), + ("inspect", "job-1", 0), + ("inspect", "job-0", 30), + ("inspect", "job-1", 30), + ("submit", "job-2", 30), + ("submit", "job-3", 30), + ("inspect", "job-2", 30), + ("inspect", "job-3", 30), + ("inspect", "job-2", 60), + ("inspect", "job-3", 60), + ] + fingerprint = hashlib.sha256( + "\0".join(task["bundle_sha256"] for task in result["tasks"]).encode() + ).hexdigest()[:16] + assert result["shared_spend"]["reservation_id"] == "cohort-v1:" + fingerprint + ledger = json.loads(cohort.ledger.read_text()) + assert ( + ledger["reservations"]["cohort-v1:" + fingerprint]["reserved_usd"] + == result["estimate"]["reservation_usd_rounded_up"] + ) + text = (cohort.output / "manifest.json").read_text() + assert all( + value not in text + for value in (*SECRETS.values(), "launch_spec", "command", "coding-config") + ) + + +@pytest.mark.parametrize("global_cap", [False, True]) +def test_aggregate_caps_stop_all_submissions(cohort, monkeypatch, global_cap): + if global_cap: + spend.reserve(cohort.ledger, "prior", "500", "Prior unresolved work") + else: + cohort.arguments["budget_usd"] = full_run.estimate(cohort.configs[:3])[ + "reservation_usd_rounded_up" + ] + credentials = Mock(side_effect=AssertionError("No token IO before budget gate")) + monkeypatch.setattr(cloud, "_runtime_secrets", credentials) + with pytest.raises( + RuntimeError, match="during reserve" if global_cap else "during budget" + ): + prepared_batch.run(**cohort.arguments, submit=True) + assert saved(cohort)["status"] == "failed" + cohort.launch.assert_not_called() + cohort.api.inspect_job.assert_not_called() + credentials.assert_not_called() + + +@pytest.mark.parametrize( + "defect", + [ + "empty", + "bad-json", + "duplicate-key", + "changed-plan", + "bad-bundle", + "duplicate-task", + "duplicate-run", + "absolute", + "escape", + "symlink", + "receipt", + "dangling-receipt", + ], +) +def test_invalid_cohort_stops_before_spend_or_credentials(cohort, monkeypatch, defect): + if defect == "empty": + cohort.rows.clear() + elif defect == "changed-plan": + cohort.plans[-1].write_text("changed") + elif defect == "bad-bundle": + (cohort.plans[-1].parent / "bundle.zip").write_bytes(b"changed") + elif defect == "duplicate-task": + cohort.rows[-1]["task_id"] = cohort.rows[0]["task_id"] + elif defect == "duplicate-run": + shutil.rmtree(cohort.plans[-1].parent) + shutil.copytree(cohort.plans[0].parent, cohort.plans[-1].parent) + cohort.rows[-1]["plan_sha256"] = cohort.rows[0]["plan_sha256"] + elif defect == "absolute": + cohort.rows[-1]["plan_path"] = str(cohort.plans[-1]) + elif defect == "escape": + cohort.rows[-1]["plan_path"] = "../task/plan.json" + elif defect == "symlink": + outside = cohort.source.parent.parent / "outside" + shutil.copytree(cohort.plans[-1].parent, outside) + shutil.rmtree(cohort.plans[-1].parent) + cohort.plans[-1].parent.symlink_to(outside, target_is_directory=True) + elif defect in {"receipt", "dangling-receipt"}: + receipt = cohort.plans[-1].parent / "submission.json" + if defect == "receipt": + receipt.write_text('{"status":"launch_unknown"}') + else: + receipt.symlink_to(receipt.parent / "missing") + rewrite_manifest(cohort) + if defect == "bad-json": + cohort.source.write_text("{") + elif defect == "duplicate-key": + cohort.source.write_text('{"tasks":[],"tasks":[]}') + forbidden = Mock(side_effect=AssertionError("No paid calls/credentials")) + monkeypatch.setattr(spend, "reserve", forbidden) + monkeypatch.setattr(cloud, "_runtime_secrets", forbidden) + with pytest.raises(RuntimeError, match="during plans"): + prepared_batch.run(**cohort.arguments, submit=True) + forbidden.assert_not_called() + cohort.launch.assert_not_called() + + +@pytest.mark.parametrize("which", ["plan", "manifest"]) +def test_change_after_estimate_is_caught_before_reservation(cohort, monkeypatch, which): + estimate = full_run.estimate + + def change(configs): + result = estimate(configs) + (cohort.source if which == "manifest" else cohort.plans[-1]).write_text( + "changed" + ) + return result + + monkeypatch.setattr(full_run, "estimate", change) + reserve = Mock(side_effect=AssertionError("No reservation")) + monkeypatch.setattr(spend, "reserve", reserve) + with pytest.raises(RuntimeError, match="during budget"): + prepared_batch.run(**cohort.arguments, submit=True) + reserve.assert_not_called() + cohort.launch.assert_not_called() + + +@pytest.mark.parametrize("stage", ["ERROR", "CANCELED", "DELETED", "NEW_STATE"]) +def test_terminal_failure_and_unknown_stop_further_launches(cohort, stage): + cohort.api.inspect_job.side_effect = lambda **kw: SimpleNamespace( + id=kw["job_id"], + docker_image=cloud.WORKER_IMAGE, + owner=SimpleNamespace(name="owner"), + status=SimpleNamespace(stage=stage), + ) + with pytest.raises(RuntimeError, match="during poll"): + prepared_batch.run(**cohort.arguments, submit=True) + result = saved(cohort) + assert cohort.launch.call_count == 2 + assert result["tasks"][0]["controller_status"] == ( + stage if stage in prepared_batch.STAGES else "UNKNOWN" + ) + assert [task.get("job_id") for task in result["tasks"]] == [ + "job-0", + "job-1", + None, + None, + ] + assert all((path.parent / "submission.json").exists() for path in cohort.plans[:2]) + + +@pytest.mark.parametrize("field", ["id", "docker_image", "owner", "api-error"]) +def test_inspection_failure_or_wrong_controller_stops(cohort, field): + inspect = cohort.api.inspect_job.side_effect + + def bad(**kwargs): + if field == "api-error": + raise RuntimeError("private-inspection-error") + job = inspect(**kwargs) + if field == "owner": + job.owner.name = "other" + else: + setattr(job, field, "wrong") + return job + + cohort.api.inspect_job.side_effect = bad + with pytest.raises(RuntimeError, match="during poll") as error: + prepared_batch.run(**cohort.arguments, submit=True) + assert cohort.launch.call_count == 2 + assert saved(cohort)["tasks"][0]["controller_status"] == "UNKNOWN" + assert ( + "private-inspection-error" + not in str(error.value) + (cohort.output / "manifest.json").read_text() + ) + + +def test_timeout_uses_hard_controller_cap_plus_startup(cohort): + cohort.api.inspect_job.side_effect = lambda **kw: SimpleNamespace( + id=kw["job_id"], + docker_image=cloud.WORKER_IMAGE, + owner=SimpleNamespace(name="owner"), + status=SimpleNamespace(stage="SCHEDULING"), + ) + with pytest.raises(RuntimeError, match="during poll"): + prepared_batch.run(**cohort.arguments, submit=True) + assert cohort.now[0] == 60 + full_run.RATEBOOK["startup_allowance_seconds"] + assert cohort.sleeps == [30] * 12 + assert cohort.launch.call_count == 2 + assert saved(cohort)["tasks"][0]["controller_status"] == "TIMEOUT" + + +@pytest.mark.parametrize("wrong_owner", [False, True]) +def test_default_namespace_still_checks_owner_from_submission_url(cohort, wrong_owner): + config = cohort.configs[0] + config["cloud"].pop("namespace") + shutil.rmtree(cohort.plans[0].parent) + root = cohort.source.parent.parent + cloud.prepare( + root / "task", config, cohort.plans[0].parent, repo_root=root / "repo" + ) + cohort.rows[:] = cohort.rows[:1] + cohort.rows[0]["plan_sha256"] = full_run._hash(cohort.plans[0])["sha256"] + rewrite_manifest(cohort) + + def inspect_job(*, job_id, namespace, token): + assert namespace is None and token == SECRETS["HF_SANDBOX_TOKEN"] + return SimpleNamespace( + id=job_id, + docker_image=cloud.WORKER_IMAGE, + owner=SimpleNamespace(name="other" if wrong_owner else "owner"), + status=SimpleNamespace(stage="COMPLETED"), + ) + + cohort.api.inspect_job.side_effect = inspect_job + if wrong_owner: + with pytest.raises(RuntimeError, match="during poll"): + prepared_batch.run(**cohort.arguments, submit=True) + else: + assert ( + prepared_batch.run(**cohort.arguments, submit=True)["status"] + == "controllers_completed_unmeasured" + ) + + +@pytest.mark.parametrize("receipt_kind", ["unknown", "known-id", "partial"]) +def test_uncertain_launch_preserves_every_receipt_and_never_retries( + cohort, receipt_kind +): + launch = cohort.launch.side_effect + + def fail(path, **kwargs): + if path != cohort.plans[1]: + return launch(path, **kwargs) + receipt = dict( + status="launch_unknown", run_id=cloud._loads(path.read_bytes())["run_id"] + ) + if receipt_kind == "known-id": + receipt.update( + job_id="job-uncertain", + job_url="https://huggingface.co/jobs/owner/job-uncertain", + ) + (path.parent / "submission.json").write_bytes( + b"{" if receipt_kind == "partial" else cloud._json_bytes(receipt) + ) + raise RuntimeError("private-launch-error") + + cohort.launch.side_effect = fail + with pytest.raises(RuntimeError, match="during submit") as error: + prepared_batch.run(**cohort.arguments, submit=True) + result = saved(cohort) + assert cohort.launch.call_count == 2 + assert result["tasks"][0]["job_id"] == "job-0" + assert result["tasks"][1]["status"] == "pending_or_unknown" + if receipt_kind == "known-id": + assert result["tasks"][1]["job_id"] == "job-uncertain" + assert (cohort.plans[1].parent / "submission.json").exists() + assert ( + "private-launch-error" + not in str(error.value) + (cohort.output / "manifest.json").read_text() + ) + with pytest.raises(FileExistsError): + prepared_batch.run(**cohort.arguments, submit=True) + cohort.arguments["output"] = cohort.output.parent / "different-output" + with pytest.raises(RuntimeError, match="during plans"): + prepared_batch.run(**cohort.arguments, submit=True) + assert cohort.launch.call_count == 2 + + +def test_cli_requires_all_binding_arguments_and_defaults_to_prepare(cohort, capsys): + arguments = [ + "--prepared-manifest", + str(cohort.source), + "--output", + str(cohort.output), + "--spend-ledger", + str(cohort.ledger), + "--reservation-id", + "cohort", + "--budget-usd", + "500", + ] + for index in range(0, len(arguments), 2): + with pytest.raises(SystemExit): + prepared_batch.main(arguments[:index] + arguments[index + 2 :]) + assert not cohort.output.exists() + prepared_batch.main(arguments) + assert "prepared:" in capsys.readouterr().out + cohort.launch.assert_not_called() + with pytest.raises(FileExistsError): + prepared_batch.main(arguments) + + +def test_replacement_after_last_budget_check_cannot_launch(cohort, monkeypatch): + path = cohort.plans[0] + replacement = cohort.source.parent / "replacement" + config = dict(cohort.configs[0]) + config["cloud"] = dict(config["cloud"], timeout_seconds=86400) + cloud.prepare( + cohort.source.parent.parent / "task", + config, + replacement, + repo_root=cohort.source.parent.parent / "repo", + ) + original = cloud._runtime_secrets + calls = 0 + + def replace_during_submit(secrets): + nonlocal calls + calls += 1 + if calls == 2: + for name in ("plan.json", "bundle.zip"): + shutil.copyfile(replacement / name, path.parent / name) + return original(secrets) + + monkeypatch.setattr(cloud, "_runtime_secrets", replace_during_submit) + monkeypatch.setattr(cloud, "submit", cohort.actual_submit) + with pytest.raises(RuntimeError, match="during submit"): + prepared_batch.run(**cohort.arguments, submit=True) + assert calls == 2 + cohort.api.run_job.assert_not_called() + cohort.api.create_commit.assert_not_called() + assert not (path.parent / "submission.json").exists() + assert saved(cohort)["shared_spend"]["status"] == "reserved" + + +@pytest.mark.parametrize("runner", ["prepared", "native"]) +def test_initial_parse_swap_cannot_underprice_reviewed_plan( + cohort, monkeypatch, runner +): + path = cohort.plans[0] + cheap = { + name: (path.parent / name).read_bytes() for name in ("plan.json", "bundle.zip") + } + replacement = cohort.source.parent / "replacement" + config = dict(cohort.configs[0]) + config["cloud"] = dict(config["cloud"], timeout_seconds=86400) + cloud.prepare( + cohort.source.parent.parent / "task", + config, + replacement, + repo_root=cohort.source.parent.parent / "repo", + ) + expensive = {name: (replacement / name).read_bytes() for name in cheap} + for name, content in expensive.items(): + (path.parent / name).write_bytes(content) + cohort.rows[:] = cohort.rows[:1] + cohort.rows[0]["plan_sha256"] = full_run._hash(path)["sha256"] + rewrite_manifest(cohort) + original = cloud._read_plan + swapped = False + + def swap_at_parse(candidate, **kwargs): + nonlocal swapped + if Path(candidate).resolve() == path and not swapped: + swapped = True + for name, content in cheap.items(): + (path.parent / name).write_bytes(content) + try: + return original(candidate, **kwargs) + finally: + for name, content in expensive.items(): + (path.parent / name).write_bytes(content) + return original(candidate, **kwargs) + + monkeypatch.setattr(cloud, "_read_plan", swap_at_parse) + cohort.api.dataset_info.return_value = SimpleNamespace(private=True, sha="a" * 40) + cohort.api.list_repo_files.return_value = [] + cohort.api.create_commit.return_value = SimpleNamespace(oid="b" * 40) + cohort.api.run_job.return_value = SimpleNamespace( + id="job-0", url="https://huggingface.co/jobs/owner/job-0" + ) + cap = full_run.estimate(cohort.configs[:1])["reservation_usd_rounded_up"] + if runner == "prepared": + monkeypatch.setattr(cloud, "submit", cohort.actual_submit) + + def invocation(): + return prepared_batch.run( + **dict(cohort.arguments, budget_usd=cap), submit=True + ) + else: + monkeypatch.setattr( + full_run.real_tasks, + "_prepare", + lambda *args, **kwargs: { + "tasks": [{"task_id": "toy", "bundle_plan": str(path)}] + }, + ) + monkeypatch.setattr(full_run, "_catalog", lambda *args: {}) + monkeypatch.setattr( + cloud, + "submit", + lambda candidate, **kwargs: cohort.actual_submit( + candidate, api=cohort.api, secrets=SECRETS, **kwargs + ), + ) + + def invocation(): + return full_run.run( + cohort.output, + cohort.source, + cohort.source, + cap, + panel_path=write_panel(cohort.source.parent), + submit=True, + spend_ledger=cohort.ledger, + reservation_id="native-toy", + ) + + with pytest.raises(RuntimeError, match="during plans"): + invocation() + assert swapped + assert full_run._hash(path)["sha256"] == cohort.rows[0]["plan_sha256"] + assert spend.summary(cohort.ledger)["reserved_usd_exact"] == "0" + cohort.api.run_job.assert_not_called() + cohort.api.create_commit.assert_not_called() + + +def test_preparation_manifest_digest_binds_the_bytes_parsed(cohort, monkeypatch): + original = cohort.source.read_bytes() + replacement = cloud._json_bytes({"tasks": cohort.rows[:1]}) + read_bytes = Path.read_bytes + changed = False + + def replace_only_while_reading(path): + nonlocal changed + if path.resolve() == cohort.source and not changed: + changed = True + path.write_bytes(replacement) + try: + return read_bytes(path) + finally: + path.write_bytes(original) + return read_bytes(path) + + monkeypatch.setattr(Path, "read_bytes", replace_only_while_reading) + with pytest.raises(RuntimeError, match="during budget"): + prepared_batch.run(**cohort.arguments, submit=True) + assert changed and read_bytes(cohort.source) == original + cohort.launch.assert_not_called() + assert spend.summary(cohort.ledger)["reserved_usd_exact"] == "0" + + +def test_blocking_inspection_is_interrupted_by_deadline(cohort): + def blocked(**kwargs): + time.sleep(5) + pytest.fail("inspection was not interrupted") + + cohort.api.inspect_job.side_effect = blocked + start = time.monotonic() + result = prepared_batch._status( + {"job_id": "owned"}, + {"image": cloud.WORKER_IMAGE}, + "owner", + start + 0.05, + cohort.api, + SECRETS, + time.monotonic, + ) + assert result == "TIMEOUT" + assert time.monotonic() - start < 1 + + +@pytest.mark.parametrize( + "url", + [ + "https://huggingface.co/jobs//job-0", + "https://foreign.invalid/jobs/owner/job-0", + "https://huggingface.co/jobs/owner/different-job", + ], +) +def test_malformed_controller_owner_url_never_bypasses_identity(cohort, url): + submit = cohort.launch.side_effect + + def malformed(*args, **kwargs): + receipt = submit(*args, **kwargs) + receipt["job_url"] = url + return receipt + + cohort.launch.side_effect = malformed + with pytest.raises(RuntimeError, match="during submit"): + prepared_batch.run(**cohort.arguments, submit=True) + cohort.api.inspect_job.assert_not_called() + assert cohort.launch.call_count == 1 diff --git a/examples/hub_validation_pipeline/test_real_tasks.py b/examples/hub_validation_pipeline/test_real_tasks.py new file mode 100644 index 000000000..81b0269d7 --- /dev/null +++ b/examples/hub_validation_pipeline/test_real_tasks.py @@ -0,0 +1,577 @@ +"""CPU-only operator-panel, acquisition and SDK transfer-contract checks.""" + +import hashlib +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from examples.coding_audit.hf import HFTaskBackend +from examples.hub_validation_pipeline import cloud, real_tasks +from huggingface_hub import RepoFile, RepoFolder +from huggingface_hub.errors import SandboxError +from openenv.validation.runner import source_digest + + +TOY_NAMES = ("toy-alpha", "toy-beta") +TOY_SOURCE = {"kind": "hf", "repo_id": "example/toy-tasks", "revision": "b" * 40} + + +def toy_config(name=TOY_NAMES[0]): + config = cloud._loads( + (Path(__file__).parents[1] / "coding_audit/config.json").read_bytes() + ) + config["execution"].update( + agent_image="example/toy@sha256:" + "c" * 64, + verifier_image="example/toy@sha256:" + "c" * 64, + submission_paths=[f"/app/{name}.txt"], + timeout_seconds=60, + max_tool_output_bytes=1048576, + ) + config["negative_cases"] = [ + {"id": "toy-empty", "script": f": > /app/{name}.txt"}, + {"id": "toy-wrong", "script": f"printf pear > /app/{name}.txt"}, + ] + config["validation"].update( + required_negative_cases=[case["id"] for case in config["negative_cases"]], + target_attempts=2, + reference_repeats=2, + ) + config["seeds"] = [17, 29] + config["generation"]["candidate_count"] = 1 + config["target"].update(max_steps=3, max_tokens=64, max_total_tokens=256) + config["generator"] = dict(config["target"], max_steps=1) + config["cloud"].update(results_repo="example/results", timeout_seconds=60) + return cloud._config(config)[0] + + +def write_panel(directory, names=TOY_NAMES): + tasks = [] + for name in names: + config = toy_config(name) + tasks.append( + { + "name": name, + "task_path": f"fixtures/{name}", + "image": config["execution"]["agent_image"], + "submission_path": config["execution"]["submission_paths"][0], + "config": config, + "negative_counterexamples": [{"id": "toy-wrong", "expected": "apple"}], + "annotations": {"caveat": "Synthetic fixture only"}, + } + ) + path = directory / "panel.json" + path.write_bytes( + cloud._json_bytes( + { + "schema": "openenv-native-evaluation-panel-v1", + "evaluation_only": True, + "source": TOY_SOURCE, + "tasks": tasks, + } + ) + ) + return path + + +def source_files(name): + return { + "task.toml": ( + '[task]\nname = "toy/' + name + '"\n' + "[agent]\ntimeout_sec = 900.0\n[verifier]\ntimeout_sec = 900.0\n" + '[environment]\ndocker_image = "example/' + name + ':fixture"\n' + "cpus = 1\nmemory_mb = 2048\nstorage_mb = 10240\ngpus = 0\n" + ).encode(), + "instruction.md": b"Public task instruction\n", + "solution/solve.sh": b"#!/bin/bash\necho native-reference\n", + "tests/test.sh": b"#!/bin/bash\necho 0 > /logs/verifier/reward.txt\n", + "tests/test_outputs.py": b"# native verifier asset\n", + "environment/fixture/task.toml": b"nested asset, never a separate task\n", + } + + +def hf_fakes(files=None): + files = ( + files + if files is not None + else { + f"fixtures/{name}/{path}": data + for name in TOY_NAMES + for path, data in source_files(name).items() + } + ) + api = Mock() + api.dataset_info.return_value = SimpleNamespace( + sha=TOY_SOURCE["revision"], gated=False, disabled=False + ) + + def listing(repo_id, **kwargs): + prefix = kwargs["path_in_repo"] + "/" + yield RepoFolder(path=prefix + "tests", oid="0" * 40) + for path, data in files.items(): + if path.startswith(prefix): + yield RepoFile( + path=path, + size=len(data), + oid=hashlib.sha1(f"blob {len(data)}\0".encode() + data).hexdigest(), + ) + + api.list_repo_tree.side_effect = listing + + def download(repo_id, filename, **kwargs): + destination = kwargs["local_dir"] / filename + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(files[filename]) + return str(destination) + + return api, Mock(side_effect=download) + + +@pytest.fixture(autouse=True) +def no_live_network(monkeypatch): + import socket + + monkeypatch.setattr( + socket.socket, "connect", Mock(side_effect=AssertionError("network forbidden")) + ) + monkeypatch.setattr( + cloud, "submit", Mock(side_effect=AssertionError("submission forbidden")) + ) + for name in cloud.SECRET_NAMES: + monkeypatch.delenv(name, raising=False) + + +def test_prepare_pinned_selected_native_trees_and_consumable_manifest(tmp_path): + api, download = hf_fakes() + panel_path = write_panel(tmp_path) + panel = cloud._loads(panel_path.read_bytes()) + output = tmp_path / "recipes" + manifest = real_tasks.prepare( + output, panel_path=panel_path, api=api, download=download + ) + assert manifest == cloud._loads((output / "manifest.json").read_bytes()) + assert manifest["evaluation_only"] is True and manifest["status"] == "prepared" + assert [row["name"] for row in manifest["tasks"]] == list(TOY_NAMES) + panel_sha = hashlib.sha256(panel_path.read_bytes()).hexdigest() + assert manifest["evaluation_panel"] == { + "path": str(panel_path), + "sha256": panel_sha, + "bytes": panel_path.stat().st_size, + } + for row, task in zip(manifest["tasks"], panel["tasks"]): + root = Path(row["task_root"]) + assert row["source"] == { + **TOY_SOURCE, + "task_path": task["task_path"], + "evaluation_panel_sha256": panel_sha, + "task_file_hashes": { + path: record["sha256"] for path, record in row["files"].items() + }, + } + assert row["source_digest"] == source_digest(root) + assert row["source_image"].endswith(":fixture") + assert row["evaluation_only"] is True + for path, data in source_files(row["name"]).items(): + assert (root / path).read_bytes() == data + assert not (root / path).is_symlink() + assert row["files"][path]["sha256"] == hashlib.sha256(data).hexdigest() + config = cloud._loads(Path(row["config_path"]).read_bytes()) + assert config == task["config"] + assert row["budget"] == cloud._config(config)[1] + assert row["negative_counterexamples"] == task["negative_counterexamples"] + assert row["annotations"] == task["annotations"] + assert "negative_counterexamples" not in config + for call in api.list_repo_tree.call_args_list: + assert call.args[0] == TOY_SOURCE["repo_id"] + assert call.kwargs["revision"] == TOY_SOURCE["revision"] + assert call.kwargs["path_in_repo"] in {"fixtures/" + name for name in TOY_NAMES} + assert call.kwargs["recursive"] is True and call.kwargs["token"] is False + for call in download.call_args_list: + assert call.kwargs["revision"] == TOY_SOURCE["revision"] + assert call.kwargs["repo_type"] == "dataset" and call.kwargs["token"] is False + with pytest.raises(FileExistsError): + real_tasks.prepare(output, panel_path=panel_path, api=api, download=download) + + +def test_bundle_and_cli_pass_exact_preparation_contract(tmp_path, monkeypatch, capsys): + api, download = hf_fakes() + panel_path = write_panel(tmp_path) + prepare = Mock() + monkeypatch.setattr(cloud, "prepare", prepare) + monkeypatch.setattr(real_tasks, "HfApi", Mock(return_value=api)) + monkeypatch.setattr(real_tasks, "hf_hub_download", download) + output = tmp_path / "cli" + real_tasks.main( + [ + "--panel", + str(panel_path), + "--output", + str(output), + "--task", + "toy-alpha", + "--bundle", + ] + ) + row = json.loads((output / "manifest.json").read_text())["tasks"][0] + call = prepare.call_args + assert call.args[0] == Path(row["task_root"]) + assert call.args[1]["cloud"]["results_repo"] == "example/results" + assert call.kwargs["source"] == row["source"] + assert row["bundle_plan"] == str(call.args[2] / "plan.json") + assert json.loads(capsys.readouterr().out)["status"] == "prepared" + + +def test_real_cloud_bundle_roundtrip_preserves_source_and_native_files(tmp_path): + api, download = hf_fakes() + panel_path = write_panel(tmp_path) + manifest = real_tasks.prepare( + tmp_path / "bundled", + panel_path=panel_path, + task_names=["toy-alpha"], + bundle=True, + api=api, + download=download, + ) + row = manifest["tasks"][0] + plan, config, data, bundle_manifest = cloud._read_plan(row["bundle_plan"]) + _, files = cloud._checked_archive(data, plan["bundle_sha256"]) + assert plan["task_source"] == bundle_manifest["task_source"] == row["source"] + assert config["execution"]["agent_image"] == row["execution_image"] + for path, data in source_files("toy-alpha").items(): + assert files["task/" + path] == data + + +def test_bundle_rejects_native_file_change_and_restore_after_acquisition( + tmp_path, monkeypatch +): + api, download = hf_fakes() + panel_path = write_panel(tmp_path) + original_tree = cloud._tree + + def changed_verifier(root, **kwargs): + root = Path(root) + if root.name != "toy-alpha": + return original_tree(root, **kwargs) + verifier = root / "tests/test.sh" + original = verifier.read_bytes() + verifier.write_text("#!/bin/bash\necho 1 > /logs/verifier/reward.txt\n") + try: + return original_tree(root, **kwargs) + finally: + verifier.write_bytes(original) + + monkeypatch.setattr(cloud, "_tree", changed_verifier) + output = tmp_path / "recipes" + with pytest.raises(ValueError, match="acquisition"): + real_tasks.prepare( + output, + panel_path=panel_path, + task_names=["toy-alpha"], + bundle=True, + api=api, + download=download, + ) + assert not (output / "bundles/toy-alpha/plan.json").exists() + + +@pytest.mark.parametrize( + "case", + [ + "revision", + "gated", + "escape", + "environment", + "file-cap", + "total-cap", + "hash", + "symlink", + "directory", + "missing", + ], +) +def test_acquisition_rejects_bad_source_before_materializing( + tmp_path, monkeypatch, case +): + name = "toy-alpha" + api, download = hf_fakes() + panel_path = write_panel(tmp_path) + if case == "revision": + api.dataset_info.return_value.sha = "main" + elif case == "gated": + api.dataset_info.return_value.gated = "manual" + elif case in ("escape", "environment"): + path = ( + "fixtures/elsewhere/task.toml" + if case == "escape" + else f"fixtures/{name}/.env" + ) + api.list_repo_tree.side_effect = None + api.list_repo_tree.return_value = [RepoFile(path=path, size=0, oid="0" * 40)] + elif case in ("file-cap", "total-cap"): + monkeypatch.setattr( + real_tasks, "MAX_FILE_BYTES" if case == "file-cap" else "MAX_TOTAL_BYTES", 1 + ) + elif case == "missing": + api.list_repo_tree.side_effect = None + api.list_repo_tree.return_value = [] + else: + original = download.side_effect + + def changed(*args, **kwargs): + path = Path(original(*args, **kwargs)) + if case == "hash": + path.write_bytes(b"x" * path.stat().st_size) + elif case == "symlink": + target = tmp_path / "outside" + target.write_text("outside") + path.unlink() + path.symlink_to(target) + else: + path.unlink() + path.mkdir() + return path + + download.side_effect = changed + destination = tmp_path / "task" + with pytest.raises(ValueError): + panel, _ = real_tasks._load_panel(panel_path) + real_tasks.acquire_task( + panel["tasks"][0], + destination, + source=panel["source"], + api=api, + download=download, + ) + assert not destination.exists() + if case in ( + "revision", + "gated", + "escape", + "environment", + "file-cap", + "total-cap", + "missing", + ): + download.assert_not_called() + + +@pytest.mark.parametrize("name", TOY_NAMES) +def test_same_image_fresh_solver_grader_transfer_with_sdk_fakes(tmp_path, name): + api, download = hf_fakes() + panel_path = write_panel(tmp_path) + root = tmp_path / "native" + panel, _ = real_tasks._load_panel(panel_path) + task = next(task for task in panel["tasks"] if task["name"] == name) + real_tasks.acquire_task( + task, root, source=panel["source"], api=api, download=download + ) + original_digest = source_digest(root) + config = toy_config(name) + execution = config["execution"] + exported = task["submission_path"] + submission_bytes = b"candidate file bytes\n" + verifier_output = "APT: fetching native verifier dependencies 雪\n" * 2048 + assert 16384 < len(verifier_output.encode()) < 1048576 + jobs, created = {}, [] + jobs_api = Mock() + + def inspect_job(*, job_id, **kwargs): + return SimpleNamespace( + id=job_id, + docker_image=execution["agent_image"], + status=SimpleNamespace(stage=jobs[job_id]), + owner=SimpleNamespace(name="owner"), + ) + + jobs_api.inspect_job.side_effect = inspect_job + jobs_api.cancel_job.side_effect = lambda *, job_id, **kwargs: jobs.update( + {job_id: "CANCELED"} + ) + + def factory(**kwargs): + assert kwargs["image"] == execution["agent_image"] + assert kwargs["forward_hf_token"] is False + assert ( + "env" not in kwargs and "secrets" not in kwargs and "volumes" not in kwargs + ) + identifier = f"fake-{len(created)}" + jobs[identifier] = "RUNNING" + data = {} + directories = {"/", "/tmp"} + + def stat_file(path): + if path in directories: + return SimpleNamespace(type="dir", size=0) + if path not in data: + raise SandboxError("missing", status_code=404) + return SimpleNamespace(type="file", size=len(data[path])) + + files = SimpleNamespace( + MAX_READ_BYTES=1048576, + data=data, + stat=stat_file, + exists=lambda path: path in data or path in directories, + read=lambda path: data[path], + write=lambda path, raw, **kwargs: data.update( + {path: raw.encode() if isinstance(raw, str) else raw} + ), + mkdir=directories.add, + delete=lambda path: data.pop(path), + ) + + def run(argv, **kwargs): + assert kwargs["cwd"] == "/app" + assert 0 < kwargs["timeout"] <= 60 + if argv == ["/bin/bash", "/tests/test.sh"]: + assert data[exported] == submission_bytes + assert data["/tests/test.sh"] == source_files(name)["tests/test.sh"] + assert not any(path.startswith("/solution") for path in data) + kwargs["on_stdout"](verifier_output) + kwargs["on_stderr"]("Native verifier completed.\n") + data["/logs/verifier/reward.txt"] = b"0\n" + else: + assert not any( + path.startswith(("/tests", "/solution")) for path in data + ) + data[exported] = submission_bytes + data["/app/not-exported.txt"] = b"not transferred" + return SimpleNamespace(exit_code=0, timed_out=False) + + sandbox = SimpleNamespace(id=identifier, files=files, run=run) + created.append(sandbox) + return sandbox + + backend = HFTaskBackend( + root, execution, token="execution-only", api=jobs_api, sandbox_factory=factory + ) + submission = backend.execute_script( + config["negative_cases"][0]["script"], "negative" + ) + grade = backend.grade(submission) + assert submission["error"] is None and grade["error"] is None + assert submission["files"] == {exported: submission_bytes} + assert grade["reward"] == 0 and grade["rawreward"] == b"0\n" + assert grade["transcript"] == [ + { + "exit_code": 0, + "output": verifier_output + "Native verifier completed.\n", + "output_truncated": False, + } + ] + assert submission["cleanup_verified"] and grade["cleanup_verified"] + assert len(created) == 2 and created[0].id != created[1].id + assert "/app/not-exported.txt" not in created[1].files.data + assert source_digest(root) == original_digest + + +def test_recipe_requires_explicit_panel_before_acquisition(tmp_path, monkeypatch): + acquisition = Mock(side_effect=AssertionError("No acquisition")) + monkeypatch.setattr(real_tasks, "HfApi", acquisition) + with pytest.raises(TypeError, match="panel_path"): + real_tasks.prepare(tmp_path / "recipes") + with pytest.raises(SystemExit): + real_tasks.main(["--output", str(tmp_path / "recipes")]) + acquisition.assert_not_called() + assert not (tmp_path / "recipes").exists() + + +@pytest.mark.parametrize( + "case", + [ + "schema", + "evaluation", + "source", + "repo", + "revision", + "empty", + "duplicate", + "name", + "path", + "image", + "submission", + "counterexamples", + "annotations", + "credentials", + "duplicate-json", + ], +) +def test_invalid_operator_panel_fails_before_acquisition(tmp_path, monkeypatch, case): + panel_path = write_panel(tmp_path) + panel = cloud._loads(panel_path.read_bytes()) + task = panel["tasks"][0] + if case == "schema": + panel["schema"] = "unknown" + elif case == "evaluation": + panel["evaluation_only"] = False + elif case == "source": + panel["source"]["kind"] = "local" + elif case == "repo": + panel["source"]["repo_id"] = "../escape/name" + elif case == "revision": + panel["source"]["revision"] = "main" + elif case == "empty": + panel["tasks"] = [] + elif case == "duplicate": + panel["tasks"].append(task) + elif case == "name": + task["name"] = "../escape" + elif case == "path": + task["task_path"] = "fixtures/.env-private" + elif case == "image": + task["image"] = "example/other@sha256:" + "d" * 64 + elif case == "submission": + task["submission_path"] = "/app/other.txt" + elif case == "counterexamples": + task["negative_counterexamples"] = "invalid" + elif case == "annotations": + task["annotations"] = [] + elif case == "credentials": + task["annotations"] = {"credential": "hf_" + "Z" * 32} + panel_path.write_bytes(cloud._json_bytes(panel)) + if case == "duplicate-json": + panel_path.write_bytes( + panel_path.read_bytes().replace( + b'"evaluation_only":true', + b'"evaluation_only":true,"evaluation_only":true', + ) + ) + acquisition = Mock(side_effect=AssertionError("No acquisition")) + monkeypatch.setattr(real_tasks, "HfApi", acquisition) + with pytest.raises(ValueError): + real_tasks.prepare(tmp_path / "recipes", panel_path=panel_path) + acquisition.assert_not_called() + assert not (tmp_path / "recipes").exists() + + +def test_recipe_hashes_exact_panel_bytes_and_loads_only_once(tmp_path, monkeypatch): + panel_path = write_panel(tmp_path) + # A noncanonical encoding must be hashed as supplied, rather than reserialized. + original_bytes = b" " + panel_path.read_bytes() + b"\n\n" + panel_path.write_bytes(original_bytes) + read_file = cloud._text_file + reads = [] + + def replace_after_read(path): + result = read_file(path) + if path == panel_path: + reads.append(path) + panel_path.write_text('{"replacement": true}') + return result + + monkeypatch.setattr(cloud, "_text_file", replace_after_read) + api, download = hf_fakes() + manifest = real_tasks.prepare( + tmp_path / "recipes", panel_path=panel_path, api=api, download=download + ) + expected = hashlib.sha256(original_bytes).hexdigest() + assert reads == [panel_path] + assert manifest["evaluation_panel"]["sha256"] == expected + assert manifest["evaluation_panel"]["bytes"] == len(original_bytes) + assert all( + row["source"]["evaluation_panel_sha256"] == expected + for row in manifest["tasks"] + ) + assert Path(manifest["tasks"][0]["config_path"]).read_bytes() == cloud._json_bytes( + toy_config() + ) diff --git a/examples/hub_validation_pipeline/test_sources.py b/examples/hub_validation_pipeline/test_sources.py new file mode 100644 index 000000000..12e10e5dc --- /dev/null +++ b/examples/hub_validation_pipeline/test_sources.py @@ -0,0 +1,411 @@ +"""Network-free checks for discovery, immutable locators, and task assets.""" + +import json +from fnmatch import fnmatchcase +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest +from examples.hub_validation_pipeline import sources + +SHA = "a" * 40 +DIGEST = "sha256:" + "b" * 64 +TASK_DIGEST = "sha256:" + "c" * 64 + + +def hf_candidate(**metadata): + return {"source": "hf", "id": "org/bundle", "revision": SHA, "metadata": metadata} + + +def fake_hf(monkeypatch, files): + api = Mock() + api.dataset_info.return_value = SimpleNamespace( + sha=SHA, gated=False, disabled=False + ) + api.list_repo_files.return_value = list(files) + factory = Mock(return_value=api) + monkeypatch.setattr(sources, "HfApi", factory) + + def download(repo_id, **kwargs): + root = kwargs["local_dir"] + for name, content in files.items(): + if any(fnmatchcase(name, pattern) for pattern in kwargs["allow_patterns"]): + destination = root / name + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(content) + return str(root) + + downloader = Mock(side_effect=download) + monkeypatch.setattr(sources, "snapshot_download", downloader) + return api, factory, downloader + + +def test_hf_exact_conjunction_trending_and_exhausted_pagination(monkeypatch): + api = Mock() + consumed = [] + + def pages(): + for number, tags in enumerate( + [ + ["harbor", "rl-environment"], + ["library:harbor"], + ["library:harbor", "rl-environment"], + ["library:harbor", "rl-environment"], + ] + ): + consumed.append(number) + yield SimpleNamespace( + id=f"org/{number}", sha=SHA, tags=tags, gated=False, disabled=False + ) + + api.list_datasets.return_value = pages() + monkeypatch.setattr(sources, "HfApi", Mock(return_value=api)) + candidates = sources.discover_hf(1) + assert [candidate["id"] for candidate in candidates] == ["org/2"] + assert candidates[0]["revision"] == SHA + assert consumed == [0, 1, 2, 3] + api.list_datasets.assert_called_once_with( + filter=("library:harbor", "rl-environment"), + sort="trending_score", + expand=["sha", "tags", "gated", "disabled"], + limit=None, + ) + + +def test_hf_bounded_assets_pinned_and_fresh(monkeypatch, tmp_path): + files = {"registry.json": "{}", "shared/data[train].txt": "shared"} + for number in range(5000): + prefix = f"environment/tasks/{number:04d}/" + files[prefix + "task.toml"] = "version = '1.0'" + files[prefix + "instruction.md"] = "do a task" + files[prefix + "environment/assets/data.txt"] = "actual asset" + api, factory, downloader = fake_hf(monkeypatch, files) + first = sources.acquire(hf_candidate(), tmp_path, max_tasks=1)[0] + second = sources.acquire(hf_candidate(), tmp_path, max_tasks=1)[0] + assert first["task_id"] == f"hf:org/bundle@{SHA}:environment/tasks/0000" + assert first["task_id"] == second["task_id"] + assert first["path"] != second["path"] + path = Path(first["path"]) + assert path.is_absolute() and path == path.resolve() + assert (path / "environment/assets/data.txt").read_text() == "actual asset" + assert not (path.parent / "0001").exists() + root = downloader.call_args.kwargs["local_dir"] + assert (root / "shared/data[train].txt").read_text() == "shared" + assert len(first["download_patterns"]) == 5 + assert first["revision"] == SHA and first["source_id"] == "org/bundle" + assert SHA in path.parts + factory.assert_called_with(token=False) + api.dataset_info.assert_called_with("org/bundle", revision=SHA) + api.list_repo_files.assert_called_with( + "org/bundle", repo_type="dataset", revision=SHA + ) + assert downloader.call_args.kwargs["revision"] == SHA + assert downloader.call_args.kwargs["repo_type"] == "dataset" + assert downloader.call_args.kwargs["force_download"] is True + + +@pytest.mark.parametrize("prefix", ["", "environment/tasks/solo/"]) +def test_hf_individual_and_nested_tasks(monkeypatch, tmp_path, prefix): + fake_hf(monkeypatch, {prefix + "task.toml": "", prefix + "instruction.md": ""}) + [record] = sources.acquire(hf_candidate(), tmp_path, 1) + assert record["task_path"] == (prefix.rstrip("/") or ".") + assert (Path(record["path"]) / "instruction.md").is_file() + + +def test_hf_same_basename_has_distinct_identity(monkeypatch, tmp_path): + fake_hf(monkeypatch, {"a/same/task.toml": "", "b/same/task.toml": ""}) + records = sources.acquire(hf_candidate(), tmp_path, 2) + assert len({record["task_id"] for record in records}) == 2 + assert {record["task_path"] for record in records} == {"a/same", "b/same"} + + +def test_hf_gated_malformed_and_floating_sources_fail(monkeypatch, tmp_path): + api, factory, download = fake_hf(monkeypatch, {"README.md": "no tasks"}) + with pytest.raises(PermissionError, match="gated"): + sources.acquire(hf_candidate(gated="manual"), tmp_path, 1) + factory.assert_not_called() + with pytest.raises(ValueError, match="No task.toml"): + sources.acquire(hf_candidate(), tmp_path, 1) + download.assert_not_called() + candidate = {**hf_candidate(), "revision": "main"} + with pytest.raises(ValueError, match="immutable"): + sources.acquire(candidate, tmp_path, 1) + api.dataset_info.return_value.sha = "d" * 40 + with pytest.raises(ValueError, match="disagrees"): + sources.acquire(hf_candidate(), tmp_path, 1) + + +def test_unsafe_source_paths_and_symlinks_fail(monkeypatch, tmp_path): + fake_hf(monkeypatch, {"../escape/task.toml": ""}) + with pytest.raises(ValueError, match="Unsafe source path"): + sources.acquire(hf_candidate(), tmp_path, 1) + outside = tmp_path / "outside" + outside.write_text("original") + root = tmp_path / "tree" + root.mkdir() + (root / "escape").symlink_to(outside) + with pytest.raises(ValueError, match="Unsafe task symlink"): + sources._real_files(root) + assert outside.read_text() == "original" + + +def test_contained_links_materialize_as_real_files(tmp_path): + root = tmp_path / "tree" + (root / "assets").mkdir(parents=True) + (root / "assets/data").write_text("real data") + (root / "linked-assets").symlink_to("assets", target_is_directory=True) + sources._real_files(root) + assert (root / "linked-assets/data").read_text() == "real data" + assert not any(path.is_symlink() for path in root.rglob("*")) + + +class PackageTaskId: + def __init__(self, org, name, ref): + self.org, self.name, self.ref = org, name, ref + + +def fake_harbor(monkeypatch): + tasks = [ + PackageTaskId("a", "same", TASK_DIGEST), + PackageTaskId("b", "same", "sha256:" + "d" * 64), + ] + registry = SimpleNamespace( + list_datasets=AsyncMock( + return_value=[ + SimpleNamespace( + name="org/dataset", + version=DIGEST, + description="bundle", + task_count=2, + ) + ] + ), + get_dataset_metadata=AsyncMock( + return_value=SimpleNamespace(version=DIGEST, task_ids=tasks, files=[]) + ), + download_dataset_files=AsyncMock(return_value={}), + ) + + async def download(*, task_ids, output_dir, overwrite, export): + results = [] + for task in task_ids: + digest = task.ref.removeprefix("sha256:") + path = output_dir / task.org / task.name / digest + (path / "environment/assets").mkdir(parents=True) + (path / "task.toml").write_text("") + (path / "instruction.md").write_text("task") + (path / "environment/assets/data").write_text("asset") + results.append(SimpleNamespace(path=path, content_hash=digest)) + return SimpleNamespace(results=results) + + task_client = SimpleNamespace(download_tasks=AsyncMock(side_effect=download)) + monkeypatch.setattr( + sources, + "_harbor_api", + lambda: ( + Mock(return_value=registry), + PackageTaskId, + Mock(return_value=task_client), + ), + ) + return registry, task_client + + +def test_harbor_discovery_and_bounded_digest_acquisition(monkeypatch, tmp_path): + registry, client = fake_harbor(monkeypatch) + [candidate] = sources.discover_harbor(1) + assert candidate["revision"] == DIGEST + [record] = sources.acquire(candidate, tmp_path, 1) + registry.get_dataset_metadata.assert_awaited_once_with(f"org/dataset@{DIGEST}") + assert len(client.download_tasks.call_args.kwargs["task_ids"]) == 1 + assert client.download_tasks.call_args.kwargs["export"] is False + assert record["revision"] == TASK_DIGEST + assert record["task_id"] == f"harbor:a/same@{TASK_DIGEST}:." + assert record["metadata"]["candidate_id"] == "org/dataset" + assert record["metadata"]["candidate_revision"] == DIGEST + assert (Path(record["path"]) / "environment/assets/data").read_text() == "asset" + + +@pytest.mark.parametrize("missing_root", [False, True]) +def test_harbor_nested_fixture_cannot_replace_package_root( + monkeypatch, tmp_path, missing_root +): + _, client = fake_harbor(monkeypatch) + download = client.download_tasks.side_effect + + async def with_nested_fixture(**kwargs): + result = await download(**kwargs) + root = result.results[0].path + nested = root / "environment/fixture" + nested.mkdir() + (nested / "task.toml").write_text("fixture") + if missing_root: + (root / "task.toml").unlink() + return result + + client.download_tasks.side_effect = with_nested_fixture + [candidate] = sources.discover_harbor(1) + if missing_root: + with pytest.raises(ValueError, match="no root task.toml"): + sources.acquire(candidate, tmp_path, 1) + else: + [record] = sources.acquire(candidate, tmp_path, 1) + assert record["task_path"] == "." + assert (Path(record["path"]) / "environment/fixture/task.toml").is_file() + + +def test_hf_nested_fixture_remains_parent_task_asset(monkeypatch, tmp_path): + fake_hf( + monkeypatch, + { + "tasks/parent/task.toml": "parent", + "tasks/parent/environment/fixture/task.toml": "fixture", + "tasks/sibling/task.toml": "sibling", + }, + ) + records = sources.acquire(hf_candidate(), tmp_path, 2) + assert [row["task_path"] for row in records] == ["tasks/parent", "tasks/sibling"] + assert ( + Path(records[0]["path"]) / "environment/fixture/task.toml" + ).read_text() == "fixture" + + +def test_harbor_single_task_and_cross_org_identity(monkeypatch, tmp_path): + registry, _ = fake_harbor(monkeypatch) + candidate = { + "source": "harbor", + "id": "a/same", + "revision": TASK_DIGEST, + "metadata": {"kind": "task"}, + } + [record] = sources.acquire(candidate, tmp_path, 1) + registry.get_dataset_metadata.assert_not_awaited() + assert record["source_id"] == "a/same" and record["revision"] == TASK_DIGEST + [dataset] = sources.discover_harbor(1) + records = sources.acquire(dataset, tmp_path, 2) + assert len({record["task_id"] for record in records}) == 2 + assert {record["source_id"] for record in records} == {"a/same", "b/same"} + + +def test_harbor_inaccessible_or_floating_tasks_fail(monkeypatch, tmp_path): + registry, client = fake_harbor(monkeypatch) + [candidate] = sources.discover_harbor(1) + registry.get_dataset_metadata.side_effect = PermissionError("unavailable task") + with pytest.raises(PermissionError, match="unavailable"): + sources.acquire(candidate, tmp_path, 1) + client.download_tasks.assert_not_awaited() + registry.get_dataset_metadata.side_effect = None + registry.get_dataset_metadata.return_value.task_ids[0].ref = "latest" + with pytest.raises(ValueError, match="immutable"): + sources.acquire(candidate, tmp_path, 1) + client.download_tasks.assert_not_awaited() + + +def test_harbor_wheel_mismatch_reports_pinned_install(monkeypatch): + # The PyPI 0.25.0 wheel shares the version string but lacks native listing. + monkeypatch.setattr( + sources, + "distribution", + lambda name: SimpleNamespace(version="0.25.0", read_text=lambda name: None), + ) + with pytest.raises(RuntimeError, match="source capability mismatch") as error: + sources.discover_harbor(1) + assert sources.HARBOR_INSTALL in str(error.value) + assert sources.HARBOR_GIT_REVISION in str(error.value) + + +def test_harbor_unsupported_listing_reports_install(monkeypatch): + registry, _ = fake_harbor(monkeypatch) + registry.list_datasets.side_effect = NotImplementedError("not yet supported") + with pytest.raises(RuntimeError, match="listing is unsupported") as error: + sources.discover_harbor(1) + assert sources.HARBOR_INSTALL in str(error.value) + + +def test_harbor_native_cli_json_and_model_contract(monkeypatch, capsys): + # Exercise the real Git-pinned API/CLI conversion, mocking only its DB read. + pytest.importorskip("harbor") + from harbor.cli.datasets import list_datasets + from harbor.db.client import RegistryDB + + native_rows = [ + { + "content_hash": "b" * 64, + "description": "bundle", + "package": {"org": {"name": "org"}, "name": "dataset"}, + "dataset_version_task": [{"count": 2}], + }, + { + "content_hash": "d" * 64, + "description": None, + "package": {"org": {"name": "z"}, "name": "files-only"}, + "dataset_version_task": [], + }, + ] + # A wheel without this method must fail this contract, rather than pass mocks. + monkeypatch.setattr( + RegistryDB, "list_dataset_versions", AsyncMock(return_value=native_rows) + ) + list_datasets(as_json=True) + cli_rows = json.loads(capsys.readouterr().out) + assert cli_rows == [ + { + "name": "org/dataset", + "version": DIGEST, + "description": "bundle", + "task_count": 2, + }, + { + "name": "z/files-only", + "version": "sha256:" + "d" * 64, + "description": "", + "task_count": 0, + }, + ] + [candidate] = sources.discover_harbor(1) + assert candidate["id"] == cli_rows[0]["name"] + assert candidate["revision"] == cli_rows[0]["version"] + assert candidate["metadata"]["task_count"] == cli_rows[0]["task_count"] + + +def test_harbor_native_acquisition_models(monkeypatch, tmp_path): + pytest.importorskip("harbor") + from harbor.models.registry import DatasetMetadata + from harbor.models.task.id import PackageTaskId as NativeTaskId + from harbor.registry.client.package import PackageDatasetClient + from harbor.tasks.client import BatchDownloadResult, TaskClient, TaskDownloadResult + + task = NativeTaskId(org="a", name="same", ref=TASK_DIGEST) + bundle = DatasetMetadata( + name="org/dataset", version=DIGEST, description="", task_ids=[task] + ) + monkeypatch.setattr( + PackageDatasetClient, "get_dataset_metadata", AsyncMock(return_value=bundle) + ) + + async def download(self, *, task_ids, output_dir, overwrite, export): + assert task_ids == [task] and overwrite and not export + bare_digest = task.ref.removeprefix("sha256:") + path = output_dir / task.org / task.name / bare_digest + path.mkdir(parents=True) + (path / "task.toml").write_text("") + (path / "instruction.md").write_text("native task") + return BatchDownloadResult( + results=[ + TaskDownloadResult( + path=path, + download_time_sec=0, + cached=False, + content_hash=bare_digest, + ) + ], + total_time_sec=0, + ) + + monkeypatch.setattr(TaskClient, "download_tasks", download) + [record] = sources.acquire( + {"source": "harbor", "id": "org/dataset", "revision": DIGEST}, tmp_path, 1 + ) + assert record["revision"] == TASK_DIGEST + assert (Path(record["path"]) / "instruction.md").read_text() == "native task" diff --git a/examples/hub_validation_pipeline/test_spend.py b/examples/hub_validation_pipeline/test_spend.py new file mode 100644 index 000000000..34f70c3f2 --- /dev/null +++ b/examples/hub_validation_pipeline/test_spend.py @@ -0,0 +1,714 @@ +"""Offline spend accounting, durability and real process/thread coordination.""" + +import json +import multiprocessing +import stat +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from decimal import Decimal, localcontext +from fractions import Fraction + +import pytest +from examples.hub_validation_pipeline import spend + + +@pytest.mark.parametrize("operation", ["reserve", "set_limit"]) +def test_idempotent_retry_must_confirm_directory_durability( + ledger, monkeypatch, operation +): + real_fsync = spend.os.fsync + calls = [] + + def fail_directory(fd): + directory = stat.S_ISDIR(spend.os.fstat(fd).st_mode) + calls.append("directory" if directory else "file") + if directory: + raise OSError("injected directory fsync failure") + return real_fsync(fd) + + monkeypatch.setattr(spend.os, "fsync", fail_directory) + + def retry(): + if operation == "reserve": + return spend.reserve(ledger, "durability", "1", "work") + return spend.set_limit(ledger, "5000", "reviewed example budget increase") + + with pytest.raises(OSError): + retry() + assert calls == ["file", "directory"] + before = ledger.read_bytes() + calls.clear() + with pytest.raises(OSError): + retry() + assert calls == ["directory"] + assert ledger.read_bytes() == before + monkeypatch.setattr(spend.os, "fsync", real_fsync) + report = retry() + assert ledger.read_bytes() == before + if operation == "reserve": + assert report["charged_usd_exact"] == "1" + else: + assert report["limit_usd_exact"] == "5000" + assert len(json.loads(ledger.read_text())["limit_changes"]) == 1 + + +@pytest.mark.parametrize("operation", ["summary", "reserve", "settle", "set_limit"]) +def test_saved_settlement_without_evidence_cannot_release_capacity(ledger, operation): + spend.reserve(ledger, "unfinished", "1", "unknown cost") + data = json.loads(ledger.read_text()) + data["reservations"]["unfinished"].update(status="settled", spent_usd="0") + ledger.write_text(json.dumps(data)) + original = ledger.read_bytes() + with pytest.raises(ValueError): + if operation == "summary": + spend.summary(ledger) + elif operation == "reserve": + spend.reserve(ledger, "new", "1", "work") + elif operation == "set_limit": + spend.set_limit(ledger, "5000", "reviewed example budget increase") + else: + spend.settle(ledger, "unfinished", "0", "new receipt") + assert ledger.read_bytes() == original + + +def _reserve_worker(path, barrier, results, amount, attempts, limit=None): + barrier.wait(timeout=10) + if limit is not None: + spend.set_limit(path, limit, "reviewed example budget increase") + successes = 0 + for index in range(attempts): + try: + spend.reserve( + path, + f"{multiprocessing.current_process().name}-{index}", + amount, + "race", + ) + except ValueError: + continue + successes += 1 + results.put(successes) + + +@pytest.fixture +def ledger(tmp_path): + path = tmp_path / "ledger.json" + spend.init(path, "10") + return path + + +def test_existing_schema_allocations_and_unknown_metadata_are_preserved(tmp_path): + path = tmp_path / "ledger.json" + data = { + "schema": "openenv-shared-spend-v1", + "limit_usd": "500", + "created_utc": "original creation", + "updated_utc": "original update", + "checked_utc": "keep this timestamp", + "metadata": {"nested": [1, "kept", None]}, + "reservations": { + identifier: dict( + reserved_usd=amount, + spent_usd=None, + status="reserved", + purpose="existing allocation", + evidence=None, + reserved_utc="original reservation", + metadata={"preserve": True}, + ) + for identifier, amount in ( + ("cohort-a", "48.123456"), + ("cohort-b", "150"), + ("image-probe", "5"), + ("generation-cohort", "100"), + ) + }, + } + path.write_text(json.dumps(data)) + before = path.read_bytes() + report = spend.summary(path) + assert Fraction(report["charged_usd_exact"]) == Fraction("303.123456") + assert Fraction(report["remaining_usd_exact"]) == Fraction("196.876544") + assert report["remaining_usd_rounded_up"] == "196.876544" + assert spend.init(path, "500.000") == report + assert path.read_bytes() == before + with pytest.raises(ValueError, match="limit differs"): + spend.init(path, "501") + assert path.read_bytes() == before + spend.reserve(path, "last", "196.876544", "remaining allocation") + before = path.read_bytes() + with pytest.raises(ValueError): + spend.reserve(path, "too-much", "1e-300", "must fail exactly") + assert path.read_bytes() == before + reservations = json.loads(before)["reservations"] + report = spend.set_limit(path, "5000", "reviewed example budget increase") + assert report["remaining_usd_exact"] == "4500" + raised = json.loads(path.read_text()) + assert raised["reservations"] == reservations + assert raised["reservations"]["generation-cohort"]["reserved_usd"] == "100" + # The existing writer's reservation serialization is unchanged by a limit update. + reservation_bytes = json.dumps({"reservations": reservations}, indent=2)[1:-2] + assert reservation_bytes.encode() in before + assert reservation_bytes.encode() in path.read_bytes() + history = raised["limit_changes"] + assert len(history) == 1 + assert history[0] == { + "previous_limit_usd": "500", + "limit_usd": "5000", + "evidence": "reviewed example budget increase", + "changed_utc": history[0]["changed_utc"], + } + assert datetime.fromisoformat(history[0]["changed_utc"]).tzinfo == timezone.utc + spend.reserve(path, "additional", "4500", "newly approved capacity") + before = path.read_bytes() + with pytest.raises(ValueError): + spend.reserve(path, "still-too-much", "1e-300", "must fail exactly") + assert path.read_bytes() == before + spend.settle(path, "cohort-a", "40", "receipt") + saved = json.loads(path.read_text()) + assert saved["limit_usd"] == "5000" + assert saved["limit_changes"] == history + for key in ("schema", "created_utc", "checked_utc", "metadata"): + assert saved[key] == data[key] + assert saved["updated_utc"] != data["updated_utc"] + for identifier, original in data["reservations"].items(): + row = saved["reservations"][identifier] + if identifier != "cohort-a": + assert row == original + else: + assert row["reserved_utc"] == original["reserved_utc"] + assert row["metadata"] == original["metadata"] + assert row["reserved_usd"] == original["reserved_usd"] + + +def test_reservation_and_settlement_are_idempotent_without_reusing_ids(ledger): + first = spend.reserve(ledger, "run", "1.0", "purpose", "plan") + before = ledger.read_bytes() + assert spend.reserve(ledger, "run", Decimal("1.000"), "retry", "other") == first + assert ledger.read_bytes() == before + with pytest.raises(ValueError, match="fresh ID"): + spend.reserve(ledger, "run", "2", "purpose") + assert ledger.read_bytes() == before + settled = spend.settle(ledger, "run", "0.7", "receipt") + before = ledger.read_bytes() + assert spend.settle(ledger, "run", "0.700", "retry receipt") == settled + assert ledger.read_bytes() == before + with pytest.raises(ValueError, match="manual reconciliation"): + spend.settle(ledger, "run", "0.8", "different receipt") + with pytest.raises(ValueError, match="fresh ID"): + spend.reserve(ledger, "run", "1.0", "reused") + assert ledger.read_bytes() == before + + +def test_actual_overspend_is_recorded_and_blocks_new_reservations(ledger): + spend.reserve(ledger, "a", "6", "first") + spend.reserve(ledger, "b", "4", "second") + report = spend.settle(ledger, "a", "12.000000000000000000000000000001", "receipt") + actual = Fraction("12.000000000000000000000000000001") + assert Fraction(report["charged_usd_exact"]) == actual + 4 + assert Fraction(report["remaining_usd_exact"]) == 6 - actual + assert report["over_budget"] is True + saved = json.loads(ledger.read_text()) + assert saved["over_budget"] is True + assert Fraction(saved["reservations"]["a"]["spent_usd"]) == actual + before = ledger.read_bytes() + with pytest.raises(ValueError, match="blocked"): + spend.reserve(ledger, "c", "0", "blocked even at zero") + assert ledger.read_bytes() == before + # Other real costs can still be settled after the cap is exceeded. + report = spend.settle(ledger, "b", "0", "cancelled before billing") + assert Fraction(report["charged_usd_exact"]) == actual + assert report["statuses"] == {"a": "settled", "b": "settled"} + for key in ("limit", "reserved", "spent", "charged", "remaining"): + exact = Fraction(report[f"{key}_usd_exact"]) + displayed = Fraction(report[f"{key}_usd_rounded_up"]) + assert exact <= displayed < exact + Fraction(1, 1000000) + + +def test_reservation_overrun_below_shared_cap_also_freezes_new_work(ledger): + spend.reserve(ledger, "a", "1", "first") + report = spend.settle(ledger, "a", "2", "receipt") + assert report["over_budget"] is True + assert report["remaining_usd_exact"] == "8" + before = json.loads(ledger.read_text()) + report = spend.set_limit(ledger, "5000", "reviewed example budget increase") + assert report["over_budget"] is True + assert report["remaining_usd_exact"] == "4998" + saved = json.loads(ledger.read_text()) + assert saved["over_budget"] is True + assert saved["reservations"] == before["reservations"] + with pytest.raises(ValueError): + spend.reserve(ledger, "b", "1", "blocked") + + +def test_normal_settlement_releases_unused_reservation(ledger): + spend.reserve(ledger, "a", "10", "first") + assert spend.settle(ledger, "a", "2", "receipt")["over_budget"] is False + report = spend.reserve(ledger, "b", "8", "released capacity") + assert report["charged_usd_exact"] == "10" + assert report["reserved_usd_exact"] == "8" + assert report["spent_usd_exact"] == "2" + + +@pytest.mark.parametrize( + "bad", + [ + True, + False, + 0.1, + float("nan"), + float("inf"), + Decimal("NaN"), + Decimal("sNaN"), + Decimal("Infinity"), + "NaN", + "Infinity", + "-1", + "-0", + "1/2", + "1_0", + " 1 ", + "", + "malformed", + None, + [], + {}, + ], +) +def test_invalid_money_is_rejected_without_partial_writes(ledger, tmp_path, bad): + missing = tmp_path / "never-created.json" + with pytest.raises(ValueError): + spend.init(missing, bad) + assert not missing.exists() + spend.reserve(ledger, "a", "1", "valid") + before = ledger.read_bytes() + with pytest.raises(ValueError): + spend.reserve(ledger, "bad", bad, "invalid") + with pytest.raises(ValueError): + spend.settle(ledger, "a", bad, "receipt") + with pytest.raises(ValueError): + spend.set_limit(ledger, bad, "reviewed example budget increase") + assert ledger.read_bytes() == before + + +def test_tiny_values_large_integers_and_decimal_context_do_not_round(tmp_path): + path = tmp_path / "precise.json" + with localcontext() as context: + context.prec = 2 + spend.init(path, Decimal("1.0000000000000000000000000000000000000001")) + spend.reserve(path, "whole", "1", "exact") + spend.reserve(path, "remainder", "1e-40", "last exact digit") + before = path.read_bytes() + with pytest.raises(ValueError): + spend.reserve(path, "tiny-over", "1e-300", "cannot round down") + assert path.read_bytes() == before + tiny = tmp_path / "tiny.json" + spend.init(tiny, "1e-300") + spend.reserve(tiny, "one", "1e-301", "tiny") + report = spend.reserve(tiny, "nine", "9e-301", "tiny") + assert report["remaining_usd_exact"] == "0" + assert Fraction(report["charged_usd_exact"]) == Fraction(Decimal("1e-300")) + assert report["charged_usd_rounded_up"] == "0.000001" + large = tmp_path / "large.json" + spend.init(large, 9007199254740993) + report = spend.reserve(large, "large", 9007199254740992, "exact integer") + assert report["remaining_usd_exact"] == "1" + + +def test_limit_changes_are_exact_append_only_and_idempotent(ledger, tmp_path): + missing = tmp_path / "missing.json" + with pytest.raises(ValueError, match="initialized"): + spend.set_limit(missing, "5000", "reviewed example budget increase") + assert not missing.exists() + with localcontext() as context: + context.prec = 2 + exact = Decimal("5000.0000000000000000000000000000000000000001") + report = spend.set_limit(ledger, exact, "reviewed example budget increase") + assert Fraction(report["limit_usd_exact"]) == Fraction(exact) + before = ledger.read_bytes() + assert spend.set_limit(ledger, str(exact), "retry evidence") == report + assert ledger.read_bytes() == before + first = json.loads(before)["limit_changes"][0] + report = spend.set_limit(ledger, "1e-300", "another decision") + assert Fraction(report["limit_usd_exact"]) == Fraction("1e-300") + changes = json.loads(ledger.read_text())["limit_changes"] + assert len(changes) == 2 + assert changes[0] == first + assert changes[1]["previous_limit_usd"] == str(exact) + report = spend.set_limit(ledger, 9007199254740993, "large exact decision") + assert report["limit_usd_exact"] == "9007199254740993" + + +def test_limit_reduction_latches_over_budget_and_raise_cannot_clear_it(ledger): + spend.reserve(ledger, "held", "5", "pending billing") + assert spend.set_limit(ledger, "4", "reduced ceiling")["over_budget"] is True + assert spend.set_limit(ledger, "5000", "reviewed example budget increase")[ + "over_budget" + ] + with pytest.raises(ValueError, match="blocked"): + spend.reserve(ledger, "blocked", "0", "still blocked") + + +@pytest.mark.parametrize("saved_flag", [False, True]) +def test_saved_over_budget_condition_survives_limit_increase(ledger, saved_flag): + spend.reserve(ledger, "held", "10", "existing hold") + data = json.loads(ledger.read_text()) + data["limit_usd"] = "9" if not saved_flag else "10" + if saved_flag: + data["over_budget"] = True + ledger.write_text(json.dumps(data)) + assert spend.set_limit(ledger, "5000", "reviewed example budget increase")[ + "over_budget" + ] + assert json.loads(ledger.read_text())["over_budget"] is True + + +@pytest.mark.parametrize( + "history", + [ + None, + {}, + "invalid", + [None], + [{}], + [{"previous_limit_usd": "500", "limit_usd": "10", "evidence": "decision"}], + ], +) +def test_malformed_limit_history_fails_closed(ledger, history): + data = json.loads(ledger.read_text()) + data["limit_changes"] = history + ledger.write_text(json.dumps(data)) + before = ledger.read_bytes() + with pytest.raises(ValueError): + spend.set_limit(ledger, "5000", "reviewed example budget increase") + with pytest.raises(ValueError): + spend.summary(ledger) + assert ledger.read_bytes() == before + + +@pytest.mark.parametrize( + "corruption", + [ + {"limit_usd": 10.0}, + {"previous_limit_usd": "-1"}, + {"evidence": " "}, + {"changed_utc": "bad timestamp"}, + {"changed_utc": "2026-11-10T00:00:00"}, + {"changed_utc": "2026-11-10T00:00:00+01:00"}, + {"limit_usd": "9"}, + {"previous_limit_usd": "10"}, + ], +) +def test_corrupt_limit_change_row_fails_closed(ledger, corruption): + change = dict( + previous_limit_usd="500", + limit_usd="10", + evidence="decision", + changed_utc="2026-11-10T00:00:00+00:00", + ) + test_malformed_limit_history_fails_closed(ledger, [change | corruption]) + + +def test_discontinuous_limit_history_fails_closed(ledger): + change = dict( + previous_limit_usd="500", + limit_usd="9", + evidence="decision", + changed_utc="2026-11-10T00:00:00+00:00", + ) + test_malformed_limit_history_fails_closed( + ledger, [change, change | {"previous_limit_usd": "8", "limit_usd": "10"}] + ) + + +@pytest.mark.parametrize( + "indent,newline,ensure_ascii", [(None, "\n", True), (4, "\r\n", False)] +) +def test_limit_change_preserves_original_reservation_bytes( + tmp_path, indent, newline, ensure_ascii +): + path = tmp_path / "ledger.json" + reservations = json.dumps( + { + "tasksmith": dict( + reserved_usd="100.0000", + spent_usd=None, + status="reserved", + purpose='original "reservations": note', + evidence="original évidence", + metadata={"keep": 1.25}, + ) + }, + indent=indent, + ensure_ascii=ensure_ascii, + ) + reservations = reservations.replace("\n", newline).replace( + "1.25", "1.25000000000000000000000000001" + ) + path.write_text( + '{"schema":"openenv-shared-spend-v1","limit_usd":"500",' + '"metadata":{"reservations":"unrelated"},"reservations":' + reservations + "}" + ) + spend.set_limit(path, "5000", "reviewed example budget increase") + assert reservations.encode() in path.read_bytes() + assert spend.summary(path)["reserved_usd_exact"] == "100" + + +@pytest.mark.parametrize("field", ["limit_usd", "reserved_usd"]) +def test_float_ledger_amounts_cannot_be_rewritten_by_limit_change(ledger, field): + spend.reserve(ledger, "held", "1", "existing hold") + data = json.loads(ledger.read_text()) + if field == "limit_usd": + data[field] = 10.0 + else: + data["reservations"]["held"][field] = 1.0 + ledger.write_text(json.dumps(data)) + before = ledger.read_bytes() + with pytest.raises(ValueError): + spend.set_limit(ledger, "5000", "reviewed example budget increase") + assert ledger.read_bytes() == before + + +@pytest.mark.parametrize("amount,attempts,expected", [("0.6", 1, 1), ("0.03", 20, 33)]) +def test_process_budget_race_and_no_lost_updates(tmp_path, amount, attempts, expected): + path = tmp_path / "shared.json" + spend.init(path, "1") + context = multiprocessing.get_context("spawn") + barrier, results = context.Barrier(2), context.Queue() + processes = [ + context.Process( + target=_reserve_worker, args=(path, barrier, results, amount, attempts) + ) + for _ in range(2) + ] + try: + for process in processes: + process.start() + for process in processes: + process.join(timeout=15) + assert process.exitcode == 0 + successes = sum(results.get(timeout=2) for _ in processes) + report = spend.summary(path) + assert successes == expected == len(report["statuses"]) + assert Fraction(report["charged_usd_exact"]) == expected * Fraction(amount) <= 1 + finally: + for process in processes: + if process.is_alive(): + process.terminate() + process.join(timeout=2) + results.close() + results.join_thread() + + +def test_threads_share_the_same_budget(ledger): + def reserve(index): + try: + spend.reserve(ledger, f"thread-{index}", "1", "thread race") + return 1 + except ValueError: + return 0 + + with ThreadPoolExecutor(max_workers=8) as executor: + successes = sum(executor.map(reserve, range(32))) + report = spend.summary(ledger) + assert successes == 10 == len(report["statuses"]) + assert report["charged_usd_exact"] == "10" + + +def test_concurrent_limit_retry_and_reservations_share_the_lock(tmp_path): + path = tmp_path / "shared.json" + spend.init(path, "500") + spend.reserve(path, "tasksmith", "100", "existing allocation") + context = multiprocessing.get_context("spawn") + barrier, results = context.Barrier(2), context.Queue() + processes = [ + context.Process( + target=_reserve_worker, + args=(path, barrier, results, "3000", 1, "5000"), + ) + for _ in range(2) + ] + try: + for process in processes: + process.start() + for process in processes: + process.join(timeout=15) + assert process.exitcode == 0 + assert sum(results.get(timeout=2) for _ in processes) == 1 + report = spend.summary(path) + assert report["charged_usd_exact"] == "3100" + saved = json.loads(path.read_text()) + assert len(saved["limit_changes"]) == 1 + assert saved["reservations"]["tasksmith"]["reserved_usd"] == "100" + finally: + for process in processes: + if process.is_alive(): + process.terminate() + process.join(timeout=2) + results.close() + results.join_thread() + + +def test_concurrent_different_limit_changes_keep_all_history(ledger): + with ThreadPoolExecutor(max_workers=8) as executor: + list( + executor.map( + lambda limit: spend.set_limit(ledger, limit, "reviewed decision"), + range(11, 27), + ) + ) + data = json.loads(ledger.read_text()) + changes = data["limit_changes"] + assert len(changes) == 16 + assert {row["limit_usd"] for row in changes} == {str(n) for n in range(11, 27)} + previous = "10" + for row in changes: + assert row["previous_limit_usd"] == previous + previous = row["limit_usd"] + assert data["limit_usd"] == previous + + +@pytest.mark.parametrize("evidence", [None, "", " ", True, {}]) +def test_settlement_requires_evidence_and_existing_id(ledger, evidence): + spend.reserve(ledger, "a", "1", "purpose") + before = ledger.read_bytes() + with pytest.raises(ValueError): + spend.settle(ledger, "a", "1", evidence) + with pytest.raises(ValueError, match="Unknown reservation"): + spend.settle(ledger, "missing", "1", "receipt") + with pytest.raises(ValueError): + spend.set_limit(ledger, "5000", evidence) + with pytest.raises(ValueError): + spend.set_limit(ledger, "10", evidence) + assert ledger.read_bytes() == before + + +@pytest.mark.parametrize("operation", ["reserve", "set_limit"]) +def test_failed_atomic_replace_keeps_original_and_cleans_temporary_file( + ledger, monkeypatch, operation +): + before, files = ledger.read_bytes(), set(ledger.parent.iterdir()) + + def fail_replace(*args): + raise OSError("private diagnostic payload") + + monkeypatch.setattr(spend.os, "replace", fail_replace) + with pytest.raises(OSError): + if operation == "reserve": + spend.reserve(ledger, "a", "1", "purpose") + else: + spend.set_limit(ledger, "5000", "reviewed example budget increase") + assert ledger.read_bytes() == before + assert set(ledger.parent.iterdir()) == files + + +def test_duplicate_keys_and_corrupt_costs_fail_closed(ledger): + ledger.write_text( + '{"schema":"openenv-shared-spend-v1","limit_usd":"10","limit_usd":"100","reservations":{}}' + ) + before = ledger.read_bytes() + with pytest.raises(ValueError, match="Duplicate"): + spend.reserve(ledger, "a", "1", "purpose") + assert ledger.read_bytes() == before + ledger.write_text( + json.dumps( + dict( + schema="openenv-shared-spend-v1", + limit_usd="10", + reservations={ + "bad": dict(status="reserved", reserved_usd=True, spent_usd=None) + }, + ) + ) + ) + before = ledger.read_bytes() + with pytest.raises(ValueError): + spend.reserve(ledger, "a", "1", "purpose") + assert ledger.read_bytes() == before + + +def test_module_cli_outputs_totals_and_statuses_without_freeform_evidence( + ledger, capsys +): + command = [ + sys.executable, + "-B", + "-m", + "examples.hub_validation_pipeline.spend", + "--ledger", + str(ledger), + ] + result = subprocess.run( + command + + [ + "reserve", + "--id", + "cli", + "--usd", + "3", + "--purpose", + "purpose", + "--evidence", + "private evidence", + ], + check=True, + text=True, + capture_output=True, + ) + assert json.loads(result.stdout)["charged_usd_exact"] == "3" + assert "private evidence" not in result.stdout + assert not result.stderr + result = subprocess.run( + command + + ["settle", "--id", "cli", "--usd", "2", "--evidence", "private receipt"], + check=True, + text=True, + capture_output=True, + ) + assert json.loads(result.stdout)["statuses"] == {"cli": "settled"} + result = subprocess.run( + command + ["summary"], check=True, text=True, capture_output=True + ) + assert json.loads(result.stdout)["spent_usd_exact"] == "2" + result = subprocess.run( + command + + [ + "set-limit", + "--usd", + "5000", + "--evidence", + "reviewed example budget increase", + ], + check=True, + text=True, + capture_output=True, + ) + assert json.loads(result.stdout)["limit_usd_exact"] == "5000" + assert json.loads(result.stdout)["spent_usd_exact"] == "2" + assert "user decision" not in result.stdout + before = ledger.read_bytes() + result = subprocess.run( + command + ["set-limit", "--usd", "6000"], text=True, capture_output=True + ) + assert result.returncode == 2 + assert ledger.read_bytes() == before + with pytest.raises(SystemExit) as error: + spend.main( + [ + "--ledger", + str(ledger), + "reserve", + "--id", + "bad", + "--usd", + "private invalid amount", + "--purpose", + "purpose", + ] + ) + assert error.value.code == 2 + output = capsys.readouterr() + assert not output.out + assert "private invalid amount" not in output.err From c1fc62a5177b1b2499c743d67211a85296404f52 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 18:56:31 +0100 Subject: [PATCH 03/19] Parse candidate Bash fences and retain raw response hashes --- examples/coding_audit/README.md | 2 ++ examples/coding_audit/hf.py | 25 +++++++++++++++++++++++ examples/coding_audit/test_hf.py | 34 ++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/examples/coding_audit/README.md b/examples/coding_audit/README.md index 1e0cd52c8..c589b0936 100644 --- a/examples/coding_audit/README.md +++ b/examples/coding_audit/README.md @@ -35,6 +35,8 @@ alongside request parameters. Support depends on the selected provider/model. Candidate generation streams bounded content and reasoning; only a complete `stop` response with validated usage can produce an executable candidate. A `length` response remains rejected even when its terminal usage is retained. +Recognized Bash/sh Markdown fences are parsed before execution; the raw response +and both raw/script hashes are retained. Other language fences are rejected. Candidate generation can opt into public task API context by adding `"context_paths": ["environment/source/package/api.py"]` to the `generator` diff --git a/examples/coding_audit/hf.py b/examples/coding_audit/hf.py index fa8614090..29d337a0d 100644 --- a/examples/coding_audit/hf.py +++ b/examples/coding_audit/hf.py @@ -108,6 +108,28 @@ def _under(path, parent): return path == parent or path.startswith(parent + "/") +def _candidate_script(raw): + lines = raw.lstrip().splitlines(keepends=True) + opening = lines[0].strip() + if not opening.startswith("```"): + return raw, None + if opening not in {"```bash", "```sh", "```shell"}: + raise HFBackendError("candidate code fence must declare Bash or sh") + end = len(lines) + while end > 1 and not lines[end - 1].strip(): + end -= 1 + closed = end > 1 and lines[end - 1].strip() == "```" + script = "".join(lines[1 : end - 1] if closed else lines[1:]) + if not script.strip(): + raise HFBackendError("candidate response must contain a shell script") + return script, { + "wrapper": "markdown_bash", + "closing_fence_present": closed, + "raw_sha256": hashlib.sha256(raw.encode()).hexdigest(), + "script_sha256": hashlib.sha256(script.encode()).hexdigest(), + } + + def submission_digest(files): """Hash sorted relative paths, NUL, content, NUL as source_digest does.""" digest = hashlib.sha256() @@ -984,6 +1006,9 @@ def generate_candidate(self, model: dict, seed: int) -> dict: ) if len(script.encode()) > self.max_script: raise HFBackendError("candidate script exceeds byte limit") + script, formatting = _candidate_script(script) + if formatting is not None: + record["script_format"] = formatting record["script"] = script except Exception as exc: record["error"] = self._error(exc) diff --git a/examples/coding_audit/test_hf.py b/examples/coding_audit/test_hf.py index 7aaa0ab6b..2acf9c0db 100644 --- a/examples/coding_audit/test_hf.py +++ b/examples/coding_audit/test_hf.py @@ -1696,3 +1696,37 @@ def test_thinking_flag_requires_an_explicit_boolean(harness, value): record = h.backend.generate_candidate({**MODEL, "enable_thinking": value}, 0) assert "enable_thinking must be a boolean" in record["error"] assert not h.inference.requests and record["script"] == "" + + +@pytest.mark.parametrize("language", ["bash", "sh", "shell"]) +@pytest.mark.parametrize("closed", [False, True]) +def test_fenced_candidate_keeps_raw_response_and_executes_only_bash( + harness, language, closed +): + h = harness + body = "printf answer > /app/answer.txt\n" + raw = "```" + language + "\n" + body + ("```\n" if closed else "") + h.inference.responses = [response(raw)] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] is None and record["script"] == body + assert record["transcript"][-1]["content"] == raw + metadata = record["script_format"] + assert metadata["raw_sha256"] == hashlib.sha256(raw.encode()).hexdigest() + assert ( + metadata["script_sha256"] + == hashlib.sha256(record["script"].encode()).hexdigest() + ) + assert metadata["closing_fence_present"] is closed + submission = h.backend.execute_script(record["script"], "generated") + assert submission["error"] is None + assert h.factory.created[0].commands[-1][0] == ["/bin/bash", "-c", record["script"]] + + +@pytest.mark.parametrize( + "raw", ["```bash\n```", "```python\nprint('wrong language')\n```"] +) +def test_empty_or_non_bash_fence_does_not_create_a_candidate(harness, raw): + h = harness + h.inference.responses = [response(raw)] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] and record["script"] == "" and not h.factory.created From 728350dc8abbf37e35cf5a6c8fc2df88870891dd Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:20:37 +0100 Subject: [PATCH 04/19] fix(coding-audit): reject linked artifacts while snapshotting Read original task files through the descriptor-based reader before writing the frozen copy, so copying cannot erase hard-link evidence. Cover context, instruction, verifier and reference artifacts through the runner with offline SDK fakes. --- examples/coding_audit/runner.py | 8 +++++ examples/coding_audit/test_runner.py | 54 ++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/examples/coding_audit/runner.py b/examples/coding_audit/runner.py index 2ea17cc51..bd3762ece 100644 --- a/examples/coding_audit/runner.py +++ b/examples/coding_audit/runner.py @@ -489,12 +489,20 @@ def persist(): _write(output_dir / "evidence.json", sanitize(evidence)) try: + from .hf import _local_file + + def copy_artifact(source, destination): + raw, _ = _local_file(source, Path(source).stat().st_size) + Path(destination).write_bytes(raw) + shutil.copystat(source, destination) + with tempfile.TemporaryDirectory(prefix="openenv-coding-frozen-") as temporary: frozen = (Path(temporary) / "task").resolve() shutil.copytree( task_root, frozen, symlinks=True, + copy_function=copy_artifact, ignore=shutil.ignore_patterns(*_DIGEST_EXCLUDED_DIRS), ) if source_digest(frozen) != original_digest: diff --git a/examples/coding_audit/test_runner.py b/examples/coding_audit/test_runner.py index f6bc7dbf6..122e34c6b 100644 --- a/examples/coding_audit/test_runner.py +++ b/examples/coding_audit/test_runner.py @@ -463,6 +463,60 @@ def test_in_task_output_rejected_before_inference(task): assert not (task / "audit.artifacts").exists() +@pytest.mark.parametrize( + "artifact", + [ + "environment/source/api.py", + "instruction.md", + "tests/test.sh", + "solution/solve.sh", + ], +) +def test_snapshot_rejects_hardlinked_artifacts_before_inference( + task, tmp_path, monkeypatch, artifact +): + import huggingface_hub + from examples.coding_audit import hf + + m = SimpleNamespace(**runpy.run_path(str(Path(__file__).with_name("test_hf.py")))) + api, inference = m.API(), m.InferenceFactory() + factory = m.Factory(api) + api.model_info = lambda *args, **kwargs: SimpleNamespace( + sha=m.MODEL["revision"], + inference_provider_mapping={ + "deepinfra": {"provider_id": m.MODEL["model_id"], "status": "live"} + }, + ) + monkeypatch.setattr(huggingface_hub, "HfApi", lambda **kwargs: api) + monkeypatch.setattr(hf, "HfApi", lambda **kwargs: api) + monkeypatch.setattr(hf.Sandbox, "create", factory) + monkeypatch.setattr(hf, "InferenceClient", inference) + for name in ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN"): + monkeypatch.setenv(name, "synthetic-test-" + name) + context_path = "environment/source/api.py" + source = task / context_path + source.parent.mkdir(parents=True) + if artifact == context_path: + source.hardlink_to(task / "solution/solve.sh") + else: + source.write_text("def public_answer(): return 42\n") + (tmp_path / "linked-artifact").hardlink_to(task / artifact) + assert (task / artifact).stat().st_nlink == 2 + config = configuration() + config["generator"] = {**config["target"], "context_paths": [context_path]} + config["execution"].update( + agent_image=m.AGENT, verifier_image=m.VERIFIER, namespace="owner" + ) + inference.responses = [m.response("true")] + output = tmp_path / "audit" + report = run_coding_validation(task, config, output) + assert not inference.requests and not inference.kwargs + assert not factory.created + assert not report["accepted"] + evidence = json.loads((output / "evidence.json").read_text()) + assert evidence["collection_error"] == "HFBackendError" + + @pytest.mark.parametrize("with_context", [False, True]) def test_real_backend_contract_reaches_difficulty_with_sdk_fakes( task, tmp_path, monkeypatch, with_context From b33494f585f196db25fca0c0f174f382b1627c7d Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:21:55 +0100 Subject: [PATCH 05/19] fix(coding-audit): reject empty tool-call completions Require a nonempty tool-call list when the provider finishes with tool_calls. Exercise null, missing and empty calls through collection and scoring after a valid terminal action, and verify failed targets are never graded. --- examples/coding_audit/hf.py | 4 +++ examples/coding_audit/test_runner.py | 39 +++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/examples/coding_audit/hf.py b/examples/coding_audit/hf.py index 29d337a0d..f615bd382 100644 --- a/examples/coding_audit/hf.py +++ b/examples/coding_audit/hf.py @@ -742,6 +742,10 @@ def _chat(self, client, model, allowed, limits, messages, record, seed, *, tools raise HFBackendError("model output exceeded byte limit") if choices[0].get("finish_reason") not in {"stop", "tool_calls"}: raise HFBackendError("model response was incomplete") + if choices[0]["finish_reason"] == "tool_calls" and ( + not isinstance(message.get("tool_calls"), list) or not message["tool_calls"] + ): + raise HFBackendError("invalid terminal function calls") return message def _candidate_response(self, client, request, allowed, limits, record): diff --git a/examples/coding_audit/test_runner.py b/examples/coding_audit/test_runner.py index 122e34c6b..b97384b05 100644 --- a/examples/coding_audit/test_runner.py +++ b/examples/coding_audit/test_runner.py @@ -518,8 +518,11 @@ def test_snapshot_rejects_hardlinked_artifacts_before_inference( @pytest.mark.parametrize("with_context", [False, True]) +@pytest.mark.parametrize( + "completion", ["stop", "null_calls", "missing_calls", "empty_calls"] +) def test_real_backend_contract_reaches_difficulty_with_sdk_fakes( - task, tmp_path, monkeypatch, with_context + task, tmp_path, monkeypatch, with_context, completion ): """Join the actual collector/backend/scorer, not just each seam in isolation.""" from examples.coding_audit.hf import HFTaskBackend @@ -555,12 +558,22 @@ def run(sandbox, argv, **kwargs): config["execution"].update( agent_image=m.AGENT, verifier_image=m.VERIFIER, namespace="owner" ) + completions = [m.response("done"), m.response("done")] + if completion != "stop": + for reply in completions: + choice = reply["choices"][0] + choice["finish_reason"] = "tool_calls" + choice["message"]["content"] = None + if completion == "missing_calls": + del choice["message"]["tool_calls"] + elif completion == "empty_calls": + choice["message"]["tool_calls"] = [] inference.responses = [ m.response('printf "42\\n" > /app/answer.txt'), m.response(None, calls=[m.terminal('printf "41\\n" > /app/answer.txt')]), - m.response("done"), + completions[0], m.response(None, calls=[m.terminal('printf "42\\n" > /app/answer.txt')]), - m.response("done"), + completions[1], ] backend = HFTaskBackend( task.resolve(), @@ -573,11 +586,25 @@ def run(sandbox, argv, **kwargs): ) output = tmp_path / "audit" report = run_coding_validation(task, config, output, backend=backend) - assert report["accepted"], report - assert report["checks"]["target_difficulty"]["solve_rate"] == 0.5 + assert report["accepted"] is (completion == "stop"), report + if completion == "stop": + assert report["checks"]["target_difficulty"]["solve_rate"] == 0.5 + else: + assert report["checks"]["target_difficulty"]["status"] == "error" assert len(inference.requests) == 5 evidence_path = output / "evidence.json" - generated = json.loads(evidence_path.read_text())["generation"][0] + evidence = json.loads(evidence_path.read_text()) + if completion != "stop": + targets = [row for row in evidence["trials"] if row["role"] == "target"] + assert len(targets) == 2 + assert all( + row["error"] == "invalid terminal function calls" + and row["cleanup_verified"] + and "grade_sandbox_id" not in row + and row.get("reward") is None + for row in targets + ) + generated = evidence["generation"][0] if with_context: assert generated["candidate_context"] == [ { From 4278afa641e74d51a7fd3be32d3cad88c0975485 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:21:55 +0100 Subject: [PATCH 06/19] fix(coding-audit): correct offline test import path Include the repository root in PYTHONPATH so pytest can import the examples package. Verified the documented command with all 417 offline tests passing. --- examples/coding_audit/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/coding_audit/README.md b/examples/coding_audit/README.md index c589b0936..b8b8a5ca0 100644 --- a/examples/coding_audit/README.md +++ b/examples/coding_audit/README.md @@ -62,5 +62,5 @@ its frozen source and task bundle; its tests stay outside the bundle. Run the offline suite with: ```bash -PYTHONPATH=src:envs uv run pytest examples/coding_audit -q +PYTHONPATH=.:src:envs uv run pytest examples/coding_audit -q ``` From 6bea568ff5b8bb38ec53293582a396028b951d08 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:22:29 +0100 Subject: [PATCH 07/19] fix(hub-validation): gate every paid launch on a bound reservation Remove the unguarded CLI and discovery submission paths. Bind reviewed plan hashes and cost allocations into shared reservations, and verify the active hold under the ledger lock before claiming or launching a Job. --- .../hub_validation_pipeline/CODING_AUDIT.md | 16 +- examples/hub_validation_pipeline/cloud.py | 70 ++++--- examples/hub_validation_pipeline/full_run.py | 9 +- examples/hub_validation_pipeline/pipeline.py | 18 +- .../hub_validation_pipeline/prepared_batch.py | 8 + examples/hub_validation_pipeline/spend.py | 17 +- .../hub_validation_pipeline/test_cloud.py | 180 ++++++++++++++++-- .../hub_validation_pipeline/test_full_run.py | 9 +- .../test_prepared_batch.py | 6 +- 9 files changed, 265 insertions(+), 68 deletions(-) diff --git a/examples/hub_validation_pipeline/CODING_AUDIT.md b/examples/hub_validation_pipeline/CODING_AUDIT.md index e18291ee8..7fa2de8c1 100644 --- a/examples/hub_validation_pipeline/CODING_AUDIT.md +++ b/examples/hub_validation_pipeline/CODING_AUDIT.md @@ -200,15 +200,13 @@ release them merely because a controller stopped. Jobs are never automatically resumed or blindly retried. `prepared_batch` provides shared reservation and bounded controller concurrency -for an already reviewed preparation manifest. A direct single-plan submission is: - -```bash -uv run --no-sync python -m examples.hub_validation_pipeline.cloud submit \ - --plan outputs/audit-plan/plan.json -``` - -Before using that lower-level command, reserve its reviewed cost in the shared -ledger. Its per-plan workload checks do not replace aggregate spending control. +for an already reviewed preparation manifest. Use these guarded cohort launchers +for paid work; the cloud CLI and discovery pipeline only prepare inputs. +`cloud.submit()` requires a ledger, reservation ID and expected plan SHA256. +Each reservation binds the reviewed plan hashes to their modeled cost allocations; +allocations must fit the reserved total. Submission rechecks the active hold, +budget block and parsed plan bytes while holding the shared ledger lock through +launch. Unbound legacy reservations cannot authorize a launch. The submitter verifies private dataset access, stages immutable inputs, claims the run identity and persists the returned Job identifier and receipt. `prepare` never makes model calls or launches paid Jobs. diff --git a/examples/hub_validation_pipeline/cloud.py b/examples/hub_validation_pipeline/cloud.py index ddab8c0a0..15b7f7e3e 100644 --- a/examples/hub_validation_pipeline/cloud.py +++ b/examples/hub_validation_pipeline/cloud.py @@ -20,6 +20,7 @@ import urllib.request import uuid import zipfile +from fractions import Fraction from pathlib import Path, PurePosixPath WORKER_IMAGE = "docker.io/library/python@sha256:48d884813cf44e927cf64c5529dd630dd3b18338c6dc727991eef9b71e934a20" @@ -856,31 +857,56 @@ def stage_inputs(plan_path, *, api=None, secrets=None, expected_plan_sha256=None return receipt -def submit(plan_path, *, api=None, secrets=None, expected_plan_sha256=None): - """Upload/claim immutable inputs and launch once, resolving actual secrets.""" +def submit( + plan_path, + *, + api=None, + secrets=None, + expected_plan_sha256=None, + spend_ledger=None, + reservation_id=None, +): + """Launch once under an active shared reservation bound to the reviewed plan.""" from huggingface_hub import HfApi + from . import full_run, spend + + if not spend_ledger or not reservation_id or expected_plan_sha256 is None: + raise ValueError("Submission requires a reservation and reviewed plan SHA256") + _, config, _, _ = _read_plan(plan_path, expected_sha256=expected_plan_sha256) + cost = Fraction(full_run.estimate([config])["reservation_usd_exact"]) secrets = _runtime_secrets(secrets) api = api or HfApi(token=secrets["HF_TOKEN"]) - receipt = stage_inputs( - plan_path, - api=api, - secrets=secrets, - expected_plan_sha256=expected_plan_sha256, - ) - receipt_path = Path(plan_path).parent / "submission.json" - receipt["status"] = "launch_pending" - _write_receipt(receipt_path, receipt) - try: - job = api.run_job( - **receipt["launch_spec"], secrets=secrets, token=secrets["HF_SANDBOX_TOKEN"] + # Keep settlement and budget changes outside the claim/launch boundary. + with spend._ledger(spend_ledger) as ledger: + row = ledger["reservations"].get(reservation_id, {}) + if ( + spend._report(ledger)["over_budget"] + or row.get("status") != "reserved" + or Fraction(row.get("plan_costs", {}).get(expected_plan_sha256, "0")) < cost + ): + raise ValueError("Shared reservation is blocked, insufficient or unbound") + receipt = stage_inputs( + plan_path, + api=api, + secrets=secrets, + expected_plan_sha256=expected_plan_sha256, ) - except Exception as error: - receipt.update(status="launch_unknown", error_type=type(error).__name__) + receipt_path = Path(plan_path).parent / "submission.json" + receipt["status"] = "launch_pending" _write_receipt(receipt_path, receipt) - raise RuntimeError( - "Job launch outcome unknown; inspect the submission receipt and HF Jobs before any manual retry" - ) from None + try: + job = api.run_job( + **receipt["launch_spec"], + secrets=secrets, + token=secrets["HF_SANDBOX_TOKEN"], + ) + except Exception as error: + receipt.update(status="launch_unknown", error_type=type(error).__name__) + _write_receipt(receipt_path, receipt) + raise RuntimeError( + "Job launch outcome unknown; inspect the submission receipt and HF Jobs before any manual retry" + ) from None receipt.update(status="submitted", job_id=job.id, job_url=job.url) _write_receipt(receipt_path, receipt) try: @@ -1028,18 +1054,12 @@ def main(argv=None): local.add_argument("--task", type=Path, required=True) local.add_argument("--config", type=Path, required=True) local.add_argument("--output", type=Path, required=True) - launch = commands.add_parser( - "submit", help="Upload private inputs and launch one paid HF Job" - ) - launch.add_argument("--plan", type=Path, required=True) internal = commands.add_parser("worker", help=argparse.SUPPRESS) internal.add_argument("--root", type=Path, required=True) internal.add_argument("--bundle-sha", required=True) args = parser.parse_args(argv) if args.command == "prepare": result = prepare(args.task, _loads(args.config.read_bytes()), args.output) - elif args.command == "submit": - result = submit(args.plan) else: result = worker(args.root, bundle_sha=args.bundle_sha) print(_json_bytes(result).decode(), end="") diff --git a/examples/hub_validation_pipeline/full_run.py b/examples/hub_validation_pipeline/full_run.py index 3e16bd98f..1e34b9cdc 100644 --- a/examples/hub_validation_pipeline/full_run.py +++ b/examples/hub_validation_pipeline/full_run.py @@ -278,6 +278,10 @@ def run( manifest["estimate"]["reservation_usd_rounded_up"], "Reviewed native Hub audit cohort", evidence=str(manifest_path), + plan_costs={ + task["plan_sha256"]: estimate([config])["reservation_usd_exact"] + for task, config in zip(manifest["tasks"], configs, strict=True) + }, ) manifest["shared_spend"] = { "ledger": str(Path(spend_ledger).absolute()), @@ -295,7 +299,10 @@ def run( } _write(manifest_path, manifest) receipt = cloud.submit( - task["plan"], expected_plan_sha256=task["plan_sha256"] + task["plan"], + expected_plan_sha256=task["plan_sha256"], + spend_ledger=spend_ledger, + reservation_id=identifier, ) task["submission"].update( { diff --git a/examples/hub_validation_pipeline/pipeline.py b/examples/hub_validation_pipeline/pipeline.py index 3574bb865..4a9609e1d 100644 --- a/examples/hub_validation_pipeline/pipeline.py +++ b/examples/hub_validation_pipeline/pipeline.py @@ -586,6 +586,8 @@ def local_tasks(root, limit): def run(args): + if vars(args).get("submit_hf_jobs", False): + raise ValueError("Use prepared_batch with a reviewed shared reservation") output = args.output.resolve() output.mkdir(parents=True, exist_ok=False) coding_config = getattr(args, "coding_config", None) @@ -642,7 +644,7 @@ def run(args): if subject not in resolve_task_dirs(str(subject.parent), refresh=True): raise ValueError("OpenEnv did not load the acquired task") if coding_config is not None: - from examples.hub_validation_pipeline.cloud import prepare, submit + from examples.hub_validation_pipeline.cloud import prepare plan_dir = output / "plans" / key plan = prepare( @@ -667,13 +669,6 @@ def run(args): "state": "prepared", "accepted": False, } - if getattr(args, "submit_hf_jobs", False): - receipt = submit(plan_dir / "plan.json") - row.update( - state=receipt["status"], - job_id=receipt.get("job_id"), - job_url=receipt.get("job_url"), - ) plans.append(row) write_rows(output / "planned.jsonl", plans) continue @@ -783,11 +778,6 @@ def main(): type=Path, help="Prepare the supplemental coding audit on HF Jobs instead of local Docker validation", ) - parser.add_argument( - "--submit-hf-jobs", - action="store_true", - help="Submit prepared coding audits; incurs cloud/inference charges", - ) parser.add_argument("--profile", choices=["full", PILOT], default="full") parser.add_argument("--source", choices=["hf", "harbor", "both"], default="both") parser.add_argument("--dataset-limit", type=int, default=2) @@ -807,8 +797,6 @@ def main(): args = parser.parse_args() if args.coding_config is None and args.contract is None: parser.error("--contract or --coding-config is required") - if args.submit_hf_jobs and args.coding_config is None: - parser.error("--submit-hf-jobs requires --coding-config") if args.coding_config is not None and (args.contract is not None or args.llm_url): parser.error( "--coding-config selects its own contract/model and cannot combine --contract or --llm-url" diff --git a/examples/hub_validation_pipeline/prepared_batch.py b/examples/hub_validation_pipeline/prepared_batch.py index 0574734aa..41e5f3c51 100644 --- a/examples/hub_validation_pipeline/prepared_batch.py +++ b/examples/hub_validation_pipeline/prepared_batch.py @@ -159,6 +159,12 @@ def save(**state): amount, "Frozen prepared cohort", evidence=str(destination), + plan_costs={ + task["plan_sha256"]: full_run.estimate([config])[ + "reservation_usd_exact" + ] + for task, config in zip(manifest["tasks"], configs, strict=True) + }, ) manifest["shared_spend"]["status"] = "reserved" save() @@ -190,6 +196,8 @@ def save(**state): api=api, secrets=secrets, expected_plan_sha256=task["plan_sha256"], + spend_ledger=spend_ledger, + reservation_id=manifest["shared_spend"]["reservation_id"], ) except BaseException: local = path.parent / "submission.json" diff --git a/examples/hub_validation_pipeline/spend.py b/examples/hub_validation_pipeline/spend.py index 65a98a3ed..c0135eb85 100644 --- a/examples/hub_validation_pipeline/spend.py +++ b/examples/hub_validation_pipeline/spend.py @@ -68,6 +68,18 @@ def _report(data): statuses = {} for identifier, row in data["reservations"].items(): amount = _amount(row["reserved_usd"]) + costs = row.get("plan_costs", {}) + if ( + not isinstance(costs, dict) + or any( + not re.fullmatch(r"[0-9a-f]{64}", key) + or not isinstance(value, str) + or Fraction(value) < 0 + for key, value in costs.items() + ) + or sum((Fraction(value) for value in costs.values()), Fraction(0)) > amount + ): + raise ValueError("Invalid reserved plan costs") statuses[identifier] = row["status"] if row["status"] == "reserved" and row["spent_usd"] is None: reserved += amount @@ -192,7 +204,7 @@ def set_limit(path, limit_usd, evidence): return _report(data) -def reserve(path, id, amount_usd, purpose, evidence=None): +def reserve(path, id, amount_usd, purpose, evidence=None, *, plan_costs=None): """Reserve before paid work; settled IDs cannot be reused.""" amount = _amount(amount_usd) _text(id), _text(purpose) @@ -205,6 +217,7 @@ def reserve(path, id, amount_usd, purpose, evidence=None): if ( rows[id]["status"] == "reserved" and _amount(rows[id]["reserved_usd"]) == amount + and rows[id].get("plan_costs") == plan_costs ): return report raise ValueError("Reservation ID already used; use a fresh ID") @@ -217,6 +230,8 @@ def reserve(path, id, amount_usd, purpose, evidence=None): purpose=purpose, evidence=evidence, ) + if plan_costs is not None: + rows[id]["plan_costs"] = plan_costs return _report(data) diff --git a/examples/hub_validation_pipeline/test_cloud.py b/examples/hub_validation_pipeline/test_cloud.py index 8949bbfcc..f294bdd89 100644 --- a/examples/hub_validation_pipeline/test_cloud.py +++ b/examples/hub_validation_pipeline/test_cloud.py @@ -14,7 +14,7 @@ from unittest.mock import Mock import pytest -from examples.hub_validation_pipeline import cloud +from examples.hub_validation_pipeline import cloud, full_run, spend SHA = "a" * 40 UPLOAD_SHA = "b" * 40 @@ -55,12 +55,13 @@ def inputs(tmp_path, monkeypatch): config = { "schema_version": 1, "target": { - "model_id": "org/model", - "revision": SHA, - "provider": "example-provider", + "model_id": full_run.MODEL_SOURCE[0], + "revision": full_run.MODEL_SOURCE[1], + "provider": full_run.MODEL_SOURCE[2], "max_steps": 3, }, "execution": { + "flavor": "cpu-basic", "agent_image": "registry/agent@sha256:" + "c" * 64, "verifier_image": "registry/verifier@sha256:" + "d" * 64, "submission_paths": ["/app/answer"], @@ -97,6 +98,23 @@ def prepared(inputs, tmp_path, name="prepared"): return output, plan +def launch_options(output, config): + ledger = output.parent / "spend.json" + spend.init(ledger, "1000") + digest = hashlib.sha256((output / "plan.json").read_bytes()).hexdigest() + cost = full_run.estimate([config]) + spend.reserve( + ledger, + output.name, + cost["reservation_usd_rounded_up"], + "Reviewed test plan", + plan_costs={digest: cost["reservation_usd_exact"]}, + ) + return dict( + spend_ledger=ledger, reservation_id=output.name, expected_plan_sha256=digest + ) + + def fake_api(): api = Mock() api.dataset_info.return_value = SimpleNamespace(private=True, sha=SHA) @@ -159,7 +177,7 @@ def test_prepare_is_deterministic_offline_and_complete(inputs, tmp_path, monkeyp } <= files.keys() assert "examples/coding_audit/test_excluded.py" not in files assert manifest["git"] == {"commit": SHA, "dirty": True} - assert manifest["target_source"]["revision"] == SHA + assert manifest["target_source"]["revision"] == full_run.MODEL_SOURCE[1] assert manifest["task_source"]["identity"].startswith("sha256:") assert manifest["files"]["task/solution/solve.sh"]["mode"] == 0o755 assert not any( @@ -323,9 +341,7 @@ def test_submit_real_runtime_secrets_pins_and_one_shot_receipt(inputs, tmp_path) output / "plan.json", api=api, secrets=SECRETS, - expected_plan_sha256=hashlib.sha256( - (output / "plan.json").read_bytes() - ).hexdigest(), + **launch_options(output, inputs[2]), ) assert receipt["job_id"] == "job-123" launch = api.run_job.call_args.kwargs @@ -346,7 +362,12 @@ def test_submit_real_runtime_secrets_pins_and_one_shot_receipt(inputs, tmp_path) for value in SECRETS.values(): assert value.encode() not in path.read_bytes() with pytest.raises(FileExistsError, match="receipt"): - cloud.submit(output / "plan.json", api=api, secrets=SECRETS) + cloud.submit( + output / "plan.json", + api=api, + secrets=SECRETS, + **launch_options(output, inputs[2]), + ) assert api.run_job.call_count == 1 @@ -363,7 +384,12 @@ def test_no_launch_until_private_repo_and_exclusive_claim(inputs, tmp_path, case else: api.create_commit.side_effect = RuntimeError("parent conflict") with pytest.raises((ValueError, RuntimeError, FileExistsError)): - cloud.submit(output / "plan.json", api=api, secrets=SECRETS) + cloud.submit( + output / "plan.json", + api=api, + secrets=SECRETS, + **launch_options(output, inputs[2]), + ) api.run_job.assert_not_called() if case in ("public", "missing", "duplicate"): api.create_commit.assert_not_called() @@ -374,12 +400,22 @@ def test_uncertain_launch_never_retries_or_leaks_exception(inputs, tmp_path): api = fake_api() api.run_job.side_effect = RuntimeError(SECRETS["HF_TOKEN"]) with pytest.raises(RuntimeError, match="unknown") as caught: - cloud.submit(output / "plan.json", api=api, secrets=SECRETS) + cloud.submit( + output / "plan.json", + api=api, + secrets=SECRETS, + **launch_options(output, inputs[2]), + ) assert SECRETS["HF_TOKEN"] not in str(caught.value) receipt = json.loads((output / "submission.json").read_text()) assert receipt["status"] == "launch_unknown" with pytest.raises(FileExistsError): - cloud.submit(output / "plan.json", api=api, secrets=SECRETS) + cloud.submit( + output / "plan.json", + api=api, + secrets=SECRETS, + **launch_options(output, inputs[2]), + ) assert api.run_job.call_count == 1 @@ -395,7 +431,12 @@ def test_secret_values_required_and_distinct(inputs, tmp_path, values): output, _ = prepared(inputs, tmp_path) api = fake_api() with pytest.raises(ValueError): - cloud.submit(output / "plan.json", api=api, secrets=values) + cloud.submit( + output / "plan.json", + api=api, + secrets=values, + **launch_options(output, inputs[2]), + ) api.dataset_info.assert_not_called() api.run_job.assert_not_called() @@ -414,7 +455,9 @@ def test_plan_tampering_stops_before_hub(inputs, tmp_path, section): path.write_text(json.dumps(plan)) api = fake_api() with pytest.raises(ValueError, match="immutable"): - cloud.submit(path, api=api, secrets=SECRETS) + cloud.submit( + path, api=api, secrets=SECRETS, **launch_options(output, inputs[2]) + ) api.dataset_info.assert_not_called() @@ -752,3 +795,112 @@ def prepare(subject, config, output, *, source): assert planned["accepted"] is False and planned["state"] == "prepared" assert (args.output / "accepted.jsonl").read_text() == "" assert calls[0][1]["revision"] + + +@pytest.mark.parametrize("ledger_state", ["missing", "exhausted", "blocked"]) +@pytest.mark.parametrize("entry", ["cloud_cli", "pipeline", "submit"]) +def test_public_launch_paths_require_spending_gates( + inputs, tmp_path, monkeypatch, ledger_state, entry +): + from argparse import Namespace + + import huggingface_hub + from examples.hub_validation_pipeline import pipeline, spend + + repo, task, config = inputs + ledger = tmp_path / "spend.json" + if ledger_state != "missing": + spend.init(ledger, "1") + spend.reserve(ledger, "prior", "1", "Prior work") + if ledger_state == "blocked": + spend.settle(ledger, "prior", "2", "Overrun") + api = fake_api() + monkeypatch.setattr(huggingface_hub, "HfApi", lambda **kwargs: api) + for name, value in SECRETS.items(): + monkeypatch.setenv(name, value) + output, _ = prepared(inputs, tmp_path) + if entry == "cloud_cli": + with pytest.raises(SystemExit): + cloud.main(["submit", "--plan", str(output / "plan.json")]) + elif entry == "pipeline": + config_path = tmp_path / "coding.json" + config_path.write_bytes(cloud._json_bytes(config)) + real_prepare = cloud.prepare + monkeypatch.setattr( + cloud, + "prepare", + lambda *args, **kwargs: real_prepare(*args, repo_root=repo, **kwargs), + ) + args = Namespace( + output=tmp_path / "run", + contract=None, + coding_config=config_path, + submit_hf_jobs=True, + hf_dataset=[], + harbor_dataset=[], + local_dataset=[task], + source="both", + dataset_limit=0, + tasks_per_dataset=1, + ) + with pytest.raises(ValueError, match="prepared_batch"): + pipeline.run(args) + else: + with pytest.raises(ValueError, match="reservation"): + cloud.submit(output / "plan.json", api=api, secrets=SECRETS) + api.run_job.assert_not_called() + api.create_commit.assert_not_called() + + +@pytest.mark.parametrize( + "case", + [ + "missing_ledger", + "unknown", + "blocked", + "settled", + "unbound", + "underpriced", + "missing_hash", + "wrong_hash", + "overallocated", + "exhausted", + ], +) +def test_submit_rechecks_reserved_plan_cost_and_status(inputs, tmp_path, case): + output, _ = prepared(inputs, tmp_path) + options = launch_options(output, inputs[2]) + ledger, identifier = options["spend_ledger"], options["reservation_id"] + if case == "missing_ledger": + ledger.unlink() + elif case == "unknown": + options["reservation_id"] = "unreserved" + elif case == "blocked": + spend.reserve(ledger, "other", "1", "Other work") + spend.settle(ledger, "other", "2", "Overrun") + elif case == "settled": + spend.settle(ledger, identifier, "0", "Completed work") + elif case == "missing_hash": + del options["expected_plan_sha256"] + elif case == "wrong_hash": + options["expected_plan_sha256"] = "0" * 64 + else: + data = json.loads(ledger.read_bytes()) + row = data["reservations"][identifier] + if case == "exhausted": + data["reservations"] = {} + elif case == "unbound": + del row["plan_costs"] + else: + row["plan_costs"][options["expected_plan_sha256"]] = ( + "0" if case == "underpriced" else "1001" + ) + ledger.write_text(json.dumps(data)) + if case == "exhausted": + spend.reserve(ledger, "other", "1000", "Other work") + api = fake_api() + with pytest.raises(ValueError): + cloud.submit(output / "plan.json", api=api, secrets=SECRETS, **options) + api.run_job.assert_not_called() + api.create_commit.assert_not_called() + assert not (output / "submission.json").exists() diff --git a/examples/hub_validation_pipeline/test_full_run.py b/examples/hub_validation_pipeline/test_full_run.py index 1edd93367..05f1b9ea0 100644 --- a/examples/hub_validation_pipeline/test_full_run.py +++ b/examples/hub_validation_pipeline/test_full_run.py @@ -300,7 +300,10 @@ def test_selection_budgets_and_submits_only_selected_task(local_run): ) assert task["submission"]["status"] == "submitted" submit.assert_called_once_with( - task["plan"], expected_plan_sha256=task["plan_sha256"] + task["plan"], + expected_plan_sha256=task["plan_sha256"], + spend_ledger=ledger, + reservation_id=manifest["shared_spend"]["reservation_id"], ) assert not (output / "recipes/toy-beta").exists() assert "toy-beta" not in (output / "manifest.json").read_text() @@ -459,7 +462,7 @@ def test_partial_ambiguous_submission_is_durable_safe_and_never_resumed( output, hf, harbor, prepare, submit, _ = local_run observed = [] - def launch(path, *, expected_plan_sha256): + def launch(path, *, expected_plan_sha256, spend_ledger, reservation_id): assert full_run._hash(Path(path))["sha256"] == expected_plan_sha256 durable = cloud._loads((output / "manifest.json").read_bytes()) assert durable["shared_spend"]["ledger"] == str( @@ -468,6 +471,8 @@ def launch(path, *, expected_plan_sha256): assert Fraction( spend.summary(paid_reservation["spend_ledger"])["reserved_usd_exact"] ) == Fraction(durable["estimate"]["reservation_usd_rounded_up"]) + assert spend_ledger == paid_reservation["spend_ledger"] + assert reservation_id == durable["shared_spend"]["reservation_id"] index = len(observed) observed.append(path) assert durable["tasks"][index]["submission"]["status"] == "pending_or_unknown" diff --git a/examples/hub_validation_pipeline/test_prepared_batch.py b/examples/hub_validation_pipeline/test_prepared_batch.py index 83175e7fd..173707010 100644 --- a/examples/hub_validation_pipeline/test_prepared_batch.py +++ b/examples/hub_validation_pipeline/test_prepared_batch.py @@ -89,10 +89,14 @@ def sleep(seconds): actual_submit = cloud.submit - def submit(path, *, api, secrets, expected_plan_sha256): + def submit( + path, *, api, secrets, expected_plan_sha256, spend_ledger, reservation_id + ): assert secrets == SECRETS assert full_run._hash(path)["sha256"] == expected_plan_sha256 durable = cloud._loads((output / "manifest.json").read_bytes()) + assert spend_ledger == ledger + assert reservation_id == durable["shared_spend"]["reservation_id"] index = plans.index(path) assert durable["tasks"][index]["status"] == "pending_or_unknown" if index: From 92897017173160e6f4f44a586bc27f5fa1c3a48c Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:24:30 +0100 Subject: [PATCH 08/19] fix(hub-validation): verify captured tasks against source identities Hash the frozen task tree rather than a restored live source. Require verified file inventories for remote identities and carry acquisition hashes through the discovery pipeline. --- .../hub_validation_pipeline/CODING_AUDIT.md | 4 +- examples/hub_validation_pipeline/cloud.py | 13 +-- examples/hub_validation_pipeline/pipeline.py | 28 +++---- .../hub_validation_pipeline/test_cloud.py | 81 +++++++++++++++++-- 4 files changed, 98 insertions(+), 28 deletions(-) diff --git a/examples/hub_validation_pipeline/CODING_AUDIT.md b/examples/hub_validation_pipeline/CODING_AUDIT.md index 7fa2de8c1..9c9849fb5 100644 --- a/examples/hub_validation_pipeline/CODING_AUDIT.md +++ b/examples/hub_validation_pipeline/CODING_AUDIT.md @@ -145,7 +145,9 @@ Each task declares: The panel's exact bytes are read once. Manifests record its path, byte length and SHA256; each prepared bundle's source provenance includes the same panel SHA256. Native task/verifier bytes remain unchanged and bind to the original acquired -file inventory. Configuration and annotations come from the operator input. +file inventory. Remote source identities require a complete, verified acquisition +hash inventory; a revision string alone cannot authorize a bundle. Local revisions +are checked against the captured bundle bytes, including the verifier. Configuration and annotations come from the operator input. ```bash uv run --no-sync python -m examples.hub_validation_pipeline.real_tasks \ diff --git a/examples/hub_validation_pipeline/cloud.py b/examples/hub_validation_pipeline/cloud.py index 15b7f7e3e..0b97289e2 100644 --- a/examples/hub_validation_pipeline/cloud.py +++ b/examples/hub_validation_pipeline/cloud.py @@ -250,8 +250,10 @@ def _positive(value, name, *, maximum=10000, minimum=1): def _check_acquisition_files(source, task_files): - """Bind a captured task to optional hashes verified during acquisition.""" + """Bind remote source identities to hashes verified during acquisition.""" if "task_file_hashes" not in source: + if source.get("kind", source.get("source")) in {"hf", "harbor"}: + raise ValueError("Remote source requires verified acquisition file hashes") return expected = source["task_file_hashes"] if ( @@ -515,6 +517,7 @@ def prepare(task_root, config, output_dir, *, repo_root=None, source=None): raise ValueError("Output directory must be outside the task") config, budget = _config(config) from examples.coding_audit.runner import validate_coding_task + from openenv.validation.runner import source_digest validate_coding_task(task_root, config) files = {} @@ -565,6 +568,7 @@ def prepare(task_root, config, output_dir, *, repo_root=None, source=None): path.write_bytes(content) path.chmod(mode) validate_coding_task(frozen, config) + captured_digest = source_digest(frozen) files.update({"task/" + name: value for name, value in task_files.items()}) files["coding-config.json"] = (_json_bytes(config), 0o644) if source is None: @@ -603,11 +607,8 @@ def prepare(task_root, config, output_dir, *, repo_root=None, source=None): raise ValueError("Dataset source requires its full immutable revision") if source.get("task_path") != ".": _safe_name(source["task_path"]) - if kind == "local": - from openenv.validation.runner import source_digest - - if source_digest(task_root) != revision: - raise ValueError("Local source revision differs from the prepared task") + if kind == "local" and captured_digest != revision: + raise ValueError("Local source revision differs from the prepared task") source["kind"] = kind _check_acquisition_files( source, {name: value[0] for name, value in task_files.items()} diff --git a/examples/hub_validation_pipeline/pipeline.py b/examples/hub_validation_pipeline/pipeline.py index 4a9609e1d..a62723bcb 100644 --- a/examples/hub_validation_pipeline/pipeline.py +++ b/examples/hub_validation_pipeline/pipeline.py @@ -647,21 +647,19 @@ def run(args): from examples.hub_validation_pipeline.cloud import prepare plan_dir = output / "plans" / key - plan = prepare( - subject, - coding_config, - plan_dir, - source={ - name: task[name] - for name in ( - "source", - "source_id", - "revision", - "task_path", - "task_id", - ) - }, - ) + provenance = { + name: task[name] + for name in ( + "source", + "source_id", + "revision", + "task_path", + "task_id", + ) + } + if "task_file_hashes" in task: + provenance["task_file_hashes"] = task["task_file_hashes"] + plan = prepare(subject, coding_config, plan_dir, source=provenance) row = { "task_id": task["task_id"], "run_id": plan["run_id"], diff --git a/examples/hub_validation_pipeline/test_cloud.py b/examples/hub_validation_pipeline/test_cloud.py index f294bdd89..ab35b161b 100644 --- a/examples/hub_validation_pipeline/test_cloud.py +++ b/examples/hub_validation_pipeline/test_cloud.py @@ -554,6 +554,10 @@ def test_pinned_discovery_sources_bind_into_cloud_bundle(inputs, tmp_path, kind) "revision": revision, "task_path": ".", } + source["task_file_hashes"] = { + name: hashlib.sha256(data).hexdigest() + for name, (data, _) in cloud._tree(task).items() + } plan = cloud.prepare(task, config, tmp_path / kind, repo_root=repo, source=source) assert plan["task_source"]["kind"] == kind assert plan["task_source"]["revision"] == revision @@ -750,14 +754,30 @@ def test_bundle_retains_available_full_run_reproduction_helpers(inputs, tmp_path assert files[relative] == (repo / relative).read_bytes() +@pytest.mark.parametrize("source_kind", ["local", "hf", "harbor"]) def test_hub_driver_prepares_coding_audits_without_admitting_or_running_docker( - inputs, tmp_path, monkeypatch + inputs, tmp_path, monkeypatch, source_kind ): from argparse import Namespace from examples.hub_validation_pipeline import cloud, pipeline - _, task, raw_config = inputs + repo, task, raw_config = inputs + hashes = { + name: hashlib.sha256(data).hexdigest() + for name, (data, _) in cloud._tree(task).items() + } + if source_kind != "local": + record = dict( + task_id="remote-test-task", + path=str(task), + source=source_kind, + source_id="org/tasks", + task_path=".", + task_file_hashes=hashes, + revision=SHA if source_kind == "hf" else "sha256:" + "b" * 64, + ) + monkeypatch.setattr(pipeline.sources, "acquire", lambda *args: [record]) config = tmp_path / "coding.json" config.write_text(json.dumps(raw_config)) calls = [] @@ -766,7 +786,9 @@ def test_hub_driver_prepares_coding_audits_without_admitting_or_running_docker( def prepare(subject, config, output, *, source): calls.append((subject, source)) assert (subject / "task.toml").is_file() - return real_prepare(subject, config, output, source=source) + if source_kind != "local": + assert source["task_file_hashes"] == hashes + return real_prepare(subject, config, output, repo_root=repo, source=source) monkeypatch.setattr(cloud, "prepare", prepare) monkeypatch.setattr( @@ -779,9 +801,11 @@ def prepare(subject, config, output, *, source): contract=None, coding_config=config, submit_hf_jobs=False, - hf_dataset=[], - harbor_dataset=[], - local_dataset=[task], + hf_dataset=["org/tasks@" + SHA] if source_kind == "hf" else [], + harbor_dataset=["org/tasks@sha256:" + "b" * 64] + if source_kind == "harbor" + else [], + local_dataset=[task] if source_kind == "local" else [], source="both", dataset_limit=0, tasks_per_dataset=1, @@ -904,3 +928,48 @@ def test_submit_rechecks_reserved_plan_cost_and_status(inputs, tmp_path, case): api.run_job.assert_not_called() api.create_commit.assert_not_called() assert not (output / "submission.json").exists() + + +def test_local_revision_checks_captured_verifier_not_restored_live_tree( + inputs, tmp_path, monkeypatch +): + from openenv.validation.runner import source_digest + + repo, task, config = inputs + revision = source_digest(task) + original_tree = cloud._tree + verifier = task / "tests/test.sh" + original = verifier.read_bytes() + captured = [] + + def mutate_during_capture(root, **kwargs): + if root != task: + return original_tree(root, **kwargs) + verifier.write_bytes(b"echo 1 > /logs/verifier/reward.txt\n") + try: + files = original_tree(root, **kwargs) + captured.append(files["tests/test.sh"][0]) + return files + finally: + verifier.write_bytes(original) + + monkeypatch.setattr(cloud, "_tree", mutate_during_capture) + source = {"kind": "local", "revision": revision, "task_path": "."} + with pytest.raises(ValueError, match="Local source revision"): + cloud.prepare(task, config, tmp_path / "race", repo_root=repo, source=source) + assert source_digest(task) == revision + assert captured == [b"echo 1 > /logs/verifier/reward.txt\n"] + assert not (tmp_path / "race").exists() + + +@pytest.mark.parametrize("kind", ["hf", "harbor"]) +def test_remote_identity_requires_verified_acquisition_hashes(inputs, tmp_path, kind): + repo, task, config = inputs + source = dict( + kind=kind, + revision="a" * 40 if kind == "hf" else "sha256:" + "b" * 64, + task_path=".", + ) + with pytest.raises(ValueError, match="verified acquisition"): + cloud.prepare(task, config, tmp_path / kind, repo_root=repo, source=source) + assert not (tmp_path / kind).exists() From 36560650c1626632dcb7b01f5516860271272bd1 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:31:21 +0100 Subject: [PATCH 09/19] fix(hub-validation): bound source downloads and Harbor archives Reuse native file-count and byte caps before downloading selected tasks and shared assets. Verify pinned file inventories, stream Harbor downloads under a byte cap, and check bounded tar contents before materializing task files. --- examples/hub_validation_pipeline/README.md | 10 +- .../hub_validation_pipeline/real_tasks.py | 4 +- examples/hub_validation_pipeline/sources.py | 300 ++++++++++----- .../hub_validation_pipeline/test_sources.py | 352 ++++++++++++++---- 4 files changed, 506 insertions(+), 160 deletions(-) diff --git a/examples/hub_validation_pipeline/README.md b/examples/hub_validation_pipeline/README.md index aa1571154..f5e70d362 100644 --- a/examples/hub_validation_pipeline/README.md +++ b/examples/hub_validation_pipeline/README.md @@ -103,7 +103,15 @@ shared assets; the runnable profile requires self-contained tasks. The example uses the HF SDK's `repo_type`, `revision`, `local_dir` and selection arguments, corresponding to `hf download --repo-type dataset --revision ... --local-dir ...`. It deliberately resolves mutable source aliases before execution, -uses anonymous public reads (`token=False`) and applies explicit byte/task limits. +uses anonymous public reads (`token=False`) and applies the native acquisition +limits: 128 files, 2 MiB per file and 8 MiB total across selected tasks and shared +assets. Size-bearing metadata is required before downloads. HF inputs must be +regular Git files (no LFS); each downloaded blob is checked against the pinned +tree. Harbor requires complete per-file size/hash inventories, caps compressed +transfers and expanded tar bytes at 8 MiB, and rejects links, sparse files and +uninventoried members. Archive directory entries count toward the 128-member cap. +Legacy packages without these inventories are refused. Verified task hashes follow +the captured bytes into cloud plans; a remote revision alone is insufficient. Uploads use ordinary SDK commits with explicit additions; they do not mirror-delete remote files. As with `hf upload`, deletion requires a separate explicit operation. diff --git a/examples/hub_validation_pipeline/real_tasks.py b/examples/hub_validation_pipeline/real_tasks.py index 37bace582..882114bfe 100644 --- a/examples/hub_validation_pipeline/real_tasks.py +++ b/examples/hub_validation_pipeline/real_tasks.py @@ -19,10 +19,8 @@ from openenv.validation.runner import source_digest from . import cloud, sources +from .sources import MAX_FILE_BYTES, MAX_FILES, MAX_TOTAL_BYTES -MAX_FILES = 128 -MAX_FILE_BYTES = 2 * 1024 * 1024 -MAX_TOTAL_BYTES = 8 * 1024 * 1024 _REQUIRED = {"task.toml", "instruction.md", "solution/solve.sh", "tests/test.sh"} diff --git a/examples/hub_validation_pipeline/sources.py b/examples/hub_validation_pipeline/sources.py index 0648c5c36..31be97c0d 100644 --- a/examples/hub_validation_pipeline/sources.py +++ b/examples/hub_validation_pipeline/sources.py @@ -1,15 +1,22 @@ """Discover public Hub bundles and acquire pinned task files without executing them.""" import asyncio +import gzip import hashlib +import io import json import re -import shutil +import stat +import tarfile import tempfile from importlib.metadata import distribution, PackageNotFoundError from pathlib import Path, PurePosixPath -from huggingface_hub import HfApi, snapshot_download +from huggingface_hub import HfApi, RepoFile, RepoFolder, snapshot_download + +MAX_FILES = 128 +MAX_FILE_BYTES = 2 * 1024 * 1024 +MAX_TOTAL_BYTES = 8 * 1024 * 1024 HARBOR_GIT_REVISION = "9c7a39b991d7e2205019f6be0598cbd22d84bfc3" HARBOR_INSTALL = ( @@ -65,14 +72,14 @@ def _harbor_api(): f"native Hub listing requires official Git {HARBOR_GIT_REVISION}. " f"Install: {HARBOR_INSTALL}" ) + from harbor.db.client import RegistryDB from harbor.models.task.id import PackageTaskId from harbor.registry.client.package import PackageDatasetClient - from harbor.tasks.client import TaskClient except (ImportError, PackageNotFoundError) as error: raise RuntimeError( f"Pinned Harbor Hub package APIs unavailable. Install: {HARBOR_INSTALL}" ) from error - return PackageDatasetClient, PackageTaskId, TaskClient + return PackageDatasetClient, PackageTaskId, RegistryDB def discover_harbor(limit: int) -> list[dict]: @@ -158,26 +165,6 @@ def _fresh(cache_dir: Path, source: str, identity: str, revision: str) -> Path: return Path(tempfile.mkdtemp(prefix="selection-", dir=parent)).resolve() -def _real_files(root: Path) -> None: - """Reject escaping links and materialize contained links as real files.""" - has_links = False - for path in root.rglob("*"): - if path.is_symlink(): - target = path.resolve(strict=True) - if not target.is_relative_to(root): - raise ValueError(f"Unsafe task symlink: {path}") - has_links = True - elif not path.is_dir() and not path.is_file(): - raise ValueError(f"Unsupported task file: {path}") - if has_links: - # copytree also rejects dangling links and directory-link cycles. - with tempfile.TemporaryDirectory(dir=root.parent) as temporary: - materialized = Path(temporary) / "files" - shutil.copytree(root, materialized, symlinks=False) - shutil.rmtree(root) - materialized.rename(root) - - def _record(source, identity, revision, root, task_path, metadata, **extra): return { "task_id": f"{source}:{identity}@{revision}:{task_path}", @@ -216,21 +203,27 @@ def _acquire_hf(candidate, cache_dir, revision, max_tasks): raise ValueError("HF revision resolution disagrees with the selected commit") if info.gated or info.disabled: raise PermissionError(f"HF source is gated or disabled: {candidate['id']}") - files = sorted( - { - _relative(name).as_posix() - for name in api.list_repo_files( - candidate["id"], repo_type="dataset", revision=revision - ) - } - ) + files = {} + for item in api.list_repo_tree( + candidate["id"], + repo_type="dataset", + revision=revision, + recursive=True, + token=False, + ): + name = _relative(item.path).as_posix() + if isinstance(item, RepoFolder): + continue + if not isinstance(item, RepoFile) or name in files: + raise ValueError("HF source requires distinct regular files") + files[name] = item roots = set(task_roots(files)) selected = sorted(roots)[:max_tasks] if not selected: raise ValueError(f"No task.toml files in {candidate['id']}@{revision}") # Retain shared repository assets while excluding every unselected task tree. selected_files = [] - for name in files: + for name in sorted(files): owner = next( ( str(parent) @@ -246,20 +239,44 @@ def _acquire_hf(candidate, cache_dir, revision, max_tasks): "".join({"[": "[[]", "?": "[?]", "*": "[*]"}.get(char, char) for char in name) for name in selected_files ] + _check_sizes([files[name].size for name in selected_files]) + if any(files[name].lfs for name in selected_files): + raise ValueError("Only native regular Git source files are supported") + hashes = {} + payloads = {} + with tempfile.TemporaryDirectory(prefix="openenv-hf-acquisition-") as temporary: + staging = Path(temporary).resolve() + snapshot_download( + candidate["id"], + repo_type="dataset", + revision=revision, + local_dir=staging, + allow_patterns=patterns, + token=False, + force_download=True, + ) + for name in selected_files: + path = staging / name + if path.resolve() != path or not stat.S_ISREG(path.lstat().st_mode): + raise ValueError( + "HF download must materialize a contained regular file" + ) + if path.stat().st_size != files[name].size: + raise ValueError("HF source size disagrees with pinned tree") + with path.open("rb") as stream: + data = stream.read(MAX_FILE_BYTES + 1) + if len(data) != files[name].size: + raise ValueError("HF source byte cap or size mismatch") + blob = hashlib.sha1(f"blob {len(data)}\0".encode() + data).hexdigest() + if blob != files[name].blob_id: + raise ValueError("HF source Git blob hash mismatch") + payloads[name] = data + hashes[name] = hashlib.sha256(data).hexdigest() root = _fresh(cache_dir, "hf", candidate["id"], revision) - snapshot_download( - candidate["id"], - repo_type="dataset", - revision=revision, - local_dir=root, - allow_patterns=patterns, - token=False, - force_download=True, - ) - _real_files(root) - for name in selected_files: - if not (root / name).is_file(): - raise FileNotFoundError(f"HF download did not materialize {name}") + for name, data in payloads.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) return [ _record( "hf", @@ -269,13 +286,100 @@ def _acquire_hf(candidate, cache_dir, revision, max_tasks): task_path, metadata, download_patterns=patterns, + task_file_hashes={ + PurePosixPath(name).relative_to(task_path).as_posix(): digest + for name, digest in hashes.items() + if PurePosixPath(name).is_relative_to(task_path) + }, ) for task_path in selected ] +def _check_sizes(sizes): + if ( + len(sizes) > MAX_FILES + or any( + type(size) is not int or not 0 <= size <= MAX_FILE_BYTES for size in sizes + ) + or sum(sizes) > MAX_TOTAL_BYTES + ): + raise ValueError("Source file count or byte cap exceeded") + + +async def _download_harbor_file(remote_path, destination, byte_limit): + import httpx + from harbor.auth.client import create_authenticated_client + from harbor.storage.supabase import BUCKET + + client = await create_authenticated_client() + signed = await client.storage.from_(BUCKET).create_signed_url(remote_path, 300) + async with httpx.AsyncClient(timeout=120, follow_redirects=False) as http: + async with http.stream("GET", signed["signedURL"]) as response: + response.raise_for_status() + if int(response.headers.get("content-length", "0")) > byte_limit: + raise ValueError("Harbor download byte cap exceeded") + total = 0 + with destination.open("xb") as stream: + async for chunk in response.aiter_bytes(chunk_size=65536): + total += len(chunk) + if total > byte_limit: + raise ValueError("Harbor download byte cap exceeded") + stream.write(chunk) + return total + + +def _harbor_inventory(rows): + files = {} + for row in rows: + name = _relative(row["path"]).as_posix() + digest = row["content_hash"].removeprefix("sha256:") + if name.casefold() in {key.casefold() for key in files} or not re.fullmatch( + r"[0-9a-f]{64}", digest + ): + raise ValueError("Invalid Harbor file inventory") + files[name] = dict(row, sha256=digest) + _check_sizes([row["size_bytes"] for row in files.values()]) + return files + + +def _read_harbor_archive(archive_path, inventory): + payloads = {} + names = set() + with gzip.open(archive_path, "rb") as stream: + data = stream.read(MAX_TOTAL_BYTES + 1) + if len(data) > MAX_TOTAL_BYTES: + raise ValueError("Harbor archive expanded byte cap exceeded") + with tarfile.open(fileobj=io.BytesIO(data), mode="r:") as archive: + for member in archive: + name = _relative(member.name).as_posix() + if name in names or len(names) >= MAX_FILES: + raise ValueError( + "Harbor archive path collision or file count cap exceeded" + ) + names.add(name) + if member.isdir(): + continue + if not member.isfile() or member.issparse() or name not in inventory: + raise ValueError("Harbor archive must match its regular file inventory") + row = inventory[name] + if member.size != row["size_bytes"]: + raise ValueError("Harbor archive byte cap or size mismatch") + with archive.extractfile(member) as stream: + data = stream.read(MAX_FILE_BYTES + 1) + if ( + len(data) != member.size + or hashlib.sha256(data).hexdigest() != row["sha256"] + ): + raise ValueError("Harbor archive file hash or size mismatch") + payloads[name] = (data, member.mode & 0o777) + if set(payloads) != set(inventory): + raise ValueError("Harbor archive file inventory is incomplete") + return payloads + + async def _acquire_harbor(candidate, cache_dir, revision, max_tasks): - registry_type, task_id_type, task_client_type = _harbor_api() + registry_type, task_id_type, db_type = _harbor_api() _package_name(candidate["id"]) registry = registry_type() metadata = candidate.get("metadata", {}) @@ -300,44 +404,74 @@ async def _acquire_harbor(candidate, cache_dir, revision, max_tasks): raise ValueError("Current Harbor Hub acquisition requires package task IDs") _package_name(f"{task.org}/{task.name}") _pin(task.ref, "harbor") - root = _fresh(cache_dir, "harbor", candidate["id"], revision) - dataset_files = {} - if bundle is not None: - for file in bundle.files: - _relative(file.path) - dataset_files = await registry.download_dataset_files( - bundle, output_dir=root / "dataset", overwrite=True + db = db_type() + shared = {} + if bundle is not None and bundle.files: + shared = _harbor_inventory( + await db.get_dataset_version_files(bundle.dataset_version_id) ) - downloaded = await task_client_type().download_tasks( - task_ids=tasks, output_dir=root / "tasks", overwrite=True, export=False - ) - _real_files(root) - records = [] - for task, result in zip(tasks, downloaded.results, strict=True): - if result.content_hash != task.ref.removeprefix("sha256:"): + if set(shared) != {file.path for file in bundle.files}: + raise ValueError("Harbor shared file inventory is incomplete") + packages = [] + sizes = [row["size_bytes"] for row in shared.values()] + for task in tasks: + resolved = await db.resolve_task_version(task.org, task.name, task.ref) + if resolved.content_hash != task.ref.removeprefix("sha256:"): raise ValueError("Harbor task resolution disagrees with its digest") - task_root = result.path.resolve(strict=True) - if not task_root.is_relative_to(root): - raise ValueError("Harbor downloaded a task outside the selected cache") - if not (task_root / "task.toml").is_file(): - raise ValueError( - f"Harbor package contains no root task.toml: {task.org}/{task.name}" + inventory = _harbor_inventory(await db.get_task_version_files(resolved.id)) + if "task.toml" not in inventory: + raise ValueError("Harbor package contains no root task.toml") + sizes.extend(row["size_bytes"] for row in inventory.values()) + _check_sizes(sizes) + packages.append((task, resolved, inventory)) + root = _fresh(cache_dir, "harbor", candidate["id"], revision) + dataset_files, records = {}, [] + remaining = MAX_TOTAL_BYTES + with tempfile.TemporaryDirectory(prefix="openenv-harbor-acquisition-") as temporary: + staging = Path(temporary) + for index, (name, row) in enumerate(shared.items()): + path = staging / f"shared-{index}" + remaining -= await _download_harbor_file( + row["storage_path"], path, row["size_bytes"] + ) + data = path.read_bytes() + if ( + len(data) != row["size_bytes"] + or hashlib.sha256(data).hexdigest() != row["sha256"] + ): + raise ValueError("Harbor shared file hash or size mismatch") + destination = root / "dataset" / name + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(data) + dataset_files[name] = str(destination) + for index, (task, resolved, inventory) in enumerate(packages): + archive = staging / f"task-{index}.tar.gz" + remaining -= await _download_harbor_file( + resolved.archive_path, archive, remaining ) - records.append( - _record( - "harbor", - f"{task.org}/{task.name}", - task.ref, - task_root, - ".", - { - **metadata, - "candidate_id": candidate["id"], - "candidate_revision": revision, - "dataset_files": { - name: str(path) for name, path in dataset_files.items() + payloads = _read_harbor_archive(archive, inventory) + task_root = root / "tasks" / task.org / task.name / resolved.content_hash + for name, (data, mode) in payloads.items(): + path = task_root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + path.chmod(mode) + records.append( + _record( + "harbor", + f"{task.org}/{task.name}", + task.ref, + task_root, + ".", + { + **metadata, + "candidate_id": candidate["id"], + "candidate_revision": revision, + "dataset_files": dataset_files, }, - }, + task_file_hashes={ + name: row["sha256"] for name, row in inventory.items() + }, + ) ) - ) - return records[:max_tasks] + return records diff --git a/examples/hub_validation_pipeline/test_sources.py b/examples/hub_validation_pipeline/test_sources.py index 12e10e5dc..12aaa09ca 100644 --- a/examples/hub_validation_pipeline/test_sources.py +++ b/examples/hub_validation_pipeline/test_sources.py @@ -1,13 +1,18 @@ """Network-free checks for discovery, immutable locators, and task assets.""" +import asyncio +import hashlib +import io import json +import tarfile from fnmatch import fnmatchcase from pathlib import Path from types import SimpleNamespace -from unittest.mock import AsyncMock, Mock +from unittest.mock import AsyncMock, MagicMock, Mock import pytest -from examples.hub_validation_pipeline import sources +from examples.hub_validation_pipeline import real_tasks, sources +from huggingface_hub import RepoFile SHA = "a" * 40 DIGEST = "sha256:" + "b" * 64 @@ -24,6 +29,16 @@ def fake_hf(monkeypatch, files): sha=SHA, gated=False, disabled=False ) api.list_repo_files.return_value = list(files) + api.list_repo_tree.return_value = [ + RepoFile( + path=name, + size=len(content.encode()), + oid=hashlib.sha1( + f"blob {len(content.encode())}\0".encode() + content.encode() + ).hexdigest(), + ) + for name, content in files.items() + ] factory = Mock(return_value=api) monkeypatch.setattr(sources, "HfApi", factory) @@ -90,15 +105,15 @@ def test_hf_bounded_assets_pinned_and_fresh(monkeypatch, tmp_path): assert path.is_absolute() and path == path.resolve() assert (path / "environment/assets/data.txt").read_text() == "actual asset" assert not (path.parent / "0001").exists() - root = downloader.call_args.kwargs["local_dir"] + root = path.parents[2] assert (root / "shared/data[train].txt").read_text() == "shared" assert len(first["download_patterns"]) == 5 assert first["revision"] == SHA and first["source_id"] == "org/bundle" assert SHA in path.parts factory.assert_called_with(token=False) api.dataset_info.assert_called_with("org/bundle", revision=SHA) - api.list_repo_files.assert_called_with( - "org/bundle", repo_type="dataset", revision=SHA + api.list_repo_tree.assert_called_with( + "org/bundle", repo_type="dataset", revision=SHA, recursive=True, token=False ) assert downloader.call_args.kwargs["revision"] == SHA assert downloader.call_args.kwargs["repo_type"] == "dataset" @@ -142,22 +157,28 @@ def test_unsafe_source_paths_and_symlinks_fail(monkeypatch, tmp_path): sources.acquire(hf_candidate(), tmp_path, 1) outside = tmp_path / "outside" outside.write_text("original") - root = tmp_path / "tree" - root.mkdir() - (root / "escape").symlink_to(outside) - with pytest.raises(ValueError, match="Unsafe task symlink"): - sources._real_files(root) + _, _, download = fake_hf(monkeypatch, {"task.toml": "original"}) + + def link(repo, **kwargs): + (kwargs["local_dir"] / "task.toml").symlink_to(outside) + + download.side_effect = link + with pytest.raises(ValueError, match="contained regular file"): + sources.acquire(hf_candidate(), tmp_path / "cache", 1) assert outside.read_text() == "original" -def test_contained_links_materialize_as_real_files(tmp_path): - root = tmp_path / "tree" - (root / "assets").mkdir(parents=True) - (root / "assets/data").write_text("real data") - (root / "linked-assets").symlink_to("assets", target_is_directory=True) - sources._real_files(root) - assert (root / "linked-assets/data").read_text() == "real data" - assert not any(path.is_symlink() for path in root.rglob("*")) +def test_hf_acquisition_rejects_contained_links(monkeypatch, tmp_path): + _, _, download = fake_hf(monkeypatch, {"task.toml": "original"}) + + def link(repo, **kwargs): + root = kwargs["local_dir"] + (root / "actual").write_text("original") + (root / "task.toml").symlink_to(root / "actual") + + download.side_effect = link + with pytest.raises(ValueError, match="contained regular file"): + sources.acquire(hf_candidate(), tmp_path, 1) class PackageTaskId: @@ -187,29 +208,59 @@ def fake_harbor(monkeypatch): download_dataset_files=AsyncMock(return_value={}), ) - async def download(*, task_ids, output_dir, overwrite, export): - results = [] - for task in task_ids: - digest = task.ref.removeprefix("sha256:") - path = output_dir / task.org / task.name / digest - (path / "environment/assets").mkdir(parents=True) - (path / "task.toml").write_text("") - (path / "instruction.md").write_text("task") - (path / "environment/assets/data").write_text("asset") - results.append(SimpleNamespace(path=path, content_hash=digest)) - return SimpleNamespace(results=results) - - task_client = SimpleNamespace(download_tasks=AsyncMock(side_effect=download)) + payloads = { + "task.toml": b"", + "instruction.md": b"task", + "environment/assets/data": b"asset", + } + + def rows(): + return [ + dict( + path=name, + size_bytes=len(data), + content_hash=hashlib.sha256(data).hexdigest(), + ) + for name, data in client.payloads.items() + ] + + async def resolve(org, name, ref): + return SimpleNamespace( + id=org + "/" + name, + content_hash=ref.removeprefix("sha256:"), + archive_path=org + "/" + name, + ) + + async def bounded_download(remote_path, destination, byte_limit): + with tarfile.open(destination, "w:gz") as archive: + for name, data in client.payloads.items(): + member = tarfile.TarInfo(name) + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + for member, data in client.extra_members: + archive.addfile(member, io.BytesIO(data)) + return destination.stat().st_size + + client = SimpleNamespace( + payloads=payloads, + extra_members=[], + download_tasks=AsyncMock( + side_effect=AssertionError("Unbounded task download forbidden") + ), + resolve_task_version=AsyncMock(side_effect=resolve), + get_task_version_files=AsyncMock(side_effect=lambda *args: rows()), + get_dataset_version_files=AsyncMock(return_value=[]), + download=AsyncMock(side_effect=bounded_download), + ) + monkeypatch.setattr( + sources, "_download_harbor_file", client.download, raising=False + ) monkeypatch.setattr( sources, "_harbor_api", - lambda: ( - Mock(return_value=registry), - PackageTaskId, - Mock(return_value=task_client), - ), + lambda: (Mock(return_value=registry), PackageTaskId, Mock(return_value=client)), ) - return registry, task_client + return registry, client def test_harbor_discovery_and_bounded_digest_acquisition(monkeypatch, tmp_path): @@ -218,8 +269,9 @@ def test_harbor_discovery_and_bounded_digest_acquisition(monkeypatch, tmp_path): assert candidate["revision"] == DIGEST [record] = sources.acquire(candidate, tmp_path, 1) registry.get_dataset_metadata.assert_awaited_once_with(f"org/dataset@{DIGEST}") - assert len(client.download_tasks.call_args.kwargs["task_ids"]) == 1 - assert client.download_tasks.call_args.kwargs["export"] is False + client.resolve_task_version.assert_awaited_once_with("a", "same", TASK_DIGEST) + client.download.assert_awaited_once() + client.download_tasks.assert_not_awaited() assert record["revision"] == TASK_DIGEST assert record["task_id"] == f"harbor:a/same@{TASK_DIGEST}:." assert record["metadata"]["candidate_id"] == "org/dataset" @@ -232,19 +284,9 @@ def test_harbor_nested_fixture_cannot_replace_package_root( monkeypatch, tmp_path, missing_root ): _, client = fake_harbor(monkeypatch) - download = client.download_tasks.side_effect - - async def with_nested_fixture(**kwargs): - result = await download(**kwargs) - root = result.results[0].path - nested = root / "environment/fixture" - nested.mkdir() - (nested / "task.toml").write_text("fixture") - if missing_root: - (root / "task.toml").unlink() - return result - - client.download_tasks.side_effect = with_nested_fixture + client.payloads["environment/fixture/task.toml"] = b"fixture" + if missing_root: + del client.payloads["task.toml"] [candidate] = sources.discover_harbor(1) if missing_root: with pytest.raises(ValueError, match="no root task.toml"): @@ -371,10 +413,11 @@ def test_harbor_native_cli_json_and_model_contract(monkeypatch, capsys): def test_harbor_native_acquisition_models(monkeypatch, tmp_path): pytest.importorskip("harbor") + from harbor.db.client import RegistryDB, ResolvedTaskVersion from harbor.models.registry import DatasetMetadata from harbor.models.task.id import PackageTaskId as NativeTaskId from harbor.registry.client.package import PackageDatasetClient - from harbor.tasks.client import BatchDownloadResult, TaskClient, TaskDownloadResult + from harbor.tasks.client import TaskClient task = NativeTaskId(org="a", name="same", ref=TASK_DIGEST) bundle = DatasetMetadata( @@ -383,29 +426,192 @@ def test_harbor_native_acquisition_models(monkeypatch, tmp_path): monkeypatch.setattr( PackageDatasetClient, "get_dataset_metadata", AsyncMock(return_value=bundle) ) - - async def download(self, *, task_ids, output_dir, overwrite, export): - assert task_ids == [task] and overwrite and not export - bare_digest = task.ref.removeprefix("sha256:") - path = output_dir / task.org / task.name / bare_digest - path.mkdir(parents=True) - (path / "task.toml").write_text("") - (path / "instruction.md").write_text("native task") - return BatchDownloadResult( - results=[ - TaskDownloadResult( - path=path, - download_time_sec=0, - cached=False, - content_hash=bare_digest, + monkeypatch.setattr( + RegistryDB, + "resolve_task_version", + AsyncMock( + return_value=ResolvedTaskVersion( + id="version-id", + archive_path="archive", + content_hash=TASK_DIGEST.removeprefix("sha256:"), + ) + ), + ) + payloads = {"task.toml": b"", "instruction.md": b"native task"} + monkeypatch.setattr( + RegistryDB, + "get_task_version_files", + AsyncMock( + return_value=[ + dict( + path=name, + size_bytes=len(data), + content_hash=hashlib.sha256(data).hexdigest(), + storage_path=None, ) - ], - total_time_sec=0, - ) - - monkeypatch.setattr(TaskClient, "download_tasks", download) + for name, data in payloads.items() + ] + ), + ) + unbounded = AsyncMock( + side_effect=AssertionError("Unbounded archive downloader forbidden") + ) + monkeypatch.setattr(TaskClient, "download_tasks", unbounded) + + async def download(remote_path, destination, byte_limit): + assert remote_path == "archive" and byte_limit == real_tasks.MAX_TOTAL_BYTES + with tarfile.open(destination, "w:gz") as archive: + for name, data in payloads.items(): + member = tarfile.TarInfo(name) + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + return destination.stat().st_size + + monkeypatch.setattr(sources, "_download_harbor_file", download) [record] = sources.acquire( {"source": "harbor", "id": "org/dataset", "revision": DIGEST}, tmp_path, 1 ) assert record["revision"] == TASK_DIGEST assert (Path(record["path"]) / "instruction.md").read_text() == "native task" + assert record["task_file_hashes"] == { + name: hashlib.sha256(data).hexdigest() for name, data in payloads.items() + } + unbounded.assert_not_awaited() + + +@pytest.mark.parametrize( + "case", ["file_bytes", "total_bytes", "file_count", "unknown_size"] +) +def test_hf_acquisition_caps_shared_assets_before_download(monkeypatch, tmp_path, case): + files = {"task/task.toml": "", "shared/large": "x"} + if case == "file_count": + files.update({f"shared/{index}": "" for index in range(real_tasks.MAX_FILES)}) + if case == "total_bytes": + files.update({f"shared/{index}": "" for index in range(5)}) + api, _, download = fake_hf(monkeypatch, files) + if case == "file_bytes": + api.list_repo_tree.return_value[1].size = 1024**3 + elif case == "total_bytes": + for item in api.list_repo_tree.return_value[1:]: + item.size = real_tasks.MAX_FILE_BYTES + elif case == "unknown_size": + api.list_repo_tree.return_value[1].size = None + with pytest.raises(ValueError, match="cap"): + sources.acquire(hf_candidate(), tmp_path, 1) + download.assert_not_called() + + +def test_hf_acquisition_requires_pinned_blob_bytes(monkeypatch, tmp_path): + api, _, download = fake_hf(monkeypatch, {"task/task.toml": "original"}) + api.list_repo_tree.return_value[0].blob_id = "0" * 40 + with pytest.raises(ValueError, match="blob"): + sources.acquire(hf_candidate(), tmp_path, 1) + download.assert_called_once() + + +@pytest.mark.parametrize( + "case", ["file_bytes", "total_bytes", "file_count", "unknown_size"] +) +@pytest.mark.parametrize("scope", ["task", "shared"]) +def test_harbor_caps_inventory_before_download(monkeypatch, tmp_path, case, scope): + registry, client = fake_harbor(monkeypatch) + rows = [dict(path="task.toml", size_bytes=0, content_hash="a" * 64)] + if case == "file_bytes": + rows[0]["size_bytes"] = 1024**3 + elif case == "total_bytes": + rows = [ + dict(rows[0], path=f"file-{index}", size_bytes=real_tasks.MAX_FILE_BYTES) + for index in range(5) + ] + elif case == "file_count": + rows = [ + dict(rows[0], path=f"file-{index}") + for index in range(real_tasks.MAX_FILES + 1) + ] + else: + rows[0]["size_bytes"] = None + if scope == "task": + client.get_task_version_files.side_effect = None + client.get_task_version_files.return_value = rows + else: + registry.get_dataset_metadata.return_value.files = [ + SimpleNamespace(path="shared") + ] + registry.get_dataset_metadata.return_value.dataset_version_id = "dataset-id" + client.get_dataset_version_files.return_value = rows + [candidate] = sources.discover_harbor(1) + with pytest.raises(ValueError, match="cap"): + sources.acquire(candidate, tmp_path, 1) + client.download.assert_not_awaited() + client.download_tasks.assert_not_awaited() + registry.download_dataset_files.assert_not_awaited() + + +@pytest.mark.parametrize( + "case", ["extra", "link", "oversized", "hash", "missing", "expanded"] +) +def test_harbor_archive_must_match_bounded_inventory(monkeypatch, tmp_path, case): + _, client = fake_harbor(monkeypatch) + if case == "expanded": + member = tarfile.TarInfo("padding") + member.size = real_tasks.MAX_TOTAL_BYTES + 1 + client.extra_members = [(member, b"x" * member.size)] + elif case in {"extra", "link"}: + member = tarfile.TarInfo("unexpected") + if case == "link": + member.type = tarfile.SYMTYPE + member.linkname = "/tmp/outside" + client.extra_members = [(member, b"")] + else: + original = client.get_task_version_files.side_effect + rows = original("version") + if case == "oversized": + client.payloads["instruction.md"] = b"x" * (real_tasks.MAX_FILE_BYTES + 1) + elif case == "hash": + client.payloads["instruction.md"] = b"evil" + else: + del client.payloads["instruction.md"] + client.get_task_version_files.side_effect = None + client.get_task_version_files.return_value = rows + [candidate] = sources.discover_harbor(1) + with pytest.raises(ValueError, match="Harbor archive"): + sources.acquire(candidate, tmp_path, 1) + assert not list(tmp_path.rglob("instruction.md")) + + +@pytest.mark.parametrize("declared", [True, False]) +def test_harbor_download_caps_headers_and_stream(monkeypatch, tmp_path, declared): + import httpx + from harbor.auth import client as auth + + storage = SimpleNamespace( + create_signed_url=AsyncMock( + return_value={"signedURL": "https://example.invalid/archive"} + ) + ) + hub = SimpleNamespace(storage=SimpleNamespace(from_=lambda bucket: storage)) + monkeypatch.setattr( + auth, "create_authenticated_client", AsyncMock(return_value=hub) + ) + response = MagicMock() + response.__aenter__.return_value = response + response.headers = {"content-length": "11"} if declared else {} + chunks = [] + + async def stream(**kwargs): + for data in [b"123456", b"78901"]: + chunks.append(data) + yield data + + response.aiter_bytes = stream + http = MagicMock() + http.__aenter__.return_value = http + http.stream.return_value = response + monkeypatch.setattr(httpx, "AsyncClient", lambda **kwargs: http) + target = tmp_path / "archive" + with pytest.raises(ValueError, match="byte cap"): + asyncio.run(sources._download_harbor_file("archive-path", target, 10)) + if declared: + assert chunks == [] and not target.exists() + else: + assert target.read_bytes() == b"123456" From f49321808d9d793f500fc81ac5302e28ed467732 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 20:35:15 +0100 Subject: [PATCH 10/19] fix(coding-audit): budget target prompts and permit bounded tool repair --- examples/coding_audit/README.md | 23 ++ examples/coding_audit/hf.py | 255 ++++++++++++++---- examples/coding_audit/test_hf.py | 6 +- examples/coding_audit/test_target_budget.py | 213 +++++++++++++++ examples/coding_audit/test_target_tools.py | 277 ++++++++++++++++++++ 5 files changed, 717 insertions(+), 57 deletions(-) create mode 100644 examples/coding_audit/test_target_budget.py create mode 100644 examples/coding_audit/test_target_tools.py diff --git a/examples/coding_audit/README.md b/examples/coding_audit/README.md index b8b8a5ca0..345e6e5ba 100644 --- a/examples/coding_audit/README.md +++ b/examples/coding_audit/README.md @@ -54,6 +54,29 @@ and `size`, bound to the frozen task and checksummed evidence. This option does not enumerate task directories or include verifier, reference or specification files. + +Target requests reserve their prompt as well as completion allowance before dispatch. +The frozen policy uses encoded controller-request bytes plus an 8,192-token framing +allowance. This conservative scheduling assumption can stop early; it is not a +serving-tokenizer measurement or a guarantee about remote billing. Evidence records +the reservation, actual per-response usage, request hashes and dispatch decisions. +Reported input above the reservation, output above the request cap, inconsistent +usage, or cumulative usage above the configured total remains an error. + +After a valid interaction prefix, a pre-dispatch budget stop collects only the +allowlisted files actually saved, then uses the normal fresh verifier. A funded, +validated API response ending in `length` is an explicit truncated target attempt: +its partial calls are discarded and existing files are graded. An initial prompt +that cannot fit remains unmeasured. Generator responses still require complete +`stop` output. No failed API, cleanup or grading operation becomes a synthetic +zero reward. + +A bindable target tool request with an unknown function name or invalid JSON +argument content gets bounded tool-error feedback and can be repaired within the +same call and token limits. It never executes an unknown method. Malformed call +identities/types, duplicate IDs in one response and backend failures remain errors; +ambiguous batches are rejected before any command executes. + The HF backend launches paid services when invoked without a supplied test backend. Use reviewed tasks and distinct execution and inference credentials. The Hub pipeline's cloud worker packages these production example modules with diff --git a/examples/coding_audit/hf.py b/examples/coding_audit/hf.py index f615bd382..bf0c1d971 100644 --- a/examples/coding_audit/hf.py +++ b/examples/coding_audit/hf.py @@ -28,11 +28,13 @@ _IMAGE = re.compile(r"[^\s@]+@sha256:[0-9a-f]{64}\Z") _REVISION = re.compile(r"[0-9a-f]{40}\Z") _TERMINAL = {"COMPLETED", "CANCELED", "ERROR", "DELETED"} +_PROMPT_FRAMING_ALLOWANCE = 8192 _LIMITATIONS = [ "Immutable images are control-plane references, not measured runtime digests.", "Sandbox RPC exposes the execution bearer to the workload server; hostile-code credential isolation is not qualified.", "The verifier executes submitted code as the same user; verifier integrity against hostile submissions is not qualified.", "Inference Providers do not pin served weights to the recorded model source revision.", + "Target prompt reservation uses a byte/framing assumption, not a serving-tokenizer or billing guarantee.", ] _TOOL = { "type": "function", @@ -57,6 +59,13 @@ class _DeadlineExceeded(HFBackendError): pass +class _TargetBudgetStop(HFBackendError): + def __init__(self, reason, *, response_received=False): + super().__init__(reason) + self.reason = reason + self.response_received = response_received + + @contextmanager def _deadline(seconds): # SDK control-plane calls have no timeout argument. A controller must run on @@ -174,6 +183,25 @@ def _plain(value): return json.loads(json.dumps(value)) +def _retain_usage(receipt, usage): + try: + encoded = json.dumps(usage, allow_nan=False).encode() + if len(encoded) > 65536: + raise ValueError("oversized usage") + except (TypeError, ValueError): + receipt["usage_receipt_error"] = ( + "nonfinite, non-JSON or oversized usage details" + ) + if isinstance(usage, dict): + receipt["usage_counts"] = { + key: usage[key] + for key in ("prompt_tokens", "completion_tokens", "total_tokens") + if type(usage.get(key)) is int and usage[key] >= 0 + } + else: + receipt["usage"] = usage + + def _unique_pairs(pairs): result = {} for key, value in pairs: @@ -658,13 +686,15 @@ def _model(self, model, seed, *, tools=False): def _chat(self, client, model, allowed, limits, messages, record, seed, *, tools): used = record["input_tokens"] + record["output_tokens"] - # Bound the request body as well as provider-reported usage. Providers do - # not expose a preflight tokenizer through this API. message_bytes = len(json.dumps(messages).encode()) if message_bytes > limits["max_total_tokens"] * 4: + if tools: + raise _TargetBudgetStop("context_byte_limit") raise HFBackendError("chat context exceeds byte budget") remaining = limits["max_total_tokens"] - used if remaining <= 0: + if tools: + raise _TargetBudgetStop("total_token_limit") raise HFBackendError("model total token budget exhausted") request = { "messages": self._redact(messages), @@ -688,32 +718,65 @@ def _chat(self, client, model, allowed, limits, messages, record, seed, *, tools } if model.get("supports_seed", False): request["seed"] = seed + metadata = record["model_metadata"] + preflight = None if tools: request.update(tools=[_TOOL], tool_choice="auto") - if not tools: + encoded = json.dumps(request, allow_nan=False).encode() + reserved_input = len(encoded) + _PROMPT_FRAMING_ALLOWANCE + output_allowance = remaining - reserved_input + preflight = { + "method": "request_utf8_bytes_plus_framing_v1", + "reservation_bytes": len(encoded), + "request_bytes": len(encoded), + "request_sha256": hashlib.sha256(encoded).hexdigest(), + "framing_allowance": _PROMPT_FRAMING_ALLOWANCE, + "input_token_reservation": reserved_input, + "actual_tokens_before": used, + "remaining_tokens": remaining, + "output_token_cap": max(0, min(limits["max_tokens"], output_allowance)), + "dispatched": False, + } + metadata.setdefault("preflights", []).append(preflight) + if output_allowance <= 0: + raise _TargetBudgetStop("input_reservation_limit") + request["max_tokens"] = preflight["output_token_cap"] + encoded = json.dumps(request, allow_nan=False).encode() + preflight["request_bytes"] = len(encoded) + preflight["request_sha256"] = hashlib.sha256(encoded).hexdigest() + else: request.update(stream=True, stream_options={"include_usage": True}) - metadata = record["model_metadata"] metadata.setdefault("requests", []).append( {key: value for key, value in request.items() if key != "messages"} ) if tools: - response = _plain(client.chat_completion(**request)) - metadata.setdefault("responses", []).append( - { - "model": response.get("model"), - "system_fingerprint": response.get("system_fingerprint"), - "id": response.get("id"), - "seed_sent": request.get("seed"), - } - ) + preflight["dispatched"] = True + parsed = client.chat_completion(**request) + response = _plain(parsed) + # Access the typed nested value: outer SDK dicts can omit extras. + response["usage"] = _plain(parsed.usage) + choices = response.get("choices") + receipt = { + "model": response.get("model"), + "system_fingerprint": response.get("system_fingerprint"), + "id": response.get("id"), + "seed_sent": request.get("seed"), + "finish_reason": choices[0].get("finish_reason") + if isinstance(choices, list) + and len(choices) == 1 + and isinstance(choices[0], dict) + else None, + "response_validated": False, + "discarded": True, + "tool_calls_processed": 0, + } + metadata.setdefault("responses", []).append(receipt) + _retain_usage(receipt, response["usage"]) else: response = self._candidate_response( client, request, allowed, limits, record ) - if response.get("model") not in allowed: - raise HFBackendError( - "provider response model does not match configured exact identities" - ) + receipt = metadata["responses"][-1] usage = response.get("usage") if not isinstance(usage, dict): raise HFBackendError("provider response omitted token usage") @@ -728,24 +791,57 @@ def _chat(self, client, model, allowed, limits, messages, record, seed, *, tools record[target] += value record["usage"][target] += value if ( - usage["completion_tokens"] > request["max_tokens"] - or record["input_tokens"] + record["output_tokens"] + type(usage.get("total_tokens")) is not int + or usage["total_tokens"] < 0 + or usage["total_tokens"] + != usage["prompt_tokens"] + usage["completion_tokens"] + ): + raise HFBackendError("provider returned inconsistent token usage") + if receipt.get("usage_receipt_error"): + raise HFBackendError("provider returned invalid usage details") + violations = [] + if usage["completion_tokens"] > request["max_tokens"]: + violations.append("response_output_cap") + if ( + record["input_tokens"] + record["output_tokens"] > limits["max_total_tokens"] ): - raise HFBackendError("provider exceeded token budget") + violations.append("cumulative_total_cap") + if tools and usage["prompt_tokens"] > preflight["input_token_reservation"]: + violations.append("input_reservation_contract") + if violations: + receipt["budget_violations"] = violations + raise HFBackendError( + "provider exceeded token budget: " + ", ".join(violations) + ) + if response.get("model") not in allowed: + raise HFBackendError( + "provider response model does not match configured exact identities" + ) choices = response.get("choices", []) - if len(choices) != 1 or not isinstance(choices[0].get("message"), dict): + if ( + not isinstance(choices, list) + or len(choices) != 1 + or not isinstance(choices[0].get("message"), dict) + ): raise HFBackendError("provider must return exactly one assistant message") message = choices[0]["message"] + if message.get("role") != "assistant": + raise HFBackendError("provider must return an assistant message") record["transcript"].append(message) if len(json.dumps(message).encode()) > limits["max_output_bytes"]: raise HFBackendError("model output exceeded byte limit") - if choices[0].get("finish_reason") not in {"stop", "tool_calls"}: + finish = choices[0].get("finish_reason") + if finish == "length" and tools: + receipt["response_validated"] = True + raise _TargetBudgetStop("response_output_limit", response_received=True) + if finish not in {"stop", "tool_calls"}: raise HFBackendError("model response was incomplete") - if choices[0]["finish_reason"] == "tool_calls" and ( + if finish == "tool_calls" and ( not isinstance(message.get("tool_calls"), list) or not message["tool_calls"] ): raise HFBackendError("invalid terminal function calls") + receipt["response_validated"] = True return message def _candidate_response(self, client, request, allowed, limits, record): @@ -859,6 +955,10 @@ def _candidate_response(self, client, request, allowed, limits, record): ) event_usage = event.get("usage") if event_usage is not None: + event_usage = _plain(parsed.usage) + _retain_usage( + record["model_metadata"]["responses"][-1], event_usage + ) if not stopped: raise HFBackendError("provider returned usage before stop") if not isinstance(event_usage, dict) or any( @@ -1042,20 +1142,34 @@ def solve(self, model: dict, seed: int, role="target") -> dict: record["transcript"] = messages sandbox = self._create(self.execution["agent_image"], record) self._audit_solver(sandbox) - calls_used = 0 + calls_used = validated_responses = 0 + record["model_metadata"]["completed_response_turns"] = 0 for _ in range(limits["max_steps"]): - message = self._chat( - client, - model, - allowed, - limits, - messages, - record, - seed, - tools=True, - ) + try: + message = self._chat( + client, + model, + allowed, + limits, + messages, + record, + seed, + tools=True, + ) + except _TargetBudgetStop as stop: + if not validated_responses and not stop.response_received: + raise HFBackendError( + "initial target prompt cannot fit token budget" + ) from None + record.update(truncated=True, termination_reason=stop.reason) + break calls = message.get("tool_calls") or [] if not calls: + record["model_metadata"]["responses"][-1]["discarded"] = False + validated_responses += 1 + record["model_metadata"]["completed_response_turns"] = ( + validated_responses + ) break if not isinstance(calls, list): raise HFBackendError("invalid terminal function calls") @@ -1065,42 +1179,73 @@ def solve(self, model: dict, seed: int, role="target") -> dict: ) break ids = set() + # Reject ambiguous identities before executing any call in a batch. for call in calls: if ( - call.get("type") != "function" - or call.get("function", {}).get("name") != "terminal" + not isinstance(call, dict) + or call.get("type") != "function" or not isinstance(call.get("id"), str) or not call["id"] or call["id"] in ids + or not isinstance(call.get("function"), dict) + or not isinstance(call["function"].get("name"), str) + or not isinstance(call["function"].get("arguments"), str) ): raise HFBackendError("invalid terminal function call") ids.add(call["id"]) - arguments = json.loads( - call["function"]["arguments"], - object_pairs_hook=_unique_pairs, - ) - if ( - not isinstance(arguments, dict) - or set(arguments) != {"command"} - or not isinstance(arguments["command"], str) - or len(arguments["command"].encode()) > self.max_script - ): - raise HFBackendError("invalid terminal function arguments") + for call in calls: calls_used += 1 - exit_code, output = self._run( - sandbox, - ["/bin/bash", "-c", arguments["command"]], - started + self.timeout, - ) + problem = None + if call["function"]["name"] != "terminal": + problem = "Only the terminal function is available; use terminal(command: string)." + else: + try: + arguments = json.loads( + call["function"]["arguments"], + object_pairs_hook=_unique_pairs, + ) + if ( + not isinstance(arguments, dict) + or set(arguments) != {"command"} + or not isinstance(arguments["command"], str) + or len(arguments["command"].encode()) + > self.max_script + ): + problem = "terminal requires exactly one bounded string argument named command." + except (ValueError, TypeError): + problem = "terminal arguments must be a unique JSON object with one command string." + if problem: + content = json.dumps({"error": problem}) + if len(content.encode()) > self.max_output: + raise HFBackendError( + "tool error feedback exceeds byte limit" + ) + record["model_metadata"]["tool_error_count"] = ( + record["model_metadata"].get("tool_error_count", 0) + 1 + ) + else: + exit_code, output = self._run( + sandbox, + ["/bin/bash", "-c", arguments["command"]], + started + self.timeout, + ) + content = json.dumps( + {"exit_code": exit_code, "output": output} + ) messages.append( { "role": "tool", "tool_call_id": call["id"], - "content": json.dumps( - {"exit_code": exit_code, "output": output} - ), + "content": content, } ) + receipt = record["model_metadata"]["responses"][-1] + receipt["tool_calls_processed"] += 1 + receipt["discarded"] = False + validated_responses += 1 + record["model_metadata"]["completed_response_turns"] = ( + validated_responses + ) else: record.update(truncated=True, termination_reason="step_limit") record["files"] = self._collect(sandbox) diff --git a/examples/coding_audit/test_hf.py b/examples/coding_audit/test_hf.py index 2acf9c0db..95484e097 100644 --- a/examples/coding_audit/test_hf.py +++ b/examples/coding_audit/test_hf.py @@ -1368,9 +1368,11 @@ def test_invalid_tool_name_is_not_executed(harness): h = harness call = terminal() call["function"]["name"] = "provider_request" - h.inference.responses = [response(None, calls=[call])] + h.inference.responses = [response(None, calls=[call]), response("cannot proceed")] record = h.backend.solve(MODEL, 0) - assert "invalid terminal" in record["error"] and not h.factory.created[0].commands + assert record["error"] is None and not h.factory.created[0].commands + assert record["model_metadata"]["tool_error_count"] == 1 + assert json.loads(h.inference.requests[1]["messages"][-1]["content"])["error"] def test_output_and_token_limits_are_fail_closed(harness): diff --git a/examples/coding_audit/test_target_budget.py b/examples/coding_audit/test_target_budget.py new file mode 100644 index 000000000..a383a75d2 --- /dev/null +++ b/examples/coding_audit/test_target_budget.py @@ -0,0 +1,213 @@ +"""Budget stops preserve valid prefixes; provider violations remain errors.""" + +import hashlib +import json + +import pytest +from examples.coding_audit.hf import HFBackendError +from examples.coding_audit.test_hf import harness as harness, MODEL, response, terminal + + +def model(**values): + return { + **MODEL, + "max_steps": 48, + "max_tokens": 4096, + "max_total_tokens": 32768, + **values, + } + + +def test_next_prompt_is_reserved_before_dispatch_and_saved_files_are_graded(harness): + h = harness + (h.root / "instruction.md").write_text("public requirement " * 700) + h.inference.responses = [ + response( + None, + calls=[terminal(identifier="first")], + usage={ + "prompt_tokens": 15000, + "completion_tokens": 4000, + "total_tokens": 19000, + }, + ), + response( + None, + calls=[terminal(identifier="second")], + usage={ + "prompt_tokens": 15000, + "completion_tokens": 4000, + "total_tokens": 19000, + }, + ), + ] + solved = h.backend.solve(model(), 0) + assert solved["error"] is None and solved["truncated"] + assert solved["termination_reason"] == "input_reservation_limit" + assert len(h.inference.requests) == 1 and len(h.factory.created[0].commands) == 1 + assert solved["usage"] == {"input_tokens": 15000, "output_tokens": 4000} + assert solved["input_tokens"] + solved["output_tokens"] <= 32768 + preflights = solved["model_metadata"]["preflights"] + assert preflights[0]["dispatched"] and not preflights[1]["dispatched"] + assert preflights[1]["output_token_cap"] == 0 + payload = json.dumps(h.inference.requests[0], allow_nan=False).encode() + assert preflights[0]["request_sha256"] == hashlib.sha256(payload).hexdigest() + assert preflights[0]["request_bytes"] == len(payload) + assert solved["files"] == {"/app/answer.txt": b"answer"} + grade = h.backend.grade(solved) + assert grade["error"] is None and grade["cleanup_verified"] + assert grade["grade_sandbox_id"] != solved["sandbox_id"] + + +def test_insufficient_first_prompt_has_no_model_measurement_or_grade(harness): + h = harness + solved = h.backend.solve(model(max_total_tokens=8192), 0) + assert "initial target prompt" in solved["error"] + assert not h.inference.requests and not h.factory.created[0].commands + assert solved["cleanup_verified"] and solved["files"] == {} + assert len(h.factory.created) == 1 and not solved["truncated"] + + +@pytest.mark.parametrize("failure", ["output", "total", "input_estimate"]) +def test_actual_overruns_are_errors_with_billed_usage_not_truncation(harness, failure): + h = harness + prompt, completion = { + "output": (10, 4097), + "total": (33000, 5), + "input_estimate": (12000, 5), + }[failure] + h.inference.responses = [ + response( + None, + calls=[terminal()], + usage={ + "prompt_tokens": prompt, + "completion_tokens": completion, + "total_tokens": prompt + completion, + }, + ) + ] + solved = h.backend.solve(model(), 0) + assert "provider exceeded token budget" in solved["error"] + assert not solved["truncated"] and not h.factory.created[0].commands + assert solved["usage"] == {"input_tokens": prompt, "output_tokens": completion} + receipt = solved["model_metadata"]["responses"][0] + assert receipt["usage"]["total_tokens"] == prompt + completion + expected = { + "output": "response_output_cap", + "total": "cumulative_total_cap", + "input_estimate": "input_reservation_contract", + }[failure] + assert expected in receipt["budget_violations"] and receipt["discarded"] + assert len(h.factory.created) == 1 and solved["cleanup_verified"] + + +def test_nested_reasoning_usage_is_retained_without_double_charging(harness): + h = harness + usage = { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "completion_tokens_details": {"reasoning_tokens": 4}, + } + h.inference.responses = [response("saved", usage=usage)] + solved = h.backend.solve(model(), 0) + assert solved["error"] is None + assert solved["usage"] == {"input_tokens": 10, "output_tokens": 5} + receipt = solved["model_metadata"]["responses"][0] + assert receipt["usage"]["completion_tokens_details"]["reasoning_tokens"] == 4 + assert receipt["response_validated"] and not receipt["discarded"] + + +@pytest.mark.parametrize( + "problem", ["missing_total", "bad_total", "nonfinite_detail", "oversized_detail"] +) +def test_invalid_usage_is_never_a_normal_budget_stop(harness, problem): + h = harness + usage = {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15} + if problem == "missing_total": + usage.pop("total_tokens") + elif problem == "bad_total": + usage["total_tokens"] = 999 + elif problem == "nonfinite_detail": + usage["detail"] = float("nan") + else: + usage["detail"] = "x" * 65537 + h.inference.responses = [response(None, calls=[terminal()], usage=usage)] + solved = h.backend.solve(model(), 0) + assert solved["error"] and not solved["truncated"] + assert not h.factory.created[0].commands and solved["cleanup_verified"] + assert len(h.factory.created) == 1 + json.dumps(solved, allow_nan=False) + + +@pytest.mark.parametrize("prefix", [False, True]) +def test_funded_length_discards_partial_calls_and_grades_existing_state( + harness, prefix +): + h = harness + h.factory.submitted = None if not prefix else b"answer" + h.factory.reward = b"0" if not prefix else b"1" + partial = terminal("synthetic-must-not-execute", "partial") + partial["function"]["arguments"] = '{"command":"unterminated' + ended = response(None, calls=[partial]) + ended["choices"][0]["finish_reason"] = "length" + h.inference.responses = ( + [response(None, calls=[terminal(identifier="saved")])] if prefix else [] + ) + [ended] + solved = h.backend.solve(model(), 0) + assert solved["error"] is None and solved["truncated"] + assert solved["termination_reason"] == "response_output_limit" + assert len(h.factory.created[0].commands) == int(prefix) + assert solved["files"] == ({"/app/answer.txt": b"answer"} if prefix else {}) + assert solved["model_metadata"]["responses"][-1]["finish_reason"] == "length" + assert solved["model_metadata"]["responses"][-1]["discarded"] + assert "reward" not in solved + grade = h.backend.grade(solved) + assert grade["error"] is None and grade["reward"] == int(prefix) + assert ( + grade["grade_sandbox_id"] != solved["sandbox_id"] and grade["cleanup_verified"] + ) + + +@pytest.mark.parametrize("violation", ["model", "role", "usage", "bytes"]) +def test_length_does_not_mask_invalid_response(harness, violation): + h = harness + ended = response(None, calls=[terminal()]) + ended["choices"][0]["finish_reason"] = "length" + if violation == "model": + ended["model"] = "wrong/model" + elif violation == "role": + ended["choices"][0]["message"]["role"] = "user" + elif violation == "usage": + ended = response( + None, + calls=[terminal()], + usage={ + "prompt_tokens": 10, + "completion_tokens": 4097, + "total_tokens": 4107, + }, + ) + ended["choices"][0]["finish_reason"] = "length" + else: + ended["choices"][0]["message"]["content"] = "x" * 16385 + h.inference.responses = [ended] + solved = h.backend.solve(model(), 0) + assert solved["error"] and not solved["truncated"] + assert not h.factory.created[0].commands and len(h.factory.created) == 1 + assert solved["cleanup_verified"] + + +def test_provider_exception_with_old_budget_message_is_not_recovered(harness): + h = harness + + def failure(**kwargs): + raise HFBackendError("provider exceeded token budget") + + h.inference.chat_completion = failure + solved = h.backend.solve(model(), 0) + assert ( + solved["error"] == "provider exceeded token budget" and not solved["truncated"] + ) + assert solved["cleanup_verified"] and len(h.factory.created) == 1 diff --git a/examples/coding_audit/test_target_tools.py b/examples/coding_audit/test_target_tools.py new file mode 100644 index 000000000..95d06bf56 --- /dev/null +++ b/examples/coding_audit/test_target_tools.py @@ -0,0 +1,277 @@ +"""Target tool repair at the real solve, provider-message, and grading seams.""" + +import json +import time + +import pytest +from examples.coding_audit.test_hf import ( + FakeSandbox, + harness as harness, + MODEL, + response, + terminal, +) + +REJECTED_COMMAND = "synthetic-command-that-must-not-run" +REPAIR_COMMAND = "printf answer > /app/answer.txt" + + +def rejected_call(kind, identifier="rejected-call"): + call = terminal(REJECTED_COMMAND, identifier) + if kind in {"unknown-empty", "unknown-command", "provider", "reset"}: + call["function"]["name"] = { + "provider": "provider_request", + "reset": "reset", + }.get(kind, "command") + if kind in {"unknown-empty", "empty"}: + call["function"]["arguments"] = "{}" + elif kind == "malformed-json": + call["function"]["arguments"] = '{"command":"unterminated' + elif kind == "duplicate-json-key": + call["function"]["arguments"] = ( + '{"command":"synthetic-first","command":"synthetic-second"}' + ) + elif kind == "argument-object": + call["function"]["arguments"] = {"command": REJECTED_COMMAND} + elif kind == "command-number": + call["function"]["arguments"] = '{"command":7}' + elif kind == "argument-list": + call["function"]["arguments"] = "[]" + elif kind == "oversized-command": + call["function"]["arguments"] = json.dumps({"command": "x" * 129}) + return call + + +def assert_feedback(message, identifier, limit): + assert set(message) == {"role", "tool_call_id", "content"} + assert message["role"] == "tool" and message["tool_call_id"] == identifier + assert isinstance(message["content"], str) + assert len(message["content"].encode()) <= limit + payload = json.loads(message["content"]) + assert isinstance(payload, dict) + assert isinstance(payload.get("error"), str) and payload["error"] + + +def assert_target_wire(requests): + for request in requests: + assert "stream" not in request and request["tool_choice"] == "auto" + assert [tool["function"]["name"] for tool in request["tools"]] == ["terminal"] + assert "PRIVATE_SOLUTION_SENTINEL" not in repr(request) + assert "PRIVATE_CANARY_SENTINEL" not in repr(request) + + +@pytest.mark.parametrize( + "kind", + [ + "unknown-empty", + "unknown-command", + "empty", + "malformed-json", + "duplicate-json-key", + "command-number", + "argument-list", + "oversized-command", + "provider", + "reset", + ], +) +def test_bindable_rejection_reaches_provider_as_feedback_then_repairs_and_grades( + harness, kind +): + h = harness + h.backend.max_output = 256 + h.backend.max_script = 128 + bad = rejected_call(kind) + h.inference.responses = [ + response(None, calls=[bad]), + response(None, calls=[terminal(REPAIR_COMMAND, "repair-call")]), + response("saved"), + ] + original = h.inference.chat_completion + + def provider(**request): + if len(h.inference.requests) == 1: + assert not h.factory.created[0].commands + assert request["messages"][-2]["role"] == "assistant" + assert_feedback(request["messages"][-1], bad["id"], h.backend.max_output) + return original(**request) + + h.inference.chat_completion = provider + solved = h.backend.solve({**MODEL, "max_steps": 4}, 0) + assert solved["error"] is None and solved["cleanup_verified"] + assert not solved["truncated"] and len(h.inference.requests) == 3 + assert [argv for argv, _ in h.factory.created[0].commands] == [ + ["/bin/bash", "-c", REPAIR_COMMAND] + ] + assert solved["files"] == {"/app/answer.txt": b"answer"} + assert "reward" not in solved + assert_target_wire(h.inference.requests) + grade = h.backend.grade(solved) + assert grade["error"] is None and grade["reward"] == 1 + assert ( + grade["cleanup_verified"] and grade["grade_sandbox_id"] != solved["sandbox_id"] + ) + assert h.factory.created[1].files.data["/app/answer.txt"] == b"answer" + assert all(stage == "CANCELED" for stage in h.api.stages.values()) + + +def test_all_invalid_single_step_is_bounded_and_grades_actual_empty_submission(harness): + h = harness + h.factory.submitted = None + h.factory.reward = b"0" + h.inference.responses = [response(None, calls=[rejected_call("unknown-empty")])] + solved = h.backend.solve({**MODEL, "max_steps": 1}, 0) + assert solved["error"] is None and solved["cleanup_verified"] + assert solved["truncated"] and solved["termination_reason"] == "step_limit" + assert len(h.inference.requests) == 1 and not h.factory.created[0].commands + assert_feedback(solved["transcript"][-1], "rejected-call", h.backend.max_output) + assert solved["files"] == {} and "reward" not in solved + grade = h.backend.grade(solved) + assert ( + grade["error"] is None and grade["reward"] == 0 and grade["rawreward"] == b"0" + ) + assert ( + grade["cleanup_verified"] and grade["grade_sandbox_id"] != solved["sandbox_id"] + ) + assert "/app/answer.txt" not in h.factory.created[1].files.data + + +def test_rejected_calls_count_toward_multi_call_budget_before_any_execution(harness): + h = harness + h.factory.submitted = None + h.inference.responses = [ + response(None, calls=[rejected_call("empty", "first")]), + response( + None, calls=[rejected_call("empty", "second"), terminal(REPAIR_COMMAND)] + ), + ] + solved = h.backend.solve({**MODEL, "max_steps": 2}, 0) + assert solved["error"] is None and solved["cleanup_verified"] + assert solved["truncated"] and solved["termination_reason"] == "tool_call_limit" + assert len(h.inference.requests) == 2 and solved["files"] == {} + assert not h.factory.created[0].commands + assert_feedback( + h.inference.requests[1]["messages"][-1], "first", h.backend.max_output + ) + + +def test_rejection_at_call_cap_preserves_saved_files_for_normal_grading(harness): + h = harness + h.inference.responses = [ + response(None, calls=[terminal(REPAIR_COMMAND, "saved-call")]), + response(None, calls=[rejected_call("empty")]), + ] + solved = h.backend.solve({**MODEL, "max_steps": 2}, 0) + assert solved["error"] is None and solved["cleanup_verified"] + assert solved["truncated"] and solved["termination_reason"] == "step_limit" + assert len(h.inference.requests) == 2 + assert len(h.factory.created[0].commands) == 1 + assert solved["files"] == {"/app/answer.txt": b"answer"} and "reward" not in solved + assert_feedback(solved["transcript"][-1], "rejected-call", h.backend.max_output) + grade = h.backend.grade(solved) + assert grade["error"] is None and grade["reward"] == 1 and grade["cleanup_verified"] + assert grade["grade_sandbox_id"] != solved["sandbox_id"] + + +@pytest.mark.parametrize( + "malformation", + [ + "missing-id", + "empty-id", + "numeric-id", + "wrong-type", + "missing-type", + "duplicate-id", + "nonstring-name", + "argument-object", + ], +) +def test_unbindable_or_duplicate_identity_is_fatal_before_batch_execution( + harness, malformation +): + h = harness + first = terminal(REPAIR_COMMAND, "first") + second = terminal(REJECTED_COMMAND, "second") + if malformation == "missing-id": + second.pop("id") + elif malformation == "empty-id": + second["id"] = "" + elif malformation == "numeric-id": + second["id"] = 7 + elif malformation == "wrong-type": + second["type"] = "reset" + elif malformation == "missing-type": + second.pop("type") + elif malformation == "duplicate-id": + second["id"] = first["id"] + elif malformation == "argument-object": + second["function"]["arguments"] = {"command": REJECTED_COMMAND} + else: + second["function"]["name"] = None + h.inference.responses = [response(None, calls=[first, second])] + solved = h.backend.solve(MODEL, 0) + assert solved["error"] and solved["cleanup_verified"] + assert not h.factory.created[0].commands and len(h.inference.requests) == 1 + assert not any(message.get("role") == "tool" for message in solved["transcript"]) + assert "reward" not in solved + + +def test_tool_execution_platform_failure_after_feedback_remains_error( + harness, monkeypatch +): + h = harness + h.inference.responses = [ + response(None, calls=[rejected_call("empty")]), + response(None, calls=[terminal(REPAIR_COMMAND)]), + ] + + def failed_run(*args, **kwargs): + raise RuntimeError("synthetic platform failure") + + monkeypatch.setattr(FakeSandbox, "run", failed_run) + solved = h.backend.solve(MODEL, 0) + assert solved["error"] == "HF backend operation failed (RuntimeError)" + assert solved["cleanup_verified"] and len(h.inference.requests) == 2 + assert ( + len( + [ + message + for message in solved["transcript"] + if message.get("role") == "tool" + ] + ) + == 1 + ) + + +def test_provider_deadline_after_feedback_remains_error(harness): + h = harness + h.backend.timeout = 0.02 + h.inference.responses = [response(None, calls=[rejected_call("empty")])] + original = h.inference.chat_completion + + def blocked_provider(**request): + if h.inference.requests: + time.sleep(1) + return original(**request) + + h.inference.chat_completion = blocked_provider + started = time.monotonic() + solved = h.backend.solve(MODEL, 0) + assert time.monotonic() - started < 0.5 + assert "deadline" in solved["error"] and solved["cleanup_verified"] + assert not h.factory.created[0].commands + + +def test_cleanup_failure_after_tool_repair_remains_error(harness): + h = harness + h.api.ignore_cancel = True + h.inference.responses = [ + response(None, calls=[rejected_call("empty")]), + response(None, calls=[terminal(REPAIR_COMMAND)]), + response("saved"), + ] + solved = h.backend.solve(MODEL, 0) + assert solved["error"] == "sandbox cleanup could not be verified" + assert not solved["cleanup_verified"] and len(h.inference.requests) == 3 + assert len(h.factory.created[0].commands) == 1 and "reward" not in solved From 6f7df1b615b0ac9f6bc5c1d4d9353612f0c7a326 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 20:40:55 +0100 Subject: [PATCH 11/19] fix(hub-validation): honor recipe defaults and persist launch receipts --- examples/hub_validation_pipeline/cloud.py | 10 +++- examples/hub_validation_pipeline/full_run.py | 7 ++- .../hub_validation_pipeline/prepared_batch.py | 5 +- .../hub_validation_pipeline/real_tasks.py | 15 ++++-- .../hub_validation_pipeline/test_cloud.py | 34 ++++++++++++++ .../hub_validation_pipeline/test_full_run.py | 25 ++++++++++ .../test_prepared_batch.py | 23 ++++++++++ .../test_real_tasks.py | 46 +++++++++++++++++++ 8 files changed, 156 insertions(+), 9 deletions(-) diff --git a/examples/hub_validation_pipeline/cloud.py b/examples/hub_validation_pipeline/cloud.py index 0b97289e2..e24956118 100644 --- a/examples/hub_validation_pipeline/cloud.py +++ b/examples/hub_validation_pipeline/cloud.py @@ -769,8 +769,16 @@ def _private_repo(api, repo, token): def _write_receipt(path, value): temporary = path.with_suffix(".tmp") - temporary.write_bytes(_json_bytes(value)) + with temporary.open("wb") as stream: + stream.write(_json_bytes(value)) + stream.flush() + os.fsync(stream.fileno()) temporary.replace(path) + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) def stage_inputs(plan_path, *, api=None, secrets=None, expected_plan_sha256=None): diff --git a/examples/hub_validation_pipeline/full_run.py b/examples/hub_validation_pipeline/full_run.py index 1e34b9cdc..97684c48a 100644 --- a/examples/hub_validation_pipeline/full_run.py +++ b/examples/hub_validation_pipeline/full_run.py @@ -61,7 +61,10 @@ def estimate(configs): != RATEBOOK["child_lifetime_seconds"] ): raise ValueError("A different child lifetime requires a different ratebook") - if any(config[key]["flavor"] != "cpu-basic" for key in ("cloud", "execution")): + if any( + config[key].get("flavor", "cpu-basic") != "cpu-basic" + for key in ("cloud", "execution") + ): raise ValueError("A different flavor requires a different ratebook") for key in ("target", "generator"): model = config[key] @@ -260,7 +263,7 @@ def run( manifest["stage"] = "budget" manifest["estimate"] = estimate(configs) manifest["within_budget"] = ( - Fraction(manifest["estimate"]["reservation_usd_exact"]) <= cap + Fraction(manifest["estimate"]["reservation_usd_rounded_up"]) <= cap ) manifest["status"] = "prepared" _write(manifest_path, manifest) diff --git a/examples/hub_validation_pipeline/prepared_batch.py b/examples/hub_validation_pipeline/prepared_batch.py index 41e5f3c51..605b98b78 100644 --- a/examples/hub_validation_pipeline/prepared_batch.py +++ b/examples/hub_validation_pipeline/prepared_batch.py @@ -54,9 +54,12 @@ def _status(task, spec, owner, deadline, api, secrets, clock): or job.owner.name != owner ): raise ValueError("Controller identity mismatch") + stage = job.status.stage + if stage in {"COMPLETED", "ERROR", "CANCELED", "DELETED"}: + return stage if clock() >= deadline: return "TIMEOUT" - return job.status.stage if job.status.stage in STAGES else "UNKNOWN" + return stage if stage in STAGES else "UNKNOWN" def run( diff --git a/examples/hub_validation_pipeline/real_tasks.py b/examples/hub_validation_pipeline/real_tasks.py index 882114bfe..e23bd7f47 100644 --- a/examples/hub_validation_pipeline/real_tasks.py +++ b/examples/hub_validation_pipeline/real_tasks.py @@ -13,7 +13,7 @@ import stat import tempfile import tomllib -from pathlib import Path +from pathlib import Path, PurePosixPath from huggingface_hub import hf_hub_download, HfApi, RepoFile, RepoFolder from openenv.validation.runner import source_digest @@ -52,7 +52,11 @@ def _load_panel(panel_path): if not isinstance(task, dict): raise ValueError("Panel tasks must be objects") name = cloud._safe_name(task.get("name", "")) - task_path = cloud._safe_name(task.get("task_path", "")) + task_path = ( + PurePosixPath(".") + if task.get("task_path") == "." + else cloud._safe_name(task.get("task_path", "")) + ) if ( len(name.parts) != 1 or name.as_posix().casefold() in names @@ -101,7 +105,8 @@ def acquire_task(task, destination, *, source, api=None, download=None): """Acquire only one pinned native tree, with size and Git-blob checks.""" revision = sources._pin(source["revision"], "hf") repo_id, task_path = source["repo_id"], task["task_path"] - cloud._safe_name(task_path) + if task_path != ".": + cloud._safe_name(task_path) destination = Path(destination) if destination.exists() or destination.is_symlink(): raise FileExistsError("Acquisition requires a fresh task directory") @@ -115,7 +120,7 @@ def acquire_task(task, destination, *, source, api=None, download=None): total = 0 for item in api.list_repo_tree( repo_id, - path_in_repo=task_path, + path_in_repo=None if task_path == "." else task_path, repo_type="dataset", revision=revision, recursive=True, @@ -250,7 +255,7 @@ def _prepare( "evaluation_only": True, "negative_counterexamples": task["negative_counterexamples"], "annotations": task.get("annotations", {}), - "source_image": native["environment"]["docker_image"], + "source_image": native.get("environment", {}).get("docker_image"), "execution_image": task["image"], "budget": cloud._config(config)[1], } diff --git a/examples/hub_validation_pipeline/test_cloud.py b/examples/hub_validation_pipeline/test_cloud.py index ab35b161b..df339e91e 100644 --- a/examples/hub_validation_pipeline/test_cloud.py +++ b/examples/hub_validation_pipeline/test_cloud.py @@ -973,3 +973,37 @@ def test_remote_identity_requires_verified_acquisition_hashes(inputs, tmp_path, with pytest.raises(ValueError, match="verified acquisition"): cloud.prepare(task, config, tmp_path / kind, repo_root=repo, source=source) assert not (tmp_path / kind).exists() + + +def test_receipt_is_synced_before_and_after_atomic_publication(tmp_path, monkeypatch): + import os + + path = tmp_path / "submission.json" + events = [] + fsync, replace = os.fsync, Path.replace + + def sync(fd): + events.append("directory" if stat.S_ISDIR(os.fstat(fd).st_mode) else "file") + fsync(fd) + + def publish(source, target): + events.append("replace") + return replace(source, target) + + monkeypatch.setattr(os, "fsync", sync) + monkeypatch.setattr(Path, "replace", publish) + cloud._write_receipt(path, {"job_id": "owned-job"}) + assert events == ["file", "replace", "directory"] + assert cloud._loads(path.read_bytes()) == {"job_id": "owned-job"} + + +def test_receipt_sync_failure_preserves_previous_receipt(tmp_path, monkeypatch): + import os + + path = tmp_path / "submission.json" + path.write_text('{"status":"launch_pending"}\n') + previous = path.read_bytes() + monkeypatch.setattr(os, "fsync", Mock(side_effect=OSError("disk failure"))) + with pytest.raises(OSError, match="disk failure"): + cloud._write_receipt(path, {"status": "submitted", "job_id": "owned-job"}) + assert path.read_bytes() == previous diff --git a/examples/hub_validation_pipeline/test_full_run.py b/examples/hub_validation_pipeline/test_full_run.py index 05f1b9ea0..eb31f2413 100644 --- a/examples/hub_validation_pipeline/test_full_run.py +++ b/examples/hub_validation_pipeline/test_full_run.py @@ -821,3 +821,28 @@ def test_full_run_requires_explicit_panel_before_preparation(local_run): assert not output.exists() prepare.assert_not_called() submit.assert_not_called() + + +def test_missing_execution_flavor_uses_backend_cpu_basic_default(): + config = toy_config() + expected = full_run.estimate([config]) + config["execution"].pop("flavor", None) + assert full_run.estimate([config]) == expected + + +def test_budget_gate_compares_actual_rounded_reservation(local_run, monkeypatch): + from decimal import Decimal + + output, hf, harbor, _, submit, _ = local_run + monkeypatch.setitem(full_run.RATEBOOK, "cpu_basic_usd_per_hour", "0.010000000001") + estimated = full_run.estimate([toy_config(name) for name in TOY_NAMES]) + exact = Fraction(estimated["reservation_usd_exact"]) + rounded = Fraction(estimated["reservation_usd_rounded_up"]) + assert exact < rounded + halfway = (exact + rounded) / 2 + cap = str(Decimal(halfway.numerator) / Decimal(halfway.denominator)) + result = full_run.run( + output, hf, harbor, cap, panel_path=output.parent / "panel.json" + ) + assert result["within_budget"] is False + submit.assert_not_called() diff --git a/examples/hub_validation_pipeline/test_prepared_batch.py b/examples/hub_validation_pipeline/test_prepared_batch.py index 173707010..33ef60f8d 100644 --- a/examples/hub_validation_pipeline/test_prepared_batch.py +++ b/examples/hub_validation_pipeline/test_prepared_batch.py @@ -691,3 +691,26 @@ def malformed(*args, **kwargs): prepared_batch.run(**cohort.arguments, submit=True) cohort.api.inspect_job.assert_not_called() assert cohort.launch.call_count == 1 + + +@pytest.mark.parametrize( + "stage", ["COMPLETED", "ERROR", "CANCELED", "DELETED", "RUNNING"] +) +def test_status_preserves_terminal_observation_at_deadline(stage): + job = SimpleNamespace( + id="job", + docker_image=cloud.WORKER_IMAGE, + owner=SimpleNamespace(name="owner"), + status=SimpleNamespace(stage=stage), + ) + clock = iter([0.0, 0.1, 1.1]) + result = prepared_batch._status( + {"job_id": "job"}, + {"image": cloud.WORKER_IMAGE}, + "owner", + 1, + SimpleNamespace(inspect_job=lambda **kwargs: job), + SECRETS, + lambda: next(clock), + ) + assert result == ("TIMEOUT" if stage == "RUNNING" else stage) diff --git a/examples/hub_validation_pipeline/test_real_tasks.py b/examples/hub_validation_pipeline/test_real_tasks.py index 81b0269d7..0dc0997a4 100644 --- a/examples/hub_validation_pipeline/test_real_tasks.py +++ b/examples/hub_validation_pipeline/test_real_tasks.py @@ -575,3 +575,49 @@ def replace_after_read(path): assert Path(manifest["tasks"][0]["config_path"]).read_bytes() == cloud._json_bytes( toy_config() ) + + +def test_dockerfile_only_recipe_preserves_native_manifest(tmp_path): + name = TOY_NAMES[0] + files = source_files(name) + files["task.toml"] = b'version = "1.0"\n[environment]\ncpus = 1\n' + api, download = hf_fakes({f"fixtures/{name}/{p}": raw for p, raw in files.items()}) + result = real_tasks.prepare( + tmp_path / "recipe", + panel_path=write_panel(tmp_path, [name]), + api=api, + download=download, + ) + row = result["tasks"][0] + assert row["source_image"] is None + assert row["execution_image"] == toy_config(name)["execution"]["agent_image"] + assert (Path(row["task_root"]) / "task.toml").read_bytes() == files["task.toml"] + + +def test_root_task_panel_acquires_and_bundles_exact_root_files(tmp_path): + name = TOY_NAMES[0] + files = source_files(name) + api, download = hf_fakes(files) + + def listing(repo_id, **kwargs): + assert kwargs["path_in_repo"] is None + for path, data in files.items(): + yield RepoFile( + path=path, + size=len(data), + oid=hashlib.sha1(f"blob {len(data)}\0".encode() + data).hexdigest(), + ) + + api.list_repo_tree.side_effect = listing + panel = write_panel(tmp_path, [name]) + value = cloud._loads(panel.read_bytes()) + value["tasks"][0]["task_path"] = "." + panel.write_bytes(cloud._json_bytes(value)) + result = real_tasks.prepare( + tmp_path / "recipe", panel_path=panel, api=api, download=download + ) + row = result["tasks"][0] + assert row["source"]["task_path"] == "." + assert set(row["files"]) == set(files) + for path, content in files.items(): + assert (Path(row["task_root"]) / path).read_bytes() == content From a55f117acbadb803cf1331807bd8d310f3f30a57 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 20:43:14 +0100 Subject: [PATCH 12/19] test(hub-validation): use a generic archive escape target --- examples/hub_validation_pipeline/test_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/hub_validation_pipeline/test_sources.py b/examples/hub_validation_pipeline/test_sources.py index 12aaa09ca..ad997135e 100644 --- a/examples/hub_validation_pipeline/test_sources.py +++ b/examples/hub_validation_pipeline/test_sources.py @@ -560,7 +560,7 @@ def test_harbor_archive_must_match_bounded_inventory(monkeypatch, tmp_path, case member = tarfile.TarInfo("unexpected") if case == "link": member.type = tarfile.SYMTYPE - member.linkname = "/tmp/outside" + member.linkname = "/outside" client.extra_members = [(member, b"")] else: original = client.get_task_version_files.side_effect From e795ea275e0ea0cec51f8bcd14134798d9574601 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 21:56:19 +0100 Subject: [PATCH 13/19] fix(coding-audit): retain safe HTTP failure status --- examples/coding_audit/README.md | 2 ++ examples/coding_audit/hf.py | 4 ++- examples/coding_audit/test_hf.py | 45 ++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/examples/coding_audit/README.md b/examples/coding_audit/README.md index 345e6e5ba..62661bf10 100644 --- a/examples/coding_audit/README.md +++ b/examples/coding_audit/README.md @@ -62,6 +62,8 @@ serving-tokenizer measurement or a guarantee about remote billing. Evidence reco the reservation, actual per-response usage, request hashes and dispatch decisions. Reported input above the reservation, output above the request cap, inconsistent usage, or cumulative usage above the configured total remains an error. +HTTP failures retain their numeric status without response bodies, URLs or headers. +They are not retried automatically; failed-request billing remains unknown. After a valid interaction prefix, a pre-dispatch budget stop collects only the allowlisted files actually saved, then uses the normal fresh verifier. A funded, diff --git a/examples/coding_audit/hf.py b/examples/coding_audit/hf.py index bf0c1d971..84f92234b 100644 --- a/examples/coding_audit/hf.py +++ b/examples/coding_audit/hf.py @@ -21,7 +21,7 @@ from pathlib import Path, PurePosixPath from huggingface_hub import HfApi, InferenceClient, Sandbox -from huggingface_hub.errors import SandboxError +from huggingface_hub.errors import HfHubHTTPError, SandboxError from .runner import _candidate_context_paths @@ -364,6 +364,8 @@ def _record(self): @staticmethod def _error(exc): + if isinstance(exc, HfHubHTTPError): + return f"HF backend operation failed ({type(exc).__name__}; HTTP {exc.response.status_code})" return ( str(exc) if isinstance(exc, HFBackendError) diff --git a/examples/coding_audit/test_hf.py b/examples/coding_audit/test_hf.py index 95484e097..468ac3a01 100644 --- a/examples/coding_audit/test_hf.py +++ b/examples/coding_audit/test_hf.py @@ -1732,3 +1732,48 @@ def test_empty_or_non_bash_fence_does_not_create_a_candidate(harness, raw): h.inference.responses = [response(raw)] record = h.backend.generate_candidate(MODEL, 0) assert record["error"] and record["script"] == "" and not h.factory.created + + +@pytest.mark.parametrize("phase", ["target", "candidate"]) +@pytest.mark.parametrize("status", [400, 401, 403, 429, 502, 503]) +def test_http_failure_retains_only_numeric_status_without_retry( + harness, monkeypatch, phase, status +): + from unittest.mock import Mock + + from huggingface_hub.errors import HfHubHTTPError + + request = httpx.Request( + "POST", + "https://router.invalid/chat?private=url-sentinel", + headers={"Authorization": "Bearer inference-token-secret"}, + ) + response = httpx.Response(status, request=request, text="private-body-sentinel") + failure = HfHubHTTPError( + "private-body-sentinel inference-token-secret url-sentinel", response=response + ) + call = Mock(side_effect=failure) + monkeypatch.setattr(harness.inference, "chat_completion", call) + record = ( + harness.backend.solve(MODEL, 0) + if phase == "target" + else harness.backend.generate_candidate(MODEL, 0) + ) + assert ( + record["error"] + == f"HF backend operation failed (HfHubHTTPError; HTTP {status})" + ) + assert len(record["model_metadata"]["requests"]) == 1 and call.call_count == 1 + assert not record.get("files") and record.get("reward") is None + assert record["usage"] == {"input_tokens": 0, "output_tokens": 0} + assert "private-body-sentinel" not in json.dumps(record) + assert "url-sentinel" not in json.dumps(record) + assert "inference-token-secret" not in json.dumps(record) + if phase == "target": + assert record["cleanup_verified"] + assert ( + len(harness.factory.created) == 1 + and not harness.factory.created[0].commands + ) + else: + assert not harness.factory.created From b1fe5e658624a1821c4467da030080101e533d8b Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 22:03:48 +0100 Subject: [PATCH 14/19] fix(coding-audit): declare candidate file-writing contract --- examples/coding_audit/README.md | 3 +++ examples/coding_audit/hf.py | 14 ++++++++++++-- examples/coding_audit/test_hf.py | 10 ++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/examples/coding_audit/README.md b/examples/coding_audit/README.md index 62661bf10..f2c31c395 100644 --- a/examples/coding_audit/README.md +++ b/examples/coding_audit/README.md @@ -35,6 +35,9 @@ alongside request parameters. Support depends on the selected provider/model. Candidate generation streams bounded content and reasoning; only a complete `stop` response with validated usage can produce an executable candidate. A `length` response remains rejected even when its terminal usage is retained. +The candidate prompt declares the allowed output files and requests a Bash script +that writes them. Its versioned contract is recorded with the model metadata; +standalone source-code fences remain errors. Recognized Bash/sh Markdown fences are parsed before execution; the raw response and both raw/script hashes are retained. Other language fences are rejected. diff --git a/examples/coding_audit/hf.py b/examples/coding_audit/hf.py index 84f92234b..5de01a1d7 100644 --- a/examples/coding_audit/hf.py +++ b/examples/coding_audit/hf.py @@ -1090,12 +1090,22 @@ def generate_candidate(self, model: dict, seed: int) -> dict: instruction = self._candidate_instruction(model, record) model, allowed, limits, client = self._model(model, seed) record["model_metadata"] = self._model_metadata(model, seed) + record["model_metadata"]["candidate_output_contract"] = { + "format": "bash_script", + "version": 2, + "submission_paths": list(self.submission_paths), + } messages = [ { "role": "system", - "content": "Return only a bash script that attempts the task. Do not use Markdown fences.", + "content": "Return only an executable /bin/bash script that creates or edits the required submission files. For source-code solutions, write the source into those files with a quoted heredoc or equivalent; do not return bare Python or other source code. Do not use Markdown fences.", + }, + { + "role": "user", + "content": instruction + + "\n\nRequired submission files: " + + json.dumps(self.submission_paths), }, - {"role": "user", "content": instruction}, ] record["transcript"] = messages message = self._chat( diff --git a/examples/coding_audit/test_hf.py b/examples/coding_audit/test_hf.py index 468ac3a01..6bfb26bb8 100644 --- a/examples/coding_audit/test_hf.py +++ b/examples/coding_audit/test_hf.py @@ -744,6 +744,16 @@ def test_provider_requests_base_id_with_distinct_token_and_records_source_limit( assert "PRIVATE_CANARY_SENTINEL" not in repr(request) assert "test.sh" not in repr(request) metadata = generated["model_metadata"] + assert metadata["candidate_output_contract"] == { + "format": "bash_script", + "version": 2, + "submission_paths": ["/app/answer.txt"], + } + assert "creates or edits" in request["messages"][0]["content"] + assert ( + 'Required submission files: ["/app/answer.txt"]' + in request["messages"][1]["content"] + ) assert metadata["source_revision"] == MODEL["revision"] assert metadata["served_weight_revision_pinned"] is False assert metadata["seed_supported"] is False From 8c6097467fd98caff863f3abe22b502ffeda568e Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 22:05:36 +0100 Subject: [PATCH 15/19] test(coding-audit): assert explicit candidate outputs and unchanged target prompt --- examples/coding_audit/test_hf.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/coding_audit/test_hf.py b/examples/coding_audit/test_hf.py index 6bfb26bb8..b1292fb79 100644 --- a/examples/coding_audit/test_hf.py +++ b/examples/coding_audit/test_hf.py @@ -1219,17 +1219,18 @@ def read(path, limit): assert "context_paths" not in h.inference.requests[0] -def test_default_candidate_prompt_and_target_protocol_are_unchanged(harness): +def test_candidate_contract_names_outputs_without_changing_target_instruction(harness): h = harness h.inference.responses = [response("true"), response("done")] generated = h.backend.generate_candidate(MODEL, 0) solved = h.backend.solve(MODEL, 1) assert generated["error"] is None and solved["error"] is None assert "candidate_context" not in generated - assert all( - request["messages"][1]["content"] == (h.root / "instruction.md").read_text() - for request in h.inference.requests + instruction = (h.root / "instruction.md").read_text() + assert h.inference.requests[0]["messages"][1]["content"] == ( + instruction + '\n\nRequired submission files: ["/app/answer.txt"]' ) + assert h.inference.requests[1]["messages"][1]["content"] == instruction @pytest.mark.parametrize( From 505edbd68d8986590a2b6c0ebe7b5a8ff4e70be6 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 06:26:44 +0100 Subject: [PATCH 16/19] fix(examples): align native admission with validator contract --- .../hub-environment-validation-pipeline.md | 11 ++ examples/hub_validation_pipeline/README.md | 20 ++- examples/hub_validation_pipeline/pipeline.py | 21 ++- .../hub_validation_pipeline/test_pipeline.py | 141 +++++++++++++++++- 4 files changed, 182 insertions(+), 11 deletions(-) diff --git a/docs/plans/hub-environment-validation-pipeline.md b/docs/plans/hub-environment-validation-pipeline.md index 5c829fe2b..5ff94c180 100644 --- a/docs/plans/hub-environment-validation-pipeline.md +++ b/docs/plans/hub-environment-validation-pipeline.md @@ -79,6 +79,12 @@ The explicitly selected `harbor-semantic-v1` profile requires: configuration, unchanged task identities and verified cleanup. - Bound raw reward files and successful oracle exits. Recompute replay equality, the declared oracle maximum, the NOP gap and canary floors from those files. +- Scale tolerance and margin by `high - low`: require + `oracle >= high - oracle_tolerance * (high - low)` and both `high - floor` and + `oracle - floor` to be at least `floor_margin * (high - low)`. +- Read the retained tag from `runtime.startup.measured.retained_image_tag` and + require equality with checksummed native/trial evidence. The validator retains + this tag only for runs without FAIL, ERROR or interruption. Other standard checks remain named SKIPs and can leave the underlying report WARN. The admission envelope records these limits beside the unchanged report; it does @@ -92,6 +98,11 @@ both the original source digest and the effective validation digest. The current native adapter supports self-contained, single-step Linux tasks with one Dockerfile, shared verifier execution and public networking. Unsupported resource, credential, multi-service and extension contracts stay named SKIPs. +Tasks declaring `environment.docker_image` are also unsupported, even alongside +a Dockerfile: native trials must build the reviewed Dockerfile. Their startup +SKIP excludes them before model handoff; acquisition preserves their original +bytes. A missing `solution/solve.sh` still produces `semantic.oracle_max` FAIL. +Supplemental cloud audits remain a separate profile and cannot waive this gate. ```bash uv run --no-sync openenv validate inputs/task --level semantic --policy v2 \ diff --git a/examples/hub_validation_pipeline/README.md b/examples/hub_validation_pipeline/README.md index f5e70d362..5e69ddd9f 100644 --- a/examples/hub_validation_pipeline/README.md +++ b/examples/hub_validation_pipeline/README.md @@ -61,6 +61,11 @@ Trials, followed by one Trial per declared canary. It checks exact task, retaine image and inspected configuration identities, raw verifier rewards, replay equality, zero oracle exit codes and independently verified cleanup. It recomputes the oracle maximum, floor gap and canary floors from the bound raw evidence. +For declared range `[low, high]`, let `width = high - low`. Oracle acceptance +requires `oracle >= high - oracle_tolerance * width`; both `high - floor` and +`oracle - floor` must be at least `floor_margin * width`. These are exact numeric +comparisons, and tolerance/margin are fractions of the range rather than raw +reward units. This establishes bounded executable/verifier sanity and exact-submission replay. It does not establish complete semantic coverage, hostile-code containment, @@ -82,6 +87,14 @@ The native adapter supports self-contained single-step Linux tasks with one Dockerfile, shared verifier execution, public networking and no GPU, task-defined environment variables, healthchecks, external MCP/credential requirements or task-authored Compose. Unsupported shapes remain explicit SKIPs. +The native trials must build the reviewed Dockerfile. A task that declares +`environment.docker_image` is unsupported even when it also contains a Dockerfile; +`runtime.startup` remains SKIP and the pipeline excludes it before model execution. +This includes prebuilt-image benchmark tasks. Acquisition leaves their native +bytes unchanged; the driver does not remove the image declaration to bypass the +gate. A missing `solution/solve.sh` remains `semantic.oracle_max` FAIL even when +startup is unsupported. The supplemental HF Jobs coding audit is a separate +operator profile and does not waive these native admission requirements. ## Search both hubs and validate a small selection @@ -163,8 +176,11 @@ Requested model runs additionally retain rollouts and native trial/capture evide its task, image, contract and execution copies. Cleanup targets exact run-owned Docker containers, networks and volumes and is -independently rechecked. Validated images are retained for model handoff; their -exact tags are recorded in native evidence. Remove only those recorded tags with +independently rechecked. The validator removes its retained image tag on FAIL, +ERROR or interruption. For an unfailed run, admission reads the tag from +`runtime.startup.measured.retained_image_tag` and binds it to the checksummed native +and trial evidence. The model corpus pins the actual immutable image ID. +Remove only the recorded retained tags with `docker image rm ` when no longer needed. Sources and excluded reports remain available; reruns use fresh output directories. diff --git a/examples/hub_validation_pipeline/pipeline.py b/examples/hub_validation_pipeline/pipeline.py index a62723bcb..737b01963 100644 --- a/examples/hub_validation_pipeline/pipeline.py +++ b/examples/hub_validation_pipeline/pipeline.py @@ -265,6 +265,11 @@ def admission(report, exit_code, subject, artifact_dir, profile, original_digest raise ValueError("required validation coverage is incomplete") if not {1, 2, 3}.issubset(report["levels_run"]): raise ValueError("semantic validation was not executed") + retained_tag = ( + results["runtime.startup"].get("measured", {}).get("retained_image_tag") + ) + if not re.fullmatch(r"openenv-validation:[a-f0-9]{32}", retained_tag or ""): + raise ValueError("startup retained image measurement is missing or invalid") if profile != PILOT and ( report["verdict"] != "pass" or 3 not in report["levels_run"] ): @@ -283,9 +288,7 @@ def admission(report, exit_code, subject, artifact_dir, profile, original_digest or report["manifest"]["execution"]["profile"] != PILOT or report["manifest"]["capabilities"]["canaries"] != parsed_contract.canaries or not re.fullmatch(r"sha256:[0-9a-f]{64}", native.get("image_ref") or "") - or not re.fullmatch( - r"openenv-validation:[a-f0-9]{32}", native.get("retained_image_tag") or "" - ) + or native.get("retained_image_tag") != retained_tag or native["cleanup"]["verified"] is not True or "failure" in native ): @@ -302,7 +305,7 @@ def admission(report, exit_code, subject, artifact_dir, profile, original_digest or any( not re.fullmatch(r"[0-9a-f]{64}", row["container_id"] or "") or row["image_ref"] != native["image_ref"] - or row["retained_image_tag"] != native["retained_image_tag"] + or row["retained_image_tag"] != retained_tag or row["unmodified_task_digest"] != report["source_digest"] or row["configuration"] != trials[0]["configuration"] or row["cleanup"]["verified"] is not True @@ -318,13 +321,14 @@ def admission(report, exit_code, subject, artifact_dir, profile, original_digest low, high = map(reward_fraction, reward_contract["range"]) margin = reward_fraction(reward_contract["floor_margin"]) tolerance = reward_fraction(reward_contract["oracle_tolerance"]) + width = high - low rewards = [] for index, trial in enumerate(trials): overlay = ( {} if index == 0 else { - "environment.docker_image": native["retained_image_tag"], + "environment.docker_image": retained_tag, } ) if index >= 3: @@ -352,9 +356,9 @@ def admission(report, exit_code, subject, artifact_dir, profile, original_digest if ( floor != reward_fraction(native.get("floor_reward")) or oracle != reward_fraction(native.get("oracle_reward")) - or oracle < high - tolerance - or oracle - floor < margin - or high - floor < margin + or oracle < high - tolerance * width + or oracle - floor < margin * width + or high - floor < margin * width or any(row[native["reward_key"]] > floor for row in rewards[3:]) ): raise ValueError("raw native rewards do not satisfy the oracle/floor contract") @@ -364,6 +368,7 @@ def admission(report, exit_code, subject, artifact_dir, profile, original_digest "original_source_digest": original_digest, "task_effective_digest": effective_digest, "image_ref": native["image_ref"], + "retained_image_tag": retained_tag, "reward_key": native["reward_key"], "floor_reward": native["floor_reward"], "oracle_reward": native["oracle_reward"], diff --git a/examples/hub_validation_pipeline/test_pipeline.py b/examples/hub_validation_pipeline/test_pipeline.py index c81404683..e823c290f 100644 --- a/examples/hub_validation_pipeline/test_pipeline.py +++ b/examples/hub_validation_pipeline/test_pipeline.py @@ -86,6 +86,9 @@ def make_evidence(tmp_path, canary_count=2): "cleanup": {"verified": True}, "trials": [], } + next(row for row in report["results"] if row["check_id"] == "runtime.startup")[ + "measured" + ] = {"retained_image_tag": native["retained_image_tag"]} artifacts = tmp_path / "evidence" roles = ["nop", "oracle", "oracle-replay"] + [ f"canary-{index}" for index in range(1, canary_count + 1) @@ -160,6 +163,7 @@ def test_explicit_pilot_admits_but_full_gate_stays_incomplete(evidence): admitted = admit(evidence) assert admitted["out_of_profile_skips"] == ["runtime.host_containment"] assert admitted["rfc008_complete"] is False + assert admitted["retained_image_tag"] == evidence[4]["retained_image_tag"] with pytest.raises(ValueError, match="coverage"): admit(evidence, "full") @@ -168,7 +172,14 @@ def test_full_profile_requires_every_policy_check_to_pass(evidence): _, report, artifacts, _, native = evidence report["verdict"] = "pass" report["results"] = [ - {"check_id": key, "status": "pass", "duration_s": 0} + { + "check_id": key, + "status": "pass", + "duration_s": 0, + "measured": {"retained_image_tag": native["retained_image_tag"]} + if key == "runtime.startup" + else {}, + } for key in pipeline.load_policy("v2").entries_for_lane(pipeline.Lane.LOCAL) ] save_bundle(artifacts, report, native) @@ -400,6 +411,134 @@ def test_admission_recomputes_oracle_floor_from_raw_rewards(evidence): admit(evidence) +@pytest.mark.parametrize( + "fault", ["missing-measured", "missing-tag", "malformed", "different"] +) +def test_startup_measurement_binds_retained_image(evidence, fault): + _, report, artifacts, _, native = evidence + startup = next( + row for row in report["results"] if row["check_id"] == "runtime.startup" + ) + if fault == "missing-measured": + startup.pop("measured") + elif fault == "missing-tag": + startup["measured"].pop("retained_image_tag") + else: + startup["measured"]["retained_image_tag"] = ( + "unowned:tag" if fault == "malformed" else "openenv-validation:" + "b" * 32 + ) + save_bundle(artifacts, report, native) + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize( + "reward_range,floor,oracle,margin,tolerance,accepted", + [ + ([0, 100], 0, 95, 0.1, 0.05, True), + ([50, 150], 50, 145, 0.1, 0.05, True), + ([0, 0.01], 0, 0.0095, 0.1, 0.05, True), + ([0, 100], 90, 100, 0.1, 0, True), + ([0, 100], 99, 100, 0.1, 0, False), + ([50, 150], 149, 150, 0.1, 0, False), + ([0, 0.01], 0.0099, 0.01, 0.1, 0, False), + ([0, 100], 0, 94.99, 0.1, 0.05, False), + ([0, 0.01], 0, 0.009499, 0.1, 0.05, False), + ], +) +def test_admission_scales_thresholds_by_reward_range( + evidence, reward_range, floor, oracle, margin, tolerance, accepted +): + subject, report, artifacts, _, native = evidence + path = subject / "validation/openenv.json" + contract = pipeline.loads(path.read_text()) + reward = { + "range": reward_range, + "floor_margin": margin, + "oracle_tolerance": tolerance, + } + contract["reward"].update(reward) + pipeline.write_json(path, contract) + report["manifest"]["reward"].update(reward) + report["source_digest"] = source_digest(subject) + native["task_effective_digest"] = report["source_digest"] + for trial in native["trials"]: + trial["unmodified_task_digest"] = report["source_digest"] + trial["rewards"]["reward"] = ( + oracle if trial["role"] in {"oracle", "oracle-replay"} else floor + ) + trial["rewards"]["aux"] = reward_range[0] + set_raw(artifacts, trial, "reward.json", pipeline.dumps(trial["rewards"])) + native.update(floor_reward=floor, oracle_reward=oracle) + save_bundle(artifacts, report, native) + # Recompute the contract even if all bundled semantic results claim PASS. + if accepted: + assert pipeline.reward_fraction(admit(evidence)["oracle_reward"]) == ( + pipeline.reward_fraction(Decimal(str(oracle))) + ) + else: + with pytest.raises(ValueError, match="oracle/floor contract"): + admit(evidence) + + +@pytest.mark.parametrize("missing_oracle", [False, True]) +def test_native_prebuilt_task_is_excluded_without_model_or_docker( + tmp_path, monkeypatch, missing_oracle +): + from openenv.validation.harbor import DockerInspector + from openenv.validation.runner import run_validation + + task = demo.create_tasks(tmp_path / "input") / "good" + manifest = task / "task.toml" + config = pipeline.tomllib.loads(manifest.read_text()) + config.setdefault("environment", {})["docker_image"] = ( + "registry.example/task:latest" + ) + manifest.write_text(pipeline.tomli_w.dumps(config)) + if missing_oracle: + (task / "solution/solve.sh").unlink() + original = manifest.read_bytes() + reports = [] + + def validate(subject, report_path, profile, timeout): + report = run_validation(subject).model_dump(mode="json") + reports.append(report) + return 0, report + + def forbidden(*args, **kwargs): + pytest.fail("excluded native task must not execute Docker or a model") + + monkeypatch.setattr(pipeline, "validate_task", validate) + monkeypatch.setattr(pipeline, "evaluate", forbidden) + monkeypatch.setattr(DockerInspector, "_docker", forbidden) + output = tmp_path / "output" + args = SimpleNamespace( + output=output, + contract=Path(pipeline.__file__).with_name("contract.json"), + local_dataset=[task], + hf_dataset=[], + harbor_dataset=[], + tasks_per_dataset=1, + profile=pipeline.PILOT, + validation_timeout=1, + llm_url="http://unused-model/v1", + ) + assert pipeline.run(args) == 1 + checks = {row["check_id"]: row for row in reports[0]["results"]} + assert checks["runtime.startup"]["status"] == "skip" + assert "docker_image" in checks["runtime.startup"]["evidence"][0] + if missing_oracle: + assert checks["semantic.oracle_max"]["status"] == "fail" + assert manifest.read_bytes() == original + assert list((output / "subjects").glob("*/task.toml"))[0].read_bytes() == original + summary = pipeline.loads((output / "summary.json").read_text()) + assert (summary["accepted"], summary["excluded"], summary["model_trials"]) == ( + 0, + 1, + 0, + ) + + @pytest.mark.parametrize( "floor,oracle,margin,tolerance", [ From 1c1405859fbf1ac0d33de22f4ab5e5e8bf4299aa Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 10:29:04 +0100 Subject: [PATCH 17/19] Fix prepared cohort reproducibility and bounded audit evidence --- examples/coding_audit/hf.py | 34 +++-- examples/coding_audit/test_hf.py | 37 +++++ .../hub_validation_pipeline/CODING_AUDIT.md | 11 ++ examples/hub_validation_pipeline/README.md | 4 + examples/hub_validation_pipeline/cloud.py | 72 ++++++++-- examples/hub_validation_pipeline/full_run.py | 3 + .../hub_validation_pipeline/real_tasks.py | 7 +- .../hub_validation_pipeline/test_cloud.py | 128 ++++++++++++++++-- .../test_prepared_batch.py | 118 +++++++++++++++- .../test_real_tasks.py | 12 +- 10 files changed, 385 insertions(+), 41 deletions(-) diff --git a/examples/coding_audit/hf.py b/examples/coding_audit/hf.py index 6502f4cb6..ed7c0956e 100644 --- a/examples/coding_audit/hf.py +++ b/examples/coding_audit/hf.py @@ -72,6 +72,8 @@ def _deadline(seconds): # the main Unix thread so a blocked synchronous SDK call can be interrupted. if threading.current_thread() is not threading.main_thread(): raise HFBackendError("HF coding controller requires the main thread") + if not math.isfinite(seconds) or seconds <= 0: + raise _DeadlineExceeded("controller deadline exceeded") previous_handler = signal.getsignal(signal.SIGALRM) previous_timer = signal.getitimer(signal.ITIMER_REAL) started = time.monotonic() @@ -751,26 +753,32 @@ def _chat(self, client, model, allowed, limits, messages, record, seed, *, tools if tools: preflight["dispatched"] = True parsed = client.chat_completion(**request) - response = _plain(parsed) - # Access the typed nested value: outer SDK dicts can omit extras. - response["usage"] = _plain(parsed.usage) - choices = response.get("choices") + # Preserve billing evidence before normalizing arbitrary provider + # extras: JSON conversion itself can fail after a billed response. receipt = { - "model": response.get("model"), - "system_fingerprint": response.get("system_fingerprint"), - "id": response.get("id"), "seed_sent": request.get("seed"), - "finish_reason": choices[0].get("finish_reason") - if isinstance(choices, list) - and len(choices) == 1 - and isinstance(choices[0], dict) - else None, "response_validated": False, "discarded": True, "tool_calls_processed": 0, } metadata.setdefault("responses", []).append(receipt) - _retain_usage(receipt, response["usage"]) + _retain_usage(receipt, parsed.usage) + response = _plain(parsed) + # Access the typed nested value: outer SDK dicts can omit extras. + response["usage"] = _plain(parsed.usage) + choices = response.get("choices") + receipt.update( + { + "model": response.get("model"), + "system_fingerprint": response.get("system_fingerprint"), + "id": response.get("id"), + "finish_reason": choices[0].get("finish_reason") + if isinstance(choices, list) + and len(choices) == 1 + and isinstance(choices[0], dict) + else None, + } + ) else: response = self._candidate_response( client, request, allowed, limits, record diff --git a/examples/coding_audit/test_hf.py b/examples/coding_audit/test_hf.py index 28e2e4e98..cc8282c7d 100644 --- a/examples/coding_audit/test_hf.py +++ b/examples/coding_audit/test_hf.py @@ -1506,6 +1506,43 @@ def test_deadline_interrupts_blocking_controller_and_restores_handler(): assert signal.getitimer(signal.ITIMER_REAL)[0] == 0 +@pytest.mark.parametrize("seconds", [0, -0.1, float("inf"), float("nan")]) +def test_expired_deadline_does_not_enter_or_change_parent_alarm(monkeypatch, seconds): + timer_calls = [] + monkeypatch.setattr(signal, "getitimer", lambda *_: (10, 0)) + monkeypatch.setattr(signal, "setitimer", lambda *args: timer_calls.append(args)) + handler = signal.getsignal(signal.SIGALRM) + entered = [] + with pytest.raises(HFBackendError, match="deadline"): + with _deadline(seconds): + entered.append(True) + assert not entered and not timer_calls + assert signal.getsignal(signal.SIGALRM) == handler + + +def test_target_retains_billed_counts_before_malformed_usage_serialization(harness): + usage = { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "provider_extra": object(), + } + harness.inference.responses = [response(usage=usage)] + record = harness.backend.solve(MODEL, 0) + assert record["error"] and record["cleanup_verified"] + receipts = record["model_metadata"].get("responses", []) + assert len(receipts) == 1 + assert receipts[0]["usage_counts"] == { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + } + assert receipts[0]["usage_receipt_error"] + assert receipts[0]["discarded"] and not receipts[0]["response_validated"] + assert not harness.factory.created[0].commands + json.dumps(record, allow_nan=False) + + def test_strong_isolation_and_equal_credentials_rejected(harness): h = harness with pytest.raises(HFBackendError, match="strong"): diff --git a/examples/hub_validation_pipeline/CODING_AUDIT.md b/examples/hub_validation_pipeline/CODING_AUDIT.md index 9c9849fb5..056489cfb 100644 --- a/examples/hub_validation_pipeline/CODING_AUDIT.md +++ b/examples/hub_validation_pipeline/CODING_AUDIT.md @@ -204,6 +204,17 @@ resumed or blindly retried. `prepared_batch` provides shared reservation and bounded controller concurrency for an already reviewed preparation manifest. Use these guarded cohort launchers for paid work; the cloud CLI and discovery pipeline only prepare inputs. +Both preparation producers emit `plan_path` relative to that manifest's directory +and `plan_sha256` for the exact generated plan bytes. Historical manifests without +these fields require fresh preparation and review; the launcher does not infer a +new digest for an old plan. + +Plans bind `runtime_versions` for uv and `huggingface_hub`. The remote bootstrap +runs the frozen dependency sync, installs the declared `huggingface_hub==1.33.0` +overlay with that environment's interpreter, then verifies the installed version +before starting the audit worker. A missing or incompatible runtime pin requires +fresh preparation; changing the package's general dependency range is unnecessary. + `cloud.submit()` requires a ledger, reservation ID and expected plan SHA256. Each reservation binds the reviewed plan hashes to their modeled cost allocations; allocations must fit the reserved total. Submission rechecks the active hold, diff --git a/examples/hub_validation_pipeline/README.md b/examples/hub_validation_pipeline/README.md index 5e69ddd9f..e5eb00548 100644 --- a/examples/hub_validation_pipeline/README.md +++ b/examples/hub_validation_pipeline/README.md @@ -125,6 +125,10 @@ transfers and expanded tar bytes at 8 MiB, and rejects links, sparse files and uninventoried members. Archive directory entries count toward the 128-member cap. Legacy packages without these inventories are refused. Verified task hashes follow the captured bytes into cloud plans; a remote revision alone is insufficient. +Acquired tasks containing native assets below `.git`, `__pycache__`, `.venv`, +`.worktrees` or `outputs` are unsupported by this bounded execution profile. +Preparation rejects their complete acquisition inventory explicitly rather than +dropping those files or weakening the source/hash comparison. Uploads use ordinary SDK commits with explicit additions; they do not mirror-delete remote files. As with `hf upload`, deletion requires a separate explicit operation. diff --git a/examples/hub_validation_pipeline/cloud.py b/examples/hub_validation_pipeline/cloud.py index e24956118..46ff3d566 100644 --- a/examples/hub_validation_pipeline/cloud.py +++ b/examples/hub_validation_pipeline/cloud.py @@ -25,6 +25,7 @@ WORKER_IMAGE = "docker.io/library/python@sha256:48d884813cf44e927cf64c5529dd630dd3b18338c6dc727991eef9b71e934a20" UV_VERSION = "0.9.3" +HF_HUB_VERSION = "1.33.0" SECRET_NAMES = ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN") MAX_FILES = 5000 MAX_FILE_BYTES = 16 * 1024 * 1024 @@ -249,20 +250,32 @@ def _positive(value, name, *, maximum=10000, minimum=1): return value -def _check_acquisition_files(source, task_files): - """Bind remote source identities to hashes verified during acquisition.""" +def _acquisition_hashes(source): + """Reject acquired assets that execution would silently omit.""" if "task_file_hashes" not in source: if source.get("kind", source.get("source")) in {"hf", "harbor"}: raise ValueError("Remote source requires verified acquisition file hashes") - return + return None expected = source["task_file_hashes"] - if ( - not isinstance(expected, dict) - or set(expected) != set(task_files) - or any( - not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value) - for value in expected.values() - ) + if not isinstance(expected, dict) or any( + not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value) + for value in expected.values() + ): + raise ValueError("Invalid verified acquisition file hashes") + for name in expected: + if any(part in _EXCLUDED for part in _safe_name(name).parts): + raise ValueError( + "Acquired task contains unsupported excluded assets; " + "native files under execution exclusions cannot be omitted safely" + ) + return expected + + +def _check_acquisition_files(source, task_files): + """Bind remote source identities to hashes verified during acquisition.""" + expected = _acquisition_hashes(source) + if expected is not None and ( + set(expected) != set(task_files) or any( hashlib.sha256(data).hexdigest() != expected[name] for name, data in task_files.items() @@ -351,7 +364,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): return redirected -def _bootstrap_main(uv_version): +def _bootstrap_main(uv_version, hub_version): import base64 import tempfile @@ -391,10 +404,36 @@ def _bootstrap_main(uv_version): ) stage = "sync" subprocess.run(["uv", "sync", "--frozen", "--no-dev"], cwd=root, check=True) + python = str(root / ".venv/bin/python") + stage = "install_sdk" + subprocess.run( + [ + "uv", + "pip", + "install", + "--python", + python, + "huggingface_hub==" + hub_version, + ], + cwd=root, + check=True, + ) + stage = "verify_sdk" + subprocess.run( + [ + python, + "-c", + "from importlib.metadata import version\n" + f"if version('huggingface_hub') != {hub_version!r}:\n" + " raise RuntimeError('Installed Hugging Face SDK differs from declared pin')\n", + ], + cwd=root, + check=True, + ) stage = "worker" subprocess.run( [ - str(root / ".venv/bin/python"), + python, "-m", "examples.hub_validation_pipeline.cloud", "worker", @@ -480,7 +519,7 @@ def _bootstrap_code(): _bootstrap_main, ) ) - return header + f"\n_bootstrap_main({UV_VERSION!r})\n" + return header + f"\n_bootstrap_main({UV_VERSION!r}, {HF_HUB_VERSION!r})\n" def _launch_spec(config, digest, revision=None): @@ -515,6 +554,7 @@ def prepare(task_root, config, output_dir, *, repo_root=None, source=None): raise FileExistsError("Prepare requires a fresh output directory") if output_dir.resolve().is_relative_to(task_root.resolve()): raise ValueError("Output directory must be outside the task") + _acquisition_hashes(source or {}) config, budget = _config(config) from examples.coding_audit.runner import validate_coding_task from openenv.validation.runner import source_digest @@ -616,6 +656,7 @@ def prepare(task_root, config, output_dir, *, repo_root=None, source=None): manifest = { "schema": "openenv-coding-bundle-v1", "git": _git_provenance(repo_root), + "runtime_versions": {"uv": UV_VERSION, "huggingface_hub": HF_HUB_VERSION}, "task_source": source, "target_source": { name: config["target"][name] @@ -650,6 +691,7 @@ def prepare(task_root, config, output_dir, *, repo_root=None, source=None): "results_repo": config["cloud"]["results_repo"], "manifest_sha256": hashlib.sha256(files[MANIFEST][0]).hexdigest(), "git": manifest["git"], + "runtime_versions": manifest["runtime_versions"], "task_source": source, "target_source": manifest["target_source"], "budget": budget, @@ -701,8 +743,10 @@ def _read_plan(plan_path, *, expected_sha256=None): or plan["manifest_sha256"] != hashlib.sha256(files[MANIFEST]).hexdigest() or any( plan.get(name) != manifest.get(name) - for name in ("git", "task_source", "target_source") + for name in ("git", "runtime_versions", "task_source", "target_source") ) + or manifest.get("runtime_versions") + != {"uv": UV_VERSION, "huggingface_hub": HF_HUB_VERSION} or manifest["target_source"] != { name: config["target"][name] diff --git a/examples/hub_validation_pipeline/full_run.py b/examples/hub_validation_pipeline/full_run.py index 97684c48a..1774c38a4 100644 --- a/examples/hub_validation_pipeline/full_run.py +++ b/examples/hub_validation_pipeline/full_run.py @@ -249,6 +249,9 @@ def run( { "task_id": row["task_id"], "plan": row["bundle_plan"], + "plan_path": Path(row["bundle_plan"]) + .relative_to(output) + .as_posix(), "plan_sha256": plan_sha256, "run_id": plan["run_id"], "bundle_sha256": plan["bundle_sha256"], diff --git a/examples/hub_validation_pipeline/real_tasks.py b/examples/hub_validation_pipeline/real_tasks.py index e23bd7f47..1a7f2d2d9 100644 --- a/examples/hub_validation_pipeline/real_tasks.py +++ b/examples/hub_validation_pipeline/real_tasks.py @@ -262,7 +262,12 @@ def _prepare( if bundle: directory = output / "bundles" / name cloud.prepare(task_root, config, directory, source=source) - row["bundle_plan"] = str(directory / "plan.json") + plan_path = directory / "plan.json" + row.update( + bundle_plan=str(plan_path), + plan_path=plan_path.relative_to(output).as_posix(), + plan_sha256=hashlib.sha256(plan_path.read_bytes()).hexdigest(), + ) rows.append(row) manifest = { "schema": "openenv-real-task-recipes-v1", diff --git a/examples/hub_validation_pipeline/test_cloud.py b/examples/hub_validation_pipeline/test_cloud.py index df339e91e..4da377f64 100644 --- a/examples/hub_validation_pipeline/test_cloud.py +++ b/examples/hub_validation_pipeline/test_cloud.py @@ -6,6 +6,7 @@ import shutil import stat import subprocess +import sys import tempfile import urllib.request import zipfile @@ -302,38 +303,102 @@ def test_archive_rejects_path_collisions_and_nonregular_files( assert not (tmp_path / "bad").exists() -def test_bootstrap_uses_pinned_download_and_frozen_sync(inputs, tmp_path, monkeypatch): +@pytest.mark.parametrize("installed_version", ["1.33.0", "1.32.0"]) +def test_bootstrap_installs_and_checks_declared_sdk_before_worker( + inputs, tmp_path, monkeypatch, installed_version +): output, plan = prepared(inputs, tmp_path) data = (output / "bundle.zip").read_bytes() - request = Mock(return_value=io.BytesIO(data)) + + def open_request(request, **kwargs): + if "/resolve/" in request.full_url: + return io.BytesIO(data) + return io.BytesIO(b'{"private":true}') + + request = Mock(side_effect=open_request) monkeypatch.setattr( urllib.request, "build_opener", lambda *args: SimpleNamespace(open=request) ) destination = tmp_path / "bootstrap" destination.mkdir() monkeypatch.setattr(tempfile, "mkdtemp", lambda **kwargs: str(destination)) - run = Mock() + real_run = subprocess.run + metadata = tmp_path / "metadata" + metadata.mkdir() + + def run_command(command, **kwargs): + if command[:3] == ["uv", "pip", "install"]: + assert command[-1] == "huggingface_hub==1.33.0" + dist = metadata / f"huggingface_hub-{installed_version}.dist-info" + dist.mkdir() + (dist / "METADATA").write_text( + f"Name: huggingface_hub\nVersion: {installed_version}\n" + ) + elif command[1] == "-c": + # Run the production version check over isolated package metadata. + return real_run( + [sys.executable, "-S", *command[1:]], + env={"PYTHONPATH": str(metadata)}, + **kwargs, + ) + + run = Mock(side_effect=run_command) monkeypatch.setattr(subprocess, "run", run) spec = cloud._launch_spec( cloud._config(inputs[2])[0], plan["bundle_sha256"], UPLOAD_SHA ) for name, value in {**spec["env"], **SECRETS}.items(): monkeypatch.setenv(name, value) - exec(spec["command"][2], {}) - assert "/resolve/" + UPLOAD_SHA + "/inputs/" in request.call_args.args[0].full_url - assert ( - request.call_args.args[0].get_header("Authorization") - == "Bearer " + SECRETS["HF_TOKEN"] - ) + if installed_version == "1.33.0": + exec(spec["command"][2], {}) + else: + with pytest.raises(RuntimeError, match="verify_sdk"): + exec(spec["command"][2], {}) + download = request.call_args_list[0].args[0] + assert "/resolve/" + UPLOAD_SHA + "/inputs/" in download.full_url + assert download.get_header("Authorization") == "Bearer " + SECRETS["HF_TOKEN"] calls = [call.args[0] for call in run.call_args_list] assert calls[0][-2:] == ["install", "uv==0.9.3"] assert calls[1] == ["uv", "sync", "--frozen", "--no-dev"] - assert calls[2][1:4] == ["-m", "examples.hub_validation_pipeline.cloud", "worker"] + python = str(destination / "bundle/.venv/bin/python") + assert calls[2] == [ + "uv", + "pip", + "install", + "--python", + python, + "huggingface_hub==1.33.0", + ] + assert calls[3][:2] == [python, "-c"] + if installed_version == "1.33.0": + assert calls[4][1:4] == [ + "-m", + "examples.hub_validation_pipeline.cloud", + "worker", + ] + else: + assert len(calls) == 4 assert ( destination / "bundle/task/instruction.md" ).read_bytes() == b"Write an answer\n" +def test_runtime_versions_are_bound_to_plan_and_bundle(inputs, tmp_path): + output, plan = prepared(inputs, tmp_path) + manifest, _ = cloud._checked_archive( + (output / "bundle.zip").read_bytes(), plan["bundle_sha256"] + ) + assert ( + manifest["runtime_versions"] + == plan["runtime_versions"] + == {"uv": "0.9.3", "huggingface_hub": "1.33.0"} + ) + plan["runtime_versions"]["huggingface_hub"] = "1.32.0" + (output / "plan.json").write_text(json.dumps(plan)) + with pytest.raises(ValueError, match="immutable bundle"): + cloud._read_plan(output / "plan.json") + + def test_submit_real_runtime_secrets_pins_and_one_shot_receipt(inputs, tmp_path): output, plan = prepared(inputs, tmp_path) api = fake_api() @@ -684,6 +749,49 @@ def test_task_exclusions_match_runner_source_identity(inputs, tmp_path): assert not (destination / "task/outputs").exists() +@pytest.mark.parametrize("excluded", sorted(cloud._EXCLUDED)) +@pytest.mark.parametrize("copied", [False, True]) +def test_acquired_excluded_assets_are_explicitly_unsupported_before_snapshot( + inputs, tmp_path, monkeypatch, excluded, copied +): + from examples.hub_validation_pipeline.pipeline import copy_subject + from openenv.validation.runner import source_digest + + repo, task, config = inputs + asset = task / "environment" / excluded / "setup.txt" + asset.parent.mkdir(parents=True) + asset.write_text("Native task setup asset\n") + hashes = { + path.relative_to(task).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in task.rglob("*") + if path.is_file() + } + source = dict(kind="hf", revision=SHA, task_path=".", task_file_hashes=hashes) + if copied: + subject = tmp_path / "subject" + copy_subject(task, subject, source_digest(task)) + assert not (subject / "environment" / excluded).exists() + else: + subject = task + original_tree = cloud._tree + snapshots = [] + + def snapshot(root, **kwargs): + if Path(root) == subject: + snapshots.append(root) + return original_tree(root, **kwargs) + + monkeypatch.setattr(cloud, "_tree", snapshot) + with pytest.raises( + ValueError, match="Acquired task contains unsupported excluded assets" + ): + cloud.prepare(subject, config, tmp_path / "plan", repo_root=repo, source=source) + assert snapshots == [] + assert source["task_file_hashes"] == hashes + assert f"environment/{excluded}/setup.txt" in hashes + assert not (tmp_path / "plan").exists() + + @pytest.mark.parametrize( "fault", ["compose", "steps", "private-network", "phase-environment", "missing-verifier"], diff --git a/examples/hub_validation_pipeline/test_prepared_batch.py b/examples/hub_validation_pipeline/test_prepared_batch.py index 33ef60f8d..ccf3a514a 100644 --- a/examples/hub_validation_pipeline/test_prepared_batch.py +++ b/examples/hub_validation_pipeline/test_prepared_batch.py @@ -11,8 +11,14 @@ from unittest.mock import Mock import pytest -from examples.hub_validation_pipeline import cloud, full_run, prepared_batch, spend -from examples.hub_validation_pipeline.test_real_tasks import write_panel +from examples.hub_validation_pipeline import ( + cloud, + full_run, + prepared_batch, + real_tasks, + spend, +) +from examples.hub_validation_pipeline.test_real_tasks import hf_fakes, write_panel SECRETS = dict( HF_TOKEN="local-results", @@ -176,6 +182,95 @@ def rewrite_manifest(cohort): cohort.source.write_bytes(cloud._json_bytes({"tasks": cohort.rows})) +@pytest.mark.parametrize("producer", ["recipes", "full-run"]) +def test_producer_manifest_launches_without_rewriting_rows( + cohort, monkeypatch, tmp_path, producer +): + source_api, download = hf_fakes() + monkeypatch.setattr(real_tasks, "HfApi", Mock(return_value=source_api)) + monkeypatch.setattr(real_tasks, "hf_hub_download", download) + prepare = cloud.prepare + monkeypatch.setattr( + cloud, + "prepare", + lambda *args, **kwargs: prepare(*args, repo_root=tmp_path / "repo", **kwargs), + ) + output = tmp_path / producer + panel = write_panel(tmp_path) + if producer == "recipes": + produced = real_tasks.prepare(output, panel_path=panel, bundle=True) + else: + hf, harbor = tmp_path / "hf.json", tmp_path / "harbor.json" + hf.write_bytes( + cloud._json_bytes( + dict( + scope="fixture", + counts={}, + compact_repo_inventory="missing", + compact_identity_inventory="missing", + ) + ) + ) + harbor.write_bytes( + cloud._json_bytes( + dict( + scope="fixture", + counts={}, + artifacts={ + name: {"path": "missing"} + for name in ( + "task_identity_inventory", + "dataset_version_membership_index", + ) + }, + ) + ) + ) + produced = full_run.run(output, hf, harbor, "500", panel_path=panel) + source = output / "manifest.json" + source_bytes = source.read_bytes() + api = Mock() + api.dataset_info.return_value = SimpleNamespace(private=True, sha="a" * 40) + api.list_repo_files.return_value = [] + api.create_commit.return_value = SimpleNamespace(oid="b" * 40) + api.run_job.side_effect = [ + SimpleNamespace( + id=f"job-{index}", url=f"https://huggingface.co/jobs/owner/job-{index}" + ) + for index in range(len(produced["tasks"])) + ] + api.inspect_job.side_effect = lambda *, job_id, **kwargs: SimpleNamespace( + id=job_id, + docker_image=cloud.WORKER_IMAGE, + owner=SimpleNamespace(name="owner"), + status=SimpleNamespace(stage="COMPLETED"), + ) + monkeypatch.setattr(cloud, "submit", cohort.actual_submit) + arguments = dict( + cohort.arguments, + prepared_manifest=source, + output=tmp_path / "launched", + api=api, + ) + result = prepared_batch.run(**arguments, submit=True) + assert result["status"] == "controllers_completed_unmeasured" + assert source.read_bytes() == source_bytes + assert result["prepared_manifest"] == full_run._hash(source) + assert ( + api.run_job.call_count == api.inspect_job.call_count == len(produced["tasks"]) + ) + for row, task in zip(produced["tasks"], result["tasks"], strict=True): + path = source.parent / row["plan_path"] + assert not Path(row["plan_path"]).is_absolute() + assert full_run._hash(path)["sha256"] == row["plan_sha256"] + receipt = cloud._loads((path.parent / "submission.json").read_bytes()) + assert receipt["status"] == "submitted" and receipt["job_id"] == task["job_id"] + launched = api.run_job.call_count + with pytest.raises(RuntimeError, match="during plans"): + prepared_batch.run(**dict(arguments, output=tmp_path / "retry"), submit=True) + assert api.run_job.call_count == launched + + def test_prepare_only_has_no_credential_network_or_ledger_io(cohort, monkeypatch): forbidden = Mock(side_effect=AssertionError("No credentials/ledger IO")) monkeypatch.setattr(cloud, "_runtime_secrets", forbidden) @@ -670,6 +765,25 @@ def blocked(**kwargs): assert time.monotonic() - start < 1 +@pytest.mark.parametrize("observed", [1.0, 1.1]) +def test_inspection_is_not_dispatched_when_deadline_expires_between_reads(observed): + clock = iter([0.9, observed]) + api = SimpleNamespace(inspect_job=Mock()) + assert ( + prepared_batch._status( + {"job_id": "job"}, + {"image": cloud.WORKER_IMAGE}, + "owner", + 1, + api, + SECRETS, + lambda: next(clock), + ) + == "TIMEOUT" + ) + api.inspect_job.assert_not_called() + + @pytest.mark.parametrize( "url", [ diff --git a/examples/hub_validation_pipeline/test_real_tasks.py b/examples/hub_validation_pipeline/test_real_tasks.py index 0dc0997a4..f9fc93f28 100644 --- a/examples/hub_validation_pipeline/test_real_tasks.py +++ b/examples/hub_validation_pipeline/test_real_tasks.py @@ -197,7 +197,12 @@ def test_prepare_pinned_selected_native_trees_and_consumable_manifest(tmp_path): def test_bundle_and_cli_pass_exact_preparation_contract(tmp_path, monkeypatch, capsys): api, download = hf_fakes() panel_path = write_panel(tmp_path) - prepare = Mock() + + def write_plan(task, config, directory, **kwargs): + directory.mkdir(parents=True) + (directory / "plan.json").write_bytes(cloud._json_bytes({"fixture": True})) + + prepare = Mock(side_effect=write_plan) monkeypatch.setattr(cloud, "prepare", prepare) monkeypatch.setattr(real_tasks, "HfApi", Mock(return_value=api)) monkeypatch.setattr(real_tasks, "hf_hub_download", download) @@ -219,6 +224,11 @@ def test_bundle_and_cli_pass_exact_preparation_contract(tmp_path, monkeypatch, c assert call.args[1]["cloud"]["results_repo"] == "example/results" assert call.kwargs["source"] == row["source"] assert row["bundle_plan"] == str(call.args[2] / "plan.json") + assert row["plan_path"] == "bundles/toy-alpha/plan.json" + assert ( + row["plan_sha256"] + == hashlib.sha256(Path(row["bundle_plan"]).read_bytes()).hexdigest() + ) assert json.loads(capsys.readouterr().out)["status"] == "prepared" From 9c2c0b3db9700ec766cb7711db0168792749ef61 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 10:40:45 +0100 Subject: [PATCH 18/19] Address cohort code quality review without weakening failure evidence --- examples/coding_audit/test_target_budget.py | 5 ++++- examples/coding_audit/test_target_tools.py | 5 ++++- examples/hub_validation_pipeline/cloud.py | 2 ++ examples/hub_validation_pipeline/full_run.py | 4 +++- examples/hub_validation_pipeline/prepared_batch.py | 2 +- examples/hub_validation_pipeline/test_cloud.py | 7 ++++--- 6 files changed, 18 insertions(+), 7 deletions(-) diff --git a/examples/coding_audit/test_target_budget.py b/examples/coding_audit/test_target_budget.py index a383a75d2..973ff0192 100644 --- a/examples/coding_audit/test_target_budget.py +++ b/examples/coding_audit/test_target_budget.py @@ -1,4 +1,7 @@ -"""Budget stops preserve valid prefixes; provider violations remain errors.""" +"""Budget stops preserve valid prefixes; provider violations remain errors. + +The shared harness is imported to register its pytest fixture in this module. +""" import hashlib import json diff --git a/examples/coding_audit/test_target_tools.py b/examples/coding_audit/test_target_tools.py index 95d06bf56..608ba7d1e 100644 --- a/examples/coding_audit/test_target_tools.py +++ b/examples/coding_audit/test_target_tools.py @@ -1,4 +1,7 @@ -"""Target tool repair at the real solve, provider-message, and grading seams.""" +"""Target tool repair at the real solve, provider-message, and grading seams. + +The shared harness is imported to register its pytest fixture in this module. +""" import json import time diff --git a/examples/hub_validation_pipeline/cloud.py b/examples/hub_validation_pipeline/cloud.py index 46ff3d566..e27a967fa 100644 --- a/examples/hub_validation_pipeline/cloud.py +++ b/examples/hub_validation_pipeline/cloud.py @@ -1042,6 +1042,8 @@ def worker(root, *, bundle_sha=None, api=None, runner=None): ) report["accepted"] = result.get("accepted") is True except BaseException as error: + # Interrupts must persist the same sanitized failure evidence before the + # worker exits unsuccessfully; they must never look like a completed run. failure = error report.update(status="error", accepted=False, error_type=type(error).__name__) # SDK exceptions can contain credentials; persist type, never raw repr. diff --git a/examples/hub_validation_pipeline/full_run.py b/examples/hub_validation_pipeline/full_run.py index 1774c38a4..f5caa56a3 100644 --- a/examples/hub_validation_pipeline/full_run.py +++ b/examples/hub_validation_pipeline/full_run.py @@ -45,7 +45,9 @@ def _budget(value): if number.is_finite() and number > 0: return Fraction(number) except InvalidOperation: - pass + raise ValueError( + "budget_usd must be a finite positive decimal string" + ) from None raise ValueError("budget_usd must be a finite positive decimal string") diff --git a/examples/hub_validation_pipeline/prepared_batch.py b/examples/hub_validation_pipeline/prepared_batch.py index 605b98b78..b82d88d51 100644 --- a/examples/hub_validation_pipeline/prepared_batch.py +++ b/examples/hub_validation_pipeline/prepared_batch.py @@ -113,7 +113,7 @@ def save(**state): for row in rows: if not isinstance(row["task_id"], str) or not row["task_id"].strip(): raise ValueError("Task IDs must be nonempty strings") - path, (plan, config, _, _) = _plan(root, row) + _, (plan, config, _, _) = _plan(root, row) if row["task_id"] in task_ids or plan["run_id"] in run_ids: raise ValueError("Task IDs, run IDs and plan paths must be distinct") task_ids.add(row["task_id"]) diff --git a/examples/hub_validation_pipeline/test_cloud.py b/examples/hub_validation_pipeline/test_cloud.py index 4da377f64..f7dd93fd6 100644 --- a/examples/hub_validation_pipeline/test_cloud.py +++ b/examples/hub_validation_pipeline/test_cloud.py @@ -341,6 +341,7 @@ def run_command(command, **kwargs): env={"PYTHONPATH": str(metadata)}, **kwargs, ) + return subprocess.CompletedProcess(command, 0) run = Mock(side_effect=run_command) monkeypatch.setattr(subprocess, "run", run) @@ -526,7 +527,7 @@ def test_plan_tampering_stops_before_hub(inputs, tmp_path, section): api.dataset_info.assert_not_called() -@pytest.mark.parametrize("fail", [False, True]) +@pytest.mark.parametrize("fail", [None, RuntimeError, KeyboardInterrupt, SystemExit]) def test_worker_persists_report_and_partial_evidence_on_failure( inputs, tmp_path, monkeypatch, fail ): @@ -539,7 +540,7 @@ def runner(task, config, output): output.mkdir() (output / "evidence.json").write_text('{"attempted":1}\n') if fail: - raise RuntimeError(SECRETS["HF_TOKEN"]) + raise fail(SECRETS["HF_TOKEN"]) return {"accepted": False, "verdict": "skip"} if fail: @@ -559,7 +560,7 @@ def runner(task, config, output): assert report["status"] == ("error" if fail else "completed") assert report["accepted"] is False if fail: - assert report["error_type"] == "RuntimeError" + assert report["error_type"] == fail.__name__ assert all(name.startswith(prefix) for name in operations) assert all(SECRETS["HF_TOKEN"].encode() not in data for data in operations.values()) From baab8461c806c23d3832b978cc5d166e49dafb87 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 11:18:29 +0100 Subject: [PATCH 19/19] Align Hub example documentation with pinned Harbor image support --- .../hub-environment-validation-pipeline.md | 15 +++++++++------ examples/hub_validation_pipeline/README.md | 19 +++++++++++-------- .../hub_validation_pipeline/test_pipeline.py | 2 +- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/docs/plans/hub-environment-validation-pipeline.md b/docs/plans/hub-environment-validation-pipeline.md index 5ff94c180..bac14ae9d 100644 --- a/docs/plans/hub-environment-validation-pipeline.md +++ b/docs/plans/hub-environment-validation-pipeline.md @@ -95,13 +95,16 @@ The contract lives at `validation/openenv.json` in the validation subject and explicitly declares the reward key/range, tolerance, floor margin, resource allocation and optional canary directory. A pipeline-supplied overlay records both the original source digest and the effective validation digest. The current -native adapter supports self-contained, single-step Linux tasks with one -Dockerfile, shared verifier execution and public networking. Unsupported resource, +native adapter supports self-contained, single-step Linux tasks with one main +Docker service, shared verifier execution and public networking. Unsupported resource, credential, multi-service and extension contracts stay named SKIPs. -Tasks declaring `environment.docker_image` are also unsupported, even alongside -a Dockerfile: native trials must build the reviewed Dockerfile. Their startup -SKIP excludes them before model handoff; acquisition preserves their original -bytes. A missing `solution/solve.sh` still produces `semantic.oracle_max` FAIL. +The first trial builds the reviewed Dockerfile or pulls a task-declared +`environment.docker_image` pinned as `name@sha256:`, recorded as +`source_image`. Later trials and model handoff use the inspected immutable image +kept by the startup-recorded retained tag. Mutable tags and tasks with neither +a Dockerfile nor a pinned image remain unsupported; their startup SKIP excludes +them before model handoff. Acquisition preserves the original bytes. A missing +`solution/solve.sh` still produces `semantic.oracle_max` FAIL. Supplemental cloud audits remain a separate profile and cannot waive this gate. ```bash diff --git a/examples/hub_validation_pipeline/README.md b/examples/hub_validation_pipeline/README.md index e5eb00548..728bfa3fa 100644 --- a/examples/hub_validation_pipeline/README.md +++ b/examples/hub_validation_pipeline/README.md @@ -83,16 +83,19 @@ uv run --no-sync openenv validate inputs/task --level semantic --policy v2 \ --json --output outputs/task-report.json ``` -The native adapter supports self-contained single-step Linux tasks with one -Dockerfile, shared verifier execution, public networking and no GPU, task-defined +The native adapter supports self-contained single-step Linux tasks with one main +Docker service, shared verifier execution, public networking and no GPU, task-defined environment variables, healthchecks, external MCP/credential requirements or task-authored Compose. Unsupported shapes remain explicit SKIPs. -The native trials must build the reviewed Dockerfile. A task that declares -`environment.docker_image` is unsupported even when it also contains a Dockerfile; -`runtime.startup` remains SKIP and the pipeline excludes it before model execution. -This includes prebuilt-image benchmark tasks. Acquisition leaves their native -bytes unchanged; the driver does not remove the image declaration to bypass the -gate. A missing `solution/solve.sh` remains `semantic.oracle_max` FAIL even when +The first native trial builds the reviewed Dockerfile or pulls the declared +`environment.docker_image` pinned as `name@sha256:`. The validator records +that declared pin as `source_image`; all subsequent trials and model handoff use +the inspected immutable image kept by the startup-recorded retained tag. +Mutable image tags and tasks with neither a Dockerfile nor a pinned image remain +unsupported: `runtime.startup` SKIPs and the pipeline excludes them before model +execution. Acquisition preserves the native bytes; the driver does not remove an +image declaration to bypass the gate. A missing `solution/solve.sh` remains +`semantic.oracle_max` FAIL even when startup is unsupported. The supplemental HF Jobs coding audit is a separate operator profile and does not waive these native admission requirements. diff --git a/examples/hub_validation_pipeline/test_pipeline.py b/examples/hub_validation_pipeline/test_pipeline.py index e823c290f..96ca47062 100644 --- a/examples/hub_validation_pipeline/test_pipeline.py +++ b/examples/hub_validation_pipeline/test_pipeline.py @@ -482,7 +482,7 @@ def test_admission_scales_thresholds_by_reward_range( @pytest.mark.parametrize("missing_oracle", [False, True]) -def test_native_prebuilt_task_is_excluded_without_model_or_docker( +def test_native_unpinned_image_is_excluded_without_model_or_docker( tmp_path, monkeypatch, missing_oracle ): from openenv.validation.harbor import DockerInspector