diff --git a/examples/coding_audit/README.md b/examples/coding_audit/README.md new file mode 100644 index 000000000..b8b8a5ca0 --- /dev/null +++ b/examples/coding_audit/README.md @@ -0,0 +1,66 @@ +# Operator coding audit + +This example collects and scores bounded coding-task evidence. It belongs to the +operator workflow, separate from the installed OpenEnv validator and its RFC 008 +checks. Acceptance here does not establish full RFC 008 conformance, containment, +served-model revision attestation, or training benefit. + +From a repository checkout, install the tested SDK and run the example: + +```bash +uv sync --group dev +uv pip install 'huggingface_hub==1.33.0' +PYTHONPATH=src:envs uv run --no-sync python -m examples.coding_audit \ + --task /path/to/task \ + --config examples/coding_audit/config.json \ + --output /path/to/new-audit-directory \ + --json +``` + +The output directory is required and must be new. It retains `report.json`, +`evidence.json`, `config.json`, raw evidence artifacts, and `SHA256SUMS`. Exit codes +are `0` for accepted evidence, `1` for rejected or incomplete evidence, and `2` for +configuration or input failure. `--json` prints the report as well. + +`scoring.py` provides pure `assess_coding_audit` and `validate_audit_config` +functions. `runner.py` provides `normalize_coding_config`, `coding_budget`, +`validate_coding_task`, and `run_coding_validation`. `hf.py` contains +`HFTaskBackend`, the bounded Sandbox and Inference Providers implementation. +The schemas bind task, verifier, immutable image references and independent execution identities. + +Models may declare an explicit boolean `enable_thinking`, forwarded through the +HF client's `extra_body.chat_template_kwargs`. It is omitted by default. A +candidate-only override leaves the target protocol unchanged and is recorded +alongside request parameters. Support depends on the selected provider/model. +Candidate generation streams bounded content and reasoning; only a complete +`stop` response with validated usage can produce an executable candidate. A +`length` response remains rejected even when its terminal usage is retained. +Recognized Bash/sh Markdown fences are parsed before execution; the raw response +and both raw/script hashes are retained. Other language fences are rejected. + +Candidate generation can opt into public task API context by adding +`"context_paths": ["environment/source/package/api.py"]` to the `generator` +model configuration. Only those declared regular UTF-8 files are read; paths must +be unique, normalized, and below `environment/source/`. Up to 32 files and 65,536 +combined bytes are allowed. Linked files or parents, hidden or private paths, +NUL bytes, invalid UTF-8 and oversized context are rejected before inference. +The existing message and token budgets still apply. No source files are added by +default, and `target.context_paths` is rejected. + +The candidate prompt labels each source path and its runtime filename relative to +the configured workdir, with `environment/source/` removed. Each generation record +retains `candidate_context` entries containing `path`, `runtime_path`, `sha256`, +and `size`, bound to the frozen task and checksummed evidence. This option does +not enumerate task directories or include verifier, reference or specification +files. + +The HF backend launches paid services when invoked without a supplied test +backend. Use reviewed tasks and distinct execution and inference credentials. +The Hub pipeline's cloud worker packages these production example modules with +its frozen source and task bundle; its tests stay outside the bundle. + +Run the offline suite with: + +```bash +PYTHONPATH=.:src:envs uv run pytest examples/coding_audit -q +``` diff --git a/examples/coding_audit/__init__.py b/examples/coding_audit/__init__.py new file mode 100644 index 000000000..1169e0fb8 --- /dev/null +++ b/examples/coding_audit/__init__.py @@ -0,0 +1 @@ +"""Operator-owned coding audits, separate from core RFC 008 validation.""" diff --git a/examples/coding_audit/__main__.py b/examples/coding_audit/__main__.py new file mode 100644 index 000000000..5ace417df --- /dev/null +++ b/examples/coding_audit/__main__.py @@ -0,0 +1,63 @@ +"""Run a bounded operator coding audit and retain its evidence bundle.""" + +import argparse +import signal +import sys +import threading +from contextlib import contextmanager +from pathlib import Path + +from openenv.validation.runtime.json_values import dumps, loads + +from .runner import run_coding_validation + + +@contextmanager +def _sigterm_as_interrupt(): + if threading.current_thread() is not threading.main_thread(): + yield + return + + def interrupt(signum, frame): + raise KeyboardInterrupt + + previous = signal.signal(signal.SIGTERM, interrupt) + try: + yield + finally: + signal.signal(signal.SIGTERM, previous) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--task", type=Path, required=True) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument( + "--output", + type=Path, + required=True, + help="New directory for the report, evidence and checksummed artifacts", + ) + parser.add_argument("--json", action="store_true", dest="json_output") + args = parser.parse_args(argv) + try: + config = loads(args.config.read_text()) + with _sigterm_as_interrupt(): + report = run_coding_validation(args.task, config, args.output) + except (OSError, ValueError) as error: + print( + f"Coding audit configuration/input failure ({type(error).__name__})", + file=sys.stderr, + ) + return 2 + if args.json_output: + print(dumps(report)) + else: + print(f"Coding audit: {report['verdict'].upper()} ({report['profile']})") + for name, check in report["checks"].items(): + print(f" {check['status'].upper():5s} {name}") + return 0 if report["accepted"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/coding_audit/config.json b/examples/coding_audit/config.json new file mode 100644 index 000000000..bfb7b3912 --- /dev/null +++ b/examples/coding_audit/config.json @@ -0,0 +1,83 @@ +{ + "schema_version": 1, + "profile": "harbor-coding-audit-v1", + "target": { + "model_id": "Qwen/Qwen3.8-27B", + "revision": "37561816e66b5b9153b1b64c10968c18c1e6f4a0", + "provider": "deepinfra", + "supports_seed": false, + "supports_tools": true, + "temperature": 0.7, + "max_steps": 16, + "max_tokens": 2048, + "max_total_tokens": 16384 + }, + "execution": { + "agent_image": "docker.io/library/python@sha256:48d884813cf44e927cf64c5529dd630dd3b18338c6dc727991eef9b71e934a20", + "verifier_image": "docker.io/library/python@sha256:48d884813cf44e927cf64c5529dd630dd3b18338c6dc727991eef9b71e934a20", + "flavor": "cpu-basic", + "timeout_seconds": 120, + "workdir": "/app", + "submission_paths": [ + "/app/answer.txt" + ], + "private_paths": [ + "/tests", + "/solution" + ], + "verifier_command": [ + "/bin/bash", + "/tests/test.sh" + ], + "reward_path": "/logs/verifier/reward.txt", + "reward_key": "reward" + }, + "validation": { + "reference_repeats": 3, + "target_attempts": 8, + "reward_min": 0, + "reward_max": 1, + "oracle_tolerance": 0, + "floor_margin": 0.1, + "wrong_answer_max": 0, + "required_negative_cases": [ + "wrong-value", + "empty-answer", + "near-miss" + ] + }, + "generation": { + "candidate_count": 2 + }, + "negative_cases": [ + { + "id": "wrong-value", + "script": "mkdir -p /app; printf '0\\n' > /app/answer.txt" + }, + { + "id": "empty-answer", + "script": "mkdir -p /app; : > /app/answer.txt" + }, + { + "id": "near-miss", + "script": "mkdir -p /app; printf '41\\n' > /app/answer.txt" + } + ], + "seeds": [ + 1001, + 1002, + 1003, + 1004, + 1005, + 1006, + 1007, + 1008 + ], + "cloud": { + "results_repo": "example-org/private-audit-results", + "flavor": "cpu-basic", + "timeout_seconds": 1800, + "max_sandboxes": 64, + "max_inference_calls": 160 + } +} diff --git a/examples/coding_audit/hf.py b/examples/coding_audit/hf.py new file mode 100644 index 000000000..f615bd382 --- /dev/null +++ b/examples/coding_audit/hf.py @@ -0,0 +1,1290 @@ +"""Bounded HF Jobs coding trials for reviewed tasks, with controller-side inference. + +Image receipts attest only the requested immutable control-plane reference. The +Sandbox RPC bearer reaches a workload-controlled server; this is not a hostile +code security boundary or a served-model weight revision guarantee. +""" + +import hashlib +import json +import math +import os +import re +import signal +import stat +import threading +import time +import uuid +from contextlib import closing, contextmanager +from decimal import Decimal +from importlib.metadata import version +from pathlib import Path, PurePosixPath + +from huggingface_hub import HfApi, InferenceClient, Sandbox +from huggingface_hub.errors import SandboxError + +from .runner import _candidate_context_paths + +_IMAGE = re.compile(r"[^\s@]+@sha256:[0-9a-f]{64}\Z") +_REVISION = re.compile(r"[0-9a-f]{40}\Z") +_TERMINAL = {"COMPLETED", "CANCELED", "ERROR", "DELETED"} +_LIMITATIONS = [ + "Immutable images are control-plane references, not measured runtime digests.", + "Sandbox RPC exposes the execution bearer to the workload server; hostile-code credential isolation is not qualified.", + "The verifier executes submitted code as the same user; verifier integrity against hostile submissions is not qualified.", + "Inference Providers do not pin served weights to the recorded model source revision.", +] +_TOOL = { + "type": "function", + "function": { + "name": "terminal", + "description": "Run a shell command in the task workspace.", + "parameters": { + "type": "object", + "properties": {"command": {"type": "string"}}, + "required": ["command"], + "additionalProperties": False, + }, + }, +} + + +class HFBackendError(ValueError): + """A credential-free controller diagnostic.""" + + +class _DeadlineExceeded(HFBackendError): + pass + + +@contextmanager +def _deadline(seconds): + # SDK control-plane calls have no timeout argument. A controller must run on + # the main Unix thread so a blocked synchronous SDK call can be interrupted. + if threading.current_thread() is not threading.main_thread(): + raise HFBackendError("HF coding controller requires the main thread") + previous_handler = signal.getsignal(signal.SIGALRM) + previous_timer = signal.getitimer(signal.ITIMER_REAL) + started = time.monotonic() + + def expired(signum, frame): + raise _DeadlineExceeded("controller deadline exceeded") + + signal.signal(signal.SIGALRM, expired) + signal.setitimer( + signal.ITIMER_REAL, + min(seconds, previous_timer[0]) if previous_timer[0] else seconds, + 0.1, + ) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_handler) + if previous_timer[0]: + signal.setitimer( + signal.ITIMER_REAL, + max(0.000001, previous_timer[0] - (time.monotonic() - started)), + previous_timer[1], + ) + + +def _path(value): + if not isinstance(value, str) or "\x00" in value: + raise HFBackendError("paths must be absolute normalized file paths") + path = PurePosixPath(value) + if ( + not path.is_absolute() + or str(path) != value + or ".." in path.parts + or value == "/" + or value.startswith("//") + ): + raise HFBackendError("paths must be absolute normalized file paths") + return value + + +def _under(path, parent): + return path == parent or path.startswith(parent + "/") + + +def _candidate_script(raw): + lines = raw.lstrip().splitlines(keepends=True) + opening = lines[0].strip() + if not opening.startswith("```"): + return raw, None + if opening not in {"```bash", "```sh", "```shell"}: + raise HFBackendError("candidate code fence must declare Bash or sh") + end = len(lines) + while end > 1 and not lines[end - 1].strip(): + end -= 1 + closed = end > 1 and lines[end - 1].strip() == "```" + script = "".join(lines[1 : end - 1] if closed else lines[1:]) + if not script.strip(): + raise HFBackendError("candidate response must contain a shell script") + return script, { + "wrapper": "markdown_bash", + "closing_fence_present": closed, + "raw_sha256": hashlib.sha256(raw.encode()).hexdigest(), + "script_sha256": hashlib.sha256(script.encode()).hexdigest(), + } + + +def submission_digest(files): + """Hash sorted relative paths, NUL, content, NUL as source_digest does.""" + digest = hashlib.sha256() + for name, content in sorted(files.items()): + digest.update(name.lstrip("/").encode()) + digest.update(b"\0") + digest.update(content) + digest.update(b"\0") + return digest.hexdigest() + + +def _local_file(path, limit): + # Walk descriptors so neither a final link nor a linked parent is followed. + path = Path(path).absolute() + fd = os.open("/", os.O_RDONLY | os.O_DIRECTORY) + try: + for component in path.parts[1:-1]: + next_fd = os.open( + component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd + ) + os.close(fd) + fd = next_fd + file_fd = os.open( + path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd + ) + with os.fdopen(file_fd, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise HFBackendError("task artifacts must be regular unlinked files") + if info.st_size > limit: + raise HFBackendError("task artifact exceeds byte limit") + content = stream.read(limit + 1) + if len(content) > limit: + raise HFBackendError("task artifact exceeds byte limit") + return content, format(stat.S_IMODE(info.st_mode), "o") + finally: + os.close(fd) + + +def _plain(value): + # HF generated output types derive from dict and preserve extra response fields. + return json.loads(json.dumps(value)) + + +def _unique_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise HFBackendError("duplicate JSON key") + result[key] = value + return result + + +class HFTaskBackend: + """Execute each solve and grade in a fresh, dedicated HF Jobs Sandbox.""" + + def __init__( + self, + task_root: Path, + execution: dict, + *, + token=None, + api=None, + sandbox_factory=None, + inference_factory=None, + canary: str | None = None, + inference_token=None, + ): + if version("huggingface_hub") != "1.33.0": + raise HFBackendError( + "this execution adapter requires huggingface_hub==1.33.0" + ) + self.task_root = Path(task_root) + self.execution = dict(execution) + self.token = token if token is not None else os.environ.get("HF_SANDBOX_TOKEN") + self.inference_token = ( + inference_token + if inference_token is not None + else os.environ.get("HF_INFERENCE_TOKEN") + ) + credentials = [ + value + for value in (self.token, self.inference_token, os.environ.get("HF_TOKEN")) + if value + ] + if len(set(credentials)) != len(credentials): + raise HFBackendError( + "execution, inference and persistence credentials must differ" + ) + if execution.get("strong_isolation") or execution.get( + "require_strong_isolation" + ): + raise HFBackendError("strong verifier isolation is unsupported") + for key in ("agent_image", "verifier_image"): + if not isinstance(execution.get(key), str) or not _IMAGE.fullmatch( + execution[key] + ): + raise HFBackendError( + f"{key} must be an immutable OCI name@sha256 reference" + ) + self.workdir = _path(execution.get("workdir", "/app")) + self.private_paths = list( + dict.fromkeys( + _path(item) + for item in ["/tests", "/solution", *execution.get("private_paths", [])] + ) + ) + self.reward_path = _path( + execution.get("reward_path", "/logs/verifier/reward.txt") + ) + paths = execution.get("submission_paths") + if not isinstance(paths, list) or not paths: + raise HFBackendError("submission_paths must be a nonempty file allowlist") + self.submission_paths = [_path(item) for item in paths] + for index, path in enumerate(self.submission_paths): + if any( + _under(path, protected) or _under(protected, path) + for protected in [ + "/tests", + "/solution", + self.reward_path, + *self.private_paths, + ] + ): + raise HFBackendError("submission path overlaps private or reward paths") + if any( + _under(path, other) or _under(other, path) + for other in self.submission_paths[:index] + ): + raise HFBackendError("submission paths overlap or repeat") + self.timeout = self._bound(execution, "timeout_seconds", 120, 3600) + self.max_bytes = self._bound( + execution, "max_submission_bytes", 1048576, 16 * 1048576 + ) + self.max_script = self._bound(execution, "max_script_bytes", 65536, 1048576) + self.max_output = self._bound( + execution, "max_tool_output_bytes", 16384, 1048576 + ) + self.cleanup_attempts = self._bound(execution, "cleanup_attempts", 3, 10) + self.cleanup_timeout = self._bound(execution, "cleanup_timeout_seconds", 10, 60) + self.api = api if api is not None else HfApi(token=self.token) + self.sandbox_factory = sandbox_factory or Sandbox.create + self.inference_factory = inference_factory or InferenceClient + self.canary = canary if canary is not None else uuid.uuid4().hex + if not isinstance(self.canary, str) or not self.canary: + raise HFBackendError("canary must be nonempty text") + self.canary_digest = hashlib.sha256(self.canary.encode()).hexdigest() + self.marker_path = str( + PurePosixPath(self.workdir) / (".openenv-state-" + uuid.uuid4().hex) + ) + self._audit = [] + self._transcripts = [] + self._owned_ids = set() + self._credentials = tuple(credentials) + + def _redact(self, value): + if isinstance(value, str): + for secret in self._credentials: + value = value.replace(secret, "[REDACTED]") + elif isinstance(value, bytes): + for secret in self._credentials: + value = value.replace(secret.encode(), b"[REDACTED]") + elif isinstance(value, dict): + value = { + self._redact(key): self._redact(item) for key, item in value.items() + } + elif isinstance(value, list): + value = [self._redact(item) for item in value] + return value + + def _has_canary(self, value): + if isinstance(value, str): + return self.canary in value + if isinstance(value, dict): + return any(self._has_canary(item) for item in value.values()) + if isinstance(value, list): + return any(self._has_canary(item) for item in value) + return False + + @staticmethod + def _bound(config, key, default, maximum): + value = config.get(key, default) + if type(value) is not int or not 0 < value <= maximum: + raise HFBackendError(f"{key} must be an integer from 1 to {maximum}") + return value + + def _record(self): + return { + "files": {}, + "submission_digest": submission_digest({}), + "sandbox_id": None, + "image_ref": self.execution["agent_image"], + "image_reference_kind": "control_plane_immutable_reference", + "cleanup_verified": False, + "error": None, + "transcript": [], + "usage": {"input_tokens": 0, "output_tokens": 0}, + "input_tokens": 0, + "output_tokens": 0, + "latency_seconds": 0.0, + "model_metadata": {}, + "limitations": list(_LIMITATIONS), + } + + @staticmethod + def _error(exc): + return ( + str(exc) + if isinstance(exc, HFBackendError) + else f"HF backend operation failed ({type(exc).__name__})" + ) + + def _inspect(self, sandbox_id): + job = self.api.inspect_job( + job_id=sandbox_id, + namespace=self.execution.get("namespace"), + token=self.token, + ) + if job.id != sandbox_id or ( + self.execution.get("namespace") + and job.owner.name != self.execution["namespace"] + ): + raise HFBackendError("control-plane job identity mismatch") + return job + + def _create(self, image, record, *, grader=False): + if not self.token: + raise HFBackendError("HF_SANDBOX_TOKEN execution credential is required") + record["creation_label"] = uuid.uuid4().hex + sandbox = self.sandbox_factory( + image=image, + flavor=self.execution.get("flavor", "cpu-basic"), + idle_timeout=self.timeout, + namespace=self.execution.get("namespace"), + forward_hf_token=False, + token=self.token, + start_timeout=self.timeout, + labels={ + "name": "openenv-coding-" + record["creation_label"], + "openenv-coding-attempt": record["creation_label"], + }, + ) + sandbox_id = sandbox.id + # Register before inspection so image/identity failures still clean up. + record["grade_sandbox_id" if grader else "sandbox_id"] = sandbox_id + if sandbox_id in self._owned_ids: + raise HFBackendError("sandbox identity was reused") + self._owned_ids.add(sandbox_id) + # SDK readiness checks the RPC server; the Jobs status can still lag. + # Reconcile only SCHEDULING within the caller's existing deadline, and + # recheck identity and the immutable image on every observation. + for attempt in range(20): + job = self._inspect(sandbox_id) + if job.docker_image != image: + raise HFBackendError("sandbox control-plane image mismatch") + if job.status.stage == "RUNNING": + break + if job.status.stage != "SCHEDULING": + raise HFBackendError("sandbox control-plane state is not runnable") + if attempt < 19: + time.sleep(0.5) + else: + raise HFBackendError("sandbox control-plane readiness did not converge") + # The SDK stat/read pair otherwise permits up to 512 MiB per read. + sandbox.files.MAX_READ_BYTES = self.max_bytes + sandbox.files.mkdir(self.workdir) + return sandbox + + def _recover_startup_jobs(self, record, pending): + # create() can allocate a job before failing during readiness. The label + # is unique to this attempt and is filtered again before any cancellation. + jobs = self.api.list_jobs( + labels={"openenv-coding-attempt": record["creation_label"]}, + namespace=self.execution.get("namespace"), + token=self.token, + timeout=self.cleanup_timeout, + ) + for index, job in enumerate(jobs): + if ( + index >= 4 + or (job.labels or {}).get("openenv-coding-attempt") + != record["creation_label"] + ): + raise HFBackendError("startup cleanup ownership could not be verified") + if ( + self.execution.get("namespace") + and job.owner.name != self.execution["namespace"] + ): + raise HFBackendError("startup cleanup namespace mismatch") + pending.add(job.id) + self._owned_ids.add(job.id) + record["cleanup_job_ids"] = sorted(pending) + record[ + "grade_sandbox_id" if "grade_sandbox_id" in record else "sandbox_id" + ] = sorted(pending)[0] + + def _cleanup(self, record, sandbox): + sandbox_id = record.get("grade_sandbox_id") or record.get("sandbox_id") + if sandbox_id is None and not record.get("creation_label"): + return + pending = {sandbox_id} if sandbox_id is not None else set() + recovery_complete = sandbox_id is not None + try: + with _deadline(self.cleanup_timeout): + for _ in range(self.cleanup_attempts): + if not recovery_complete: + try: + self._recover_startup_jobs(record, pending) + recovery_complete = bool(pending) + except _DeadlineExceeded: + raise + except Exception: + pass + if not pending: + continue + for owned_id in list(pending): + try: + self.api.cancel_job( + job_id=owned_id, + namespace=self.execution.get("namespace"), + token=self.token, + ) + except _DeadlineExceeded: + raise + except Exception: + if sandbox is not None and sandbox.id == owned_id: + try: + sandbox.kill() + except _DeadlineExceeded: + raise + except Exception: + pass + try: + if self._inspect(owned_id).status.stage in _TERMINAL: + pending.remove(owned_id) + except _DeadlineExceeded: + raise + except Exception: + continue + if not pending and recovery_complete: + record["cleanup_verified"] = True + break + except Exception: + pass + if not record["cleanup_verified"]: + record["error"] = record["error"] or "sandbox cleanup could not be verified" + if "reward" in record: + record["reward"] = None + + def _audit_solver(self, sandbox): + evidence = { + "sandbox_id": sandbox.id, + "private_paths_absent": False, + "fresh_state_absent": False, + } + self._audit.append(evidence) + for path in self.private_paths: + if sandbox.files.exists(path): + raise HFBackendError("private path exists in solver sandbox") + evidence["private_paths_absent"] = True + if sandbox.files.exists(self.marker_path): + raise HFBackendError("fresh sandbox contains prior-state marker") + evidence["fresh_state_absent"] = True + sandbox.files.write(self.marker_path, self.canary_digest.encode()) + + def _run(self, sandbox, argv, end, *, cwd=None, trace=None): + output = [] + size = 0 + + def collect(chunk): + nonlocal size + raw = chunk.encode() + if size + len(raw) > self.max_output: + if trace is not None: + trace["output_truncated"] = True + raise HFBackendError("terminal output exceeded byte limit") + size += len(raw) + output.append(chunk) + + try: + remaining = end - time.monotonic() + if remaining <= 0: + raise HFBackendError("controller deadline exceeded") + result = sandbox.run( + argv, + shell=False, + cwd=cwd or self.workdir, + timeout=remaining, + check=False, + capture_output=False, + on_stdout=collect, + on_stderr=collect, + ) + if trace is not None: + trace["exit_code"] = result.exit_code + if result.timed_out: + raise HFBackendError("sandbox command timed out") + return result.exit_code, "".join(output) + finally: + if trace is not None: + trace["output"] = "" if trace["output_truncated"] else "".join(output) + + def _collect(self, sandbox): + files = {} + remaining = self.max_bytes + for path in self.submission_paths: + try: + entry = sandbox.files.stat(path) + except SandboxError as exc: + if exc.status_code == 404: + continue + raise + if ( + entry.type != "file" + or type(entry.size) is not int + or not 0 <= entry.size <= remaining + ): + raise HFBackendError("submission is not a bounded regular file") + sandbox.files.MAX_READ_BYTES = remaining + data = sandbox.files.read(path) + if ( + not isinstance(data, bytes) + or len(data) != entry.size + or len(data) > remaining + ): + raise HFBackendError("submission changed size or exceeded byte limit") + files[path] = data + if any(secret.encode() in data for secret in self._credentials): + raise HFBackendError("credential detected in submission artifact") + remaining -= len(data) + return files + + def execute_script(self, script: str, role: str) -> dict: + """Execute a reference/candidate script without exposing controller credentials.""" + record = self._record() + record["role"] = role + sandbox = None + started = time.monotonic() + try: + with _deadline(self.timeout): + if ( + not isinstance(script, str) + or len(script.encode()) > self.max_script + ): + raise HFBackendError("script exceeds byte limit or is not text") + sandbox = self._create(self.execution["agent_image"], record) + self._audit_solver(sandbox) + exit_code, output = self._run( + sandbox, ["/bin/bash", "-c", script], started + self.timeout + ) + record["transcript"].append( + { + "role": role, + "script": script, + "output": output, + "exit_code": exit_code, + } + ) + if exit_code != 0: + raise HFBackendError("task script exited nonzero") + record["files"] = self._collect(sandbox) + record["submission_digest"] = submission_digest(record["files"]) + except Exception as exc: + record["error"] = self._error(exc) + finally: + self._cleanup(record, sandbox) + record["latency_seconds"] = time.monotonic() - started + self._transcripts.append(record["transcript"]) + return self._redact(record) + + def _model(self, model, seed, *, tools=False): + model = dict(model) + if tools and "context_paths" in model: + raise HFBackendError("target.context_paths is unsupported") + if ( + not isinstance(model.get("model_id"), str) + or not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", model["model_id"]) + or not isinstance(model.get("revision"), str) + or not _REVISION.fullmatch(model["revision"]) + or not isinstance(model.get("provider"), str) + or not model["provider"] + or model["provider"] in {"auto", "cheapest", "fastest", "preferred"} + ): + raise HFBackendError( + "model_id, pinned source revision and explicit provider are required" + ) + if type(seed) is not int or seed < 0: + raise HFBackendError("seed must be a nonnegative integer") + if tools and model.get("supports_tools") is False: + raise HFBackendError( + "configured model does not support terminal function calling" + ) + if "enable_thinking" in model and type(model["enable_thinking"]) is not bool: + raise HFBackendError("enable_thinking must be a boolean") + if type(model.get("supports_seed", False)) is not bool: + raise HFBackendError("supports_seed must be an explicit boolean") + aliases = model.get("response_model_aliases", []) + if not isinstance(aliases, list) or any( + not isinstance(item, str) or not item for item in aliases + ): + raise HFBackendError( + "response_model_aliases must be exact nonempty strings" + ) + allowed = {model["model_id"], *aliases} + if model.get("provider_model_id"): + if not isinstance(model["provider_model_id"], str): + raise HFBackendError("provider_model_id must be text") + allowed.add(model["provider_model_id"]) + limits = { + key: self._bound(model, key, default, maximum) + for key, default, maximum in ( + ("max_steps", 8, 128), + ("max_tokens", 2048, 32768), + ("max_total_tokens", 16384, 1048576), + ("max_output_bytes", 16384, 1048576), + ) + } + if not self.inference_token: + raise HFBackendError("controller HF_INFERENCE_TOKEN is required") + client = self.inference_factory( + provider=model["provider"], + api_key=self.inference_token, + timeout=self.timeout, + ) + return model, allowed, limits, client + + def _chat(self, client, model, allowed, limits, messages, record, seed, *, tools): + used = record["input_tokens"] + record["output_tokens"] + # Bound the request body as well as provider-reported usage. Providers do + # not expose a preflight tokenizer through this API. + message_bytes = len(json.dumps(messages).encode()) + if message_bytes > limits["max_total_tokens"] * 4: + raise HFBackendError("chat context exceeds byte budget") + remaining = limits["max_total_tokens"] - used + if remaining <= 0: + raise HFBackendError("model total token budget exhausted") + request = { + "messages": self._redact(messages), + "model": model["model_id"], + "max_tokens": min(limits["max_tokens"], remaining), + "temperature": model.get("temperature", 0), + } + if "top_p" in model: + value = model["top_p"] + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or not 0 < value <= 1 + ): + raise HFBackendError("top_p must be a finite number in (0, 1]") + request["top_p"] = value + if "enable_thinking" in model: + request["extra_body"] = { + "chat_template_kwargs": {"enable_thinking": model["enable_thinking"]} + } + if model.get("supports_seed", False): + request["seed"] = seed + if tools: + request.update(tools=[_TOOL], tool_choice="auto") + if not tools: + request.update(stream=True, stream_options={"include_usage": True}) + metadata = record["model_metadata"] + metadata.setdefault("requests", []).append( + {key: value for key, value in request.items() if key != "messages"} + ) + if tools: + response = _plain(client.chat_completion(**request)) + metadata.setdefault("responses", []).append( + { + "model": response.get("model"), + "system_fingerprint": response.get("system_fingerprint"), + "id": response.get("id"), + "seed_sent": request.get("seed"), + } + ) + else: + response = self._candidate_response( + client, request, allowed, limits, record + ) + if response.get("model") not in allowed: + raise HFBackendError( + "provider response model does not match configured exact identities" + ) + usage = response.get("usage") + if not isinstance(usage, dict): + raise HFBackendError("provider response omitted token usage") + for source, target in ( + ("prompt_tokens", "input_tokens"), + ("completion_tokens", "output_tokens"), + ): + value = usage.get(source) + if type(value) is not int or value < 0: + raise HFBackendError("provider returned invalid token usage") + if tools: + record[target] += value + record["usage"][target] += value + if ( + usage["completion_tokens"] > request["max_tokens"] + or record["input_tokens"] + record["output_tokens"] + > limits["max_total_tokens"] + ): + raise HFBackendError("provider exceeded token budget") + choices = response.get("choices", []) + if len(choices) != 1 or not isinstance(choices[0].get("message"), dict): + raise HFBackendError("provider must return exactly one assistant message") + message = choices[0]["message"] + record["transcript"].append(message) + if len(json.dumps(message).encode()) > limits["max_output_bytes"]: + raise HFBackendError("model output exceeded byte limit") + if choices[0].get("finish_reason") not in {"stop", "tool_calls"}: + raise HFBackendError("model response was incomplete") + if choices[0]["finish_reason"] == "tool_calls" and ( + not isinstance(message.get("tool_calls"), list) or not message["tool_calls"] + ): + raise HFBackendError("invalid terminal function calls") + return message + + def _candidate_response(self, client, request, allowed, limits, record): + # The pinned SDK yields parsed deltas but owns HTTP contexts on the client. + response = {} + parts = {"content": []} + size = script_size = 0 + stopped = False + finish_reason = None + usage = None + try: + with closing(client.chat_completion(**request)) as stream: + for parsed in stream: + event = _plain(parsed) + if not isinstance(event, dict) or event.get("error") is not None: + raise HFBackendError("provider returned a stream error") + # Outer SDK dicts omit extra fields on nested dataclasses. + if isinstance(event.get("choices"), list): + for index, choice in enumerate(parsed.choices): + event["choices"][index]["delta"] = _plain(choice.delta) + if not response: + response = { + key: event.get(key) + for key in ("model", "id", "system_fingerprint") + } + record["model_metadata"]["responses"].append( + {**response, "seed_sent": request.get("seed")} + ) + if event.get("model") not in allowed: + raise HFBackendError( + "provider response model does not match configured exact identities" + ) + if ( + not isinstance(event.get("id"), str) + or not event["id"] + or any( + event.get(key) != value for key, value in response.items() + ) + ): + raise HFBackendError( + "provider stream response identity changed" + ) + if usage is not None: + raise HFBackendError("provider returned data after final usage") + choices = event.get("choices") + if not isinstance(choices, list) or len(choices) > 1: + raise HFBackendError( + "provider must stream exactly one assistant choice" + ) + if choices: + if stopped: + raise HFBackendError( + "provider returned a choice after stop" + ) + choice = choices[0] + if ( + not isinstance(choice, dict) + or type(choice.get("index")) is not int + or choice["index"] != 0 + or not isinstance(choice.get("delta"), dict) + ): + raise HFBackendError( + "provider must stream exactly one assistant choice at index 0" + ) + delta = choice["delta"] + if delta.get("role") not in {None, "assistant"} or ( + not parts["content"] and delta.get("role") != "assistant" + ): + raise HFBackendError( + "provider must stream an assistant message" + ) + for key, value in delta.items(): + if key == "role" or value is None: + continue + if key == "tool_calls" and value == []: + continue + if key not in {"content", "reasoning", "reasoning_content"}: + raise HFBackendError( + "candidate stream contains tool calls or unsupported deltas" + ) + if not isinstance(value, str): + raise HFBackendError( + "provider stream delta must be text" + ) + length = len(value.encode()) + size += length + if key == "content": + script_size += length + if ( + size > limits["max_output_bytes"] + or script_size > self.max_script + ): + raise HFBackendError("model output exceeded byte limit") + if value: + parts.setdefault(key, []).append(value) + # Empty assistant role events still establish the message. + if not parts["content"]: + parts["content"].append("") + finish = choice.get("finish_reason") + if finish is not None: + finish_reason = finish + record["model_metadata"]["responses"][-1][ + "finish_reason" + ] = finish + if finish not in {"stop", "length"}: + raise HFBackendError("model response was incomplete") + stopped = True + elif not stopped: + raise HFBackendError( + "model response was incomplete before usage" + ) + event_usage = event.get("usage") + if event_usage is not None: + if not stopped: + raise HFBackendError("provider returned usage before stop") + if not isinstance(event_usage, dict) or any( + type(event_usage.get(key)) is not int + or event_usage[key] < 0 + for key in ( + "prompt_tokens", + "completion_tokens", + "total_tokens", + ) + ): + raise HFBackendError( + "provider returned invalid token usage" + ) + if event_usage["total_tokens"] != ( + event_usage["prompt_tokens"] + + event_usage["completion_tokens"] + ): + raise HFBackendError( + "provider returned inconsistent token usage" + ) + usage = event_usage + # Usage already observed remains evidence if the stream + # fails later. The caller accounts only non-streamed calls. + for source, target in ( + ("prompt_tokens", "input_tokens"), + ("completion_tokens", "output_tokens"), + ): + record[target] += usage[source] + record["usage"][target] += usage[source] + if ( + usage["completion_tokens"] > request["max_tokens"] + or record["input_tokens"] + record["output_tokens"] + > limits["max_total_tokens"] + ): + raise HFBackendError("provider exceeded token budget") + elif not choices: + raise HFBackendError("provider response omitted token usage") + if not stopped: + raise HFBackendError("model response was incomplete") + if usage is None: + raise HFBackendError("provider response omitted token usage") + if finish_reason != "stop": + raise HFBackendError("model response was incomplete") + response.update( + usage=usage, + choices=[ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + **{key: "".join(value) for key, value in parts.items()}, + }, + } + ], + ) + return response + except Exception: + if parts["content"]: + record["transcript"].append( + { + "role": "assistant", + **{key: "".join(value) for key, value in parts.items()}, + } + ) + raise + finally: + client.close() + + def _model_metadata(self, model, seed): + return { + "requested_model": model["model_id"], + "source_revision": model["revision"], + "provider": model["provider"], + "provider_model_id": model.get("provider_model_id"), + "response_model_aliases": model.get("response_model_aliases", []), + "seed": seed, + "seed_supported": model.get("supports_seed", False), + "served_weight_revision_pinned": False, + "responses": [], + } + + def _instruction(self): + raw, _ = _local_file(self.task_root / "instruction.md", self.max_script) + return raw.decode("utf-8") + + def _candidate_instruction(self, model, record): + instruction = self._instruction() + if "context_paths" not in model: + return instruction + paths = _candidate_context_paths(model["context_paths"]) + record["candidate_context"] = [] + parts = [instruction] + remaining = 65536 + for path in paths: + raw, _ = _local_file(self.task_root / path, remaining) + remaining -= len(raw) + if b"\0" in raw: + raise HFBackendError("candidate context contains NUL") + text = raw.decode("utf-8") + runtime_path = ( + PurePosixPath(path).relative_to("environment/source").as_posix() + ) + record["candidate_context"].append( + { + "path": path, + "runtime_path": runtime_path, + "size": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + } + ) + parts.append( + f"Public source path: {path}\n" + f"Runtime filename relative to {self.workdir}: {runtime_path}\n" + f"---\n{text}\n---" + ) + return "\n\n".join(parts) + + def generate_candidate(self, model: dict, seed: int) -> dict: + """Generate a shell script from the instruction and explicit public context.""" + record = self._record() + record["script"] = "" + started = time.monotonic() + try: + with _deadline(self.timeout): + instruction = self._candidate_instruction(model, record) + model, allowed, limits, client = self._model(model, seed) + record["model_metadata"] = self._model_metadata(model, seed) + messages = [ + { + "role": "system", + "content": "Return only a bash script that attempts the task. Do not use Markdown fences.", + }, + {"role": "user", "content": instruction}, + ] + record["transcript"] = messages + message = self._chat( + client, model, allowed, limits, messages, record, seed, tools=False + ) + script = message.get("content") + if ( + not isinstance(script, str) + or not script.strip() + or message.get("tool_calls") + ): + raise HFBackendError( + "candidate response must contain a shell script" + ) + if len(script.encode()) > self.max_script: + raise HFBackendError("candidate script exceeds byte limit") + script, formatting = _candidate_script(script) + if formatting is not None: + record["script_format"] = formatting + record["script"] = script + except Exception as exc: + record["error"] = self._error(exc) + finally: + record["latency_seconds"] = time.monotonic() - started + self._transcripts.append(record["transcript"]) + return self._redact(record) + + def solve(self, model: dict, seed: int, role="target") -> dict: + """Run a bounded terminal-tool agent with inference kept in the controller.""" + record = self._record() + record["role"] = role + record.update(truncated=False, termination_reason="completed") + sandbox = None + started = time.monotonic() + try: + with _deadline(self.timeout): + model, allowed, limits, client = self._model(model, seed, tools=True) + record["model_metadata"] = self._model_metadata(model, seed) + messages = [ + { + "role": "system", + "content": "Complete the task using the terminal function. Commands run in the task workspace. Finish with a short message after saving your solution files.", + }, + {"role": "user", "content": self._instruction()}, + ] + record["transcript"] = messages + sandbox = self._create(self.execution["agent_image"], record) + self._audit_solver(sandbox) + calls_used = 0 + for _ in range(limits["max_steps"]): + message = self._chat( + client, + model, + allowed, + limits, + messages, + record, + seed, + tools=True, + ) + calls = message.get("tool_calls") or [] + if not calls: + break + if not isinstance(calls, list): + raise HFBackendError("invalid terminal function calls") + if calls_used + len(calls) > limits["max_steps"]: + record.update( + truncated=True, termination_reason="tool_call_limit" + ) + break + ids = set() + for call in calls: + if ( + call.get("type") != "function" + or call.get("function", {}).get("name") != "terminal" + or not isinstance(call.get("id"), str) + or not call["id"] + or call["id"] in ids + ): + raise HFBackendError("invalid terminal function call") + ids.add(call["id"]) + arguments = json.loads( + call["function"]["arguments"], + object_pairs_hook=_unique_pairs, + ) + if ( + not isinstance(arguments, dict) + or set(arguments) != {"command"} + or not isinstance(arguments["command"], str) + or len(arguments["command"].encode()) > self.max_script + ): + raise HFBackendError("invalid terminal function arguments") + calls_used += 1 + exit_code, output = self._run( + sandbox, + ["/bin/bash", "-c", arguments["command"]], + started + self.timeout, + ) + messages.append( + { + "role": "tool", + "tool_call_id": call["id"], + "content": json.dumps( + {"exit_code": exit_code, "output": output} + ), + } + ) + else: + record.update(truncated=True, termination_reason="step_limit") + record["files"] = self._collect(sandbox) + record["submission_digest"] = submission_digest(record["files"]) + except Exception as exc: + record["error"] = self._error(exc) + finally: + self._cleanup(record, sandbox) + record["latency_seconds"] = time.monotonic() - started + self._transcripts.append(record["transcript"]) + return self._redact(record) + + def _test_files(self): + root = self.task_root / "tests" + if root.is_symlink() or not root.is_dir(): + raise HFBackendError("task tests must be a regular directory") + files = [] + remaining = 16 * 1048576 + for index, path in enumerate(root.rglob("*")): + if index >= 4096: + raise HFBackendError("too many verifier filesystem entries") + info = path.lstat() + if stat.S_ISDIR(info.st_mode): + continue + relative = path.relative_to(root).as_posix() + raw, mode = _local_file(path, remaining) + remaining -= len(raw) + files.append(("/tests/" + relative, raw, mode)) + return sorted(files) + + def grade(self, submission: dict) -> dict: + """Grade exact allowlisted submission bytes in a separate verifier Sandbox.""" + record = { + "grade_sandbox_id": None, + "verifier_image_ref": self.execution["verifier_image"], + "image_reference_kind": "control_plane_immutable_reference", + "reward": None, + "rawreward": b"", + "transcript": [], + "cleanup_verified": False, + "error": None, + "limitations": list(_LIMITATIONS), + } + sandbox = None + started = time.monotonic() + try: + with _deadline(self.timeout): + files = submission["files"] + if ( + submission.get("error") + or not isinstance(files, dict) + or any( + path not in self.submission_paths or not isinstance(raw, bytes) + for path, raw in files.items() + ) + or sum(map(len, files.values())) > self.max_bytes + ): + raise HFBackendError("invalid submission transfer") + if submission.get("submission_digest") != submission_digest(files): + raise HFBackendError("submission digest mismatch") + files = dict(files) + tests = self._test_files() + command = self.execution.get( + "verifier_command", ["/bin/bash", "/tests/test.sh"] + ) + if ( + not isinstance(command, list) + or not command + or any(not isinstance(arg, str) or "\x00" in arg for arg in command) + ): + raise HFBackendError("verifier_command must be an argv list") + sandbox = self._create( + self.execution["verifier_image"], record, grader=True + ) + if sandbox.files.exists("/tests") or sandbox.files.exists("/solution"): + raise HFBackendError( + "verifier image contains preexisting private paths" + ) + for path in self.submission_paths: + if sandbox.files.exists(path): + if sandbox.files.stat(path).type != "file": + raise HFBackendError( + "verifier submission destination is not a regular file" + ) + sandbox.files.delete(path) + for path, raw, mode in tests: + sandbox.files.write(path, raw, mode=mode) + sandbox.files.write( + "/tests/.openenv-canary-" + uuid.uuid4().hex, self.canary.encode() + ) + for path, raw in files.items(): + sandbox.files.write(path, raw) + # Harbor normally mounts this directory. Fresh Sandbox graders + # need it before a native verifier can write its reward file. + sandbox.files.mkdir(str(PurePosixPath(self.reward_path).parent)) + # A stale image-provided reward must not become new evidence. + if sandbox.files.exists(self.reward_path): + sandbox.files.delete(self.reward_path) + trace = {"exit_code": None, "output": "", "output_truncated": False} + record["transcript"].append(trace) + exit_code, _ = self._run( + sandbox, command, started + self.timeout, trace=trace + ) + if exit_code != 0: + raise HFBackendError("verifier command exited nonzero") + entry = sandbox.files.stat(self.reward_path) + if ( + entry.type != "file" + or type(entry.size) is not int + or not 0 < entry.size <= min(self.max_bytes, 65536) + ): + raise HFBackendError("reward is not a bounded regular file") + sandbox.files.MAX_READ_BYTES = min(self.max_bytes, 65536) + raw = sandbox.files.read(self.reward_path) + if len(raw) != entry.size: + raise HFBackendError("reward changed during collection") + if any(secret.encode() in raw for secret in self._credentials): + raise HFBackendError("credential detected in reward artifact") + record["rawreward"] = raw + text = raw.decode("utf-8").strip() + if text.startswith("{"): + value = json.loads( + text, parse_float=Decimal, object_pairs_hook=_unique_pairs + )[self.execution.get("reward_key", "reward")] + else: + value = Decimal(text) + if ( + isinstance(value, bool) + or not isinstance(value, (Decimal, int)) + or not Decimal(value).is_finite() + or not math.isfinite(float(value)) + ): + raise HFBackendError("reward must be finite numeric data") + record["reward"] = float(value) + except Exception as exc: + record["error"] = self._error(exc) + record["reward"] = None + finally: + self._cleanup(record, sandbox) + record["latency_seconds"] = time.monotonic() - started + self._transcripts.append(record.get("transcript", [])) + record = self._redact(record) + for trace in record["transcript"]: + if len(trace["output"].encode()) > self.max_output: + trace["output"] = "" + trace["output_truncated"] = True + return record + + def leakage_probes(self) -> list: + """Audit private paths, two fresh solver instances, and accumulated transcripts.""" + probes = [] + while len(self._audit) < 2: + record = self.execute_script(":", "leakage_probe") + if record["error"]: + probes.append( + { + "name": "fresh_state_absent", + "passed": False, + "error": record["error"], + "evidence": list(self._audit), + } + ) + break + for name in ("private_paths_absent", "fresh_state_absent"): + if any(probe["name"] == name for probe in probes): + continue + probes.append( + { + "name": name, + "error": None, + "passed": len(self._audit) >= 2 + and all(item[name] for item in self._audit), + "evidence": list(self._audit), + } + ) + probes.append( + { + "name": "canary_not_in_transcript", + "error": None, + "passed": not self._has_canary(self._transcripts), + "evidence": { + "canary_digest": self.canary_digest, + "transcript_count": len(self._transcripts), + }, + } + ) + return probes diff --git a/examples/coding_audit/runner.py b/examples/coding_audit/runner.py new file mode 100644 index 000000000..bd3762ece --- /dev/null +++ b/examples/coding_audit/runner.py @@ -0,0 +1,744 @@ +"""Collect a bounded coding audit using fresh, credential-scoped HF Sandboxes. + +This supplemental profile does not replace RFC 008 admission or demonstrate RL +training benefit. Inference runs in the controller; task/verifier data cross only +the explicit file boundary in ``hf``. +""" + +import hashlib +import json +import os +import re +import shutil +import tempfile +import tomllib +from copy import deepcopy +from fractions import Fraction +from pathlib import Path, PurePosixPath + +from openenv.validation.runner import _DIGEST_EXCLUDED_DIRS, source_digest +from openenv.validation.runtime.json_values import dumps, loads + +from .scoring import assess_coding_audit, validate_audit_config + +PROFILE = "harbor-coding-audit-v1" +TARGET_MODEL = "Qwen/Qwen3.8-27B" +TARGET_REVISION = "37561816e66b5b9153b1b64c10968c18c1e6f4a0" +_IMAGE = re.compile(r"[^\s@]+@sha256:[0-9a-f]{64}\Z") + + +def _integer(value, name, low, high): + if type(value) is not int or not low <= value <= high: + raise ValueError(f"{name} must be an integer in [{low}, {high}]") + return value + + +def _candidate_context_paths(paths): + message = ( + "generator.context_paths must list at most 32 unique normalized public " + "files under environment/source/" + ) + if not isinstance(paths, list) or len(paths) > 32: + raise ValueError(message) + result = [] + private = { + "tests", + "test", + "solution", + "validation", + "verifier", + "reference", + "references", + "spec", + "specs", + "secret", + "secrets", + "credential", + "credentials", + "private", + } + for value in paths: + if ( + not isinstance(value, str) + or "\\" in value + or ":" in value + or any(ord(char) < 32 or ord(char) == 127 for char in value) + ): + raise ValueError(message) + path = PurePosixPath(value) + if ( + path.as_posix() != value + or path.parts[:2] != ("environment", "source") + or len(path.parts) < 3 + or any( + part.startswith(".") or part.lower() in private for part in path.parts + ) + or value in result + ): + raise ValueError(message) + result.append(value) + return result + + +def normalize_coding_config(config): + """Validate operator inputs without launching Jobs, Sandboxes or inference.""" + if not isinstance(config, dict): + raise ValueError("coding configuration must be an object") + quality = deepcopy(config.get("validation", {})) + config = json.loads(dumps(config)) + allowed = { + "schema_version", + "profile", + "target", + "generator", + "execution", + "validation", + "generation", + "negative_cases", + "seeds", + "cloud", + } + if ( + set(config) - allowed + or type(config.get("schema_version")) is not int + or config["schema_version"] != 1 + ): + raise ValueError("unsupported coding audit configuration") + if config.get("profile", PROFILE) != PROFILE: + raise ValueError("unsupported coding audit profile") + config["profile"] = PROFILE + config["validation"] = validate_audit_config(quality) + _integer(config["validation"]["reference_repeats"], "reference_repeats", 2, 16) + _integer(config["validation"]["target_attempts"], "target_attempts", 2, 128) + for name in ("target", "generator"): + model = config.get(name, config.get("target")) + if not isinstance(model, dict): + raise ValueError(f"{name} model configuration is required") + model = dict(model) + model_keys = { + "model_id", + "revision", + "provider", + "provider_model_id", + "response_model_aliases", + "max_steps", + "max_tokens", + "max_total_tokens", + "max_output_bytes", + "temperature", + "supports_seed", + "supports_tools", + "top_p", + "enable_thinking", + "context_paths", + } + if name == "target" and "context_paths" in model: + raise ValueError("target.context_paths is unsupported") + if set(model) - model_keys: + raise ValueError( + f"unknown {name} model option; credentials belong in environment secrets" + ) + if not re.fullmatch(r"[^\s/@]+/[^\s/@]+", model.get("model_id", "")): + raise ValueError(f"{name}.model_id must be a canonical Hub repository") + if not re.fullmatch(r"[0-9a-f]{40}", model.get("revision", "")): + raise ValueError(f"{name}.revision must be a full source commit") + if ( + not isinstance(model.get("provider"), str) + or not re.fullmatch(r"[a-z0-9-]+", model["provider"]) + or model["provider"] == "auto" + ): + raise ValueError(f"{name}.provider must be selected explicitly") + if "enable_thinking" in model and type(model["enable_thinking"]) is not bool: + raise ValueError(f"{name}.enable_thinking must be a boolean") + if "context_paths" in model: + model["context_paths"] = _candidate_context_paths(model["context_paths"]) + model.setdefault("max_steps", 8) + model.setdefault("max_tokens", 1024) + model.setdefault("max_total_tokens", 16384) + model.setdefault("supports_seed", False) + model.setdefault("supports_tools", True) + for key, high in ( + ("max_steps", 128), + ("max_tokens", 32768), + ("max_total_tokens", 262144), + ): + _integer(model[key], f"{name}.{key}", 1, high) + config[name] = model + execution = config.get("execution", {}) + if not isinstance(execution, dict): + raise ValueError("execution must be an object") + execution_keys = { + "agent_image", + "verifier_image", + "submission_paths", + "private_paths", + "workdir", + "flavor", + "namespace", + "timeout_seconds", + "verifier_command", + "reward_path", + "reward_key", + "max_submission_bytes", + "cleanup_attempts", + "cleanup_timeout_seconds", + "max_tool_output_bytes", + "max_script_bytes", + "strong_isolation", + "require_strong_isolation", + } + if set(execution) - execution_keys: + raise ValueError( + "unknown execution option; task credentials cannot be embedded" + ) + for key in ("agent_image", "verifier_image"): + if not _IMAGE.fullmatch(execution.get(key, "")): + raise ValueError(f"execution.{key} must be an immutable OCI reference") + if not execution.get("submission_paths"): + raise ValueError( + "execution.submission_paths must declare the artifact boundary" + ) + generation = config.setdefault("generation", {}) + if not isinstance(generation, dict): + raise ValueError("generation must be an object") + if set(generation) - {"candidate_count"}: + raise ValueError("unknown generation option") + generation.setdefault("candidate_count", 2) + _integer(generation["candidate_count"], "candidate_count", 1, 16) + negatives = config.get("negative_cases", []) + expected = config["validation"]["required_negative_cases"] + if ( + not isinstance(negatives, list) + or any( + not isinstance(case, dict) or set(case) != {"id", "script"} + for case in negatives + ) + or [case["id"] for case in negatives] != expected + or any( + not isinstance(case["script"], str) or not case["script"].strip() + for case in negatives + ) + ): + raise ValueError( + "negative_cases must supply the trusted required IDs and scripts in order" + ) + attempts = config["validation"]["target_attempts"] + seeds = config.setdefault("seeds", list(range(attempts))) + if ( + not isinstance(seeds, list) + or len(seeds) != attempts + or any(type(seed) is not int for seed in seeds) + or len(set(seeds)) != len(seeds) + ): + raise ValueError( + "one distinct seed/attempt identifier per target attempt is required" + ) + for seed in seeds: + _integer(seed, "seed", 0, 2**32 - 1) + cloud = config.setdefault("cloud", {}) + if not isinstance(cloud, dict): + raise ValueError("cloud must be an object") + if set(cloud) - { + "results_repo", + "namespace", + "flavor", + "timeout_seconds", + "max_sandboxes", + "max_inference_calls", + }: + raise ValueError("unknown cloud option") + cloud.setdefault("max_sandboxes", 100) + cloud.setdefault("max_inference_calls", 256) + _integer(cloud["max_sandboxes"], "max_sandboxes", 1, 10000) + _integer(cloud["max_inference_calls"], "max_inference_calls", 1, 10000) + return config + + +def coding_budget(config, *, has_reference=True): + """A conservative, finite launch ceiling, including leakage probes.""" + repeats = config["validation"]["reference_repeats"] + candidates = config["generation"]["candidate_count"] + references = candidates + int(has_reference) + executions = ( + 1 + references * repeats + len(config["negative_cases"]) + len(config["seeds"]) + ) + return { + "sandbox_jobs_max": 2 * executions + 6, + "inference_calls_max": candidates + + len(config["seeds"]) * config["target"]["max_steps"], + "generated_candidates": candidates, + "target_attempts": len(config["seeds"]), + } + + +def _credentials(): + names = ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN") + values = {name: os.environ.get(name) for name in names} + if not all(values.values()): + raise ValueError( + "HF_TOKEN, HF_INFERENCE_TOKEN and HF_SANDBOX_TOKEN are required" + ) + if len(set(values.values())) != len(values): + raise ValueError( + "artifact, inference and sandbox credentials must be distinct and separately scoped" + ) + return values + + +def _write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(dumps(value) + "\n") + + +def live_provider_id(info, provider): + """Read both Hub SDK provider-map representations without selecting an alias.""" + mappings = info.inference_provider_mapping + if isinstance(mappings, dict): + candidates = [mappings[provider]] if provider in mappings else [] + elif isinstance(mappings, list): + candidates = [ + entry + for entry in mappings + if ( + entry.get("provider") + if isinstance(entry, dict) + else getattr(entry, "provider", None) + ) + == provider + ] + else: + candidates = [] + if len(candidates) != 1: + raise ValueError("selected model/provider mapping is missing or ambiguous") + entry = candidates[0] + status = ( + entry.get("status") + if isinstance(entry, dict) + else getattr(entry, "status", None) + ) + identity = ( + entry.get("provider_id") + if isinstance(entry, dict) + else getattr(entry, "provider_id", None) + ) + if status != "live" or not isinstance(identity, str) or not identity: + raise ValueError("selected model/provider is not live") + return identity + + +def validate_coding_task(task_root, config): + """Check supported native task contracts locally, before controller spending. + + Returns normalized audit configuration. Passing this check is preparation, + not evidence that the task, image, or grader passes runtime validation. + """ + config = normalize_coding_config(config) + task_root = Path(task_root).resolve() + task_config = tomllib.loads((task_root / "task.toml").read_text()) + if any((task_root / "steps").glob("*")): + raise ValueError("multi-step Harbor tasks are not supported by this profile") + for section in ("environment", "agent", "verifier", "solution"): + section_config = task_config.get(section, {}) + if not isinstance(section_config, dict): + raise ValueError("task phase configuration must be an object") + if section_config.get("environment"): + raise ValueError("separate phase environment contracts are unsupported") + if section_config.get("allow_internet") is False: + raise ValueError("non-public phase network contracts are unsupported") + if section_config.get("allowed_hosts"): + raise ValueError("network allowlist contracts are unsupported") + if section_config.get("env"): + raise ValueError( + "task environment/credential substitutions are unsupported" + ) + if section_config.get("network_mode", "public") != "public": + raise ValueError("non-public phase network contracts are unsupported") + environment = task_config.get("environment", {}) + if task_config.get("verifier", {}).get("collect"): + raise ValueError("verifier collection commands are unsupported") + if environment.get("workdir") not in ( + None, + config["execution"].get("workdir", "/app"), + ): + raise ValueError("source and audit workdirs disagree") + if any( + environment.get(key) + for key in ("gpus", "tpu", "mcp_servers", "healthcheck", "skills_dir") + ): + raise ValueError( + "accelerator, healthcheck, skills and external MCP requirements are unsupported" + ) + if any( + (task_root / "environment" / name).exists() + for name in ( + "compose.yaml", + "compose.yml", + "docker-compose.yaml", + "docker-compose.yml", + ) + ): + raise ValueError("multi-service Compose tasks are unsupported") + if ( + not (task_root / "instruction.md").is_file() + or not (task_root / "tests/test.sh").is_file() + ): + raise ValueError("a native instruction and verifier are required") + budget = coding_budget( + config, has_reference=(task_root / "solution/solve.sh").is_file() + ) + if ( + budget["sandbox_jobs_max"] > config["cloud"]["max_sandboxes"] + or budget["inference_calls_max"] > config["cloud"]["max_inference_calls"] + ): + raise ValueError( + "configured launch budget is too small for the requested audit" + ) + return config + + +def run_coding_validation(task_root, config, output_dir, *, backend=None): + """Generate, challenge, replay and measure a single prepared native task. + + Every result is bound to frozen task/verifier bytes and immutable requested + image references. Provider source revisions are recorded without pretending + they pin the provider's served weights. Outputs must be a new directory. + """ + config = validate_coding_task(task_root, config) + task_root, output_dir = Path(task_root).resolve(), Path(output_dir).resolve() + if output_dir.is_relative_to(task_root) and not any( + part in _DIGEST_EXCLUDED_DIRS + for part in output_dir.relative_to(task_root).parts + ): + raise ValueError("audit output must be outside the digested task tree") + secrets = [ + value + for name in ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN") + if (value := os.environ.get(name)) + ] + if any(secret in dumps(config) for secret in secrets): + raise ValueError("credentials may not be embedded in audit metadata") + original_digest = source_digest(task_root) + budget = coding_budget( + config, has_reference=(task_root / "solution/solve.sh").is_file() + ) + output_dir.mkdir(parents=True, exist_ok=False) + evidence = { + "task_digest": original_digest, + "verifier_digest": None, + "agent_image": config["execution"]["agent_image"], + "verifier_image": config["execution"]["verifier_image"], + "target": { + "model": config["target"]["model_id"], + "revision": config["target"]["revision"], + "provider": config["target"]["provider"], + }, + "references": [], + "trials": [], + "leakage_probes": [], + "generation": [], + "budget": budget, + "configuration_digest": hashlib.sha256(dumps(config).encode()).hexdigest(), + "requested_configuration_digest": hashlib.sha256( + dumps(config).encode() + ).hexdigest(), + "limitations": [ + "Supplemental coding audit; full RFC 008 qualification is not established.", + "Provider model source SHA does not attest served weights or deterministic sampling.", + "Image identities are control-plane immutable references, not runtime image attestations.", + "Sandbox RPC exposes its execution credential to the workload server; only reviewed workloads are supported.", + "Separate verifier Jobs do not prove submitted code cannot read or tamper with its grader.", + ], + } + + def sanitize(value): + if isinstance(value, str): + for secret in secrets: + value = value.replace(secret, "[REDACTED]") + return value + if isinstance(value, dict): + return {sanitize(key): sanitize(child) for key, child in value.items()} + if isinstance(value, list): + return [sanitize(child) for child in value] + return value + + def artifact(relative, data): + if isinstance(data, str): + data = data.encode() + digest = hashlib.sha256(data).hexdigest() + if any(secret.encode() in data for secret in secrets): + evidence["leakage_probes"].append( + {"name": "credential_in_artifact", "passed": False, "error": None} + ) + return {"sha256": digest, "omitted": "credential detected"} + path = output_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return {"path": relative, "sha256": digest} + + def persist(): + evidence["configuration_digest"] = hashlib.sha256( + dumps(config).encode() + ).hexdigest() + if any(secret in dumps(evidence) for secret in secrets) and not any( + probe.get("name") == "credential_in_metadata" + for probe in evidence["leakage_probes"] + ): + evidence["leakage_probes"].append( + {"name": "credential_in_metadata", "passed": False, "error": None} + ) + _write(output_dir / "evidence.json", sanitize(evidence)) + + try: + from .hf import _local_file + + def copy_artifact(source, destination): + raw, _ = _local_file(source, Path(source).stat().st_size) + Path(destination).write_bytes(raw) + shutil.copystat(source, destination) + + with tempfile.TemporaryDirectory(prefix="openenv-coding-frozen-") as temporary: + frozen = (Path(temporary) / "task").resolve() + shutil.copytree( + task_root, + frozen, + symlinks=True, + copy_function=copy_artifact, + ignore=shutil.ignore_patterns(*_DIGEST_EXCLUDED_DIRS), + ) + if source_digest(frozen) != original_digest: + raise ValueError("task changed during snapshot") + evidence["verifier_digest"] = source_digest(frozen / "tests") + if backend is None: + from .hf import HFTaskBackend + + credentials = _credentials() + # Provider preflight is read-only and precedes every paid sandbox. + from huggingface_hub import HfApi + + api = HfApi(token=credentials["HF_INFERENCE_TOKEN"]) + for model in (config["target"], config["generator"]): + info = api.model_info(model["model_id"], revision=model["revision"]) + if info.sha != model["revision"]: + raise ValueError("model source revision could not be verified") + current = api.model_info( + model["model_id"], expand=["inferenceProviderMapping"] + ) + model.setdefault( + "provider_model_id", + live_provider_id(current, model["provider"]), + ) + backend = HFTaskBackend( + frozen, config["execution"], token=credentials["HF_SANDBOX_TOKEN"] + ) + + def trial(role, case_id, script=None, seed=None): + if source_digest(frozen) != original_digest: + raise ValueError("frozen task source changed") + number = len(evidence["trials"]) + submission = ( + backend.solve(config["target"], seed, role="target") + if role == "target" + else backend.execute_script(script, role) + ) + row = { + "role": role, + "case_id": case_id, + "task_digest": original_digest, + "verifier_digest": evidence["verifier_digest"], + "image_ref": submission.get("image_ref"), + "submission_digest": submission.get("submission_digest"), + "sandbox_id": submission.get("sandbox_id"), + "error": submission["error"] + if "error" in submission + else "execution error status missing", + "cleanup_verified": submission.get("cleanup_verified") is True, + "latency_seconds": submission.get("latency_seconds"), + "usage": submission.get("usage"), + "seed": seed, + "model_metadata": submission.get("model_metadata"), + "truncated": submission.get("truncated", False), + "termination_reason": submission.get("termination_reason"), + } + row["transcript"] = artifact( + f"trials/{number:04d}/transcript.json", + dumps(submission.get("transcript", [])), + ) + for index, (name, data) in enumerate( + sorted(submission.get("files", {}).items()) + ): + item = artifact(f"trials/{number:04d}/submission/{index:04d}", data) + row.setdefault("submission_files", []).append( + {"destination": name, **item} + ) + if row["error"] is None and row["cleanup_verified"]: + grade = backend.grade(submission) + row["grader_transcript"] = artifact( + f"trials/{number:04d}/grader-transcript.json", + dumps(grade.get("transcript", [])), + ) + row.update( + { + "grade_sandbox_id": grade.get("grade_sandbox_id"), + "verifier_image_ref": grade.get("verifier_image_ref"), + "reward": grade.get("reward"), + "error": grade["error"] + if "error" in grade + else "grader error status missing", + "cleanup_verified": grade.get("cleanup_verified") is True, + } + ) + if grade.get("rawreward"): + row["raw_reward"] = artifact( + f"trials/{number:04d}/reward.raw", grade["rawreward"] + ) + try: + raw_value = loads(grade["rawreward"]) + if isinstance(raw_value, dict): + raw_value = raw_value[ + config["execution"].get("reward_key", "reward") + ] + if isinstance(raw_value, (bool, str)) or isinstance( + row["reward"], bool + ): + raise ValueError("reward is not numeric") + if Fraction(str(raw_value)) != Fraction(str(row["reward"])): + raise ValueError( + "raw reward and parsed reward disagree" + ) + except (ValueError, TypeError, KeyError, ZeroDivisionError): + row.update( + error=row["error"] + if row["error"] is not None + else "raw reward fidelity check failed", + reward=None, + ) + elif row["error"] is None: + row.update(error="raw reward evidence missing", reward=None) + if role == "target": + row.update( + { + "model": config["target"]["model_id"], + "revision": config["target"]["revision"], + "provider": config["target"]["provider"], + } + ) + evidence["trials"].append(row) + persist() + if not row["cleanup_verified"]: + raise ValueError("stop collection after unverified cleanup") + return row + + trial("nop", "nop", "true") + candidates = [] + provided = frozen / "solution/solve.sh" + if provided.is_file(): + candidates.append(("provided", "provided", provided.read_text())) + for index in range(config["generation"]["candidate_count"]): + generated = backend.generate_candidate(config["generator"], index) + script = generated.get("script", "") + entry = { + key: value for key, value in generated.items() if key != "script" + } + entry["artifact"] = artifact(f"candidates/generated-{index}.sh", script) + evidence["generation"].append(entry) + if generated.get("error") or not script.strip(): + evidence["generation_error"] = True + continue + candidates.append((f"generated-{index}", "generated", script)) + for case_id, origin, script in candidates: + records = [ + trial("reference", case_id, script) + for _ in range(config["validation"]["reference_repeats"]) + ] + evidence["references"].append( + { + "case_id": case_id, + "origin": origin, + "submission_digest": records[0].get("submission_digest"), + "candidate_script_digest": hashlib.sha256( + script.encode() + ).hexdigest(), + } + ) + for case in config["negative_cases"]: + trial("negative", case["id"], case["script"]) + evidence["leakage_probes"].extend(backend.leakage_probes()) + if evidence.get("generation_error"): + evidence["leakage_probes"].append( + { + "name": "generation_complete", + "passed": False, + "error": "candidate generation incomplete", + } + ) + persist() + screening = assess_coding_audit(config["validation"], evidence) + if all( + check["status"] == "pass" + for name, check in screening["checks"].items() + if name != "target_difficulty" + ): + for index, seed in enumerate(config["seeds"]): + trial("target", f"target-{index}", seed=seed) + # Include target transcripts in the final leakage measurement. + names = { + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", + } + evidence["leakage_probes"] = [ + probe + for probe in evidence["leakage_probes"] + if probe.get("name") not in names + ] + evidence["leakage_probes"].extend(backend.leakage_probes()) + else: + evidence["target_not_run_reason"] = ( + "reference/verifier/reliability/leakage screening did not pass" + ) + if ( + source_digest(task_root) != original_digest + or source_digest(frozen) != original_digest + ): + raise ValueError("task source changed during audit") + except (Exception, KeyboardInterrupt) as exc: + # Provider exceptions may include request URLs or credentials. + evidence["collection_error"] = type(exc).__name__ + evidence["leakage_probes"].append( + { + "name": "collection_complete", + "passed": False, + "error": type(exc).__name__, + } + ) + persist() + report = assess_coding_audit(config["validation"], evidence) + report.update( + { + "task_digest": original_digest, + "verifier_digest": evidence["verifier_digest"], + "target": evidence["target"], + "budget": budget, + "limitations": evidence["limitations"], + "configuration_digest": evidence["configuration_digest"], + } + ) + report["checks"]["target_difficulty"]["truncated_attempts"] = sum( + trial.get("truncated") is True + for trial in evidence["trials"] + if trial["role"] == "target" + ) + report = sanitize(report) + _write(output_dir / "config.json", sanitize(config)) + _write(output_dir / "report.json", report) + files = sorted(path for path in output_dir.rglob("*") if path.is_file()) + (output_dir / "SHA256SUMS").write_text( + "".join( + hashlib.sha256(path.read_bytes()).hexdigest() + + " " + + path.relative_to(output_dir).as_posix() + + "\n" + for path in files + ) + ) + return report diff --git a/examples/coding_audit/scoring.py b/examples/coding_audit/scoring.py new file mode 100644 index 000000000..ff04abeb4 --- /dev/null +++ b/examples/coding_audit/scoring.py @@ -0,0 +1,506 @@ +"""Pure scoring of bounded coding-audit evidence collected by a trusted runner. + +This profile measures reference replay, known-negative rejection, explicit leakage +probes, and target difficulty. It does not establish containment or trainability. +""" + +import math +import re +from collections import Counter, defaultdict +from decimal import Decimal +from fractions import Fraction +from statistics import pvariance +from typing import Any + +PROFILE = "harbor-coding-audit-v1" +_CHECKS = ( + "references", + "verifier_rejection", + "reliability", + "leakage", + "target_difficulty", +) +_PROBES = ( + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", +) +_RANK = {"pass": 0, "skip": 1, "fail": 2, "error": 3} +_SHA256 = re.compile(r"(?:sha256:)?[0-9a-f]{64}\Z") +_IMAGE = re.compile(r"(?:[^\s@]+@)?sha256:[0-9a-f]{64}\Z") +_REVISION = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z") + + +def _number(value: Any) -> bool: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + try: + return math.isfinite(value) + except OverflowError: + return False + + +def _text(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _matches(pattern: re.Pattern[str], value: Any) -> bool: + return isinstance(value, str) and pattern.fullmatch(value) is not None + + +def _sha(value: str) -> str: + return value.removeprefix("sha256:") + + +def _exact(value: int | float) -> Fraction: + return Fraction(str(value)) + + +def validate_audit_config(config: dict[str, Any]) -> dict[str, Any]: + """Return quality thresholds, raising `ValueError` for invalid configuration. + + `required_negative_cases` must come from a trusted reviewed configuration; + generated labels in evidence cannot supply this rejection coverage. + """ + defaults = { + "reference_repeats": 2, + "target_attempts": 2, + "reward_min": 0, + "reward_max": 1, + "oracle_tolerance": 0, + "floor_margin": 0.1, + "wrong_answer_max": 0, + "required_negative_cases": None, + } + if not isinstance(config, dict) or set(config) - defaults.keys(): + raise ValueError("config must contain only documented quality thresholds") + result = defaults | config + for name in ("reference_repeats", "target_attempts"): + if type(result[name]) is not int or result[name] < 2: + raise ValueError(f"{name} must be an integer >= 2") + for name in ( + "reward_min", + "reward_max", + "oracle_tolerance", + "floor_margin", + "wrong_answer_max", + ): + if not ( + _number(result[name]) + or isinstance(result[name], Decimal) + and result[name].is_finite() + ): + raise ValueError(f"{name} must be a finite nonbool number") + low, high = (_exact(result[name]) for name in ("reward_min", "reward_max")) + tolerance, margin = ( + _exact(result[name]) for name in ("oracle_tolerance", "floor_margin") + ) + if not ( + low < high + and 0 <= tolerance < high - low + and 0 < margin <= high - low + and low <= _exact(result["wrong_answer_max"]) < high - tolerance + and low + tolerance < high - tolerance + ): + raise ValueError("reward bounds, tolerance, margin or negative limit conflict") + cases = result["required_negative_cases"] + if ( + not isinstance(cases, list) + or not cases + or any(not _text(case) for case in cases) + or len(set(cases)) != len(cases) + ): + raise ValueError("required_negative_cases must be nonempty unique case IDs") + result["required_negative_cases"] = list(cases) + return result + + +def _issue(check: dict[str, Any], status: str, reason: str) -> None: + if _RANK[status] > _RANK[check["status"]]: + check["status"] = status + check["reasons"].append(reason) + + +def _finish(checks: dict[str, Any]) -> dict[str, Any]: + verdict = max((check["status"] for check in checks.values()), key=_RANK.get) + return { + "profile": PROFILE, + "accepted": verdict == "pass", + "verdict": verdict, + "checks": checks, + "rfc008_complete": False, + "trainability_established": False, + } + + +def _wilson(solved: int, count: int) -> list[float] | None: + if not count: + return None + z = 1.959963984540054 + rate = solved / count + denominator = 1 + z * z / count + center = (rate + z * z / (2 * count)) / denominator + half = z * math.sqrt(rate * (1 - rate) / count + z * z / (4 * count**2)) + return [ + max(0.0, center - half / denominator), + min(1.0, center + half / denominator), + ] + + +def _target_metrics( + raw: list[dict[str, Any]], trials: list[dict[str, Any]], config: dict[str, Any] +) -> dict[str, Any]: + rewards = [trial["reward"] for trial in trials] + count = len(rewards) + solved = sum( + _exact(reward) + >= _exact(config["reward_max"]) - _exact(config["oracle_tolerance"]) + for reward in rewards + ) + telemetry_missing = {"latency_seconds": 0, "input_tokens": 0, "output_tokens": 0} + telemetry_invalid = dict(telemetry_missing) + telemetry_errors = [] + latency: list[Fraction] = [] + usage: dict[str, list[int]] = {"input_tokens": [], "output_tokens": []} + # Failed executions can still consume tokens/time. Account for all records. + for trial in raw: + seconds = trial.get("latency_seconds") + if "latency_seconds" not in trial: + telemetry_missing["latency_seconds"] += 1 + elif _number(seconds) and seconds >= 0: + latency.append(_exact(seconds)) + else: + telemetry_invalid["latency_seconds"] += 1 + telemetry_errors.append("invalid target latency_seconds") + tokens = trial.get("usage") + if "usage" in trial and not isinstance(tokens, dict): + telemetry_errors.append("invalid target usage object") + for name in usage: + value = tokens.get(name) if isinstance(tokens, dict) else None + if "usage" not in trial or isinstance(tokens, dict) and name not in tokens: + telemetry_missing[name] += 1 + elif type(value) is int and value >= 0: + usage[name].append(value) + else: + telemetry_invalid[name] += 1 + telemetry_errors.append(f"invalid target {name}") + latency_total = None + if latency: + try: + latency_total = float(sum(latency)) + except OverflowError: + telemetry_errors.append("latency total exceeds finite report precision") + return { + "attempted": len(raw), + "completed": count, + "failed": len(raw) - count, + "infrastructure_errors": sum(trial.get("error") is not None for trial in raw), + "missing": max(0, config["target_attempts"] - count), + "solve_count": solved, + "solve_rate": solved / count if count else None, + "solve_rate_wilson95": _wilson(solved, count), + "reward_variance": _variance(rewards), + "mixed_success": 0 < solved < count if count else None, + "difficulty": ( + "unmeasured" + if not count + else "mixed" + if 0 < solved < count + else "all_solved" + if solved == count + else "none_solved" + ), + "latency_seconds_total": latency_total, + "usage_totals": { + name: sum(values) if values else None for name, values in usage.items() + }, + "telemetry_missing": telemetry_missing, + "telemetry_invalid": telemetry_invalid, + "telemetry_errors": telemetry_errors, + } + + +def _variance(rewards: list[float]) -> float | None: + if not rewards: + return None + try: + value = pvariance(rewards) + except OverflowError: + return None + return value if _number(value) else None + + +def assess_coding_audit( + config: dict[str, Any], evidence: dict[str, Any] +) -> dict[str, Any]: + """Score a trusted runner's evidence without executing tasks or making IO calls. + + Trials bind to the common task, verifier, images and pinned target identity. + Reference inventories bind replay groups by `(case_id, submission_digest)`. + Missing coverage is `skip`, contradictions are `fail`, and malformed or + infrastructure evidence is `error`; only five passing checks admit an audit. + """ + checks = {name: {"status": "pass", "reasons": []} for name in _CHECKS} + try: + config = validate_audit_config(config) + except ValueError as exc: + for check in checks.values(): + _issue(check, "error", str(exc)) + return _finish(checks) + if not isinstance(evidence, dict): + for check in checks.values(): + _issue(check, "error", "evidence must be an object") + return _finish(checks) + + reliability = checks["reliability"] + references = checks["references"] + rejection = checks["verifier_rejection"] + target = checks["target_difficulty"] + low, high, tolerance, margin, negative_max = ( + _exact(config[name]) + for name in ( + "reward_min", + "reward_max", + "oracle_tolerance", + "floor_margin", + "wrong_answer_max", + ) + ) + common_ok = True + for name, pattern in ( + ("task_digest", _SHA256), + ("verifier_digest", _SHA256), + ("agent_image", _IMAGE), + ("verifier_image", _IMAGE), + ): + if not _matches(pattern, evidence.get(name)): + _issue(reliability, "error", f"invalid or missing common {name}") + common_ok = False + identity = evidence.get("target") + if not ( + isinstance(identity, dict) + and all(_text(identity.get(name)) for name in ("model", "provider")) + and _matches(_REVISION, identity.get("revision")) + ): + _issue( + target, "error", "target requires model, provider and immutable revision" + ) + identity = None + + inventory = evidence.get("references", []) + candidates: dict[tuple[str, str], dict[str, Any]] = {} + if not isinstance(inventory, list): + _issue(references, "error", "references must be a list") + inventory = [] + for candidate in inventory: + if not ( + isinstance(candidate, dict) + and _text(candidate.get("case_id")) + and _matches(_SHA256, candidate.get("submission_digest")) + and candidate.get("origin") in ("provided", "generated") + and ( + candidate["origin"] != "generated" + or _matches(_SHA256, candidate.get("candidate_script_digest")) + ) + ): + _issue(references, "error", "invalid reference candidate metadata") + continue + key = (candidate["case_id"], _sha(candidate["submission_digest"])) + if key in candidates: + _issue(references, "error", f"duplicate reference candidate {key[0]}") + candidates[key] = candidate + if not candidates: + _issue(references, "skip", "no declared reference candidates") + + raw_trials = evidence.get("trials", []) + if not isinstance(raw_trials, list): + _issue(reliability, "error", "trials must be a list") + raw_trials = [] + sandbox_counts = Counter( + trial[name] + for trial in raw_trials + if isinstance(trial, dict) + for name in ("sandbox_id", "grade_sandbox_id") + if _text(trial.get(name)) + ) + target_cases = Counter( + trial["case_id"] + for trial in raw_trials + if isinstance(trial, dict) + and trial.get("role") == "target" + and _text(trial.get("case_id")) + ) + by_role: dict[str, list[dict[str, Any]]] = defaultdict(list) + raw_target = [] + for index, trial in enumerate(raw_trials): + if not isinstance(trial, dict) or trial.get("role") not in ( + "nop", + "reference", + "negative", + "target", + ): + _issue(reliability, "error", f"trial {index}: invalid role or record") + continue + role = trial["role"] + role_check = checks[ + { + "nop": "verifier_rejection", + "negative": "verifier_rejection", + "reference": "references", + "target": "target_difficulty", + }[role] + ] + if role == "target": + raw_target.append(trial) + errors = [] + if not common_ok: + errors.append("common binding unavailable") + for name, expected in ( + ("task_digest", evidence.get("task_digest")), + ("verifier_digest", evidence.get("verifier_digest")), + ("image_ref", evidence.get("agent_image")), + ("verifier_image_ref", evidence.get("verifier_image")), + ): + actual = trial.get(name) + if name in ("task_digest", "verifier_digest"): + matches = ( + _matches(_SHA256, actual) + and _matches(_SHA256, expected) + and _sha(actual) == _sha(expected) + ) + else: + matches = actual == expected + if not matches: + errors.append(f"{name} binding mismatch") + submission_ok = _text(trial.get("case_id")) and _matches( + _SHA256, trial.get("submission_digest") + ) + if not submission_ok: + errors.append("missing case ID or submission digest") + for name in ("sandbox_id", "grade_sandbox_id"): + value = trial.get(name) + if not _text(value) or sandbox_counts[value] != 1: + errors.append(f"{name} must be fresh and distinct") + if trial.get("cleanup_verified") is not True: + errors.append("cleanup not verified") + if "error" not in trial or trial["error"] is not None: + errors.append("execution error or missing error status") + reward = trial.get("reward") + if not _number(reward) or not low <= _exact(reward) <= high: + errors.append("reward must be finite nonbool and in bounds") + if role == "target": + if identity is None or any( + trial.get(name) != identity[name] + for name in ("model", "revision", "provider") + ): + errors.append("target identity mismatch") + if _text(trial.get("case_id")) and target_cases[trial["case_id"]] != 1: + errors.append("duplicate target attempt case ID") + if ( + role == "reference" + and submission_ok + and (trial["case_id"], _sha(trial["submission_digest"])) not in candidates + ): + errors.append("undeclared reference submission") + if errors: + message = f"trial {index}: " + "; ".join(errors) + _issue(role_check, "error", message) + _issue(reliability, "error", message) + else: + by_role[role].append(trial) + if not raw_trials: + _issue(reliability, "skip", "no trial evidence") + + floor_rewards = [_exact(trial["reward"]) for trial in by_role["nop"]] + if not floor_rewards: + _issue(rejection, "skip", "NOP floor evidence missing") + _issue(references, "skip", "NOP floor evidence missing") + elif max(floor_rewards) > low + tolerance: + _issue(rejection, "fail", "NOP reward exceeds configured floor") + + groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for trial in by_role["reference"]: + groups[(trial["case_id"], _sha(trial["submission_digest"]))].append(trial) + validated = [] + excluded = [] + for key, candidate in candidates.items(): + trials = groups[key] + rewards = [_exact(trial["reward"]) for trial in trials] + if not rewards: + _issue(references, "skip", f"candidate {key[0]} execution missing") + continue + stable = max(rewards) - min(rewards) <= tolerance + if not stable: + _issue(reliability, "fail", f"reference {key[0]} replay reward drift") + correct = min(rewards) >= high - tolerance + if not correct: + if candidate["origin"] == "generated": + excluded.append({"case_id": key[0], "submission_digest": key[1]}) + else: + _issue( + references, "fail", f"provided reference {key[0]} reward too low" + ) + continue + if len(trials) < config["reference_repeats"]: + _issue(references, "skip", f"reference {key[0]} replay coverage incomplete") + _issue( + reliability, "skip", f"reference {key[0]} replay coverage incomplete" + ) + continue + if floor_rewards and min(rewards) - max(floor_rewards) < margin: + _issue(references, "fail", f"reference {key[0]} lacks floor separation") + continue + if stable and floor_rewards: + validated.append({"case_id": key[0], "submission_digest": key[1]}) + references.update(validated=validated, excluded=excluded) + if not validated: + _issue(references, "skip", "no validated reference") + _issue(reliability, "skip", "no validated reference replay group") + + negatives = defaultdict(list) + for trial in by_role["negative"]: + negatives[trial["case_id"]].append(_exact(trial["reward"])) + covered = [] + for case in config["required_negative_cases"]: + if not negatives[case]: + _issue(rejection, "skip", f"trusted negative {case} execution missing") + elif max(negatives[case]) > negative_max: + _issue(rejection, "fail", f"trusted negative {case} was rewarded") + else: + covered.append(case) + rejection["covered_negative_cases"] = covered + + leakage = checks["leakage"] + leakage["scope"] = "bounded probes; universal containment is not established" + probes = evidence.get("leakage_probes", []) + if not isinstance(probes, list): + _issue(leakage, "error", "leakage_probes must be a list") + probes = [] + seen = set() + for probe in probes: + if not isinstance(probe, dict) or not _text(probe.get("name")): + _issue(leakage, "error", "invalid leakage probe record") + continue + name = probe["name"] + if name in seen: + _issue(leakage, "error", f"duplicate leakage probe {name}") + seen.add(name) + if type(probe.get("passed")) is not bool or "error" not in probe: + _issue(leakage, "error", f"invalid leakage probe result {name}") + elif probe["error"] is not None: + _issue(leakage, "error", f"leakage probe {name} execution error") + elif not probe["passed"]: + _issue(leakage, "fail", f"leakage probe {name} failed") + for name in _PROBES: + if name not in seen: + _issue(leakage, "skip", f"required leakage probe {name} missing") + leakage["observed_probes"] = sorted(seen) + + target.update(_target_metrics(raw_target, by_role["target"], config)) + if target["completed"] and target["reward_variance"] is None: + _issue(target, "error", "reward variance exceeds finite report precision") + for message in target["telemetry_errors"]: + _issue(target, "error", message) + if len(by_role["target"]) < config["target_attempts"]: + _issue(target, "skip", "target attempt coverage incomplete") + return _finish(checks) diff --git a/examples/coding_audit/test_hf.py b/examples/coding_audit/test_hf.py new file mode 100644 index 000000000..2acf9c0db --- /dev/null +++ b/examples/coding_audit/test_hf.py @@ -0,0 +1,1732 @@ +"""Offline SDK-shaped tests for the HF coding task execution boundary.""" + +import hashlib +import json +import os +import signal +import time +from pathlib import PurePosixPath +from types import SimpleNamespace + +import httpx +import pytest +from examples.coding_audit.hf import ( + _deadline, + HFBackendError, + HFTaskBackend, + submission_digest, +) +from huggingface_hub import hf_api, InferenceClient, Sandbox +from huggingface_hub.errors import SandboxError +from huggingface_hub.inference._generated.types.chat_completion import ( + ChatCompletionOutput, + ChatCompletionStreamOutput, +) +from huggingface_hub.inference._providers.deepinfra import DeepInfraConversationalTask +from openenv.validation.runner import source_digest + +AGENT = "python@sha256:" + "a" * 64 +VERIFIER = "python@sha256:" + "b" * 64 +MODEL = { + "model_id": "Qwen/Qwen3.8-27B", + "revision": "37561816e66b5b9153b1b64c10968c18c1e6f4a0", + "provider": "deepinfra", + "provider_model_id": "Qwen/Qwen3.8-27B", +} + + +def test_real_sandbox_sdk_serializes_short_name_for_long_image(harness, monkeypatch): + captured = {} + image = ( + "registry.hf.space/example-organization-a-long-starter-image-name" + "@sha256:" + "a" * 64 + ) + + def capture(request): + assert request.method == "POST" and request.url.host == "huggingface.co" + captured.update(json.loads(request.content)) + return httpx.Response(400, json={"error": "offline request capture"}) + + harness.backend.execution["agent_image"] = image + harness.backend.sandbox_factory = Sandbox.create + with httpx.Client(transport=httpx.MockTransport(capture)) as client: + monkeypatch.setattr(hf_api, "get_session", lambda: client) + record = harness.backend.execute_script(":", "nop") + assert "BadRequestError" in record["error"] + assert captured["dockerImage"] == image + assert captured["labels"]["name"] == ( + "openenv-coding-" + captured["labels"]["openenv-coding-attempt"] + ) + assert 0 < len(captured["labels"]["name"]) <= 100 + + +def response( + content="done", *, calls=None, model=None, fingerprint="fp-123", usage=None +): + return ChatCompletionOutput.parse_obj_as_instance( + { + "id": "chatcmpl-1", + "created": 1, + "object": "chat.completion", + "model": model or MODEL["model_id"], + "system_fingerprint": fingerprint, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls" if calls else "stop", + "message": { + "role": "assistant", + "content": content, + "tool_calls": calls, + }, + } + ], + "usage": usage + or {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + ) + + +def terminal(command="printf answer > /app/answer.txt", identifier="call-1"): + return { + "id": identifier, + "type": "function", + "function": {"name": "terminal", "arguments": json.dumps({"command": command})}, + } + + +def stream_event(*, delta=None, finish=None, choices=None, usage=None, **identity): + return ChatCompletionStreamOutput.parse_obj_as_instance( + { + "id": "chatcmpl-1", + "created": 1, + "model": MODEL["model_id"], + "system_fingerprint": "fp-123", + **identity, + "choices": choices + if choices is not None + else [{"index": 0, "delta": delta or {}, "finish_reason": finish}], + "usage": usage, + } + ) + + +def stream_response(reply): + yield stream_event( + delta=reply["choices"][0]["message"], + finish=reply["choices"][0]["finish_reason"], + model=reply["model"], + system_fingerprint=reply["system_fingerprint"], + ) + yield stream_event( + choices=[], + usage=reply["usage"], + model=reply["model"], + system_fingerprint=reply["system_fingerprint"], + ) + + +class Files: + MAX_READ_BYTES = 512 * 1048576 + + def __init__(self, initial=None): + self.data = dict(initial or {}) + self.types = {} + self.directories = {"/", "/tmp"} + self.writes = [] + self.reads = [] + self.grow_on_read = False + + def stat(self, path): + if path in self.directories: + return SimpleNamespace(type="dir", size=0) + if path not in self.data: + raise SandboxError("missing", status_code=404) + return SimpleNamespace( + type=self.types.get(path, "file"), size=len(self.data[path]) + ) + + def exists(self, path): + try: + self.stat(path) + return True + except SandboxError as exc: + if exc.status_code == 404: + return False + raise + + def read(self, path): + self.reads.append(path) + data = self.data[path] + if self.grow_on_read: + data = data + b"x" * self.MAX_READ_BYTES + if len(data) > self.MAX_READ_BYTES: + raise SandboxError("too large") + return data + + def write(self, path, data, mode=None): + assert isinstance(data, (str, bytes)) + data = data.encode() if isinstance(data, str) else data + self.data[path] = data + self.writes.append((path, data, mode)) + + def mkdir(self, path): + self.directories.add(path) + + def delete(self, path): + del self.data[path] + + +class FakeSandbox: + def __init__(self, identity, image, api, factory): + self.id = identity + self.image = image + self.files = Files(factory.initial) + self.api = api + self.factory = factory + self.commands = [] + self.killed = False + + def run(self, argv, **kwargs): + self.commands.append((argv, kwargs)) + assert isinstance(argv, list) + assert kwargs["shell"] is False + assert kwargs["check"] is False + assert kwargs["capture_output"] is False + assert kwargs["cwd"] in self.files.directories + assert 0 < kwargs["timeout"] <= 120 + assert "env" not in kwargs + if self.factory.output: + kwargs["on_stdout"](self.factory.output) + if self.image == VERIFIER: + self.files.data["/logs/verifier/reward.txt"] = self.factory.reward + elif self.factory.submitted is not None: + self.files.data["/app/answer.txt"] = self.factory.submitted + return SimpleNamespace( + exit_code=self.factory.exit_code, + stdout="", + stderr="", + timed_out=self.factory.timed_out, + ) + + def kill(self): + self.killed = True + if not self.api.ignore_cancel: + self.api.stages[self.id] = "CANCELED" + + +class API: + def __init__(self): + self.images = {} + self.stages = {} + self.cancelled = [] + self.inspections = [] + self.ignore_cancel = False + self.cancel_error = False + self.inspect_error_after_cancel = False + self.owner = "owner" + self.image_override = None + self.labels = {} + + def inspect_job(self, *, job_id, namespace=None, token=None): + self.inspections.append((job_id, namespace, token)) + if self.inspect_error_after_cancel and job_id in self.cancelled: + raise RuntimeError("inspection contains an execution-token-secret") + return SimpleNamespace( + id=job_id, + docker_image=self.image_override or self.images[job_id], + status=SimpleNamespace(stage=self.stages[job_id]), + owner=SimpleNamespace(name=self.owner), + labels=self.labels.get(job_id, {}), + ) + + def list_jobs(self, *, labels=None, timeout=None, namespace=None, token=None): + return ( + self.inspect_job(job_id=job_id, namespace=namespace, token=token) + for job_id in self.images + if all( + self.labels.get(job_id, {}).get(key) == value + for key, value in (labels or {}).items() + ) + ) + + def cancel_job(self, *, job_id, namespace=None, token=None): + self.cancelled.append(job_id) + if self.cancel_error: + raise RuntimeError("cancel failed") + if not self.ignore_cancel: + self.stages[job_id] = "CANCELED" + + +class Factory: + def __init__(self, api): + self.api = api + self.created = [] + self.kwargs = [] + self.initial = {} + self.output = "" + self.submitted = b"answer" + self.reward = b"1.0\n" + self.exit_code = 0 + self.timed_out = False + self.fail_after_create = False + + def __call__(self, **kwargs): + identity = f"job-{len(self.created)}" + self.kwargs.append(kwargs) + self.api.images[identity] = kwargs["image"] + self.api.stages[identity] = "RUNNING" + self.api.labels[identity] = kwargs.get("labels", {}) + sandbox = FakeSandbox(identity, kwargs["image"], self.api, self) + self.created.append(sandbox) + if self.fail_after_create: + raise RuntimeError("readiness failed after allocation") + return sandbox + + +class InferenceFactory: + def __init__(self): + self.kwargs = [] + self.requests = [] + self.responses = [] + self.closed = 0 + + def __call__(self, **kwargs): + self.kwargs.append(kwargs) + return self + + def chat_completion(self, **kwargs): + self.requests.append(json.loads(json.dumps(kwargs))) + reply = self.responses.pop(0) + if kwargs.get("stream") and isinstance(reply, ChatCompletionOutput): + return stream_response(reply) + return reply + + def close(self): + self.closed += 1 + + +@pytest.fixture +def harness(tmp_path, monkeypatch): + for name in ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN"): + monkeypatch.delenv(name, raising=False) + root = tmp_path.resolve() + (root / "instruction.md").write_text("Write /app/answer.txt containing answer.") + (root / "tests").mkdir() + (root / "tests/test.sh").write_bytes( + b"#!/bin/bash\nprintf 1 > /logs/verifier/reward.txt\n" + ) + (root / "solution").mkdir() + (root / "solution/solve.sh").write_text("PRIVATE_SOLUTION_SENTINEL") + api = API() + factory = Factory(api) + inference = InferenceFactory() + execution = { + "agent_image": AGENT, + "verifier_image": VERIFIER, + "submission_paths": ["/app/answer.txt"], + "namespace": "owner", + } + backend = HFTaskBackend( + root, + execution, + token="execution-token-secret", + inference_token="inference-token-secret", + api=api, + sandbox_factory=factory, + inference_factory=inference, + canary="PRIVATE_CANARY_SENTINEL", + ) + return SimpleNamespace( + root=root, + backend=backend, + api=api, + factory=factory, + inference=inference, + execution=execution, + ) + + +def test_execution_grade_are_fresh_and_credentials_stay_in_controller(harness): + h = harness + submission = h.backend.execute_script( + "printf answer > /app/answer.txt", "reference" + ) + grade = h.backend.grade(submission) + assert submission["error"] is None and submission["cleanup_verified"] + assert grade["error"] is None and grade["cleanup_verified"] + assert grade["reward"] == 1 and grade["rawreward"] == b"1.0\n" + assert submission["sandbox_id"] != grade["grade_sandbox_id"] + assert [item["image"] for item in h.factory.kwargs] == [AGENT, VERIFIER] + for kwargs in h.factory.kwargs: + assert kwargs["forward_hf_token"] is False + assert kwargs["token"] == "execution-token-secret" + assert ( + "env" not in kwargs and "secrets" not in kwargs and "volumes" not in kwargs + ) + solver, verifier = h.factory.created + assert not any( + path.startswith(("/tests", "/solution")) for path in solver.files.data + ) + assert ( + verifier.files.data["/tests/test.sh"] == (h.root / "tests/test.sh").read_bytes() + ) + assert verifier.files.data["/app/answer.txt"] == b"answer" + assert not any("solution" in path for path in verifier.files.data) + assert b"PRIVATE_CANARY_SENTINEL" in verifier.files.data.values() + assert submission["image_reference_kind"] == "control_plane_immutable_reference" + assert len(submission["limitations"]) >= 4 + assert "token-secret" not in repr(submission) + repr(grade) + assert h.api.cancelled == ["job-0", "job-1"] + + +def test_absent_submission_is_empty_and_can_legitimately_grade_zero(harness): + h = harness + h.factory.submitted = None + submission = h.backend.execute_script(":", "nop") + assert submission["files"] == {} and submission["error"] is None + h.factory.reward = b"0" + grade = h.backend.grade(submission) + assert grade["error"] is None and grade["reward"] == 0 + + +def test_native_verifier_reward_directory_exists_before_execution(harness, monkeypatch): + h = harness + submission = h.backend.execute_script(":", "nop") + original_run = FakeSandbox.run + + def native_run(sandbox, argv, **kwargs): + if sandbox.image == VERIFIER: + assert "/logs/verifier" in sandbox.files.directories + return original_run(sandbox, argv, **kwargs) + + monkeypatch.setattr(FakeSandbox, "run", native_run) + grade = h.backend.grade(submission) + assert grade["error"] is None + assert grade["cleanup_verified"] + + +@pytest.mark.parametrize("phase", ["script", "grade"]) +def test_nonzero_is_error_not_synthesized_reward(harness, phase): + h = harness + submission = h.backend.execute_script(":", "reference") + h.factory.exit_code = 3 + record = ( + h.backend.grade(submission) + if phase == "grade" + else h.backend.execute_script("false", "target") + ) + assert "nonzero" in record["error"] + assert record.get("reward") is None and record["cleanup_verified"] + + +@pytest.mark.parametrize("failure", ["overflow", "timeout", "nonzero", "exception"]) +def test_failed_grader_keeps_logs_only_without_overflow(harness, monkeypatch, failure): + h = harness + submission = h.backend.execute_script(":", "reference") + h.backend.max_output = 17 + output = "setup\n雪💥雪💥" if failure == "overflow" else "setup\nfailed\n" + + def failed_run(sandbox, argv, **kwargs): + kwargs["on_stdout"]("setup\n") + kwargs["on_stderr"](output.removeprefix("setup\n")) + if failure == "exception": + raise RuntimeError("RPC failed after output") + return SimpleNamespace( + exit_code=3 if failure == "nonzero" else 0, + timed_out=failure == "timeout", + ) + + monkeypatch.setattr(FakeSandbox, "run", failed_run) + grade = h.backend.grade(submission) + assert grade["error"] and grade["cleanup_verified"] + assert grade["reward"] is None and grade["rawreward"] == b"" + trace = grade["transcript"][0] + assert trace["output"] == ("" if failure == "overflow" else output) + assert len(trace["output"].encode()) <= 17 + assert trace["output_truncated"] is (failure == "overflow") + assert trace["exit_code"] == ( + 3 if failure == "nonzero" else 0 if failure == "timeout" else None + ) + + +@pytest.mark.parametrize("cap", [17, 18]) +def test_grader_output_stays_bounded_after_redaction(harness, cap): + h = harness + submission = h.backend.execute_script(":", "nop") + backend = HFTaskBackend( + h.root, + dict(h.execution, max_tool_output_bytes=cap), + token="secret", + api=h.api, + sandbox_factory=h.factory, + ) + h.factory.output = "雪secret💥\n" + h.factory.reward = b"0" + grade = backend.grade(submission) + assert grade["error"] is None and grade["reward"] == 0 + assert grade["transcript"] == [ + { + "exit_code": 0, + "output": "" if cap == 17 else "雪[REDACTED]💥\n", + "output_truncated": cap == 17, + } + ] + assert len(grade["transcript"][0]["output"].encode()) <= cap + assert grade["cleanup_verified"] + + +@pytest.mark.parametrize("callbacks", ["single", "separate", "split_secret"]) +@pytest.mark.parametrize( + "credential", ["execution", "inference", "overlapping", "repeated", "intersecting"] +) +@pytest.mark.parametrize("retained", [1, 28]) +def test_grader_overflow_discards_output( + harness, monkeypatch, callbacks, credential, retained +): + h = harness + submission = h.backend.execute_script(":", "reference") + credentials = { + "execution": "hf_SYNTHETIC_EXECUTION_TOKEN_FOR_REVIEW_ONLY", + "inference": "hf_SYNTHETIC_INFERENCE_TOKEN_FOR_REVIEW_ONLY", + } + if credential == "overlapping": + credentials["execution"] = "SYNTHETIC_EXECUTION_TOKEN" + credentials["inference"] = "hf_SYNTHETIC_EXECUTION_TOKEN_FOR_REVIEW_ONLY" + elif credential == "repeated": + credentials["execution"] = "hf_" + "X" * 22 + "hf_" + credential = "execution" + elif credential == "intersecting": + credentials["execution"] = "hf_SYNTHETIC_EXECUTION_TOKEN_REVIEW_ONLYh" + credentials["inference"] = "hf_SYNTHETIC_INFERENCE_TOKEN_REVIEW_ONLY" + cap = 1048576 + backend = HFTaskBackend( + h.root, + dict(h.execution, max_tool_output_bytes=cap), + token=credentials["execution"], + inference_token=credentials["inference"], + api=h.api, + sandbox_factory=h.factory, + ) + secret = credentials[ + "inference" if credential in ("overlapping", "intersecting") else credential + ] + prefix = "雪" + "A" * (cap - len("雪".encode()) - retained) + if credential == "intersecting": + complete = credentials["execution"] + prefix = prefix[: -len(complete[:-1])] + complete[:-1] + assert complete in prefix + secret + + def overflow(sandbox, argv, **kwargs): + if callbacks == "single": + kwargs["on_stdout"](prefix + secret + "\n") + else: + split = min(7, retained) if callbacks == "split_secret" else 0 + kwargs["on_stdout"](prefix + secret[:split]) + kwargs["on_stderr"](secret[split:] + "\n") + pytest.fail("grader output overflow was not rejected") + + monkeypatch.setattr(FakeSandbox, "run", overflow) + grade = backend.grade(submission) + assert grade["error"] == "terminal output exceeded byte limit" + assert grade["reward"] is None and grade["rawreward"] == b"" + assert grade["cleanup_verified"] + trace = grade["transcript"][0] + assert trace == { + "exit_code": None, + "output": "", + "output_truncated": True, + } + assert len(trace["output"].encode()) <= cap + assert secret[:retained] not in trace["output"] + assert h.api.stages and all(stage == "CANCELED" for stage in h.api.stages.values()) + + +def test_image_mismatch_cancels_before_any_workload_command(harness): + h = harness + h.api.image_override = VERIFIER + record = h.backend.execute_script("dangerous", "reference") + assert "image" in record["error"] and record["cleanup_verified"] + assert not h.factory.created[0].commands + assert h.api.cancelled == ["job-0"] + + +def test_rpc_ready_sandbox_waits_for_running_control_plane(harness, monkeypatch): + h = harness + inspect = h.api.inspect_job + observed = [] + + def lagged_inspect(**kwargs): + job = inspect(**kwargs) + if not h.api.cancelled: + job.status.stage = "SCHEDULING" if not observed else "RUNNING" + observed.append(job.status.stage) + return job + + monkeypatch.setattr(h.api, "inspect_job", lagged_inspect) + monkeypatch.setattr("examples.coding_audit.hf.time.sleep", lambda _: None) + record = h.backend.execute_script(":", "reference") + assert record["error"] is None + assert observed == ["SCHEDULING", "RUNNING"] + assert record["cleanup_verified"] + assert h.factory.created[0].commands + + +@pytest.mark.parametrize("stage", ["SCHEDULING", "ERROR", "COMPLETED", "UNKNOWN"]) +def test_nonrunning_control_plane_never_executes_and_is_cleaned_up( + harness, monkeypatch, stage +): + h = harness + inspect = h.api.inspect_job + attempts = [] + + def never_running(**kwargs): + job = inspect(**kwargs) + if not h.api.cancelled: + attempts.append(job.id) + job.status.stage = stage + return job + + monkeypatch.setattr(h.api, "inspect_job", never_running) + monkeypatch.setattr("examples.coding_audit.hf.time.sleep", lambda _: None) + record = h.backend.execute_script("dangerous", "reference") + assert record["error"] is not None and record["cleanup_verified"] + assert not h.factory.created[0].commands + assert len(attempts) <= 20 + + +def test_startup_reconciliation_rechecks_image_before_execution(harness, monkeypatch): + h = harness + inspect = h.api.inspect_job + attempts = [] + + def changed_image(**kwargs): + job = inspect(**kwargs) + if not h.api.cancelled: + if attempts: + job.docker_image = VERIFIER + else: + job.status.stage = "SCHEDULING" + attempts.append(job.id) + return job + + monkeypatch.setattr(h.api, "inspect_job", changed_image) + monkeypatch.setattr("examples.coding_audit.hf.time.sleep", lambda _: None) + record = h.backend.execute_script("dangerous", "reference") + assert "image" in record["error"] and record["cleanup_verified"] + assert not h.factory.created[0].commands + assert len(attempts) == 2 + + +@pytest.mark.parametrize("failure", ["nonterminal", "inspection"]) +def test_cleanup_cannot_pass_from_cancel_return_alone(harness, failure): + h = harness + submission = h.backend.execute_script(":", "reference") + h.api.ignore_cancel = failure == "nonterminal" + h.api.inspect_error_after_cancel = failure == "inspection" + grade = h.backend.grade(submission) + assert grade["cleanup_verified"] is False + assert grade["reward"] is None and "cleanup" in grade["error"] + assert h.api.cancelled.count("job-1") == 3 + + +def test_exact_kill_fallback_requires_independent_terminal_inspection(harness): + h = harness + h.api.cancel_error = True + record = h.backend.execute_script(":", "reference") + assert record["cleanup_verified"] and h.factory.created[0].killed + assert h.api.inspections[-1][0] == record["sandbox_id"] + + +@pytest.mark.parametrize( + "path", + [ + "relative", + "/app/../tests/x", + "/app//x", + "/app/./x", + "/tests/x", + "/solution/x", + "/logs/verifier/reward.txt", + "/", + "//app/x", + ], +) +def test_submission_path_preflight(harness, path): + h = harness + with pytest.raises(HFBackendError): + HFTaskBackend(h.root, {**h.execution, "submission_paths": [path]}) + + +@pytest.mark.parametrize("paths", [["/app/a", "/app/a/b"], ["/app/a", "/app/a"]]) +def test_overlapping_paths_rejected(harness, paths): + with pytest.raises(HFBackendError, match="overlap|repeat"): + HFTaskBackend(harness.root, {**harness.execution, "submission_paths": paths}) + + +@pytest.mark.parametrize( + "mutation", ["extra_path", "tampered_digest", "not_bytes", "oversize"] +) +def test_grade_rejects_unsafe_transfer_before_launch(harness, mutation): + h = harness + files = {"/app/answer.txt": b"answer"} + submission = {"files": files, "submission_digest": submission_digest(files)} + if mutation == "extra_path": + files["/tests/test.sh"] = b"fake reward" + elif mutation == "tampered_digest": + files["/app/answer.txt"] = b"changed" + elif mutation == "not_bytes": + files["/app/answer.txt"] = "answer" + else: + files["/app/answer.txt"] = b"x" * (h.backend.max_bytes + 1) + grade = h.backend.grade(submission) + assert grade["error"] and grade["reward"] is None + assert not h.factory.created + + +@pytest.mark.parametrize("kind", ["symlink", "hardlink", "fifo", "symlink_parent"]) +def test_local_verifier_artifacts_reject_nonregular_or_linked_paths(harness, kind): + h = harness + path = h.root / "tests/extra" + if kind == "symlink": + path.symlink_to(h.root / "instruction.md") + elif kind == "hardlink": + os.link(h.root / "instruction.md", path) + elif kind == "fifo": + os.mkfifo(path) + else: + path.symlink_to(h.root / "solution", target_is_directory=True) + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert grade["error"] and not h.factory.created + + +@pytest.mark.parametrize( + "reward", + [ + b"NaN", + b"Infinity", + b"1e9999", + b'{"reward": true}', + b'{"reward": "1"}', + b'{"reward":1,"reward":0}', + b"garbage", + ], +) +def test_reward_requires_finite_trusted_numeric_bytes(harness, reward): + h = harness + h.factory.reward = reward + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert grade["error"] and grade["reward"] is None and grade["cleanup_verified"] + + +def test_json_reward_collects_original_bytes(harness): + h = harness + h.factory.reward = b'{"reward":0.5,"detail":"ok"}\n' + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert grade["reward"] == 0.5 and grade["rawreward"] == h.factory.reward + + +def test_provider_requests_base_id_with_distinct_token_and_records_source_limit( + harness, +): + h = harness + h.inference.responses = [response("printf answer > /app/answer.txt")] + generated = h.backend.generate_candidate(MODEL, 42) + assert generated["error"] is None + assert not h.factory.created + request = h.inference.requests[0] + assert request["model"] == MODEL["model_id"] and "@" not in request["model"] + assert "seed" not in request and "tools" not in request + assert h.inference.kwargs == [ + {"provider": "deepinfra", "api_key": "inference-token-secret", "timeout": 120} + ] + assert "PRIVATE_SOLUTION_SENTINEL" not in repr(request) + assert "PRIVATE_CANARY_SENTINEL" not in repr(request) + assert "test.sh" not in repr(request) + metadata = generated["model_metadata"] + assert metadata["source_revision"] == MODEL["revision"] + assert metadata["served_weight_revision_pinned"] is False + assert metadata["seed_supported"] is False + assert metadata["responses"][0]["system_fingerprint"] == "fp-123" + assert generated["usage"] == {"input_tokens": 10, "output_tokens": 5} + + +def test_seed_and_exact_provider_alias_are_explicit(harness): + h = harness + h.inference.responses = [response("true", model="provider/exact")] + record = h.backend.generate_candidate( + {**MODEL, "supports_seed": True, "provider_model_id": "provider/exact"}, 7 + ) + assert record["error"] is None + assert h.inference.requests[0]["seed"] == 7 + h.inference.responses = [response("true", model="provider/unlisted")] + record = h.backend.generate_candidate(MODEL, 7) + assert "response model" in record["error"] + assert record["model_metadata"]["responses"][0]["model"] == "provider/unlisted" + + +def test_top_p_is_forwarded_and_recorded(harness): + h = harness + h.inference.responses = [response("true")] + record = h.backend.generate_candidate({**MODEL, "top_p": 0.1}, 7) + assert record["error"] is None + assert h.inference.requests[0]["top_p"] == 0.1 + assert record["model_metadata"]["requests"][0]["top_p"] == 0.1 + + +def test_candidate_stream_assembles_text_and_final_usage_without_changing_recipe( + harness, +): + h = harness + events = [ + stream_event(delta={"role": "assistant", "content": "printf "}), + stream_event(delta={"reasoning_content": "Think 雪"}), + stream_event(delta={"content": "answer > /app/answer.txt"}, finish="stop"), + stream_event( + choices=[], + usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + ), + ] + h.inference.responses = [(event for event in events)] + record = h.backend.generate_candidate( + {**MODEL, "max_tokens": 8192, "max_total_tokens": 32768, "top_p": 0.1}, 7 + ) + assert record["error"] is None + assert record["script"] == "printf answer > /app/answer.txt" + assert record["transcript"][-1]["reasoning_content"] == "Think 雪" + assert record["usage"] == {"input_tokens": 10, "output_tokens": 5} + request = h.inference.requests[0] + assert request["stream"] is True + assert request["stream_options"] == {"include_usage": True} + assert request["max_tokens"] == 8192 and request["top_p"] == 0.1 + assert request["model"] == MODEL["model_id"] and request["temperature"] == 0 + assert "tools" not in request and "seed" not in request + assert record["model_metadata"]["requests"] == [ + {key: value for key, value in request.items() if key != "messages"} + ] + assert h.inference.closed == 1 and not h.factory.created + + +@pytest.mark.parametrize( + "outcome", ["complete", "early_close", "provider_error", "bad_json"] +) +def test_real_sdk_stream_serializes_request_and_closes_http_response( + harness, monkeypatch, outcome +): + h = harness + captured = {} + closed = [] + events = list(stream_response(response("printf answer > /app/answer.txt"))) + + class Body(httpx.SyncByteStream): + def __iter__(self): + for event in events[:1] if outcome == "early_close" else events: + yield ("data: " + json.dumps(event) + "\n\n").encode() + if outcome == "provider_error": + yield b'data: {"error": "provider failed"}\n\n' + elif outcome == "bad_json": + yield b"data: {broken\n\n" + elif outcome == "complete": + yield b"data: [DONE]\n\n" + + def close(self): + closed.append(True) + + def capture(request): + assert request.url.host == "api.deepinfra.com" + captured.update(json.loads(request.content)) + return httpx.Response(200, stream=Body()) + + monkeypatch.setattr( + DeepInfraConversationalTask, + "_prepare_mapping_info", + lambda self, model: hf_api.InferenceProviderMapping( + provider="deepinfra", + hf_model_id=model, + providerId=model, + task="conversational", + status="live", + ), + ) + h.backend.inference_factory = InferenceClient + with httpx.Client(transport=httpx.MockTransport(capture)) as client: + monkeypatch.setattr( + "huggingface_hub.inference._client.get_session", lambda: client + ) + record = h.backend.generate_candidate({**MODEL, "enable_thinking": False}, 0) + assert closed == [True] + assert captured["chat_template_kwargs"] == {"enable_thinking": False} + assert captured["stream"] is True + assert captured["stream_options"] == {"include_usage": True} + assert captured["model"] == MODEL["model_id"] + if outcome == "complete": + assert record["error"] is None and record["script"].startswith("printf answer") + assert record["usage"] == {"input_tokens": 10, "output_tokens": 5} + else: + assert record["error"] and record["script"] == "" + + +@pytest.mark.parametrize( + "mutation,diagnostic", + [ + ("model_drift", "response model"), + ("allowed_model_drift", "identity"), + ("id_drift", "identity"), + ("fingerprint_drift", "identity"), + ("two_choices", "one assistant"), + ("wrong_index", "one assistant"), + ("boolean_index", "one assistant"), + ("wrong_role", "assistant"), + ("missing_role", "assistant"), + ("nontext", "text"), + ("nontext_list", "text"), + ("tool_call", "tool"), + ("no_stop", "incomplete"), + ("length_finish", "incomplete"), + ("no_usage", "usage"), + ("missing_usage_field", "usage"), + ("boolean_usage", "usage"), + ("negative_usage", "usage"), + ("inconsistent_usage", "usage"), + ("early_usage", "usage"), + ("extra_content", "stop"), + ("duplicate_usage", "usage"), + ("provider_error", "provider"), + ("completion_budget", "token budget"), + ("total_budget", "token budget"), + ], +) +def test_candidate_stream_rejects_invalid_or_incomplete_sdk_events( + harness, mutation, diagnostic +): + h = harness + events = [ + stream_event(delta={"role": "assistant", "content": "true"}), + stream_event(finish="stop"), + stream_event( + choices=[], + usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + ), + ] + events = json.loads(json.dumps(events)) + if mutation == "model_drift": + events[1]["model"] = "unlisted/model" + elif mutation == "allowed_model_drift": + events[1]["model"] = "configured/alias" + elif mutation == "id_drift": + events[1]["id"] = "chatcmpl-other" + elif mutation == "fingerprint_drift": + events[1]["system_fingerprint"] = "fp-other" + elif mutation == "two_choices": + events[0]["choices"] *= 2 + elif mutation in {"wrong_index", "boolean_index"}: + events[0]["choices"][0]["index"] = True if mutation == "boolean_index" else 1 + elif mutation in {"wrong_role", "missing_role"}: + events[0]["choices"][0]["delta"]["role"] = ( + None if mutation == "missing_role" else "user" + ) + elif mutation == "nontext": + events[0]["choices"][0]["delta"]["content"] = 7 + elif mutation == "nontext_list": + events[0]["choices"][0]["delta"]["content"] = [] + elif mutation == "tool_call": + events[0]["choices"][0]["delta"]["tool_calls"] = [terminal()] + elif mutation == "no_stop": + events = events[:1] + elif mutation == "length_finish": + events[1]["choices"][0]["finish_reason"] = "length" + elif mutation == "no_usage": + events = events[:2] + elif mutation == "missing_usage_field": + del events[-1]["usage"]["total_tokens"] + elif mutation == "boolean_usage": + events[-1]["usage"]["prompt_tokens"] = True + elif mutation == "negative_usage": + events[-1]["usage"].update(prompt_tokens=-1, total_tokens=4) + elif mutation == "inconsistent_usage": + events[-1]["usage"]["total_tokens"] = 16 + elif mutation == "early_usage": + events[0]["usage"] = events[-1]["usage"] + elif mutation == "extra_content": + events.insert(2, stream_event(delta={"content": "false"})) + elif mutation == "duplicate_usage": + events.append(events[-1]) + elif mutation == "provider_error": + events[-1]["error"] = "provider failed after partial output" + elif mutation == "completion_budget": + events[-1]["usage"].update(completion_tokens=2049, total_tokens=2059) + elif mutation == "total_budget": + events[-1]["usage"].update(prompt_tokens=16384, total_tokens=16389) + closed = [] + + def streaming(): + try: + for event in events: + yield ChatCompletionStreamOutput.parse_obj_as_instance(event) + finally: + closed.append(True) + + h.inference.responses = [streaming()] + record = h.backend.generate_candidate( + {**MODEL, "response_model_aliases": ["configured/alias"]}, 0 + ) + assert diagnostic in record["error"], record + assert record["script"] == "" and not h.factory.created + assert closed == [True] and h.inference.closed == 1 + assert record["model_metadata"]["requests"][0]["stream"] is True + + +@pytest.mark.parametrize("stage", ["before_first", "partial", "after_stop"]) +def test_candidate_stream_provider_failures_preserve_request_and_partial_error( + harness, stage +): + h = harness + closed = [] + + def streaming(): + try: + if stage != "before_first": + yield stream_event(delta={"role": "assistant", "content": "true"}) + if stage == "after_stop": + yield stream_event(finish="stop") + raise RuntimeError("provider error with inference-token-secret") + finally: + closed.append(True) + + h.inference.responses = [streaming()] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] == "HF backend operation failed (RuntimeError)" + assert record["script"] == "" and not h.factory.created + assert "inference-token-secret" not in repr(record) + assert record["model_metadata"]["requests"][0]["stream"] is True + assert closed == [True] and h.inference.closed == 1 + + +@pytest.mark.parametrize("ending", ["provider_error", "extra_event", "over_budget"]) +def test_failed_candidate_stream_retains_observed_final_usage(harness, ending): + h = harness + output_tokens = 2049 if ending == "over_budget" else 5 + + def streaming(): + yield stream_event(delta={"role": "assistant", "content": "true"}) + yield stream_event(finish="stop") + yield stream_event( + choices=[], + usage={ + "prompt_tokens": 10, + "completion_tokens": output_tokens, + "total_tokens": 10 + output_tokens, + }, + ) + if ending == "provider_error": + raise RuntimeError("provider interrupted after usage") + if ending == "extra_event": + yield stream_event(choices=[]) + + h.inference.responses = [streaming()] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] and record["script"] == "" + assert record["usage"] == {"input_tokens": 10, "output_tokens": output_tokens} + assert record["input_tokens"] == 10 and record["output_tokens"] == output_tokens + assert h.inference.closed == 1 and not h.factory.created + + +def test_failed_candidate_request_retains_metadata_and_closes_client(harness): + h = harness + + def fail(**kwargs): + raise RuntimeError("provider unavailable") + + h.inference.chat_completion = fail + record = h.backend.generate_candidate({**MODEL, "top_p": 0.1}, 0) + assert record["error"] == "HF backend operation failed (RuntimeError)" + assert record["model_metadata"]["requests"][0]["top_p"] == 0.1 + assert record["model_metadata"]["requests"][0]["stream"] is True + assert h.inference.closed == 1 and record["script"] == "" + + +@pytest.mark.parametrize("field", ["content", "reasoning", "reasoning_content"]) +def test_candidate_stream_byte_bounds_stop_consuming_and_close(harness, field): + h = harness + h.backend.max_script = 16 if field == "content" else 65536 + (h.root / "instruction.md").write_text("write answer") + continued = [] + + def streaming(): + yield stream_event(delta={"role": "assistant", field: "雪" * 6}) + continued.append(True) + pytest.fail("oversized stream was consumed after its byte limit") + + h.inference.responses = [streaming()] + record = h.backend.generate_candidate( + {**MODEL, "max_output_bytes": 512 if field == "content" else 16}, 0 + ) + assert "byte limit" in record["error"] and record["script"] == "" + assert not continued and h.inference.closed == 1 + + +def test_candidate_stream_keeps_serialized_output_byte_gate(harness): + h = harness + h.inference.responses = [response("雪" * 10)] + record = h.backend.generate_candidate({**MODEL, "max_output_bytes": 64}, 0) + assert "output exceeded byte limit" in record["error"] + assert record["script"] == "" and h.inference.closed == 1 + + +def test_candidate_stream_accepts_usage_on_final_stop(harness): + h = harness + h.inference.responses = [ + ( + event + for event in [ + stream_event( + delta={"role": "assistant", "content": "true"}, + finish="stop", + usage={ + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + }, + ) + ] + ) + ] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] is None and record["script"] == "true" + assert record["usage"] == {"input_tokens": 10, "output_tokens": 5} + + +def test_candidate_stream_is_interrupted_by_root_deadline_and_closes(harness): + h = harness + h.backend.timeout = 0.02 + closed = [] + + def streaming(): + try: + yield stream_event(delta={"role": "assistant", "content": "true"}) + time.sleep(1) + finally: + closed.append(True) + + h.inference.responses = [streaming()] + started = time.monotonic() + record = h.backend.generate_candidate(MODEL, 0) + assert time.monotonic() - started < 0.5 + assert "deadline" in record["error"] and record["script"] == "" + assert closed == [True] and h.inference.closed == 1 + + +def test_unqualified_sdk_version_refused_before_launch(harness, monkeypatch): + monkeypatch.setattr("examples.coding_audit.hf.version", lambda name: "0.0.0") + with pytest.raises(HFBackendError, match="requires huggingface_hub"): + HFTaskBackend( + harness.backend.task_root, + harness.backend.execution, + sandbox_factory=harness.factory, + ) + assert not harness.factory.created + + +@pytest.mark.parametrize( + "model", + [ + {}, + {**MODEL, "revision": "main"}, + {**MODEL, "model_id": MODEL["model_id"] + "@" + MODEL["revision"]}, + {**MODEL, "provider": "auto"}, + {**MODEL, "supports_tools": False}, + ], +) +def test_invalid_model_preflight_never_launches_or_calls_provider(harness, model): + record = harness.backend.solve(model, 0) + assert record["error"] + assert not harness.factory.created and not harness.inference.requests + + +def test_target_tool_loop_is_bounded_and_uses_instruction_only(harness): + h = harness + h.inference.responses = [response(None, calls=[terminal()]), response("saved")] + record = h.backend.solve(MODEL, 1) + assert record["error"] is None and record["cleanup_verified"] + assert record["files"] == {"/app/answer.txt": b"answer"} + assert len(h.factory.created) == 1 + assert h.factory.created[0].commands[0][0] == [ + "/bin/bash", + "-c", + "printf answer > /app/answer.txt", + ] + assert h.inference.requests[0]["tools"][0]["function"]["name"] == "terminal" + assert all("stream" not in request for request in h.inference.requests) + assert h.inference.closed == 0 + assert "PRIVATE_SOLUTION_SENTINEL" not in repr(h.inference.requests) + assert "PRIVATE_CANARY_SENTINEL" not in repr(h.inference.requests) + assert record["input_tokens"] == 20 and record["output_tokens"] == 10 + + +def test_candidate_context_binds_only_declared_public_bytes_to_prompt( + harness, monkeypatch +): + from examples.coding_audit import hf + + h = harness + path = "environment/source/pkg/api.py" + source = h.root / path + source.parent.mkdir(parents=True) + raw = 'def answer():\n return "雪"\n'.encode() + source.write_bytes(raw) + (source.parent / "unused.py").write_text("UNDECLARED_PUBLIC_SENTINEL") + (source.parent / ".hidden").mkdir() + (source.parent / ".hidden/private.txt").write_text("HIDDEN_SENTINEL") + reads = [] + original = hf._local_file + + def read(path, limit): + reads.append(path.relative_to(h.root).as_posix()) + return original(path, limit) + + monkeypatch.setattr(hf, "_local_file", read) + h.backend.workdir = "/workspace/runtime" + h.inference.responses = [response("true")] + record = h.backend.generate_candidate({**MODEL, "context_paths": [path]}, 0) + assert record["error"] is None + assert reads == ["instruction.md", path] + prompt = h.inference.requests[0]["messages"][1]["content"] + assert prompt.startswith((h.root / "instruction.md").read_text()) + assert path in prompt and "pkg/api.py" in prompt and "/workspace/runtime" in prompt + assert raw.decode() in prompt + assert ( + "UNDECLARED_PUBLIC_SENTINEL" not in prompt and "HIDDEN_SENTINEL" not in prompt + ) + assert "PRIVATE_SOLUTION_SENTINEL" not in prompt + assert record["candidate_context"] == [ + { + "path": path, + "runtime_path": "pkg/api.py", + "size": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + } + ] + assert "context_paths" not in h.inference.requests[0] + + +def test_default_candidate_prompt_and_target_protocol_are_unchanged(harness): + h = harness + h.inference.responses = [response("true"), response("done")] + generated = h.backend.generate_candidate(MODEL, 0) + solved = h.backend.solve(MODEL, 1) + assert generated["error"] is None and solved["error"] is None + assert "candidate_context" not in generated + assert all( + request["messages"][1]["content"] == (h.root / "instruction.md").read_text() + for request in h.inference.requests + ) + + +@pytest.mark.parametrize( + "kind", ["symlink", "symlink_parent", "hardlink", "directory", "fifo"] +) +def test_candidate_context_rejects_linked_or_nonregular_files_before_inference( + harness, kind +): + h = harness + parent = h.root / "environment/source" + parent.mkdir(parents=True) + source = parent / "api.py" + if kind == "symlink": + source.symlink_to(h.root / "solution/solve.sh") + elif kind == "symlink_parent": + (h.root / "solution/api.py").write_text("PRIVATE_API_SENTINEL") + (parent / "linked").symlink_to(h.root / "solution", target_is_directory=True) + source = parent / "linked/api.py" + elif kind == "hardlink": + os.link(h.root / "solution/solve.sh", source) + elif kind == "directory": + source.mkdir() + else: + os.mkfifo(source) + record = h.backend.generate_candidate( + {**MODEL, "context_paths": [source.relative_to(h.root).as_posix()]}, 0 + ) + assert record["error"] and record["script"] == "" + assert not h.inference.requests and not h.inference.kwargs + assert not h.factory.created + + +@pytest.mark.parametrize( + "contents", + [ + [b"public\x00hidden"], + [b"\xff"], + [b"x" * 65537], + [b"x" * 32768, b"y" * 32769], + ], +) +def test_candidate_context_requires_bounded_utf8_without_nul(harness, contents): + h = harness + parent = h.root / "environment/source" + parent.mkdir(parents=True) + paths = [] + for index, raw in enumerate(contents): + path = parent / f"public-{index}.txt" + path.write_bytes(raw) + paths.append(path.relative_to(h.root).as_posix()) + record = h.backend.generate_candidate({**MODEL, "context_paths": paths}, 0) + assert record["error"] and record["script"] == "" + assert not h.inference.requests and not h.inference.kwargs + + +@pytest.mark.parametrize( + "size,budget,accepted", [(65536, 32768, True), (1024, 64, False)] +) +def test_candidate_context_uses_combined_byte_and_existing_message_budget( + harness, size, budget, accepted +): + h = harness + path = h.root / "environment/source/api.py" + path.parent.mkdir(parents=True) + path.write_bytes(b"x" * size) + h.inference.responses = [response("true")] + record = h.backend.generate_candidate( + { + **MODEL, + "context_paths": ["environment/source/api.py"], + "max_total_tokens": budget, + }, + 0, + ) + assert (record["error"] is None) is accepted + assert bool(h.inference.requests) is accepted + if not accepted: + assert "context exceeds byte budget" in record["error"] + + +def test_target_context_is_rejected_before_sandbox_or_inference(harness): + h = harness + record = h.backend.solve( + {**MODEL, "context_paths": ["environment/source/api.py"]}, 0 + ) + assert record["error"] and not h.factory.created + assert not h.inference.requests and not h.inference.kwargs + + +@pytest.mark.parametrize( + "path", + [ + "solution/solve.sh", + "environment/source/../../solution/solve.sh", + "environment/source/tests/hidden.py", + "environment/source/validation/spec.json", + "environment/source/.hidden/secret.py", + "environment/source/secrets/token", + "environment/source/api\x00.py", + ], +) +def test_invalid_candidate_context_is_rejected_without_reading_private_files( + harness, monkeypatch, path +): + from examples.coding_audit import hf + + h = harness + reads = [] + original = hf._local_file + + def read(filename, limit): + assert filename == h.root / "instruction.md" + reads.append(filename) + return original(filename, limit) + + monkeypatch.setattr(hf, "_local_file", read) + record = h.backend.generate_candidate({**MODEL, "context_paths": [path]}, 0) + assert record["error"] and record["script"] == "" + assert reads == [h.root / "instruction.md"] + assert not h.inference.requests and not h.inference.kwargs + + +def test_excess_parallel_tool_calls_execute_nothing_and_cleanup(harness): + h = harness + h.inference.responses = [ + response(None, calls=[terminal(identifier="a"), terminal(identifier="b")]) + ] + record = h.backend.solve({**MODEL, "max_steps": 1}, 0) + assert record["error"] is None and record["cleanup_verified"] + assert record["truncated"] and record["termination_reason"] == "tool_call_limit" + assert not h.factory.created[0].commands + + +def test_step_exhaustion_hands_actual_submission_to_grader(harness): + h = harness + h.inference.responses = [response(None, calls=[terminal()])] + record = h.backend.solve({**MODEL, "max_steps": 1}, 0) + assert record["error"] is None and record["files"] == {"/app/answer.txt": b"answer"} + assert record["truncated"] and record["termination_reason"] == "step_limit" + assert len(h.inference.requests) == 1 and record["cleanup_verified"] + grade = h.backend.grade(record) + assert grade["error"] is None and grade["reward"] == 1 + + +def test_invalid_tool_name_is_not_executed(harness): + h = harness + call = terminal() + call["function"]["name"] = "provider_request" + h.inference.responses = [response(None, calls=[call])] + record = h.backend.solve(MODEL, 0) + assert "invalid terminal" in record["error"] and not h.factory.created[0].commands + + +def test_output_and_token_limits_are_fail_closed(harness): + h = harness + h.factory.output = "💥" * 4097 + record = h.backend.execute_script(":", "reference") + assert "output exceeded" in record["error"] and record["cleanup_verified"] + h.factory.output = "" + h.inference.responses = [ + response( + "true", + usage={ + "prompt_tokens": 16000, + "completion_tokens": 400, + "total_tokens": 16400, + }, + ) + ] + record = h.backend.generate_candidate(MODEL, 0) + assert "token budget" in record["error"] + + +def test_private_path_audit_blocks_before_execution(harness): + h = harness + h.factory.initial["/tests"] = b"private" + record = h.backend.execute_script("true", "reference") + assert "private path" in record["error"] and record["cleanup_verified"] + assert not h.factory.created[0].commands + + +def test_leakage_probes_recheck_transcripts_without_extra_jobs(harness): + h = harness + h.factory.submitted = None + probes = h.backend.leakage_probes() + assert {row["name"] for row in probes} == { + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", + } + assert all(row["passed"] and row["error"] is None for row in probes) + assert len(h.factory.created) == 2 + h.inference.responses = [response("# PRIVATE_CANARY_SENTINEL\ntrue")] + h.backend.generate_candidate(MODEL, 0) + probes = h.backend.leakage_probes() + assert len(h.factory.created) == 2 + assert not next(row for row in probes if row["name"] == "canary_not_in_transcript")[ + "passed" + ] + + +def test_fresh_state_marker_blocks_reused_filesystem(harness): + h = harness + first = h.backend.execute_script(":", "reference") + h.factory.initial.update(h.factory.created[0].files.data) + second = h.backend.execute_script(":", "target") + assert first["error"] is None and "prior-state" in second["error"] + assert not h.factory.created[1].commands + assert not next( + row for row in h.backend.leakage_probes() if row["name"] == "fresh_state_absent" + )["passed"] + + +def test_state_marker_uses_configured_workdir(harness): + h = harness + h.execution["workdir"] = "/workspace/task" + backend = HFTaskBackend( + h.root, + h.execution, + token="execution-token-secret", + inference_token="inference-token-secret", + api=h.api, + sandbox_factory=h.factory, + inference_factory=h.inference, + ) + assert PurePosixPath(backend.marker_path).parent == PurePosixPath("/workspace/task") + assert PurePosixPath(backend.marker_path).name.startswith(".openenv-state-") + first = backend.execute_script(":", "reference") + h.factory.initial.update(h.factory.created[0].files.data) + second = backend.execute_script(":", "target") + assert first["error"] is None and "prior-state" in second["error"] + + +def test_submission_tree_digest_matches_canonical_algorithm(tmp_path): + (tmp_path / "app").mkdir() + (tmp_path / "app/a").write_bytes(b"alpha") + (tmp_path / "app/z").write_bytes(b"omega") + assert submission_digest({"/app/z": b"omega", "/app/a": b"alpha"}) == source_digest( + tmp_path + ) + assert submission_digest({}) == hashlib.sha256(b"").hexdigest() + + +def test_deadline_interrupts_blocking_controller_and_restores_handler(): + handler = signal.getsignal(signal.SIGALRM) + started = time.monotonic() + with pytest.raises(HFBackendError, match="deadline"): + with _deadline(0.02): + time.sleep(1) + assert time.monotonic() - started < 0.5 + assert signal.getsignal(signal.SIGALRM) == handler + assert signal.getitimer(signal.ITIMER_REAL)[0] == 0 + + +def test_strong_isolation_and_equal_credentials_rejected(harness): + h = harness + with pytest.raises(HFBackendError, match="strong"): + HFTaskBackend(h.root, {**h.execution, "require_strong_isolation": True}) + with pytest.raises(HFBackendError, match="credentials must differ"): + HFTaskBackend(h.root, h.execution, token="same", inference_token="same") + + +def test_runtime_tokens_are_redacted_and_credential_files_rejected(harness): + h = harness + h.factory.output = "server echoed execution-token-secret" + record = h.backend.execute_script(":", "reference") + assert "execution-token-secret" not in repr(record) + assert "[REDACTED]" in repr(record["transcript"]) + h.factory.output = "" + h.factory.submitted = b"inference-token-secret" + record = h.backend.execute_script(":", "reference") + assert "credential detected" in record["error"] and record["files"] == {} + assert "inference-token-secret" not in repr(record) + + +def test_empty_submission_does_not_inherit_verifier_image_answer(harness): + h = harness + h.factory.initial["/app/answer.txt"] = b"image-provided-answer" + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert grade["error"] is None + assert "/app/answer.txt" not in h.factory.created[0].files.data + + +def test_verifier_private_image_paths_are_not_overlaid(harness): + h = harness + h.factory.initial["/tests"] = b"stale-tests" + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert "preexisting private" in grade["error"] and grade["cleanup_verified"] + assert not h.factory.created[0].commands + + +def test_remote_regular_file_and_read_growth_checks(harness): + h = harness + sandbox = h.factory(image=AGENT) + sandbox.files.data["/app/answer.txt"] = b"answer" + sandbox.files.types["/app/answer.txt"] = "symlink" + with pytest.raises(HFBackendError, match="regular"): + h.backend._collect(sandbox) + sandbox.files.types.clear() + sandbox.files.grow_on_read = True + with pytest.raises(SandboxError): + h.backend._collect(sandbox) + assert sandbox.files.MAX_READ_BYTES == h.backend.max_bytes + + +def test_missing_execution_or_inference_credentials_never_fall_back_to_hf_token( + harness, monkeypatch +): + h = harness + monkeypatch.setenv("HF_TOKEN", "persistence-token") + backend = HFTaskBackend( + h.root, + h.execution, + api=h.api, + sandbox_factory=h.factory, + inference_factory=h.inference, + ) + assert "HF_SANDBOX_TOKEN" in backend.execute_script(":", "nop")["error"] + assert "HF_INFERENCE_TOKEN" in backend.solve(MODEL, 0)["error"] + assert not h.factory.created and not h.inference.requests + + +def test_timed_out_command_with_zero_exit_is_error(harness): + harness.factory.timed_out = True + record = harness.backend.execute_script(":", "reference") + assert "timed out" in record["error"] and record["cleanup_verified"] + + +def test_cleanup_deadline_is_not_swallowed_by_retry_loop(harness): + h = harness + original = h.api.cancel_job + + def blocked_cancel(**kwargs): + time.sleep(2) + return original(**kwargs) + + h.api.cancel_job = blocked_cancel + h.backend.cleanup_timeout = 0.02 + started = time.monotonic() + record = h.backend.execute_script(":", "reference") + assert time.monotonic() - started < 0.5 + assert record["error"] and not record["cleanup_verified"] + + +def test_canary_probe_checks_actual_unicode_message_text(harness): + h = harness + h.backend.canary = "private\n雪" + h.backend._transcripts = [[{"content": "private\n雪"}]] + assert not h.backend.leakage_probes()[-1]["passed"] + + +def test_failed_shell_command_is_observed_then_agent_repairs(harness, monkeypatch): + h = harness + original = FakeSandbox.run + calls = [] + + def run_with_failure(sandbox, argv, **kwargs): + calls.append(argv) + result = original(sandbox, argv, **kwargs) + result.exit_code = 1 if len(calls) == 1 else 0 + return result + + monkeypatch.setattr(FakeSandbox, "run", run_with_failure) + h.inference.responses = [ + response(None, calls=[terminal("grep missing absent", "a")]), + response(None, calls=[terminal("printf answer > /app/answer.txt", "b")]), + response("saved"), + ] + record = h.backend.solve(MODEL, 0) + assert record["error"] is None and not record["truncated"] + assert record["files"] == {"/app/answer.txt": b"answer"} + observation = h.inference.requests[1]["messages"][-1] + assert json.loads(observation["content"])["exit_code"] == 1 + assert h.backend.grade(record)["reward"] == 1 + + +def test_execution_token_in_terminal_output_never_reaches_provider(harness): + h = harness + h.factory.output = "server echoed execution-token-secret" + h.inference.responses = [response(None, calls=[terminal()]), response("done")] + record = h.backend.solve(MODEL, 0) + assert record["error"] is None + assert "execution-token-secret" not in repr(h.inference.requests) + assert "[REDACTED]" in repr(h.inference.requests[1]) + + +def test_failed_startup_recovers_exact_labeled_job_and_independently_cleans(harness): + h = harness + h.factory.fail_after_create = True + original = h.api.cancel_job + cancelled = [] + + def transient_cancel(**kwargs): + cancelled.append(kwargs["job_id"]) + if len(cancelled) == 1: + raise RuntimeError("transient controller failure") + return original(**kwargs) + + h.api.cancel_job = transient_cancel + record = h.backend.execute_script(":", "reference") + assert record["error"] and record["cleanup_verified"] + assert record["sandbox_id"] == "job-0" and cancelled == ["job-0", "job-0"] + assert record["cleanup_job_ids"] == ["job-0"] + assert h.api.stages["job-0"] == "CANCELED" + + +def test_transient_kill_exception_does_not_abort_cleanup_retries(harness, monkeypatch): + h = harness + h.api.cancel_error = True + original = FakeSandbox.kill + killed = [] + + def transient_kill(sandbox): + killed.append(sandbox.id) + if len(killed) == 1: + raise RuntimeError("transient kill failure") + original(sandbox) + + monkeypatch.setattr(FakeSandbox, "kill", transient_kill) + record = h.backend.execute_script(":", "reference") + assert record["error"] is None and record["cleanup_verified"] + assert killed == ["job-0", "job-0"] + + +def test_startup_recovery_retries_transient_listing_error(harness): + h = harness + h.factory.fail_after_create = True + original = h.api.list_jobs + calls = [] + + def transient_listing(**kwargs): + calls.append(kwargs) + if len(calls) == 1: + raise RuntimeError("transient lookup failure") + return original(**kwargs) + + h.api.list_jobs = transient_listing + record = h.backend.execute_script(":", "reference") + assert record["error"] and record["cleanup_verified"] + assert len(calls) == 2 and h.api.cancelled == ["job-0"] + + +def test_rejected_length_stream_keeps_finish_and_terminal_usage(harness): + h = harness + h.inference.responses = [ + ( + ChatCompletionStreamOutput.parse_obj_as_instance(event) + for event in ( + [ + stream_event(delta={"role": "assistant", "content": "partial"}), + stream_event(finish="length"), + stream_event( + choices=[], + usage={ + "prompt_tokens": 10, + "completion_tokens": 2048, + "total_tokens": 2058, + }, + ), + ] + ) + ) + ] + record = h.backend.generate_candidate(MODEL, 0) + assert "incomplete" in record["error"] and record["script"] == "" + assert record["model_metadata"]["responses"][0]["finish_reason"] == "length" + assert record["usage"] == {"input_tokens": 10, "output_tokens": 2048} + assert h.inference.closed == 1 and not h.factory.created + + +@pytest.mark.parametrize("value", [0, 1, "false", None]) +def test_thinking_flag_requires_an_explicit_boolean(harness, value): + h = harness + record = h.backend.generate_candidate({**MODEL, "enable_thinking": value}, 0) + assert "enable_thinking must be a boolean" in record["error"] + assert not h.inference.requests and record["script"] == "" + + +@pytest.mark.parametrize("language", ["bash", "sh", "shell"]) +@pytest.mark.parametrize("closed", [False, True]) +def test_fenced_candidate_keeps_raw_response_and_executes_only_bash( + harness, language, closed +): + h = harness + body = "printf answer > /app/answer.txt\n" + raw = "```" + language + "\n" + body + ("```\n" if closed else "") + h.inference.responses = [response(raw)] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] is None and record["script"] == body + assert record["transcript"][-1]["content"] == raw + metadata = record["script_format"] + assert metadata["raw_sha256"] == hashlib.sha256(raw.encode()).hexdigest() + assert ( + metadata["script_sha256"] + == hashlib.sha256(record["script"].encode()).hexdigest() + ) + assert metadata["closing_fence_present"] is closed + submission = h.backend.execute_script(record["script"], "generated") + assert submission["error"] is None + assert h.factory.created[0].commands[-1][0] == ["/bin/bash", "-c", record["script"]] + + +@pytest.mark.parametrize( + "raw", ["```bash\n```", "```python\nprint('wrong language')\n```"] +) +def test_empty_or_non_bash_fence_does_not_create_a_candidate(harness, raw): + h = harness + h.inference.responses = [response(raw)] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] and record["script"] == "" and not h.factory.created diff --git a/examples/coding_audit/test_runner.py b/examples/coding_audit/test_runner.py new file mode 100644 index 000000000..b97384b05 --- /dev/null +++ b/examples/coding_audit/test_runner.py @@ -0,0 +1,838 @@ +"""Exercise collection, gating and artifact binding without paid services.""" + +import hashlib +import json +import runpy +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +from examples.coding_audit.runner import ( + coding_budget, + live_provider_id, + normalize_coding_config, + run_coding_validation, +) + + +@pytest.mark.parametrize("representation", ["dict", "list", "json-list"]) +def test_live_hub_provider_mapping_shapes(representation): + entry = SimpleNamespace( + provider="deepinfra", provider_id="Qwen/Qwen3.8-27B", status="live" + ) + mappings = ( + {"deepinfra": entry} + if representation == "dict" + else [vars(entry)] + if representation == "json-list" + else [entry] + ) + assert ( + live_provider_id( + SimpleNamespace(inference_provider_mapping=mappings), "deepinfra" + ) + == "Qwen/Qwen3.8-27B" + ) + + +@pytest.mark.parametrize( + "mappings", + [ + None, + [], + [SimpleNamespace(provider="deepinfra", provider_id="model", status="staging")], + [SimpleNamespace(provider="deepinfra", provider_id="model", status="live")] * 2, + ], +) +def test_provider_mapping_must_be_unique_and_live(mappings): + with pytest.raises(ValueError): + live_provider_id( + SimpleNamespace(inference_provider_mapping=mappings), "deepinfra" + ) + + +def configuration(): + config = json.loads(Path(__file__).with_name("config.json").read_text()) + config["generation"]["candidate_count"] = 1 + config["validation"].update(reference_repeats=2, target_attempts=2) + config["seeds"] = [1, 2] + return config + + +class Backend: + def __init__(self): + self.number = 0 + self.target_calls = 0 + self.generated = "mkdir -p /app; echo 42 > /app/answer.txt" + self.bad_verifier = False + self.bad_raw = False + self.target_error = False + self.mutate = None + self.transcript = [] + + def execute_script(self, script, role): + self.number += 1 + value = b"42\n" if "42" in script else b"41\n" + files = {} if script == "true" else {"/app/answer.txt": value} + digest = hashlib.sha256( + json.dumps({k: v.hex() for k, v in files.items()}, sort_keys=True).encode() + ).hexdigest() + image = configuration()["execution"]["agent_image"] + return { + "files": files, + "submission_digest": digest, + "sandbox_id": f"solver-{self.number}", + "image_ref": image, + "cleanup_verified": True, + "error": None, + "transcript": list(self.transcript), + "usage": {"input_tokens": 10, "output_tokens": 5}, + "latency_seconds": 1, + } + + def solve(self, model, seed, role): + self.target_calls += 1 + result = self.execute_script("42" if seed == 2 else "41", role) + if self.target_error: + result["error"] = "provider failure" + return result + + def generate_candidate(self, model, seed): + if self.mutate: + self.mutate() + return {"script": self.generated, "error": None} + + def grade(self, submission): + self.number += 1 + reward = int( + submission["files"].get("/app/answer.txt") == b"42\n" or self.bad_verifier + ) + return { + "grade_sandbox_id": f"grader-{self.number}", + "verifier_image_ref": configuration()["execution"]["verifier_image"], + "reward": reward, + "rawreward": b"0.999999999999999999999999" + if self.bad_raw + else str(reward).encode(), + "cleanup_verified": True, + "error": None, + "transcript": [ + { + "exit_code": 0, + "output": f"Native reward: {reward}\n", + "output_truncated": False, + } + ], + } + + def leakage_probes(self): + return [ + {"name": name, "passed": True, "error": None} + for name in ( + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", + ) + ] + + +@pytest.fixture +def task(tmp_path): + root = tmp_path / "task" + for name in ("tests", "solution", "environment"): + (root / name).mkdir(parents=True) + (root / "task.toml").write_text('version = "1.0"\n') + (root / "instruction.md").write_text("Write exactly 42 into /app/answer.txt.\n") + (root / "tests/test.sh").write_text("#!/bin/bash\nprintf 1\n") + (root / "solution/solve.sh").write_text("printf 42 > /app/answer.txt\n") + (root / "environment/Dockerfile").write_text( + "FROM python:3.12-slim\nWORKDIR /app\n" + ) + return root + + +def test_complete_collection_is_bound_and_keeps_model_difficulty(task, tmp_path): + backend = Backend() + output = tmp_path / "audit" + report = run_coding_validation(task, configuration(), output, backend=backend) + assert report["accepted"] + assert not report["rfc008_complete"] + assert not report["trainability_established"] + assert backend.target_calls == 2 + assert report["checks"]["target_difficulty"]["solve_rate"] == 0.5 + evidence = json.loads((output / "evidence.json").read_text()) + assert {r["origin"] for r in evidence["references"]} == {"provided", "generated"} + assert len(evidence["generation"]) == 1 + for row in evidence["trials"]: + artifact = row["grader_transcript"] + assert artifact["path"].endswith("/grader-transcript.json") + data = (output / artifact["path"]).read_bytes() + assert hashlib.sha256(data).hexdigest() == artifact["sha256"] + assert json.loads(data)[0]["output"] == f"Native reward: {row['reward']}\n" + assert evidence["trials"][0]["reward"] == 0 + for line in (output / "SHA256SUMS").read_text().splitlines(): + digest, name = line.split(" ", 1) + assert hashlib.sha256((output / name).read_bytes()).hexdigest() == digest + + +def test_bad_verifier_blocks_target_inference(task, tmp_path): + backend = Backend() + backend.bad_verifier = True + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + assert report["checks"]["verifier_rejection"]["status"] == "fail" + assert backend.target_calls == 0 + + +def test_failed_clean_candidate_excluded_without_overwriting_frozen_verifier( + task, tmp_path +): + backend = Backend() + backend.generated = "echo wrong" + before = (task / "tests/test.sh").read_bytes() + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert report["accepted"] + assert len(report["checks"]["references"]["excluded"]) == 1 + assert (task / "tests/test.sh").read_bytes() == before + + +def test_raw_reward_precision_disagreement_cannot_pass(task, tmp_path): + backend = Backend() + backend.bad_raw = True + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + assert backend.target_calls == 0 + assert report["checks"]["reliability"]["status"] == "error" + + +def test_target_infra_failure_preserves_missing_not_zero(task, tmp_path): + backend = Backend() + backend.target_error = True + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + check = report["checks"]["target_difficulty"] + assert not report["accepted"] + assert check["completed"] == 0 + assert check["solve_rate"] is None + + +def test_changed_source_cannot_be_admitted(task, tmp_path): + backend = Backend() + backend.mutate = lambda: (task / "instruction.md").write_text("Changed task") + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + evidence = json.loads((tmp_path / "audit/evidence.json").read_text()) + assert evidence["collection_error"] == "ValueError" + + +def test_credentials_in_transcript_are_omitted_and_block_acceptance( + task, tmp_path, monkeypatch +): + token = "test-secret-not-a-real-token" + monkeypatch.setenv("HF_SANDBOX_TOKEN", token) + backend = Backend() + backend.transcript = [{"content": token}] + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + for path in (tmp_path / "audit").rglob("*"): + if path.is_file(): + assert token.encode() not in path.read_bytes() + + +@pytest.mark.parametrize( + "path,value", [("agent_image", "python:latest"), ("api_key", "secret")] +) +def test_mutable_images_and_embedded_credentials_rejected(path, value): + config = configuration() + config["execution"][path] = value + with pytest.raises(ValueError): + normalize_coding_config(config) + + +def test_budget_is_bounded_before_any_remote_execution(task, tmp_path): + config = configuration() + config["cloud"]["max_sandboxes"] = 1 + with pytest.raises(ValueError, match="budget"): + run_coding_validation(task, config, tmp_path / "audit", backend=Backend()) + assert not (tmp_path / "audit").exists() + assert ( + coding_budget(normalize_coding_config(configuration()))["inference_calls_max"] + == 33 + ) + + +@pytest.mark.parametrize("accepted", [True, False]) +def test_cli_dispatch_uses_example_runner_and_retains_evidence( + task, tmp_path, monkeypatch, capsys, accepted +): + config = tmp_path / "config.json" + config.write_text(json.dumps(configuration())) + original = run_coding_validation + backend = Backend() + backend.bad_verifier = not accepted + monkeypatch.setattr( + "examples.coding_audit.runner.run_coding_validation", + lambda task, config, output: original(task, config, output, backend=backend), + ) + output = tmp_path / "audit" + monkeypatch.setattr( + sys, + "argv", + [ + "examples.coding_audit", + "--task", + str(task), + "--config", + str(config), + "--output", + str(output), + "--json", + ], + ) + with pytest.raises(SystemExit) as result: + runpy.run_module("examples.coding_audit", run_name="__main__") + assert result.value.code == (0 if accepted else 1) + report = json.loads((output / "report.json").read_text()) + assert report["accepted"] is accepted + assert json.loads(capsys.readouterr().out) == report + assert (output / "evidence.json").is_file() + for line in (output / "SHA256SUMS").read_text().splitlines(): + digest, name = line.split(" ", 1) + assert hashlib.sha256((output / name).read_bytes()).hexdigest() == digest + + +def test_cli_requires_persisted_evidence(task, tmp_path, monkeypatch, capsys): + monkeypatch.setattr( + sys, + "argv", + [ + "examples.coding_audit", + "--task", + str(task), + "--config", + str(tmp_path / "missing.json"), + ], + ) + with pytest.raises(SystemExit) as result: + runpy.run_module("examples.coding_audit", run_name="__main__") + assert result.value.code == 2 + assert "--output" in capsys.readouterr().err + + +def test_cli_input_failure_exits_two_without_starting_backend( + task, tmp_path, monkeypatch, capsys +): + from examples.coding_audit.__main__ import main + + monkeypatch.setattr( + "examples.coding_audit.__main__.run_coding_validation", + lambda *args: pytest.fail("input failure reached backend"), + ) + assert ( + main( + [ + "--task", + str(task), + "--config", + str(tmp_path / "missing.json"), + "--output", + str(tmp_path / "audit"), + ] + ) + == 2 + ) + assert "input failure (FileNotFoundError)" in capsys.readouterr().err + + +def test_exact_json_threshold_cannot_round_into_a_pass(task, tmp_path): + from decimal import Decimal + + from openenv.validation.runtime.json_values import loads + + config = loads( + json.dumps(configuration()).replace( + '"reward_max": 1,', '"reward_max": 1.00000000000000000001,' + ) + ) + normalized = normalize_coding_config(config) + assert normalized["validation"]["reward_max"] == Decimal("1.00000000000000000001") + report = run_coding_validation(task, config, tmp_path / "audit", backend=Backend()) + assert not report["accepted"] + saved = loads((tmp_path / "audit/config.json").read_text()) + assert saved["validation"]["reward_max"] == normalized["validation"]["reward_max"] + + +@pytest.mark.parametrize("value", ["", "missing"]) +def test_execution_error_status_cannot_be_overwritten(task, tmp_path, value): + class BrokenStatus(Backend): + def execute_script(self, script, role): + row = super().execute_script(script, role) + if value == "missing": + del row["error"] + else: + row["error"] = value + return row + + backend = BrokenStatus() + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + assert backend.target_calls == 0 + rows = json.loads((tmp_path / "audit/evidence.json").read_text())["trials"] + assert all(row["error"] is not None for row in rows) + + +def test_verifier_identity_comes_from_verified_snapshot(task, tmp_path, monkeypatch): + from examples.coding_audit import runner as coding_runner + + original = coding_runner.source_digest + expected = original(task / "tests") + + def raced(path): + if path == task / "tests": + raise AssertionError( + "mutable original verifier must not be hashed independently" + ) + return original(path) + + monkeypatch.setattr(coding_runner, "source_digest", raced) + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=Backend() + ) + assert report["accepted"] + assert report["verifier_digest"] == expected + + +def test_credential_metadata_is_failed_and_report_is_sanitized( + task, tmp_path, monkeypatch +): + secret = "dummy-execution-credential-for-regression" + monkeypatch.setenv("HF_SANDBOX_TOKEN", secret) + + class DiagnosticBackend(Backend): + def leakage_probes(self): + return super().leakage_probes() + [ + { + "name": "diagnostic:" + secret, + "passed": True, + "error": None, + secret: "key", + } + ] + + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=DiagnosticBackend() + ) + assert not report["accepted"] + assert secret not in json.dumps(report) + for path in (tmp_path / "audit").rglob("*"): + if path.is_file(): + assert secret.encode() not in path.read_bytes() + + +def test_credential_in_config_is_rejected_before_execution(task, tmp_path, monkeypatch): + secret = "dummy-config-credential-for-regression" + monkeypatch.setenv("HF_SANDBOX_TOKEN", secret) + config = configuration() + config["negative_cases"][0]["id"] = secret + config["validation"]["required_negative_cases"][0] = secret + with pytest.raises(ValueError, match="credentials"): + run_coding_validation(task, config, tmp_path / "audit", backend=Backend()) + + +def test_in_task_output_rejected_before_inference(task): + backend = Backend() + with pytest.raises(ValueError, match="outside"): + run_coding_validation( + task, configuration(), task / "audit.artifacts", backend=backend + ) + assert backend.target_calls == 0 + assert not (task / "audit.artifacts").exists() + + +@pytest.mark.parametrize( + "artifact", + [ + "environment/source/api.py", + "instruction.md", + "tests/test.sh", + "solution/solve.sh", + ], +) +def test_snapshot_rejects_hardlinked_artifacts_before_inference( + task, tmp_path, monkeypatch, artifact +): + import huggingface_hub + from examples.coding_audit import hf + + m = SimpleNamespace(**runpy.run_path(str(Path(__file__).with_name("test_hf.py")))) + api, inference = m.API(), m.InferenceFactory() + factory = m.Factory(api) + api.model_info = lambda *args, **kwargs: SimpleNamespace( + sha=m.MODEL["revision"], + inference_provider_mapping={ + "deepinfra": {"provider_id": m.MODEL["model_id"], "status": "live"} + }, + ) + monkeypatch.setattr(huggingface_hub, "HfApi", lambda **kwargs: api) + monkeypatch.setattr(hf, "HfApi", lambda **kwargs: api) + monkeypatch.setattr(hf.Sandbox, "create", factory) + monkeypatch.setattr(hf, "InferenceClient", inference) + for name in ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN"): + monkeypatch.setenv(name, "synthetic-test-" + name) + context_path = "environment/source/api.py" + source = task / context_path + source.parent.mkdir(parents=True) + if artifact == context_path: + source.hardlink_to(task / "solution/solve.sh") + else: + source.write_text("def public_answer(): return 42\n") + (tmp_path / "linked-artifact").hardlink_to(task / artifact) + assert (task / artifact).stat().st_nlink == 2 + config = configuration() + config["generator"] = {**config["target"], "context_paths": [context_path]} + config["execution"].update( + agent_image=m.AGENT, verifier_image=m.VERIFIER, namespace="owner" + ) + inference.responses = [m.response("true")] + output = tmp_path / "audit" + report = run_coding_validation(task, config, output) + assert not inference.requests and not inference.kwargs + assert not factory.created + assert not report["accepted"] + evidence = json.loads((output / "evidence.json").read_text()) + assert evidence["collection_error"] == "HFBackendError" + + +@pytest.mark.parametrize("with_context", [False, True]) +@pytest.mark.parametrize( + "completion", ["stop", "null_calls", "missing_calls", "empty_calls"] +) +def test_real_backend_contract_reaches_difficulty_with_sdk_fakes( + task, tmp_path, monkeypatch, with_context, completion +): + """Join the actual collector/backend/scorer, not just each seam in isolation.""" + from examples.coding_audit.hf import HFTaskBackend + + m = SimpleNamespace(**runpy.run_path(str(Path(__file__).with_name("test_hf.py")))) + api, inference = m.API(), m.InferenceFactory() + factory = m.Factory(api) + original = m.FakeSandbox.run + + def run(sandbox, argv, **kwargs): + if sandbox.image == m.VERIFIER: + sandbox.factory.reward = ( + b"1" if sandbox.files.data.get("/app/answer.txt") == b"42\n" else b"0" + ) + else: + command = " ".join(argv) + sandbox.factory.submitted = ( + b"42\n" if "42" in command else b"41\n" if "41" in command else None + ) + return original(sandbox, argv, **kwargs) + + monkeypatch.setattr(m.FakeSandbox, "run", run) + config = configuration() + context_path = "environment/source/pkg/api.py" + context = b"def public_answer():\n return 42\n" + if with_context: + source = task / context_path + source.parent.mkdir(parents=True) + source.write_bytes(context) + config.setdefault("generator", dict(config["target"]))["context_paths"] = [ + context_path + ] + config["execution"].update( + agent_image=m.AGENT, verifier_image=m.VERIFIER, namespace="owner" + ) + completions = [m.response("done"), m.response("done")] + if completion != "stop": + for reply in completions: + choice = reply["choices"][0] + choice["finish_reason"] = "tool_calls" + choice["message"]["content"] = None + if completion == "missing_calls": + del choice["message"]["tool_calls"] + elif completion == "empty_calls": + choice["message"]["tool_calls"] = [] + inference.responses = [ + m.response('printf "42\\n" > /app/answer.txt'), + m.response(None, calls=[m.terminal('printf "41\\n" > /app/answer.txt')]), + completions[0], + m.response(None, calls=[m.terminal('printf "42\\n" > /app/answer.txt')]), + completions[1], + ] + backend = HFTaskBackend( + task.resolve(), + config["execution"], + token="execution-only-test-token", + inference_token="inference-only-test-token", + api=api, + sandbox_factory=factory, + inference_factory=inference, + ) + output = tmp_path / "audit" + report = run_coding_validation(task, config, output, backend=backend) + assert report["accepted"] is (completion == "stop"), report + if completion == "stop": + assert report["checks"]["target_difficulty"]["solve_rate"] == 0.5 + else: + assert report["checks"]["target_difficulty"]["status"] == "error" + assert len(inference.requests) == 5 + evidence_path = output / "evidence.json" + evidence = json.loads(evidence_path.read_text()) + if completion != "stop": + targets = [row for row in evidence["trials"] if row["role"] == "target"] + assert len(targets) == 2 + assert all( + row["error"] == "invalid terminal function calls" + and row["cleanup_verified"] + and "grade_sandbox_id" not in row + and row.get("reward") is None + for row in targets + ) + generated = evidence["generation"][0] + if with_context: + assert generated["candidate_context"] == [ + { + "path": context_path, + "runtime_path": "pkg/api.py", + "size": len(context), + "sha256": hashlib.sha256(context).hexdigest(), + } + ] + assert context.decode() in inference.requests[0]["messages"][1]["content"] + else: + assert "candidate_context" not in generated + assert all( + request["messages"][1]["content"] == (task / "instruction.md").read_text() + for request in inference.requests[1:] + ) + checksums = dict( + line.split(" ", 1)[::-1] + for line in (output / "SHA256SUMS").read_text().splitlines() + ) + assert ( + checksums["evidence.json"] + == hashlib.sha256(evidence_path.read_bytes()).hexdigest() + ) + assert factory.created + assert all(stage == "CANCELED" for stage in api.stages.values()) + + +def test_grader_exception_keeps_sanitized_sha_bound_logs(task, tmp_path, monkeypatch): + from examples.coding_audit.hf import HFTaskBackend + + m = SimpleNamespace(**runpy.run_path(str(Path(__file__).with_name("test_hf.py")))) + api, inference = m.API(), m.InferenceFactory() + factory = m.Factory(api) + original = m.FakeSandbox.run + secret = "execution-only-test-token" + + def run(sandbox, argv, **kwargs): + if sandbox.image == m.VERIFIER: + kwargs["on_stdout"]("APT: partial output 雪 " + secret) + raise RuntimeError("grader RPC failed: " + secret) + return original(sandbox, argv, **kwargs) + + monkeypatch.setattr(m.FakeSandbox, "run", run) + config = configuration() + config["execution"].update( + agent_image=m.AGENT, verifier_image=m.VERIFIER, namespace="owner" + ) + inference.responses = [m.response('printf "42\\n" > /app/answer.txt')] + backend = HFTaskBackend( + task.resolve(), + config["execution"], + token=secret, + inference_token="inference-only-test-token", + api=api, + sandbox_factory=factory, + inference_factory=inference, + ) + output = tmp_path / "audit" + report = run_coding_validation(task, config, output, backend=backend) + assert not report["accepted"] + evidence = json.loads((output / "evidence.json").read_text()) + assert evidence["target_not_run_reason"] + for row in evidence["trials"]: + assert row["error"] == "HF backend operation failed (RuntimeError)" + assert row["reward"] is None and row["cleanup_verified"] + artifact = row["grader_transcript"] + data = (output / artifact["path"]).read_bytes() + assert hashlib.sha256(data).hexdigest() == artifact["sha256"] + trace = json.loads(data)[0] + assert trace["output"] == "APT: partial output 雪 [REDACTED]" + assert trace["exit_code"] is None and not trace["output_truncated"] + for path in output.rglob("*"): + if path.is_file(): + assert secret.encode() not in path.read_bytes() + assert all(stage == "CANCELED" for stage in api.stages.values()) + + +@pytest.mark.parametrize("callbacks", ["single", "separate", "split_secret"]) +@pytest.mark.parametrize("intersecting", [False, True]) +def test_grader_overflow_discards_output_in_sha_bound_artifact( + task, tmp_path, monkeypatch, callbacks, intersecting +): + from examples.coding_audit.hf import HFTaskBackend + + m = SimpleNamespace(**runpy.run_path(str(Path(__file__).with_name("test_hf.py")))) + api, inference = m.API(), m.InferenceFactory() + factory = m.Factory(api) + original = m.FakeSandbox.run + execution_secret = ( + "hf_SYNTHETIC_EXECUTION_TOKEN_REVIEW_ONLYh" + if intersecting + else "hf_SYNTHETIC_EXECUTION_TOKEN_FOR_REVIEW_ONLY" + ) + inference_secret = "hf_SYNTHETIC_INFERENCE_TOKEN_REVIEW_ONLY" + secret = inference_secret if intersecting else execution_secret + cap, retained = 1048576, 28 + prefix = "雪" + "A" * (cap - len("雪".encode()) - retained) + if intersecting: + prefix = prefix[: -len(execution_secret[:-1])] + execution_secret[:-1] + assert execution_secret in prefix + secret + + def run(sandbox, argv, **kwargs): + if sandbox.image == m.VERIFIER: + if callbacks == "single": + kwargs["on_stdout"](prefix + secret) + else: + split = 7 if callbacks == "split_secret" else 0 + kwargs["on_stdout"](prefix + secret[:split]) + kwargs["on_stderr"](secret[split:]) + pytest.fail("grader output overflow was not rejected") + return original(sandbox, argv, **kwargs) + + monkeypatch.setattr(m.FakeSandbox, "run", run) + config = configuration() + config["execution"].update( + agent_image=m.AGENT, + verifier_image=m.VERIFIER, + namespace="owner", + max_tool_output_bytes=cap, + ) + inference.responses = [m.response('printf "42\\\\n" > /app/answer.txt')] + backend = HFTaskBackend( + task.resolve(), + config["execution"], + token=execution_secret, + inference_token=inference_secret, + api=api, + sandbox_factory=factory, + inference_factory=inference, + ) + output = tmp_path / "audit" + report = run_coding_validation(task, config, output, backend=backend) + assert not report["accepted"] + evidence = json.loads((output / "evidence.json").read_text()) + assert evidence["target_not_run_reason"] and evidence["trials"] + for row in evidence["trials"]: + assert row["error"] == "terminal output exceeded byte limit" + assert row["reward"] is None and row["cleanup_verified"] + assert "raw_reward" not in row + artifact = row["grader_transcript"] + data = (output / artifact["path"]).read_bytes() + assert hashlib.sha256(data).hexdigest() == artifact["sha256"] + assert json.loads(data) == [ + { + "exit_code": None, + "output": "", + "output_truncated": True, + } + ] + assert len(json.loads(data)[0]["output"].encode()) <= cap + for line in (output / "SHA256SUMS").read_text().splitlines(): + digest, name = line.split(" ", 1) + data = (output / name).read_bytes() + assert hashlib.sha256(data).hexdigest() == digest + assert secret[:retained].encode() not in data + assert execution_secret[:-1].encode() not in data + assert api.stages and all(stage == "CANCELED" for stage in api.stages.values()) + + +@pytest.mark.parametrize("value", [False, True]) +def test_explicit_candidate_thinking_control_preserves_target(value): + raw = configuration() + raw["generator"] = {**raw["target"], "enable_thinking": value} + normalized = normalize_coding_config(raw) + assert normalized["generator"]["enable_thinking"] is value + assert "enable_thinking" not in normalized["target"] + + +@pytest.mark.parametrize("value", [0, 1, "false", None]) +def test_thinking_configuration_rejects_non_boolean(value): + raw = configuration() + raw["generator"] = {**raw["target"], "enable_thinking": value} + with pytest.raises(ValueError, match="enable_thinking"): + normalize_coding_config(raw) + + +@pytest.mark.parametrize("count", [0, 1, 32]) +def test_generator_context_is_optional_explicit_and_preserves_target(count): + raw = configuration() + paths = [f"environment/source/pkg/api-{index}.py" for index in range(count)] + raw["generator"] = {**raw["target"], "context_paths": paths} + normalized = normalize_coding_config(raw) + assert normalized["generator"]["context_paths"] == paths + assert "context_paths" not in normalized["target"] + assert "context_paths" not in normalize_coding_config(configuration())["generator"] + + +@pytest.mark.parametrize( + "paths", + [ + "environment/source/api.py", + None, + [1], + ["environment/source/api.py"] * 2, + [f"environment/source/{index}.py" for index in range(33)], + ["environment/source"], + ["environment/source/api.py/"], + ["/environment/source/api.py"], + ["environment/source/../tests/test.sh"], + ["environment/source//api.py"], + ["environment/source/./api.py"], + ["environment/source/api\x00.py"], + ["environment/source/a\nb.py"], + ["environment/source/a\\b.py"], + ["environment/source/a:b.py"], + ["tests/test.sh"], + ["solution/solve.sh"], + ["validation/spec.json"], + ["environment/source/tests/test.py"], + ["environment/source/solution/solve.sh"], + ["environment/source/validation/spec.json"], + ["environment/source/.hidden/api.py"], + ["environment/source/.env"], + ["environment/source/secrets/token"], + ["environment/source/reference/solve.py"], + ["environment/source/spec/task.md"], + ], +) +def test_generator_context_rejects_unsafe_paths(paths): + raw = configuration() + raw["generator"] = {**raw["target"], "context_paths": paths} + with pytest.raises(ValueError, match="context_paths"): + normalize_coding_config(raw) + + +def test_target_cannot_declare_context_paths(): + raw = configuration() + raw["target"]["context_paths"] = [] + with pytest.raises(ValueError, match="target.context_paths"): + normalize_coding_config(raw) diff --git a/examples/coding_audit/test_scoring.py b/examples/coding_audit/test_scoring.py new file mode 100644 index 000000000..a30cd8746 --- /dev/null +++ b/examples/coding_audit/test_scoring.py @@ -0,0 +1,575 @@ +"""Admission tests for pure coding-audit evidence scoring; no execution backends.""" + +import copy +import json + +import pytest +from examples.coding_audit.scoring import assess_coding_audit, validate_audit_config + +TASK = "a" * 64 +VERIFIER = "b" * 64 +IMAGE = "registry/agent@sha256:" + "c" * 64 +VERIFIER_IMAGE = "registry/verifier@sha256:" + "d" * 64 +TARGET = {"model": "Qwen/Qwen3.8-27B", "revision": "e" * 40, "provider": "hf-inference"} +CONFIG = {"required_negative_cases": ["wrong_answer", "empty_answer"]} + + +def trial(role, case_id, reward, number, submission_digest=None): + result = { + "role": role, + "case_id": case_id, + "task_digest": TASK, + "verifier_digest": VERIFIER, + "image_ref": IMAGE, + "verifier_image_ref": VERIFIER_IMAGE, + "submission_digest": submission_digest or "f" * 64, + "sandbox_id": f"agent-{number}", + "grade_sandbox_id": f"grade-{number}", + "reward": reward, + "cleanup_verified": True, + "error": None, + } + if role == "target": + result.update(TARGET) + result.update( + latency_seconds=2.5, usage={"input_tokens": 100, "output_tokens": 20} + ) + return result + + +@pytest.fixture +def evidence(): + return { + "task_digest": TASK, + "verifier_digest": VERIFIER, + "agent_image": IMAGE, + "verifier_image": VERIFIER_IMAGE, + "target": dict(TARGET), + "references": [ + { + "case_id": "oracle", + "submission_digest": "f" * 64, + "origin": "provided", + } + ], + "trials": [ + trial("nop", "nop", 0, 0), + trial("reference", "oracle", 1, 1), + trial("reference", "oracle", 1, 2), + trial("negative", "wrong_answer", 0, 3), + trial("negative", "empty_answer", 0, 4), + trial("target", "attempt-1", 0, 5), + trial("target", "attempt-2", 1, 6), + ], + "leakage_probes": [ + {"name": name, "passed": True, "error": None} + for name in ( + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", + ) + ], + } + + +def test_complete_audit_reports_bounded_measurements_without_mutating_inputs(evidence): + original = copy.deepcopy(evidence) + report = assess_coding_audit(CONFIG, evidence) + assert report["profile"] == "harbor-coding-audit-v1" + assert report["accepted"] is True + assert report["verdict"] == "pass" + assert not report["rfc008_complete"] + assert not report["trainability_established"] + assert {check["status"] for check in report["checks"].values()} == {"pass"} + target = report["checks"]["target_difficulty"] + assert target["completed"] == 2 + assert target["failed"] == target["missing"] == 0 + assert target["solve_rate"] == 0.5 + assert target["solve_rate_wilson95"] == pytest.approx([0.0945312057, 0.9054687943]) + assert target["reward_variance"] == 0.25 + assert target["mixed_success"] is True + assert target["difficulty"] == "mixed" + assert target["latency_seconds_total"] == 5 + assert target["usage_totals"] == {"input_tokens": 200, "output_tokens": 40} + assert all(value == 0 for value in target["telemetry_missing"].values()) + assert "bounded probes" in report["checks"]["leakage"]["scope"] + assert evidence == original + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize("reward,difficulty", [(0, "none_solved"), (1, "all_solved")]) +def test_uniform_target_rewards_are_valid_difficulty_measurements( + evidence, reward, difficulty +): + for record in evidence["trials"][-2:]: + record["reward"] = reward + report = assess_coding_audit(CONFIG, evidence) + assert report["accepted"] + target = report["checks"]["target_difficulty"] + assert target["difficulty"] == difficulty + assert target["mixed_success"] is False + assert target["reward_variance"] == 0 + assert target["solve_rate"] == reward + + +def test_target_infrastructure_error_is_missing_evidence_not_a_zero_reward(evidence): + failed = evidence["trials"][-2] + failed.update(error="provider timeout", reward=None) + report = assess_coding_audit(CONFIG, evidence) + assert not report["accepted"] + target = report["checks"]["target_difficulty"] + assert target["status"] == "error" + assert target["attempted"] == 2 + assert target["completed"] == target["failed"] == target["missing"] == 1 + assert target["infrastructure_errors"] == 1 + assert target["solve_rate"] == 1 + assert target["reward_variance"] == 0 + assert target["usage_totals"]["input_tokens"] == 200 + + +def test_missing_target_attempt_is_skip(evidence): + evidence["trials"].pop() + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "skip" + assert report["checks"]["target_difficulty"]["missing"] == 1 + + +def test_configured_repeat_and_attempt_counts_are_honored(evidence): + config = CONFIG | {"reference_repeats": 3, "target_attempts": 3} + report = assess_coding_audit(config, evidence) + assert report["checks"]["references"]["status"] == "skip" + assert report["checks"]["target_difficulty"]["missing"] == 1 + evidence["trials"].extend( + [trial("reference", "oracle", 1, 10), trial("target", "attempt-3", 0, 11)] + ) + assert assess_coding_audit(config, evidence)["accepted"] + + +@pytest.mark.parametrize( + "duplicate", ["record", "case_id", "sandbox_id", "grade_sandbox_id", "cross_role"] +) +def test_duplicate_target_attempts_never_count_as_coverage(evidence, duplicate): + first, second = evidence["trials"][-2:] + if duplicate == "record": + evidence["trials"][-1] = dict(first) + elif duplicate == "cross_role": + second["sandbox_id"] = evidence["trials"][1]["grade_sandbox_id"] + else: + second[duplicate] = first[duplicate] + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "error" + assert report["checks"]["target_difficulty"]["completed"] < 2 + + +def test_no_target_data_reports_null_metrics(evidence): + evidence["trials"] = [ + record for record in evidence["trials"] if record["role"] != "target" + ] + report = assess_coding_audit(CONFIG, evidence) + target = report["checks"]["target_difficulty"] + assert target["status"] == "skip" + for name in ( + "solve_rate", + "solve_rate_wilson95", + "reward_variance", + "mixed_success", + ): + assert target[name] is None + assert target["usage_totals"] == {"input_tokens": None, "output_tokens": None} + assert target["difficulty"] == "unmeasured" + + +def test_missing_telemetry_is_explicit_and_does_not_invalidate_reward(evidence): + for record in evidence["trials"][-2:]: + del record["usage"] + del record["latency_seconds"] + report = assess_coding_audit(CONFIG, evidence) + assert report["accepted"] + target = report["checks"]["target_difficulty"] + assert target["telemetry_missing"] == { + "latency_seconds": 2, + "input_tokens": 2, + "output_tokens": 2, + } + assert target["latency_seconds_total"] is None + + +@pytest.mark.parametrize( + "field,value", + [ + ("latency_seconds", -1), + ("latency_seconds", None), + ("latency_seconds", True), + ("latency_seconds", float("inf")), + ("usage", None), + ("usage", {"input_tokens": True, "output_tokens": -5}), + ], +) +def test_supplied_malformed_telemetry_is_error_not_missing(evidence, field, value): + evidence["trials"][-1][field] = value + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "error" + target = report["checks"]["target_difficulty"] + assert target["status"] == "error" + assert target["telemetry_errors"] + assert all(value == 0 for value in target["telemetry_missing"].values()) + + +def test_latency_total_overflow_returns_error_report(evidence): + evidence["trials"].append(trial("target", "attempt-3", 1, 10)) + for record, seconds in zip(evidence["trials"][-3:], [10**308, 10**308, 0.0]): + record["latency_seconds"] = seconds + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "error" + assert report["checks"]["target_difficulty"]["latency_seconds_total"] is None + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize( + "reward", + [True, False, float("nan"), float("inf"), -float("inf"), None, "1", -0.01, 1.01], +) +@pytest.mark.parametrize("index", [0, 1, 3, 5]) +def test_invalid_rewards_block_all_roles(evidence, reward, index): + evidence["trials"][index]["reward"] = reward + report = assess_coding_audit(CONFIG, evidence) + assert not report["accepted"] + assert report["checks"]["reliability"]["status"] == "error" + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize("cleanup", [False, 1, "true", None]) +def test_cleanup_requires_literal_true(evidence, cleanup): + evidence["trials"][1]["cleanup_verified"] = cleanup + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +@pytest.mark.parametrize( + "field", + [ + "error", + "cleanup_verified", + "sandbox_id", + "grade_sandbox_id", + "submission_digest", + ], +) +def test_missing_trial_provenance_never_passes(evidence, field): + del evidence["trials"][1][field] + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +@pytest.mark.parametrize("field", ["case_id", "submission_digest"]) +@pytest.mark.parametrize("value", [[], {}, True]) +def test_malformed_reference_group_keys_return_error_without_crashing( + evidence, field, value +): + evidence["trials"][1][field] = value + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +def test_sha256_prefix_does_not_split_identical_reference_replays(evidence): + evidence["references"][0]["submission_digest"] = "sha256:" + "f" * 64 + evidence["trials"][1]["submission_digest"] = "sha256:" + "f" * 64 + evidence["trials"][0]["task_digest"] = "sha256:" + TASK + evidence["trials"][3]["verifier_digest"] = "sha256:" + VERIFIER + assert assess_coding_audit(CONFIG, evidence)["accepted"] + + +def test_unrepresentable_reward_variance_blocks_acceptance_without_nonfinite_json( + evidence, +): + config = CONFIG | {"reward_max": 1e308} + for record in evidence["trials"]: + if record["reward"] == 1: + record["reward"] = 1e308 + report = assess_coding_audit(config, evidence) + assert report["verdict"] == "error" + assert report["checks"]["target_difficulty"]["reward_variance"] is None + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize("index", [0, 1, 3, 5]) +@pytest.mark.parametrize( + "field,value", + [ + ("task_digest", "0" * 64), + ("verifier_digest", "1" * 64), + ("image_ref", "other@sha256:" + "2" * 64), + ("verifier_image_ref", "other@sha256:" + "3" * 64), + ], +) +def test_every_trial_binds_to_expected_task_verifier_and_images( + evidence, index, field, value +): + evidence["trials"][index][field] = value + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "error" + assert any( + "binding mismatch" in reason + for reason in report["checks"]["reliability"]["reasons"] + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("task_digest", "a" * 63), + ("verifier_digest", "b" * 65), + ("agent_image", "agent:latest"), + ("verifier_image", "sha256:abc"), + ], +) +def test_common_binding_requires_full_digests_and_immutable_images( + evidence, field, value +): + evidence[field] = value + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +@pytest.mark.parametrize( + "field,value", + [("model", "other/model"), ("revision", "a" * 40), ("provider", "other-provider")], +) +def test_target_identity_is_bound(evidence, field, value): + evidence["trials"][-1][field] = value + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["target_difficulty"]["status"] == "error" + assert report["checks"]["target_difficulty"]["completed"] == 1 + + +@pytest.mark.parametrize("revision", ["main", "abc", "a" * 39, None]) +def test_common_target_revision_must_be_immutable(evidence, revision): + evidence["target"]["revision"] = revision + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +def test_reference_replay_drift_blocks_acceptance(evidence): + evidence["trials"][2]["reward"] = 0 + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["references"]["status"] == "fail" + assert report["checks"]["reliability"]["status"] == "fail" + + +def test_separate_generated_submissions_are_not_deterministic_replays(evidence): + evidence["references"] = [ + { + "case_id": "oracle", + "submission_digest": digest * 64, + "origin": "generated", + "candidate_script_digest": digest * 64, + } + for digest in ("0", "1") + ] + evidence["trials"][1]["submission_digest"] = "0" * 64 + evidence["trials"][2]["submission_digest"] = "1" * 64 + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "skip" + assert report["checks"]["references"]["validated"] == [] + assert report["checks"]["reliability"]["status"] == "skip" + + +def test_clean_failed_generated_candidate_is_excluded_but_must_execute(evidence): + evidence["references"].append( + { + "case_id": "generated", + "submission_digest": "0" * 64, + "origin": "generated", + "candidate_script_digest": "1" * 64, + } + ) + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["references"]["status"] == "skip" + evidence["trials"].append(trial("reference", "generated", 0, 10, "0" * 64)) + report = assess_coding_audit(CONFIG, evidence) + assert report["accepted"] + assert report["checks"]["references"]["excluded"] == [ + {"case_id": "generated", "submission_digest": "0" * 64} + ] + evidence["trials"][-1]["error"] = "execution failed" + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +def test_generated_reference_can_be_validated_without_provided_oracle(evidence): + evidence["references"][0].update( + origin="generated", candidate_script_digest="1" * 64 + ) + assert assess_coding_audit(CONFIG, evidence)["accepted"] + del evidence["references"][0]["candidate_script_digest"] + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +@pytest.mark.parametrize("kind", ["nop", "wrong_answer", "contradictory_negative"]) +def test_verifier_must_reject_nop_and_every_observed_known_negative(evidence, kind): + if kind == "nop": + evidence["trials"][0]["reward"] = 1 + elif kind == "wrong_answer": + evidence["trials"][3]["reward"] = 1 + else: + evidence["trials"].append(trial("negative", "wrong_answer", 1, 10)) + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["verifier_rejection"]["status"] == "fail" + + +def test_generated_negative_labels_do_not_establish_trusted_coverage(evidence): + evidence["trials"][3]["case_id"] = "model_says_wrong" + evidence["required_negative_cases"] = ["model_says_wrong", "empty_answer"] + evidence["trials"][3]["reviewed"] = True + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["verifier_rejection"]["status"] == "skip" + assert report["checks"]["verifier_rejection"]["covered_negative_cases"] == [ + "empty_answer" + ] + + +@pytest.mark.parametrize("name", ["nop", "reference", "negative"]) +def test_missing_role_is_incomplete(evidence, name): + evidence["trials"] = [ + record for record in evidence["trials"] if record["role"] != name + ] + report = assess_coding_audit(CONFIG, evidence) + assert not report["accepted"] + assert report["verdict"] == "skip" + + +def test_floor_margin_is_required(evidence): + config = CONFIG | {"floor_margin": 0.95, "oracle_tolerance": 0.1} + evidence["trials"][0]["reward"] = 0.1 + report = assess_coding_audit(config, evidence) + assert report["checks"]["references"]["status"] == "fail" + + +def test_floor_gap_does_not_round_away_tiny_nop_reward(evidence): + config = CONFIG | {"oracle_tolerance": 1e-300, "floor_margin": 1} + evidence["trials"][0]["reward"] = 1e-300 + report = assess_coding_audit(config, evidence) + assert report["checks"]["verifier_rejection"]["status"] == "pass" + assert report["checks"]["references"]["status"] == "fail" + assert not report["accepted"] + + +def test_margin_must_fit_exact_reward_range(): + config = CONFIG | { + "reward_min": 1e-300, + "wrong_answer_max": 1e-300, + "floor_margin": 1, + } + with pytest.raises(ValueError): + validate_audit_config(config) + + +def test_zero_float_tolerance_preserves_large_integer_reference_and_solve_threshold( + evidence, +): + config = CONFIG | {"reward_max": 2**53 + 1, "oracle_tolerance": 0.0} + for record in evidence["trials"]: + if record["role"] in ("reference", "target"): + record["reward"] = 2**53 + report = assess_coding_audit(config, evidence) + assert report["checks"]["references"]["status"] == "fail" + target = report["checks"]["target_difficulty"] + assert target["solve_count"] == 0 + assert target["difficulty"] == "none_solved" + assert not report["accepted"] + + +def test_floor_separation_uses_decimal_values_without_float_subtraction_error(evidence): + config = CONFIG | { + "reward_min": 0.7, + "reward_max": 1, + "oracle_tolerance": 0.1, + "wrong_answer_max": 0.8, + "floor_margin": 0.1, + } + for record in evidence["trials"]: + record["reward"] = 0.9 if record["role"] == "reference" else 0.8 + assert assess_coding_audit(config, evidence)["accepted"] + + +@pytest.mark.parametrize( + "change,status", + [ + ("missing", "skip"), + ("unknown", "skip"), + ("duplicate", "error"), + ("false", "fail"), + ("truthy", "error"), + ("error", "error"), + ("missing_error", "error"), + ], +) +def test_leakage_requires_explicit_named_successful_probes(evidence, change, status): + if change == "missing": + evidence["leakage_probes"].pop() + elif change == "unknown": + evidence["leakage_probes"][-1]["name"] = "unknown_probe" + elif change == "duplicate": + evidence["leakage_probes"].append(dict(evidence["leakage_probes"][0])) + elif change == "false": + evidence["leakage_probes"][0]["passed"] = False + elif change == "truthy": + evidence["leakage_probes"][0]["passed"] = 1 + elif change == "error": + evidence["leakage_probes"][0]["error"] = "probe failed to execute" + else: + del evidence["leakage_probes"][0]["error"] + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["leakage"]["status"] == status + assert not report["accepted"] + + +def test_unknown_probe_cannot_replace_required_names_but_can_be_diagnostic(evidence): + evidence["leakage_probes"].append({"name": "extra", "passed": True, "error": None}) + report = assess_coding_audit(CONFIG, evidence) + assert report["accepted"] + assert "extra" in report["checks"]["leakage"]["observed_probes"] + + +@pytest.mark.parametrize( + "config", + [ + {}, + {"reference_repeats": 1}, + {"reference_repeats": True}, + {"target_attempts": 1}, + {"reward_min": True}, + {"reward_max": float("nan")}, + {"oracle_tolerance": 1}, + {"floor_margin": 0}, + {"floor_margin": float("inf")}, + {"wrong_answer_max": 1}, + {"required_negative_cases": []}, + {"required_negative_cases": ["a", "a"]}, + {"required_negative_cases": [1]}, + {"unknown": True}, + ], +) +def test_invalid_config_raises_and_assessment_returns_error(evidence, config): + config = config if not config else CONFIG | config + with pytest.raises(ValueError): + validate_audit_config(config) + report = assess_coding_audit(config, evidence) + assert report["verdict"] == "error" + assert not report["accepted"] + + +def test_empty_evidence_cannot_be_accepted(): + report = assess_coding_audit(CONFIG, {}) + assert not report["accepted"] + assert report["checks"]["leakage"]["status"] == "skip" + assert report["checks"]["references"]["status"] == "skip" + + +@pytest.mark.parametrize( + "field,value", + [ + ("trials", {}), + ("references", {}), + ("leakage_probes", {}), + ("trials", [None]), + ("references", [None]), + ], +) +def test_malformed_containers_return_error(evidence, field, value): + evidence[field] = value + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error"