From 8570efd74916856fd8b44541a1d2ece9299db153 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Mon, 9 Nov 2026 17:20:54 +0100 Subject: [PATCH 01/40] feat(validation): add native Harbor oracle-floor pilot Ported onto main from the Hub validation pipeline branch. Keeps main's duplicate-key handling in runtime/json_values.py (e4ca57b7) instead of this commit's helper, and drops this commit's duplicate-key test, which main's artifact tests already cover. (cherry picked from commit f60512b5ec0f808e8657edbbb052fa67cfc35397) --- docs/source/reference/cli.md | 10 +- rfcs/008-environment-auto-validation.md | 36 + src/openenv/cli/commands/validate.py | 6 +- src/openenv/validation/harbor.py | 760 ++++++++++++++++++ src/openenv/validation/manifest.py | 14 +- src/openenv/validation/parsers/harbor_toml.py | 144 ++++ src/openenv/validation/runner.py | 18 + .../schemas/manifest-v2.schema.json | 49 +- .../validation/schemas/report-v2.schema.json | 49 +- src/openenv/validation/signature.py | 1 + tests/test_validation/test_cli_validate.py | 7 +- .../test_validation/test_harbor_validation.py | 565 +++++++++++++ .../test_signature_detection.py | 11 +- tests/test_validation/test_types.py | 5 +- 14 files changed, 1664 insertions(+), 11 deletions(-) create mode 100644 src/openenv/validation/harbor.py create mode 100644 src/openenv/validation/parsers/harbor_toml.py create mode 100644 tests/test_validation/test_harbor_validation.py diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index 977e72c85..d16ff6fc2 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -60,7 +60,7 @@ openenv validate ./my_env --level static openenv validate ./my_env --level runtime --local --output report.json ``` -Runtime validation requires Docker and a `validation.execution` declaration in +For served packages, runtime validation requires Docker and a `validation.execution` declaration in `openenv.yaml`. The declaration points to a bounded JSON replay plan containing one reset and a sequence of actions. The validator builds an immutable image, collects the original episode, repeats the plan in a fresh WebSocket session and @@ -69,6 +69,14 @@ supports that capability. Docker validation uses two containers concurrently during the fresh-container replay. Both use the same immutable image and are removed even if a check fails. +Native Harbor `task.toml` directories are also recognized. Supply an explicit +`validation/openenv.json` reward/resource contract and the optional Harbor +dependency. The `harbor-oracle-floor-v1` binding runs fresh native NOP and oracle +trials in Docker, verifies raw rewards, actual task image/configuration and cleanup, +and retains `harbor-evidence.json` in the report bundle. Other standard checks +remain named SKIPs, so a passing pilot still produces WARN. This does not qualify +the outer `harbor_env` server or claim full RFC 008 admission. + The 16 implemented runtime checks cover startup, rewards, observation schemas, state continuity, network policy, host containment, resource bounds, episode isolation, oracle containment, seed forwarding, determinism, emitted trajectory records, tool diff --git a/rfcs/008-environment-auto-validation.md b/rfcs/008-environment-auto-validation.md index c0f40b720..c045febf4 100644 --- a/rfcs/008-environment-auto-validation.md +++ b/rfcs/008-environment-auto-validation.md @@ -718,6 +718,42 @@ findings: a WARN exit 0 does not prove completeness. Product publish gating and operator certification remain outside this feature; the test workflow only proves the implementation's own acceptance cases. +### Native Harbor pilot binding + +`task.toml` packages may opt into the explicit `harbor-oracle-floor-v1` pilot +through an operator-supplied `validation/openenv.json` contract. The contract +declares the selected reward key, range, oracle tolerance, floor margin and finite +resource budgets; Harbor metadata and observed rewards do not supply these +declarations. The normalized schema-2 execution kind is `harbor_task`, with a +`process` agent boundary. Parsing uses Harbor's data-only Task loader and never +creates a Trial or executes package code. + +This initial profile supports single-step Linux tasks, one Dockerfile, one main +Docker service, a shared verifier, public network policies and no accelerators, +external MCP services or credentials. Native execution runs fresh NOP and oracle +Harbor Trials, inspects the actual agent container at `AGENT_START`, retains its +immutable image under a run-owned tag, and pins the oracle to that image. Source, +explicit execution overlays and effective container configuration are bound to +the evidence. Supported `delete=False` and `keep_containers=True` trial settings +leave image retention and resource removal to the native supervisor. Cleanup +independently enumerates the exact run-owned Compose projects, registers anonymous +volumes from those containers' actual mounts, removes only those owned resources, +and verifies their absence. The trusted oracle exit marker is captured at +`AGENT_END`, before later sandbox log synchronization can overwrite host logs. + +`harbor-evidence.json` and preserved raw reward/exit files are checksummed bundle +artifacts. The pilot requires finite nonboolean in-range rewards, an oracle +without a nonzero exit marker or Trial exception, oracle reward at least the +declared maximum minus tolerance, and oracle-minus-NOP and maximum-minus-NOP gaps +at least the declared floor margin. Failures or unverifiable cleanup block the +normal report. `static.manifest`, `runtime.startup` and +`runtime.reward_well_formed` may PASS with actual evidence; other requested checks +remain named SKIPs. In particular this binding supplies no synthetic WebSocket +exchange, generic semantic PASS, containment qualification or measured resource +enforcement. A WARN report is admissible only to a driver explicitly choosing this +pilot and checking its native gates; full RFC validation still rejects required +SKIPs. + ## Explicitly out of scope No hub/runner/queue/coordinator; no submission or auth APIs; no statistical-level implementations diff --git a/src/openenv/cli/commands/validate.py b/src/openenv/cli/commands/validate.py index 5b16ba156..25a7ab8d3 100644 --- a/src/openenv/cli/commands/validate.py +++ b/src/openenv/cli/commands/validate.py @@ -231,10 +231,14 @@ def validate( Validate a local package or a running OpenEnv server. Local validation detects the package format by its well-known file (the - formats this build can parse; currently `openenv.yaml`), parses it into the + formats this build can parse: `openenv.yaml` or Harbor `task.toml`), parses it into the normalized manifest, runs the applicable graders up to the requested level, applies the severity policy, and emits a report. + Native Harbor tasks require the optional Harbor dependency and an explicit + `validation/openenv.json` contract. The oracle/floor pilot preserves unsupported + checks as SKIP; its WARN report does not establish full RFC 008 qualification. + Exit codes: 0 pass/warn · 1 fail · 2 unrecognized/unsupported package · 3 internal or policy error. diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py new file mode 100644 index 000000000..ab1ed8fbf --- /dev/null +++ b/src/openenv/validation/harbor.py @@ -0,0 +1,760 @@ +"""Native Harbor oracle/floor pilot; no served-protocol evidence is synthesized.""" + +import asyncio +import hashlib +import json +import os +import re +import shutil +import stat +import subprocess +import tempfile +import time +import uuid +from decimal import Decimal +from fractions import Fraction +from importlib.metadata import distribution +from pathlib import Path + +from .parsers.harbor_toml import HARBOR_PROFILE, read_harbor_contract, read_harbor_task +from .policy import apply_policy +from .report import CheckResult, ValidationReportV2 +from .runtime.artifacts import write_runtime_bundle +from .runtime.json_values import dumps +from .types import CheckStatus, Lane, Level + +_IMAGE_ID = re.compile(r"sha256:[0-9a-f]{64}\Z") +_RESOURCE_ID = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*\Z") +_MAX_REWARD_BYTES = 1024 * 1024 + + +class NativeEvidenceError(ValueError): + """A credential-safe diagnostic written by the trusted native collector.""" + + +def _digest(value): + return hashlib.sha256( + json.dumps(value, sort_keys=True, allow_nan=False).encode() + ).hexdigest() + + +class DockerInspector: + """Trusted host inspection and removal limited to exact run-owned projects.""" + + def __init__(self): + self._owned_volumes = {} + + def _docker(self, *args): + result = subprocess.run( + ["docker", *args], capture_output=True, timeout=30, check=True + ) + if len(result.stdout) > 4 * 1024 * 1024: + raise NativeEvidenceError("Docker inspection exceeded its output limit") + return result.stdout.decode() + + def preflight(self): + info = json.loads(self._docker("info", "--format", "{{json .}}")) + if info["OSType"] != "linux": + raise NativeEvidenceError("the Harbor pilot requires a Linux Docker daemon") + self._docker("compose", "version") + return { + "os": info["OSType"], + "daemon_id": info["ID"], + "server_version": info["ServerVersion"], + } + + def inventory(self, project): + if not re.fullmatch(r"openenv-validation-[a-f0-9]+-(nop|oracle)__env", project): + raise NativeEvidenceError("not a run-owned Harbor project") + label = f"label=com.docker.compose.project={project}" + inventory = {} + for kind, args in ( + ("containers", ("ps", "-aq", "--no-trunc")), + ("networks", ("network", "ls", "-q", "--no-trunc")), + ("volumes", ("volume", "ls", "-q")), + ): + ids = self._docker(*args, "--filter", label).split() + if any(not _RESOURCE_ID.fullmatch(item) for item in ids): + raise NativeEvidenceError("invalid Docker resource identity") + inventory[kind] = ids + return inventory + + def capture(self, project, retained_tag, *, retain=True): + inventory = self.inventory(project) + if len(inventory["containers"]) != 1: + raise NativeEvidenceError( + "the Harbor pilot requires exactly one main container" + ) + item = json.loads(self._docker("inspect", inventory["containers"][0]))[0] + labels = item["Config"]["Labels"] + if ( + labels.get("com.docker.compose.project") != project + or labels.get("com.docker.compose.service") != "main" + ): + raise NativeEvidenceError( + "container does not match the native main service" + ) + if not item["State"]["Running"] or not _IMAGE_ID.fullmatch(item["Image"]): + raise NativeEvidenceError( + "native container is not running with an immutable image" + ) + if retain: + self._docker("image", "tag", item["Image"], retained_tag) + retained = json.loads(self._docker("image", "inspect", retained_tag))[0] + if (retain and retained["Id"] != item["Image"]) or retained["Os"] != "linux": + raise NativeEvidenceError("retained image binding changed") + host = item["HostConfig"] + # Dockerfile VOLUME creates anonymous volumes without Compose labels. + volumes = [ + mount["Name"] for mount in item["Mounts"] if mount["Type"] == "volume" + ] + if any(not _RESOURCE_ID.fullmatch(name) for name in volumes): + raise NativeEvidenceError("invalid mounted volume identity") + self._owned_volumes[project] = volumes + configuration = { + "image_ref": item["Image"], + "user": item["Config"]["User"], + "workdir": item["Config"]["WorkingDir"], + "environment_digest": _digest(sorted(item["Config"].get("Env") or [])), + "entrypoint": item["Config"].get("Entrypoint"), + "command": item["Config"].get("Cmd"), + "host": { + key: host.get(key) + for key in ( + "NanoCpus", + "Memory", + "MemorySwap", + "PidsLimit", + "ReadonlyRootfs", + "Privileged", + "CapAdd", + "CapDrop", + "SecurityOpt", + "Devices", + ) + }, + "network_mode": "compose-default" + if host["NetworkMode"] == project + "_default" + else host["NetworkMode"], + "mounts": sorted( + [ + { + "type": mount["Type"], + "destination": mount["Destination"], + "rw": mount["RW"], + } + for mount in item["Mounts"] + ], + key=lambda mount: mount["destination"], + ), + } + return { + "container_id": item["Id"], + "project": project, + "image_ref": item["Image"], + "retained_image_tag": retained_tag, + "configuration": configuration, + "configuration_digest": _digest(configuration), + "owned_volumes": volumes, + } + + def cleanup(self, project): + before = self.inventory(project) + for container in before["containers"]: + item = json.loads(self._docker("inspect", container))[0] + if item["Config"]["Labels"].get("com.docker.compose.project") != project: + raise NativeEvidenceError("cleanup container ownership changed") + volumes = [ + mount["Name"] for mount in item["Mounts"] if mount["Type"] == "volume" + ] + if any(not _RESOURCE_ID.fullmatch(name) for name in volumes): + raise NativeEvidenceError("invalid cleanup volume identity") + self._owned_volumes.setdefault(project, []).extend(volumes) + # Even a successful docker rm -v can leave an anonymous volume behind. + registered = self._owned_volumes.get(project, []) + existing = ( + set(self._docker("volume", "ls", "-q").split()) if registered else set() + ) + before["volumes"] = sorted( + set(before["volumes"]) | (set(registered) & existing) + ) + for key, args in ( + ("containers", ("rm", "-f", "-v")), + ("networks", ("network", "rm")), + ("volumes", ("volume", "rm")), + ): + remaining = before[key] + if key == "volumes" and remaining: + existing = set(self._docker("volume", "ls", "-q").split()) + remaining = [name for name in remaining if name in existing] + if remaining: + self._docker(*args, *remaining) + after = self.inventory(project) + if registered: + existing = set(self._docker("volume", "ls", "-q").split()) + after["volumes"] = sorted( + set(after["volumes"]) | (set(registered) & existing) + ) + return {"verified": not any(after.values()), "before": before, "after": after} + + def verify_image(self, tag, image_ref): + image = json.loads(self._docker("image", "inspect", tag))[0] + return image["Id"] == image_ref + + +def _unsupported(task, contract): + config = task.config + env = config.environment + if task.has_steps: + return "multi-step Harbor tasks" + if env.os.value != "linux": + return "non-Linux Harbor tasks" + if config.verifier.environment is not None or ( + config.verifier.environment_mode is not None + and config.verifier.environment_mode.value != "shared" + ): + return "separate Harbor verifier environments" + if not (task.task_dir / "environment/Dockerfile").is_file() or any( + path.name.lower() + in {"docker-compose.yaml", "docker-compose.yml", "compose.yaml", "compose.yml"} + for path in (task.task_dir / "environment").rglob("*") + ): + return "tasks without one Dockerfile or with task-authored Compose" + if contract.resources.gpus or env.gpus or env.tpu or env.mcp_servers: + return "GPU/TPU or external MCP requirements" + if contract.requires_credentials: + return "external credential requirements" + if env.network_mode.value != "public" or any( + phase.network_mode is not None and phase.network_mode.value != "public" + for phase in (config.agent, config.verifier) + ): + return "non-public or differing phase network policies" + if env.env or config.verifier.env or config.solution.env: + return "task environment variables (including host credential substitutions)" + if env.healthcheck or env.skills_dir or config.verifier.collect or config.artifacts: + return "healthcheck, skills, collect or artifact extensions" + return None + + +async def _create_trial(config): + from harbor.trial.trial import Trial + + return await Trial.create(config) + + +def _read_output(path, root): + relative = path.relative_to(root) + current = root + for part in relative.parts: + current = current / part + if current.is_symlink(): + raise NativeEvidenceError("native output may not contain symbolic links") + try: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + except FileNotFoundError: + return None + with os.fdopen(fd, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size > _MAX_REWARD_BYTES: + raise NativeEvidenceError( + "native reward output must be a bounded regular file" + ) + raw = stream.read(_MAX_REWARD_BYTES + 1) + if len(raw) > _MAX_REWARD_BYTES: + raise NativeEvidenceError("native reward output exceeded its size bound") + return raw + + +def _reward(raw_files, parsed, contract): + from .parsers.harbor_toml import _unique_object + + if "reward.json" in raw_files: + raw = json.loads( + raw_files["reward.json"], + object_pairs_hook=_unique_object, + parse_float=Decimal, + ) + elif "reward.txt" in raw_files: + raw = {"reward": Decimal(raw_files["reward.txt"].decode().strip())} + else: + raise NativeEvidenceError("verifier produced no raw reward file") + if not isinstance(raw, dict) or not raw or contract.reward_key not in raw: + raise NativeEvidenceError( + "verifier reward map is empty or lacks the declared key" + ) + low, high = (Decimal(str(value)) for value in contract.reward.range) + for value in raw.values(): + if ( + isinstance(value, bool) + or not isinstance(value, (int, Decimal)) + or not Decimal(value).is_finite() + or not low <= value <= high + ): + raise NativeEvidenceError( + "every reward must be finite, nonboolean and within the declared range" + ) + if ( + not isinstance(parsed, dict) + or any(isinstance(value, bool) for value in parsed.values()) + or raw != {key: Decimal(str(value)) for key, value in parsed.items()} + ): + raise NativeEvidenceError("raw verifier rewards differ from the Harbor result") + return raw + + +async def _trial( + root, role, work, contract, inspector, run_id, source_binding, expected=None +): + from harbor.models.environment_type import EnvironmentType + from harbor.models.trial.config import ( + AgentConfig, + EnvironmentConfig, + TaskConfig, + TrialConfig, + VerifierConfig, + ) + from harbor.trial.hooks import TrialEvent + + from .runner import _DIGEST_EXCLUDED_DIRS, source_digest + + name = f"openenv-validation-{run_id}-{role}" + project = name + "__env" + retained_tag = f"openenv-validation:{run_id}" + record = { + "role": role, + "project": project, + "raw_rewards": {}, + "cleanup": {"verified": False}, + } + raw_files = {} + trial = None + # Register and verify ownership before Trial.create can allocate anything. + if any(inspector.inventory(project).values()): + raise NativeEvidenceError("run-owned project was not empty before allocation") + try: + task_copy = work / role / "task" + shutil.copytree( + root, + task_copy, + symlinks=True, + ignore=shutil.ignore_patterns(*_DIGEST_EXCLUDED_DIRS), + ) + copied_digest = source_digest(task_copy) + record["unmodified_task_digest"] = copied_digest + if copied_digest != source_binding: + raise NativeEvidenceError( + "private execution task differs from the validated source digest" + ) + if expected: + task = read_harbor_task(task_copy) + task.config.environment.docker_image = retained_tag + (task_copy / "task.toml").write_text(task.config.model_dump_toml()) + digest_before = source_digest(task_copy) + record["execution_task_digest"] = digest_before + record["task_overlay"] = ( + {"environment.docker_image": retained_tag} if expected else {} + ) + budget = contract.resources.episode_timeout_s + config = TrialConfig( + task=TaskConfig(path=task_copy.resolve()), + agent=AgentConfig( + name=role, + override_timeout_sec=budget, + override_setup_timeout_sec=budget, + ), + environment=EnvironmentConfig( + type=EnvironmentType.DOCKER, + delete=False, + force_build=False, + override_cpus=contract.resources.cpu, + override_memory_mb=contract.resources.memory_mb, + override_storage_mb=contract.resources.disk_mb, + kwargs={"auto_mounting": False, "keep_containers": True}, + ), + verifier=VerifierConfig(override_timeout_sec=budget), + trial_name=name, + trials_dir=(work / role / "trials").resolve(), + ) + effective_config = config.model_dump(mode="json") + effective_config["task"]["path"] = "" + effective_config["trials_dir"] = "" + effective_config.pop("trial_name") + record["trial_config_digest"] = _digest(effective_config) + record["resources"] = contract.resources.model_dump() + async with asyncio.timeout(budget): + trial = await _create_trial(config) + + async def capture(_event): + if trial.agent_environment.session_id != project: + raise NativeEvidenceError( + "Harbor sandbox identity differs from the registered project" + ) + identity = inspector.capture(project, retained_tag, retain=not expected) + record.update(identity) + if expected and ( + identity["image_ref"] != expected["image_ref"] + or identity["configuration_digest"] + != expected["configuration_digest"] + ): + raise NativeEvidenceError( + "fresh native trial image or effective container configuration drifted" + ) + + async def capture_exit(_event): + if role == "oracle": + marker = _read_output( + trial.paths.agent_dir / "exit-code.txt", trial.paths.trial_dir + ) + record["oracle_exit_observed_at_agent_end"] = True + record["oracle_exit_code"] = ( + int(marker) if marker is not None else 0 + ) + if marker is not None: + raw_files["oracle-exit-code.txt"] = marker + if record["oracle_exit_code"] != 0: + raise NativeEvidenceError( + "oracle exit-code.txt records a nonzero exit" + ) + + trial.add_hook(TrialEvent.AGENT_START, capture) + trial.add_hook(TrialEvent.AGENT_END, capture_exit) + result = await trial.run() + if source_digest(task_copy) != digest_before: + raise NativeEvidenceError( + "private execution task changed during validation" + ) + if result.exception_info is not None: + raise NativeEvidenceError("Harbor Trial returned exception_info") + if "container_id" not in record: + raise NativeEvidenceError("Harbor Trial has no actual AGENT_START identity") + for filename, path in ( + ("reward.json", trial.paths.reward_json_path), + ("reward.txt", trial.paths.reward_text_path), + ): + raw = _read_output(path, trial.paths.trial_dir) + if raw is not None: + raw_files[filename] = raw + if role == "oracle": + if not record.get("oracle_exit_observed_at_agent_end"): + raise NativeEvidenceError( + "oracle has no trusted AGENT_END exit observation" + ) + marker = _read_output( + trial.paths.agent_dir / "exit-code.txt", trial.paths.trial_dir + ) + if marker is not None and int(marker) != 0: + raise NativeEvidenceError("oracle exit-code.txt records a nonzero exit") + parsed = result.verifier_result.rewards if result.verifier_result else None + record["reward_error"] = True + record["rewards"] = _reward(raw_files, parsed, contract) + record.pop("reward_error") + except (Exception, KeyboardInterrupt) as exc: + record["failure"] = type(exc).__name__ + # Error categories are safe; task-authored exception messages/logs are not. + if isinstance(exc, NativeEvidenceError): + record["reason"] = str(exc) + finally: + if trial is not None: + try: + async with asyncio.timeout(30): + await trial.agent_environment.stop(delete=False) + except (Exception, KeyboardInterrupt): + record["failure"] = "HarborTeardownError" + try: + record["cleanup"] = inspector.cleanup(project) + if not record["cleanup"]["verified"]: + record["failure"] = "CleanupNotVerified" + except (Exception, KeyboardInterrupt) as exc: + record["cleanup"] = {"verified": False, "failure": type(exc).__name__} + record["failure"] = "CleanupNotVerified" + if trial is not None: + # Preserve raw verifier files even when Harbor rejects their values. + for filename, path in ( + ("reward.json", trial.paths.reward_json_path), + ("reward.txt", trial.paths.reward_text_path), + ("oracle-exit-code.txt", trial.paths.agent_dir / "exit-code.txt"), + ): + try: + raw = _read_output(path, trial.paths.trial_dir) + if raw is not None: + if filename in raw_files and raw_files[filename] != raw: + record["failure"] = "RewardArtifactDrift" + if filename == "oracle-exit-code.txt" and filename in raw_files: + continue + raw_files[filename] = raw + except (ValueError, OSError): + record["failure"] = "UnsafeRewardArtifact" + return record, raw_files + + +async def _collect(root, contract, work, inspector, source_binding): + run_id = uuid.uuid4().hex + daemon = inspector.preflight() + trials, files = [], {} + for role in ("nop", "oracle"): + trial, raw = await _trial( + root, + role, + work, + contract, + inspector, + run_id, + source_binding, + trials[0] if trials else None, + ) + trials.append(trial) + for name, value in raw.items(): + relative = f"harbor-trials/{role}/{name}" + files[relative] = value + trial["raw_rewards"][name] = { + "path": relative, + "sha256": hashlib.sha256(value).hexdigest(), + } + if "failure" in trial: + break + if trials and "image_ref" in trials[0]: + try: + if not inspector.verify_image( + f"openenv-validation:{run_id}", trials[0]["image_ref"] + ): + trials[-1]["failure"] = "RetainedImageDrift" + except Exception: + trials[-1]["failure"] = "RetainedImageUnverified" + return daemon, trials, files + + +def _result(check_id, status, reason, measured=None): + return CheckResult( + check_id=check_id, + status=status, + evidence=[reason], + measured=measured or {}, + duration_s=0, + ) + + +def validate_harbor( + subject, + *, + max_level, + policy, + results, + digest_before, + artifacts_dir, + skip_build=False, + provider=None, +): + """Return the usual report plus explicitly bounded native pilot evidence.""" + from .runner import source_digest + + started = time.monotonic() + contract = read_harbor_contract(subject.root) + record = { + "schema_version": "1", + "profile": HARBOR_PROFILE, + "task_effective_digest": digest_before, + "original_source_digest": contract.original_source_digest, + "contract_digest": _digest(contract.model_dump(mode="json")), + "reward_key": contract.reward_key, + "oracle_max": {"passed": False}, + "floor_gap": {"passed": False}, + "cleanup": {"verified": False}, + "trials": [], + } + harbor_dist = distribution("harbor") + record["harbor"] = {"version": harbor_dist.version} + direct_url = harbor_dist.read_text("direct_url.json") + if direct_url: + installed = json.loads(direct_url) + record["harbor"]["commit_id"] = installed.get("vcs_info", {}).get("commit_id") + files = {} + attempted = False + problem = _unsupported(read_harbor_task(subject.root), contract) + if skip_build: + problem = "image build skipped by --skip-build" + if provider is not None: + problem = "native Harbor binding requires its own Docker Trial provider" + if any(result.status is not CheckStatus.PASS for result in results): + problem = "unmet dependency: static.manifest" + startup = _result( + "runtime.startup", + CheckStatus.SKIP, + f"unsupported by {HARBOR_PROFILE}: {problem}", + ) + reward_result = _result( + "runtime.reward_well_formed", + CheckStatus.SKIP, + "unmet dependency: native Harbor trials", + ) + if not problem: + if subject.manifest.capabilities.oracle is None: + startup = _result( + "runtime.startup", + CheckStatus.FAIL, + "native pilot requires solution/solve.sh", + ) + else: + attempted = True + try: + with tempfile.TemporaryDirectory( + prefix="openenv-harbor-validation-" + ) as temp: + daemon, trials, files = asyncio.run( + _collect( + subject.root, + contract, + Path(temp), + DockerInspector(), + digest_before, + ) + ) + record.update({"daemon": daemon, "trials": trials}) + record["cleanup"] = { + "verified": bool(trials) + and all(trial["cleanup"]["verified"] for trial in trials) + } + failed = [trial for trial in trials if "failure" in trial] + if failed or len(trials) != 2: + startup = _result( + "runtime.startup", + CheckStatus.ERROR, + "native Harbor trial, binding or cleanup failed", + { + "failures": [ + trial.get("reason", trial.get("failure")) + for trial in failed + ] + }, + ) + if any(trial.get("reward_error") for trial in failed): + reward_result = _result( + "runtime.reward_well_formed", + CheckStatus.FAIL, + "native verifier reward violates the explicit contract", + ) + else: + floor, oracle = ( + trial["rewards"][contract.reward_key] for trial in trials + ) + maximum = Decimal(str(contract.reward.range[1])) + oracle_pass = Fraction(oracle) >= Fraction(maximum) - Fraction( + str(contract.reward.oracle_tolerance) + ) + gap_pass = Fraction(oracle) - Fraction(floor) >= Fraction( + str(contract.reward.floor_margin) + ) and Fraction(maximum) - Fraction(floor) >= Fraction( + str(contract.reward.floor_margin) + ) + record.update( + { + "image_ref": trials[0]["image_ref"], + "retained_image_tag": trials[0]["retained_image_tag"], + "floor_reward": floor, + "oracle_reward": oracle, + "oracle_max": { + "passed": oracle_pass, + "declared_max": maximum, + "tolerance": contract.reward.oracle_tolerance, + }, + "floor_gap": { + "passed": gap_pass, + "comparison": "exact-rational", + "measured_gap": oracle - floor, + "declared_margin": contract.reward.floor_margin, + }, + } + ) + distinct = trials[0]["container_id"] != trials[1]["container_id"] + startup = _result( + "runtime.startup", + CheckStatus.PASS + if oracle_pass and gap_pass and distinct + else CheckStatus.FAIL, + "fresh native Harbor trials with verified cleanup; pilot oracle/floor gates evaluated", + { + key: json.loads(dumps(record[key])) + for key in ( + "profile", + "image_ref", + "floor_reward", + "oracle_reward", + "oracle_max", + "floor_gap", + "cleanup", + "task_effective_digest", + ) + }, + ) + reward_result = _result( + "runtime.reward_well_formed", + CheckStatus.PASS, + "both raw verifier reward maps match Harbor and satisfy the declared key/range", + {"profile": HARBOR_PROFILE, "reward_key": contract.reward_key}, + ) + except (Exception, KeyboardInterrupt) as exc: + record["failure"] = type(exc).__name__ + startup = _result( + "runtime.startup", + CheckStatus.ERROR, + f"native Harbor collection failed ({type(exc).__name__})", + ) + try: + unchanged = source_digest(subject.root) == digest_before + except (ValueError, OSError): + unchanged = False + if not unchanged: + record["failure"] = "SourceDrift" + startup = _result( + "runtime.startup", + CheckStatus.ERROR, + "task source changed during native validation", + ) + reward_result = _result( + "runtime.reward_well_formed", + CheckStatus.SKIP, + "unmet dependency: unchanged native task source", + ) + startup.duration_s = time.monotonic() - started + record["cleanup"].update( + {"required": attempted, "completed": record["cleanup"]["verified"]} + ) + results = [*results, startup, reward_result] + present = {result.check_id for result in results} + for entry in policy.entries_for_lane(Lane.LOCAL).values(): + if entry.level <= max_level and entry.check_id not in present: + results.append( + _result( + entry.check_id, + CheckStatus.SKIP, + f"out of profile: {HARBOR_PROFILE}; native process binding has no implemented evidence for this check", + ) + ) + report = ValidationReportV2( + report_schema_version="2", + target=str(subject.root), + source_digest=digest_before, + signature=subject.manifest.signature, + manifest=subject.manifest, + policy_version=policy.policy_version, + lane=Lane.LOCAL, + levels_run=[Level.STATIC, Level.RUNTIME] if attempted else [Level.STATIC], + results=results, + verdict=apply_policy(results, policy, Lane.LOCAL), + ) + if artifacts_dir is not None: + write_runtime_bundle( + artifacts_dir, + report, + cleanup=record["cleanup"], + provider={"binding": "harbor_task", "profile": HARBOR_PROFILE}, + ) + files["harbor-evidence.json"] = (dumps(record) + "\n").encode() + checksums = (artifacts_dir / "SHA256SUMS").read_text() + for name, raw in sorted(files.items()): + path = artifacts_dir / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(raw) + path.chmod(0o600) + checksums += f"{hashlib.sha256(raw).hexdigest()} {name}\n" + (artifacts_dir / "SHA256SUMS").write_text(checksums) + return report diff --git a/src/openenv/validation/manifest.py b/src/openenv/validation/manifest.py index d5509d6d3..ca5fa12f3 100644 --- a/src/openenv/validation/manifest.py +++ b/src/openenv/validation/manifest.py @@ -380,6 +380,18 @@ def _file_path(cls, value: str) -> str: return value +class HarborExecutionDeclaration(BaseModel): + """Native process execution for the bounded Harbor oracle/floor pilot.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: Literal["harbor_task"] + agent_boundary: Literal["process"] = "process" + profile: Literal["harbor-oracle-floor-v1"] = "harbor-oracle-floor-v1" + contract_path: Literal["validation/openenv.json"] = "validation/openenv.json" + requires_credentials: bool = Field(default=False, strict=True) + + class NormalizedManifestV2(NormalizedManifest): """ Version 2 adds the runtime execution declaration without changing schema 1. @@ -390,7 +402,7 @@ class NormalizedManifestV2(NormalizedManifest): """ manifest_schema_version: Literal["2"] - execution: ExecutionDeclaration + execution: ExecutionDeclaration | HarborExecutionDeclaration @model_validator(mode="after") def _finite_declarations(self) -> "NormalizedManifestV2": diff --git a/src/openenv/validation/parsers/harbor_toml.py b/src/openenv/validation/parsers/harbor_toml.py new file mode 100644 index 000000000..664c0ab15 --- /dev/null +++ b/src/openenv/validation/parsers/harbor_toml.py @@ -0,0 +1,144 @@ +"""Pure Harbor task parsing and the explicit, operator-supplied pilot contract.""" + +import json +import math +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from ..manifest import ( + ManifestError, + NormalizedManifestV2, + ResourceDeclaration, + RewardDeclaration, + TypeSpec, +) +from ..types import SignatureKind + +HARBOR_PROFILE = "harbor-oracle-floor-v1" + + +class HarborContract(BaseModel): + """Trusted declarations absent from Harbor's task format; never inferred.""" + + model_config = ConfigDict(extra="forbid") + + profile: Literal["harbor-oracle-floor-v1"] + reward_key: str = Field(min_length=1) + reward: RewardDeclaration + resources: ResourceDeclaration + types: TypeSpec + original_source_digest: str | None = Field(default=None, pattern="^[0-9a-f]{64}$") + requires_credentials: bool = Field(default=False, strict=True) + + +def _unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON object key") + result[key] = value + return result + + +def read_harbor_contract(package_root: Path) -> HarborContract: + """Read `validation/openenv.json` without importing or executing task code.""" + try: + raw = json.loads( + (package_root / "validation/openenv.json").read_text(), + object_pairs_hook=_unique_object, + ) + if not isinstance(raw, dict): + raise ValueError("contract must be an object") + if ( + not {"range", "oracle_tolerance", "floor_margin"} + <= raw.get("reward", {}).keys() + ): + raise ValueError( + "reward range, oracle_tolerance and floor_margin must be explicit" + ) + for section in ("reward", "resources"): + pending = [raw.get(section)] + while pending: + item = pending.pop() + if isinstance(item, dict): + pending.extend(item.values()) + elif isinstance(item, list): + pending.extend(item) + elif isinstance(item, bool) or ( + isinstance(item, float) and not math.isfinite(item) + ): + raise ValueError( + "numeric declarations must be finite and nonboolean" + ) + return HarborContract.model_validate(raw) + except (OSError, ValueError, TypeError, AttributeError) as exc: + raise ManifestError( + [f"invalid trusted Harbor contract ({type(exc).__name__})"], + remediation="supply explicit reward/resources in validation/openenv.json", + ) from exc + + +def read_harbor_task(package_root: Path): + """Use the pinned Harbor data loader; constructing a Trial is runtime-only.""" + try: + if version("harbor") != "0.25.0": + raise ValueError("the native pilot requires Harbor 0.25.0") + from harbor.models.task.task import Task + + return Task(package_root) + except ( + ImportError, + PackageNotFoundError, + OSError, + ValueError, + RuntimeError, + ) as exc: + raise ManifestError( + [f"invalid Harbor 0.25.0 task ({type(exc).__name__})"], + remediation="install Harbor 0.25.0 and supply a valid task.toml/task layout", + ) from exc + + +class HarborTaskParser: + """Normalize declarations only; no environment, Trial, build or subprocess.""" + + signature = SignatureKind.HARBOR_TASK + + def parse(self, package_root: Path) -> NormalizedManifestV2: + contract = read_harbor_contract(package_root) + task = read_harbor_task(package_root) + oracle = ( + {"form": "script", "location": "solution/solve.sh"} + if (package_root / "solution/solve.sh").is_file() + else None + ) + try: + return NormalizedManifestV2.model_validate( + { + "manifest_schema_version": "2", + "signature": self.signature, + "name": task.name, + "version": task.config.task.version if task.config.task else None, + "reward": contract.reward.model_dump(), + "resources": contract.resources.model_dump(), + "types": contract.types.model_dump(), + "network": {"mode": task.config.environment.network_mode.value}, + "capabilities": { + "verifier": {"kind": "script", "entry": "tests/test.sh"}, + "oracle": oracle, + }, + "execution": { + "kind": "harbor_task", + "agent_boundary": "process", + "profile": contract.profile, + "requires_credentials": contract.requires_credentials, + }, + } + ) + except ValidationError as exc: + raise ManifestError( + ["Harbor declarations do not match the manifest schema"] + ) from exc diff --git a/src/openenv/validation/runner.py b/src/openenv/validation/runner.py index d1ec09c39..a8968a6f9 100644 --- a/src/openenv/validation/runner.py +++ b/src/openenv/validation/runner.py @@ -39,6 +39,7 @@ from .graders.static import StaticManifestGrader from .manifest import ManifestError, NormalizedManifest, NormalizedManifestV2 from .parsers import ParserRegistry +from .parsers.harbor_toml import HarborTaskParser from .parsers.openenv_yaml import OpenEnvYamlParser from .policy import apply_policy, load_policy, PolicyError, SeverityPolicy from .providers import CleanupError, ProviderError, StartupError, UnsupportedCapability @@ -121,6 +122,7 @@ def default_parser_registry() -> ParserRegistry: """The parsers shipped in this build.""" registry = ParserRegistry() registry.register(OpenEnvYamlParser()) + registry.register(HarborTaskParser()) return registry @@ -685,6 +687,22 @@ def run_validation( ) ) if wants_runtime: + if ( + isinstance(manifest, NormalizedManifestV2) + and manifest.execution.kind == "harbor_task" + ): + from .harbor import validate_harbor + + return validate_harbor( + subject, + max_level=max_level, + policy=policy, + results=results, + digest_before=digest_before, + artifacts_dir=artifacts_dir, + skip_build=skip_build, + provider=provider, + ) ( runtime_results, attempted, diff --git a/src/openenv/validation/schemas/manifest-v2.schema.json b/src/openenv/validation/schemas/manifest-v2.schema.json index 93525bff6..cc89df38a 100644 --- a/src/openenv/validation/schemas/manifest-v2.schema.json +++ b/src/openenv/validation/schemas/manifest-v2.schema.json @@ -111,6 +111,45 @@ "title": "ExecutionDeclaration", "type": "object" }, + "HarborExecutionDeclaration": { + "additionalProperties": false, + "description": "Native process execution for the bounded Harbor oracle/floor pilot.", + "properties": { + "agent_boundary": { + "const": "process", + "default": "process", + "title": "Agent Boundary", + "type": "string" + }, + "contract_path": { + "const": "validation/openenv.json", + "default": "validation/openenv.json", + "title": "Contract Path", + "type": "string" + }, + "kind": { + "const": "harbor_task", + "title": "Kind", + "type": "string" + }, + "profile": { + "const": "harbor-oracle-floor-v1", + "default": "harbor-oracle-floor-v1", + "title": "Profile", + "type": "string" + }, + "requires_credentials": { + "default": false, + "title": "Requires Credentials", + "type": "boolean" + } + }, + "required": [ + "kind" + ], + "title": "HarborExecutionDeclaration", + "type": "object" + }, "JudgePin": { "additionalProperties": false, "description": "Pinned judge configuration. Required iff `capabilities.llm_judged`.\n\nThe pin lives in the manifest, not the rubric object: `LLMJudge.state_dict()` does\nnot serialize model/version/params, so validation checks the declared pin.", @@ -398,7 +437,15 @@ "$ref": "#/$defs/CapabilitiesSpec" }, "execution": { - "$ref": "#/$defs/ExecutionDeclaration" + "anyOf": [ + { + "$ref": "#/$defs/ExecutionDeclaration" + }, + { + "$ref": "#/$defs/HarborExecutionDeclaration" + } + ], + "title": "Execution" }, "judge": { "anyOf": [ diff --git a/src/openenv/validation/schemas/report-v2.schema.json b/src/openenv/validation/schemas/report-v2.schema.json index 4f4d16e31..57e10928e 100644 --- a/src/openenv/validation/schemas/report-v2.schema.json +++ b/src/openenv/validation/schemas/report-v2.schema.json @@ -172,6 +172,45 @@ "title": "ExecutionDeclaration", "type": "object" }, + "HarborExecutionDeclaration": { + "additionalProperties": false, + "description": "Native process execution for the bounded Harbor oracle/floor pilot.", + "properties": { + "agent_boundary": { + "const": "process", + "default": "process", + "title": "Agent Boundary", + "type": "string" + }, + "contract_path": { + "const": "validation/openenv.json", + "default": "validation/openenv.json", + "title": "Contract Path", + "type": "string" + }, + "kind": { + "const": "harbor_task", + "title": "Kind", + "type": "string" + }, + "profile": { + "const": "harbor-oracle-floor-v1", + "default": "harbor-oracle-floor-v1", + "title": "Profile", + "type": "string" + }, + "requires_credentials": { + "default": false, + "title": "Requires Credentials", + "type": "boolean" + } + }, + "required": [ + "kind" + ], + "title": "HarborExecutionDeclaration", + "type": "object" + }, "JudgePin": { "additionalProperties": false, "description": "Pinned judge configuration. Required iff `capabilities.llm_judged`.\n\nThe pin lives in the manifest, not the rubric object: `LLMJudge.state_dict()` does\nnot serialize model/version/params, so validation checks the declared pin.", @@ -328,7 +367,15 @@ "$ref": "#/$defs/CapabilitiesSpec" }, "execution": { - "$ref": "#/$defs/ExecutionDeclaration" + "anyOf": [ + { + "$ref": "#/$defs/ExecutionDeclaration" + }, + { + "$ref": "#/$defs/HarborExecutionDeclaration" + } + ], + "title": "Execution" }, "judge": { "anyOf": [ diff --git a/src/openenv/validation/signature.py b/src/openenv/validation/signature.py index d6b9c6d15..80312ae59 100644 --- a/src/openenv/validation/signature.py +++ b/src/openenv/validation/signature.py @@ -6,6 +6,7 @@ WELL_KNOWN_FILES: dict[SignatureKind, str] = { SignatureKind.OPENENV_SERVED: "openenv.yaml", + SignatureKind.HARBOR_TASK: "task.toml", } """Formats this build can parse. Values are the enum filenames.""" diff --git a/tests/test_validation/test_cli_validate.py b/tests/test_validation/test_cli_validate.py index a1aee9c6e..3958e97dd 100644 --- a/tests/test_validation/test_cli_validate.py +++ b/tests/test_validation/test_cli_validate.py @@ -100,12 +100,13 @@ def test_unrecognized_package_exits_two(): assert "unrecognized" in result.stderr -def test_format_without_a_parser_exits_two(): +def test_harbor_without_trusted_contract_fails_manifest_validation(): result = _validate( str(FIXTURES / "harbor_task_min"), "--level", "static", "--skip-build" ) - assert result.returncode == 2, result.stdout + result.stderr - assert "unrecognized" in result.stderr + assert result.returncode == 1, result.stdout + result.stderr + assert "static.manifest" in result.stdout + assert "Harbor contract" in result.stdout def test_nonexistent_path_exits_two(tmp_path): diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py new file mode 100644 index 000000000..88fcdfe86 --- /dev/null +++ b/tests/test_validation/test_harbor_validation.py @@ -0,0 +1,565 @@ +"""Native pilot checks mock only the Trial boundary and trusted host inspector.""" + +import hashlib +import json +from types import SimpleNamespace + +import pytest +from openenv.validation import harbor +from openenv.validation.manifest import ManifestError +from openenv.validation.parsers.harbor_toml import ( + HarborTaskParser, + read_harbor_contract, +) +from openenv.validation.runner import run_validation, source_digest +from openenv.validation.types import CheckStatus, Level, SignatureKind, Verdict + +pytest.importorskip("harbor.models.task.task") + +IMAGE = "sha256:" + "a" * 64 + + +@pytest.fixture +def task(tmp_path): + root = tmp_path / "task" + for directory in ("environment", "solution", "tests", "validation"): + (root / directory).mkdir(parents=True) + (root / "task.toml").write_text( + 'schema_version = "1.4"\n[verifier]\nenvironment_mode = "shared"\n' + ) + (root / "instruction.md").write_text("Write the correct answer.\n") + (root / "environment/Dockerfile").write_text("FROM debian:bookworm-slim\n") + (root / "tests/test.sh").write_text("#!/bin/sh\nexit 0\n") + (root / "solution/solve.sh").write_text("#!/bin/sh\nexit 0\n") + contract = { + "profile": "harbor-oracle-floor-v1", + "reward_key": "reward", + "reward": {"range": [0, 1], "oracle_tolerance": 0, "floor_margin": 0.1}, + "resources": { + "cpu": 1, + "memory_mb": 512, + "disk_mb": 1024, + "episode_timeout_s": 10, + }, + "types": {"tags": ["swe"]}, + "original_source_digest": "b" * 64, + "requires_credentials": False, + } + (root / "validation/openenv.json").write_text(json.dumps(contract)) + return root + + +@pytest.fixture +def boundary(monkeypatch, task): + state = SimpleNamespace( + rewards={"nop": b'{"reward":0}', "oracle": b'{"reward":1}'}, + trials=[], + cleanup_ok=True, + mutate=False, + private_mutate=False, + image_drift=False, + config_drift=False, + oracle_exit=None, + exception=False, + no_identity=False, + timeout=False, + retained=True, + overwrite_exit=False, + excluded_copied=[], + ) + + class Inspector: + def preflight(self): + return {"os": "linux", "daemon_id": "mock-host"} + + def inventory(self, project): + return {"containers": [], "networks": [], "volumes": []} + + def capture(self, project, retained_tag, *, retain=True): + role = "oracle" if "-oracle__env" in project else "nop" + return { + "container_id": role + "-container", + "image_ref": "sha256:" + "c" * 64 + if state.image_drift and role == "oracle" + else IMAGE, + "configuration_digest": "drift" + if state.config_drift and role == "oracle" + else "stable", + "retained_image_tag": retained_tag, + } + + def cleanup(self, project): + return { + "verified": state.cleanup_ok, + "after": { + "containers": [] if state.cleanup_ok else ["leftover"], + "networks": [], + "volumes": [], + }, + } + + def verify_image(self, tag, image_ref): + return state.retained + + class Trial: + def __init__(self, config): + self.config = config + self.role = config.agent.name + directory = config.trials_dir / config.trial_name + (directory / "verifier").mkdir(parents=True) + (directory / "agent").mkdir() + self.paths = SimpleNamespace( + trial_dir=directory, + reward_json_path=directory / "verifier/reward.json", + reward_text_path=directory / "verifier/reward.txt", + agent_dir=directory / "agent", + ) + self.agent_environment = SimpleNamespace( + session_id=config.trial_name + "__env", stop=self.stop + ) + self.hooks = {} + state.excluded_copied.append( + (config.task.path / "environment/.worktrees").exists() + ) + state.trials.append(self) + + async def stop(self, *, delete): + assert not delete + + def add_hook(self, event, callback): + self.hooks.setdefault(event.value, []).append(callback) + + async def run(self): + if not state.no_identity: + for hook in self.hooks.get("agent-start", []): + await hook(None) + if state.timeout: + raise TimeoutError() + raw = state.rewards[self.role] + if raw is not None: + self.paths.reward_json_path.write_bytes(raw) + if self.role == "oracle" and state.oracle_exit is not None: + (self.paths.agent_dir / "exit-code.txt").write_text( + str(state.oracle_exit) + ) + try: + for hook in self.hooks.get("agent-end", []): + await hook(None) + finally: + if self.role == "oracle" and state.overwrite_exit: + (self.paths.agent_dir / "exit-code.txt").write_text("0") + if state.mutate: + (task / "instruction.md").write_text("changed source") + if state.private_mutate: + (self.config.task.path / "instruction.md").write_text( + "changed execution source" + ) + try: + rewards = json.loads(raw) if raw else None + except ValueError: + rewards = None + return SimpleNamespace( + exception_info={"exception_type": "TestError"} + if state.exception + else None, + verifier_result=SimpleNamespace(rewards=rewards), + ) + + async def create(config): + return Trial(config) + + monkeypatch.setattr(harbor, "DockerInspector", Inspector) + monkeypatch.setattr(harbor, "_create_trial", create) + return state + + +def result(report, check_id): + return next(check for check in report.results if check.check_id == check_id) + + +def test_parser_is_pure_and_describes_process_binding(task, monkeypatch): + monkeypatch.setattr( + harbor, "_create_trial", lambda config: pytest.fail("Trial created by parser") + ) + manifest = HarborTaskParser().parse(task) + assert manifest.signature is SignatureKind.HARBOR_TASK + assert manifest.execution.kind == "harbor_task" + assert manifest.execution.agent_boundary == "process" + assert manifest.reward.floor_margin == 0.1 + + +def test_success_is_bounded_warn_with_native_raw_evidence(task, boundary, tmp_path): + original = source_digest(task) + artifacts = tmp_path / "artifacts" + report = run_validation(task, artifacts_dir=artifacts) + assert report.verdict is Verdict.WARN + assert source_digest(task) == original + required = {"static.manifest", "runtime.startup", "runtime.reward_well_formed"} + assert { + check.check_id for check in report.results if check.status is CheckStatus.PASS + } == required + assert all( + check.status is CheckStatus.SKIP + for check in report.results + if check.check_id not in required + ) + assert result(report, "semantic.oracle_max").status is CheckStatus.SKIP + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert native["profile"] == "harbor-oracle-floor-v1" + assert native["task_effective_digest"] == report.source_digest == original + assert native["original_source_digest"] == "b" * 64 + assert native["image_ref"] == IMAGE + assert native["floor_reward"] == 0 and native["oracle_reward"] == 1 + assert native["oracle_max"]["passed"] and native["floor_gap"]["passed"] + assert native["cleanup"]["verified"] + assert [trial.config.agent.name for trial in boundary.trials] == ["nop", "oracle"] + assert all(not trial.config.environment.delete for trial in boundary.trials) + assert all( + trial.config.environment.kwargs["keep_containers"] for trial in boundary.trials + ) + assert boundary.trials[0].config.task.path != boundary.trials[1].config.task.path + assert json.loads((artifacts / "report.json").read_text()) == json.loads( + report.model_dump_json() + ) + inventory = {} + for line in (artifacts / "SHA256SUMS").read_text().splitlines(): + expected, name = line.split(" ", 1) + assert hashlib.sha256((artifacts / name).read_bytes()).hexdigest() == expected + inventory[name] = expected + assert { + "harbor-evidence.json", + "harbor-trials/nop/reward.json", + "harbor-trials/oracle/reward.json", + } <= inventory.keys() + assert (artifacts / "harbor-trials/nop/reward.json").read_bytes() == b'{"reward":0}' + + +@pytest.mark.parametrize( + "raw", + [ + None, + b"{}", + b'{"other":0}', + b'{"reward":true}', + b'{"reward":NaN}', + b'{"reward":Infinity}', + b'{"reward":1.01}', + b'{"reward":-0.01}', + b'{"reward":"0"}', + b'{"reward":0,"aux":true}', + b'{"reward":0,"reward":1}', + b'{"reward":1.0000000000000000001}', + ], +) +def test_bad_rewards_block_and_preserve_raw(task, boundary, tmp_path, raw): + boundary.rewards["nop"] = raw + artifacts = tmp_path / "bad" + report = run_validation(task, artifacts_dir=artifacts) + assert report.verdict is Verdict.FAIL + assert result(report, "runtime.reward_well_formed").status is not CheckStatus.PASS + assert len(boundary.trials) == 1 + if raw is not None: + assert (artifacts / "harbor-trials/nop/reward.json").read_bytes() == raw + + +@pytest.mark.parametrize( + "floor,oracle,oracle_max,floor_gap", + [(1, 1, True, False), (0, 0.8, False, True), (0.95, 1, True, False)], +) +def test_actual_oracle_floor_arithmetic( + task, boundary, tmp_path, floor, oracle, oracle_max, floor_gap +): + boundary.rewards = { + "nop": json.dumps({"reward": floor}).encode(), + "oracle": json.dumps({"reward": oracle}).encode(), + } + artifacts = tmp_path / "arithmetic" + report = run_validation(task, artifacts_dir=artifacts) + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert report.verdict is Verdict.FAIL + assert native["oracle_max"]["passed"] is oracle_max + assert native["floor_gap"]["passed"] is floor_gap + assert result(report, "runtime.reward_well_formed").status is CheckStatus.PASS + + +def test_tiny_positive_floor_does_not_round_up_to_margin(task, boundary, tmp_path): + # Decimal's default precision and float subtraction both round this gap up. + raw_contract = json.loads((task / "validation/openenv.json").read_text()) + raw_contract["reward"]["oracle_tolerance"] = 0.1 + raw_contract["reward"]["floor_margin"] = 0.9 + (task / "validation/openenv.json").write_text(json.dumps(raw_contract)) + boundary.rewards = {"nop": b'{"reward":1e-300}', "oracle": b'{"reward":0.9}'} + artifacts = tmp_path / "exact-gap" + report = run_validation(task, artifacts_dir=artifacts) + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert native["oracle_max"]["passed"] + assert not native["floor_gap"]["passed"] + assert report.verdict is Verdict.FAIL + + +@pytest.mark.parametrize( + "flag", + [ + "mutate", + "private_mutate", + "image_drift", + "config_drift", + "exception", + "no_identity", + "timeout", + ], +) +def test_source_identity_exception_timeout_fail_closed(task, boundary, flag): + setattr(boundary, flag, True) + report = run_validation(task) + assert report.verdict is Verdict.FAIL + assert result(report, "runtime.startup").status is CheckStatus.ERROR + + +def test_cleanup_failure_blocks(task, boundary, tmp_path): + boundary.cleanup_ok = False + artifacts = tmp_path / "cleanup" + report = run_validation(task, artifacts_dir=artifacts) + assert report.verdict is Verdict.FAIL + assert not json.loads((artifacts / "harbor-evidence.json").read_text())["cleanup"][ + "verified" + ] + + +def test_oracle_nonzero_exit_blocks_and_retains_marker(task, boundary, tmp_path): + boundary.oracle_exit = 7 + artifacts = tmp_path / "exit" + report = run_validation(task, artifacts_dir=artifacts) + assert report.verdict is Verdict.FAIL + assert ( + artifacts / "harbor-trials/oracle/oracle-exit-code.txt" + ).read_bytes() == b"7" + + +def test_sandbox_log_sync_cannot_erase_trusted_oracle_exit(task, boundary, tmp_path): + boundary.oracle_exit = 7 + boundary.overwrite_exit = True + artifacts = tmp_path / "overwritten-exit" + report = run_validation(task, artifacts_dir=artifacts) + assert report.verdict is Verdict.FAIL + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert native["trials"][1]["oracle_exit_code"] == 7 + assert ( + artifacts / "harbor-trials/oracle/oracle-exit-code.txt" + ).read_bytes() == b"7" + + +def test_excluded_symlink_tree_is_never_dereferenced_or_copied( + task, boundary, tmp_path +): + sentinel = tmp_path / "outside-secret" + sentinel.write_text("must not enter a build context") + excluded = task / "environment/.worktrees" + excluded.mkdir() + (excluded / "payload").symlink_to(sentinel) + assert run_validation(task).verdict is Verdict.WARN + assert boundary.excluded_copied == [False, False] + + +def test_copy_change_and_restore_cannot_escape_source_binding( + task, boundary, tmp_path, monkeypatch +): + solution = task / "solution/solve.sh" + solution.write_text("exit 7\n") + digest = source_digest(task) + copytree = harbor.shutil.copytree + copies = 0 + + def copying(*args, **kwargs): + nonlocal copies + if args[0] == task: + copies += 1 + if copies == 2: + solution.write_text("exit 0\n") + try: + return copytree(*args, **kwargs) + finally: + solution.write_text("exit 7\n") + return copytree(*args, **kwargs) + + monkeypatch.setattr(harbor.shutil, "copytree", copying) + artifacts = tmp_path / "copy-binding" + report = run_validation(task, artifacts_dir=artifacts) + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert source_digest(task) == report.source_digest == digest + assert report.verdict is Verdict.FAIL + assert result(report, "runtime.startup").status is CheckStatus.ERROR + assert len(boundary.trials) == 1 + assert native["trials"][1]["unmodified_task_digest"] != digest + assert "validated source digest" in native["trials"][1]["reason"] + + +def test_missing_oracle_blocks_without_starting_trial(task, boundary): + (task / "solution/solve.sh").unlink() + report = run_validation(task) + assert report.verdict is Verdict.FAIL + assert result(report, "runtime.startup").status is CheckStatus.FAIL + assert not boundary.trials + + +def test_retained_image_drift_blocks(task, boundary): + boundary.retained = False + assert run_validation(task).verdict is Verdict.FAIL + + +def test_host_inspector_scopes_inventory_and_cleanup_to_owned_project(monkeypatch): + inspector = harbor.DockerInspector() + project = "openenv-validation-" + "d" * 32 + "-nop__env" + commands = [] + resources = { + "containers": ["c" * 64], + "networks": ["e" * 64], + "volumes": ["owned-volume"], + } + + def docker(*args): + commands.append(args) + if args == ("volume", "ls", "-q"): + return "\n".join(resources["volumes"]) + if args[0] == "inspect": + return json.dumps( + [ + { + "Config": {"Labels": {"com.docker.compose.project": project}}, + "Mounts": [], + } + ] + ) + if args[0] == "ps": + assert args[-2:] == ( + "--filter", + f"label=com.docker.compose.project={project}", + ) + return "\n".join(resources["containers"]) + if args[:2] in (("network", "ls"), ("volume", "ls")): + assert args[-2:] == ( + "--filter", + f"label=com.docker.compose.project={project}", + ) + return "\n".join( + resources["networks" if args[0] == "network" else "volumes"] + ) + if args[0] == "rm": + assert args == ("rm", "-f", "-v", "c" * 64) + resources["containers"] = [] + elif args[:2] == ("network", "rm"): + assert args == ("network", "rm", "e" * 64) + resources["networks"] = [] + elif args[:2] == ("volume", "rm"): + assert args == ("volume", "rm", "owned-volume") + resources["volumes"] = [] + else: + pytest.fail(f"unexpected Docker command: {args}") + return "" + + monkeypatch.setattr(inspector, "_docker", docker) + assert inspector.cleanup(project)["verified"] + assert commands[-3:] == [ + ( + "ps", + "-aq", + "--no-trunc", + "--filter", + f"label=com.docker.compose.project={project}", + ), + ( + "network", + "ls", + "-q", + "--no-trunc", + "--filter", + f"label=com.docker.compose.project={project}", + ), + ( + "volume", + "ls", + "-q", + "--filter", + f"label=com.docker.compose.project={project}", + ), + ] + with pytest.raises(ValueError, match="run-owned"): + inspector.cleanup("somebody-elses-project") + + +@pytest.mark.parametrize("volume_survives_removal", [False, True]) +@pytest.mark.parametrize("automatic_volume_removal", [False, True]) +def test_anonymous_volumes_are_registered_removed_and_rechecked( + monkeypatch, volume_survives_removal, automatic_volume_removal +): + inspector = harbor.DockerInspector() + project = "openenv-validation-" + "d" * 32 + "-nop__env" + containers = ["c" * 64] + volumes = {"anonymous-owned", "unrelated-volume"} + + def inventory(value): + assert value == project + # The anonymous volume deliberately has no Compose project label. + return {"containers": list(containers), "networks": [], "volumes": []} + + def docker(*args): + if args == ("inspect", "c" * 64): + return json.dumps( + [ + { + "Config": {"Labels": {"com.docker.compose.project": project}}, + "Mounts": [{"Type": "volume", "Name": "anonymous-owned"}], + } + ] + ) + if args == ("volume", "ls", "-q"): + return "\n".join(volumes) + if args == ("rm", "-f", "-v", "c" * 64): + containers.clear() # Docker may log volume-removal errors and return 0. + if automatic_volume_removal: + volumes.remove("anonymous-owned") + elif args == ("volume", "rm", "anonymous-owned"): + if not volume_survives_removal: + volumes.remove("anonymous-owned") + else: + pytest.fail(f"unexpected command: {args}") + return "" + + monkeypatch.setattr(inspector, "inventory", inventory) + monkeypatch.setattr(inspector, "_docker", docker) + cleanup = inspector.cleanup(project) + assert cleanup["verified"] is ( + automatic_volume_removal or not volume_survives_removal + ) + assert "unrelated-volume" in volumes + assert "anonymous-owned" in cleanup["before"]["volumes"] + + +def test_static_and_skip_build_do_not_start_trials(task, boundary): + assert run_validation(task, max_level=Level.STATIC).verdict is Verdict.PASS + report = run_validation(task, skip_build=True) + assert result(report, "runtime.startup").status is CheckStatus.SKIP + assert not boundary.trials + + +def test_separate_verifier_is_named_unsupported(task, boundary): + (task / "task.toml").write_text('[verifier]\nenvironment_mode="separate"\n') + report = run_validation(task) + assert result(report, "runtime.startup").status is CheckStatus.SKIP + assert "separate" in result(report, "runtime.startup").evidence[0] + assert not boundary.trials + + +def test_explicit_contract_and_valid_toml_required(task, boundary): + (task / "validation/openenv.json").unlink() + with pytest.raises(ManifestError): + read_harbor_contract(task) + assert run_validation(task).verdict is Verdict.FAIL + assert not boundary.trials + + +def test_malformed_toml_is_manifest_failure(task, boundary): + (task / "task.toml").write_text("invalid = [") + report = run_validation(task) + assert result(report, "static.manifest").status is CheckStatus.FAIL + assert not boundary.trials diff --git a/tests/test_validation/test_signature_detection.py b/tests/test_validation/test_signature_detection.py index 3363d82f2..8a5451a04 100644 --- a/tests/test_validation/test_signature_detection.py +++ b/tests/test_validation/test_signature_detection.py @@ -15,10 +15,17 @@ def test_openenv_yaml_detects_the_served_format(): def test_the_table_lists_exactly_the_implemented_parsers(): - assert set(WELL_KNOWN_FILES) == {SignatureKind.OPENENV_SERVED} + assert set(WELL_KNOWN_FILES) == { + SignatureKind.OPENENV_SERVED, + SignatureKind.HARBOR_TASK, + } -@pytest.mark.parametrize("fixture", ["harbor_task_min", "posttrain_task_min"]) +def test_harbor_task_detects_the_native_format(): + assert detect_signature(FIXTURES / "harbor_task_min") is SignatureKind.HARBOR_TASK + + +@pytest.mark.parametrize("fixture", ["posttrain_task_min"]) def test_formats_without_parsers_are_unrecognized_not_guessed(fixture): with pytest.raises(SignatureError, match="unrecognized"): detect_signature(FIXTURES / fixture) diff --git a/tests/test_validation/test_types.py b/tests/test_validation/test_types.py index 911eea0a2..c2b4d417d 100644 --- a/tests/test_validation/test_types.py +++ b/tests/test_validation/test_types.py @@ -30,4 +30,7 @@ def test_lanes(): def test_well_known_files_lists_implemented_parsers(): - assert WELL_KNOWN_FILES == {SignatureKind.OPENENV_SERVED: "openenv.yaml"} + assert WELL_KNOWN_FILES == { + SignatureKind.OPENENV_SERVED: "openenv.yaml", + SignatureKind.HARBOR_TASK: "task.toml", + } From baefcba05bb762a3379a7c188b0010a6872f86dd Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 17:18:43 +0100 Subject: [PATCH 02/40] feat(validation): name every unimplemented local check in reports Static runs reported only static.manifest, so a valid package read as PASS while nine static checks never ran. Every requested local check without a grader is now a named SKIP at every level, which makes such runs WARN (exit code 0). Served packages also state why semantic checks skip: oracle execution is not implemented for them. Co-Authored-By: Claude Opus 5.5 --- src/openenv/validation/runner.py | 46 +++++++++++----------- tests/test_cli/test_init.py | 2 +- tests/test_validation/test_cli_validate.py | 8 ++-- tests/test_validation/test_runner.py | 23 ++++++++--- 4 files changed, 45 insertions(+), 34 deletions(-) diff --git a/src/openenv/validation/runner.py b/src/openenv/validation/runner.py index a8968a6f9..1a47a100c 100644 --- a/src/openenv/validation/runner.py +++ b/src/openenv/validation/runner.py @@ -726,31 +726,29 @@ def run_validation( if attempted: levels.append(Level.RUNTIME) + # Policy IDs are an inventory, not evidence that a grader exists. Name every + # requested check that did not run, so a PASS never hides missing coverage. + present = {result.check_id for result in results} + for entry in policy.entries_for_lane(Lane.LOCAL).values(): + grader = graders.get(entry.check_id) + applicable = ( + manifest is None + or entry.check_id in selected + or (grader is None and _applicable(entry.check_id, manifest)) + ) + if entry.level <= max_level and entry.check_id not in present and applicable: + reason = "grader not implemented in this build" + if manifest is None: + reason = "unmet dependency: valid manifest" + elif entry.level is Level.SEMANTIC: + reason = "oracle execution is not implemented for served packages" + elif grader is not None and grader.depends_on: + reason = "unmet dependency: " + ", ".join(grader.depends_on) + elif grader is not None: + reason = "grader did not complete" + results.append(_outcome(entry.check_id, CheckStatus.SKIP, reason)) + if wants_runtime: - # Policy IDs are an inventory, not evidence that a grader exists. Keep the - # incomplete surface explicit throughout the staged implementation. - present = {result.check_id for result in results} - for entry in policy.entries_for_lane(Lane.LOCAL).values(): - grader = graders.get(entry.check_id) - applicable = ( - manifest is None - or entry.check_id in selected - or (grader is None and _applicable(entry.check_id, manifest)) - ) - if ( - entry.level in {Level.RUNTIME, Level.SEMANTIC} - and entry.level <= max_level - and entry.check_id not in present - and applicable - ): - reason = "grader not implemented in this build" - if manifest is None: - reason = "unmet dependency: valid manifest" - elif grader is not None and grader.depends_on: - reason = "unmet dependency: " + ", ".join(grader.depends_on) - elif grader is not None: - reason = "grader did not complete" - results.append(_outcome(entry.check_id, CheckStatus.SKIP, reason)) source_problem = None if not digest_before: source_problem = "package source could not be verified before validation" diff --git a/tests/test_cli/test_init.py b/tests/test_cli/test_init.py index 8dd4bdeaf..e596aad91 100644 --- a/tests/test_cli/test_init.py +++ b/tests/test_cli/test_init.py @@ -132,7 +132,7 @@ def test_init_generates_openenv_yaml(tmp_path: Path) -> None: ], ) assert validate_result.exit_code == 0, validate_result.output - assert "Verdict: PASS" in validate_result.output + assert "PASS static.manifest" in validate_result.output environment_file = env_dir / "server" / f"{env_name}_environment.py" assert "min(length * 0.1, 1.0)" in environment_file.read_text() diff --git a/tests/test_validation/test_cli_validate.py b/tests/test_validation/test_cli_validate.py index 3958e97dd..190cc2c02 100644 --- a/tests/test_validation/test_cli_validate.py +++ b/tests/test_validation/test_cli_validate.py @@ -22,7 +22,7 @@ def test_valid_package_exits_zero(): str(FIXTURES / "served_min_pass"), "--level", "static", "--skip-build" ) assert result.returncode == 0, result.stdout + result.stderr - assert "Verdict: PASS" in result.stdout + assert "Verdict: WARN" in result.stdout def test_echo_env_exits_zero(): @@ -35,7 +35,8 @@ def test_echo_env_exits_zero(): ) assert result.returncode == 0, result.stdout + result.stderr report = ValidationReport.model_validate_json(result.stdout) - assert report.verdict.value == "pass" + # Static graders other than static.manifest are named SKIPs: WARN, exit 0. + assert report.verdict.value == "warn" assert any(r.check_id == "static.manifest" for r in report.results) @@ -45,7 +46,8 @@ def test_json_report_is_schema_valid(): ) assert result.returncode == 0, result.stdout + result.stderr report = ValidationReport.model_validate_json(result.stdout) - assert report.verdict.value == "pass" + # Static graders other than static.manifest are named SKIPs: WARN, exit 0. + assert report.verdict.value == "warn" def test_output_writes_the_json_report(tmp_path): diff --git a/tests/test_validation/test_runner.py b/tests/test_validation/test_runner.py index a77826e7b..48b171851 100644 --- a/tests/test_validation/test_runner.py +++ b/tests/test_validation/test_runner.py @@ -89,16 +89,24 @@ def collect(*args, **kwargs): ] -def test_valid_package_passes_static_level(): +def test_valid_package_names_unimplemented_static_checks(): report = run_validation( FIXTURES / "served_min_pass", max_level=Level.STATIC, skip_build=True ) - assert report.verdict is Verdict.PASS + # A PASS on static.manifest alone must not read as full static coverage. + assert report.verdict is Verdict.WARN assert report.lane is Lane.LOCAL assert report.levels_run == [Level.STATIC] assert report.manifest is not None and report.manifest.name == "served-min-pass" by_id = {r.check_id: r for r in report.results} - assert by_id["static.manifest"].status is CheckStatus.PASS + assert by_id.pop("static.manifest").status is CheckStatus.PASS + assert len(by_id) == 9 + assert all( + r.check_id.startswith("static.") + and r.status is CheckStatus.SKIP + and r.evidence == ["grader not implemented in this build"] + for r in by_id.values() + ) def test_report_round_trips_through_its_schema(): @@ -114,10 +122,11 @@ def test_broken_manifest_fails_static_manifest_with_evidence(): ) assert report.verdict is Verdict.FAIL assert report.manifest is None - (result,) = report.results + result, *others = report.results assert result.check_id == "static.manifest" assert result.status is CheckStatus.FAIL assert result.evidence, "schema errors must surface as evidence" + assert all(r.evidence == ["unmet dependency: valid manifest"] for r in others) def test_out_of_bounds_declaration_fails_static_manifest(tmp_path): @@ -127,7 +136,7 @@ def test_out_of_bounds_declaration_fails_static_manifest(tmp_path): ) report = run_validation(tmp_path, max_level=Level.STATIC, skip_build=True) assert report.verdict is Verdict.FAIL - (result,) = report.results + result = next(r for r in report.results if r.check_id == "static.manifest") assert result.status is CheckStatus.FAIL assert "floor_margin" in "\n".join(result.evidence) @@ -192,7 +201,9 @@ def fail_parse(*args): ) report = run_validation(FIXTURES / "served_min_pass", max_level=Level.STATIC) - (result,) = report.results + result, *others = report.results + assert result.check_id == "static.manifest" + assert all(r.status is CheckStatus.SKIP for r in others) assert report.source_digest == "" assert result.status is CheckStatus.ERROR assert ( From 2599cf2176ede1508beff51d2ed9f1a1c102c82c Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 17:31:00 +0100 Subject: [PATCH 03/40] feat(validation): grade Harbor tasks with RFC 008 semantic checks The native Harbor binding now reports its oracle and floor results under the RFC check ids instead of folding them into runtime.startup: - semantic.oracle_max: every oracle run exits 0 and earns max - tolerance. A task without solution/solve.sh fails it, as the RFC requires. - semantic.floor_gap: the do-nothing reward sits the declared margin below the maximum and below the oracle. - semantic.canary_floor: operator-declared canaries (known-wrong solve.sh replacements in validation/canaries/) earn no more than doing nothing. - runtime.episode_determinism: an oracle replay in a fresh container on the same image must reproduce every reward. It gates the semantic checks. runtime.startup now means the trials ran and cleaned up. A malformed verifier reward fails runtime.reward_well_formed rather than erroring startup. The graders read a format-neutral OracleEvidence, so a future served binding can reuse them. The contract profile becomes harbor-semantic-v1 because the trials and report changed. Co-Authored-By: Claude Opus 5.5 --- src/openenv/validation/graders/__init__.py | 10 +- .../validation/graders/semantic/__init__.py | 5 + .../validation/graders/semantic/oracle.py | 145 ++++++++ src/openenv/validation/harbor.py | 333 +++++++++++++----- src/openenv/validation/manifest.py | 4 +- src/openenv/validation/parsers/harbor_toml.py | 63 +++- src/openenv/validation/runtime/contracts.py | 27 ++ .../schemas/manifest-v2.schema.json | 6 +- .../validation/schemas/report-v2.schema.json | 6 +- .../test_validation/test_harbor_validation.py | 196 +++++++++-- .../test_validation/test_semantic_graders.py | 94 +++++ 11 files changed, 758 insertions(+), 131 deletions(-) create mode 100644 src/openenv/validation/graders/semantic/__init__.py create mode 100644 src/openenv/validation/graders/semantic/oracle.py create mode 100644 tests/test_validation/test_semantic_graders.py diff --git a/src/openenv/validation/graders/__init__.py b/src/openenv/validation/graders/__init__.py index e52a6f7f5..88bbd0222 100644 --- a/src/openenv/validation/graders/__init__.py +++ b/src/openenv/validation/graders/__init__.py @@ -13,7 +13,12 @@ from ..manifest import CapabilitiesSpec, NormalizedManifest from ..providers import RunningSubject from ..report import CheckResult -from ..runtime.contracts import HttpProbe, IsolationEvidence, RuntimeEvidence +from ..runtime.contracts import ( + HttpProbe, + IsolationEvidence, + OracleEvidence, + RuntimeEvidence, +) from ..types import Level, ProviderCapability ENTRY_POINT_GROUP = "openenv.validation.graders" @@ -46,6 +51,8 @@ class Subject: Same-session and fresh episodes on the running subject. oracle_http (`tuple[HttpProbe, ...]`): GET probes for the declared oracle path on the control endpoint. + oracle_evidence ([`~openenv.validation.runtime.contracts.OracleEvidence`], *optional*): + Oracle, do-nothing and canary rewards for the Level 3 checks. """ root: Path @@ -59,6 +66,7 @@ class Subject: resource_evidence_json: str | None = None isolation_evidence: IsolationEvidence | None = None oracle_http: tuple[HttpProbe, ...] = () + oracle_evidence: OracleEvidence | None = None @runtime_checkable diff --git a/src/openenv/validation/graders/semantic/__init__.py b/src/openenv/validation/graders/semantic/__init__.py new file mode 100644 index 000000000..08aadc2e4 --- /dev/null +++ b/src/openenv/validation/graders/semantic/__init__.py @@ -0,0 +1,5 @@ +"""Semantic-level graders.""" + +from .oracle import CanaryFloorGrader, FloorGapGrader, OracleMaxGrader + +__all__ = ["CanaryFloorGrader", "FloorGapGrader", "OracleMaxGrader"] diff --git a/src/openenv/validation/graders/semantic/oracle.py b/src/openenv/validation/graders/semantic/oracle.py new file mode 100644 index 000000000..881274eac --- /dev/null +++ b/src/openenv/validation/graders/semantic/oracle.py @@ -0,0 +1,145 @@ +"""Verifier sanity over oracle, do-nothing and canary rewards (#778 tests #25, #28). + +The execution binding measures the rewards; these graders only compare them with the +manifest's declared range, tolerance and margin. Comparisons use exact rationals so a +tiny floor cannot round into the margin. +""" + +import time +from fractions import Fraction + +from ...report import CheckResult +from ...types import CheckStatus, Level + + +class _SemanticGrader: + level = Level.SEMANTIC + requires_capabilities = frozenset() + requires_provider = frozenset() + # RFC 008: episode determinism is a precondition of every semantic check. + depends_on = ("runtime.episode_determinism",) + remediation = "" + + def applies_to(self, manifest) -> bool: + return True + + def run(self, subject) -> CheckResult: + started = time.monotonic() + evidence = subject.oracle_evidence + if evidence is None: + return CheckResult( + check_id=self.check_id, + status=CheckStatus.SKIP, + evidence=["oracle evidence is unavailable"], + duration_s=0, + ) + reward = subject.manifest.reward + problems, measured = self.check( + evidence, + maximum=Fraction(str(reward.range[1])), + tolerance=Fraction(str(reward.oracle_tolerance)), + margin=Fraction(str(reward.floor_margin)), + ) + return CheckResult( + check_id=self.check_id, + status=CheckStatus.FAIL if problems else CheckStatus.PASS, + measured=measured, + evidence=problems or [self.passed], + remediation=self.remediation if problems else None, + duration_s=time.monotonic() - started, + ) + + +class OracleMaxGrader(_SemanticGrader): + """Every oracle run exits cleanly and earns at least `max - oracle_tolerance`.""" + + check_id = "semantic.oracle_max" + passed = "every oracle run reached the declared maximum within tolerance" + remediation = ( + "Fix the oracle or the verifier so a correct solution exits cleanly and " + "earns the declared maximum reward." + ) + + def check(self, evidence, *, maximum, tolerance, margin): + threshold = maximum - tolerance + problems = [] + for run, (reward, code) in enumerate( + zip(evidence.oracle, evidence.oracle_exit_codes, strict=True), start=1 + ): + if code != 0: + problems.append(f"oracle run {run} exited with code {code}") + if reward < threshold: + problems.append( + f"oracle run {run} earned {float(reward)}, below {float(threshold)}" + ) + return problems, { + "oracle_rewards": [float(reward) for reward in evidence.oracle], + "oracle_exit_codes": list(evidence.oracle_exit_codes), + "declared_max": float(maximum), + "oracle_tolerance": float(tolerance), + } + + +class FloorGapGrader(_SemanticGrader): + """ + The do-nothing reward sits at least `floor_margin` below the maximum and below + every oracle run. The oracle may sit up to the tolerance below the maximum, so + both gaps are required. + """ + + check_id = "semantic.floor_gap" + passed = "the do-nothing reward leaves the declared margin" + remediation = ( + "The verifier rewards doing nothing. Check for vacuous success conditions, " + "such as comparisons against empty output." + ) + + def check(self, evidence, *, maximum, tolerance, margin): + floor = evidence.floor + problems = [] + if maximum - floor < margin: + problems.append( + f"do-nothing reward {float(floor)} is within {float(maximum - floor)} " + f"of the maximum, below the margin {float(margin)}" + ) + lowest = min(evidence.oracle) + if lowest - floor < margin: + problems.append( + f"oracle reward {float(lowest)} is within {float(lowest - floor)} " + f"of the do-nothing reward, below the margin {float(margin)}" + ) + return problems, { + "measured_floor": float(floor), + "oracle_reward": float(lowest), + "declared_max": float(maximum), + "declared_margin": float(margin), + } + + +class CanaryFloorGrader(_SemanticGrader): + """Every declared canary, a known-wrong solution, earns no more than doing nothing.""" + + check_id = "semantic.canary_floor" + requires_capabilities = frozenset({"canaries"}) + passed = "every canary stayed at the do-nothing reward" + remediation = ( + "The verifier rewards a known-wrong solution. Tighten the tests so each " + "canary earns no more than doing nothing." + ) + + def check(self, evidence, *, maximum, tolerance, margin): + problems = [ + f"canary {name} earned {float(reward)}, above the do-nothing reward " + f"{float(evidence.floor)}" + for name, reward in evidence.canaries + if reward > evidence.floor + ] + # An empty run list must not pass vacuously. + if not evidence.canaries: + problems.append("no canary run was recorded") + return problems, { + "measured_floor": float(evidence.floor), + "canary_rewards": { + name: float(reward) for name, reward in evidence.canaries + }, + } diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index ab1ed8fbf..50804c563 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -1,4 +1,8 @@ -"""Native Harbor oracle/floor pilot; no served-protocol evidence is synthesized.""" +"""Native Harbor execution: fresh Docker trials graded through RFC 008 check ids. + +No served-protocol evidence is synthesized. A do-nothing trial builds the image, and +the oracle, its replay and any canaries run pinned to that image. +""" import asyncio import hashlib @@ -11,21 +15,50 @@ import tempfile import time import uuid +from dataclasses import replace from decimal import Decimal from fractions import Fraction from importlib.metadata import distribution from pathlib import Path -from .parsers.harbor_toml import HARBOR_PROFILE, read_harbor_contract, read_harbor_task +from .graders import GraderRegistry +from .graders.semantic import CanaryFloorGrader, FloorGapGrader, OracleMaxGrader +from .parsers.harbor_toml import ( + canary_scripts, + HARBOR_PROFILE, + read_harbor_contract, + read_harbor_task, +) from .policy import apply_policy from .report import CheckResult, ValidationReportV2 from .runtime.artifacts import write_runtime_bundle +from .runtime.contracts import OracleEvidence from .runtime.json_values import dumps +from .runtime.scheduler import execute_graders from .types import CheckStatus, Lane, Level _IMAGE_ID = re.compile(r"sha256:[0-9a-f]{64}\Z") _RESOURCE_ID = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*\Z") +_PROJECT = re.compile( + r"openenv-validation-[a-f0-9]+-(nop|oracle|oracle-replay|canary-[1-8])__env" +) _MAX_REWARD_BYTES = 1024 * 1024 +# Failures the trial's teardown records over any earlier one. +_HOST_FAILURES = { + "HarborTeardownError", + "CleanupNotVerified", + "RewardArtifactDrift", + "UnsafeRewardArtifact", +} +# Checks this binding measures; every other local check is a named SKIP. +_IMPLEMENTED = { + "runtime.startup", + "runtime.reward_well_formed", + "runtime.episode_determinism", + "semantic.oracle_max", + "semantic.floor_gap", + "semantic.canary_floor", +} class NativeEvidenceError(ValueError): @@ -64,7 +97,7 @@ def preflight(self): } def inventory(self, project): - if not re.fullmatch(r"openenv-validation-[a-f0-9]+-(nop|oracle)__env", project): + if not _PROJECT.fullmatch(project): raise NativeEvidenceError("not a run-owned Harbor project") label = f"label=com.docker.compose.project={project}" inventory = {} @@ -303,7 +336,16 @@ def _reward(raw_files, parsed, contract): async def _trial( - root, role, work, contract, inspector, run_id, source_binding, expected=None + root, + role, + agent, + work, + contract, + inspector, + run_id, + source_binding, + expected=None, + canary=None, ): from harbor.models.environment_type import EnvironmentType from harbor.models.trial.config import ( @@ -322,6 +364,7 @@ async def _trial( retained_tag = f"openenv-validation:{run_id}" record = { "role": role, + "agent": agent, "project": project, "raw_rewards": {}, "cleanup": {"verified": False}, @@ -345,20 +388,28 @@ async def _trial( raise NativeEvidenceError( "private execution task differs from the validated source digest" ) + overlay = {} if expected: task = read_harbor_task(task_copy) task.config.environment.docker_image = retained_tag (task_copy / "task.toml").write_text(task.config.model_dump_toml()) + overlay["environment.docker_image"] = retained_tag + if canary is not None: + # The oracle agent runs the canary in place of the reference solution. + script = (task_copy / canary).read_bytes() + (task_copy / "solution/solve.sh").write_bytes(script) + overlay["solution/solve.sh"] = { + "canary": canary, + "sha256": hashlib.sha256(script).hexdigest(), + } digest_before = source_digest(task_copy) record["execution_task_digest"] = digest_before - record["task_overlay"] = ( - {"environment.docker_image": retained_tag} if expected else {} - ) + record["task_overlay"] = overlay budget = contract.resources.episode_timeout_s config = TrialConfig( task=TaskConfig(path=task_copy.resolve()), agent=AgentConfig( - name=role, + name=agent, override_timeout_sec=budget, override_setup_timeout_sec=budget, ), @@ -401,7 +452,9 @@ async def capture(_event): ) async def capture_exit(_event): - if role == "oracle": + # Harbor writes the marker only for a nonzero exit. Later sandbox + # log synchronization can overwrite it, so this reading is trusted. + if agent == "oracle": marker = _read_output( trial.paths.agent_dir / "exit-code.txt", trial.paths.trial_dir ) @@ -411,10 +464,6 @@ async def capture_exit(_event): ) if marker is not None: raw_files["oracle-exit-code.txt"] = marker - if record["oracle_exit_code"] != 0: - raise NativeEvidenceError( - "oracle exit-code.txt records a nonzero exit" - ) trial.add_hook(TrialEvent.AGENT_START, capture) trial.add_hook(TrialEvent.AGENT_END, capture_exit) @@ -434,7 +483,7 @@ async def capture_exit(_event): raw = _read_output(path, trial.paths.trial_dir) if raw is not None: raw_files[filename] = raw - if role == "oracle": + if agent == "oracle": if not record.get("oracle_exit_observed_at_agent_end"): raise NativeEvidenceError( "oracle has no trusted AGENT_END exit observation" @@ -442,8 +491,8 @@ async def capture_exit(_event): marker = _read_output( trial.paths.agent_dir / "exit-code.txt", trial.paths.trial_dir ) - if marker is not None and int(marker) != 0: - raise NativeEvidenceError("oracle exit-code.txt records a nonzero exit") + if marker is not None and int(marker) != record["oracle_exit_code"]: + raise NativeEvidenceError("oracle exit marker changed after AGENT_END") parsed = result.verifier_result.rewards if result.verifier_result else None record["reward_error"] = True record["rewards"] = _reward(raw_files, parsed, contract) @@ -487,20 +536,32 @@ async def capture_exit(_event): return record, raw_files -async def _collect(root, contract, work, inspector, source_binding): +async def _collect(root, contract, work, inspector, source_binding, canaries=()): run_id = uuid.uuid4().hex daemon = inspector.preflight() + # The replay makes episode determinism measurable before any semantic check. + plan = [ + ("nop", "nop", None), + ("oracle", "oracle", None), + ("oracle-replay", "oracle", None), + *( + (f"canary-{index}", "oracle", canary) + for index, canary in enumerate(canaries, start=1) + ), + ] trials, files = [], {} - for role in ("nop", "oracle"): + for role, agent, canary in plan: trial, raw = await _trial( root, role, + agent, work, contract, inspector, run_id, source_binding, trials[0] if trials else None, + canary, ) trials.append(trial) for name, value in raw.items(): @@ -533,6 +594,22 @@ def _result(check_id, status, reason, measured=None): ) +def _reward_only(trial): + # `reward_error` marks a failure raised while parsing the verifier's reward: + # the package's defect, unless teardown or cleanup later found a host problem. + return trial.get("reward_error") and trial.get("failure") not in _HOST_FAILURES + + +def _skip_reason(entry, contract): + if entry.check_id not in _IMPLEMENTED: + if entry.level is Level.STATIC: + return "grader not implemented in this build" + return f"not implemented for the harbor_task binding ({HARBOR_PROFILE})" + if entry.check_id == "semantic.canary_floor" and contract.canaries is None: + return "no canaries declared in validation/openenv.json" + return "unmet dependency: native Harbor trials" + + def validate_harbor( subject, *, @@ -544,20 +621,24 @@ def validate_harbor( skip_build=False, provider=None, ): - """Return the usual report plus explicitly bounded native pilot evidence.""" + """Run fresh native Harbor trials and grade them up to the requested level.""" from .runner import source_digest started = time.monotonic() contract = read_harbor_contract(subject.root) + semantic = max_level >= Level.SEMANTIC + canaries = ( + canary_scripts(subject.root, contract.canaries) + if semantic and contract.canaries is not None + else [] + ) record = { - "schema_version": "1", + "schema_version": "2", "profile": HARBOR_PROFILE, "task_effective_digest": digest_before, "original_source_digest": contract.original_source_digest, "contract_digest": _digest(contract.model_dump(mode="json")), "reward_key": contract.reward_key, - "oracle_max": {"passed": False}, - "floor_gap": {"passed": False}, "cleanup": {"verified": False}, "trials": [], } @@ -586,13 +667,34 @@ def validate_harbor( CheckStatus.SKIP, "unmet dependency: native Harbor trials", ) + determinism = _result( + "runtime.episode_determinism", + CheckStatus.SKIP, + "unmet dependency: native Harbor trials", + ) + evidence = None + checks = [] if not problem: if subject.manifest.capabilities.oracle is None: startup = _result( "runtime.startup", - CheckStatus.FAIL, - "native pilot requires solution/solve.sh", + CheckStatus.SKIP, + "native Harbor trials need solution/solve.sh", ) + if semantic: + # RFC 008: a missing oracle is a graded FAIL, never a SKIP. + checks.append( + CheckResult( + check_id="semantic.oracle_max", + status=CheckStatus.FAIL, + evidence=["the task has no oracle (solution/solve.sh)"], + remediation=( + "Add solution/solve.sh that drives the task to a " + "correct solution." + ), + duration_s=0, + ) + ) else: attempted = True try: @@ -606,6 +708,7 @@ def validate_harbor( Path(temp), DockerInspector(), digest_before, + canaries, ) ) record.update({"daemon": daemon, "trials": trials}) @@ -613,8 +716,18 @@ def validate_harbor( "verified": bool(trials) and all(trial["cleanup"]["verified"] for trial in trials) } - failed = [trial for trial in trials if "failure" in trial] - if failed or len(trials) != 2: + broken = [ + trial + for trial in trials + if "failure" in trial and not _reward_only(trial) + ] + malformed = any(_reward_only(trial) for trial in trials) + containers = [trial.get("container_id") for trial in trials] + if ( + broken + or (not malformed and len(trials) != 3 + len(canaries)) + or len(set(containers)) != len(containers) + ): startup = _result( "runtime.startup", CheckStatus.ERROR, @@ -622,75 +735,108 @@ def validate_harbor( { "failures": [ trial.get("reason", trial.get("failure")) - for trial in failed + for trial in broken ] }, ) - if any(trial.get("reward_error") for trial in failed): + if malformed: reward_result = _result( "runtime.reward_well_formed", CheckStatus.FAIL, "native verifier reward violates the explicit contract", ) else: - floor, oracle = ( - trial["rewards"][contract.reward_key] for trial in trials - ) - maximum = Decimal(str(contract.reward.range[1])) - oracle_pass = Fraction(oracle) >= Fraction(maximum) - Fraction( - str(contract.reward.oracle_tolerance) - ) - gap_pass = Fraction(oracle) - Fraction(floor) >= Fraction( - str(contract.reward.floor_margin) - ) and Fraction(maximum) - Fraction(floor) >= Fraction( - str(contract.reward.floor_margin) - ) record.update( { "image_ref": trials[0]["image_ref"], "retained_image_tag": trials[0]["retained_image_tag"], - "floor_reward": floor, - "oracle_reward": oracle, - "oracle_max": { - "passed": oracle_pass, - "declared_max": maximum, - "tolerance": contract.reward.oracle_tolerance, - }, - "floor_gap": { - "passed": gap_pass, - "comparison": "exact-rational", - "measured_gap": oracle - floor, - "declared_margin": contract.reward.floor_margin, - }, } ) - distinct = trials[0]["container_id"] != trials[1]["container_id"] startup = _result( "runtime.startup", - CheckStatus.PASS - if oracle_pass and gap_pass and distinct - else CheckStatus.FAIL, - "fresh native Harbor trials with verified cleanup; pilot oracle/floor gates evaluated", + CheckStatus.PASS, + "fresh native Harbor trials ran from one immutable image " + "with verified cleanup", { - key: json.loads(dumps(record[key])) - for key in ( - "profile", - "image_ref", - "floor_reward", - "oracle_reward", - "oracle_max", - "floor_gap", - "cleanup", - "task_effective_digest", - ) + "profile": HARBOR_PROFILE, + "image_ref": trials[0]["image_ref"], + "roles": [trial["role"] for trial in trials], + "task_effective_digest": digest_before, }, ) - reward_result = _result( - "runtime.reward_well_formed", - CheckStatus.PASS, - "both raw verifier reward maps match Harbor and satisfy the declared key/range", - {"profile": HARBOR_PROFILE, "reward_key": contract.reward_key}, - ) + if malformed: + reward_result = _result( + "runtime.reward_well_formed", + CheckStatus.FAIL, + "native verifier reward violates the explicit contract", + ) + determinism = _result( + "runtime.episode_determinism", + CheckStatus.SKIP, + "unmet dependency: runtime.reward_well_formed", + ) + else: + reward_result = _result( + "runtime.reward_well_formed", + CheckStatus.PASS, + "every raw verifier reward map matches Harbor and " + "satisfies the declared key/range", + { + "profile": HARBOR_PROFILE, + "reward_key": contract.reward_key, + }, + ) + nop, oracle, replay, *canary_trials = trials + same = ( + oracle["rewards"] == replay["rewards"] + and oracle["oracle_exit_code"] == replay["oracle_exit_code"] + ) + determinism = _result( + "runtime.episode_determinism", + CheckStatus.PASS if same else CheckStatus.FAIL, + "the oracle replay in a fresh container " + + ( + "reproduced every reward and the exit code" + if same + else "produced different rewards or a different exit code" + ), + { + "rewards": [ + json.loads(dumps(trial["rewards"])) + for trial in (oracle, replay) + ], + "exit_codes": [ + trial["oracle_exit_code"] + for trial in (oracle, replay) + ], + }, + ) + key = contract.reward_key + record.update( + { + "floor_reward": nop["rewards"][key], + "oracle_reward": oracle["rewards"][key], + } + ) + evidence = OracleEvidence( + floor=Fraction(nop["rewards"][key]), + oracle=tuple( + Fraction(trial["rewards"][key]) + for trial in (oracle, replay) + ), + oracle_exit_codes=tuple( + trial["oracle_exit_code"] for trial in (oracle, replay) + ), + canaries=tuple( + ( + trial["task_overlay"]["solution/solve.sh"][ + "canary" + ], + Fraction(trial["rewards"][key]), + ) + for trial in canary_trials + ), + ) except (Exception, KeyboardInterrupt) as exc: record["failure"] = type(exc).__name__ startup = _result( @@ -698,6 +844,21 @@ def validate_harbor( CheckStatus.ERROR, f"native Harbor collection failed ({type(exc).__name__})", ) + if semantic: + registry = GraderRegistry() + for grader in ( + OracleMaxGrader(), + FloorGapGrader(), + CanaryFloorGrader(), + ): + registry.register(grader) + checks.extend( + execute_graders( + registry.select(subject.manifest, max_level), + replace(subject, oracle_evidence=evidence), + prior=[startup, reward_result, determinism], + ) + ) try: unchanged = source_digest(subject.root) == digest_before except (ValueError, OSError): @@ -709,16 +870,19 @@ def validate_harbor( CheckStatus.ERROR, "task source changed during native validation", ) - reward_result = _result( - "runtime.reward_well_formed", - CheckStatus.SKIP, - "unmet dependency: unchanged native task source", + reward_result, determinism, *checks = ( + _result( + check.check_id, + CheckStatus.SKIP, + "unmet dependency: unchanged native task source", + ) + for check in (reward_result, determinism, *checks) ) startup.duration_s = time.monotonic() - started record["cleanup"].update( {"required": attempted, "completed": record["cleanup"]["verified"]} ) - results = [*results, startup, reward_result] + results = [*results, startup, reward_result, determinism, *checks] present = {result.check_id for result in results} for entry in policy.entries_for_lane(Lane.LOCAL).values(): if entry.level <= max_level and entry.check_id not in present: @@ -726,9 +890,14 @@ def validate_harbor( _result( entry.check_id, CheckStatus.SKIP, - f"out of profile: {HARBOR_PROFILE}; native process binding has no implemented evidence for this check", + _skip_reason(entry, contract), ) ) + levels = [Level.STATIC] + if attempted: + levels.append(Level.RUNTIME) + if semantic: + levels.append(Level.SEMANTIC) report = ValidationReportV2( report_schema_version="2", target=str(subject.root), @@ -737,7 +906,7 @@ def validate_harbor( manifest=subject.manifest, policy_version=policy.policy_version, lane=Lane.LOCAL, - levels_run=[Level.STATIC, Level.RUNTIME] if attempted else [Level.STATIC], + levels_run=levels, results=results, verdict=apply_policy(results, policy, Lane.LOCAL), ) diff --git a/src/openenv/validation/manifest.py b/src/openenv/validation/manifest.py index ca5fa12f3..08eba70bb 100644 --- a/src/openenv/validation/manifest.py +++ b/src/openenv/validation/manifest.py @@ -381,13 +381,13 @@ def _file_path(cls, value: str) -> str: class HarborExecutionDeclaration(BaseModel): - """Native process execution for the bounded Harbor oracle/floor pilot.""" + """Native process execution of a Harbor task through fresh Docker trials.""" model_config = ConfigDict(extra="forbid", frozen=True) kind: Literal["harbor_task"] agent_boundary: Literal["process"] = "process" - profile: Literal["harbor-oracle-floor-v1"] = "harbor-oracle-floor-v1" + profile: Literal["harbor-semantic-v1"] = "harbor-semantic-v1" contract_path: Literal["validation/openenv.json"] = "validation/openenv.json" requires_credentials: bool = Field(default=False, strict=True) diff --git a/src/openenv/validation/parsers/harbor_toml.py b/src/openenv/validation/parsers/harbor_toml.py index 664c0ab15..3b935c6ce 100644 --- a/src/openenv/validation/parsers/harbor_toml.py +++ b/src/openenv/validation/parsers/harbor_toml.py @@ -2,11 +2,12 @@ import json import math +import stat from importlib.metadata import PackageNotFoundError, version -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Literal -from pydantic import BaseModel, ConfigDict, Field, ValidationError +from pydantic import BaseModel, ConfigDict, Field, field_validator, ValidationError from ..manifest import ( ManifestError, @@ -17,7 +18,8 @@ ) from ..types import SignatureKind -HARBOR_PROFILE = "harbor-oracle-floor-v1" +HARBOR_PROFILE = "harbor-semantic-v1" +MAX_CANARIES = 8 class HarborContract(BaseModel): @@ -25,13 +27,30 @@ class HarborContract(BaseModel): model_config = ConfigDict(extra="forbid") - profile: Literal["harbor-oracle-floor-v1"] + profile: Literal["harbor-semantic-v1"] reward_key: str = Field(min_length=1) reward: RewardDeclaration resources: ResourceDeclaration types: TypeSpec original_source_digest: str | None = Field(default=None, pattern="^[0-9a-f]{64}$") requires_credentials: bool = Field(default=False, strict=True) + # Directory of known-wrong `solve.sh` replacements for `semantic.canary_floor`. + canaries: str | None = None + + @field_validator("canaries") + @classmethod + def _package_relative(cls, value): + path = PurePosixPath(value) + if ( + "\\" in value + or ":" in value + or "\x00" in value + or path.is_absolute() + or ".." in path.parts + or path == PurePosixPath(".") + ): + raise ValueError("canaries must be a portable package-relative directory") + return value def _unique_object(pairs): @@ -81,6 +100,33 @@ def read_harbor_contract(package_root: Path) -> HarborContract: ) from exc +def canary_scripts(package_root: Path, directory: str) -> list[str]: + """ + List the declared canary scripts without reading or running them. + + Args: + package_root (`Path`): + The task directory. + directory (`str`): + Package-relative directory holding one `*.sh` file per canary. + + Returns: + `list[str]`: package-relative script paths, sorted by name. + """ + root = package_root / directory + if root.is_symlink() or not root.is_dir(): + raise ValueError("the canaries directory does not exist") + scripts = [] + for path in sorted(root.iterdir()): + info = path.lstat() + if path.suffix != ".sh" or not stat.S_ISREG(info.st_mode): + raise ValueError("the canaries directory may hold only regular *.sh files") + scripts.append((PurePosixPath(directory) / path.name).as_posix()) + if not 1 <= len(scripts) <= MAX_CANARIES: + raise ValueError(f"declare between 1 and {MAX_CANARIES} canaries") + return scripts + + def read_harbor_task(package_root: Path): """Use the pinned Harbor data loader; constructing a Trial is runtime-only.""" try: @@ -110,6 +156,14 @@ class HarborTaskParser: def parse(self, package_root: Path) -> NormalizedManifestV2: contract = read_harbor_contract(package_root) task = read_harbor_task(package_root) + if contract.canaries is not None: + try: + canary_scripts(package_root, contract.canaries) + except (OSError, ValueError) as exc: + raise ManifestError( + [f"invalid canaries declaration ({exc})"], + remediation="declare a directory of 1 to 8 *.sh canary scripts", + ) from exc oracle = ( {"form": "script", "location": "solution/solve.sh"} if (package_root / "solution/solve.sh").is_file() @@ -129,6 +183,7 @@ def parse(self, package_root: Path) -> NormalizedManifestV2: "capabilities": { "verifier": {"kind": "script", "entry": "tests/test.sh"}, "oracle": oracle, + "canaries": contract.canaries, }, "execution": { "kind": "harbor_task", diff --git a/src/openenv/validation/runtime/contracts.py b/src/openenv/validation/runtime/contracts.py index 7b4164293..478ba53ac 100644 --- a/src/openenv/validation/runtime/contracts.py +++ b/src/openenv/validation/runtime/contracts.py @@ -4,6 +4,7 @@ import math import re from dataclasses import dataclass +from fractions import Fraction from pathlib import Path from typing import Any, Literal @@ -685,6 +686,32 @@ class IsolationEvidence: fresh: RuntimeEvidence +@dataclass(frozen=True) +class OracleEvidence: + """ + Rewards for the Level 3 checks, measured by the execution binding. + + The binding runs every episode in a fresh subject from one immutable image and + selects the graded reward. Values are exact rationals parsed from the raw + verifier output. + + Attributes: + floor (`Fraction`): + Reward of the do-nothing episode. + oracle (`tuple[Fraction, ...]`): + Reward of each oracle run, in run order. + oracle_exit_codes (`tuple[int, ...]`): + Exit code of each oracle run, in run order. + canaries (`tuple[tuple[str, Fraction], ...]`, *optional*): + Name and reward of each declared canary run. + """ + + floor: Fraction + oracle: tuple[Fraction, ...] + oracle_exit_codes: tuple[int, ...] + canaries: tuple[tuple[str, Fraction], ...] = () + + @dataclass(frozen=True) class HttpProbe: """ diff --git a/src/openenv/validation/schemas/manifest-v2.schema.json b/src/openenv/validation/schemas/manifest-v2.schema.json index cc89df38a..228f0d27a 100644 --- a/src/openenv/validation/schemas/manifest-v2.schema.json +++ b/src/openenv/validation/schemas/manifest-v2.schema.json @@ -113,7 +113,7 @@ }, "HarborExecutionDeclaration": { "additionalProperties": false, - "description": "Native process execution for the bounded Harbor oracle/floor pilot.", + "description": "Native process execution of a Harbor task through fresh Docker trials.", "properties": { "agent_boundary": { "const": "process", @@ -133,8 +133,8 @@ "type": "string" }, "profile": { - "const": "harbor-oracle-floor-v1", - "default": "harbor-oracle-floor-v1", + "const": "harbor-semantic-v1", + "default": "harbor-semantic-v1", "title": "Profile", "type": "string" }, diff --git a/src/openenv/validation/schemas/report-v2.schema.json b/src/openenv/validation/schemas/report-v2.schema.json index 57e10928e..dc5b1f185 100644 --- a/src/openenv/validation/schemas/report-v2.schema.json +++ b/src/openenv/validation/schemas/report-v2.schema.json @@ -174,7 +174,7 @@ }, "HarborExecutionDeclaration": { "additionalProperties": false, - "description": "Native process execution for the bounded Harbor oracle/floor pilot.", + "description": "Native process execution of a Harbor task through fresh Docker trials.", "properties": { "agent_boundary": { "const": "process", @@ -194,8 +194,8 @@ "type": "string" }, "profile": { - "const": "harbor-oracle-floor-v1", - "default": "harbor-oracle-floor-v1", + "const": "harbor-semantic-v1", + "default": "harbor-semantic-v1", "title": "Profile", "type": "string" }, diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 88fcdfe86..9e1089966 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -1,4 +1,4 @@ -"""Native pilot checks mock only the Trial boundary and trusted host inspector.""" +"""Native Harbor checks mock only the Trial boundary and trusted host inspector.""" import hashlib import json @@ -32,7 +32,7 @@ def task(tmp_path): (root / "tests/test.sh").write_text("#!/bin/sh\nexit 0\n") (root / "solution/solve.sh").write_text("#!/bin/sh\nexit 0\n") contract = { - "profile": "harbor-oracle-floor-v1", + "profile": "harbor-semantic-v1", "reward_key": "reward", "reward": {"range": [0, 1], "oracle_tolerance": 0, "floor_margin": 0.1}, "resources": { @@ -76,7 +76,7 @@ def inventory(self, project): return {"containers": [], "networks": [], "volumes": []} def capture(self, project, retained_tag, *, retain=True): - role = "oracle" if "-oracle__env" in project else "nop" + role = _role(project.removesuffix("__env")) return { "container_id": role + "-container", "image_ref": "sha256:" + "c" * 64 @@ -104,7 +104,9 @@ def verify_image(self, tag, image_ref): class Trial: def __init__(self, config): self.config = config - self.role = config.agent.name + self.role = _role(config.trial_name) + self.agent = config.agent.name + self.solve = (config.task.path / "solution/solve.sh").read_bytes() directory = config.trials_dir / config.trial_name (directory / "verifier").mkdir(parents=True) (directory / "agent").mkdir() @@ -135,10 +137,10 @@ async def run(self): await hook(None) if state.timeout: raise TimeoutError() - raw = state.rewards[self.role] + raw = state.rewards.get(self.role, state.rewards[self.agent]) if raw is not None: self.paths.reward_json_path.write_bytes(raw) - if self.role == "oracle" and state.oracle_exit is not None: + if self.agent == "oracle" and state.oracle_exit is not None: (self.paths.agent_dir / "exit-code.txt").write_text( str(state.oracle_exit) ) @@ -146,7 +148,7 @@ async def run(self): for hook in self.hooks.get("agent-end", []): await hook(None) finally: - if self.role == "oracle" and state.overwrite_exit: + if self.agent == "oracle" and state.overwrite_exit: (self.paths.agent_dir / "exit-code.txt").write_text("0") if state.mutate: (task / "instruction.md").write_text("changed source") @@ -173,10 +175,19 @@ async def create(config): return state +def _role(trial_name): + # openenv-validation-- + return trial_name.split("-", 3)[3] + + def result(report, check_id): return next(check for check in report.results if check.check_id == check_id) +def statuses(report): + return {check.check_id: check.status for check in report.results} + + def test_parser_is_pure_and_describes_process_binding(task, monkeypatch): monkeypatch.setattr( harbor, "_create_trial", lambda config: pytest.fail("Trial created by parser") @@ -188,36 +199,59 @@ def test_parser_is_pure_and_describes_process_binding(task, monkeypatch): assert manifest.reward.floor_margin == 0.1 -def test_success_is_bounded_warn_with_native_raw_evidence(task, boundary, tmp_path): +def test_success_grades_semantic_checks_with_native_raw_evidence( + task, boundary, tmp_path +): original = source_digest(task) artifacts = tmp_path / "artifacts" report = run_validation(task, artifacts_dir=artifacts) + # Checks this binding does not implement remain named SKIPs. assert report.verdict is Verdict.WARN + assert report.levels_run == [Level.STATIC, Level.RUNTIME, Level.SEMANTIC] assert source_digest(task) == original - required = {"static.manifest", "runtime.startup", "runtime.reward_well_formed"} - assert { - check.check_id for check in report.results if check.status is CheckStatus.PASS - } == required + passed = { + "static.manifest", + "runtime.startup", + "runtime.reward_well_formed", + "runtime.episode_determinism", + "semantic.oracle_max", + "semantic.floor_gap", + } + found = statuses(report) + assert {check for check, status in found.items() if status is CheckStatus.PASS} == ( + passed + ) assert all( - check.status is CheckStatus.SKIP - for check in report.results - if check.check_id not in required + status is CheckStatus.SKIP + for check, status in found.items() + if check not in passed ) - assert result(report, "semantic.oracle_max").status is CheckStatus.SKIP + assert result(report, "semantic.canary_floor").evidence == [ + "no canaries declared in validation/openenv.json" + ] + assert result(report, "static.sbom").evidence == [ + "grader not implemented in this build" + ] + assert "harbor_task" in result(report, "semantic.replayability").evidence[0] + assert result(report, "semantic.floor_gap").measured["measured_floor"] == 0 native = json.loads((artifacts / "harbor-evidence.json").read_text()) - assert native["profile"] == "harbor-oracle-floor-v1" + assert native["profile"] == "harbor-semantic-v1" assert native["task_effective_digest"] == report.source_digest == original assert native["original_source_digest"] == "b" * 64 assert native["image_ref"] == IMAGE assert native["floor_reward"] == 0 and native["oracle_reward"] == 1 - assert native["oracle_max"]["passed"] and native["floor_gap"]["passed"] assert native["cleanup"]["verified"] - assert [trial.config.agent.name for trial in boundary.trials] == ["nop", "oracle"] + assert [trial["role"] for trial in native["trials"]] == [ + "nop", + "oracle", + "oracle-replay", + ] + assert [trial.agent for trial in boundary.trials] == ["nop", "oracle", "oracle"] assert all(not trial.config.environment.delete for trial in boundary.trials) assert all( trial.config.environment.kwargs["keep_containers"] for trial in boundary.trials ) - assert boundary.trials[0].config.task.path != boundary.trials[1].config.task.path + assert len({trial.config.task.path for trial in boundary.trials}) == 3 assert json.loads((artifacts / "report.json").read_text()) == json.loads( report.model_dump_json() ) @@ -230,6 +264,7 @@ def test_success_is_bounded_warn_with_native_raw_evidence(task, boundary, tmp_pa "harbor-evidence.json", "harbor-trials/nop/reward.json", "harbor-trials/oracle/reward.json", + "harbor-trials/oracle-replay/reward.json", } <= inventory.keys() assert (artifacts / "harbor-trials/nop/reward.json").read_bytes() == b'{"reward":0}' @@ -256,7 +291,9 @@ def test_bad_rewards_block_and_preserve_raw(task, boundary, tmp_path, raw): artifacts = tmp_path / "bad" report = run_validation(task, artifacts_dir=artifacts) assert report.verdict is Verdict.FAIL - assert result(report, "runtime.reward_well_formed").status is not CheckStatus.PASS + assert result(report, "runtime.startup").status is CheckStatus.PASS + assert result(report, "runtime.reward_well_formed").status is CheckStatus.FAIL + assert result(report, "semantic.oracle_max").status is CheckStatus.SKIP assert len(boundary.trials) == 1 if raw is not None: assert (artifacts / "harbor-trials/nop/reward.json").read_bytes() == raw @@ -273,12 +310,11 @@ def test_actual_oracle_floor_arithmetic( "nop": json.dumps({"reward": floor}).encode(), "oracle": json.dumps({"reward": oracle}).encode(), } - artifacts = tmp_path / "arithmetic" - report = run_validation(task, artifacts_dir=artifacts) - native = json.loads((artifacts / "harbor-evidence.json").read_text()) + report = run_validation(task) + expected = {True: CheckStatus.PASS, False: CheckStatus.FAIL} assert report.verdict is Verdict.FAIL - assert native["oracle_max"]["passed"] is oracle_max - assert native["floor_gap"]["passed"] is floor_gap + assert result(report, "semantic.oracle_max").status is expected[oracle_max] + assert result(report, "semantic.floor_gap").status is expected[floor_gap] assert result(report, "runtime.reward_well_formed").status is CheckStatus.PASS @@ -289,11 +325,9 @@ def test_tiny_positive_floor_does_not_round_up_to_margin(task, boundary, tmp_pat raw_contract["reward"]["floor_margin"] = 0.9 (task / "validation/openenv.json").write_text(json.dumps(raw_contract)) boundary.rewards = {"nop": b'{"reward":1e-300}', "oracle": b'{"reward":0.9}'} - artifacts = tmp_path / "exact-gap" - report = run_validation(task, artifacts_dir=artifacts) - native = json.loads((artifacts / "harbor-evidence.json").read_text()) - assert native["oracle_max"]["passed"] - assert not native["floor_gap"]["passed"] + report = run_validation(task) + assert result(report, "semantic.oracle_max").status is CheckStatus.PASS + assert result(report, "semantic.floor_gap").status is CheckStatus.FAIL assert report.verdict is Verdict.FAIL @@ -326,11 +360,17 @@ def test_cleanup_failure_blocks(task, boundary, tmp_path): ] -def test_oracle_nonzero_exit_blocks_and_retains_marker(task, boundary, tmp_path): +def test_oracle_nonzero_exit_fails_oracle_max_and_retains_marker( + task, boundary, tmp_path +): boundary.oracle_exit = 7 artifacts = tmp_path / "exit" report = run_validation(task, artifacts_dir=artifacts) assert report.verdict is Verdict.FAIL + assert result(report, "runtime.startup").status is CheckStatus.PASS + oracle_max = result(report, "semantic.oracle_max") + assert oracle_max.status is CheckStatus.FAIL + assert "oracle run 1 exited with code 7" in oracle_max.evidence assert ( artifacts / "harbor-trials/oracle/oracle-exit-code.txt" ).read_bytes() == b"7" @@ -342,6 +382,7 @@ def test_sandbox_log_sync_cannot_erase_trusted_oracle_exit(task, boundary, tmp_p artifacts = tmp_path / "overwritten-exit" report = run_validation(task, artifacts_dir=artifacts) assert report.verdict is Verdict.FAIL + assert result(report, "runtime.startup").status is CheckStatus.ERROR native = json.loads((artifacts / "harbor-evidence.json").read_text()) assert native["trials"][1]["oracle_exit_code"] == 7 assert ( @@ -358,7 +399,7 @@ def test_excluded_symlink_tree_is_never_dereferenced_or_copied( excluded.mkdir() (excluded / "payload").symlink_to(sentinel) assert run_validation(task).verdict is Verdict.WARN - assert boundary.excluded_copied == [False, False] + assert boundary.excluded_copied == [False, False, False] def test_copy_change_and_restore_cannot_escape_source_binding( @@ -394,11 +435,13 @@ def copying(*args, **kwargs): assert "validated source digest" in native["trials"][1]["reason"] -def test_missing_oracle_blocks_without_starting_trial(task, boundary): +def test_missing_oracle_fails_oracle_max_without_starting_trial(task, boundary): (task / "solution/solve.sh").unlink() report = run_validation(task) assert report.verdict is Verdict.FAIL - assert result(report, "runtime.startup").status is CheckStatus.FAIL + # RFC 008: no oracle is a graded FAIL, not a SKIP. + assert result(report, "semantic.oracle_max").status is CheckStatus.FAIL + assert result(report, "runtime.startup").status is CheckStatus.SKIP assert not boundary.trials @@ -536,7 +579,8 @@ def docker(*args): def test_static_and_skip_build_do_not_start_trials(task, boundary): - assert run_validation(task, max_level=Level.STATIC).verdict is Verdict.PASS + static = run_validation(task, max_level=Level.STATIC) + assert result(static, "static.manifest").status is CheckStatus.PASS report = run_validation(task, skip_build=True) assert result(report, "runtime.startup").status is CheckStatus.SKIP assert not boundary.trials @@ -563,3 +607,83 @@ def test_malformed_toml_is_manifest_failure(task, boundary): report = run_validation(task) assert result(report, "static.manifest").status is CheckStatus.FAIL assert not boundary.trials + + +def test_replay_divergence_fails_determinism_and_skips_semantics(task, boundary): + boundary.rewards["oracle-replay"] = b'{"reward":0.5}' + report = run_validation(task) + assert report.verdict is Verdict.FAIL + assert result(report, "runtime.episode_determinism").status is CheckStatus.FAIL + for check_id in ("semantic.oracle_max", "semantic.floor_gap"): + skipped = result(report, check_id) + assert skipped.status is CheckStatus.SKIP + assert skipped.evidence == ["unmet dependencies: runtime.episode_determinism"] + + +@pytest.fixture +def canaries(task): + directory = task / "validation/canaries" + directory.mkdir() + (directory / "a-empty.sh").write_bytes(b"#!/bin/sh\ntrue\n") + (directory / "b-hardcoded.sh").write_bytes(b"#!/bin/sh\necho 42 > /app/out\n") + contract = json.loads((task / "validation/openenv.json").read_text()) + contract["canaries"] = "validation/canaries" + (task / "validation/openenv.json").write_text(json.dumps(contract)) + return directory + + +def test_canaries_replace_the_solution_and_must_stay_at_the_floor( + task, boundary, canaries, tmp_path +): + boundary.rewards.update({"canary-1": b'{"reward":0}', "canary-2": b'{"reward":1}'}) + artifacts = tmp_path / "canaries" + report = run_validation(task, artifacts_dir=artifacts) + assert report.verdict is Verdict.FAIL + canary = result(report, "semantic.canary_floor") + assert canary.status is CheckStatus.FAIL + assert canary.evidence == [ + "canary validation/canaries/b-hardcoded.sh earned 1.0, " + "above the do-nothing reward 0.0" + ] + by_role = {trial.role: trial for trial in boundary.trials} + assert list(by_role) == ["nop", "oracle", "oracle-replay", "canary-1", "canary-2"] + assert by_role["oracle"].solve == (task / "solution/solve.sh").read_bytes() + assert by_role["canary-1"].solve == (canaries / "a-empty.sh").read_bytes() + assert by_role["canary-2"].solve == (canaries / "b-hardcoded.sh").read_bytes() + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + overlay = native["trials"][4]["task_overlay"]["solution/solve.sh"] + assert overlay["canary"] == "validation/canaries/b-hardcoded.sh" + + +def test_canaries_at_the_floor_pass(task, boundary, canaries): + boundary.rewards.update({"canary-1": b'{"reward":0}', "canary-2": b'{"reward":0}'}) + report = run_validation(task) + assert result(report, "semantic.canary_floor").status is CheckStatus.PASS + + +def test_canaries_run_only_at_the_semantic_level(task, boundary, canaries): + report = run_validation(task, max_level=Level.RUNTIME) + assert [trial.role for trial in boundary.trials] == [ + "nop", + "oracle", + "oracle-replay", + ] + assert result(report, "runtime.episode_determinism").status is CheckStatus.PASS + assert not any(check.check_id.startswith("semantic.") for check in report.results) + + +@pytest.mark.parametrize("declared", ["../outside", "/abs", "validation/missing"]) +def test_invalid_canaries_declaration_is_manifest_failure(task, boundary, declared): + contract = json.loads((task / "validation/openenv.json").read_text()) + contract["canaries"] = declared + (task / "validation/openenv.json").write_text(json.dumps(contract)) + report = run_validation(task) + assert result(report, "static.manifest").status is CheckStatus.FAIL + assert not boundary.trials + + +def test_canaries_directory_holds_only_scripts(task, boundary, canaries): + (canaries / "notes.txt").write_text("not a canary") + report = run_validation(task) + assert result(report, "static.manifest").status is CheckStatus.FAIL + assert not boundary.trials diff --git a/tests/test_validation/test_semantic_graders.py b/tests/test_validation/test_semantic_graders.py new file mode 100644 index 000000000..d0a93b1f7 --- /dev/null +++ b/tests/test_validation/test_semantic_graders.py @@ -0,0 +1,94 @@ +"""Semantic graders compare binding-measured rewards with the declared contract.""" + +from fractions import Fraction +from pathlib import Path + +import pytest +from openenv.validation.graders import Subject +from openenv.validation.graders.semantic import ( + CanaryFloorGrader, + FloorGapGrader, + OracleMaxGrader, +) +from openenv.validation.manifest import NormalizedManifest +from openenv.validation.runtime.contracts import OracleEvidence +from openenv.validation.types import CheckStatus + +FIXTURE = Path(__file__).parent.parent / "fixtures/validation/served_min_pass" + + +def subject(evidence, **reward): + manifest = NormalizedManifest.model_validate_json( + (FIXTURE / "normalized_manifest.json").read_text() + ) + manifest = manifest.model_copy( + update={"reward": manifest.reward.model_copy(update=reward)} + ) + return Subject( + root=FIXTURE, + manifest=manifest, + image_ref=None, + running=None, + outputs_dir=FIXTURE, + oracle_evidence=evidence, + ) + + +def evidence(floor="0", oracle=("1", "1"), exits=(0, 0), canaries=()): + return OracleEvidence( + floor=Fraction(floor), + oracle=tuple(Fraction(value) for value in oracle), + oracle_exit_codes=exits, + canaries=tuple((name, Fraction(value)) for name, value in canaries), + ) + + +@pytest.mark.parametrize( + "oracle,exits,tolerance,status", + [ + (("1", "1"), (0, 0), 0.0, CheckStatus.PASS), + (("0.95", "0.95"), (0, 0), 0.05, CheckStatus.PASS), + (("0.95", "0.95"), (0, 0), 0.0, CheckStatus.FAIL), + (("1", "1"), (3, 3), 0.0, CheckStatus.FAIL), + ], +) +def test_oracle_max(oracle, exits, tolerance, status): + found = OracleMaxGrader().run( + subject(evidence(oracle=oracle, exits=exits), oracle_tolerance=tolerance) + ) + assert found.status is status + assert (found.remediation is None) is (status is CheckStatus.PASS) + + +@pytest.mark.parametrize( + "floor,oracle,status", + [ + ("0", ("1", "1"), CheckStatus.PASS), + # Doing nothing earns the maximum: the motivating defect class. + ("1", ("1", "1"), CheckStatus.FAIL), + # The oracle barely beats doing nothing, though max - floor clears the margin. + ("0.4", ("0.5", "0.5"), CheckStatus.FAIL), + # Exact rationals: a tiny floor never rounds into the margin. + ("1e-300", ("0.5", "0.5"), CheckStatus.FAIL), + ], +) +def test_floor_gap(floor, oracle, status): + found = FloorGapGrader().run(subject(evidence(floor=floor, oracle=oracle))) + assert found.status is status + + +def test_canaries_must_not_earn_more_than_doing_nothing(): + grader = CanaryFloorGrader() + held = grader.run(subject(evidence(canaries=[("empty.sh", "0")]))) + assert held.status is CheckStatus.PASS + leaked = grader.run(subject(evidence(canaries=[("hardcoded.sh", "0.25")]))) + assert leaked.status is CheckStatus.FAIL + assert "hardcoded.sh" in leaked.evidence[0] + # An empty run list must not pass vacuously. + assert grader.run(subject(evidence())).status is CheckStatus.FAIL + + +def test_missing_evidence_skips(): + for grader in (OracleMaxGrader(), FloorGapGrader(), CanaryFloorGrader()): + assert grader.run(subject(None)).status is CheckStatus.SKIP + assert grader.depends_on == ("runtime.episode_determinism",) From 5e5c342cf6de895a781442492d45ec30c3d9d14d Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 17:33:18 +0100 Subject: [PATCH 04/40] feat(harbor): let trusted callers hook a trial before it runs run_rollout and run_batch accept on_trial_created, a trusted callback invoked with the native Harbor trial after Trial.create and before trial.run(). An exception fails the rollout before execution, so a caller can verify the trial's image and settings first. Both functions also forward environment_kwargs to Harbor's environment config; run_batch forwards agent_timeout_sec and agent_step_limit to run_rollout. None of these is exposed through MCP. The defaults keep current behaviour. Ported from bf0b78ee on the Hub validation pipeline branch: these two modules and their test only. Co-Authored-By: Claude Opus 5.5 --- src/openenv/harbor/rollout.py | 22 ++ src/openenv/harbor/runner.py | 18 ++ .../envs/test_harbor_validated_trial_hook.py | 239 ++++++++++++++++++ 3 files changed, 279 insertions(+) create mode 100644 tests/envs/test_harbor_validated_trial_hook.py diff --git a/src/openenv/harbor/rollout.py b/src/openenv/harbor/rollout.py index a8d5eb2ff..496acc4d7 100644 --- a/src/openenv/harbor/rollout.py +++ b/src/openenv/harbor/rollout.py @@ -251,12 +251,17 @@ def build_trial_config( force_build: bool = False, training: bool = False, harness_profile: str | None = None, + environment_kwargs: dict[str, Any] | None = None, ) -> Any: """Assemble Harbor's `TrialConfig` for one rollout. The seam decides how this particular agent learns about the proxy — an env var, a constructor kwarg, or a config file written by a subclass. That is the only per-agent knowledge involved; everything downstream is identical. + + Args: + environment_kwargs (`dict[str, Any]`, *optional*): + Backend kwargs passed to Harbor. `None` keeps Harbor's defaults. """ from harbor.models.trial.config import ( AgentConfig, @@ -312,6 +317,9 @@ def build_trial_config( ), delete=not keep_sandbox, force_build=force_build, + **( + {"kwargs": environment_kwargs} if environment_kwargs is not None else {} + ), ), verifier=VerifierConfig(), trial_name=trial_name, @@ -344,6 +352,8 @@ async def run_rollout( eval_sampling: dict[str, Any] | None = None, harness_profile: str | None = None, on_session_created: Callable[[str], None] | None = None, + on_trial_created: Callable[[Any], None] | None = None, + environment_kwargs: dict[str, Any] | None = None, ) -> HarborRolloutResult: """Run one rollout end to end. Never raises. @@ -386,6 +396,11 @@ async def run_rollout( on_session_created (`Callable[[str], None]`, *optional*): Receive this rollout's capture session ID before the agent starts. Live views must follow this ID rather than discover unrelated sessions from the shared registry. + on_trial_created (`Callable[[Any], None]`, *optional*): + Trusted infrastructure callback invoked with the native Harbor trial before it runs. + Exceptions fail the rollout and prevent trial execution. Not exposed through MCP. + environment_kwargs (`dict[str, Any]`, *optional*): + Backend kwargs passed to Harbor. `None` keeps Harbor's defaults. Returns: [`HarborRolloutResult`]: Reward, per-turn token ids and logprobs, and validation findings. @@ -488,6 +503,7 @@ async def run_rollout( force_build=force_build, training=sampling is not None, harness_profile=harness_profile, + environment_kwargs=environment_kwargs, ) # Held across construction because Harbor's wrappers read `os.environ` while building the @@ -503,6 +519,8 @@ async def run_rollout( if not proc_env: config = build_trial_config(**config_kwargs) trial = await Trial.create(config) + if on_trial_created is not None: + on_trial_created(trial) trial_result = await trial.run() else: # The override has to span `run()`, not just construction. Harbor's goose wrapper reads @@ -535,6 +553,8 @@ async def run_rollout( with ctx_env_overlay(proc_env): config = build_trial_config(**config_kwargs) trial = await Trial.create(config) + if on_trial_created is not None: + on_trial_created(trial) trial_result = await trial.run() else: # Fallback: mutate the real environment under a lock, one rollout at a time. @@ -543,6 +563,8 @@ async def run_rollout( with process_env(seam.name, proc_env): config = build_trial_config(**config_kwargs) trial = await Trial.create(config) + if on_trial_created is not None: + on_trial_created(trial) trial_result = await trial.run() finally: _PROC_ENV_LOCK.release() diff --git a/src/openenv/harbor/runner.py b/src/openenv/harbor/runner.py index 93225a361..aea527625 100644 --- a/src/openenv/harbor/runner.py +++ b/src/openenv/harbor/runner.py @@ -11,6 +11,7 @@ import os import secrets from pathlib import Path +from typing import Any, Callable from openenv.core.harness.capture import CaptureServer @@ -44,6 +45,10 @@ async def run_batch( auth_header: str = "Authorization", provider: str = "openai", admin_key: str | None = None, + agent_timeout_sec: float | None = None, + agent_step_limit: int | None = None, + on_trial_created: Callable[[Any], None] | None = None, + environment_kwargs: dict[str, Any] | None = None, ) -> list[HarborRolloutResult]: """Run `task_indices` from `dataset` and print a per-rollout report. @@ -64,6 +69,15 @@ async def run_batch( Key the capture proxy's session-management routes require. Defaults to `$OPENENV_CAPTURE_ADMIN_KEY`, else a random one minted for this batch. The proxy is published on a public URL, so these routes are never left open. + agent_timeout_sec (`float`, *optional*): + Override each agent's execution timeout in seconds. + agent_step_limit (`int`, *optional*): + Limit each rollout's agent steps and captured model calls. + on_trial_created (`Callable[[Any], None]`, *optional*): + Trusted infrastructure callback invoked with each native Harbor trial before it runs. + Exceptions fail that rollout and prevent trial execution. Not exposed through MCP. + environment_kwargs (`dict[str, Any]`, *optional*): + Backend kwargs passed to Harbor. `None` keeps Harbor's defaults. Returns: `list[HarborRolloutResult]`: One per index, in order. @@ -163,9 +177,13 @@ async def run_batch( # training path (`run_rollout`'s default) still refuses to pick a signal. require_reward=False, keep_sandbox=keep_sandbox, + agent_timeout_sec=agent_timeout_sec, + agent_step_limit=agent_step_limit, force_build=force_build, capture_level=capture_level, inference=capture.inference, + on_trial_created=on_trial_created, + environment_kwargs=environment_kwargs, ) results.append(result) print(" " + _summarise(result)) diff --git a/tests/envs/test_harbor_validated_trial_hook.py b/tests/envs/test_harbor_validated_trial_hook.py new file mode 100644 index 000000000..e338cf15a --- /dev/null +++ b/tests/envs/test_harbor_validated_trial_hook.py @@ -0,0 +1,239 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Trusted trial hooks run before agent execution, without model or sandbox calls.""" + +from __future__ import annotations + +import asyncio +import os +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest + +rollout = pytest.importorskip("openenv.harbor.rollout") +runner = pytest.importorskip("openenv.harbor.runner") +trial_module = pytest.importorskip("harbor.trial.trial") + +from openenv.core.harness.capture.sessions import SessionRegistry +from openenv.harbor import proc_env_context, shared_template +from openenv.harbor.models import HarborRolloutResult + +ENVIRONMENT_KWARGS = {"auto_mounting": False, "keep_containers": True} + + +@pytest.mark.parametrize("environment_kwargs", [None, {}, ENVIRONMENT_KWARGS]) +@pytest.mark.parametrize("keep_sandbox", [False, True]) +def test_trial_config_environment_kwargs( + monkeypatch, tmp_path, environment_kwargs, keep_sandbox +): + monkeypatch.delenv("OPENENV_HARBOR_AGENT_VERSIONS", raising=False) + options = ( + {} if environment_kwargs is None else {"environment_kwargs": environment_kwargs} + ) + config = rollout.build_trial_config( + task_dir=tmp_path, + harness="opencode", + sandbox="docker", + intercept_url="http://capture.invalid", + session_id="test", + model="m", + trial_name="test", + trials_dir=tmp_path, + keep_sandbox=keep_sandbox, + **options, + ) + + assert config.environment.kwargs == (environment_kwargs or {}) + assert ("kwargs" in config.environment.model_fields_set) is ( + environment_kwargs is not None + ) + assert config.environment.delete is (not keep_sandbox) + + +@pytest.mark.parametrize("credential_path", ["agent-env", "context-env", "locked-env"]) +@pytest.mark.parametrize("hook_mode", ["attach", "reject", "omitted"]) +def test_trial_hook_precedes_execution( + monkeypatch, tmp_path, credential_path, hook_mode +): + # Restore the global proxy as well as its install state after exercising the real contexts. + monkeypatch.setattr(os, "environ", os.environ) + monkeypatch.setattr(proc_env_context, "_installed", False) + monkeypatch.setattr(proc_env_context, "_real_environ", None) + monkeypatch.setenv( + "OPENENV_CONCURRENT_PROC_ENV", "0" if credential_path == "locked-env" else "1" + ) + env_key = "OPENENV_TRIAL_HOOK_TEST_KEY" + monkeypatch.setenv(env_key, "baseline") + proc_env = {} if credential_path == "agent-env" else {env_key: "trial-value"} + expected_env = proc_env.get(env_key, "baseline") + seam = SimpleNamespace( + name="fake", + resolve=lambda **kwargs: ("m", {}, {}, proc_env), + ) + monkeypatch.setattr(rollout.seams, "get", lambda *args, **kwargs: seam) + monkeypatch.setattr(shared_template, "enable_shared_templates", lambda: False) + config = object() + build = Mock(return_value=config) + monkeypatch.setattr(rollout, "build_trial_config", build) + monkeypatch.setattr( + rollout, + "export_session", + lambda *args, **kwargs: {"stats": {}, "sequences": [], "turns": []}, + ) + monkeypatch.setattr(rollout, "load_trace", lambda path: (None, "")) + monkeypatch.setattr( + rollout, + "reconcile", + lambda *args: SimpleNamespace(ok=True, findings=[], aux_node_ids=[]), + ) + events = [] + owned_trials = [] + agent_run = AsyncMock() + + class Trial: + guarded = False + + @classmethod + async def create(cls, trial_config): + assert trial_config is config + assert os.environ[env_key] == expected_env + events.append("create") + return trial + + async def run(self): + assert os.environ[env_key] == expected_env + assert hook_mode == "omitted" or self.guarded + events.append("run") + await agent_run() + return SimpleNamespace( + verifier_result=SimpleNamespace(rewards={"reward": 1.0}) + ) + + trial = Trial() + monkeypatch.setattr(trial_module, "Trial", Trial) + + def on_trial_created(created): + assert created is trial + assert os.environ[env_key] == expected_env + events.append("hook") + owned_trials.append(created) + if hook_mode == "reject": + raise ValueError("image guard rejected trial") + created.guarded = True + + registry = SessionRegistry() + options = ( + {} + if hook_mode == "omitted" + else { + "on_trial_created": on_trial_created, + "environment_kwargs": ENVIRONMENT_KWARGS, + } + ) + result = asyncio.run( + rollout.run_rollout( + task_dir=tmp_path, + harness="fake", + sandbox="docker", + registry=registry, + intercept_url="http://capture.invalid", + model="m", + trials_dir=tmp_path, + purpose="eval", + capture_level="text", + agent_timeout_sec=12.5, + agent_step_limit=3, + **options, + ) + ) + + assert build.call_args.kwargs["agent_timeout_sec"] == 12.5 + assert build.call_args.kwargs["agent_step_limit"] == 3 + assert build.call_args.kwargs["environment_kwargs"] is ( + None if hook_mode == "omitted" else ENVIRONMENT_KWARGS + ) + assert registry.list_ids() == [] + assert os.environ[env_key] == "baseline" + assert not rollout._PROC_ENV_LOCK.locked() + assert owned_trials == ([] if hook_mode == "omitted" else [trial]) + if hook_mode == "reject": + assert events == ["create", "hook"] + agent_run.assert_not_awaited() + assert result.ok is False + assert result.reward is None + assert result.error == "image guard rejected trial" + assert result.exception_type == "ValueError" + else: + assert events == ( + ["create", "run"] if hook_mode == "omitted" else ["create", "hook", "run"] + ) + agent_run.assert_awaited_once() + assert result.ok is True + assert result.reward == 1.0 + + +@pytest.mark.parametrize("bounded", [False, True]) +def test_batch_forwards_trial_hook_and_agent_budgets(monkeypatch, tmp_path, bounded): + monkeypatch.setattr( + "openenv.harbor.startup.prepare", + lambda **kwargs: SimpleNamespace( + llm={"model": "m", "capture_level": "text"}, + available_sandboxes=("docker",), + ), + ) + monkeypatch.setattr( + runner, "resolve_task_dirs", lambda dataset: [tmp_path, tmp_path] + ) + capture = SimpleNamespace( + registry=object(), inference=None, start=Mock(), stop=Mock() + ) + monkeypatch.setattr(runner, "CaptureServer", lambda **kwargs: capture) + forwarder = SimpleNamespace( + name="direct", start=Mock(return_value="http://capture.invalid"), stop=Mock() + ) + monkeypatch.setattr( + "openenv.core.harness.capture.forwarding.make_forwarder", + lambda expose: forwarder, + ) + run = AsyncMock(return_value=HarborRolloutResult(rollout_type="eval")) + monkeypatch.setattr(runner, "run_rollout", run) + hook = Mock() + options = ( + { + "on_trial_created": hook, + "agent_timeout_sec": 12.5, + "agent_step_limit": 3, + "environment_kwargs": ENVIRONMENT_KWARGS, + } + if bounded + else {} + ) + + results = asyncio.run( + runner.run_batch( + llm_url="http://llm.invalid/v1", + dataset="local", + task_indices=[0, 1], + sandbox="docker", + expose="direct", + trials_dir=tmp_path, + **options, + ) + ) + + assert len(results) == 2 + assert run.await_count == 2 + for call in run.await_args_list: + assert call.kwargs["on_trial_created"] is (hook if bounded else None) + assert call.kwargs["agent_timeout_sec"] == (12.5 if bounded else None) + assert call.kwargs["agent_step_limit"] == (3 if bounded else None) + assert call.kwargs["environment_kwargs"] is ( + ENVIRONMENT_KWARGS if bounded else None + ) + capture.stop.assert_called_once() + forwarder.stop.assert_called_once() From 04ace6d3615fb607e6e4a51c2f3da078fa247237 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 17:36:35 +0100 Subject: [PATCH 05/40] docs(validation): document the Harbor task binding and remaining RFC 008 work Replace the RFC 008 pilot amendment with the harbor-semantic-v1 binding: trial order, image pinning, the check-id mapping, canaries and the floor-gap clarification. Update the CLI reference and command help. Add a plan that records where validation stands and ranks the remaining workstreams. Co-Authored-By: Claude Opus 5.5 --- docs/plans/rfc008-validation-completion.md | 302 +++++++++++++++++++++ docs/source/reference/cli.md | 21 +- rfcs/008-environment-auto-validation.md | 94 ++++--- src/openenv/cli/commands/validate.py | 5 +- 4 files changed, 377 insertions(+), 45 deletions(-) create mode 100644 docs/plans/rfc008-validation-completion.md diff --git a/docs/plans/rfc008-validation-completion.md b/docs/plans/rfc008-validation-completion.md new file mode 100644 index 000000000..85099b024 --- /dev/null +++ b/docs/plans/rfc008-validation-completion.md @@ -0,0 +1,302 @@ +# Completing RFC 008 validation + +Planning baseline: 2026-11-10, `openenv/OpenEnv` main at +`fb36d7d62ff9548ebd44f3323386cbea908e8706`. + +Trackers: [RFC 008 #778](https://github.com/openenv/OpenEnv/issues/778), +[Level 2 umbrella #1177](https://github.com/openenv/OpenEnv/issues/1177) and +[task packages #898](https://github.com/openenv/OpenEnv/issues/898). + +RFC 008 promises that `openenv validate` grades an environment against the 44 +acceptance tests of #778. At the baseline, main ran every Level 2 runtime check for +served environments on Docker, but no Level 3 semantic check. That includes the +check the RFC opens with: a correct solution must earn maximum reward, and doing +nothing must not. The PR that adds this document delivers that check for Harbor +task packages and makes every report name the checks it did not run. This document +records where validation stands after that PR, defines what "complete" means, and +ranks the remaining work as PR-sized workstreams. + +## Where validation stands + +Each RFC check has a stable id, a level and a lane. The lane says who can run the +check. `local` checks run inside `openenv validate`. `hub` and `statistical` ids are +reserved for operators, the teams that run validation hubs or lab intake. The table +counts the local ids in the committed severity policy and the ids each execution +binding grades. + +| Level | Local ids | Served packages (`openenv_ws`) | Harbor tasks (`harbor_task`) | +|---|---|---|---| +| 1 Static | 10 | 1 (`static.manifest`) | 1 (`static.manifest`) | +| 2 Runtime | 16 | 16, on Docker-local | 3: startup, reward well-formedness, episode determinism | +| 3 Semantic | 8 | 0 | 3: oracle reaches max, floor gap, canary floor | +| 4 Statistical | 0 (7 reserved) | Operator-side by design | Operator-side by design | +| Hub | 0 (5 reserved) | Operator-side by design | Operator-side by design | + +Every requested local check that does not run is a named SKIP with its reason, at +every level. A report therefore reads WARN until all requested checks pass, and a +PASS on a subset cannot pass for full coverage. + +Three limits apply across bindings: + +- **Parsers.** `openenv.yaml` and Harbor `task.toml` ship. PostTrain `task.md` + packages are refused as unrecognized. +- **Providers.** Docker-local enforces `public` and `no-network` networking and + refuses allowlists and GPUs. The HF Sandbox provider refuses every launch with + `missing provider capability: immutable_image`, because nothing yet binds a Space + revision to an image digest. +- **Served oracles.** RFC 008 does not yet say how to execute an oracle against a + served WebSocket environment, so served packages report their semantic checks as + named SKIPs. + +Work outside main measures more than the RFC's local checks. A cloud audit on HF +Jobs adds generated reference solutions, wrong-answer controls, repeat runs, +leakage probes and target-model pass rates. +[#1582](https://github.com/openenv/OpenEnv/pull/1582) adds training and validator +performance experiments. These results report under their own fields rather than +RFC check ids, so they do not count toward RFC coverage yet. + +## What complete means + +The RFC splits responsibility between the repo and operators, so completion has two +targets. The repo can reach the first one alone. The second one needs an operator +procedure that the repo documents but does not run. + +**Local completeness.** Every local-lane id has a grader for each shipped execution +binding where it applies, and reports name every inapplicable or unimplemented id +with its reason. For served packages this means nine static graders, an oracle +execution contract and eight semantic graders. For Harbor tasks it means the same +static graders, the remaining semantic graders, and a decision on which runtime ids +apply to one-shot tasks. + +**Operator completeness.** Each reserved `statistical.*` id has at least one +documented reference procedure that emits results in the shared report schema. The +repo still ships no hub, as the RFC requires, so the procedure lives in an example or +an operator repository. Comparable operator results also need the metric namespace +(`pass@k`, headroom, variance bound) that the RFC defers to a future RFC. + +Validation still certifies that an environment can be trained on, not that it should +be. Measuring training signal comes last, and it belongs with the evaluation work in +[RFC 013 (#1256)](https://github.com/openenv/OpenEnv/pull/1256). + +## Proposed workstreams + +The workstreams are ranked by dependency and by how much each one improves +environment selection. Each one is one to three PRs. Costs are relative: S needs no +build or inference, M needs Docker builds and replays, and L needs model inference +or GPU time. Owners are suggestions for discussion. + +| Rank | Workstream | #778 tests | Status | Cost | Suggested owner | +|---|---|---|---|---|---| +| 1 | Verifier sanity graders | #11, #25 | Harbor done; envelope and served open | M | Validation core | +| 2 | Harbor binding on main | Harbor parser slice | Done | M | Validation core | +| 3 | Verifier strength | #28, #26 procedure | Harbor canaries done; panel open | M, then L | Validation core and Hub pipeline | +| 4 | Determinism, replay and leakage graders | #30, #39, #40, #44 | Open | M | Validation core | +| 5 | Static graders and complete reports | #1, #3, #4, #8, #9, #10, #12, #41, #42 | Reports done; graders open | S to M | Any contributor | +| 6 | Cloud execution on HF Sandbox | Provider qualification | Open | M | Hub pipeline | +| 7 | Statistical reference procedure | #13 to #16, #27 | Open | L | Hub pipeline | +| 8 | Wider task shapes | Workspace handoff, multi-container, credentials | Open | M to L | Hub pipeline and validation core | +| 9 | Training signal | #17 | Open | L | Validation experiments and RFC 013 | + +### 1. Verifier sanity graders + +`semantic.oracle_max` and `semantic.floor_gap` now grade Harbor tasks. The graders +read a format-neutral `OracleEvidence`, so another binding only has to measure the +rewards. + +**Remaining.** Served packages need an oracle execution contract first: the RFC +names injected state and executable scripts, but not how either drives a served +WebSocket session or what doing nothing means there. That needs an RFC amendment. +`semantic.resource_envelope` also remains: record peak CPU, memory and disk during +the oracle run and compare them with the declared resources. + +**Evidence.** Oracle and do-nothing rewards, the declared margin and tolerance, the +image digest and the cleanup outcome. + +**Cost and owner.** M: Docker builds and short episodes, with no inference. +Validation core. + +### 2. Harbor binding on main + +Done. Harbor `task.toml` packages validate through the `harbor-semantic-v1` +profile and an operator-supplied `validation/openenv.json` contract. Fresh Docker +trials run a do-nothing agent, the oracle, an oracle replay and declared canaries, +all pinned to one immutable image. + +**Remaining.** Decide which runtime ids apply to one-shot tasks. Containment, +network policy, resource bounds and oracle containment could apply. Observation +schema, state contract and tool declaration accuracy do not. + +**Cost and owner.** M. Validation core. + +### 3. Verifier strength: canaries and an adversarial panel + +A verifier can pass the oracle and floor checks and still reward wrong answers. +`semantic.canary_floor` now runs operator-declared canaries on Harbor tasks: each +known-wrong `solve.sh` replacement must earn no more than doing nothing. + +**Remaining.** The reserved `statistical.adversarial_floor` (#26) still has no +procedure; the RFC notes that the panel "is not yet specified anywhere". Write an +RFC 008 amendment that specifies: + +- where wrong answers come from: mutations of the reference solution, known-fault + corpora, empty and partial outputs, and plausible solutions from a weaker model + that an independent check confirms are wrong; +- how many are needed for a verdict; +- the pass rule, which is that every wrong answer scores at the floor. + +The cloud audit's wrong-answer controls are a working prototype of this panel. + +**Evidence.** Each wrong answer's provenance and reward, plus the verifier version. + +**Cost and owner.** L for generated panels, which need inference. Validation core +writes the amendment, and the Hub pipeline team implements the operator panel. + +### 4. Determinism, replay and leakage graders + +`semantic.verifier_determinism`, `semantic.verifier_portability`, +`semantic.replayability` and `semantic.no_solution_leakage` have no graders. +Harbor tasks now establish `runtime.episode_determinism` with an oracle replay in a +fresh container, which these checks depend on. + +**Approach.** Verifier determinism runs the verifier on one final state several +times and requires identical rewards. For `llm_judged` environments it requires +variance within the declared bound, reusing the Level 2 variance path. Portability +reruns the verifier with networking denied, as the RFC's hermeticity rule for #40 +requires. Replayability re-executes a recorded trajectory and compares rewards. The +leakage check scans observations and agent-visible files for oracle artifacts such +as solution text, expected outputs and test files. + +**Evidence.** Replay transcripts and reward vectors, the network mode of each +verifier run, and every leakage match with its location. + +**Cost and owner.** M. Validation core. + +### 5. Static graders and complete reports + +Reports now name every unimplemented static check as a SKIP, so a static run reads +WARN rather than PASS after one check. + +**Remaining.** Add the nine graders in two batches. The first batch reads the +manifest and lockfiles and needs no build: `static.dependency_pinning`, +`static.task_distribution_pinning`, `static.resource_declaration` and +`static.timeout_ceiling`. The second batch inspects built images: +`static.oci_labels`, `static.sbom`, `static.image_hygiene`, `static.layout` and +`static.reproducible_build`, which builds twice and compares digests. + +**Evidence.** The files and image metadata each grader read. + +**Cost and owner.** S for the first batch and M for the image batch. Any +contributor; these make good first issues. + +### 6. Cloud execution on HF Sandbox + +The HF Sandbox provider refuses every launch because it has no immutable image +binding. Runtime and semantic checks therefore cannot run in the cloud, or on GPUs, +through the RFC provider interface. Cloud audits work around this with their own +runner. + +**Approach.** Resolve a Space revision to the registry digest of the image it +builds, verify it, and launch by digest. Qualify public Space images first, then add +an authenticated pull path for private ones. Once launches work, the cloud audit can +call the RFC graders instead of its own runner. + +**Evidence.** The Space revision, the resolved digest, the pull receipt and the same +cleanup evidence Docker-local records. + +**Cost and owner.** M. Hub pipeline team. + +### 7. Statistical reference procedure + +The learnable tests (#13 to #16) and the gameability gap (#27) are reserved ids with +no reference procedure. The RFC also defers the shared metric namespace. + +**Approach.** Write the metric-namespace RFC, with definitions, sample sizes and +confidence intervals for `pass@k`, headroom and variance bound. Then publish a +reference operator procedure, outside the core package, that maps target-model +attempts onto the reserved ids: + +- `statistical.reward_reachability`: some attempt reaches maximum reward; +- `statistical.difficulty_separation`: pass rates differ across model tiers; +- `statistical.headroom`: the target model's pass rate leaves room to improve; +- `statistical.variance_bound`: reward noise across repeats stays within a bound. + +The cloud audit already measures target-model pass rates with confidence intervals. +It needs to emit them under these ids in hub-lane reports. + +**Evidence.** Model identity and provider, attempt counts, per-attempt rewards and +intervals. + +**Cost and owner.** L, since every task needs inference. Hub pipeline team, with the +metric-namespace RFC from validation core. + +### 8. Wider task shapes + +The Harbor binding refuses multi-step tasks, task-authored Docker Compose files and +tasks that need credentials. The cloud audit also hands off only individual output +files. The RFC defers credential delivery explicitly. + +**Approach.** Add a workspace or git-patch handoff so repository tasks are graded on +the agent's full change. Treat multi-container support as a provider capability, as +the RFC does for GPUs, so providers without it SKIP with the capability named. Write +the credential-delivery amendment the RFC asks for, covering delivery, access +isolation, lifetime, network access and evidence redaction. + +**Evidence.** Handoff manifests with hashes, and provider capability reports. + +**Cost and owner.** M for workspace handoff and L for multi-container and +credentials. The Hub pipeline team builds the adapters, and validation core writes +the amendment. + +### 9. Training signal + +`statistical.training_signal` (#17, advisory) asks whether training on an environment +improves a model. Nothing measures it yet. + +**Approach.** Use the evaluation harness proposed in RFC 013 to run short training +jobs on a sample of validated environments, then report the change in held-out +success. The training experiments in #1582 are a first measurement. Keep the result +advisory, as the RFC's severity policy says, because a short run cannot show that an +environment is unlearnable. + +**Evidence.** Training configuration, seeds, and held-out success before and after +training. + +**Cost and owner.** L: GPU time. Validation experiments, with the RFC 013 authors. + +## Sequencing + +The verifier sanity graders came first because canaries, verifier determinism and +replay all reuse oracle replay. The order of the remaining work follows from its +dependencies: + +- Workstream 4 builds on the oracle replay the Harbor binding now runs. +- Workstream 5 is independent and can start now. +- Workstream 6 moves the Harbor and served checks to the cloud and is needed before + workstream 7 runs at scale. +- The served half of workstream 1 waits on the oracle execution amendment. +- Workstreams 8 and 9 widen coverage after the core lands. + +## Open questions for maintainers + +These questions change the scope of a workstream, so they need an answer before its +first PR: + +1. How should an oracle drive a served WebSocket environment, and what is its + do-nothing episode? +2. Should `semantic.canary_floor` also accept canaries the validator generates, or + only canaries shipped with the package? +3. Should the adversarial panel be an RFC 008 amendment or its own RFC? +4. Where should operator reference procedures live: `examples/` or a separate + repository? +5. Should the runtime applicability table for one-shot task packages be normative + in the RFC? + +## Sources + +- `rfcs/008-environment-auto-validation.md`, including the Harbor task binding + section +- `docs/plans/rfc008-level2-implementation.md` +- `tests/fixtures/validation/runtime/cases.json` +- `src/openenv/validation/runner.py`, `harbor.py`, `graders/semantic/` and + `providers/hf_sandbox.py` +- Issues #778, #898 and #1177; PRs #1256 and #1582 diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index d16ff6fc2..349f187de 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -69,13 +69,17 @@ supports that capability. Docker validation uses two containers concurrently during the fresh-container replay. Both use the same immutable image and are removed even if a check fails. -Native Harbor `task.toml` directories are also recognized. Supply an explicit -`validation/openenv.json` reward/resource contract and the optional Harbor -dependency. The `harbor-oracle-floor-v1` binding runs fresh native NOP and oracle -trials in Docker, verifies raw rewards, actual task image/configuration and cleanup, -and retains `harbor-evidence.json` in the report bundle. Other standard checks -remain named SKIPs, so a passing pilot still produces WARN. This does not qualify -the outer `harbor_env` server or claim full RFC 008 admission. +Harbor `task.toml` directories are also recognized. Install the `harbor` extra +and add a `validation/openenv.json` contract that declares the reward key, range, +oracle tolerance, floor margin and resources. The `harbor-semantic-v1` binding runs +fresh Harbor trials in Docker: a do-nothing agent, the oracle (`solution/solve.sh`), +an oracle replay and any canaries declared in the contract. A canary is a known-wrong +`solve.sh` replacement. The results report under `runtime.startup`, +`runtime.reward_well_formed`, `runtime.episode_determinism`, `semantic.oracle_max`, +`semantic.floor_gap` and `semantic.canary_floor`, and `harbor-evidence.json` keeps +the raw trial evidence in the report bundle. Other checks are named SKIPs, so a +passing task still reports WARN. This validates task packages, not the outer +`harbor_env` server. The 16 implemented runtime checks cover startup, rewards, observation schemas, state continuity, network policy, host containment, resource bounds, episode isolation, @@ -85,7 +89,8 @@ with unsupported capabilities or incomplete evidence appear as `SKIP`; they make the result `WARN`, which exits zero and does not mean Level 2 is complete. `FAIL` exits 1, unsupported package formats exit 2, and internal or policy errors exit 3. `--level semantic` -includes the available lower-level checks but does not claim semantic execution. +runs the semantic checks for Harbor tasks. Served packages have no oracle execution +yet, so their semantic checks are named SKIPs. `--skip-build` runs declaration checks and skips runtime execution entirely. The Docker provider supports CPU workloads in `public` and `no-network` modes; `allowlist` and GPU requirements skip runtime before building. A `no-network` diff --git a/rfcs/008-environment-auto-validation.md b/rfcs/008-environment-auto-validation.md index c045febf4..dade23e1c 100644 --- a/rfcs/008-environment-auto-validation.md +++ b/rfcs/008-environment-auto-validation.md @@ -718,41 +718,65 @@ findings: a WARN exit 0 does not prove completeness. Product publish gating and operator certification remain outside this feature; the test workflow only proves the implementation's own acceptance cases. -### Native Harbor pilot binding - -`task.toml` packages may opt into the explicit `harbor-oracle-floor-v1` pilot -through an operator-supplied `validation/openenv.json` contract. The contract -declares the selected reward key, range, oracle tolerance, floor margin and finite -resource budgets; Harbor metadata and observed rewards do not supply these -declarations. The normalized schema-2 execution kind is `harbor_task`, with a -`process` agent boundary. Parsing uses Harbor's data-only Task loader and never -creates a Trial or executes package code. - -This initial profile supports single-step Linux tasks, one Dockerfile, one main -Docker service, a shared verifier, public network policies and no accelerators, -external MCP services or credentials. Native execution runs fresh NOP and oracle -Harbor Trials, inspects the actual agent container at `AGENT_START`, retains its -immutable image under a run-owned tag, and pins the oracle to that image. Source, -explicit execution overlays and effective container configuration are bound to -the evidence. Supported `delete=False` and `keep_containers=True` trial settings -leave image retention and resource removal to the native supervisor. Cleanup -independently enumerates the exact run-owned Compose projects, registers anonymous -volumes from those containers' actual mounts, removes only those owned resources, -and verifies their absence. The trusted oracle exit marker is captured at -`AGENT_END`, before later sandbox log synchronization can overwrite host logs. - -`harbor-evidence.json` and preserved raw reward/exit files are checksummed bundle -artifacts. The pilot requires finite nonboolean in-range rewards, an oracle -without a nonzero exit marker or Trial exception, oracle reward at least the -declared maximum minus tolerance, and oracle-minus-NOP and maximum-minus-NOP gaps -at least the declared floor margin. Failures or unverifiable cleanup block the -normal report. `static.manifest`, `runtime.startup` and -`runtime.reward_well_formed` may PASS with actual evidence; other requested checks -remain named SKIPs. In particular this binding supplies no synthetic WebSocket -exchange, generic semantic PASS, containment qualification or measured resource -enforcement. A WARN report is admissible only to a driver explicitly choosing this -pilot and checking its native gates; full RFC validation still rejects required -SKIPs. +### Harbor task binding + +`task.toml` packages opt into the `harbor-semantic-v1` profile through an +operator-supplied `validation/openenv.json` contract. The contract declares the +reward key, range, oracle tolerance, floor margin, finite resource budgets and, +optionally, a directory of canaries. Harbor metadata and observed rewards never +supply these declarations. The normalized schema-2 execution kind is `harbor_task`, +with a `process` agent boundary. Parsing uses Harbor's data-only Task loader and +never creates a Trial or executes package code. + +The profile supports single-step Linux tasks with one Dockerfile, one main Docker +service, a shared verifier and public networking. Accelerators, external MCP +services, task environment variables, credentials and Harbor extensions such as +healthchecks or artifacts are refused with the reason named. + +Execution runs fresh Harbor Trials in a fixed order: a do-nothing (NOP) trial, the +oracle (`solution/solve.sh`), an oracle replay, and one trial per declared canary. +The NOP trial builds the image. The validator inspects the actual agent container +at `AGENT_START`, retains its immutable image under a run-owned tag, and pins every +later trial to that image and its effective container configuration. Each trial +runs on a private copy of the task whose digest must equal the validated source +digest before any overlay. A canary trial replaces `solution/solve.sh` in its copy +with the canary script and records the script's digest. Trials run with +`delete=False` and `keep_containers=True`, so the validator removes the exact +run-owned Compose projects, including anonymous volumes registered from the +containers' actual mounts, and verifies their absence. The oracle's exit code is +read at `AGENT_END`, before sandbox log synchronization can overwrite it, and a +later change fails the run. + +Results map to RFC check ids: + +| Check | Evidence | +|---|---| +| `runtime.startup` | Every trial ran from the pinned image in a fresh container, and cleanup was verified. | +| `runtime.reward_well_formed` | Every raw reward map matches Harbor's result and is finite, nonboolean and in range. | +| `runtime.episode_determinism` | The oracle replay reproduces every reward and the exit code. | +| `semantic.oracle_max` | Both oracle runs exit 0 and earn at least `max - oracle_tolerance`. A task without `solution/solve.sh` fails. | +| `semantic.floor_gap` | `max - floor >= floor_margin`, and `oracle - floor >= floor_margin` because the oracle may sit up to the tolerance below the maximum. | +| `semantic.canary_floor` | Every declared canary earns no more than the measured floor. Selected only when canaries are declared. | + +The semantic ids depend on `runtime.episode_determinism`, so a diverging replay +SKIPs them with the dependency named. Reward comparisons use exact rationals parsed +from the raw verifier output. Every other local check is a named SKIP for this +binding. It produces no WebSocket exchange, containment or resource measurement, +second verification of one final state, or leakage scan. A passing run is therefore +WARN, and full RFC qualification still rejects these SKIPs. + +Canaries are operator-declared, known-wrong `solve.sh` replacements, at most eight +per task. They give local validation the package-shipped form of test #28. +Generating wrong answers, for example by mutating the oracle, belongs to the +adversarial panel of test #26, which remains unspecified. + +`harbor-evidence.json` records the contract digest, the Harbor version, and each +trial's role, image identity, private-copy digests, overlays, raw reward file hashes +and cleanup outcome. It and the raw reward and exit files are checksummed bundle +artifacts beside the report. + +Served `openenv_ws` packages have no oracle execution contract yet, so their +semantic checks are named SKIPs. ## Explicitly out of scope diff --git a/src/openenv/cli/commands/validate.py b/src/openenv/cli/commands/validate.py index 25a7ab8d3..b7e716014 100644 --- a/src/openenv/cli/commands/validate.py +++ b/src/openenv/cli/commands/validate.py @@ -236,8 +236,9 @@ def validate( applies the severity policy, and emits a report. Native Harbor tasks require the optional Harbor dependency and an explicit - `validation/openenv.json` contract. The oracle/floor pilot preserves unsupported - checks as SKIP; its WARN report does not establish full RFC 008 qualification. + `validation/openenv.json` contract. Fresh Docker trials grade the oracle, the + do-nothing floor and any declared canaries under the semantic check ids. Checks + without an implementation are named SKIPs, so the report is WARN at best. Exit codes: 0 pass/warn · 1 fail · 2 unrecognized/unsupported package · 3 internal or policy error. From abf12d022727c3ce87a52addf413754152c861b0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 10 Nov 2026 16:54:23 +0000 Subject: [PATCH 06/40] fix(validation): handle Harbor validation edge cases --- envs/grid_world_env/uv.lock | 2 +- pyproject.toml | 2 +- src/openenv/validation/harbor.py | 9 ++++--- .../test_validation/test_harbor_validation.py | 27 +++++++++++++++++++ tests/validation_runtime/uv.lock | 2 +- 5 files changed, 36 insertions(+), 6 deletions(-) diff --git a/envs/grid_world_env/uv.lock b/envs/grid_world_env/uv.lock index 753795aa7..896d19e19 100644 --- a/envs/grid_world_env/uv.lock +++ b/envs/grid_world_env/uv.lock @@ -1343,7 +1343,7 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.116.0" }, { name = "fastmcp", specifier = ">=3.0.0,<4.0.0" }, { name = "gradio", specifier = ">=4.0.0" }, - { name = "harbor", extras = ["e2b", "modal", "daytona", "gke", "ec2", "runloop", "novita", "blaxel", "beam", "islo", "opensandbox", "cwsandbox", "use-computer", "cua"], marker = "python_full_version >= '3.12' and extra == 'harbor'", specifier = ">=0.22.0" }, + { name = "harbor", extras = ["e2b", "modal", "daytona", "gke", "ec2", "runloop", "novita", "blaxel", "beam", "islo", "opensandbox", "cwsandbox", "use-computer", "cua"], marker = "python_full_version >= '3.12' and extra == 'harbor'", specifier = "==0.25.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "huggingface-hub", specifier = ">=1.29.0" }, { name = "inspect-ai", marker = "extra == 'inspect'", specifier = ">=0.3.0" }, diff --git a/pyproject.toml b/pyproject.toml index dde48a4e8..75e656ba8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ harbor = [ # (websockets>=15) and `tensorlake` (websockets>=13,<14) together. # # Harbor requires Python >= 3.12, so this extra is excluded from the 3.11 CI leg. - "harbor[e2b,modal,daytona,gke,ec2,runloop,novita,blaxel,beam,islo,opensandbox,cwsandbox,use-computer,cua]>=0.22.0; python_version >= '3.12'", + "harbor[e2b,modal,daytona,gke,ec2,runloop,novita,blaxel,beam,islo,opensandbox,cwsandbox,use-computer,cua]==0.25.0; python_version >= '3.12'", ] [project.scripts] diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index 50804c563..63676dd0d 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -674,6 +674,7 @@ def validate_harbor( ) evidence = None checks = [] + semantic_graded = False if not problem: if subject.manifest.capabilities.oracle is None: startup = _result( @@ -683,6 +684,7 @@ def validate_harbor( ) if semantic: # RFC 008: a missing oracle is a graded FAIL, never a SKIP. + semantic_graded = True checks.append( CheckResult( check_id="semantic.oracle_max", @@ -699,7 +701,8 @@ def validate_harbor( attempted = True try: with tempfile.TemporaryDirectory( - prefix="openenv-harbor-validation-" + prefix="openenv-harbor-validation-", + ignore_cleanup_errors=True, ) as temp: daemon, trials, files = asyncio.run( _collect( @@ -896,8 +899,8 @@ def validate_harbor( levels = [Level.STATIC] if attempted: levels.append(Level.RUNTIME) - if semantic: - levels.append(Level.SEMANTIC) + if semantic and (attempted or semantic_graded): + levels.append(Level.SEMANTIC) report = ValidationReportV2( report_schema_version="2", target=str(subject.root), diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 9e1089966..98ef394ff 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -360,6 +360,32 @@ def test_cleanup_failure_blocks(task, boundary, tmp_path): ] +def test_temp_cleanup_failure_does_not_discard_successful_trials( + task, boundary, monkeypatch +): + temporary_directory = harbor.tempfile.TemporaryDirectory + + class CleanupSensitiveTemporaryDirectory: + def __init__(self, *args, ignore_cleanup_errors=False, **kwargs): + self.directory = temporary_directory(*args, **kwargs) + self.ignore_cleanup_errors = ignore_cleanup_errors + + def __enter__(self): + return self.directory.__enter__() + + def __exit__(self, *args): + self.directory.__exit__(*args) + if not self.ignore_cleanup_errors: + raise PermissionError("container-owned trial file") + + monkeypatch.setattr( + harbor.tempfile, "TemporaryDirectory", CleanupSensitiveTemporaryDirectory + ) + report = run_validation(task) + assert result(report, "runtime.startup").status is CheckStatus.PASS + assert result(report, "semantic.oracle_max").status is CheckStatus.PASS + + def test_oracle_nonzero_exit_fails_oracle_max_and_retains_marker( task, boundary, tmp_path ): @@ -439,6 +465,7 @@ def test_missing_oracle_fails_oracle_max_without_starting_trial(task, boundary): (task / "solution/solve.sh").unlink() report = run_validation(task) assert report.verdict is Verdict.FAIL + assert report.levels_run == [Level.STATIC, Level.SEMANTIC] # RFC 008: no oracle is a graded FAIL, not a SKIP. assert result(report, "semantic.oracle_max").status is CheckStatus.FAIL assert result(report, "runtime.startup").status is CheckStatus.SKIP diff --git a/tests/validation_runtime/uv.lock b/tests/validation_runtime/uv.lock index f140d77a8..9e0422619 100644 --- a/tests/validation_runtime/uv.lock +++ b/tests/validation_runtime/uv.lock @@ -952,7 +952,7 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.116.0" }, { name = "fastmcp", specifier = ">=3.0.0,<4.0.0" }, { name = "gradio", specifier = ">=4.0.0" }, - { name = "harbor", extras = ["e2b", "modal", "daytona", "gke", "ec2", "runloop", "novita", "blaxel", "beam", "islo", "opensandbox", "cwsandbox", "use-computer", "cua"], marker = "python_full_version >= '3.12' and extra == 'harbor'", specifier = ">=0.22.0" }, + { name = "harbor", extras = ["e2b", "modal", "daytona", "gke", "ec2", "runloop", "novita", "blaxel", "beam", "islo", "opensandbox", "cwsandbox", "use-computer", "cua"], marker = "python_full_version >= '3.12' and extra == 'harbor'", specifier = "==0.25.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "huggingface-hub", specifier = ">=1.29.0" }, { name = "inspect-ai", marker = "extra == 'inspect'", specifier = ">=0.3.0" }, From 015f3aae1ff0e228a84f822724d716890584bd04 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 17:59:23 +0100 Subject: [PATCH 07/40] fix(validation): reject Harbor contract numbers that change as floats The contract reader parsed JSON numbers as floats, so a floor_margin of 0.100000000000000001 became 0.1. A measured gap of exactly 0.1 then passed semantic.floor_gap although it is below the declared margin. The reader now parses reward and resource numbers as exact decimals and rejects any whose float value differs from the declared token. Every value the graders compare is therefore the operator's own. Co-Authored-By: Claude Opus 5.5 --- src/openenv/validation/parsers/harbor_toml.py | 10 ++++++++ .../test_validation/test_harbor_validation.py | 23 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/openenv/validation/parsers/harbor_toml.py b/src/openenv/validation/parsers/harbor_toml.py index 3b935c6ce..2d20af68d 100644 --- a/src/openenv/validation/parsers/harbor_toml.py +++ b/src/openenv/validation/parsers/harbor_toml.py @@ -3,6 +3,7 @@ import json import math import stat +from decimal import Decimal from importlib.metadata import PackageNotFoundError, version from pathlib import Path, PurePosixPath from typing import Literal @@ -68,6 +69,7 @@ def read_harbor_contract(package_root: Path) -> HarborContract: raw = json.loads( (package_root / "validation/openenv.json").read_text(), object_pairs_hook=_unique_object, + parse_float=Decimal, ) if not isinstance(raw, dict): raise ValueError("contract must be an object") @@ -92,6 +94,14 @@ def read_harbor_contract(package_root: Path) -> HarborContract: raise ValueError( "numeric declarations must be finite and nonboolean" ) + # Graders compare against the stored float, so it must keep the + # declared value exactly: 0.100000000000000001 would become 0.1. + elif isinstance(item, (int, Decimal)) and Decimal( + repr(float(item)) + ) != Decimal(item): + raise ValueError( + "numeric declarations must keep their exact value as floats" + ) return HarborContract.model_validate(raw) except (OSError, ValueError, TypeError, AttributeError) as exc: raise ManifestError( diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 98ef394ff..92929c9ab 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -629,6 +629,29 @@ def test_explicit_contract_and_valid_toml_required(task, boundary): assert not boundary.trials +@pytest.mark.parametrize( + "field,token", + [ + # As a float this margin is 0.1, so a measured gap of 0.1 would pass. + ("floor_margin", "0.100000000000000001"), + ("oracle_tolerance", "1e-999"), + ("range", "[0, 1.0000000000000001]"), + ], +) +def test_contract_numbers_must_keep_their_exact_value(task, boundary, field, token): + path = task / "validation/openenv.json" + contract = json.loads(path.read_text()) + contract["reward"][field] = "TOKEN" + path.write_text(json.dumps(contract).replace('"TOKEN"', token)) + boundary.rewards["nop"] = b'{"reward":0.9}' + with pytest.raises(ManifestError): + read_harbor_contract(task) + report = run_validation(task) + assert report.verdict is Verdict.FAIL + assert result(report, "static.manifest").status is CheckStatus.FAIL + assert not boundary.trials + + def test_malformed_toml_is_manifest_failure(task, boundary): (task / "task.toml").write_text("invalid = [") report = run_validation(task) From e328ab3dc112a0d1a9bfd402b1c1cf5dddea0d51 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 18:03:03 +0100 Subject: [PATCH 08/40] fix(validation): keep the harbor extra unpinned and trial cleanup strict Keeps the levels_run fix from abf12d02 and reverts its other two changes: - Pinning the harbor extra to ==0.25.0 would hold every Harbor-based feature in OpenEnv to that version. The Harbor task parser already refuses other versions with a named error. - Ignoring temp-directory cleanup errors guards against root-owned trial files. The binding runs Harbor with auto_mounting disabled, so trial outputs are downloaded by the host and owned by it. Ignoring errors would also leave private task copies on disk without notice. Also rename the version error, which still called the binding a pilot. Co-Authored-By: Claude Opus 5.5 --- envs/grid_world_env/uv.lock | 2 +- pyproject.toml | 2 +- src/openenv/validation/harbor.py | 3 +-- src/openenv/validation/parsers/harbor_toml.py | 2 +- .../test_validation/test_harbor_validation.py | 26 ------------------- tests/validation_runtime/uv.lock | 2 +- 6 files changed, 5 insertions(+), 32 deletions(-) diff --git a/envs/grid_world_env/uv.lock b/envs/grid_world_env/uv.lock index 896d19e19..753795aa7 100644 --- a/envs/grid_world_env/uv.lock +++ b/envs/grid_world_env/uv.lock @@ -1343,7 +1343,7 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.116.0" }, { name = "fastmcp", specifier = ">=3.0.0,<4.0.0" }, { name = "gradio", specifier = ">=4.0.0" }, - { name = "harbor", extras = ["e2b", "modal", "daytona", "gke", "ec2", "runloop", "novita", "blaxel", "beam", "islo", "opensandbox", "cwsandbox", "use-computer", "cua"], marker = "python_full_version >= '3.12' and extra == 'harbor'", specifier = "==0.25.0" }, + { name = "harbor", extras = ["e2b", "modal", "daytona", "gke", "ec2", "runloop", "novita", "blaxel", "beam", "islo", "opensandbox", "cwsandbox", "use-computer", "cua"], marker = "python_full_version >= '3.12' and extra == 'harbor'", specifier = ">=0.22.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "huggingface-hub", specifier = ">=1.29.0" }, { name = "inspect-ai", marker = "extra == 'inspect'", specifier = ">=0.3.0" }, diff --git a/pyproject.toml b/pyproject.toml index 75e656ba8..dde48a4e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ harbor = [ # (websockets>=15) and `tensorlake` (websockets>=13,<14) together. # # Harbor requires Python >= 3.12, so this extra is excluded from the 3.11 CI leg. - "harbor[e2b,modal,daytona,gke,ec2,runloop,novita,blaxel,beam,islo,opensandbox,cwsandbox,use-computer,cua]==0.25.0; python_version >= '3.12'", + "harbor[e2b,modal,daytona,gke,ec2,runloop,novita,blaxel,beam,islo,opensandbox,cwsandbox,use-computer,cua]>=0.22.0; python_version >= '3.12'", ] [project.scripts] diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index 63676dd0d..de4b4c485 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -701,8 +701,7 @@ def validate_harbor( attempted = True try: with tempfile.TemporaryDirectory( - prefix="openenv-harbor-validation-", - ignore_cleanup_errors=True, + prefix="openenv-harbor-validation-" ) as temp: daemon, trials, files = asyncio.run( _collect( diff --git a/src/openenv/validation/parsers/harbor_toml.py b/src/openenv/validation/parsers/harbor_toml.py index 2d20af68d..cc905dd94 100644 --- a/src/openenv/validation/parsers/harbor_toml.py +++ b/src/openenv/validation/parsers/harbor_toml.py @@ -141,7 +141,7 @@ def read_harbor_task(package_root: Path): """Use the pinned Harbor data loader; constructing a Trial is runtime-only.""" try: if version("harbor") != "0.25.0": - raise ValueError("the native pilot requires Harbor 0.25.0") + raise ValueError("Harbor task validation requires Harbor 0.25.0") from harbor.models.task.task import Task return Task(package_root) diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 92929c9ab..02176029f 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -360,32 +360,6 @@ def test_cleanup_failure_blocks(task, boundary, tmp_path): ] -def test_temp_cleanup_failure_does_not_discard_successful_trials( - task, boundary, monkeypatch -): - temporary_directory = harbor.tempfile.TemporaryDirectory - - class CleanupSensitiveTemporaryDirectory: - def __init__(self, *args, ignore_cleanup_errors=False, **kwargs): - self.directory = temporary_directory(*args, **kwargs) - self.ignore_cleanup_errors = ignore_cleanup_errors - - def __enter__(self): - return self.directory.__enter__() - - def __exit__(self, *args): - self.directory.__exit__(*args) - if not self.ignore_cleanup_errors: - raise PermissionError("container-owned trial file") - - monkeypatch.setattr( - harbor.tempfile, "TemporaryDirectory", CleanupSensitiveTemporaryDirectory - ) - report = run_validation(task) - assert result(report, "runtime.startup").status is CheckStatus.PASS - assert result(report, "semantic.oracle_max").status is CheckStatus.PASS - - def test_oracle_nonzero_exit_fails_oracle_max_and_retains_marker( task, boundary, tmp_path ): diff --git a/tests/validation_runtime/uv.lock b/tests/validation_runtime/uv.lock index 9e0422619..f140d77a8 100644 --- a/tests/validation_runtime/uv.lock +++ b/tests/validation_runtime/uv.lock @@ -952,7 +952,7 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.116.0" }, { name = "fastmcp", specifier = ">=3.0.0,<4.0.0" }, { name = "gradio", specifier = ">=4.0.0" }, - { name = "harbor", extras = ["e2b", "modal", "daytona", "gke", "ec2", "runloop", "novita", "blaxel", "beam", "islo", "opensandbox", "cwsandbox", "use-computer", "cua"], marker = "python_full_version >= '3.12' and extra == 'harbor'", specifier = "==0.25.0" }, + { name = "harbor", extras = ["e2b", "modal", "daytona", "gke", "ec2", "runloop", "novita", "blaxel", "beam", "islo", "opensandbox", "cwsandbox", "use-computer", "cua"], marker = "python_full_version >= '3.12' and extra == 'harbor'", specifier = ">=0.22.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "huggingface-hub", specifier = ">=1.29.0" }, { name = "inspect-ai", marker = "extra == 'inspect'", specifier = ">=0.3.0" }, From 7ce0f91da33860eae8e1f4a6dc4c45b925e6b4b9 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 18:10:17 +0100 Subject: [PATCH 09/40] test(validation): install Harbor in the runtime reference project The fast reference suite runs tests/test_validation in its locked project and fails on any skip. Without Harbor installed, the Harbor task binding's tests skipped. Adding harbor==0.25.0 runs them. Harbor's dependencies cap websockets below 16, so the lock moves websockets from 17.1 to 15.0.1, which is within OpenEnv's websockets>=15.0.1. The fast suite (1,711 tests) and the protocol suite (138 tests) pass with no skips under the new lock. Co-Authored-By: Claude Opus 5.5 --- tests/validation_runtime/pyproject.toml | 2 + tests/validation_runtime/uv.lock | 674 ++++++++++++++++++++++-- 2 files changed, 640 insertions(+), 36 deletions(-) diff --git a/tests/validation_runtime/pyproject.toml b/tests/validation_runtime/pyproject.toml index fe02e980d..de190c4ef 100644 --- a/tests/validation_runtime/pyproject.toml +++ b/tests/validation_runtime/pyproject.toml @@ -4,6 +4,8 @@ version = "0.1.0" requires-python = "==3.12.*" dependencies = [ "openenv", + # The Harbor task binding's tests import Harbor; a skip fails --require-complete. + "harbor==0.25.0", "pytest==8.4.2", "pytest-asyncio==1.2.0", "jsonschema==4.25.1", diff --git a/tests/validation_runtime/uv.lock b/tests/validation_runtime/uv.lock index f140d77a8..ffb7f703b 100644 --- a/tests/validation_runtime/uv.lock +++ b/tests/validation_runtime/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = "==3.12.*" resolution-markers = [ "sys_platform == 'win32'", @@ -19,6 +19,65 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4e/79/6e45e778c4c3cab39e0937b007b720c15f76c50c6453d153282d0fcc3588/aiofile-3.12.3-py3-none-any.whl", hash = "sha256:5c1bcc9e929c50834608e8cc1a4cc1d7503eb60c15a535b779fd39e2f372c017", size = 22122, upload-time = "2026-08-04T22:59:25.838Z" }, ] +[[package]] +name = "aiohappyeyeballs" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.14.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "typing-extensions" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/4c/bdccd81e9ee225b69c60e7766c9a5b05364f118f4d383713b89a682d772d/aiohttp-3.14.5.tar.gz", hash = "sha256:5558a7f5a05af9ecf744af91e5baefc436f93c9333e656c27ec253f9a6bbe178", size = 8078052, upload-time = "2026-10-11T01:05:12.408Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/94/6ba86efddcb616c811b40e6a0dfdd862738f647e4e3860a961075d5e9ed8/aiohttp-3.14.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:df37b620684e19b5e25724412518ccafc3b1a49cdac706fdbd2f983fad943450", size = 817499, upload-time = "2026-10-11T01:00:18.901Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e1/7bca6d84dabd228aa8eb4b7f9feac2586aaa9be5505d7d65bf287a615c43/aiohttp-3.14.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ef60869969180ec2464f1349aff07138ae35ca2200f0946cb3552e49e8f301a8", size = 549467, upload-time = "2026-10-11T01:00:20.854Z" }, + { url = "https://files.pythonhosted.org/packages/64/91/11b89f45ca486252dd67dd5f3231fec04bf5518da39a95cb3997619f17fb/aiohttp-3.14.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d079c0a0135c36e7beb6f1c88087c8f108dc5891cdd0b5eafa778421bda70ed2", size = 548067, upload-time = "2026-10-11T01:00:22.659Z" }, + { url = "https://files.pythonhosted.org/packages/22/ff/c6615806c14aab34f82b9424ccde8ce6e417315fd57ce1c5b4d4747888e1/aiohttp-3.14.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:abfda5cb094a829f7bc25216a32f7db2e85cc65bd59910f8e7b40b3d9b224764", size = 1896245, upload-time = "2026-10-11T01:00:24.638Z" }, + { url = "https://files.pythonhosted.org/packages/39/b2/25a8c971ae6a92c8394d77e42422d7f38989e05cf41a5ceb92d73d67ab7e/aiohttp-3.14.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9cc882cf8619109583c906b4d4a85d6a111a98afa34b7a450d1e08118d016820", size = 1872180, upload-time = "2026-10-11T01:00:26.838Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d5/99f93ea36cc5205e47c1e5a803e087f2ad21b5430b5db2e942cb6e988a37/aiohttp-3.14.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7457580535e019e1247ea35d6a02bf081ad30c26d0cbc210c93f6c3ab67a0835", size = 1937261, upload-time = "2026-10-11T01:00:28.74Z" }, + { url = "https://files.pythonhosted.org/packages/40/a6/9ac9c9e6695040bd73d2584a1b59a9388f6433c76d5294a7bf591e21ffe5/aiohttp-3.14.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c5ed596aedb9c42afd3fe0aae3117725378ac73d2cc5ddc735056fbdb96c5d02", size = 2064994, upload-time = "2026-10-11T01:00:30.623Z" }, + { url = "https://files.pythonhosted.org/packages/da/e4/aa172eb534b7f02f1f8ff1c3213347eaf3cf218db91a727c6863c22f1035/aiohttp-3.14.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20f085697d7e911f1f73c43ed03fafbed1e7121797e2eb5428efa80398060584", size = 1919630, upload-time = "2026-10-11T01:00:32.548Z" }, + { url = "https://files.pythonhosted.org/packages/06/7d/4eedafc5bababa8932636c141e346806966eade12c0b7e5946d43bf8218b/aiohttp-3.14.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:74b0a9c8270f9b0a11410e124ff8d4f18bfc1f1837440ec84da5ae7b50927b5d", size = 1710016, upload-time = "2026-10-11T01:00:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/b2/94/eee018537ba19da0ceb2ac79cab83faed4ac49568e08376e2799043f2538/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:19e2ba471507c34f8252402ab50f5ab512398b9ea8c8f1cb26beb3f75793ba30", size = 1858112, upload-time = "2026-10-11T01:00:36.581Z" }, + { url = "https://files.pythonhosted.org/packages/68/76/354653a306547238f3427283905972d796ba7c292ba9977abec9f2b6f260/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d418ce2af40c6bb685b3f663e9e8de27cb0a22431d8e88a167348d7f01878073", size = 1896371, upload-time = "2026-10-11T01:00:38.478Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ec/63e8c7136b570e356345ad3174e3820fdc973ea10712cb6c649bf875755a/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:70cb4008ac2ed1e0ca9e824deb4b53d3aa0d939109698ebf1e723a84337bd794", size = 1905836, upload-time = "2026-10-11T01:00:40.527Z" }, + { url = "https://files.pythonhosted.org/packages/57/d8/11365bda144b127928cd42533d0eff78a55613c9e28f81941bd6630ea887/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a23fe35d776bc03cb495938b9594450d047e3bc08c5255315a82323e9cb7d2dd", size = 1696534, upload-time = "2026-10-11T01:00:42.686Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3d/82df0461b18e00b2998f205c03e0d3010222478c43640aceb8e03dcd7e8f/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3e0eb43bed3c6801a6cee315195377789e90b2a72c2277a475b578535312488d", size = 1933970, upload-time = "2026-10-11T01:00:44.71Z" }, + { url = "https://files.pythonhosted.org/packages/03/ad/6ddfe0aacd931c17b53533336d97e9d11a98b96d6ae815a9da0b19f82ccf/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be7dd397d64ca3e1869626fa9318aaebb54b7bf93bc72d7a205448d83e4f748", size = 1899501, upload-time = "2026-10-11T01:00:46.629Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8e/189bdd9ae4793059bb09f6dc880f211a6c6c7859cebcbf33912dbfab7dd7/aiohttp-3.14.5-cp312-cp312-win32.whl", hash = "sha256:eb324e2009fb54db30a071dad7caf6998ee2879c4704007efb244514dad1fec1", size = 480370, upload-time = "2026-10-11T01:00:48.468Z" }, + { url = "https://files.pythonhosted.org/packages/ae/ce/1f08114679d49655b30a6e0a29858375c94b82c1de1a0bd0a20c2fee8b02/aiohttp-3.14.5-cp312-cp312-win_amd64.whl", hash = "sha256:2cc38a4f2b516bef1714e690df87a0e043faf1a7693c82d860091684453d5111", size = 506141, upload-time = "2026-10-11T01:00:50.272Z" }, + { url = "https://files.pythonhosted.org/packages/79/d4/c7b4f60b16a1b7e43249fa9031ae05e7e8c341ba4b7d1866f914dafeaa0e/aiohttp-3.14.5-cp312-cp312-win_arm64.whl", hash = "sha256:a63afd1f757de949028387e65a7127b61ad0f775432dbb0e62816ae619fe69ac", size = 488108, upload-time = "2026-10-11T01:00:52.321Z" }, + { url = "https://files.pythonhosted.org/packages/68/30/173960c42b05a6c59f7558e4b12a4b0d9ba376cf6aa9bde7f9e08a30ca8d/aiohttp-3.14.5-py3-none-any.whl", hash = "sha256:efc21a454892828368b11c2c780de0ff8bc991f73f6b99c6b66e56205470929b", size = 279517, upload-time = "2026-10-11T01:05:08.523Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + [[package]] name = "annotated-doc" version = "0.0.5" @@ -81,6 +140,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl", hash = "sha256:d16c9bbc61ea14637596c5f6fbff2ee99cbe3573e46a716401734ef50c3060c2", size = 1333658, upload-time = "2025-12-13T06:50:28.266Z" }, ] +[[package]] +name = "boto3" +version = "1.43.127" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ca/6b/2bc953d6f0b87505636d0a7c16e46a47ab6c49df4a278ac9c0b1c1732e65/boto3-1.43.127.tar.gz", hash = "sha256:b5417e81d439b35ccf3a1eb51fa06cf9c97167c9a6beaa1929175ee9840d505a", size = 112783, upload-time = "2026-11-09T20:52:17.411Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/48/f562f1240da499614a19bdf5d54dbdbfea5cffabbdd2703c3a413e0c13a1/boto3-1.43.127-py3-none-any.whl", hash = "sha256:47f87f89b10e62f5191bd98674beb0dd52ed368c19fcdfef2ef38ead31126f49", size = 140187, upload-time = "2026-11-09T20:52:15.845Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.127" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/45/b4e606b0f66c11c6bd44e91381aa562c6b4dacf478b97b461a429036a89a/botocore-1.43.127.tar.gz", hash = "sha256:93bae360fb3a6f1b8e36593faa8bbbfa05b19866f7a37e9606a98f13394156f8", size = 16559749, upload-time = "2026-11-09T20:52:12.475Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/db/bee0401fd3f8cf1e2404044a4e3156dc7e064c2b1fc7162df2d795941cd8/botocore-1.43.127-py3-none-any.whl", hash = "sha256:aa0e3188679a2164a2098a14ea9b59520520cb573aaee9999d852480691e134a", size = 16257630, upload-time = "2026-11-09T20:52:09.488Z" }, +] + [[package]] name = "brotli" version = "1.2.1" @@ -132,11 +219,11 @@ wheels = [ [[package]] name = "certifi" -version = "2026.10.18" +version = "2026.10.23" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/eb/a2aa1ac3ea30dd3f30ca57f40232138b6b0040719316ecb85daf61167771/certifi-2026.10.18.tar.gz", hash = "sha256:b9995d97b64d0431a5af837d8ca8cd6238104d1f319f04bd93ea43124d6c861e", size = 135227, upload-time = "2026-10-18T16:21:07.655Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/08/ba/95dd616042484f7c83a06974388b37b837732740ee70bc5dc715b5fcd55a/certifi-2026.10.18-py3-none-any.whl", hash = "sha256:73191c47aaa79da394e8fee2dbc2ec827432cab56311122c9e47abf79e390f62", size = 134093, upload-time = "2026-10-18T16:21:06.276Z" }, + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, ] [[package]] @@ -273,6 +360,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/0e/eadf712ca776fd96982c09348dd8f141ae6f96746a4eabc3e3fb54dea738/cyclopts-4.25.2-py3-none-any.whl", hash = "sha256:51b42513eea5e4ba6a08b68acf187b81f1191032cf77b9631086cfa2c88fa7a4", size = 242400, upload-time = "2026-09-08T16:58:02.696Z" }, ] +[[package]] +name = "deprecation" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178, upload-time = "2020-04-20T14:23:36.581Z" }, +] + +[[package]] +name = "dirhash" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "scantree" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/70/49f93897f3a4f7ab5f20a854ebc91aad47854e9fb2cd169e3a4452fa3f5e/dirhash-0.5.0.tar.gz", hash = "sha256:e60760f0ab2e935d8cb088923ea2c6492398dca42cec785df778985fd4cd5386", size = 21377, upload-time = "2024-08-03T22:14:13.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/1f/c8bf92552b7f0a13b9f12b85e3de8df6d9814240e0f8ce8f37433df028b3/dirhash-0.5.0-py3-none-any.whl", hash = "sha256:523dfd6b058c64f45b31604376926c6e2bd2ea301d0df23095d4055674e38b09", size = 13119, upload-time = "2024-08-03T22:14:11.688Z" }, +] + [[package]] name = "dnspython" version = "2.9.0" @@ -395,6 +506,25 @@ server = [ { name = "websockets" }, ] +[[package]] +name = "fastuuid" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/7d/d9daedf0f2ebcacd20d599928f8913e9d2aea1d56d2d355a93bfa2b611d7/fastuuid-0.14.0.tar.gz", hash = "sha256:178947fc2f995b38497a74172adee64fdeb8b7ec18f2a5934d037641ba265d26", size = 18232, upload-time = "2025-10-19T22:19:22.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/a2/e78fcc5df65467f0d207661b7ef86c5b7ac62eea337c0c0fcedbeee6fb13/fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77e94728324b63660ebf8adb27055e92d2e4611645bf12ed9d88d30486471d0a", size = 510164, upload-time = "2025-10-19T22:31:45.635Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b3/c846f933f22f581f558ee63f81f29fa924acd971ce903dab1a9b6701816e/fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:caa1f14d2102cb8d353096bc6ef6c13b2c81f347e6ab9d6fbd48b9dea41c153d", size = 261837, upload-time = "2025-10-19T22:38:38.53Z" }, + { url = "https://files.pythonhosted.org/packages/54/ea/682551030f8c4fa9a769d9825570ad28c0c71e30cf34020b85c1f7ee7382/fastuuid-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d23ef06f9e67163be38cece704170486715b177f6baae338110983f99a72c070", size = 251370, upload-time = "2025-10-19T22:40:26.07Z" }, + { url = "https://files.pythonhosted.org/packages/14/dd/5927f0a523d8e6a76b70968e6004966ee7df30322f5fc9b6cdfb0276646a/fastuuid-0.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c9ec605ace243b6dbe3bd27ebdd5d33b00d8d1d3f580b39fdd15cd96fd71796", size = 277766, upload-time = "2025-10-19T22:37:23.779Z" }, + { url = "https://files.pythonhosted.org/packages/16/6e/c0fb547eef61293153348f12e0f75a06abb322664b34a1573a7760501336/fastuuid-0.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:808527f2407f58a76c916d6aa15d58692a4a019fdf8d4c32ac7ff303b7d7af09", size = 278105, upload-time = "2025-10-19T22:26:56.821Z" }, + { url = "https://files.pythonhosted.org/packages/2d/b1/b9c75e03b768f61cf2e84ee193dc18601aeaf89a4684b20f2f0e9f52b62c/fastuuid-0.14.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fb3c0d7fef6674bbeacdd6dbd386924a7b60b26de849266d1ff6602937675c8", size = 301564, upload-time = "2025-10-19T22:30:31.604Z" }, + { url = "https://files.pythonhosted.org/packages/fc/fa/f7395fdac07c7a54f18f801744573707321ca0cee082e638e36452355a9d/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab3f5d36e4393e628a4df337c2c039069344db5f4b9d2a3c9cea48284f1dd741", size = 459659, upload-time = "2025-10-19T22:31:32.341Z" }, + { url = "https://files.pythonhosted.org/packages/66/49/c9fd06a4a0b1f0f048aacb6599e7d96e5d6bc6fa680ed0d46bf111929d1b/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b9a0ca4f03b7e0b01425281ffd44e99d360e15c895f1907ca105854ed85e2057", size = 478430, upload-time = "2025-10-19T22:26:22.962Z" }, + { url = "https://files.pythonhosted.org/packages/be/9c/909e8c95b494e8e140e8be6165d5fc3f61fdc46198c1554df7b3e1764471/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3acdf655684cc09e60fb7e4cf524e8f42ea760031945aa8086c7eae2eeeabeb8", size = 450894, upload-time = "2025-10-19T22:27:01.647Z" }, + { url = "https://files.pythonhosted.org/packages/90/eb/d29d17521976e673c55ef7f210d4cdd72091a9ec6755d0fd4710d9b3c871/fastuuid-0.14.0-cp312-cp312-win32.whl", hash = "sha256:9579618be6280700ae36ac42c3efd157049fe4dd40ca49b021280481c78c3176", size = 154374, upload-time = "2025-10-19T22:29:19.879Z" }, + { url = "https://files.pythonhosted.org/packages/cc/fc/f5c799a6ea6d877faec0472d0b27c079b47c86b1cdc577720a5386483b36/fastuuid-0.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:d9e4332dc4ba054434a9594cbfaf7823b57993d7d8e7267831c3e059857cf397", size = 156550, upload-time = "2025-10-19T22:27:49.658Z" }, +] + [[package]] name = "filelock" version = "4.2.1" @@ -404,6 +534,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/00/db89d339bc4ecdc8087fa7b8607ba22ffc27bf96286f340c9620bea428b3/filelock-4.2.1-py3-none-any.whl", hash = "sha256:7be2662e9e826b9bba28afc0f843a123280d799a2ea44bb5d35ccec67905ae44", size = 162455, upload-time = "2026-10-17T16:48:12.937Z" }, ] +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, + { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, + { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, + { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, + { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, + { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, + { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, + { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, + { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, + { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, + { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, +] + [[package]] name = "fsspec" version = "2026.7.0" @@ -494,6 +649,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "h2" +version = "4.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hpack" }, + { name = "hyperframe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/85/7c366e69d84c17bb778fe41419e1fbcce3033d5b7ce29bbffff0a98b859f/h2-4.4.1.tar.gz", hash = "sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516", size = 2157281, upload-time = "2026-08-03T11:45:09.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/22/e85faf23bd72a92d1921e37d674ca56eb298a3c8be31fdecef0ff2b3aaac/h2-4.4.1-py3-none-any.whl", hash = "sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6", size = 62636, upload-time = "2026-08-03T11:44:59.164Z" }, +] + +[[package]] +name = "harbor" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "dirhash" }, + { name = "fastapi" }, + { name = "filelock" }, + { name = "httpx" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "litellm" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, + { name = "pydantic" }, + { name = "pyjwt" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "rich" }, + { name = "semver" }, + { name = "shortuuid" }, + { name = "supabase" }, + { name = "tenacity" }, + { name = "toml" }, + { name = "typer" }, + { name = "uvicorn" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/8f/624ed4b2763565d50c78e9278d2800037c9147d56ee977d3b8ed6c5b81b1/harbor-0.25.0.tar.gz", hash = "sha256:a66543a223539cd853889ee0eea77d15a3e5222963418a87c0f079b63dadb4b2", size = 2309418, upload-time = "2026-10-25T20:27:27.643Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/72/2ead84b6d43e4737682dd489b107be9d828bf5ae4869d0d4f2aa72243149/harbor-0.25.0-py3-none-any.whl", hash = "sha256:51923ac567fc57b03279e93468b105f689628bcac2cd3d541d4e44aa5c0ec309", size = 2535481, upload-time = "2026-10-25T20:27:25.575Z" }, +] + [[package]] name = "hf-gradio" version = "0.4.1" @@ -523,6 +726,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, ] +[[package]] +name = "hpack" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/5b/fcabf6028144a8723726318b07a32c2f3314acdff6265743cf08a344b18e/hpack-4.2.0.tar.gz", hash = "sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0", size = 51300, upload-time = "2026-06-23T18:34:46.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/b4/4a9fcfb2aef6ba44d9073ecd301443aa00b3dac95de5619f2a7de7ec8a91/hpack-4.2.0-py3-none-any.whl", hash = "sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986", size = 34246, upload-time = "2026-06-23T18:34:45.472Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -541,8 +753,8 @@ name = "httpcore2" version = "2.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "h11" }, - { name = "truststore" }, + { name = "h11", marker = "sys_platform != 'emscripten'" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/15/8c/e925b1c92018abb3a1863ce1549d76d2381e334d21d65d4ac8f65dabd78a/httpcore2-2.13.0.tar.gz", hash = "sha256:2adc8be4fb285fbcd6d894298db3b52c177e74b6674eda3a76bd36be3292a3db", size = 67740, upload-time = "2026-09-14T14:18:04.717Z" } wheels = [ @@ -564,6 +776,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[package.optional-dependencies] +http2 = [ + { name = "h2" }, +] + [[package]] name = "httpx-sse" version = "0.4.3" @@ -619,6 +836,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c3/7f/3f886a625043b77312b80da2f2bf00b5ecbf5a73061af1aa0259cd258c9d/huggingface_hub-1.31.0-py3-none-any.whl", hash = "sha256:9dbb6a503cbe2494ea666695207e7262d410659e09134059deb83e5480864667", size = 798313, upload-time = "2026-09-10T10:27:20.798Z" }, ] +[[package]] +name = "hyperframe" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, +] + [[package]] name = "idna" version = "3.19" @@ -628,6 +854,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, ] +[[package]] +name = "importlib-metadata" +version = "8.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/72/c600ae4f68c28fc19f9c31b9403053e5dbb8cace2e6842c7b7c3e4d42fe9/importlib_metadata-8.9.0.tar.gz", hash = "sha256:58850626cef4bd2df100378b0f2aea9724a7b92f10770d547725b047078f99ee", size = 56140, upload-time = "2026-03-20T16:56:26.362Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/f9/97f2ca8bb3ec6e4b1d64f983ebe98b9a192faddff67fac3d6303a537e670/importlib_metadata-8.9.0-py3-none-any.whl", hash = "sha256:e0f761b6ea91ced3b0844c14c9d955224d538105921f8e6754c00f6ca79fba7f", size = 27220, upload-time = "2026-03-20T16:56:25.07Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -717,6 +955,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/8b/d76219ebdbcf3d4209d9d21a0810db4c8d0a6f88e3ee87d30bdea4e90d30/jiter-0.17.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2c0bf24c72fd0491405dce5d40194f2070e9021ce648c1a1d46234b93d848ff", size = 307147, upload-time = "2026-09-12T15:14:12.897Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "joserfc" version = "1.7.5" @@ -797,6 +1044,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, ] +[[package]] +name = "litellm" +version = "1.108.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "boto3" }, + { name = "certifi" }, + { name = "click" }, + { name = "fastuuid" }, + { name = "filelock" }, + { name = "httpx", extra = ["http2"] }, + { name = "huggingface-hub" }, + { name = "importlib-metadata" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "openai" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "tiktoken" }, + { name = "tokenizers" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2a/1e/ffa920bca1ee09bf8b8c089888f0ffd7b0653c8a5dd7ce95c19b633c8d5b/litellm-1.108.4.tar.gz", hash = "sha256:192974c6a17ef20ddbc3851f2151cdb0ad81eb8b67bf043cc90cea8714bbed83", size = 21465884, upload-time = "2026-11-09T22:57:03.391Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/61/7dea0ad7bfd8415e7916f8fb37a663b61a1e558205e136b5eab3b342f012/litellm-1.108.4-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:08e56d2f17c69e7df053fdde45fa3a536d685021b036e68813b52c6f2ad43131", size = 36501280, upload-time = "2026-11-09T22:56:38.121Z" }, + { url = "https://files.pythonhosted.org/packages/c4/60/24e4f460f534c4e619def2ac170ffbd5478e80d8c8f410cfa21999aa6515/litellm-1.108.4-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:2a815ddc987b18382b799c230a1ce43111a1c2f915dd79c9a4759009b1ec5e4c", size = 35812361, upload-time = "2026-11-09T22:56:42.265Z" }, + { url = "https://files.pythonhosted.org/packages/ac/3c/57bdb20df4c37a5835b348d377ee8af87a363758bebdee9f577c7e215df7/litellm-1.108.4-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e3d907c211651f0a0dc86b50480cef65076f4538c388a557766ac04c5257b236", size = 36308756, upload-time = "2026-11-09T22:56:45.9Z" }, + { url = "https://files.pythonhosted.org/packages/62/64/bf5eebf56b5f1d95ce70ebb2bf1036e8e31acc2864f86cbcf6e795ed360c/litellm-1.108.4-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:d51df0a967fb271c943994f8f1e941982d3ceff538298dc2134dcb03ef01b86a", size = 36999649, upload-time = "2026-11-09T22:56:50.31Z" }, + { url = "https://files.pythonhosted.org/packages/20/33/6f8fcfc2c5d4e61646c087a790d39705bf5878d00e7ad3cff6198598ad2e/litellm-1.108.4-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ad7c68fea674141f3a9ec8b4a0441a0c1007cb0e689b23c59878e60f50c7c807", size = 36445716, upload-time = "2026-11-09T22:56:53.63Z" }, + { url = "https://files.pythonhosted.org/packages/e3/fc/125fe7e4ceda8d9a6469f143e42499be3a52d3b8611c26964bef5ce21d2b/litellm-1.108.4-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:913d2bc64477cf5f62159a02eff353e8cc2f48da41758550bfabf39892e896c3", size = 37077338, upload-time = "2026-11-09T22:56:57.072Z" }, + { url = "https://files.pythonhosted.org/packages/e1/96/f8c8f74939c9a424fd4ffa0af0ea4b040c2d349ed5c4035887512a2d85e0/litellm-1.108.4-cp310-abi3-win_amd64.whl", hash = "sha256:09ebd0b793b942277660c8f2e9dce90efddf88222231d85cdec0b78114a2aaeb", size = 37949824, upload-time = "2026-11-09T22:57:00.207Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -871,6 +1155,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192", size = 72226, upload-time = "2026-05-22T14:14:28.824Z" }, ] +[[package]] +name = "multidict" +version = "7.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6b/29ef8e752f5854dcc267ded0652c475bec99b86f255957a0d4fcad657b18/multidict-7.2.1.tar.gz", hash = "sha256:e8af3e6f1bdbc31fd762b094133b478b89988eaebc63cbe82b1d9e3bbd15cede", size = 369994, upload-time = "2026-10-24T20:41:35.457Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/21/53c299f1ea289f5fd438e70d4f1f053f59b4485f5a410bcc85eecb553394/multidict-7.2.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:cb43bb34346155d479acdc5423afccdff79ce1fc66adfcc9de995cbec8a13acf", size = 172948, upload-time = "2026-10-24T20:40:10.189Z" }, + { url = "https://files.pythonhosted.org/packages/9d/17/faacc27a01e364abed3c6218d8a32bc42fa0688c95f82c6b11e3ffe99a07/multidict-7.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbda2baafe08f8431fd3333be02c261023fd454607e3bde5e7dc00565f7bebcb", size = 98039, upload-time = "2026-10-24T20:40:13.633Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e4/bd8838244fd1309165c79e74af9a6b9e9e42fd0f2c24c75bee665fc9ba69/multidict-7.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d37773029ba034a2cc970b6eeb67dad3bff9d894393c51f6bb1c6b4f563a032c", size = 103911, upload-time = "2026-10-24T20:40:13.517Z" }, + { url = "https://files.pythonhosted.org/packages/0c/39/21dbe0be29cd44de626736c6f0915c0b75f153e0155340a253f375ac5747/multidict-7.2.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c85a0a54a19429a17ef449e967baffe7b5089cb4f6ef2894c9168a69fa024d05", size = 493430, upload-time = "2026-10-24T20:40:49.104Z" }, + { url = "https://files.pythonhosted.org/packages/72/13/c081a1af93258029471636ea1c741141ece10f1045578a5f73ba99b694a0/multidict-7.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fa0ecd064b0ae18d235283abb5adbc4e07304dba8f2aa7c0d61f28bebcc70d4c", size = 506972, upload-time = "2026-10-24T20:40:11.68Z" }, + { url = "https://files.pythonhosted.org/packages/3d/3e/e9982c6b1e0d4bb19c4551cd56e50c1fbe90492bdf7893adf8d1ea39f373/multidict-7.2.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f278f118f9fdb5faba5a6792be397fbda094d1568efa4dfe8c3595ce4ce89eea", size = 470406, upload-time = "2026-10-24T20:40:14.984Z" }, + { url = "https://files.pythonhosted.org/packages/6d/2e/ab4136028044825995b3c9de4cf8f9ef82c2076b09b5c44c65e7e3eef0a1/multidict-7.2.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3d28ac22fe658ad7486d1532b3a9084fbf65b428db31f47212c5b0b6da826f1", size = 524691, upload-time = "2026-10-24T20:40:15.205Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ed/0a57de7a6b6444f301fc4cfc4b8eb0d1c36a33b8ff05717fab7700f205b1/multidict-7.2.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2f3cf710144b1859f4128051cc27ea86eee83caac5648cf2ca0c1b835261cbbe", size = 549512, upload-time = "2026-10-24T20:40:50.756Z" }, + { url = "https://files.pythonhosted.org/packages/43/50/6bd8600b4843f0a9d939e0cc0f4160f4c727bd6c69e4f7e294a1caa93db8/multidict-7.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fb8c1a3480c130c3a6422d5f16f3cad6552bf08c2089dd288ba7134649183b5", size = 516796, upload-time = "2026-10-24T20:40:13.481Z" }, + { url = "https://files.pythonhosted.org/packages/c2/77/31e78862081f54a06115927c1c3d81b1a57522fa0209fe1992fac25c6e75/multidict-7.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b81ff0f2d375988152614eaea4d5023c98ecb08f77df4bb1d96762504bb86fca", size = 451174, upload-time = "2026-10-24T20:40:16.492Z" }, + { url = "https://files.pythonhosted.org/packages/e8/bf/30a0c0911e4b61accb9995165acfdc5107dd335ad3ce003007afaf168dec/multidict-7.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b4821bd30b38144d8ab75df56af0b07e9d71327c9cf94c0c7a1965f865bb9615", size = 492916, upload-time = "2026-10-24T20:40:16.619Z" }, + { url = "https://files.pythonhosted.org/packages/5a/80/2192317e096489e9361f732beeee2dd213b827de71b038011dd8176e262f/multidict-7.2.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0eb7fa012ab47b13e96bffc1369642f520adcc4e3d3e170d7f1a2b39fb953bce", size = 468924, upload-time = "2026-10-24T20:40:52.584Z" }, + { url = "https://files.pythonhosted.org/packages/65/fe/92a104d1d9731a97053193a3f3646a2ee286929fdb7f9c2d5077de782b33/multidict-7.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f9e78d98a29e969a001e953eb09167a48bbc71249e804dae5db987ee7e6531c9", size = 492920, upload-time = "2026-10-24T20:40:15.165Z" }, + { url = "https://files.pythonhosted.org/packages/c6/6b/165323c76d383afbf60da76a4e67f4b78635b09a77c2b83d7033e72d5ee8/multidict-7.2.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:eb5c4a5b1281985271b6c29be53f2c1a9decf11e30a0e2733439c4d9d61b9ebb", size = 510473, upload-time = "2026-10-24T20:40:17.824Z" }, + { url = "https://files.pythonhosted.org/packages/ff/61/e25786f40ab8a862102c198ad0e39cbca704aeeac096edd9cffc85122f8c/multidict-7.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2c4e6a97f79699f33e6dc13bddb5631781ee79a4a94c1822116d100e7d5c136c", size = 445954, upload-time = "2026-10-24T20:40:18.14Z" }, + { url = "https://files.pythonhosted.org/packages/a5/bb/deb193198dd1913e8feb54bc83dd13a0cf5425b1aaf7ac913e980537aca6/multidict-7.2.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5324411117bdb67cfaeb3d1d28fb783710fcca3d88b855696af99044d7219e4b", size = 528252, upload-time = "2026-10-24T20:40:54.068Z" }, + { url = "https://files.pythonhosted.org/packages/7b/9f/ae4d6c442e11aea5ff631f36e4e8d5ff4d1f4c330d2c2721cb729fa5eed6/multidict-7.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d615377a8e7bfc4f1983ede3f411c2ac2b0f7c3053f262d0437b7d8bcc4485e3", size = 508675, upload-time = "2026-10-24T20:40:17.249Z" }, + { url = "https://files.pythonhosted.org/packages/06/ca/bff250b81ada0df548da2b9c0abaf12b4fd864f40ee39e983cd9bada5799/multidict-7.2.1-cp312-cp312-win32.whl", hash = "sha256:30572b61c7932e4d3c2ad1e580de7db1f35b7a35d36db38f89397ce3f2e69096", size = 84510, upload-time = "2026-10-24T20:40:19.481Z" }, + { url = "https://files.pythonhosted.org/packages/68/d5/64108a33c21bde81454b5a400d16baa1378a7d01599de64b131c3e3e3114/multidict-7.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:781ca5dfc2f1d58e47f52216a0c1c7e1ba91ce8aa235459e20de2d402af77fb3", size = 95372, upload-time = "2026-10-24T20:40:19.679Z" }, + { url = "https://files.pythonhosted.org/packages/e0/00/b746ff0f4f70589be9eca08d37a211c3257d602e0ba95224dd0ee0577801/multidict-7.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:9306e0c5595f91874e685be0a9eb743d48ba069c888eea051bec3c67c08b5b8d", size = 92629, upload-time = "2026-10-24T20:40:55.897Z" }, + { url = "https://files.pythonhosted.org/packages/9d/96/1aaad0e5df33f1f5af21775c411349c0234a6edd581ea403bc81919a6eab/multidict-7.2.1-py3-none-any.whl", hash = "sha256:08ea94663d9e7c25d14d78222f7aee906c472d4837766bec62477c38f6e419dc", size = 28145, upload-time = "2026-10-24T20:40:59.159Z" }, +] + [[package]] name = "numpy" version = "2.5.3" @@ -991,6 +1304,7 @@ name = "openenv-validation-runtime-tests" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "harbor" }, { name = "jsonschema" }, { name = "openenv" }, { name = "pip" }, @@ -1002,6 +1316,7 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "harbor", specifier = "==0.25.0" }, { name = "jsonschema", specifier = "==4.25.1" }, { name = "openenv", directory = "../../" }, { name = "pip", specifier = "==26.2.1" }, @@ -1082,6 +1397,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/e8/6d75ffd9784bce2e93d1ae4415649427e39a53bb172d4672b2b59c6f0a7b/pathable-0.6.0-py3-none-any.whl", hash = "sha256:82c4ca6c98c502ad12e0d4e9779b6210afee93c38990988c8c5d1b49bdcdf566", size = 18983, upload-time = "2026-05-19T18:15:10.728Z" }, ] +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + [[package]] name = "pillow" version = "12.3.0" @@ -1126,6 +1450,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "postgrest" +version = "2.34.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecation" }, + { name = "httpx", extra = ["http2"] }, + { name = "pydantic" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/79/3309b0d26a098776946942372dcb6ad7b7cc46ed00069016b047d8cc2666/postgrest-2.34.1.tar.gz", hash = "sha256:fb3f7b225601b9bd6dfbf439a341f67302b3b7fbb43868a347fc438440d77601", size = 16697, upload-time = "2026-11-06T15:23:03.838Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/7c/b10fcd7f292ec1c40894af583e51026b9b9b4ead792711c3492fe188bb98/postgrest-2.34.1-py3-none-any.whl", hash = "sha256:7abb123b050011408f61410e334b4288c5c69768acea4eef8c4c1e518853c09a", size = 25732, upload-time = "2026-11-06T15:22:57.425Z" }, +] + +[[package]] +name = "propcache" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/9a/9fbf4e4ec0c2d7f1c32519fff782ef467859b8faa9fbc5331a96f6395d43/propcache-0.5.4.tar.gz", hash = "sha256:ff6b113f50bc066a698db5d944d2c6dc7507168dd3341e255a8892fd0715a558", size = 61545, upload-time = "2026-09-16T00:17:14.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/cd/348d58f142aebc4873345c6b31087629182ca6e0f2b3caeaa528cf882eba/propcache-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b28f41fa3b8c6900457f858ec5b03998f3a6d535fbc1bb2edec5961ea05ec429", size = 87285, upload-time = "2026-09-16T00:14:29.362Z" }, + { url = "https://files.pythonhosted.org/packages/df/f4/f3ffaee281b276da854ac1d7a6a506d26cbc62ea2e623756f1d0a4a1ba1a/propcache-0.5.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dcbf346a318a5e30063f547630b02bb787ce2f45b6368d5da143660b6a3835d8", size = 50984, upload-time = "2026-09-16T00:14:30.473Z" }, + { url = "https://files.pythonhosted.org/packages/25/88/1d7df7201750b37765ef2b23bc1c526c028dadde80afa0f57a118fc01182/propcache-0.5.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:87a3caecf8095e48dc72f84bfa42e23a848cf410cc9cc13031fba4869b706a21", size = 52460, upload-time = "2026-09-16T00:14:31.692Z" }, + { url = "https://files.pythonhosted.org/packages/83/4f/48865bd02a16ee5236bc46166b2946f37b93e07b0eae355dac0be0b216ca/propcache-0.5.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60a64cbccaa11b7760ce705a14ada17ba459e7ca9f23ba587eb013821032d7ef", size = 251768, upload-time = "2026-09-16T00:14:32.908Z" }, + { url = "https://files.pythonhosted.org/packages/b0/19/3742a5eed62317b03b4002ee865dc9fd720308bdd0da1f29a5786c630311/propcache-0.5.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a74bfa37147cc08fb29df10bd9c16f40fa7f860cd3a6d2fff853323a94f6e17f", size = 257723, upload-time = "2026-09-16T00:14:34.267Z" }, + { url = "https://files.pythonhosted.org/packages/cb/d5/ee6350fb0be9122bb6c67082a876d34b90d980d100c106af4b81023e04f4/propcache-0.5.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a4d7a54719b67338a305dca2ce6aafe366817df94ddfd4b5514374356f5ca546", size = 265597, upload-time = "2026-09-16T00:14:35.56Z" }, + { url = "https://files.pythonhosted.org/packages/85/9f/83a07b6ec0e043c050cfdd35fb0cf1b7897b91d554d6eea293740309afe7/propcache-0.5.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2814ecd8e818f487bee4b0f921bc4d1c176cc5fc71ac0f072d0fa67eda4ac14b", size = 250424, upload-time = "2026-09-16T00:14:36.894Z" }, + { url = "https://files.pythonhosted.org/packages/33/2c/a763a8251f50fba042af0fb1f02bfec4b31381e40aff760db2be7b2e1f84/propcache-0.5.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6af4693716bfb03f1752ef1b30faa593db2c01d5272e9b8564a1549452a979ab", size = 216748, upload-time = "2026-09-16T00:14:38.369Z" }, + { url = "https://files.pythonhosted.org/packages/6a/e2/4d11bea8fd6a777149c6c20645f873952eab5de3a2497aa11648ec9ab6ab/propcache-0.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4fbc1a15dc8cd1689508758d626b372b1f09d28d9577667feaf9e6bfcd8efcbc", size = 246533, upload-time = "2026-09-16T00:14:39.82Z" }, + { url = "https://files.pythonhosted.org/packages/9f/36/6683597de4907e70c717e3588c541202c66086a72ff3db58be49de66e72c/propcache-0.5.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cdee8205a44d0be91bbac4c41b95d86641b72dfc7aef1279400e4fda3f26a937", size = 238173, upload-time = "2026-09-16T00:14:41.259Z" }, + { url = "https://files.pythonhosted.org/packages/85/84/cb08d79f1762daafeb2b030c470cd0c725c97b8ad67412457c6f35c53e9d/propcache-0.5.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9a2a8a50a93dee0268a860a07fa3b4bd968f8ce4dbd794957da772f395368526", size = 251128, upload-time = "2026-09-16T00:14:42.652Z" }, + { url = "https://files.pythonhosted.org/packages/c2/0d/41b848036db6621370c1f2e5471a7da8149c730f8552a5257567721f4576/propcache-0.5.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7ffafcbfc7b549ab940047e505c831eabac5e67de53e1bc174adbc5285c55944", size = 214821, upload-time = "2026-09-16T00:14:44.112Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/adfae4bf9c63bccf12e2d9690a175c6579047a6eec3b5a6a5f51428c15e2/propcache-0.5.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:d1f5a500bfcbb2c0ab85e98a0dcd70f5899d34efe365a0187700369a79603031", size = 254793, upload-time = "2026-09-16T00:14:45.429Z" }, + { url = "https://files.pythonhosted.org/packages/51/6f/eeca9647245d5f92e87d53e5f14335bb42fce1a7e6842c8045b364eded8b/propcache-0.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8a235f73d6e020855dc29dff012d920c02ee0feab8d73a24185a7569f4be1161", size = 247134, upload-time = "2026-09-16T00:14:46.976Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a9/424e38838793d37160b4379c702f61c74c598fc6cd17204adbe3c554f7a8/propcache-0.5.4-cp312-cp312-win32.whl", hash = "sha256:b3083bfe87f95c756e610bd8025f26cbd1cd4aaa03a422f2d65efb7a97cd53d8", size = 43073, upload-time = "2026-09-16T00:14:48.338Z" }, + { url = "https://files.pythonhosted.org/packages/58/7b/6e8ef26f6d510a7916064fec68d55fcbfbdf7eb01e377480d66a122152d8/propcache-0.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:98914de2c4d7f0f9f4a8c6ea4bf05841f4175796941e3ef7d47eb718f22311fb", size = 46190, upload-time = "2026-09-16T00:14:49.99Z" }, + { url = "https://files.pythonhosted.org/packages/08/b9/72028c5b56ced97f456de6aefa79435ca64d7f77af78ea8cf3c76fc5195f/propcache-0.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:8876b39961e33d912afe3c1bee18ee564fdad0206f873cc15d522756b7f50737", size = 43075, upload-time = "2026-09-16T00:14:51.155Z" }, + { url = "https://files.pythonhosted.org/packages/f5/cd/785c64ed382f3f04201870267b02783f63b4678c2acfddc177a3ebcc2727/propcache-0.5.4-py3-none-any.whl", hash = "sha256:62c60aec739ed00124573cce1178138fd690c7676352d67a37328c1cf51d7468", size = 16338, upload-time = "2026-09-16T00:17:13.106Z" }, +] + [[package]] name = "py-key-value-aio" version = "0.4.5" @@ -1244,11 +1609,11 @@ wheels = [ [[package]] name = "pyjwt" -version = "2.15.0" +version = "2.15.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/a5/5197bfd06417837ac079921c66fa6393f1dea3557272a263cebfef69e432/pyjwt-2.15.0.tar.gz", hash = "sha256:b11c5f9791d7bf51c2b39a81ed669f6b2dbbd669df2942f6c60167e9e3d1abe4", size = 120513, upload-time = "2026-09-23T16:56:00.689Z" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ea/5194e52748b0da83d71e082d75496eaec6e58f419f5e184786ded517e6a9/pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8", size = 121252, upload-time = "2026-09-28T18:40:42.598Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/55/40e45bf052ee8ee12a4dfd785519660f8effa7b065442b91646ec6828619/pyjwt-2.15.0-py3-none-any.whl", hash = "sha256:7a3742debf6b879e912dbb9819ceec1594be812452b78c5f2e2dfc56564954f8", size = 33680, upload-time = "2026-09-23T16:55:59.241Z" }, + { url = "https://files.pythonhosted.org/packages/50/ca/44de4e75f8aadc457f0634be3b542815078ded46dca30efb960edeecad6e/pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193", size = 33860, upload-time = "2026-09-28T18:40:41.429Z" }, ] [package.optional-dependencies] @@ -1370,6 +1735,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, ] +[[package]] +name = "realtime" +version = "2.34.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx", extra = ["http2"] }, + { name = "pydantic" }, + { name = "typing-extensions" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/25/7f1c3a40f76c0375f9860d5e988d73e934eb8e9186694918204a221e55ec/realtime-2.34.1.tar.gz", hash = "sha256:88c10d9f4acfc524d613e4f52d5e34596b5519079b3260dfefd7feab2dab92e2", size = 24729, upload-time = "2026-11-06T15:23:04.646Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/24/ff19f90404852130c0406cde640e2c1aabb860ae66d5367500d2f8bd0b77/realtime-2.34.1-py3-none-any.whl", hash = "sha256:74f1c1839a4b74c40fdb83afc105e7eb5c359a1a31d2082751e58b87d5f936ce", size = 27381, upload-time = "2026-11-06T15:22:58.502Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -1384,6 +1764,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] +[[package]] +name = "regex" +version = "2026.11.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/61/27/a5afeb952577b84ee5fe48b8b9ce47ae662dbb69cc5b76c28eff98f83b1e/regex-2026.11.9.tar.gz", hash = "sha256:316a70fbd70e999c9664681cb0eb5b4982b2cc0f623c7c86a8a112e1bd4c6277", size = 419513, upload-time = "2026-11-09T00:42:09.786Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/81/f9c9857e531301761b612129ca24fc2d6a9f83c5d804510f0a86d97a0ce6/regex-2026.11.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9538d0bc19f87d273c2a5397a08fcd2646d02ace31ba7974483d997e36654dcf", size = 498602, upload-time = "2026-11-09T00:40:00.967Z" }, + { url = "https://files.pythonhosted.org/packages/e5/3b/58288749f3fb7303b1ee293a598813c4d08a440cc14c0c55a8fe4f7723ae/regex-2026.11.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a919f782789b9b3460bc51a1c69125551bb6164a0e401865a037397b219705b1", size = 296962, upload-time = "2026-11-09T00:40:02.404Z" }, + { url = "https://files.pythonhosted.org/packages/13/c6/32671781f2e61f40c76cf81df160ef5471264ba8a5f28cbd342cd4887a8c/regex-2026.11.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2255e173ffb8da59f26ec3529bf7e8531320e71575b27c3fb062747eb3a12e1f", size = 294199, upload-time = "2026-11-09T00:40:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/cc/ba/80e68a4d9a3cbbfd7734744a25e26e42976f50baaec786ba7c420b4855b1/regex-2026.11.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e2513028297550486dffc6e0799f24f194957418b75cd0ac2c7ad484e0d12a2", size = 804924, upload-time = "2026-11-09T00:40:04.54Z" }, + { url = "https://files.pythonhosted.org/packages/9f/57/029cdaaed8616a6088e39e0fb5c4f688e1821470f2004a90ef4738fd91d2/regex-2026.11.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:16165b67939c4fbaff46bb0380828d4fdda6936d92861baa47257399459145a4", size = 878334, upload-time = "2026-11-09T00:40:05.807Z" }, + { url = "https://files.pythonhosted.org/packages/62/48/cbbe309d95982fb72a09eb8e31a32ec719de04d061dbacba94ca892a6ae1/regex-2026.11.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b7073a7a0134621c5cfcf2c63b55cfd569cd88a6417c08cf829babba200b989d", size = 919365, upload-time = "2026-11-09T00:40:07.474Z" }, + { url = "https://files.pythonhosted.org/packages/46/c2/59c5b6cf4b762d80de04cf60f2d0deba1da2f4172a91c41eaee8933080a8/regex-2026.11.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ab4a0c5cd3df4b93841ba4a419a5438b2a4c78a4e418682fc674e3f50d56b857", size = 808826, upload-time = "2026-11-09T00:40:08.864Z" }, + { url = "https://files.pythonhosted.org/packages/e1/7d/18e82a2dc59bbebbea7426d3d5034cb9790230c2826d55ce090e6c8fc38b/regex-2026.11.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e3fe4474aeaa34535077bd3cdee6294756e99f4ed498b09712b5f3dc0b2947a5", size = 783421, upload-time = "2026-11-09T00:40:10.376Z" }, + { url = "https://files.pythonhosted.org/packages/40/a3/4c561718974dad242b481c2d9399a2a2b5d29b357cb6bed52427d6c0c2d9/regex-2026.11.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b47bd645e6305d22a31dc063635de40796e93e72cfeb9a86108dfa806ce866fa", size = 791492, upload-time = "2026-11-09T00:40:11.582Z" }, + { url = "https://files.pythonhosted.org/packages/fc/70/80a9f43944dd81012c2e10f4ae295d91f8db6929e79b10f1232e00b10d78/regex-2026.11.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ad0795929a24ad49f096c464f0ff7fb4d66488c6f20c2b1da724e2067e3d5c2d", size = 866863, upload-time = "2026-11-09T00:40:13.188Z" }, + { url = "https://files.pythonhosted.org/packages/26/d0/93464b1455c35000058c368fa989f21d09e9fceeb2289d1be4df93ded78f/regex-2026.11.9-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:f1ed6b50f54b8467bb3085fb6d701f3eb77c9ad81071da498cdc7725b4867f92", size = 770521, upload-time = "2026-11-09T00:40:14.703Z" }, + { url = "https://files.pythonhosted.org/packages/1f/5d/34c5aec48033f7929beaad8623e19ad7957b0de7ebffeae96b9689f1fedd/regex-2026.11.9-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e6f945b6fcbf1a51760ab41b61159dba8bbcf3614ed921bd175d62529be5ec26", size = 861457, upload-time = "2026-11-09T00:40:16.091Z" }, + { url = "https://files.pythonhosted.org/packages/94/59/80c4299ca9422b44738f8559d850e96d09a201334be41499863877980cb9/regex-2026.11.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ac0114fa49c175803fce562ed243bfaf7f6943edf4b2a01ef82f0f61e7e397", size = 795558, upload-time = "2026-11-09T00:40:17.222Z" }, + { url = "https://files.pythonhosted.org/packages/4c/cc/f14bf9e6e0ed44f29402bd8319ca231f2dce52c76084fc8561abaeda2e45/regex-2026.11.9-cp312-cp312-win32.whl", hash = "sha256:2f1c997fcb7a74006e3c4f09199d9e835bb46eb12c1c909f8b8259627f7b5ec7", size = 269777, upload-time = "2026-11-09T00:40:18.606Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ef/e03271355a802f38f7510e31e006667fc8b95b771058d75a36e2afda61b2/regex-2026.11.9-cp312-cp312-win_amd64.whl", hash = "sha256:ae42ad9825cde4cd374abfce6bc82233b91099170c138b349554813e97da5100", size = 280891, upload-time = "2026-11-09T00:40:19.698Z" }, + { url = "https://files.pythonhosted.org/packages/a3/08/551312e6599ac812b4052b5c9d3b3097b716f853c01e8ae6a150f805345b/regex-2026.11.9-cp312-cp312-win_arm64.whl", hash = "sha256:3c3886844cf8d2ce2b2cddb849eff6c404bfa951c77bd63fb4bd7ed270f135dc", size = 280256, upload-time = "2026-11-09T00:40:20.726Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -1448,6 +1852,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, ] +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + [[package]] name = "safehttpx" version = "0.1.7" @@ -1460,13 +1876,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2e/a3/0f0b7d78e2f1eb9e8e1afbff1d2bff8d60144aee17aca51c065b516743dd/safehttpx-0.1.7-py3-none-any.whl", hash = "sha256:c4f4a162db6993464d7ca3d7cc4af0ffc6515a606dfd220b9f82c6945d869cde", size = 8959, upload-time = "2025-10-24T18:30:08.733Z" }, ] +[[package]] +name = "scantree" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "pathspec" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/e4/40998faefc72ba1ddeb640a44fba92935353525dba110488806da8339c0b/scantree-0.0.4.tar.gz", hash = "sha256:15bd5cb24483b04db2c70653604e8ea3522e98087db7e38ab8482f053984c0ac", size = 24643, upload-time = "2024-08-03T20:08:59.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/ce/828467ddfa0d2fe473673026442d2032d552a168e42cfbf25fd0e5264e0c/scantree-0.0.4-py3-none-any.whl", hash = "sha256:7616ab65aa6b7f16fcf8e6fa1d9afaa99a27ab72bba05c61b691853b96763174", size = 20690, upload-time = "2024-08-03T20:08:58.137Z" }, +] + [[package]] name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography" }, - { name = "jeepney" }, + { name = "cryptography", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "jeepney", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -1482,6 +1911,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177", size = 15552, upload-time = "2022-05-26T13:35:21.206Z" }, ] +[[package]] +name = "semver" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0e/50/2932d6434608c7d954d0d4f4b71b697260e3b40c802425fc84c1eed12440/semver-3.1.1.tar.gz", hash = "sha256:149b26cabada6f63212f97e94505d21578cb1d280cf714827f328d5e70765fea", size = 292174, upload-time = "2026-11-09T11:38:46.145Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/1d/793b054cbd0a2e0b4694dacc22bad066d6635ce85dd6cdd1f98d95169e9b/semver-3.1.1-py3-none-any.whl", hash = "sha256:c00cf7dac88a352bc6a4a997d6ac0bf95ba835c589bc0c54f7b76f9e2a45123a", size = 18905, upload-time = "2026-11-09T11:38:44.556Z" }, +] + [[package]] name = "setuptools" version = "84.1.0" @@ -1500,6 +1938,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "shortuuid" +version = "1.0.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/e2/bcf761f3bff95856203f9559baf3741c416071dd200c0fc19fad7f078f86/shortuuid-1.0.13.tar.gz", hash = "sha256:3bb9cf07f606260584b1df46399c0b87dd84773e7b25912b7e391e30797c5e72", size = 9662, upload-time = "2024-03-11T20:11:06.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/44/21d6bf170bf40b41396480d8d49ad640bca3f2b02139cd52aa1e272830a5/shortuuid-1.0.13-py3-none-any.whl", hash = "sha256:a482a497300b49b4953e15108a7913244e1bb0d41f9d332f5e9925dba33a3c5a", size = 10529, upload-time = "2024-03-11T20:11:04.807Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -1544,6 +1991,130 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/cb/6a6a47d5b464bd08695d254f3da6e7986cc70c9fa5d778eda57538edfe56/starlette-1.6.0-py3-none-any.whl", hash = "sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c", size = 75969, upload-time = "2026-08-08T18:27:56.196Z" }, ] +[[package]] +name = "storage3" +version = "2.34.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecation" }, + { name = "httpx", extra = ["http2"] }, + { name = "pydantic" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/b2/1fdee833f9d3db231feda833c6969b876ac31e64bcd42cb2cd102596c3f7/storage3-2.34.1.tar.gz", hash = "sha256:e5462b05f095ba6ea59d95f70c5841dfc67922401aa7f9570e151e9df96c05bd", size = 20483, upload-time = "2026-11-06T15:23:05.552Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/fa/194142a00be8cc39329991d648d3418e12a8939296366164dab7424d0931/storage3-2.34.1-py3-none-any.whl", hash = "sha256:0f2615d75bd9415b804274a430bb89ee0bead1bec0c0a26717aecbdb0ee05b30", size = 28641, upload-time = "2026-11-06T15:22:59.529Z" }, +] + +[[package]] +name = "supabase" +version = "2.34.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "postgrest" }, + { name = "realtime" }, + { name = "storage3" }, + { name = "supabase-auth" }, + { name = "supabase-functions" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/c5/958cd70894e414eb00da45d2c202c3e66c62eb00905c08fa458f6c7dc7fd/supabase-2.34.1.tar.gz", hash = "sha256:122cb31eb917e7a61817e4b51a754c7a858eecc882d44af4f1c3b26f1ebd0a5c", size = 10077, upload-time = "2026-11-06T15:23:06.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/50/cc59d5a7e0278a137e957a959627f5f9d988a45c4867b1162665eab7bb0d/supabase-2.34.1-py3-none-any.whl", hash = "sha256:bf06e59d0b37fbbdbc4d4d1f6b60844c517b744aa9bc34432deae06f59a6cc1d", size = 17096, upload-time = "2026-11-06T15:23:00.531Z" }, +] + +[[package]] +name = "supabase-auth" +version = "2.34.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx", extra = ["http2"] }, + { name = "pydantic" }, + { name = "pyjwt", extra = ["crypto"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/70/52956c5ee237bffafb1cb3c99af9491b37ab67e481eadd0002ddead97e17/supabase_auth-2.34.1.tar.gz", hash = "sha256:d4d7d89ef16f55d1562b6408a361c1982d5f9764c43a64d18222655ee58c3207", size = 41293, upload-time = "2026-11-06T15:23:08.205Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/0f/64b8e050da58a527f00f14787b8bf0f1aa2b091810e13cf1fb4002b336ce/supabase_auth-2.34.1-py3-none-any.whl", hash = "sha256:45aa3334335926a8940beb89a9438386a9af0561d7315797e46c47b50d0c9620", size = 50455, upload-time = "2026-11-06T15:23:01.689Z" }, +] + +[[package]] +name = "supabase-functions" +version = "2.34.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx", extra = ["http2"] }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/4f/c481b8fc17d136b20892adba018e7156ceccbba4cc3e9e5f25d9e1dbd121/supabase_functions-2.34.1.tar.gz", hash = "sha256:873947ae6015be3bca68bf6da6eab58ae0b24b19ff9136ba31b07ad38a730d98", size = 5546, upload-time = "2026-11-06T15:23:08.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/a6/41051983ca00712bea46da8749b0c2dcf75db33a9df69f9bbbb289e772b1/supabase_functions-2.34.1-py3-none-any.whl", hash = "sha256:6829e5138de7af07197fb2e3dc9f067a50a6bd5615476494b92fa49d4f429c6a", size = 10258, upload-time = "2026-11-06T15:23:02.879Z" }, +] + +[[package]] +name = "tenacity" +version = "9.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/9e/497c1c8ebe5a5b5d1d4a7511aea22c0bb1a97e3170d98abdef0e1b34265a/tenacity-9.2.1.tar.gz", hash = "sha256:a606b5c808d0cded4a359d5b9932d867ff2a6a6b64d37350260fd01bbdf83839", size = 58261, upload-time = "2026-10-07T12:13:01.633Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/26/1ff2b0721ac66a3ec5b1402b333110b352ab0a8724052ac279a7b82d40c4/tenacity-9.2.1-py3-none-any.whl", hash = "sha256:9e56f17539296baab7beabb08b92f6ee3d7be92d8be72d763360677c2ad6580e", size = 32310, upload-time = "2026-10-07T12:13:00.102Z" }, +] + +[[package]] +name = "tiktoken" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/62/167a842aa0429d45f5e797354fd4343a96f6043d67d0513c675c7b8d36e6/tiktoken-0.14.0.tar.gz", hash = "sha256:231dec90efcdccf1b565a1416107736f1e09b1a08fe736ef9d6363e626d03874", size = 38898, upload-time = "2026-08-17T19:49:49.514Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/da/e273746b9d24a63c776bc60fba914351573ad9c575b52601eb5e60632564/tiktoken-0.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8e947aefe98ef74cce94923f90e48c98fe34eb1ec0a6bfdfadfc5a96359bfc36", size = 1094408, upload-time = "2026-08-17T19:48:49.269Z" }, + { url = "https://files.pythonhosted.org/packages/69/9f/fe6b1aca23331aa5271df5a4bd07bf68a7059254d47faee1b8272592a777/tiktoken-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d6cebe67765569df3dafac8474e4eccf5c19d24140492567a5e58a11445732a4", size = 1038499, upload-time = "2026-08-17T19:48:50.666Z" }, + { url = "https://files.pythonhosted.org/packages/0b/35/e9f47647c9e163bd1de30fe1a491669b7248cfc67b7404c35c009a701e1a/tiktoken-0.14.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:7db45b98e94adf4173a5cd7422b150999a7ee11ff847783a14f6e1b80cc38cb6", size = 1186355, upload-time = "2026-08-17T19:48:51.93Z" }, + { url = "https://files.pythonhosted.org/packages/51/11/9976ad86980a00cdef05e730a0127a2578a1bc6d11644d8d47246de2eb26/tiktoken-0.14.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7896eea257fe497a2b7134474d909156c6744ce8da35bce88011a960e008aa0d", size = 1204197, upload-time = "2026-08-17T19:48:53.18Z" }, + { url = "https://files.pythonhosted.org/packages/d4/9c/7035b0bcfaa68d1ee4803fc5be5214ad865669b05bd20e7105ae8a18afc6/tiktoken-0.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b950248272f1b303dc32986396e2dccfa10cf6d1e83ec8f0bba1776660305482", size = 1250635, upload-time = "2026-08-17T19:48:54.392Z" }, + { url = "https://files.pythonhosted.org/packages/bc/1d/69cabf18bed7f4366da076735816abce0d4db3fae491ae338a6612128777/tiktoken-0.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3de75343041a1c57333b1e707ac8a9769738241d7d6a55d39e12cf84548337c6", size = 1316085, upload-time = "2026-08-17T19:48:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/bd/bd/a2e884fb1402cba5be08836590320012b2d8ada0e2eef9911a64df4bcd2d/tiktoken-0.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:087538c080e5ff421abd3a0785ed63c5111d06af98e6cd0d374dbe5969147ca3", size = 941208, upload-time = "2026-08-17T19:48:56.938Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.23.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/7c/2cabb2174e772636683008f2c5621949b645da7d303c596589e84516a184/tokenizers-0.23.3.tar.gz", hash = "sha256:cded33237c77caeef62944d32aa9a7ef42bdce2b3497e18d137e072a8c4be438", size = 385286, upload-time = "2026-10-09T10:16:55.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/2e/4ce5b9716f26e526eff6b0502ebed4ea8d7161f03b3c77617c9f25528e97/tokenizers-0.23.3-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9d2b5c97daf61688c2ad1803ca851800feaba50fb68d5821779e9ea5880d968c", size = 3148800, upload-time = "2026-10-09T10:00:51.457Z" }, + { url = "https://files.pythonhosted.org/packages/b2/72/01e49f032bb346e5aaf06c10c74fe8aeec847173adbadd66eb7c53054bf2/tokenizers-0.23.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:68649e97d5b43c44c031d8d848874a6eecae8f8fe40ea989aa777a5a83aca716", size = 3101381, upload-time = "2026-10-09T10:00:54.063Z" }, + { url = "https://files.pythonhosted.org/packages/15/fc/ae987741829b1cd547668c4c94be732ae3eefd1d74344e64c3d2ca714acd/tokenizers-0.23.3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec82e80e65a862275b97c3d90b7a523df8d9519ee48aeb4e9625b2cc909274e0", size = 3519944, upload-time = "2026-10-09T10:00:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/1c/da/cc8f6c030afaf05fbddc608158fbb761dca46913cbeba6b112e59fc82e2a/tokenizers-0.23.3-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c64a0713180ff16829d4e7f39a658b77ea11443af4e1aa46523692943c9b1414", size = 3397695, upload-time = "2026-10-09T10:00:57.444Z" }, + { url = "https://files.pythonhosted.org/packages/ec/f1/256f78d1365fa2cd3ea6db716883d74667c8cbb6a21f15fa5b89a773cdc2/tokenizers-0.23.3-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddedfd4b3b4be6be24ff6ca645c4a37fddfd305f6f3e354c54cf10b715c48215", size = 3753125, upload-time = "2026-10-09T10:01:00.165Z" }, + { url = "https://files.pythonhosted.org/packages/60/93/eee007ac2fcbf4ecfce7fbc354826cf3611f56bdb886f3e91b1f7dd06b8f/tokenizers-0.23.3-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a89614730d7b80940a5d2ed9320e1ec8add5a745c6151d8d05071b7215505b6", size = 4018598, upload-time = "2026-10-09T10:01:02.05Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f9/0c96c4739461fce9d8d865b416728081bf6230022d7163bd6244f35f4b31/tokenizers-0.23.3-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e88646b8580c5ad7f4361477f1298e9cc01771a1ee9aecfe32c47b8ff614cc38", size = 3602442, upload-time = "2026-10-09T10:01:03.77Z" }, + { url = "https://files.pythonhosted.org/packages/3a/40/6706b82693715581457c6d5423eaa7faae576bb0526c5738a57085eb4449/tokenizers-0.23.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:376851d22bcf9d650a5c3090bb83e6cf9e895fbf0595369fa4cd43c1f69b5f87", size = 3396193, upload-time = "2026-10-09T10:01:05.48Z" }, + { url = "https://files.pythonhosted.org/packages/fe/0c/85946de40e25b7364b8f1bcf56def129069acd5bb364b7c86a32919e1a23/tokenizers-0.23.3-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:bf501c40b72d2d5c8623620210430e9cac1ce47a46e45b34107b70a1557d46b0", size = 3553483, upload-time = "2026-10-09T10:01:07.387Z" }, + { url = "https://files.pythonhosted.org/packages/f1/6b/8d615d92cad1d511ca5ab188d1c7c167f0b3d295cc0d96207f9f82d486d8/tokenizers-0.23.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:114e2b55ed177179d59f4ab98200a4471e11e78f9e4b5a922d146740f96fcf52", size = 9972248, upload-time = "2026-10-09T10:01:09.437Z" }, + { url = "https://files.pythonhosted.org/packages/c9/7d/a922e37ddd58d1b463bbc2ad08120c8f59c60b814cd353519a116b24f8ba/tokenizers-0.23.3-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:d3407fb7b9c4d75dd68850ffd7180bc0a5d2dbaf0762d888e612f31fec3f9c6b", size = 9802957, upload-time = "2026-10-09T10:01:11.869Z" }, + { url = "https://files.pythonhosted.org/packages/4b/06/5d3f506a86ae0699a0e4ea05c05978f9aee169ef2c1d844e68c971cf8194/tokenizers-0.23.3-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:84513ef0aeb8bf8f4ea11a2e8a7ac163ec5288aa115e649a59b470ac5c3107df", size = 10145487, upload-time = "2026-10-09T10:01:14.268Z" }, + { url = "https://files.pythonhosted.org/packages/26/e5/065625317690ea3548d834dad81f48ea1fd32e4964610e658e195d7fe28e/tokenizers-0.23.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e05ab7baf7f47b406a95fea6f3b0a484b2ddcd9e1d14b68844c457eb755085a3", size = 10266026, upload-time = "2026-10-09T10:16:33.054Z" }, + { url = "https://files.pythonhosted.org/packages/77/4e/babede85d0d19f5e3deeef0063e01848141329934d3d77c31b5cab5ac2b4/tokenizers-0.23.3-cp310-abi3-win32.whl", hash = "sha256:1ebf28794e7e4954e20a7f70fbea410b2d1f0418f7dbbca97ca384fcfef38c25", size = 2588086, upload-time = "2026-10-09T10:16:35.686Z" }, + { url = "https://files.pythonhosted.org/packages/d1/6c/24f074c9a0efb98e61b20aafe6b2641922d5db24e447d5d6daffd9e17555/tokenizers-0.23.3-cp310-abi3-win_amd64.whl", hash = "sha256:1f0823bb00c5fdc98e487354d54dd55a03848d61a1a0bf29a68c77f24f3b26c3", size = 2872101, upload-time = "2026-10-09T10:16:37.533Z" }, + { url = "https://files.pythonhosted.org/packages/53/77/a476b6f73a661c11d113a342d2326b91506cf2285f0995d1212a6bb2022d/tokenizers-0.23.3-cp310-abi3-win_arm64.whl", hash = "sha256:7e48734d2de9260d86f03ab056d2cfeeff3869f61dbd49aaa15a2793b5f3458b", size = 2742580, upload-time = "2026-10-09T10:16:39.244Z" }, +] + +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -1704,31 +2275,22 @@ wheels = [ [[package]] name = "websockets" -version = "17.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/72/fba934cb3dff7a85d811820efffcd141ddd52b5a2a01637f64551373ff4d/websockets-17.1.tar.gz", hash = "sha256:acfea4c20bf54384883ea33b1240fc1db4f52e190823a4e2b334bc3e8bfca96a", size = 187520, upload-time = "2026-08-26T17:25:33.063Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/0d/098f23c4c858e5de9459ffc554fa07d5493fbcfca7f040b5800cf1cecc35/websockets-17.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:76dd004f59115087c7b700474cb18f01325e37250032e19396c08ae41448e4b3", size = 217015, upload-time = "2026-08-26T14:55:45.194Z" }, - { url = "https://files.pythonhosted.org/packages/13/86/bc1317b1a4d8c4688e2a7e564b5e004dab44c2534d7ca05de6ae9a863fca/websockets-17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:581fa678ef46f4277cc8491312468e582f8ad609dbab907ba6096a08c6a0ff98", size = 214692, upload-time = "2026-08-26T14:55:46.366Z" }, - { url = "https://files.pythonhosted.org/packages/8f/e7/df821761772beaa48c211ee0e234930b35c1473778470773823f56d3911b/websockets-17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:87f0d5e77548b0c40c8464cdb6108792e7e53f487c6400028a4ec28a8afbe5ab", size = 214959, upload-time = "2026-08-26T14:55:47.885Z" }, - { url = "https://files.pythonhosted.org/packages/3e/92/c3fb72f11764812fc648bf3838d224972427b348e8b3989d9e0a9df87da3/websockets-17.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:882af300d2c6a092b93767d5de03c7bb56dfb06314140c8e872d3f48e09f7b74", size = 224278, upload-time = "2026-08-26T14:55:49.241Z" }, - { url = "https://files.pythonhosted.org/packages/fb/05/9f82d090c8d2d861604147ef6dfb938a90b039f9358d5193f1df62558593/websockets-17.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0c863507ada5805517ca6dff1c524dcd42942efe6304dacf06700878398d21a6", size = 224557, upload-time = "2026-08-26T14:55:50.348Z" }, - { url = "https://files.pythonhosted.org/packages/8a/50/5cbf677b865290fe36819ff00615826e7edc1df38786f770123ff39a933d/websockets-17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d41ef69d5416fbc1d98cf96c37be6192d10fd101c3e0f8b3ddc36e09432b3c08", size = 225791, upload-time = "2026-08-26T14:55:51.75Z" }, - { url = "https://files.pythonhosted.org/packages/c1/1c/eb8a032285243381b09a221ae384c972d5000453ad136add4d1595cec798/websockets-17.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5aefe78e6a3077fe22b5e64b04666a85a3eb8b934d40e8595a693adcbceb6f11", size = 228574, upload-time = "2026-08-26T14:55:52.922Z" }, - { url = "https://files.pythonhosted.org/packages/69/85/413736251cb3ac04ce84cbd90e893d9a36a9698d4820b323aff3aa187e50/websockets-17.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f64e001bb7fa89b9f32cfa600bf8e9ac8ca26759d9b92ae01453ee303d9cd7b4", size = 226428, upload-time = "2026-08-26T14:55:54.263Z" }, - { url = "https://files.pythonhosted.org/packages/d2/2b/a08bcc7fa1ca81a10f84ba32b6e6edd73a913f4b0c2640eed1fd626efacd/websockets-17.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:677014a073bcb1fbaa7e21144786864f16c08f856d66834f611eceb9006cbab8", size = 225184, upload-time = "2026-08-26T14:55:55.943Z" }, - { url = "https://files.pythonhosted.org/packages/e5/8a/3bd2d0cf6b148c8c866d5d9fdcde30c04bfd81fdfac86813e69377eb4448/websockets-17.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0de501b7f2db11e83739ac20e2d33d46da4604b829f506c24be80e7def069391", size = 222430, upload-time = "2026-08-26T14:55:57.103Z" }, - { url = "https://files.pythonhosted.org/packages/3b/c9/8e891ae342668735eabbbc669895e15195e4b45f24a4beeb58af76f414c7/websockets-17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f62114a54117e4948a1e414e89521f7fe1e3c2f83f2a571a06a4fc6718b0900a", size = 225227, upload-time = "2026-08-26T14:55:58.375Z" }, - { url = "https://files.pythonhosted.org/packages/e1/6f/c816f332dca11425e9bda7c07f7573eb5c5f8a735849d02b0d81e8ee20fa/websockets-17.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:eec113a5b41d124ef42ff56b0d74a6da3fd986400038eab9e58ee42a4024e837", size = 223831, upload-time = "2026-08-26T14:55:59.664Z" }, - { url = "https://files.pythonhosted.org/packages/53/67/5e91d5308ce24fc1ec74f56536c12f4888bad45ff5ea50f3180f8c518c57/websockets-17.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5f051f8030a51815dc00e24bd2e5f1435af095c1cc111d747ac6e2a3620d7641", size = 224600, upload-time = "2026-08-26T14:56:00.873Z" }, - { url = "https://files.pythonhosted.org/packages/bb/96/faa298ecf2570d35b0eb37caddf4992178d907e108ed74bfffb6bc092c29/websockets-17.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:655a8e28010f09fd6fa317e857afab3af7647f33e41dee88fa421e92086d1090", size = 225707, upload-time = "2026-08-26T14:56:02.001Z" }, - { url = "https://files.pythonhosted.org/packages/0b/12/5710d2482ca5061c1eec5eb46f6313837c760d4115b1795c85b6c08be4e3/websockets-17.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:dc2b79afc074d2f3e64b26539350f697fe1b85ea1c49ea24eb588f247b053ce1", size = 223263, upload-time = "2026-08-26T14:56:03.092Z" }, - { url = "https://files.pythonhosted.org/packages/27/47/0c30f4eebfd1d93fae779d268f678d48847fb98516f5200849574eee8820/websockets-17.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e4bd7eacb87d8cf3ed70d6392c770a0d92441f05d7d2a3efafb5bc171d5e3067", size = 224244, upload-time = "2026-08-26T14:56:04.321Z" }, - { url = "https://files.pythonhosted.org/packages/41/33/46c256195a1255079ae23d1b1267b2e1843dc5f46a67f973cdf2a3523dff/websockets-17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ccbf3f4a9890d50b3a08ee04029fde30a03bfdeffaa19977628bf17251764e60", size = 224520, upload-time = "2026-08-26T14:56:05.521Z" }, - { url = "https://files.pythonhosted.org/packages/06/9a/aef0792731df4352e5f417369b532b3325fe434765ca90c193f594ae1e67/websockets-17.1-cp312-cp312-win32.whl", hash = "sha256:7e724f843fa6a0614aece65a7c73e51d0f4412ca41dccac13c3caf98e69536bb", size = 217485, upload-time = "2026-08-26T14:56:06.715Z" }, - { url = "https://files.pythonhosted.org/packages/50/23/493ecfdaf32898e5ea24dc900e33e5e317f9662d5d9ab2d44b2e111b4e1c/websockets-17.1-cp312-cp312-win_amd64.whl", hash = "sha256:617243e19a0992095956f406ee9cd3bc4ba92862d83cb1d83bb59ce574412bec", size = 217786, upload-time = "2026-08-26T14:56:08.055Z" }, - { url = "https://files.pythonhosted.org/packages/97/3d/91954e2f7876f74ce1213e9b92c65a63b559cc4b942a931ebeb351cd9932/websockets-17.1-cp312-cp312-win_arm64.whl", hash = "sha256:9f4a08ff7cb68c27b18e09223cc6304e01d0f82d5a240d251266dfd2e6e44729", size = 217711, upload-time = "2026-08-26T14:56:09.267Z" }, - { url = "https://files.pythonhosted.org/packages/41/63/23572870e01836a98346075b9e17a8bc24a6ddd9800a3204ceee58677f3c/websockets-17.1-py3-none-any.whl", hash = "sha256:f221081107b8c48184d99f7019604486376e7ef826037e70aad6b02540732c23", size = 211134, upload-time = "2026-08-26T17:25:31.397Z" }, +version = "15.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/e6/26d09fab466b7ca9c7737474c52be4f76a40301b08362eb2dbc19dcc16c1/websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee", size = 177016, upload-time = "2025-03-05T20:03:41.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/6b/4545a0d843594f5d0771e86463606a3988b5a09ca5123136f8a76580dd63/websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3", size = 175437, upload-time = "2025-03-05T20:02:16.706Z" }, + { url = "https://files.pythonhosted.org/packages/f4/71/809a0f5f6a06522af902e0f2ea2757f71ead94610010cf570ab5c98e99ed/websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665", size = 173096, upload-time = "2025-03-05T20:02:18.832Z" }, + { url = "https://files.pythonhosted.org/packages/3d/69/1a681dd6f02180916f116894181eab8b2e25b31e484c5d0eae637ec01f7c/websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2", size = 173332, upload-time = "2025-03-05T20:02:20.187Z" }, + { url = "https://files.pythonhosted.org/packages/a6/02/0073b3952f5bce97eafbb35757f8d0d54812b6174ed8dd952aa08429bcc3/websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215", size = 183152, upload-time = "2025-03-05T20:02:22.286Z" }, + { url = "https://files.pythonhosted.org/packages/74/45/c205c8480eafd114b428284840da0b1be9ffd0e4f87338dc95dc6ff961a1/websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5", size = 182096, upload-time = "2025-03-05T20:02:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/14/8f/aa61f528fba38578ec553c145857a181384c72b98156f858ca5c8e82d9d3/websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65", size = 182523, upload-time = "2025-03-05T20:02:25.669Z" }, + { url = "https://files.pythonhosted.org/packages/ec/6d/0267396610add5bc0d0d3e77f546d4cd287200804fe02323797de77dbce9/websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe", size = 182790, upload-time = "2025-03-05T20:02:26.99Z" }, + { url = "https://files.pythonhosted.org/packages/02/05/c68c5adbf679cf610ae2f74a9b871ae84564462955d991178f95a1ddb7dd/websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4", size = 182165, upload-time = "2025-03-05T20:02:30.291Z" }, + { url = "https://files.pythonhosted.org/packages/29/93/bb672df7b2f5faac89761cb5fa34f5cec45a4026c383a4b5761c6cea5c16/websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597", size = 182160, upload-time = "2025-03-05T20:02:31.634Z" }, + { url = "https://files.pythonhosted.org/packages/ff/83/de1f7709376dc3ca9b7eeb4b9a07b4526b14876b6d372a4dc62312bebee0/websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9", size = 176395, upload-time = "2025-03-05T20:02:33.017Z" }, + { url = "https://files.pythonhosted.org/packages/7d/71/abf2ebc3bbfa40f391ce1428c7168fb20582d0ff57019b69ea20fa698043/websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7", size = 176841, upload-time = "2025-03-05T20:02:34.498Z" }, + { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, ] [[package]] @@ -1742,3 +2304,43 @@ sdist = { url = "https://files.pythonhosted.org/packages/d0/20/50ed6bdf27dec98b5 wheels = [ { url = "https://files.pythonhosted.org/packages/2e/29/69cfbb602cd91690c55d38ba9fe53e6a7e76a6fa647bf38f19c138d25449/wheel-0.48.0-py3-none-any.whl", hash = "sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab", size = 33320, upload-time = "2026-08-11T22:02:26.1Z" }, ] + +[[package]] +name = "yarl" +version = "1.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/16/e8be8e2fb175bbf41a0680381a319f1199fae256588241a2ac8677eafb49/yarl-1.25.1.tar.gz", hash = "sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7", size = 246245, upload-time = "2026-09-15T19:35:02.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/b3/cd32ac66ae622b854c2df0ac52106dda220d361b65a64fde7d5b3684aa3f/yarl-1.25.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4", size = 144798, upload-time = "2026-09-15T19:31:01.821Z" }, + { url = "https://files.pythonhosted.org/packages/61/fb/a2c52a8007c2051ba74662afb112ecf3d00346af4c25e33df9d80fd14fb8/yarl-1.25.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d", size = 104583, upload-time = "2026-09-15T19:31:04.05Z" }, + { url = "https://files.pythonhosted.org/packages/be/dd/ee38aec8e09fdf957e50d4085453fbe202f56c6c3b4cf07b81cdb4f09ee9/yarl-1.25.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6", size = 104325, upload-time = "2026-09-15T19:31:06.338Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b3/058dbfb1857b484c9cf9cc135659f50b85ce66e03c99e44dc2f7b6161f55/yarl-1.25.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58", size = 115358, upload-time = "2026-09-15T19:31:08.593Z" }, + { url = "https://files.pythonhosted.org/packages/db/39/29693446cf0cf6b15a0e2f75a5d40f93c56819b05b0622196f45e95b5cc0/yarl-1.25.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad", size = 107658, upload-time = "2026-09-15T19:31:10.802Z" }, + { url = "https://files.pythonhosted.org/packages/86/b3/3c4dd7e1af43b931fba95e0a722737f2ea94a6d199c802585282831d7abd/yarl-1.25.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40", size = 122660, upload-time = "2026-09-15T19:31:12.84Z" }, + { url = "https://files.pythonhosted.org/packages/bd/b5/1b60dbc3cfc9c5712b15148c206748f2bc93953ffdbe25ea75b63dfc89c9/yarl-1.25.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c", size = 126506, upload-time = "2026-09-15T19:31:15.088Z" }, + { url = "https://files.pythonhosted.org/packages/bc/7b/ca212cbe170ac8b96e45317ecbcf9c3c3ecf0cdec98d5b088a9c4088929b/yarl-1.25.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d", size = 117050, upload-time = "2026-09-15T19:31:17.241Z" }, + { url = "https://files.pythonhosted.org/packages/cb/c3/72b4938cdbe619ad71ac156182faef4908846b84dc3ca4dbb4c4e6f84014/yarl-1.25.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54", size = 114174, upload-time = "2026-09-15T19:31:19.294Z" }, + { url = "https://files.pythonhosted.org/packages/e8/43/268717870f9ba0cc9701a95181587f6dc8c5f387aab4aeecc83158f38a79/yarl-1.25.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f", size = 114944, upload-time = "2026-09-15T19:31:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/da/84/baa5bf504d51fe062c4bcaf62936da97fffb43285978d0b39984824231fd/yarl-1.25.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc", size = 108263, upload-time = "2026-09-15T19:31:23.388Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/779a2ed9e0152a601a27039bed9aead3f0b79797a67e2c44bfa444622dd8/yarl-1.25.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d", size = 122184, upload-time = "2026-09-15T19:31:25.343Z" }, + { url = "https://files.pythonhosted.org/packages/f8/1f/118e9e5b8f07694d63fd3222e801d7782270003f1a222aa798df3f8d5933/yarl-1.25.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1", size = 114001, upload-time = "2026-09-15T19:31:27.465Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/a4cf1cf372313734b17996d4007f9f73596e7a178b9485802e5494ecf484/yarl-1.25.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68", size = 120565, upload-time = "2026-09-15T19:31:29.47Z" }, + { url = "https://files.pythonhosted.org/packages/05/79/ad94f93ca731bc9e44d321833ab96b82a4f9f5f63cf773f81a4aeea5ecc1/yarl-1.25.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707", size = 117060, upload-time = "2026-09-15T19:31:31.367Z" }, + { url = "https://files.pythonhosted.org/packages/bb/cc/51a7b4abf4ac593b8e7eb3794b28e5a35ae26eed8bc04787628d215af82f/yarl-1.25.1-cp312-cp312-win_amd64.whl", hash = "sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03", size = 102593, upload-time = "2026-09-15T19:31:33.329Z" }, + { url = "https://files.pythonhosted.org/packages/9d/21/0941a6b93a58b59a1ec75e5333bf06929b671309c43c0cd201c172d9c39f/yarl-1.25.1-cp312-cp312-win_arm64.whl", hash = "sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85", size = 97697, upload-time = "2026-09-15T19:31:35.855Z" }, + { url = "https://files.pythonhosted.org/packages/54/22/318c7980066769c6bcd9221ed2248294f5698811da099013098c670565ed/yarl-1.25.1-py3-none-any.whl", hash = "sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3", size = 63617, upload-time = "2026-09-15T19:34:59.616Z" }, +] + +[[package]] +name = "zipp" +version = "4.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/05/ba1458dee0ac485b2a01071c1b88f5199fe878a485df482e10e1ef2205cc/zipp-4.1.2.tar.gz", hash = "sha256:cf474b90b07ea932637e17c6b9479de2b2882900b0d311bdaf4f5c000a15a97f", size = 27864, upload-time = "2026-10-21T18:57:04.743Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/44/7bd2007260da11cccca29ba2d4c345f1287aa663db38ee7cefeabf6386a5/zipp-4.1.2-py3-none-any.whl", hash = "sha256:b30cb6cd253122169c47de245ca24a86bac2a5d3802ba15bf7ba6414eb9e144d", size = 10581, upload-time = "2026-10-21T18:57:03.79Z" }, +] From 156e411eeab7beb359c0f7d3fd567263a730203c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 10 Nov 2026 17:14:49 +0000 Subject: [PATCH 10/40] fix(validation): keep Harbor out of fixture export --- tests/validation_runtime/pyproject.toml | 8 ++++++-- tests/validation_runtime/uv.lock | 10 ++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/validation_runtime/pyproject.toml b/tests/validation_runtime/pyproject.toml index de190c4ef..733d002b7 100644 --- a/tests/validation_runtime/pyproject.toml +++ b/tests/validation_runtime/pyproject.toml @@ -4,8 +4,6 @@ version = "0.1.0" requires-python = "==3.12.*" dependencies = [ "openenv", - # The Harbor task binding's tests import Harbor; a skip fails --require-complete. - "harbor==0.25.0", "pytest==8.4.2", "pytest-asyncio==1.2.0", "jsonschema==4.25.1", @@ -17,5 +15,11 @@ dependencies = [ [tool.uv] package = false +[dependency-groups] +dev = [ + # The Harbor task binding's tests import Harbor; a skip fails --require-complete. + "harbor==0.25.0", +] + [tool.uv.sources] openenv = { path = "../..", editable = false } diff --git a/tests/validation_runtime/uv.lock b/tests/validation_runtime/uv.lock index ffb7f703b..800989ac6 100644 --- a/tests/validation_runtime/uv.lock +++ b/tests/validation_runtime/uv.lock @@ -1304,7 +1304,6 @@ name = "openenv-validation-runtime-tests" version = "0.1.0" source = { virtual = "." } dependencies = [ - { name = "harbor" }, { name = "jsonschema" }, { name = "openenv" }, { name = "pip" }, @@ -1314,9 +1313,13 @@ dependencies = [ { name = "wheel" }, ] +[package.dev-dependencies] +dev = [ + { name = "harbor" }, +] + [package.metadata] requires-dist = [ - { name = "harbor", specifier = "==0.25.0" }, { name = "jsonschema", specifier = "==4.25.1" }, { name = "openenv", directory = "../../" }, { name = "pip", specifier = "==26.2.1" }, @@ -1326,6 +1329,9 @@ requires-dist = [ { name = "wheel", specifier = "==0.48.0" }, ] +[package.metadata.requires-dev] +dev = [{ name = "harbor", specifier = "==0.25.0" }] + [[package]] name = "opentelemetry-api" version = "1.45.1" From 3148709dd81c30ce84f2453e6e1923c39852be6d Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 18:46:50 +0100 Subject: [PATCH 11/40] test(validation): expect named static SKIPs in the Docker acceptance suite Every report now names the nine static checks without a grader as SKIPs. A served run where every runtime check passes therefore reads WARN until those graders land. The acceptance tests derive the expected verdict from a STATIC_PENDING set, check that every report names those SKIPs with their reason, and include them in coverage.json's incomplete list. Co-Authored-By: Claude Opus 5.5 --- .../integration/test_runtime_cli.py | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/test_validation/integration/test_runtime_cli.py b/tests/test_validation/integration/test_runtime_cli.py index 5a8e94013..738a96da8 100644 --- a/tests/test_validation/integration/test_runtime_cli.py +++ b/tests/test_validation/integration/test_runtime_cli.py @@ -37,6 +37,19 @@ "runtime.oracle_containment", } PENDING = set() +# Static checks without a grader: every report names them as SKIPs, so a run +# where every runtime check passes reads WARN until these graders exist. +STATIC_PENDING = { + "static.reproducible_build", + "static.image_hygiene", + "static.layout", + "static.sbom", + "static.oci_labels", + "static.resource_declaration", + "static.timeout_ceiling", + "static.dependency_pinning", + "static.task_distribution_pinning", +} def _copy_asset(source, destination): @@ -295,6 +308,10 @@ def _invoke_cli( assert {key for key in checks if key.startswith("runtime.")} == applicable assert all(checks[key]["status"] == "skip" for key in PENDING) assert all(checks[key]["evidence"] for key in PENDING) + assert all( + checks[key]["evidence"] == ["grader not implemented in this build"] + for key in STATIC_PENDING + ) assert set(artifacts["coverage.json"]["requested_runtime_checks"]) == applicable assert artifacts["cleanup.json"]["completed"] is True return result, report, checks, artifacts @@ -408,9 +425,11 @@ def test_cli_runtime_contract_findings(cli_context, tmp_path, mode, failed_check assert checks[failed_check]["evidence"] else: assert result.returncode == 0 - assert report["verdict"] == "pass" + assert report["verdict"] == ("warn" if STATIC_PENDING else "pass") assert all(checks[key]["status"] == "pass" for key in IMPLEMENTED) - assert set(artifacts["coverage.json"]["incomplete"]) == PENDING + assert set(artifacts["coverage.json"]["incomplete"]) == ( + PENDING | STATIC_PENDING + ) def _assert_partial_episode(artifacts, expected_failure, checks): @@ -556,7 +575,8 @@ def test_cli_controlled_judge_requires_twenty_samples( assert len(variances) == 2 if mode == "judged_stable": assert variances == [0.0, 0.0] - assert result.returncode == 0 and report["verdict"] == "pass" + assert result.returncode == 0 + assert report["verdict"] == ("warn" if STATIC_PENDING else "pass") else: assert all(value > 0.01 for value in variances) assert result.returncode == 1 and report["verdict"] == "fail" From 71099021475632c6c7b67f738b18ca4e276ebc25 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:21:20 +0100 Subject: [PATCH 12/40] test(validation): keep the pop out of an assert Python skips assert statements under -O, so the side effect would vanish and the later count check would change meaning. Co-Authored-By: Claude Opus 5.5 --- tests/test_validation/test_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_validation/test_runner.py b/tests/test_validation/test_runner.py index 48b171851..2498a6fa0 100644 --- a/tests/test_validation/test_runner.py +++ b/tests/test_validation/test_runner.py @@ -99,7 +99,8 @@ def test_valid_package_names_unimplemented_static_checks(): assert report.levels_run == [Level.STATIC] assert report.manifest is not None and report.manifest.name == "served-min-pass" by_id = {r.check_id: r for r in report.results} - assert by_id.pop("static.manifest").status is CheckStatus.PASS + manifest_check = by_id.pop("static.manifest") + assert manifest_check.status is CheckStatus.PASS assert len(by_id) == 9 assert all( r.check_id.startswith("static.") From 6d05868b9f357f70938357f0b132fa5d4bbc3ff0 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:45:36 +0100 Subject: [PATCH 13/40] fix(validation): refuse task-declared Harbor images Require the NOP trial to build the reviewed Dockerfile. Refuse environment.docker_image before any trial can retain and reuse a registry image. --- src/openenv/validation/harbor.py | 2 ++ tests/test_validation/test_harbor_validation.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index de4b4c485..860a8575e 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -242,6 +242,8 @@ def _unsupported(task, contract): return "multi-step Harbor tasks" if env.os.value != "linux": return "non-Linux Harbor tasks" + if env.docker_image is not None: + return "task-declared environment.docker_image instead of a Dockerfile build" if config.verifier.environment is not None or ( config.verifier.environment_mode is not None and config.verifier.environment_mode.value != "shared" diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 02176029f..5c61ec56f 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -595,6 +595,16 @@ def test_separate_verifier_is_named_unsupported(task, boundary): assert not boundary.trials +def test_task_declared_image_is_refused_before_trials(task, boundary): + with (task / "task.toml").open("a") as stream: + stream.write('[environment]\ndocker_image="registry.example/image:latest"\n') + report = run_validation(task) + startup = result(report, "runtime.startup") + assert startup.status is CheckStatus.SKIP + assert "docker_image" in startup.evidence[0] + assert not boundary.trials + + def test_explicit_contract_and_valid_toml_required(task, boundary): (task / "validation/openenv.json").unlink() with pytest.raises(ManifestError): From e0a901f779886f926011a35513eb9b9343941a07 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:46:07 +0100 Subject: [PATCH 14/40] fix(validation): fail missing Harbor oracles before execution gates Grade a missing solution whenever semantic validation is requested and static checks pass, including skipped builds and unsupported providers. --- src/openenv/validation/harbor.py | 33 ++++++++++--------- .../test_validation/test_harbor_validation.py | 13 ++++++-- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index 860a8575e..dab3104b5 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -677,6 +677,24 @@ def validate_harbor( evidence = None checks = [] semantic_graded = False + if ( + semantic + and subject.manifest.capabilities.oracle is None + and all(result.status is CheckStatus.PASS for result in results) + ): + # RFC 008: a missing oracle is a graded FAIL, never a SKIP. + semantic_graded = True + checks.append( + CheckResult( + check_id="semantic.oracle_max", + status=CheckStatus.FAIL, + evidence=["the task has no oracle (solution/solve.sh)"], + remediation=( + "Add solution/solve.sh that drives the task to a correct solution." + ), + duration_s=0, + ) + ) if not problem: if subject.manifest.capabilities.oracle is None: startup = _result( @@ -684,21 +702,6 @@ def validate_harbor( CheckStatus.SKIP, "native Harbor trials need solution/solve.sh", ) - if semantic: - # RFC 008: a missing oracle is a graded FAIL, never a SKIP. - semantic_graded = True - checks.append( - CheckResult( - check_id="semantic.oracle_max", - status=CheckStatus.FAIL, - evidence=["the task has no oracle (solution/solve.sh)"], - remediation=( - "Add solution/solve.sh that drives the task to a " - "correct solution." - ), - duration_s=0, - ) - ) else: attempted = True try: diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 5c61ec56f..6303a1018 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -435,9 +435,18 @@ def copying(*args, **kwargs): assert "validated source digest" in native["trials"][1]["reason"] -def test_missing_oracle_fails_oracle_max_without_starting_trial(task, boundary): +@pytest.mark.parametrize("blocked", [None, "skip-build", "unsupported", "provider"]) +def test_missing_oracle_fails_oracle_max_without_starting_trial( + task, boundary, blocked +): (task / "solution/solve.sh").unlink() - report = run_validation(task) + if blocked == "unsupported": + (task / "task.toml").write_text('[environment]\ndocker_image="prebuilt"\n') + report = run_validation( + task, + skip_build=blocked == "skip-build", + provider=object() if blocked == "provider" else None, + ) assert report.verdict is Verdict.FAIL assert report.levels_run == [Level.STATIC, Level.SEMANTIC] # RFC 008: no oracle is a graded FAIL, not a SKIP. From 47fcf5f112a879253b2a21991d056c3cedb4f171 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:47:12 +0100 Subject: [PATCH 15/40] fix(validation): classify Harbor package failures by phase Treat Harbor reward-file and parse exceptions as reward failures, preserving raw output. Report Dockerfile failures and subject timeouts as phase-specific startup FAILs while keeping validator and cleanup failures as ERRORs. --- src/openenv/validation/harbor.py | 56 ++++++++++++++++++- .../test_validation/test_harbor_validation.py | 47 +++++++++++++++- 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index dab3104b5..f9251fc48 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -373,6 +373,7 @@ async def _trial( } raw_files = {} trial = None + phase = "trial creation" # Register and verify ownership before Trial.create can allocate anything. if any(inspector.inventory(project).values()): raise NativeEvidenceError("run-owned project was not empty before allocation") @@ -437,7 +438,13 @@ async def _trial( async with asyncio.timeout(budget): trial = await _create_trial(config) + async def environment_start(_event): + nonlocal phase + phase = "environment build/start" + async def capture(_event): + nonlocal phase + phase = "agent" if trial.agent_environment.session_id != project: raise NativeEvidenceError( "Harbor sandbox identity differs from the registered project" @@ -467,14 +474,46 @@ async def capture_exit(_event): if marker is not None: raw_files["oracle-exit-code.txt"] = marker + async def verification_start(_event): + nonlocal phase + phase = "verifier" + + trial.add_hook(TrialEvent.ENVIRONMENT_START, environment_start) trial.add_hook(TrialEvent.AGENT_START, capture) trial.add_hook(TrialEvent.AGENT_END, capture_exit) + trial.add_hook(TrialEvent.VERIFICATION_START, verification_start) result = await trial.run() if source_digest(task_copy) != digest_before: raise NativeEvidenceError( "private execution task changed during validation" ) if result.exception_info is not None: + failure = result.exception_info.exception_type + if failure in { + "RewardFileNotFoundError", + "RewardFileEmptyError", + "VerifierOutputParseError", + }: + if "container_id" not in record: + raise NativeEvidenceError( + "Harbor reward failure has no container identity" + ) + record["reward_error"] = True + raise NativeEvidenceError(f"Harbor verifier reward failed ({failure})") + subject_phase = { + "EnvironmentStartTimeoutError": "environment build/start", + "SandboxBuildFailedError": "environment build/start", + "AgentSetupTimeoutError": "agent setup", + "AgentTimeoutError": "agent", + "VerifierTimeoutError": "verifier", + }.get(failure) + if failure == "RuntimeError" and phase == "environment build/start": + subject_phase = phase + if subject_phase: + record["subject_error"] = subject_phase + raise NativeEvidenceError( + f"Harbor subject failed during {subject_phase} ({failure})" + ) raise NativeEvidenceError("Harbor Trial returned exception_info") if "container_id" not in record: raise NativeEvidenceError("Harbor Trial has no actual AGENT_START identity") @@ -501,6 +540,9 @@ async def capture_exit(_event): record.pop("reward_error") except (Exception, KeyboardInterrupt) as exc: record["failure"] = type(exc).__name__ + if isinstance(exc, TimeoutError): + record["subject_error"] = phase + record["reason"] = f"Harbor subject timed out during {phase}" # Error categories are safe; task-authored exception messages/logs are not. if isinstance(exc, NativeEvidenceError): record["reason"] = str(exc) @@ -735,10 +777,20 @@ def validate_harbor( or (not malformed and len(trials) != 3 + len(canaries)) or len(set(containers)) != len(containers) ): + subject_failures = broken and all( + trial.get("subject_error") + and trial["failure"] not in _HOST_FAILURES + for trial in broken + ) startup = _result( "runtime.startup", - CheckStatus.ERROR, - "native Harbor trial, binding or cleanup failed", + CheckStatus.FAIL if subject_failures else CheckStatus.ERROR, + "native Harbor subject failed during " + + ", ".join( + sorted({trial["subject_error"] for trial in broken}) + ) + if subject_failures + else "native Harbor trial, binding or cleanup failed", { "failures": [ trial.get("reason", trial.get("failure")) diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 6303a1018..f2fab6c4b 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -61,6 +61,7 @@ def boundary(monkeypatch, task): config_drift=False, oracle_exit=None, exception=False, + exception_type=None, no_identity=False, timeout=False, retained=True, @@ -132,6 +133,13 @@ def add_hook(self, event, callback): self.hooks.setdefault(event.value, []).append(callback) async def run(self): + for hook in self.hooks.get("environment-start", []): + await hook(None) + if state.exception_type in ("RuntimeError", "EnvironmentStartTimeoutError"): + return SimpleNamespace( + exception_info=SimpleNamespace(exception_type=state.exception_type), + verifier_result=None, + ) if not state.no_identity: for hook in self.hooks.get("agent-start", []): await hook(None) @@ -156,13 +164,24 @@ async def run(self): (self.config.task.path / "instruction.md").write_text( "changed execution source" ) + for hook in self.hooks.get("verification-start", []): + await hook(None) try: rewards = json.loads(raw) if raw else None except ValueError: rewards = None + exception_type = state.exception_type + if raw is None: + exception_type = "RewardFileNotFoundError" + elif raw == b"": + exception_type = "RewardFileEmptyError" + elif raw in (b'{"reward":NaN}', b'{"reward":Infinity}', b'{"reward":"0"}'): + exception_type = "VerifierOutputParseError" + if state.exception: + exception_type = "TestError" return SimpleNamespace( - exception_info={"exception_type": "TestError"} - if state.exception + exception_info=SimpleNamespace(exception_type=exception_type) + if exception_type else None, verifier_result=SimpleNamespace(rewards=rewards), ) @@ -273,6 +292,7 @@ def test_success_grades_semantic_checks_with_native_raw_evidence( "raw", [ None, + b"", b"{}", b'{"other":0}', b'{"reward":true}', @@ -347,7 +367,28 @@ def test_source_identity_exception_timeout_fail_closed(task, boundary, flag): setattr(boundary, flag, True) report = run_validation(task) assert report.verdict is Verdict.FAIL - assert result(report, "runtime.startup").status is CheckStatus.ERROR + assert result(report, "runtime.startup").status is ( + CheckStatus.FAIL if flag == "timeout" else CheckStatus.ERROR + ) + + +@pytest.mark.parametrize( + "exception_type,phase", + [ + ("RuntimeError", "environment"), + ("EnvironmentStartTimeoutError", "environment"), + ("AgentTimeoutError", "agent"), + ("AgentSetupTimeoutError", "agent setup"), + ("VerifierTimeoutError", "verifier"), + ], +) +def test_subject_failures_name_the_phase(task, boundary, exception_type, phase): + boundary.exception_type = exception_type + report = run_validation(task) + startup = result(report, "runtime.startup") + assert startup.status is CheckStatus.FAIL + assert phase in startup.evidence[0] + assert result(report, "runtime.reward_well_formed").status is CheckStatus.SKIP def test_cleanup_failure_blocks(task, boundary, tmp_path): From 51af19cf637a2dbfa998300cec4439d99e9c0880 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:48:12 +0100 Subject: [PATCH 16/40] fix(validation): preserve completed Harbor semantic evidence Grade the verified NOP, oracle and replay prefix even if a later canary fails. Keep incomplete canary collections ungraded and preserve the startup cleanup error. --- src/openenv/validation/harbor.py | 121 ++++++++++-------- .../test_validation/test_harbor_validation.py | 34 ++++- 2 files changed, 103 insertions(+), 52 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index f9251fc48..c63488b34 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -845,57 +845,64 @@ def validate_harbor( "reward_key": contract.reward_key, }, ) - nop, oracle, replay, *canary_trials = trials - same = ( - oracle["rewards"] == replay["rewards"] - and oracle["oracle_exit_code"] == replay["oracle_exit_code"] - ) - determinism = _result( - "runtime.episode_determinism", - CheckStatus.PASS if same else CheckStatus.FAIL, - "the oracle replay in a fresh container " - + ( - "reproduced every reward and the exit code" - if same - else "produced different rewards or a different exit code" - ), - { - "rewards": [ - json.loads(dumps(trial["rewards"])) - for trial in (oracle, replay) - ], - "exit_codes": [ - trial["oracle_exit_code"] - for trial in (oracle, replay) - ], - }, - ) - key = contract.reward_key - record.update( - { - "floor_reward": nop["rewards"][key], - "oracle_reward": oracle["rewards"][key], - } - ) - evidence = OracleEvidence( - floor=Fraction(nop["rewards"][key]), - oracle=tuple( - Fraction(trial["rewards"][key]) + complete_baseline = ( + len(trials) >= 3 + and all( + "failure" not in trial and trial["cleanup"]["verified"] + for trial in trials[:3] + ) + and len(set(containers[:3])) == 3 + ) + if complete_baseline: + nop, oracle, replay, *canary_trials = trials + same = ( + oracle["rewards"] == replay["rewards"] + and oracle["oracle_exit_code"] == replay["oracle_exit_code"] + ) + determinism = _result( + "runtime.episode_determinism", + CheckStatus.PASS if same else CheckStatus.FAIL, + "the oracle replay in a fresh container " + + ( + "reproduced every reward and the exit code" + if same + else "produced different rewards or a different exit code" + ), + { + "rewards": [ + json.loads(dumps(trial["rewards"])) for trial in (oracle, replay) - ), - oracle_exit_codes=tuple( + ], + "exit_codes": [ trial["oracle_exit_code"] for trial in (oracle, replay) - ), - canaries=tuple( - ( - trial["task_overlay"]["solution/solve.sh"][ - "canary" - ], - Fraction(trial["rewards"][key]), - ) - for trial in canary_trials - ), - ) + ], + }, + ) + key = contract.reward_key + record.update( + { + "floor_reward": nop["rewards"][key], + "oracle_reward": oracle["rewards"][key], + } + ) + evidence = OracleEvidence( + floor=Fraction(nop["rewards"][key]), + oracle=tuple( + Fraction(trial["rewards"][key]) + for trial in (oracle, replay) + ), + oracle_exit_codes=tuple( + trial["oracle_exit_code"] for trial in (oracle, replay) + ), + canaries=tuple( + ( + trial["task_overlay"]["solution/solve.sh"]["canary"], + Fraction(trial["rewards"][key]), + ) + for trial in canary_trials + if "failure" not in trial and trial["cleanup"]["verified"] + ), + ) except (Exception, KeyboardInterrupt) as exc: record["failure"] = type(exc).__name__ startup = _result( @@ -911,9 +918,23 @@ def validate_harbor( CanaryFloorGrader(), ): registry.register(grader) + selected = registry.select(subject.manifest, max_level) + if evidence is not None and len(evidence.canaries) != len(canaries): + selected = [ + grader + for grader in selected + if grader.check_id != "semantic.canary_floor" + ] + checks.append( + _result( + "semantic.canary_floor", + CheckStatus.SKIP, + "unmet dependency: complete canary trials with verified cleanup", + ) + ) checks.extend( execute_graders( - registry.select(subject.manifest, max_level), + selected, replace(subject, oracle_evidence=evidence), prior=[startup, reward_result, determinism], ) diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index f2fab6c4b..c1863f74c 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -55,6 +55,7 @@ def boundary(monkeypatch, task): rewards={"nop": b'{"reward":0}', "oracle": b'{"reward":1}'}, trials=[], cleanup_ok=True, + cleanup_fail_role=None, mutate=False, private_mutate=False, image_drift=False, @@ -90,10 +91,14 @@ def capture(self, project, retained_tag, *, retain=True): } def cleanup(self, project): + verified = ( + state.cleanup_ok + and _role(project.removesuffix("__env")) != state.cleanup_fail_role + ) return { - "verified": state.cleanup_ok, + "verified": verified, "after": { - "containers": [] if state.cleanup_ok else ["leftover"], + "containers": [] if verified else ["leftover"], "networks": [], "volumes": [], }, @@ -745,6 +750,31 @@ def test_canaries_at_the_floor_pass(task, boundary, canaries): assert result(report, "semantic.canary_floor").status is CheckStatus.PASS +@pytest.mark.parametrize("floor,oracle", [(1, 1), (0, 0.8)]) +def test_canary_cleanup_failure_preserves_completed_semantics( + task, boundary, canaries, tmp_path, floor, oracle +): + boundary.rewards["nop"] = json.dumps({"reward": floor}).encode() + boundary.rewards["oracle"] = json.dumps({"reward": oracle}).encode() + boundary.cleanup_fail_role = "canary-2" + artifacts = tmp_path / "partial-semantics" + report = run_validation(task, artifacts_dir=artifacts) + assert result(report, "runtime.startup").status is CheckStatus.ERROR + assert result(report, "runtime.reward_well_formed").status is CheckStatus.SKIP + assert result(report, "runtime.episode_determinism").status is CheckStatus.PASS + assert result(report, "semantic.floor_gap").status is ( + CheckStatus.FAIL if floor else CheckStatus.PASS + ) + assert result(report, "semantic.oracle_max").status is ( + CheckStatus.PASS if oracle == 1 else CheckStatus.FAIL + ) + assert result(report, "semantic.canary_floor").status is CheckStatus.SKIP + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert len(native["trials"]) == 5 + assert not native["cleanup"]["verified"] + assert native["floor_reward"] == floor + + def test_canaries_run_only_at_the_semantic_level(task, boundary, canaries): report = run_validation(task, max_level=Level.RUNTIME) assert [trial.role for trial in boundary.trials] == [ From 964ac4b47c70bc7809d4653b8d126e55b9b29ab7 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:49:22 +0100 Subject: [PATCH 17/40] fix(validation): prioritize retained Harbor image failures Classify retained-image drift and failed image verification as host failures, even when the same trial has malformed rewards. --- src/openenv/validation/harbor.py | 2 ++ tests/test_validation/test_harbor_validation.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index c63488b34..32fc7d4a0 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -49,6 +49,8 @@ "CleanupNotVerified", "RewardArtifactDrift", "UnsafeRewardArtifact", + "RetainedImageDrift", + "RetainedImageUnverified", } # Checks this binding measures; every other local check is a named SKIP. _IMPLEMENTED = { diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index c1863f74c..eac03cd25 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -66,6 +66,7 @@ def boundary(monkeypatch, task): no_identity=False, timeout=False, retained=True, + retained_error=False, overwrite_exit=False, excluded_copied=[], ) @@ -105,6 +106,8 @@ def cleanup(self, project): } def verify_image(self, tag, image_ref): + if state.retained_error: + raise OSError("inspect failed") return state.retained class Trial: @@ -506,6 +509,18 @@ def test_retained_image_drift_blocks(task, boundary): assert run_validation(task).verdict is Verdict.FAIL +@pytest.mark.parametrize("inspect_error", [False, True]) +def test_retained_image_failure_is_not_hidden_by_bad_reward( + task, boundary, inspect_error +): + boundary.rewards["nop"] = b'{"reward":true}' + boundary.retained = False + boundary.retained_error = inspect_error + report = run_validation(task) + assert result(report, "runtime.startup").status is CheckStatus.ERROR + assert result(report, "runtime.reward_well_formed").status is CheckStatus.SKIP + + def test_host_inspector_scopes_inventory_and_cleanup_to_owned_project(monkeypatch): inspector = harbor.DockerInspector() project = "openenv-validation-" + "d" * 32 + "-nop__env" From 23338b3cab0589abaaef8b0ee4c0a95d3fbd0acf Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:49:35 +0100 Subject: [PATCH 18/40] fix(validation): preserve resolved targets in Harbor reports Forward report_target from the runner to the native binding so Space reports and their bundles identify the resolved Space revision. --- src/openenv/validation/harbor.py | 3 ++- src/openenv/validation/runner.py | 1 + tests/test_validation/test_harbor_validation.py | 8 ++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index 32fc7d4a0..2dadb8091 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -664,6 +664,7 @@ def validate_harbor( results, digest_before, artifacts_dir, + report_target=None, skip_build=False, provider=None, ): @@ -982,7 +983,7 @@ def validate_harbor( levels.append(Level.SEMANTIC) report = ValidationReportV2( report_schema_version="2", - target=str(subject.root), + target=str(subject.root) if report_target is None else report_target, source_digest=digest_before, signature=subject.manifest.signature, manifest=subject.manifest, diff --git a/src/openenv/validation/runner.py b/src/openenv/validation/runner.py index 1a47a100c..75ced47bc 100644 --- a/src/openenv/validation/runner.py +++ b/src/openenv/validation/runner.py @@ -700,6 +700,7 @@ def run_validation( results=results, digest_before=digest_before, artifacts_dir=artifacts_dir, + report_target=report_target, skip_build=skip_build, provider=provider, ) diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index eac03cd25..3b412483e 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -657,6 +657,14 @@ def test_static_and_skip_build_do_not_start_trials(task, boundary): assert not boundary.trials +def test_harbor_report_preserves_resolved_space_target(task, boundary, tmp_path): + target = "spaces/openenv/example@" + "d" * 40 + artifacts = tmp_path / "space-report" + report = run_validation(task, report_target=target, artifacts_dir=artifacts) + assert report.target == target + assert json.loads((artifacts / "report.json").read_text())["target"] == target + + def test_separate_verifier_is_named_unsupported(task, boundary): (task / "task.toml").write_text('[verifier]\nenvironment_mode="separate"\n') report = run_validation(task) From 3d0791ee07106202f60344c6937ff672a623cffc Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:53:01 +0100 Subject: [PATCH 19/40] fix(validation): supervise Harbor builds outside episode budgets Reuse the Docker validator build budget for NOP build/start and resume the episode deadline afterward. Kill and reap Compose process groups on cancellation, including spawn races. Keep daemon completion explicitly unverified for cancelled allocating commands and record their project and operation before cleanup can claim success. --- src/openenv/validation/harbor.py | 116 +++++++++++++++++- .../test_harbor_binding_supervision.py | 100 +++++++++++++++ .../test_validation/test_harbor_validation.py | 75 ++++++++++- 3 files changed, 288 insertions(+), 3 deletions(-) create mode 100644 tests/test_validation/test_harbor_binding_supervision.py diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index 2dadb8091..04ea8fa9f 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -10,6 +10,7 @@ import os import re import shutil +import signal import stat import subprocess import tempfile @@ -18,6 +19,7 @@ from dataclasses import replace from decimal import Decimal from fractions import Fraction +from functools import partial from importlib.metadata import distribution from pathlib import Path @@ -276,7 +278,85 @@ def _unsupported(task, contract): async def _create_trial(config): from harbor.trial.trial import Trial - return await Trial.create(config) + trial = await Trial.create(config) + environment = trial.agent_environment + trial._openenv_cancelled_compose = [] + environment._run_docker_compose_command = partial( + _run_compose, environment, cancelled=trial._openenv_cancelled_compose + ) + return trial + + +async def _run_compose( + environment, + command, + check=True, + timeout_sec=None, + stdin_data=None, + on_output=None, + *, + cancelled, +): + # Harbor's Compose collector does not reap the client on outer cancellation. + runtime = type(environment).runtime() + argv = [*runtime.compose, "--project-name", environment.session_id] + if runtime.supports_compose_project_directory: + argv.extend(["--project-directory", str(environment.environment_dir.resolve())]) + for path in environment._docker_compose_paths: + argv.extend(["-f", str(path.resolve())]) + spawn = asyncio.create_task( + asyncio.create_subprocess_exec( + *argv, + *command, + env=environment._compose_env_vars(include_os_env=True), + cwd=str(environment.environment_dir.resolve()), + stdin=asyncio.subprocess.PIPE + if stdin_data is not None + else asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + start_new_session=True, + ) + ) + try: + process = await asyncio.shield(spawn) + if on_output is not None: + result = await environment._collect_streamed_output( + process, + timeout_sec=timeout_sec, + stdin_data=stdin_data, + on_output=on_output, + ) + else: + result = await environment._collect_buffered_output( + process, timeout_sec=timeout_sec, stdin_data=stdin_data + ) + if check and result.return_code != 0: + raise RuntimeError("Harbor Compose command failed") + return result + except BaseException as exc: + # Finish spawning before cleanup, even if cancellation raced with exec. + process = await spawn + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + await process.wait() + if isinstance(exc, asyncio.CancelledError) and command[0] in { + "build", + "pull", + "up", + }: + cancelled.append( + { + "command": command[0], + "project": environment.session_id, + "client_group_killed": True, + "client_reaped": True, + "daemon_completion_verified": False, + } + ) + raise def _read_output(path, root): @@ -361,6 +441,7 @@ async def _trial( ) from harbor.trial.hooks import TrialEvent + from .providers.docker import DockerValidationProvider from .runner import _DIGEST_EXCLUDED_DIRS, source_digest name = f"openenv-validation-{run_id}-{role}" @@ -411,6 +492,7 @@ async def _trial( record["execution_task_digest"] = digest_before record["task_overlay"] = overlay budget = contract.resources.episode_timeout_s + build_budget = DockerValidationProvider().build_timeout_s config = TrialConfig( task=TaskConfig(path=task_copy.resolve()), agent=AgentConfig( @@ -437,8 +519,28 @@ async def _trial( effective_config.pop("trial_name") record["trial_config_digest"] = _digest(effective_config) record["resources"] = contract.resources.model_dump() - async with asyncio.timeout(budget): + async with asyncio.timeout(budget) as deadline: trial = await _create_trial(config) + trial._environment_build_timeout_sec = ( + build_budget if not expected else budget + ) + if not expected: + record["build_timeout_s"] = build_budget + start_environment = trial.agent_environment.start + + async def build_and_start(*, force_build): + nonlocal phase + loop = asyncio.get_running_loop() + remaining = deadline.when() - loop.time() + deadline.reschedule(None) + try: + async with asyncio.timeout(build_budget): + await start_environment(force_build=force_build) + finally: + deadline.reschedule(loop.time() + remaining) + phase = "agent setup" + + trial.agent_environment.start = build_and_start async def environment_start(_event): nonlocal phase @@ -557,6 +659,16 @@ async def verification_start(_event): record["failure"] = "HarborTeardownError" try: record["cleanup"] = inspector.cleanup(project) + if trial is not None and trial._openenv_cancelled_compose: + record["cleanup"].update( + { + "verified": False, + "pending_operations": trial._openenv_cancelled_compose, + } + ) + record["reason"] = ( + "Docker daemon completion after cancelled Compose allocation is unverified" + ) if not record["cleanup"]["verified"]: record["failure"] = "CleanupNotVerified" except (Exception, KeyboardInterrupt) as exc: diff --git a/tests/test_validation/test_harbor_binding_supervision.py b/tests/test_validation/test_harbor_binding_supervision.py new file mode 100644 index 000000000..f27255d7b --- /dev/null +++ b/tests/test_validation/test_harbor_binding_supervision.py @@ -0,0 +1,100 @@ +"""Compose cancellation tests use fake processes and never contact Docker.""" + +import asyncio +import signal +from types import SimpleNamespace + +import pytest +from openenv.validation import harbor + +pytest.importorskip("harbor.trial.trial") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("during_spawn", [False, True]) +@pytest.mark.parametrize("command", ["build", "up", "pull"]) +async def test_compose_cancellation_reaps_the_process_group( + monkeypatch, tmp_path, during_spawn, command +): + from harbor.environments.docker.docker import DockerEnvironment + from harbor.trial.trial import Trial + + started = asyncio.Event() + release_spawn = asyncio.Event() + events = [] + options = {} + + class Process: + pid = 123456 + returncode = None + + async def communicate(self, input=None): + started.set() + await asyncio.Event().wait() + + async def wait(self): + assert self.returncode == -signal.SIGKILL + events.append("reaped") + + process = Process() + + async def spawn(*args, **kwargs): + options.update(kwargs) + if during_spawn: + started.set() + await release_spawn.wait() + return process + + class Environment: + session_id = "openenv-validation-" + "d" * 32 + "-nop__env" + environment_name = "test" + environment_dir = tmp_path + _docker_compose_paths = [tmp_path / "compose.yaml"] + _run_docker_compose_command = DockerEnvironment._run_docker_compose_command + _collect_buffered_output = staticmethod( + DockerEnvironment._collect_buffered_output + ) + + @classmethod + def runtime(cls): + return SimpleNamespace( + compose=["docker", "compose"], supports_compose_project_directory=True + ) + + def _compose_env_vars(self, *, include_os_env): + return {} + + environment = Environment() + trial = SimpleNamespace(agent_environment=environment) + + async def create(config): + return trial + + def killpg(pid, sig): + assert pid == process.pid and sig == signal.SIGKILL + process.returncode = -sig + events.append("killed") + + monkeypatch.setattr(Trial, "create", create) + monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn) + monkeypatch.setattr(harbor.os, "killpg", killpg) + created = await harbor._create_trial(None) + running = asyncio.create_task( + created.agent_environment._run_docker_compose_command([command]) + ) + await started.wait() + running.cancel() + release_spawn.set() + with pytest.raises(asyncio.CancelledError): + await running + assert options.get("start_new_session") is True + assert events == ["killed", "reaped"] + assert created._openenv_cancelled_compose == [ + { + "command": command, + "project": environment.session_id, + "client_group_killed": True, + "client_reaped": True, + "daemon_completion_verified": False, + } + ] diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 3b412483e..3472c9c2e 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -1,5 +1,6 @@ """Native Harbor checks mock only the Trial boundary and trusted host inspector.""" +import asyncio import hashlib import json from types import SimpleNamespace @@ -69,6 +70,9 @@ def boundary(monkeypatch, task): retained_error=False, overwrite_exit=False, excluded_copied=[], + start_delay=0, + agent_delay=0, + pending_compose=False, ) class Inspector: @@ -126,9 +130,10 @@ def __init__(self, config): agent_dir=directory / "agent", ) self.agent_environment = SimpleNamespace( - session_id=config.trial_name + "__env", stop=self.stop + session_id=config.trial_name + "__env", start=self.start, stop=self.stop ) self.hooks = {} + self._openenv_cancelled_compose = [] state.excluded_copied.append( (config.task.path / "environment/.worktrees").exists() ) @@ -137,6 +142,21 @@ def __init__(self, config): async def stop(self, *, delete): assert not delete + async def start(self, *, force_build): + if state.pending_compose: + self._openenv_cancelled_compose.append( + { + "command": "build", + "project": self.agent_environment.session_id, + "client_group_killed": True, + "client_reaped": True, + "daemon_completion_verified": False, + } + ) + raise TimeoutError() + if self.role == "nop": + await asyncio.sleep(state.start_delay) + def add_hook(self, event, callback): self.hooks.setdefault(event.value, []).append(callback) @@ -148,9 +168,11 @@ async def run(self): exception_info=SimpleNamespace(exception_type=state.exception_type), verifier_result=None, ) + await self.agent_environment.start(force_build=False) if not state.no_identity: for hook in self.hooks.get("agent-start", []): await hook(None) + await asyncio.sleep(state.agent_delay) if state.timeout: raise TimeoutError() raw = state.rewards.get(self.role, state.rewards[self.agent]) @@ -657,6 +679,42 @@ def test_static_and_skip_build_do_not_start_trials(task, boundary): assert not boundary.trials +@pytest.mark.parametrize("timeout_phase", [None, "build", "agent"]) +def test_nop_build_has_a_separate_supervisor_budget( + task, boundary, monkeypatch, tmp_path, timeout_phase +): + from openenv.validation.providers import docker + + path = task / "validation/openenv.json" + contract = json.loads(path.read_text()) + contract["resources"]["episode_timeout_s"] = 0.1 + path.write_text(json.dumps(contract)) + build_budget = 0.02 if timeout_phase == "build" else 0.5 + monkeypatch.setattr( + docker, + "DockerValidationProvider", + lambda: SimpleNamespace(build_timeout_s=build_budget), + ) + boundary.start_delay = 0.2 + boundary.agent_delay = 0.2 if timeout_phase == "agent" else 0 + artifacts = tmp_path / "build-budget" + report = run_validation(task, artifacts_dir=artifacts) + startup = result(report, "runtime.startup") + assert startup.status is (CheckStatus.FAIL if timeout_phase else CheckStatus.PASS) + if timeout_phase: + assert len(boundary.trials) == 1 + phase = "environment" if timeout_phase == "build" else "agent" + assert phase in startup.evidence[0] + assert ( + result(report, "runtime.episode_determinism").status is not CheckStatus.PASS + ) + else: + assert result(report, "semantic.oracle_max").status is CheckStatus.PASS + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert native["cleanup"]["verified"] + assert native["trials"][0]["build_timeout_s"] == build_budget + + def test_harbor_report_preserves_resolved_space_target(task, boundary, tmp_path): target = "spaces/openenv/example@" + "d" * 40 artifacts = tmp_path / "space-report" @@ -665,6 +723,21 @@ def test_harbor_report_preserves_resolved_space_target(task, boundary, tmp_path) assert json.loads((artifacts / "report.json").read_text())["target"] == target +def test_cancelled_compose_keeps_daemon_cleanup_unverified(task, boundary, tmp_path): + boundary.pending_compose = True + artifacts = tmp_path / "pending-compose" + report = run_validation(task, artifacts_dir=artifacts) + assert result(report, "runtime.startup").status is CheckStatus.ERROR + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + cleanup = native["trials"][0]["cleanup"] + assert not cleanup["verified"] + assert not any(cleanup["after"].values()) + (pending,) = cleanup["pending_operations"] + assert pending["command"] == "build" + assert pending["project"] == boundary.trials[0].agent_environment.session_id + assert pending["client_reaped"] and not pending["daemon_completion_verified"] + + def test_separate_verifier_is_named_unsupported(task, boundary): (task / "task.toml").write_text('[verifier]\nenvironment_mode="separate"\n') report = run_validation(task) From 6fdc8ef8973e7f9a92689b81c47222c16ad9891e Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:56:11 +0100 Subject: [PATCH 20/40] fix(validation): cover Harbor host inspection guards Exercise image, label, retained-tag and configuration checks with fake Docker responses. Cover symlink, FIFO, bounded-read and size guards with temporary local files, and verify guard-removal mutants fail. --- .../test_harbor_binding_inspection.py | 290 ++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 tests/test_validation/test_harbor_binding_inspection.py diff --git a/tests/test_validation/test_harbor_binding_inspection.py b/tests/test_validation/test_harbor_binding_inspection.py new file mode 100644 index 000000000..663339a39 --- /dev/null +++ b/tests/test_validation/test_harbor_binding_inspection.py @@ -0,0 +1,290 @@ +"""Trusted host checks against fake Docker replies and real local output files.""" + +import json +import os +import stat +from contextlib import contextmanager +from types import SimpleNamespace + +import pytest +from openenv.validation.harbor import ( + _MAX_REWARD_BYTES, + _read_output, + DockerInspector, + NativeEvidenceError, +) + +PROJECT = "openenv-validation-" + "d" * 32 + "-nop__env" +IMAGE = "sha256:" + "a" * 64 +TAG = "openenv-validation:" + "d" * 32 + + +@pytest.fixture +def inspection(monkeypatch): + inspector = DockerInspector() + container = { + "Id": "c" * 64, + "Image": IMAGE, + "State": {"Running": True}, + "Config": { + "Labels": { + "com.docker.compose.project": PROJECT, + "com.docker.compose.service": "main", + }, + "User": "1000", + "WorkingDir": "/app", + "Env": ["B=2", "A=1"], + "Entrypoint": ["/bin/sh"], + "Cmd": ["-c", "sleep infinity"], + }, + "HostConfig": { + "NetworkMode": PROJECT + "_default", + "Memory": 512 * 1024 * 1024, + }, + "Mounts": [ + { + "Type": "volume", + "Name": "owned-volume", + "Destination": "/data", + "RW": True, + }, + ], + } + retained = {"Id": IMAGE, "Os": "linux"} + containers = [container["Id"]] + commands = [] + + def docker(*args): + commands.append(args) + if args[0] == "ps": + return "\n".join(containers) + if args[:2] in (("network", "ls"), ("volume", "ls")): + return "" + if args == ("inspect", container["Id"]): + return json.dumps([container]) + if args[:2] == ("image", "tag"): + assert args[3] == TAG + return "" + if args == ("image", "inspect", TAG): + return json.dumps([retained]) + pytest.fail(f"unexpected Docker command: {args}") + + monkeypatch.setattr(inspector, "_docker", docker) + return inspector, container, retained, containers, commands + + +@pytest.mark.parametrize("retain", [False, True]) +def test_capture_binds_the_image_configuration_and_anonymous_volumes( + inspection, retain +): + inspector, container, _, _, commands = inspection + identity = inspector.capture(PROJECT, TAG, retain=retain) + assert identity["container_id"] == container["Id"] + assert identity["image_ref"] == IMAGE + assert identity["retained_image_tag"] == TAG + assert identity["configuration"]["network_mode"] == "compose-default" + assert identity["owned_volumes"] == ["owned-volume"] + assert inspector._owned_volumes[PROJECT] == ["owned-volume"] + assert (("image", "tag", IMAGE, TAG) in commands) is retain + container["Config"]["Env"].reverse() + assert ( + inspector.capture(PROJECT, TAG, retain=False)["configuration_digest"] + == identity["configuration_digest"] + ) + + +@pytest.mark.parametrize( + "defect", + [ + "empty", + "multiple", + "project", + "service", + "stopped", + "mutable", + "tag", + "os", + "volume", + ], +) +def test_capture_rejects_untrusted_identity_and_binding(inspection, defect): + inspector, container, retained, containers, _ = inspection + if defect == "empty": + containers.clear() + elif defect == "multiple": + containers.append("e" * 64) + elif defect == "project": + container["Config"]["Labels"]["com.docker.compose.project"] = "other" + elif defect == "service": + container["Config"]["Labels"]["com.docker.compose.service"] = "other" + elif defect == "stopped": + container["State"]["Running"] = False + elif defect == "mutable": + container["Image"] = "registry.example/mutable:latest" + elif defect == "tag": + retained["Id"] = "sha256:" + "b" * 64 + elif defect == "os": + retained["Os"] = "windows" + elif defect == "volume": + container["Mounts"][0]["Name"] = "--unowned" + reason = { + "empty": "exactly one main container", + "multiple": "exactly one main container", + "project": "native main service", + "service": "native main service", + "stopped": "not running with an immutable image", + "mutable": "not running with an immutable image", + "tag": "retained image binding changed", + "os": "retained image binding changed", + "volume": "invalid mounted volume identity", + }[defect] + with pytest.raises(NativeEvidenceError, match=reason): + inspector.capture(PROJECT, TAG) + + +@pytest.mark.parametrize( + "section,key,value", + [ + (None, "Image", "sha256:" + "b" * 64), + ("Config", "User", "root"), + ("Config", "WorkingDir", "/other"), + ("Config", "Env", ["A=changed"]), + ("Config", "Entrypoint", ["/other"]), + ("Config", "Cmd", ["other"]), + ("HostConfig", "NanoCpus", 2_000_000_000), + ("HostConfig", "Memory", 1024), + ("HostConfig", "MemorySwap", -1), + ("HostConfig", "PidsLimit", 512), + ("HostConfig", "ReadonlyRootfs", True), + ("HostConfig", "Privileged", True), + ("HostConfig", "CapAdd", ["ALL"]), + ("HostConfig", "CapDrop", ["ALL"]), + ("HostConfig", "SecurityOpt", ["no-new-privileges"]), + ("HostConfig", "Devices", ["/dev/example"]), + ("HostConfig", "NetworkMode", "host"), + ("Mounts", "Type", "bind"), + ("Mounts", "Destination", "/other"), + ("Mounts", "RW", False), + ], +) +def test_capture_digest_covers_effective_configuration(inspection, section, key, value): + inspector, container, _, _, _ = inspection + before = inspector.capture(PROJECT, TAG) + target = container if section is None else container[section] + if section == "Mounts": + target = target[0] + target[key] = value + after = inspector.capture(PROJECT, TAG, retain=False) + assert after["configuration_digest"] != before["configuration_digest"] + + +@pytest.mark.parametrize("linux", [False, True]) +def test_preflight_requires_linux_and_compose(monkeypatch, linux): + inspector = DockerInspector() + commands = [] + + def docker(*args): + commands.append(args) + if args == ("info", "--format", "{{json .}}"): + return json.dumps( + { + "OSType": "linux" if linux else "windows", + "ID": "daemon", + "ServerVersion": "29", + } + ) + assert args == ("compose", "version") + return "Docker Compose" + + monkeypatch.setattr(inspector, "_docker", docker) + if linux: + assert inspector.preflight() == { + "os": "linux", + "daemon_id": "daemon", + "server_version": "29", + } + assert commands[-1] == ("compose", "version") + else: + with pytest.raises(NativeEvidenceError, match="requires a Linux Docker daemon"): + inspector.preflight() + assert len(commands) == 1 + + +@pytest.mark.parametrize("matches", [False, True]) +def test_verify_image_checks_the_retained_tag(inspection, matches): + inspector, _, retained, _, commands = inspection + retained["Id"] = IMAGE if matches else "sha256:" + "b" * 64 + assert inspector.verify_image(TAG, IMAGE) is matches + assert commands == [("image", "inspect", TAG)] + + +@pytest.mark.parametrize("kind", ["file", "parent", "broken"]) +def test_read_output_refuses_symlinks(tmp_path, kind): + outside = tmp_path / "outside" + outside.mkdir() + (outside / "reward.json").write_bytes(b"secret") + root = tmp_path / "trial" + root.mkdir() + if kind == "parent": + (root / "verifier").symlink_to(outside, target_is_directory=True) + path = root / "verifier/reward.json" + else: + path = root / "reward.json" + path.symlink_to(outside / ("missing" if kind == "broken" else "reward.json")) + with pytest.raises(NativeEvidenceError, match="symbolic links"): + _read_output(path, root) + + +def test_read_output_refuses_a_real_fifo_without_blocking(tmp_path, monkeypatch): + path = tmp_path / "reward.json" + os.mkfifo(path) + real_open = os.open + + def opened(path, flags): + assert flags & os.O_NONBLOCK and flags & os.O_NOFOLLOW + return real_open(path, flags) + + monkeypatch.setattr(os, "open", opened) + with pytest.raises(NativeEvidenceError, match="bounded regular file"): + _read_output(path, tmp_path) + + +@pytest.mark.parametrize("grows_after_stat", [False, True]) +def test_read_output_bounds_both_file_size_and_bytes_read( + tmp_path, monkeypatch, grows_after_stat +): + path = tmp_path / "reward.json" + path.write_bytes(b"x" * (_MAX_REWARD_BYTES + 1)) + if grows_after_stat: + actual_stat = os.fstat + + def fstat(fd): + info = list(actual_stat(fd)) + info[stat.ST_SIZE] = _MAX_REWARD_BYTES + return os.stat_result(info) + + monkeypatch.setattr(os, "fstat", fstat) + reason = "exceeded its size bound" if grows_after_stat else "bounded regular file" + with pytest.raises(NativeEvidenceError, match=reason): + _read_output(path, tmp_path) + + +def test_read_output_preserves_raw_bytes_and_missing_files(tmp_path, monkeypatch): + real_fdopen = os.fdopen + + @contextmanager + def fdopen(fd, mode): + with real_fdopen(fd, mode) as stream: + + def read(size=-1): + assert size == _MAX_REWARD_BYTES + 1 + return stream.read(size) + + yield SimpleNamespace(fileno=stream.fileno, read=read) + + monkeypatch.setattr(os, "fdopen", fdopen) + path = tmp_path / "reward.json" + assert _read_output(path, tmp_path) is None + raw = b'{"reward":0}\n' + path.write_bytes(raw) + assert _read_output(path, tmp_path) == raw From efe49fd259dc68201883f84fd447ef0018011c3b Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:57:05 +0100 Subject: [PATCH 21/40] fix(validation): preserve Harbor inventory failures and safe reasons Keep pre-allocation inventory checks inside trial evidence collection without cleaning projects whose ownership was not verified. Preserve trusted NativeEvidenceError reasons, remove stale pilot wording, and import the JSON object hook once. --- src/openenv/validation/harbor.py | 64 +++++++++++-------- .../test_validation/test_harbor_validation.py | 48 ++++++++++++++ 2 files changed, 86 insertions(+), 26 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index 04ea8fa9f..40d08006b 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -26,6 +26,7 @@ from .graders import GraderRegistry from .graders.semantic import CanaryFloorGrader, FloorGapGrader, OracleMaxGrader from .parsers.harbor_toml import ( + _unique_object, canary_scripts, HARBOR_PROFILE, read_harbor_contract, @@ -92,7 +93,7 @@ def _docker(self, *args): def preflight(self): info = json.loads(self._docker("info", "--format", "{{json .}}")) if info["OSType"] != "linux": - raise NativeEvidenceError("the Harbor pilot requires a Linux Docker daemon") + raise NativeEvidenceError("Harbor requires a Linux Docker daemon") self._docker("compose", "version") return { "os": info["OSType"], @@ -119,9 +120,7 @@ def inventory(self, project): def capture(self, project, retained_tag, *, retain=True): inventory = self.inventory(project) if len(inventory["containers"]) != 1: - raise NativeEvidenceError( - "the Harbor pilot requires exactly one main container" - ) + raise NativeEvidenceError("Harbor requires exactly one main container") item = json.loads(self._docker("inspect", inventory["containers"][0]))[0] labels = item["Config"]["Labels"] if ( @@ -383,8 +382,6 @@ def _read_output(path, root): def _reward(raw_files, parsed, contract): - from .parsers.harbor_toml import _unique_object - if "reward.json" in raw_files: raw = json.loads( raw_files["reward.json"], @@ -457,10 +454,15 @@ async def _trial( raw_files = {} trial = None phase = "trial creation" - # Register and verify ownership before Trial.create can allocate anything. - if any(inspector.inventory(project).values()): - raise NativeEvidenceError("run-owned project was not empty before allocation") + registered = False try: + # Verify ownership before Trial.create can allocate anything. + record["pre_allocation_inventory"] = inspector.inventory(project) + if any(record["pre_allocation_inventory"].values()): + raise NativeEvidenceError( + "run-owned project was not empty before allocation" + ) + registered = True task_copy = work / role / "task" shutil.copytree( root, @@ -657,23 +659,29 @@ async def verification_start(_event): await trial.agent_environment.stop(delete=False) except (Exception, KeyboardInterrupt): record["failure"] = "HarborTeardownError" - try: - record["cleanup"] = inspector.cleanup(project) - if trial is not None and trial._openenv_cancelled_compose: - record["cleanup"].update( - { - "verified": False, - "pending_operations": trial._openenv_cancelled_compose, - } - ) - record["reason"] = ( - "Docker daemon completion after cancelled Compose allocation is unverified" - ) - if not record["cleanup"]["verified"]: + if registered: + try: + record["cleanup"] = inspector.cleanup(project) + if trial is not None and trial._openenv_cancelled_compose: + record["cleanup"].update( + { + "verified": False, + "pending_operations": trial._openenv_cancelled_compose, + } + ) + record["reason"] = ( + "Docker daemon completion after cancelled Compose allocation is unverified" + ) + if not record["cleanup"]["verified"]: + record["failure"] = "CleanupNotVerified" + except (Exception, KeyboardInterrupt) as exc: + record["cleanup"] = {"verified": False, "failure": type(exc).__name__} record["failure"] = "CleanupNotVerified" - except (Exception, KeyboardInterrupt) as exc: - record["cleanup"] = {"verified": False, "failure": type(exc).__name__} - record["failure"] = "CleanupNotVerified" + else: + record["cleanup"] = { + "verified": False, + "reason": "project ownership was not verified before allocation", + } if trial is not None: # Preserve raw verifier files even when Harbor rejects their values. for filename, path in ( @@ -1020,10 +1028,14 @@ def validate_harbor( ) except (Exception, KeyboardInterrupt) as exc: record["failure"] = type(exc).__name__ + reason = f"native Harbor collection failed ({type(exc).__name__})" + if isinstance(exc, NativeEvidenceError): + record["reason"] = str(exc) + reason += ": " + str(exc) startup = _result( "runtime.startup", CheckStatus.ERROR, - f"native Harbor collection failed ({type(exc).__name__})", + reason, ) if semantic: registry = GraderRegistry() diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 3472c9c2e..d4179b455 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -73,13 +73,22 @@ def boundary(monkeypatch, task): start_delay=0, agent_delay=0, pending_compose=False, + inventory_problem=None, + cleanup_calls=[], + preflight_error=None, ) class Inspector: def preflight(self): + if state.preflight_error is not None: + raise state.preflight_error return {"os": "linux", "daemon_id": "mock-host"} def inventory(self, project): + if state.inventory_problem and "canary-1__env" in project: + if state.inventory_problem == "error": + raise OSError("Docker inventory failed") + return {"containers": ["preexisting"], "networks": [], "volumes": []} return {"containers": [], "networks": [], "volumes": []} def capture(self, project, retained_tag, *, retain=True): @@ -96,6 +105,7 @@ def capture(self, project, retained_tag, *, retain=True): } def cleanup(self, project): + state.cleanup_calls.append(project) verified = ( state.cleanup_ok and _role(project.removesuffix("__env")) != state.cleanup_fail_role @@ -871,6 +881,44 @@ def test_canary_cleanup_failure_preserves_completed_semantics( assert native["floor_reward"] == floor +@pytest.mark.parametrize("problem", ["error", "collision"]) +def test_preallocation_inventory_failure_preserves_completed_trials( + task, boundary, canaries, tmp_path, problem +): + boundary.inventory_problem = problem + boundary.rewards["nop"] = b'{"reward":1}' + artifacts = tmp_path / "inventory-error" + report = run_validation(task, artifacts_dir=artifacts) + assert result(report, "runtime.startup").status is CheckStatus.ERROR + assert result(report, "runtime.episode_determinism").status is CheckStatus.PASS + assert result(report, "semantic.floor_gap").status is CheckStatus.FAIL + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert len(native["trials"]) == 4 + assert len(boundary.trials) == len(boundary.cleanup_calls) == 3 + assert all(trial["cleanup"]["verified"] for trial in native["trials"][:3]) + assert not native["trials"][-1]["cleanup"]["verified"] + assert ( + artifacts / "harbor-trials/oracle/reward.json" + ).read_bytes() == b'{"reward":1}' + + +@pytest.mark.parametrize("safe", [False, True]) +def test_collection_failure_keeps_only_safe_native_reasons( + task, boundary, tmp_path, safe +): + reason = "Harbor requires a Linux Docker daemon" if safe else "token=do-not-persist" + boundary.preflight_error = ( + harbor.NativeEvidenceError(reason) if safe else RuntimeError(reason) + ) + artifacts = tmp_path / "preflight" + report = run_validation(task, artifacts_dir=artifacts) + startup = result(report, "runtime.startup") + assert startup.status is CheckStatus.ERROR + assert (reason in startup.evidence[0]) is safe + native = (artifacts / "harbor-evidence.json").read_text() + assert (reason in native) is safe + + def test_canaries_run_only_at_the_semantic_level(task, boundary, canaries): report = run_validation(task, max_level=Level.RUNTIME) assert [trial.role for trial in boundary.trials] == [ From 50a4a27a53d78d154d910d2a048e481c43627bbb Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 10 Nov 2026 19:59:05 +0100 Subject: [PATCH 22/40] examples: add bounded coding audits with HF Sandboxes (#1592) * Add bounded operator coding audits with HF Sandboxes * Parse candidate Bash fences and retain raw response hashes * fix(coding-audit): reject linked artifacts while snapshotting Read original task files through the descriptor-based reader before writing the frozen copy, so copying cannot erase hard-link evidence. Cover context, instruction, verifier and reference artifacts through the runner with offline SDK fakes. * fix(coding-audit): reject empty tool-call completions Require a nonempty tool-call list when the provider finishes with tool_calls. Exercise null, missing and empty calls through collection and scoring after a valid terminal action, and verify failed targets are never graded. * fix(coding-audit): correct offline test import path Include the repository root in PYTHONPATH so pytest can import the examples package. Verified the documented command with all 417 offline tests passing. --- examples/coding_audit/README.md | 66 + examples/coding_audit/__init__.py | 1 + examples/coding_audit/__main__.py | 63 + examples/coding_audit/config.json | 83 ++ examples/coding_audit/hf.py | 1290 ++++++++++++++++++ examples/coding_audit/runner.py | 744 +++++++++++ examples/coding_audit/scoring.py | 506 ++++++++ examples/coding_audit/test_hf.py | 1732 +++++++++++++++++++++++++ examples/coding_audit/test_runner.py | 838 ++++++++++++ examples/coding_audit/test_scoring.py | 575 ++++++++ 10 files changed, 5898 insertions(+) create mode 100644 examples/coding_audit/README.md create mode 100644 examples/coding_audit/__init__.py create mode 100644 examples/coding_audit/__main__.py create mode 100644 examples/coding_audit/config.json create mode 100644 examples/coding_audit/hf.py create mode 100644 examples/coding_audit/runner.py create mode 100644 examples/coding_audit/scoring.py create mode 100644 examples/coding_audit/test_hf.py create mode 100644 examples/coding_audit/test_runner.py create mode 100644 examples/coding_audit/test_scoring.py diff --git a/examples/coding_audit/README.md b/examples/coding_audit/README.md new file mode 100644 index 000000000..b8b8a5ca0 --- /dev/null +++ b/examples/coding_audit/README.md @@ -0,0 +1,66 @@ +# Operator coding audit + +This example collects and scores bounded coding-task evidence. It belongs to the +operator workflow, separate from the installed OpenEnv validator and its RFC 008 +checks. Acceptance here does not establish full RFC 008 conformance, containment, +served-model revision attestation, or training benefit. + +From a repository checkout, install the tested SDK and run the example: + +```bash +uv sync --group dev +uv pip install 'huggingface_hub==1.33.0' +PYTHONPATH=src:envs uv run --no-sync python -m examples.coding_audit \ + --task /path/to/task \ + --config examples/coding_audit/config.json \ + --output /path/to/new-audit-directory \ + --json +``` + +The output directory is required and must be new. It retains `report.json`, +`evidence.json`, `config.json`, raw evidence artifacts, and `SHA256SUMS`. Exit codes +are `0` for accepted evidence, `1` for rejected or incomplete evidence, and `2` for +configuration or input failure. `--json` prints the report as well. + +`scoring.py` provides pure `assess_coding_audit` and `validate_audit_config` +functions. `runner.py` provides `normalize_coding_config`, `coding_budget`, +`validate_coding_task`, and `run_coding_validation`. `hf.py` contains +`HFTaskBackend`, the bounded Sandbox and Inference Providers implementation. +The schemas bind task, verifier, immutable image references and independent execution identities. + +Models may declare an explicit boolean `enable_thinking`, forwarded through the +HF client's `extra_body.chat_template_kwargs`. It is omitted by default. A +candidate-only override leaves the target protocol unchanged and is recorded +alongside request parameters. Support depends on the selected provider/model. +Candidate generation streams bounded content and reasoning; only a complete +`stop` response with validated usage can produce an executable candidate. A +`length` response remains rejected even when its terminal usage is retained. +Recognized Bash/sh Markdown fences are parsed before execution; the raw response +and both raw/script hashes are retained. Other language fences are rejected. + +Candidate generation can opt into public task API context by adding +`"context_paths": ["environment/source/package/api.py"]` to the `generator` +model configuration. Only those declared regular UTF-8 files are read; paths must +be unique, normalized, and below `environment/source/`. Up to 32 files and 65,536 +combined bytes are allowed. Linked files or parents, hidden or private paths, +NUL bytes, invalid UTF-8 and oversized context are rejected before inference. +The existing message and token budgets still apply. No source files are added by +default, and `target.context_paths` is rejected. + +The candidate prompt labels each source path and its runtime filename relative to +the configured workdir, with `environment/source/` removed. Each generation record +retains `candidate_context` entries containing `path`, `runtime_path`, `sha256`, +and `size`, bound to the frozen task and checksummed evidence. This option does +not enumerate task directories or include verifier, reference or specification +files. + +The HF backend launches paid services when invoked without a supplied test +backend. Use reviewed tasks and distinct execution and inference credentials. +The Hub pipeline's cloud worker packages these production example modules with +its frozen source and task bundle; its tests stay outside the bundle. + +Run the offline suite with: + +```bash +PYTHONPATH=.:src:envs uv run pytest examples/coding_audit -q +``` diff --git a/examples/coding_audit/__init__.py b/examples/coding_audit/__init__.py new file mode 100644 index 000000000..1169e0fb8 --- /dev/null +++ b/examples/coding_audit/__init__.py @@ -0,0 +1 @@ +"""Operator-owned coding audits, separate from core RFC 008 validation.""" diff --git a/examples/coding_audit/__main__.py b/examples/coding_audit/__main__.py new file mode 100644 index 000000000..5ace417df --- /dev/null +++ b/examples/coding_audit/__main__.py @@ -0,0 +1,63 @@ +"""Run a bounded operator coding audit and retain its evidence bundle.""" + +import argparse +import signal +import sys +import threading +from contextlib import contextmanager +from pathlib import Path + +from openenv.validation.runtime.json_values import dumps, loads + +from .runner import run_coding_validation + + +@contextmanager +def _sigterm_as_interrupt(): + if threading.current_thread() is not threading.main_thread(): + yield + return + + def interrupt(signum, frame): + raise KeyboardInterrupt + + previous = signal.signal(signal.SIGTERM, interrupt) + try: + yield + finally: + signal.signal(signal.SIGTERM, previous) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--task", type=Path, required=True) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument( + "--output", + type=Path, + required=True, + help="New directory for the report, evidence and checksummed artifacts", + ) + parser.add_argument("--json", action="store_true", dest="json_output") + args = parser.parse_args(argv) + try: + config = loads(args.config.read_text()) + with _sigterm_as_interrupt(): + report = run_coding_validation(args.task, config, args.output) + except (OSError, ValueError) as error: + print( + f"Coding audit configuration/input failure ({type(error).__name__})", + file=sys.stderr, + ) + return 2 + if args.json_output: + print(dumps(report)) + else: + print(f"Coding audit: {report['verdict'].upper()} ({report['profile']})") + for name, check in report["checks"].items(): + print(f" {check['status'].upper():5s} {name}") + return 0 if report["accepted"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/coding_audit/config.json b/examples/coding_audit/config.json new file mode 100644 index 000000000..bfb7b3912 --- /dev/null +++ b/examples/coding_audit/config.json @@ -0,0 +1,83 @@ +{ + "schema_version": 1, + "profile": "harbor-coding-audit-v1", + "target": { + "model_id": "Qwen/Qwen3.8-27B", + "revision": "37561816e66b5b9153b1b64c10968c18c1e6f4a0", + "provider": "deepinfra", + "supports_seed": false, + "supports_tools": true, + "temperature": 0.7, + "max_steps": 16, + "max_tokens": 2048, + "max_total_tokens": 16384 + }, + "execution": { + "agent_image": "docker.io/library/python@sha256:48d884813cf44e927cf64c5529dd630dd3b18338c6dc727991eef9b71e934a20", + "verifier_image": "docker.io/library/python@sha256:48d884813cf44e927cf64c5529dd630dd3b18338c6dc727991eef9b71e934a20", + "flavor": "cpu-basic", + "timeout_seconds": 120, + "workdir": "/app", + "submission_paths": [ + "/app/answer.txt" + ], + "private_paths": [ + "/tests", + "/solution" + ], + "verifier_command": [ + "/bin/bash", + "/tests/test.sh" + ], + "reward_path": "/logs/verifier/reward.txt", + "reward_key": "reward" + }, + "validation": { + "reference_repeats": 3, + "target_attempts": 8, + "reward_min": 0, + "reward_max": 1, + "oracle_tolerance": 0, + "floor_margin": 0.1, + "wrong_answer_max": 0, + "required_negative_cases": [ + "wrong-value", + "empty-answer", + "near-miss" + ] + }, + "generation": { + "candidate_count": 2 + }, + "negative_cases": [ + { + "id": "wrong-value", + "script": "mkdir -p /app; printf '0\\n' > /app/answer.txt" + }, + { + "id": "empty-answer", + "script": "mkdir -p /app; : > /app/answer.txt" + }, + { + "id": "near-miss", + "script": "mkdir -p /app; printf '41\\n' > /app/answer.txt" + } + ], + "seeds": [ + 1001, + 1002, + 1003, + 1004, + 1005, + 1006, + 1007, + 1008 + ], + "cloud": { + "results_repo": "example-org/private-audit-results", + "flavor": "cpu-basic", + "timeout_seconds": 1800, + "max_sandboxes": 64, + "max_inference_calls": 160 + } +} diff --git a/examples/coding_audit/hf.py b/examples/coding_audit/hf.py new file mode 100644 index 000000000..f615bd382 --- /dev/null +++ b/examples/coding_audit/hf.py @@ -0,0 +1,1290 @@ +"""Bounded HF Jobs coding trials for reviewed tasks, with controller-side inference. + +Image receipts attest only the requested immutable control-plane reference. The +Sandbox RPC bearer reaches a workload-controlled server; this is not a hostile +code security boundary or a served-model weight revision guarantee. +""" + +import hashlib +import json +import math +import os +import re +import signal +import stat +import threading +import time +import uuid +from contextlib import closing, contextmanager +from decimal import Decimal +from importlib.metadata import version +from pathlib import Path, PurePosixPath + +from huggingface_hub import HfApi, InferenceClient, Sandbox +from huggingface_hub.errors import SandboxError + +from .runner import _candidate_context_paths + +_IMAGE = re.compile(r"[^\s@]+@sha256:[0-9a-f]{64}\Z") +_REVISION = re.compile(r"[0-9a-f]{40}\Z") +_TERMINAL = {"COMPLETED", "CANCELED", "ERROR", "DELETED"} +_LIMITATIONS = [ + "Immutable images are control-plane references, not measured runtime digests.", + "Sandbox RPC exposes the execution bearer to the workload server; hostile-code credential isolation is not qualified.", + "The verifier executes submitted code as the same user; verifier integrity against hostile submissions is not qualified.", + "Inference Providers do not pin served weights to the recorded model source revision.", +] +_TOOL = { + "type": "function", + "function": { + "name": "terminal", + "description": "Run a shell command in the task workspace.", + "parameters": { + "type": "object", + "properties": {"command": {"type": "string"}}, + "required": ["command"], + "additionalProperties": False, + }, + }, +} + + +class HFBackendError(ValueError): + """A credential-free controller diagnostic.""" + + +class _DeadlineExceeded(HFBackendError): + pass + + +@contextmanager +def _deadline(seconds): + # SDK control-plane calls have no timeout argument. A controller must run on + # the main Unix thread so a blocked synchronous SDK call can be interrupted. + if threading.current_thread() is not threading.main_thread(): + raise HFBackendError("HF coding controller requires the main thread") + previous_handler = signal.getsignal(signal.SIGALRM) + previous_timer = signal.getitimer(signal.ITIMER_REAL) + started = time.monotonic() + + def expired(signum, frame): + raise _DeadlineExceeded("controller deadline exceeded") + + signal.signal(signal.SIGALRM, expired) + signal.setitimer( + signal.ITIMER_REAL, + min(seconds, previous_timer[0]) if previous_timer[0] else seconds, + 0.1, + ) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_handler) + if previous_timer[0]: + signal.setitimer( + signal.ITIMER_REAL, + max(0.000001, previous_timer[0] - (time.monotonic() - started)), + previous_timer[1], + ) + + +def _path(value): + if not isinstance(value, str) or "\x00" in value: + raise HFBackendError("paths must be absolute normalized file paths") + path = PurePosixPath(value) + if ( + not path.is_absolute() + or str(path) != value + or ".." in path.parts + or value == "/" + or value.startswith("//") + ): + raise HFBackendError("paths must be absolute normalized file paths") + return value + + +def _under(path, parent): + return path == parent or path.startswith(parent + "/") + + +def _candidate_script(raw): + lines = raw.lstrip().splitlines(keepends=True) + opening = lines[0].strip() + if not opening.startswith("```"): + return raw, None + if opening not in {"```bash", "```sh", "```shell"}: + raise HFBackendError("candidate code fence must declare Bash or sh") + end = len(lines) + while end > 1 and not lines[end - 1].strip(): + end -= 1 + closed = end > 1 and lines[end - 1].strip() == "```" + script = "".join(lines[1 : end - 1] if closed else lines[1:]) + if not script.strip(): + raise HFBackendError("candidate response must contain a shell script") + return script, { + "wrapper": "markdown_bash", + "closing_fence_present": closed, + "raw_sha256": hashlib.sha256(raw.encode()).hexdigest(), + "script_sha256": hashlib.sha256(script.encode()).hexdigest(), + } + + +def submission_digest(files): + """Hash sorted relative paths, NUL, content, NUL as source_digest does.""" + digest = hashlib.sha256() + for name, content in sorted(files.items()): + digest.update(name.lstrip("/").encode()) + digest.update(b"\0") + digest.update(content) + digest.update(b"\0") + return digest.hexdigest() + + +def _local_file(path, limit): + # Walk descriptors so neither a final link nor a linked parent is followed. + path = Path(path).absolute() + fd = os.open("/", os.O_RDONLY | os.O_DIRECTORY) + try: + for component in path.parts[1:-1]: + next_fd = os.open( + component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd + ) + os.close(fd) + fd = next_fd + file_fd = os.open( + path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd + ) + with os.fdopen(file_fd, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise HFBackendError("task artifacts must be regular unlinked files") + if info.st_size > limit: + raise HFBackendError("task artifact exceeds byte limit") + content = stream.read(limit + 1) + if len(content) > limit: + raise HFBackendError("task artifact exceeds byte limit") + return content, format(stat.S_IMODE(info.st_mode), "o") + finally: + os.close(fd) + + +def _plain(value): + # HF generated output types derive from dict and preserve extra response fields. + return json.loads(json.dumps(value)) + + +def _unique_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise HFBackendError("duplicate JSON key") + result[key] = value + return result + + +class HFTaskBackend: + """Execute each solve and grade in a fresh, dedicated HF Jobs Sandbox.""" + + def __init__( + self, + task_root: Path, + execution: dict, + *, + token=None, + api=None, + sandbox_factory=None, + inference_factory=None, + canary: str | None = None, + inference_token=None, + ): + if version("huggingface_hub") != "1.33.0": + raise HFBackendError( + "this execution adapter requires huggingface_hub==1.33.0" + ) + self.task_root = Path(task_root) + self.execution = dict(execution) + self.token = token if token is not None else os.environ.get("HF_SANDBOX_TOKEN") + self.inference_token = ( + inference_token + if inference_token is not None + else os.environ.get("HF_INFERENCE_TOKEN") + ) + credentials = [ + value + for value in (self.token, self.inference_token, os.environ.get("HF_TOKEN")) + if value + ] + if len(set(credentials)) != len(credentials): + raise HFBackendError( + "execution, inference and persistence credentials must differ" + ) + if execution.get("strong_isolation") or execution.get( + "require_strong_isolation" + ): + raise HFBackendError("strong verifier isolation is unsupported") + for key in ("agent_image", "verifier_image"): + if not isinstance(execution.get(key), str) or not _IMAGE.fullmatch( + execution[key] + ): + raise HFBackendError( + f"{key} must be an immutable OCI name@sha256 reference" + ) + self.workdir = _path(execution.get("workdir", "/app")) + self.private_paths = list( + dict.fromkeys( + _path(item) + for item in ["/tests", "/solution", *execution.get("private_paths", [])] + ) + ) + self.reward_path = _path( + execution.get("reward_path", "/logs/verifier/reward.txt") + ) + paths = execution.get("submission_paths") + if not isinstance(paths, list) or not paths: + raise HFBackendError("submission_paths must be a nonempty file allowlist") + self.submission_paths = [_path(item) for item in paths] + for index, path in enumerate(self.submission_paths): + if any( + _under(path, protected) or _under(protected, path) + for protected in [ + "/tests", + "/solution", + self.reward_path, + *self.private_paths, + ] + ): + raise HFBackendError("submission path overlaps private or reward paths") + if any( + _under(path, other) or _under(other, path) + for other in self.submission_paths[:index] + ): + raise HFBackendError("submission paths overlap or repeat") + self.timeout = self._bound(execution, "timeout_seconds", 120, 3600) + self.max_bytes = self._bound( + execution, "max_submission_bytes", 1048576, 16 * 1048576 + ) + self.max_script = self._bound(execution, "max_script_bytes", 65536, 1048576) + self.max_output = self._bound( + execution, "max_tool_output_bytes", 16384, 1048576 + ) + self.cleanup_attempts = self._bound(execution, "cleanup_attempts", 3, 10) + self.cleanup_timeout = self._bound(execution, "cleanup_timeout_seconds", 10, 60) + self.api = api if api is not None else HfApi(token=self.token) + self.sandbox_factory = sandbox_factory or Sandbox.create + self.inference_factory = inference_factory or InferenceClient + self.canary = canary if canary is not None else uuid.uuid4().hex + if not isinstance(self.canary, str) or not self.canary: + raise HFBackendError("canary must be nonempty text") + self.canary_digest = hashlib.sha256(self.canary.encode()).hexdigest() + self.marker_path = str( + PurePosixPath(self.workdir) / (".openenv-state-" + uuid.uuid4().hex) + ) + self._audit = [] + self._transcripts = [] + self._owned_ids = set() + self._credentials = tuple(credentials) + + def _redact(self, value): + if isinstance(value, str): + for secret in self._credentials: + value = value.replace(secret, "[REDACTED]") + elif isinstance(value, bytes): + for secret in self._credentials: + value = value.replace(secret.encode(), b"[REDACTED]") + elif isinstance(value, dict): + value = { + self._redact(key): self._redact(item) for key, item in value.items() + } + elif isinstance(value, list): + value = [self._redact(item) for item in value] + return value + + def _has_canary(self, value): + if isinstance(value, str): + return self.canary in value + if isinstance(value, dict): + return any(self._has_canary(item) for item in value.values()) + if isinstance(value, list): + return any(self._has_canary(item) for item in value) + return False + + @staticmethod + def _bound(config, key, default, maximum): + value = config.get(key, default) + if type(value) is not int or not 0 < value <= maximum: + raise HFBackendError(f"{key} must be an integer from 1 to {maximum}") + return value + + def _record(self): + return { + "files": {}, + "submission_digest": submission_digest({}), + "sandbox_id": None, + "image_ref": self.execution["agent_image"], + "image_reference_kind": "control_plane_immutable_reference", + "cleanup_verified": False, + "error": None, + "transcript": [], + "usage": {"input_tokens": 0, "output_tokens": 0}, + "input_tokens": 0, + "output_tokens": 0, + "latency_seconds": 0.0, + "model_metadata": {}, + "limitations": list(_LIMITATIONS), + } + + @staticmethod + def _error(exc): + return ( + str(exc) + if isinstance(exc, HFBackendError) + else f"HF backend operation failed ({type(exc).__name__})" + ) + + def _inspect(self, sandbox_id): + job = self.api.inspect_job( + job_id=sandbox_id, + namespace=self.execution.get("namespace"), + token=self.token, + ) + if job.id != sandbox_id or ( + self.execution.get("namespace") + and job.owner.name != self.execution["namespace"] + ): + raise HFBackendError("control-plane job identity mismatch") + return job + + def _create(self, image, record, *, grader=False): + if not self.token: + raise HFBackendError("HF_SANDBOX_TOKEN execution credential is required") + record["creation_label"] = uuid.uuid4().hex + sandbox = self.sandbox_factory( + image=image, + flavor=self.execution.get("flavor", "cpu-basic"), + idle_timeout=self.timeout, + namespace=self.execution.get("namespace"), + forward_hf_token=False, + token=self.token, + start_timeout=self.timeout, + labels={ + "name": "openenv-coding-" + record["creation_label"], + "openenv-coding-attempt": record["creation_label"], + }, + ) + sandbox_id = sandbox.id + # Register before inspection so image/identity failures still clean up. + record["grade_sandbox_id" if grader else "sandbox_id"] = sandbox_id + if sandbox_id in self._owned_ids: + raise HFBackendError("sandbox identity was reused") + self._owned_ids.add(sandbox_id) + # SDK readiness checks the RPC server; the Jobs status can still lag. + # Reconcile only SCHEDULING within the caller's existing deadline, and + # recheck identity and the immutable image on every observation. + for attempt in range(20): + job = self._inspect(sandbox_id) + if job.docker_image != image: + raise HFBackendError("sandbox control-plane image mismatch") + if job.status.stage == "RUNNING": + break + if job.status.stage != "SCHEDULING": + raise HFBackendError("sandbox control-plane state is not runnable") + if attempt < 19: + time.sleep(0.5) + else: + raise HFBackendError("sandbox control-plane readiness did not converge") + # The SDK stat/read pair otherwise permits up to 512 MiB per read. + sandbox.files.MAX_READ_BYTES = self.max_bytes + sandbox.files.mkdir(self.workdir) + return sandbox + + def _recover_startup_jobs(self, record, pending): + # create() can allocate a job before failing during readiness. The label + # is unique to this attempt and is filtered again before any cancellation. + jobs = self.api.list_jobs( + labels={"openenv-coding-attempt": record["creation_label"]}, + namespace=self.execution.get("namespace"), + token=self.token, + timeout=self.cleanup_timeout, + ) + for index, job in enumerate(jobs): + if ( + index >= 4 + or (job.labels or {}).get("openenv-coding-attempt") + != record["creation_label"] + ): + raise HFBackendError("startup cleanup ownership could not be verified") + if ( + self.execution.get("namespace") + and job.owner.name != self.execution["namespace"] + ): + raise HFBackendError("startup cleanup namespace mismatch") + pending.add(job.id) + self._owned_ids.add(job.id) + record["cleanup_job_ids"] = sorted(pending) + record[ + "grade_sandbox_id" if "grade_sandbox_id" in record else "sandbox_id" + ] = sorted(pending)[0] + + def _cleanup(self, record, sandbox): + sandbox_id = record.get("grade_sandbox_id") or record.get("sandbox_id") + if sandbox_id is None and not record.get("creation_label"): + return + pending = {sandbox_id} if sandbox_id is not None else set() + recovery_complete = sandbox_id is not None + try: + with _deadline(self.cleanup_timeout): + for _ in range(self.cleanup_attempts): + if not recovery_complete: + try: + self._recover_startup_jobs(record, pending) + recovery_complete = bool(pending) + except _DeadlineExceeded: + raise + except Exception: + pass + if not pending: + continue + for owned_id in list(pending): + try: + self.api.cancel_job( + job_id=owned_id, + namespace=self.execution.get("namespace"), + token=self.token, + ) + except _DeadlineExceeded: + raise + except Exception: + if sandbox is not None and sandbox.id == owned_id: + try: + sandbox.kill() + except _DeadlineExceeded: + raise + except Exception: + pass + try: + if self._inspect(owned_id).status.stage in _TERMINAL: + pending.remove(owned_id) + except _DeadlineExceeded: + raise + except Exception: + continue + if not pending and recovery_complete: + record["cleanup_verified"] = True + break + except Exception: + pass + if not record["cleanup_verified"]: + record["error"] = record["error"] or "sandbox cleanup could not be verified" + if "reward" in record: + record["reward"] = None + + def _audit_solver(self, sandbox): + evidence = { + "sandbox_id": sandbox.id, + "private_paths_absent": False, + "fresh_state_absent": False, + } + self._audit.append(evidence) + for path in self.private_paths: + if sandbox.files.exists(path): + raise HFBackendError("private path exists in solver sandbox") + evidence["private_paths_absent"] = True + if sandbox.files.exists(self.marker_path): + raise HFBackendError("fresh sandbox contains prior-state marker") + evidence["fresh_state_absent"] = True + sandbox.files.write(self.marker_path, self.canary_digest.encode()) + + def _run(self, sandbox, argv, end, *, cwd=None, trace=None): + output = [] + size = 0 + + def collect(chunk): + nonlocal size + raw = chunk.encode() + if size + len(raw) > self.max_output: + if trace is not None: + trace["output_truncated"] = True + raise HFBackendError("terminal output exceeded byte limit") + size += len(raw) + output.append(chunk) + + try: + remaining = end - time.monotonic() + if remaining <= 0: + raise HFBackendError("controller deadline exceeded") + result = sandbox.run( + argv, + shell=False, + cwd=cwd or self.workdir, + timeout=remaining, + check=False, + capture_output=False, + on_stdout=collect, + on_stderr=collect, + ) + if trace is not None: + trace["exit_code"] = result.exit_code + if result.timed_out: + raise HFBackendError("sandbox command timed out") + return result.exit_code, "".join(output) + finally: + if trace is not None: + trace["output"] = "" if trace["output_truncated"] else "".join(output) + + def _collect(self, sandbox): + files = {} + remaining = self.max_bytes + for path in self.submission_paths: + try: + entry = sandbox.files.stat(path) + except SandboxError as exc: + if exc.status_code == 404: + continue + raise + if ( + entry.type != "file" + or type(entry.size) is not int + or not 0 <= entry.size <= remaining + ): + raise HFBackendError("submission is not a bounded regular file") + sandbox.files.MAX_READ_BYTES = remaining + data = sandbox.files.read(path) + if ( + not isinstance(data, bytes) + or len(data) != entry.size + or len(data) > remaining + ): + raise HFBackendError("submission changed size or exceeded byte limit") + files[path] = data + if any(secret.encode() in data for secret in self._credentials): + raise HFBackendError("credential detected in submission artifact") + remaining -= len(data) + return files + + def execute_script(self, script: str, role: str) -> dict: + """Execute a reference/candidate script without exposing controller credentials.""" + record = self._record() + record["role"] = role + sandbox = None + started = time.monotonic() + try: + with _deadline(self.timeout): + if ( + not isinstance(script, str) + or len(script.encode()) > self.max_script + ): + raise HFBackendError("script exceeds byte limit or is not text") + sandbox = self._create(self.execution["agent_image"], record) + self._audit_solver(sandbox) + exit_code, output = self._run( + sandbox, ["/bin/bash", "-c", script], started + self.timeout + ) + record["transcript"].append( + { + "role": role, + "script": script, + "output": output, + "exit_code": exit_code, + } + ) + if exit_code != 0: + raise HFBackendError("task script exited nonzero") + record["files"] = self._collect(sandbox) + record["submission_digest"] = submission_digest(record["files"]) + except Exception as exc: + record["error"] = self._error(exc) + finally: + self._cleanup(record, sandbox) + record["latency_seconds"] = time.monotonic() - started + self._transcripts.append(record["transcript"]) + return self._redact(record) + + def _model(self, model, seed, *, tools=False): + model = dict(model) + if tools and "context_paths" in model: + raise HFBackendError("target.context_paths is unsupported") + if ( + not isinstance(model.get("model_id"), str) + or not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", model["model_id"]) + or not isinstance(model.get("revision"), str) + or not _REVISION.fullmatch(model["revision"]) + or not isinstance(model.get("provider"), str) + or not model["provider"] + or model["provider"] in {"auto", "cheapest", "fastest", "preferred"} + ): + raise HFBackendError( + "model_id, pinned source revision and explicit provider are required" + ) + if type(seed) is not int or seed < 0: + raise HFBackendError("seed must be a nonnegative integer") + if tools and model.get("supports_tools") is False: + raise HFBackendError( + "configured model does not support terminal function calling" + ) + if "enable_thinking" in model and type(model["enable_thinking"]) is not bool: + raise HFBackendError("enable_thinking must be a boolean") + if type(model.get("supports_seed", False)) is not bool: + raise HFBackendError("supports_seed must be an explicit boolean") + aliases = model.get("response_model_aliases", []) + if not isinstance(aliases, list) or any( + not isinstance(item, str) or not item for item in aliases + ): + raise HFBackendError( + "response_model_aliases must be exact nonempty strings" + ) + allowed = {model["model_id"], *aliases} + if model.get("provider_model_id"): + if not isinstance(model["provider_model_id"], str): + raise HFBackendError("provider_model_id must be text") + allowed.add(model["provider_model_id"]) + limits = { + key: self._bound(model, key, default, maximum) + for key, default, maximum in ( + ("max_steps", 8, 128), + ("max_tokens", 2048, 32768), + ("max_total_tokens", 16384, 1048576), + ("max_output_bytes", 16384, 1048576), + ) + } + if not self.inference_token: + raise HFBackendError("controller HF_INFERENCE_TOKEN is required") + client = self.inference_factory( + provider=model["provider"], + api_key=self.inference_token, + timeout=self.timeout, + ) + return model, allowed, limits, client + + def _chat(self, client, model, allowed, limits, messages, record, seed, *, tools): + used = record["input_tokens"] + record["output_tokens"] + # Bound the request body as well as provider-reported usage. Providers do + # not expose a preflight tokenizer through this API. + message_bytes = len(json.dumps(messages).encode()) + if message_bytes > limits["max_total_tokens"] * 4: + raise HFBackendError("chat context exceeds byte budget") + remaining = limits["max_total_tokens"] - used + if remaining <= 0: + raise HFBackendError("model total token budget exhausted") + request = { + "messages": self._redact(messages), + "model": model["model_id"], + "max_tokens": min(limits["max_tokens"], remaining), + "temperature": model.get("temperature", 0), + } + if "top_p" in model: + value = model["top_p"] + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or not 0 < value <= 1 + ): + raise HFBackendError("top_p must be a finite number in (0, 1]") + request["top_p"] = value + if "enable_thinking" in model: + request["extra_body"] = { + "chat_template_kwargs": {"enable_thinking": model["enable_thinking"]} + } + if model.get("supports_seed", False): + request["seed"] = seed + if tools: + request.update(tools=[_TOOL], tool_choice="auto") + if not tools: + request.update(stream=True, stream_options={"include_usage": True}) + metadata = record["model_metadata"] + metadata.setdefault("requests", []).append( + {key: value for key, value in request.items() if key != "messages"} + ) + if tools: + response = _plain(client.chat_completion(**request)) + metadata.setdefault("responses", []).append( + { + "model": response.get("model"), + "system_fingerprint": response.get("system_fingerprint"), + "id": response.get("id"), + "seed_sent": request.get("seed"), + } + ) + else: + response = self._candidate_response( + client, request, allowed, limits, record + ) + if response.get("model") not in allowed: + raise HFBackendError( + "provider response model does not match configured exact identities" + ) + usage = response.get("usage") + if not isinstance(usage, dict): + raise HFBackendError("provider response omitted token usage") + for source, target in ( + ("prompt_tokens", "input_tokens"), + ("completion_tokens", "output_tokens"), + ): + value = usage.get(source) + if type(value) is not int or value < 0: + raise HFBackendError("provider returned invalid token usage") + if tools: + record[target] += value + record["usage"][target] += value + if ( + usage["completion_tokens"] > request["max_tokens"] + or record["input_tokens"] + record["output_tokens"] + > limits["max_total_tokens"] + ): + raise HFBackendError("provider exceeded token budget") + choices = response.get("choices", []) + if len(choices) != 1 or not isinstance(choices[0].get("message"), dict): + raise HFBackendError("provider must return exactly one assistant message") + message = choices[0]["message"] + record["transcript"].append(message) + if len(json.dumps(message).encode()) > limits["max_output_bytes"]: + raise HFBackendError("model output exceeded byte limit") + if choices[0].get("finish_reason") not in {"stop", "tool_calls"}: + raise HFBackendError("model response was incomplete") + if choices[0]["finish_reason"] == "tool_calls" and ( + not isinstance(message.get("tool_calls"), list) or not message["tool_calls"] + ): + raise HFBackendError("invalid terminal function calls") + return message + + def _candidate_response(self, client, request, allowed, limits, record): + # The pinned SDK yields parsed deltas but owns HTTP contexts on the client. + response = {} + parts = {"content": []} + size = script_size = 0 + stopped = False + finish_reason = None + usage = None + try: + with closing(client.chat_completion(**request)) as stream: + for parsed in stream: + event = _plain(parsed) + if not isinstance(event, dict) or event.get("error") is not None: + raise HFBackendError("provider returned a stream error") + # Outer SDK dicts omit extra fields on nested dataclasses. + if isinstance(event.get("choices"), list): + for index, choice in enumerate(parsed.choices): + event["choices"][index]["delta"] = _plain(choice.delta) + if not response: + response = { + key: event.get(key) + for key in ("model", "id", "system_fingerprint") + } + record["model_metadata"]["responses"].append( + {**response, "seed_sent": request.get("seed")} + ) + if event.get("model") not in allowed: + raise HFBackendError( + "provider response model does not match configured exact identities" + ) + if ( + not isinstance(event.get("id"), str) + or not event["id"] + or any( + event.get(key) != value for key, value in response.items() + ) + ): + raise HFBackendError( + "provider stream response identity changed" + ) + if usage is not None: + raise HFBackendError("provider returned data after final usage") + choices = event.get("choices") + if not isinstance(choices, list) or len(choices) > 1: + raise HFBackendError( + "provider must stream exactly one assistant choice" + ) + if choices: + if stopped: + raise HFBackendError( + "provider returned a choice after stop" + ) + choice = choices[0] + if ( + not isinstance(choice, dict) + or type(choice.get("index")) is not int + or choice["index"] != 0 + or not isinstance(choice.get("delta"), dict) + ): + raise HFBackendError( + "provider must stream exactly one assistant choice at index 0" + ) + delta = choice["delta"] + if delta.get("role") not in {None, "assistant"} or ( + not parts["content"] and delta.get("role") != "assistant" + ): + raise HFBackendError( + "provider must stream an assistant message" + ) + for key, value in delta.items(): + if key == "role" or value is None: + continue + if key == "tool_calls" and value == []: + continue + if key not in {"content", "reasoning", "reasoning_content"}: + raise HFBackendError( + "candidate stream contains tool calls or unsupported deltas" + ) + if not isinstance(value, str): + raise HFBackendError( + "provider stream delta must be text" + ) + length = len(value.encode()) + size += length + if key == "content": + script_size += length + if ( + size > limits["max_output_bytes"] + or script_size > self.max_script + ): + raise HFBackendError("model output exceeded byte limit") + if value: + parts.setdefault(key, []).append(value) + # Empty assistant role events still establish the message. + if not parts["content"]: + parts["content"].append("") + finish = choice.get("finish_reason") + if finish is not None: + finish_reason = finish + record["model_metadata"]["responses"][-1][ + "finish_reason" + ] = finish + if finish not in {"stop", "length"}: + raise HFBackendError("model response was incomplete") + stopped = True + elif not stopped: + raise HFBackendError( + "model response was incomplete before usage" + ) + event_usage = event.get("usage") + if event_usage is not None: + if not stopped: + raise HFBackendError("provider returned usage before stop") + if not isinstance(event_usage, dict) or any( + type(event_usage.get(key)) is not int + or event_usage[key] < 0 + for key in ( + "prompt_tokens", + "completion_tokens", + "total_tokens", + ) + ): + raise HFBackendError( + "provider returned invalid token usage" + ) + if event_usage["total_tokens"] != ( + event_usage["prompt_tokens"] + + event_usage["completion_tokens"] + ): + raise HFBackendError( + "provider returned inconsistent token usage" + ) + usage = event_usage + # Usage already observed remains evidence if the stream + # fails later. The caller accounts only non-streamed calls. + for source, target in ( + ("prompt_tokens", "input_tokens"), + ("completion_tokens", "output_tokens"), + ): + record[target] += usage[source] + record["usage"][target] += usage[source] + if ( + usage["completion_tokens"] > request["max_tokens"] + or record["input_tokens"] + record["output_tokens"] + > limits["max_total_tokens"] + ): + raise HFBackendError("provider exceeded token budget") + elif not choices: + raise HFBackendError("provider response omitted token usage") + if not stopped: + raise HFBackendError("model response was incomplete") + if usage is None: + raise HFBackendError("provider response omitted token usage") + if finish_reason != "stop": + raise HFBackendError("model response was incomplete") + response.update( + usage=usage, + choices=[ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + **{key: "".join(value) for key, value in parts.items()}, + }, + } + ], + ) + return response + except Exception: + if parts["content"]: + record["transcript"].append( + { + "role": "assistant", + **{key: "".join(value) for key, value in parts.items()}, + } + ) + raise + finally: + client.close() + + def _model_metadata(self, model, seed): + return { + "requested_model": model["model_id"], + "source_revision": model["revision"], + "provider": model["provider"], + "provider_model_id": model.get("provider_model_id"), + "response_model_aliases": model.get("response_model_aliases", []), + "seed": seed, + "seed_supported": model.get("supports_seed", False), + "served_weight_revision_pinned": False, + "responses": [], + } + + def _instruction(self): + raw, _ = _local_file(self.task_root / "instruction.md", self.max_script) + return raw.decode("utf-8") + + def _candidate_instruction(self, model, record): + instruction = self._instruction() + if "context_paths" not in model: + return instruction + paths = _candidate_context_paths(model["context_paths"]) + record["candidate_context"] = [] + parts = [instruction] + remaining = 65536 + for path in paths: + raw, _ = _local_file(self.task_root / path, remaining) + remaining -= len(raw) + if b"\0" in raw: + raise HFBackendError("candidate context contains NUL") + text = raw.decode("utf-8") + runtime_path = ( + PurePosixPath(path).relative_to("environment/source").as_posix() + ) + record["candidate_context"].append( + { + "path": path, + "runtime_path": runtime_path, + "size": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + } + ) + parts.append( + f"Public source path: {path}\n" + f"Runtime filename relative to {self.workdir}: {runtime_path}\n" + f"---\n{text}\n---" + ) + return "\n\n".join(parts) + + def generate_candidate(self, model: dict, seed: int) -> dict: + """Generate a shell script from the instruction and explicit public context.""" + record = self._record() + record["script"] = "" + started = time.monotonic() + try: + with _deadline(self.timeout): + instruction = self._candidate_instruction(model, record) + model, allowed, limits, client = self._model(model, seed) + record["model_metadata"] = self._model_metadata(model, seed) + messages = [ + { + "role": "system", + "content": "Return only a bash script that attempts the task. Do not use Markdown fences.", + }, + {"role": "user", "content": instruction}, + ] + record["transcript"] = messages + message = self._chat( + client, model, allowed, limits, messages, record, seed, tools=False + ) + script = message.get("content") + if ( + not isinstance(script, str) + or not script.strip() + or message.get("tool_calls") + ): + raise HFBackendError( + "candidate response must contain a shell script" + ) + if len(script.encode()) > self.max_script: + raise HFBackendError("candidate script exceeds byte limit") + script, formatting = _candidate_script(script) + if formatting is not None: + record["script_format"] = formatting + record["script"] = script + except Exception as exc: + record["error"] = self._error(exc) + finally: + record["latency_seconds"] = time.monotonic() - started + self._transcripts.append(record["transcript"]) + return self._redact(record) + + def solve(self, model: dict, seed: int, role="target") -> dict: + """Run a bounded terminal-tool agent with inference kept in the controller.""" + record = self._record() + record["role"] = role + record.update(truncated=False, termination_reason="completed") + sandbox = None + started = time.monotonic() + try: + with _deadline(self.timeout): + model, allowed, limits, client = self._model(model, seed, tools=True) + record["model_metadata"] = self._model_metadata(model, seed) + messages = [ + { + "role": "system", + "content": "Complete the task using the terminal function. Commands run in the task workspace. Finish with a short message after saving your solution files.", + }, + {"role": "user", "content": self._instruction()}, + ] + record["transcript"] = messages + sandbox = self._create(self.execution["agent_image"], record) + self._audit_solver(sandbox) + calls_used = 0 + for _ in range(limits["max_steps"]): + message = self._chat( + client, + model, + allowed, + limits, + messages, + record, + seed, + tools=True, + ) + calls = message.get("tool_calls") or [] + if not calls: + break + if not isinstance(calls, list): + raise HFBackendError("invalid terminal function calls") + if calls_used + len(calls) > limits["max_steps"]: + record.update( + truncated=True, termination_reason="tool_call_limit" + ) + break + ids = set() + for call in calls: + if ( + call.get("type") != "function" + or call.get("function", {}).get("name") != "terminal" + or not isinstance(call.get("id"), str) + or not call["id"] + or call["id"] in ids + ): + raise HFBackendError("invalid terminal function call") + ids.add(call["id"]) + arguments = json.loads( + call["function"]["arguments"], + object_pairs_hook=_unique_pairs, + ) + if ( + not isinstance(arguments, dict) + or set(arguments) != {"command"} + or not isinstance(arguments["command"], str) + or len(arguments["command"].encode()) > self.max_script + ): + raise HFBackendError("invalid terminal function arguments") + calls_used += 1 + exit_code, output = self._run( + sandbox, + ["/bin/bash", "-c", arguments["command"]], + started + self.timeout, + ) + messages.append( + { + "role": "tool", + "tool_call_id": call["id"], + "content": json.dumps( + {"exit_code": exit_code, "output": output} + ), + } + ) + else: + record.update(truncated=True, termination_reason="step_limit") + record["files"] = self._collect(sandbox) + record["submission_digest"] = submission_digest(record["files"]) + except Exception as exc: + record["error"] = self._error(exc) + finally: + self._cleanup(record, sandbox) + record["latency_seconds"] = time.monotonic() - started + self._transcripts.append(record["transcript"]) + return self._redact(record) + + def _test_files(self): + root = self.task_root / "tests" + if root.is_symlink() or not root.is_dir(): + raise HFBackendError("task tests must be a regular directory") + files = [] + remaining = 16 * 1048576 + for index, path in enumerate(root.rglob("*")): + if index >= 4096: + raise HFBackendError("too many verifier filesystem entries") + info = path.lstat() + if stat.S_ISDIR(info.st_mode): + continue + relative = path.relative_to(root).as_posix() + raw, mode = _local_file(path, remaining) + remaining -= len(raw) + files.append(("/tests/" + relative, raw, mode)) + return sorted(files) + + def grade(self, submission: dict) -> dict: + """Grade exact allowlisted submission bytes in a separate verifier Sandbox.""" + record = { + "grade_sandbox_id": None, + "verifier_image_ref": self.execution["verifier_image"], + "image_reference_kind": "control_plane_immutable_reference", + "reward": None, + "rawreward": b"", + "transcript": [], + "cleanup_verified": False, + "error": None, + "limitations": list(_LIMITATIONS), + } + sandbox = None + started = time.monotonic() + try: + with _deadline(self.timeout): + files = submission["files"] + if ( + submission.get("error") + or not isinstance(files, dict) + or any( + path not in self.submission_paths or not isinstance(raw, bytes) + for path, raw in files.items() + ) + or sum(map(len, files.values())) > self.max_bytes + ): + raise HFBackendError("invalid submission transfer") + if submission.get("submission_digest") != submission_digest(files): + raise HFBackendError("submission digest mismatch") + files = dict(files) + tests = self._test_files() + command = self.execution.get( + "verifier_command", ["/bin/bash", "/tests/test.sh"] + ) + if ( + not isinstance(command, list) + or not command + or any(not isinstance(arg, str) or "\x00" in arg for arg in command) + ): + raise HFBackendError("verifier_command must be an argv list") + sandbox = self._create( + self.execution["verifier_image"], record, grader=True + ) + if sandbox.files.exists("/tests") or sandbox.files.exists("/solution"): + raise HFBackendError( + "verifier image contains preexisting private paths" + ) + for path in self.submission_paths: + if sandbox.files.exists(path): + if sandbox.files.stat(path).type != "file": + raise HFBackendError( + "verifier submission destination is not a regular file" + ) + sandbox.files.delete(path) + for path, raw, mode in tests: + sandbox.files.write(path, raw, mode=mode) + sandbox.files.write( + "/tests/.openenv-canary-" + uuid.uuid4().hex, self.canary.encode() + ) + for path, raw in files.items(): + sandbox.files.write(path, raw) + # Harbor normally mounts this directory. Fresh Sandbox graders + # need it before a native verifier can write its reward file. + sandbox.files.mkdir(str(PurePosixPath(self.reward_path).parent)) + # A stale image-provided reward must not become new evidence. + if sandbox.files.exists(self.reward_path): + sandbox.files.delete(self.reward_path) + trace = {"exit_code": None, "output": "", "output_truncated": False} + record["transcript"].append(trace) + exit_code, _ = self._run( + sandbox, command, started + self.timeout, trace=trace + ) + if exit_code != 0: + raise HFBackendError("verifier command exited nonzero") + entry = sandbox.files.stat(self.reward_path) + if ( + entry.type != "file" + or type(entry.size) is not int + or not 0 < entry.size <= min(self.max_bytes, 65536) + ): + raise HFBackendError("reward is not a bounded regular file") + sandbox.files.MAX_READ_BYTES = min(self.max_bytes, 65536) + raw = sandbox.files.read(self.reward_path) + if len(raw) != entry.size: + raise HFBackendError("reward changed during collection") + if any(secret.encode() in raw for secret in self._credentials): + raise HFBackendError("credential detected in reward artifact") + record["rawreward"] = raw + text = raw.decode("utf-8").strip() + if text.startswith("{"): + value = json.loads( + text, parse_float=Decimal, object_pairs_hook=_unique_pairs + )[self.execution.get("reward_key", "reward")] + else: + value = Decimal(text) + if ( + isinstance(value, bool) + or not isinstance(value, (Decimal, int)) + or not Decimal(value).is_finite() + or not math.isfinite(float(value)) + ): + raise HFBackendError("reward must be finite numeric data") + record["reward"] = float(value) + except Exception as exc: + record["error"] = self._error(exc) + record["reward"] = None + finally: + self._cleanup(record, sandbox) + record["latency_seconds"] = time.monotonic() - started + self._transcripts.append(record.get("transcript", [])) + record = self._redact(record) + for trace in record["transcript"]: + if len(trace["output"].encode()) > self.max_output: + trace["output"] = "" + trace["output_truncated"] = True + return record + + def leakage_probes(self) -> list: + """Audit private paths, two fresh solver instances, and accumulated transcripts.""" + probes = [] + while len(self._audit) < 2: + record = self.execute_script(":", "leakage_probe") + if record["error"]: + probes.append( + { + "name": "fresh_state_absent", + "passed": False, + "error": record["error"], + "evidence": list(self._audit), + } + ) + break + for name in ("private_paths_absent", "fresh_state_absent"): + if any(probe["name"] == name for probe in probes): + continue + probes.append( + { + "name": name, + "error": None, + "passed": len(self._audit) >= 2 + and all(item[name] for item in self._audit), + "evidence": list(self._audit), + } + ) + probes.append( + { + "name": "canary_not_in_transcript", + "error": None, + "passed": not self._has_canary(self._transcripts), + "evidence": { + "canary_digest": self.canary_digest, + "transcript_count": len(self._transcripts), + }, + } + ) + return probes diff --git a/examples/coding_audit/runner.py b/examples/coding_audit/runner.py new file mode 100644 index 000000000..bd3762ece --- /dev/null +++ b/examples/coding_audit/runner.py @@ -0,0 +1,744 @@ +"""Collect a bounded coding audit using fresh, credential-scoped HF Sandboxes. + +This supplemental profile does not replace RFC 008 admission or demonstrate RL +training benefit. Inference runs in the controller; task/verifier data cross only +the explicit file boundary in ``hf``. +""" + +import hashlib +import json +import os +import re +import shutil +import tempfile +import tomllib +from copy import deepcopy +from fractions import Fraction +from pathlib import Path, PurePosixPath + +from openenv.validation.runner import _DIGEST_EXCLUDED_DIRS, source_digest +from openenv.validation.runtime.json_values import dumps, loads + +from .scoring import assess_coding_audit, validate_audit_config + +PROFILE = "harbor-coding-audit-v1" +TARGET_MODEL = "Qwen/Qwen3.8-27B" +TARGET_REVISION = "37561816e66b5b9153b1b64c10968c18c1e6f4a0" +_IMAGE = re.compile(r"[^\s@]+@sha256:[0-9a-f]{64}\Z") + + +def _integer(value, name, low, high): + if type(value) is not int or not low <= value <= high: + raise ValueError(f"{name} must be an integer in [{low}, {high}]") + return value + + +def _candidate_context_paths(paths): + message = ( + "generator.context_paths must list at most 32 unique normalized public " + "files under environment/source/" + ) + if not isinstance(paths, list) or len(paths) > 32: + raise ValueError(message) + result = [] + private = { + "tests", + "test", + "solution", + "validation", + "verifier", + "reference", + "references", + "spec", + "specs", + "secret", + "secrets", + "credential", + "credentials", + "private", + } + for value in paths: + if ( + not isinstance(value, str) + or "\\" in value + or ":" in value + or any(ord(char) < 32 or ord(char) == 127 for char in value) + ): + raise ValueError(message) + path = PurePosixPath(value) + if ( + path.as_posix() != value + or path.parts[:2] != ("environment", "source") + or len(path.parts) < 3 + or any( + part.startswith(".") or part.lower() in private for part in path.parts + ) + or value in result + ): + raise ValueError(message) + result.append(value) + return result + + +def normalize_coding_config(config): + """Validate operator inputs without launching Jobs, Sandboxes or inference.""" + if not isinstance(config, dict): + raise ValueError("coding configuration must be an object") + quality = deepcopy(config.get("validation", {})) + config = json.loads(dumps(config)) + allowed = { + "schema_version", + "profile", + "target", + "generator", + "execution", + "validation", + "generation", + "negative_cases", + "seeds", + "cloud", + } + if ( + set(config) - allowed + or type(config.get("schema_version")) is not int + or config["schema_version"] != 1 + ): + raise ValueError("unsupported coding audit configuration") + if config.get("profile", PROFILE) != PROFILE: + raise ValueError("unsupported coding audit profile") + config["profile"] = PROFILE + config["validation"] = validate_audit_config(quality) + _integer(config["validation"]["reference_repeats"], "reference_repeats", 2, 16) + _integer(config["validation"]["target_attempts"], "target_attempts", 2, 128) + for name in ("target", "generator"): + model = config.get(name, config.get("target")) + if not isinstance(model, dict): + raise ValueError(f"{name} model configuration is required") + model = dict(model) + model_keys = { + "model_id", + "revision", + "provider", + "provider_model_id", + "response_model_aliases", + "max_steps", + "max_tokens", + "max_total_tokens", + "max_output_bytes", + "temperature", + "supports_seed", + "supports_tools", + "top_p", + "enable_thinking", + "context_paths", + } + if name == "target" and "context_paths" in model: + raise ValueError("target.context_paths is unsupported") + if set(model) - model_keys: + raise ValueError( + f"unknown {name} model option; credentials belong in environment secrets" + ) + if not re.fullmatch(r"[^\s/@]+/[^\s/@]+", model.get("model_id", "")): + raise ValueError(f"{name}.model_id must be a canonical Hub repository") + if not re.fullmatch(r"[0-9a-f]{40}", model.get("revision", "")): + raise ValueError(f"{name}.revision must be a full source commit") + if ( + not isinstance(model.get("provider"), str) + or not re.fullmatch(r"[a-z0-9-]+", model["provider"]) + or model["provider"] == "auto" + ): + raise ValueError(f"{name}.provider must be selected explicitly") + if "enable_thinking" in model and type(model["enable_thinking"]) is not bool: + raise ValueError(f"{name}.enable_thinking must be a boolean") + if "context_paths" in model: + model["context_paths"] = _candidate_context_paths(model["context_paths"]) + model.setdefault("max_steps", 8) + model.setdefault("max_tokens", 1024) + model.setdefault("max_total_tokens", 16384) + model.setdefault("supports_seed", False) + model.setdefault("supports_tools", True) + for key, high in ( + ("max_steps", 128), + ("max_tokens", 32768), + ("max_total_tokens", 262144), + ): + _integer(model[key], f"{name}.{key}", 1, high) + config[name] = model + execution = config.get("execution", {}) + if not isinstance(execution, dict): + raise ValueError("execution must be an object") + execution_keys = { + "agent_image", + "verifier_image", + "submission_paths", + "private_paths", + "workdir", + "flavor", + "namespace", + "timeout_seconds", + "verifier_command", + "reward_path", + "reward_key", + "max_submission_bytes", + "cleanup_attempts", + "cleanup_timeout_seconds", + "max_tool_output_bytes", + "max_script_bytes", + "strong_isolation", + "require_strong_isolation", + } + if set(execution) - execution_keys: + raise ValueError( + "unknown execution option; task credentials cannot be embedded" + ) + for key in ("agent_image", "verifier_image"): + if not _IMAGE.fullmatch(execution.get(key, "")): + raise ValueError(f"execution.{key} must be an immutable OCI reference") + if not execution.get("submission_paths"): + raise ValueError( + "execution.submission_paths must declare the artifact boundary" + ) + generation = config.setdefault("generation", {}) + if not isinstance(generation, dict): + raise ValueError("generation must be an object") + if set(generation) - {"candidate_count"}: + raise ValueError("unknown generation option") + generation.setdefault("candidate_count", 2) + _integer(generation["candidate_count"], "candidate_count", 1, 16) + negatives = config.get("negative_cases", []) + expected = config["validation"]["required_negative_cases"] + if ( + not isinstance(negatives, list) + or any( + not isinstance(case, dict) or set(case) != {"id", "script"} + for case in negatives + ) + or [case["id"] for case in negatives] != expected + or any( + not isinstance(case["script"], str) or not case["script"].strip() + for case in negatives + ) + ): + raise ValueError( + "negative_cases must supply the trusted required IDs and scripts in order" + ) + attempts = config["validation"]["target_attempts"] + seeds = config.setdefault("seeds", list(range(attempts))) + if ( + not isinstance(seeds, list) + or len(seeds) != attempts + or any(type(seed) is not int for seed in seeds) + or len(set(seeds)) != len(seeds) + ): + raise ValueError( + "one distinct seed/attempt identifier per target attempt is required" + ) + for seed in seeds: + _integer(seed, "seed", 0, 2**32 - 1) + cloud = config.setdefault("cloud", {}) + if not isinstance(cloud, dict): + raise ValueError("cloud must be an object") + if set(cloud) - { + "results_repo", + "namespace", + "flavor", + "timeout_seconds", + "max_sandboxes", + "max_inference_calls", + }: + raise ValueError("unknown cloud option") + cloud.setdefault("max_sandboxes", 100) + cloud.setdefault("max_inference_calls", 256) + _integer(cloud["max_sandboxes"], "max_sandboxes", 1, 10000) + _integer(cloud["max_inference_calls"], "max_inference_calls", 1, 10000) + return config + + +def coding_budget(config, *, has_reference=True): + """A conservative, finite launch ceiling, including leakage probes.""" + repeats = config["validation"]["reference_repeats"] + candidates = config["generation"]["candidate_count"] + references = candidates + int(has_reference) + executions = ( + 1 + references * repeats + len(config["negative_cases"]) + len(config["seeds"]) + ) + return { + "sandbox_jobs_max": 2 * executions + 6, + "inference_calls_max": candidates + + len(config["seeds"]) * config["target"]["max_steps"], + "generated_candidates": candidates, + "target_attempts": len(config["seeds"]), + } + + +def _credentials(): + names = ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN") + values = {name: os.environ.get(name) for name in names} + if not all(values.values()): + raise ValueError( + "HF_TOKEN, HF_INFERENCE_TOKEN and HF_SANDBOX_TOKEN are required" + ) + if len(set(values.values())) != len(values): + raise ValueError( + "artifact, inference and sandbox credentials must be distinct and separately scoped" + ) + return values + + +def _write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(dumps(value) + "\n") + + +def live_provider_id(info, provider): + """Read both Hub SDK provider-map representations without selecting an alias.""" + mappings = info.inference_provider_mapping + if isinstance(mappings, dict): + candidates = [mappings[provider]] if provider in mappings else [] + elif isinstance(mappings, list): + candidates = [ + entry + for entry in mappings + if ( + entry.get("provider") + if isinstance(entry, dict) + else getattr(entry, "provider", None) + ) + == provider + ] + else: + candidates = [] + if len(candidates) != 1: + raise ValueError("selected model/provider mapping is missing or ambiguous") + entry = candidates[0] + status = ( + entry.get("status") + if isinstance(entry, dict) + else getattr(entry, "status", None) + ) + identity = ( + entry.get("provider_id") + if isinstance(entry, dict) + else getattr(entry, "provider_id", None) + ) + if status != "live" or not isinstance(identity, str) or not identity: + raise ValueError("selected model/provider is not live") + return identity + + +def validate_coding_task(task_root, config): + """Check supported native task contracts locally, before controller spending. + + Returns normalized audit configuration. Passing this check is preparation, + not evidence that the task, image, or grader passes runtime validation. + """ + config = normalize_coding_config(config) + task_root = Path(task_root).resolve() + task_config = tomllib.loads((task_root / "task.toml").read_text()) + if any((task_root / "steps").glob("*")): + raise ValueError("multi-step Harbor tasks are not supported by this profile") + for section in ("environment", "agent", "verifier", "solution"): + section_config = task_config.get(section, {}) + if not isinstance(section_config, dict): + raise ValueError("task phase configuration must be an object") + if section_config.get("environment"): + raise ValueError("separate phase environment contracts are unsupported") + if section_config.get("allow_internet") is False: + raise ValueError("non-public phase network contracts are unsupported") + if section_config.get("allowed_hosts"): + raise ValueError("network allowlist contracts are unsupported") + if section_config.get("env"): + raise ValueError( + "task environment/credential substitutions are unsupported" + ) + if section_config.get("network_mode", "public") != "public": + raise ValueError("non-public phase network contracts are unsupported") + environment = task_config.get("environment", {}) + if task_config.get("verifier", {}).get("collect"): + raise ValueError("verifier collection commands are unsupported") + if environment.get("workdir") not in ( + None, + config["execution"].get("workdir", "/app"), + ): + raise ValueError("source and audit workdirs disagree") + if any( + environment.get(key) + for key in ("gpus", "tpu", "mcp_servers", "healthcheck", "skills_dir") + ): + raise ValueError( + "accelerator, healthcheck, skills and external MCP requirements are unsupported" + ) + if any( + (task_root / "environment" / name).exists() + for name in ( + "compose.yaml", + "compose.yml", + "docker-compose.yaml", + "docker-compose.yml", + ) + ): + raise ValueError("multi-service Compose tasks are unsupported") + if ( + not (task_root / "instruction.md").is_file() + or not (task_root / "tests/test.sh").is_file() + ): + raise ValueError("a native instruction and verifier are required") + budget = coding_budget( + config, has_reference=(task_root / "solution/solve.sh").is_file() + ) + if ( + budget["sandbox_jobs_max"] > config["cloud"]["max_sandboxes"] + or budget["inference_calls_max"] > config["cloud"]["max_inference_calls"] + ): + raise ValueError( + "configured launch budget is too small for the requested audit" + ) + return config + + +def run_coding_validation(task_root, config, output_dir, *, backend=None): + """Generate, challenge, replay and measure a single prepared native task. + + Every result is bound to frozen task/verifier bytes and immutable requested + image references. Provider source revisions are recorded without pretending + they pin the provider's served weights. Outputs must be a new directory. + """ + config = validate_coding_task(task_root, config) + task_root, output_dir = Path(task_root).resolve(), Path(output_dir).resolve() + if output_dir.is_relative_to(task_root) and not any( + part in _DIGEST_EXCLUDED_DIRS + for part in output_dir.relative_to(task_root).parts + ): + raise ValueError("audit output must be outside the digested task tree") + secrets = [ + value + for name in ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN") + if (value := os.environ.get(name)) + ] + if any(secret in dumps(config) for secret in secrets): + raise ValueError("credentials may not be embedded in audit metadata") + original_digest = source_digest(task_root) + budget = coding_budget( + config, has_reference=(task_root / "solution/solve.sh").is_file() + ) + output_dir.mkdir(parents=True, exist_ok=False) + evidence = { + "task_digest": original_digest, + "verifier_digest": None, + "agent_image": config["execution"]["agent_image"], + "verifier_image": config["execution"]["verifier_image"], + "target": { + "model": config["target"]["model_id"], + "revision": config["target"]["revision"], + "provider": config["target"]["provider"], + }, + "references": [], + "trials": [], + "leakage_probes": [], + "generation": [], + "budget": budget, + "configuration_digest": hashlib.sha256(dumps(config).encode()).hexdigest(), + "requested_configuration_digest": hashlib.sha256( + dumps(config).encode() + ).hexdigest(), + "limitations": [ + "Supplemental coding audit; full RFC 008 qualification is not established.", + "Provider model source SHA does not attest served weights or deterministic sampling.", + "Image identities are control-plane immutable references, not runtime image attestations.", + "Sandbox RPC exposes its execution credential to the workload server; only reviewed workloads are supported.", + "Separate verifier Jobs do not prove submitted code cannot read or tamper with its grader.", + ], + } + + def sanitize(value): + if isinstance(value, str): + for secret in secrets: + value = value.replace(secret, "[REDACTED]") + return value + if isinstance(value, dict): + return {sanitize(key): sanitize(child) for key, child in value.items()} + if isinstance(value, list): + return [sanitize(child) for child in value] + return value + + def artifact(relative, data): + if isinstance(data, str): + data = data.encode() + digest = hashlib.sha256(data).hexdigest() + if any(secret.encode() in data for secret in secrets): + evidence["leakage_probes"].append( + {"name": "credential_in_artifact", "passed": False, "error": None} + ) + return {"sha256": digest, "omitted": "credential detected"} + path = output_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return {"path": relative, "sha256": digest} + + def persist(): + evidence["configuration_digest"] = hashlib.sha256( + dumps(config).encode() + ).hexdigest() + if any(secret in dumps(evidence) for secret in secrets) and not any( + probe.get("name") == "credential_in_metadata" + for probe in evidence["leakage_probes"] + ): + evidence["leakage_probes"].append( + {"name": "credential_in_metadata", "passed": False, "error": None} + ) + _write(output_dir / "evidence.json", sanitize(evidence)) + + try: + from .hf import _local_file + + def copy_artifact(source, destination): + raw, _ = _local_file(source, Path(source).stat().st_size) + Path(destination).write_bytes(raw) + shutil.copystat(source, destination) + + with tempfile.TemporaryDirectory(prefix="openenv-coding-frozen-") as temporary: + frozen = (Path(temporary) / "task").resolve() + shutil.copytree( + task_root, + frozen, + symlinks=True, + copy_function=copy_artifact, + ignore=shutil.ignore_patterns(*_DIGEST_EXCLUDED_DIRS), + ) + if source_digest(frozen) != original_digest: + raise ValueError("task changed during snapshot") + evidence["verifier_digest"] = source_digest(frozen / "tests") + if backend is None: + from .hf import HFTaskBackend + + credentials = _credentials() + # Provider preflight is read-only and precedes every paid sandbox. + from huggingface_hub import HfApi + + api = HfApi(token=credentials["HF_INFERENCE_TOKEN"]) + for model in (config["target"], config["generator"]): + info = api.model_info(model["model_id"], revision=model["revision"]) + if info.sha != model["revision"]: + raise ValueError("model source revision could not be verified") + current = api.model_info( + model["model_id"], expand=["inferenceProviderMapping"] + ) + model.setdefault( + "provider_model_id", + live_provider_id(current, model["provider"]), + ) + backend = HFTaskBackend( + frozen, config["execution"], token=credentials["HF_SANDBOX_TOKEN"] + ) + + def trial(role, case_id, script=None, seed=None): + if source_digest(frozen) != original_digest: + raise ValueError("frozen task source changed") + number = len(evidence["trials"]) + submission = ( + backend.solve(config["target"], seed, role="target") + if role == "target" + else backend.execute_script(script, role) + ) + row = { + "role": role, + "case_id": case_id, + "task_digest": original_digest, + "verifier_digest": evidence["verifier_digest"], + "image_ref": submission.get("image_ref"), + "submission_digest": submission.get("submission_digest"), + "sandbox_id": submission.get("sandbox_id"), + "error": submission["error"] + if "error" in submission + else "execution error status missing", + "cleanup_verified": submission.get("cleanup_verified") is True, + "latency_seconds": submission.get("latency_seconds"), + "usage": submission.get("usage"), + "seed": seed, + "model_metadata": submission.get("model_metadata"), + "truncated": submission.get("truncated", False), + "termination_reason": submission.get("termination_reason"), + } + row["transcript"] = artifact( + f"trials/{number:04d}/transcript.json", + dumps(submission.get("transcript", [])), + ) + for index, (name, data) in enumerate( + sorted(submission.get("files", {}).items()) + ): + item = artifact(f"trials/{number:04d}/submission/{index:04d}", data) + row.setdefault("submission_files", []).append( + {"destination": name, **item} + ) + if row["error"] is None and row["cleanup_verified"]: + grade = backend.grade(submission) + row["grader_transcript"] = artifact( + f"trials/{number:04d}/grader-transcript.json", + dumps(grade.get("transcript", [])), + ) + row.update( + { + "grade_sandbox_id": grade.get("grade_sandbox_id"), + "verifier_image_ref": grade.get("verifier_image_ref"), + "reward": grade.get("reward"), + "error": grade["error"] + if "error" in grade + else "grader error status missing", + "cleanup_verified": grade.get("cleanup_verified") is True, + } + ) + if grade.get("rawreward"): + row["raw_reward"] = artifact( + f"trials/{number:04d}/reward.raw", grade["rawreward"] + ) + try: + raw_value = loads(grade["rawreward"]) + if isinstance(raw_value, dict): + raw_value = raw_value[ + config["execution"].get("reward_key", "reward") + ] + if isinstance(raw_value, (bool, str)) or isinstance( + row["reward"], bool + ): + raise ValueError("reward is not numeric") + if Fraction(str(raw_value)) != Fraction(str(row["reward"])): + raise ValueError( + "raw reward and parsed reward disagree" + ) + except (ValueError, TypeError, KeyError, ZeroDivisionError): + row.update( + error=row["error"] + if row["error"] is not None + else "raw reward fidelity check failed", + reward=None, + ) + elif row["error"] is None: + row.update(error="raw reward evidence missing", reward=None) + if role == "target": + row.update( + { + "model": config["target"]["model_id"], + "revision": config["target"]["revision"], + "provider": config["target"]["provider"], + } + ) + evidence["trials"].append(row) + persist() + if not row["cleanup_verified"]: + raise ValueError("stop collection after unverified cleanup") + return row + + trial("nop", "nop", "true") + candidates = [] + provided = frozen / "solution/solve.sh" + if provided.is_file(): + candidates.append(("provided", "provided", provided.read_text())) + for index in range(config["generation"]["candidate_count"]): + generated = backend.generate_candidate(config["generator"], index) + script = generated.get("script", "") + entry = { + key: value for key, value in generated.items() if key != "script" + } + entry["artifact"] = artifact(f"candidates/generated-{index}.sh", script) + evidence["generation"].append(entry) + if generated.get("error") or not script.strip(): + evidence["generation_error"] = True + continue + candidates.append((f"generated-{index}", "generated", script)) + for case_id, origin, script in candidates: + records = [ + trial("reference", case_id, script) + for _ in range(config["validation"]["reference_repeats"]) + ] + evidence["references"].append( + { + "case_id": case_id, + "origin": origin, + "submission_digest": records[0].get("submission_digest"), + "candidate_script_digest": hashlib.sha256( + script.encode() + ).hexdigest(), + } + ) + for case in config["negative_cases"]: + trial("negative", case["id"], case["script"]) + evidence["leakage_probes"].extend(backend.leakage_probes()) + if evidence.get("generation_error"): + evidence["leakage_probes"].append( + { + "name": "generation_complete", + "passed": False, + "error": "candidate generation incomplete", + } + ) + persist() + screening = assess_coding_audit(config["validation"], evidence) + if all( + check["status"] == "pass" + for name, check in screening["checks"].items() + if name != "target_difficulty" + ): + for index, seed in enumerate(config["seeds"]): + trial("target", f"target-{index}", seed=seed) + # Include target transcripts in the final leakage measurement. + names = { + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", + } + evidence["leakage_probes"] = [ + probe + for probe in evidence["leakage_probes"] + if probe.get("name") not in names + ] + evidence["leakage_probes"].extend(backend.leakage_probes()) + else: + evidence["target_not_run_reason"] = ( + "reference/verifier/reliability/leakage screening did not pass" + ) + if ( + source_digest(task_root) != original_digest + or source_digest(frozen) != original_digest + ): + raise ValueError("task source changed during audit") + except (Exception, KeyboardInterrupt) as exc: + # Provider exceptions may include request URLs or credentials. + evidence["collection_error"] = type(exc).__name__ + evidence["leakage_probes"].append( + { + "name": "collection_complete", + "passed": False, + "error": type(exc).__name__, + } + ) + persist() + report = assess_coding_audit(config["validation"], evidence) + report.update( + { + "task_digest": original_digest, + "verifier_digest": evidence["verifier_digest"], + "target": evidence["target"], + "budget": budget, + "limitations": evidence["limitations"], + "configuration_digest": evidence["configuration_digest"], + } + ) + report["checks"]["target_difficulty"]["truncated_attempts"] = sum( + trial.get("truncated") is True + for trial in evidence["trials"] + if trial["role"] == "target" + ) + report = sanitize(report) + _write(output_dir / "config.json", sanitize(config)) + _write(output_dir / "report.json", report) + files = sorted(path for path in output_dir.rglob("*") if path.is_file()) + (output_dir / "SHA256SUMS").write_text( + "".join( + hashlib.sha256(path.read_bytes()).hexdigest() + + " " + + path.relative_to(output_dir).as_posix() + + "\n" + for path in files + ) + ) + return report diff --git a/examples/coding_audit/scoring.py b/examples/coding_audit/scoring.py new file mode 100644 index 000000000..ff04abeb4 --- /dev/null +++ b/examples/coding_audit/scoring.py @@ -0,0 +1,506 @@ +"""Pure scoring of bounded coding-audit evidence collected by a trusted runner. + +This profile measures reference replay, known-negative rejection, explicit leakage +probes, and target difficulty. It does not establish containment or trainability. +""" + +import math +import re +from collections import Counter, defaultdict +from decimal import Decimal +from fractions import Fraction +from statistics import pvariance +from typing import Any + +PROFILE = "harbor-coding-audit-v1" +_CHECKS = ( + "references", + "verifier_rejection", + "reliability", + "leakage", + "target_difficulty", +) +_PROBES = ( + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", +) +_RANK = {"pass": 0, "skip": 1, "fail": 2, "error": 3} +_SHA256 = re.compile(r"(?:sha256:)?[0-9a-f]{64}\Z") +_IMAGE = re.compile(r"(?:[^\s@]+@)?sha256:[0-9a-f]{64}\Z") +_REVISION = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z") + + +def _number(value: Any) -> bool: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + try: + return math.isfinite(value) + except OverflowError: + return False + + +def _text(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _matches(pattern: re.Pattern[str], value: Any) -> bool: + return isinstance(value, str) and pattern.fullmatch(value) is not None + + +def _sha(value: str) -> str: + return value.removeprefix("sha256:") + + +def _exact(value: int | float) -> Fraction: + return Fraction(str(value)) + + +def validate_audit_config(config: dict[str, Any]) -> dict[str, Any]: + """Return quality thresholds, raising `ValueError` for invalid configuration. + + `required_negative_cases` must come from a trusted reviewed configuration; + generated labels in evidence cannot supply this rejection coverage. + """ + defaults = { + "reference_repeats": 2, + "target_attempts": 2, + "reward_min": 0, + "reward_max": 1, + "oracle_tolerance": 0, + "floor_margin": 0.1, + "wrong_answer_max": 0, + "required_negative_cases": None, + } + if not isinstance(config, dict) or set(config) - defaults.keys(): + raise ValueError("config must contain only documented quality thresholds") + result = defaults | config + for name in ("reference_repeats", "target_attempts"): + if type(result[name]) is not int or result[name] < 2: + raise ValueError(f"{name} must be an integer >= 2") + for name in ( + "reward_min", + "reward_max", + "oracle_tolerance", + "floor_margin", + "wrong_answer_max", + ): + if not ( + _number(result[name]) + or isinstance(result[name], Decimal) + and result[name].is_finite() + ): + raise ValueError(f"{name} must be a finite nonbool number") + low, high = (_exact(result[name]) for name in ("reward_min", "reward_max")) + tolerance, margin = ( + _exact(result[name]) for name in ("oracle_tolerance", "floor_margin") + ) + if not ( + low < high + and 0 <= tolerance < high - low + and 0 < margin <= high - low + and low <= _exact(result["wrong_answer_max"]) < high - tolerance + and low + tolerance < high - tolerance + ): + raise ValueError("reward bounds, tolerance, margin or negative limit conflict") + cases = result["required_negative_cases"] + if ( + not isinstance(cases, list) + or not cases + or any(not _text(case) for case in cases) + or len(set(cases)) != len(cases) + ): + raise ValueError("required_negative_cases must be nonempty unique case IDs") + result["required_negative_cases"] = list(cases) + return result + + +def _issue(check: dict[str, Any], status: str, reason: str) -> None: + if _RANK[status] > _RANK[check["status"]]: + check["status"] = status + check["reasons"].append(reason) + + +def _finish(checks: dict[str, Any]) -> dict[str, Any]: + verdict = max((check["status"] for check in checks.values()), key=_RANK.get) + return { + "profile": PROFILE, + "accepted": verdict == "pass", + "verdict": verdict, + "checks": checks, + "rfc008_complete": False, + "trainability_established": False, + } + + +def _wilson(solved: int, count: int) -> list[float] | None: + if not count: + return None + z = 1.959963984540054 + rate = solved / count + denominator = 1 + z * z / count + center = (rate + z * z / (2 * count)) / denominator + half = z * math.sqrt(rate * (1 - rate) / count + z * z / (4 * count**2)) + return [ + max(0.0, center - half / denominator), + min(1.0, center + half / denominator), + ] + + +def _target_metrics( + raw: list[dict[str, Any]], trials: list[dict[str, Any]], config: dict[str, Any] +) -> dict[str, Any]: + rewards = [trial["reward"] for trial in trials] + count = len(rewards) + solved = sum( + _exact(reward) + >= _exact(config["reward_max"]) - _exact(config["oracle_tolerance"]) + for reward in rewards + ) + telemetry_missing = {"latency_seconds": 0, "input_tokens": 0, "output_tokens": 0} + telemetry_invalid = dict(telemetry_missing) + telemetry_errors = [] + latency: list[Fraction] = [] + usage: dict[str, list[int]] = {"input_tokens": [], "output_tokens": []} + # Failed executions can still consume tokens/time. Account for all records. + for trial in raw: + seconds = trial.get("latency_seconds") + if "latency_seconds" not in trial: + telemetry_missing["latency_seconds"] += 1 + elif _number(seconds) and seconds >= 0: + latency.append(_exact(seconds)) + else: + telemetry_invalid["latency_seconds"] += 1 + telemetry_errors.append("invalid target latency_seconds") + tokens = trial.get("usage") + if "usage" in trial and not isinstance(tokens, dict): + telemetry_errors.append("invalid target usage object") + for name in usage: + value = tokens.get(name) if isinstance(tokens, dict) else None + if "usage" not in trial or isinstance(tokens, dict) and name not in tokens: + telemetry_missing[name] += 1 + elif type(value) is int and value >= 0: + usage[name].append(value) + else: + telemetry_invalid[name] += 1 + telemetry_errors.append(f"invalid target {name}") + latency_total = None + if latency: + try: + latency_total = float(sum(latency)) + except OverflowError: + telemetry_errors.append("latency total exceeds finite report precision") + return { + "attempted": len(raw), + "completed": count, + "failed": len(raw) - count, + "infrastructure_errors": sum(trial.get("error") is not None for trial in raw), + "missing": max(0, config["target_attempts"] - count), + "solve_count": solved, + "solve_rate": solved / count if count else None, + "solve_rate_wilson95": _wilson(solved, count), + "reward_variance": _variance(rewards), + "mixed_success": 0 < solved < count if count else None, + "difficulty": ( + "unmeasured" + if not count + else "mixed" + if 0 < solved < count + else "all_solved" + if solved == count + else "none_solved" + ), + "latency_seconds_total": latency_total, + "usage_totals": { + name: sum(values) if values else None for name, values in usage.items() + }, + "telemetry_missing": telemetry_missing, + "telemetry_invalid": telemetry_invalid, + "telemetry_errors": telemetry_errors, + } + + +def _variance(rewards: list[float]) -> float | None: + if not rewards: + return None + try: + value = pvariance(rewards) + except OverflowError: + return None + return value if _number(value) else None + + +def assess_coding_audit( + config: dict[str, Any], evidence: dict[str, Any] +) -> dict[str, Any]: + """Score a trusted runner's evidence without executing tasks or making IO calls. + + Trials bind to the common task, verifier, images and pinned target identity. + Reference inventories bind replay groups by `(case_id, submission_digest)`. + Missing coverage is `skip`, contradictions are `fail`, and malformed or + infrastructure evidence is `error`; only five passing checks admit an audit. + """ + checks = {name: {"status": "pass", "reasons": []} for name in _CHECKS} + try: + config = validate_audit_config(config) + except ValueError as exc: + for check in checks.values(): + _issue(check, "error", str(exc)) + return _finish(checks) + if not isinstance(evidence, dict): + for check in checks.values(): + _issue(check, "error", "evidence must be an object") + return _finish(checks) + + reliability = checks["reliability"] + references = checks["references"] + rejection = checks["verifier_rejection"] + target = checks["target_difficulty"] + low, high, tolerance, margin, negative_max = ( + _exact(config[name]) + for name in ( + "reward_min", + "reward_max", + "oracle_tolerance", + "floor_margin", + "wrong_answer_max", + ) + ) + common_ok = True + for name, pattern in ( + ("task_digest", _SHA256), + ("verifier_digest", _SHA256), + ("agent_image", _IMAGE), + ("verifier_image", _IMAGE), + ): + if not _matches(pattern, evidence.get(name)): + _issue(reliability, "error", f"invalid or missing common {name}") + common_ok = False + identity = evidence.get("target") + if not ( + isinstance(identity, dict) + and all(_text(identity.get(name)) for name in ("model", "provider")) + and _matches(_REVISION, identity.get("revision")) + ): + _issue( + target, "error", "target requires model, provider and immutable revision" + ) + identity = None + + inventory = evidence.get("references", []) + candidates: dict[tuple[str, str], dict[str, Any]] = {} + if not isinstance(inventory, list): + _issue(references, "error", "references must be a list") + inventory = [] + for candidate in inventory: + if not ( + isinstance(candidate, dict) + and _text(candidate.get("case_id")) + and _matches(_SHA256, candidate.get("submission_digest")) + and candidate.get("origin") in ("provided", "generated") + and ( + candidate["origin"] != "generated" + or _matches(_SHA256, candidate.get("candidate_script_digest")) + ) + ): + _issue(references, "error", "invalid reference candidate metadata") + continue + key = (candidate["case_id"], _sha(candidate["submission_digest"])) + if key in candidates: + _issue(references, "error", f"duplicate reference candidate {key[0]}") + candidates[key] = candidate + if not candidates: + _issue(references, "skip", "no declared reference candidates") + + raw_trials = evidence.get("trials", []) + if not isinstance(raw_trials, list): + _issue(reliability, "error", "trials must be a list") + raw_trials = [] + sandbox_counts = Counter( + trial[name] + for trial in raw_trials + if isinstance(trial, dict) + for name in ("sandbox_id", "grade_sandbox_id") + if _text(trial.get(name)) + ) + target_cases = Counter( + trial["case_id"] + for trial in raw_trials + if isinstance(trial, dict) + and trial.get("role") == "target" + and _text(trial.get("case_id")) + ) + by_role: dict[str, list[dict[str, Any]]] = defaultdict(list) + raw_target = [] + for index, trial in enumerate(raw_trials): + if not isinstance(trial, dict) or trial.get("role") not in ( + "nop", + "reference", + "negative", + "target", + ): + _issue(reliability, "error", f"trial {index}: invalid role or record") + continue + role = trial["role"] + role_check = checks[ + { + "nop": "verifier_rejection", + "negative": "verifier_rejection", + "reference": "references", + "target": "target_difficulty", + }[role] + ] + if role == "target": + raw_target.append(trial) + errors = [] + if not common_ok: + errors.append("common binding unavailable") + for name, expected in ( + ("task_digest", evidence.get("task_digest")), + ("verifier_digest", evidence.get("verifier_digest")), + ("image_ref", evidence.get("agent_image")), + ("verifier_image_ref", evidence.get("verifier_image")), + ): + actual = trial.get(name) + if name in ("task_digest", "verifier_digest"): + matches = ( + _matches(_SHA256, actual) + and _matches(_SHA256, expected) + and _sha(actual) == _sha(expected) + ) + else: + matches = actual == expected + if not matches: + errors.append(f"{name} binding mismatch") + submission_ok = _text(trial.get("case_id")) and _matches( + _SHA256, trial.get("submission_digest") + ) + if not submission_ok: + errors.append("missing case ID or submission digest") + for name in ("sandbox_id", "grade_sandbox_id"): + value = trial.get(name) + if not _text(value) or sandbox_counts[value] != 1: + errors.append(f"{name} must be fresh and distinct") + if trial.get("cleanup_verified") is not True: + errors.append("cleanup not verified") + if "error" not in trial or trial["error"] is not None: + errors.append("execution error or missing error status") + reward = trial.get("reward") + if not _number(reward) or not low <= _exact(reward) <= high: + errors.append("reward must be finite nonbool and in bounds") + if role == "target": + if identity is None or any( + trial.get(name) != identity[name] + for name in ("model", "revision", "provider") + ): + errors.append("target identity mismatch") + if _text(trial.get("case_id")) and target_cases[trial["case_id"]] != 1: + errors.append("duplicate target attempt case ID") + if ( + role == "reference" + and submission_ok + and (trial["case_id"], _sha(trial["submission_digest"])) not in candidates + ): + errors.append("undeclared reference submission") + if errors: + message = f"trial {index}: " + "; ".join(errors) + _issue(role_check, "error", message) + _issue(reliability, "error", message) + else: + by_role[role].append(trial) + if not raw_trials: + _issue(reliability, "skip", "no trial evidence") + + floor_rewards = [_exact(trial["reward"]) for trial in by_role["nop"]] + if not floor_rewards: + _issue(rejection, "skip", "NOP floor evidence missing") + _issue(references, "skip", "NOP floor evidence missing") + elif max(floor_rewards) > low + tolerance: + _issue(rejection, "fail", "NOP reward exceeds configured floor") + + groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for trial in by_role["reference"]: + groups[(trial["case_id"], _sha(trial["submission_digest"]))].append(trial) + validated = [] + excluded = [] + for key, candidate in candidates.items(): + trials = groups[key] + rewards = [_exact(trial["reward"]) for trial in trials] + if not rewards: + _issue(references, "skip", f"candidate {key[0]} execution missing") + continue + stable = max(rewards) - min(rewards) <= tolerance + if not stable: + _issue(reliability, "fail", f"reference {key[0]} replay reward drift") + correct = min(rewards) >= high - tolerance + if not correct: + if candidate["origin"] == "generated": + excluded.append({"case_id": key[0], "submission_digest": key[1]}) + else: + _issue( + references, "fail", f"provided reference {key[0]} reward too low" + ) + continue + if len(trials) < config["reference_repeats"]: + _issue(references, "skip", f"reference {key[0]} replay coverage incomplete") + _issue( + reliability, "skip", f"reference {key[0]} replay coverage incomplete" + ) + continue + if floor_rewards and min(rewards) - max(floor_rewards) < margin: + _issue(references, "fail", f"reference {key[0]} lacks floor separation") + continue + if stable and floor_rewards: + validated.append({"case_id": key[0], "submission_digest": key[1]}) + references.update(validated=validated, excluded=excluded) + if not validated: + _issue(references, "skip", "no validated reference") + _issue(reliability, "skip", "no validated reference replay group") + + negatives = defaultdict(list) + for trial in by_role["negative"]: + negatives[trial["case_id"]].append(_exact(trial["reward"])) + covered = [] + for case in config["required_negative_cases"]: + if not negatives[case]: + _issue(rejection, "skip", f"trusted negative {case} execution missing") + elif max(negatives[case]) > negative_max: + _issue(rejection, "fail", f"trusted negative {case} was rewarded") + else: + covered.append(case) + rejection["covered_negative_cases"] = covered + + leakage = checks["leakage"] + leakage["scope"] = "bounded probes; universal containment is not established" + probes = evidence.get("leakage_probes", []) + if not isinstance(probes, list): + _issue(leakage, "error", "leakage_probes must be a list") + probes = [] + seen = set() + for probe in probes: + if not isinstance(probe, dict) or not _text(probe.get("name")): + _issue(leakage, "error", "invalid leakage probe record") + continue + name = probe["name"] + if name in seen: + _issue(leakage, "error", f"duplicate leakage probe {name}") + seen.add(name) + if type(probe.get("passed")) is not bool or "error" not in probe: + _issue(leakage, "error", f"invalid leakage probe result {name}") + elif probe["error"] is not None: + _issue(leakage, "error", f"leakage probe {name} execution error") + elif not probe["passed"]: + _issue(leakage, "fail", f"leakage probe {name} failed") + for name in _PROBES: + if name not in seen: + _issue(leakage, "skip", f"required leakage probe {name} missing") + leakage["observed_probes"] = sorted(seen) + + target.update(_target_metrics(raw_target, by_role["target"], config)) + if target["completed"] and target["reward_variance"] is None: + _issue(target, "error", "reward variance exceeds finite report precision") + for message in target["telemetry_errors"]: + _issue(target, "error", message) + if len(by_role["target"]) < config["target_attempts"]: + _issue(target, "skip", "target attempt coverage incomplete") + return _finish(checks) diff --git a/examples/coding_audit/test_hf.py b/examples/coding_audit/test_hf.py new file mode 100644 index 000000000..2acf9c0db --- /dev/null +++ b/examples/coding_audit/test_hf.py @@ -0,0 +1,1732 @@ +"""Offline SDK-shaped tests for the HF coding task execution boundary.""" + +import hashlib +import json +import os +import signal +import time +from pathlib import PurePosixPath +from types import SimpleNamespace + +import httpx +import pytest +from examples.coding_audit.hf import ( + _deadline, + HFBackendError, + HFTaskBackend, + submission_digest, +) +from huggingface_hub import hf_api, InferenceClient, Sandbox +from huggingface_hub.errors import SandboxError +from huggingface_hub.inference._generated.types.chat_completion import ( + ChatCompletionOutput, + ChatCompletionStreamOutput, +) +from huggingface_hub.inference._providers.deepinfra import DeepInfraConversationalTask +from openenv.validation.runner import source_digest + +AGENT = "python@sha256:" + "a" * 64 +VERIFIER = "python@sha256:" + "b" * 64 +MODEL = { + "model_id": "Qwen/Qwen3.8-27B", + "revision": "37561816e66b5b9153b1b64c10968c18c1e6f4a0", + "provider": "deepinfra", + "provider_model_id": "Qwen/Qwen3.8-27B", +} + + +def test_real_sandbox_sdk_serializes_short_name_for_long_image(harness, monkeypatch): + captured = {} + image = ( + "registry.hf.space/example-organization-a-long-starter-image-name" + "@sha256:" + "a" * 64 + ) + + def capture(request): + assert request.method == "POST" and request.url.host == "huggingface.co" + captured.update(json.loads(request.content)) + return httpx.Response(400, json={"error": "offline request capture"}) + + harness.backend.execution["agent_image"] = image + harness.backend.sandbox_factory = Sandbox.create + with httpx.Client(transport=httpx.MockTransport(capture)) as client: + monkeypatch.setattr(hf_api, "get_session", lambda: client) + record = harness.backend.execute_script(":", "nop") + assert "BadRequestError" in record["error"] + assert captured["dockerImage"] == image + assert captured["labels"]["name"] == ( + "openenv-coding-" + captured["labels"]["openenv-coding-attempt"] + ) + assert 0 < len(captured["labels"]["name"]) <= 100 + + +def response( + content="done", *, calls=None, model=None, fingerprint="fp-123", usage=None +): + return ChatCompletionOutput.parse_obj_as_instance( + { + "id": "chatcmpl-1", + "created": 1, + "object": "chat.completion", + "model": model or MODEL["model_id"], + "system_fingerprint": fingerprint, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls" if calls else "stop", + "message": { + "role": "assistant", + "content": content, + "tool_calls": calls, + }, + } + ], + "usage": usage + or {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + ) + + +def terminal(command="printf answer > /app/answer.txt", identifier="call-1"): + return { + "id": identifier, + "type": "function", + "function": {"name": "terminal", "arguments": json.dumps({"command": command})}, + } + + +def stream_event(*, delta=None, finish=None, choices=None, usage=None, **identity): + return ChatCompletionStreamOutput.parse_obj_as_instance( + { + "id": "chatcmpl-1", + "created": 1, + "model": MODEL["model_id"], + "system_fingerprint": "fp-123", + **identity, + "choices": choices + if choices is not None + else [{"index": 0, "delta": delta or {}, "finish_reason": finish}], + "usage": usage, + } + ) + + +def stream_response(reply): + yield stream_event( + delta=reply["choices"][0]["message"], + finish=reply["choices"][0]["finish_reason"], + model=reply["model"], + system_fingerprint=reply["system_fingerprint"], + ) + yield stream_event( + choices=[], + usage=reply["usage"], + model=reply["model"], + system_fingerprint=reply["system_fingerprint"], + ) + + +class Files: + MAX_READ_BYTES = 512 * 1048576 + + def __init__(self, initial=None): + self.data = dict(initial or {}) + self.types = {} + self.directories = {"/", "/tmp"} + self.writes = [] + self.reads = [] + self.grow_on_read = False + + def stat(self, path): + if path in self.directories: + return SimpleNamespace(type="dir", size=0) + if path not in self.data: + raise SandboxError("missing", status_code=404) + return SimpleNamespace( + type=self.types.get(path, "file"), size=len(self.data[path]) + ) + + def exists(self, path): + try: + self.stat(path) + return True + except SandboxError as exc: + if exc.status_code == 404: + return False + raise + + def read(self, path): + self.reads.append(path) + data = self.data[path] + if self.grow_on_read: + data = data + b"x" * self.MAX_READ_BYTES + if len(data) > self.MAX_READ_BYTES: + raise SandboxError("too large") + return data + + def write(self, path, data, mode=None): + assert isinstance(data, (str, bytes)) + data = data.encode() if isinstance(data, str) else data + self.data[path] = data + self.writes.append((path, data, mode)) + + def mkdir(self, path): + self.directories.add(path) + + def delete(self, path): + del self.data[path] + + +class FakeSandbox: + def __init__(self, identity, image, api, factory): + self.id = identity + self.image = image + self.files = Files(factory.initial) + self.api = api + self.factory = factory + self.commands = [] + self.killed = False + + def run(self, argv, **kwargs): + self.commands.append((argv, kwargs)) + assert isinstance(argv, list) + assert kwargs["shell"] is False + assert kwargs["check"] is False + assert kwargs["capture_output"] is False + assert kwargs["cwd"] in self.files.directories + assert 0 < kwargs["timeout"] <= 120 + assert "env" not in kwargs + if self.factory.output: + kwargs["on_stdout"](self.factory.output) + if self.image == VERIFIER: + self.files.data["/logs/verifier/reward.txt"] = self.factory.reward + elif self.factory.submitted is not None: + self.files.data["/app/answer.txt"] = self.factory.submitted + return SimpleNamespace( + exit_code=self.factory.exit_code, + stdout="", + stderr="", + timed_out=self.factory.timed_out, + ) + + def kill(self): + self.killed = True + if not self.api.ignore_cancel: + self.api.stages[self.id] = "CANCELED" + + +class API: + def __init__(self): + self.images = {} + self.stages = {} + self.cancelled = [] + self.inspections = [] + self.ignore_cancel = False + self.cancel_error = False + self.inspect_error_after_cancel = False + self.owner = "owner" + self.image_override = None + self.labels = {} + + def inspect_job(self, *, job_id, namespace=None, token=None): + self.inspections.append((job_id, namespace, token)) + if self.inspect_error_after_cancel and job_id in self.cancelled: + raise RuntimeError("inspection contains an execution-token-secret") + return SimpleNamespace( + id=job_id, + docker_image=self.image_override or self.images[job_id], + status=SimpleNamespace(stage=self.stages[job_id]), + owner=SimpleNamespace(name=self.owner), + labels=self.labels.get(job_id, {}), + ) + + def list_jobs(self, *, labels=None, timeout=None, namespace=None, token=None): + return ( + self.inspect_job(job_id=job_id, namespace=namespace, token=token) + for job_id in self.images + if all( + self.labels.get(job_id, {}).get(key) == value + for key, value in (labels or {}).items() + ) + ) + + def cancel_job(self, *, job_id, namespace=None, token=None): + self.cancelled.append(job_id) + if self.cancel_error: + raise RuntimeError("cancel failed") + if not self.ignore_cancel: + self.stages[job_id] = "CANCELED" + + +class Factory: + def __init__(self, api): + self.api = api + self.created = [] + self.kwargs = [] + self.initial = {} + self.output = "" + self.submitted = b"answer" + self.reward = b"1.0\n" + self.exit_code = 0 + self.timed_out = False + self.fail_after_create = False + + def __call__(self, **kwargs): + identity = f"job-{len(self.created)}" + self.kwargs.append(kwargs) + self.api.images[identity] = kwargs["image"] + self.api.stages[identity] = "RUNNING" + self.api.labels[identity] = kwargs.get("labels", {}) + sandbox = FakeSandbox(identity, kwargs["image"], self.api, self) + self.created.append(sandbox) + if self.fail_after_create: + raise RuntimeError("readiness failed after allocation") + return sandbox + + +class InferenceFactory: + def __init__(self): + self.kwargs = [] + self.requests = [] + self.responses = [] + self.closed = 0 + + def __call__(self, **kwargs): + self.kwargs.append(kwargs) + return self + + def chat_completion(self, **kwargs): + self.requests.append(json.loads(json.dumps(kwargs))) + reply = self.responses.pop(0) + if kwargs.get("stream") and isinstance(reply, ChatCompletionOutput): + return stream_response(reply) + return reply + + def close(self): + self.closed += 1 + + +@pytest.fixture +def harness(tmp_path, monkeypatch): + for name in ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN"): + monkeypatch.delenv(name, raising=False) + root = tmp_path.resolve() + (root / "instruction.md").write_text("Write /app/answer.txt containing answer.") + (root / "tests").mkdir() + (root / "tests/test.sh").write_bytes( + b"#!/bin/bash\nprintf 1 > /logs/verifier/reward.txt\n" + ) + (root / "solution").mkdir() + (root / "solution/solve.sh").write_text("PRIVATE_SOLUTION_SENTINEL") + api = API() + factory = Factory(api) + inference = InferenceFactory() + execution = { + "agent_image": AGENT, + "verifier_image": VERIFIER, + "submission_paths": ["/app/answer.txt"], + "namespace": "owner", + } + backend = HFTaskBackend( + root, + execution, + token="execution-token-secret", + inference_token="inference-token-secret", + api=api, + sandbox_factory=factory, + inference_factory=inference, + canary="PRIVATE_CANARY_SENTINEL", + ) + return SimpleNamespace( + root=root, + backend=backend, + api=api, + factory=factory, + inference=inference, + execution=execution, + ) + + +def test_execution_grade_are_fresh_and_credentials_stay_in_controller(harness): + h = harness + submission = h.backend.execute_script( + "printf answer > /app/answer.txt", "reference" + ) + grade = h.backend.grade(submission) + assert submission["error"] is None and submission["cleanup_verified"] + assert grade["error"] is None and grade["cleanup_verified"] + assert grade["reward"] == 1 and grade["rawreward"] == b"1.0\n" + assert submission["sandbox_id"] != grade["grade_sandbox_id"] + assert [item["image"] for item in h.factory.kwargs] == [AGENT, VERIFIER] + for kwargs in h.factory.kwargs: + assert kwargs["forward_hf_token"] is False + assert kwargs["token"] == "execution-token-secret" + assert ( + "env" not in kwargs and "secrets" not in kwargs and "volumes" not in kwargs + ) + solver, verifier = h.factory.created + assert not any( + path.startswith(("/tests", "/solution")) for path in solver.files.data + ) + assert ( + verifier.files.data["/tests/test.sh"] == (h.root / "tests/test.sh").read_bytes() + ) + assert verifier.files.data["/app/answer.txt"] == b"answer" + assert not any("solution" in path for path in verifier.files.data) + assert b"PRIVATE_CANARY_SENTINEL" in verifier.files.data.values() + assert submission["image_reference_kind"] == "control_plane_immutable_reference" + assert len(submission["limitations"]) >= 4 + assert "token-secret" not in repr(submission) + repr(grade) + assert h.api.cancelled == ["job-0", "job-1"] + + +def test_absent_submission_is_empty_and_can_legitimately_grade_zero(harness): + h = harness + h.factory.submitted = None + submission = h.backend.execute_script(":", "nop") + assert submission["files"] == {} and submission["error"] is None + h.factory.reward = b"0" + grade = h.backend.grade(submission) + assert grade["error"] is None and grade["reward"] == 0 + + +def test_native_verifier_reward_directory_exists_before_execution(harness, monkeypatch): + h = harness + submission = h.backend.execute_script(":", "nop") + original_run = FakeSandbox.run + + def native_run(sandbox, argv, **kwargs): + if sandbox.image == VERIFIER: + assert "/logs/verifier" in sandbox.files.directories + return original_run(sandbox, argv, **kwargs) + + monkeypatch.setattr(FakeSandbox, "run", native_run) + grade = h.backend.grade(submission) + assert grade["error"] is None + assert grade["cleanup_verified"] + + +@pytest.mark.parametrize("phase", ["script", "grade"]) +def test_nonzero_is_error_not_synthesized_reward(harness, phase): + h = harness + submission = h.backend.execute_script(":", "reference") + h.factory.exit_code = 3 + record = ( + h.backend.grade(submission) + if phase == "grade" + else h.backend.execute_script("false", "target") + ) + assert "nonzero" in record["error"] + assert record.get("reward") is None and record["cleanup_verified"] + + +@pytest.mark.parametrize("failure", ["overflow", "timeout", "nonzero", "exception"]) +def test_failed_grader_keeps_logs_only_without_overflow(harness, monkeypatch, failure): + h = harness + submission = h.backend.execute_script(":", "reference") + h.backend.max_output = 17 + output = "setup\n雪💥雪💥" if failure == "overflow" else "setup\nfailed\n" + + def failed_run(sandbox, argv, **kwargs): + kwargs["on_stdout"]("setup\n") + kwargs["on_stderr"](output.removeprefix("setup\n")) + if failure == "exception": + raise RuntimeError("RPC failed after output") + return SimpleNamespace( + exit_code=3 if failure == "nonzero" else 0, + timed_out=failure == "timeout", + ) + + monkeypatch.setattr(FakeSandbox, "run", failed_run) + grade = h.backend.grade(submission) + assert grade["error"] and grade["cleanup_verified"] + assert grade["reward"] is None and grade["rawreward"] == b"" + trace = grade["transcript"][0] + assert trace["output"] == ("" if failure == "overflow" else output) + assert len(trace["output"].encode()) <= 17 + assert trace["output_truncated"] is (failure == "overflow") + assert trace["exit_code"] == ( + 3 if failure == "nonzero" else 0 if failure == "timeout" else None + ) + + +@pytest.mark.parametrize("cap", [17, 18]) +def test_grader_output_stays_bounded_after_redaction(harness, cap): + h = harness + submission = h.backend.execute_script(":", "nop") + backend = HFTaskBackend( + h.root, + dict(h.execution, max_tool_output_bytes=cap), + token="secret", + api=h.api, + sandbox_factory=h.factory, + ) + h.factory.output = "雪secret💥\n" + h.factory.reward = b"0" + grade = backend.grade(submission) + assert grade["error"] is None and grade["reward"] == 0 + assert grade["transcript"] == [ + { + "exit_code": 0, + "output": "" if cap == 17 else "雪[REDACTED]💥\n", + "output_truncated": cap == 17, + } + ] + assert len(grade["transcript"][0]["output"].encode()) <= cap + assert grade["cleanup_verified"] + + +@pytest.mark.parametrize("callbacks", ["single", "separate", "split_secret"]) +@pytest.mark.parametrize( + "credential", ["execution", "inference", "overlapping", "repeated", "intersecting"] +) +@pytest.mark.parametrize("retained", [1, 28]) +def test_grader_overflow_discards_output( + harness, monkeypatch, callbacks, credential, retained +): + h = harness + submission = h.backend.execute_script(":", "reference") + credentials = { + "execution": "hf_SYNTHETIC_EXECUTION_TOKEN_FOR_REVIEW_ONLY", + "inference": "hf_SYNTHETIC_INFERENCE_TOKEN_FOR_REVIEW_ONLY", + } + if credential == "overlapping": + credentials["execution"] = "SYNTHETIC_EXECUTION_TOKEN" + credentials["inference"] = "hf_SYNTHETIC_EXECUTION_TOKEN_FOR_REVIEW_ONLY" + elif credential == "repeated": + credentials["execution"] = "hf_" + "X" * 22 + "hf_" + credential = "execution" + elif credential == "intersecting": + credentials["execution"] = "hf_SYNTHETIC_EXECUTION_TOKEN_REVIEW_ONLYh" + credentials["inference"] = "hf_SYNTHETIC_INFERENCE_TOKEN_REVIEW_ONLY" + cap = 1048576 + backend = HFTaskBackend( + h.root, + dict(h.execution, max_tool_output_bytes=cap), + token=credentials["execution"], + inference_token=credentials["inference"], + api=h.api, + sandbox_factory=h.factory, + ) + secret = credentials[ + "inference" if credential in ("overlapping", "intersecting") else credential + ] + prefix = "雪" + "A" * (cap - len("雪".encode()) - retained) + if credential == "intersecting": + complete = credentials["execution"] + prefix = prefix[: -len(complete[:-1])] + complete[:-1] + assert complete in prefix + secret + + def overflow(sandbox, argv, **kwargs): + if callbacks == "single": + kwargs["on_stdout"](prefix + secret + "\n") + else: + split = min(7, retained) if callbacks == "split_secret" else 0 + kwargs["on_stdout"](prefix + secret[:split]) + kwargs["on_stderr"](secret[split:] + "\n") + pytest.fail("grader output overflow was not rejected") + + monkeypatch.setattr(FakeSandbox, "run", overflow) + grade = backend.grade(submission) + assert grade["error"] == "terminal output exceeded byte limit" + assert grade["reward"] is None and grade["rawreward"] == b"" + assert grade["cleanup_verified"] + trace = grade["transcript"][0] + assert trace == { + "exit_code": None, + "output": "", + "output_truncated": True, + } + assert len(trace["output"].encode()) <= cap + assert secret[:retained] not in trace["output"] + assert h.api.stages and all(stage == "CANCELED" for stage in h.api.stages.values()) + + +def test_image_mismatch_cancels_before_any_workload_command(harness): + h = harness + h.api.image_override = VERIFIER + record = h.backend.execute_script("dangerous", "reference") + assert "image" in record["error"] and record["cleanup_verified"] + assert not h.factory.created[0].commands + assert h.api.cancelled == ["job-0"] + + +def test_rpc_ready_sandbox_waits_for_running_control_plane(harness, monkeypatch): + h = harness + inspect = h.api.inspect_job + observed = [] + + def lagged_inspect(**kwargs): + job = inspect(**kwargs) + if not h.api.cancelled: + job.status.stage = "SCHEDULING" if not observed else "RUNNING" + observed.append(job.status.stage) + return job + + monkeypatch.setattr(h.api, "inspect_job", lagged_inspect) + monkeypatch.setattr("examples.coding_audit.hf.time.sleep", lambda _: None) + record = h.backend.execute_script(":", "reference") + assert record["error"] is None + assert observed == ["SCHEDULING", "RUNNING"] + assert record["cleanup_verified"] + assert h.factory.created[0].commands + + +@pytest.mark.parametrize("stage", ["SCHEDULING", "ERROR", "COMPLETED", "UNKNOWN"]) +def test_nonrunning_control_plane_never_executes_and_is_cleaned_up( + harness, monkeypatch, stage +): + h = harness + inspect = h.api.inspect_job + attempts = [] + + def never_running(**kwargs): + job = inspect(**kwargs) + if not h.api.cancelled: + attempts.append(job.id) + job.status.stage = stage + return job + + monkeypatch.setattr(h.api, "inspect_job", never_running) + monkeypatch.setattr("examples.coding_audit.hf.time.sleep", lambda _: None) + record = h.backend.execute_script("dangerous", "reference") + assert record["error"] is not None and record["cleanup_verified"] + assert not h.factory.created[0].commands + assert len(attempts) <= 20 + + +def test_startup_reconciliation_rechecks_image_before_execution(harness, monkeypatch): + h = harness + inspect = h.api.inspect_job + attempts = [] + + def changed_image(**kwargs): + job = inspect(**kwargs) + if not h.api.cancelled: + if attempts: + job.docker_image = VERIFIER + else: + job.status.stage = "SCHEDULING" + attempts.append(job.id) + return job + + monkeypatch.setattr(h.api, "inspect_job", changed_image) + monkeypatch.setattr("examples.coding_audit.hf.time.sleep", lambda _: None) + record = h.backend.execute_script("dangerous", "reference") + assert "image" in record["error"] and record["cleanup_verified"] + assert not h.factory.created[0].commands + assert len(attempts) == 2 + + +@pytest.mark.parametrize("failure", ["nonterminal", "inspection"]) +def test_cleanup_cannot_pass_from_cancel_return_alone(harness, failure): + h = harness + submission = h.backend.execute_script(":", "reference") + h.api.ignore_cancel = failure == "nonterminal" + h.api.inspect_error_after_cancel = failure == "inspection" + grade = h.backend.grade(submission) + assert grade["cleanup_verified"] is False + assert grade["reward"] is None and "cleanup" in grade["error"] + assert h.api.cancelled.count("job-1") == 3 + + +def test_exact_kill_fallback_requires_independent_terminal_inspection(harness): + h = harness + h.api.cancel_error = True + record = h.backend.execute_script(":", "reference") + assert record["cleanup_verified"] and h.factory.created[0].killed + assert h.api.inspections[-1][0] == record["sandbox_id"] + + +@pytest.mark.parametrize( + "path", + [ + "relative", + "/app/../tests/x", + "/app//x", + "/app/./x", + "/tests/x", + "/solution/x", + "/logs/verifier/reward.txt", + "/", + "//app/x", + ], +) +def test_submission_path_preflight(harness, path): + h = harness + with pytest.raises(HFBackendError): + HFTaskBackend(h.root, {**h.execution, "submission_paths": [path]}) + + +@pytest.mark.parametrize("paths", [["/app/a", "/app/a/b"], ["/app/a", "/app/a"]]) +def test_overlapping_paths_rejected(harness, paths): + with pytest.raises(HFBackendError, match="overlap|repeat"): + HFTaskBackend(harness.root, {**harness.execution, "submission_paths": paths}) + + +@pytest.mark.parametrize( + "mutation", ["extra_path", "tampered_digest", "not_bytes", "oversize"] +) +def test_grade_rejects_unsafe_transfer_before_launch(harness, mutation): + h = harness + files = {"/app/answer.txt": b"answer"} + submission = {"files": files, "submission_digest": submission_digest(files)} + if mutation == "extra_path": + files["/tests/test.sh"] = b"fake reward" + elif mutation == "tampered_digest": + files["/app/answer.txt"] = b"changed" + elif mutation == "not_bytes": + files["/app/answer.txt"] = "answer" + else: + files["/app/answer.txt"] = b"x" * (h.backend.max_bytes + 1) + grade = h.backend.grade(submission) + assert grade["error"] and grade["reward"] is None + assert not h.factory.created + + +@pytest.mark.parametrize("kind", ["symlink", "hardlink", "fifo", "symlink_parent"]) +def test_local_verifier_artifacts_reject_nonregular_or_linked_paths(harness, kind): + h = harness + path = h.root / "tests/extra" + if kind == "symlink": + path.symlink_to(h.root / "instruction.md") + elif kind == "hardlink": + os.link(h.root / "instruction.md", path) + elif kind == "fifo": + os.mkfifo(path) + else: + path.symlink_to(h.root / "solution", target_is_directory=True) + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert grade["error"] and not h.factory.created + + +@pytest.mark.parametrize( + "reward", + [ + b"NaN", + b"Infinity", + b"1e9999", + b'{"reward": true}', + b'{"reward": "1"}', + b'{"reward":1,"reward":0}', + b"garbage", + ], +) +def test_reward_requires_finite_trusted_numeric_bytes(harness, reward): + h = harness + h.factory.reward = reward + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert grade["error"] and grade["reward"] is None and grade["cleanup_verified"] + + +def test_json_reward_collects_original_bytes(harness): + h = harness + h.factory.reward = b'{"reward":0.5,"detail":"ok"}\n' + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert grade["reward"] == 0.5 and grade["rawreward"] == h.factory.reward + + +def test_provider_requests_base_id_with_distinct_token_and_records_source_limit( + harness, +): + h = harness + h.inference.responses = [response("printf answer > /app/answer.txt")] + generated = h.backend.generate_candidate(MODEL, 42) + assert generated["error"] is None + assert not h.factory.created + request = h.inference.requests[0] + assert request["model"] == MODEL["model_id"] and "@" not in request["model"] + assert "seed" not in request and "tools" not in request + assert h.inference.kwargs == [ + {"provider": "deepinfra", "api_key": "inference-token-secret", "timeout": 120} + ] + assert "PRIVATE_SOLUTION_SENTINEL" not in repr(request) + assert "PRIVATE_CANARY_SENTINEL" not in repr(request) + assert "test.sh" not in repr(request) + metadata = generated["model_metadata"] + assert metadata["source_revision"] == MODEL["revision"] + assert metadata["served_weight_revision_pinned"] is False + assert metadata["seed_supported"] is False + assert metadata["responses"][0]["system_fingerprint"] == "fp-123" + assert generated["usage"] == {"input_tokens": 10, "output_tokens": 5} + + +def test_seed_and_exact_provider_alias_are_explicit(harness): + h = harness + h.inference.responses = [response("true", model="provider/exact")] + record = h.backend.generate_candidate( + {**MODEL, "supports_seed": True, "provider_model_id": "provider/exact"}, 7 + ) + assert record["error"] is None + assert h.inference.requests[0]["seed"] == 7 + h.inference.responses = [response("true", model="provider/unlisted")] + record = h.backend.generate_candidate(MODEL, 7) + assert "response model" in record["error"] + assert record["model_metadata"]["responses"][0]["model"] == "provider/unlisted" + + +def test_top_p_is_forwarded_and_recorded(harness): + h = harness + h.inference.responses = [response("true")] + record = h.backend.generate_candidate({**MODEL, "top_p": 0.1}, 7) + assert record["error"] is None + assert h.inference.requests[0]["top_p"] == 0.1 + assert record["model_metadata"]["requests"][0]["top_p"] == 0.1 + + +def test_candidate_stream_assembles_text_and_final_usage_without_changing_recipe( + harness, +): + h = harness + events = [ + stream_event(delta={"role": "assistant", "content": "printf "}), + stream_event(delta={"reasoning_content": "Think 雪"}), + stream_event(delta={"content": "answer > /app/answer.txt"}, finish="stop"), + stream_event( + choices=[], + usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + ), + ] + h.inference.responses = [(event for event in events)] + record = h.backend.generate_candidate( + {**MODEL, "max_tokens": 8192, "max_total_tokens": 32768, "top_p": 0.1}, 7 + ) + assert record["error"] is None + assert record["script"] == "printf answer > /app/answer.txt" + assert record["transcript"][-1]["reasoning_content"] == "Think 雪" + assert record["usage"] == {"input_tokens": 10, "output_tokens": 5} + request = h.inference.requests[0] + assert request["stream"] is True + assert request["stream_options"] == {"include_usage": True} + assert request["max_tokens"] == 8192 and request["top_p"] == 0.1 + assert request["model"] == MODEL["model_id"] and request["temperature"] == 0 + assert "tools" not in request and "seed" not in request + assert record["model_metadata"]["requests"] == [ + {key: value for key, value in request.items() if key != "messages"} + ] + assert h.inference.closed == 1 and not h.factory.created + + +@pytest.mark.parametrize( + "outcome", ["complete", "early_close", "provider_error", "bad_json"] +) +def test_real_sdk_stream_serializes_request_and_closes_http_response( + harness, monkeypatch, outcome +): + h = harness + captured = {} + closed = [] + events = list(stream_response(response("printf answer > /app/answer.txt"))) + + class Body(httpx.SyncByteStream): + def __iter__(self): + for event in events[:1] if outcome == "early_close" else events: + yield ("data: " + json.dumps(event) + "\n\n").encode() + if outcome == "provider_error": + yield b'data: {"error": "provider failed"}\n\n' + elif outcome == "bad_json": + yield b"data: {broken\n\n" + elif outcome == "complete": + yield b"data: [DONE]\n\n" + + def close(self): + closed.append(True) + + def capture(request): + assert request.url.host == "api.deepinfra.com" + captured.update(json.loads(request.content)) + return httpx.Response(200, stream=Body()) + + monkeypatch.setattr( + DeepInfraConversationalTask, + "_prepare_mapping_info", + lambda self, model: hf_api.InferenceProviderMapping( + provider="deepinfra", + hf_model_id=model, + providerId=model, + task="conversational", + status="live", + ), + ) + h.backend.inference_factory = InferenceClient + with httpx.Client(transport=httpx.MockTransport(capture)) as client: + monkeypatch.setattr( + "huggingface_hub.inference._client.get_session", lambda: client + ) + record = h.backend.generate_candidate({**MODEL, "enable_thinking": False}, 0) + assert closed == [True] + assert captured["chat_template_kwargs"] == {"enable_thinking": False} + assert captured["stream"] is True + assert captured["stream_options"] == {"include_usage": True} + assert captured["model"] == MODEL["model_id"] + if outcome == "complete": + assert record["error"] is None and record["script"].startswith("printf answer") + assert record["usage"] == {"input_tokens": 10, "output_tokens": 5} + else: + assert record["error"] and record["script"] == "" + + +@pytest.mark.parametrize( + "mutation,diagnostic", + [ + ("model_drift", "response model"), + ("allowed_model_drift", "identity"), + ("id_drift", "identity"), + ("fingerprint_drift", "identity"), + ("two_choices", "one assistant"), + ("wrong_index", "one assistant"), + ("boolean_index", "one assistant"), + ("wrong_role", "assistant"), + ("missing_role", "assistant"), + ("nontext", "text"), + ("nontext_list", "text"), + ("tool_call", "tool"), + ("no_stop", "incomplete"), + ("length_finish", "incomplete"), + ("no_usage", "usage"), + ("missing_usage_field", "usage"), + ("boolean_usage", "usage"), + ("negative_usage", "usage"), + ("inconsistent_usage", "usage"), + ("early_usage", "usage"), + ("extra_content", "stop"), + ("duplicate_usage", "usage"), + ("provider_error", "provider"), + ("completion_budget", "token budget"), + ("total_budget", "token budget"), + ], +) +def test_candidate_stream_rejects_invalid_or_incomplete_sdk_events( + harness, mutation, diagnostic +): + h = harness + events = [ + stream_event(delta={"role": "assistant", "content": "true"}), + stream_event(finish="stop"), + stream_event( + choices=[], + usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + ), + ] + events = json.loads(json.dumps(events)) + if mutation == "model_drift": + events[1]["model"] = "unlisted/model" + elif mutation == "allowed_model_drift": + events[1]["model"] = "configured/alias" + elif mutation == "id_drift": + events[1]["id"] = "chatcmpl-other" + elif mutation == "fingerprint_drift": + events[1]["system_fingerprint"] = "fp-other" + elif mutation == "two_choices": + events[0]["choices"] *= 2 + elif mutation in {"wrong_index", "boolean_index"}: + events[0]["choices"][0]["index"] = True if mutation == "boolean_index" else 1 + elif mutation in {"wrong_role", "missing_role"}: + events[0]["choices"][0]["delta"]["role"] = ( + None if mutation == "missing_role" else "user" + ) + elif mutation == "nontext": + events[0]["choices"][0]["delta"]["content"] = 7 + elif mutation == "nontext_list": + events[0]["choices"][0]["delta"]["content"] = [] + elif mutation == "tool_call": + events[0]["choices"][0]["delta"]["tool_calls"] = [terminal()] + elif mutation == "no_stop": + events = events[:1] + elif mutation == "length_finish": + events[1]["choices"][0]["finish_reason"] = "length" + elif mutation == "no_usage": + events = events[:2] + elif mutation == "missing_usage_field": + del events[-1]["usage"]["total_tokens"] + elif mutation == "boolean_usage": + events[-1]["usage"]["prompt_tokens"] = True + elif mutation == "negative_usage": + events[-1]["usage"].update(prompt_tokens=-1, total_tokens=4) + elif mutation == "inconsistent_usage": + events[-1]["usage"]["total_tokens"] = 16 + elif mutation == "early_usage": + events[0]["usage"] = events[-1]["usage"] + elif mutation == "extra_content": + events.insert(2, stream_event(delta={"content": "false"})) + elif mutation == "duplicate_usage": + events.append(events[-1]) + elif mutation == "provider_error": + events[-1]["error"] = "provider failed after partial output" + elif mutation == "completion_budget": + events[-1]["usage"].update(completion_tokens=2049, total_tokens=2059) + elif mutation == "total_budget": + events[-1]["usage"].update(prompt_tokens=16384, total_tokens=16389) + closed = [] + + def streaming(): + try: + for event in events: + yield ChatCompletionStreamOutput.parse_obj_as_instance(event) + finally: + closed.append(True) + + h.inference.responses = [streaming()] + record = h.backend.generate_candidate( + {**MODEL, "response_model_aliases": ["configured/alias"]}, 0 + ) + assert diagnostic in record["error"], record + assert record["script"] == "" and not h.factory.created + assert closed == [True] and h.inference.closed == 1 + assert record["model_metadata"]["requests"][0]["stream"] is True + + +@pytest.mark.parametrize("stage", ["before_first", "partial", "after_stop"]) +def test_candidate_stream_provider_failures_preserve_request_and_partial_error( + harness, stage +): + h = harness + closed = [] + + def streaming(): + try: + if stage != "before_first": + yield stream_event(delta={"role": "assistant", "content": "true"}) + if stage == "after_stop": + yield stream_event(finish="stop") + raise RuntimeError("provider error with inference-token-secret") + finally: + closed.append(True) + + h.inference.responses = [streaming()] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] == "HF backend operation failed (RuntimeError)" + assert record["script"] == "" and not h.factory.created + assert "inference-token-secret" not in repr(record) + assert record["model_metadata"]["requests"][0]["stream"] is True + assert closed == [True] and h.inference.closed == 1 + + +@pytest.mark.parametrize("ending", ["provider_error", "extra_event", "over_budget"]) +def test_failed_candidate_stream_retains_observed_final_usage(harness, ending): + h = harness + output_tokens = 2049 if ending == "over_budget" else 5 + + def streaming(): + yield stream_event(delta={"role": "assistant", "content": "true"}) + yield stream_event(finish="stop") + yield stream_event( + choices=[], + usage={ + "prompt_tokens": 10, + "completion_tokens": output_tokens, + "total_tokens": 10 + output_tokens, + }, + ) + if ending == "provider_error": + raise RuntimeError("provider interrupted after usage") + if ending == "extra_event": + yield stream_event(choices=[]) + + h.inference.responses = [streaming()] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] and record["script"] == "" + assert record["usage"] == {"input_tokens": 10, "output_tokens": output_tokens} + assert record["input_tokens"] == 10 and record["output_tokens"] == output_tokens + assert h.inference.closed == 1 and not h.factory.created + + +def test_failed_candidate_request_retains_metadata_and_closes_client(harness): + h = harness + + def fail(**kwargs): + raise RuntimeError("provider unavailable") + + h.inference.chat_completion = fail + record = h.backend.generate_candidate({**MODEL, "top_p": 0.1}, 0) + assert record["error"] == "HF backend operation failed (RuntimeError)" + assert record["model_metadata"]["requests"][0]["top_p"] == 0.1 + assert record["model_metadata"]["requests"][0]["stream"] is True + assert h.inference.closed == 1 and record["script"] == "" + + +@pytest.mark.parametrize("field", ["content", "reasoning", "reasoning_content"]) +def test_candidate_stream_byte_bounds_stop_consuming_and_close(harness, field): + h = harness + h.backend.max_script = 16 if field == "content" else 65536 + (h.root / "instruction.md").write_text("write answer") + continued = [] + + def streaming(): + yield stream_event(delta={"role": "assistant", field: "雪" * 6}) + continued.append(True) + pytest.fail("oversized stream was consumed after its byte limit") + + h.inference.responses = [streaming()] + record = h.backend.generate_candidate( + {**MODEL, "max_output_bytes": 512 if field == "content" else 16}, 0 + ) + assert "byte limit" in record["error"] and record["script"] == "" + assert not continued and h.inference.closed == 1 + + +def test_candidate_stream_keeps_serialized_output_byte_gate(harness): + h = harness + h.inference.responses = [response("雪" * 10)] + record = h.backend.generate_candidate({**MODEL, "max_output_bytes": 64}, 0) + assert "output exceeded byte limit" in record["error"] + assert record["script"] == "" and h.inference.closed == 1 + + +def test_candidate_stream_accepts_usage_on_final_stop(harness): + h = harness + h.inference.responses = [ + ( + event + for event in [ + stream_event( + delta={"role": "assistant", "content": "true"}, + finish="stop", + usage={ + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + }, + ) + ] + ) + ] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] is None and record["script"] == "true" + assert record["usage"] == {"input_tokens": 10, "output_tokens": 5} + + +def test_candidate_stream_is_interrupted_by_root_deadline_and_closes(harness): + h = harness + h.backend.timeout = 0.02 + closed = [] + + def streaming(): + try: + yield stream_event(delta={"role": "assistant", "content": "true"}) + time.sleep(1) + finally: + closed.append(True) + + h.inference.responses = [streaming()] + started = time.monotonic() + record = h.backend.generate_candidate(MODEL, 0) + assert time.monotonic() - started < 0.5 + assert "deadline" in record["error"] and record["script"] == "" + assert closed == [True] and h.inference.closed == 1 + + +def test_unqualified_sdk_version_refused_before_launch(harness, monkeypatch): + monkeypatch.setattr("examples.coding_audit.hf.version", lambda name: "0.0.0") + with pytest.raises(HFBackendError, match="requires huggingface_hub"): + HFTaskBackend( + harness.backend.task_root, + harness.backend.execution, + sandbox_factory=harness.factory, + ) + assert not harness.factory.created + + +@pytest.mark.parametrize( + "model", + [ + {}, + {**MODEL, "revision": "main"}, + {**MODEL, "model_id": MODEL["model_id"] + "@" + MODEL["revision"]}, + {**MODEL, "provider": "auto"}, + {**MODEL, "supports_tools": False}, + ], +) +def test_invalid_model_preflight_never_launches_or_calls_provider(harness, model): + record = harness.backend.solve(model, 0) + assert record["error"] + assert not harness.factory.created and not harness.inference.requests + + +def test_target_tool_loop_is_bounded_and_uses_instruction_only(harness): + h = harness + h.inference.responses = [response(None, calls=[terminal()]), response("saved")] + record = h.backend.solve(MODEL, 1) + assert record["error"] is None and record["cleanup_verified"] + assert record["files"] == {"/app/answer.txt": b"answer"} + assert len(h.factory.created) == 1 + assert h.factory.created[0].commands[0][0] == [ + "/bin/bash", + "-c", + "printf answer > /app/answer.txt", + ] + assert h.inference.requests[0]["tools"][0]["function"]["name"] == "terminal" + assert all("stream" not in request for request in h.inference.requests) + assert h.inference.closed == 0 + assert "PRIVATE_SOLUTION_SENTINEL" not in repr(h.inference.requests) + assert "PRIVATE_CANARY_SENTINEL" not in repr(h.inference.requests) + assert record["input_tokens"] == 20 and record["output_tokens"] == 10 + + +def test_candidate_context_binds_only_declared_public_bytes_to_prompt( + harness, monkeypatch +): + from examples.coding_audit import hf + + h = harness + path = "environment/source/pkg/api.py" + source = h.root / path + source.parent.mkdir(parents=True) + raw = 'def answer():\n return "雪"\n'.encode() + source.write_bytes(raw) + (source.parent / "unused.py").write_text("UNDECLARED_PUBLIC_SENTINEL") + (source.parent / ".hidden").mkdir() + (source.parent / ".hidden/private.txt").write_text("HIDDEN_SENTINEL") + reads = [] + original = hf._local_file + + def read(path, limit): + reads.append(path.relative_to(h.root).as_posix()) + return original(path, limit) + + monkeypatch.setattr(hf, "_local_file", read) + h.backend.workdir = "/workspace/runtime" + h.inference.responses = [response("true")] + record = h.backend.generate_candidate({**MODEL, "context_paths": [path]}, 0) + assert record["error"] is None + assert reads == ["instruction.md", path] + prompt = h.inference.requests[0]["messages"][1]["content"] + assert prompt.startswith((h.root / "instruction.md").read_text()) + assert path in prompt and "pkg/api.py" in prompt and "/workspace/runtime" in prompt + assert raw.decode() in prompt + assert ( + "UNDECLARED_PUBLIC_SENTINEL" not in prompt and "HIDDEN_SENTINEL" not in prompt + ) + assert "PRIVATE_SOLUTION_SENTINEL" not in prompt + assert record["candidate_context"] == [ + { + "path": path, + "runtime_path": "pkg/api.py", + "size": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + } + ] + assert "context_paths" not in h.inference.requests[0] + + +def test_default_candidate_prompt_and_target_protocol_are_unchanged(harness): + h = harness + h.inference.responses = [response("true"), response("done")] + generated = h.backend.generate_candidate(MODEL, 0) + solved = h.backend.solve(MODEL, 1) + assert generated["error"] is None and solved["error"] is None + assert "candidate_context" not in generated + assert all( + request["messages"][1]["content"] == (h.root / "instruction.md").read_text() + for request in h.inference.requests + ) + + +@pytest.mark.parametrize( + "kind", ["symlink", "symlink_parent", "hardlink", "directory", "fifo"] +) +def test_candidate_context_rejects_linked_or_nonregular_files_before_inference( + harness, kind +): + h = harness + parent = h.root / "environment/source" + parent.mkdir(parents=True) + source = parent / "api.py" + if kind == "symlink": + source.symlink_to(h.root / "solution/solve.sh") + elif kind == "symlink_parent": + (h.root / "solution/api.py").write_text("PRIVATE_API_SENTINEL") + (parent / "linked").symlink_to(h.root / "solution", target_is_directory=True) + source = parent / "linked/api.py" + elif kind == "hardlink": + os.link(h.root / "solution/solve.sh", source) + elif kind == "directory": + source.mkdir() + else: + os.mkfifo(source) + record = h.backend.generate_candidate( + {**MODEL, "context_paths": [source.relative_to(h.root).as_posix()]}, 0 + ) + assert record["error"] and record["script"] == "" + assert not h.inference.requests and not h.inference.kwargs + assert not h.factory.created + + +@pytest.mark.parametrize( + "contents", + [ + [b"public\x00hidden"], + [b"\xff"], + [b"x" * 65537], + [b"x" * 32768, b"y" * 32769], + ], +) +def test_candidate_context_requires_bounded_utf8_without_nul(harness, contents): + h = harness + parent = h.root / "environment/source" + parent.mkdir(parents=True) + paths = [] + for index, raw in enumerate(contents): + path = parent / f"public-{index}.txt" + path.write_bytes(raw) + paths.append(path.relative_to(h.root).as_posix()) + record = h.backend.generate_candidate({**MODEL, "context_paths": paths}, 0) + assert record["error"] and record["script"] == "" + assert not h.inference.requests and not h.inference.kwargs + + +@pytest.mark.parametrize( + "size,budget,accepted", [(65536, 32768, True), (1024, 64, False)] +) +def test_candidate_context_uses_combined_byte_and_existing_message_budget( + harness, size, budget, accepted +): + h = harness + path = h.root / "environment/source/api.py" + path.parent.mkdir(parents=True) + path.write_bytes(b"x" * size) + h.inference.responses = [response("true")] + record = h.backend.generate_candidate( + { + **MODEL, + "context_paths": ["environment/source/api.py"], + "max_total_tokens": budget, + }, + 0, + ) + assert (record["error"] is None) is accepted + assert bool(h.inference.requests) is accepted + if not accepted: + assert "context exceeds byte budget" in record["error"] + + +def test_target_context_is_rejected_before_sandbox_or_inference(harness): + h = harness + record = h.backend.solve( + {**MODEL, "context_paths": ["environment/source/api.py"]}, 0 + ) + assert record["error"] and not h.factory.created + assert not h.inference.requests and not h.inference.kwargs + + +@pytest.mark.parametrize( + "path", + [ + "solution/solve.sh", + "environment/source/../../solution/solve.sh", + "environment/source/tests/hidden.py", + "environment/source/validation/spec.json", + "environment/source/.hidden/secret.py", + "environment/source/secrets/token", + "environment/source/api\x00.py", + ], +) +def test_invalid_candidate_context_is_rejected_without_reading_private_files( + harness, monkeypatch, path +): + from examples.coding_audit import hf + + h = harness + reads = [] + original = hf._local_file + + def read(filename, limit): + assert filename == h.root / "instruction.md" + reads.append(filename) + return original(filename, limit) + + monkeypatch.setattr(hf, "_local_file", read) + record = h.backend.generate_candidate({**MODEL, "context_paths": [path]}, 0) + assert record["error"] and record["script"] == "" + assert reads == [h.root / "instruction.md"] + assert not h.inference.requests and not h.inference.kwargs + + +def test_excess_parallel_tool_calls_execute_nothing_and_cleanup(harness): + h = harness + h.inference.responses = [ + response(None, calls=[terminal(identifier="a"), terminal(identifier="b")]) + ] + record = h.backend.solve({**MODEL, "max_steps": 1}, 0) + assert record["error"] is None and record["cleanup_verified"] + assert record["truncated"] and record["termination_reason"] == "tool_call_limit" + assert not h.factory.created[0].commands + + +def test_step_exhaustion_hands_actual_submission_to_grader(harness): + h = harness + h.inference.responses = [response(None, calls=[terminal()])] + record = h.backend.solve({**MODEL, "max_steps": 1}, 0) + assert record["error"] is None and record["files"] == {"/app/answer.txt": b"answer"} + assert record["truncated"] and record["termination_reason"] == "step_limit" + assert len(h.inference.requests) == 1 and record["cleanup_verified"] + grade = h.backend.grade(record) + assert grade["error"] is None and grade["reward"] == 1 + + +def test_invalid_tool_name_is_not_executed(harness): + h = harness + call = terminal() + call["function"]["name"] = "provider_request" + h.inference.responses = [response(None, calls=[call])] + record = h.backend.solve(MODEL, 0) + assert "invalid terminal" in record["error"] and not h.factory.created[0].commands + + +def test_output_and_token_limits_are_fail_closed(harness): + h = harness + h.factory.output = "💥" * 4097 + record = h.backend.execute_script(":", "reference") + assert "output exceeded" in record["error"] and record["cleanup_verified"] + h.factory.output = "" + h.inference.responses = [ + response( + "true", + usage={ + "prompt_tokens": 16000, + "completion_tokens": 400, + "total_tokens": 16400, + }, + ) + ] + record = h.backend.generate_candidate(MODEL, 0) + assert "token budget" in record["error"] + + +def test_private_path_audit_blocks_before_execution(harness): + h = harness + h.factory.initial["/tests"] = b"private" + record = h.backend.execute_script("true", "reference") + assert "private path" in record["error"] and record["cleanup_verified"] + assert not h.factory.created[0].commands + + +def test_leakage_probes_recheck_transcripts_without_extra_jobs(harness): + h = harness + h.factory.submitted = None + probes = h.backend.leakage_probes() + assert {row["name"] for row in probes} == { + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", + } + assert all(row["passed"] and row["error"] is None for row in probes) + assert len(h.factory.created) == 2 + h.inference.responses = [response("# PRIVATE_CANARY_SENTINEL\ntrue")] + h.backend.generate_candidate(MODEL, 0) + probes = h.backend.leakage_probes() + assert len(h.factory.created) == 2 + assert not next(row for row in probes if row["name"] == "canary_not_in_transcript")[ + "passed" + ] + + +def test_fresh_state_marker_blocks_reused_filesystem(harness): + h = harness + first = h.backend.execute_script(":", "reference") + h.factory.initial.update(h.factory.created[0].files.data) + second = h.backend.execute_script(":", "target") + assert first["error"] is None and "prior-state" in second["error"] + assert not h.factory.created[1].commands + assert not next( + row for row in h.backend.leakage_probes() if row["name"] == "fresh_state_absent" + )["passed"] + + +def test_state_marker_uses_configured_workdir(harness): + h = harness + h.execution["workdir"] = "/workspace/task" + backend = HFTaskBackend( + h.root, + h.execution, + token="execution-token-secret", + inference_token="inference-token-secret", + api=h.api, + sandbox_factory=h.factory, + inference_factory=h.inference, + ) + assert PurePosixPath(backend.marker_path).parent == PurePosixPath("/workspace/task") + assert PurePosixPath(backend.marker_path).name.startswith(".openenv-state-") + first = backend.execute_script(":", "reference") + h.factory.initial.update(h.factory.created[0].files.data) + second = backend.execute_script(":", "target") + assert first["error"] is None and "prior-state" in second["error"] + + +def test_submission_tree_digest_matches_canonical_algorithm(tmp_path): + (tmp_path / "app").mkdir() + (tmp_path / "app/a").write_bytes(b"alpha") + (tmp_path / "app/z").write_bytes(b"omega") + assert submission_digest({"/app/z": b"omega", "/app/a": b"alpha"}) == source_digest( + tmp_path + ) + assert submission_digest({}) == hashlib.sha256(b"").hexdigest() + + +def test_deadline_interrupts_blocking_controller_and_restores_handler(): + handler = signal.getsignal(signal.SIGALRM) + started = time.monotonic() + with pytest.raises(HFBackendError, match="deadline"): + with _deadline(0.02): + time.sleep(1) + assert time.monotonic() - started < 0.5 + assert signal.getsignal(signal.SIGALRM) == handler + assert signal.getitimer(signal.ITIMER_REAL)[0] == 0 + + +def test_strong_isolation_and_equal_credentials_rejected(harness): + h = harness + with pytest.raises(HFBackendError, match="strong"): + HFTaskBackend(h.root, {**h.execution, "require_strong_isolation": True}) + with pytest.raises(HFBackendError, match="credentials must differ"): + HFTaskBackend(h.root, h.execution, token="same", inference_token="same") + + +def test_runtime_tokens_are_redacted_and_credential_files_rejected(harness): + h = harness + h.factory.output = "server echoed execution-token-secret" + record = h.backend.execute_script(":", "reference") + assert "execution-token-secret" not in repr(record) + assert "[REDACTED]" in repr(record["transcript"]) + h.factory.output = "" + h.factory.submitted = b"inference-token-secret" + record = h.backend.execute_script(":", "reference") + assert "credential detected" in record["error"] and record["files"] == {} + assert "inference-token-secret" not in repr(record) + + +def test_empty_submission_does_not_inherit_verifier_image_answer(harness): + h = harness + h.factory.initial["/app/answer.txt"] = b"image-provided-answer" + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert grade["error"] is None + assert "/app/answer.txt" not in h.factory.created[0].files.data + + +def test_verifier_private_image_paths_are_not_overlaid(harness): + h = harness + h.factory.initial["/tests"] = b"stale-tests" + grade = h.backend.grade({"files": {}, "submission_digest": submission_digest({})}) + assert "preexisting private" in grade["error"] and grade["cleanup_verified"] + assert not h.factory.created[0].commands + + +def test_remote_regular_file_and_read_growth_checks(harness): + h = harness + sandbox = h.factory(image=AGENT) + sandbox.files.data["/app/answer.txt"] = b"answer" + sandbox.files.types["/app/answer.txt"] = "symlink" + with pytest.raises(HFBackendError, match="regular"): + h.backend._collect(sandbox) + sandbox.files.types.clear() + sandbox.files.grow_on_read = True + with pytest.raises(SandboxError): + h.backend._collect(sandbox) + assert sandbox.files.MAX_READ_BYTES == h.backend.max_bytes + + +def test_missing_execution_or_inference_credentials_never_fall_back_to_hf_token( + harness, monkeypatch +): + h = harness + monkeypatch.setenv("HF_TOKEN", "persistence-token") + backend = HFTaskBackend( + h.root, + h.execution, + api=h.api, + sandbox_factory=h.factory, + inference_factory=h.inference, + ) + assert "HF_SANDBOX_TOKEN" in backend.execute_script(":", "nop")["error"] + assert "HF_INFERENCE_TOKEN" in backend.solve(MODEL, 0)["error"] + assert not h.factory.created and not h.inference.requests + + +def test_timed_out_command_with_zero_exit_is_error(harness): + harness.factory.timed_out = True + record = harness.backend.execute_script(":", "reference") + assert "timed out" in record["error"] and record["cleanup_verified"] + + +def test_cleanup_deadline_is_not_swallowed_by_retry_loop(harness): + h = harness + original = h.api.cancel_job + + def blocked_cancel(**kwargs): + time.sleep(2) + return original(**kwargs) + + h.api.cancel_job = blocked_cancel + h.backend.cleanup_timeout = 0.02 + started = time.monotonic() + record = h.backend.execute_script(":", "reference") + assert time.monotonic() - started < 0.5 + assert record["error"] and not record["cleanup_verified"] + + +def test_canary_probe_checks_actual_unicode_message_text(harness): + h = harness + h.backend.canary = "private\n雪" + h.backend._transcripts = [[{"content": "private\n雪"}]] + assert not h.backend.leakage_probes()[-1]["passed"] + + +def test_failed_shell_command_is_observed_then_agent_repairs(harness, monkeypatch): + h = harness + original = FakeSandbox.run + calls = [] + + def run_with_failure(sandbox, argv, **kwargs): + calls.append(argv) + result = original(sandbox, argv, **kwargs) + result.exit_code = 1 if len(calls) == 1 else 0 + return result + + monkeypatch.setattr(FakeSandbox, "run", run_with_failure) + h.inference.responses = [ + response(None, calls=[terminal("grep missing absent", "a")]), + response(None, calls=[terminal("printf answer > /app/answer.txt", "b")]), + response("saved"), + ] + record = h.backend.solve(MODEL, 0) + assert record["error"] is None and not record["truncated"] + assert record["files"] == {"/app/answer.txt": b"answer"} + observation = h.inference.requests[1]["messages"][-1] + assert json.loads(observation["content"])["exit_code"] == 1 + assert h.backend.grade(record)["reward"] == 1 + + +def test_execution_token_in_terminal_output_never_reaches_provider(harness): + h = harness + h.factory.output = "server echoed execution-token-secret" + h.inference.responses = [response(None, calls=[terminal()]), response("done")] + record = h.backend.solve(MODEL, 0) + assert record["error"] is None + assert "execution-token-secret" not in repr(h.inference.requests) + assert "[REDACTED]" in repr(h.inference.requests[1]) + + +def test_failed_startup_recovers_exact_labeled_job_and_independently_cleans(harness): + h = harness + h.factory.fail_after_create = True + original = h.api.cancel_job + cancelled = [] + + def transient_cancel(**kwargs): + cancelled.append(kwargs["job_id"]) + if len(cancelled) == 1: + raise RuntimeError("transient controller failure") + return original(**kwargs) + + h.api.cancel_job = transient_cancel + record = h.backend.execute_script(":", "reference") + assert record["error"] and record["cleanup_verified"] + assert record["sandbox_id"] == "job-0" and cancelled == ["job-0", "job-0"] + assert record["cleanup_job_ids"] == ["job-0"] + assert h.api.stages["job-0"] == "CANCELED" + + +def test_transient_kill_exception_does_not_abort_cleanup_retries(harness, monkeypatch): + h = harness + h.api.cancel_error = True + original = FakeSandbox.kill + killed = [] + + def transient_kill(sandbox): + killed.append(sandbox.id) + if len(killed) == 1: + raise RuntimeError("transient kill failure") + original(sandbox) + + monkeypatch.setattr(FakeSandbox, "kill", transient_kill) + record = h.backend.execute_script(":", "reference") + assert record["error"] is None and record["cleanup_verified"] + assert killed == ["job-0", "job-0"] + + +def test_startup_recovery_retries_transient_listing_error(harness): + h = harness + h.factory.fail_after_create = True + original = h.api.list_jobs + calls = [] + + def transient_listing(**kwargs): + calls.append(kwargs) + if len(calls) == 1: + raise RuntimeError("transient lookup failure") + return original(**kwargs) + + h.api.list_jobs = transient_listing + record = h.backend.execute_script(":", "reference") + assert record["error"] and record["cleanup_verified"] + assert len(calls) == 2 and h.api.cancelled == ["job-0"] + + +def test_rejected_length_stream_keeps_finish_and_terminal_usage(harness): + h = harness + h.inference.responses = [ + ( + ChatCompletionStreamOutput.parse_obj_as_instance(event) + for event in ( + [ + stream_event(delta={"role": "assistant", "content": "partial"}), + stream_event(finish="length"), + stream_event( + choices=[], + usage={ + "prompt_tokens": 10, + "completion_tokens": 2048, + "total_tokens": 2058, + }, + ), + ] + ) + ) + ] + record = h.backend.generate_candidate(MODEL, 0) + assert "incomplete" in record["error"] and record["script"] == "" + assert record["model_metadata"]["responses"][0]["finish_reason"] == "length" + assert record["usage"] == {"input_tokens": 10, "output_tokens": 2048} + assert h.inference.closed == 1 and not h.factory.created + + +@pytest.mark.parametrize("value", [0, 1, "false", None]) +def test_thinking_flag_requires_an_explicit_boolean(harness, value): + h = harness + record = h.backend.generate_candidate({**MODEL, "enable_thinking": value}, 0) + assert "enable_thinking must be a boolean" in record["error"] + assert not h.inference.requests and record["script"] == "" + + +@pytest.mark.parametrize("language", ["bash", "sh", "shell"]) +@pytest.mark.parametrize("closed", [False, True]) +def test_fenced_candidate_keeps_raw_response_and_executes_only_bash( + harness, language, closed +): + h = harness + body = "printf answer > /app/answer.txt\n" + raw = "```" + language + "\n" + body + ("```\n" if closed else "") + h.inference.responses = [response(raw)] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] is None and record["script"] == body + assert record["transcript"][-1]["content"] == raw + metadata = record["script_format"] + assert metadata["raw_sha256"] == hashlib.sha256(raw.encode()).hexdigest() + assert ( + metadata["script_sha256"] + == hashlib.sha256(record["script"].encode()).hexdigest() + ) + assert metadata["closing_fence_present"] is closed + submission = h.backend.execute_script(record["script"], "generated") + assert submission["error"] is None + assert h.factory.created[0].commands[-1][0] == ["/bin/bash", "-c", record["script"]] + + +@pytest.mark.parametrize( + "raw", ["```bash\n```", "```python\nprint('wrong language')\n```"] +) +def test_empty_or_non_bash_fence_does_not_create_a_candidate(harness, raw): + h = harness + h.inference.responses = [response(raw)] + record = h.backend.generate_candidate(MODEL, 0) + assert record["error"] and record["script"] == "" and not h.factory.created diff --git a/examples/coding_audit/test_runner.py b/examples/coding_audit/test_runner.py new file mode 100644 index 000000000..b97384b05 --- /dev/null +++ b/examples/coding_audit/test_runner.py @@ -0,0 +1,838 @@ +"""Exercise collection, gating and artifact binding without paid services.""" + +import hashlib +import json +import runpy +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +from examples.coding_audit.runner import ( + coding_budget, + live_provider_id, + normalize_coding_config, + run_coding_validation, +) + + +@pytest.mark.parametrize("representation", ["dict", "list", "json-list"]) +def test_live_hub_provider_mapping_shapes(representation): + entry = SimpleNamespace( + provider="deepinfra", provider_id="Qwen/Qwen3.8-27B", status="live" + ) + mappings = ( + {"deepinfra": entry} + if representation == "dict" + else [vars(entry)] + if representation == "json-list" + else [entry] + ) + assert ( + live_provider_id( + SimpleNamespace(inference_provider_mapping=mappings), "deepinfra" + ) + == "Qwen/Qwen3.8-27B" + ) + + +@pytest.mark.parametrize( + "mappings", + [ + None, + [], + [SimpleNamespace(provider="deepinfra", provider_id="model", status="staging")], + [SimpleNamespace(provider="deepinfra", provider_id="model", status="live")] * 2, + ], +) +def test_provider_mapping_must_be_unique_and_live(mappings): + with pytest.raises(ValueError): + live_provider_id( + SimpleNamespace(inference_provider_mapping=mappings), "deepinfra" + ) + + +def configuration(): + config = json.loads(Path(__file__).with_name("config.json").read_text()) + config["generation"]["candidate_count"] = 1 + config["validation"].update(reference_repeats=2, target_attempts=2) + config["seeds"] = [1, 2] + return config + + +class Backend: + def __init__(self): + self.number = 0 + self.target_calls = 0 + self.generated = "mkdir -p /app; echo 42 > /app/answer.txt" + self.bad_verifier = False + self.bad_raw = False + self.target_error = False + self.mutate = None + self.transcript = [] + + def execute_script(self, script, role): + self.number += 1 + value = b"42\n" if "42" in script else b"41\n" + files = {} if script == "true" else {"/app/answer.txt": value} + digest = hashlib.sha256( + json.dumps({k: v.hex() for k, v in files.items()}, sort_keys=True).encode() + ).hexdigest() + image = configuration()["execution"]["agent_image"] + return { + "files": files, + "submission_digest": digest, + "sandbox_id": f"solver-{self.number}", + "image_ref": image, + "cleanup_verified": True, + "error": None, + "transcript": list(self.transcript), + "usage": {"input_tokens": 10, "output_tokens": 5}, + "latency_seconds": 1, + } + + def solve(self, model, seed, role): + self.target_calls += 1 + result = self.execute_script("42" if seed == 2 else "41", role) + if self.target_error: + result["error"] = "provider failure" + return result + + def generate_candidate(self, model, seed): + if self.mutate: + self.mutate() + return {"script": self.generated, "error": None} + + def grade(self, submission): + self.number += 1 + reward = int( + submission["files"].get("/app/answer.txt") == b"42\n" or self.bad_verifier + ) + return { + "grade_sandbox_id": f"grader-{self.number}", + "verifier_image_ref": configuration()["execution"]["verifier_image"], + "reward": reward, + "rawreward": b"0.999999999999999999999999" + if self.bad_raw + else str(reward).encode(), + "cleanup_verified": True, + "error": None, + "transcript": [ + { + "exit_code": 0, + "output": f"Native reward: {reward}\n", + "output_truncated": False, + } + ], + } + + def leakage_probes(self): + return [ + {"name": name, "passed": True, "error": None} + for name in ( + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", + ) + ] + + +@pytest.fixture +def task(tmp_path): + root = tmp_path / "task" + for name in ("tests", "solution", "environment"): + (root / name).mkdir(parents=True) + (root / "task.toml").write_text('version = "1.0"\n') + (root / "instruction.md").write_text("Write exactly 42 into /app/answer.txt.\n") + (root / "tests/test.sh").write_text("#!/bin/bash\nprintf 1\n") + (root / "solution/solve.sh").write_text("printf 42 > /app/answer.txt\n") + (root / "environment/Dockerfile").write_text( + "FROM python:3.12-slim\nWORKDIR /app\n" + ) + return root + + +def test_complete_collection_is_bound_and_keeps_model_difficulty(task, tmp_path): + backend = Backend() + output = tmp_path / "audit" + report = run_coding_validation(task, configuration(), output, backend=backend) + assert report["accepted"] + assert not report["rfc008_complete"] + assert not report["trainability_established"] + assert backend.target_calls == 2 + assert report["checks"]["target_difficulty"]["solve_rate"] == 0.5 + evidence = json.loads((output / "evidence.json").read_text()) + assert {r["origin"] for r in evidence["references"]} == {"provided", "generated"} + assert len(evidence["generation"]) == 1 + for row in evidence["trials"]: + artifact = row["grader_transcript"] + assert artifact["path"].endswith("/grader-transcript.json") + data = (output / artifact["path"]).read_bytes() + assert hashlib.sha256(data).hexdigest() == artifact["sha256"] + assert json.loads(data)[0]["output"] == f"Native reward: {row['reward']}\n" + assert evidence["trials"][0]["reward"] == 0 + for line in (output / "SHA256SUMS").read_text().splitlines(): + digest, name = line.split(" ", 1) + assert hashlib.sha256((output / name).read_bytes()).hexdigest() == digest + + +def test_bad_verifier_blocks_target_inference(task, tmp_path): + backend = Backend() + backend.bad_verifier = True + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + assert report["checks"]["verifier_rejection"]["status"] == "fail" + assert backend.target_calls == 0 + + +def test_failed_clean_candidate_excluded_without_overwriting_frozen_verifier( + task, tmp_path +): + backend = Backend() + backend.generated = "echo wrong" + before = (task / "tests/test.sh").read_bytes() + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert report["accepted"] + assert len(report["checks"]["references"]["excluded"]) == 1 + assert (task / "tests/test.sh").read_bytes() == before + + +def test_raw_reward_precision_disagreement_cannot_pass(task, tmp_path): + backend = Backend() + backend.bad_raw = True + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + assert backend.target_calls == 0 + assert report["checks"]["reliability"]["status"] == "error" + + +def test_target_infra_failure_preserves_missing_not_zero(task, tmp_path): + backend = Backend() + backend.target_error = True + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + check = report["checks"]["target_difficulty"] + assert not report["accepted"] + assert check["completed"] == 0 + assert check["solve_rate"] is None + + +def test_changed_source_cannot_be_admitted(task, tmp_path): + backend = Backend() + backend.mutate = lambda: (task / "instruction.md").write_text("Changed task") + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + evidence = json.loads((tmp_path / "audit/evidence.json").read_text()) + assert evidence["collection_error"] == "ValueError" + + +def test_credentials_in_transcript_are_omitted_and_block_acceptance( + task, tmp_path, monkeypatch +): + token = "test-secret-not-a-real-token" + monkeypatch.setenv("HF_SANDBOX_TOKEN", token) + backend = Backend() + backend.transcript = [{"content": token}] + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + for path in (tmp_path / "audit").rglob("*"): + if path.is_file(): + assert token.encode() not in path.read_bytes() + + +@pytest.mark.parametrize( + "path,value", [("agent_image", "python:latest"), ("api_key", "secret")] +) +def test_mutable_images_and_embedded_credentials_rejected(path, value): + config = configuration() + config["execution"][path] = value + with pytest.raises(ValueError): + normalize_coding_config(config) + + +def test_budget_is_bounded_before_any_remote_execution(task, tmp_path): + config = configuration() + config["cloud"]["max_sandboxes"] = 1 + with pytest.raises(ValueError, match="budget"): + run_coding_validation(task, config, tmp_path / "audit", backend=Backend()) + assert not (tmp_path / "audit").exists() + assert ( + coding_budget(normalize_coding_config(configuration()))["inference_calls_max"] + == 33 + ) + + +@pytest.mark.parametrize("accepted", [True, False]) +def test_cli_dispatch_uses_example_runner_and_retains_evidence( + task, tmp_path, monkeypatch, capsys, accepted +): + config = tmp_path / "config.json" + config.write_text(json.dumps(configuration())) + original = run_coding_validation + backend = Backend() + backend.bad_verifier = not accepted + monkeypatch.setattr( + "examples.coding_audit.runner.run_coding_validation", + lambda task, config, output: original(task, config, output, backend=backend), + ) + output = tmp_path / "audit" + monkeypatch.setattr( + sys, + "argv", + [ + "examples.coding_audit", + "--task", + str(task), + "--config", + str(config), + "--output", + str(output), + "--json", + ], + ) + with pytest.raises(SystemExit) as result: + runpy.run_module("examples.coding_audit", run_name="__main__") + assert result.value.code == (0 if accepted else 1) + report = json.loads((output / "report.json").read_text()) + assert report["accepted"] is accepted + assert json.loads(capsys.readouterr().out) == report + assert (output / "evidence.json").is_file() + for line in (output / "SHA256SUMS").read_text().splitlines(): + digest, name = line.split(" ", 1) + assert hashlib.sha256((output / name).read_bytes()).hexdigest() == digest + + +def test_cli_requires_persisted_evidence(task, tmp_path, monkeypatch, capsys): + monkeypatch.setattr( + sys, + "argv", + [ + "examples.coding_audit", + "--task", + str(task), + "--config", + str(tmp_path / "missing.json"), + ], + ) + with pytest.raises(SystemExit) as result: + runpy.run_module("examples.coding_audit", run_name="__main__") + assert result.value.code == 2 + assert "--output" in capsys.readouterr().err + + +def test_cli_input_failure_exits_two_without_starting_backend( + task, tmp_path, monkeypatch, capsys +): + from examples.coding_audit.__main__ import main + + monkeypatch.setattr( + "examples.coding_audit.__main__.run_coding_validation", + lambda *args: pytest.fail("input failure reached backend"), + ) + assert ( + main( + [ + "--task", + str(task), + "--config", + str(tmp_path / "missing.json"), + "--output", + str(tmp_path / "audit"), + ] + ) + == 2 + ) + assert "input failure (FileNotFoundError)" in capsys.readouterr().err + + +def test_exact_json_threshold_cannot_round_into_a_pass(task, tmp_path): + from decimal import Decimal + + from openenv.validation.runtime.json_values import loads + + config = loads( + json.dumps(configuration()).replace( + '"reward_max": 1,', '"reward_max": 1.00000000000000000001,' + ) + ) + normalized = normalize_coding_config(config) + assert normalized["validation"]["reward_max"] == Decimal("1.00000000000000000001") + report = run_coding_validation(task, config, tmp_path / "audit", backend=Backend()) + assert not report["accepted"] + saved = loads((tmp_path / "audit/config.json").read_text()) + assert saved["validation"]["reward_max"] == normalized["validation"]["reward_max"] + + +@pytest.mark.parametrize("value", ["", "missing"]) +def test_execution_error_status_cannot_be_overwritten(task, tmp_path, value): + class BrokenStatus(Backend): + def execute_script(self, script, role): + row = super().execute_script(script, role) + if value == "missing": + del row["error"] + else: + row["error"] = value + return row + + backend = BrokenStatus() + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=backend + ) + assert not report["accepted"] + assert backend.target_calls == 0 + rows = json.loads((tmp_path / "audit/evidence.json").read_text())["trials"] + assert all(row["error"] is not None for row in rows) + + +def test_verifier_identity_comes_from_verified_snapshot(task, tmp_path, monkeypatch): + from examples.coding_audit import runner as coding_runner + + original = coding_runner.source_digest + expected = original(task / "tests") + + def raced(path): + if path == task / "tests": + raise AssertionError( + "mutable original verifier must not be hashed independently" + ) + return original(path) + + monkeypatch.setattr(coding_runner, "source_digest", raced) + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=Backend() + ) + assert report["accepted"] + assert report["verifier_digest"] == expected + + +def test_credential_metadata_is_failed_and_report_is_sanitized( + task, tmp_path, monkeypatch +): + secret = "dummy-execution-credential-for-regression" + monkeypatch.setenv("HF_SANDBOX_TOKEN", secret) + + class DiagnosticBackend(Backend): + def leakage_probes(self): + return super().leakage_probes() + [ + { + "name": "diagnostic:" + secret, + "passed": True, + "error": None, + secret: "key", + } + ] + + report = run_coding_validation( + task, configuration(), tmp_path / "audit", backend=DiagnosticBackend() + ) + assert not report["accepted"] + assert secret not in json.dumps(report) + for path in (tmp_path / "audit").rglob("*"): + if path.is_file(): + assert secret.encode() not in path.read_bytes() + + +def test_credential_in_config_is_rejected_before_execution(task, tmp_path, monkeypatch): + secret = "dummy-config-credential-for-regression" + monkeypatch.setenv("HF_SANDBOX_TOKEN", secret) + config = configuration() + config["negative_cases"][0]["id"] = secret + config["validation"]["required_negative_cases"][0] = secret + with pytest.raises(ValueError, match="credentials"): + run_coding_validation(task, config, tmp_path / "audit", backend=Backend()) + + +def test_in_task_output_rejected_before_inference(task): + backend = Backend() + with pytest.raises(ValueError, match="outside"): + run_coding_validation( + task, configuration(), task / "audit.artifacts", backend=backend + ) + assert backend.target_calls == 0 + assert not (task / "audit.artifacts").exists() + + +@pytest.mark.parametrize( + "artifact", + [ + "environment/source/api.py", + "instruction.md", + "tests/test.sh", + "solution/solve.sh", + ], +) +def test_snapshot_rejects_hardlinked_artifacts_before_inference( + task, tmp_path, monkeypatch, artifact +): + import huggingface_hub + from examples.coding_audit import hf + + m = SimpleNamespace(**runpy.run_path(str(Path(__file__).with_name("test_hf.py")))) + api, inference = m.API(), m.InferenceFactory() + factory = m.Factory(api) + api.model_info = lambda *args, **kwargs: SimpleNamespace( + sha=m.MODEL["revision"], + inference_provider_mapping={ + "deepinfra": {"provider_id": m.MODEL["model_id"], "status": "live"} + }, + ) + monkeypatch.setattr(huggingface_hub, "HfApi", lambda **kwargs: api) + monkeypatch.setattr(hf, "HfApi", lambda **kwargs: api) + monkeypatch.setattr(hf.Sandbox, "create", factory) + monkeypatch.setattr(hf, "InferenceClient", inference) + for name in ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN"): + monkeypatch.setenv(name, "synthetic-test-" + name) + context_path = "environment/source/api.py" + source = task / context_path + source.parent.mkdir(parents=True) + if artifact == context_path: + source.hardlink_to(task / "solution/solve.sh") + else: + source.write_text("def public_answer(): return 42\n") + (tmp_path / "linked-artifact").hardlink_to(task / artifact) + assert (task / artifact).stat().st_nlink == 2 + config = configuration() + config["generator"] = {**config["target"], "context_paths": [context_path]} + config["execution"].update( + agent_image=m.AGENT, verifier_image=m.VERIFIER, namespace="owner" + ) + inference.responses = [m.response("true")] + output = tmp_path / "audit" + report = run_coding_validation(task, config, output) + assert not inference.requests and not inference.kwargs + assert not factory.created + assert not report["accepted"] + evidence = json.loads((output / "evidence.json").read_text()) + assert evidence["collection_error"] == "HFBackendError" + + +@pytest.mark.parametrize("with_context", [False, True]) +@pytest.mark.parametrize( + "completion", ["stop", "null_calls", "missing_calls", "empty_calls"] +) +def test_real_backend_contract_reaches_difficulty_with_sdk_fakes( + task, tmp_path, monkeypatch, with_context, completion +): + """Join the actual collector/backend/scorer, not just each seam in isolation.""" + from examples.coding_audit.hf import HFTaskBackend + + m = SimpleNamespace(**runpy.run_path(str(Path(__file__).with_name("test_hf.py")))) + api, inference = m.API(), m.InferenceFactory() + factory = m.Factory(api) + original = m.FakeSandbox.run + + def run(sandbox, argv, **kwargs): + if sandbox.image == m.VERIFIER: + sandbox.factory.reward = ( + b"1" if sandbox.files.data.get("/app/answer.txt") == b"42\n" else b"0" + ) + else: + command = " ".join(argv) + sandbox.factory.submitted = ( + b"42\n" if "42" in command else b"41\n" if "41" in command else None + ) + return original(sandbox, argv, **kwargs) + + monkeypatch.setattr(m.FakeSandbox, "run", run) + config = configuration() + context_path = "environment/source/pkg/api.py" + context = b"def public_answer():\n return 42\n" + if with_context: + source = task / context_path + source.parent.mkdir(parents=True) + source.write_bytes(context) + config.setdefault("generator", dict(config["target"]))["context_paths"] = [ + context_path + ] + config["execution"].update( + agent_image=m.AGENT, verifier_image=m.VERIFIER, namespace="owner" + ) + completions = [m.response("done"), m.response("done")] + if completion != "stop": + for reply in completions: + choice = reply["choices"][0] + choice["finish_reason"] = "tool_calls" + choice["message"]["content"] = None + if completion == "missing_calls": + del choice["message"]["tool_calls"] + elif completion == "empty_calls": + choice["message"]["tool_calls"] = [] + inference.responses = [ + m.response('printf "42\\n" > /app/answer.txt'), + m.response(None, calls=[m.terminal('printf "41\\n" > /app/answer.txt')]), + completions[0], + m.response(None, calls=[m.terminal('printf "42\\n" > /app/answer.txt')]), + completions[1], + ] + backend = HFTaskBackend( + task.resolve(), + config["execution"], + token="execution-only-test-token", + inference_token="inference-only-test-token", + api=api, + sandbox_factory=factory, + inference_factory=inference, + ) + output = tmp_path / "audit" + report = run_coding_validation(task, config, output, backend=backend) + assert report["accepted"] is (completion == "stop"), report + if completion == "stop": + assert report["checks"]["target_difficulty"]["solve_rate"] == 0.5 + else: + assert report["checks"]["target_difficulty"]["status"] == "error" + assert len(inference.requests) == 5 + evidence_path = output / "evidence.json" + evidence = json.loads(evidence_path.read_text()) + if completion != "stop": + targets = [row for row in evidence["trials"] if row["role"] == "target"] + assert len(targets) == 2 + assert all( + row["error"] == "invalid terminal function calls" + and row["cleanup_verified"] + and "grade_sandbox_id" not in row + and row.get("reward") is None + for row in targets + ) + generated = evidence["generation"][0] + if with_context: + assert generated["candidate_context"] == [ + { + "path": context_path, + "runtime_path": "pkg/api.py", + "size": len(context), + "sha256": hashlib.sha256(context).hexdigest(), + } + ] + assert context.decode() in inference.requests[0]["messages"][1]["content"] + else: + assert "candidate_context" not in generated + assert all( + request["messages"][1]["content"] == (task / "instruction.md").read_text() + for request in inference.requests[1:] + ) + checksums = dict( + line.split(" ", 1)[::-1] + for line in (output / "SHA256SUMS").read_text().splitlines() + ) + assert ( + checksums["evidence.json"] + == hashlib.sha256(evidence_path.read_bytes()).hexdigest() + ) + assert factory.created + assert all(stage == "CANCELED" for stage in api.stages.values()) + + +def test_grader_exception_keeps_sanitized_sha_bound_logs(task, tmp_path, monkeypatch): + from examples.coding_audit.hf import HFTaskBackend + + m = SimpleNamespace(**runpy.run_path(str(Path(__file__).with_name("test_hf.py")))) + api, inference = m.API(), m.InferenceFactory() + factory = m.Factory(api) + original = m.FakeSandbox.run + secret = "execution-only-test-token" + + def run(sandbox, argv, **kwargs): + if sandbox.image == m.VERIFIER: + kwargs["on_stdout"]("APT: partial output 雪 " + secret) + raise RuntimeError("grader RPC failed: " + secret) + return original(sandbox, argv, **kwargs) + + monkeypatch.setattr(m.FakeSandbox, "run", run) + config = configuration() + config["execution"].update( + agent_image=m.AGENT, verifier_image=m.VERIFIER, namespace="owner" + ) + inference.responses = [m.response('printf "42\\n" > /app/answer.txt')] + backend = HFTaskBackend( + task.resolve(), + config["execution"], + token=secret, + inference_token="inference-only-test-token", + api=api, + sandbox_factory=factory, + inference_factory=inference, + ) + output = tmp_path / "audit" + report = run_coding_validation(task, config, output, backend=backend) + assert not report["accepted"] + evidence = json.loads((output / "evidence.json").read_text()) + assert evidence["target_not_run_reason"] + for row in evidence["trials"]: + assert row["error"] == "HF backend operation failed (RuntimeError)" + assert row["reward"] is None and row["cleanup_verified"] + artifact = row["grader_transcript"] + data = (output / artifact["path"]).read_bytes() + assert hashlib.sha256(data).hexdigest() == artifact["sha256"] + trace = json.loads(data)[0] + assert trace["output"] == "APT: partial output 雪 [REDACTED]" + assert trace["exit_code"] is None and not trace["output_truncated"] + for path in output.rglob("*"): + if path.is_file(): + assert secret.encode() not in path.read_bytes() + assert all(stage == "CANCELED" for stage in api.stages.values()) + + +@pytest.mark.parametrize("callbacks", ["single", "separate", "split_secret"]) +@pytest.mark.parametrize("intersecting", [False, True]) +def test_grader_overflow_discards_output_in_sha_bound_artifact( + task, tmp_path, monkeypatch, callbacks, intersecting +): + from examples.coding_audit.hf import HFTaskBackend + + m = SimpleNamespace(**runpy.run_path(str(Path(__file__).with_name("test_hf.py")))) + api, inference = m.API(), m.InferenceFactory() + factory = m.Factory(api) + original = m.FakeSandbox.run + execution_secret = ( + "hf_SYNTHETIC_EXECUTION_TOKEN_REVIEW_ONLYh" + if intersecting + else "hf_SYNTHETIC_EXECUTION_TOKEN_FOR_REVIEW_ONLY" + ) + inference_secret = "hf_SYNTHETIC_INFERENCE_TOKEN_REVIEW_ONLY" + secret = inference_secret if intersecting else execution_secret + cap, retained = 1048576, 28 + prefix = "雪" + "A" * (cap - len("雪".encode()) - retained) + if intersecting: + prefix = prefix[: -len(execution_secret[:-1])] + execution_secret[:-1] + assert execution_secret in prefix + secret + + def run(sandbox, argv, **kwargs): + if sandbox.image == m.VERIFIER: + if callbacks == "single": + kwargs["on_stdout"](prefix + secret) + else: + split = 7 if callbacks == "split_secret" else 0 + kwargs["on_stdout"](prefix + secret[:split]) + kwargs["on_stderr"](secret[split:]) + pytest.fail("grader output overflow was not rejected") + return original(sandbox, argv, **kwargs) + + monkeypatch.setattr(m.FakeSandbox, "run", run) + config = configuration() + config["execution"].update( + agent_image=m.AGENT, + verifier_image=m.VERIFIER, + namespace="owner", + max_tool_output_bytes=cap, + ) + inference.responses = [m.response('printf "42\\\\n" > /app/answer.txt')] + backend = HFTaskBackend( + task.resolve(), + config["execution"], + token=execution_secret, + inference_token=inference_secret, + api=api, + sandbox_factory=factory, + inference_factory=inference, + ) + output = tmp_path / "audit" + report = run_coding_validation(task, config, output, backend=backend) + assert not report["accepted"] + evidence = json.loads((output / "evidence.json").read_text()) + assert evidence["target_not_run_reason"] and evidence["trials"] + for row in evidence["trials"]: + assert row["error"] == "terminal output exceeded byte limit" + assert row["reward"] is None and row["cleanup_verified"] + assert "raw_reward" not in row + artifact = row["grader_transcript"] + data = (output / artifact["path"]).read_bytes() + assert hashlib.sha256(data).hexdigest() == artifact["sha256"] + assert json.loads(data) == [ + { + "exit_code": None, + "output": "", + "output_truncated": True, + } + ] + assert len(json.loads(data)[0]["output"].encode()) <= cap + for line in (output / "SHA256SUMS").read_text().splitlines(): + digest, name = line.split(" ", 1) + data = (output / name).read_bytes() + assert hashlib.sha256(data).hexdigest() == digest + assert secret[:retained].encode() not in data + assert execution_secret[:-1].encode() not in data + assert api.stages and all(stage == "CANCELED" for stage in api.stages.values()) + + +@pytest.mark.parametrize("value", [False, True]) +def test_explicit_candidate_thinking_control_preserves_target(value): + raw = configuration() + raw["generator"] = {**raw["target"], "enable_thinking": value} + normalized = normalize_coding_config(raw) + assert normalized["generator"]["enable_thinking"] is value + assert "enable_thinking" not in normalized["target"] + + +@pytest.mark.parametrize("value", [0, 1, "false", None]) +def test_thinking_configuration_rejects_non_boolean(value): + raw = configuration() + raw["generator"] = {**raw["target"], "enable_thinking": value} + with pytest.raises(ValueError, match="enable_thinking"): + normalize_coding_config(raw) + + +@pytest.mark.parametrize("count", [0, 1, 32]) +def test_generator_context_is_optional_explicit_and_preserves_target(count): + raw = configuration() + paths = [f"environment/source/pkg/api-{index}.py" for index in range(count)] + raw["generator"] = {**raw["target"], "context_paths": paths} + normalized = normalize_coding_config(raw) + assert normalized["generator"]["context_paths"] == paths + assert "context_paths" not in normalized["target"] + assert "context_paths" not in normalize_coding_config(configuration())["generator"] + + +@pytest.mark.parametrize( + "paths", + [ + "environment/source/api.py", + None, + [1], + ["environment/source/api.py"] * 2, + [f"environment/source/{index}.py" for index in range(33)], + ["environment/source"], + ["environment/source/api.py/"], + ["/environment/source/api.py"], + ["environment/source/../tests/test.sh"], + ["environment/source//api.py"], + ["environment/source/./api.py"], + ["environment/source/api\x00.py"], + ["environment/source/a\nb.py"], + ["environment/source/a\\b.py"], + ["environment/source/a:b.py"], + ["tests/test.sh"], + ["solution/solve.sh"], + ["validation/spec.json"], + ["environment/source/tests/test.py"], + ["environment/source/solution/solve.sh"], + ["environment/source/validation/spec.json"], + ["environment/source/.hidden/api.py"], + ["environment/source/.env"], + ["environment/source/secrets/token"], + ["environment/source/reference/solve.py"], + ["environment/source/spec/task.md"], + ], +) +def test_generator_context_rejects_unsafe_paths(paths): + raw = configuration() + raw["generator"] = {**raw["target"], "context_paths": paths} + with pytest.raises(ValueError, match="context_paths"): + normalize_coding_config(raw) + + +def test_target_cannot_declare_context_paths(): + raw = configuration() + raw["target"]["context_paths"] = [] + with pytest.raises(ValueError, match="target.context_paths"): + normalize_coding_config(raw) diff --git a/examples/coding_audit/test_scoring.py b/examples/coding_audit/test_scoring.py new file mode 100644 index 000000000..a30cd8746 --- /dev/null +++ b/examples/coding_audit/test_scoring.py @@ -0,0 +1,575 @@ +"""Admission tests for pure coding-audit evidence scoring; no execution backends.""" + +import copy +import json + +import pytest +from examples.coding_audit.scoring import assess_coding_audit, validate_audit_config + +TASK = "a" * 64 +VERIFIER = "b" * 64 +IMAGE = "registry/agent@sha256:" + "c" * 64 +VERIFIER_IMAGE = "registry/verifier@sha256:" + "d" * 64 +TARGET = {"model": "Qwen/Qwen3.8-27B", "revision": "e" * 40, "provider": "hf-inference"} +CONFIG = {"required_negative_cases": ["wrong_answer", "empty_answer"]} + + +def trial(role, case_id, reward, number, submission_digest=None): + result = { + "role": role, + "case_id": case_id, + "task_digest": TASK, + "verifier_digest": VERIFIER, + "image_ref": IMAGE, + "verifier_image_ref": VERIFIER_IMAGE, + "submission_digest": submission_digest or "f" * 64, + "sandbox_id": f"agent-{number}", + "grade_sandbox_id": f"grade-{number}", + "reward": reward, + "cleanup_verified": True, + "error": None, + } + if role == "target": + result.update(TARGET) + result.update( + latency_seconds=2.5, usage={"input_tokens": 100, "output_tokens": 20} + ) + return result + + +@pytest.fixture +def evidence(): + return { + "task_digest": TASK, + "verifier_digest": VERIFIER, + "agent_image": IMAGE, + "verifier_image": VERIFIER_IMAGE, + "target": dict(TARGET), + "references": [ + { + "case_id": "oracle", + "submission_digest": "f" * 64, + "origin": "provided", + } + ], + "trials": [ + trial("nop", "nop", 0, 0), + trial("reference", "oracle", 1, 1), + trial("reference", "oracle", 1, 2), + trial("negative", "wrong_answer", 0, 3), + trial("negative", "empty_answer", 0, 4), + trial("target", "attempt-1", 0, 5), + trial("target", "attempt-2", 1, 6), + ], + "leakage_probes": [ + {"name": name, "passed": True, "error": None} + for name in ( + "private_paths_absent", + "fresh_state_absent", + "canary_not_in_transcript", + ) + ], + } + + +def test_complete_audit_reports_bounded_measurements_without_mutating_inputs(evidence): + original = copy.deepcopy(evidence) + report = assess_coding_audit(CONFIG, evidence) + assert report["profile"] == "harbor-coding-audit-v1" + assert report["accepted"] is True + assert report["verdict"] == "pass" + assert not report["rfc008_complete"] + assert not report["trainability_established"] + assert {check["status"] for check in report["checks"].values()} == {"pass"} + target = report["checks"]["target_difficulty"] + assert target["completed"] == 2 + assert target["failed"] == target["missing"] == 0 + assert target["solve_rate"] == 0.5 + assert target["solve_rate_wilson95"] == pytest.approx([0.0945312057, 0.9054687943]) + assert target["reward_variance"] == 0.25 + assert target["mixed_success"] is True + assert target["difficulty"] == "mixed" + assert target["latency_seconds_total"] == 5 + assert target["usage_totals"] == {"input_tokens": 200, "output_tokens": 40} + assert all(value == 0 for value in target["telemetry_missing"].values()) + assert "bounded probes" in report["checks"]["leakage"]["scope"] + assert evidence == original + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize("reward,difficulty", [(0, "none_solved"), (1, "all_solved")]) +def test_uniform_target_rewards_are_valid_difficulty_measurements( + evidence, reward, difficulty +): + for record in evidence["trials"][-2:]: + record["reward"] = reward + report = assess_coding_audit(CONFIG, evidence) + assert report["accepted"] + target = report["checks"]["target_difficulty"] + assert target["difficulty"] == difficulty + assert target["mixed_success"] is False + assert target["reward_variance"] == 0 + assert target["solve_rate"] == reward + + +def test_target_infrastructure_error_is_missing_evidence_not_a_zero_reward(evidence): + failed = evidence["trials"][-2] + failed.update(error="provider timeout", reward=None) + report = assess_coding_audit(CONFIG, evidence) + assert not report["accepted"] + target = report["checks"]["target_difficulty"] + assert target["status"] == "error" + assert target["attempted"] == 2 + assert target["completed"] == target["failed"] == target["missing"] == 1 + assert target["infrastructure_errors"] == 1 + assert target["solve_rate"] == 1 + assert target["reward_variance"] == 0 + assert target["usage_totals"]["input_tokens"] == 200 + + +def test_missing_target_attempt_is_skip(evidence): + evidence["trials"].pop() + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "skip" + assert report["checks"]["target_difficulty"]["missing"] == 1 + + +def test_configured_repeat_and_attempt_counts_are_honored(evidence): + config = CONFIG | {"reference_repeats": 3, "target_attempts": 3} + report = assess_coding_audit(config, evidence) + assert report["checks"]["references"]["status"] == "skip" + assert report["checks"]["target_difficulty"]["missing"] == 1 + evidence["trials"].extend( + [trial("reference", "oracle", 1, 10), trial("target", "attempt-3", 0, 11)] + ) + assert assess_coding_audit(config, evidence)["accepted"] + + +@pytest.mark.parametrize( + "duplicate", ["record", "case_id", "sandbox_id", "grade_sandbox_id", "cross_role"] +) +def test_duplicate_target_attempts_never_count_as_coverage(evidence, duplicate): + first, second = evidence["trials"][-2:] + if duplicate == "record": + evidence["trials"][-1] = dict(first) + elif duplicate == "cross_role": + second["sandbox_id"] = evidence["trials"][1]["grade_sandbox_id"] + else: + second[duplicate] = first[duplicate] + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "error" + assert report["checks"]["target_difficulty"]["completed"] < 2 + + +def test_no_target_data_reports_null_metrics(evidence): + evidence["trials"] = [ + record for record in evidence["trials"] if record["role"] != "target" + ] + report = assess_coding_audit(CONFIG, evidence) + target = report["checks"]["target_difficulty"] + assert target["status"] == "skip" + for name in ( + "solve_rate", + "solve_rate_wilson95", + "reward_variance", + "mixed_success", + ): + assert target[name] is None + assert target["usage_totals"] == {"input_tokens": None, "output_tokens": None} + assert target["difficulty"] == "unmeasured" + + +def test_missing_telemetry_is_explicit_and_does_not_invalidate_reward(evidence): + for record in evidence["trials"][-2:]: + del record["usage"] + del record["latency_seconds"] + report = assess_coding_audit(CONFIG, evidence) + assert report["accepted"] + target = report["checks"]["target_difficulty"] + assert target["telemetry_missing"] == { + "latency_seconds": 2, + "input_tokens": 2, + "output_tokens": 2, + } + assert target["latency_seconds_total"] is None + + +@pytest.mark.parametrize( + "field,value", + [ + ("latency_seconds", -1), + ("latency_seconds", None), + ("latency_seconds", True), + ("latency_seconds", float("inf")), + ("usage", None), + ("usage", {"input_tokens": True, "output_tokens": -5}), + ], +) +def test_supplied_malformed_telemetry_is_error_not_missing(evidence, field, value): + evidence["trials"][-1][field] = value + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "error" + target = report["checks"]["target_difficulty"] + assert target["status"] == "error" + assert target["telemetry_errors"] + assert all(value == 0 for value in target["telemetry_missing"].values()) + + +def test_latency_total_overflow_returns_error_report(evidence): + evidence["trials"].append(trial("target", "attempt-3", 1, 10)) + for record, seconds in zip(evidence["trials"][-3:], [10**308, 10**308, 0.0]): + record["latency_seconds"] = seconds + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "error" + assert report["checks"]["target_difficulty"]["latency_seconds_total"] is None + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize( + "reward", + [True, False, float("nan"), float("inf"), -float("inf"), None, "1", -0.01, 1.01], +) +@pytest.mark.parametrize("index", [0, 1, 3, 5]) +def test_invalid_rewards_block_all_roles(evidence, reward, index): + evidence["trials"][index]["reward"] = reward + report = assess_coding_audit(CONFIG, evidence) + assert not report["accepted"] + assert report["checks"]["reliability"]["status"] == "error" + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize("cleanup", [False, 1, "true", None]) +def test_cleanup_requires_literal_true(evidence, cleanup): + evidence["trials"][1]["cleanup_verified"] = cleanup + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +@pytest.mark.parametrize( + "field", + [ + "error", + "cleanup_verified", + "sandbox_id", + "grade_sandbox_id", + "submission_digest", + ], +) +def test_missing_trial_provenance_never_passes(evidence, field): + del evidence["trials"][1][field] + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +@pytest.mark.parametrize("field", ["case_id", "submission_digest"]) +@pytest.mark.parametrize("value", [[], {}, True]) +def test_malformed_reference_group_keys_return_error_without_crashing( + evidence, field, value +): + evidence["trials"][1][field] = value + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +def test_sha256_prefix_does_not_split_identical_reference_replays(evidence): + evidence["references"][0]["submission_digest"] = "sha256:" + "f" * 64 + evidence["trials"][1]["submission_digest"] = "sha256:" + "f" * 64 + evidence["trials"][0]["task_digest"] = "sha256:" + TASK + evidence["trials"][3]["verifier_digest"] = "sha256:" + VERIFIER + assert assess_coding_audit(CONFIG, evidence)["accepted"] + + +def test_unrepresentable_reward_variance_blocks_acceptance_without_nonfinite_json( + evidence, +): + config = CONFIG | {"reward_max": 1e308} + for record in evidence["trials"]: + if record["reward"] == 1: + record["reward"] = 1e308 + report = assess_coding_audit(config, evidence) + assert report["verdict"] == "error" + assert report["checks"]["target_difficulty"]["reward_variance"] is None + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize("index", [0, 1, 3, 5]) +@pytest.mark.parametrize( + "field,value", + [ + ("task_digest", "0" * 64), + ("verifier_digest", "1" * 64), + ("image_ref", "other@sha256:" + "2" * 64), + ("verifier_image_ref", "other@sha256:" + "3" * 64), + ], +) +def test_every_trial_binds_to_expected_task_verifier_and_images( + evidence, index, field, value +): + evidence["trials"][index][field] = value + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "error" + assert any( + "binding mismatch" in reason + for reason in report["checks"]["reliability"]["reasons"] + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("task_digest", "a" * 63), + ("verifier_digest", "b" * 65), + ("agent_image", "agent:latest"), + ("verifier_image", "sha256:abc"), + ], +) +def test_common_binding_requires_full_digests_and_immutable_images( + evidence, field, value +): + evidence[field] = value + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +@pytest.mark.parametrize( + "field,value", + [("model", "other/model"), ("revision", "a" * 40), ("provider", "other-provider")], +) +def test_target_identity_is_bound(evidence, field, value): + evidence["trials"][-1][field] = value + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["target_difficulty"]["status"] == "error" + assert report["checks"]["target_difficulty"]["completed"] == 1 + + +@pytest.mark.parametrize("revision", ["main", "abc", "a" * 39, None]) +def test_common_target_revision_must_be_immutable(evidence, revision): + evidence["target"]["revision"] = revision + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +def test_reference_replay_drift_blocks_acceptance(evidence): + evidence["trials"][2]["reward"] = 0 + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["references"]["status"] == "fail" + assert report["checks"]["reliability"]["status"] == "fail" + + +def test_separate_generated_submissions_are_not_deterministic_replays(evidence): + evidence["references"] = [ + { + "case_id": "oracle", + "submission_digest": digest * 64, + "origin": "generated", + "candidate_script_digest": digest * 64, + } + for digest in ("0", "1") + ] + evidence["trials"][1]["submission_digest"] = "0" * 64 + evidence["trials"][2]["submission_digest"] = "1" * 64 + report = assess_coding_audit(CONFIG, evidence) + assert report["verdict"] == "skip" + assert report["checks"]["references"]["validated"] == [] + assert report["checks"]["reliability"]["status"] == "skip" + + +def test_clean_failed_generated_candidate_is_excluded_but_must_execute(evidence): + evidence["references"].append( + { + "case_id": "generated", + "submission_digest": "0" * 64, + "origin": "generated", + "candidate_script_digest": "1" * 64, + } + ) + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["references"]["status"] == "skip" + evidence["trials"].append(trial("reference", "generated", 0, 10, "0" * 64)) + report = assess_coding_audit(CONFIG, evidence) + assert report["accepted"] + assert report["checks"]["references"]["excluded"] == [ + {"case_id": "generated", "submission_digest": "0" * 64} + ] + evidence["trials"][-1]["error"] = "execution failed" + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +def test_generated_reference_can_be_validated_without_provided_oracle(evidence): + evidence["references"][0].update( + origin="generated", candidate_script_digest="1" * 64 + ) + assert assess_coding_audit(CONFIG, evidence)["accepted"] + del evidence["references"][0]["candidate_script_digest"] + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" + + +@pytest.mark.parametrize("kind", ["nop", "wrong_answer", "contradictory_negative"]) +def test_verifier_must_reject_nop_and_every_observed_known_negative(evidence, kind): + if kind == "nop": + evidence["trials"][0]["reward"] = 1 + elif kind == "wrong_answer": + evidence["trials"][3]["reward"] = 1 + else: + evidence["trials"].append(trial("negative", "wrong_answer", 1, 10)) + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["verifier_rejection"]["status"] == "fail" + + +def test_generated_negative_labels_do_not_establish_trusted_coverage(evidence): + evidence["trials"][3]["case_id"] = "model_says_wrong" + evidence["required_negative_cases"] = ["model_says_wrong", "empty_answer"] + evidence["trials"][3]["reviewed"] = True + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["verifier_rejection"]["status"] == "skip" + assert report["checks"]["verifier_rejection"]["covered_negative_cases"] == [ + "empty_answer" + ] + + +@pytest.mark.parametrize("name", ["nop", "reference", "negative"]) +def test_missing_role_is_incomplete(evidence, name): + evidence["trials"] = [ + record for record in evidence["trials"] if record["role"] != name + ] + report = assess_coding_audit(CONFIG, evidence) + assert not report["accepted"] + assert report["verdict"] == "skip" + + +def test_floor_margin_is_required(evidence): + config = CONFIG | {"floor_margin": 0.95, "oracle_tolerance": 0.1} + evidence["trials"][0]["reward"] = 0.1 + report = assess_coding_audit(config, evidence) + assert report["checks"]["references"]["status"] == "fail" + + +def test_floor_gap_does_not_round_away_tiny_nop_reward(evidence): + config = CONFIG | {"oracle_tolerance": 1e-300, "floor_margin": 1} + evidence["trials"][0]["reward"] = 1e-300 + report = assess_coding_audit(config, evidence) + assert report["checks"]["verifier_rejection"]["status"] == "pass" + assert report["checks"]["references"]["status"] == "fail" + assert not report["accepted"] + + +def test_margin_must_fit_exact_reward_range(): + config = CONFIG | { + "reward_min": 1e-300, + "wrong_answer_max": 1e-300, + "floor_margin": 1, + } + with pytest.raises(ValueError): + validate_audit_config(config) + + +def test_zero_float_tolerance_preserves_large_integer_reference_and_solve_threshold( + evidence, +): + config = CONFIG | {"reward_max": 2**53 + 1, "oracle_tolerance": 0.0} + for record in evidence["trials"]: + if record["role"] in ("reference", "target"): + record["reward"] = 2**53 + report = assess_coding_audit(config, evidence) + assert report["checks"]["references"]["status"] == "fail" + target = report["checks"]["target_difficulty"] + assert target["solve_count"] == 0 + assert target["difficulty"] == "none_solved" + assert not report["accepted"] + + +def test_floor_separation_uses_decimal_values_without_float_subtraction_error(evidence): + config = CONFIG | { + "reward_min": 0.7, + "reward_max": 1, + "oracle_tolerance": 0.1, + "wrong_answer_max": 0.8, + "floor_margin": 0.1, + } + for record in evidence["trials"]: + record["reward"] = 0.9 if record["role"] == "reference" else 0.8 + assert assess_coding_audit(config, evidence)["accepted"] + + +@pytest.mark.parametrize( + "change,status", + [ + ("missing", "skip"), + ("unknown", "skip"), + ("duplicate", "error"), + ("false", "fail"), + ("truthy", "error"), + ("error", "error"), + ("missing_error", "error"), + ], +) +def test_leakage_requires_explicit_named_successful_probes(evidence, change, status): + if change == "missing": + evidence["leakage_probes"].pop() + elif change == "unknown": + evidence["leakage_probes"][-1]["name"] = "unknown_probe" + elif change == "duplicate": + evidence["leakage_probes"].append(dict(evidence["leakage_probes"][0])) + elif change == "false": + evidence["leakage_probes"][0]["passed"] = False + elif change == "truthy": + evidence["leakage_probes"][0]["passed"] = 1 + elif change == "error": + evidence["leakage_probes"][0]["error"] = "probe failed to execute" + else: + del evidence["leakage_probes"][0]["error"] + report = assess_coding_audit(CONFIG, evidence) + assert report["checks"]["leakage"]["status"] == status + assert not report["accepted"] + + +def test_unknown_probe_cannot_replace_required_names_but_can_be_diagnostic(evidence): + evidence["leakage_probes"].append({"name": "extra", "passed": True, "error": None}) + report = assess_coding_audit(CONFIG, evidence) + assert report["accepted"] + assert "extra" in report["checks"]["leakage"]["observed_probes"] + + +@pytest.mark.parametrize( + "config", + [ + {}, + {"reference_repeats": 1}, + {"reference_repeats": True}, + {"target_attempts": 1}, + {"reward_min": True}, + {"reward_max": float("nan")}, + {"oracle_tolerance": 1}, + {"floor_margin": 0}, + {"floor_margin": float("inf")}, + {"wrong_answer_max": 1}, + {"required_negative_cases": []}, + {"required_negative_cases": ["a", "a"]}, + {"required_negative_cases": [1]}, + {"unknown": True}, + ], +) +def test_invalid_config_raises_and_assessment_returns_error(evidence, config): + config = config if not config else CONFIG | config + with pytest.raises(ValueError): + validate_audit_config(config) + report = assess_coding_audit(config, evidence) + assert report["verdict"] == "error" + assert not report["accepted"] + + +def test_empty_evidence_cannot_be_accepted(): + report = assess_coding_audit(CONFIG, {}) + assert not report["accepted"] + assert report["checks"]["leakage"]["status"] == "skip" + assert report["checks"]["references"]["status"] == "skip" + + +@pytest.mark.parametrize( + "field,value", + [ + ("trials", {}), + ("references", {}), + ("leakage_probes", {}), + ("trials", [None]), + ("references", [None]), + ], +) +def test_malformed_containers_return_error(evidence, field, value): + evidence[field] = value + assert assess_coding_audit(CONFIG, evidence)["verdict"] == "error" From 1a165f35979447c796cd75e96bfa86c3d2f58ea8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 10 Nov 2026 19:08:39 +0000 Subject: [PATCH 23/40] fix(validation): preserve Harbor verifier timeout --- src/openenv/validation/harbor.py | 64 +++++++++---------- .../test_validation/test_harbor_validation.py | 19 ++++++ 2 files changed, 49 insertions(+), 34 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index de4b4c485..81dbec259 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -432,42 +432,38 @@ async def _trial( effective_config.pop("trial_name") record["trial_config_digest"] = _digest(effective_config) record["resources"] = contract.resources.model_dump() - async with asyncio.timeout(budget): - trial = await _create_trial(config) + trial = await _create_trial(config) - async def capture(_event): - if trial.agent_environment.session_id != project: - raise NativeEvidenceError( - "Harbor sandbox identity differs from the registered project" - ) - identity = inspector.capture(project, retained_tag, retain=not expected) - record.update(identity) - if expected and ( - identity["image_ref"] != expected["image_ref"] - or identity["configuration_digest"] - != expected["configuration_digest"] - ): - raise NativeEvidenceError( - "fresh native trial image or effective container configuration drifted" - ) - - async def capture_exit(_event): - # Harbor writes the marker only for a nonzero exit. Later sandbox - # log synchronization can overwrite it, so this reading is trusted. - if agent == "oracle": - marker = _read_output( - trial.paths.agent_dir / "exit-code.txt", trial.paths.trial_dir - ) - record["oracle_exit_observed_at_agent_end"] = True - record["oracle_exit_code"] = ( - int(marker) if marker is not None else 0 - ) - if marker is not None: - raw_files["oracle-exit-code.txt"] = marker + async def capture(_event): + if trial.agent_environment.session_id != project: + raise NativeEvidenceError( + "Harbor sandbox identity differs from the registered project" + ) + identity = inspector.capture(project, retained_tag, retain=not expected) + record.update(identity) + if expected and ( + identity["image_ref"] != expected["image_ref"] + or identity["configuration_digest"] != expected["configuration_digest"] + ): + raise NativeEvidenceError( + "fresh native trial image or effective container configuration drifted" + ) - trial.add_hook(TrialEvent.AGENT_START, capture) - trial.add_hook(TrialEvent.AGENT_END, capture_exit) - result = await trial.run() + async def capture_exit(_event): + # Harbor writes the marker only for a nonzero exit. Later sandbox + # log synchronization can overwrite it, so this reading is trusted. + if agent == "oracle": + marker = _read_output( + trial.paths.agent_dir / "exit-code.txt", trial.paths.trial_dir + ) + record["oracle_exit_observed_at_agent_end"] = True + record["oracle_exit_code"] = int(marker) if marker is not None else 0 + if marker is not None: + raw_files["oracle-exit-code.txt"] = marker + + trial.add_hook(TrialEvent.AGENT_START, capture) + trial.add_hook(TrialEvent.AGENT_END, capture_exit) + result = await trial.run() if source_digest(task_copy) != digest_before: raise NativeEvidenceError( "private execution task changed during validation" diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 02176029f..31937439c 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -269,6 +269,25 @@ def test_success_grades_semantic_checks_with_native_raw_evidence( assert (artifacts / "harbor-trials/nop/reward.json").read_bytes() == b'{"reward":0}' +def test_trial_does_not_share_phase_timeout(task, boundary, monkeypatch): + timeouts = [] + timeout = harbor.asyncio.timeout + + def record_timeout(delay): + timeouts.append(delay) + return timeout(delay) + + monkeypatch.setattr(harbor.asyncio, "timeout", record_timeout) + assert run_validation(task).verdict is Verdict.WARN + assert timeouts == [30, 30, 30] + assert all( + trial.config.agent.override_timeout_sec == 10 + and trial.config.agent.override_setup_timeout_sec == 10 + and trial.config.verifier.override_timeout_sec == 10 + for trial in boundary.trials + ) + + @pytest.mark.parametrize( "raw", [ From 751eaf70ddf57dd33d6c23a89ebd81148fd5ce77 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Wed, 11 Nov 2026 15:21:40 +0100 Subject: [PATCH 24/40] fix(validation): reject strings in numeric Harbor declarations Check every reward field and numeric resource field before coercion. Preserve string GPU types and cover the precision bypass. --- src/openenv/validation/parsers/harbor_toml.py | 11 ++-- tests/test_validation/test_harbor_contract.py | 61 +++++++++++++++++++ 2 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 tests/test_validation/test_harbor_contract.py diff --git a/src/openenv/validation/parsers/harbor_toml.py b/src/openenv/validation/parsers/harbor_toml.py index cc905dd94..926bdd520 100644 --- a/src/openenv/validation/parsers/harbor_toml.py +++ b/src/openenv/validation/parsers/harbor_toml.py @@ -80,19 +80,22 @@ def read_harbor_contract(package_root: Path) -> HarborContract: raise ValueError( "reward range, oracle_tolerance and floor_margin must be explicit" ) - for section in ("reward", "resources"): - pending = [raw.get(section)] + for section, fields in ( + ("reward", RewardDeclaration.model_fields), + ("resources", ("cpu", "memory_mb", "disk_mb", "episode_timeout_s", "gpus")), + ): + pending = [raw.get(section, {}).get(field) for field in fields] while pending: item = pending.pop() if isinstance(item, dict): pending.extend(item.values()) elif isinstance(item, list): pending.extend(item) - elif isinstance(item, bool) or ( + elif isinstance(item, (bool, str)) or ( isinstance(item, float) and not math.isfinite(item) ): raise ValueError( - "numeric declarations must be finite and nonboolean" + "numeric declarations must be finite, nonboolean JSON numbers" ) # Graders compare against the stored float, so it must keep the # declared value exactly: 0.100000000000000001 would become 0.1. diff --git a/tests/test_validation/test_harbor_contract.py b/tests/test_validation/test_harbor_contract.py new file mode 100644 index 000000000..abb9b72b9 --- /dev/null +++ b/tests/test_validation/test_harbor_contract.py @@ -0,0 +1,61 @@ +"""The trusted Harbor contract accepts explicit numeric JSON declarations.""" + +import json + +import pytest +from openenv.validation.manifest import ManifestError +from openenv.validation.parsers.harbor_toml import read_harbor_contract + + +@pytest.fixture +def contract_package(tmp_path): + (tmp_path / "validation").mkdir() + contract = { + "profile": "harbor-semantic-v1", + "reward_key": "reward", + "reward": {"range": [0, 1], "oracle_tolerance": 0, "floor_margin": 0.1}, + "resources": { + "cpu": 1, + "memory_mb": 512, + "disk_mb": 1024, + "episode_timeout_s": 10, + }, + "types": {"tags": ["swe"]}, + } + (tmp_path / "validation/openenv.json").write_text(json.dumps(contract)) + return tmp_path + + +@pytest.mark.parametrize( + "section,field,value", + [ + ("reward", "range", ["0", "1"]), + ("reward", "oracle_tolerance", "0.1"), + ("reward", "floor_margin", "0.100000000000000001"), + ("reward", "variance_tolerance", "0.01"), + ("resources", "cpu", "1"), + ("resources", "memory_mb", "512"), + ("resources", "disk_mb", "1024"), + ("resources", "episode_timeout_s", "10"), + ("resources", "gpus", "1"), + ], +) +def test_numeric_strings_are_rejected(contract_package, section, field, value): + path = contract_package / "validation/openenv.json" + contract = json.loads(path.read_text()) + contract[section][field] = value + path.write_text(json.dumps(contract)) + + with pytest.raises(ManifestError): + read_harbor_contract(contract_package) + + +def test_gpu_type_strings_are_preserved(contract_package): + path = contract_package / "validation/openenv.json" + contract = json.loads(path.read_text()) + contract["resources"].update(gpus=1, gpu_types=["A100", "H100"]) + path.write_text(json.dumps(contract)) + + loaded = read_harbor_contract(contract_package) + assert loaded.resources.gpu_types == ["A100", "H100"] + assert loaded.resources.gpus == 1 From 63ba6d1f449afbd64495c296b9a6a82aa508d4af Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Wed, 11 Nov 2026 15:22:59 +0100 Subject: [PATCH 25/40] test(validation): cover Harbor host failures over malformed rewards The binding fixes already classify all six teardown, cleanup and retained-image failure labels as host failures. Add regressions for teardown errors, unverified cleanup, reward artifact drift and unsafe artifacts overriding an earlier malformed reward. Removing the host classifications makes all four tests fail; restoring them passes these and both retained-image regressions. --- .../test_validation/test_harbor_validation.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 794b7a388..bb2edd5c3 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -57,6 +57,7 @@ def boundary(monkeypatch, task): trials=[], cleanup_ok=True, cleanup_fail_role=None, + teardown_defect=None, mutate=False, private_mutate=False, image_drift=False, @@ -151,6 +152,13 @@ def __init__(self, config): async def stop(self, *, delete): assert not delete + if state.teardown_defect == "error": + raise OSError("Harbor teardown failed") + if state.teardown_defect == "drift": + self.paths.reward_json_path.write_bytes(b'{"reward":0}') + if state.teardown_defect == "unsafe": + self.paths.reward_json_path.unlink() + self.paths.reward_json_path.symlink_to("outside-reward.json") async def start(self, *, force_build): if state.pending_compose: @@ -463,6 +471,28 @@ def test_cleanup_failure_blocks(task, boundary, tmp_path): ] +@pytest.mark.parametrize( + "defect,failure", + [ + ("error", "HarborTeardownError"), + ("cleanup", "CleanupNotVerified"), + ("drift", "RewardArtifactDrift"), + ("unsafe", "UnsafeRewardArtifact"), + ], +) +def test_host_failure_overrides_reward_error(task, boundary, tmp_path, defect, failure): + boundary.rewards["nop"] = b'{"reward":true}' + boundary.teardown_defect = defect + boundary.cleanup_ok = defect != "cleanup" + artifacts = tmp_path / "host-over-reward" + report = run_validation(task, artifacts_dir=artifacts) + assert result(report, "runtime.startup").status is CheckStatus.ERROR + assert result(report, "runtime.reward_well_formed").status is CheckStatus.SKIP + trial = json.loads((artifacts / "harbor-evidence.json").read_text())["trials"][0] + assert trial["failure"] == failure + assert trial["reward_error"] + + def test_oracle_nonzero_exit_fails_oracle_max_and_retains_marker( task, boundary, tmp_path ): From 3b24b79369f9c490c0ac4e6c65e8bc0b5fd2e26c Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Wed, 11 Nov 2026 15:28:32 +0100 Subject: [PATCH 26/40] fix(validation): keep the retained Harbor image only for unfailed runs Track the run-owned image binding before tagging so failures during capture can release it. Remove only that tag after any FAIL or ERROR, verify its image identity before removal, and disable parent-image pruning. A completion guard also releases the tag if grading or report writing is interrupted or raises. Publish retained_image_tag in runtime.startup.measured only for completed runs without failed or errored checks. Cover retention, semantic failure, startup error, interruption, reporting failure, partial capture and ownership refusal with fake Docker responses. The report schema already allows measured keys. --- src/openenv/validation/harbor.py | 627 ++++++++++-------- .../test_harbor_binding_inspection.py | 41 ++ .../test_validation/test_harbor_validation.py | 129 ++++ 3 files changed, 502 insertions(+), 295 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index 74ee2d52c..6418d4d8e 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -77,10 +77,11 @@ def _digest(value): class DockerInspector: - """Trusted host inspection and removal limited to exact run-owned projects.""" + """Trusted host inspection and removal limited to run-owned resources.""" def __init__(self): self._owned_volumes = {} + self._retained_image = None def _docker(self, *args): result = subprocess.run( @@ -119,6 +120,10 @@ def inventory(self, project): def capture(self, project, retained_tag, *, retain=True): inventory = self.inventory(project) + if retained_tag != "openenv-validation:" + project.split("-", 3)[2]: + raise NativeEvidenceError( + "retained image tag differs from its run-owned project" + ) if len(inventory["containers"]) != 1: raise NativeEvidenceError("Harbor requires exactly one main container") item = json.loads(self._docker("inspect", inventory["containers"][0]))[0] @@ -135,6 +140,7 @@ def capture(self, project, retained_tag, *, retain=True): "native container is not running with an immutable image" ) if retain: + self._retained_image = (retained_tag, item["Image"]) self._docker("image", "tag", item["Image"], retained_tag) retained = json.loads(self._docker("image", "inspect", retained_tag))[0] if (retain and retained["Id"] != item["Image"]) or retained["Os"] != "linux": @@ -237,6 +243,16 @@ def verify_image(self, tag, image_ref): image = json.loads(self._docker("image", "inspect", tag))[0] return image["Id"] == image_ref + def remove_image(self): + if self._retained_image is not None: + tag, image_ref = self._retained_image + if not self.verify_image(tag, image_ref): + raise NativeEvidenceError( + "retained image ownership changed before removal" + ) + self._docker("image", "rm", "--no-prune", tag) + self._retained_image = None + def _unsupported(task, contract): config = task.config @@ -804,321 +820,342 @@ def validate_harbor( installed = json.loads(direct_url) record["harbor"]["commit_id"] = installed.get("vcs_info", {}).get("commit_id") files = {} - attempted = False - problem = _unsupported(read_harbor_task(subject.root), contract) - if skip_build: - problem = "image build skipped by --skip-build" - if provider is not None: - problem = "native Harbor binding requires its own Docker Trial provider" - if any(result.status is not CheckStatus.PASS for result in results): - problem = "unmet dependency: static.manifest" - startup = _result( - "runtime.startup", - CheckStatus.SKIP, - f"unsupported by {HARBOR_PROFILE}: {problem}", - ) - reward_result = _result( - "runtime.reward_well_formed", - CheckStatus.SKIP, - "unmet dependency: native Harbor trials", - ) - determinism = _result( - "runtime.episode_determinism", - CheckStatus.SKIP, - "unmet dependency: native Harbor trials", - ) - evidence = None - checks = [] - semantic_graded = False - if ( - semantic - and subject.manifest.capabilities.oracle is None - and all(result.status is CheckStatus.PASS for result in results) - ): - # RFC 008: a missing oracle is a graded FAIL, never a SKIP. - semantic_graded = True - checks.append( - CheckResult( - check_id="semantic.oracle_max", - status=CheckStatus.FAIL, - evidence=["the task has no oracle (solution/solve.sh)"], - remediation=( - "Add solution/solve.sh that drives the task to a correct solution." - ), - duration_s=0, - ) + inspector = DockerInspector() + completed = False + try: + attempted = False + problem = _unsupported(read_harbor_task(subject.root), contract) + if skip_build: + problem = "image build skipped by --skip-build" + if provider is not None: + problem = "native Harbor binding requires its own Docker Trial provider" + if any(result.status is not CheckStatus.PASS for result in results): + problem = "unmet dependency: static.manifest" + startup = _result( + "runtime.startup", + CheckStatus.SKIP, + f"unsupported by {HARBOR_PROFILE}: {problem}", ) - if not problem: - if subject.manifest.capabilities.oracle is None: - startup = _result( - "runtime.startup", - CheckStatus.SKIP, - "native Harbor trials need solution/solve.sh", + reward_result = _result( + "runtime.reward_well_formed", + CheckStatus.SKIP, + "unmet dependency: native Harbor trials", + ) + determinism = _result( + "runtime.episode_determinism", + CheckStatus.SKIP, + "unmet dependency: native Harbor trials", + ) + evidence = None + checks = [] + semantic_graded = False + if ( + semantic + and subject.manifest.capabilities.oracle is None + and all(result.status is CheckStatus.PASS for result in results) + ): + # RFC 008: a missing oracle is a graded FAIL, never a SKIP. + semantic_graded = True + checks.append( + CheckResult( + check_id="semantic.oracle_max", + status=CheckStatus.FAIL, + evidence=["the task has no oracle (solution/solve.sh)"], + remediation=( + "Add solution/solve.sh that drives the task to a correct solution." + ), + duration_s=0, + ) ) - else: - attempted = True - try: - with tempfile.TemporaryDirectory( - prefix="openenv-harbor-validation-" - ) as temp: - daemon, trials, files = asyncio.run( - _collect( - subject.root, - contract, - Path(temp), - DockerInspector(), - digest_before, - canaries, - ) - ) - record.update({"daemon": daemon, "trials": trials}) - record["cleanup"] = { - "verified": bool(trials) - and all(trial["cleanup"]["verified"] for trial in trials) - } - broken = [ - trial - for trial in trials - if "failure" in trial and not _reward_only(trial) - ] - malformed = any(_reward_only(trial) for trial in trials) - containers = [trial.get("container_id") for trial in trials] - if ( - broken - or (not malformed and len(trials) != 3 + len(canaries)) - or len(set(containers)) != len(containers) - ): - subject_failures = broken and all( - trial.get("subject_error") - and trial["failure"] not in _HOST_FAILURES - for trial in broken - ) - startup = _result( - "runtime.startup", - CheckStatus.FAIL if subject_failures else CheckStatus.ERROR, - "native Harbor subject failed during " - + ", ".join( - sorted({trial["subject_error"] for trial in broken}) - ) - if subject_failures - else "native Harbor trial, binding or cleanup failed", - { - "failures": [ - trial.get("reason", trial.get("failure")) - for trial in broken - ] - }, - ) - if malformed: - reward_result = _result( - "runtime.reward_well_formed", - CheckStatus.FAIL, - "native verifier reward violates the explicit contract", + if not problem: + if subject.manifest.capabilities.oracle is None: + startup = _result( + "runtime.startup", + CheckStatus.SKIP, + "native Harbor trials need solution/solve.sh", + ) + else: + attempted = True + try: + with tempfile.TemporaryDirectory( + prefix="openenv-harbor-validation-" + ) as temp: + daemon, trials, files = asyncio.run( + _collect( + subject.root, + contract, + Path(temp), + inspector, + digest_before, + canaries, + ) ) - else: - record.update( - { - "image_ref": trials[0]["image_ref"], - "retained_image_tag": trials[0]["retained_image_tag"], - } - ) - startup = _result( - "runtime.startup", - CheckStatus.PASS, - "fresh native Harbor trials ran from one immutable image " - "with verified cleanup", - { - "profile": HARBOR_PROFILE, - "image_ref": trials[0]["image_ref"], - "roles": [trial["role"] for trial in trials], - "task_effective_digest": digest_before, - }, - ) - if malformed: - reward_result = _result( - "runtime.reward_well_formed", - CheckStatus.FAIL, - "native verifier reward violates the explicit contract", + record.update({"daemon": daemon, "trials": trials}) + record["cleanup"] = { + "verified": bool(trials) + and all(trial["cleanup"]["verified"] for trial in trials) + } + broken = [ + trial + for trial in trials + if "failure" in trial and not _reward_only(trial) + ] + malformed = any(_reward_only(trial) for trial in trials) + containers = [trial.get("container_id") for trial in trials] + if ( + broken + or (not malformed and len(trials) != 3 + len(canaries)) + or len(set(containers)) != len(containers) + ): + subject_failures = broken and all( + trial.get("subject_error") + and trial["failure"] not in _HOST_FAILURES + for trial in broken ) - determinism = _result( - "runtime.episode_determinism", - CheckStatus.SKIP, - "unmet dependency: runtime.reward_well_formed", + startup = _result( + "runtime.startup", + CheckStatus.FAIL if subject_failures else CheckStatus.ERROR, + "native Harbor subject failed during " + + ", ".join( + sorted({trial["subject_error"] for trial in broken}) + ) + if subject_failures + else "native Harbor trial, binding or cleanup failed", + { + "failures": [ + trial.get("reason", trial.get("failure")) + for trial in broken + ] + }, ) + if malformed: + reward_result = _result( + "runtime.reward_well_formed", + CheckStatus.FAIL, + "native verifier reward violates the explicit contract", + ) else: - reward_result = _result( - "runtime.reward_well_formed", + record["image_ref"] = trials[0]["image_ref"] + startup = _result( + "runtime.startup", CheckStatus.PASS, - "every raw verifier reward map matches Harbor and " - "satisfies the declared key/range", + "fresh native Harbor trials ran from one immutable image " + "with verified cleanup", { "profile": HARBOR_PROFILE, - "reward_key": contract.reward_key, + "image_ref": trials[0]["image_ref"], + "roles": [trial["role"] for trial in trials], + "task_effective_digest": digest_before, }, ) - complete_baseline = ( - len(trials) >= 3 - and all( - "failure" not in trial and trial["cleanup"]["verified"] - for trial in trials[:3] - ) - and len(set(containers[:3])) == 3 - ) - if complete_baseline: - nop, oracle, replay, *canary_trials = trials - same = ( - oracle["rewards"] == replay["rewards"] - and oracle["oracle_exit_code"] == replay["oracle_exit_code"] + if malformed: + reward_result = _result( + "runtime.reward_well_formed", + CheckStatus.FAIL, + "native verifier reward violates the explicit contract", + ) + determinism = _result( + "runtime.episode_determinism", + CheckStatus.SKIP, + "unmet dependency: runtime.reward_well_formed", + ) + else: + reward_result = _result( + "runtime.reward_well_formed", + CheckStatus.PASS, + "every raw verifier reward map matches Harbor and " + "satisfies the declared key/range", + { + "profile": HARBOR_PROFILE, + "reward_key": contract.reward_key, + }, + ) + complete_baseline = ( + len(trials) >= 3 + and all( + "failure" not in trial and trial["cleanup"]["verified"] + for trial in trials[:3] + ) + and len(set(containers[:3])) == 3 ) - determinism = _result( - "runtime.episode_determinism", - CheckStatus.PASS if same else CheckStatus.FAIL, - "the oracle replay in a fresh container " - + ( - "reproduced every reward and the exit code" - if same - else "produced different rewards or a different exit code" - ), - { - "rewards": [ - json.loads(dumps(trial["rewards"])) + if complete_baseline: + nop, oracle, replay, *canary_trials = trials + same = ( + oracle["rewards"] == replay["rewards"] + and oracle["oracle_exit_code"] == replay["oracle_exit_code"] + ) + determinism = _result( + "runtime.episode_determinism", + CheckStatus.PASS if same else CheckStatus.FAIL, + "the oracle replay in a fresh container " + + ( + "reproduced every reward and the exit code" + if same + else "produced different rewards or a different exit code" + ), + { + "rewards": [ + json.loads(dumps(trial["rewards"])) + for trial in (oracle, replay) + ], + "exit_codes": [ + trial["oracle_exit_code"] + for trial in (oracle, replay) + ], + }, + ) + key = contract.reward_key + record.update( + { + "floor_reward": nop["rewards"][key], + "oracle_reward": oracle["rewards"][key], + } + ) + evidence = OracleEvidence( + floor=Fraction(nop["rewards"][key]), + oracle=tuple( + Fraction(trial["rewards"][key]) for trial in (oracle, replay) - ], - "exit_codes": [ + ), + oracle_exit_codes=tuple( trial["oracle_exit_code"] for trial in (oracle, replay) - ], - }, - ) - key = contract.reward_key - record.update( - { - "floor_reward": nop["rewards"][key], - "oracle_reward": oracle["rewards"][key], - } + ), + canaries=tuple( + ( + trial["task_overlay"]["solution/solve.sh"][ + "canary" + ], + Fraction(trial["rewards"][key]), + ) + for trial in canary_trials + if "failure" not in trial + and trial["cleanup"]["verified"] + ), + ) + except (Exception, KeyboardInterrupt) as exc: + record["failure"] = type(exc).__name__ + reason = f"native Harbor collection failed ({type(exc).__name__})" + if isinstance(exc, NativeEvidenceError): + record["reason"] = str(exc) + reason += ": " + str(exc) + startup = _result( + "runtime.startup", + CheckStatus.ERROR, + reason, ) - evidence = OracleEvidence( - floor=Fraction(nop["rewards"][key]), - oracle=tuple( - Fraction(trial["rewards"][key]) - for trial in (oracle, replay) - ), - oracle_exit_codes=tuple( - trial["oracle_exit_code"] for trial in (oracle, replay) - ), - canaries=tuple( - ( - trial["task_overlay"]["solution/solve.sh"]["canary"], - Fraction(trial["rewards"][key]), + if semantic: + registry = GraderRegistry() + for grader in ( + OracleMaxGrader(), + FloorGapGrader(), + CanaryFloorGrader(), + ): + registry.register(grader) + selected = registry.select(subject.manifest, max_level) + if evidence is not None and len(evidence.canaries) != len(canaries): + selected = [ + grader + for grader in selected + if grader.check_id != "semantic.canary_floor" + ] + checks.append( + _result( + "semantic.canary_floor", + CheckStatus.SKIP, + "unmet dependency: complete canary trials with verified cleanup", ) - for trial in canary_trials - if "failure" not in trial and trial["cleanup"]["verified"] - ), - ) - except (Exception, KeyboardInterrupt) as exc: - record["failure"] = type(exc).__name__ - reason = f"native Harbor collection failed ({type(exc).__name__})" - if isinstance(exc, NativeEvidenceError): - record["reason"] = str(exc) - reason += ": " + str(exc) - startup = _result( - "runtime.startup", - CheckStatus.ERROR, - reason, - ) - if semantic: - registry = GraderRegistry() - for grader in ( - OracleMaxGrader(), - FloorGapGrader(), - CanaryFloorGrader(), - ): - registry.register(grader) - selected = registry.select(subject.manifest, max_level) - if evidence is not None and len(evidence.canaries) != len(canaries): - selected = [ - grader - for grader in selected - if grader.check_id != "semantic.canary_floor" - ] - checks.append( - _result( - "semantic.canary_floor", - CheckStatus.SKIP, - "unmet dependency: complete canary trials with verified cleanup", + ) + checks.extend( + execute_graders( + selected, + replace(subject, oracle_evidence=evidence), + prior=[startup, reward_result, determinism], ) ) - checks.extend( - execute_graders( - selected, - replace(subject, oracle_evidence=evidence), - prior=[startup, reward_result, determinism], - ) - ) - try: - unchanged = source_digest(subject.root) == digest_before - except (ValueError, OSError): - unchanged = False - if not unchanged: - record["failure"] = "SourceDrift" - startup = _result( - "runtime.startup", - CheckStatus.ERROR, - "task source changed during native validation", - ) - reward_result, determinism, *checks = ( - _result( - check.check_id, - CheckStatus.SKIP, - "unmet dependency: unchanged native task source", + try: + unchanged = source_digest(subject.root) == digest_before + except (ValueError, OSError): + unchanged = False + if not unchanged: + record["failure"] = "SourceDrift" + startup = _result( + "runtime.startup", + CheckStatus.ERROR, + "task source changed during native validation", ) - for check in (reward_result, determinism, *checks) - ) - startup.duration_s = time.monotonic() - started - record["cleanup"].update( - {"required": attempted, "completed": record["cleanup"]["verified"]} - ) - results = [*results, startup, reward_result, determinism, *checks] - present = {result.check_id for result in results} - for entry in policy.entries_for_lane(Lane.LOCAL).values(): - if entry.level <= max_level and entry.check_id not in present: - results.append( + reward_result, determinism, *checks = ( _result( - entry.check_id, + check.check_id, CheckStatus.SKIP, - _skip_reason(entry, contract), + "unmet dependency: unchanged native task source", ) + for check in (reward_result, determinism, *checks) ) - levels = [Level.STATIC] - if attempted: - levels.append(Level.RUNTIME) - if semantic and (attempted or semantic_graded): - levels.append(Level.SEMANTIC) - report = ValidationReportV2( - report_schema_version="2", - target=str(subject.root) if report_target is None else report_target, - source_digest=digest_before, - signature=subject.manifest.signature, - manifest=subject.manifest, - policy_version=policy.policy_version, - lane=Lane.LOCAL, - levels_run=levels, - results=results, - verdict=apply_policy(results, policy, Lane.LOCAL), - ) - if artifacts_dir is not None: - write_runtime_bundle( - artifacts_dir, - report, - cleanup=record["cleanup"], - provider={"binding": "harbor_task", "profile": HARBOR_PROFILE}, + startup.duration_s = time.monotonic() - started + record["cleanup"].update( + {"required": attempted, "completed": record["cleanup"]["verified"]} + ) + results = [*results, startup, reward_result, determinism, *checks] + present = {result.check_id for result in results} + for entry in policy.entries_for_lane(Lane.LOCAL).values(): + if entry.level <= max_level and entry.check_id not in present: + results.append( + _result( + entry.check_id, + CheckStatus.SKIP, + _skip_reason(entry, contract), + ) + ) + if any( + check.status in {CheckStatus.FAIL, CheckStatus.ERROR} for check in results + ): + try: + inspector.remove_image() + except (Exception, KeyboardInterrupt) as exc: + startup.status = CheckStatus.ERROR + startup.evidence.append( + f"retained image removal failed ({type(exc).__name__})" + ) + record["cleanup"].update(verified=False, completed=False) + elif "image_ref" in record: + tag = trials[0]["retained_image_tag"] + startup.measured["retained_image_tag"] = tag + record["retained_image_tag"] = tag + levels = [Level.STATIC] + if attempted: + levels.append(Level.RUNTIME) + if semantic and (attempted or semantic_graded): + levels.append(Level.SEMANTIC) + report = ValidationReportV2( + report_schema_version="2", + target=str(subject.root) if report_target is None else report_target, + source_digest=digest_before, + signature=subject.manifest.signature, + manifest=subject.manifest, + policy_version=policy.policy_version, + lane=Lane.LOCAL, + levels_run=levels, + results=results, + verdict=apply_policy(results, policy, Lane.LOCAL), ) - files["harbor-evidence.json"] = (dumps(record) + "\n").encode() - checksums = (artifacts_dir / "SHA256SUMS").read_text() - for name, raw in sorted(files.items()): - path = artifacts_dir / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(raw) - path.chmod(0o600) - checksums += f"{hashlib.sha256(raw).hexdigest()} {name}\n" - (artifacts_dir / "SHA256SUMS").write_text(checksums) - return report + if artifacts_dir is not None: + write_runtime_bundle( + artifacts_dir, + report, + cleanup=record["cleanup"], + provider={"binding": "harbor_task", "profile": HARBOR_PROFILE}, + ) + files["harbor-evidence.json"] = (dumps(record) + "\n").encode() + checksums = (artifacts_dir / "SHA256SUMS").read_text() + for name, raw in sorted(files.items()): + path = artifacts_dir / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(raw) + path.chmod(0o600) + checksums += f"{hashlib.sha256(raw).hexdigest()} {name}\n" + (artifacts_dir / "SHA256SUMS").write_text(checksums) + completed = True + return report + finally: + if not completed: + inspector.remove_image() diff --git a/tests/test_validation/test_harbor_binding_inspection.py b/tests/test_validation/test_harbor_binding_inspection.py index 663339a39..9602d3d44 100644 --- a/tests/test_validation/test_harbor_binding_inspection.py +++ b/tests/test_validation/test_harbor_binding_inspection.py @@ -67,6 +67,8 @@ def docker(*args): return "" if args == ("image", "inspect", TAG): return json.dumps([retained]) + if args == ("image", "rm", "--no-prune", TAG): + return "" pytest.fail(f"unexpected Docker command: {args}") monkeypatch.setattr(inspector, "_docker", docker) @@ -218,6 +220,45 @@ def test_verify_image_checks_the_retained_tag(inspection, matches): assert commands == [("image", "inspect", TAG)] +@pytest.mark.parametrize("tag", ["unrelated:tag", "openenv-validation:" + "e" * 32]) +def test_capture_refuses_a_tag_outside_its_run(inspection, tag): + inspector, _, _, _, commands = inspection + with pytest.raises(NativeEvidenceError, match="run-owned project"): + inspector.capture(PROJECT, tag) + assert not any(cmd[:2] == ("image", "tag") for cmd in commands) + + +def test_image_removal_requires_a_tag_created_by_this_inspector(inspection): + inspector, _, _, _, commands = inspection + inspector.capture(PROJECT, TAG, retain=False) + commands.clear() + inspector.remove_image() + assert not commands + + +@pytest.mark.parametrize("drift", [False, True]) +def test_image_removal_verifies_ownership_and_removes_only_the_run_tag( + inspection, drift +): + inspector, _, retained, _, commands = inspection + inspector.capture(PROJECT, TAG) + commands.clear() + if drift: + retained["Id"] = "sha256:" + "b" * 64 + with pytest.raises(NativeEvidenceError, match="ownership changed"): + inspector.remove_image() + assert commands == [("image", "inspect", TAG)] + else: + inspector.remove_image() + assert commands == [ + ("image", "inspect", TAG), + ("image", "rm", "--no-prune", TAG), + ] + commands.clear() + inspector.remove_image() + assert not commands + + @pytest.mark.parametrize("kind", ["file", "parent", "broken"]) def test_read_output_refuses_symlinks(tmp_path, kind): outside = tmp_path / "outside" diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index bb2edd5c3..a8ff5d573 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -7,6 +7,7 @@ import pytest from openenv.validation import harbor +from openenv.validation.harbor import DockerInspector from openenv.validation.manifest import ManifestError from openenv.validation.parsers.harbor_toml import ( HarborTaskParser, @@ -77,6 +78,7 @@ def boundary(monkeypatch, task): inventory_problem=None, cleanup_calls=[], preflight_error=None, + interrupt_role=None, ) class Inspector: @@ -120,6 +122,9 @@ def cleanup(self, project): }, } + def remove_image(self): + pass + def verify_image(self, tag, image_ref): if state.retained_error: raise OSError("inspect failed") @@ -190,6 +195,8 @@ async def run(self): if not state.no_identity: for hook in self.hooks.get("agent-start", []): await hook(None) + if self.role == state.interrupt_role: + raise KeyboardInterrupt await asyncio.wait_for( asyncio.sleep(state.agent_delay), timeout=self.config.agent.override_timeout_sec, @@ -245,6 +252,68 @@ async def create(config): return state +@pytest.fixture +def image_docker(boundary, monkeypatch): + state = SimpleNamespace( + images={"unrelated:tag": IMAGE}, removed=set(), commands=[], image_os="linux" + ) + + def docker(self, *args): + state.commands.append(args) + if args == ("info", "--format", "{{json .}}"): + return json.dumps({"OSType": "linux", "ID": "fake", "ServerVersion": "29"}) + if args == ("compose", "version"): + return "Docker Compose" + if args[0] == "ps": + project = args[-1].removeprefix("label=com.docker.compose.project=") + return "\n".join( + trial.role + "-container" + for trial in boundary.trials + if trial.agent_environment.session_id == project + and trial.role not in state.removed + ) + if args[:2] in (("network", "ls"), ("volume", "ls")): + return "" + if args[0] == "inspect": + trial = next(t for t in boundary.trials if t.role + "-container" == args[1]) + project = trial.agent_environment.session_id + return json.dumps( + [ + { + "Id": args[1], + "Image": IMAGE, + "State": {"Running": True}, + "Config": { + "Labels": { + "com.docker.compose.project": project, + "com.docker.compose.service": "main", + }, + "User": "1000", + "WorkingDir": "/app", + }, + "HostConfig": {"NetworkMode": project + "_default"}, + "Mounts": [], + } + ] + ) + if args[:2] == ("image", "tag"): + state.images[args[3]] = args[2] + return "" + if args[:2] == ("image", "inspect"): + return json.dumps([{"Id": state.images[args[2]], "Os": state.image_os}]) + if args[:3] == ("image", "rm", "--no-prune"): + del state.images[args[3]] + return "" + if args[:3] == ("rm", "-f", "-v"): + state.removed.add(args[3].removesuffix("-container")) + return "" + pytest.fail(f"unexpected Docker command: {args}") + + monkeypatch.setattr(DockerInspector, "_docker", docker) + monkeypatch.setattr(harbor, "DockerInspector", DockerInspector) + return state + + def _role(trial_name): # openenv-validation-- return trial_name.split("-", 3)[3] @@ -358,6 +427,66 @@ def record_timeout(delay): ) +@pytest.mark.parametrize( + "outcome", ["kept", "fail", "error", "interrupted", "capture_error"] +) +def test_retained_image_policy(task, boundary, image_docker, tmp_path, outcome): + if outcome == "fail": + boundary.rewards["nop"] = b'{"reward":1}' + elif outcome == "error": + boundary.exception = True + elif outcome == "interrupted": + boundary.interrupt_role = "oracle" + elif outcome == "capture_error": + image_docker.image_os = "windows" + artifacts = tmp_path / "image-policy" + report = run_validation(task, artifacts_dir=artifacts) + tag = "openenv-validation:" + boundary.trials[0].config.trial_name.split("-")[2] + measured = result(report, "runtime.startup").measured + removals = [cmd for cmd in image_docker.commands if cmd[:2] == ("image", "rm")] + if outcome == "kept": + assert report.verdict is Verdict.WARN + assert measured["retained_image_tag"] == tag + assert image_docker.images == {"unrelated:tag": IMAGE, tag: IMAGE} + assert not removals + else: + assert report.verdict is Verdict.FAIL + if outcome == "fail": + assert result(report, "semantic.floor_gap").status is CheckStatus.FAIL + else: + assert result(report, "runtime.startup").status is CheckStatus.ERROR + assert "retained_image_tag" not in measured + assert image_docker.images == {"unrelated:tag": IMAGE} + assert removals == [("image", "rm", "--no-prune", tag)] + assert json.loads((artifacts / "report.json").read_text()) == json.loads( + report.model_dump_json() + ) + + +@pytest.mark.parametrize("error", [KeyboardInterrupt, OSError]) +def test_retained_image_removed_when_reporting_does_not_complete( + task, boundary, image_docker, tmp_path, monkeypatch, error +): + def interrupted(*args, **kwargs): + raise error + + monkeypatch.setattr(harbor, "write_runtime_bundle", interrupted) + with pytest.raises(error): + run_validation(task, artifacts_dir=tmp_path / "interrupted-report") + assert image_docker.images == {"unrelated:tag": IMAGE} + assert len(boundary.trials) == 3 + + +def test_build_failure_has_no_retained_tag_to_remove(task, boundary, image_docker): + boundary.exception_type = "RuntimeError" + report = run_validation(task) + startup = result(report, "runtime.startup") + assert startup.status is CheckStatus.FAIL + assert "retained_image_tag" not in startup.measured + assert image_docker.images == {"unrelated:tag": IMAGE} + assert not any(cmd[:2] == ("image", "rm") for cmd in image_docker.commands) + + @pytest.mark.parametrize( "raw", [ From e34c0f9bf4502726d9ecc3f38e349d051bfe981a Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Wed, 11 Nov 2026 15:22:30 +0100 Subject: [PATCH 27/40] fix(validation): reject Harbor execution in served manifests Validate served execution declarations as openenv_ws before normalization. Report a static.manifest failure instead of entering Harbor execution. --- src/openenv/validation/parsers/openenv_yaml.py | 17 +++++++++++++++-- tests/test_validation/test_cli_validate.py | 18 +++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/openenv/validation/parsers/openenv_yaml.py b/src/openenv/validation/parsers/openenv_yaml.py index 6d82d3cac..90722125c 100644 --- a/src/openenv/validation/parsers/openenv_yaml.py +++ b/src/openenv/validation/parsers/openenv_yaml.py @@ -5,7 +5,12 @@ import yaml from pydantic import ValidationError -from ..manifest import ManifestError, NormalizedManifest, NormalizedManifestV2 +from ..manifest import ( + ExecutionDeclaration, + ManifestError, + NormalizedManifest, + NormalizedManifestV2, +) from ..types import SignatureKind VALIDATION_BLOCK_REMEDIATION = ( @@ -89,7 +94,15 @@ def parse(self, package_root: Path) -> NormalizedManifest | NormalizedManifestV2 data[key] = value if has_execution: - data["execution"] = validation["execution"] + try: + data["execution"] = ExecutionDeclaration.model_validate( + validation["execution"] + ) + except ValidationError as exc: + raise ManifestError( + [f"execution: {error}" for error in _format_validation_error(exc)], + remediation=SCHEMA_REMEDIATION, + ) from exc try: model = NormalizedManifestV2 if has_execution else NormalizedManifest diff --git a/tests/test_validation/test_cli_validate.py b/tests/test_validation/test_cli_validate.py index 190cc2c02..1a5975bbe 100644 --- a/tests/test_validation/test_cli_validate.py +++ b/tests/test_validation/test_cli_validate.py @@ -3,7 +3,7 @@ from pathlib import Path from conftest import FIXTURES -from openenv.validation.report import ValidationReport +from openenv.validation.report import ValidationReport, ValidationReportV2 REPO_ROOT = Path(__file__).parent.parent.parent @@ -132,3 +132,19 @@ def test_unpinned_judge_fails_static_manifest(): ) assert result.returncode == 1 assert "judge pin" in result.stdout + + +def test_served_package_cannot_claim_harbor_execution(tmp_path): + source = (FIXTURES / "served_min_pass/openenv.yaml").read_text() + (tmp_path / "openenv.yaml").write_text( + source + " execution:\n kind: harbor_task\n agent_boundary: process\n" + ) + + result = _validate(str(tmp_path), "--json") + + assert result.returncode == 1, result.stdout + result.stderr + report = ValidationReportV2.model_validate_json(result.stdout) + manifest_result = next(r for r in report.results if r.check_id == "static.manifest") + assert manifest_result.status.value == "fail" + assert "openenv_ws" in " ".join(manifest_result.evidence) + assert "Internal error" not in result.stderr From 802b3891c27daeb5b2dd15e5304719ffcf2d0cd8 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Wed, 11 Nov 2026 15:23:11 +0100 Subject: [PATCH 28/40] fix(validation): grade hostile Harbor contract JSON as invalid Convert numeric overflow and excessive JSON recursion into ManifestError. Keep malformed declarations on the package-failure path. --- src/openenv/validation/parsers/harbor_toml.py | 9 ++++++++- tests/test_validation/test_harbor_contract.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/openenv/validation/parsers/harbor_toml.py b/src/openenv/validation/parsers/harbor_toml.py index 926bdd520..76c0ca05a 100644 --- a/src/openenv/validation/parsers/harbor_toml.py +++ b/src/openenv/validation/parsers/harbor_toml.py @@ -106,7 +106,14 @@ def read_harbor_contract(package_root: Path) -> HarborContract: "numeric declarations must keep their exact value as floats" ) return HarborContract.model_validate(raw) - except (OSError, ValueError, TypeError, AttributeError) as exc: + except ( + OSError, + ValueError, + TypeError, + AttributeError, + OverflowError, + RecursionError, + ) as exc: raise ManifestError( [f"invalid trusted Harbor contract ({type(exc).__name__})"], remediation="supply explicit reward/resources in validation/openenv.json", diff --git a/tests/test_validation/test_harbor_contract.py b/tests/test_validation/test_harbor_contract.py index abb9b72b9..74f91febc 100644 --- a/tests/test_validation/test_harbor_contract.py +++ b/tests/test_validation/test_harbor_contract.py @@ -59,3 +59,16 @@ def test_gpu_type_strings_are_preserved(contract_package): loaded = read_harbor_contract(contract_package) assert loaded.resources.gpu_types == ["A100", "H100"] assert loaded.resources.gpus == 1 + + +@pytest.mark.parametrize("hostile_input", ["huge_integer", "deep_nesting"]) +def test_hostile_json_is_a_manifest_failure(contract_package, hostile_input): + path = contract_package / "validation/openenv.json" + if hostile_input == "huge_integer": + payload = path.read_text().replace('"cpu": 1', '"cpu": ' + "9" * 400) + else: + payload = "[" * 100_000 + "0" + "]" * 100_000 + path.write_text(payload) + + with pytest.raises(ManifestError): + read_harbor_contract(contract_package) From b98a3ac879215e19c141bc46603d1a3a2da8b68a Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Wed, 11 Nov 2026 15:24:02 +0100 Subject: [PATCH 29/40] fix(validation): report Harbor version mismatch as validator error Check the installed Harbor version outside package parsing. Exit through the existing internal-error path with the required and installed versions. --- src/openenv/validation/parsers/harbor_toml.py | 13 +++++-- tests/test_validation/test_harbor_contract.py | 37 +++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/openenv/validation/parsers/harbor_toml.py b/src/openenv/validation/parsers/harbor_toml.py index 76c0ca05a..ee0f1eb43 100644 --- a/src/openenv/validation/parsers/harbor_toml.py +++ b/src/openenv/validation/parsers/harbor_toml.py @@ -150,14 +150,21 @@ def canary_scripts(package_root: Path, directory: str) -> list[str]: def read_harbor_task(package_root: Path): """Use the pinned Harbor data loader; constructing a Trial is runtime-only.""" try: - if version("harbor") != "0.25.0": - raise ValueError("Harbor task validation requires Harbor 0.25.0") + installed = version("harbor") + except PackageNotFoundError as exc: + raise RuntimeError( + "Harbor task validation requires harbor==0.25.0; Harbor is not installed" + ) from exc + if installed != "0.25.0": + raise RuntimeError( + f"Harbor task validation requires harbor==0.25.0; installed {installed}" + ) + try: from harbor.models.task.task import Task return Task(package_root) except ( ImportError, - PackageNotFoundError, OSError, ValueError, RuntimeError, diff --git a/tests/test_validation/test_harbor_contract.py b/tests/test_validation/test_harbor_contract.py index 74f91febc..9d274edd2 100644 --- a/tests/test_validation/test_harbor_contract.py +++ b/tests/test_validation/test_harbor_contract.py @@ -1,10 +1,14 @@ """The trusted Harbor contract accepts explicit numeric JSON declarations.""" import json +from importlib.metadata import PackageNotFoundError import pytest +from openenv.cli.__main__ import app from openenv.validation.manifest import ManifestError +from openenv.validation.parsers import harbor_toml from openenv.validation.parsers.harbor_toml import read_harbor_contract +from typer.testing import CliRunner @pytest.fixture @@ -72,3 +76,36 @@ def test_hostile_json_is_a_manifest_failure(contract_package, hostile_input): with pytest.raises(ManifestError): read_harbor_contract(contract_package) + + +@pytest.mark.parametrize("installed_version", ["0.22.0", "0.26.0"]) +def test_harbor_version_mismatch_is_a_validator_error( + contract_package, monkeypatch, installed_version +): + (contract_package / "task.toml").write_text('schema_version = "1.4"\n') + monkeypatch.setattr(harbor_toml, "version", lambda name: installed_version) + + result = CliRunner().invoke( + app, ["validate", str(contract_package), "--level", "static", "--skip-build"] + ) + + assert result.exit_code == 3, result.output + assert "harbor==0.25.0" in result.output + assert installed_version in result.output + assert "static.manifest" not in result.output + + +def test_missing_harbor_is_a_validator_error(contract_package, monkeypatch): + (contract_package / "task.toml").write_text('schema_version = "1.4"\n') + + def missing_version(name): + raise PackageNotFoundError(name) + + monkeypatch.setattr(harbor_toml, "version", missing_version) + result = CliRunner().invoke( + app, ["validate", str(contract_package), "--level", "static", "--skip-build"] + ) + + assert result.exit_code == 3, result.output + assert "harbor==0.25.0" in result.output + assert "static.manifest" not in result.output From b3db1573e056e7e69cb6f9bf7e23dc056c2384b7 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Wed, 11 Nov 2026 15:24:33 +0100 Subject: [PATCH 30/40] docs(validation): document a complete pinned Harbor contract State the Python and Harbor version requirements. Load the CLI reference JSON example in the contract-reader tests. --- docs/source/reference/cli.md | 34 +++++++++++++++++-- tests/test_validation/test_harbor_contract.py | 21 ++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index 349f187de..80e699031 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -69,9 +69,37 @@ supports that capability. Docker validation uses two containers concurrently during the fresh-container replay. Both use the same immutable image and are removed even if a check fails. -Harbor `task.toml` directories are also recognized. Install the `harbor` extra -and add a `validation/openenv.json` contract that declares the reward key, range, -oracle tolerance, floor margin and resources. The `harbor-semantic-v1` binding runs +Harbor `task.toml` directories are also recognized. Validation requires Python 3.12 or newer +and `harbor==0.25.0` installed alongside `openenv[harbor]`: + +```bash +pip install "openenv[harbor]" "harbor==0.25.0" +``` + +Add a `validation/openenv.json` contract with `profile: "harbor-semantic-v1"`, +`reward_key`, `reward`, `resources` and `types`. The reward range, oracle tolerance +and floor margin must be explicit. This is a minimal complete contract: + +```json +{ + "profile": "harbor-semantic-v1", + "reward_key": "reward", + "reward": { + "range": [0, 1], + "oracle_tolerance": 0, + "floor_margin": 0.1 + }, + "resources": { + "cpu": 1, + "memory_mb": 512, + "disk_mb": 1024, + "episode_timeout_s": 300 + }, + "types": {"tags": ["swe"]} +} +``` + +The `harbor-semantic-v1` binding runs fresh Harbor trials in Docker: a do-nothing agent, the oracle (`solution/solve.sh`), an oracle replay and any canaries declared in the contract. A canary is a known-wrong `solve.sh` replacement. The results report under `runtime.startup`, diff --git a/tests/test_validation/test_harbor_contract.py b/tests/test_validation/test_harbor_contract.py index 9d274edd2..e7e601d84 100644 --- a/tests/test_validation/test_harbor_contract.py +++ b/tests/test_validation/test_harbor_contract.py @@ -1,7 +1,9 @@ """The trusted Harbor contract accepts explicit numeric JSON declarations.""" import json +import re from importlib.metadata import PackageNotFoundError +from pathlib import Path import pytest from openenv.cli.__main__ import app @@ -109,3 +111,22 @@ def missing_version(name): assert result.exit_code == 3, result.output assert "harbor==0.25.0" in result.output assert "static.manifest" not in result.output + + +def test_cli_reference_contract_loads(tmp_path): + reference = Path(__file__).resolve().parents[2] / "docs/source/reference/cli.md" + section = reference.read_text().split("Harbor `task.toml` directories", 1)[1] + section = section.split("\n## ", 1)[0] + example = re.search(r"```json\n(.*?)\n```", section, re.DOTALL) + assert example is not None, "document a complete validation/openenv.json contract" + assert "Python 3.12 or newer" in section + assert "harbor==0.25.0" in section + assert "openenv[harbor]" in section + (tmp_path / "validation").mkdir() + (tmp_path / "validation/openenv.json").write_text(example[1]) + + contract = read_harbor_contract(tmp_path) + + assert contract.profile == "harbor-semantic-v1" + assert contract.reward_key == "reward" + assert contract.types.tags == ["swe"] From 913e632294d3a9123ee05b548d83b3ce5be24ba8 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Wed, 11 Nov 2026 15:25:45 +0100 Subject: [PATCH 31/40] fix(validation): normalize semantic thresholds by reward range Scale oracle tolerance and both floor gaps by the declared width using exact rationals. Document fractional policy bounds and test non-unit ranges and exact boundaries. --- docs/source/reference/cli.md | 3 +- rfcs/008-environment-auto-validation.md | 22 +++++++--- .../validation/graders/semantic/oracle.py | 29 ++++++------ src/openenv/validation/manifest.py | 7 ++- .../schemas/manifest-v2.schema.json | 2 +- .../test_validation/test_semantic_graders.py | 44 +++++++++++++++++++ 6 files changed, 84 insertions(+), 23 deletions(-) diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index 80e699031..8d2bca78e 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -78,7 +78,8 @@ pip install "openenv[harbor]" "harbor==0.25.0" Add a `validation/openenv.json` contract with `profile: "harbor-semantic-v1"`, `reward_key`, `reward`, `resources` and `types`. The reward range, oracle tolerance -and floor margin must be explicit. This is a minimal complete contract: +and floor margin must be explicit. Oracle tolerance and floor margin are fractions +of the declared reward range. This is a minimal complete contract: ```json { diff --git a/rfcs/008-environment-auto-validation.md b/rfcs/008-environment-auto-validation.md index dade23e1c..2acda8472 100644 --- a/rfcs/008-environment-auto-validation.md +++ b/rfcs/008-environment-auto-validation.md @@ -182,7 +182,8 @@ The core promise is #778's verifier-sanity test, made precise: - **Floor is measured, not declared.** The floor is the measured reward of the reset/do-nothing state — the declared range minimum may sit below it (an agent that deletes the repo earns a legal sub-floor reward, not a validation anomaly). The contract is a **gap check**: - `reward(oracle) ≥ max − tolerance`, `reward(reset) = floor`, `max − floor ≥ margin`. + `reward(oracle) >= max - tolerance * width`, `reward(reset) = floor`, + `max - floor >= margin * width`, where `width = max - min`. - **Oracle containment is required.** The oracle (state or script) must be withheld from or isolated against the agent at serve time, and validation checks that it is. - **Reward range is declared** in the manifest, defaulting to `[0, 1]`; graders normalize against @@ -193,7 +194,7 @@ The core promise is #778's verifier-sanity test, made precise: **LLM-as-judge is an allowed reward.** Judged environments declare `llm_judged` and pin the judge configuration (model, version, params) in the manifest; the oracle check becomes -`≥ max − declared tolerance` and determinism becomes variance-within-declared-bound rather than +`>= max - declared tolerance * width` and determinism becomes variance-within-declared-bound rather than bit-exact. RFC 004 rubrics are **leveraged, not required**: the contract stays spec-neutral (graders read the manifest), but for the served OpenEnv format the rubric tree is the native satisfaction path — `LLMJudge` is the in-repo `llm_judged` implementation, and the @@ -204,6 +205,14 @@ the rubric object does not serialize model/version/params today. Tolerances, margins, and variance bounds are author-declared in the manifest, **bounded by the versioned severity policy**, and carried verbatim in reports so hubs can apply stricter ceilings. +Oracle tolerance and floor margin are fractions of the declared reward range. +Grader comparisons multiply each fraction by `width = max - min` using exact +rational arithmetic. The declared range must be strictly increasing. + +| Semantic declaration | Policy bound | Reward-unit threshold | +|---|---|---| +| `oracle_tolerance` | At most 0.1 of the range | `oracle_tolerance * width` below max | +| `floor_margin` | At least 0.1 of the range | `floor_margin * width` between rewards | ### Report and severity policy @@ -754,13 +763,14 @@ Results map to RFC check ids: | `runtime.startup` | Every trial ran from the pinned image in a fresh container, and cleanup was verified. | | `runtime.reward_well_formed` | Every raw reward map matches Harbor's result and is finite, nonboolean and in range. | | `runtime.episode_determinism` | The oracle replay reproduces every reward and the exit code. | -| `semantic.oracle_max` | Both oracle runs exit 0 and earn at least `max - oracle_tolerance`. A task without `solution/solve.sh` fails. | -| `semantic.floor_gap` | `max - floor >= floor_margin`, and `oracle - floor >= floor_margin` because the oracle may sit up to the tolerance below the maximum. | +| `semantic.oracle_max` | Both oracle runs exit 0 and earn at least `max - oracle_tolerance * width`. A task without `solution/solve.sh` fails. | +| `semantic.floor_gap` | `max - floor >= floor_margin * width`, and `oracle - floor >= floor_margin * width` because the oracle may sit up to the tolerance below the maximum. | | `semantic.canary_floor` | Every declared canary earns no more than the measured floor. Selected only when canaries are declared. | The semantic ids depend on `runtime.episode_determinism`, so a diverging replay -SKIPs them with the dependency named. Reward comparisons use exact rationals parsed -from the raw verifier output. Every other local check is a named SKIP for this +SKIPs them with the dependency named. Oracle tolerance and floor margin are fractions +of the declared range, with `width = max - min`. Reward comparisons use exact +rationals parsed from the raw verifier output. Every other local check is a named SKIP for this binding. It produces no WebSocket exchange, containment or resource measurement, second verification of one final state, or leakage scan. A passing run is therefore WARN, and full RFC qualification still rejects these SKIPs. diff --git a/src/openenv/validation/graders/semantic/oracle.py b/src/openenv/validation/graders/semantic/oracle.py index 881274eac..91b660894 100644 --- a/src/openenv/validation/graders/semantic/oracle.py +++ b/src/openenv/validation/graders/semantic/oracle.py @@ -34,9 +34,11 @@ def run(self, subject) -> CheckResult: duration_s=0, ) reward = subject.manifest.reward + low, high = (Fraction(str(value)) for value in reward.range) problems, measured = self.check( evidence, - maximum=Fraction(str(reward.range[1])), + maximum=high, + width=high - low, tolerance=Fraction(str(reward.oracle_tolerance)), margin=Fraction(str(reward.floor_margin)), ) @@ -51,7 +53,7 @@ def run(self, subject) -> CheckResult: class OracleMaxGrader(_SemanticGrader): - """Every oracle run exits cleanly and earns at least `max - oracle_tolerance`.""" + """Every oracle run exits cleanly and meets the range-normalized tolerance.""" check_id = "semantic.oracle_max" passed = "every oracle run reached the declared maximum within tolerance" @@ -60,8 +62,8 @@ class OracleMaxGrader(_SemanticGrader): "earns the declared maximum reward." ) - def check(self, evidence, *, maximum, tolerance, margin): - threshold = maximum - tolerance + def check(self, evidence, *, maximum, width, tolerance, margin): + threshold = maximum - tolerance * width problems = [] for run, (reward, code) in enumerate( zip(evidence.oracle, evidence.oracle_exit_codes, strict=True), start=1 @@ -82,9 +84,9 @@ def check(self, evidence, *, maximum, tolerance, margin): class FloorGapGrader(_SemanticGrader): """ - The do-nothing reward sits at least `floor_margin` below the maximum and below - every oracle run. The oracle may sit up to the tolerance below the maximum, so - both gaps are required. + The do-nothing reward sits at least `floor_margin` of the declared range below + the maximum and every oracle run. The oracle may sit up to the tolerance below + the maximum, so both gaps are required. """ check_id = "semantic.floor_gap" @@ -94,19 +96,20 @@ class FloorGapGrader(_SemanticGrader): "such as comparisons against empty output." ) - def check(self, evidence, *, maximum, tolerance, margin): + def check(self, evidence, *, maximum, width, tolerance, margin): floor = evidence.floor + required_gap = margin * width problems = [] - if maximum - floor < margin: + if maximum - floor < required_gap: problems.append( f"do-nothing reward {float(floor)} is within {float(maximum - floor)} " - f"of the maximum, below the margin {float(margin)}" + f"of the maximum, below the required gap {float(required_gap)}" ) lowest = min(evidence.oracle) - if lowest - floor < margin: + if lowest - floor < required_gap: problems.append( f"oracle reward {float(lowest)} is within {float(lowest - floor)} " - f"of the do-nothing reward, below the margin {float(margin)}" + f"of the do-nothing reward, below the required gap {float(required_gap)}" ) return problems, { "measured_floor": float(floor), @@ -127,7 +130,7 @@ class CanaryFloorGrader(_SemanticGrader): "canary earns no more than doing nothing." ) - def check(self, evidence, *, maximum, tolerance, margin): + def check(self, evidence, *, maximum, width, tolerance, margin): problems = [ f"canary {name} earned {float(reward)}, above the do-nothing reward " f"{float(evidence.floor)}" diff --git a/src/openenv/validation/manifest.py b/src/openenv/validation/manifest.py index 08eba70bb..3d829b49c 100644 --- a/src/openenv/validation/manifest.py +++ b/src/openenv/validation/manifest.py @@ -54,9 +54,12 @@ class RewardDeclaration(BaseModel): range (`tuple[float, float]`): Declared reward range, defaults to `(0.0, 1.0)`. Must be strictly increasing. oracle_tolerance (`float`): - The oracle check passes when `reward(oracle) >= max - oracle_tolerance`. + Fraction of the declared range allowed below the maximum. + The oracle check requires `reward(oracle) >= max - oracle_tolerance * width`, + where `width = max - min`. floor_margin (`float`): - The gap check requires `max - measured_floor >= floor_margin`. + Fraction of the declared range required between the maximum and the floor. + The gap check requires `max - measured_floor >= floor_margin * width`. variance_tolerance (`float`, *optional*): Required iff the environment declares `llm_judged`; bounds reward variance for variance-mode determinism checks. diff --git a/src/openenv/validation/schemas/manifest-v2.schema.json b/src/openenv/validation/schemas/manifest-v2.schema.json index 228f0d27a..6bb1d8c0e 100644 --- a/src/openenv/validation/schemas/manifest-v2.schema.json +++ b/src/openenv/validation/schemas/manifest-v2.schema.json @@ -273,7 +273,7 @@ }, "RewardDeclaration": { "additionalProperties": false, - "description": "Author-declared reward contract. Graders normalize measurements against it.\n\nDeclared tolerances are bounded by the severity policy; reports carry the declared\nvalues so hubs can apply stricter ceilings.\n\nAttributes:\n range (`tuple[float, float]`):\n Declared reward range, defaults to `(0.0, 1.0)`. Must be strictly increasing.\n oracle_tolerance (`float`):\n The oracle check passes when `reward(oracle) >= max - oracle_tolerance`.\n floor_margin (`float`):\n The gap check requires `max - measured_floor >= floor_margin`.\n variance_tolerance (`float`, *optional*):\n Required iff the environment declares `llm_judged`; bounds reward variance\n for variance-mode determinism checks.", + "description": "Author-declared reward contract. Graders normalize measurements against it.\n\nDeclared tolerances are bounded by the severity policy; reports carry the declared\nvalues so hubs can apply stricter ceilings.\n\nAttributes:\n range (`tuple[float, float]`):\n Declared reward range, defaults to `(0.0, 1.0)`. Must be strictly increasing.\n oracle_tolerance (`float`):\n Fraction of the declared range allowed below the maximum.\n The oracle check requires `reward(oracle) >= max - oracle_tolerance * width`,\n where `width = max - min`.\n floor_margin (`float`):\n Fraction of the declared range required between the maximum and the floor.\n The gap check requires `max - measured_floor >= floor_margin * width`.\n variance_tolerance (`float`, *optional*):\n Required iff the environment declares `llm_judged`; bounds reward variance\n for variance-mode determinism checks.", "properties": { "floor_margin": { "minimum": 0.0, diff --git a/tests/test_validation/test_semantic_graders.py b/tests/test_validation/test_semantic_graders.py index d0a93b1f7..1e65a07b8 100644 --- a/tests/test_validation/test_semantic_graders.py +++ b/tests/test_validation/test_semantic_graders.py @@ -92,3 +92,47 @@ def test_missing_evidence_skips(): for grader in (OracleMaxGrader(), FloorGapGrader(), CanaryFloorGrader()): assert grader.run(subject(None)).status is CheckStatus.SKIP assert grader.depends_on == ("runtime.episode_determinism",) + + +@pytest.mark.parametrize( + "reward_range,oracle,tolerance,status", + [ + ((0, 100), "95", 0.05, CheckStatus.PASS), + ((0, 100), "94.999999999999999999999", 0.05, CheckStatus.FAIL), + ((50, 150), "145", 0.05, CheckStatus.PASS), + ((0, 0.01), "0.009", 0.05, CheckStatus.FAIL), + ], +) +def test_oracle_tolerance_is_a_fraction_of_range( + reward_range, oracle, tolerance, status +): + found = OracleMaxGrader().run( + subject( + evidence(oracle=(oracle, oracle)), + range=reward_range, + oracle_tolerance=tolerance, + ) + ) + assert found.status is status + assert found.measured["oracle_tolerance"] == tolerance + + +@pytest.mark.parametrize( + "floor,oracle,status", + [ + ("90", "100", CheckStatus.PASS), + ("99.9", "100", CheckStatus.FAIL), + ("90", "99", CheckStatus.FAIL), + ("90.000000000000000000001", "100", CheckStatus.FAIL), + ], +) +def test_floor_margin_is_a_fraction_of_range(floor, oracle, status): + found = FloorGapGrader().run( + subject( + evidence(floor=floor, oracle=(oracle, oracle)), + range=(0, 100), + floor_margin=0.1, + ) + ) + assert found.status is status + assert found.measured["declared_margin"] == 0.1 From 8178528e56be72a01efbaeef1bf59858c40b6301 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Wed, 11 Nov 2026 15:27:01 +0100 Subject: [PATCH 32/40] fix(validation): close rejected trial logs and fail missing evidence Share the hook lifecycle across credential paths and close Harbor log handlers on rejection. Remove the empty-canary fallback, fail missing evidence closed, and clean up contract and hook documentation. --- src/openenv/harbor/rollout.py | 27 ++++++++++--------- .../validation/graders/semantic/oracle.py | 5 +--- src/openenv/validation/parsers/harbor_toml.py | 2 +- .../envs/test_harbor_validated_trial_hook.py | 16 +++++++++++ .../test_validation/test_semantic_graders.py | 12 ++++----- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/openenv/harbor/rollout.py b/src/openenv/harbor/rollout.py index 496acc4d7..493f14c20 100644 --- a/src/openenv/harbor/rollout.py +++ b/src/openenv/harbor/rollout.py @@ -259,9 +259,7 @@ def build_trial_config( kwarg, or a config file written by a subclass. That is the only per-agent knowledge involved; everything downstream is identical. - Args: - environment_kwargs (`dict[str, Any]`, *optional*): - Backend kwargs passed to Harbor. `None` keeps Harbor's defaults. + Backend kwargs in `environment_kwargs` pass to Harbor. `None` keeps Harbor's defaults. """ from harbor.models.trial.config import ( AgentConfig, @@ -327,6 +325,17 @@ def build_trial_config( ) +async def _run_trial(trial: Any, on_trial_created: Callable[[Any], None] | None) -> Any: + """Close Harbor's trial log when a trusted hook prevents execution.""" + if on_trial_created is not None: + try: + on_trial_created(trial) + except BaseException: + trial._close_logger_handler() + raise + return await trial.run() + + async def run_rollout( *, task_dir: Path | str, @@ -519,9 +528,7 @@ async def run_rollout( if not proc_env: config = build_trial_config(**config_kwargs) trial = await Trial.create(config) - if on_trial_created is not None: - on_trial_created(trial) - trial_result = await trial.run() + trial_result = await _run_trial(trial, on_trial_created) else: # The override has to span `run()`, not just construction. Harbor's goose wrapper reads # `os.environ["OPENAI_API_KEY"]` inside its own `run` (goose.py:653), long after @@ -553,9 +560,7 @@ async def run_rollout( with ctx_env_overlay(proc_env): config = build_trial_config(**config_kwargs) trial = await Trial.create(config) - if on_trial_created is not None: - on_trial_created(trial) - trial_result = await trial.run() + trial_result = await _run_trial(trial, on_trial_created) else: # Fallback: mutate the real environment under a lock, one rollout at a time. await _acquire_proc_env() @@ -563,9 +568,7 @@ async def run_rollout( with process_env(seam.name, proc_env): config = build_trial_config(**config_kwargs) trial = await Trial.create(config) - if on_trial_created is not None: - on_trial_created(trial) - trial_result = await trial.run() + trial_result = await _run_trial(trial, on_trial_created) finally: _PROC_ENV_LOCK.release() except Exception as exc: # noqa: BLE001 - a rollout failure is a RESULT, never an exception diff --git a/src/openenv/validation/graders/semantic/oracle.py b/src/openenv/validation/graders/semantic/oracle.py index 91b660894..9dd8a8d9f 100644 --- a/src/openenv/validation/graders/semantic/oracle.py +++ b/src/openenv/validation/graders/semantic/oracle.py @@ -29,7 +29,7 @@ def run(self, subject) -> CheckResult: if evidence is None: return CheckResult( check_id=self.check_id, - status=CheckStatus.SKIP, + status=CheckStatus.ERROR, evidence=["oracle evidence is unavailable"], duration_s=0, ) @@ -137,9 +137,6 @@ def check(self, evidence, *, maximum, width, tolerance, margin): for name, reward in evidence.canaries if reward > evidence.floor ] - # An empty run list must not pass vacuously. - if not evidence.canaries: - problems.append("no canary run was recorded") return problems, { "measured_floor": float(evidence.floor), "canary_rewards": { diff --git a/src/openenv/validation/parsers/harbor_toml.py b/src/openenv/validation/parsers/harbor_toml.py index ee0f1eb43..39bce5d73 100644 --- a/src/openenv/validation/parsers/harbor_toml.py +++ b/src/openenv/validation/parsers/harbor_toml.py @@ -1,4 +1,4 @@ -"""Pure Harbor task parsing and the explicit, operator-supplied pilot contract.""" +"""Pure Harbor task parsing and the explicit, operator-supplied semantic contract.""" import json import math diff --git a/tests/envs/test_harbor_validated_trial_hook.py b/tests/envs/test_harbor_validated_trial_hook.py index e338cf15a..015578cd0 100644 --- a/tests/envs/test_harbor_validated_trial_hook.py +++ b/tests/envs/test_harbor_validated_trial_hook.py @@ -97,6 +97,12 @@ def test_trial_hook_precedes_execution( class Trial: guarded = False + _close_logger_handler = trial_module.Trial._close_logger_handler + + def __init__(self): + self.config = SimpleNamespace(trial_name=tmp_path.name) + self.paths = SimpleNamespace(log_path=tmp_path / "trial.log") + trial_module.Trial._init_logger(self) @classmethod async def create(cls, trial_config): @@ -110,11 +116,14 @@ async def run(self): assert hook_mode == "omitted" or self.guarded events.append("run") await agent_run() + self._close_logger_handler() return SimpleNamespace( verifier_result=SimpleNamespace(rewards={"reward": 1.0}) ) trial = Trial() + handler = trial._log_handler + stream = handler.stream monkeypatch.setattr(trial_module, "Trial", Trial) def on_trial_created(created): @@ -152,6 +161,13 @@ def on_trial_created(created): ) ) + try: + assert stream.closed + assert handler not in trial.logger.handlers + assert trial._log_handler is None + finally: + trial._close_logger_handler() + assert build.call_args.kwargs["agent_timeout_sec"] == 12.5 assert build.call_args.kwargs["agent_step_limit"] == 3 assert build.call_args.kwargs["environment_kwargs"] is ( diff --git a/tests/test_validation/test_semantic_graders.py b/tests/test_validation/test_semantic_graders.py index 1e65a07b8..ec32cc06a 100644 --- a/tests/test_validation/test_semantic_graders.py +++ b/tests/test_validation/test_semantic_graders.py @@ -84,14 +84,14 @@ def test_canaries_must_not_earn_more_than_doing_nothing(): leaked = grader.run(subject(evidence(canaries=[("hardcoded.sh", "0.25")]))) assert leaked.status is CheckStatus.FAIL assert "hardcoded.sh" in leaked.evidence[0] - # An empty run list must not pass vacuously. - assert grader.run(subject(evidence())).status is CheckStatus.FAIL -def test_missing_evidence_skips(): - for grader in (OracleMaxGrader(), FloorGapGrader(), CanaryFloorGrader()): - assert grader.run(subject(None)).status is CheckStatus.SKIP - assert grader.depends_on == ("runtime.episode_determinism",) +@pytest.mark.parametrize( + "grader", [OracleMaxGrader(), FloorGapGrader(), CanaryFloorGrader()] +) +def test_missing_evidence_is_an_error(grader): + assert grader.run(subject(None)).status is CheckStatus.ERROR + assert grader.depends_on == ("runtime.episode_determinism",) @pytest.mark.parametrize( From d617a4910b6d476d14ed4c851f260dbe8d8c0415 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Wed, 11 Nov 2026 15:27:52 +0100 Subject: [PATCH 33/40] docs(validation): reconcile Harbor semantic and image contracts Keep both floor gaps, scope reward replay determinism to one-shot Harbor tasks, and specify conditional image retention. Refuse task-declared images and preserve the Dockerfile build requirement. --- docs/plans/rfc008-validation-completion.md | 18 ++++-- docs/source/reference/cli.md | 5 ++ rfcs/008-environment-auto-validation.md | 56 +++++++++++++------ src/openenv/validation/manifest.py | 5 +- .../schemas/manifest-v2.schema.json | 2 +- 5 files changed, 61 insertions(+), 25 deletions(-) diff --git a/docs/plans/rfc008-validation-completion.md b/docs/plans/rfc008-validation-completion.md index 85099b024..754996006 100644 --- a/docs/plans/rfc008-validation-completion.md +++ b/docs/plans/rfc008-validation-completion.md @@ -101,7 +101,9 @@ or GPU time. Owners are suggestions for discussion. `semantic.oracle_max` and `semantic.floor_gap` now grade Harbor tasks. The graders read a format-neutral `OracleEvidence`, so another binding only has to measure the -rewards. +rewards. Oracle tolerance and floor margin are fractions of the declared reward +range. Both the maximum and every oracle reward must exceed the measured floor +by at least the declared margin times the range width. **Remaining.** Served packages need an oracle execution contract first: the RFC names injected state and executable scripts, but not how either drives a served @@ -120,7 +122,12 @@ Validation core. Done. Harbor `task.toml` packages validate through the `harbor-semantic-v1` profile and an operator-supplied `validation/openenv.json` contract. Fresh Docker trials run a do-nothing agent, the oracle, an oracle replay and declared canaries, -all pinned to one immutable image. +all pinned to one immutable image. The binding requires Python 3.12 or newer and +`harbor==0.25.0` alongside `openenv[harbor]`. It refuses task-declared `docker_image` +values, so the NOP trial builds from the task's Dockerfile. The validator keeps +`openenv-validation:` only when the run has no FAIL or ERROR and was not +interrupted. It records the tag in `runtime.startup.measured` as +`retained_image_tag`. Otherwise it removes the tag. **Remaining.** Decide which runtime ids apply to one-shot tasks. Containment, network policy, resource bounds and oracle containment could apply. Observation @@ -155,8 +162,11 @@ writes the amendment, and the Hub pipeline team implements the operator panel. `semantic.verifier_determinism`, `semantic.verifier_portability`, `semantic.replayability` and `semantic.no_solution_leakage` have no graders. -Harbor tasks now establish `runtime.episode_determinism` with an oracle replay in a -fresh container, which these checks depend on. +Harbor tasks now establish `runtime.episode_determinism` when an oracle replay in a +fresh container reproduces every reward and the oracle exit code. Each trial is a +one-shot episode with no per-step observations. Served environments retain the +general comparison of observation, reward, done, tool output and state. The semantic +checks depend on the determinism finding. **Approach.** Verifier determinism runs the verifier on one final state several times and requires identical rewards. For `llm_judged` environments it requires diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index 8d2bca78e..edc5b3245 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -110,6 +110,11 @@ the raw trial evidence in the report bundle. Other checks are named SKIPs, so a passing task still reports WARN. This validates task packages, not the outer `harbor_env` server. +Task-declared `docker_image` values are refused. The do-nothing trial builds from +the task's Dockerfile. The validator keeps `openenv-validation:` only when +the run has no FAIL or ERROR and was not interrupted. It records the tag in +`runtime.startup.measured` as `retained_image_tag`. Otherwise it removes the tag. + The 16 implemented runtime checks cover startup, rewards, observation schemas, state continuity, network policy, host containment, resource bounds, episode isolation, oracle containment, seed forwarding, determinism, emitted trajectory records, tool diff --git a/rfcs/008-environment-auto-validation.md b/rfcs/008-environment-auto-validation.md index 2acda8472..3d9afaf95 100644 --- a/rfcs/008-environment-auto-validation.md +++ b/rfcs/008-environment-auto-validation.md @@ -183,7 +183,9 @@ The core promise is #778's verifier-sanity test, made precise: state — the declared range minimum may sit below it (an agent that deletes the repo earns a legal sub-floor reward, not a validation anomaly). The contract is a **gap check**: `reward(oracle) >= max - tolerance * width`, `reward(reset) = floor`, - `max - floor >= margin * width`, where `width = max - min`. + `max - floor >= margin * width`, and + `reward(oracle) - floor >= margin * width`, where `width = max - min`. + Both floor gaps must hold for every oracle run. - **Oracle containment is required.** The oracle (state or script) must be withheld from or isolated against the agent at serve time, and validation checks that it is. - **Reward range is declared** in the manifest, defaulting to `[0, 1]`; graders normalize against @@ -194,8 +196,9 @@ The core promise is #778's verifier-sanity test, made precise: **LLM-as-judge is an allowed reward.** Judged environments declare `llm_judged` and pin the judge configuration (model, version, params) in the manifest; the oracle check becomes -`>= max - declared tolerance * width` and determinism becomes variance-within-declared-bound rather than -bit-exact. RFC 004 rubrics are **leveraged, not required**: the contract stays spec-neutral +`>= max - declared tolerance * width`. Determinism becomes variance within the +declared bound rather than bit-exact. RFC 004 rubrics are optional. The contract stays +spec-neutral (graders read the manifest), but for the served OpenEnv format the rubric tree is the native satisfaction path — `LLMJudge` is the in-repo `llm_judged` implementation, and the introspectability and reward-attribution graders read `named_rubrics()`, explicit @@ -619,9 +622,10 @@ parsing these declarations does not claim they are enforced. Seed acceptance and empirical determinism are separate findings. A reset that silently drops its seed does not establish seed control, while a deterministic -environment may legitimately behave identically under different seeds. Replay -comparison includes observation, reward, done, tool output and state; only -policy-owned volatile metadata can be excluded. Authors cannot exclude fields. +environment may legitimately behave identically under different seeds. For served +environments, replay comparison includes observation, reward, done, tool output and +state. Only policy-owned volatile metadata can be excluded. Authors cannot exclude +fields. The one-shot Harbor comparison is specified in the Harbor task binding below. For `llm_judged`, the bounded variance path uses 20 completed identical-input fresh replays and population reward variance in reward-squared units, compared to the declared bound. The total run budget bounds all samples; fewer than 20 is incomplete. @@ -737,25 +741,41 @@ supply these declarations. The normalized schema-2 execution kind is `harbor_tas with a `process` agent boundary. Parsing uses Harbor's data-only Task loader and never creates a Trial or executes package code. +The profile requires Python 3.12 or newer and `harbor==0.25.0` alongside +`openenv[harbor]`. A missing or mismatched Harbor installation is a validator +environment error, not a package failure. + The profile supports single-step Linux tasks with one Dockerfile, one main Docker -service, a shared verifier and public networking. Accelerators, external MCP -services, task environment variables, credentials and Harbor extensions such as -healthchecks or artifacts are refused with the reason named. +service, a shared verifier and public networking. Task-declared `docker_image` +values are refused. The NOP trial must build from the task's Dockerfile. +Accelerators, external MCP services, task environment variables, credentials and +Harbor extensions such as healthchecks or artifacts are refused with the reason named. Execution runs fresh Harbor Trials in a fixed order: a do-nothing (NOP) trial, the oracle (`solution/solve.sh`), an oracle replay, and one trial per declared canary. The NOP trial builds the image. The validator inspects the actual agent container at `AGENT_START`, retains its immutable image under a run-owned tag, and pins every -later trial to that image and its effective container configuration. Each trial -runs on a private copy of the task whose digest must equal the validated source -digest before any overlay. A canary trial replaces `solution/solve.sh` in its copy -with the canary script and records the script's digest. Trials run with +later trial to that image and its effective container configuration. + +The validator keeps `openenv-validation:` only when the run has no FAIL or +ERROR and was not interrupted. It records the tag in `runtime.startup.measured` +as `retained_image_tag`. Otherwise it removes the tag. + +Each trial runs on a private copy of the task whose digest must equal the validated +source digest before any overlay. A canary trial replaces `solution/solve.sh` in its +copy with the canary script and records the script's digest. Trials run with `delete=False` and `keep_containers=True`, so the validator removes the exact run-owned Compose projects, including anonymous volumes registered from the containers' actual mounts, and verifies their absence. The oracle's exit code is read at `AGENT_END`, before sandbox log synchronization can overwrite it, and a later change fails the run. +For `harbor-semantic-v1`, one trial is a one-shot episode with no per-step +observations. A fresh oracle replay that reproduces every reward and the oracle +exit code establishes `runtime.episode_determinism`. This rule is specific to the +Harbor binding. Served environments retain the general replay comparison of +observation, reward, done, tool output and state. + Results map to RFC check ids: | Check | Evidence | @@ -763,16 +783,16 @@ Results map to RFC check ids: | `runtime.startup` | Every trial ran from the pinned image in a fresh container, and cleanup was verified. | | `runtime.reward_well_formed` | Every raw reward map matches Harbor's result and is finite, nonboolean and in range. | | `runtime.episode_determinism` | The oracle replay reproduces every reward and the exit code. | -| `semantic.oracle_max` | Both oracle runs exit 0 and earn at least `max - oracle_tolerance * width`. A task without `solution/solve.sh` fails. | -| `semantic.floor_gap` | `max - floor >= floor_margin * width`, and `oracle - floor >= floor_margin * width` because the oracle may sit up to the tolerance below the maximum. | +| `semantic.oracle_max` | Both oracle runs exit 0 and meet the oracle bound in the semantic contract above. A task without `solution/solve.sh` fails. | +| `semantic.floor_gap` | Both floor-gap conditions in the semantic contract above hold for every oracle run. | | `semantic.canary_floor` | Every declared canary earns no more than the measured floor. Selected only when canaries are declared. | The semantic ids depend on `runtime.episode_determinism`, so a diverging replay SKIPs them with the dependency named. Oracle tolerance and floor margin are fractions of the declared range, with `width = max - min`. Reward comparisons use exact -rationals parsed from the raw verifier output. Every other local check is a named SKIP for this -binding. It produces no WebSocket exchange, containment or resource measurement, -second verification of one final state, or leakage scan. A passing run is therefore +rationals parsed from the raw verifier output. Every other local check is a named +SKIP for this binding. It produces no WebSocket exchange, containment or resource +measurement, second verification of one final state, or leakage scan. A passing run is therefore WARN, and full RFC qualification still rejects these SKIPs. Canaries are operator-declared, known-wrong `solve.sh` replacements, at most eight diff --git a/src/openenv/validation/manifest.py b/src/openenv/validation/manifest.py index 3d829b49c..f54c75462 100644 --- a/src/openenv/validation/manifest.py +++ b/src/openenv/validation/manifest.py @@ -58,8 +58,9 @@ class RewardDeclaration(BaseModel): The oracle check requires `reward(oracle) >= max - oracle_tolerance * width`, where `width = max - min`. floor_margin (`float`): - Fraction of the declared range required between the maximum and the floor. - The gap check requires `max - measured_floor >= floor_margin * width`. + Fraction of the declared range required below the maximum and every oracle. + Both `max - measured_floor` and `reward(oracle) - measured_floor` must be + at least `floor_margin * width`. variance_tolerance (`float`, *optional*): Required iff the environment declares `llm_judged`; bounds reward variance for variance-mode determinism checks. diff --git a/src/openenv/validation/schemas/manifest-v2.schema.json b/src/openenv/validation/schemas/manifest-v2.schema.json index 6bb1d8c0e..541b5d8c0 100644 --- a/src/openenv/validation/schemas/manifest-v2.schema.json +++ b/src/openenv/validation/schemas/manifest-v2.schema.json @@ -273,7 +273,7 @@ }, "RewardDeclaration": { "additionalProperties": false, - "description": "Author-declared reward contract. Graders normalize measurements against it.\n\nDeclared tolerances are bounded by the severity policy; reports carry the declared\nvalues so hubs can apply stricter ceilings.\n\nAttributes:\n range (`tuple[float, float]`):\n Declared reward range, defaults to `(0.0, 1.0)`. Must be strictly increasing.\n oracle_tolerance (`float`):\n Fraction of the declared range allowed below the maximum.\n The oracle check requires `reward(oracle) >= max - oracle_tolerance * width`,\n where `width = max - min`.\n floor_margin (`float`):\n Fraction of the declared range required between the maximum and the floor.\n The gap check requires `max - measured_floor >= floor_margin * width`.\n variance_tolerance (`float`, *optional*):\n Required iff the environment declares `llm_judged`; bounds reward variance\n for variance-mode determinism checks.", + "description": "Author-declared reward contract. Graders normalize measurements against it.\n\nDeclared tolerances are bounded by the severity policy; reports carry the declared\nvalues so hubs can apply stricter ceilings.\n\nAttributes:\n range (`tuple[float, float]`):\n Declared reward range, defaults to `(0.0, 1.0)`. Must be strictly increasing.\n oracle_tolerance (`float`):\n Fraction of the declared range allowed below the maximum.\n The oracle check requires `reward(oracle) >= max - oracle_tolerance * width`,\n where `width = max - min`.\n floor_margin (`float`):\n Fraction of the declared range required below the maximum and every oracle.\n Both `max - measured_floor` and `reward(oracle) - measured_floor` must be\n at least `floor_margin * width`.\n variance_tolerance (`float`, *optional*):\n Required iff the environment declares `llm_judged`; bounds reward variance\n for variance-mode determinism checks.", "properties": { "floor_margin": { "minimum": 0.0, From 7a5af24c1225d93261b6dcbde1fd0b39929c6339 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Wed, 11 Nov 2026 18:52:36 +0100 Subject: [PATCH 34/40] chore(validation): sync generated schema descriptions Regenerates the RewardDeclaration description in the derived manifest and report schemas with scripts/sync_validation_schemas.py --fix after the range-normalized margin wording. Co-Authored-By: Claude Opus 5.5 --- src/openenv/validation/schemas/manifest.schema.json | 2 +- src/openenv/validation/schemas/report-v2.schema.json | 2 +- src/openenv/validation/schemas/report.schema.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/openenv/validation/schemas/manifest.schema.json b/src/openenv/validation/schemas/manifest.schema.json index d91a469c9..1c342e034 100644 --- a/src/openenv/validation/schemas/manifest.schema.json +++ b/src/openenv/validation/schemas/manifest.schema.json @@ -194,7 +194,7 @@ }, "RewardDeclaration": { "additionalProperties": false, - "description": "Author-declared reward contract. Graders normalize measurements against it.\n\nDeclared tolerances are bounded by the severity policy; reports carry the declared\nvalues so hubs can apply stricter ceilings.\n\nAttributes:\n range (`tuple[float, float]`):\n Declared reward range, defaults to `(0.0, 1.0)`. Must be strictly increasing.\n oracle_tolerance (`float`):\n The oracle check passes when `reward(oracle) >= max - oracle_tolerance`.\n floor_margin (`float`):\n The gap check requires `max - measured_floor >= floor_margin`.\n variance_tolerance (`float`, *optional*):\n Required iff the environment declares `llm_judged`; bounds reward variance\n for variance-mode determinism checks.", + "description": "Author-declared reward contract. Graders normalize measurements against it.\n\nDeclared tolerances are bounded by the severity policy; reports carry the declared\nvalues so hubs can apply stricter ceilings.\n\nAttributes:\n range (`tuple[float, float]`):\n Declared reward range, defaults to `(0.0, 1.0)`. Must be strictly increasing.\n oracle_tolerance (`float`):\n Fraction of the declared range allowed below the maximum.\n The oracle check requires `reward(oracle) >= max - oracle_tolerance * width`,\n where `width = max - min`.\n floor_margin (`float`):\n Fraction of the declared range required below the maximum and every oracle.\n Both `max - measured_floor` and `reward(oracle) - measured_floor` must be\n at least `floor_margin * width`.\n variance_tolerance (`float`, *optional*):\n Required iff the environment declares `llm_judged`; bounds reward variance\n for variance-mode determinism checks.", "properties": { "floor_margin": { "minimum": 0.0, diff --git a/src/openenv/validation/schemas/report-v2.schema.json b/src/openenv/validation/schemas/report-v2.schema.json index dc5b1f185..9da03461a 100644 --- a/src/openenv/validation/schemas/report-v2.schema.json +++ b/src/openenv/validation/schemas/report-v2.schema.json @@ -522,7 +522,7 @@ }, "RewardDeclaration": { "additionalProperties": false, - "description": "Author-declared reward contract. Graders normalize measurements against it.\n\nDeclared tolerances are bounded by the severity policy; reports carry the declared\nvalues so hubs can apply stricter ceilings.\n\nAttributes:\n range (`tuple[float, float]`):\n Declared reward range, defaults to `(0.0, 1.0)`. Must be strictly increasing.\n oracle_tolerance (`float`):\n The oracle check passes when `reward(oracle) >= max - oracle_tolerance`.\n floor_margin (`float`):\n The gap check requires `max - measured_floor >= floor_margin`.\n variance_tolerance (`float`, *optional*):\n Required iff the environment declares `llm_judged`; bounds reward variance\n for variance-mode determinism checks.", + "description": "Author-declared reward contract. Graders normalize measurements against it.\n\nDeclared tolerances are bounded by the severity policy; reports carry the declared\nvalues so hubs can apply stricter ceilings.\n\nAttributes:\n range (`tuple[float, float]`):\n Declared reward range, defaults to `(0.0, 1.0)`. Must be strictly increasing.\n oracle_tolerance (`float`):\n Fraction of the declared range allowed below the maximum.\n The oracle check requires `reward(oracle) >= max - oracle_tolerance * width`,\n where `width = max - min`.\n floor_margin (`float`):\n Fraction of the declared range required below the maximum and every oracle.\n Both `max - measured_floor` and `reward(oracle) - measured_floor` must be\n at least `floor_margin * width`.\n variance_tolerance (`float`, *optional*):\n Required iff the environment declares `llm_judged`; bounds reward variance\n for variance-mode determinism checks.", "properties": { "floor_margin": { "minimum": 0.0, diff --git a/src/openenv/validation/schemas/report.schema.json b/src/openenv/validation/schemas/report.schema.json index 104c69bd0..ca908b59d 100644 --- a/src/openenv/validation/schemas/report.schema.json +++ b/src/openenv/validation/schemas/report.schema.json @@ -353,7 +353,7 @@ }, "RewardDeclaration": { "additionalProperties": false, - "description": "Author-declared reward contract. Graders normalize measurements against it.\n\nDeclared tolerances are bounded by the severity policy; reports carry the declared\nvalues so hubs can apply stricter ceilings.\n\nAttributes:\n range (`tuple[float, float]`):\n Declared reward range, defaults to `(0.0, 1.0)`. Must be strictly increasing.\n oracle_tolerance (`float`):\n The oracle check passes when `reward(oracle) >= max - oracle_tolerance`.\n floor_margin (`float`):\n The gap check requires `max - measured_floor >= floor_margin`.\n variance_tolerance (`float`, *optional*):\n Required iff the environment declares `llm_judged`; bounds reward variance\n for variance-mode determinism checks.", + "description": "Author-declared reward contract. Graders normalize measurements against it.\n\nDeclared tolerances are bounded by the severity policy; reports carry the declared\nvalues so hubs can apply stricter ceilings.\n\nAttributes:\n range (`tuple[float, float]`):\n Declared reward range, defaults to `(0.0, 1.0)`. Must be strictly increasing.\n oracle_tolerance (`float`):\n Fraction of the declared range allowed below the maximum.\n The oracle check requires `reward(oracle) >= max - oracle_tolerance * width`,\n where `width = max - min`.\n floor_margin (`float`):\n Fraction of the declared range required below the maximum and every oracle.\n Both `max - measured_floor` and `reward(oracle) - measured_floor` must be\n at least `floor_margin * width`.\n variance_tolerance (`float`, *optional*):\n Required iff the environment declares `llm_judged`; bounds reward variance\n for variance-mode determinism checks.", "properties": { "floor_margin": { "minimum": 0.0, From b5683b1224234e067a5ea56d9fea51a750f47dc8 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 06:20:19 +0100 Subject: [PATCH 35/40] fix(validation): remove run-owned Compose build tags Inventory the native main-service tag before allocation and during cleanup. Verify its image identity and remove it with --no-prune after removing containers, including on rejected runs. --- src/openenv/validation/harbor.py | 19 +++ .../test_harbor_binding_inspection.py | 2 + .../test_validation/test_harbor_validation.py | 109 +++++++++++++++++- 3 files changed, 125 insertions(+), 5 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index 6418d4d8e..a35753089 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -116,6 +116,12 @@ def inventory(self, project): if any(not _RESOURCE_ID.fullmatch(item) for item in ids): raise NativeEvidenceError("invalid Docker resource identity") inventory[kind] = ids + images = self._docker( + "image", "ls", "-q", "--no-trunc", project + "-main:latest" + ).split() + if len(images) > 1 or any(not _IMAGE_ID.fullmatch(image) for image in images): + raise NativeEvidenceError("invalid Compose image identity") + inventory["images"] = images return inventory def capture(self, project, retained_tag, *, retain=True): @@ -231,6 +237,19 @@ def cleanup(self, project): remaining = [name for name in remaining if name in existing] if remaining: self._docker(*args, *remaining) + for image_ref in before["images"]: + tag = project + "-main:latest" + image = json.loads(self._docker("image", "inspect", tag))[0] + labels = image["Config"]["Labels"] + if ( + image["Id"] != image_ref + or labels.get("com.docker.compose.project") != project + or labels.get("com.docker.compose.service") != "main" + ): + raise NativeEvidenceError( + "Compose image ownership changed before removal" + ) + self._docker("image", "rm", "--no-prune", tag) after = self.inventory(project) if registered: existing = set(self._docker("volume", "ls", "-q").split()) diff --git a/tests/test_validation/test_harbor_binding_inspection.py b/tests/test_validation/test_harbor_binding_inspection.py index 9602d3d44..e39ab4aaa 100644 --- a/tests/test_validation/test_harbor_binding_inspection.py +++ b/tests/test_validation/test_harbor_binding_inspection.py @@ -58,6 +58,8 @@ def docker(*args): commands.append(args) if args[0] == "ps": return "\n".join(containers) + if args[:2] == ("image", "ls"): + return "" if args[:2] in (("network", "ls"), ("volume", "ls")): return "" if args == ("inspect", container["Id"]): diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index a8ff5d573..8a24cd8ad 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -79,6 +79,7 @@ def boundary(monkeypatch, task): cleanup_calls=[], preflight_error=None, interrupt_role=None, + on_build=None, ) class Inspector: @@ -166,6 +167,8 @@ async def stop(self, *, delete): self.paths.reward_json_path.symlink_to("outside-reward.json") async def start(self, *, force_build): + if self.role == "nop" and state.on_build is not None: + state.on_build(self.agent_environment.session_id) if state.pending_compose: self._openenv_cancelled_compose.append( { @@ -255,9 +258,22 @@ async def create(config): @pytest.fixture def image_docker(boundary, monkeypatch): state = SimpleNamespace( - images={"unrelated:tag": IMAGE}, removed=set(), commands=[], image_os="linux" + images={"unrelated:tag": IMAGE}, + removed=set(), + commands=[], + image_os="linux", + remove_error=False, + image_drift=False, + foreign_image=False, ) + def built(project): + state.images[project + "-main:latest"] = ( + "sha256:" + "e" * 64 if state.foreign_image else IMAGE + ) + + boundary.on_build = built + def docker(self, *args): state.commands.append(args) if args == ("info", "--format", "{{json .}}"): @@ -299,9 +315,33 @@ def docker(self, *args): if args[:2] == ("image", "tag"): state.images[args[3]] = args[2] return "" + if args[:2] == ("image", "ls"): + assert args[2:4] == ("-q", "--no-trunc") + return state.images.get(args[4], "") if args[:2] == ("image", "inspect"): - return json.dumps([{"Id": state.images[args[2]], "Os": state.image_os}]) + image = state.images[args[2]] + if state.image_drift and args[2].endswith("-main:latest"): + image = "sha256:" + "e" * 64 + project = args[2].removesuffix("-main:latest") + return json.dumps( + [ + { + "Id": image, + "Os": state.image_os, + "Config": { + "Labels": { + "com.docker.compose.project": "unrelated" + if state.foreign_image + else project, + "com.docker.compose.service": "main", + } + }, + } + ] + ) if args[:3] == ("image", "rm", "--no-prune"): + if state.remove_error and args[3].endswith("-main:latest"): + raise OSError("image removal failed") del state.images[args[3]] return "" if args[:3] == ("rm", "-f", "-v"): @@ -443,7 +483,12 @@ def test_retained_image_policy(task, boundary, image_docker, tmp_path, outcome): report = run_validation(task, artifacts_dir=artifacts) tag = "openenv-validation:" + boundary.trials[0].config.trial_name.split("-")[2] measured = result(report, "runtime.startup").measured - removals = [cmd for cmd in image_docker.commands if cmd[:2] == ("image", "rm")] + compose_tag = boundary.trials[0].agent_environment.session_id + "-main:latest" + removals = [ + cmd + for cmd in image_docker.commands + if cmd[:2] == ("image", "rm") and cmd[3] != compose_tag + ] if outcome == "kept": assert report.verdict is Verdict.WARN assert measured["retained_image_tag"] == tag @@ -463,6 +508,51 @@ def test_retained_image_policy(task, boundary, image_docker, tmp_path, outcome): ) +@pytest.mark.parametrize("keep", [True, False], ids=["kept", "removed"]) +def test_compose_build_tag_is_removed_on_every_run( + task, boundary, image_docker, tmp_path, keep +): + if not keep: + boundary.rewards["nop"] = b'{"reward":1}' + report = run_validation(task, artifacts_dir=tmp_path / "compose-tags") + project = boundary.trials[0].agent_environment.session_id + compose_tag = project + "-main:latest" + retained_tag = "openenv-validation:" + project.split("-")[2] + expected = {"unrelated:tag": IMAGE} + if keep: + expected[retained_tag] = IMAGE + assert ( + result(report, "runtime.startup").measured["retained_image_tag"] + == retained_tag + ) + assert image_docker.images == expected + removal = ("image", "rm", "--no-prune", compose_tag) + index = image_docker.commands.index(removal) + assert image_docker.commands[index - 1] == ("image", "inspect", compose_tag) + assert image_docker.removed == {trial.role for trial in boundary.trials} + + +@pytest.mark.parametrize("defect", ["remove_error", "image_drift", "foreign_image"]) +def test_compose_tag_cleanup_failure_is_a_startup_error( + task, boundary, image_docker, tmp_path, defect +): + setattr(image_docker, defect, True) + artifacts = tmp_path / "compose-cleanup" + report = run_validation(task, artifacts_dir=artifacts) + startup = result(report, "runtime.startup") + assert startup.status is CheckStatus.ERROR + assert "retained_image_tag" not in startup.measured + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert not native["cleanup"]["verified"] + assert native["trials"][0]["failure"] == "CleanupNotVerified" + assert "unrelated:tag" in image_docker.images + if defect in {"image_drift", "foreign_image"}: + assert not any( + cmd[:2] == ("image", "rm") and cmd[3].endswith("-main:latest") + for cmd in image_docker.commands + ) + + @pytest.mark.parametrize("error", [KeyboardInterrupt, OSError]) def test_retained_image_removed_when_reporting_does_not_complete( task, boundary, image_docker, tmp_path, monkeypatch, error @@ -746,6 +836,9 @@ def test_host_inspector_scopes_inventory_and_cleanup_to_owned_project(monkeypatc def docker(*args): commands.append(args) + if args[:2] == ("image", "ls"): + assert args == ("image", "ls", "-q", "--no-trunc", project + "-main:latest") + return "" if args == ("volume", "ls", "-q"): return "\n".join(resources["volumes"]) if args[0] == "inspect": @@ -786,7 +879,7 @@ def docker(*args): monkeypatch.setattr(inspector, "_docker", docker) assert inspector.cleanup(project)["verified"] - assert commands[-3:] == [ + assert commands[-4:] == [ ( "ps", "-aq", @@ -809,6 +902,7 @@ def docker(*args): "--filter", f"label=com.docker.compose.project={project}", ), + ("image", "ls", "-q", "--no-trunc", project + "-main:latest"), ] with pytest.raises(ValueError, match="run-owned"): inspector.cleanup("somebody-elses-project") @@ -827,7 +921,12 @@ def test_anonymous_volumes_are_registered_removed_and_rechecked( def inventory(value): assert value == project # The anonymous volume deliberately has no Compose project label. - return {"containers": list(containers), "networks": [], "volumes": []} + return { + "containers": list(containers), + "networks": [], + "volumes": [], + "images": [], + } def docker(*args): if args == ("inspect", "c" * 64): From 5cfa06acd8da2e9ac2aaea04edcb055f6b923387 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 06:21:16 +0100 Subject: [PATCH 36/40] fix(validation): preserve errors during image cleanup Keep the original reporting exception when retained-image removal also fails. Mark cleanup incomplete and attach a safe cleanup diagnostic to the original exception. --- src/openenv/validation/harbor.py | 14 ++++++--- .../test_validation/test_harbor_validation.py | 31 ++++++++++++++++++- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index a35753089..e15e083f0 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -840,7 +840,6 @@ def validate_harbor( record["harbor"]["commit_id"] = installed.get("vcs_info", {}).get("commit_id") files = {} inspector = DockerInspector() - completed = False try: attempted = False problem = _unsupported(read_harbor_task(subject.root), contract) @@ -1173,8 +1172,15 @@ def validate_harbor( path.chmod(0o600) checksums += f"{hashlib.sha256(raw).hexdigest()} {name}\n" (artifacts_dir / "SHA256SUMS").write_text(checksums) - completed = True return report - finally: - if not completed: + except BaseException as original: + try: inspector.remove_image() + except (Exception, KeyboardInterrupt) as exc: + record["cleanup"].update( + verified=False, completed=False, failure=type(exc).__name__ + ) + original.add_note( + f"retained image removal failed ({type(exc).__name__}); cleanup incomplete" + ) + raise diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 8a24cd8ad..c9521306a 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -263,6 +263,7 @@ def image_docker(boundary, monkeypatch): commands=[], image_os="linux", remove_error=False, + retained_remove_error=False, image_drift=False, foreign_image=False, ) @@ -340,7 +341,10 @@ def docker(self, *args): ] ) if args[:3] == ("image", "rm", "--no-prune"): - if state.remove_error and args[3].endswith("-main:latest"): + if (state.remove_error and args[3].endswith("-main:latest")) or ( + state.retained_remove_error + and args[3].startswith("openenv-validation:") + ): raise OSError("image removal failed") del state.images[args[3]] return "" @@ -567,6 +571,31 @@ def interrupted(*args, **kwargs): assert len(boundary.trials) == 3 +@pytest.mark.parametrize("error", [KeyboardInterrupt, OSError]) +def test_image_removal_failure_preserves_the_reporting_exception( + task, boundary, image_docker, tmp_path, monkeypatch, error +): + original = error() + cleanup = {} + + def interrupted(*args, **kwargs): + nonlocal cleanup + cleanup = kwargs["cleanup"] + raise original + + image_docker.retained_remove_error = True + monkeypatch.setattr(harbor, "write_runtime_bundle", interrupted) + with pytest.raises(error) as caught: + run_validation(task, artifacts_dir=tmp_path / "interrupted-report") + assert caught.value is original + assert not cleanup["verified"] and not cleanup["completed"] + assert cleanup["failure"] == "OSError" + assert ( + "retained image removal failed (OSError); cleanup incomplete" + in original.__notes__ + ) + + def test_build_failure_has_no_retained_tag_to_remove(task, boundary, image_docker): boundary.exception_type = "RuntimeError" report = run_validation(task) From c03bbe9804897e50af8041f1c3777a582b2f0bf8 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 06:22:31 +0100 Subject: [PATCH 37/40] fix(validation): grade Harbor supervisor timeouts as errors Name the effective validator build/start budget in startup errors. Keep Harbor setup, agent and verifier timeouts as phase-specific package failures, and treat unexpected timeouts as validator errors. --- src/openenv/validation/harbor.py | 21 +++++-- .../test_validation/test_harbor_validation.py | 60 +++++++++++++++---- 2 files changed, 65 insertions(+), 16 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index e15e083f0..6654e1eec 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -529,7 +529,9 @@ async def _trial( record["execution_task_digest"] = digest_before record["task_overlay"] = overlay budget = contract.resources.episode_timeout_s - build_budget = DockerValidationProvider().build_timeout_s + build_budget = ( + DockerValidationProvider().build_timeout_s if not expected else budget + ) config = TrialConfig( task=TaskConfig(path=task_copy.resolve()), agent=AgentConfig( @@ -557,7 +559,7 @@ async def _trial( record["trial_config_digest"] = _digest(effective_config) record["resources"] = contract.resources.model_dump() trial = await _create_trial(config) - trial._environment_build_timeout_sec = build_budget if not expected else budget + trial._environment_build_timeout_sec = build_budget if not expected: record["build_timeout_s"] = build_budget start_environment = trial.agent_environment.start @@ -629,8 +631,11 @@ async def verification_start(_event): ) record["reward_error"] = True raise NativeEvidenceError(f"Harbor verifier reward failed ({failure})") + if failure == "EnvironmentStartTimeoutError": + raise NativeEvidenceError( + f"validator build/start exceeded its {build_budget:g}-second budget" + ) subject_phase = { - "EnvironmentStartTimeoutError": "environment build/start", "SandboxBuildFailedError": "environment build/start", "AgentSetupTimeoutError": "agent setup", "AgentTimeoutError": "agent", @@ -670,8 +675,11 @@ async def verification_start(_event): except (Exception, KeyboardInterrupt) as exc: record["failure"] = type(exc).__name__ if isinstance(exc, TimeoutError): - record["subject_error"] = phase - record["reason"] = f"Harbor subject timed out during {phase}" + record["reason"] = ( + f"validator build/start exceeded its {build_budget:g}-second budget" + if phase == "environment build/start" + else f"unexpected Harbor timeout during {phase}" + ) # Error categories are safe; task-authored exception messages/logs are not. if isinstance(exc, NativeEvidenceError): record["reason"] = str(exc) @@ -946,6 +954,9 @@ def validate_harbor( ] }, ) + startup.evidence.extend( + trial["reason"] for trial in broken if "reason" in trial + ) if malformed: reward_result = _result( "runtime.reward_well_formed", diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index c9521306a..e2783343d 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -200,10 +200,16 @@ async def run(self): await hook(None) if self.role == state.interrupt_role: raise KeyboardInterrupt - await asyncio.wait_for( - asyncio.sleep(state.agent_delay), - timeout=self.config.agent.override_timeout_sec, - ) + try: + await asyncio.wait_for( + asyncio.sleep(state.agent_delay), + timeout=self.config.agent.override_timeout_sec, + ) + except TimeoutError: + return SimpleNamespace( + exception_info=SimpleNamespace(exception_type="AgentTimeoutError"), + verifier_result=None, + ) if state.timeout: raise TimeoutError() raw = state.rewards.get(self.role, state.rewards[self.agent]) @@ -685,16 +691,13 @@ def test_source_identity_exception_timeout_fail_closed(task, boundary, flag): setattr(boundary, flag, True) report = run_validation(task) assert report.verdict is Verdict.FAIL - assert result(report, "runtime.startup").status is ( - CheckStatus.FAIL if flag == "timeout" else CheckStatus.ERROR - ) + assert result(report, "runtime.startup").status is CheckStatus.ERROR @pytest.mark.parametrize( "exception_type,phase", [ ("RuntimeError", "environment"), - ("EnvironmentStartTimeoutError", "environment"), ("AgentTimeoutError", "agent"), ("AgentSetupTimeoutError", "agent setup"), ("VerifierTimeoutError", "verifier"), @@ -1019,11 +1022,17 @@ def test_nop_build_has_a_separate_supervisor_budget( artifacts = tmp_path / "build-budget" report = run_validation(task, artifacts_dir=artifacts) startup = result(report, "runtime.startup") - assert startup.status is (CheckStatus.FAIL if timeout_phase else CheckStatus.PASS) + assert startup.status is ( + CheckStatus.ERROR + if timeout_phase == "build" + else CheckStatus.FAIL + if timeout_phase == "agent" + else CheckStatus.PASS + ) if timeout_phase: assert len(boundary.trials) == 1 - phase = "environment" if timeout_phase == "build" else "agent" - assert phase in startup.evidence[0] + phase = "build/start" if timeout_phase == "build" else "agent" + assert any(phase in reason for reason in startup.evidence) assert ( result(report, "runtime.episode_determinism").status is not CheckStatus.PASS ) @@ -1034,6 +1043,35 @@ def test_nop_build_has_a_separate_supervisor_budget( assert native["trials"][0]["build_timeout_s"] == build_budget +@pytest.mark.parametrize("timeout_source", ["supervisor", "harbor", "unexpected"]) +def test_validator_build_start_timeout_is_an_error( + task, boundary, monkeypatch, timeout_source +): + from openenv.validation.providers import docker + + monkeypatch.setattr( + docker, + "DockerValidationProvider", + lambda: SimpleNamespace(build_timeout_s=0.01), + ) + if timeout_source == "supervisor": + boundary.start_delay = 0.1 + elif timeout_source == "harbor": + boundary.exception_type = "EnvironmentStartTimeoutError" + else: + boundary.timeout = True + report = run_validation(task) + startup = result(report, "runtime.startup") + assert startup.status is CheckStatus.ERROR + failures = startup.measured["failures"] + if timeout_source == "unexpected": + assert failures == ["unexpected Harbor timeout during agent"] + else: + assert failures == ["validator build/start exceeded its 0.01-second budget"] + assert failures[0] in startup.evidence + assert len(boundary.trials) == 1 + + def test_harbor_report_preserves_resolved_space_target(task, boundary, tmp_path): target = "spaces/openenv/example@" + "d" * 40 artifacts = tmp_path / "space-report" From bf8744686c23b9df67e87776e12ed5d9d2a7dbb6 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 06:23:52 +0100 Subject: [PATCH 38/40] test(validation): exercise Harbor startup budgets and phase errors Run startup through Harbor 0.25.0 Trial._start_agent_environment and catch regressions to the NOP build budget or RuntimeError phase gate. Read and record the existing Harbor timeout before overriding it so a missing binding fails closed. --- src/openenv/validation/harbor.py | 1 + .../test_validation/test_harbor_validation.py | 76 ++++++++++++++++++- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index 6654e1eec..cb1afdae9 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -559,6 +559,7 @@ async def _trial( record["trial_config_digest"] = _digest(effective_config) record["resources"] = contract.resources.model_dump() trial = await _create_trial(config) + record["harbor_build_timeout_s"] = trial._environment_build_timeout_sec trial._environment_build_timeout_sec = build_budget if not expected: record["build_timeout_s"] = build_budget diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index e2783343d..9f2316ada 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -12,6 +12,7 @@ from openenv.validation.parsers.harbor_toml import ( HarborTaskParser, read_harbor_contract, + read_harbor_task, ) from openenv.validation.runner import run_validation, source_digest from openenv.validation.types import CheckStatus, Level, SignatureKind, Verdict @@ -66,6 +67,7 @@ def boundary(monkeypatch, task): oracle_exit=None, exception=False, exception_type=None, + exception_phase="environment build/start", no_identity=False, timeout=False, retained=True, @@ -151,6 +153,9 @@ def __init__(self, config): ) self.hooks = {} self._openenv_cancelled_compose = [] + self._environment_build_timeout_sec = read_harbor_task( + config.task.path + ).config.environment.build_timeout_sec state.excluded_copied.append( (config.task.path / "environment/.worktrees").exists() ) @@ -189,12 +194,28 @@ def add_hook(self, event, callback): async def run(self): for hook in self.hooks.get("environment-start", []): await hook(None) - if state.exception_type in ("RuntimeError", "EnvironmentStartTimeoutError"): + if ( + state.exception_type in ("RuntimeError", "EnvironmentStartTimeoutError") + and state.exception_phase == "environment build/start" + ): return SimpleNamespace( exception_info=SimpleNamespace(exception_type=state.exception_type), verifier_result=None, ) - await self.agent_environment.start(force_build=False) + from harbor.trial.trial import ( + EnvironmentStartTimeoutError, + Trial as HarborTrial, + ) + + try: + await HarborTrial._start_agent_environment(self) + except EnvironmentStartTimeoutError: + return SimpleNamespace( + exception_info=SimpleNamespace( + exception_type="EnvironmentStartTimeoutError" + ), + verifier_result=None, + ) if not state.no_identity: for hook in self.hooks.get("agent-start", []): await hook(None) @@ -712,6 +733,16 @@ def test_subject_failures_name_the_phase(task, boundary, exception_type, phase): assert result(report, "runtime.reward_well_formed").status is CheckStatus.SKIP +def test_runtime_error_outside_build_start_is_a_validator_error(task, boundary): + boundary.exception_type = "RuntimeError" + boundary.exception_phase = "verifier" + report = run_validation(task) + startup = result(report, "runtime.startup") + assert startup.status is CheckStatus.ERROR + assert startup.measured["failures"] == ["Harbor Trial returned exception_info"] + assert result(report, "runtime.reward_well_formed").status is CheckStatus.SKIP + + def test_cleanup_failure_blocks(task, boundary, tmp_path): boundary.cleanup_ok = False artifacts = tmp_path / "cleanup" @@ -1072,6 +1103,47 @@ def test_validator_build_start_timeout_is_an_error( assert len(boundary.trials) == 1 +def test_real_harbor_start_uses_build_budget_then_episode_budgets( + task, boundary, monkeypatch +): + timeouts = [] + wait_for = asyncio.wait_for + + async def record_timeout(awaitable, timeout): + timeouts.append(timeout) + return await wait_for(awaitable, timeout) + + monkeypatch.setattr(asyncio, "wait_for", record_timeout) + report = run_validation(task) + assert result(report, "runtime.startup").status is CheckStatus.PASS + # Harbor starts each environment, then runs its agent under a separate budget. + assert timeouts == [600, 10, 10, 10, 10, 10] + assert [trial.role for trial in boundary.trials] == [ + "nop", + "oracle", + "oracle-replay", + ] + + +def test_missing_harbor_build_timeout_binding_is_a_startup_error( + task, boundary, monkeypatch +): + create = harbor._create_trial + + async def missing_binding(config): + trial = await create(config) + del trial._environment_build_timeout_sec + return trial + + monkeypatch.setattr(harbor, "_create_trial", missing_binding) + report = run_validation(task) + startup = result(report, "runtime.startup") + assert startup.status is CheckStatus.ERROR + assert startup.measured["failures"] == ["AttributeError"] + assert "retained_image_tag" not in startup.measured + assert len(boundary.trials) == 1 + + def test_harbor_report_preserves_resolved_space_target(task, boundary, tmp_path): target = "spaces/openenv/example@" + "d" * 40 artifacts = tmp_path / "space-report" From 7df6474e5a022cb2279db1d812fad54f9f8ba0b0 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 06:25:18 +0100 Subject: [PATCH 39/40] docs(validation): specify Harbor limits and cleanup outcomes Document unsupported task shapes and providers, validator and phase budgets, contract input restrictions, and the pinned Harbor requirement. Record the prebuilt-only task follow-up and image cleanup behavior. --- docs/plans/rfc008-validation-completion.md | 4 +++ docs/source/reference/cli.md | 36 +++++++++++++++++++--- rfcs/008-environment-auto-validation.md | 33 +++++++++++++++----- 3 files changed, 61 insertions(+), 12 deletions(-) diff --git a/docs/plans/rfc008-validation-completion.md b/docs/plans/rfc008-validation-completion.md index 754996006..050aba5a2 100644 --- a/docs/plans/rfc008-validation-completion.md +++ b/docs/plans/rfc008-validation-completion.md @@ -129,6 +129,10 @@ values, so the NOP trial builds from the task's Dockerfile. The validator keeps interrupted. It records the tag in `runtime.startup.measured` as `retained_image_tag`. Otherwise it removes the tag. +**Follow-up.** Tasks that ship only a prebuilt image cannot run under +`harbor-semantic-v1`. Some Terminal Bench tasks are affected. Supporting them needs +a reviewable Dockerfile path or digest-pinned image support. + **Remaining.** Decide which runtime ids apply to one-shot tasks. Containment, network policy, resource bounds and oracle containment could apply. Observation schema, state contract and tool declaration accuracy do not. diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index edc5b3245..ca8084495 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -76,10 +76,15 @@ and `harbor==0.25.0` installed alongside `openenv[harbor]`: pip install "openenv[harbor]" "harbor==0.25.0" ``` +A missing or mismatched Harbor installation exits 3 and names `harbor==0.25.0`. + Add a `validation/openenv.json` contract with `profile: "harbor-semantic-v1"`, `reward_key`, `reward`, `resources` and `types`. The reward range, oracle tolerance and floor margin must be explicit. Oracle tolerance and floor margin are fractions -of the declared reward range. This is a minimal complete contract: +of the declared reward range. Numeric strings in numeric fields, oversized integers +and excessive JSON nesting fail `static.manifest`. Reward and resource numbers must +retain their exact value when stored as floats. `resources.gpu_types` remains a list +of strings. This is a minimal complete contract: ```json { @@ -110,10 +115,31 @@ the raw trial evidence in the report bundle. Other checks are named SKIPs, so a passing task still reports WARN. This validates task packages, not the outer `harbor_env` server. -Task-declared `docker_image` values are refused. The do-nothing trial builds from -the task's Dockerfile. The validator keeps `openenv-validation:` only when -the run has no FAIL or ERROR and was not interrupted. It records the tag in -`runtime.startup.measured` as `retained_image_tag`. Otherwise it removes the tag. +Supported tasks use Linux, a single step, one Dockerfile, a shared verifier and +public networking. Unsupported shapes include multi-step or non-Linux tasks, +task-declared `environment.docker_image`, missing Dockerfiles, task-authored Compose +files and separate verifier environments. GPU/TPU or external MCP requirements, +credentials, non-public or differing phase networks, task environment variables, +healthchecks, skills, verifier collection and artifact extensions are also +unsupported. A supplied provider, including `--space`, is unsupported. Each case +gives `runtime.startup` SKIP with a named reason and runs no trials. With valid +static declarations and an oracle, the verdict is WARN and the exit code is 0. +A missing `solution/solve.sh` still FAILs `semantic.oracle_max` at the semantic level, +including with `--skip-build` or `--space`. + +The initial image build and environment start share a separate 600-second validator +budget. Exceeding it gives `runtime.startup` ERROR. Later environment starts use the +episode budget. There is no shared outer trial deadline. Harbor bounds setup, agent +and verifier separately with the declared `resources.episode_timeout_s`. A timeout +in one of these phases is a package FAIL that names the phase. + +The do-nothing trial builds from the task's Dockerfile. The validator keeps +`openenv-validation:` only when the run has no FAIL or ERROR and was not +interrupted. It records the tag in `runtime.startup.measured` as +`retained_image_tag`. Otherwise it removes the tag. The run-owned Compose build tag +is removed on every run. Tag removal verifies image identity and uses `--no-prune`. +A removal failure marks cleanup incomplete and gives a startup ERROR. During an +abort, cleanup preserves the original exception and records the removal failure. The 16 implemented runtime checks cover startup, rewards, observation schemas, state continuity, network policy, host containment, resource bounds, episode isolation, diff --git a/rfcs/008-environment-auto-validation.md b/rfcs/008-environment-auto-validation.md index 3d9afaf95..9d1183a1a 100644 --- a/rfcs/008-environment-auto-validation.md +++ b/rfcs/008-environment-auto-validation.md @@ -739,17 +739,26 @@ reward key, range, oracle tolerance, floor margin, finite resource budgets and, optionally, a directory of canaries. Harbor metadata and observed rewards never supply these declarations. The normalized schema-2 execution kind is `harbor_task`, with a `process` agent boundary. Parsing uses Harbor's data-only Task loader and -never creates a Trial or executes package code. +never creates a Trial or executes package code. Numeric strings in numeric fields, +oversized integers and excessive JSON nesting fail `static.manifest`. Reward and +resource numbers must retain their exact value when stored as floats. +`resources.gpu_types` remains a list of strings. The profile requires Python 3.12 or newer and `harbor==0.25.0` alongside `openenv[harbor]`. A missing or mismatched Harbor installation is a validator -environment error, not a package failure. +environment error. It exits 3 and names `harbor==0.25.0`. The profile supports single-step Linux tasks with one Dockerfile, one main Docker -service, a shared verifier and public networking. Task-declared `docker_image` -values are refused. The NOP trial must build from the task's Dockerfile. -Accelerators, external MCP services, task environment variables, credentials and -Harbor extensions such as healthchecks or artifacts are refused with the reason named. +service, a shared verifier and public networking. Unsupported shapes include +multi-step or non-Linux tasks, task-declared `environment.docker_image`, missing +Dockerfiles, task-authored Compose files and separate verifier environments. +GPU/TPU or external MCP requirements, credentials, non-public or differing phase +networks, task environment variables, healthchecks, skills, verifier collection and +artifact extensions are also unsupported. A supplied provider, including `--space`, +is unsupported. Each case gives `runtime.startup` SKIP with a named reason and runs +no trials. With valid static declarations and an oracle, the verdict is WARN and +the exit code is 0. A missing `solution/solve.sh` still FAILs `semantic.oracle_max` +at the semantic level, including with `--skip-build` or `--space`. Execution runs fresh Harbor Trials in a fixed order: a do-nothing (NOP) trial, the oracle (`solution/solve.sh`), an oracle replay, and one trial per declared canary. @@ -757,9 +766,19 @@ The NOP trial builds the image. The validator inspects the actual agent containe at `AGENT_START`, retains its immutable image under a run-owned tag, and pins every later trial to that image and its effective container configuration. +The initial image build and environment start share a separate 600-second validator +budget. Exceeding it gives `runtime.startup` ERROR. Later environment starts use the +episode budget. There is no shared outer trial deadline. Harbor bounds setup, agent +and verifier separately with the declared `resources.episode_timeout_s`. A timeout +in one of these phases is a package FAIL that names the phase. + The validator keeps `openenv-validation:` only when the run has no FAIL or ERROR and was not interrupted. It records the tag in `runtime.startup.measured` -as `retained_image_tag`. Otherwise it removes the tag. +as `retained_image_tag`. Otherwise it removes the tag. The run-owned Compose build +tag is removed on every run. Tag removal verifies image identity and uses +`--no-prune`. A removal failure marks cleanup incomplete and gives a startup ERROR. +During an abort, cleanup preserves the original exception and records the removal +failure. Each trial runs on a private copy of the task whose digest must equal the validated source digest before any overlay. A canary trial replaces `solution/solve.sh` in its From 46301f8a240c909e0d061942b0932b4ae4426f01 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 12 Nov 2026 08:39:50 +0000 Subject: [PATCH 40/40] Honor coding audit deadline during sandbox readiness --- examples/coding_audit/hf.py | 7 ++----- examples/coding_audit/test_hf.py | 25 ++++++++++++++++++++++--- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/examples/coding_audit/hf.py b/examples/coding_audit/hf.py index f615bd382..7868cf34e 100644 --- a/examples/coding_audit/hf.py +++ b/examples/coding_audit/hf.py @@ -381,7 +381,7 @@ def _create(self, image, record, *, grader=False): # SDK readiness checks the RPC server; the Jobs status can still lag. # Reconcile only SCHEDULING within the caller's existing deadline, and # recheck identity and the immutable image on every observation. - for attempt in range(20): + while True: job = self._inspect(sandbox_id) if job.docker_image != image: raise HFBackendError("sandbox control-plane image mismatch") @@ -389,10 +389,7 @@ def _create(self, image, record, *, grader=False): break if job.status.stage != "SCHEDULING": raise HFBackendError("sandbox control-plane state is not runnable") - if attempt < 19: - time.sleep(0.5) - else: - raise HFBackendError("sandbox control-plane readiness did not converge") + time.sleep(0.5) # The SDK stat/read pair otherwise permits up to 512 MiB per read. sandbox.files.MAX_READ_BYTES = self.max_bytes sandbox.files.mkdir(self.workdir) diff --git a/examples/coding_audit/test_hf.py b/examples/coding_audit/test_hf.py index 2acf9c0db..a2a994b24 100644 --- a/examples/coding_audit/test_hf.py +++ b/examples/coding_audit/test_hf.py @@ -559,7 +559,7 @@ def test_rpc_ready_sandbox_waits_for_running_control_plane(harness, monkeypatch) def lagged_inspect(**kwargs): job = inspect(**kwargs) if not h.api.cancelled: - job.status.stage = "SCHEDULING" if not observed else "RUNNING" + job.status.stage = "SCHEDULING" if len(observed) < 25 else "RUNNING" observed.append(job.status.stage) return job @@ -567,12 +567,31 @@ def lagged_inspect(**kwargs): monkeypatch.setattr("examples.coding_audit.hf.time.sleep", lambda _: None) record = h.backend.execute_script(":", "reference") assert record["error"] is None - assert observed == ["SCHEDULING", "RUNNING"] + assert observed == ["SCHEDULING"] * 25 + ["RUNNING"] assert record["cleanup_verified"] assert h.factory.created[0].commands -@pytest.mark.parametrize("stage", ["SCHEDULING", "ERROR", "COMPLETED", "UNKNOWN"]) +def test_scheduling_reconciliation_stops_at_caller_deadline(harness): + h = harness + inspect = h.api.inspect_job + + def never_running(**kwargs): + job = inspect(**kwargs) + if not h.api.cancelled: + job.status.stage = "SCHEDULING" + return job + + h.api.inspect_job = never_running + h.backend.timeout = 0.02 + started = time.monotonic() + record = h.backend.execute_script("dangerous", "reference") + assert time.monotonic() - started < 0.5 + assert "deadline" in record["error"] and record["cleanup_verified"] + assert not h.factory.created[0].commands + + +@pytest.mark.parametrize("stage", ["ERROR", "COMPLETED", "UNKNOWN"]) def test_nonrunning_control_plane_never_executes_and_is_cleaned_up( harness, monkeypatch, stage ):