From 37469738b4e77453ff4e95c7bf9b41a27c587de7 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 14:34:11 +0100 Subject: [PATCH 1/7] feat(validation): run Harbor validation trials on HF Sandbox `openenv validate --sandbox hf-sandbox [--flavor F] [--namespace N]` runs each harbor-semantic-v1 trial (do-nothing, oracle, oracle replay and canaries) as its own Hugging Face Job through Harbor's hf-sandbox environment, instead of on the local Docker daemon. The task must pin environment.docker_image by digest, so nothing is built and every trial starts at once. HFJobsInspector labels each job with the run and trial role, reads it back at AGENT_START (pinned image, requested flavor, RUNNING), and after each trial cancels any labelled job still running and verifies that all of them stop. Check ids, graders and thresholds are unchanged. Docker validation behaves as before. Co-Authored-By: Claude Opus 5.5 --- docs/plans/rfc008-validation-completion.md | 16 +- docs/source/reference/cli.md | 37 ++- rfcs/008-environment-auto-validation.md | 18 +- src/openenv/cli/commands/validate.py | 60 +++- src/openenv/validation/harbor.py | 306 +++++++++++++++--- src/openenv/validation/runner.py | 24 +- tests/test_validation/test_cli_validate.py | 39 +++ .../test_harbor_binding_supervision.py | 80 +++++ .../test_validation/test_harbor_validation.py | 253 ++++++++++++++- 9 files changed, 780 insertions(+), 53 deletions(-) diff --git a/docs/plans/rfc008-validation-completion.md b/docs/plans/rfc008-validation-completion.md index db0fb36a84..84303e606b 100644 --- a/docs/plans/rfc008-validation-completion.md +++ b/docs/plans/rfc008-validation-completion.md @@ -43,7 +43,8 @@ Three limits apply across bindings: - **Providers.** Docker-local enforces `public` and `no-network` networking and refuses allowlists and GPUs. The HF Sandbox provider refuses every launch with `missing provider capability: immutable_image`, because nothing yet binds a Space - revision to an image digest. + revision to an image digest. Harbor tasks with a digest-pinned image can run + their trials on HF Jobs with `--sandbox hf-sandbox`. - **Served oracles.** RFC 008 does not yet say how to execute an oracle against a served WebSocket environment, so served packages report their semantic checks as named SKIPs. @@ -92,7 +93,7 @@ or GPU time. Owners are suggestions for discussion. | 3 | Verifier strength | #28, #26 procedure | Harbor canaries done; panel open | M, then L | Validation core and Hub pipeline | | 4 | Determinism, replay and leakage graders | #30, #39, #40, #44 | Open | M | Validation core | | 5 | Static graders and complete reports | #1, #3, #4, #8, #9, #10, #12, #41, #42 | Reports done; graders open | S to M | Any contributor | -| 6 | Cloud execution on HF Sandbox | Provider qualification | Open | M | Hub pipeline | +| 6 | Cloud execution on HF Sandbox | Provider qualification | Harbor done; Spaces open | M | Hub pipeline | | 7 | Statistical reference procedure | #13 to #16, #27 | Open | L | Hub pipeline | | 8 | Wider task shapes | Workspace handoff, multi-container, credentials | Open | M to L | Hub pipeline and validation core | | 9 | Training signal | #17 | Open | L | Validation experiments and RFC 013 | @@ -208,9 +209,14 @@ contributor; these make good first issues. ### 6. Cloud execution on HF Sandbox The HF Sandbox provider refuses every launch because it has no immutable image -binding. Runtime and semantic checks therefore cannot run in the cloud, or on GPUs, -through the RFC provider interface. Cloud audits work around this with their own -runner. +binding. Runtime and semantic checks for served Spaces therefore cannot run in the +cloud, or on GPUs, through the RFC provider interface. + +Harbor tasks do not have this gap when they pin `environment.docker_image` by +digest, because the task declares its own immutable image. `openenv validate +--sandbox hf-sandbox` now runs their trials as HF Jobs through Harbor's +`hf-sandbox` environment, so cloud audits can call `openenv validate` instead of +their own runner. The remaining work below covers served Spaces. **Approach.** Resolve a Space revision to the registry digest of the image it builds, verify it, and launch by digest. Qualify public Space images first, then add diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index 753756074e..6a8cf30a4f 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -106,7 +106,8 @@ of strings. This is a minimal complete contract: ``` The `harbor-semantic-v1` binding runs -fresh Harbor trials in Docker: a do-nothing agent, the oracle (`solution/solve.sh`), +fresh Harbor trials, in local Docker unless `--sandbox hf-sandbox` moves them to +HF Jobs: a do-nothing agent, the oracle (`solution/solve.sh`), an oracle replay and any canaries declared in the contract. A canary is a known-wrong `solve.sh` replacement. The results report under `runtime.startup`, `runtime.reward_well_formed`, `runtime.episode_determinism`, `semantic.oracle_max`, @@ -126,7 +127,7 @@ Dockerfile nor a pinned image, task-authored Compose files and separate verifier environments. GPU/TPU or external MCP requirements, credentials, non-public or differing phase networks, task environment variables, healthchecks, skills, verifier collection and artifact extensions are also -unsupported. A supplied provider, including `--space`, is unsupported. Each case +unsupported. A served-package provider, including `--space`, is unsupported. Each case gives `runtime.startup` SKIP with a named reason and runs no trials. With valid static declarations and an oracle, the verdict is WARN and the exit code is 0. A missing `solution/solve.sh` still FAILs `semantic.oracle_max` at the semantic level, @@ -146,6 +147,38 @@ is removed on every run. Tag removal verifies image identity and uses `--no-prun A removal failure marks cleanup incomplete and gives a startup ERROR. During an abort, cleanup preserves the original exception and records the removal failure. +`--sandbox hf-sandbox` runs the same trials on Hugging Face Jobs instead, one job +per trial, through Harbor's `hf-sandbox` environment: + +```bash +openenv validate path/to/task --sandbox hf-sandbox +openenv validate path/to/task --sandbox hf-sandbox --flavor cpu-upgrade --namespace my-org +``` + +A sandbox cannot build a Dockerfile, so the task must pin a pullable image as +`environment.docker_image = "name@sha256:"`. Build and push the image +first, then pin its digest. The image needs `/bin/bash`, which Harbor uses to run +commands, and either `sha256sum` or `openssl`, which the sandbox server uses to +verify its own download. With nothing +to build, every trial starts at once. `--flavor` and `--namespace` mean what they +mean for `hf jobs run`: the flavor sets the hardware, and the namespace is the user +or organization that owns and pays for the jobs, by default the token's user. The +flavor must provide the CPU, memory and disk the contract declares, or +`runtime.startup` reports ERROR before any job starts. Tasks that set an agent or +verifier user are unsupported, because the sandbox cannot switch users. + +Each job carries `openenv-validation-run` and `openenv-validation-role` labels. At +`AGENT_START` the validator reads the job back and requires the pinned image, the +requested flavor and a `RUNNING` stage. After each trial it cancels any labelled +job that is still running and waits up to 60 seconds for all of them to stop. +Otherwise cleanup is unverified and `runtime.startup` reports ERROR. +`runtime.startup.measured` records the flavor, namespace and job IDs, and +`harbor-evidence.json` states that the image is the digest requested from HF Jobs, +not one measured inside the sandbox. Check IDs, graders and thresholds are the +same as on Docker, and no image is retained. A served package gives a named +`runtime.startup` SKIP with `--sandbox hf-sandbox`; validate a served Space with +`--space` instead. + The 16 implemented runtime checks cover startup, rewards, observation schemas, state continuity, network policy, host containment, resource bounds, episode isolation, oracle containment, seed forwarding, determinism, emitted trajectory records, tool diff --git a/rfcs/008-environment-auto-validation.md b/rfcs/008-environment-auto-validation.md index b209fd1662..d058a447b8 100644 --- a/rfcs/008-environment-auto-validation.md +++ b/rfcs/008-environment-auto-validation.md @@ -756,8 +756,8 @@ an `environment.docker_image` without a digest, tasks with neither a Dockerfile a pinned image, task-authored Compose files and separate verifier environments. GPU/TPU or external MCP requirements, credentials, non-public or differing phase networks, task environment variables, healthchecks, skills, verifier collection and -artifact extensions are also unsupported. A supplied provider, including `--space`, -is unsupported. Each case gives `runtime.startup` SKIP with a named reason and runs +artifact extensions are also unsupported. A served-package provider, including +`--space`, is unsupported. Each case gives `runtime.startup` SKIP with a named reason and runs no trials. With valid static declarations and an oracle, the verdict is WARN and the exit code is 0. A missing `solution/solve.sh` still FAILs `semantic.oracle_max` at the semantic level, including with `--skip-build` or `--space`. @@ -792,6 +792,20 @@ containers' actual mounts, and verifies their absence. The oracle's exit code is read at `AGENT_END`, before sandbox log synchronization can overwrite it, and a later change fails the run. +With `--sandbox hf-sandbox`, each trial runs as its own Hugging Face Job through +Harbor's `hf-sandbox` environment, on the flavor and in the namespace the operator +names. The task must declare a digest-pinned `environment.docker_image`. Tasks that +only ship a Dockerfile, and tasks that set agent or verifier users, give +`runtime.startup` SKIP. A flavor smaller than the contract's resources gives +`runtime.startup` ERROR before any job starts. All trials start at once from the +pinned image. Every job carries run and role labels. At `AGENT_START` the validator +reads the job back and requires the pinned image, the requested flavor and a +`RUNNING` stage. After each trial it cancels any labelled job still running and +verifies that every one reaches a terminal stage. The image identity is the digest +requested from HF Jobs and read back from the job, not a digest measured inside the +sandbox, and the evidence states this limitation. Check ids, graders and thresholds +are unchanged. + For `harbor-semantic-v1`, one trial is a one-shot episode with no per-step observations. A fresh oracle replay that reproduces every reward and the oracle exit code establishes `runtime.episode_determinism`. This rule is specific to the diff --git a/src/openenv/cli/commands/validate.py b/src/openenv/cli/commands/validate.py index b7e716014d..f4bcc831a7 100644 --- a/src/openenv/cli/commands/validate.py +++ b/src/openenv/cli/commands/validate.py @@ -223,9 +223,29 @@ def validate( str, typer.Option( "--flavor", - help="HF Jobs hardware flavor for --space, e.g. cpu-basic or t4-small", + help=( + "HF Jobs hardware flavor for --space or --sandbox hf-sandbox, " + "e.g. cpu-basic or t4-small" + ), ), ] = "cpu-basic", + sandbox: Annotated[ + str, + typer.Option( + "--sandbox", + help=( + "Where Harbor task trials run: docker (local daemon) or " + "hf-sandbox (one HF Job per trial)" + ), + ), + ] = "docker", + namespace: Annotated[ + str | None, + typer.Option( + "--namespace", + help="User or organization that owns the --sandbox hf-sandbox jobs", + ), + ] = None, ) -> None: """ Validate a local package or a running OpenEnv server. @@ -236,9 +256,11 @@ def validate( applies the severity policy, and emits a report. Native Harbor tasks require the optional Harbor dependency and an explicit - `validation/openenv.json` contract. Fresh Docker trials grade the oracle, the - do-nothing floor and any declared canaries under the semantic check ids. Checks - without an implementation are named SKIPs, so the report is WARN at best. + `validation/openenv.json` contract. Fresh trials grade the oracle, the + do-nothing floor and any declared canaries under the semantic check ids. They + run on the local Docker daemon, or with `--sandbox hf-sandbox` as one Hugging + Face Job per trial from the task's digest-pinned image. Checks without an + implementation are named SKIPs, so the report is WARN at best. Exit codes: 0 pass/warn · 1 fail · 2 unrecognized/unsupported package · 3 internal or policy error. @@ -260,6 +282,9 @@ def validate( # Validate a Hub Space on HF Jobs GPU hardware openenv validate --space openenv/echo_env --level runtime --flavor t4-small + + # Run a Harbor task's trials as HF Jobs instead of local Docker + openenv validate path/to/task --sandbox hf-sandbox --flavor cpu-upgrade ``` """ if space is not None and (target is not None or url is not None or local): @@ -267,8 +292,28 @@ def validate( "Error: --space cannot be combined with a path, --url or --local", err=True ) raise typer.Exit(EXIT_FAIL) - if space is None and (revision is not None or flavor != "cpu-basic"): - typer.echo("Error: --revision and --flavor require --space", err=True) + if sandbox not in ("docker", "hf-sandbox"): + typer.echo( + f"Error: unknown sandbox {sandbox!r}; expected docker or hf-sandbox", + err=True, + ) + raise typer.Exit(EXIT_FAIL) + hf_sandbox = sandbox == "hf-sandbox" + running_url = url is not None or (target is not None and _looks_like_url(target)) + if hf_sandbox and (space is not None or running_url): + typer.echo( + "Error: --sandbox cannot be combined with --space or a running URL", + err=True, + ) + raise typer.Exit(EXIT_FAIL) + if space is None and revision is not None: + typer.echo("Error: --revision requires --space", err=True) + raise typer.Exit(EXIT_FAIL) + if space is None and not hf_sandbox and flavor != "cpu-basic": + typer.echo("Error: --flavor requires --space or --sandbox hf-sandbox", err=True) + raise typer.Exit(EXIT_FAIL) + if namespace is not None and not hf_sandbox: + typer.echo("Error: --namespace requires --sandbox hf-sandbox", err=True) raise typer.Exit(EXIT_FAIL) runtime_target = url @@ -367,6 +412,9 @@ def validate( report_target=f"spaces/{space}@{revision}" if space is not None else None, + sandbox=sandbox, + flavor=flavor, + namespace=namespace, ) report_json = write_report(validation_report, output) except (SignatureError, UnsupportedPackageError) as exc: diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index e0210ce52c..83f190b2b6 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -1,13 +1,15 @@ -"""Native Harbor execution: fresh Docker trials graded through RFC 008 check ids. +"""Native Harbor execution: fresh trials graded through RFC 008 check ids. -No served-protocol evidence is synthesized. A do-nothing trial builds the image, or -pulls the task's digest-pinned one, and the oracle, its replay and any canaries run -pinned to that image. +No served-protocol evidence is synthesized. On local Docker, a do-nothing trial +builds the image, or pulls the task's digest-pinned one, and the oracle, its replay +and any canaries run pinned to that image. On `hf-sandbox`, every trial starts at +once from the task's digest-pinned image, each in its own Hugging Face Job. """ import asyncio import hashlib import json +import math import os import re import shutil @@ -17,6 +19,7 @@ import tempfile import time import uuid +from concurrent.futures import ThreadPoolExecutor from dataclasses import replace from decimal import Decimal from fractions import Fraction @@ -58,6 +61,10 @@ "RetainedImageDrift", "RetainedImageUnverified", } +_TERMINAL_STAGES = {"COMPLETED", "CANCELED", "ERROR", "DELETED"} +_JOB_STOP_TIMEOUT_S = 60 +# Inside the 600-second validator start budget, so Sandbox.create finishes first. +_SANDBOX_START_TIMEOUT_S = 540 # Checks this binding measures; every other local check is a named SKIP. _IMPLEMENTED = { "runtime.startup", @@ -276,7 +283,134 @@ def remove_image(self): self._retained_image = None -def _unsupported(task, contract): +def _job_labels(trial_name): + # openenv-validation-- + run_id, role = trial_name.split("-", 3)[2:] + return {"openenv-validation-run": run_id, "openenv-validation-role": role} + + +class HFJobsInspector: + """ + Trusted HF Jobs checks limited to sandbox jobs labelled with this run. + + The image identity is the digest requested from HF Jobs and read back from the + job; no runtime digest is measured inside the sandbox. + """ + + def __init__(self, image, flavor, namespace, resources): + self.image = image + self.flavor = flavor + self.namespace = namespace + self._resources = resources + self._api = None + + def preflight(self): + from huggingface_hub import HfApi + + self._api = HfApi() + if self.namespace is None: + self.namespace = self._api.whoami()["name"] + hardware = {item.name: item for item in self._api.list_jobs_hardware()} + if self.flavor not in hardware: + raise NativeEvidenceError(f"unknown HF Jobs flavor {self.flavor}") + item = hardware[self.flavor] + # The Hub lists specs as "2 vCPU", "16 GB" and "50 GB". + cpu, memory_gb, disk_gb = ( + int(spec.split()[0]) + for spec in (item.cpu, item.ram, item.ephemeral_storage) + ) + if ( + cpu < self._resources.cpu + or memory_gb * 1000 < self._resources.memory_mb + or disk_gb * 1000 < self._resources.disk_mb + ): + raise NativeEvidenceError( + f"HF Jobs flavor {self.flavor} ({item.cpu}, {item.ram} RAM, " + + f"{item.ephemeral_storage} disk) is smaller than the contract resources" + ) + return { + "flavor": self.flavor, + "namespace": self.namespace, + "hardware": { + "cpu": item.cpu, + "ram": item.ram, + "disk": item.ephemeral_storage, + }, + } + + def _jobs(self, project): + if not _PROJECT.fullmatch(project): + raise NativeEvidenceError("not a run-owned Harbor project") + labels = _job_labels(project.removesuffix("__env")) + jobs = list(self._api.list_jobs(labels=labels, namespace=self.namespace)) + # Cleanup cancels what this returns, so recheck the server-side filter. + if any( + not labels.items() <= (job.labels or {}).items() + or job.owner.name != self.namespace + for job in jobs + ): + raise NativeEvidenceError("HF Jobs returned a job outside this run") + return jobs + + def inventory(self, project): + return {"jobs": sorted(job.id for job in self._jobs(project))} + + def capture(self, project, retained_tag, *, retain=True): + jobs = self._jobs(project) + if len(jobs) != 1: + raise NativeEvidenceError("HF Sandbox trial requires exactly one job") + job = jobs[0] + if ( + job.status.stage != "RUNNING" + or job.docker_image != self.image + or job.flavor != self.flavor + ): + raise NativeEvidenceError( + "HF Sandbox job is not running the pinned image on the requested flavor" + ) + configuration = { + "image_ref": job.docker_image, + "flavor": self.flavor, + "namespace": self.namespace, + } + return { + "container_id": job.id, + "project": project, + "image_ref": job.docker_image, + "configuration": configuration, + "configuration_digest": _digest(configuration), + } + + def cleanup(self, project): + # A start abandoned at its deadline can still create a job, so keep looking. + deadline = time.monotonic() + _JOB_STOP_TIMEOUT_S + cancelled = set() + while True: + stages = {job.id: job.status.stage for job in self._jobs(project)} + running = { + job_id + for job_id, stage in stages.items() + if stage not in _TERMINAL_STAGES + } + for job_id in running - cancelled: + self._api.cancel_job(job_id=job_id, namespace=self.namespace) + cancelled.add(job_id) + if not running or time.monotonic() > deadline: + return { + "verified": not running, + "jobs": stages, + "cancelled": sorted(cancelled), + } + time.sleep(2) + + def verify_image(self, tag, image_ref): + return image_ref == self.image + + def remove_image(self): + pass # Nothing is pulled or tagged on this host. + + +def _unsupported(task, contract, sandbox): config = task.config env = config.environment if task.has_steps: @@ -312,21 +446,57 @@ def _unsupported(task, contract): return "task environment variables (including host credential substitutions)" if env.healthcheck or env.skills_dir or config.verifier.collect or config.artifacts: return "healthcheck, skills, collect or artifact extensions" + if sandbox == "hf-sandbox": + if env.docker_image is None: + return ( + "hf-sandbox tasks without environment.docker_image; build and push " + + "the image, then pin it as name@sha256:" + ) + if config.agent.user is not None or config.verifier.user is not None: + return "agent or verifier users on hf-sandbox, which cannot switch users" return None async def _create_trial(config): + from harbor.environments.hf_sandbox import HFSandboxEnvironment from harbor.trial.trial import Trial trial = await Trial.create(config) environment = trial.agent_environment trial._openenv_cancelled_compose = [] - environment._run_docker_compose_command = partial( - _run_compose, environment, cancelled=trial._openenv_cancelled_compose - ) + if isinstance(environment, HFSandboxEnvironment): + environment.start = partial( + _start_sandbox, + environment, + labels=_job_labels(config.trial_name), + namespace=config.environment.kwargs["namespace"], + ) + else: + environment._run_docker_compose_command = partial( + _run_compose, environment, cancelled=trial._openenv_cancelled_compose + ) return trial +async def _start_sandbox(environment, *, force_build, labels, namespace): + # Harbor's hf-sandbox start, plus labels that let cleanup find every run-owned job. + from huggingface_hub import Sandbox + + environment._sandbox = await asyncio.to_thread( + Sandbox.create, + image=environment.task_env_config.docker_image, + flavor=environment._flavor, + idle_timeout=environment._idle_timeout, + namespace=namespace, + labels=labels, + start_timeout=_SANDBOX_START_TIMEOUT_S, + ) + if workdir := environment.task_env_config.workdir: + await asyncio.to_thread(environment._sandbox.files.mkdir, workdir) + await environment.ensure_dirs(environment._mount_targets(writable_only=True)) + await environment._upload_environment_dir_after_start() + + async def _run_compose( environment, command, @@ -499,9 +669,12 @@ async def _trial( trial = None phase = "trial creation" registered = False + hf = isinstance(inspector, HFJobsInspector) try: # Verify ownership before Trial.create can allocate anything. - record["pre_allocation_inventory"] = inspector.inventory(project) + record["pre_allocation_inventory"] = await asyncio.to_thread( + inspector.inventory, project + ) if any(record["pre_allocation_inventory"].values()): raise NativeEvidenceError( "run-owned project was not empty before allocation" @@ -549,6 +722,17 @@ async def _trial( override_setup_timeout_sec=budget, ), environment=EnvironmentConfig( + type=EnvironmentType.HF_SANDBOX, + delete=True, + kwargs={ + "flavor": inspector.flavor, + "namespace": inspector.namespace, + # Idle shutdown must outlast the longest single command. + "job_timeout": math.ceil(budget) + 600, + }, + ) + if hf + else EnvironmentConfig( type=EnvironmentType.DOCKER, delete=False, force_build=False, @@ -593,7 +777,9 @@ async def capture(_event): raise NativeEvidenceError( "Harbor sandbox identity differs from the registered project" ) - identity = inspector.capture(project, retained_tag, retain=not expected) + identity = await asyncio.to_thread( + inspector.capture, project, retained_tag, retain=not expected + ) record.update(identity) if expected and ( identity["image_ref"] != expected["image_ref"] @@ -697,12 +883,12 @@ async def verification_start(_event): if trial is not None: try: async with asyncio.timeout(30): - await trial.agent_environment.stop(delete=False) + await trial.agent_environment.stop(delete=hf) except (Exception, KeyboardInterrupt): record["failure"] = "HarborTeardownError" if registered: try: - record["cleanup"] = inspector.cleanup(project) + record["cleanup"] = await asyncio.to_thread(inspector.cleanup, project) if trial is not None and trial._openenv_cancelled_compose: record["cleanup"].update( { @@ -745,7 +931,6 @@ async def verification_start(_event): async def _collect(root, contract, work, inspector, source_binding, canaries=()): run_id = uuid.uuid4().hex - daemon = inspector.preflight() # The replay makes episode determinism measurable before any semantic check. plan = [ ("nop", "nop", None), @@ -756,30 +941,46 @@ async def _collect(root, contract, work, inspector, source_binding, canaries=()) for index, canary in enumerate(canaries, start=1) ), ] - trials, files = [], {} - for role, agent, canary in plan: - trial, raw = await _trial( - root, - role, - agent, - work, - contract, - inspector, - run_id, - source_binding, - trials[0] if trials else None, - canary, + if isinstance(inspector, HFJobsInspector): + # Each running trial blocks a worker thread for every sandbox command, and + # a trial waiting for a free thread would spend its own phase budget. + asyncio.get_running_loop().set_default_executor( + ThreadPoolExecutor(max_workers=len(plan) + 4) + ) + host = await asyncio.to_thread(inspector.preflight) + run_trial = partial( + _trial, + root, + work=work, + contract=contract, + inspector=inspector, + run_id=run_id, + source_binding=source_binding, + ) + if isinstance(inspector, HFJobsInspector): + # A pinned image needs no build, so every trial starts at once. + outcomes = await asyncio.gather( + *(run_trial(role, agent, canary=canary) for role, agent, canary in plan) ) + else: + outcomes = [] + for role, agent, canary in plan: + expected = outcomes[0][0] if outcomes else None + outcomes.append( + await run_trial(role, agent, expected=expected, canary=canary) + ) + if "failure" in outcomes[-1][0]: + break + trials, files = [], {} + for trial, raw in outcomes: trials.append(trial) for name, value in raw.items(): - relative = f"harbor-trials/{role}/{name}" + relative = f"harbor-trials/{trial['role']}/{name}" files[relative] = value trial["raw_rewards"][name] = { "path": relative, "sha256": hashlib.sha256(value).hexdigest(), } - if "failure" in trial: - break if trials and "image_ref" in trials[0]: try: if not inspector.verify_image( @@ -788,7 +989,7 @@ async def _collect(root, contract, work, inspector, source_binding, canaries=()) trials[-1]["failure"] = "RetainedImageDrift" except Exception: trials[-1]["failure"] = "RetainedImageUnverified" - return daemon, trials, files + return host, trials, files def _result(check_id, status, reason, measured=None): @@ -828,6 +1029,9 @@ def validate_harbor( report_target=None, skip_build=False, provider=None, + sandbox="docker", + flavor="cpu-basic", + namespace=None, ): """Run fresh native Harbor trials and grade them up to the requested level.""" from .runner import source_digest @@ -843,6 +1047,7 @@ def validate_harbor( record = { "schema_version": "2", "profile": HARBOR_PROFILE, + "sandbox": sandbox, "task_effective_digest": digest_before, "original_source_digest": contract.original_source_digest, "contract_digest": _digest(contract.model_dump(mode="json")), @@ -861,11 +1066,21 @@ def validate_harbor( try: attempted = False task = read_harbor_task(subject.root) - problem = _unsupported(task, contract) + problem = _unsupported(task, contract, sandbox) + if sandbox == "hf-sandbox": + inspector = HFJobsInspector( + task.config.environment.docker_image, + flavor, + namespace, + contract.resources, + ) if skip_build: problem = "image build skipped by --skip-build" if provider is not None: - problem = "native Harbor binding requires its own Docker Trial provider" + problem = ( + "served-package providers, including --space; " + + "run Harbor trials on HF Jobs with --sandbox hf-sandbox" + ) if any(result.status is not CheckStatus.PASS for result in results): problem = "unmet dependency: static.manifest" startup = _result( @@ -917,7 +1132,7 @@ def validate_harbor( with tempfile.TemporaryDirectory( prefix="openenv-harbor-validation-" ) as temp: - daemon, trials, files = asyncio.run( + host, trials, files = asyncio.run( _collect( subject.root, contract, @@ -927,7 +1142,15 @@ def validate_harbor( canaries, ) ) - record.update({"daemon": daemon, "trials": trials}) + record["trials"] = trials + if sandbox == "hf-sandbox": + record["hf_jobs"] = host + record["limitations"] = [ + "The image is the digest requested from HF Jobs and read " + + "back from each job, not a digest measured at runtime." + ] + else: + record["daemon"] = host record["cleanup"] = { "verified": bool(trials) and all(trial["cleanup"]["verified"] for trial in trials) @@ -992,6 +1215,13 @@ def validate_harbor( if source_image is not None: record["source_image"] = source_image startup.measured["source_image"] = source_image + if sandbox == "hf-sandbox": + startup.measured.update( + sandbox=sandbox, + flavor=host["flavor"], + namespace=host["namespace"], + job_ids=containers, + ) if malformed: reward_result = _result( "runtime.reward_well_formed", @@ -1161,7 +1391,7 @@ def validate_harbor( f"retained image removal failed ({type(exc).__name__})" ) record["cleanup"].update(verified=False, completed=False) - elif "image_ref" in record: + elif "image_ref" in record and sandbox == "docker": tag = trials[0]["retained_image_tag"] startup.measured["retained_image_tag"] = tag record["retained_image_tag"] = tag @@ -1187,7 +1417,11 @@ def validate_harbor( artifacts_dir, report, cleanup=record["cleanup"], - provider={"binding": "harbor_task", "profile": HARBOR_PROFILE}, + provider={ + "binding": "harbor_task", + "profile": HARBOR_PROFILE, + "sandbox": sandbox, + }, ) files["harbor-evidence.json"] = (dumps(record) + "\n").encode() checksums = (artifacts_dir / "SHA256SUMS").read_text() diff --git a/src/openenv/validation/runner.py b/src/openenv/validation/runner.py index 75ced47bc8..8ccce7461e 100644 --- a/src/openenv/validation/runner.py +++ b/src/openenv/validation/runner.py @@ -274,7 +274,7 @@ def _applicable(check_id, manifest): return True -def _runtime(subject, graders, *, skip_build, provider): +def _runtime(subject, graders, *, skip_build, provider, sandbox): """Own build/start/collection/stop and retain failure evidence through teardown.""" manifest = subject.manifest plan = None @@ -293,6 +293,11 @@ def _runtime(subject, graders, *, skip_build, provider): raise UnsupportedCapability( "--skip-build: build-dependent runtime checks skipped" ) + if sandbox != "docker": + raise UnsupportedCapability( + f"{sandbox} runs native Harbor tasks only; " + + "validate a served Space with --space" + ) if not isinstance(manifest, NormalizedManifestV2): raise UnsupportedCapability( "missing validation.execution declaration and runtime plan" @@ -586,6 +591,9 @@ def run_validation( provider=None, artifacts_dir: Path | None = None, report_target: str | None = None, + sandbox: str = "docker", + flavor: str = "cpu-basic", + namespace: str | None = None, ) -> ValidationReport | ValidationReportV2: """ Validate a package end to end and return the report. @@ -612,6 +620,16 @@ def run_validation( Write a redacted reproduction bundle to this directory. report_target (`str`, *optional*): Durable source identity for the report; defaults to the package path. + sandbox (`str`, *optional*, defaults to `"docker"`): + Where native Harbor trials run: `"docker"` on the local daemon or + `"hf-sandbox"`, one Hugging Face Job per trial. Served packages run + on `"docker"` or `provider` only. + flavor (`str`, *optional*, defaults to `"cpu-basic"`): + HF Jobs hardware flavor for `"hf-sandbox"` trials, as listed by + `hf jobs hardware`. + namespace (`str`, *optional*): + User or organization that owns the `"hf-sandbox"` jobs; defaults to + the token's user. Returns: [`~openenv.validation.report.ValidationReport`]: the completed report. @@ -703,6 +721,9 @@ def run_validation( report_target=report_target, skip_build=skip_build, provider=provider, + sandbox=sandbox, + flavor=flavor, + namespace=namespace, ) ( runtime_results, @@ -722,6 +743,7 @@ def run_validation( ], skip_build=skip_build, provider=provider, + sandbox=sandbox, ) results.extend(runtime_results) if attempted: diff --git a/tests/test_validation/test_cli_validate.py b/tests/test_validation/test_cli_validate.py index 1a5975bbec..a71aceefb7 100644 --- a/tests/test_validation/test_cli_validate.py +++ b/tests/test_validation/test_cli_validate.py @@ -1,7 +1,9 @@ +import json import subprocess import sys from pathlib import Path +import pytest from conftest import FIXTURES from openenv.validation.report import ValidationReport, ValidationReportV2 @@ -148,3 +150,40 @@ def test_served_package_cannot_claim_harbor_execution(tmp_path): assert manifest_result.status.value == "fail" assert "openenv_ws" in " ".join(manifest_result.evidence) assert "Internal error" not in result.stderr + + +@pytest.mark.parametrize( + "args,message", + [ + (["--sandbox", "nope"], "unknown sandbox"), + (["--sandbox", "hf-sandbox", "--space", "org/env"], "--sandbox cannot be"), + (["--sandbox", "hf-sandbox", "--url", "http://localhost:8000"], "--sandbox"), + (["http://localhost:8000", "--sandbox", "hf-sandbox"], "--sandbox cannot be"), + (["--namespace", "org"], "--namespace requires --sandbox hf-sandbox"), + (["--flavor", "t4-small"], "--flavor requires --space or --sandbox"), + (["--revision", "main"], "--revision requires --space"), + ], +) +def test_sandbox_flags_must_be_used_together(args, message): + result = _validate(*args) + assert result.returncode == 1, result.stdout + result.stderr + assert message in result.stderr + + +def test_hf_sandbox_reports_served_packages_as_unsupported(): + result = _validate( + str(FIXTURES / "served_min_pass"), + "--sandbox", + "hf-sandbox", + "--level", + "runtime", + "--json", + ) + assert result.returncode == 0, result.stdout + result.stderr + startup = next( + check + for check in json.loads(result.stdout)["results"] + if check["check_id"] == "runtime.startup" + ) + assert startup["status"] == "skip" + assert "runs native Harbor tasks only" in startup["evidence"][0] diff --git a/tests/test_validation/test_harbor_binding_supervision.py b/tests/test_validation/test_harbor_binding_supervision.py index 0904e30cfa..dea0b97fbc 100644 --- a/tests/test_validation/test_harbor_binding_supervision.py +++ b/tests/test_validation/test_harbor_binding_supervision.py @@ -135,3 +135,83 @@ async def _collect_buffered_output(self, process, *, timeout_sec, stdin_data): assert argv[argv.index("--project-directory") + 1] == str( Environment.environment_dir.resolve() ) + + +@pytest.mark.asyncio +async def test_hf_sandbox_start_labels_its_job_and_keeps_harbor_setup( + monkeypatch, tmp_path +): + import huggingface_hub + from harbor.models.environment_type import EnvironmentType + from harbor.models.trial.config import ( + AgentConfig, + EnvironmentConfig, + TaskConfig, + TrialConfig, + ) + + calls = [] + + class Sandbox: + id = "job-1" + files = SimpleNamespace(mkdir=lambda path: calls.append(("mkdir", path))) + + def run(self, argv, **kwargs): + calls.append(("run", argv)) + return SimpleNamespace(stdout="", stderr="", exit_code=0) + + def kill(self): + calls.append(("kill",)) + + def create(**kwargs): + calls.append(("create", kwargs)) + return Sandbox() + + monkeypatch.setattr(huggingface_hub.Sandbox, "create", create) + image = "registry.example/image@sha256:" + "d" * 64 + task = tmp_path / "task" + for directory in ("solution", "tests"): + (task / directory).mkdir(parents=True) + (task / "task.toml").write_text( + f'schema_version = "1.4"\n[environment]\ndocker_image = "{image}"\n' + + 'workdir = "/app"\n' + ) + (task / "instruction.md").write_text("Write the correct answer.\n") + (task / "tests/test.sh").write_text("#!/bin/sh\nexit 0\n") + (task / "solution/solve.sh").write_text("#!/bin/sh\nexit 0\n") + trial = await harbor._create_trial( + TrialConfig( + task=TaskConfig(path=task), + agent=AgentConfig(name="oracle"), + environment=EnvironmentConfig( + type=EnvironmentType.HF_SANDBOX, + kwargs={ + "flavor": "cpu-upgrade", + "namespace": "org", + "job_timeout": 900, + }, + ), + trial_name="openenv-validation-" + "d" * 32 + "-oracle-replay", + trials_dir=tmp_path / "trials", + ) + ) + await trial.agent_environment.start(force_build=False) + assert calls[0] == ( + "create", + { + "image": image, + "flavor": "cpu-upgrade", + "idle_timeout": 900, + "namespace": "org", + "labels": { + "openenv-validation-run": "d" * 32, + "openenv-validation-role": "oracle-replay", + }, + "start_timeout": harbor._SANDBOX_START_TIMEOUT_S, + }, + ) + assert calls[1] == ("mkdir", "/app") + # Harbor's own setup still creates the log directories the verifier writes. + assert "/logs/verifier" in calls[2][1][-1] + await trial.agent_environment.stop(delete=True) + assert calls[-1] == ("kill",) diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index a297c9c8b1..67a5d7f3ad 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -3,6 +3,8 @@ import asyncio import hashlib import json +import os +import threading from types import SimpleNamespace import pytest @@ -82,6 +84,9 @@ def boundary(monkeypatch, task): preflight_error=None, interrupt_role=None, on_build=None, + on_start=None, + on_stop=None, + start_barrier=None, ) class Inspector: @@ -162,7 +167,10 @@ def __init__(self, config): state.trials.append(self) async def stop(self, *, delete): - assert not delete + # Docker cleanup removes run-owned resources itself; HF Sandbox is killed. + assert delete is (self.config.environment.type == "hf-sandbox") + if state.on_stop is not None: + state.on_stop(self) if state.teardown_defect == "error": raise OSError("Harbor teardown failed") if state.teardown_defect == "drift": @@ -174,6 +182,11 @@ async def stop(self, *, delete): async def start(self, *, force_build): if self.role == "nop" and state.on_build is not None: state.on_build(self.agent_environment.session_id) + if state.on_start is not None: + state.on_start(self) + if state.start_barrier is not None: + # Like Sandbox.create, block a worker thread until every trial starts. + await asyncio.to_thread(state.start_barrier.wait) if state.pending_compose: self._openenv_cancelled_compose.append( { @@ -1385,3 +1398,241 @@ def test_canaries_directory_holds_only_scripts(task, boundary, canaries): report = run_validation(task) assert result(report, "static.manifest").status is CheckStatus.FAIL assert not boundary.trials + + +PINNED = "registry.example/image@sha256:" + "d" * 64 + + +@pytest.fixture +def hf_jobs(task, boundary, monkeypatch): + """A fake HF Jobs control plane; each fake Trial start creates one labelled job.""" + import huggingface_hub + + with (task / "task.toml").open("a") as stream: + stream.write(f'[environment]\ndocker_image="{PINNED}"\n') + state = SimpleNamespace( + jobs={}, + by_trial={}, + cancelled=[], + kill_fails=False, + cancel_fails=False, + image=None, + max_running=0, + ) + + def start(trial): + kwargs = trial.config.environment.kwargs + job = SimpleNamespace( + id=f"job-{len(state.jobs)}", + labels={"hf-sandbox": "1", **harbor._job_labels(trial.config.trial_name)}, + docker_image=state.image or trial.docker_image, + flavor=kwargs["flavor"], + owner=SimpleNamespace(name=kwargs["namespace"]), + status=SimpleNamespace(stage="RUNNING"), + ) + state.jobs[job.id] = state.by_trial[trial.config.trial_name] = job + running = [job for job in state.jobs.values() if job.status.stage == "RUNNING"] + state.max_running = max(state.max_running, len(running)) + + def stop(trial): + job = state.by_trial.get(trial.config.trial_name) + if job is not None and not state.kill_fails: + job.status.stage = "CANCELED" + + class Api: + def whoami(self): + return {"name": "me"} + + def list_jobs_hardware(self): + return [ + SimpleNamespace( + name="cpu-basic", + cpu="2 vCPU", + ram="16 GB", + ephemeral_storage="50 GB", + ) + ] + + def list_jobs(self, *, labels, namespace): + return [ + job + for job in state.jobs.values() + if labels.items() <= job.labels.items() and job.owner.name == namespace + ] + + def cancel_job(self, *, job_id, namespace): + state.cancelled.append(job_id) + if not state.cancel_fails: + state.jobs[job_id].status.stage = "CANCELED" + + boundary.on_start = start + boundary.on_stop = stop + monkeypatch.setattr(huggingface_hub, "HfApi", Api) + return state + + +def test_hf_sandbox_runs_every_trial_at_once_as_a_labelled_job( + task, boundary, hf_jobs, canaries, tmp_path +): + boundary.agent_delay = 0.05 + boundary.rewards.update({"canary-1": b'{"reward":0}', "canary-2": b'{"reward":0}'}) + artifacts = tmp_path / "hf" + report = run_validation( + task, sandbox="hf-sandbox", namespace="org", artifacts_dir=artifacts + ) + assert report.verdict is Verdict.WARN + for check_id in ( + "runtime.startup", + "runtime.reward_well_formed", + "runtime.episode_determinism", + "semantic.oracle_max", + "semantic.floor_gap", + "semantic.canary_floor", + ): + assert result(report, check_id).status is CheckStatus.PASS, check_id + # Nothing is built, so all five trials overlap. + assert hf_jobs.max_running == 5 + assert all(job.status.stage == "CANCELED" for job in hf_jobs.jobs.values()) + assert not hf_jobs.cancelled and not boundary.cleanup_calls + assert all( + trial.config.environment.type == "hf-sandbox" + and trial.config.environment.delete + and trial.config.environment.kwargs + == {"flavor": "cpu-basic", "namespace": "org", "job_timeout": 610} + and trial.docker_image == PINNED + for trial in boundary.trials + ) + measured = result(report, "runtime.startup").measured + assert measured["image_ref"] == measured["source_image"] == PINNED + assert (measured["sandbox"], measured["flavor"], measured["namespace"]) == ( + "hf-sandbox", + "cpu-basic", + "org", + ) + assert sorted(measured["job_ids"]) == sorted(hf_jobs.jobs) + assert "retained_image_tag" not in measured + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert native["sandbox"] == "hf-sandbox" + assert native["hf_jobs"]["hardware"]["cpu"] == "2 vCPU" + assert native["limitations"] and "daemon" not in native + assert [trial["role"] for trial in native["trials"]] == [ + "nop", + "oracle", + "oracle-replay", + "canary-1", + "canary-2", + ] + assert all(trial["cleanup"]["verified"] for trial in native["trials"]) + assert measured["job_ids"] == [trial["container_id"] for trial in native["trials"]] + + +@pytest.mark.parametrize( + "shape,reason", + [("dockerfile", "build and push the image"), ("user", "cannot switch users")], +) +def test_hf_sandbox_refuses_tasks_it_cannot_run(task, boundary, shape, reason): + if shape == "user": + with (task / "task.toml").open("a") as stream: + stream.write( + f'[environment]\ndocker_image="{PINNED}"\n[agent]\nuser="1000"\n' + ) + report = run_validation(task, sandbox="hf-sandbox") + startup = result(report, "runtime.startup") + assert startup.status is CheckStatus.SKIP + assert reason in startup.evidence[0] + assert not boundary.trials + + +@pytest.mark.parametrize( + "flavor,cpu,reason", + [ + ("cpu-basic", 4, "smaller than the contract resources"), + ("cpu-unknown", 1, "unknown HF Jobs flavor"), + ], +) +def test_hf_sandbox_flavor_must_exist_and_cover_the_contract( + task, boundary, hf_jobs, flavor, cpu, reason +): + path = task / "validation/openenv.json" + contract = json.loads(path.read_text()) + contract["resources"]["cpu"] = cpu + path.write_text(json.dumps(contract)) + report = run_validation(task, sandbox="hf-sandbox", flavor=flavor) + startup = result(report, "runtime.startup") + assert startup.status is CheckStatus.ERROR + assert reason in startup.evidence[0] + assert not boundary.trials and not hf_jobs.jobs + + +def test_hf_sandbox_job_must_run_the_pinned_image(task, boundary, hf_jobs): + hf_jobs.image = "registry.example/image@sha256:" + "e" * 64 + report = run_validation(task, sandbox="hf-sandbox") + startup = result(report, "runtime.startup") + assert startup.status is CheckStatus.ERROR + assert any("pinned image" in line for line in startup.evidence) + assert all(job.status.stage == "CANCELED" for job in hf_jobs.jobs.values()) + + +@pytest.mark.parametrize("cancel_works", [True, False]) +def test_hf_sandbox_cleanup_cancels_jobs_that_survive_teardown( + task, boundary, hf_jobs, tmp_path, monkeypatch, cancel_works +): + hf_jobs.kill_fails = True + hf_jobs.cancel_fails = not cancel_works + monkeypatch.setattr(harbor, "_JOB_STOP_TIMEOUT_S", 5 if cancel_works else 0) + artifacts = tmp_path / "hf-cleanup" + report = run_validation(task, sandbox="hf-sandbox", artifacts_dir=artifacts) + assert sorted(hf_jobs.cancelled) == sorted(hf_jobs.jobs) + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert native["cleanup"]["verified"] is cancel_works + startup = result(report, "runtime.startup") + if cancel_works: + assert startup.status is CheckStatus.PASS + else: + assert startup.status is CheckStatus.ERROR + assert {trial["failure"] for trial in native["trials"]} == { + "CleanupNotVerified" + } + + +@pytest.mark.parametrize( + "labels,owner", + [ + # The Hub reports an unlabelled job as labels=None. + (None, "me"), + (harbor._job_labels("openenv-validation-" + "d" * 32 + "-nop"), "you"), + ], + ids=["unlabelled", "other-owner"], +) +def test_hf_jobs_cleanup_never_cancels_jobs_outside_its_run(labels, owner): + # A control plane that ignored the label or namespace filter. + cancelled = [] + inspector = harbor.HFJobsInspector(PINNED, "cpu-basic", "me", None) + job = SimpleNamespace( + id="someone-elses-job", + labels=labels, + owner=SimpleNamespace(name=owner), + status=SimpleNamespace(stage="RUNNING"), + ) + inspector._api = SimpleNamespace( + list_jobs=lambda **kwargs: [job], + cancel_job=lambda **kwargs: cancelled.append(kwargs["job_id"]), + ) + with pytest.raises(harbor.NativeEvidenceError, match="outside this run"): + inspector.cleanup("openenv-validation-" + "d" * 32 + "-nop__env") + assert not cancelled + + +def test_hf_sandbox_gives_every_trial_its_own_worker_thread( + task, boundary, hf_jobs, canaries, monkeypatch +): + # Eight canaries make eleven trials, more than the default thread pool of a + # two-CPU machine. A trial left waiting for a thread would spend its budget. + for index in range(3, 9): + (canaries / f"c{index}-empty.sh").write_bytes(b"#!/bin/sh\ntrue\n") + monkeypatch.setattr(os, "cpu_count", lambda: 2) + monkeypatch.setattr(os, "process_cpu_count", lambda: 2, raising=False) + boundary.start_barrier = threading.Barrier(11, timeout=5) + report = run_validation(task, sandbox="hf-sandbox") + assert result(report, "runtime.startup").status is CheckStatus.PASS + assert hf_jobs.max_running == 11 From 51434931b134989d333e91106bd4275d61f25641 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 15:18:53 +0100 Subject: [PATCH 2/7] fix(validation): preserve HF job ownership through cancelled startup Drain sandbox creation before teardown, register the actual job handle before setup failure can clear it, and inspect known jobs omitted from listings before verifying cleanup. Reject unsupported sandbox names at the Python API boundary. Co-Authored-By: OpenAI Codex --- src/openenv/validation/harbor.py | 52 ++++++++--- src/openenv/validation/runner.py | 4 +- .../test_harbor_binding_supervision.py | 38 ++++++++ .../test_validation/test_harbor_validation.py | 86 ++++++++++++++++++- tests/test_validation/test_runner.py | 6 ++ 5 files changed, 172 insertions(+), 14 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index 83f190b2b6..01a0dedbbb 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -63,7 +63,7 @@ } _TERMINAL_STAGES = {"COMPLETED", "CANCELED", "ERROR", "DELETED"} _JOB_STOP_TIMEOUT_S = 60 -# Inside the 600-second validator start budget, so Sandbox.create finishes first. +# Readiness budget inside the 600-second start budget; allocation can take longer. _SANDBOX_START_TIMEOUT_S = 540 # Checks this binding measures; every other local check is a named SKIP. _IMPLEMENTED = { @@ -303,6 +303,7 @@ def __init__(self, image, flavor, namespace, resources): self.namespace = namespace self._resources = resources self._api = None + self._observed_jobs = {} def preflight(self): from huggingface_hub import HfApi @@ -338,11 +339,22 @@ def preflight(self): }, } + def register_job(self, project, job_id): + # The Sandbox handle is authoritative even before AGENT_START or listing. + self._observed_jobs.setdefault(project, set()).add(job_id) + def _jobs(self, project): if not _PROJECT.fullmatch(project): raise NativeEvidenceError("not a run-owned Harbor project") labels = _job_labels(project.removesuffix("__env")) jobs = list(self._api.list_jobs(labels=labels, namespace=self.namespace)) + # A listing omission does not prove a previously observed job terminated. + listed_ids = {job.id for job in jobs} + known_ids = self._observed_jobs.get(project, set()) + jobs.extend( + self._api.inspect_job(job_id=job_id, namespace=self.namespace) + for job_id in sorted(known_ids - listed_ids) + ) # Cleanup cancels what this returns, so recheck the server-side filter. if any( not labels.items() <= (job.labels or {}).items() @@ -350,6 +362,7 @@ def _jobs(self, project): for job in jobs ): raise NativeEvidenceError("HF Jobs returned a job outside this run") + self._observed_jobs[project] = known_ids | listed_ids return jobs def inventory(self, project): @@ -382,7 +395,7 @@ def capture(self, project, retained_tag, *, retain=True): } def cleanup(self, project): - # A start abandoned at its deadline can still create a job, so keep looking. + # Startup settles allocation before teardown; inspect until each job ends. deadline = time.monotonic() + _JOB_STOP_TIMEOUT_S cancelled = set() while True: @@ -482,15 +495,24 @@ async def _start_sandbox(environment, *, force_build, labels, namespace): # Harbor's hf-sandbox start, plus labels that let cleanup find every run-owned job. from huggingface_hub import Sandbox - environment._sandbox = await asyncio.to_thread( - Sandbox.create, - image=environment.task_env_config.docker_image, - flavor=environment._flavor, - idle_timeout=environment._idle_timeout, - namespace=namespace, - labels=labels, - start_timeout=_SANDBOX_START_TIMEOUT_S, + creating = asyncio.create_task( + asyncio.to_thread( + Sandbox.create, + image=environment.task_env_config.docker_image, + flavor=environment._flavor, + idle_timeout=environment._idle_timeout, + namespace=namespace, + labels=labels, + start_timeout=_SANDBOX_START_TIMEOUT_S, + ) ) + try: + environment._sandbox = await asyncio.shield(creating) + except asyncio.CancelledError: + # A cancelled worker can still allocate a job. Settle it and preserve its + # handle before Harbor teardown and the final ownership/terminal checks. + environment._sandbox = await creating + raise if workdir := environment.task_env_config.workdir: await asyncio.to_thread(environment._sandbox.files.mkdir, workdir) await environment.ensure_dirs(environment._mount_targets(writable_only=True)) @@ -760,8 +782,14 @@ async def _trial( async def build_and_start(*, force_build): nonlocal phase - async with asyncio.timeout(build_budget): - await start_environment(force_build=force_build) + try: + async with asyncio.timeout(build_budget): + await start_environment(force_build=force_build) + finally: + if hf and trial.agent_environment._sandbox is not None: + inspector.register_job( + project, trial.agent_environment._sandbox.id + ) phase = "agent setup" trial.agent_environment.start = build_and_start diff --git a/src/openenv/validation/runner.py b/src/openenv/validation/runner.py index 8ccce7461e..4fd19388f7 100644 --- a/src/openenv/validation/runner.py +++ b/src/openenv/validation/runner.py @@ -603,7 +603,7 @@ def run_validation( [`~openenv.validation.signature.UnsupportedPackageError`] for recognized-but-unsupported ones (CLI exit code 2). A package whose declarations fail the manifest schema yields a normal report with a `static.manifest` FAIL - (exit code 1). + (exit code 1). An unknown `sandbox` raises `ValueError` before validation. Args: target (`Path`): @@ -634,6 +634,8 @@ def run_validation( Returns: [`~openenv.validation.report.ValidationReport`]: the completed report. """ + if sandbox not in ("docker", "hf-sandbox"): + raise ValueError(f"unknown sandbox {sandbox!r}; expected docker or hf-sandbox") target = Path(target) wants_runtime = max_level >= Level.RUNTIME policy = policy or load_policy("v2" if wants_runtime else "v1") diff --git a/tests/test_validation/test_harbor_binding_supervision.py b/tests/test_validation/test_harbor_binding_supervision.py index dea0b97fbc..46ae1a2acf 100644 --- a/tests/test_validation/test_harbor_binding_supervision.py +++ b/tests/test_validation/test_harbor_binding_supervision.py @@ -2,6 +2,7 @@ import asyncio import signal +import threading from types import SimpleNamespace import pytest @@ -215,3 +216,40 @@ def create(**kwargs): assert "/logs/verifier" in calls[2][1][-1] await trial.agent_environment.stop(delete=True) assert calls[-1] == ("kill",) + + +@pytest.mark.asyncio +async def test_hf_sandbox_cancelled_start_retains_the_late_job(monkeypatch): + import huggingface_hub + + entered = threading.Event() + release = threading.Event() + sandbox = SimpleNamespace(id="late-job") + environment = SimpleNamespace( + task_env_config=SimpleNamespace(docker_image="pinned-image"), + _flavor="cpu-basic", + _idle_timeout=600, + _sandbox=None, + ) + + def create(**kwargs): + entered.set() + assert release.wait(5) + return sandbox + + monkeypatch.setattr(huggingface_hub.Sandbox, "create", create) + running = asyncio.create_task( + harbor._start_sandbox(environment, force_build=False, labels={}, namespace="me") + ) + try: + assert await asyncio.to_thread(entered.wait, 5) + running.cancel() + await asyncio.sleep(0) + cancelled_before_allocation_finished = running.done() + finally: + release.set() + with pytest.raises(asyncio.CancelledError): + await running + # Outer teardown must not run before creation settles or lose the job handle. + assert not cancelled_before_allocation_finished + assert environment._sandbox is sandbox diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 67a5d7f3ad..511bb5602e 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -154,7 +154,10 @@ def __init__(self, config): agent_dir=directory / "agent", ) self.agent_environment = SimpleNamespace( - session_id=config.trial_name + "__env", start=self.start, stop=self.stop + session_id=config.trial_name + "__env", + start=self.start, + stop=self.stop, + _sandbox=None, ) self.hooks = {} self._openenv_cancelled_compose = [] @@ -1431,6 +1434,7 @@ def start(trial): status=SimpleNamespace(stage="RUNNING"), ) state.jobs[job.id] = state.by_trial[trial.config.trial_name] = job + trial.agent_environment._sandbox = SimpleNamespace(id=job.id) running = [job for job in state.jobs.values() if job.status.stage == "RUNNING"] state.max_running = max(state.max_running, len(running)) @@ -1465,6 +1469,9 @@ def cancel_job(self, *, job_id, namespace): if not state.cancel_fails: state.jobs[job_id].status.stage = "CANCELED" + def inspect_job(self, *, job_id, namespace): + return state.jobs[job_id] + boundary.on_start = start boundary.on_stop = stop monkeypatch.setattr(huggingface_hub, "HfApi", Api) @@ -1573,6 +1580,36 @@ def test_hf_sandbox_job_must_run_the_pinned_image(task, boundary, hf_jobs): assert all(job.status.stage == "CANCELED" for job in hf_jobs.jobs.values()) +def test_hf_sandbox_failed_setup_registers_the_actual_job_before_teardown( + task, boundary, hf_jobs, monkeypatch, tmp_path +): + import huggingface_hub + + start = boundary.on_start + + def failed_setup(trial): + start(trial) + raise OSError("sandbox setup failed") + + def failed_stop(trial): + # Harbor clears its handle even when kill fails. + trial.agent_environment._sandbox = None + + boundary.on_start = failed_setup + boundary.on_stop = failed_stop + monkeypatch.setattr(huggingface_hub.HfApi, "list_jobs", lambda self, **kwargs: []) + monkeypatch.setattr(harbor.time, "sleep", lambda seconds: None) + artifacts = tmp_path / "failed-hf-setup" + report = run_validation(task, sandbox="hf-sandbox", artifacts_dir=artifacts) + assert result(report, "runtime.startup").status is CheckStatus.ERROR + assert len(hf_jobs.jobs) == 3 + assert sorted(hf_jobs.cancelled) == sorted(hf_jobs.jobs) + assert all(job.status.stage == "CANCELED" for job in hf_jobs.jobs.values()) + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert native["cleanup"]["verified"] + assert all(trial["cleanup"]["jobs"] for trial in native["trials"]) + + @pytest.mark.parametrize("cancel_works", [True, False]) def test_hf_sandbox_cleanup_cancels_jobs_that_survive_teardown( task, boundary, hf_jobs, tmp_path, monkeypatch, cancel_works @@ -1623,6 +1660,53 @@ def test_hf_jobs_cleanup_never_cancels_jobs_outside_its_run(labels, owner): assert not cancelled +@pytest.mark.parametrize("inspect_works", [True, False]) +def test_hf_jobs_cleanup_checks_observed_jobs_missing_from_listing( + monkeypatch, inspect_works +): + project = "openenv-validation-" + "d" * 32 + "-nop__env" + job = SimpleNamespace( + id="known-job", + labels=harbor._job_labels(project.removesuffix("__env")), + owner=SimpleNamespace(name="me"), + status=SimpleNamespace(stage="RUNNING"), + docker_image=PINNED, + flavor="cpu-basic", + ) + listing = [job] + cancelled = [] + + def inspect(*, job_id, namespace): + assert job_id == job.id and namespace == "me" + if not inspect_works: + raise OSError("job readback unavailable") + return job + + def cancel(*, job_id, namespace): + assert job_id == job.id and namespace == "me" + cancelled.append(job_id) + job.status.stage = "CANCELED" + + inspector = harbor.HFJobsInspector(PINNED, "cpu-basic", "me", None) + inspector._api = SimpleNamespace( + list_jobs=lambda **kwargs: list(listing), + inspect_job=inspect, + cancel_job=cancel, + ) + monkeypatch.setattr(harbor.time, "sleep", lambda seconds: None) + assert inspector.capture(project, None)["container_id"] == job.id + listing.clear() + if inspect_works: + cleanup = inspector.cleanup(project) + assert cleanup["verified"] + assert cleanup["jobs"] == {job.id: "CANCELED"} + assert cancelled == [job.id] + else: + with pytest.raises(OSError, match="readback unavailable"): + inspector.cleanup(project) + assert not cancelled + + def test_hf_sandbox_gives_every_trial_its_own_worker_thread( task, boundary, hf_jobs, canaries, monkeypatch ): diff --git a/tests/test_validation/test_runner.py b/tests/test_validation/test_runner.py index 2498a6fa0d..4bdd2b5cea 100644 --- a/tests/test_validation/test_runner.py +++ b/tests/test_validation/test_runner.py @@ -13,6 +13,12 @@ from support.runtime import evidence, FakeRuntimeProvider +@pytest.mark.parametrize("level", list(Level)) +def test_unknown_sandbox_is_rejected_before_validation(tmp_path, level): + with pytest.raises(ValueError, match="unknown sandbox"): + run_validation(tmp_path, sandbox="hf_sandbox", max_level=level) + + @pytest.mark.parametrize( "mode", ["run", "skip-build", "source-change", "capability", "applies-to"] ) From 95050bc6347d3e5c87d95a84740400daa416d543 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 15:21:14 +0100 Subject: [PATCH 3/7] fix(validation): shield allocation drain from nested timeouts Harbor and validator timeout supervisors can both cancel startup. Preserve the pending worker and its eventual job handle through repeated cancellation, then propagate cancellation for owned teardown. Co-Authored-By: OpenAI Codex --- src/openenv/validation/harbor.py | 8 +++++++- .../test_validation/test_harbor_binding_supervision.py | 10 +++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index 01a0dedbbb..851ba29948 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -511,7 +511,13 @@ async def _start_sandbox(environment, *, force_build, labels, namespace): except asyncio.CancelledError: # A cancelled worker can still allocate a job. Settle it and preserve its # handle before Harbor teardown and the final ownership/terminal checks. - environment._sandbox = await creating + # Both Harbor and the validator can cancel the same start at its deadline. + while not creating.done(): + try: + await asyncio.shield(creating) + except asyncio.CancelledError: + continue + environment._sandbox = creating.result() raise if workdir := environment.task_env_config.workdir: await asyncio.to_thread(environment._sandbox.files.mkdir, workdir) diff --git a/tests/test_validation/test_harbor_binding_supervision.py b/tests/test_validation/test_harbor_binding_supervision.py index 46ae1a2acf..4bf4dd65d1 100644 --- a/tests/test_validation/test_harbor_binding_supervision.py +++ b/tests/test_validation/test_harbor_binding_supervision.py @@ -219,7 +219,10 @@ def create(**kwargs): @pytest.mark.asyncio -async def test_hf_sandbox_cancelled_start_retains_the_late_job(monkeypatch): +@pytest.mark.parametrize("cancellations", [1, 2]) +async def test_hf_sandbox_cancelled_start_retains_the_late_job( + monkeypatch, cancellations +): import huggingface_hub entered = threading.Event() @@ -243,8 +246,9 @@ def create(**kwargs): ) try: assert await asyncio.to_thread(entered.wait, 5) - running.cancel() - await asyncio.sleep(0) + for _ in range(cancellations): + running.cancel() + await asyncio.sleep(0) cancelled_before_allocation_finished = running.done() finally: release.set() From fc92066f0809d4e5080d2419dffde3c5019a0bd7 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 15:31:18 +0100 Subject: [PATCH 4/7] test(validation): bound cancelled sandbox task completion Wait for the cancelled startup task with an explicit deadline while preserving the cancellation assertion and avoiding a CodeQL no-effect finding. Co-Authored-By: OpenAI Codex --- tests/test_validation/test_harbor_binding_supervision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validation/test_harbor_binding_supervision.py b/tests/test_validation/test_harbor_binding_supervision.py index 4bf4dd65d1..bf8a0e2315 100644 --- a/tests/test_validation/test_harbor_binding_supervision.py +++ b/tests/test_validation/test_harbor_binding_supervision.py @@ -253,7 +253,7 @@ def create(**kwargs): finally: release.set() with pytest.raises(asyncio.CancelledError): - await running + await asyncio.wait_for(running, timeout=5) # Outer teardown must not run before creation settles or lose the job handle. assert not cancelled_before_allocation_finished assert environment._sandbox is sandbox From 36e373ea411e1d23c8de9f13f44d667c29d77fd7 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 15:33:34 +0100 Subject: [PATCH 5/7] fix(validation): fail cleanup closed after unbound HF creation HF Sandbox creation can raise after allocating a job and failing its best-effort cancellation. Record missing startup handles and require recovery of a labelled job before cleanup can be verified. Exercise both recovered and invisible allocations with fakes. Co-Authored-By: OpenAI Codex --- src/openenv/validation/harbor.py | 20 ++++++++-- .../test_validation/test_harbor_validation.py | 40 +++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index 851ba29948..aa44c462e1 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -792,10 +792,11 @@ async def build_and_start(*, force_build): async with asyncio.timeout(build_budget): await start_environment(force_build=force_build) finally: - if hf and trial.agent_environment._sandbox is not None: - inspector.register_job( - project, trial.agent_environment._sandbox.id - ) + if hf: + sandbox = trial.agent_environment._sandbox + record["job_handle_missing_after_start"] = sandbox is None + if sandbox is not None: + inspector.register_job(project, sandbox.id) phase = "agent setup" trial.agent_environment.start = build_and_start @@ -923,6 +924,17 @@ async def verification_start(_event): if registered: try: record["cleanup"] = await asyncio.to_thread(inspector.cleanup, project) + if ( + hf + and record.get("job_handle_missing_after_start") + and not record["cleanup"]["jobs"] + ): + # Sandbox.create can fail after allocation and swallow a failed + # cancellation. An empty listing cannot prove no job was created. + record["cleanup"].update( + verified=False, + reason="sandbox creation returned no handle and no labelled job was recovered", + ) if trial is not None and trial._openenv_cancelled_compose: record["cleanup"].update( { diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 511bb5602e..1d769accf2 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -1610,6 +1610,46 @@ def failed_stop(trial): assert all(trial["cleanup"]["jobs"] for trial in native["trials"]) +@pytest.mark.parametrize("visible", [True, False]) +def test_hf_sandbox_creation_without_a_handle_requires_job_recovery( + task, boundary, hf_jobs, monkeypatch, tmp_path, visible +): + import huggingface_hub + + start = boundary.on_start + + def failed_creation(trial): + start(trial) + # The SDK can allocate a job, fail readiness and its best-effort kill, + # then raise without returning the handle to Harbor. + trial.agent_environment._sandbox = None + raise OSError("sandbox readiness failed") + + boundary.on_start = failed_creation + hf_jobs.kill_fails = True + if not visible: + monkeypatch.setattr( + huggingface_hub.HfApi, "list_jobs", lambda self, **kwargs: [] + ) + monkeypatch.setattr(harbor.time, "sleep", lambda seconds: None) + artifacts = tmp_path / "hf-creation-failure" + report = run_validation(task, sandbox="hf-sandbox", artifacts_dir=artifacts) + assert result(report, "runtime.startup").status is CheckStatus.ERROR + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert native["cleanup"]["verified"] is visible + assert len(hf_jobs.jobs) == 3 + if visible: + assert sorted(hf_jobs.cancelled) == sorted(hf_jobs.jobs) + assert all(job.status.stage == "CANCELED" for job in hf_jobs.jobs.values()) + else: + assert not hf_jobs.cancelled + assert all(job.status.stage == "RUNNING" for job in hf_jobs.jobs.values()) + assert {trial["failure"] for trial in native["trials"]} == { + "CleanupNotVerified" + } + assert all(not trial["cleanup"]["verified"] for trial in native["trials"]) + + @pytest.mark.parametrize("cancel_works", [True, False]) def test_hf_sandbox_cleanup_cancels_jobs_that_survive_teardown( task, boundary, hf_jobs, tmp_path, monkeypatch, cancel_works From 6d1c97182d47779d1946672bf89e61acb9ec8c5b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 12 Nov 2026 14:33:22 +0000 Subject: [PATCH 6/7] fix(validation): compare HF flavor capacity in MiB --- src/openenv/validation/harbor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index aa44c462e1..f0a3755f78 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -322,8 +322,8 @@ def preflight(self): ) if ( cpu < self._resources.cpu - or memory_gb * 1000 < self._resources.memory_mb - or disk_gb * 1000 < self._resources.disk_mb + or memory_gb * 1024 < self._resources.memory_mb + or disk_gb * 1024 < self._resources.disk_mb ): raise NativeEvidenceError( f"HF Jobs flavor {self.flavor} ({item.cpu}, {item.ram} RAM, " From c85a653c9240b3e58c59efebaa921a27507de44e Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Thu, 12 Nov 2026 15:36:30 +0100 Subject: [PATCH 7/7] test(validation): cover HF flavor capacity boundaries Accept the advertised memory and storage capacities expressed in contract MiB, and reject one MiB above each limit before job allocation. Co-Authored-By: OpenAI Codex --- tests/test_validation/test_harbor_validation.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 1d769accf2..2bafc8cd39 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -1571,6 +1571,23 @@ def test_hf_sandbox_flavor_must_exist_and_cover_the_contract( assert not boundary.trials and not hf_jobs.jobs +@pytest.mark.parametrize( + "resource,capacity", [("memory_mb", 16 * 1024), ("disk_mb", 50 * 1024)] +) +@pytest.mark.parametrize("excess", [0, 1]) +def test_hf_sandbox_flavor_capacity_uses_contract_mib( + task, boundary, hf_jobs, resource, capacity, excess +): + path = task / "validation/openenv.json" + contract = json.loads(path.read_text()) + contract["resources"][resource] = capacity + excess + path.write_text(json.dumps(contract)) + report = run_validation(task, sandbox="hf-sandbox") + startup = result(report, "runtime.startup") + assert startup.status is (CheckStatus.ERROR if excess else CheckStatus.PASS) + assert bool(hf_jobs.jobs) is (not excess) + + def test_hf_sandbox_job_must_run_the_pinned_image(task, boundary, hf_jobs): hf_jobs.image = "registry.example/image@sha256:" + "e" * 64 report = run_validation(task, sandbox="hf-sandbox")