diff --git a/.claude/docs/REPO_WALKTHROUGH.md b/.claude/docs/REPO_WALKTHROUGH.md index bbec02843..1c542410e 100644 --- a/.claude/docs/REPO_WALKTHROUGH.md +++ b/.claude/docs/REPO_WALKTHROUGH.md @@ -284,6 +284,11 @@ ThinkingBox examples: `example_usage.py` is a public-client smoke test, while `e ## Harbor integration +`examples/coding_audit/` owns the supplemental operator coding audit, its standalone +`python -m examples.coding_audit` CLI, and offline tests. It is separate from core +RFC 008 validation. `examples/hub_validation_pipeline/` prepares and submits its +frozen cloud bundles. + `src/openenv/core/harness/capture/` records inference calls independently of a trainer or tokenizer. `src/openenv/harbor/` runs Harbor tasks, reconciles their trajectories, and exposes the shared training contract through clients and the Gradio playground. `envs/harbor_env/` is the installable environment wrapper. `examples/harbor/nemo_shell_profile/` provides the explicitly qualified NeMo shell workflow. See `docs/source/guides/harbor-provider-qualification.md` for provider and adapter evidence, `rfcs/012-harbor-capture-providers.md` for explicit evaluation/training and session ownership, and `tests/envs/test_harbor*.py` / `test_capture*.py` for deterministic capture regressions. diff --git a/docs/plans/hub-environment-trainability-validation.md b/docs/plans/hub-environment-trainability-validation.md new file mode 100644 index 000000000..a2dfe5976 --- /dev/null +++ b/docs/plans/hub-environment-trainability-validation.md @@ -0,0 +1,291 @@ +# Selecting OpenEnv/Harbor environments for training + +This research plan separates native validation, supplemental audit evidence and +controlled learning experiments. Its scientific recommendations do not qualify +a private corpus, a provider or an OpenEnv package. + +Choose environments for **trustworthy rewards, useful policy-dependent difficulty, +diversity, and measured held-out learning per unit cost**. Generate missing solutions, +tests, and scaffolding as candidates, then validate them independently. A generated +solution passing a generated grader establishes agreement between those components; +it does not establish task correctness or trainability. + +## Current evidence boundary + +The [example README](../../examples/hub_validation_pipeline/README.md), +[`pipeline.py`](../../examples/hub_validation_pipeline/pipeline.py), and +[`harbor.py`](../../src/openenv/validation/harbor.py) define the native boundary: + +- `harbor-semantic-v1` requires `static.manifest`, `runtime.startup`, + `runtime.reward_well_formed`, `runtime.episode_determinism`, + `semantic.oracle_max` and `semantic.floor_gap` to PASS. Declared canaries also + require `semantic.canary_floor` to PASS. Any FAIL/ERROR, required SKIP or missing + required evidence blocks admission; other standard checks remain named SKIPs. +- Schema 2 native evidence binds fresh NOP, oracle, oracle-replay and declared + canary Trials to original/effective task digests, the same retained image and + inspected configuration, raw verifier rewards, successful oracle exits and + verified cleanup. Replay checks concern the same exact submission. +- The maximum, floor gap and canary outcomes are checked against an explicit + operator contract. They measure the frozen controls, not general specification + correctness. The complete `full` admission inventory remains unqualified. +- `evaluate()` requests a configured model attempt per admitted task. It does not + perform optimizer updates or a controlled multi-model training experiment. +- The native adapter rejects separate verifier environments, declared artifact + extensions and non-public/differing phase network policies. Upstream capability + alone does not qualify those contracts in this adapter. + +The supplemental `harbor-coding-audit-v1` implementation lives in +[`examples/coding_audit`](../../examples/coding_audit/) and is invoked with +`python -m examples.coding_audit --task TASK --config CONFIG --output REPORT`. +Cloud and discovery orchestration import this example package. It generates bounded +candidate references, tests operator-reviewed negatives, repeats verifier runs and +measures target attempts after screening. Its reports always retain +`rfc008_complete: false` and `trainability_established: false`. + +Real-task recipes require an explicit `--panel PATH`, including pinned source +identity, execution boundaries, full audit configuration and privately reviewed +counterexamples. The exact panel byte snapshot is hashed into preparation/bundle +provenance; native task/verifier files stay unchanged. Keep evaluation-only tasks, +private controls and populated results outside public source and training data. + +Keep semantic checks in the validator and corpus/experiment selection in examples. +[The #1591 contract](https://github.com/openenv/OpenEnv/pull/1591) versions the +native evidence; new trial roles or guarantees require a versioned extension. +[RFC 008](../../rfcs/008-environment-auto-validation.md) also names containment, +verifier determinism, leakage, canaries and replayability. Implemented IDs or a +supplemental PASS cannot substitute for complete evidence of those contracts. + +The example pins Harbor at `9c7a39b991d7e2205019f6be0598cbd22d84bfc3`. +Its Oracle agent executes a supplied solution script; its verifier executes +provided tests and reads their rewards. Those roles do not independently certify +the solution or verifier. [Pinned Oracle source][harbor-oracle], +[pinned verifier source][harbor-source] + +## What the research supports + +- **Verifier quality needs both acceptance and rejection controls.** SWE-bench's + original verification process reviews task clarity and tests that reject valid + implementations; its evaluation uses hidden issue-fixing and regression tests. + Human screening improves reliability without proving complete correctness. + [Original SWE-bench Verified methodology][verified] +- **Generated tests can find false positives, but are fallible.** UTBoost reports + 345 previously accepted erroneous patch evaluations exposed through test + augmentation **and parser corrections**. Its behavioral comparisons depend on + reference patches, so they do not independently prove specification completeness. + R2E-Gym separately documents low discrimination of generated tests and biases in + learned verifiers. [UTBoost][utboost], [R2E-Gym][r2e] +- **Public benchmark success needs contamination checks.** OpenAI's February 2026 + audit reports residual test-design defects and elicitable task/solution knowledge. + Its 59.4% defect figure concerns a selected set of 138 difficult instances, + not the full benchmark or arbitrary Harbor tasks. [2026 SWE-bench audit][audit] +- **Synthetic tasks have demonstrated training value in particular settings.** + SWE-smith generates bugs that break previously passing repository tests and + reports improved external benchmark performance after supervised fine-tuning. + SWE-Gym trains agents and outcome verifiers; its repeated-attempt selection gains + are distinct from single-attempt training gains. Neither result establishes + online RL benefit for this corpus. [SWE-smith][smith], [SWE-Gym][gym] +- **RL and self-evolving tasks have direct, bounded evidence.** DAPO reports math + RL results and filters uniform-reward groups to retain useful relative-advantage + signal; this does not establish coding transfer or universal cost savings. + Self-play SWE-RL reports coding RL gains on external benchmarks, validates + injected bugs through execution/inverse mutation checks, and restores oracle + tests before grading. It also discusses degenerate challengers and narrow + curricula: solve-rate targeting alone can be gamed. These are results for the + papers' recipes, not qualification of OpenEnv packages. [DAPO][dapo], + [Self-play SWE-RL, June 2026 revision][ssr] + +## Proposed stages, in priority order + +### 1. Establish the specification and a frozen verifier + +Select a small, relevant, training-permitted corpus. Record provenance, licenses, +repository/release, task lineage, bug/template family, and intended skills. Split +by repository **and task-generation/bug family before generating variants**; +near-duplicate prompts, related patches, and descendants of one source task stay +together. Keep an evaluation-only benchmark outside the training candidate pool. + +Review each task's behavioral requirements before using its reward. Assemble: + +- A known-bad starting state/NOP; known-good repairs checked against the requirements; + and an independently implemented valid alternative where feasible. Reject + implementation-specific assertions that the instructions do not justify. +- Issue-specific `FAIL_TO_PASS` checks plus `PASS_TO_PASS` regression coverage. + Validate required test discovery/counts and individual results; a zero-test run, + skipped required tests, swallowed collection error, or stale reward cannot pass. +- Hidden edge cases, property/metamorphic checks, and counterexamples justified by + the specification. Differential comparison with a reference implementation is a + useful signal; agreement with one potentially wrong reference is insufficient. + +Freeze the task specification, verifier source/image/dependencies, test inventory, +and reward aggregation before comparative rollouts. Repeat grading of the **same +submission** in fresh environments to measure verifier flakiness separately from +policy sampling variance. Test ordering, filesystem state, clock/seed dependence, +and parser behavior need controls. These recommendations extend the failure modes +documented in [Verified][verified] and [UTBoost][utboost]. + +### 2. Separate public scaffolding from private evaluation + +Use three explicit artifact classes: + +- **Solver-visible:** instructions, starter repository, dependencies, public examples + and development tests, generated API stubs/documentation, and tool scaffolding. +- **Training evaluator-private:** reference repairs, hidden reward tests, expected + outputs, counterexamples, grader configuration, and candidate-generation traces. + The policy may receive the declared reward/limited feedback, not these artifacts. +- **Sealed evaluation:** held-out task families and their independent verifier/tests; + inaccessible to training, task generators, candidate selection, and tuning. + +Harbor uploads shared-mode tests after the agent runs and has a separate-verifier +mode with explicit output artifact transfer. A clean verifier image can also avoid +runtime dependency installation. Qualify this adapter's support for that mode with +both agent and verifier image/configuration identities and run-owned cleanup. +Artifact contents remain untrusted. [Harbor verifier][harbor-verifier], +[separate verifier and artifact boundary][harbor-separate] + +For code execution, a separate container alone is insufficient: submitted code +must not run with the grader's authority to read answer keys or write rewards. +Use a restricted execution worker and a protected result collector, with only +declared submission files crossing the boundary. Probe copied answers, forged +reward files, test-runner/PATH/import shadowing, disabled assertions, symlink/path +escapes, background-process interference, and grader-file modification. Inspect +the solver image, mounts, history/caches, logs, network access, and captured +observations for solutions, generation prompts, upstream fixed commits, or private +test material. Public-network v1 cannot establish an offline leakage boundary. +Runtime isolation and pretraining contamination are separate concerns; hidden +files do not remove public-source memorization. [Harbor boundary][harbor-separate], +[contamination evidence][audit] + +### 3. Generate candidates and challenge them + +Prefer existing reviewed solutions. When one is missing, generate a bounded number +of candidate repairs from solver-visible inputs, and record generator/model revision, +prompt/input digests, visibility, seeds, dependencies, cost, and patch digest. Replay +each from the same clean start; inspect privileged oracle scripts so writing reward +files or modifying tests cannot masquerade as solving the task. Independently +check a passing repair before promoting it to a reference candidate. +Generated components create a derived task version: retain the parent source digest +and new content/component digests, then revalidate its bound artifacts. + +Generate counterexamples and task-relevant mutants: boundary-condition changes, +removed error handling, partial fixes, constant/hard-coded outputs, regressions, +and exploit submissions. Record which known-invalid behaviors the verifier rejects +and which valid alternatives it accepts. Review surviving mutants: equivalent or +out-of-scope mutations are not verifier failures. A mutation score is relative to +that panel, not a correctness probability. Search especially for submissions that +pass training rewards but fail the independent verifier. [UTBoost][utboost], +[SWE-smith][smith], [R2E-Gym][r2e] + +**Prevent circular validation:** a component author cannot certify their own +solution/grader pair. Generate challenger tests from requirements and development +failures, independently review their assertions, then freeze them. Do not rewrite +the grader until a generated solution passes or obtain expected outputs solely +from that solution. A different generator model is useful diversity, not sufficient +independence. Disagreements trigger specification review/quarantine and a new +version; never silently repair the evaluator during a comparison. Final held-out +feedback must not enter this generation loop. + +Start with generated wrappers/public scaffolds and candidate repairs; expand next +to validated mutants of working repositories. Try adaptive/self-evolving task +generation only after the static pipeline shows useful transfer. Keep generator +changes on training families, and audit diversity and exploitability as well as +difficulty. [SWE-smith][smith], [Self-play SWE-RL][ssr] + +### 4. Measure rollout difficulty, variance, and efficiency + +Run repeated attempts with the intended trainable policy and at least one other +model family. Pin model/checkpoint, harness, prompt/tool schemas, sampling settings, +seed where supported, task/verifier/image digests, token/step/time limits, hardware, +and concurrency. Randomize task/run ordering. Retain every attempt, including +failures and retries; provider seeds do not guarantee reproducibility. + +Report per task/model: single-attempt solve rate with uncertainty, bounded-budget +pass@k separately, reward distribution, mixed-success group frequency, solution +diversity, truncation, environment/verifier error rates, and disagreement with the +independent evaluator. Regrade identical submissions to isolate verifier noise. +Infrastructure errors stay explicit rather than becoming model reward zero. + +For binary rewards, `p(1-p)` describes Bernoulli reward variance; high variance +could reflect useful policy uncertainty **or a broken verifier**. Uniform-reward +groups have no relative reward advantage in GRPO-style updates, motivating DAPO's +dynamic sampling. Neither a tiny all-zero sample proves impossibility nor an +all-one sample proves the task useless to every policy. Select a mixture of +reliable easy/moderate/hard tasks; refresh difficulty as the policy changes. +[DAPO][dapo] + +Measure build/cold-start, reset, agent, and verification time separately, with +median/tail latency, tokens, sandbox/GPU time, cache/storage, retries, and human +review. Rank reliable, diverse tasks by useful rollout yield and total cost; +include discarded generations and failed builds. Cache immutable builds and +preinstall verifier dependencies, while keeping fresh episode state. Timeouts and +long-horizon skills deserve explicit strata rather than deletion for cost alone. +[Harbor separate verifier][harbor-separate] + +### 5. Demonstrate trainability in a controlled RL pilot + +Pre-register the split, metric, budgets, and comparisons. Use identical initialization, +harness, optimizer recipe, and inference budgets. Compare the unchanged base model, +RL on a reviewed reference curriculum, and RL on a cost-matched curriculum with +the candidate subset replacing part of that reference. A matched-data SFT control +helps distinguish RL benefit from imitation of generated solutions. Repeat training +seeds and use development families for checkpoint selection. + +Evaluate on the sealed family/repository split using the frozen independent +verifier. Report baseline-to-trained **pass@1 transfer**, uncertainty across seeds +and families, regressions, reward-versus-independent-score divergence, and total +cost/time. Cluster uncertainty by task family; repeated attempts on one task are +not independent new tasks. Report best-of-k selection separately so additional +inference spending cannot appear as a learning gain. These controls are proposed +experimental design, not a protocol established wholesale by the cited papers. + +Advance only with positive held-out transfer against the base and useful improvement +against the matched reference, within a declared cost budget. Report incremental +cost per additional held-out solve and performance at equal total spend, including +generation/qualification/training/evaluation. If uncertainty includes no gain, +report inconclusive; if only training reward rises, trainability remains unproven. +Evidence qualifies the tested model/task distribution/recipe, not an environment +universally. [SWE-Gym][gym], [Self-play SWE-RL][ssr] + +## Suggested starting budgets and implementation seam + +These are **engineering suggestions, not scientific acceptance thresholds**: + +- Screen about 20 tasks across several families. Try up to 2–3 repair candidates + for missing oracles, 3 fresh replays per control, and 10–20 relevant faulty mutants + per task. Expand targeted panels when survivors expose gaps. +- Begin with 2 models × 4 attempts × 20 tasks = 160 rollouts. On roughly 10 + finalists, target 3 models × 8 attempts = 240 observations, reusing compatible + screening runs. Small samples provide coarse estimates; increase repeats/tasks + according to observed uncertainty and cost. +- For an RL feasibility pilot, reserve disjoint training, development, and held-out + families; for example, 60–100 training tasks, 20 development tasks, and 40 held-out + tasks across multiple repositories. Start with 256–512 attempted training episodes + per RL arm/seed and 2 training seeds; expand seeds and held-out families before + making a broad claim. Fix an attempted-rollout/token/wall-clock ceiling so dynamic + sampling cannot create an unbounded budget. Fewer tasks/seeds justify weaker claims. + +Implement incrementally: first native determinism/containment/replay evidence and +separate-verifier support; then private artifact, mutation, and leakage audits; +then repeated `evaluate()` orchestration; finally an external training experiment +consuming the frozen corpus. Keep training-framework-specific machinery in examples. +Suggested sidecars are `verifier-audit.json`, `generated-candidates.jsonl`, +`rollout-matrix.jsonl`, and `trainability.json`, bound to corpus/profile/verifier +digests. Keep explicit exclusions and incomplete outcomes. + +The resulting evidence labels should distinguish **runtime sanity**, **audited +verifier**, **rollout-characterized**, and **training benefit demonstrated for a +specified experiment**. These are proposed research labels, not new RFC PASS +statuses. No combined quality score should let a cheap rollout or oracle pass +compensate for leakage, an unreliable verifier, or missing required validation. + +[verified]: https://openai.com/index/introducing-swe-bench-verified/ +[audit]: https://openai.com/index/why-we-no-longer-evaluate-swe-bench-verified/ +[utboost]: https://aclanthology.org/2025.acl-long.189.pdf +[r2e]: https://arxiv.org/html/2504.07164v1 +[smith]: https://arxiv.org/html/2504.21798v1 +[gym]: https://arxiv.org/html/2412.21139v2 +[dapo]: https://arxiv.org/html/2503.14476v1 +[ssr]: https://arxiv.org/html/2512.18552v3 +[harbor-verifier]: https://docs.harborframework.com/tasks/verifier +[harbor-separate]: https://docs.harborframework.com/tasks/separate-verifier +[harbor-oracle]: https://github.com/harbor-framework/harbor/blob/9c7a39b991d7e2205019f6be0598cbd22d84bfc3/src/harbor/agents/oracle.py +[harbor-source]: https://github.com/harbor-framework/harbor/blob/9c7a39b991d7e2205019f6be0598cbd22d84bfc3/src/harbor/verifier/verifier.py diff --git a/docs/plans/hub-environment-validation-pipeline.md b/docs/plans/hub-environment-validation-pipeline.md new file mode 100644 index 000000000..bac14ae9d --- /dev/null +++ b/docs/plans/hub-environment-validation-pipeline.md @@ -0,0 +1,203 @@ +# Hub discovery, native validation and model execution + +The runnable implementation is [the Hub validation example](../../examples/hub_validation_pipeline/README.md). +It acquires immutable Hugging Face or Harbor Hub task sources, invokes OpenEnv's +native validator, admits tasks against an explicit profile, and optionally runs a +configured model on a new corpus containing only admitted tasks. Source packages +and exclusions are retained. This document describes the reusable implementation +and its qualification boundary, without operational corpus or run records. + +Native semantic validation uses the `harbor-semantic-v1` contract and schema 2 +evidence introduced by [#1591](https://github.com/openenv/OpenEnv/pull/1591). +The [supplemental coding audit](../../examples/hub_validation_pipeline/CODING_AUDIT.md) +is example code in `examples/coding_audit`, with orchestration in +`examples/hub_validation_pipeline`. Neither a bounded native admission nor a +supplemental audit establishes complete RFC 008 coverage or training benefit. + +## Pipeline and ownership + +| Stage | Responsibility | Retained output | +| --- | --- | --- | +| Discover | Query both hubs, exhaust pagination and freeze candidate identities. | Candidate metadata and source aliases | +| Acquire | Resolve immutable identities and bound selected downloads. | Original task files, inventories and hashes | +| Enumerate | Identify task roots and preserve ordered canonical identities. | Per-task source records | +| Validate | Use OpenEnv's parser, native collector and standard graders. | JSON reports, raw evidence and checksums | +| Admit | Verify required PASS results and provenance for the selected profile. | Accepted/excluded records with reasons | +| Materialize | Copy admitted tasks with explicit image/resource overlays. | Frozen corpus and index-to-task mapping | +| Evaluate | Use the existing Harbor harness and capture runner. | Every attempted outcome and cleanup evidence | + +Core native validation remains in OpenEnv. Discovery, admission policy, cloud +launching, model selection, ratebooks and research experiments remain in examples. +A sibling sandbox or outer `harbor_env` server cannot establish properties of the +actual task sandbox. Every execution copy must bind to its acquired source and +validated image/configuration. + +## Acquisition and HF conventions + +HF discovery uses both `library:harbor` and `rl-environment`, checks those tags in +returned metadata, and consumes listing pagination before applying a candidate +limit. Trending rank selects discovery priority; it is not a stable task identity. +HF sources require full commit revisions and Harbor packages require full `sha256:` +references. Resolve a mutable HF alias once and record the resulting commit before +acquisition. Never infer byte identity from matching dataset names or task counts. + +The SDK's `snapshot_download`/`hf_hub_download` arguments correspond to +`hf download --repo-type dataset --revision ... --local-dir ...`. The example +adds anonymous public-source reads (`token=False`), explicit selected-file/task +bounds and immutable revision checks. It materializes regular files for Harbor +rather than relying on cache links. Gated, disabled, ambiguous or unsupported +sources are exclusions; they do not silently fall back to another registry. +Harbor's Git source pin is checked because a matching package version alone does +not establish that the required listing API exists. + +Check Git blob/content hashes and declared file sizes, reject escaping paths, +symlinks, special files and secret-file inputs, and retain the original acquired +inventory. Parsing/acquisition never executes downloaded task scripts on the +controller host. Source revisions do not pin container bases, package downloads +or external services; those need separate image/runtime evidence. + +HF publication uses explicit SDK add operations and retained immutable commit IDs. +It does not synchronize a local folder by deleting remote files. This follows the +`hf upload` convention that deletion is explicit. The cloud workflow additionally +requires an existing private result dataset and a fresh content-derived run claim; +ordinary upload success does not authorize duplicate paid execution. + +## Native semantic admission + +The default `full` profile requires the complete pinned static/runtime/semantic +inventory. Missing or skipped required coverage prevents full admission. +The explicitly selected `harbor-semantic-v1` profile requires: + +- `static.manifest`, `runtime.startup`, `runtime.reward_well_formed` and + `runtime.episode_determinism` to PASS. +- `semantic.oracle_max` and `semantic.floor_gap` to PASS, plus + `semantic.canary_floor` when the operator declares canaries. +- No FAIL or ERROR anywhere in the report; no duplicate, unknown or missing + required result, and execution of all required validation levels. +- Schema 2 native evidence for fresh NOP, oracle and oracle-replay Trials, + followed by the declared canaries, all with the same retained image and inspected + configuration, unchanged task identities and verified cleanup. +- Bound raw reward files and successful oracle exits. Recompute replay equality, + the declared oracle maximum, the NOP gap and canary floors from those files. +- Scale tolerance and margin by `high - low`: require + `oracle >= high - oracle_tolerance * (high - low)` and both `high - floor` and + `oracle - floor` to be at least `floor_margin * (high - low)`. +- Read the retained tag from `runtime.startup.measured.retained_image_tag` and + require equality with checksummed native/trial evidence. The validator retains + this tag only for runs without FAIL, ERROR or interruption. + +Other standard checks remain named SKIPs and can leave the underlying report WARN. +The admission envelope records these limits beside the unchanged report; it does +not turn the report into full RFC qualification. Missing required evidence is +incomplete, not PASS, and any confirmed failure keeps FAIL precedence. + +The contract lives at `validation/openenv.json` in the validation subject and +explicitly declares the reward key/range, tolerance, floor margin, resource +allocation and optional canary directory. A pipeline-supplied overlay records +both the original source digest and the effective validation digest. The current +native adapter supports self-contained, single-step Linux tasks with one main +Docker service, shared verifier execution and public networking. Unsupported resource, +credential, multi-service and extension contracts stay named SKIPs. +The first trial builds the reviewed Dockerfile or pulls a task-declared +`environment.docker_image` pinned as `name@sha256:`, recorded as +`source_image`. Later trials and model handoff use the inspected immutable image +kept by the startup-recorded retained tag. Mutable tags and tasks with neither +a Dockerfile nor a pinned image remain unsupported; their startup SKIP excludes +them before model handoff. Acquisition preserves the original bytes. A missing +`solution/solve.sh` still produces `semantic.oracle_max` FAIL. +Supplemental cloud audits remain a separate profile and cannot waive this gate. + +```bash +uv run --no-sync openenv validate inputs/task --level semantic --policy v2 \ + --json --output outputs/task-report.json + +uv run --no-sync python -m examples.hub_validation_pipeline.pipeline \ + --contract inputs/native-contract.json --profile harbor-semantic-v1 \ + --hf-dataset 'org/repo@FULL_COMMIT_SHA' --tasks-per-dataset 1 \ + --output outputs/native-selection +``` + +Use [the README](../../examples/hub_validation_pipeline/README.md) for installation, +Docker requirements and the synthetic demo. The generic example contract is not +a substitute for reviewing another task's reward/resource/canary declarations. + +## Safe model handoff and evidence + +Build a new run-owned corpus from admitted tasks only. Record its ordered mapping +from transport index to canonical task ID; filtering must not change identities +or expose excluded siblings. Materialize only the validated immutable image and +explicit resource overlay. A trusted pre-execution Trial hook checks each task +copy, source digest, actual image and inspected container configuration. + +Pin the model/harness/backend and sampling, token, step and time limits. Hosted +model IDs and returned fingerprints describe available provider evidence; a +public source revision does not attest the weights served remotely. Keep inference +credentials outside validation workers and task arguments. + +Record reward maps, the selected headline metric, execution status, errors, +traces, timing and cleanup. A valid reward of zero is a model outcome. Missing +rewards, failed infrastructure and partial result sets remain ungraded/incomplete. +Reconcile expected and returned task IDs. Do not replace failures with zero or +delete their source/evidence. Clean up exact run-owned containers, networks and +volumes; retained image tags are recorded for explicit later removal. + +## Supplemental coding audit + +`examples/coding_audit/{scoring.py,hf.py,runner.py,__main__.py}` contains the +operator-run `harbor-coding-audit-v1` workflow. It generates bounded candidate +references, repeats exact submissions against fresh verifier instances, tests +reviewed known-wrong cases, probes limited leakage/stale-state behavior and then +measures repeated target attempts. Cloud and pipeline orchestration import this +example package. The controller command is: + +```bash +uv run --no-sync python -m examples.coding_audit \ + --task inputs/task --config inputs/audit.json --output outputs/audit-report.json +``` + +Reference solutions, verifier-private files, known-wrong submissions and their +counterexamples remain operator-controlled evaluation inputs. Real-task recipes +require `--panel PATH`; they contain no built-in benchmark selection or negative +panel. The operator panel declares a pinned HF source, task paths, immutable +execution boundaries and complete audit configuration. Its exact byte snapshot +is read once and SHA256-bound to preparation and bundle provenance. Acquisition +preserves every native task/verifier byte. + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.real_tasks \ + --panel "$PANEL_PATH" --output outputs/native-recipes --bundle + +uv run --no-sync python -m examples.hub_validation_pipeline.full_run \ + --panel "$PANEL_PATH" --hf-census "$HF_CENSUS_PATH" \ + --harbor-census "$HARBOR_CENSUS_PATH" --budget-usd "$BUDGET_USD" \ + --output outputs/native-plans +``` + +These commands prepare inputs without paid execution. Operator-selected +`--task NAME` values are validated against the supplied panel. A metadata census +is an inventory, not a deduplicated or runtime-qualified corpus. Evaluation-only +panels never become training data. + +Paid native-cohort submission additionally requires a shared initialized ledger, +a reservation identifier and `--submit`. The aggregate estimate reserves the +configured request/token ceilings, controller allowance and each child's remote +maximum lifetime before the first submission. The fixed ratebook is a declared +modeling assumption; it is neither a current-price guarantee nor an account-enforced +billing cap. Every initial budget-bearing plan parse and final submission checks +the same expected plan-byte SHA256. Partial or uncertain receipts and unresolved +reservations remain durable for manual reconciliation; launchers never blindly +resume or retry. + +## Qualification and research boundary + +Keep source-level tests, Linux Docker evidence and live-provider qualification +separate. Framework fakes exercise binding, parsing, incomplete outcomes, budget +races and cleanup accounting; they do not establish a provider's execution or +billing behavior. Supplemental reports always retain `rfc008_complete: false` +and `trainability_established: false`. + +Verifier acceptance/rejection controls measure the reviewed panel, not complete +specification correctness. Repeated attempts measure policy-dependent difficulty; +they do not demonstrate learning. Training value requires a separate controlled +experiment with training-permitted data, sealed held-out task families, matched +budgets and independent evaluation. See [the trainability research plan](hub-environment-trainability-validation.md). diff --git a/examples/coding_audit/README.md b/examples/coding_audit/README.md index b8b8a5ca0..f2c31c395 100644 --- a/examples/coding_audit/README.md +++ b/examples/coding_audit/README.md @@ -35,6 +35,9 @@ alongside request parameters. Support depends on the selected provider/model. Candidate generation streams bounded content and reasoning; only a complete `stop` response with validated usage can produce an executable candidate. A `length` response remains rejected even when its terminal usage is retained. +The candidate prompt declares the allowed output files and requests a Bash script +that writes them. Its versioned contract is recorded with the model metadata; +standalone source-code fences remain errors. Recognized Bash/sh Markdown fences are parsed before execution; the raw response and both raw/script hashes are retained. Other language fences are rejected. @@ -54,6 +57,31 @@ and `size`, bound to the frozen task and checksummed evidence. This option does not enumerate task directories or include verifier, reference or specification files. + +Target requests reserve their prompt as well as completion allowance before dispatch. +The frozen policy uses encoded controller-request bytes plus an 8,192-token framing +allowance. This conservative scheduling assumption can stop early; it is not a +serving-tokenizer measurement or a guarantee about remote billing. Evidence records +the reservation, actual per-response usage, request hashes and dispatch decisions. +Reported input above the reservation, output above the request cap, inconsistent +usage, or cumulative usage above the configured total remains an error. +HTTP failures retain their numeric status without response bodies, URLs or headers. +They are not retried automatically; failed-request billing remains unknown. + +After a valid interaction prefix, a pre-dispatch budget stop collects only the +allowlisted files actually saved, then uses the normal fresh verifier. A funded, +validated API response ending in `length` is an explicit truncated target attempt: +its partial calls are discarded and existing files are graded. An initial prompt +that cannot fit remains unmeasured. Generator responses still require complete +`stop` output. No failed API, cleanup or grading operation becomes a synthetic +zero reward. + +A bindable target tool request with an unknown function name or invalid JSON +argument content gets bounded tool-error feedback and can be repaired within the +same call and token limits. It never executes an unknown method. Malformed call +identities/types, duplicate IDs in one response and backend failures remain errors; +ambiguous batches are rejected before any command executes. + The HF backend launches paid services when invoked without a supplied test backend. Use reviewed tasks and distinct execution and inference credentials. The Hub pipeline's cloud worker packages these production example modules with diff --git a/examples/coding_audit/hf.py b/examples/coding_audit/hf.py index 7868cf34e..ed7c0956e 100644 --- a/examples/coding_audit/hf.py +++ b/examples/coding_audit/hf.py @@ -21,18 +21,20 @@ from pathlib import Path, PurePosixPath from huggingface_hub import HfApi, InferenceClient, Sandbox -from huggingface_hub.errors import SandboxError +from huggingface_hub.errors import HfHubHTTPError, SandboxError from .runner import _candidate_context_paths _IMAGE = re.compile(r"[^\s@]+@sha256:[0-9a-f]{64}\Z") _REVISION = re.compile(r"[0-9a-f]{40}\Z") _TERMINAL = {"COMPLETED", "CANCELED", "ERROR", "DELETED"} +_PROMPT_FRAMING_ALLOWANCE = 8192 _LIMITATIONS = [ "Immutable images are control-plane references, not measured runtime digests.", "Sandbox RPC exposes the execution bearer to the workload server; hostile-code credential isolation is not qualified.", "The verifier executes submitted code as the same user; verifier integrity against hostile submissions is not qualified.", "Inference Providers do not pin served weights to the recorded model source revision.", + "Target prompt reservation uses a byte/framing assumption, not a serving-tokenizer or billing guarantee.", ] _TOOL = { "type": "function", @@ -57,12 +59,21 @@ class _DeadlineExceeded(HFBackendError): pass +class _TargetBudgetStop(HFBackendError): + def __init__(self, reason, *, response_received=False): + super().__init__(reason) + self.reason = reason + self.response_received = response_received + + @contextmanager def _deadline(seconds): # SDK control-plane calls have no timeout argument. A controller must run on # the main Unix thread so a blocked synchronous SDK call can be interrupted. if threading.current_thread() is not threading.main_thread(): raise HFBackendError("HF coding controller requires the main thread") + if not math.isfinite(seconds) or seconds <= 0: + raise _DeadlineExceeded("controller deadline exceeded") previous_handler = signal.getsignal(signal.SIGALRM) previous_timer = signal.getitimer(signal.ITIMER_REAL) started = time.monotonic() @@ -174,6 +185,25 @@ def _plain(value): return json.loads(json.dumps(value)) +def _retain_usage(receipt, usage): + try: + encoded = json.dumps(usage, allow_nan=False).encode() + if len(encoded) > 65536: + raise ValueError("oversized usage") + except (TypeError, ValueError): + receipt["usage_receipt_error"] = ( + "nonfinite, non-JSON or oversized usage details" + ) + if isinstance(usage, dict): + receipt["usage_counts"] = { + key: usage[key] + for key in ("prompt_tokens", "completion_tokens", "total_tokens") + if type(usage.get(key)) is int and usage[key] >= 0 + } + else: + receipt["usage"] = usage + + def _unique_pairs(pairs): result = {} for key, value in pairs: @@ -336,6 +366,8 @@ def _record(self): @staticmethod def _error(exc): + if isinstance(exc, HfHubHTTPError): + return f"HF backend operation failed ({type(exc).__name__}; HTTP {exc.response.status_code})" return ( str(exc) if isinstance(exc, HFBackendError) @@ -655,13 +687,15 @@ def _model(self, model, seed, *, tools=False): def _chat(self, client, model, allowed, limits, messages, record, seed, *, tools): used = record["input_tokens"] + record["output_tokens"] - # Bound the request body as well as provider-reported usage. Providers do - # not expose a preflight tokenizer through this API. message_bytes = len(json.dumps(messages).encode()) if message_bytes > limits["max_total_tokens"] * 4: + if tools: + raise _TargetBudgetStop("context_byte_limit") raise HFBackendError("chat context exceeds byte budget") remaining = limits["max_total_tokens"] - used if remaining <= 0: + if tools: + raise _TargetBudgetStop("total_token_limit") raise HFBackendError("model total token budget exhausted") request = { "messages": self._redact(messages), @@ -685,32 +719,71 @@ def _chat(self, client, model, allowed, limits, messages, record, seed, *, tools } if model.get("supports_seed", False): request["seed"] = seed + metadata = record["model_metadata"] + preflight = None if tools: request.update(tools=[_TOOL], tool_choice="auto") - if not tools: + encoded = json.dumps(request, allow_nan=False).encode() + reserved_input = len(encoded) + _PROMPT_FRAMING_ALLOWANCE + output_allowance = remaining - reserved_input + preflight = { + "method": "request_utf8_bytes_plus_framing_v1", + "reservation_bytes": len(encoded), + "request_bytes": len(encoded), + "request_sha256": hashlib.sha256(encoded).hexdigest(), + "framing_allowance": _PROMPT_FRAMING_ALLOWANCE, + "input_token_reservation": reserved_input, + "actual_tokens_before": used, + "remaining_tokens": remaining, + "output_token_cap": max(0, min(limits["max_tokens"], output_allowance)), + "dispatched": False, + } + metadata.setdefault("preflights", []).append(preflight) + if output_allowance <= 0: + raise _TargetBudgetStop("input_reservation_limit") + request["max_tokens"] = preflight["output_token_cap"] + encoded = json.dumps(request, allow_nan=False).encode() + preflight["request_bytes"] = len(encoded) + preflight["request_sha256"] = hashlib.sha256(encoded).hexdigest() + else: request.update(stream=True, stream_options={"include_usage": True}) - metadata = record["model_metadata"] metadata.setdefault("requests", []).append( {key: value for key, value in request.items() if key != "messages"} ) if tools: - response = _plain(client.chat_completion(**request)) - metadata.setdefault("responses", []).append( + preflight["dispatched"] = True + parsed = client.chat_completion(**request) + # Preserve billing evidence before normalizing arbitrary provider + # extras: JSON conversion itself can fail after a billed response. + receipt = { + "seed_sent": request.get("seed"), + "response_validated": False, + "discarded": True, + "tool_calls_processed": 0, + } + metadata.setdefault("responses", []).append(receipt) + _retain_usage(receipt, parsed.usage) + response = _plain(parsed) + # Access the typed nested value: outer SDK dicts can omit extras. + response["usage"] = _plain(parsed.usage) + choices = response.get("choices") + receipt.update( { "model": response.get("model"), "system_fingerprint": response.get("system_fingerprint"), "id": response.get("id"), - "seed_sent": request.get("seed"), + "finish_reason": choices[0].get("finish_reason") + if isinstance(choices, list) + and len(choices) == 1 + and isinstance(choices[0], dict) + else None, } ) else: response = self._candidate_response( client, request, allowed, limits, record ) - if response.get("model") not in allowed: - raise HFBackendError( - "provider response model does not match configured exact identities" - ) + receipt = metadata["responses"][-1] usage = response.get("usage") if not isinstance(usage, dict): raise HFBackendError("provider response omitted token usage") @@ -725,24 +798,57 @@ def _chat(self, client, model, allowed, limits, messages, record, seed, *, tools record[target] += value record["usage"][target] += value if ( - usage["completion_tokens"] > request["max_tokens"] - or record["input_tokens"] + record["output_tokens"] + type(usage.get("total_tokens")) is not int + or usage["total_tokens"] < 0 + or usage["total_tokens"] + != usage["prompt_tokens"] + usage["completion_tokens"] + ): + raise HFBackendError("provider returned inconsistent token usage") + if receipt.get("usage_receipt_error"): + raise HFBackendError("provider returned invalid usage details") + violations = [] + if usage["completion_tokens"] > request["max_tokens"]: + violations.append("response_output_cap") + if ( + record["input_tokens"] + record["output_tokens"] > limits["max_total_tokens"] ): - raise HFBackendError("provider exceeded token budget") + violations.append("cumulative_total_cap") + if tools and usage["prompt_tokens"] > preflight["input_token_reservation"]: + violations.append("input_reservation_contract") + if violations: + receipt["budget_violations"] = violations + raise HFBackendError( + "provider exceeded token budget: " + ", ".join(violations) + ) + if response.get("model") not in allowed: + raise HFBackendError( + "provider response model does not match configured exact identities" + ) choices = response.get("choices", []) - if len(choices) != 1 or not isinstance(choices[0].get("message"), dict): + if ( + not isinstance(choices, list) + or len(choices) != 1 + or not isinstance(choices[0].get("message"), dict) + ): raise HFBackendError("provider must return exactly one assistant message") message = choices[0]["message"] + if message.get("role") != "assistant": + raise HFBackendError("provider must return an assistant message") record["transcript"].append(message) if len(json.dumps(message).encode()) > limits["max_output_bytes"]: raise HFBackendError("model output exceeded byte limit") - if choices[0].get("finish_reason") not in {"stop", "tool_calls"}: + finish = choices[0].get("finish_reason") + if finish == "length" and tools: + receipt["response_validated"] = True + raise _TargetBudgetStop("response_output_limit", response_received=True) + if finish not in {"stop", "tool_calls"}: raise HFBackendError("model response was incomplete") - if choices[0]["finish_reason"] == "tool_calls" and ( + if finish == "tool_calls" and ( not isinstance(message.get("tool_calls"), list) or not message["tool_calls"] ): raise HFBackendError("invalid terminal function calls") + receipt["response_validated"] = True return message def _candidate_response(self, client, request, allowed, limits, record): @@ -856,6 +962,10 @@ def _candidate_response(self, client, request, allowed, limits, record): ) event_usage = event.get("usage") if event_usage is not None: + event_usage = _plain(parsed.usage) + _retain_usage( + record["model_metadata"]["responses"][-1], event_usage + ) if not stopped: raise HFBackendError("provider returned usage before stop") if not isinstance(event_usage, dict) or any( @@ -985,12 +1095,22 @@ def generate_candidate(self, model: dict, seed: int) -> dict: instruction = self._candidate_instruction(model, record) model, allowed, limits, client = self._model(model, seed) record["model_metadata"] = self._model_metadata(model, seed) + record["model_metadata"]["candidate_output_contract"] = { + "format": "bash_script", + "version": 2, + "submission_paths": list(self.submission_paths), + } messages = [ { "role": "system", - "content": "Return only a bash script that attempts the task. Do not use Markdown fences.", + "content": "Return only an executable /bin/bash script that creates or edits the required submission files. For source-code solutions, write the source into those files with a quoted heredoc or equivalent; do not return bare Python or other source code. Do not use Markdown fences.", + }, + { + "role": "user", + "content": instruction + + "\n\nRequired submission files: " + + json.dumps(self.submission_paths), }, - {"role": "user", "content": instruction}, ] record["transcript"] = messages message = self._chat( @@ -1039,20 +1159,34 @@ def solve(self, model: dict, seed: int, role="target") -> dict: record["transcript"] = messages sandbox = self._create(self.execution["agent_image"], record) self._audit_solver(sandbox) - calls_used = 0 + calls_used = validated_responses = 0 + record["model_metadata"]["completed_response_turns"] = 0 for _ in range(limits["max_steps"]): - message = self._chat( - client, - model, - allowed, - limits, - messages, - record, - seed, - tools=True, - ) + try: + message = self._chat( + client, + model, + allowed, + limits, + messages, + record, + seed, + tools=True, + ) + except _TargetBudgetStop as stop: + if not validated_responses and not stop.response_received: + raise HFBackendError( + "initial target prompt cannot fit token budget" + ) from None + record.update(truncated=True, termination_reason=stop.reason) + break calls = message.get("tool_calls") or [] if not calls: + record["model_metadata"]["responses"][-1]["discarded"] = False + validated_responses += 1 + record["model_metadata"]["completed_response_turns"] = ( + validated_responses + ) break if not isinstance(calls, list): raise HFBackendError("invalid terminal function calls") @@ -1062,42 +1196,73 @@ def solve(self, model: dict, seed: int, role="target") -> dict: ) break ids = set() + # Reject ambiguous identities before executing any call in a batch. for call in calls: if ( - call.get("type") != "function" - or call.get("function", {}).get("name") != "terminal" + not isinstance(call, dict) + or call.get("type") != "function" or not isinstance(call.get("id"), str) or not call["id"] or call["id"] in ids + or not isinstance(call.get("function"), dict) + or not isinstance(call["function"].get("name"), str) + or not isinstance(call["function"].get("arguments"), str) ): raise HFBackendError("invalid terminal function call") ids.add(call["id"]) - arguments = json.loads( - call["function"]["arguments"], - object_pairs_hook=_unique_pairs, - ) - if ( - not isinstance(arguments, dict) - or set(arguments) != {"command"} - or not isinstance(arguments["command"], str) - or len(arguments["command"].encode()) > self.max_script - ): - raise HFBackendError("invalid terminal function arguments") + for call in calls: calls_used += 1 - exit_code, output = self._run( - sandbox, - ["/bin/bash", "-c", arguments["command"]], - started + self.timeout, - ) + problem = None + if call["function"]["name"] != "terminal": + problem = "Only the terminal function is available; use terminal(command: string)." + else: + try: + arguments = json.loads( + call["function"]["arguments"], + object_pairs_hook=_unique_pairs, + ) + if ( + not isinstance(arguments, dict) + or set(arguments) != {"command"} + or not isinstance(arguments["command"], str) + or len(arguments["command"].encode()) + > self.max_script + ): + problem = "terminal requires exactly one bounded string argument named command." + except (ValueError, TypeError): + problem = "terminal arguments must be a unique JSON object with one command string." + if problem: + content = json.dumps({"error": problem}) + if len(content.encode()) > self.max_output: + raise HFBackendError( + "tool error feedback exceeds byte limit" + ) + record["model_metadata"]["tool_error_count"] = ( + record["model_metadata"].get("tool_error_count", 0) + 1 + ) + else: + exit_code, output = self._run( + sandbox, + ["/bin/bash", "-c", arguments["command"]], + started + self.timeout, + ) + content = json.dumps( + {"exit_code": exit_code, "output": output} + ) messages.append( { "role": "tool", "tool_call_id": call["id"], - "content": json.dumps( - {"exit_code": exit_code, "output": output} - ), + "content": content, } ) + receipt = record["model_metadata"]["responses"][-1] + receipt["tool_calls_processed"] += 1 + receipt["discarded"] = False + validated_responses += 1 + record["model_metadata"]["completed_response_turns"] = ( + validated_responses + ) else: record.update(truncated=True, termination_reason="step_limit") record["files"] = self._collect(sandbox) diff --git a/examples/coding_audit/test_hf.py b/examples/coding_audit/test_hf.py index a2a994b24..cc8282c7d 100644 --- a/examples/coding_audit/test_hf.py +++ b/examples/coding_audit/test_hf.py @@ -763,6 +763,16 @@ def test_provider_requests_base_id_with_distinct_token_and_records_source_limit( assert "PRIVATE_CANARY_SENTINEL" not in repr(request) assert "test.sh" not in repr(request) metadata = generated["model_metadata"] + assert metadata["candidate_output_contract"] == { + "format": "bash_script", + "version": 2, + "submission_paths": ["/app/answer.txt"], + } + assert "creates or edits" in request["messages"][0]["content"] + assert ( + 'Required submission files: ["/app/answer.txt"]' + in request["messages"][1]["content"] + ) assert metadata["source_revision"] == MODEL["revision"] assert metadata["served_weight_revision_pinned"] is False assert metadata["seed_supported"] is False @@ -1228,17 +1238,18 @@ def read(path, limit): assert "context_paths" not in h.inference.requests[0] -def test_default_candidate_prompt_and_target_protocol_are_unchanged(harness): +def test_candidate_contract_names_outputs_without_changing_target_instruction(harness): h = harness h.inference.responses = [response("true"), response("done")] generated = h.backend.generate_candidate(MODEL, 0) solved = h.backend.solve(MODEL, 1) assert generated["error"] is None and solved["error"] is None assert "candidate_context" not in generated - assert all( - request["messages"][1]["content"] == (h.root / "instruction.md").read_text() - for request in h.inference.requests + instruction = (h.root / "instruction.md").read_text() + assert h.inference.requests[0]["messages"][1]["content"] == ( + instruction + '\n\nRequired submission files: ["/app/answer.txt"]' ) + assert h.inference.requests[1]["messages"][1]["content"] == instruction @pytest.mark.parametrize( @@ -1387,9 +1398,11 @@ def test_invalid_tool_name_is_not_executed(harness): h = harness call = terminal() call["function"]["name"] = "provider_request" - h.inference.responses = [response(None, calls=[call])] + h.inference.responses = [response(None, calls=[call]), response("cannot proceed")] record = h.backend.solve(MODEL, 0) - assert "invalid terminal" in record["error"] and not h.factory.created[0].commands + assert record["error"] is None and not h.factory.created[0].commands + assert record["model_metadata"]["tool_error_count"] == 1 + assert json.loads(h.inference.requests[1]["messages"][-1]["content"])["error"] def test_output_and_token_limits_are_fail_closed(harness): @@ -1493,6 +1506,43 @@ def test_deadline_interrupts_blocking_controller_and_restores_handler(): assert signal.getitimer(signal.ITIMER_REAL)[0] == 0 +@pytest.mark.parametrize("seconds", [0, -0.1, float("inf"), float("nan")]) +def test_expired_deadline_does_not_enter_or_change_parent_alarm(monkeypatch, seconds): + timer_calls = [] + monkeypatch.setattr(signal, "getitimer", lambda *_: (10, 0)) + monkeypatch.setattr(signal, "setitimer", lambda *args: timer_calls.append(args)) + handler = signal.getsignal(signal.SIGALRM) + entered = [] + with pytest.raises(HFBackendError, match="deadline"): + with _deadline(seconds): + entered.append(True) + assert not entered and not timer_calls + assert signal.getsignal(signal.SIGALRM) == handler + + +def test_target_retains_billed_counts_before_malformed_usage_serialization(harness): + usage = { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "provider_extra": object(), + } + harness.inference.responses = [response(usage=usage)] + record = harness.backend.solve(MODEL, 0) + assert record["error"] and record["cleanup_verified"] + receipts = record["model_metadata"].get("responses", []) + assert len(receipts) == 1 + assert receipts[0]["usage_counts"] == { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + } + assert receipts[0]["usage_receipt_error"] + assert receipts[0]["discarded"] and not receipts[0]["response_validated"] + assert not harness.factory.created[0].commands + json.dumps(record, allow_nan=False) + + def test_strong_isolation_and_equal_credentials_rejected(harness): h = harness with pytest.raises(HFBackendError, match="strong"): @@ -1749,3 +1799,48 @@ def test_empty_or_non_bash_fence_does_not_create_a_candidate(harness, raw): h.inference.responses = [response(raw)] record = h.backend.generate_candidate(MODEL, 0) assert record["error"] and record["script"] == "" and not h.factory.created + + +@pytest.mark.parametrize("phase", ["target", "candidate"]) +@pytest.mark.parametrize("status", [400, 401, 403, 429, 502, 503]) +def test_http_failure_retains_only_numeric_status_without_retry( + harness, monkeypatch, phase, status +): + from unittest.mock import Mock + + from huggingface_hub.errors import HfHubHTTPError + + request = httpx.Request( + "POST", + "https://router.invalid/chat?private=url-sentinel", + headers={"Authorization": "Bearer inference-token-secret"}, + ) + response = httpx.Response(status, request=request, text="private-body-sentinel") + failure = HfHubHTTPError( + "private-body-sentinel inference-token-secret url-sentinel", response=response + ) + call = Mock(side_effect=failure) + monkeypatch.setattr(harness.inference, "chat_completion", call) + record = ( + harness.backend.solve(MODEL, 0) + if phase == "target" + else harness.backend.generate_candidate(MODEL, 0) + ) + assert ( + record["error"] + == f"HF backend operation failed (HfHubHTTPError; HTTP {status})" + ) + assert len(record["model_metadata"]["requests"]) == 1 and call.call_count == 1 + assert not record.get("files") and record.get("reward") is None + assert record["usage"] == {"input_tokens": 0, "output_tokens": 0} + assert "private-body-sentinel" not in json.dumps(record) + assert "url-sentinel" not in json.dumps(record) + assert "inference-token-secret" not in json.dumps(record) + if phase == "target": + assert record["cleanup_verified"] + assert ( + len(harness.factory.created) == 1 + and not harness.factory.created[0].commands + ) + else: + assert not harness.factory.created diff --git a/examples/coding_audit/test_target_budget.py b/examples/coding_audit/test_target_budget.py new file mode 100644 index 000000000..973ff0192 --- /dev/null +++ b/examples/coding_audit/test_target_budget.py @@ -0,0 +1,216 @@ +"""Budget stops preserve valid prefixes; provider violations remain errors. + +The shared harness is imported to register its pytest fixture in this module. +""" + +import hashlib +import json + +import pytest +from examples.coding_audit.hf import HFBackendError +from examples.coding_audit.test_hf import harness as harness, MODEL, response, terminal + + +def model(**values): + return { + **MODEL, + "max_steps": 48, + "max_tokens": 4096, + "max_total_tokens": 32768, + **values, + } + + +def test_next_prompt_is_reserved_before_dispatch_and_saved_files_are_graded(harness): + h = harness + (h.root / "instruction.md").write_text("public requirement " * 700) + h.inference.responses = [ + response( + None, + calls=[terminal(identifier="first")], + usage={ + "prompt_tokens": 15000, + "completion_tokens": 4000, + "total_tokens": 19000, + }, + ), + response( + None, + calls=[terminal(identifier="second")], + usage={ + "prompt_tokens": 15000, + "completion_tokens": 4000, + "total_tokens": 19000, + }, + ), + ] + solved = h.backend.solve(model(), 0) + assert solved["error"] is None and solved["truncated"] + assert solved["termination_reason"] == "input_reservation_limit" + assert len(h.inference.requests) == 1 and len(h.factory.created[0].commands) == 1 + assert solved["usage"] == {"input_tokens": 15000, "output_tokens": 4000} + assert solved["input_tokens"] + solved["output_tokens"] <= 32768 + preflights = solved["model_metadata"]["preflights"] + assert preflights[0]["dispatched"] and not preflights[1]["dispatched"] + assert preflights[1]["output_token_cap"] == 0 + payload = json.dumps(h.inference.requests[0], allow_nan=False).encode() + assert preflights[0]["request_sha256"] == hashlib.sha256(payload).hexdigest() + assert preflights[0]["request_bytes"] == len(payload) + assert solved["files"] == {"/app/answer.txt": b"answer"} + grade = h.backend.grade(solved) + assert grade["error"] is None and grade["cleanup_verified"] + assert grade["grade_sandbox_id"] != solved["sandbox_id"] + + +def test_insufficient_first_prompt_has_no_model_measurement_or_grade(harness): + h = harness + solved = h.backend.solve(model(max_total_tokens=8192), 0) + assert "initial target prompt" in solved["error"] + assert not h.inference.requests and not h.factory.created[0].commands + assert solved["cleanup_verified"] and solved["files"] == {} + assert len(h.factory.created) == 1 and not solved["truncated"] + + +@pytest.mark.parametrize("failure", ["output", "total", "input_estimate"]) +def test_actual_overruns_are_errors_with_billed_usage_not_truncation(harness, failure): + h = harness + prompt, completion = { + "output": (10, 4097), + "total": (33000, 5), + "input_estimate": (12000, 5), + }[failure] + h.inference.responses = [ + response( + None, + calls=[terminal()], + usage={ + "prompt_tokens": prompt, + "completion_tokens": completion, + "total_tokens": prompt + completion, + }, + ) + ] + solved = h.backend.solve(model(), 0) + assert "provider exceeded token budget" in solved["error"] + assert not solved["truncated"] and not h.factory.created[0].commands + assert solved["usage"] == {"input_tokens": prompt, "output_tokens": completion} + receipt = solved["model_metadata"]["responses"][0] + assert receipt["usage"]["total_tokens"] == prompt + completion + expected = { + "output": "response_output_cap", + "total": "cumulative_total_cap", + "input_estimate": "input_reservation_contract", + }[failure] + assert expected in receipt["budget_violations"] and receipt["discarded"] + assert len(h.factory.created) == 1 and solved["cleanup_verified"] + + +def test_nested_reasoning_usage_is_retained_without_double_charging(harness): + h = harness + usage = { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "completion_tokens_details": {"reasoning_tokens": 4}, + } + h.inference.responses = [response("saved", usage=usage)] + solved = h.backend.solve(model(), 0) + assert solved["error"] is None + assert solved["usage"] == {"input_tokens": 10, "output_tokens": 5} + receipt = solved["model_metadata"]["responses"][0] + assert receipt["usage"]["completion_tokens_details"]["reasoning_tokens"] == 4 + assert receipt["response_validated"] and not receipt["discarded"] + + +@pytest.mark.parametrize( + "problem", ["missing_total", "bad_total", "nonfinite_detail", "oversized_detail"] +) +def test_invalid_usage_is_never_a_normal_budget_stop(harness, problem): + h = harness + usage = {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15} + if problem == "missing_total": + usage.pop("total_tokens") + elif problem == "bad_total": + usage["total_tokens"] = 999 + elif problem == "nonfinite_detail": + usage["detail"] = float("nan") + else: + usage["detail"] = "x" * 65537 + h.inference.responses = [response(None, calls=[terminal()], usage=usage)] + solved = h.backend.solve(model(), 0) + assert solved["error"] and not solved["truncated"] + assert not h.factory.created[0].commands and solved["cleanup_verified"] + assert len(h.factory.created) == 1 + json.dumps(solved, allow_nan=False) + + +@pytest.mark.parametrize("prefix", [False, True]) +def test_funded_length_discards_partial_calls_and_grades_existing_state( + harness, prefix +): + h = harness + h.factory.submitted = None if not prefix else b"answer" + h.factory.reward = b"0" if not prefix else b"1" + partial = terminal("synthetic-must-not-execute", "partial") + partial["function"]["arguments"] = '{"command":"unterminated' + ended = response(None, calls=[partial]) + ended["choices"][0]["finish_reason"] = "length" + h.inference.responses = ( + [response(None, calls=[terminal(identifier="saved")])] if prefix else [] + ) + [ended] + solved = h.backend.solve(model(), 0) + assert solved["error"] is None and solved["truncated"] + assert solved["termination_reason"] == "response_output_limit" + assert len(h.factory.created[0].commands) == int(prefix) + assert solved["files"] == ({"/app/answer.txt": b"answer"} if prefix else {}) + assert solved["model_metadata"]["responses"][-1]["finish_reason"] == "length" + assert solved["model_metadata"]["responses"][-1]["discarded"] + assert "reward" not in solved + grade = h.backend.grade(solved) + assert grade["error"] is None and grade["reward"] == int(prefix) + assert ( + grade["grade_sandbox_id"] != solved["sandbox_id"] and grade["cleanup_verified"] + ) + + +@pytest.mark.parametrize("violation", ["model", "role", "usage", "bytes"]) +def test_length_does_not_mask_invalid_response(harness, violation): + h = harness + ended = response(None, calls=[terminal()]) + ended["choices"][0]["finish_reason"] = "length" + if violation == "model": + ended["model"] = "wrong/model" + elif violation == "role": + ended["choices"][0]["message"]["role"] = "user" + elif violation == "usage": + ended = response( + None, + calls=[terminal()], + usage={ + "prompt_tokens": 10, + "completion_tokens": 4097, + "total_tokens": 4107, + }, + ) + ended["choices"][0]["finish_reason"] = "length" + else: + ended["choices"][0]["message"]["content"] = "x" * 16385 + h.inference.responses = [ended] + solved = h.backend.solve(model(), 0) + assert solved["error"] and not solved["truncated"] + assert not h.factory.created[0].commands and len(h.factory.created) == 1 + assert solved["cleanup_verified"] + + +def test_provider_exception_with_old_budget_message_is_not_recovered(harness): + h = harness + + def failure(**kwargs): + raise HFBackendError("provider exceeded token budget") + + h.inference.chat_completion = failure + solved = h.backend.solve(model(), 0) + assert ( + solved["error"] == "provider exceeded token budget" and not solved["truncated"] + ) + assert solved["cleanup_verified"] and len(h.factory.created) == 1 diff --git a/examples/coding_audit/test_target_tools.py b/examples/coding_audit/test_target_tools.py new file mode 100644 index 000000000..608ba7d1e --- /dev/null +++ b/examples/coding_audit/test_target_tools.py @@ -0,0 +1,280 @@ +"""Target tool repair at the real solve, provider-message, and grading seams. + +The shared harness is imported to register its pytest fixture in this module. +""" + +import json +import time + +import pytest +from examples.coding_audit.test_hf import ( + FakeSandbox, + harness as harness, + MODEL, + response, + terminal, +) + +REJECTED_COMMAND = "synthetic-command-that-must-not-run" +REPAIR_COMMAND = "printf answer > /app/answer.txt" + + +def rejected_call(kind, identifier="rejected-call"): + call = terminal(REJECTED_COMMAND, identifier) + if kind in {"unknown-empty", "unknown-command", "provider", "reset"}: + call["function"]["name"] = { + "provider": "provider_request", + "reset": "reset", + }.get(kind, "command") + if kind in {"unknown-empty", "empty"}: + call["function"]["arguments"] = "{}" + elif kind == "malformed-json": + call["function"]["arguments"] = '{"command":"unterminated' + elif kind == "duplicate-json-key": + call["function"]["arguments"] = ( + '{"command":"synthetic-first","command":"synthetic-second"}' + ) + elif kind == "argument-object": + call["function"]["arguments"] = {"command": REJECTED_COMMAND} + elif kind == "command-number": + call["function"]["arguments"] = '{"command":7}' + elif kind == "argument-list": + call["function"]["arguments"] = "[]" + elif kind == "oversized-command": + call["function"]["arguments"] = json.dumps({"command": "x" * 129}) + return call + + +def assert_feedback(message, identifier, limit): + assert set(message) == {"role", "tool_call_id", "content"} + assert message["role"] == "tool" and message["tool_call_id"] == identifier + assert isinstance(message["content"], str) + assert len(message["content"].encode()) <= limit + payload = json.loads(message["content"]) + assert isinstance(payload, dict) + assert isinstance(payload.get("error"), str) and payload["error"] + + +def assert_target_wire(requests): + for request in requests: + assert "stream" not in request and request["tool_choice"] == "auto" + assert [tool["function"]["name"] for tool in request["tools"]] == ["terminal"] + assert "PRIVATE_SOLUTION_SENTINEL" not in repr(request) + assert "PRIVATE_CANARY_SENTINEL" not in repr(request) + + +@pytest.mark.parametrize( + "kind", + [ + "unknown-empty", + "unknown-command", + "empty", + "malformed-json", + "duplicate-json-key", + "command-number", + "argument-list", + "oversized-command", + "provider", + "reset", + ], +) +def test_bindable_rejection_reaches_provider_as_feedback_then_repairs_and_grades( + harness, kind +): + h = harness + h.backend.max_output = 256 + h.backend.max_script = 128 + bad = rejected_call(kind) + h.inference.responses = [ + response(None, calls=[bad]), + response(None, calls=[terminal(REPAIR_COMMAND, "repair-call")]), + response("saved"), + ] + original = h.inference.chat_completion + + def provider(**request): + if len(h.inference.requests) == 1: + assert not h.factory.created[0].commands + assert request["messages"][-2]["role"] == "assistant" + assert_feedback(request["messages"][-1], bad["id"], h.backend.max_output) + return original(**request) + + h.inference.chat_completion = provider + solved = h.backend.solve({**MODEL, "max_steps": 4}, 0) + assert solved["error"] is None and solved["cleanup_verified"] + assert not solved["truncated"] and len(h.inference.requests) == 3 + assert [argv for argv, _ in h.factory.created[0].commands] == [ + ["/bin/bash", "-c", REPAIR_COMMAND] + ] + assert solved["files"] == {"/app/answer.txt": b"answer"} + assert "reward" not in solved + assert_target_wire(h.inference.requests) + grade = h.backend.grade(solved) + assert grade["error"] is None and grade["reward"] == 1 + assert ( + grade["cleanup_verified"] and grade["grade_sandbox_id"] != solved["sandbox_id"] + ) + assert h.factory.created[1].files.data["/app/answer.txt"] == b"answer" + assert all(stage == "CANCELED" for stage in h.api.stages.values()) + + +def test_all_invalid_single_step_is_bounded_and_grades_actual_empty_submission(harness): + h = harness + h.factory.submitted = None + h.factory.reward = b"0" + h.inference.responses = [response(None, calls=[rejected_call("unknown-empty")])] + solved = h.backend.solve({**MODEL, "max_steps": 1}, 0) + assert solved["error"] is None and solved["cleanup_verified"] + assert solved["truncated"] and solved["termination_reason"] == "step_limit" + assert len(h.inference.requests) == 1 and not h.factory.created[0].commands + assert_feedback(solved["transcript"][-1], "rejected-call", h.backend.max_output) + assert solved["files"] == {} and "reward" not in solved + grade = h.backend.grade(solved) + assert ( + grade["error"] is None and grade["reward"] == 0 and grade["rawreward"] == b"0" + ) + assert ( + grade["cleanup_verified"] and grade["grade_sandbox_id"] != solved["sandbox_id"] + ) + assert "/app/answer.txt" not in h.factory.created[1].files.data + + +def test_rejected_calls_count_toward_multi_call_budget_before_any_execution(harness): + h = harness + h.factory.submitted = None + h.inference.responses = [ + response(None, calls=[rejected_call("empty", "first")]), + response( + None, calls=[rejected_call("empty", "second"), terminal(REPAIR_COMMAND)] + ), + ] + solved = h.backend.solve({**MODEL, "max_steps": 2}, 0) + assert solved["error"] is None and solved["cleanup_verified"] + assert solved["truncated"] and solved["termination_reason"] == "tool_call_limit" + assert len(h.inference.requests) == 2 and solved["files"] == {} + assert not h.factory.created[0].commands + assert_feedback( + h.inference.requests[1]["messages"][-1], "first", h.backend.max_output + ) + + +def test_rejection_at_call_cap_preserves_saved_files_for_normal_grading(harness): + h = harness + h.inference.responses = [ + response(None, calls=[terminal(REPAIR_COMMAND, "saved-call")]), + response(None, calls=[rejected_call("empty")]), + ] + solved = h.backend.solve({**MODEL, "max_steps": 2}, 0) + assert solved["error"] is None and solved["cleanup_verified"] + assert solved["truncated"] and solved["termination_reason"] == "step_limit" + assert len(h.inference.requests) == 2 + assert len(h.factory.created[0].commands) == 1 + assert solved["files"] == {"/app/answer.txt": b"answer"} and "reward" not in solved + assert_feedback(solved["transcript"][-1], "rejected-call", h.backend.max_output) + grade = h.backend.grade(solved) + assert grade["error"] is None and grade["reward"] == 1 and grade["cleanup_verified"] + assert grade["grade_sandbox_id"] != solved["sandbox_id"] + + +@pytest.mark.parametrize( + "malformation", + [ + "missing-id", + "empty-id", + "numeric-id", + "wrong-type", + "missing-type", + "duplicate-id", + "nonstring-name", + "argument-object", + ], +) +def test_unbindable_or_duplicate_identity_is_fatal_before_batch_execution( + harness, malformation +): + h = harness + first = terminal(REPAIR_COMMAND, "first") + second = terminal(REJECTED_COMMAND, "second") + if malformation == "missing-id": + second.pop("id") + elif malformation == "empty-id": + second["id"] = "" + elif malformation == "numeric-id": + second["id"] = 7 + elif malformation == "wrong-type": + second["type"] = "reset" + elif malformation == "missing-type": + second.pop("type") + elif malformation == "duplicate-id": + second["id"] = first["id"] + elif malformation == "argument-object": + second["function"]["arguments"] = {"command": REJECTED_COMMAND} + else: + second["function"]["name"] = None + h.inference.responses = [response(None, calls=[first, second])] + solved = h.backend.solve(MODEL, 0) + assert solved["error"] and solved["cleanup_verified"] + assert not h.factory.created[0].commands and len(h.inference.requests) == 1 + assert not any(message.get("role") == "tool" for message in solved["transcript"]) + assert "reward" not in solved + + +def test_tool_execution_platform_failure_after_feedback_remains_error( + harness, monkeypatch +): + h = harness + h.inference.responses = [ + response(None, calls=[rejected_call("empty")]), + response(None, calls=[terminal(REPAIR_COMMAND)]), + ] + + def failed_run(*args, **kwargs): + raise RuntimeError("synthetic platform failure") + + monkeypatch.setattr(FakeSandbox, "run", failed_run) + solved = h.backend.solve(MODEL, 0) + assert solved["error"] == "HF backend operation failed (RuntimeError)" + assert solved["cleanup_verified"] and len(h.inference.requests) == 2 + assert ( + len( + [ + message + for message in solved["transcript"] + if message.get("role") == "tool" + ] + ) + == 1 + ) + + +def test_provider_deadline_after_feedback_remains_error(harness): + h = harness + h.backend.timeout = 0.02 + h.inference.responses = [response(None, calls=[rejected_call("empty")])] + original = h.inference.chat_completion + + def blocked_provider(**request): + if h.inference.requests: + time.sleep(1) + return original(**request) + + h.inference.chat_completion = blocked_provider + started = time.monotonic() + solved = h.backend.solve(MODEL, 0) + assert time.monotonic() - started < 0.5 + assert "deadline" in solved["error"] and solved["cleanup_verified"] + assert not h.factory.created[0].commands + + +def test_cleanup_failure_after_tool_repair_remains_error(harness): + h = harness + h.api.ignore_cancel = True + h.inference.responses = [ + response(None, calls=[rejected_call("empty")]), + response(None, calls=[terminal(REPAIR_COMMAND)]), + response("saved"), + ] + solved = h.backend.solve(MODEL, 0) + assert solved["error"] == "sandbox cleanup could not be verified" + assert not solved["cleanup_verified"] and len(h.inference.requests) == 3 + assert len(h.factory.created[0].commands) == 1 and "reward" not in solved diff --git a/examples/hub_validation_pipeline/CODING_AUDIT.md b/examples/hub_validation_pipeline/CODING_AUDIT.md new file mode 100644 index 000000000..056489cfb --- /dev/null +++ b/examples/hub_validation_pipeline/CODING_AUDIT.md @@ -0,0 +1,268 @@ +# Reproducible supplemental coding audits + +The operator implementation lives in `examples/coding_audit`: `scoring.py` defines +reports/checks, `hf.py` adapts fresh HF Sandboxes, `runner.py` orchestrates attempts, +and `__main__.py` exposes the controller command. Cloud and discovery orchestration +in this directory import that example package. This workflow does not add a coding +mode to the installed `openenv validate` CLI. + +`cloud.py` freezes a Harbor task, operator audit configuration, required source and +`uv.lock` into a content-addressed bundle. Submission publishes it to an existing +private results dataset, records the immutable commit, and launches an HF Job +controller. Each solution execution and verifier run uses a fresh dedicated HF +Sandbox. Docker inside the controller Job is unnecessary. + +The supplemental profile is `harbor-coding-audit-v1`. Its PASS covers the measured +reference, rejection, leakage/reliability and rollout checks. Reports retain +`rfc008_complete: false` and `trainability_established: false`. OpenEnv's native +`harbor-semantic-v1` contract and the complete `full` admission requirements keep +their own meanings. + +## What runs + +1. Freeze/hash the task, verifier and operator configuration. Reject source drift, + mutable image references, unsafe artifact paths and excessive workload bounds. +2. Run NOP in a clean solver Sandbox and transfer only allowlisted submission file + bytes to a separate clean verifier Sandbox. +3. Generate bounded candidate shell solutions from solver-visible instructions. + Execute provided/generated candidates and repeat exact submissions in fresh + sandboxes. Clean incorrect generated candidates are excluded; execution and + cleanup errors remain errors. Never rewrite a verifier to make a candidate pass. +4. Execute the operator's reviewed known-wrong cases and require the frozen + verifier to reject them. The checked-in toy cases apply only to the synthetic + demo task. A generator's claim that its own solution is wrong does not establish + trusted negative coverage for another task. +5. Probe private-path visibility, stale solver state and a verifier-private canary + in collected transcripts. Check source/image binding and terminal Job status + after cleanup. +6. Only after screening passes, measure repeated target attempts. Retain successes, + failures, infrastructure errors, rewards, variance, mixed-success groups, + uncertainty, token usage and latency. Uniform success/failure is a measured + difficulty result, not proof of task invalidity or universal usefulness. + +A candidate passing a weak verifier does not establish intended correctness. +Demonstrating training benefit requires an independent controlled learning +experiment with held-out task families; see [the research plan](../../docs/plans/hub-environment-trainability-validation.md). + +## Model and protocol + +The synthetic configuration pins public source provenance to +[`Qwen/Qwen3.8-27B`](https://huggingface.co/Qwen/Qwen3.8-27B) at +`37561816e66b5b9153b1b64c10968c18c1e6f4a0`, with `deepinfra` explicitly selected. +Its attempt/candidate/replay and token/step limits are declared in +[config.json](../coding_audit/config.json). Real-task protocols come entirely from the +operator panel and must be reviewed independently. + +Model/provider availability is checked before task Sandbox creation. A source +commit does not force a provider to serve that revision. Evidence retains requested +and returned model identities and supplied fingerprints. Provider-specific seed +behavior is not assumed; recorded attempt seeds may be identifiers only. Frozen +inputs and the same declared protocol do not imply identical hosted generations. + +## Prepare a task and configuration + +Use a reviewed, self-contained Harbor task with `task.toml`, `instruction.md` and +`tests/test.sh`. A supplied `solution/solve.sh` is optional for the supplemental +audit because it can generate candidate references. This differs from native +semantic oracle admission. Multi-service, multi-step, credential-dependent or +unsupported resource/network contracts require a qualified adapter. + +Solver/verifier images must already exist under immutable OCI registry digests; +HF Sandbox has no Dockerfile build-context API. Build required images on a trusted +builder. Keep private tests, reference solutions and answer keys out of solver +images and public layers. The generic pinned Python image in the toy configuration +is not a qualified image for arbitrary downloaded tasks. + +Declare exact `submission_paths`, `private_paths`, verifier command, reward file/key, +reward range, trusted negatives and resource/time budgets. Submission paths name +individual files; directory/glob exports are unsupported. Only those bytes cross +into the fresh verifier. Set `cloud.results_repo` to an existing private dataset +under your control. + +Create the trusted synthetic controls without executing them: + +```bash +uv run --no-sync python -c \ + 'from pathlib import Path; from examples.hub_validation_pipeline.demo import create_tasks; create_tasks(Path("outputs/coding-controls"))' +``` + +Prepare a local bundle and reviewable launch plan; this performs no paid operations +or remote writes: + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.cloud prepare \ + --task outputs/coding-controls/good \ + --config examples/coding_audit/config.json \ + --output outputs/audit-plan +``` + +The plan records hashes, source provenance, dependency lock and conservative +Sandbox/inference workload bounds. Review `plan.json`, the configured protocol and +applicable prices before submission. Child command/start/idle deadlines do not +replace the SDK's 24-hour remote Job maximum lifetime. An interrupted controller +may leave an active child billable until shutdown; inspect/cancel exact run-owned +Jobs and retain cleanup evidence. + +To prepare discovered tasks, supply a configuration appropriate for the selected +family: + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.pipeline \ + --source both --dataset-limit 1 --tasks-per-dataset 1 \ + --coding-config inputs/reviewed-audit.json --output outputs/hub-coding-plans +``` + +This pipeline option selects supplemental cloud preparation; it is not an option +on `openenv validate`. The driver writes bundles and `planned.jsonl`; preparation +or a queued Job never marks a task accepted. Access/preparation failures remain +explicit exclusions. Explicit pinned dataset inputs or `--local-dataset` reproduce +the selection without relying on a later discovery ranking. + +Acquisition uses ordinary HF SDK revision/local-directory semantics, corresponding +to `hf download --repo-type dataset --revision ... --local-dir ...`, with anonymous +public reads and additional selection/size/hash checks. Publication uses explicit +SDK additions and commit identities. Like `hf upload`, it does not delete remote +files unless deletion is explicitly requested. + +## Operator-supplied native evaluation panels + +`real_tasks.py` and `full_run.py` require `--panel PATH`. Their Python entry points +likewise require the keyword `panel_path`. There is no embedded real-task list, +reference implementation, known-wrong submission or fallback private input path. +Keep panels, private task material and populated reports/ledgers in protected +operational storage; do not include their payloads in public source or PR prose. + +A panel has schema `openenv-native-evaluation-panel-v1`, `evaluation_only: true`, +a pinned HF `source` (`kind`, `repo_id`, full `revision`) and a nonempty `tasks` list. +Each task declares: + +- A unique safe `name` and package-relative `task_path`. +- An immutable `image` and exact `submission_path`, matching its configuration. +- Complete audit `config`, including reviewed negative cases and required IDs. +- Operator-reviewed `negative_counterexamples` metadata. +- Optional `annotations` for task-specific caveats or provenance. + +The panel's exact bytes are read once. Manifests record its path, byte length and +SHA256; each prepared bundle's source provenance includes the same panel SHA256. +Native task/verifier bytes remain unchanged and bind to the original acquired +file inventory. Remote source identities require a complete, verified acquisition +hash inventory; a revision string alone cannot authorize a bundle. Local revisions +are checked against the captured bundle bytes, including the verifier. Configuration and annotations come from the operator input. + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.real_tasks \ + --panel "$PANEL_PATH" --output outputs/native-recipes --bundle + +uv run --no-sync python -m examples.hub_validation_pipeline.full_run \ + --panel "$PANEL_PATH" --hf-census "$HF_CENSUS_PATH" \ + --harbor-census "$HARBOR_CENSUS_PATH" --budget-usd "$BUDGET_USD" \ + --output outputs/native-cohort-plan +``` + +Set the variables to reviewed operator inputs. `full_run` freezes small census +summaries and hashes referenced inventories; census membership does not qualify +a task. Repeat `--task NAME` to select distinct names from the panel in that order; +omitting it selects all panel tasks. Empty, unknown and duplicate selections fail +before acquisition/spending. Native panels remain evaluation-only. + +## Shared spending and one-shot submission + +Initialize one shared ledger using your global allowance, without publishing its +contents. Set `SPEND_LEDGER` and `GLOBAL_BUDGET_USD` before this command: + +```bash +uv run --no-sync python -c \ + 'import os; from examples.hub_validation_pipeline import spend; spend.init(os.environ["SPEND_LEDGER"], os.environ["GLOBAL_BUDGET_USD"])' +``` + +Account for prior/concurrent spending and unresolved reservations in that ledger. +Use the same ledger across all cooperating launchers. For a native cohort, +`full_run(..., submit=True)` requires nonempty `spend_ledger` and `reservation_id` +before preparation; the CLI requires both flags: + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.full_run \ + --panel "$PANEL_PATH" --hf-census "$HF_CENSUS_PATH" \ + --harbor-census "$HARBOR_CENSUS_PATH" --budget-usd "$BUDGET_USD" \ + --spend-ledger "$SPEND_LEDGER" --reservation-id "$RESERVATION_ID" \ + --output outputs/native-cohort-submit --submit +``` + +The aggregate reservation covers configured request/token ceilings, controller +allowances and each child's full remote lifetime plus startup allowance. The +ratebook is a declared model, not a current-price guarantee or account billing cap; +unsupported model/provider/flavor changes require a reviewed ratebook. Both the +per-run cap and remaining shared allowance must cover the selected cohort. + +Every budget-bearing initial plan parse and final submission checks the expected +SHA256 against the same bytes it parses. A new directory or reservation identifier +does not bypass immutable run-ID duplicate protection. Partial/uncertain receipts +and unresolved reservations remain durable for manual reconciliation; never +release them merely because a controller stopped. Jobs are never automatically +resumed or blindly retried. + +`prepared_batch` provides shared reservation and bounded controller concurrency +for an already reviewed preparation manifest. Use these guarded cohort launchers +for paid work; the cloud CLI and discovery pipeline only prepare inputs. +Both preparation producers emit `plan_path` relative to that manifest's directory +and `plan_sha256` for the exact generated plan bytes. Historical manifests without +these fields require fresh preparation and review; the launcher does not infer a +new digest for an old plan. + +Plans bind `runtime_versions` for uv and `huggingface_hub`. The remote bootstrap +runs the frozen dependency sync, installs the declared `huggingface_hub==1.33.0` +overlay with that environment's interpreter, then verifies the installed version +before starting the audit worker. A missing or incompatible runtime pin requires +fresh preparation; changing the package's general dependency range is unnecessary. + +`cloud.submit()` requires a ledger, reservation ID and expected plan SHA256. +Each reservation binds the reviewed plan hashes to their modeled cost allocations; +allocations must fit the reserved total. Submission rechecks the active hold, +budget block and parsed plan bytes while holding the shared ledger lock through +launch. Unbound legacy reservations cannot authorize a launch. +The submitter verifies private dataset access, stages immutable inputs, claims the +run identity and persists the returned Job identifier and receipt. `prepare` never +makes model calls or launches paid Jobs. + +## Credentials and execution boundary + +Supply three distinct scoped credentials at runtime: `HF_TOKEN` for the private +input/result dataset, `HF_INFERENCE_TOKEN` for inference and `HF_SANDBOX_TOKEN` for +execution-only Jobs/Sandbox access. Keep their values out of configs, arguments, +bundles and Git. Submission uses HF Job secrets; they are not deliberately passed +into task environment variables or tool arguments. + +The current Sandbox RPC sends its execution bearer to the workload server. +Hostile root code could replace that server, so this backend is for reviewed +workloads; separate credentials limit authority without proving hostile-code +containment. A separate verifier Sandbox also does not establish that arbitrary +submitted code cannot read its grader or forge same-user output. Strong verifier +isolation requests are refused; these limits remain in reports. + +The worker retains reports, config, raw rewards, generated candidates, submission +hashes, sanitized transcripts and checksums in the private result dataset even +when an audit fails normally. Infrastructure termination can prevent final upload; +absent results remain incomplete. Keep original source and prior evidence intact. + +Run the same controller locally, with task execution still in HF Sandboxes: + +```bash +uv run --no-sync python -m examples.coding_audit \ + --task inputs/task --config inputs/audit.json --output outputs/audit-report.json +``` + +Missing evidence, malformed/ambiguous rewards, source/image drift, incorrect +provided references, accepted wrong answers, failed probes and unverified cleanup +block acceptance. Provider/Job failures never become synthetic zero-reward model +attempts. Checks retain PASS/FAIL/SKIP/ERROR and qualification limits. + +## Verification + +```bash +PYTHONPATH=.:src:envs uv run --no-sync python -m pytest \ + examples/hub_validation_pipeline/ -q +``` + +Injected SDK checks consume no cloud/inference credits and establish source-level +behavior. They do not establish Linux Docker, live-provider, billing or learning +qualification. A live audit needs separately authorized compute and scoped access. diff --git a/examples/hub_validation_pipeline/README.md b/examples/hub_validation_pipeline/README.md new file mode 100644 index 000000000..728bfa3fa --- /dev/null +++ b/examples/hub_validation_pipeline/README.md @@ -0,0 +1,228 @@ +# Discover, validate and evaluate Harbor tasks + +This example discovers Hugging Face datasets tagged with both `library:harbor` +and `rl-environment`, discovers Harbor Hub packages, acquires pinned task files, +and calls OpenEnv's native validator for each task. It builds an admitted corpus +before optionally running a configured model through the Harbor capture runner. +The supplemental coding audit lives in [`examples/coding_audit`](../coding_audit/); +its configuration, cloud orchestration and selection policy remain example code. + +The native contract is `harbor-semantic-v1`, with schema 2 evidence from OpenEnv's +Harbor semantic validation support. See [RFC 008](../../rfcs/008-environment-auto-validation.md) +and the [pipeline plan](../../docs/plans/hub-environment-validation-pipeline.md) +for its boundaries. Neither corpus admission nor a supplemental coding-audit PASS +establishes full RFC 008 qualification or training benefit. + +## Install + +From the repository root, with Python 3.12+ and a Linux Docker daemon/Compose: + +```bash +uv sync --python 3.13 --group dev +uv pip install 'huggingface_hub==1.33.0' +uv pip install 'harbor @ git+https://github.com/harbor-framework/harbor.git@9c7a39b991d7e2205019f6be0598cbd22d84bfc3' +uv pip check +``` + +Use `uv run --no-sync` below to preserve that exact Harbor source installation. +The PyPI 0.25.0 wheel lacks the listing API used here; the example checks the +installed Git revision. Its cloud adapter targets `huggingface_hub==1.33.0`. + +## Run the synthetic Docker demo + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.demo \ + --output outputs/hub-pipeline-demo +``` + +The output directory must be new. The demo creates three trusted toy tasks and +runs native Harbor validation in Docker. The expected result is one admitted task +and two exclusions: an always-rewarding verifier and a missing reference solution. +The valid task has NOP reward 0 and oracle reward 1; its declared wrong-answer +canaries must remain at the measured floor. This command makes no model calls. +These controls are synthetic framework examples. + +## Choose the qualification profile + +`--profile full` is the default. It requires the complete versioned static, +runtime and semantic inventory to pass. Native Harbor coverage remains incomplete, +so missing or skipped required checks block full admission. + +Select `--profile harbor-semantic-v1` explicitly for the bounded native path. +Its required results are `static.manifest`, `runtime.startup`, +`runtime.reward_well_formed`, `runtime.episode_determinism`, `semantic.oracle_max` +and `semantic.floor_gap`, plus `semantic.canary_floor` when canaries are declared. +Every required check must PASS. Any FAIL or ERROR, duplicate or missing result, +or required SKIP excludes the task. Other standard checks remain named SKIPs; +the unchanged validator report may therefore retain a WARN verdict. + +The driver verifies schema 2 native evidence for fresh NOP, oracle and oracle-replay +Trials, followed by one Trial per declared canary. It checks exact task, retained +image and inspected configuration identities, raw verifier rewards, replay equality, +zero oracle exit codes and independently verified cleanup. It recomputes the +oracle maximum, floor gap and canary floors from the bound raw evidence. +For declared range `[low, high]`, let `width = high - low`. Oracle acceptance +requires `oracle >= high - oracle_tolerance * width`; both `high - floor` and +`oracle - floor` must be at least `floor_margin * width`. These are exact numeric +comparisons, and tolerance/margin are fractions of the range rather than raw +reward units. + +This establishes bounded executable/verifier sanity and exact-submission replay. +It does not establish complete semantic coverage, hostile-code containment, +resource enforcement, network conformance or general verifier correctness. + +Supply a trusted reward/resource contract with `--contract`; see +[contract.json](contract.json). Its binary reward range and canary directory are +appropriate for the demo. For another task, review the range/resources and provide +the declared canary scripts, or omit that optional declaration. The contract is an +operator input with a recorded digest. Direct CLI use reads it from +`validation/openenv.json` inside the task: + +```bash +uv run --no-sync openenv validate inputs/task --level semantic --policy v2 \ + --json --output outputs/task-report.json +``` + +The native adapter supports self-contained single-step Linux tasks with one main +Docker service, shared verifier execution, public networking and no GPU, task-defined +environment variables, healthchecks, external MCP/credential requirements or +task-authored Compose. Unsupported shapes remain explicit SKIPs. +The first native trial builds the reviewed Dockerfile or pulls the declared +`environment.docker_image` pinned as `name@sha256:`. The validator records +that declared pin as `source_image`; all subsequent trials and model handoff use +the inspected immutable image kept by the startup-recorded retained tag. +Mutable image tags and tasks with neither a Dockerfile nor a pinned image remain +unsupported: `runtime.startup` SKIPs and the pipeline excludes them before model +execution. Acquisition preserves the native bytes; the driver does not remove an +image declaration to bypass the gate. A missing `solution/solve.sh` remains +`semantic.oracle_max` FAIL even when +startup is unsupported. The supplemental HF Jobs coding audit is a separate +operator profile and does not waive these native admission requirements. + +## Search both hubs and validate a small selection + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.pipeline \ + --output outputs/hub-pipeline-search \ + --contract examples/hub_validation_pipeline/contract.json \ + --profile harbor-semantic-v1 \ + --source both --dataset-limit 1 --tasks-per-dataset 1 +``` + +Use a contract appropriate for the selected tasks. HF listing exhausts pagination +before limiting the recorded candidates and checks the exact tag conjunction. +HF repositories are pinned to full commits; Harbor dataset/task packages use full +`sha256:` references. Gated or disabled sources become access failures. Acquisition +preserves native task/verifier bytes and downloads only selected trees and required +shared assets; the runnable profile requires self-contained tasks. + +The example uses the HF SDK's `repo_type`, `revision`, `local_dir` and selection +arguments, corresponding to `hf download --repo-type dataset --revision ... +--local-dir ...`. It deliberately resolves mutable source aliases before execution, +uses anonymous public reads (`token=False`) and applies the native acquisition +limits: 128 files, 2 MiB per file and 8 MiB total across selected tasks and shared +assets. Size-bearing metadata is required before downloads. HF inputs must be +regular Git files (no LFS); each downloaded blob is checked against the pinned +tree. Harbor requires complete per-file size/hash inventories, caps compressed +transfers and expanded tar bytes at 8 MiB, and rejects links, sparse files and +uninventoried members. Archive directory entries count toward the 128-member cap. +Legacy packages without these inventories are refused. Verified task hashes follow +the captured bytes into cloud plans; a remote revision alone is insufficient. +Acquired tasks containing native assets below `.git`, `__pycache__`, `.venv`, +`.worktrees` or `outputs` are unsupported by this bounded execution profile. +Preparation rejects their complete acquisition inventory explicitly rather than +dropping those files or weakening the source/hash comparison. +Uploads use ordinary SDK commits with explicit additions; they do not mirror-delete +remote files. As with `hf upload`, deletion requires a separate explicit operation. + +Explicit inputs replace discovery. An HF repository without `@SHA` is resolved +once to its current commit; explicit Harbor input requires an immutable digest. + +```bash +uv run --no-sync python -m examples.hub_validation_pipeline.pipeline \ + --output outputs/hub-pipeline-selected \ + --contract inputs/native-contract.json \ + --profile harbor-semantic-v1 \ + --hf-dataset 'org/repo@FULL_COMMIT_SHA' \ + --harbor-dataset 'org/dataset@sha256:FULL_PACKAGE_DIGEST' \ + --tasks-per-dataset 2 +``` + +`--local-dataset inputs/dataset` accepts already acquired files without Hub access. +The driver copies tasks into run-owned subjects, writes the trusted contract there +and retains both original and effective digests. Failures are excluded per task. + +## Evaluate a model on the admitted corpus + +Add these options to a search or explicit/local-input invocation: + +```bash + --llm-url "$LLM_URL" --model "$MODEL_ID" \ + --harness opencode --model-timeout 120 --model-step-limit 32 +``` + +Supply the inference credential through `OPENENV_LLM_API_KEY`. Validation workers +receive Docker access and a small environment whitelist; inference and Hub +credentials are not forwarded. Model execution uses the existing capture proxy and +native harness. Default forwarding is `gradio`; `--expose direct` requires sandbox +access to the host capture endpoint. + +Admitted tasks are copied into a new corpus with an explicit image/resource overlay. +Before execution, a trusted Trial hook checks the selected task, source digest, +actual image and inspected container configuration. The driver rebuilds indices +and reconciles canonical task IDs; the model runner cannot enumerate excluded +siblings. Reward 0 is a graded model outcome. Model, capture or cleanup failures +remain unsuccessful/incomplete outcomes and never become fabricated zero rewards. + +## Outputs and cleanup + +Runs retain discovery/task/admission JSONL files, per-task JSON reports and sibling +artifact bundles, run-owned subjects, an approved corpus and `summary.json`. +Requested model runs additionally retain rollouts and native trial/capture evidence. +`harbor-evidence.json`, raw reward files and checksums bind native admission to +its task, image, contract and execution copies. + +Cleanup targets exact run-owned Docker containers, networks and volumes and is +independently rechecked. The validator removes its retained image tag on FAIL, +ERROR or interruption. For an unfailed run, admission reads the tag from +`runtime.startup.measured.retained_image_tag` and binds it to the checksummed native +and trial evidence. The model corpus pins the actual immutable image ID. +Remove only the recorded retained tags with +`docker image rm ` when no longer needed. Sources and excluded +reports remain available; reruns use fresh output directories. + +## Supplemental coding audit and operator panels + +[CODING_AUDIT.md](CODING_AUDIT.md) describes generated candidate references, +operator-reviewed wrong-answer controls, fresh verifier replays, bounded leakage +probes and repeated model attempts. The local controller command is: + +```bash +uv run --no-sync python -m examples.coding_audit \ + --task inputs/task --config inputs/audit.json --output outputs/coding-report.json +``` + +This executes tasks in fresh HF Sandboxes. The `cloud` module prepares immutable +bundles and submits HF Job controllers. `real_tasks` and `full_run` additionally +require `--panel PATH`: no real corpus, reference implementation or negative panel +is embedded in their source. Keep operator panels and populated reports/ledgers +in private operational storage. + +The default model configuration records a public source revision and explicit +Inference Provider. Hosted inference does not attest an immutable served-weight +revision. Supplemental reports retain `rfc008_complete: false` and +`trainability_established: false`. + +## Focused verification + +```bash +PYTHONPATH=.:src:envs uv run --no-sync python -m pytest \ + examples/hub_validation_pipeline/ \ + tests/test_validation/test_harbor_validation.py \ + tests/envs/test_harbor_validated_trial_hook.py -q +``` + +Injected SDK tests consume no cloud/inference credits. They check source and +artifact binding, incomplete admission, raw rewards, source-mutation races, +model guards and shared-spend gates. Linux Docker execution and live provider +qualification remain separate verification steps. diff --git a/examples/hub_validation_pipeline/cloud.py b/examples/hub_validation_pipeline/cloud.py new file mode 100644 index 000000000..e27a967fa --- /dev/null +++ b/examples/hub_validation_pipeline/cloud.py @@ -0,0 +1,1139 @@ +"""Prepare immutable coding audits locally, then explicitly submit one HF Job. + +Only ``submit`` writes to the Hub or launches compute. Jobs use three distinct +runtime secrets. Native Sandbox RPC exposes its execution bearer to root task +code; use reviewed tasks. A provider's served weights are not revision-attested. +""" + +import argparse +import hashlib +import inspect +import io +import json +import os +import re +import stat +import subprocess +import sys +import tempfile +import urllib.parse +import urllib.request +import uuid +import zipfile +from fractions import Fraction +from pathlib import Path, PurePosixPath + +WORKER_IMAGE = "docker.io/library/python@sha256:48d884813cf44e927cf64c5529dd630dd3b18338c6dc727991eef9b71e934a20" +UV_VERSION = "0.9.3" +HF_HUB_VERSION = "1.33.0" +SECRET_NAMES = ("HF_TOKEN", "HF_INFERENCE_TOKEN", "HF_SANDBOX_TOKEN") +MAX_FILES = 5000 +MAX_FILE_BYTES = 16 * 1024 * 1024 +MAX_TOTAL_BYTES = 128 * 1024 * 1024 +MAX_ARCHIVE_BYTES = 160 * 1024 * 1024 +MANIFEST = "bundle-manifest.json" +LIMITATIONS = [ + "Reviewed tasks only: Native Sandbox root code can observe its execution RPC token.", + "Target revision records Hub source provenance; provider-served weights are not attested.", + "Worker image digest is a launch reference, not runtime attestation.", + "Hard termination or unavailable Hub can prevent final evidence upload.", + "This supplemental coding audit does not establish full RFC008 compliance or trainability.", +] +_EXCLUDED = {".git", "__pycache__", ".venv", ".worktrees", "outputs"} +_FORBIDDEN = { + ".env", + "env", + "secrets", + "credentials", + "logs", + "outputs", + "output", + "runs", +} + + +def _json_bytes(value): + from openenv.validation.runtime.json_values import dumps + + return (dumps(value) + "\n").encode() + + +def _loads(data): + from decimal import Decimal + + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("Duplicate JSON key") + result[key] = value + return result + + def invalid(value): + raise ValueError("Nonfinite JSON number") + + return json.loads( + data, object_pairs_hook=pairs, parse_constant=invalid, parse_float=Decimal + ) + + +def _safe_name(name): + if not isinstance(name, str): + raise ValueError("Unsafe bundle path") + path = PurePosixPath(name) + if ( + not path.parts + or path.is_absolute() + or path.as_posix() != name + or any(part in (".", "..") for part in path.parts) + or any(ord(char) < 32 for char in name) + or "\\" in name + or ":" in name + ): + raise ValueError("Unsafe bundle path") + return path + + +def _checked_archive(data, expected_sha=None): + """Check every archive member before writing or executing any payload.""" + if len(data) > MAX_ARCHIVE_BYTES: + raise ValueError("Archive size limit exceeded") + if expected_sha is not None and hashlib.sha256(data).hexdigest() != expected_sha: + raise ValueError("Bundle SHA256 mismatch") + with zipfile.ZipFile(io.BytesIO(data)) as archive: + members = archive.infolist() + if not 1 < len(members) <= MAX_FILES + 1: + raise ValueError("Bundle file count limit exceeded") + names = set() + folded = set() + total = 0 + for member in members: + name = str(_safe_name(member.filename)) + mode = member.external_attr >> 16 + if stat.S_IFMT(mode) != stat.S_IFREG or member.flag_bits & 1: + raise ValueError("Bundle members must be unencrypted regular files") + if name in names or name.casefold() in folded: + raise ValueError("Bundle path collision") + names.add(name) + folded.add(name.casefold()) + total += member.file_size + if member.file_size > MAX_FILE_BYTES or total > MAX_TOTAL_BYTES: + raise ValueError("Bundle size limit exceeded") + for name in names: + if any( + str(parent).casefold() in folded + for parent in PurePosixPath(name).parents + if str(parent) != "." + ): + raise ValueError("Bundle file/directory collision") + if MANIFEST not in names: + raise ValueError("Bundle inventory missing") + manifest = _loads(archive.read(MANIFEST)) + inventory = manifest["files"] + if not isinstance(inventory, dict) or set(inventory) != names - {MANIFEST}: + raise ValueError("Bundle inventory is incomplete") + files = {} + for member in members: + if member.filename == MANIFEST: + continue + content = archive.read(member) + record = inventory[member.filename] + if ( + not isinstance(record, dict) + or record.get("size") != len(content) + or record.get("sha256") != hashlib.sha256(content).hexdigest() + or record.get("mode") not in (0o644, 0o755) + or stat.S_IMODE(member.external_attr >> 16) != record["mode"] + ): + raise ValueError("Bundle inventory hash, size or mode mismatch") + files[member.filename] = content + files[MANIFEST] = archive.read(MANIFEST) + return manifest, files + + +def extract_bundle(data, destination, expected_sha): + """Extract a fully verified bundle into a fresh directory, without extractall.""" + manifest, files = _checked_archive(data, expected_sha) + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=False) + for name, content in files.items(): + path = destination.joinpath(*_safe_name(name).parts) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("xb") as stream: + stream.write(content) + path.chmod(manifest["files"].get(name, {}).get("mode", 0o644)) + return manifest + + +def _text_file(path): + mode = path.lstat().st_mode + if not stat.S_ISREG(mode): + raise ValueError(f"Only regular source files are allowed: {path.name}") + if path.stat().st_size > MAX_FILE_BYTES: + raise ValueError("Source file size limit exceeded") + with path.open("rb") as stream: + content = stream.read(MAX_FILE_BYTES + 1) + if len(content) > MAX_FILE_BYTES or b"\0" in content: + raise ValueError("Binary or oversized source file") + content.decode("utf-8") + _no_secrets(content) + return content, 0o755 if mode & 0o111 else 0o644 + + +def _no_secrets(content, secrets=()): + if re.search(rb"hf_[A-Za-z0-9]{16,}", content) or re.search( + rb"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----", content + ): + raise ValueError("Credential material is not allowed in persisted artifacts") + if any(secret and secret.encode() in content for secret in secrets): + raise ValueError("Runtime credential is not allowed in persisted artifacts") + for name in SECRET_NAMES: + value = os.environ.get(name) + if value and value.encode() in content: + raise ValueError("Runtime credential is not allowed in persisted artifacts") + + +def _tree(root, *, source=False): + if root.is_symlink() or not root.is_dir(): + raise ValueError("Source root must be a real directory") + result = {} + for directory, dirs, files in os.walk(root, followlinks=False): + parent = Path(directory) + excluded = _EXCLUDED | ({".pytest_cache", ".ruff_cache"} if source else set()) + dirs[:] = sorted(name for name in dirs if name not in excluded) + files = sorted(name for name in files if name not in excluded) + for name in dirs + files: + if (parent / name).is_symlink(): + raise ValueError("Symlinks are not allowed in bundles") + if any(name.lower() in _FORBIDDEN for name in dirs + files): + raise ValueError( + "Environment, secret, log or output files are not bundle inputs" + ) + for name in sorted(files): + path = parent / name + if not stat.S_ISREG(path.lstat().st_mode): + raise ValueError("Nonregular bundle input") + if source and name.endswith((".pyc", ".pyo")): + continue + relative = path.relative_to(root).as_posix() + if name.startswith(".env") or path.suffix.lower() in ( + ".pem", + ".key", + ".log", + ".zip", + ".tar", + ".gz", + ): + raise ValueError( + "Environment, secret, binary or log file is not a bundle input" + ) + if source and not ( + path.suffix == ".py" + or ( + path.suffix == ".json" + and any(part in ("schemas", "policies") for part in path.parts) + ) + ): + continue + result[relative] = _text_file(path) + if ( + len(result) > MAX_FILES + or sum(len(item[0]) for item in result.values()) > MAX_TOTAL_BYTES + ): + raise ValueError("Source tree size limit exceeded") + return result + + +def _positive(value, name, *, maximum=10000, minimum=1): + if type(value) is not int or not minimum <= value <= maximum: + raise ValueError(f"{name} must be an integer in [{minimum}, {maximum}]") + return value + + +def _acquisition_hashes(source): + """Reject acquired assets that execution would silently omit.""" + if "task_file_hashes" not in source: + if source.get("kind", source.get("source")) in {"hf", "harbor"}: + raise ValueError("Remote source requires verified acquisition file hashes") + return None + expected = source["task_file_hashes"] + if not isinstance(expected, dict) or any( + not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value) + for value in expected.values() + ): + raise ValueError("Invalid verified acquisition file hashes") + for name in expected: + if any(part in _EXCLUDED for part in _safe_name(name).parts): + raise ValueError( + "Acquired task contains unsupported excluded assets; " + "native files under execution exclusions cannot be omitted safely" + ) + return expected + + +def _check_acquisition_files(source, task_files): + """Bind remote source identities to hashes verified during acquisition.""" + expected = _acquisition_hashes(source) + if expected is not None and ( + set(expected) != set(task_files) + or any( + hashlib.sha256(data).hexdigest() != expected[name] + for name, data in task_files.items() + ) + ): + raise ValueError("Captured task files disagree with verified acquisition") + + +def _config(config): + from examples.coding_audit.runner import coding_budget, normalize_coding_config + + _no_secrets(_json_bytes(config)) + config = _loads(_json_bytes(config)) + cloud = config.setdefault("cloud", {}) + for key, value in { + "flavor": "cpu-basic", + "timeout_seconds": 1800, + "max_sandboxes": 40, + }.items(): + cloud.setdefault(key, value) + config = normalize_coding_config(config) + cloud = config["cloud"] + if "results_repo" not in cloud: + raise ValueError("Supply cloud.results_repo explicitly") + if not re.fullmatch(r"[\w.-]+/[\w.-]+", cloud["results_repo"]): + raise ValueError("cloud.results_repo must be org/repo") + if cloud["flavor"] != "cpu-basic": + raise ValueError("This bounded audit worker requires flavor cpu-basic") + if cloud.get("namespace") is not None and not re.fullmatch( + r"[\w.-]+", cloud["namespace"] + ): + raise ValueError("Invalid Job namespace") + _positive(cloud["timeout_seconds"], "timeout_seconds", maximum=86400) + budget = coding_budget(config, has_reference=True) + if ( + budget["sandbox_jobs_max"] > cloud["max_sandboxes"] + or budget["inference_calls_max"] > cloud["max_inference_calls"] + ): + raise ValueError( + "Audit upper bound exceeds max_sandboxes or max_inference_calls" + ) + budget.update( + job_timeout_seconds=cloud["timeout_seconds"], + max_sandboxes=cloud["max_sandboxes"], + max_inference_calls=cloud["max_inference_calls"], + output_token_upper_bound=config["generation"]["candidate_count"] + * config["generator"]["max_tokens"] + + len(config["seeds"]) + * config["target"]["max_steps"] + * config["target"]["max_tokens"], + cost_usd=None, + sandbox_remote_job_lifetime_seconds=86400, + concurrent_sandbox_jobs_max=1, + orphan_job_lifetime_note="SDK 1.33.0 hardcodes a 24-hour child Job lifetime; command/start/idle deadlines are shorter but cannot replace this server-side orphan cap. Collection stops after unverified cleanup.", + cost_note="Call/token counts bound requests, not current provider prices or input-token billing.", + ) + return config, budget + + +def _git_provenance(root): + def git(*args): + return ( + subprocess.check_output( + ["git", "-C", str(root), *args], stderr=subprocess.DEVNULL + ) + .decode() + .strip() + ) + + return { + "commit": git("rev-parse", "HEAD"), + "dirty": bool(git("status", "--porcelain", "--untracked-files=normal")), + } + + +class _HubRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + if urllib.parse.urlsplit(newurl).scheme != "https": + raise ValueError("Hub download redirect requires HTTPS") + redirected = super().redirect_request(req, fp, code, msg, headers, newurl) + if ( + urllib.parse.urlsplit(req.full_url).netloc + != urllib.parse.urlsplit(newurl).netloc + ): + redirected.remove_header("Authorization") + return redirected + + +def _bootstrap_main(uv_version, hub_version): + import base64 + import tempfile + + revision = os.environ["OPENENV_BUNDLE_REVISION"] + sha = os.environ["OPENENV_BUNDLE_SHA256"] + repo = os.environ["OPENENV_RESULTS_REPO"] + token = os.environ["HF_TOKEN"] + if ( + not re.fullmatch(r"[0-9a-f]{40}", revision) + or not re.fullmatch(r"[0-9a-f]{64}", sha) + or not re.fullmatch(r"[\w.-]+/[\w.-]+", repo) + ): + raise ValueError("Invalid immutable input identity") + opener = urllib.request.build_opener(_HubRedirect()) + headers = {"Authorization": "Bearer " + token} + stage = "download" + try: + url = ( + "https://huggingface.co/datasets/" + + repo + + "/resolve/" + + revision + + "/inputs/" + + sha + + "/bundle.zip" + ) + with opener.open( + urllib.request.Request(url, headers=headers), timeout=120 + ) as response: + data = response.read(MAX_ARCHIVE_BYTES + 1) + root = Path(tempfile.mkdtemp(prefix="openenv-coding-")) / "bundle" + stage = "integrity" + extract_bundle(data, root, sha) + stage = "install_uv" + subprocess.run( + [sys.executable, "-m", "pip", "install", "uv==" + uv_version], check=True + ) + stage = "sync" + subprocess.run(["uv", "sync", "--frozen", "--no-dev"], cwd=root, check=True) + python = str(root / ".venv/bin/python") + stage = "install_sdk" + subprocess.run( + [ + "uv", + "pip", + "install", + "--python", + python, + "huggingface_hub==" + hub_version, + ], + cwd=root, + check=True, + ) + stage = "verify_sdk" + subprocess.run( + [ + python, + "-c", + "from importlib.metadata import version\n" + f"if version('huggingface_hub') != {hub_version!r}:\n" + " raise RuntimeError('Installed Hugging Face SDK differs from declared pin')\n", + ], + cwd=root, + check=True, + ) + stage = "worker" + subprocess.run( + [ + python, + "-m", + "examples.hub_validation_pipeline.cloud", + "worker", + "--root", + str(root), + "--bundle-sha", + sha, + ], + cwd=root, + check=True, + ) + except BaseException as error: + # Stdlib fallback also covers dependency failures before HfApi is installed. + report = { + "status": "error", + "accepted": False, + "stage": stage, + "error_type": type(error).__name__, + "bundle_sha256": sha, + } + try: + info_url = "https://huggingface.co/api/datasets/" + repo + with opener.open( + urllib.request.Request(info_url, headers=headers), timeout=30 + ) as response: + info = _loads(response.read(1024 * 1024)) + if info.get("private") is not True: + raise ValueError("Results repository is not private") + records = [ + { + "key": "header", + "value": { + "summary": "Persist coding bootstrap error", + "description": "", + }, + }, + { + "key": "file", + "value": { + "path": "runs/coding-" + sha + "/bootstrap-error.json", + "encoding": "base64", + "content": base64.b64encode( + json.dumps(report).encode() + ).decode(), + }, + }, + ] + payload = "".join(json.dumps(row) + "\n" for row in records).encode() + request = urllib.request.Request( + info_url + "/commit/main", + data=payload, + headers={**headers, "Content-Type": "application/x-ndjson"}, + method="POST", + ) + with opener.open(request, timeout=60) as response: + response.read(1024 * 1024) + except Exception: + print("Bootstrap error evidence upload failed", file=sys.stderr) + raise RuntimeError( + "Coding bootstrap failed during " + stage + "; inspect runs/coding-" + sha + ) from None + + +def _bootstrap_code(): + # Reuse the exact stdlib-only archive validator in the remote bootstrap. + header = "import hashlib, io, json, os, re, stat, subprocess, sys, urllib.parse, urllib.request, zipfile\nfrom pathlib import Path, PurePosixPath\n" + for name in ( + "MAX_FILES", + "MAX_FILE_BYTES", + "MAX_TOTAL_BYTES", + "MAX_ARCHIVE_BYTES", + "MANIFEST", + ): + header += f"{name} = {globals()[name]!r}\n" + header += "\n".join( + inspect.getsource(function) + for function in ( + _loads, + _safe_name, + _checked_archive, + extract_bundle, + _HubRedirect, + _bootstrap_main, + ) + ) + return header + f"\n_bootstrap_main({UV_VERSION!r}, {HF_HUB_VERSION!r})\n" + + +def _launch_spec(config, digest, revision=None): + cloud = config["cloud"] + env = { + "OPENENV_RESULTS_REPO": cloud["results_repo"], + "OPENENV_BUNDLE_SHA256": digest, + "PYTHONUNBUFFERED": "1", + } + if revision is not None: + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ValueError("Input upload did not return a full commit SHA") + env["OPENENV_BUNDLE_REVISION"] = revision + spec = { + "image": WORKER_IMAGE, + "command": ["python", "-c", _bootstrap_code()], + "env": env, + "flavor": cloud["flavor"], + "timeout": f"{cloud['timeout_seconds']}s", + "labels": {"openenv_bundle": digest, "openenv_profile": "coding-audit-v1"}, + } + if cloud.get("namespace"): + spec["namespace"] = cloud["namespace"] + return spec + + +def prepare(task_root, config, output_dir, *, repo_root=None, source=None): + """Write a deterministic bundle and reviewable launch plan without network IO.""" + repo_root = Path(repo_root or Path(__file__).resolve().parents[2]) + task_root, output_dir = Path(task_root), Path(output_dir) + if output_dir.exists() or output_dir.is_symlink(): + raise FileExistsError("Prepare requires a fresh output directory") + if output_dir.resolve().is_relative_to(task_root.resolve()): + raise ValueError("Output directory must be outside the task") + _acquisition_hashes(source or {}) + config, budget = _config(config) + from examples.coding_audit.runner import validate_coding_task + from openenv.validation.runner import source_digest + + validate_coding_task(task_root, config) + files = {} + for name in ("pyproject.toml", "README.md", "LICENSE", "uv.lock"): + files[name] = _text_file(repo_root / name) + files.update( + { + "src/openenv/" + name: value + for name, value in _tree(repo_root / "src/openenv", source=True).items() + } + ) + audit_files = { + name: value + for name, value in _tree( + repo_root / "examples/coding_audit", source=True + ).items() + if not PurePosixPath(name).name.startswith("test_") + } + if ( + not {"__init__.py", "__main__.py", "scoring.py", "hf.py", "runner.py"} + <= audit_files.keys() + ): + raise ValueError("Operator coding-audit production package is incomplete") + files.update( + {"examples/coding_audit/" + name: value for name, value in audit_files.items()} + ) + worker_path = "examples/hub_validation_pipeline/cloud.py" + files[worker_path] = _text_file(repo_root / worker_path) + for name in ( + "full_run.py", + "real_tasks.py", + "sources.py", + "spend.py", + ): + relative = "examples/hub_validation_pipeline/" + name + if (repo_root / relative).exists(): + files[relative] = _text_file(repo_root / relative) + task_files = _tree(task_root) + if not task_files or "task.toml" not in task_files: + raise ValueError("Local task must contain task.toml") + # Check the exact captured bytes: a live source can change and be restored + # between its initial preflight and inventory collection. + with tempfile.TemporaryDirectory(prefix="openenv-cloud-preflight-") as temporary: + frozen = Path(temporary) + for name, (content, mode) in task_files.items(): + path = frozen / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + path.chmod(mode) + validate_coding_task(frozen, config) + captured_digest = source_digest(frozen) + files.update({"task/" + name: value for name, value in task_files.items()}) + files["coding-config.json"] = (_json_bytes(config), 0o644) + if source is None: + source = { + "kind": "local", + "identity": "sha256:" + + hashlib.sha256( + _json_bytes( + { + name: hashlib.sha256(value[0]).hexdigest() + for name, value in task_files.items() + } + ) + ).hexdigest(), + } + else: + source = dict(source) + kind = source.get("kind", source.get("source")) + if ( + source.get("kind") + and source.get("source") + and source["kind"] != source["source"] + ): + raise ValueError("Source kind declarations disagree") + revision = source.get("revision", "") + patterns = { + "hf": r"[0-9a-f]{40}", + "harbor": r"sha256:[0-9a-f]{64}", + "local": r"[0-9a-f]{64}", + } + if ( + kind not in patterns + or not isinstance(revision, str) + or not re.fullmatch(patterns[kind], revision) + ): + raise ValueError("Dataset source requires its full immutable revision") + if source.get("task_path") != ".": + _safe_name(source["task_path"]) + if kind == "local" and captured_digest != revision: + raise ValueError("Local source revision differs from the prepared task") + source["kind"] = kind + _check_acquisition_files( + source, {name: value[0] for name, value in task_files.items()} + ) + manifest = { + "schema": "openenv-coding-bundle-v1", + "git": _git_provenance(repo_root), + "runtime_versions": {"uv": UV_VERSION, "huggingface_hub": HF_HUB_VERSION}, + "task_source": source, + "target_source": { + name: config["target"][name] + for name in ("model_id", "revision", "provider") + }, + "files": { + name: { + "size": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + "mode": mode, + } + for name, (content, mode) in sorted(files.items()) + }, + "inventory_note": "Every payload file is inventoried; the outer ZIP digest also binds this manifest.", + } + files[MANIFEST] = (_json_bytes(manifest), 0o644) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_STORED) as archive: + for name, (content, mode) in sorted(files.items()): + member = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + member.create_system = 3 + member.external_attr = (stat.S_IFREG | mode) << 16 + archive.writestr(member, content) + data = buffer.getvalue() + digest = hashlib.sha256(data).hexdigest() + _checked_archive(data, digest) + plan = { + "schema": "openenv-coding-cloud-v1", + "bundle": "bundle.zip", + "bundle_sha256": digest, + "run_id": "coding-" + digest, + "results_repo": config["cloud"]["results_repo"], + "manifest_sha256": hashlib.sha256(files[MANIFEST][0]).hexdigest(), + "git": manifest["git"], + "runtime_versions": manifest["runtime_versions"], + "task_source": source, + "target_source": manifest["target_source"], + "budget": budget, + "secret_names": list(SECRET_NAMES), + "mcp_request": { + "operation": "run", + "args": _launch_spec(config, digest), + "required_env_after_upload": ["OPENENV_BUNDLE_REVISION"], + "secret_names": list(SECRET_NAMES), + }, + "limitations": LIMITATIONS, + } + output_dir.mkdir(parents=True, exist_ok=False) + (output_dir / "bundle.zip").write_bytes(data) + (output_dir / "plan.json").write_bytes(_json_bytes(plan)) + return plan + + +def _read_plan(plan_path, *, expected_sha256=None): + plan_path = Path(plan_path) + plan_bytes = plan_path.read_bytes() + if expected_sha256 is not None and ( + not isinstance(expected_sha256, str) + or not re.fullmatch(r"[0-9a-f]{64}", expected_sha256) + or hashlib.sha256(plan_bytes).hexdigest() != expected_sha256 + ): + raise ValueError("Prepared plan changed before submission") + plan = _loads(plan_bytes) + if ( + plan.get("schema") != "openenv-coding-cloud-v1" + or plan.get("bundle") != "bundle.zip" + ): + raise ValueError("Unsupported cloud plan") + data = _read_bundle(plan_path.parent / "bundle.zip") + manifest, files = _checked_archive(data, plan["bundle_sha256"]) + _check_acquisition_files( + manifest["task_source"], + { + name.removeprefix("task/"): value + for name, value in files.items() + if name.startswith("task/") + }, + ) + config, budget = _config(_loads(files["coding-config.json"])) + if ( + plan["run_id"] != "coding-" + plan["bundle_sha256"] + or plan["results_repo"] != config["cloud"]["results_repo"] + or plan["budget"] != budget + or plan["manifest_sha256"] != hashlib.sha256(files[MANIFEST]).hexdigest() + or any( + plan.get(name) != manifest.get(name) + for name in ("git", "runtime_versions", "task_source", "target_source") + ) + or manifest.get("runtime_versions") + != {"uv": UV_VERSION, "huggingface_hub": HF_HUB_VERSION} + or manifest["target_source"] + != { + name: config["target"][name] + for name in ("model_id", "revision", "provider") + } + or plan["mcp_request"]["args"] != _launch_spec(config, plan["bundle_sha256"]) + ): + raise ValueError("Plan does not match its immutable bundle") + return plan, config, data, manifest + + +def _read_bundle(path): + if ( + not stat.S_ISREG(path.lstat().st_mode) + or path.stat().st_size > MAX_ARCHIVE_BYTES + ): + raise ValueError("Bundle must be a bounded regular file") + with path.open("rb") as stream: + return stream.read(MAX_ARCHIVE_BYTES + 1) + + +def _runtime_secrets(supplied=None): + from huggingface_hub import get_token + + values = ( + dict(supplied) + if supplied is not None + else { + "HF_TOKEN": get_token(), + **{name: os.environ.get(name) for name in SECRET_NAMES[1:]}, + } + ) + if set(values) != set(SECRET_NAMES) or any( + not isinstance(values.get(name), str) + or not values[name] + or values[name].startswith("$") + for name in SECRET_NAMES + ): + raise ValueError( + "Provide HF_TOKEN, HF_INFERENCE_TOKEN and HF_SANDBOX_TOKEN as runtime secrets" + ) + if len(set(values.values())) != len(SECRET_NAMES): + raise ValueError("All three runtime secret values must be distinct") + return values + + +def _private_repo(api, repo, token): + try: + info = api.dataset_info(repo, token=token) + except Exception: + raise RuntimeError( + "Results dataset must already exist and be accessible with HF_TOKEN; create a private dataset before submission" + ) from None + if ( + info.private is not True + or not isinstance(info.sha, str) + or not re.fullmatch(r"[0-9a-f]{40}", info.sha) + ): + raise ValueError( + "Results dataset must already be private with a known commit SHA" + ) + return info + + +def _write_receipt(path, value): + temporary = path.with_suffix(".tmp") + with temporary.open("wb") as stream: + stream.write(_json_bytes(value)) + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def stage_inputs(plan_path, *, api=None, secrets=None, expected_plan_sha256=None): + """Claim a run and upload inputs; return exact launch kwargs without secrets. + + This writes to the private Hub, but does not launch compute. A stale/uncertain + receipt requires manual reconciliation. Never automatically retry a launch. + An expected plan SHA binds the parsed bytes to the caller's reviewed budget. + """ + from huggingface_hub import CommitOperationAdd, HfApi + + plan_path = Path(plan_path) + plan, config, data, manifest = _read_plan( + plan_path, expected_sha256=expected_plan_sha256 + ) + secrets = _runtime_secrets(secrets) + _no_secrets(data, secrets.values()) + api = api or HfApi(token=secrets["HF_TOKEN"]) + info = _private_repo(api, plan["results_repo"], secrets["HF_TOKEN"]) + receipt_path = plan_path.parent / "submission.json" + receipt = { + "run_id": plan["run_id"], + "status": "reserved", + "claim_nonce": uuid.uuid4().hex, + } + try: + with receipt_path.open("xb") as stream: + stream.write(_json_bytes(receipt)) + except FileExistsError: + raise FileExistsError( + "Submission receipt exists; reuse/review existing evidence explicitly instead of rerunning" + ) from None + prefix = "runs/" + plan["run_id"] + "/" + try: + names = api.list_repo_files( + plan["results_repo"], + repo_type="dataset", + revision=info.sha, + token=secrets["HF_TOKEN"], + ) + if any(name.startswith(prefix) for name in names): + raise FileExistsError( + "Run already claimed on Hub; reuse existing evidence explicitly" + ) + commit = api.create_commit( + repo_id=plan["results_repo"], + repo_type="dataset", + revision="main", + create_pr=False, + parent_commit=info.sha, + token=secrets["HF_TOKEN"], + commit_message="Claim immutable coding audit " + plan["run_id"], + operations=[ + CommitOperationAdd( + path_in_repo="inputs/" + plan["bundle_sha256"] + "/bundle.zip", + path_or_fileobj=data, + ), + CommitOperationAdd( + path_in_repo="inputs/" + plan["bundle_sha256"] + "/manifest.json", + path_or_fileobj=_json_bytes(manifest), + ), + CommitOperationAdd( + path_in_repo=prefix + "claim.json", + path_or_fileobj=_json_bytes(receipt), + ), + ], + ) + spec = _launch_spec(config, plan["bundle_sha256"], commit.oid) + receipt.update( + status="inputs_uploaded", + input_revision=commit.oid, + launch_spec=spec, + secret_names=list(SECRET_NAMES), + ) + _write_receipt(receipt_path, receipt) + (plan_path.parent / "mcp-request.json").write_bytes( + _json_bytes( + {"operation": "run", "args": spec, "secret_names": list(SECRET_NAMES)} + ) + ) + except Exception as error: + receipt.update(status="staging_failed", error_type=type(error).__name__) + _write_receipt(receipt_path, receipt) + raise + return receipt + + +def submit( + plan_path, + *, + api=None, + secrets=None, + expected_plan_sha256=None, + spend_ledger=None, + reservation_id=None, +): + """Launch once under an active shared reservation bound to the reviewed plan.""" + from huggingface_hub import HfApi + + from . import full_run, spend + + if not spend_ledger or not reservation_id or expected_plan_sha256 is None: + raise ValueError("Submission requires a reservation and reviewed plan SHA256") + _, config, _, _ = _read_plan(plan_path, expected_sha256=expected_plan_sha256) + cost = Fraction(full_run.estimate([config])["reservation_usd_exact"]) + secrets = _runtime_secrets(secrets) + api = api or HfApi(token=secrets["HF_TOKEN"]) + # Keep settlement and budget changes outside the claim/launch boundary. + with spend._ledger(spend_ledger) as ledger: + row = ledger["reservations"].get(reservation_id, {}) + if ( + spend._report(ledger)["over_budget"] + or row.get("status") != "reserved" + or Fraction(row.get("plan_costs", {}).get(expected_plan_sha256, "0")) < cost + ): + raise ValueError("Shared reservation is blocked, insufficient or unbound") + receipt = stage_inputs( + plan_path, + api=api, + secrets=secrets, + expected_plan_sha256=expected_plan_sha256, + ) + receipt_path = Path(plan_path).parent / "submission.json" + receipt["status"] = "launch_pending" + _write_receipt(receipt_path, receipt) + try: + job = api.run_job( + **receipt["launch_spec"], + secrets=secrets, + token=secrets["HF_SANDBOX_TOKEN"], + ) + except Exception as error: + receipt.update(status="launch_unknown", error_type=type(error).__name__) + _write_receipt(receipt_path, receipt) + raise RuntimeError( + "Job launch outcome unknown; inspect the submission receipt and HF Jobs before any manual retry" + ) from None + receipt.update(status="submitted", job_id=job.id, job_url=job.url) + _write_receipt(receipt_path, receipt) + try: + api.upload_file( + repo_id=receipt["launch_spec"]["env"]["OPENENV_RESULTS_REPO"], + repo_type="dataset", + path_in_repo="runs/" + receipt["run_id"] + "/submission.json", + path_or_fileobj=_json_bytes(receipt), + token=secrets["HF_TOKEN"], + commit_message="Record coding Job " + job.id, + ) + except Exception as error: + receipt.update( + receipt_sync_status="error", receipt_sync_error_type=type(error).__name__ + ) + _write_receipt(receipt_path, receipt) + return receipt + + +def worker(root, *, bundle_sha=None, api=None, runner=None): + """Run the bundled audit and persist report/evidence even after runner errors.""" + from huggingface_hub import HfApi + + root = Path(root) + secrets = _runtime_secrets() + api = api or HfApi(token=secrets["HF_TOKEN"]) + digest = bundle_sha or os.environ.get("OPENENV_BUNDLE_SHA256", "") + if not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ValueError("Worker requires bundle SHA256") + output = root / "worker-output" + output.mkdir(exist_ok=False) + run_id = "coding-" + digest + # Repo identity comes from bootstrap env so invalid config still has a sink. + repo = os.environ.get("OPENENV_RESULTS_REPO") + report = { + "status": "error", + "accepted": False, + "run_id": run_id, + "bundle_sha256": digest, + "limitations": LIMITATIONS, + } + failure = None + try: + manifest = _loads((root / MANIFEST).read_bytes()) + for name, record in manifest["files"].items(): + path = root.joinpath(*_safe_name(name).parts) + if any( + parent.is_symlink() + for parent in [ + path, + *(root / parent for parent in path.relative_to(root).parents), + ] + ): + raise ValueError("Worker payload contains a symlink") + data, mode = _text_file(path) + if ( + hashlib.sha256(data).hexdigest() != record["sha256"] + or len(data) != record["size"] + or mode != record["mode"] + ): + raise ValueError("Worker inventory mismatch") + config, budget = _config(_loads((root / "coding-config.json").read_bytes())) + if repo != config["cloud"]["results_repo"]: + raise ValueError("Worker results repository disagrees with launch") + _private_repo(api, repo, secrets["HF_TOKEN"]) + if runner is None: + from examples.coding_audit.runner import run_coding_validation + + runner = run_coding_validation + result = runner(root / "task", config, output / "audit") + if not isinstance(result, dict): + raise TypeError("Coding runner must return a report object") + report.update( + status="completed", + report=result, + budget=budget, + git=manifest["git"], + task_source=manifest["task_source"], + target_source=manifest["target_source"], + ) + report["accepted"] = result.get("accepted") is True + except BaseException as error: + # Interrupts must persist the same sanitized failure evidence before the + # worker exits unsuccessfully; they must never look like a completed run. + failure = error + report.update(status="error", accepted=False, error_type=type(error).__name__) + # SDK exceptions can contain credentials; persist type, never raw repr. + finally: + (output / "cloud-report.json").write_bytes(_json_bytes(report)) + try: + _private_repo(api, repo, secrets["HF_TOKEN"]) + # A malformed artifact must not prevent persistence of the error report. + try: + evidence = _tree(output) + for data, _ in evidence.values(): + _no_secrets(data, secrets.values()) + except Exception as error: + failure = error + report.pop("report", None) + report.update( + status="error", + accepted=False, + evidence_collection_error_type=type(error).__name__, + evidence_omitted=True, + ) + report_data = _json_bytes(report) + (output / "cloud-report.json").write_bytes(report_data) + evidence = {"cloud-report.json": (report_data, 0o644)} + from huggingface_hub import CommitOperationAdd + + api.create_commit( + repo_id=repo, + repo_type="dataset", + commit_message="Persist coding audit " + run_id, + operations=[ + CommitOperationAdd( + path_in_repo="runs/" + run_id + "/" + name, path_or_fileobj=data + ) + for name, (data, _) in sorted(evidence.items()) + ], + token=secrets["HF_TOKEN"], + ) + except Exception as error: + report.update( + status="error", + accepted=False, + persistence_status="error", + persistence_error_type=type(error).__name__, + ) + (output / "cloud-report.json").write_bytes(_json_bytes(report)) + raise RuntimeError( + "Evidence upload failed; local worker-output/cloud-report.json retained" + ) from None + if failure is not None: + raise RuntimeError( + "Coding worker failed; error report and available evidence persisted" + ) from None + return report + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + local = commands.add_parser( + "prepare", help="Build local immutable inputs; no network or paid compute" + ) + local.add_argument("--task", type=Path, required=True) + local.add_argument("--config", type=Path, required=True) + local.add_argument("--output", type=Path, required=True) + internal = commands.add_parser("worker", help=argparse.SUPPRESS) + internal.add_argument("--root", type=Path, required=True) + internal.add_argument("--bundle-sha", required=True) + args = parser.parse_args(argv) + if args.command == "prepare": + result = prepare(args.task, _loads(args.config.read_bytes()), args.output) + else: + result = worker(args.root, bundle_sha=args.bundle_sha) + print(_json_bytes(result).decode(), end="") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as error: + from huggingface_hub import get_token + + message = str(error) + for value in [get_token(), *(os.environ.get(name) for name in SECRET_NAMES)]: + if value: + message = message.replace(value, "[REDACTED]") + message = re.sub(r"hf_[A-Za-z0-9]{16,}", "[REDACTED]", message) + print( + "Cloud operation failed (" + type(error).__name__ + "): " + message[:1200], + file=sys.stderr, + ) + sys.exit(1) diff --git a/examples/hub_validation_pipeline/contract.json b/examples/hub_validation_pipeline/contract.json new file mode 100644 index 000000000..6bf3b848f --- /dev/null +++ b/examples/hub_validation_pipeline/contract.json @@ -0,0 +1,18 @@ +{ + "profile": "harbor-semantic-v1", + "reward_key": "reward", + "reward": { + "range": [0, 1], + "oracle_tolerance": 0, + "floor_margin": 0.1 + }, + "resources": { + "cpu": 1, + "memory_mb": 512, + "disk_mb": 1024, + "episode_timeout_s": 120 + }, + "types": {"tags": ["swe"]}, + "requires_credentials": false, + "canaries": "validation/canaries" +} diff --git a/examples/hub_validation_pipeline/demo.py b/examples/hub_validation_pipeline/demo.py new file mode 100644 index 000000000..c116298cf --- /dev/null +++ b/examples/hub_validation_pipeline/demo.py @@ -0,0 +1,94 @@ +"""Create three trusted tiny tasks and exercise validation/filtering with local Docker.""" + +import argparse +from pathlib import Path + +from examples.hub_validation_pipeline.pipeline import PILOT, run + + +def create_tasks(root): + """One valid task, one broken verifier, and one missing reference solution.""" + config = """version = "1.0" +[agent] +timeout_sec = 30.0 +[verifier] +timeout_sec = 10.0 +environment_mode = "shared" +[environment] +cpus = 1 +memory_mb = 512 +storage_mb = 1024 +""" + verifier = """#!/bin/bash +mkdir -p /logs/verifier +if [ "$(cat /app/answer.txt 2>/dev/null)" = "42" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi +""" + for name in ("good", "always-rewarding", "missing-oracle"): + task = root / name + (task / "environment").mkdir(parents=True) + (task / "tests").mkdir() + canaries = task / "validation/canaries" + canaries.mkdir(parents=True) + (canaries / "empty.sh").write_text("#!/bin/bash\n:\n") + (canaries / "wrong-answer.sh").write_text( + "#!/bin/bash\nprintf '41\\n' > /app/answer.txt\n" + ) + (task / "task.toml").write_text(config) + (task / "instruction.md").write_text("Write exactly 42 into /app/answer.txt.\n") + (task / "environment/Dockerfile").write_text( + "FROM python:3.12-slim\nWORKDIR /app\n" + ) + (task / "tests/test.sh").write_text( + "#!/bin/bash\nmkdir -p /logs/verifier\necho 1 > /logs/verifier/reward.txt\n" + if name == "always-rewarding" + else verifier + ) + if name != "missing-oracle": + (task / "solution").mkdir() + (task / "solution/solve.sh").write_text( + "#!/bin/bash\nprintf '42\\n' > /app/answer.txt\n" + ) + return root + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True, help="New demo directory") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + dataset = create_tasks(args.output.resolve() / "dataset") + options = argparse.Namespace( + output=args.output / "run", + contract=Path(__file__).with_name("contract.json"), + profile=PILOT, + source="both", + dataset_limit=0, + tasks_per_dataset=3, + hf_dataset=[], + harbor_dataset=[], + local_dataset=[dataset], + validation_timeout=300, + llm_url="", + model="", + harness="opencode", + expose="gradio", + model_timeout=120, + model_step_limit=32, + ) + status = run(options) + import json + + summary = json.loads((options.output / "summary.json").read_text()) + if status != 0 or summary["accepted"] != 1 or summary["excluded"] != 2: + raise SystemExit("demo failed: expected one admitted task and two exclusions") + print( + "Demo passed: good task admitted; broken verifier and missing oracle excluded." + ) + + +if __name__ == "__main__": + main() diff --git a/examples/hub_validation_pipeline/full_run.py b/examples/hub_validation_pipeline/full_run.py new file mode 100644 index 000000000..f5caa56a3 --- /dev/null +++ b/examples/hub_validation_pipeline/full_run.py @@ -0,0 +1,369 @@ +"""Freeze census metadata and prepare the reviewed cohort; --submit launches once. + +Preparation acquires selected pinned public task trees, never paid compute. Only +cloud.submit publishes existing inputs/results. No catalog-wide execution claim. +""" + +import argparse +import hashlib +import os +from decimal import Decimal, InvalidOperation +from fractions import Fraction +from math import ceil +from pathlib import Path + +from . import cloud, real_tasks, spend + +MODEL_SOURCE = ( + "Qwen/Qwen3.8-27B", + "37561816e66b5b9153b1b64c10968c18c1e6f4a0", + "deepinfra", +) +RATEBOOK = { + "as_of": "2026-11-10", + "input_usd_per_million": "0.15", + "output_usd_per_million": "1.875", + "cpu_basic_usd_per_hour": "0.01", + "sandbox_port_usd_per_hour": "0.01", + "child_lifetime_seconds": 86400, + "startup_allowance_seconds": 300, +} +ASSUMPTIONS = [ + "Modeled launch reservation at recorded prices; not an account-enforced billing limit.", + "Reservation covers the selected cohort, without builds; the shared ledger must account for prior/concurrent spending and retries.", + "Every child reserves the SDK hard 86400-second lifetime plus 300-second startup; controllers reserve configured timeout plus 300 seconds, without an exposed port.", + "Prior billed tokens stay below max_total_tokens before the last request; message bytes <= 4*total+8192, including tool/framing allowance; request context ceiling 262144.", + "Source revision does not attest provider-served weights; rate/model/flavor changes require a different reviewed ratebook.", + "Only the configured cohort has reviewed execution contracts; all remaining catalog tasks are metadata only. Native tasks are evaluation-only, never training data.", + "An uncertain submission requires manual reconciliation; never automatically resume or retry.", +] + + +def _budget(value): + try: + number = Decimal(value) if type(value) is str else Decimal("NaN") + if number.is_finite() and number > 0: + return Fraction(number) + except InvalidOperation: + raise ValueError( + "budget_usd must be a finite positive decimal string" + ) from None + raise ValueError("budget_usd must be a finite positive decimal string") + + +def estimate(configs): + """Return exact aggregate launch reservation and request/token ceilings.""" + if not configs: + raise ValueError("A configured cohort is required") + children = calls = inputs = outputs = controller_seconds = 0 + for raw in configs: + config, limits = cloud._config(raw) + if ( + limits["sandbox_remote_job_lifetime_seconds"] + != RATEBOOK["child_lifetime_seconds"] + ): + raise ValueError("A different child lifetime requires a different ratebook") + if any( + config[key].get("flavor", "cpu-basic") != "cpu-basic" + for key in ("cloud", "execution") + ): + raise ValueError("A different flavor requires a different ratebook") + for key in ("target", "generator"): + model = config[key] + if tuple( + model[k] for k in ("model_id", "revision", "provider") + ) != MODEL_SOURCE or model.get("provider_model_id"): + raise ValueError( + "A different model/source/provider requires a different ratebook" + ) + total, maximum = model["max_total_tokens"], model["max_tokens"] + count = ( + limits["target_attempts"] + if key == "target" + else limits["generated_candidates"] + ) + inputs += count * ( + min(262144, 4 * total + 8192) + (total if key == "target" else 0) + ) + outputs += count * min( + total, maximum * (model["max_steps"] if key == "target" else 1) + ) + children += limits["sandbox_jobs_max"] + calls += limits["inference_calls_max"] + controller_seconds += ( + config["cloud"]["timeout_seconds"] + RATEBOOK["startup_allowance_seconds"] + ) + inference = ( + inputs * Fraction(RATEBOOK["input_usd_per_million"]) + + outputs * Fraction(RATEBOOK["output_usd_per_million"]) + ) / 1000000 + cpu, port = ( + Fraction(RATEBOOK[key]) + for key in ("cpu_basic_usd_per_hour", "sandbox_port_usd_per_hour") + ) + child_seconds = ( + RATEBOOK["child_lifetime_seconds"] + RATEBOOK["startup_allowance_seconds"] + ) + compute = ( + children * child_seconds * (cpu + port) + controller_seconds * cpu + ) / 3600 + cost = inference + compute + micros = ceil(cost * 1000000) + return { + "sandbox_jobs_max": children, + "controller_jobs_max": len(configs), + "inference_calls_max": calls, + "input_tokens_max": inputs, + "output_tokens_max": outputs, + "inference_usd_exact": str(inference), + "compute_usd_exact": str(compute), + "reservation_usd_exact": str(cost), + "reservation_usd_rounded_up": f"{micros // 1000000}.{micros % 1000000:06d}", + } + + +def _write(path, value): + temporary = path.with_suffix(".tmp") + with temporary.open("wb") as stream: + stream.write(cloud._json_bytes(value)) + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def _hash(path): + digest, size = hashlib.sha256(), 0 + with path.open("rb") as stream: + while block := stream.read(1024 * 1024): + digest.update(block) + size += len(block) + return {"path": str(path), "sha256": digest.hexdigest(), "bytes": size} + + +def _catalog(path, destination, kind): + path = Path(path).absolute() + with path.open("rb") as stream: + data = stream.read(1024 * 1024 + 1) + if len(data) > 1024 * 1024: + raise ValueError("Supply a small census summary, not a raw census") + summary = cloud._loads(data) + destination.write_bytes(data) + references = ( + { + key: summary[key] + for key in ("compact_repo_inventory", "compact_identity_inventory") + } + if kind == "hf" + else { + key: summary["artifacts"][key]["path"] + for key in ("task_identity_inventory", "dataset_version_membership_index") + } + ) + inventories = {} + for name, reference in references.items(): + inventory = Path(reference) + if not inventory.is_absolute(): + inventory = path.parent / inventory + inventories[name] = ( + _hash(inventory) + if inventory.is_file() + else {"path": str(inventory), "status": "unavailable"} + ) + return { + **_hash(destination), + "scope": summary["scope"], + "counts": summary["counts"], + "inventories": inventories, + "runtime_qualified": False, + } + + +def run( + output, + hf_census, + harbor_census, + budget_usd, + *, + panel_path, + task_names=None, + submit=False, + spend_ledger=None, + reservation_id=None, +): + if (submit or spend_ledger is not None or reservation_id is not None) and ( + spend_ledger is None + or (isinstance(spend_ledger, str) and not spend_ledger.strip()) + or not isinstance(reservation_id, str) + or not reservation_id.strip() + ): + raise ValueError("Supply both a shared ledger and a reservation ID") + cap = _budget(budget_usd) + panel, panel_provenance = real_tasks._load_panel(panel_path) + names = [task["name"] for task in real_tasks._select_tasks(panel, task_names)] + output = Path(output).absolute() + output.mkdir(parents=True, exist_ok=False) + manifest_path = output / "manifest.json" + manifest = { + "schema": "openenv-full-run-v1", + "status": "preparing", + "stage": "catalogs", + "accepted": False, + "rfc008_qualified": False, + "trainable": False, + "evaluation_only": True, + "full_catalog_executed": False, + "budget_usd": budget_usd, + "ratebook": RATEBOOK, + "assumptions": ASSUMPTIONS, + "evaluation_panel": panel_provenance, + "catalogs": {}, + "tasks": [], + } + _write(manifest_path, manifest) + try: + for kind, source in (("hf", hf_census), ("harbor", harbor_census)): + manifest["catalogs"][kind] = _catalog( + source, output / f"{kind}-census-summary.json", kind + ) + _write(manifest_path, manifest) + manifest["stage"] = "prepare" + recipes = real_tasks._prepare( + output / "recipes", + panel=panel, + panel_provenance=panel_provenance, + task_names=names, + bundle=True, + ) + configs = [] + manifest["stage"] = "plans" + for row in recipes["tasks"]: + plan_sha256 = _hash(Path(row["bundle_plan"]))["sha256"] + plan, config, _, _ = cloud._read_plan( + row["bundle_plan"], expected_sha256=plan_sha256 + ) + configs.append(config) + manifest["tasks"].append( + { + "task_id": row["task_id"], + "plan": row["bundle_plan"], + "plan_path": Path(row["bundle_plan"]) + .relative_to(output) + .as_posix(), + "plan_sha256": plan_sha256, + "run_id": plan["run_id"], + "bundle_sha256": plan["bundle_sha256"], + "budget": plan["budget"], + "accepted": False, + "trainable": False, + "rfc008_qualified": False, + "submission": {"status": "not_submitted"}, + } + ) + _write(manifest_path, manifest) + manifest["stage"] = "budget" + manifest["estimate"] = estimate(configs) + manifest["within_budget"] = ( + Fraction(manifest["estimate"]["reservation_usd_rounded_up"]) <= cap + ) + manifest["status"] = "prepared" + _write(manifest_path, manifest) + if submit: + if not manifest["within_budget"]: + raise ValueError("Reservation exceeds budget") + manifest["stage"] = "reserve" + fingerprint = hashlib.sha256( + "\0".join(task["bundle_sha256"] for task in manifest["tasks"]).encode() + ).hexdigest()[:16] + identifier = f"{reservation_id}:{fingerprint}" + spend.reserve( + spend_ledger, + identifier, + manifest["estimate"]["reservation_usd_rounded_up"], + "Reviewed native Hub audit cohort", + evidence=str(manifest_path), + plan_costs={ + task["plan_sha256"]: estimate([config])["reservation_usd_exact"] + for task, config in zip(manifest["tasks"], configs, strict=True) + }, + ) + manifest["shared_spend"] = { + "ledger": str(Path(spend_ledger).absolute()), + "reservation_id": identifier, + "reserved_usd": manifest["estimate"]["reservation_usd_rounded_up"], + } + _write(manifest_path, manifest) + manifest.update(stage="submit", status="submitting") + for task in manifest["tasks"]: + if _hash(Path(task["plan"]))["sha256"] != task["plan_sha256"]: + raise ValueError("Prepared plan changed before submission") + task["submission"] = { + "status": "pending_or_unknown", + "receipt": str(Path(task["plan"]).parent / "submission.json"), + } + _write(manifest_path, manifest) + receipt = cloud.submit( + task["plan"], + expected_plan_sha256=task["plan_sha256"], + spend_ledger=spend_ledger, + reservation_id=identifier, + ) + task["submission"].update( + { + key: receipt[key] + for key in ( + "status", + "job_id", + "job_url", + "input_revision", + "receipt_sync_status", + "receipt_sync_error_type", + ) + if key in receipt + } + ) + _write(manifest_path, manifest) + manifest["status"] = "submitted" + manifest["stage"] = "done" + _write(manifest_path, manifest) + except BaseException as error: + manifest.update(status="failed", error_type=type(error).__name__) + _write(manifest_path, manifest) + raise RuntimeError( + f"Full run stopped during {manifest['stage']}; inspect manifest.json and per-plan submission receipts" + ) from None + return manifest + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--panel", dest="panel_path", type=Path, required=True) + parser.add_argument("--hf-census", type=Path, required=True) + parser.add_argument("--harbor-census", type=Path, required=True) + parser.add_argument("--budget-usd", required=True) + parser.add_argument("--task", action="append", dest="task_names") + parser.add_argument("--submit", action="store_true") + parser.add_argument("--spend-ledger", type=Path) + parser.add_argument("--reservation-id") + args = parser.parse_args(argv) + if args.submit and (args.spend_ledger is None or not args.reservation_id): + parser.error("Paid submission requires --spend-ledger and --reservation-id") + manifest = run(**vars(args)) + print( + cloud._json_bytes( + { + "manifest": str(args.output.absolute() / "manifest.json"), + "status": manifest["status"], + } + ).decode(), + end="", + ) + + +if __name__ == "__main__": + main() diff --git a/examples/hub_validation_pipeline/pipeline.py b/examples/hub_validation_pipeline/pipeline.py new file mode 100644 index 000000000..737b01963 --- /dev/null +++ b/examples/hub_validation_pipeline/pipeline.py @@ -0,0 +1,823 @@ +"""Discover pinned tasks, validate them, then evaluate only an admitted local corpus.""" + +import argparse +import asyncio +import hashlib +import math +import os +import re +import shutil +import subprocess +import sys +import time +import tomllib +from decimal import Decimal +from fractions import Fraction +from pathlib import Path + +import tomli_w +from examples.hub_validation_pipeline import sources +from openenv.validation.parsers.harbor_toml import ( + _unique_object, + canary_scripts, + read_harbor_contract, +) +from openenv.validation.policy import load_policy +from openenv.validation.report import ValidationReportV2 +from openenv.validation.runner import _DIGEST_EXCLUDED_DIRS, source_digest +from openenv.validation.runtime.json_values import dumps, loads +from openenv.validation.types import Lane + +PILOT = "harbor-semantic-v1" +PILOT_CHECKS = { + "static.manifest", + "runtime.startup", + "runtime.reward_well_formed", + "runtime.episode_determinism", + "semantic.oracle_max", + "semantic.floor_gap", +} +REPO = Path(__file__).resolve().parents[2] + + +def write_json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(dumps(value) + "\n") + temporary.replace(path) + + +def write_rows(path, rows): + path.write_text("".join(dumps(row) + "\n" for row in rows)) + + +def copy_subject(source, destination, expected_digest): + shutil.copytree( + source, + destination, + symlinks=True, + ignore=shutil.ignore_patterns(*_DIGEST_EXCLUDED_DIRS), + ) + if source_digest(destination) != expected_digest: + raise ValueError("source changed while copying the task") + + +def validation_environment(): + """Give the validation worker Docker access without inference or Hub secrets.""" + names = { + "PATH", + "HOME", + "TMPDIR", + "LANG", + "LC_ALL", + "DOCKER_HOST", + "DOCKER_CONTEXT", + "DOCKER_CONFIG", + "DOCKER_CERT_PATH", + "DOCKER_TLS_VERIFY", + "SSH_AUTH_SOCK", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + } + env = {key: value for key, value in os.environ.items() if key in names} + env["PYTHONPATH"] = os.pathsep.join((str(REPO / "src"), str(REPO / "envs"))) + env["PYTHONDONTWRITEBYTECODE"] = "1" + return env + + +def validate_task(subject, report_path, profile, timeout_s): + if profile not in {PILOT, "full"}: + raise ValueError("unsupported admission profile") + command = [ + sys.executable, + "-m", + "openenv.cli", + "validate", + str(subject), + "--level", + "semantic", + "--policy", + "v2", + "--json", + "--output", + str(report_path), + ] + report_path.parent.mkdir(parents=True, exist_ok=True) + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=validation_environment(), + cwd=REPO, + ) + try: + _, stderr = process.communicate(timeout=timeout_s) + except subprocess.TimeoutExpired: + # SIGTERM reaches validate's cleanup handler before a final hard stop. + process.terminate() + try: + process.communicate(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.communicate() + raise TimeoutError("validation deadline exceeded; task excluded") from None + if not report_path.is_file(): + raise RuntimeError( + f"validator exited {process.returncode} without a report: {stderr[-500:]}" + ) + return process.returncode, loads(report_path.read_text()) + + +def verified_bundle(directory): + """Verify every retained artifact before using its admission evidence.""" + entries = {} + for row in (directory / "SHA256SUMS").read_text().splitlines(): + digest, name = row.split(" ", 1) + relative = Path(name) + if ( + not re.fullmatch(r"[0-9a-f]{64}", digest) + or relative.is_absolute() + or ".." in relative.parts + or name in entries + ): + raise ValueError("invalid artifact checksum inventory") + path = directory / relative + if path.is_symlink() or not path.resolve().is_relative_to(directory.resolve()): + raise ValueError("artifact path escapes evidence directory") + if hashlib.sha256(path.read_bytes()).hexdigest() != digest: + raise ValueError(f"artifact checksum mismatch: {name}") + entries[name] = digest + if not {"report.json", "harbor-evidence.json"}.issubset(entries): + raise ValueError( + "native validation evidence is missing from checksum inventory" + ) + return entries + + +def reward_fraction(value): + if ( + isinstance(value, bool) + or not isinstance(value, (int, Decimal)) + or not Decimal(value).is_finite() + ): + raise ValueError("native reward must be finite and numeric") + return Fraction(value) + + +def trial_rewards(trial, artifact_dir, checksums, key, low, high): + """Bind every raw output, including an optional nonzero oracle exit marker.""" + raw = trial["raw_rewards"] + allowed = {"reward.json", "reward.txt", "oracle-exit-code.txt"} + if not isinstance(raw, dict) or set(raw) - allowed: + raise ValueError("unexpected native raw output inventory") + prefix = f"harbor-trials/{trial['role']}" + paths = {f"{prefix}/{name}" for name in raw} + actual = { + path.relative_to(artifact_dir).as_posix() + for path in (artifact_dir / prefix).iterdir() + if path.name in allowed + } + if paths != actual: + raise ValueError("native raw output inventory is incomplete") + outputs = {} + for name, reference in raw.items(): + if reference.get("path") != f"{prefix}/{name}" or checksums.get( + reference["path"] + ) != reference.get("sha256"): + raise ValueError( + "raw native output is not in the verified artifact inventory" + ) + outputs[name] = (artifact_dir / reference["path"]).read_text() + if "reward.json" in outputs: + values = loads(outputs["reward.json"], object_pairs_hook=_unique_object) + elif "reward.txt" in outputs: + try: + values = {"reward": Decimal(outputs["reward.txt"].strip())} + except ArithmeticError as error: + raise ValueError("invalid native reward text") from error + else: + raise ValueError("native trial has no raw rewards") + if not isinstance(values, dict) or not values or key not in values: + raise ValueError("native reward map lacks the declared key") + rewards = {name: reward_fraction(value) for name, value in values.items()} + if any(not low <= value <= high for value in rewards.values()): + raise ValueError("native reward is outside the declared range") + observed = { + name: reward_fraction(value) for name, value in trial["rewards"].items() + } + if rewards != observed: + raise ValueError("raw native rewards differ from the recorded reward map") + if trial["role"] != "nop": + code = trial.get("oracle_exit_code") + if ( + trial.get("oracle_exit_observed_at_agent_end") is not True + or isinstance(code, bool) + or not isinstance(code, int) + or code != int(outputs.get("oracle-exit-code.txt", "0")) + ): + raise ValueError("native oracle exit evidence is inconsistent") + return rewards + + +def admission(report, exit_code, subject, artifact_dir, profile, original_digest): + """Return a task/image-bound admission envelope, or reject the task.""" + if profile not in {PILOT, "full"}: + raise ValueError("unsupported admission profile") + ValidationReportV2.model_validate(report) + if exit_code != 0 or report["verdict"] not in {"pass", "warn"}: + raise ValueError("validation failed") + if ( + report["signature"] != "task.toml" + or report["policy_version"] != "v2" + or report["lane"] != "local" + ): + raise ValueError("unexpected validation signature or policy") + effective_digest = source_digest(subject) + if report["source_digest"] != effective_digest: + raise ValueError("task changed after validation") + parsed_contract = read_harbor_contract(subject) + contract = loads( + (subject / "validation/openenv.json").read_text(), + object_pairs_hook=_unique_object, + ) + canaries = ( + canary_scripts(subject, parsed_contract.canaries) + if parsed_contract.canaries is not None + else [] + ) + results = {row["check_id"]: row for row in report["results"]} + if len(results) != len(report["results"]): + raise ValueError("duplicate validation check IDs") + if any(row["status"] in {"fail", "error"} for row in results.values()): + raise ValueError("a validation check failed or errored") + known_checks = set(load_policy("v2").entries_for_lane(Lane.LOCAL)) + if set(results) - known_checks: + raise ValueError("unknown validation check ID") + required = ( + PILOT_CHECKS | ({"semantic.canary_floor"} if canaries else set()) + if profile == PILOT + else set(load_policy("v2").entries_for_lane(Lane.LOCAL)) + ) + if not required.issubset(results) or any( + results[key]["status"] != "pass" for key in required + ): + raise ValueError("required validation coverage is incomplete") + if not {1, 2, 3}.issubset(report["levels_run"]): + raise ValueError("semantic validation was not executed") + retained_tag = ( + results["runtime.startup"].get("measured", {}).get("retained_image_tag") + ) + if not re.fullmatch(r"openenv-validation:[a-f0-9]{32}", retained_tag or ""): + raise ValueError("startup retained image measurement is missing or invalid") + if profile != PILOT and ( + report["verdict"] != "pass" or 3 not in report["levels_run"] + ): + raise ValueError("full validation is incomplete") + checksums = verified_bundle(artifact_dir) + if dumps(loads((artifact_dir / "report.json").read_text())) != dumps(report): + raise ValueError("bundled report differs from the validator result") + native = loads((artifact_dir / "harbor-evidence.json").read_text()) + if ( + native.get("schema_version") != "2" + or native["profile"] != PILOT + or native["task_effective_digest"] != effective_digest + or native["original_source_digest"] != original_digest + or contract.get("original_source_digest") != original_digest + or native["reward_key"] != contract["reward_key"] + or report["manifest"]["execution"]["profile"] != PILOT + or report["manifest"]["capabilities"]["canaries"] != parsed_contract.canaries + or not re.fullmatch(r"sha256:[0-9a-f]{64}", native.get("image_ref") or "") + or native.get("retained_image_tag") != retained_tag + or native["cleanup"]["verified"] is not True + or "failure" in native + ): + raise ValueError( + "native oracle/floor, identity or cleanup evidence is incomplete" + ) + trials = native["trials"] + roles = ["nop", "oracle", "oracle-replay"] + [ + f"canary-{index}" for index in range(1, len(canaries) + 1) + ] + if ( + [row["role"] for row in trials] != roles + or len({row["container_id"] for row in trials}) != len(trials) + or any( + not re.fullmatch(r"[0-9a-f]{64}", row["container_id"] or "") + or row["image_ref"] != native["image_ref"] + or row["retained_image_tag"] != retained_tag + or row["unmodified_task_digest"] != report["source_digest"] + or row["configuration"] != trials[0]["configuration"] + or row["cleanup"]["verified"] is not True + or "failure" in row + or "reward_error" in row + for row in trials + ) + ): + raise ValueError( + "native trials do not establish fresh identical-image execution" + ) + reward_contract = contract["reward"] + low, high = map(reward_fraction, reward_contract["range"]) + margin = reward_fraction(reward_contract["floor_margin"]) + tolerance = reward_fraction(reward_contract["oracle_tolerance"]) + width = high - low + rewards = [] + for index, trial in enumerate(trials): + overlay = ( + {} + if index == 0 + else { + "environment.docker_image": retained_tag, + } + ) + if index >= 3: + canary = canaries[index - 3] + overlay["solution/solve.sh"] = { + "canary": canary, + "sha256": hashlib.sha256((subject / canary).read_bytes()).hexdigest(), + } + if trial["task_overlay"] != overlay: + raise ValueError( + "native task overlay differs from the declared source bytes" + ) + rewards.append( + trial_rewards( + trial, artifact_dir, checksums, native["reward_key"], low, high + ) + ) + if ( + rewards[1] != rewards[2] + or trials[1]["oracle_exit_code"] != trials[2]["oracle_exit_code"] + or trials[1]["oracle_exit_code"] != 0 + ): + raise ValueError("raw native oracle replay or exit code is inconsistent") + floor, oracle = (row[native["reward_key"]] for row in rewards[:2]) + if ( + floor != reward_fraction(native.get("floor_reward")) + or oracle != reward_fraction(native.get("oracle_reward")) + or oracle < high - tolerance * width + or oracle - floor < margin * width + or high - floor < margin * width + or any(row[native["reward_key"]] > floor for row in rewards[3:]) + ): + raise ValueError("raw native rewards do not satisfy the oracle/floor contract") + return { + "profile": profile, + "rfc008_complete": profile == "full", + "original_source_digest": original_digest, + "task_effective_digest": effective_digest, + "image_ref": native["image_ref"], + "retained_image_tag": retained_tag, + "reward_key": native["reward_key"], + "floor_reward": native["floor_reward"], + "oracle_reward": native["oracle_reward"], + "resources": report["manifest"]["resources"], + "container_configuration": native["trials"][0]["configuration"], + "report_sha256": checksums["report.json"], + "evidence_sha256": checksums["harbor-evidence.json"], + "out_of_profile_skips": sorted( + key for key, row in results.items() if row["status"] == "skip" + ), + } + + +def materialize(record, destination): + """Pin the execution image while retaining the validated task and overlay hashes.""" + subject = Path(record["subject"]) + if source_digest(subject) != record["task_effective_digest"]: + raise ValueError("validated subject changed before materialization") + copy_subject(subject, destination, record["task_effective_digest"]) + config_path = destination / "task.toml" + config = tomllib.loads(config_path.read_text()) + config.setdefault("environment", {})["docker_image"] = record["image_ref"] + for key, value in ( + ("cpus", record["resources"]["cpu"]), + ("memory_mb", record["resources"]["memory_mb"]), + ("storage_mb", record["resources"]["disk_mb"]), + ): + config["environment"][key] = float(value) if key == "cpus" else value + config_path.write_text(tomli_w.dumps(config)) + return { + **record, + "execution_path": str(destination), + "execution_digest": source_digest(destination), + "execution_overlay": { + "environment": config["environment"], + }, + } + + +def docker(*arguments): + return subprocess.run( + ["docker", *arguments], check=True, text=True, capture_output=True, timeout=30 + ).stdout.strip() + + +def model_guard(records, projects): + """Attach an actual-container image check before the agent can call its model.""" + by_path = {str(Path(row["execution_path"]).resolve()): row for row in records} + + def created(trial): + from harbor.trial.hooks import TrialEvent + + path = str(trial.config.task.path.resolve()) + if path not in by_path: + raise ValueError("model runner selected an unapproved task") + record = by_path[path] + if source_digest(Path(path)) != record["execution_digest"]: + raise ValueError("approved task changed before execution") + project = trial.agent_environment.session_id + if not re.fullmatch(r"[a-z0-9_-]+", project): + raise ValueError("unexpected Harbor project identity") + projects.add(project) + + async def inspect_image(event): + ids = docker( + "ps", "-aq", "--filter", f"label=com.docker.compose.project={project}" + ).splitlines() + if len(ids) != 1: + raise ValueError("expected one approved task container") + actual = loads(docker("inspect", ids[0]))[0] + if actual["Image"] != record["image_ref"]: + raise ValueError("model task image differs from validated image") + expected = record["container_configuration"] + mounts = sorted( + [ + { + "type": item["Type"], + "destination": item["Destination"], + "rw": item["RW"], + } + for item in actual["Mounts"] + ], + key=lambda item: item["destination"], + ) + network = actual["HostConfig"]["NetworkMode"] + if network == project + "_default": + network = "compose-default" + if ( + actual["Config"]["User"] != expected["user"] + or actual["Config"]["WorkingDir"] != expected["workdir"] + or {key: actual["HostConfig"].get(key) for key in expected["host"]} + != expected["host"] + or mounts != expected["mounts"] + or network != expected["network_mode"] + ): + raise ValueError("model sandbox configuration differs from validation") + + trial.add_hook(TrialEvent.AGENT_START, inspect_image) + + return created + + +def cleanup_projects(projects): + """Remove and recheck only resources belonging to this run's model trials.""" + for project in projects: + selector = f"label=com.docker.compose.project={project}" + containers = docker("ps", "-aq", "--filter", selector).splitlines() + owned_volumes = set() + if containers: + for container in loads(docker("inspect", *containers)): + for mount in container["Mounts"]: + if mount["Type"] == "volume": + name = mount["Name"] + if not re.fullmatch(r"[a-f0-9]{64}", name): + raise ValueError( + "model trial mounted a volume of unproven ownership" + ) + owned_volumes.add(name) + for kind, listing, removal in ( + ("container", ("ps", "-aq"), ("rm", "-f", "-v")), + ("network", ("network", "ls", "-q"), ("network", "rm")), + ("volume", ("volume", "ls", "-q"), ("volume", "rm")), + ): + ids = docker(*listing, "--filter", selector).splitlines() + if ids: + docker(*removal, *ids) + if docker(*listing, "--filter", selector): + raise RuntimeError(f"model {kind} cleanup could not be verified") + if owned_volumes.intersection(docker("volume", "ls", "-q").splitlines()): + raise RuntimeError("model anonymous-volume cleanup could not be verified") + + +def evaluate( + records, output, *, llm_url, model, harness, expose, timeout_s, step_limit +): + """Run existing OpenEnv Harbor capture on the frozen approved corpus.""" + from openenv.harbor.runner import run_batch + from openenv.harbor.tasks import resolve_task_dirs + + if not records: + return [] + approved = output / "approved" + expected = {str(Path(row["execution_path"]).resolve()): row for row in records} + paths = resolve_task_dirs(str(approved)) + if {str(path.resolve()) for path in paths} != set(expected): + raise ValueError("OpenEnv loader and approved corpus disagree") + results = [] + # Different tasks can choose different headline reward keys. + for index, path in enumerate(paths): + record = expected[str(path.resolve())] + projects = set() + cleanup_ok = True + try: + batch = asyncio.run( + run_batch( + llm_url=llm_url, + model=model, + dataset=str(approved), + task_indices=[index], + harness=harness, + sandbox="docker", + expose=expose, + trials_dir=output / "model-trials", + reward_key=record["reward_key"], + agent_timeout_sec=timeout_s, + agent_step_limit=step_limit, + on_trial_created=model_guard(records, projects), + keep_sandbox=True, + environment_kwargs={ + "auto_mounting": False, + "keep_containers": True, + }, + ) + ) + if len(batch) != 1 or Path(batch[0].task_id).resolve() != path.resolve(): + raise ValueError("model result accounting is incomplete") + results.append( + { + "task_id": record["task_id"], + "admission": record, + "result": batch[0].model_dump(), + } + ) + except Exception as error: + results.append( + {"task_id": record["task_id"], "error": str(error), "ok": False} + ) + finally: + try: + cleanup_projects(projects) + except Exception as error: + cleanup_ok = False + results[-1]["cleanup_error"] = str(error) + write_rows(output / "rollouts.jsonl", results) + if not cleanup_ok: + break + return results + + +def local_tasks(root, limit): + root = root.resolve(strict=True) + manifests = [ + root / path / "task.toml" + for path in sources.task_roots( + manifest.relative_to(root).as_posix() + for manifest in root.rglob("task.toml") + ) + ] + return [ + { + "task_id": f"local:{root.name}@{source_digest(manifest.parent)}:{manifest.parent.relative_to(root)}", + "path": str(manifest.parent), + "source": "local", + "source_id": str(root), + "revision": source_digest(manifest.parent), + "task_path": manifest.parent.relative_to(root).as_posix(), + } + for manifest in manifests[:limit] + ] + + +def run(args): + if vars(args).get("submit_hf_jobs", False): + raise ValueError("Use prepared_batch with a reviewed shared reservation") + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + coding_config = getattr(args, "coding_config", None) + coding_config = loads(coding_config.read_text()) if coding_config else None + contract = loads(args.contract.read_text()) if coding_config is None else None + if contract is not None and contract.get("profile") != PILOT: + raise ValueError(f"contract must declare {PILOT}") + candidates, tasks, excluded, accepted = [], [], [], [] + plans = [] + explicit = bool(args.hf_dataset or args.harbor_dataset or args.local_dataset) + if not explicit: + for source, discover in ( + ("hf", sources.discover_hf), + ("harbor", sources.discover_harbor), + ): + if args.source in {source, "both"}: + try: + candidates.extend(discover(args.dataset_limit)) + except Exception as error: + excluded.append( + {"source": source, "stage": "discovery", "reason": str(error)} + ) + for spec in args.hf_dataset: + repo, separator, revision = spec.partition("@") + if not separator: + revision = sources.HfApi(token=False).dataset_info(repo).sha + candidates.append({"source": "hf", "id": repo, "revision": revision}) + for spec in args.harbor_dataset: + name, separator, revision = spec.partition("@") + if not separator: + raise ValueError("explicit Harbor datasets require org/name@sha256:digest") + candidates.append({"source": "harbor", "id": name, "revision": revision}) + write_rows(output / "candidates.jsonl", candidates) + for candidate in candidates: + try: + tasks.extend( + sources.acquire(candidate, output / "downloads", args.tasks_per_dataset) + ) + except Exception as error: + excluded.append( + {"candidate": candidate, "stage": "acquisition", "reason": str(error)} + ) + for root in args.local_dataset: + tasks.extend(local_tasks(root, args.tasks_per_dataset)) + write_rows(output / "tasks.jsonl", tasks) + for task in tasks: + key = hashlib.sha256(task["task_id"].encode()).hexdigest()[:24] + subject = output / "subjects" / key + try: + original_digest = source_digest(Path(task["path"])) + copy_subject(task["path"], subject, original_digest) + from openenv.harbor.tasks import resolve_task_dirs + + if subject not in resolve_task_dirs(str(subject.parent), refresh=True): + raise ValueError("OpenEnv did not load the acquired task") + if coding_config is not None: + from examples.hub_validation_pipeline.cloud import prepare + + plan_dir = output / "plans" / key + provenance = { + name: task[name] + for name in ( + "source", + "source_id", + "revision", + "task_path", + "task_id", + ) + } + if "task_file_hashes" in task: + provenance["task_file_hashes"] = task["task_file_hashes"] + plan = prepare(subject, coding_config, plan_dir, source=provenance) + row = { + "task_id": task["task_id"], + "run_id": plan["run_id"], + "plan": str(plan_dir / "plan.json"), + "state": "prepared", + "accepted": False, + } + plans.append(row) + write_rows(output / "planned.jsonl", plans) + continue + write_json( + subject / "validation" / "openenv.json", + { + **contract, + "original_source_digest": original_digest, + }, + ) + report_path = output / "reports" / f"{key}.json" + exit_code, report = validate_task( + subject, report_path, args.profile, args.validation_timeout + ) + envelope = admission( + report, + exit_code, + subject, + report_path.with_suffix(".artifacts"), + args.profile, + original_digest, + ) + accepted.append( + materialize( + {**task, **envelope, "subject": str(subject)}, + output / "approved" / "tasks" / key, + ) + ) + except Exception as error: + stage = ( + "cloud_submission" + if coding_config is not None and getattr(args, "submit_hf_jobs", False) + else "cloud_preparation" + if coding_config is not None + else "validation" + ) + excluded.append({**task, "stage": stage, "reason": str(error)}) + write_rows(output / "accepted.jsonl", accepted) + write_rows(output / "excluded.jsonl", excluded) + write_rows(output / "excluded.jsonl", excluded) + write_rows(output / "accepted.jsonl", accepted) + if coding_config is not None: + write_rows(output / "planned.jsonl", plans) + summary = { + "profile": "harbor-coding-audit-v1", + "phase": "submitted" + if getattr(args, "submit_hf_jobs", False) + else "prepared", + "candidates": len(candidates), + "tasks": len(tasks), + "plans": len(plans), + "excluded": len(excluded), + "accepted": 0, + "rfc008_complete": False, + "note": "Plans/submitted Jobs are not qualification results; inspect each immutable run report.", + "finished_at": time.time(), + } + write_json(output / "summary.json", summary) + print(dumps(summary)) + return 0 if plans else 1 + results = ( + evaluate( + accepted, + output, + llm_url=args.llm_url, + model=args.model, + harness=args.harness, + expose=args.expose, + timeout_s=args.model_timeout, + step_limit=args.model_step_limit, + ) + if args.llm_url and accepted + else [] + ) + summary = { + "profile": args.profile, + "rfc008_complete": args.profile == "full" and bool(accepted), + "candidates": len(candidates), + "tasks": len(tasks), + "accepted": len(accepted), + "excluded": len(excluded), + "model_trials": len(results), + "model_requested": bool(args.llm_url), + "finished_at": time.time(), + } + write_json(output / "summary.json", summary) + print(dumps(summary)) + model_ok = len(results) == len(accepted) and all( + row.get("result", {}).get("ok") is True and "cleanup_error" not in row + for row in results + ) + return 0 if accepted and (not args.llm_url or model_ok) else 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output", type=Path, required=True, help="New run directory; must not exist" + ) + parser.add_argument( + "--contract", + type=Path, + help="Trusted reward/resource contract for local native Harbor validation", + ) + parser.add_argument( + "--coding-config", + type=Path, + help="Prepare the supplemental coding audit on HF Jobs instead of local Docker validation", + ) + parser.add_argument("--profile", choices=["full", PILOT], default="full") + parser.add_argument("--source", choices=["hf", "harbor", "both"], default="both") + parser.add_argument("--dataset-limit", type=int, default=2) + parser.add_argument("--tasks-per-dataset", type=int, default=2) + parser.add_argument("--hf-dataset", action="append", default=[]) + parser.add_argument("--harbor-dataset", action="append", default=[]) + parser.add_argument("--local-dataset", action="append", type=Path, default=[]) + parser.add_argument("--validation-timeout", type=float, default=900) + parser.add_argument("--llm-url", default="") + parser.add_argument("--model", default="") + parser.add_argument("--harness", default="opencode") + parser.add_argument( + "--expose", choices=["direct", "gradio", "cloudflare"], default="gradio" + ) + parser.add_argument("--model-timeout", type=float, default=120) + parser.add_argument("--model-step-limit", type=int, default=32) + args = parser.parse_args() + if args.coding_config is None and args.contract is None: + parser.error("--contract or --coding-config is required") + if args.coding_config is not None and (args.contract is not None or args.llm_url): + parser.error( + "--coding-config selects its own contract/model and cannot combine --contract or --llm-url" + ) + if args.dataset_limit < 0 or args.tasks_per_dataset < 0: + parser.error("discovery/task limits must be nonnegative") + if ( + not all( + math.isfinite(value) and value > 0 + for value in (args.validation_timeout, args.model_timeout) + ) + or args.model_step_limit < 1 + ): + parser.error("timeouts and model step limit must be positive") + if args.llm_url and not args.model: + parser.error("--model is required with --llm-url") + raise SystemExit(run(args)) + + +if __name__ == "__main__": + main() diff --git a/examples/hub_validation_pipeline/prepared_batch.py b/examples/hub_validation_pipeline/prepared_batch.py new file mode 100644 index 000000000..b82d88d51 --- /dev/null +++ b/examples/hub_validation_pipeline/prepared_batch.py @@ -0,0 +1,277 @@ +"""Validate a frozen cohort offline; --submit reserves and launches it once. + +Only owned controllers are polled. Completed controllers remain unmeasured; +result collection and acceptance require separate evidence review. +""" + +import argparse +import hashlib +import re +import time +from fractions import Fraction +from pathlib import Path +from urllib.parse import urlsplit + +from examples.coding_audit.hf import _deadline, _DeadlineExceeded + +from . import cloud, full_run, spend + +STAGES = "SCHEDULING RUNNING COMPLETED ERROR CANCELED DELETED".split() + + +def _plan(root, task): + path = (root / cloud._safe_name(task["plan_path"])).resolve(strict=True) + if not path.is_relative_to(root) or not path.is_file(): + raise ValueError("Plan must resolve inside the preparation directory") + receipt = path.parent / "submission.json" + if receipt.exists() or receipt.is_symlink(): + raise FileExistsError("Submission receipt exists; manual review required") + return path, cloud._read_plan(path, expected_sha256=task["plan_sha256"]) + + +def _record(task, receipt): + for key in ("job_id", "job_url", "status"): + if isinstance(receipt.get(key), str): + task["submission_status" if key == "status" else key] = receipt[key] + + +def _status(task, spec, owner, deadline, api, secrets, clock): + if clock() >= deadline: + return "TIMEOUT" + try: + with _deadline(deadline - clock()): + job = api.inspect_job( + job_id=task["job_id"], + namespace=spec.get("namespace"), + token=secrets["HF_SANDBOX_TOKEN"], + ) + except _DeadlineExceeded: + return "TIMEOUT" + if ( + job.id != task["job_id"] + or job.docker_image != spec["image"] + or not owner + or job.owner.name != owner + ): + raise ValueError("Controller identity mismatch") + stage = job.status.stage + if stage in {"COMPLETED", "ERROR", "CANCELED", "DELETED"}: + return stage + if clock() >= deadline: + return "TIMEOUT" + return stage if stage in STAGES else "UNKNOWN" + + +def run( + prepared_manifest, + output, + spend_ledger, + reservation_id, + budget_usd, + *, + submit=False, + api=None, + secrets=None, + sleep=time.sleep, + clock=time.monotonic, +): + cap = full_run._budget(budget_usd) + spend._text(reservation_id) + if not str(spend_ledger or "").strip(): + raise ValueError("Supply a shared ledger and a reservation ID") + output = Path(output).absolute() + output.mkdir(parents=True, exist_ok=False) + source = Path(prepared_manifest).resolve(strict=True) + root, destination = source.parent, output / "manifest.json" + source_bytes = source.read_bytes() + binding = { + "path": str(source), + "sha256": hashlib.sha256(source_bytes).hexdigest(), + "bytes": len(source_bytes), + } + manifest = dict( + schema="openenv-prepared-batch-v1", + status="preparing", + stage="plans", + prepared_manifest=binding, + accepted=False, + results_collected=False, + budget_usd=budget_usd, + tasks=[], + ) + + def save(**state): + manifest.update(**state) + full_run._write(destination, manifest) + + save() + try: + rows = cloud._loads(source_bytes)["tasks"] + if not isinstance(rows, list) or not rows: + raise ValueError("A nonempty prepared cohort is required") + configs, task_ids, run_ids = [], set(), set() + for row in rows: + if not isinstance(row["task_id"], str) or not row["task_id"].strip(): + raise ValueError("Task IDs must be nonempty strings") + _, (plan, config, _, _) = _plan(root, row) + if row["task_id"] in task_ids or plan["run_id"] in run_ids: + raise ValueError("Task IDs, run IDs and plan paths must be distinct") + task_ids.add(row["task_id"]) + run_ids.add(plan["run_id"]) + configs.append(config) + task = {key: row[key] for key in ("task_id", "plan_path", "plan_sha256")} + task.update( + run_id=plan["run_id"], + bundle_sha256=plan["bundle_sha256"], + status="not_submitted", + accepted=False, + ) + manifest["tasks"].append(task) + manifest["stage"] = "budget" + manifest["estimate"] = full_run.estimate(configs) + amount = manifest["estimate"]["reservation_usd_rounded_up"] + manifest["within_budget"] = Fraction(amount) <= cap + fingerprint = hashlib.sha256( + "\0".join(task["bundle_sha256"] for task in manifest["tasks"]).encode() + ).hexdigest()[:16] + manifest["shared_spend"] = dict( + ledger=str(Path(spend_ledger).absolute()), + reservation_id=f"{reservation_id}:{fingerprint}", + reserved_usd=amount, + status="not_reserved", + ) + manifest["status"] = "prepared" + save() + if not submit: + save(stage="done") + return manifest + with _deadline(1): + pass + if not manifest["within_budget"]: + raise ValueError("Aggregate reservation exceeds budget") + if full_run._hash(source) != binding: + raise ValueError("Preparation manifest changed") + for task in manifest["tasks"]: + _plan(root, task) + manifest["stage"] = "reserve" + manifest["shared_spend"]["status"] = "pending_or_unknown" + save() + spend.reserve( + spend_ledger, + manifest["shared_spend"]["reservation_id"], + amount, + "Frozen prepared cohort", + evidence=str(destination), + plan_costs={ + task["plan_sha256"]: full_run.estimate([config])[ + "reservation_usd_exact" + ] + for task, config in zip(manifest["tasks"], configs, strict=True) + }, + ) + manifest["shared_spend"]["status"] = "reserved" + save() + secrets = cloud._runtime_secrets(secrets) + if api is None: + from huggingface_hub import HfApi + + api = HfApi(token=secrets["HF_TOKEN"]) + job_ids = set() + for offset in range(0, len(rows), 2): + active = [] + for task in manifest["tasks"][offset : offset + 2]: + if any(clock() >= entry[3] for entry in active): + raise TimeoutError("Controller deadline exceeded") + if full_run._hash(source) != binding: + raise ValueError("Preparation manifest changed") + path, (plan, config, _, _) = _plan(root, task) + spec = plan["mcp_request"]["args"] + task["status"] = "pending_or_unknown" + save(stage="submit", status="submitting") + deadline = ( + clock() + + config["cloud"]["timeout_seconds"] + + full_run.RATEBOOK["startup_allowance_seconds"] + ) + try: + receipt = cloud.submit( + path, + api=api, + secrets=secrets, + expected_plan_sha256=task["plan_sha256"], + spend_ledger=spend_ledger, + reservation_id=manifest["shared_spend"]["reservation_id"], + ) + except BaseException: + local = path.parent / "submission.json" + if local.is_file(): + _record(task, cloud._loads(local.read_bytes())) + raise + _record(task, receipt) + save() + if ( + receipt["status"] != "submitted" + or receipt["run_id"] != task["run_id"] + or not task.get("job_id") + ): + raise ValueError("Submission identity or outcome unknown") + if task["job_id"] in job_ids: + raise ValueError("Duplicate controller Job ID") + job_ids.add(task["job_id"]) + url = urlsplit(receipt["job_url"]) + parts = url.path.split("/") + if ( + url.scheme != "https" + or url.netloc != "huggingface.co" + or len(parts) != 4 + or parts[:2] != ["", "jobs"] + or parts[-1] != task["job_id"] + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,95}", parts[2]) + ): + raise ValueError("Controller owner URL is invalid") + owner = spec.get("namespace") or parts[2] + if owner != parts[2]: + raise ValueError( + "Controller owner differs from the launch namespace" + ) + task["status"] = "accepted_unmeasured" + active.append((task, spec, owner, deadline)) + save() + save(stage="poll", status="waiting") + while active: + for entry in active[:]: + task, spec, owner, deadline = entry + task["controller_status"] = "UNKNOWN" + stage = task["controller_status"] = _status( + task, spec, owner, deadline, api, secrets, clock + ) + save() + if stage == "COMPLETED": + active.remove(entry) + elif stage not in {"SCHEDULING", "RUNNING"}: + raise RuntimeError("Controller failed or status unknown") + if active: + sleep(max(0, min(30, min(entry[3] - clock() for entry in active)))) + manifest["status"] = "controllers_completed_unmeasured" + save(stage="done") + except BaseException as error: + save(status="failed", error_type=type(error).__name__) + raise RuntimeError( + f"Prepared batch stopped during {manifest['stage']}; inspect manifest.json and per-plan receipts" + ) from None + return manifest + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + names = "prepared-manifest output spend-ledger reservation-id budget-usd" + for name in names.split(): + parser.add_argument("--" + name, required=True) + parser.add_argument("--submit", action="store_true") + args = parser.parse_args(argv) + manifest = run(**vars(args)) + print(f"{manifest['status']}: {Path(args.output).absolute() / 'manifest.json'}") + + +if __name__ == "__main__": + main() diff --git a/examples/hub_validation_pipeline/real_tasks.py b/examples/hub_validation_pipeline/real_tasks.py new file mode 100644 index 000000000..1a7f2d2d9 --- /dev/null +++ b/examples/hub_validation_pipeline/real_tasks.py @@ -0,0 +1,317 @@ +"""Prepare operator-supplied evaluation-only native tasks without paid work. + +Run ``python -m examples.hub_validation_pipeline.real_tasks --panel PANEL --output DIR``. +The panel supplies pinned source identity, task boundaries and audit controls. +``--bundle`` also calls the local, network-free ``cloud.prepare`` preparer. +Only the selected immutable HF task trees are downloaded anonymously. +""" + +import argparse +import hashlib +import json +import re +import stat +import tempfile +import tomllib +from pathlib import Path, PurePosixPath + +from huggingface_hub import hf_hub_download, HfApi, RepoFile, RepoFolder +from openenv.validation.runner import source_digest + +from . import cloud, sources +from .sources import MAX_FILE_BYTES, MAX_FILES, MAX_TOTAL_BYTES + +_REQUIRED = {"task.toml", "instruction.md", "solution/solve.sh", "tests/test.sh"} + + +def _load_panel(panel_path): + """Validate one exact byte snapshot and return its provenance.""" + path = Path(panel_path).absolute() + data, _ = cloud._text_file(path) + panel = cloud._loads(data) + if ( + not isinstance(panel, dict) + or panel.get("schema") != "openenv-native-evaluation-panel-v1" + or panel.get("evaluation_only") is not True + ): + raise ValueError("Supply an evaluation-only native panel v1") + source = panel.get("source", {}) + if ( + not isinstance(source, dict) + or source.get("kind") != "hf" + or not isinstance(source.get("repo_id"), str) + or not re.fullmatch(r"[\w.-]+/[\w.-]+", source["repo_id"]) + ): + raise ValueError("Panel source must identify an HF dataset") + sources._pin(source.get("revision"), "hf") + tasks = panel.get("tasks") + if not isinstance(tasks, list) or not tasks: + raise ValueError("Panel requires a nonempty task list") + names, paths = set(), set() + for task in tasks: + if not isinstance(task, dict): + raise ValueError("Panel tasks must be objects") + name = cloud._safe_name(task.get("name", "")) + task_path = ( + PurePosixPath(".") + if task.get("task_path") == "." + else cloud._safe_name(task.get("task_path", "")) + ) + if ( + len(name.parts) != 1 + or name.as_posix().casefold() in names + or task_path.as_posix().casefold() in paths + or name.as_posix().lower().startswith(".env") + or any(part.lower().startswith(".env") for part in task_path.parts) + ): + raise ValueError("Panel requires distinct safe task names and paths") + names.add(name.as_posix().casefold()) + paths.add(task_path.as_posix().casefold()) + config, _ = cloud._config(task.get("config", {})) + execution = config["execution"] + if ( + execution["agent_image"] != task.get("image") + or execution["verifier_image"] != task.get("image") + or execution["submission_paths"] != [task.get("submission_path")] + ): + raise ValueError("Panel task and execution boundaries disagree") + if not isinstance(task.get("negative_counterexamples"), list) or not all( + isinstance(item, dict) for item in task["negative_counterexamples"] + ): + raise ValueError("Panel counterexamples must be a list of objects") + if not isinstance(task.get("annotations", {}), dict): + raise ValueError("Panel annotations must be an object") + task["config"] = config + return panel, { + "path": str(path), + "sha256": hashlib.sha256(data).hexdigest(), + "bytes": len(data), + } + + +def _select_tasks(panel, task_names): + tasks = {task["name"]: task for task in panel["tasks"]} + names = list(tasks if task_names is None else task_names) + if ( + not names + or len(set(names)) != len(names) + or any(name not in tasks for name in names) + ): + raise ValueError("Select distinct configured panel task names") + return [tasks[name] for name in names] + + +def acquire_task(task, destination, *, source, api=None, download=None): + """Acquire only one pinned native tree, with size and Git-blob checks.""" + revision = sources._pin(source["revision"], "hf") + repo_id, task_path = source["repo_id"], task["task_path"] + if task_path != ".": + cloud._safe_name(task_path) + destination = Path(destination) + if destination.exists() or destination.is_symlink(): + raise FileExistsError("Acquisition requires a fresh task directory") + destination = destination.absolute() + api = api if api is not None else HfApi(token=False) + download = download if download is not None else hf_hub_download + info = api.dataset_info(repo_id, revision=revision) + if info.sha != revision or info.gated or info.disabled: + raise ValueError("Public source immutable revision could not be verified") + files = {} + total = 0 + for item in api.list_repo_tree( + repo_id, + path_in_repo=None if task_path == "." else task_path, + repo_type="dataset", + revision=revision, + recursive=True, + token=False, + ): + path = cloud._safe_name(item.path) + if not path.is_relative_to(task_path): + raise ValueError("Source listing escaped the selected task tree") + if any(part.lower().startswith(".env") for part in path.parts): + raise ValueError("Environment files are not recipe inputs") + if isinstance(item, RepoFolder): + continue + if not isinstance(item, RepoFile) or item.lfs: + raise ValueError("Only native regular Git source files are supported") + relative = path.relative_to(task_path).as_posix() + if relative.casefold() in {key.casefold() for key in files}: + raise ValueError("Source path collision") + if type(item.size) is not int or not 0 <= item.size <= MAX_FILE_BYTES: + raise ValueError("Source per-file byte cap exceeded") + total += item.size + files[relative] = item + if len(files) > MAX_FILES or total > MAX_TOTAL_BYTES: + raise ValueError("Source file count or total byte cap exceeded") + if not _REQUIRED.issubset(files): + raise ValueError("Native task files are missing") + if sources.task_roots(files) != ["."]: + raise ValueError("Selected native task root is ambiguous") + inventory, payloads = {}, {} + with tempfile.TemporaryDirectory(prefix="openenv-real-task-") as temporary: + staging = Path(temporary).resolve() + for relative, item in sorted(files.items()): + path = Path( + download( + repo_id, + item.path, + repo_type="dataset", + revision=revision, + local_dir=staging, + token=False, + force_download=True, + ) + ) + if path.is_symlink() or path.resolve() != staging / item.path: + raise ValueError("Download must materialize a contained regular file") + if ( + not stat.S_ISREG(path.lstat().st_mode) + or path.stat().st_size != item.size + ): + raise ValueError("Source file type or size disagrees with pinned tree") + data, _ = cloud._text_file(path) + blob = hashlib.sha1(f"blob {len(data)}\0".encode() + data).hexdigest() + if blob != item.blob_id: + raise ValueError("Source Git blob hash mismatch") + inventory[relative] = { + "size": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + "git_blob": blob, + } + payloads[relative] = data + destination.mkdir(parents=True) + for relative, data in payloads.items(): + path = destination / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return inventory + + +def prepare( + output_dir, + *, + panel_path, + task_names=None, + bundle=False, + api=None, + download=None, +): + """Load an explicit operator panel and prepare recipes, never Jobs.""" + panel, provenance = _load_panel(panel_path) + return _prepare( + output_dir, + panel=panel, + panel_provenance=provenance, + task_names=task_names, + bundle=bundle, + api=api, + download=download, + ) + + +def _prepare( + output_dir, + *, + panel, + panel_provenance, + task_names=None, + bundle=False, + api=None, + download=None, +): + tasks = _select_tasks(panel, task_names) + output = Path(output_dir).absolute() + output.mkdir(parents=True, exist_ok=False) + rows = [] + for task in tasks: + name, config = task["name"], task["config"] + task_root = output / "sources" / name + inventory = acquire_task( + task, task_root, source=panel["source"], api=api, download=download + ) + config_path = output / "configs" / f"{name}.json" + config_path.parent.mkdir(exist_ok=True) + config_bytes = cloud._json_bytes(config) + config_path.write_bytes(config_bytes) + native = tomllib.loads((task_root / "task.toml").read_text()) + source = { + **panel["source"], + "task_path": task["task_path"], + "evaluation_panel_sha256": panel_provenance["sha256"], + "task_file_hashes": { + path: record["sha256"] for path, record in inventory.items() + }, + } + row = { + "name": name, + "task_id": f"hf:{source['repo_id']}@{source['revision']}:{source['task_path']}", + "task_root": str(task_root), + "config_path": str(config_path), + "config_sha256": hashlib.sha256(config_bytes).hexdigest(), + "source": source, + "source_digest": source_digest(task_root), + "files": inventory, + "evaluation_only": True, + "negative_counterexamples": task["negative_counterexamples"], + "annotations": task.get("annotations", {}), + "source_image": native.get("environment", {}).get("docker_image"), + "execution_image": task["image"], + "budget": cloud._config(config)[1], + } + if bundle: + directory = output / "bundles" / name + cloud.prepare(task_root, config, directory, source=source) + plan_path = directory / "plan.json" + row.update( + bundle_plan=str(plan_path), + plan_path=plan_path.relative_to(output).as_posix(), + plan_sha256=hashlib.sha256(plan_path.read_bytes()).hexdigest(), + ) + rows.append(row) + manifest = { + "schema": "openenv-real-task-recipes-v1", + "status": "prepared", + "evaluation_only": True, + "evaluation_panel": panel_provenance, + "scope": "Operator-selected tasks only; no whole-catalog qualification or admission claim.", + "limitations": [ + "No Jobs, inference, containers, builds or pushes are performed by this recipe.", + "Hosted model source revision does not attest provider-served weights.", + "Native task and verifier bytes are preserved; dependency and resource reproducibility require separate qualification.", + ], + "tasks": rows, + } + (output / "manifest.json").write_bytes(cloud._json_bytes(manifest)) + return manifest + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--panel", dest="panel_path", type=Path, required=True) + parser.add_argument("--task", action="append", dest="task_names") + parser.add_argument( + "--bundle", + action="store_true", + help="Also prepare local bundles; does not submit", + ) + args = parser.parse_args(argv) + prepare( + args.output, + panel_path=args.panel_path, + task_names=args.task_names, + bundle=args.bundle, + ) + print( + json.dumps( + { + "manifest": str(args.output.absolute() / "manifest.json"), + "status": "prepared", + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/hub_validation_pipeline/sources.py b/examples/hub_validation_pipeline/sources.py new file mode 100644 index 000000000..31be97c0d --- /dev/null +++ b/examples/hub_validation_pipeline/sources.py @@ -0,0 +1,477 @@ +"""Discover public Hub bundles and acquire pinned task files without executing them.""" + +import asyncio +import gzip +import hashlib +import io +import json +import re +import stat +import tarfile +import tempfile +from importlib.metadata import distribution, PackageNotFoundError +from pathlib import Path, PurePosixPath + +from huggingface_hub import HfApi, RepoFile, RepoFolder, snapshot_download + +MAX_FILES = 128 +MAX_FILE_BYTES = 2 * 1024 * 1024 +MAX_TOTAL_BYTES = 8 * 1024 * 1024 + +HARBOR_GIT_REVISION = "9c7a39b991d7e2205019f6be0598cbd22d84bfc3" +HARBOR_INSTALL = ( + "uv pip install --python .venv/bin/python " + f"'harbor @ git+https://github.com/harbor-framework/harbor.git@{HARBOR_GIT_REVISION}'" +) + + +def discover_hf(limit: int) -> list[dict]: + """List the exact Harbor-library AND RL-environment view, in trending order.""" + if limit < 0: + raise ValueError("limit must be nonnegative") + if limit == 0: + return [] + required = {"library:harbor", "rl-environment"} + found = {} + # Exhaust the SDK iterator: its limit is a total cap, not a page size. + for rank, item in enumerate( + HfApi(token=False).list_datasets( + filter=("library:harbor", "rl-environment"), + sort="trending_score", + expand=["sha", "tags", "gated", "disabled"], + limit=None, + ) + ): + if not required.issubset(item.tags or []) or item.id in found: + continue + found[item.id] = { + "source": "hf", + "id": item.id, + "revision": _pin(item.sha, "hf"), + "metadata": { + "tags": item.tags, + "gated": item.gated, + "disabled": item.disabled, + "rank": rank, + }, + } + return list(found.values())[:limit] + + +def _harbor_api(): + # The PyPI 0.25.0 wheel lacks listing; the official Git pin provides it. + try: + installed = distribution("harbor") + origin = json.loads(installed.read_text("direct_url.json") or "{}") + if ( + installed.version != "0.25.0" + or origin.get("vcs_info", {}).get("commit_id") != HARBOR_GIT_REVISION + ): + raise RuntimeError( + f"Harbor source capability mismatch (installed {installed.version}); " + f"native Hub listing requires official Git {HARBOR_GIT_REVISION}. " + f"Install: {HARBOR_INSTALL}" + ) + from harbor.db.client import RegistryDB + from harbor.models.task.id import PackageTaskId + from harbor.registry.client.package import PackageDatasetClient + except (ImportError, PackageNotFoundError) as error: + raise RuntimeError( + f"Pinned Harbor Hub package APIs unavailable. Install: {HARBOR_INSTALL}" + ) from error + return PackageDatasetClient, PackageTaskId, RegistryDB + + +def discover_harbor(limit: int) -> list[dict]: + """List current Harbor Hub dataset versions, retaining their package digests.""" + if limit < 0: + raise ValueError("limit must be nonnegative") + if limit == 0: + return [] + registry, _, _ = _harbor_api() + # This Git-pinned API exhausts pages, as does `harbor dataset list --json`. + try: + rows = asyncio.run(registry().list_datasets()) + except NotImplementedError as error: + raise RuntimeError( + f"Harbor native Hub listing is unsupported. Install: {HARBOR_INSTALL}" + ) from error + found = {} + for row in sorted(rows, key=lambda row: (row.name, row.version or "")): + revision = _pin(row.version, "harbor") + _package_name(row.name) + found.setdefault( + (row.name, revision), + { + "source": "harbor", + "id": row.name, + "revision": revision, + "metadata": { + "kind": "dataset", + "description": row.description, + "task_count": row.task_count, + "harbor_source_revision": HARBOR_GIT_REVISION, + }, + }, + ) + return list(found.values())[:limit] + + +def acquire(candidate: dict, cache_dir: Path, max_tasks: int) -> list[dict]: + """Download at most max_tasks into a fresh revision-addressed directory. + + Errors propagate to the driver for exclusion. Zero selects no tasks. Harbor + single-task candidates use metadata.kind="task"; discovery returns datasets. + """ + if max_tasks < 0: + raise ValueError("max_tasks must be nonnegative") + if max_tasks == 0: + return [] + source = candidate["source"] + revision = _pin(candidate["revision"], source) + if source == "hf": + return _acquire_hf(candidate, cache_dir, revision, max_tasks) + if source == "harbor": + return asyncio.run(_acquire_harbor(candidate, cache_dir, revision, max_tasks)) + raise ValueError(f"Unknown source: {source}") + + +def _pin(value: str, source: str) -> str: + pattern = {"hf": r"[0-9a-f]{40}", "harbor": r"sha256:[0-9a-f]{64}"} + if source not in pattern or not isinstance(value, str): + raise ValueError(f"Invalid source or immutable revision: {source}, {value}") + if not re.fullmatch(pattern[source], value): + raise ValueError(f"Expected immutable {source} revision, got {value!r}") + return value + + +def _relative(value: str) -> PurePosixPath: + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts or "\\" in value or not path.parts: + raise ValueError(f"Unsafe source path: {value!r}") + return path + + +def _package_name(name: str) -> None: + path = _relative(name) + if len(path.parts) != 2 or path.as_posix() != name or re.search(r"[@:\s]", name): + raise ValueError(f"Expected Harbor org/name, got {name!r}") + + +def _fresh(cache_dir: Path, source: str, identity: str, revision: str) -> Path: + identity_hash = hashlib.sha256(identity.encode()).hexdigest()[:16] + parent = cache_dir / source / identity_hash / revision.replace(":", "-") + parent.mkdir(parents=True, exist_ok=True) + return Path(tempfile.mkdtemp(prefix="selection-", dir=parent)).resolve() + + +def _record(source, identity, revision, root, task_path, metadata, **extra): + return { + "task_id": f"{source}:{identity}@{revision}:{task_path}", + "path": str((root / task_path).resolve(strict=True)), + "source": source, + "source_id": identity, + "revision": revision, + "task_path": task_path, + "acquisition_ref": f"{source}:{identity}@{revision}", + "metadata": metadata, + **extra, + } + + +def task_roots(files): + """A nested task.toml inside an existing task is an asset, not another task.""" + roots = { + PurePosixPath(name).parent + for name in files + if PurePosixPath(name).name == "task.toml" + } + return sorted( + str(root) + for root in roots + if not any(parent in roots for parent in root.parents) + ) + + +def _acquire_hf(candidate, cache_dir, revision, max_tasks): + metadata = candidate.get("metadata", {}) + if metadata.get("gated") or metadata.get("disabled"): + raise PermissionError(f"HF source is gated or disabled: {candidate['id']}") + api = HfApi(token=False) + info = api.dataset_info(candidate["id"], revision=revision) + if info.sha != revision: + raise ValueError("HF revision resolution disagrees with the selected commit") + if info.gated or info.disabled: + raise PermissionError(f"HF source is gated or disabled: {candidate['id']}") + files = {} + for item in api.list_repo_tree( + candidate["id"], + repo_type="dataset", + revision=revision, + recursive=True, + token=False, + ): + name = _relative(item.path).as_posix() + if isinstance(item, RepoFolder): + continue + if not isinstance(item, RepoFile) or name in files: + raise ValueError("HF source requires distinct regular files") + files[name] = item + roots = set(task_roots(files)) + selected = sorted(roots)[:max_tasks] + if not selected: + raise ValueError(f"No task.toml files in {candidate['id']}@{revision}") + # Retain shared repository assets while excluding every unselected task tree. + selected_files = [] + for name in sorted(files): + owner = next( + ( + str(parent) + for parent in PurePosixPath(name).parents + if str(parent) in roots + ), + None, + ) + if owner is None or owner in selected: + selected_files.append(name) + # Literal filename patterns prevent [brackets], ? and * from widening a subset. + patterns = [ + "".join({"[": "[[]", "?": "[?]", "*": "[*]"}.get(char, char) for char in name) + for name in selected_files + ] + _check_sizes([files[name].size for name in selected_files]) + if any(files[name].lfs for name in selected_files): + raise ValueError("Only native regular Git source files are supported") + hashes = {} + payloads = {} + with tempfile.TemporaryDirectory(prefix="openenv-hf-acquisition-") as temporary: + staging = Path(temporary).resolve() + snapshot_download( + candidate["id"], + repo_type="dataset", + revision=revision, + local_dir=staging, + allow_patterns=patterns, + token=False, + force_download=True, + ) + for name in selected_files: + path = staging / name + if path.resolve() != path or not stat.S_ISREG(path.lstat().st_mode): + raise ValueError( + "HF download must materialize a contained regular file" + ) + if path.stat().st_size != files[name].size: + raise ValueError("HF source size disagrees with pinned tree") + with path.open("rb") as stream: + data = stream.read(MAX_FILE_BYTES + 1) + if len(data) != files[name].size: + raise ValueError("HF source byte cap or size mismatch") + blob = hashlib.sha1(f"blob {len(data)}\0".encode() + data).hexdigest() + if blob != files[name].blob_id: + raise ValueError("HF source Git blob hash mismatch") + payloads[name] = data + hashes[name] = hashlib.sha256(data).hexdigest() + root = _fresh(cache_dir, "hf", candidate["id"], revision) + for name, data in payloads.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return [ + _record( + "hf", + candidate["id"], + revision, + root, + task_path, + metadata, + download_patterns=patterns, + task_file_hashes={ + PurePosixPath(name).relative_to(task_path).as_posix(): digest + for name, digest in hashes.items() + if PurePosixPath(name).is_relative_to(task_path) + }, + ) + for task_path in selected + ] + + +def _check_sizes(sizes): + if ( + len(sizes) > MAX_FILES + or any( + type(size) is not int or not 0 <= size <= MAX_FILE_BYTES for size in sizes + ) + or sum(sizes) > MAX_TOTAL_BYTES + ): + raise ValueError("Source file count or byte cap exceeded") + + +async def _download_harbor_file(remote_path, destination, byte_limit): + import httpx + from harbor.auth.client import create_authenticated_client + from harbor.storage.supabase import BUCKET + + client = await create_authenticated_client() + signed = await client.storage.from_(BUCKET).create_signed_url(remote_path, 300) + async with httpx.AsyncClient(timeout=120, follow_redirects=False) as http: + async with http.stream("GET", signed["signedURL"]) as response: + response.raise_for_status() + if int(response.headers.get("content-length", "0")) > byte_limit: + raise ValueError("Harbor download byte cap exceeded") + total = 0 + with destination.open("xb") as stream: + async for chunk in response.aiter_bytes(chunk_size=65536): + total += len(chunk) + if total > byte_limit: + raise ValueError("Harbor download byte cap exceeded") + stream.write(chunk) + return total + + +def _harbor_inventory(rows): + files = {} + for row in rows: + name = _relative(row["path"]).as_posix() + digest = row["content_hash"].removeprefix("sha256:") + if name.casefold() in {key.casefold() for key in files} or not re.fullmatch( + r"[0-9a-f]{64}", digest + ): + raise ValueError("Invalid Harbor file inventory") + files[name] = dict(row, sha256=digest) + _check_sizes([row["size_bytes"] for row in files.values()]) + return files + + +def _read_harbor_archive(archive_path, inventory): + payloads = {} + names = set() + with gzip.open(archive_path, "rb") as stream: + data = stream.read(MAX_TOTAL_BYTES + 1) + if len(data) > MAX_TOTAL_BYTES: + raise ValueError("Harbor archive expanded byte cap exceeded") + with tarfile.open(fileobj=io.BytesIO(data), mode="r:") as archive: + for member in archive: + name = _relative(member.name).as_posix() + if name in names or len(names) >= MAX_FILES: + raise ValueError( + "Harbor archive path collision or file count cap exceeded" + ) + names.add(name) + if member.isdir(): + continue + if not member.isfile() or member.issparse() or name not in inventory: + raise ValueError("Harbor archive must match its regular file inventory") + row = inventory[name] + if member.size != row["size_bytes"]: + raise ValueError("Harbor archive byte cap or size mismatch") + with archive.extractfile(member) as stream: + data = stream.read(MAX_FILE_BYTES + 1) + if ( + len(data) != member.size + or hashlib.sha256(data).hexdigest() != row["sha256"] + ): + raise ValueError("Harbor archive file hash or size mismatch") + payloads[name] = (data, member.mode & 0o777) + if set(payloads) != set(inventory): + raise ValueError("Harbor archive file inventory is incomplete") + return payloads + + +async def _acquire_harbor(candidate, cache_dir, revision, max_tasks): + registry_type, task_id_type, db_type = _harbor_api() + _package_name(candidate["id"]) + registry = registry_type() + metadata = candidate.get("metadata", {}) + kind = metadata.get("kind", "dataset") + bundle = None + if kind == "dataset": + bundle = await registry.get_dataset_metadata(f"{candidate['id']}@{revision}") + if bundle.version != revision: + raise ValueError("Harbor dataset resolution disagrees with its digest") + tasks = sorted( + bundle.task_ids, key=lambda task: (task.org, task.name, task.ref) + )[:max_tasks] + elif kind == "task": + org, name = candidate["id"].split("/") + tasks = [task_id_type(org=org, name=name, ref=revision)] + else: + raise ValueError(f"Unknown Harbor package kind: {kind}") + if not tasks: + raise ValueError(f"Harbor dataset contains no tasks: {candidate['id']}") + for task in tasks: + if not isinstance(task, task_id_type): + raise ValueError("Current Harbor Hub acquisition requires package task IDs") + _package_name(f"{task.org}/{task.name}") + _pin(task.ref, "harbor") + db = db_type() + shared = {} + if bundle is not None and bundle.files: + shared = _harbor_inventory( + await db.get_dataset_version_files(bundle.dataset_version_id) + ) + if set(shared) != {file.path for file in bundle.files}: + raise ValueError("Harbor shared file inventory is incomplete") + packages = [] + sizes = [row["size_bytes"] for row in shared.values()] + for task in tasks: + resolved = await db.resolve_task_version(task.org, task.name, task.ref) + if resolved.content_hash != task.ref.removeprefix("sha256:"): + raise ValueError("Harbor task resolution disagrees with its digest") + inventory = _harbor_inventory(await db.get_task_version_files(resolved.id)) + if "task.toml" not in inventory: + raise ValueError("Harbor package contains no root task.toml") + sizes.extend(row["size_bytes"] for row in inventory.values()) + _check_sizes(sizes) + packages.append((task, resolved, inventory)) + root = _fresh(cache_dir, "harbor", candidate["id"], revision) + dataset_files, records = {}, [] + remaining = MAX_TOTAL_BYTES + with tempfile.TemporaryDirectory(prefix="openenv-harbor-acquisition-") as temporary: + staging = Path(temporary) + for index, (name, row) in enumerate(shared.items()): + path = staging / f"shared-{index}" + remaining -= await _download_harbor_file( + row["storage_path"], path, row["size_bytes"] + ) + data = path.read_bytes() + if ( + len(data) != row["size_bytes"] + or hashlib.sha256(data).hexdigest() != row["sha256"] + ): + raise ValueError("Harbor shared file hash or size mismatch") + destination = root / "dataset" / name + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(data) + dataset_files[name] = str(destination) + for index, (task, resolved, inventory) in enumerate(packages): + archive = staging / f"task-{index}.tar.gz" + remaining -= await _download_harbor_file( + resolved.archive_path, archive, remaining + ) + payloads = _read_harbor_archive(archive, inventory) + task_root = root / "tasks" / task.org / task.name / resolved.content_hash + for name, (data, mode) in payloads.items(): + path = task_root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + path.chmod(mode) + records.append( + _record( + "harbor", + f"{task.org}/{task.name}", + task.ref, + task_root, + ".", + { + **metadata, + "candidate_id": candidate["id"], + "candidate_revision": revision, + "dataset_files": dataset_files, + }, + task_file_hashes={ + name: row["sha256"] for name, row in inventory.items() + }, + ) + ) + return records diff --git a/examples/hub_validation_pipeline/spend.py b/examples/hub_validation_pipeline/spend.py new file mode 100644 index 000000000..c0135eb85 --- /dev/null +++ b/examples/hub_validation_pipeline/spend.py @@ -0,0 +1,303 @@ +"""Offline shared spend ledger; amounts are decimal strings, Decimal, or integers.""" + +import argparse +import fcntl +import json +import os +import re +import tempfile +from contextlib import contextmanager +from copy import deepcopy +from datetime import datetime, timezone +from decimal import Decimal +from fractions import Fraction +from pathlib import Path + + +def _amount(value): + if type(value) not in (str, Decimal, int) or not re.fullmatch( + r"\+?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?", str(value) + ): + raise ValueError("Amount must be a nonnegative finite decimal") + return Fraction(Decimal(value)) + + +def _text(value): + if not isinstance(value, str) or not value.strip(): + raise ValueError("ID, purpose and evidence must be nonempty strings") + return value + + +def _pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("Duplicate ledger key") + result[key] = value + return result + + +def _report(data): + if not isinstance(data, dict) or data.get("schema") != "openenv-shared-spend-v1": + raise ValueError("Invalid ledger schema") + limit, reserved, spent = _amount(data["limit_usd"]), Fraction(0), Fraction(0) + changes = data.get("limit_changes", []) + if not isinstance(changes, list): + raise ValueError("Invalid limit change history") + previous = None + for change in changes: + if ( + not isinstance(change, dict) + or not {"previous_limit_usd", "limit_usd", "evidence", "changed_utc"} + <= change.keys() + ): + raise ValueError("Invalid limit change history") + old, new = _amount(change["previous_limit_usd"]), _amount(change["limit_usd"]) + _text(change["evidence"]) + changed = datetime.fromisoformat(_text(change["changed_utc"])) + if ( + changed.utcoffset() != timezone.utc.utcoffset(None) + or old == new + or (previous is not None and previous != old) + ): + raise ValueError("Invalid limit change history") + previous = new + if changes and previous != limit: + raise ValueError("Limit change history differs from current limit") + over = data.get("over_budget", False) + statuses = {} + for identifier, row in data["reservations"].items(): + amount = _amount(row["reserved_usd"]) + costs = row.get("plan_costs", {}) + if ( + not isinstance(costs, dict) + or any( + not re.fullmatch(r"[0-9a-f]{64}", key) + or not isinstance(value, str) + or Fraction(value) < 0 + for key, value in costs.items() + ) + or sum((Fraction(value) for value in costs.values()), Fraction(0)) > amount + ): + raise ValueError("Invalid reserved plan costs") + statuses[identifier] = row["status"] + if row["status"] == "reserved" and row["spent_usd"] is None: + reserved += amount + elif row["status"] == "settled": + _text(row.get("evidence")) + actual = _amount(row["spent_usd"]) + spent += actual + over = over or actual > amount + else: + raise ValueError("Invalid reservation status") + totals = dict(limit=limit, reserved=reserved, spent=spent, charged=reserved + spent) + totals["remaining"] = limit - totals["charged"] + result = { + "over_budget": bool(over or totals["charged"] > limit), + "statuses": statuses, + } + for key, value in totals.items(): + micros = -((-value.numerator * 1000000) // value.denominator) + sign, whole = ("-" if micros < 0 else ""), abs(micros) + result[f"{key}_usd_exact"] = str(value) + result[f"{key}_usd_rounded_up"] = ( + f"{sign}{whole // 1000000}.{whole % 1000000:06d}" + ) + return result + + +@contextmanager +def _ledger(path, limit=None): + path = Path(path).resolve() + with path.with_name(path.name + ".lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + source = None + if path.exists(): + source = path.read_bytes().decode("utf-8") + data = json.loads(source, object_pairs_hook=_pairs) + elif limit is not None: + data = dict( + schema="openenv-shared-spend-v1", + limit_usd=str(limit), + reservations={}, + created_utc=datetime.now(timezone.utc).isoformat(), + ) + else: + raise ValueError("Ledger must be initialized first") + _report(data) + before = deepcopy(data) + yield data + if data != before or not path.exists(): + data["updated_utc"] = datetime.now(timezone.utc).isoformat() + with tempfile.NamedTemporaryFile( + mode="w", dir=path.parent, delete=False + ) as out: + temporary = Path(out.name) + try: + rendered = json.dumps(data, indent=2, allow_nan=False) + if ( + source is not None + and data["reservations"] == before["reservations"] + ): + # Decode top-level values to retain the original reservation bytes. + decoder = json.JSONDecoder() + index = source.index("{") + 1 + for _ in before: + index += re.match(r"\s*", source[index:]).end() + key, index = decoder.raw_decode(source, index) + index = source.index(":", index) + 1 + index += re.match(r"\s*", source[index:]).end() + _, end = decoder.raw_decode(source, index) + if key == "reservations": + field = json.dumps( + {"reservations": data["reservations"]}, indent=2 + )[1:-2] + rendered = rendered.replace( + field, '\n "reservations": ' + source[index:end], 1 + ) + break + index = source.index(",", end) + 1 + out.write(rendered) + out.write("\n") + out.flush() + os.fsync(out.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + # A prior replace may have succeeded before directory syncing failed. + # Idempotent retries must confirm that durability before acknowledging. + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def init(path, limit_usd): + """Initialize once; an existing ledger must have the same limit.""" + limit = _amount(limit_usd) + with _ledger(path, str(limit_usd)) as data: + if _amount(data["limit_usd"]) != limit: + raise ValueError("Existing ledger limit differs") + return _report(data) + + +def set_limit(path, limit_usd, evidence): + """Record an evidenced limit decision, retaining all holds and overrun blocks.""" + limit = _amount(limit_usd) + _text(evidence) + with _ledger(path) as data: + report = _report(data) + if _amount(data["limit_usd"]) == limit: + return report + data.setdefault("limit_changes", []).append( + dict( + previous_limit_usd=data["limit_usd"], + limit_usd=str(limit_usd), + evidence=evidence, + changed_utc=datetime.now(timezone.utc).isoformat(), + ) + ) + data["limit_usd"] = str(limit_usd) + if report["over_budget"] or _report(data)["over_budget"]: + data["over_budget"] = True + return _report(data) + + +def reserve(path, id, amount_usd, purpose, evidence=None, *, plan_costs=None): + """Reserve before paid work; settled IDs cannot be reused.""" + amount = _amount(amount_usd) + _text(id), _text(purpose) + if evidence is not None: + _text(evidence) + with _ledger(path) as data: + report = _report(data) + rows = data["reservations"] + if id in rows: + if ( + rows[id]["status"] == "reserved" + and _amount(rows[id]["reserved_usd"]) == amount + and rows[id].get("plan_costs") == plan_costs + ): + return report + raise ValueError("Reservation ID already used; use a fresh ID") + if report["over_budget"] or amount > Fraction(report["remaining_usd_exact"]): + raise ValueError("Shared budget exceeded or blocked") + rows[id] = dict( + reserved_usd=str(amount_usd), + spent_usd=None, + status="reserved", + purpose=purpose, + evidence=evidence, + ) + if plan_costs is not None: + rows[id]["plan_costs"] = plan_costs + return _report(data) + + +def settle(path, id, spent_usd, evidence): + """Record actual cost, freezing new reservations on any reservation/cap overrun.""" + actual = _amount(spent_usd) + _text(id), _text(evidence) + with _ledger(path) as data: + rows = data["reservations"] + if id not in rows: + raise ValueError("Unknown reservation ID") + row = rows[id] + if row["status"] == "settled": + if _amount(row["spent_usd"]) == actual: + return _report(data) + raise ValueError("Different settlement requires manual reconciliation") + row.update( + status="settled", + spent_usd=str(spent_usd), + evidence=evidence, + settled_utc=datetime.now(timezone.utc).isoformat(), + ) + report = _report(data) + data["over_budget"] = report["over_budget"] + return report + + +def summary(path): + """Return exact fraction totals, upward-rounded six-place displays and ID statuses.""" + with _ledger(path) as data: + return _report(data) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ledger", required=True) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("summary") + sub = commands.add_parser("set-limit") + sub.add_argument("--usd", required=True) + sub.add_argument("--evidence", required=True) + for command in ("reserve", "settle"): + sub = commands.add_parser(command) + sub.add_argument("--id", required=True) + sub.add_argument("--usd", required=True) + sub.add_argument("--evidence", required=command == "settle") + if command == "reserve": + sub.add_argument("--purpose", required=True) + args = parser.parse_args(argv) + try: + if args.command == "set-limit": + result = set_limit(args.ledger, args.usd, args.evidence) + elif args.command == "reserve": + result = reserve( + args.ledger, args.id, args.usd, args.purpose, args.evidence + ) + elif args.command == "settle": + result = settle(args.ledger, args.id, args.usd, args.evidence) + else: + result = summary(args.ledger) + print(json.dumps(result, sort_keys=True)) + except (ValueError, TypeError, KeyError, AttributeError, OSError): + parser.exit( + 2, "Spend operation rejected; inspect ledger and arguments locally.\n" + ) + + +if __name__ == "__main__": + main() diff --git a/examples/hub_validation_pipeline/test_cloud.py b/examples/hub_validation_pipeline/test_cloud.py new file mode 100644 index 000000000..f7dd93fd6 --- /dev/null +++ b/examples/hub_validation_pipeline/test_cloud.py @@ -0,0 +1,1118 @@ +"""Offline cloud planning, input integrity, one-shot launch and failure evidence.""" + +import hashlib +import io +import json +import shutil +import stat +import subprocess +import sys +import tempfile +import urllib.request +import zipfile +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from examples.hub_validation_pipeline import cloud, full_run, spend + +SHA = "a" * 40 +UPLOAD_SHA = "b" * 40 +SECRETS = { + "HF_TOKEN": "results-write-credential-123", + "HF_INFERENCE_TOKEN": "inference-only-credential-456", + "HF_SANDBOX_TOKEN": "execution-only-credential-789", +} + + +@pytest.fixture +def inputs(tmp_path, monkeypatch): + repo = tmp_path / "repo" + (repo / "src/openenv").mkdir(parents=True) + (repo / "src/openenv/__init__.py").write_text("# source\n") + (repo / "examples/hub_validation_pipeline").mkdir(parents=True) + (repo / "examples/hub_validation_pipeline/cloud.py").write_bytes( + Path(cloud.__file__).read_bytes() + ) + shutil.copytree( + Path(cloud.__file__).parents[1] / "coding_audit", + repo / "examples/coding_audit", + ignore=shutil.ignore_patterns("test_*", "__pycache__", ".pytest_cache"), + ) + (repo / "examples/coding_audit/test_excluded.py").write_text( + "raise AssertionError\n" + ) + for name in ("pyproject.toml", "README.md", "LICENSE", "uv.lock"): + (repo / name).write_text(name + "\n") + task = tmp_path / "task" + (task / "tests").mkdir(parents=True) + (task / "solution").mkdir() + (task / "task.toml").write_text("version = '1.0'\n") + (task / "instruction.md").write_text("Write an answer\n") + (task / "tests/test.sh").write_text("printf 1\n") + (task / "solution/solve.sh").write_text("printf answer > /app/answer\n") + (task / "solution/solve.sh").chmod(0o755) + config = { + "schema_version": 1, + "target": { + "model_id": full_run.MODEL_SOURCE[0], + "revision": full_run.MODEL_SOURCE[1], + "provider": full_run.MODEL_SOURCE[2], + "max_steps": 3, + }, + "execution": { + "flavor": "cpu-basic", + "agent_image": "registry/agent@sha256:" + "c" * 64, + "verifier_image": "registry/verifier@sha256:" + "d" * 64, + "submission_paths": ["/app/answer"], + }, + "validation": { + "reference_repeats": 2, + "target_attempts": 2, + "required_negative_cases": ["wrong", "empty"], + }, + "generation": {"candidate_count": 2}, + "negative_cases": [ + {"id": "wrong", "script": "printf wrong > /app/answer"}, + {"id": "empty", "script": "true"}, + ], + "seeds": [7, 13], + "cloud": { + "results_repo": "org/private-results", + "namespace": "org", + "flavor": "cpu-basic", + "timeout_seconds": 1800, + "max_sandboxes": 40, + }, + } + monkeypatch.setattr( + cloud, "_git_provenance", lambda root: {"commit": SHA, "dirty": True} + ) + return repo, task, config + + +def prepared(inputs, tmp_path, name="prepared"): + repo, task, config = inputs + output = tmp_path / name + plan = cloud.prepare(task, config, output, repo_root=repo) + return output, plan + + +def launch_options(output, config): + ledger = output.parent / "spend.json" + spend.init(ledger, "1000") + digest = hashlib.sha256((output / "plan.json").read_bytes()).hexdigest() + cost = full_run.estimate([config]) + spend.reserve( + ledger, + output.name, + cost["reservation_usd_rounded_up"], + "Reviewed test plan", + plan_costs={digest: cost["reservation_usd_exact"]}, + ) + return dict( + spend_ledger=ledger, reservation_id=output.name, expected_plan_sha256=digest + ) + + +def fake_api(): + api = Mock() + api.dataset_info.return_value = SimpleNamespace(private=True, sha=SHA) + api.list_repo_files.return_value = ["corpus/task.toml"] + api.create_commit.return_value = SimpleNamespace(oid=UPLOAD_SHA) + api.run_job.return_value = SimpleNamespace( + id="job-123", url="https://huggingface.co/jobs/org/job-123" + ) + return api + + +def worker_root(inputs, tmp_path, monkeypatch): + output, plan = prepared(inputs, tmp_path) + root = tmp_path / "extracted" + cloud.extract_bundle( + (output / "bundle.zip").read_bytes(), root, plan["bundle_sha256"] + ) + for name, value in SECRETS.items(): + monkeypatch.setenv(name, value) + monkeypatch.setenv("OPENENV_RESULTS_REPO", plan["results_repo"]) + return root, plan + + +def rewrite_archive(data, *, changes=None, additions=None, remove=()): + output = io.BytesIO() + with zipfile.ZipFile(io.BytesIO(data)) as old, zipfile.ZipFile(output, "w") as new: + for member in old.infolist(): + if member.filename not in remove: + new.writestr( + member, (changes or {}).get(member.filename, old.read(member)) + ) + for name, content, mode in additions or []: + member = zipfile.ZipInfo(name) + member.create_system = 3 + member.external_attr = mode << 16 + new.writestr(member, content) + return output.getvalue() + + +def test_prepare_is_deterministic_offline_and_complete(inputs, tmp_path, monkeypatch): + network = Mock(side_effect=AssertionError("No network")) + monkeypatch.setattr(urllib.request, "urlopen", network) + repo, task, config = inputs + (repo / "src/openenv/__pycache__").mkdir() + (repo / "src/openenv/__pycache__/cached.pyc").write_bytes(b"\x00binary") + (repo / ".env").write_text("never bundle me") + (repo / ".venv").mkdir() + output, plan = prepared(inputs, tmp_path) + second, again = prepared(inputs, tmp_path, "second") + assert plan == again + data = (output / "bundle.zip").read_bytes() + assert data == (second / "bundle.zip").read_bytes() + assert plan["bundle_sha256"] == hashlib.sha256(data).hexdigest() + manifest, files = cloud._checked_archive(data, plan["bundle_sha256"]) + assert set(files) == set(manifest["files"]) | {cloud.MANIFEST} + assert files["uv.lock"] == b"uv.lock\n" + assert { + "examples/coding_audit/" + name + for name in ("__init__.py", "__main__.py", "scoring.py", "hf.py", "runner.py") + } <= files.keys() + assert "examples/coding_audit/test_excluded.py" not in files + assert manifest["git"] == {"commit": SHA, "dirty": True} + assert manifest["target_source"]["revision"] == full_run.MODEL_SOURCE[1] + assert manifest["task_source"]["identity"].startswith("sha256:") + assert manifest["files"]["task/solution/solve.sh"]["mode"] == 0o755 + assert not any( + ".env" in name or ".venv" in name or "__pycache__" in name for name in files + ) + assert plan["budget"]["sandbox_jobs_max"] == 28 + assert plan["budget"]["sandbox_remote_job_lifetime_seconds"] == 86400 + assert plan["budget"]["concurrent_sandbox_jobs_max"] == 1 + assert plan["budget"]["inference_calls_max"] == 8 + assert plan["mcp_request"]["args"]["image"] == cloud.WORKER_IMAGE + assert plan["mcp_request"]["required_env_after_upload"] == [ + "OPENENV_BUNDLE_REVISION" + ] + assert "secrets" not in plan["mcp_request"]["args"] + network.assert_not_called() + (task / "instruction.md").write_text("different task\n") + _, changed = prepared(inputs, tmp_path, "changed") + assert changed["bundle_sha256"] != plan["bundle_sha256"] + + +def test_prepare_fresh_directory_and_locked_inputs(inputs, tmp_path): + repo, task, config = inputs + output, _ = prepared(inputs, tmp_path) + with pytest.raises(FileExistsError, match="fresh"): + cloud.prepare(task, config, output, repo_root=repo) + with pytest.raises(ValueError, match="outside"): + cloud.prepare(task, config, task / "output", repo_root=repo) + (repo / "uv.lock").unlink() + with pytest.raises(FileNotFoundError): + prepared(inputs, tmp_path, "missing-lock") + + +@pytest.mark.parametrize("bad", ["link", "dirlink", "binary", "secret", "env", "fifo"]) +def test_prepare_rejects_unsafe_local_inputs(inputs, tmp_path, bad): + _, task, _ = inputs + if bad == "link": + (task / "link").symlink_to(task / "instruction.md") + elif bad == "dirlink": + (task / "link").symlink_to(task / "tests", target_is_directory=True) + elif bad == "binary": + (task / "binary").write_bytes(b"\x00binary") + elif bad == "secret": + (task / "secret.txt").write_text("hf_" + "z" * 30) + elif bad == "env": + (task / ".env.local").write_text("password=123") + else: + import os + + os.mkfifo(task / "fifo") + with pytest.raises((ValueError, UnicodeDecodeError)): + prepared(inputs, tmp_path) + assert not (tmp_path / "prepared").exists() + + +@pytest.mark.parametrize( + "kind", + ["timeout", "sandboxes", "calls", "tag", "provider", "revision", "credentials"], +) +def test_preflight_rejects_unbounded_or_mutable_config(inputs, tmp_path, kind): + config = inputs[2] + if kind == "timeout": + config["cloud"]["timeout_seconds"] = 0 + elif kind == "sandboxes": + config["cloud"]["max_sandboxes"] = 1 + elif kind == "calls": + config["cloud"]["max_inference_calls"] = 1 + elif kind == "tag": + config["execution"]["agent_image"] = "python:latest" + elif kind == "provider": + config["target"]["provider"] = "auto" + elif kind == "revision": + config["target"]["revision"] = "main" + else: + config["execution"]["token"] = "do-not-persist" + with pytest.raises(ValueError): + prepared(inputs, tmp_path) + + +def test_archive_digest_inventory_and_size_limits(inputs, tmp_path, monkeypatch): + output, plan = prepared(inputs, tmp_path) + data = (output / "bundle.zip").read_bytes() + with pytest.raises(ValueError, match="SHA256"): + cloud.extract_bundle(data, tmp_path / "bad", "0" * 64) + for changed in ( + rewrite_archive(data, changes={"task/instruction.md": b"tampered"}), + rewrite_archive(data, remove=["task/instruction.md"]), + rewrite_archive(data, additions=[("extra", b"extra", stat.S_IFREG | 0o644)]), + ): + with pytest.raises(ValueError, match="inventory"): + cloud.extract_bundle( + changed, tmp_path / "bad", hashlib.sha256(changed).hexdigest() + ) + assert not (tmp_path / "bad").exists() + monkeypatch.setattr(cloud, "MAX_TOTAL_BYTES", 1) + with pytest.raises(ValueError, match="size limit"): + cloud.extract_bundle(data, tmp_path / "bad", plan["bundle_sha256"]) + + +@pytest.mark.parametrize( + "name,mode", + [ + ("../escape", stat.S_IFREG | 0o644), + ("/absolute", stat.S_IFREG | 0o644), + ("task\\escape", stat.S_IFREG | 0o644), + ("task//escape", stat.S_IFREG | 0o644), + ("C:escape", stat.S_IFREG | 0o644), + ("task/instruction.md/child", stat.S_IFREG | 0o644), + ("TASK/INSTRUCTION.MD", stat.S_IFREG | 0o644), + ("link", stat.S_IFLNK | 0o777), + ("directory", stat.S_IFDIR | 0o755), + ], +) +def test_archive_rejects_path_collisions_and_nonregular_files( + inputs, tmp_path, name, mode +): + output, _ = prepared(inputs, tmp_path) + data = rewrite_archive( + (output / "bundle.zip").read_bytes(), additions=[(name, b"payload", mode)] + ) + with pytest.raises(ValueError): + cloud.extract_bundle(data, tmp_path / "bad", hashlib.sha256(data).hexdigest()) + assert not (tmp_path / "bad").exists() + + +@pytest.mark.parametrize("installed_version", ["1.33.0", "1.32.0"]) +def test_bootstrap_installs_and_checks_declared_sdk_before_worker( + inputs, tmp_path, monkeypatch, installed_version +): + output, plan = prepared(inputs, tmp_path) + data = (output / "bundle.zip").read_bytes() + + def open_request(request, **kwargs): + if "/resolve/" in request.full_url: + return io.BytesIO(data) + return io.BytesIO(b'{"private":true}') + + request = Mock(side_effect=open_request) + monkeypatch.setattr( + urllib.request, "build_opener", lambda *args: SimpleNamespace(open=request) + ) + destination = tmp_path / "bootstrap" + destination.mkdir() + monkeypatch.setattr(tempfile, "mkdtemp", lambda **kwargs: str(destination)) + real_run = subprocess.run + metadata = tmp_path / "metadata" + metadata.mkdir() + + def run_command(command, **kwargs): + if command[:3] == ["uv", "pip", "install"]: + assert command[-1] == "huggingface_hub==1.33.0" + dist = metadata / f"huggingface_hub-{installed_version}.dist-info" + dist.mkdir() + (dist / "METADATA").write_text( + f"Name: huggingface_hub\nVersion: {installed_version}\n" + ) + elif command[1] == "-c": + # Run the production version check over isolated package metadata. + return real_run( + [sys.executable, "-S", *command[1:]], + env={"PYTHONPATH": str(metadata)}, + **kwargs, + ) + return subprocess.CompletedProcess(command, 0) + + run = Mock(side_effect=run_command) + monkeypatch.setattr(subprocess, "run", run) + spec = cloud._launch_spec( + cloud._config(inputs[2])[0], plan["bundle_sha256"], UPLOAD_SHA + ) + for name, value in {**spec["env"], **SECRETS}.items(): + monkeypatch.setenv(name, value) + if installed_version == "1.33.0": + exec(spec["command"][2], {}) + else: + with pytest.raises(RuntimeError, match="verify_sdk"): + exec(spec["command"][2], {}) + download = request.call_args_list[0].args[0] + assert "/resolve/" + UPLOAD_SHA + "/inputs/" in download.full_url + assert download.get_header("Authorization") == "Bearer " + SECRETS["HF_TOKEN"] + calls = [call.args[0] for call in run.call_args_list] + assert calls[0][-2:] == ["install", "uv==0.9.3"] + assert calls[1] == ["uv", "sync", "--frozen", "--no-dev"] + python = str(destination / "bundle/.venv/bin/python") + assert calls[2] == [ + "uv", + "pip", + "install", + "--python", + python, + "huggingface_hub==1.33.0", + ] + assert calls[3][:2] == [python, "-c"] + if installed_version == "1.33.0": + assert calls[4][1:4] == [ + "-m", + "examples.hub_validation_pipeline.cloud", + "worker", + ] + else: + assert len(calls) == 4 + assert ( + destination / "bundle/task/instruction.md" + ).read_bytes() == b"Write an answer\n" + + +def test_runtime_versions_are_bound_to_plan_and_bundle(inputs, tmp_path): + output, plan = prepared(inputs, tmp_path) + manifest, _ = cloud._checked_archive( + (output / "bundle.zip").read_bytes(), plan["bundle_sha256"] + ) + assert ( + manifest["runtime_versions"] + == plan["runtime_versions"] + == {"uv": "0.9.3", "huggingface_hub": "1.33.0"} + ) + plan["runtime_versions"]["huggingface_hub"] = "1.32.0" + (output / "plan.json").write_text(json.dumps(plan)) + with pytest.raises(ValueError, match="immutable bundle"): + cloud._read_plan(output / "plan.json") + + +def test_submit_real_runtime_secrets_pins_and_one_shot_receipt(inputs, tmp_path): + output, plan = prepared(inputs, tmp_path) + api = fake_api() + receipt = cloud.submit( + output / "plan.json", + api=api, + secrets=SECRETS, + **launch_options(output, inputs[2]), + ) + assert receipt["job_id"] == "job-123" + launch = api.run_job.call_args.kwargs + assert launch["secrets"] == SECRETS + assert launch["token"] == SECRETS["HF_SANDBOX_TOKEN"] + assert launch["env"]["OPENENV_BUNDLE_REVISION"] == UPLOAD_SHA + assert launch["timeout"] == "1800s" + assert launch["image"] == cloud.WORKER_IMAGE + commit = api.create_commit.call_args.kwargs + assert commit["parent_commit"] == SHA + assert commit["create_pr"] is False + assert [op.path_in_repo for op in commit["operations"]] == [ + "inputs/" + plan["bundle_sha256"] + "/bundle.zip", + "inputs/" + plan["bundle_sha256"] + "/manifest.json", + "runs/" + plan["run_id"] + "/claim.json", + ] + for path in output.iterdir(): + for value in SECRETS.values(): + assert value.encode() not in path.read_bytes() + with pytest.raises(FileExistsError, match="receipt"): + cloud.submit( + output / "plan.json", + api=api, + secrets=SECRETS, + **launch_options(output, inputs[2]), + ) + assert api.run_job.call_count == 1 + + +@pytest.mark.parametrize("case", ["public", "missing", "duplicate", "conflict"]) +def test_no_launch_until_private_repo_and_exclusive_claim(inputs, tmp_path, case): + output, plan = prepared(inputs, tmp_path) + api = fake_api() + if case == "public": + api.dataset_info.return_value.private = False + elif case == "missing": + api.dataset_info.side_effect = RuntimeError("missing") + elif case == "duplicate": + api.list_repo_files.return_value = ["runs/" + plan["run_id"] + "/claim.json"] + else: + api.create_commit.side_effect = RuntimeError("parent conflict") + with pytest.raises((ValueError, RuntimeError, FileExistsError)): + cloud.submit( + output / "plan.json", + api=api, + secrets=SECRETS, + **launch_options(output, inputs[2]), + ) + api.run_job.assert_not_called() + if case in ("public", "missing", "duplicate"): + api.create_commit.assert_not_called() + + +def test_uncertain_launch_never_retries_or_leaks_exception(inputs, tmp_path): + output, _ = prepared(inputs, tmp_path) + api = fake_api() + api.run_job.side_effect = RuntimeError(SECRETS["HF_TOKEN"]) + with pytest.raises(RuntimeError, match="unknown") as caught: + cloud.submit( + output / "plan.json", + api=api, + secrets=SECRETS, + **launch_options(output, inputs[2]), + ) + assert SECRETS["HF_TOKEN"] not in str(caught.value) + receipt = json.loads((output / "submission.json").read_text()) + assert receipt["status"] == "launch_unknown" + with pytest.raises(FileExistsError): + cloud.submit( + output / "plan.json", + api=api, + secrets=SECRETS, + **launch_options(output, inputs[2]), + ) + assert api.run_job.call_count == 1 + + +@pytest.mark.parametrize( + "values", + [ + {}, + {**SECRETS, "HF_INFERENCE_TOKEN": "$HF_INFERENCE_TOKEN"}, + {**SECRETS, "HF_SANDBOX_TOKEN": SECRETS["HF_TOKEN"]}, + ], +) +def test_secret_values_required_and_distinct(inputs, tmp_path, values): + output, _ = prepared(inputs, tmp_path) + api = fake_api() + with pytest.raises(ValueError): + cloud.submit( + output / "plan.json", + api=api, + secrets=values, + **launch_options(output, inputs[2]), + ) + api.dataset_info.assert_not_called() + api.run_job.assert_not_called() + + +@pytest.mark.parametrize("section", ["launch", "git", "task_source", "target_source"]) +def test_plan_tampering_stops_before_hub(inputs, tmp_path, section): + output, _ = prepared(inputs, tmp_path) + path = output / "plan.json" + plan = json.loads(path.read_text()) + if section == "launch": + plan["mcp_request"]["args"]["image"] = "python:latest" + else: + plan[section] = { + "tampered": "reviewed provenance no longer matches bundled input" + } + path.write_text(json.dumps(plan)) + api = fake_api() + with pytest.raises(ValueError, match="immutable"): + cloud.submit( + path, api=api, secrets=SECRETS, **launch_options(output, inputs[2]) + ) + api.dataset_info.assert_not_called() + + +@pytest.mark.parametrize("fail", [None, RuntimeError, KeyboardInterrupt, SystemExit]) +def test_worker_persists_report_and_partial_evidence_on_failure( + inputs, tmp_path, monkeypatch, fail +): + root, plan = worker_root(inputs, tmp_path, monkeypatch) + api = fake_api() + + def runner(task, config, output): + assert task == root / "task" + assert not output.exists() + output.mkdir() + (output / "evidence.json").write_text('{"attempted":1}\n') + if fail: + raise fail(SECRETS["HF_TOKEN"]) + return {"accepted": False, "verdict": "skip"} + + if fail: + with pytest.raises(RuntimeError, match="persisted"): + cloud.worker(root, bundle_sha=plan["bundle_sha256"], api=api, runner=runner) + else: + report = cloud.worker( + root, bundle_sha=plan["bundle_sha256"], api=api, runner=runner + ) + assert report["status"] == "completed" + assert report["accepted"] is False + commit = api.create_commit.call_args.kwargs + operations = {op.path_in_repo: op.path_or_fileobj for op in commit["operations"]} + prefix = "runs/" + plan["run_id"] + "/" + assert prefix + "audit/evidence.json" in operations + report = json.loads(operations[prefix + "cloud-report.json"]) + assert report["status"] == ("error" if fail else "completed") + assert report["accepted"] is False + if fail: + assert report["error_type"] == fail.__name__ + assert all(name.startswith(prefix) for name in operations) + assert all(SECRETS["HF_TOKEN"].encode() not in data for data in operations.values()) + + +def test_worker_inventory_failure_still_persists_error(inputs, tmp_path, monkeypatch): + root, plan = worker_root(inputs, tmp_path, monkeypatch) + (root / "task/instruction.md").write_text("modified") + api, runner = fake_api(), Mock() + with pytest.raises(RuntimeError, match="persisted"): + cloud.worker(root, bundle_sha=plan["bundle_sha256"], api=api, runner=runner) + runner.assert_not_called() + assert ( + json.loads((root / "worker-output/cloud-report.json").read_text())["status"] + == "error" + ) + api.create_commit.assert_called_once() + + +def test_worker_upload_failure_retains_local_report(inputs, tmp_path, monkeypatch): + root, plan = worker_root(inputs, tmp_path, monkeypatch) + api = fake_api() + api.create_commit.side_effect = RuntimeError("Hub unavailable") + with pytest.raises(RuntimeError, match="upload failed"): + cloud.worker( + root, + bundle_sha=plan["bundle_sha256"], + api=api, + runner=lambda *args: {"accepted": True}, + ) + report = json.loads((root / "worker-output/cloud-report.json").read_text()) + assert report["persistence_status"] == "error" + + +def test_admission_thresholds_survive_bundle_roundtrip_exactly(inputs, tmp_path): + from decimal import Decimal + + repo, task, config = inputs + config["validation"]["reward_max"] = Decimal("1.00000000000000000001") + cloud.prepare(task, config, tmp_path / "precise", repo_root=repo) + _, restored, _, _ = cloud._read_plan(tmp_path / "precise/plan.json") + assert restored["validation"]["reward_max"] == Decimal("1.00000000000000000001") + + +@pytest.mark.parametrize("kind", ["hf", "harbor", "local"]) +def test_pinned_discovery_sources_bind_into_cloud_bundle(inputs, tmp_path, kind): + from openenv.validation.runner import source_digest + + repo, task, config = inputs + revision = { + "hf": "a" * 40, + "harbor": "sha256:" + "b" * 64, + "local": source_digest(task), + }[kind] + source = { + "source": kind, + "source_id": "org/repo", + "revision": revision, + "task_path": ".", + } + source["task_file_hashes"] = { + name: hashlib.sha256(data).hexdigest() + for name, (data, _) in cloud._tree(task).items() + } + plan = cloud.prepare(task, config, tmp_path / kind, repo_root=repo, source=source) + assert plan["task_source"]["kind"] == kind + assert plan["task_source"]["revision"] == revision + + +def test_duplicate_json_keys_rejected(): + with pytest.raises(ValueError, match="Duplicate"): + cloud._loads('{"cloud":{},"cloud":{}}') + + +def test_bootstrap_persists_dependency_failure_without_sdk( + inputs, tmp_path, monkeypatch +): + import base64 + + output, plan = prepared(inputs, tmp_path) + data = (output / "bundle.zip").read_bytes() + requests = [] + + def open_request(request, **kwargs): + requests.append(request) + if request.get_method() == "POST": + return io.BytesIO(b'{"commitOid":"done"}') + if "/resolve/" in request.full_url: + return io.BytesIO(data) + return io.BytesIO(b'{"private":true}') + + monkeypatch.setattr( + urllib.request, "build_opener", lambda *args: SimpleNamespace(open=open_request) + ) + destination = tmp_path / "bootstrap" + destination.mkdir() + monkeypatch.setattr(tempfile, "mkdtemp", lambda **kwargs: str(destination)) + monkeypatch.setattr( + subprocess, "run", Mock(side_effect=RuntimeError(SECRETS["HF_TOKEN"])) + ) + spec = cloud._launch_spec( + cloud._config(inputs[2])[0], plan["bundle_sha256"], UPLOAD_SHA + ) + for name, value in {**spec["env"], **SECRETS}.items(): + monkeypatch.setenv(name, value) + with pytest.raises(RuntimeError, match="install_uv") as caught: + exec(spec["command"][2], {}) + assert SECRETS["HF_TOKEN"] not in str(caught.value) + commit = requests[-1] + assert ( + commit.full_url + == "https://huggingface.co/api/datasets/org/private-results/commit/main" + ) + rows = [json.loads(line) for line in commit.data.splitlines()] + assert ( + rows[1]["value"]["path"] == "runs/" + plan["run_id"] + "/bootstrap-error.json" + ) + report = json.loads(base64.b64decode(rows[1]["value"]["content"])) + assert report["stage"] == "install_uv" + assert report["accepted"] is False + assert report["error_type"] == "RuntimeError" + + +def test_cross_host_redirect_drops_results_token(): + redirect = cloud._HubRedirect() + request = urllib.request.Request( + "https://huggingface.co/resolve/blob", + headers={"Authorization": "Bearer private", "Range": "bytes=0-"}, + ) + target = redirect.redirect_request( + request, None, 302, "Found", {}, "https://cdn.example/blob" + ) + assert target.get_header("Authorization") is None + assert target.get_header("Range") == "bytes=0-" + same = redirect.redirect_request( + request, None, 302, "Found", {}, "https://huggingface.co/blob" + ) + assert same.get_header("Authorization") == "Bearer private" + with pytest.raises(ValueError, match="HTTPS"): + redirect.redirect_request( + request, None, 302, "Found", {}, "http://cdn.example/blob" + ) + + +@pytest.mark.parametrize("kind", ["symlink", "binary", "credential"]) +def test_worker_unsafe_evidence_still_uploads_error_only( + inputs, tmp_path, monkeypatch, kind +): + root, plan = worker_root(inputs, tmp_path, monkeypatch) + api = fake_api() + + def runner(task, config, output): + output.mkdir() + if kind == "symlink": + (output / "link").symlink_to(root / "coding-config.json") + elif kind == "binary": + (output / "binary").write_bytes(b"\x00binary") + else: + (output / "unsafe.json").write_text(json.dumps(SECRETS)) + return {"accepted": True} + + with pytest.raises(RuntimeError, match="persisted"): + cloud.worker(root, bundle_sha=plan["bundle_sha256"], api=api, runner=runner) + operations = api.create_commit.call_args.kwargs["operations"] + assert len(operations) == 1 + assert operations[0].path_in_repo.endswith("/cloud-report.json") + report = json.loads(operations[0].path_or_fileobj) + assert report["status"] == "error" + assert report["accepted"] is False + assert report["evidence_omitted"] is True + + +def test_task_exclusions_match_runner_source_identity(inputs, tmp_path): + from openenv.validation.runner import source_digest + + task = inputs[1] + for name in (".git", ".worktrees", ".venv", "__pycache__", "outputs"): + (task / name).mkdir() + (task / name / "binary-secret").write_bytes(b"\x00ignored") + before = source_digest(task) + output, plan = prepared(inputs, tmp_path) + destination = tmp_path / "snapshot" + cloud.extract_bundle( + (output / "bundle.zip").read_bytes(), destination, plan["bundle_sha256"] + ) + assert source_digest(destination / "task") == before + assert not (destination / "task/.worktrees").exists() + assert not (destination / "task/outputs").exists() + + +@pytest.mark.parametrize("excluded", sorted(cloud._EXCLUDED)) +@pytest.mark.parametrize("copied", [False, True]) +def test_acquired_excluded_assets_are_explicitly_unsupported_before_snapshot( + inputs, tmp_path, monkeypatch, excluded, copied +): + from examples.hub_validation_pipeline.pipeline import copy_subject + from openenv.validation.runner import source_digest + + repo, task, config = inputs + asset = task / "environment" / excluded / "setup.txt" + asset.parent.mkdir(parents=True) + asset.write_text("Native task setup asset\n") + hashes = { + path.relative_to(task).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in task.rglob("*") + if path.is_file() + } + source = dict(kind="hf", revision=SHA, task_path=".", task_file_hashes=hashes) + if copied: + subject = tmp_path / "subject" + copy_subject(task, subject, source_digest(task)) + assert not (subject / "environment" / excluded).exists() + else: + subject = task + original_tree = cloud._tree + snapshots = [] + + def snapshot(root, **kwargs): + if Path(root) == subject: + snapshots.append(root) + return original_tree(root, **kwargs) + + monkeypatch.setattr(cloud, "_tree", snapshot) + with pytest.raises( + ValueError, match="Acquired task contains unsupported excluded assets" + ): + cloud.prepare(subject, config, tmp_path / "plan", repo_root=repo, source=source) + assert snapshots == [] + assert source["task_file_hashes"] == hashes + assert f"environment/{excluded}/setup.txt" in hashes + assert not (tmp_path / "plan").exists() + + +@pytest.mark.parametrize( + "fault", + ["compose", "steps", "private-network", "phase-environment", "missing-verifier"], +) +def test_prepare_rejects_unsupported_task_before_controller_spending( + inputs, tmp_path, fault +): + repo, task, config = inputs + if fault == "compose": + (task / "environment").mkdir() + (task / "environment/compose.yaml").write_text("services: {}\n") + elif fault == "steps": + (task / "steps").mkdir() + (task / "steps/step.toml").write_text("name='step'\n") + elif fault == "private-network": + (task / "task.toml").write_text("[environment]\nallow_internet=false\n") + elif fault == "phase-environment": + (task / "task.toml").write_text("[verifier.environment]\nnetwork_mode='none'\n") + else: + (task / "tests/test.sh").unlink() + output = tmp_path / "plan" + with pytest.raises(ValueError): + cloud.prepare(task, config, output, repo_root=repo) + assert not output.exists() + + +def test_prepare_checks_the_frozen_task_contract_not_a_restored_source( + inputs, tmp_path, monkeypatch +): + repo, task, config = inputs + original_tree = cloud._tree + original = (task / "task.toml").read_bytes() + + def changed_during_snapshot(root, **kwargs): + if Path(root) != task: + return original_tree(root, **kwargs) + (task / "task.toml").write_text("[environment]\nallow_internet=false\n") + try: + return original_tree(root, **kwargs) + finally: + (task / "task.toml").write_bytes(original) + + monkeypatch.setattr(cloud, "_tree", changed_during_snapshot) + output = tmp_path / "plan" + with pytest.raises(ValueError, match="network"): + cloud.prepare(task, config, output, repo_root=repo) + assert (task / "task.toml").read_bytes() == original + assert not output.exists() + + +def test_bundle_retains_available_full_run_reproduction_helpers(inputs, tmp_path): + repo, _, _ = inputs + names = ( + "full_run.py", + "real_tasks.py", + "sources.py", + "spend.py", + ) + for name in names: + path = repo / "examples/hub_validation_pipeline" / name + path.write_text("{}\n" if name.endswith(".json") else "# helper\n") + output, plan = prepared(inputs, tmp_path) + _, files = cloud._checked_archive( + (output / "bundle.zip").read_bytes(), plan["bundle_sha256"] + ) + for name in names: + relative = "examples/hub_validation_pipeline/" + name + assert files[relative] == (repo / relative).read_bytes() + + +@pytest.mark.parametrize("source_kind", ["local", "hf", "harbor"]) +def test_hub_driver_prepares_coding_audits_without_admitting_or_running_docker( + inputs, tmp_path, monkeypatch, source_kind +): + from argparse import Namespace + + from examples.hub_validation_pipeline import cloud, pipeline + + repo, task, raw_config = inputs + hashes = { + name: hashlib.sha256(data).hexdigest() + for name, (data, _) in cloud._tree(task).items() + } + if source_kind != "local": + record = dict( + task_id="remote-test-task", + path=str(task), + source=source_kind, + source_id="org/tasks", + task_path=".", + task_file_hashes=hashes, + revision=SHA if source_kind == "hf" else "sha256:" + "b" * 64, + ) + monkeypatch.setattr(pipeline.sources, "acquire", lambda *args: [record]) + config = tmp_path / "coding.json" + config.write_text(json.dumps(raw_config)) + calls = [] + real_prepare = cloud.prepare + + def prepare(subject, config, output, *, source): + calls.append((subject, source)) + assert (subject / "task.toml").is_file() + if source_kind != "local": + assert source["task_file_hashes"] == hashes + return real_prepare(subject, config, output, repo_root=repo, source=source) + + monkeypatch.setattr(cloud, "prepare", prepare) + monkeypatch.setattr( + pipeline, + "validate_task", + lambda *args: pytest.fail("cloud mode must not invoke Docker validation"), + ) + args = Namespace( + output=tmp_path / "run", + contract=None, + coding_config=config, + submit_hf_jobs=False, + hf_dataset=["org/tasks@" + SHA] if source_kind == "hf" else [], + harbor_dataset=["org/tasks@sha256:" + "b" * 64] + if source_kind == "harbor" + else [], + local_dataset=[task] if source_kind == "local" else [], + source="both", + dataset_limit=0, + tasks_per_dataset=1, + ) + assert pipeline.run(args) == 0 + summary = json.loads((args.output / "summary.json").read_text()) + assert summary["phase"] == "prepared" + assert summary["plans"] == 1 and summary["accepted"] == 0 + assert not summary["rfc008_complete"] + planned = json.loads((args.output / "planned.jsonl").read_text()) + assert planned["accepted"] is False and planned["state"] == "prepared" + assert (args.output / "accepted.jsonl").read_text() == "" + assert calls[0][1]["revision"] + + +@pytest.mark.parametrize("ledger_state", ["missing", "exhausted", "blocked"]) +@pytest.mark.parametrize("entry", ["cloud_cli", "pipeline", "submit"]) +def test_public_launch_paths_require_spending_gates( + inputs, tmp_path, monkeypatch, ledger_state, entry +): + from argparse import Namespace + + import huggingface_hub + from examples.hub_validation_pipeline import pipeline, spend + + repo, task, config = inputs + ledger = tmp_path / "spend.json" + if ledger_state != "missing": + spend.init(ledger, "1") + spend.reserve(ledger, "prior", "1", "Prior work") + if ledger_state == "blocked": + spend.settle(ledger, "prior", "2", "Overrun") + api = fake_api() + monkeypatch.setattr(huggingface_hub, "HfApi", lambda **kwargs: api) + for name, value in SECRETS.items(): + monkeypatch.setenv(name, value) + output, _ = prepared(inputs, tmp_path) + if entry == "cloud_cli": + with pytest.raises(SystemExit): + cloud.main(["submit", "--plan", str(output / "plan.json")]) + elif entry == "pipeline": + config_path = tmp_path / "coding.json" + config_path.write_bytes(cloud._json_bytes(config)) + real_prepare = cloud.prepare + monkeypatch.setattr( + cloud, + "prepare", + lambda *args, **kwargs: real_prepare(*args, repo_root=repo, **kwargs), + ) + args = Namespace( + output=tmp_path / "run", + contract=None, + coding_config=config_path, + submit_hf_jobs=True, + hf_dataset=[], + harbor_dataset=[], + local_dataset=[task], + source="both", + dataset_limit=0, + tasks_per_dataset=1, + ) + with pytest.raises(ValueError, match="prepared_batch"): + pipeline.run(args) + else: + with pytest.raises(ValueError, match="reservation"): + cloud.submit(output / "plan.json", api=api, secrets=SECRETS) + api.run_job.assert_not_called() + api.create_commit.assert_not_called() + + +@pytest.mark.parametrize( + "case", + [ + "missing_ledger", + "unknown", + "blocked", + "settled", + "unbound", + "underpriced", + "missing_hash", + "wrong_hash", + "overallocated", + "exhausted", + ], +) +def test_submit_rechecks_reserved_plan_cost_and_status(inputs, tmp_path, case): + output, _ = prepared(inputs, tmp_path) + options = launch_options(output, inputs[2]) + ledger, identifier = options["spend_ledger"], options["reservation_id"] + if case == "missing_ledger": + ledger.unlink() + elif case == "unknown": + options["reservation_id"] = "unreserved" + elif case == "blocked": + spend.reserve(ledger, "other", "1", "Other work") + spend.settle(ledger, "other", "2", "Overrun") + elif case == "settled": + spend.settle(ledger, identifier, "0", "Completed work") + elif case == "missing_hash": + del options["expected_plan_sha256"] + elif case == "wrong_hash": + options["expected_plan_sha256"] = "0" * 64 + else: + data = json.loads(ledger.read_bytes()) + row = data["reservations"][identifier] + if case == "exhausted": + data["reservations"] = {} + elif case == "unbound": + del row["plan_costs"] + else: + row["plan_costs"][options["expected_plan_sha256"]] = ( + "0" if case == "underpriced" else "1001" + ) + ledger.write_text(json.dumps(data)) + if case == "exhausted": + spend.reserve(ledger, "other", "1000", "Other work") + api = fake_api() + with pytest.raises(ValueError): + cloud.submit(output / "plan.json", api=api, secrets=SECRETS, **options) + api.run_job.assert_not_called() + api.create_commit.assert_not_called() + assert not (output / "submission.json").exists() + + +def test_local_revision_checks_captured_verifier_not_restored_live_tree( + inputs, tmp_path, monkeypatch +): + from openenv.validation.runner import source_digest + + repo, task, config = inputs + revision = source_digest(task) + original_tree = cloud._tree + verifier = task / "tests/test.sh" + original = verifier.read_bytes() + captured = [] + + def mutate_during_capture(root, **kwargs): + if root != task: + return original_tree(root, **kwargs) + verifier.write_bytes(b"echo 1 > /logs/verifier/reward.txt\n") + try: + files = original_tree(root, **kwargs) + captured.append(files["tests/test.sh"][0]) + return files + finally: + verifier.write_bytes(original) + + monkeypatch.setattr(cloud, "_tree", mutate_during_capture) + source = {"kind": "local", "revision": revision, "task_path": "."} + with pytest.raises(ValueError, match="Local source revision"): + cloud.prepare(task, config, tmp_path / "race", repo_root=repo, source=source) + assert source_digest(task) == revision + assert captured == [b"echo 1 > /logs/verifier/reward.txt\n"] + assert not (tmp_path / "race").exists() + + +@pytest.mark.parametrize("kind", ["hf", "harbor"]) +def test_remote_identity_requires_verified_acquisition_hashes(inputs, tmp_path, kind): + repo, task, config = inputs + source = dict( + kind=kind, + revision="a" * 40 if kind == "hf" else "sha256:" + "b" * 64, + task_path=".", + ) + with pytest.raises(ValueError, match="verified acquisition"): + cloud.prepare(task, config, tmp_path / kind, repo_root=repo, source=source) + assert not (tmp_path / kind).exists() + + +def test_receipt_is_synced_before_and_after_atomic_publication(tmp_path, monkeypatch): + import os + + path = tmp_path / "submission.json" + events = [] + fsync, replace = os.fsync, Path.replace + + def sync(fd): + events.append("directory" if stat.S_ISDIR(os.fstat(fd).st_mode) else "file") + fsync(fd) + + def publish(source, target): + events.append("replace") + return replace(source, target) + + monkeypatch.setattr(os, "fsync", sync) + monkeypatch.setattr(Path, "replace", publish) + cloud._write_receipt(path, {"job_id": "owned-job"}) + assert events == ["file", "replace", "directory"] + assert cloud._loads(path.read_bytes()) == {"job_id": "owned-job"} + + +def test_receipt_sync_failure_preserves_previous_receipt(tmp_path, monkeypatch): + import os + + path = tmp_path / "submission.json" + path.write_text('{"status":"launch_pending"}\n') + previous = path.read_bytes() + monkeypatch.setattr(os, "fsync", Mock(side_effect=OSError("disk failure"))) + with pytest.raises(OSError, match="disk failure"): + cloud._write_receipt(path, {"status": "submitted", "job_id": "owned-job"}) + assert path.read_bytes() == previous diff --git a/examples/hub_validation_pipeline/test_full_run.py b/examples/hub_validation_pipeline/test_full_run.py new file mode 100644 index 000000000..eb31f2413 --- /dev/null +++ b/examples/hub_validation_pipeline/test_full_run.py @@ -0,0 +1,848 @@ +"""Local-only checks for modeled reservation and durable one-shot orchestration.""" + +import hashlib +import json +import socket +from copy import deepcopy +from fractions import Fraction +from pathlib import Path +from unittest.mock import Mock + +import pytest +from examples.hub_validation_pipeline import cloud, full_run, real_tasks, spend +from examples.hub_validation_pipeline.test_real_tasks import ( + toy_config, + TOY_NAMES, + TOY_SOURCE, + write_panel, +) + + +@pytest.fixture +def local_run(tmp_path, monkeypatch): + forbidden = Mock(side_effect=AssertionError("Live operations forbidden")) + monkeypatch.setattr(socket.socket, "connect", forbidden) + monkeypatch.setattr(real_tasks, "HfApi", forbidden) + monkeypatch.setattr(cloud, "_runtime_secrets", forbidden) + panel_path = write_panel(tmp_path) + panel_sha256 = full_run._hash(panel_path)["sha256"] + plans = {} + + def prepare(output, *, panel, panel_provenance, task_names, bundle): + assert panel_provenance["sha256"] == panel_sha256 + tasks = {task["name"]: task for task in panel["tasks"]} + assert bundle is True + output.mkdir() + rows = [] + for index, name in enumerate(task_names): + config, budget = cloud._config(tasks[name]["config"]) + path = output / name / "plan.json" + path.parent.mkdir() + plan = { + "run_id": f"coding-{index}", + "bundle_sha256": str(index) * 64, + "budget": budget, + } + path.write_bytes(cloud._json_bytes(plan)) + plans[str(path)] = (plan, config, b"", {}) + rows.append( + { + "task_id": f"hf:{TOY_SOURCE['repo_id']}@{TOY_SOURCE['revision']}:fixtures/{name}", + "bundle_plan": str(path), + } + ) + result = {"tasks": rows} + (output / "manifest.json").write_bytes(cloud._json_bytes(result)) + return result + + prepare = Mock(side_effect=prepare) + monkeypatch.setattr(real_tasks, "_prepare", prepare) + + def read_plan(path, *, expected_sha256=None): + if expected_sha256 is not None: + if hashlib.sha256(Path(path).read_bytes()).hexdigest() != expected_sha256: + raise ValueError("Prepared plan changed before submission") + return plans[str(path)] + + monkeypatch.setattr(cloud, "_read_plan", read_plan) + submit = Mock(return_value={"status": "submitted", "job_id": "queued-job"}) + monkeypatch.setattr(cloud, "submit", submit) + hf, harbor = tmp_path / "hf.json", tmp_path / "harbor.json" + inventory = tmp_path / "inventory.json.gz" + # Deliberately not JSON/gzip: inventory bytes must only be streamed/hashed. + inventory.write_bytes(b"unparsed inventory" * 100000) + missing = tmp_path / "missing.json" + hf.write_text( + json.dumps( + { + "scope": "Frozen loose native identities; no runtime qualification", + "counts": {"repositories": 3, "native_tasks": 7}, + "compact_repo_inventory": str(inventory), + "compact_identity_inventory": str(missing), + "raw_census": str(tmp_path / "do-not-read-raw.json"), + } + ) + ) + harbor.write_text( + json.dumps( + { + "scope": "Anonymous native registry metadata only", + "counts": { + "dataset_versions_listed": 9, + "dataset_names": 5, + "unique_task_digests": 11, + }, + "artifacts": { + key: {"path": str(inventory)} + for key in ( + "task_identity_inventory", + "dataset_version_membership_index", + ) + }, + } + ) + ) + return tmp_path / "run", hf, harbor, prepare, submit, plans + + +def assert_prepared(prepare, output, names): + prepare.assert_called_once() + assert prepare.call_args.args == (output / "recipes",) + assert prepare.call_args.kwargs["task_names"] == names + assert prepare.call_args.kwargs["bundle"] is True + assert ( + prepare.call_args.kwargs["panel_provenance"]["sha256"] + == full_run._hash(output.parent / "panel.json")["sha256"] + ) + + +@pytest.fixture +def paid_reservation(tmp_path): + ledger = tmp_path / "spend.json" + spend.init(ledger, "500") + return {"spend_ledger": ledger, "reservation_id": "test-cohort"} + + +@pytest.mark.parametrize("supplied", [False, True], ids=["omitted", "exhausted"]) +def test_paid_api_cannot_bypass_fully_reserved_global_ledger(local_run, supplied): + output, hf, harbor, prepare, submit, _ = local_run + ledger = output.parent / "spend.json" + spend.init(ledger, "500") + spend.reserve(ledger, "prior-unresolved", "500", "Prior unresolved work") + arguments = ( + {"spend_ledger": ledger, "reservation_id": "toy-selection-v3"} + if supplied + else {} + ) + error, message = ( + (RuntimeError, "during reserve") + if supplied + else (ValueError, "shared ledger and a reservation ID") + ) + with pytest.raises(error, match=message): + full_run.run( + output, + hf, + harbor, + "500", + task_names=["toy-alpha"], + submit=True, + **arguments, + panel_path=output.parent / "panel.json", + ) + submit.assert_not_called() + if not supplied: + prepare.assert_not_called() + assert not output.exists() + assert spend.summary(ledger)["reserved_usd_exact"] == "500" + assert spend.summary(ledger)["remaining_usd_exact"] == "0" + + +@pytest.mark.parametrize( + "ledger_value,identifier", + [ + (None, None), + (None, "cohort"), + ("", "cohort"), + (" ", "cohort"), + ("initialized", None), + ("initialized", ""), + ("initialized", " "), + ], +) +def test_paid_api_requires_nonempty_shared_reservation_before_preparation( + local_run, paid_reservation, monkeypatch, ledger_value, identifier +): + output, hf, harbor, prepare, submit, _ = local_run + monkeypatch.chdir(output.parent) + ledger = paid_reservation["spend_ledger"] + with pytest.raises(ValueError, match="shared ledger and a reservation ID"): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + spend_ledger=ledger if ledger_value == "initialized" else ledger_value, + reservation_id=identifier, + panel_path=output.parent / "panel.json", + ) + assert not output.exists() + prepare.assert_not_called() + submit.assert_not_called() + assert spend.summary(ledger)["reserved_usd_exact"] == "0" + + +@pytest.mark.parametrize("empty_file", [False, True], ids=["missing", "empty"]) +def test_paid_api_requires_initialized_ledger_before_submission(local_run, empty_file): + output, hf, harbor, _, submit, _ = local_run + ledger = output.parent / "spend.json" + if empty_file: + ledger.write_bytes(b"") + with pytest.raises(RuntimeError, match="during reserve"): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + spend_ledger=ledger, + reservation_id="test-cohort", + panel_path=output.parent / "panel.json", + ) + submit.assert_not_called() + + +def test_exact_aggregate_reservation_and_normalized_generator(): + configs = [toy_config(name) for name in TOY_NAMES] + for config in configs: + config.pop("generator") + result = full_run.estimate(configs) + # Missing generator inherits the target's request limits. + inputs_per_task = 2 * (4 * 256 + 8192 + 256) + (4 * 256 + 8192) + outputs_per_task = 2 * 3 * 64 + 64 + assert result["input_tokens_max"] == 2 * inputs_per_task + assert result["output_tokens_max"] == 2 * outputs_per_task + assert result["controller_jobs_max"] == 2 + budgets = [cloud._config(config)[1] for config in configs] + children = sum(budget["sandbox_jobs_max"] for budget in budgets) + assert result["sandbox_jobs_max"] == children + inference = ( + 2 * inputs_per_task * Fraction("0.15") + + 2 * outputs_per_task * Fraction("1.875") + ) / 1000000 + compute = (children * 86700 * Fraction("0.02") + 2 * 360 * Fraction("0.01")) / 3600 + assert Fraction(result["inference_usd_exact"]) == inference + assert Fraction(result["compute_usd_exact"]) == compute + assert Fraction(result["reservation_usd_exact"]) == inference + compute + assert all(isinstance(value, (str, int)) for value in result.values()) + + +def test_operator_runtime_limits_drive_reservation(): + configs = [toy_config(name) for name in TOY_NAMES] + base = full_run.estimate(configs) + configs[0]["target"]["max_total_tokens"] *= 2 + changed = full_run.estimate(configs) + assert changed["input_tokens_max"] > base["input_tokens_max"] + assert Fraction(changed["reservation_usd_exact"]) > Fraction( + base["reservation_usd_exact"] + ) + + +@pytest.mark.parametrize( + "names", [[], ["unknown"], ["toy-alpha", "unknown"], ["toy-beta"] * 2] +) +def test_invalid_task_selection_is_rejected_before_preparation(local_run, names): + output, hf, harbor, prepare, submit, _ = local_run + ledger = output.parent / "spend.json" + spend.init(ledger, "500") + with pytest.raises(ValueError, match="distinct configured panel task names"): + full_run.run( + output, + hf, + harbor, + "500", + task_names=names, + submit=True, + spend_ledger=ledger, + reservation_id="invalid-cohort", + panel_path=output.parent / "panel.json", + ) + assert not output.exists() + prepare.assert_not_called() + submit.assert_not_called() + assert spend.summary(ledger)["reserved_usd_exact"] == "0" + + +def test_selection_budgets_and_submits_only_selected_task(local_run): + output, hf, harbor, prepare, submit, _ = local_run + ledger = output.parent / "spend.json" + spend.init(ledger, "30") + spend.reserve(ledger, "prior-launch", "5", "Unreconciled prior launch") + manifest = full_run.run( + output, + hf, + harbor, + "15", + task_names=["toy-alpha"], + submit=True, + spend_ledger=ledger, + reservation_id="toy-selection-v2", + panel_path=output.parent / "panel.json", + ) + assert_prepared(prepare, output, ["toy-alpha"]) + assert manifest["status"] == "submitted" and manifest["within_budget"] is True + assert manifest == cloud._loads((output / "manifest.json").read_bytes()) + assert len(manifest["tasks"]) == 1 + task = manifest["tasks"][0] + assert task["task_id"] == ( + f"hf:{TOY_SOURCE['repo_id']}@{TOY_SOURCE['revision']}:fixtures/toy-alpha" + ) + assert task["submission"]["status"] == "submitted" + submit.assert_called_once_with( + task["plan"], + expected_plan_sha256=task["plan_sha256"], + spend_ledger=ledger, + reservation_id=manifest["shared_spend"]["reservation_id"], + ) + assert not (output / "recipes/toy-beta").exists() + assert "toy-beta" not in (output / "manifest.json").read_text() + estimate = manifest["estimate"] + assert estimate == full_run.estimate([toy_config("toy-alpha")]) + assert estimate["controller_jobs_max"] == 1 + assert Fraction(spend.summary(ledger)["reserved_usd_exact"]) == ( + 5 + Fraction(estimate["reservation_usd_rounded_up"]) + ) + assert manifest["shared_spend"]["reservation_id"].startswith("toy-selection-v2:") + with pytest.raises(FileExistsError): + full_run.run( + output, + hf, + harbor, + "15", + task_names=["toy-alpha"], + submit=True, + spend_ledger=ledger, + reservation_id="toy-selection-v2", + panel_path=output.parent / "panel.json", + ) + assert submit.call_count == prepare.call_count == 1 + + +def test_selected_task_still_obeys_global_spend_cap(local_run): + output, hf, harbor, _, submit, _ = local_run + ledger = output.parent / "spend.json" + spend.init(ledger, "15") + spend.reserve(ledger, "prior-launch", "10", "Unreconciled prior launch") + with pytest.raises(RuntimeError, match="during reserve"): + full_run.run( + output, + hf, + harbor, + "15", + task_names=["toy-alpha"], + submit=True, + spend_ledger=ledger, + reservation_id="toy-selection-v2", + panel_path=output.parent / "panel.json", + ) + submit.assert_not_called() + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert manifest["within_budget"] is True + assert len(manifest["tasks"]) == 1 + assert manifest["tasks"][0]["submission"]["status"] == "not_submitted" + assert spend.summary(ledger)["reserved_usd_exact"] == "10" + + +@pytest.mark.parametrize( + "budget", ["0", "-1", "NaN", "Infinity", "1/2", "invalid", 50.0, True, None] +) +def test_invalid_cap_is_rejected_before_preparation( + local_run, paid_reservation, budget +): + output, hf, harbor, prepare, submit, _ = local_run + with pytest.raises(ValueError, match="decimal string"): + full_run.run( + output, + hf, + harbor, + budget, + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + assert not output.exists() + prepare.assert_not_called() + submit.assert_not_called() + + +def test_prepare_only_freezes_small_summaries_and_hashes_inventories(local_run): + output, hf, harbor, prepare, submit, _ = local_run + manifest = full_run.run( + output, hf, harbor, "500", panel_path=output.parent / "panel.json" + ) + assert_prepared(prepare, output, list(TOY_NAMES)) + submit.assert_not_called() + assert manifest == cloud._loads((output / "manifest.json").read_bytes()) + assert manifest["status"] == "prepared" and manifest["within_budget"] is True + assert manifest["full_catalog_executed"] is False + assert ( + manifest["accepted"] + is manifest["rfc008_qualified"] + is manifest["trainable"] + is False + ) + assert len(manifest["tasks"]) == 2 + for kind, source in (("hf", hf), ("harbor", harbor)): + record = manifest["catalogs"][kind] + assert Path(record["path"]).read_bytes() == source.read_bytes() + assert record["sha256"] == hashlib.sha256(source.read_bytes()).hexdigest() + assert record["runtime_qualified"] is False + inventories = manifest["catalogs"]["hf"]["inventories"] + record = inventories["compact_repo_inventory"] + assert ( + record["sha256"] + == hashlib.sha256(Path(record["path"]).read_bytes()).hexdigest() + ) + assert inventories["compact_identity_inventory"]["status"] == "unavailable" + assert manifest["catalogs"]["hf"]["counts"]["native_tasks"] == 7 + assert not (output.parent / "spend.json").exists() + with pytest.raises(FileExistsError): + full_run.run(output, hf, harbor, "500", panel_path=output.parent / "panel.json") + prepare.assert_called_once() + + +@pytest.mark.parametrize( + "cap", ["1", "0.0000000000000000000000000000000000000000000000000000000001"] +) +def test_aggregate_cap_stops_before_any_submit_and_preserves_prepared_manifest( + local_run, paid_reservation, cap +): + output, hf, harbor, _, submit, _ = local_run + with pytest.raises(RuntimeError, match="during budget"): + full_run.run( + output, + hf, + harbor, + cap, + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + submit.assert_not_called() + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert manifest["within_budget"] is False and manifest["stage"] == "budget" + assert len(manifest["tasks"]) == 2 and (output / "recipes/manifest.json").exists() + assert all( + task["submission"]["status"] == "not_submitted" for task in manifest["tasks"] + ) + + +@pytest.mark.parametrize( + "section,field,value", + [ + ("target", "model_id", "other/model"), + ("generator", "provider", "other"), + ("target", "revision", "a" * 40), + ("generator", "provider_model_id", "different-model"), + ("execution", "flavor", "cpu-upgrade"), + ("cloud", "flavor", "cpu-upgrade"), + ], +) +def test_unquoted_profiles_are_rejected(section, field, value): + config = deepcopy(toy_config("toy-beta")) + config[section][field] = value + with pytest.raises(ValueError): + full_run.estimate([config]) + + +def test_partial_ambiguous_submission_is_durable_safe_and_never_resumed( + local_run, paid_reservation +): + output, hf, harbor, prepare, submit, _ = local_run + observed = [] + + def launch(path, *, expected_plan_sha256, spend_ledger, reservation_id): + assert full_run._hash(Path(path))["sha256"] == expected_plan_sha256 + durable = cloud._loads((output / "manifest.json").read_bytes()) + assert durable["shared_spend"]["ledger"] == str( + paid_reservation["spend_ledger"] + ) + assert Fraction( + spend.summary(paid_reservation["spend_ledger"])["reserved_usd_exact"] + ) == Fraction(durable["estimate"]["reservation_usd_rounded_up"]) + assert spend_ledger == paid_reservation["spend_ledger"] + assert reservation_id == durable["shared_spend"]["reservation_id"] + index = len(observed) + observed.append(path) + assert durable["tasks"][index]["submission"]["status"] == "pending_or_unknown" + if index: + assert durable["tasks"][0]["submission"]["job_id"] == "job-first" + (Path(path).parent / "submission.json").write_text( + '{"status":"launch_unknown"}' + ) + raise RuntimeError("sensitive-error-value-must-not-escape") + return { + "status": "submitted", + "job_id": "job-first", + "launch_spec": {"command": ["omitted"]}, + } + + submit.side_effect = launch + with pytest.raises(RuntimeError, match="during submit") as failure: + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert manifest["status"] == "failed" and manifest["accepted"] is False + assert manifest["tasks"][0]["submission"]["status"] == "submitted" + assert manifest["tasks"][1]["submission"]["status"] == "pending_or_unknown" + receipt = Path(manifest["tasks"][1]["submission"]["receipt"]) + assert json.loads(receipt.read_text())["status"] == "launch_unknown" + assert ( + "sensitive-error-value" + not in str(failure.value) + (output / "manifest.json").read_text() + ) + assert "launch_spec" not in manifest["tasks"][0]["submission"] + assert Fraction( + spend.summary(paid_reservation["spend_ledger"])["reserved_usd_exact"] + ) == Fraction(manifest["estimate"]["reservation_usd_rounded_up"]) + with pytest.raises(FileExistsError): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + assert submit.call_count == 2 and prepare.call_count == 1 + + +def test_submitted_jobs_remain_unaccepted(local_run, paid_reservation): + output, hf, harbor, _, submit, _ = local_run + manifest = full_run.run( + output, + hf, + harbor, + "500", + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + assert submit.call_count == 2 and manifest["status"] == "submitted" + assert manifest["shared_spend"]["ledger"] == str(paid_reservation["spend_ledger"]) + assert ( + manifest["accepted"] + is manifest["trainable"] + is manifest["rfc008_qualified"] + is False + ) + assert all( + task["accepted"] is task["trainable"] is task["rfc008_qualified"] is False + for task in manifest["tasks"] + ) + + +def test_prepare_failure_preserves_own_manifest_and_partial_recipes( + local_run, paid_reservation +): + output, hf, harbor, prepare, submit, _ = local_run + + def fail(path, **kwargs): + path.mkdir() + (path / "partial.json").write_text("{}") + raise ValueError("untrusted exception text") + + prepare.side_effect = fail + with pytest.raises(RuntimeError, match="during prepare"): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert manifest["stage"] == "prepare" and manifest["error_type"] == "ValueError" + assert (output / "recipes/partial.json").exists() + assert "untrusted exception text" not in (output / "manifest.json").read_text() + submit.assert_not_called() + + +def test_second_plan_failure_preserves_first_plan_and_stops_all_submits( + local_run, paid_reservation, monkeypatch +): + output, hf, harbor, _, submit, plans = local_run + + def read(path, *, expected_sha256=None): + if Path(path).parent.name == "toy-beta": + raise ValueError("bad plan") + return plans[str(path)] + + monkeypatch.setattr(cloud, "_read_plan", read) + with pytest.raises(RuntimeError, match="during plans"): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert len(manifest["tasks"]) == 1 and manifest["accepted"] is False + assert (output / "recipes/toy-beta/plan.json").exists() + submit.assert_not_called() + + +def test_changed_plan_cannot_reuse_reviewed_estimate( + local_run, paid_reservation, monkeypatch +): + output, hf, harbor, _, submit, plans = local_run + estimate = full_run.estimate + + def change(configs): + result = estimate(configs) + Path(next(iter(plans))).write_text("changed") + return result + + monkeypatch.setattr(full_run, "estimate", change) + with pytest.raises(RuntimeError, match="during submit"): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + **paid_reservation, + panel_path=output.parent / "panel.json", + ) + submit.assert_not_called() + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert Fraction( + spend.summary(paid_reservation["spend_ledger"])["reserved_usd_exact"] + ) == Fraction(manifest["estimate"]["reservation_usd_rounded_up"]) + + +def test_raw_census_is_rejected_without_preparation(local_run): + output, hf, harbor, prepare, _, _ = local_run + hf.write_bytes(b" " * (1024 * 1024 + 1)) + with pytest.raises(RuntimeError, match="during catalogs"): + full_run.run(output, hf, harbor, "500", panel_path=output.parent / "panel.json") + prepare.assert_not_called() + + +def test_cli_requires_explicit_budget_and_defaults_to_preparation(local_run, capsys): + output, hf, harbor, _, submit, _ = local_run + arguments = [ + "--panel", + str(output.parent / "panel.json"), + "--output", + str(output), + "--hf-census", + str(hf), + "--harbor-census", + str(harbor), + ] + with pytest.raises(SystemExit): + full_run.main(arguments) + assert not output.exists() + full_run.main(arguments + ["--budget-usd", "500"]) + assert json.loads(capsys.readouterr().out)["status"] == "prepared" + submit.assert_not_called() + + +@pytest.mark.parametrize("names", [["toy-alpha"], ["toy-beta", "toy-alpha"]]) +def test_cli_passes_repeatable_task_selection(local_run, capsys, names): + output, hf, harbor, prepare, submit, _ = local_run + arguments = [ + "--panel", + str(output.parent / "panel.json"), + "--output", + str(output), + "--hf-census", + str(hf), + "--harbor-census", + str(harbor), + "--budget-usd", + "500", + ] + for name in names: + arguments.extend(["--task", name]) + full_run.main(arguments) + assert_prepared(prepare, output, names) + manifest = cloud._loads((output / "manifest.json").read_bytes()) + assert [Path(task["plan"]).parent.name for task in manifest["tasks"]] == names + assert json.loads(capsys.readouterr().out)["status"] == "prepared" + submit.assert_not_called() + + +@pytest.mark.parametrize( + "selection", [["--task", "unknown"], ["--task", ""], ["--task"]] +) +def test_cli_rejects_unknown_or_empty_task_before_preparation(local_run, selection): + output, hf, harbor, prepare, submit, _ = local_run + with pytest.raises((SystemExit, ValueError)): + full_run.main( + [ + "--panel", + str(output.parent / "panel.json"), + "--output", + str(output), + "--hf-census", + str(hf), + "--harbor-census", + str(harbor), + "--budget-usd", + "500", + *selection, + ] + ) + assert not output.exists() + prepare.assert_not_called() + submit.assert_not_called() + + +def test_shared_ledger_blocks_before_any_paid_submission(local_run): + output, hf, harbor, _, submit, _ = local_run + ledger = output.parent / "spend.json" + spend.init(ledger, "1") + with pytest.raises(RuntimeError, match="during reserve"): + full_run.run( + output, + hf, + harbor, + "500", + submit=True, + spend_ledger=ledger, + reservation_id="cohort-v2", + panel_path=output.parent / "panel.json", + ) + submit.assert_not_called() + assert spend.summary(ledger)["reserved_usd_exact"] == "0" + + +def test_shared_reservation_is_bound_to_the_immutable_cohort(local_run): + output, hf, harbor, _, submit, _ = local_run + ledger = output.parent / "spend.json" + spend.init(ledger, "500") + manifest = full_run.run( + output, + hf, + harbor, + "500", + submit=True, + spend_ledger=ledger, + reservation_id="cohort-v2", + panel_path=output.parent / "panel.json", + ) + assert submit.call_count == 2 + assert manifest["shared_spend"]["reservation_id"].startswith("cohort-v2:") + assert Fraction(spend.summary(ledger)["reserved_usd_exact"]) == Fraction( + manifest["estimate"]["reservation_usd_rounded_up"] + ) + + +def test_paid_cli_requires_shared_reservation(local_run): + output, hf, harbor, _, submit, _ = local_run + with pytest.raises((SystemExit, ValueError)): + full_run.main( + [ + "--panel", + str(output.parent / "panel.json"), + "--output", + str(output), + "--hf-census", + str(hf), + "--harbor-census", + str(harbor), + "--budget-usd", + "500", + "--submit", + ] + ) + assert not output.exists() + submit.assert_not_called() + + +def test_full_run_reads_and_binds_one_panel_snapshot(local_run, monkeypatch): + output, hf, harbor, prepare, _, _ = local_run + panel_path = output.parent / "panel.json" + original_bytes = panel_path.read_bytes() + read_file = cloud._text_file + reads = [] + + def replace_after_read(path): + result = read_file(path) + if path == panel_path: + reads.append(path) + panel_path.write_text('{"replacement": true}') + return result + + monkeypatch.setattr(cloud, "_text_file", replace_after_read) + manifest = full_run.run(output, hf, harbor, "500", panel_path=panel_path) + assert reads == [panel_path] + assert ( + manifest["evaluation_panel"]["sha256"] + == hashlib.sha256(original_bytes).hexdigest() + ) + assert prepare.call_args.kwargs["panel_provenance"] == manifest["evaluation_panel"] + assert prepare.call_args.kwargs["panel"]["tasks"][0]["config"] == toy_config() + + +def test_full_run_requires_explicit_panel_before_preparation(local_run): + output, hf, harbor, prepare, submit, _ = local_run + with pytest.raises(TypeError, match="panel_path"): + full_run.run(output, hf, harbor, "500") + with pytest.raises(SystemExit): + full_run.main( + [ + "--output", + str(output), + "--hf-census", + str(hf), + "--harbor-census", + str(harbor), + "--budget-usd", + "500", + ] + ) + assert not output.exists() + prepare.assert_not_called() + submit.assert_not_called() + + +def test_missing_execution_flavor_uses_backend_cpu_basic_default(): + config = toy_config() + expected = full_run.estimate([config]) + config["execution"].pop("flavor", None) + assert full_run.estimate([config]) == expected + + +def test_budget_gate_compares_actual_rounded_reservation(local_run, monkeypatch): + from decimal import Decimal + + output, hf, harbor, _, submit, _ = local_run + monkeypatch.setitem(full_run.RATEBOOK, "cpu_basic_usd_per_hour", "0.010000000001") + estimated = full_run.estimate([toy_config(name) for name in TOY_NAMES]) + exact = Fraction(estimated["reservation_usd_exact"]) + rounded = Fraction(estimated["reservation_usd_rounded_up"]) + assert exact < rounded + halfway = (exact + rounded) / 2 + cap = str(Decimal(halfway.numerator) / Decimal(halfway.denominator)) + result = full_run.run( + output, hf, harbor, cap, panel_path=output.parent / "panel.json" + ) + assert result["within_budget"] is False + submit.assert_not_called() diff --git a/examples/hub_validation_pipeline/test_pipeline.py b/examples/hub_validation_pipeline/test_pipeline.py new file mode 100644 index 000000000..96ca47062 --- /dev/null +++ b/examples/hub_validation_pipeline/test_pipeline.py @@ -0,0 +1,705 @@ +"""Admission and execution regressions at the report/runner boundaries.""" + +import asyncio +import hashlib +import json +import shutil +from copy import deepcopy +from decimal import Decimal +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from examples.hub_validation_pipeline import demo, pipeline +from openenv.harbor.models import HarborRolloutResult +from openenv.validation.parsers.harbor_toml import HarborTaskParser +from openenv.validation.runner import source_digest + + +@pytest.fixture +def evidence(tmp_path): + return make_evidence(tmp_path) + + +def make_evidence(tmp_path, canary_count=2): + dataset = demo.create_tasks(tmp_path / "input") + subject = tmp_path / "subject" + shutil.copytree(dataset / "good", subject) + canaries = subject / "validation/canaries" + if canaries.exists(): + shutil.rmtree(canaries) + canaries.mkdir(parents=True) + for index in range(canary_count): + (canaries / f"wrong-{index}.sh").write_text(f"#!/bin/bash\necho {index}\n") + original = source_digest(subject) + contract = json.loads( + Path(pipeline.__file__).with_name("contract.json").read_text() + ) + if canary_count: + contract["canaries"] = "validation/canaries" + else: + contract.pop("canaries", None) + pipeline.write_json( + subject / "validation/openenv.json", + {**contract, "original_source_digest": original}, + ) + manifest = HarborTaskParser().parse(subject).model_dump(mode="json") + report = { + "report_schema_version": "2", + "target": str(subject), + "source_digest": source_digest(subject), + "signature": "task.toml", + "manifest": manifest, + "policy_version": "v2", + "lane": "local", + "levels_run": [1, 2, 3], + "verdict": "warn", + "results": [ + {"check_id": key, "status": "pass", "duration_s": 0} + for key in sorted(pipeline.PILOT_CHECKS) + ] + + ( + [{"check_id": "semantic.canary_floor", "status": "pass", "duration_s": 0}] + if canary_count + else [] + ) + + [ + { + "check_id": "runtime.host_containment", + "status": "skip", + "duration_s": 0, + "evidence": ["out of profile: native process boundary"], + }, + ], + } + native = { + "schema_version": "2", + "profile": pipeline.PILOT, + "task_effective_digest": report["source_digest"], + "original_source_digest": original, + "image_ref": "sha256:" + "a" * 64, + "retained_image_tag": "openenv-validation:" + "a" * 32, + "reward_key": "reward", + "floor_reward": 0, + "oracle_reward": 1, + "cleanup": {"verified": True}, + "trials": [], + } + next(row for row in report["results"] if row["check_id"] == "runtime.startup")[ + "measured" + ] = {"retained_image_tag": native["retained_image_tag"]} + artifacts = tmp_path / "evidence" + roles = ["nop", "oracle", "oracle-replay"] + [ + f"canary-{index}" for index in range(1, canary_count + 1) + ] + for index, role in enumerate(roles): + rewards = {"reward": int(role in {"oracle", "oracle-replay"}), "aux": 0} + trial = { + "role": role, + "image_ref": native["image_ref"], + "retained_image_tag": native["retained_image_tag"], + "container_id": f"{index + 1:064x}", + "unmodified_task_digest": report["source_digest"], + "task_overlay": {} + if role == "nop" + else { + "environment.docker_image": native["retained_image_tag"], + }, + "cleanup": {"verified": True}, + "configuration": { + "user": "", + "workdir": "/app", + "host": {"Memory": 512 * 1024**2}, + "network_mode": "compose-default", + "mounts": [], + }, + "rewards": rewards, + "raw_rewards": {}, + } + if role != "nop": + trial.update(oracle_exit_observed_at_agent_end=True, oracle_exit_code=0) + if role.startswith("canary-"): + canary = sorted(canaries.iterdir())[index - 3] + trial["task_overlay"]["solution/solve.sh"] = { + "canary": canary.relative_to(subject).as_posix(), + "sha256": hashlib.sha256(canary.read_bytes()).hexdigest(), + } + native["trials"].append(trial) + set_raw(artifacts, trial, "reward.json", pipeline.dumps(rewards) + "\n") + save_bundle(artifacts, report, native) + return subject, report, artifacts, original, native + + +def set_raw(directory, trial, filename, text): + name = f"harbor-trials/{trial['role']}/{filename}" + path = directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + trial["raw_rewards"][filename] = { + "path": name, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + + +def save_bundle(directory, report, native): + pipeline.write_json(directory / "report.json", report) + pipeline.write_json(directory / "harbor-evidence.json", native) + (directory / "SHA256SUMS").write_text( + "".join( + f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.relative_to(directory)}\n" + for path in sorted(directory.rglob("*")) + if path.is_file() and path.name != "SHA256SUMS" + ) + ) + + +def admit(evidence, profile=pipeline.PILOT): + subject, report, artifacts, original, _ = evidence + return pipeline.admission(report, 0, subject, artifacts, profile, original) + + +def test_explicit_pilot_admits_but_full_gate_stays_incomplete(evidence): + admitted = admit(evidence) + assert admitted["out_of_profile_skips"] == ["runtime.host_containment"] + assert admitted["rfc008_complete"] is False + assert admitted["retained_image_tag"] == evidence[4]["retained_image_tag"] + with pytest.raises(ValueError, match="coverage"): + admit(evidence, "full") + + +def test_full_profile_requires_every_policy_check_to_pass(evidence): + _, report, artifacts, _, native = evidence + report["verdict"] = "pass" + report["results"] = [ + { + "check_id": key, + "status": "pass", + "duration_s": 0, + "measured": {"retained_image_tag": native["retained_image_tag"]} + if key == "runtime.startup" + else {}, + } + for key in pipeline.load_policy("v2").entries_for_lane(pipeline.Lane.LOCAL) + ] + save_bundle(artifacts, report, native) + assert admit(evidence, "full")["rfc008_complete"] is True + report["results"][-1]["status"] = "skip" + with pytest.raises(ValueError, match="coverage"): + admit(evidence, "full") + + +@pytest.mark.parametrize("profile", [pipeline.PILOT, "full"]) +def test_local_validator_requests_semantic_level(tmp_path, monkeypatch, profile): + calls = [] + report = tmp_path / "report.json" + + class Process: + returncode = 0 + + def communicate(self, timeout): + report.write_text('{"validator":"result"}') + return "", "" + + def start(command, **kwargs): + calls.append((command, kwargs)) + return Process() + + monkeypatch.setattr(pipeline.subprocess, "Popen", start) + assert pipeline.validate_task(tmp_path, report, profile, 10) == ( + 0, + {"validator": "result"}, + ) + command = calls[0][0] + assert command[command.index("--level") + 1] == "semantic" + + +@pytest.mark.parametrize("fault", ["missing", "duplicate", "skip", "fail", "unknown"]) +def test_exit_zero_cannot_hide_bad_coverage(evidence, fault): + report = evidence[1] + if fault == "missing": + report["results"].pop(0) + elif fault == "duplicate": + report["results"].append(deepcopy(report["results"][0])) + elif fault == "skip": + report["results"][0]["status"] = "skip" + elif fault == "fail": + report["results"][-1]["status"] = "fail" + else: + report["results"].append( + {"check_id": "runtime.invented", "status": "pass", "duration_s": 0} + ) + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize( + "check", + [ + "runtime.episode_determinism", + "semantic.oracle_max", + "semantic.floor_gap", + "semantic.canary_floor", + ], +) +@pytest.mark.parametrize("status", [None, "skip", "fail", "error"]) +def test_semantic_coverage_is_required(evidence, check, status): + report = evidence[1] + row = next(row for row in report["results"] if row["check_id"] == check) + if status is None: + report["results"].remove(row) + else: + row["status"] = status + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize("canary_count", [0, 1, 8]) +def test_declared_canary_count_sets_the_trial_plan(tmp_path, canary_count): + sample = make_evidence(tmp_path, canary_count) + assert len(sample[-1]["trials"]) == 3 + canary_count + assert admit(sample)["profile"] == "harbor-semantic-v1" + + +@pytest.mark.parametrize( + "fault", + [ + "missing", + "reordered", + "duplicate-container", + "image", + "source", + "failed", + "cleanup", + "top-cleanup", + "configuration", + "missing-floor", + "missing-tag", + ], +) +def test_every_trial_must_complete_with_bound_identity(evidence, fault): + _, report, artifacts, _, native = evidence + trials = native["trials"] + if fault == "missing": + trials.pop() + elif fault == "reordered": + trials[1], trials[2] = trials[2], trials[1] + elif fault == "duplicate-container": + trials[-1]["container_id"] = trials[0]["container_id"] + elif fault == "image": + trials[-1]["image_ref"] = "sha256:" + "b" * 64 + elif fault == "source": + trials[-1]["unmodified_task_digest"] = "b" * 64 + elif fault == "failed": + trials[-1]["failure"] = "RewardArtifactDrift" + elif fault == "cleanup": + trials[-1]["cleanup"]["verified"] = False + elif fault == "top-cleanup": + native["cleanup"]["verified"] = False + elif fault == "configuration": + trials[-1]["configuration"]["workdir"] = "/different" + elif fault == "missing-floor": + native.pop("floor_reward") + else: + native.pop("retained_image_tag") + save_bundle(artifacts, report, native) + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize( + "fault", ["reward-map", "exit", "unobserved", "marker", "unbound-marker"] +) +def test_oracle_replay_checks_every_reward_and_trusted_exit(evidence, fault): + _, report, artifacts, _, native = evidence + replay = native["trials"][2] + if fault == "reward-map": + replay["rewards"]["aux"] = 1 + set_raw(artifacts, replay, "reward.json", pipeline.dumps(replay["rewards"])) + elif fault == "exit": + replay["oracle_exit_code"] = 1 + set_raw(artifacts, replay, "oracle-exit-code.txt", "1\n") + elif fault == "unobserved": + replay["oracle_exit_observed_at_agent_end"] = False + else: + set_raw(artifacts, replay, "oracle-exit-code.txt", "1\n") + save_bundle(artifacts, report, native) + if fault == "unbound-marker": + inventory = artifacts / "SHA256SUMS" + inventory.write_text( + "".join( + row + for row in inventory.read_text().splitlines(keepends=True) + if not row.rstrip().endswith("oracle-exit-code.txt") + ) + ) + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize("fault", ["reward", "path", "digest", "missing", "extra"]) +def test_canary_reward_and_overlay_match_declared_bytes(evidence, fault): + _, report, artifacts, _, native = evidence + canary = native["trials"][-1] + overlay = canary["task_overlay"] + if fault == "reward": + canary["rewards"]["reward"] = 1 + set_raw(artifacts, canary, "reward.json", pipeline.dumps(canary["rewards"])) + elif fault == "path": + overlay["solution/solve.sh"]["canary"] = "solution/solve.sh" + elif fault == "digest": + overlay["solution/solve.sh"]["sha256"] = "b" * 64 + elif fault == "missing": + overlay.pop("solution/solve.sh") + else: + overlay["tests/test.sh"] = "changed" + save_bundle(artifacts, report, native) + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize( + "raw", + [ + '{"reward":true}', + '{"reward":"1"}', + '{"reward":NaN}', + '{"reward":1,"aux":2}', + '{"reward":1,"reward":0}', + "{}", + ], +) +def test_raw_reward_maps_fail_closed(evidence, raw): + _, report, artifacts, _, native = evidence + set_raw(artifacts, native["trials"][1], "reward.json", raw) + save_bundle(artifacts, report, native) + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize( + "fault", + ["old-profile", "old-schema", "missing-schema", "operator-profile", "runtime-only"], +) +def test_stale_native_contract_is_rejected(evidence, fault): + _, report, artifacts, _, native = evidence + if fault == "old-profile": + native["profile"] = "harbor-oracle-floor-v1" + elif fault == "old-schema": + native["schema_version"] = "1" + elif fault == "missing-schema": + native.pop("schema_version") + elif fault == "runtime-only": + report["levels_run"] = [1, 2] + save_bundle(artifacts, report, native) + with pytest.raises(ValueError): + admit( + evidence, + "harbor-oracle-floor-v1" if fault == "operator-profile" else pipeline.PILOT, + ) + + +def test_admission_recomputes_oracle_floor_from_raw_rewards(evidence): + _, report, artifacts, _, native = evidence + trial = native["trials"][0] + trial["rewards"]["reward"] = 1 + set_raw(artifacts, trial, "reward.json", pipeline.dumps(trial["rewards"])) + native["floor_reward"] = 1 + # Even a self-consistent bundle with stale semantic PASS results is rejected. + save_bundle(artifacts, report, native) + with pytest.raises(ValueError, match="oracle/floor contract"): + admit(evidence) + + +@pytest.mark.parametrize( + "fault", ["missing-measured", "missing-tag", "malformed", "different"] +) +def test_startup_measurement_binds_retained_image(evidence, fault): + _, report, artifacts, _, native = evidence + startup = next( + row for row in report["results"] if row["check_id"] == "runtime.startup" + ) + if fault == "missing-measured": + startup.pop("measured") + elif fault == "missing-tag": + startup["measured"].pop("retained_image_tag") + else: + startup["measured"]["retained_image_tag"] = ( + "unowned:tag" if fault == "malformed" else "openenv-validation:" + "b" * 32 + ) + save_bundle(artifacts, report, native) + with pytest.raises(ValueError): + admit(evidence) + + +@pytest.mark.parametrize( + "reward_range,floor,oracle,margin,tolerance,accepted", + [ + ([0, 100], 0, 95, 0.1, 0.05, True), + ([50, 150], 50, 145, 0.1, 0.05, True), + ([0, 0.01], 0, 0.0095, 0.1, 0.05, True), + ([0, 100], 90, 100, 0.1, 0, True), + ([0, 100], 99, 100, 0.1, 0, False), + ([50, 150], 149, 150, 0.1, 0, False), + ([0, 0.01], 0.0099, 0.01, 0.1, 0, False), + ([0, 100], 0, 94.99, 0.1, 0.05, False), + ([0, 0.01], 0, 0.009499, 0.1, 0.05, False), + ], +) +def test_admission_scales_thresholds_by_reward_range( + evidence, reward_range, floor, oracle, margin, tolerance, accepted +): + subject, report, artifacts, _, native = evidence + path = subject / "validation/openenv.json" + contract = pipeline.loads(path.read_text()) + reward = { + "range": reward_range, + "floor_margin": margin, + "oracle_tolerance": tolerance, + } + contract["reward"].update(reward) + pipeline.write_json(path, contract) + report["manifest"]["reward"].update(reward) + report["source_digest"] = source_digest(subject) + native["task_effective_digest"] = report["source_digest"] + for trial in native["trials"]: + trial["unmodified_task_digest"] = report["source_digest"] + trial["rewards"]["reward"] = ( + oracle if trial["role"] in {"oracle", "oracle-replay"} else floor + ) + trial["rewards"]["aux"] = reward_range[0] + set_raw(artifacts, trial, "reward.json", pipeline.dumps(trial["rewards"])) + native.update(floor_reward=floor, oracle_reward=oracle) + save_bundle(artifacts, report, native) + # Recompute the contract even if all bundled semantic results claim PASS. + if accepted: + assert pipeline.reward_fraction(admit(evidence)["oracle_reward"]) == ( + pipeline.reward_fraction(Decimal(str(oracle))) + ) + else: + with pytest.raises(ValueError, match="oracle/floor contract"): + admit(evidence) + + +@pytest.mark.parametrize("missing_oracle", [False, True]) +def test_native_unpinned_image_is_excluded_without_model_or_docker( + tmp_path, monkeypatch, missing_oracle +): + from openenv.validation.harbor import DockerInspector + from openenv.validation.runner import run_validation + + task = demo.create_tasks(tmp_path / "input") / "good" + manifest = task / "task.toml" + config = pipeline.tomllib.loads(manifest.read_text()) + config.setdefault("environment", {})["docker_image"] = ( + "registry.example/task:latest" + ) + manifest.write_text(pipeline.tomli_w.dumps(config)) + if missing_oracle: + (task / "solution/solve.sh").unlink() + original = manifest.read_bytes() + reports = [] + + def validate(subject, report_path, profile, timeout): + report = run_validation(subject).model_dump(mode="json") + reports.append(report) + return 0, report + + def forbidden(*args, **kwargs): + pytest.fail("excluded native task must not execute Docker or a model") + + monkeypatch.setattr(pipeline, "validate_task", validate) + monkeypatch.setattr(pipeline, "evaluate", forbidden) + monkeypatch.setattr(DockerInspector, "_docker", forbidden) + output = tmp_path / "output" + args = SimpleNamespace( + output=output, + contract=Path(pipeline.__file__).with_name("contract.json"), + local_dataset=[task], + hf_dataset=[], + harbor_dataset=[], + tasks_per_dataset=1, + profile=pipeline.PILOT, + validation_timeout=1, + llm_url="http://unused-model/v1", + ) + assert pipeline.run(args) == 1 + checks = {row["check_id"]: row for row in reports[0]["results"]} + assert checks["runtime.startup"]["status"] == "skip" + assert "docker_image" in checks["runtime.startup"]["evidence"][0] + if missing_oracle: + assert checks["semantic.oracle_max"]["status"] == "fail" + assert manifest.read_bytes() == original + assert list((output / "subjects").glob("*/task.toml"))[0].read_bytes() == original + summary = pipeline.loads((output / "summary.json").read_text()) + assert (summary["accepted"], summary["excluded"], summary["model_trials"]) == ( + 0, + 1, + 0, + ) + + +@pytest.mark.parametrize( + "floor,oracle,margin,tolerance", + [ + (Decimal("0.9"), 1, Decimal("0.100000000000000001"), 0), + (0, Decimal("0.95"), 1, Decimal("0.1")), + ], +) +def test_exact_operator_margin_overrides_rounded_semantic_pass( + evidence, monkeypatch, floor, oracle, margin, tolerance +): + subject, report, artifacts, _, native = evidence + path = subject / "validation/openenv.json" + contract = pipeline.loads(path.read_text()) + contract["reward"].update(floor_margin=margin, oracle_tolerance=tolerance) + pipeline.write_json(path, contract) + report["manifest"]["reward"].update( + floor_margin=float(margin), oracle_tolerance=float(tolerance) + ) + # Exercise admission independently of the parser's representability guard. + monkeypatch.setattr( + pipeline, + "read_harbor_contract", + lambda root: SimpleNamespace(canaries="validation/canaries"), + ) + report["source_digest"] = source_digest(subject) + native["task_effective_digest"] = report["source_digest"] + for trial in native["trials"]: + trial["unmodified_task_digest"] = report["source_digest"] + trial["rewards"]["reward"] = ( + oracle if trial["role"] in {"oracle", "oracle-replay"} else floor + ) + set_raw(artifacts, trial, "reward.json", pipeline.dumps(trial["rewards"])) + native.update(floor_reward=floor, oracle_reward=oracle) + save_bundle(artifacts, report, native) + with pytest.raises(ValueError, match="oracle/floor contract"): + admit(evidence) + + +def test_bundled_report_must_equal_the_cli_result(evidence): + _, report, artifacts, _, native = evidence + bundled = deepcopy(report) + bundled["target"] = "different-subject" + save_bundle(artifacts, bundled, native) + with pytest.raises(ValueError, match="bundled report differs"): + admit(evidence) + + +def test_raw_output_cannot_redirect_to_another_trials_file(evidence): + _, report, artifacts, _, native = evidence + native["trials"][2]["raw_rewards"] = deepcopy(native["trials"][1]["raw_rewards"]) + save_bundle(artifacts, report, native) + with pytest.raises(ValueError, match="verified artifact inventory"): + admit(evidence) + + +def test_tampered_artifact_or_subject_cannot_be_reused(evidence): + subject, _, artifacts, _, _ = evidence + (artifacts / "harbor-evidence.json").write_text("{}") + with pytest.raises(ValueError, match="checksum"): + admit(evidence) + (subject / "instruction.md").write_text("changed") + with pytest.raises(ValueError, match="changed after validation"): + admit(evidence) + + +def approved_task(evidence, output): + record = {**admit(evidence), "subject": str(evidence[0]), "task_id": "good-task"} + return pipeline.materialize(record, output / "approved/tasks/good") + + +def test_execution_pins_only_declared_overlay_and_preserves_validated_source( + evidence, tmp_path +): + subject = evidence[0] + original = (subject / "task.toml").read_bytes() + record = approved_task(evidence, tmp_path / "output") + assert (subject / "task.toml").read_bytes() == original + assert source_digest(Path(record["execution_path"])) == record["execution_digest"] + assert ( + record["execution_overlay"]["environment"]["docker_image"] + == record["image_ref"] + ) + + +def test_copy_omits_unhashed_files_and_never_dereferences_links(evidence, tmp_path): + source = evidence[0] + hidden = source / ".worktrees" + hidden.mkdir() + (hidden / "host-file").symlink_to("/etc/hosts") + expected = source_digest(source) + destination = tmp_path / "copy" + pipeline.copy_subject(source, destination, expected) + assert not (destination / ".worktrees").exists() + (source / "escaped").symlink_to("/etc/hosts") + with pytest.raises(ValueError, match="symbolic links"): + pipeline.copy_subject(source, tmp_path / "unsafe", expected) + + +def test_only_approved_tasks_reach_existing_model_runner( + evidence, tmp_path, monkeypatch +): + from openenv.harbor import runner + + output = tmp_path / "output" + record = approved_task(evidence, output) + call = AsyncMock( + return_value=[ + HarborRolloutResult(task_id=record["execution_path"], reward=0, ok=True) + ] + ) + monkeypatch.setattr(runner, "run_batch", call) + rows = pipeline.evaluate( + [record], + output, + llm_url="http://model/v1", + model="test", + harness="opencode", + expose="direct", + timeout_s=10, + step_limit=4, + ) + assert len(rows) == 1 and rows[0]["result"]["reward"] == 0 + assert call.call_args.kwargs["task_indices"] == [0] + assert call.call_args.kwargs["agent_step_limit"] == 4 + # An unapproved sibling must stop the runner before another model call. + shutil.copytree(record["execution_path"], output / "approved/tasks/unapproved") + from openenv.harbor.tasks import resolve_task_dirs + + resolve_task_dirs(str(output / "approved"), refresh=True) + with pytest.raises(ValueError, match="corpus disagree"): + pipeline.evaluate( + [record], + output, + llm_url="http://model/v1", + model="test", + harness="opencode", + expose="direct", + timeout_s=10, + step_limit=4, + ) + assert call.await_count == 1 + + +def test_model_image_guard_runs_before_agent_execution(evidence, tmp_path, monkeypatch): + record = approved_task(evidence, tmp_path / "output") + hooks = [] + trial = SimpleNamespace( + config=SimpleNamespace( + task=SimpleNamespace(path=Path(record["execution_path"])) + ), + agent_environment=SimpleNamespace(session_id="oe-good__env"), + add_hook=lambda event, callback: hooks.append(callback), + ) + actual = {"Image": "sha256:" + "b" * 64} + monkeypatch.setattr( + pipeline, + "docker", + lambda *args: "container" if args[0] == "ps" else json.dumps([actual]), + ) + projects = set() + pipeline.model_guard([record], projects)(trial) + assert projects == {"oe-good__env"} + with pytest.raises(ValueError, match="image differs"): + asyncio.run(hooks[0](None)) diff --git a/examples/hub_validation_pipeline/test_prepared_batch.py b/examples/hub_validation_pipeline/test_prepared_batch.py new file mode 100644 index 000000000..ccf3a514a --- /dev/null +++ b/examples/hub_validation_pipeline/test_prepared_batch.py @@ -0,0 +1,830 @@ +"""Offline cohort binding, spend gates and one-shot controller polling.""" + +import hashlib +import json +import shutil +import socket +import time +from collections import Counter +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from examples.hub_validation_pipeline import ( + cloud, + full_run, + prepared_batch, + real_tasks, + spend, +) +from examples.hub_validation_pipeline.test_real_tasks import hf_fakes, write_panel + +SECRETS = dict( + HF_TOKEN="local-results", + HF_INFERENCE_TOKEN="local-inference", + HF_SANDBOX_TOKEN="local-execution", +) + + +@pytest.fixture +def cohort(tmp_path, monkeypatch): + forbidden = Mock(side_effect=AssertionError("Live operations forbidden")) + monkeypatch.setattr(socket.socket, "connect", forbidden) + import huggingface_hub + + monkeypatch.setattr(huggingface_hub, "HfApi", forbidden) + monkeypatch.setattr(huggingface_hub, "get_token", forbidden) + repo = tmp_path / "repo" + (repo / "src/openenv").mkdir(parents=True) + (repo / "src/openenv/__init__.py").write_text("# source\n") + (repo / "examples/hub_validation_pipeline").mkdir(parents=True) + (repo / "examples/hub_validation_pipeline/cloud.py").write_bytes( + Path(cloud.__file__).read_bytes() + ) + shutil.copytree( + Path(cloud.__file__).parents[1] / "coding_audit", + repo / "examples/coding_audit", + ignore=shutil.ignore_patterns("test_*", "__pycache__", ".pytest_cache"), + ) + for name in ("pyproject.toml", "README.md", "LICENSE", "uv.lock"): + (repo / name).write_text(name + "\n") + monkeypatch.setattr( + cloud, "_git_provenance", lambda root: {"commit": "a" * 40, "dirty": False} + ) + task = tmp_path / "task" + (task / "tests").mkdir(parents=True) + (task / "solution").mkdir() + (task / "task.toml").write_text("version = '1.0'\n") + (task / "tests/test.sh").write_text("printf 1\n") + (task / "solution/solve.sh").write_text("true\n") + prepared = tmp_path / "prepared" + configs, plans, rows = [], [], [] + for index in range(4): + (task / "instruction.md").write_text(f"Distinct task {index}\n") + config = cloud._loads( + (Path(__file__).parents[1] / "coding_audit/config.json").read_bytes() + ) + config["cloud"].update(namespace="owner", timeout_seconds=60) + path = prepared / f"task-{index}" / "plan.json" + cloud.prepare(task, config, path.parent, repo_root=repo) + configs.append(config) + plans.append(path) + rows.append( + dict( + task_id=f"task-{index}", + plan_path=str(path.relative_to(prepared)), + plan_sha256=full_run._hash(path)["sha256"], + ) + ) + source = prepared / "preparation-manifest.json" + source.write_bytes( + cloud._json_bytes({"tasks": rows, "evaluation_protocol": "test"}) + ) + ledger = tmp_path / "local-spend.json" + spend.init(ledger, "500") + output = tmp_path / "batch" + api = Mock() + now, sleeps, events = [0], [], [] + counts, launched, active = Counter(), [], set() + + def sleep(seconds): + assert 0 <= seconds <= 30 + sleeps.append(seconds) + now[0] += seconds + + actual_submit = cloud.submit + + def submit( + path, *, api, secrets, expected_plan_sha256, spend_ledger, reservation_id + ): + assert secrets == SECRETS + assert full_run._hash(path)["sha256"] == expected_plan_sha256 + durable = cloud._loads((output / "manifest.json").read_bytes()) + assert spend_ledger == ledger + assert reservation_id == durable["shared_spend"]["reservation_id"] + index = plans.index(path) + assert durable["tasks"][index]["status"] == "pending_or_unknown" + if index: + assert durable["tasks"][index - 1]["job_id"] == f"job-{index - 1}" + assert durable["tasks"][index - 1]["status"] == "accepted_unmeasured" + assert durable["shared_spend"]["status"] == "reserved" + assert ( + spend.summary(ledger)["reserved_usd_rounded_up"] + == durable["estimate"]["reservation_usd_rounded_up"] + ) + job_id = f"job-{index}" + launched.append(job_id) + active.add(job_id) + assert len(active) <= 2 + events.append(("submit", job_id, now[0])) + plan = cloud._loads(path.read_bytes()) + receipt = dict( + status="submitted", + run_id=plan["run_id"], + job_id=job_id, + job_url=f"https://huggingface.co/jobs/owner/{job_id}", + launch_spec=plan["mcp_request"]["args"], + ) + (path.parent / "submission.json").write_bytes(cloud._json_bytes(receipt)) + return receipt + + def inspect_job(*, job_id, namespace, token): + assert job_id in launched + assert namespace == "owner" and token == SECRETS["HF_SANDBOX_TOKEN"] + counts[job_id] += 1 + stage = "RUNNING" if counts[job_id] == 1 else "COMPLETED" + if stage == "COMPLETED": + active.remove(job_id) + events.append(("inspect", job_id, now[0])) + return SimpleNamespace( + id=job_id, + docker_image=cloud.WORKER_IMAGE, + owner=SimpleNamespace(name="owner"), + status=SimpleNamespace(stage=stage), + ) + + launch = Mock(side_effect=submit) + api.inspect_job.side_effect = inspect_job + monkeypatch.setattr(cloud, "submit", launch) + return SimpleNamespace( + source=source, + output=output, + ledger=ledger, + configs=configs, + plans=plans, + rows=rows, + api=api, + actual_submit=actual_submit, + launch=launch, + now=now, + sleeps=sleeps, + events=events, + arguments=dict( + prepared_manifest=source, + output=output, + spend_ledger=ledger, + reservation_id="cohort-v1", + budget_usd="500", + api=api, + secrets=SECRETS, + sleep=sleep, + clock=lambda: now[0], + ), + ) + + +def saved(cohort): + return cloud._loads((cohort.output / "manifest.json").read_bytes()) + + +def rewrite_manifest(cohort): + cohort.source.write_bytes(cloud._json_bytes({"tasks": cohort.rows})) + + +@pytest.mark.parametrize("producer", ["recipes", "full-run"]) +def test_producer_manifest_launches_without_rewriting_rows( + cohort, monkeypatch, tmp_path, producer +): + source_api, download = hf_fakes() + monkeypatch.setattr(real_tasks, "HfApi", Mock(return_value=source_api)) + monkeypatch.setattr(real_tasks, "hf_hub_download", download) + prepare = cloud.prepare + monkeypatch.setattr( + cloud, + "prepare", + lambda *args, **kwargs: prepare(*args, repo_root=tmp_path / "repo", **kwargs), + ) + output = tmp_path / producer + panel = write_panel(tmp_path) + if producer == "recipes": + produced = real_tasks.prepare(output, panel_path=panel, bundle=True) + else: + hf, harbor = tmp_path / "hf.json", tmp_path / "harbor.json" + hf.write_bytes( + cloud._json_bytes( + dict( + scope="fixture", + counts={}, + compact_repo_inventory="missing", + compact_identity_inventory="missing", + ) + ) + ) + harbor.write_bytes( + cloud._json_bytes( + dict( + scope="fixture", + counts={}, + artifacts={ + name: {"path": "missing"} + for name in ( + "task_identity_inventory", + "dataset_version_membership_index", + ) + }, + ) + ) + ) + produced = full_run.run(output, hf, harbor, "500", panel_path=panel) + source = output / "manifest.json" + source_bytes = source.read_bytes() + api = Mock() + api.dataset_info.return_value = SimpleNamespace(private=True, sha="a" * 40) + api.list_repo_files.return_value = [] + api.create_commit.return_value = SimpleNamespace(oid="b" * 40) + api.run_job.side_effect = [ + SimpleNamespace( + id=f"job-{index}", url=f"https://huggingface.co/jobs/owner/job-{index}" + ) + for index in range(len(produced["tasks"])) + ] + api.inspect_job.side_effect = lambda *, job_id, **kwargs: SimpleNamespace( + id=job_id, + docker_image=cloud.WORKER_IMAGE, + owner=SimpleNamespace(name="owner"), + status=SimpleNamespace(stage="COMPLETED"), + ) + monkeypatch.setattr(cloud, "submit", cohort.actual_submit) + arguments = dict( + cohort.arguments, + prepared_manifest=source, + output=tmp_path / "launched", + api=api, + ) + result = prepared_batch.run(**arguments, submit=True) + assert result["status"] == "controllers_completed_unmeasured" + assert source.read_bytes() == source_bytes + assert result["prepared_manifest"] == full_run._hash(source) + assert ( + api.run_job.call_count == api.inspect_job.call_count == len(produced["tasks"]) + ) + for row, task in zip(produced["tasks"], result["tasks"], strict=True): + path = source.parent / row["plan_path"] + assert not Path(row["plan_path"]).is_absolute() + assert full_run._hash(path)["sha256"] == row["plan_sha256"] + receipt = cloud._loads((path.parent / "submission.json").read_bytes()) + assert receipt["status"] == "submitted" and receipt["job_id"] == task["job_id"] + launched = api.run_job.call_count + with pytest.raises(RuntimeError, match="during plans"): + prepared_batch.run(**dict(arguments, output=tmp_path / "retry"), submit=True) + assert api.run_job.call_count == launched + + +def test_prepare_only_has_no_credential_network_or_ledger_io(cohort, monkeypatch): + forbidden = Mock(side_effect=AssertionError("No credentials/ledger IO")) + monkeypatch.setattr(cloud, "_runtime_secrets", forbidden) + monkeypatch.setattr(spend, "reserve", forbidden) + cohort.ledger.unlink() + result = prepared_batch.run(**cohort.arguments) + assert result == saved(cohort) + assert result["status"] == "prepared" + assert result["accepted"] is result["results_collected"] is False + assert result["estimate"] == full_run.estimate(cohort.configs) + assert result["prepared_manifest"] == full_run._hash(cohort.source) + assert result["shared_spend"]["status"] == "not_reserved" + assert not cohort.ledger.exists() + cohort.launch.assert_not_called() + cohort.api.inspect_job.assert_not_called() + forbidden.assert_not_called() + + +def test_exact_aggregate_is_reserved_first_and_only_two_controllers_run(cohort): + result = prepared_batch.run(**cohort.arguments, submit=True) + assert result == saved(cohort) + assert result["status"] == "controllers_completed_unmeasured" + assert result["accepted"] is result["results_collected"] is False + assert all( + task["status"] == "accepted_unmeasured" + and task["controller_status"] == "COMPLETED" + and task["accepted"] is False + for task in result["tasks"] + ) + assert cohort.sleeps == [30, 30] + assert cohort.events == [ + ("submit", "job-0", 0), + ("submit", "job-1", 0), + ("inspect", "job-0", 0), + ("inspect", "job-1", 0), + ("inspect", "job-0", 30), + ("inspect", "job-1", 30), + ("submit", "job-2", 30), + ("submit", "job-3", 30), + ("inspect", "job-2", 30), + ("inspect", "job-3", 30), + ("inspect", "job-2", 60), + ("inspect", "job-3", 60), + ] + fingerprint = hashlib.sha256( + "\0".join(task["bundle_sha256"] for task in result["tasks"]).encode() + ).hexdigest()[:16] + assert result["shared_spend"]["reservation_id"] == "cohort-v1:" + fingerprint + ledger = json.loads(cohort.ledger.read_text()) + assert ( + ledger["reservations"]["cohort-v1:" + fingerprint]["reserved_usd"] + == result["estimate"]["reservation_usd_rounded_up"] + ) + text = (cohort.output / "manifest.json").read_text() + assert all( + value not in text + for value in (*SECRETS.values(), "launch_spec", "command", "coding-config") + ) + + +@pytest.mark.parametrize("global_cap", [False, True]) +def test_aggregate_caps_stop_all_submissions(cohort, monkeypatch, global_cap): + if global_cap: + spend.reserve(cohort.ledger, "prior", "500", "Prior unresolved work") + else: + cohort.arguments["budget_usd"] = full_run.estimate(cohort.configs[:3])[ + "reservation_usd_rounded_up" + ] + credentials = Mock(side_effect=AssertionError("No token IO before budget gate")) + monkeypatch.setattr(cloud, "_runtime_secrets", credentials) + with pytest.raises( + RuntimeError, match="during reserve" if global_cap else "during budget" + ): + prepared_batch.run(**cohort.arguments, submit=True) + assert saved(cohort)["status"] == "failed" + cohort.launch.assert_not_called() + cohort.api.inspect_job.assert_not_called() + credentials.assert_not_called() + + +@pytest.mark.parametrize( + "defect", + [ + "empty", + "bad-json", + "duplicate-key", + "changed-plan", + "bad-bundle", + "duplicate-task", + "duplicate-run", + "absolute", + "escape", + "symlink", + "receipt", + "dangling-receipt", + ], +) +def test_invalid_cohort_stops_before_spend_or_credentials(cohort, monkeypatch, defect): + if defect == "empty": + cohort.rows.clear() + elif defect == "changed-plan": + cohort.plans[-1].write_text("changed") + elif defect == "bad-bundle": + (cohort.plans[-1].parent / "bundle.zip").write_bytes(b"changed") + elif defect == "duplicate-task": + cohort.rows[-1]["task_id"] = cohort.rows[0]["task_id"] + elif defect == "duplicate-run": + shutil.rmtree(cohort.plans[-1].parent) + shutil.copytree(cohort.plans[0].parent, cohort.plans[-1].parent) + cohort.rows[-1]["plan_sha256"] = cohort.rows[0]["plan_sha256"] + elif defect == "absolute": + cohort.rows[-1]["plan_path"] = str(cohort.plans[-1]) + elif defect == "escape": + cohort.rows[-1]["plan_path"] = "../task/plan.json" + elif defect == "symlink": + outside = cohort.source.parent.parent / "outside" + shutil.copytree(cohort.plans[-1].parent, outside) + shutil.rmtree(cohort.plans[-1].parent) + cohort.plans[-1].parent.symlink_to(outside, target_is_directory=True) + elif defect in {"receipt", "dangling-receipt"}: + receipt = cohort.plans[-1].parent / "submission.json" + if defect == "receipt": + receipt.write_text('{"status":"launch_unknown"}') + else: + receipt.symlink_to(receipt.parent / "missing") + rewrite_manifest(cohort) + if defect == "bad-json": + cohort.source.write_text("{") + elif defect == "duplicate-key": + cohort.source.write_text('{"tasks":[],"tasks":[]}') + forbidden = Mock(side_effect=AssertionError("No paid calls/credentials")) + monkeypatch.setattr(spend, "reserve", forbidden) + monkeypatch.setattr(cloud, "_runtime_secrets", forbidden) + with pytest.raises(RuntimeError, match="during plans"): + prepared_batch.run(**cohort.arguments, submit=True) + forbidden.assert_not_called() + cohort.launch.assert_not_called() + + +@pytest.mark.parametrize("which", ["plan", "manifest"]) +def test_change_after_estimate_is_caught_before_reservation(cohort, monkeypatch, which): + estimate = full_run.estimate + + def change(configs): + result = estimate(configs) + (cohort.source if which == "manifest" else cohort.plans[-1]).write_text( + "changed" + ) + return result + + monkeypatch.setattr(full_run, "estimate", change) + reserve = Mock(side_effect=AssertionError("No reservation")) + monkeypatch.setattr(spend, "reserve", reserve) + with pytest.raises(RuntimeError, match="during budget"): + prepared_batch.run(**cohort.arguments, submit=True) + reserve.assert_not_called() + cohort.launch.assert_not_called() + + +@pytest.mark.parametrize("stage", ["ERROR", "CANCELED", "DELETED", "NEW_STATE"]) +def test_terminal_failure_and_unknown_stop_further_launches(cohort, stage): + cohort.api.inspect_job.side_effect = lambda **kw: SimpleNamespace( + id=kw["job_id"], + docker_image=cloud.WORKER_IMAGE, + owner=SimpleNamespace(name="owner"), + status=SimpleNamespace(stage=stage), + ) + with pytest.raises(RuntimeError, match="during poll"): + prepared_batch.run(**cohort.arguments, submit=True) + result = saved(cohort) + assert cohort.launch.call_count == 2 + assert result["tasks"][0]["controller_status"] == ( + stage if stage in prepared_batch.STAGES else "UNKNOWN" + ) + assert [task.get("job_id") for task in result["tasks"]] == [ + "job-0", + "job-1", + None, + None, + ] + assert all((path.parent / "submission.json").exists() for path in cohort.plans[:2]) + + +@pytest.mark.parametrize("field", ["id", "docker_image", "owner", "api-error"]) +def test_inspection_failure_or_wrong_controller_stops(cohort, field): + inspect = cohort.api.inspect_job.side_effect + + def bad(**kwargs): + if field == "api-error": + raise RuntimeError("private-inspection-error") + job = inspect(**kwargs) + if field == "owner": + job.owner.name = "other" + else: + setattr(job, field, "wrong") + return job + + cohort.api.inspect_job.side_effect = bad + with pytest.raises(RuntimeError, match="during poll") as error: + prepared_batch.run(**cohort.arguments, submit=True) + assert cohort.launch.call_count == 2 + assert saved(cohort)["tasks"][0]["controller_status"] == "UNKNOWN" + assert ( + "private-inspection-error" + not in str(error.value) + (cohort.output / "manifest.json").read_text() + ) + + +def test_timeout_uses_hard_controller_cap_plus_startup(cohort): + cohort.api.inspect_job.side_effect = lambda **kw: SimpleNamespace( + id=kw["job_id"], + docker_image=cloud.WORKER_IMAGE, + owner=SimpleNamespace(name="owner"), + status=SimpleNamespace(stage="SCHEDULING"), + ) + with pytest.raises(RuntimeError, match="during poll"): + prepared_batch.run(**cohort.arguments, submit=True) + assert cohort.now[0] == 60 + full_run.RATEBOOK["startup_allowance_seconds"] + assert cohort.sleeps == [30] * 12 + assert cohort.launch.call_count == 2 + assert saved(cohort)["tasks"][0]["controller_status"] == "TIMEOUT" + + +@pytest.mark.parametrize("wrong_owner", [False, True]) +def test_default_namespace_still_checks_owner_from_submission_url(cohort, wrong_owner): + config = cohort.configs[0] + config["cloud"].pop("namespace") + shutil.rmtree(cohort.plans[0].parent) + root = cohort.source.parent.parent + cloud.prepare( + root / "task", config, cohort.plans[0].parent, repo_root=root / "repo" + ) + cohort.rows[:] = cohort.rows[:1] + cohort.rows[0]["plan_sha256"] = full_run._hash(cohort.plans[0])["sha256"] + rewrite_manifest(cohort) + + def inspect_job(*, job_id, namespace, token): + assert namespace is None and token == SECRETS["HF_SANDBOX_TOKEN"] + return SimpleNamespace( + id=job_id, + docker_image=cloud.WORKER_IMAGE, + owner=SimpleNamespace(name="other" if wrong_owner else "owner"), + status=SimpleNamespace(stage="COMPLETED"), + ) + + cohort.api.inspect_job.side_effect = inspect_job + if wrong_owner: + with pytest.raises(RuntimeError, match="during poll"): + prepared_batch.run(**cohort.arguments, submit=True) + else: + assert ( + prepared_batch.run(**cohort.arguments, submit=True)["status"] + == "controllers_completed_unmeasured" + ) + + +@pytest.mark.parametrize("receipt_kind", ["unknown", "known-id", "partial"]) +def test_uncertain_launch_preserves_every_receipt_and_never_retries( + cohort, receipt_kind +): + launch = cohort.launch.side_effect + + def fail(path, **kwargs): + if path != cohort.plans[1]: + return launch(path, **kwargs) + receipt = dict( + status="launch_unknown", run_id=cloud._loads(path.read_bytes())["run_id"] + ) + if receipt_kind == "known-id": + receipt.update( + job_id="job-uncertain", + job_url="https://huggingface.co/jobs/owner/job-uncertain", + ) + (path.parent / "submission.json").write_bytes( + b"{" if receipt_kind == "partial" else cloud._json_bytes(receipt) + ) + raise RuntimeError("private-launch-error") + + cohort.launch.side_effect = fail + with pytest.raises(RuntimeError, match="during submit") as error: + prepared_batch.run(**cohort.arguments, submit=True) + result = saved(cohort) + assert cohort.launch.call_count == 2 + assert result["tasks"][0]["job_id"] == "job-0" + assert result["tasks"][1]["status"] == "pending_or_unknown" + if receipt_kind == "known-id": + assert result["tasks"][1]["job_id"] == "job-uncertain" + assert (cohort.plans[1].parent / "submission.json").exists() + assert ( + "private-launch-error" + not in str(error.value) + (cohort.output / "manifest.json").read_text() + ) + with pytest.raises(FileExistsError): + prepared_batch.run(**cohort.arguments, submit=True) + cohort.arguments["output"] = cohort.output.parent / "different-output" + with pytest.raises(RuntimeError, match="during plans"): + prepared_batch.run(**cohort.arguments, submit=True) + assert cohort.launch.call_count == 2 + + +def test_cli_requires_all_binding_arguments_and_defaults_to_prepare(cohort, capsys): + arguments = [ + "--prepared-manifest", + str(cohort.source), + "--output", + str(cohort.output), + "--spend-ledger", + str(cohort.ledger), + "--reservation-id", + "cohort", + "--budget-usd", + "500", + ] + for index in range(0, len(arguments), 2): + with pytest.raises(SystemExit): + prepared_batch.main(arguments[:index] + arguments[index + 2 :]) + assert not cohort.output.exists() + prepared_batch.main(arguments) + assert "prepared:" in capsys.readouterr().out + cohort.launch.assert_not_called() + with pytest.raises(FileExistsError): + prepared_batch.main(arguments) + + +def test_replacement_after_last_budget_check_cannot_launch(cohort, monkeypatch): + path = cohort.plans[0] + replacement = cohort.source.parent / "replacement" + config = dict(cohort.configs[0]) + config["cloud"] = dict(config["cloud"], timeout_seconds=86400) + cloud.prepare( + cohort.source.parent.parent / "task", + config, + replacement, + repo_root=cohort.source.parent.parent / "repo", + ) + original = cloud._runtime_secrets + calls = 0 + + def replace_during_submit(secrets): + nonlocal calls + calls += 1 + if calls == 2: + for name in ("plan.json", "bundle.zip"): + shutil.copyfile(replacement / name, path.parent / name) + return original(secrets) + + monkeypatch.setattr(cloud, "_runtime_secrets", replace_during_submit) + monkeypatch.setattr(cloud, "submit", cohort.actual_submit) + with pytest.raises(RuntimeError, match="during submit"): + prepared_batch.run(**cohort.arguments, submit=True) + assert calls == 2 + cohort.api.run_job.assert_not_called() + cohort.api.create_commit.assert_not_called() + assert not (path.parent / "submission.json").exists() + assert saved(cohort)["shared_spend"]["status"] == "reserved" + + +@pytest.mark.parametrize("runner", ["prepared", "native"]) +def test_initial_parse_swap_cannot_underprice_reviewed_plan( + cohort, monkeypatch, runner +): + path = cohort.plans[0] + cheap = { + name: (path.parent / name).read_bytes() for name in ("plan.json", "bundle.zip") + } + replacement = cohort.source.parent / "replacement" + config = dict(cohort.configs[0]) + config["cloud"] = dict(config["cloud"], timeout_seconds=86400) + cloud.prepare( + cohort.source.parent.parent / "task", + config, + replacement, + repo_root=cohort.source.parent.parent / "repo", + ) + expensive = {name: (replacement / name).read_bytes() for name in cheap} + for name, content in expensive.items(): + (path.parent / name).write_bytes(content) + cohort.rows[:] = cohort.rows[:1] + cohort.rows[0]["plan_sha256"] = full_run._hash(path)["sha256"] + rewrite_manifest(cohort) + original = cloud._read_plan + swapped = False + + def swap_at_parse(candidate, **kwargs): + nonlocal swapped + if Path(candidate).resolve() == path and not swapped: + swapped = True + for name, content in cheap.items(): + (path.parent / name).write_bytes(content) + try: + return original(candidate, **kwargs) + finally: + for name, content in expensive.items(): + (path.parent / name).write_bytes(content) + return original(candidate, **kwargs) + + monkeypatch.setattr(cloud, "_read_plan", swap_at_parse) + cohort.api.dataset_info.return_value = SimpleNamespace(private=True, sha="a" * 40) + cohort.api.list_repo_files.return_value = [] + cohort.api.create_commit.return_value = SimpleNamespace(oid="b" * 40) + cohort.api.run_job.return_value = SimpleNamespace( + id="job-0", url="https://huggingface.co/jobs/owner/job-0" + ) + cap = full_run.estimate(cohort.configs[:1])["reservation_usd_rounded_up"] + if runner == "prepared": + monkeypatch.setattr(cloud, "submit", cohort.actual_submit) + + def invocation(): + return prepared_batch.run( + **dict(cohort.arguments, budget_usd=cap), submit=True + ) + else: + monkeypatch.setattr( + full_run.real_tasks, + "_prepare", + lambda *args, **kwargs: { + "tasks": [{"task_id": "toy", "bundle_plan": str(path)}] + }, + ) + monkeypatch.setattr(full_run, "_catalog", lambda *args: {}) + monkeypatch.setattr( + cloud, + "submit", + lambda candidate, **kwargs: cohort.actual_submit( + candidate, api=cohort.api, secrets=SECRETS, **kwargs + ), + ) + + def invocation(): + return full_run.run( + cohort.output, + cohort.source, + cohort.source, + cap, + panel_path=write_panel(cohort.source.parent), + submit=True, + spend_ledger=cohort.ledger, + reservation_id="native-toy", + ) + + with pytest.raises(RuntimeError, match="during plans"): + invocation() + assert swapped + assert full_run._hash(path)["sha256"] == cohort.rows[0]["plan_sha256"] + assert spend.summary(cohort.ledger)["reserved_usd_exact"] == "0" + cohort.api.run_job.assert_not_called() + cohort.api.create_commit.assert_not_called() + + +def test_preparation_manifest_digest_binds_the_bytes_parsed(cohort, monkeypatch): + original = cohort.source.read_bytes() + replacement = cloud._json_bytes({"tasks": cohort.rows[:1]}) + read_bytes = Path.read_bytes + changed = False + + def replace_only_while_reading(path): + nonlocal changed + if path.resolve() == cohort.source and not changed: + changed = True + path.write_bytes(replacement) + try: + return read_bytes(path) + finally: + path.write_bytes(original) + return read_bytes(path) + + monkeypatch.setattr(Path, "read_bytes", replace_only_while_reading) + with pytest.raises(RuntimeError, match="during budget"): + prepared_batch.run(**cohort.arguments, submit=True) + assert changed and read_bytes(cohort.source) == original + cohort.launch.assert_not_called() + assert spend.summary(cohort.ledger)["reserved_usd_exact"] == "0" + + +def test_blocking_inspection_is_interrupted_by_deadline(cohort): + def blocked(**kwargs): + time.sleep(5) + pytest.fail("inspection was not interrupted") + + cohort.api.inspect_job.side_effect = blocked + start = time.monotonic() + result = prepared_batch._status( + {"job_id": "owned"}, + {"image": cloud.WORKER_IMAGE}, + "owner", + start + 0.05, + cohort.api, + SECRETS, + time.monotonic, + ) + assert result == "TIMEOUT" + assert time.monotonic() - start < 1 + + +@pytest.mark.parametrize("observed", [1.0, 1.1]) +def test_inspection_is_not_dispatched_when_deadline_expires_between_reads(observed): + clock = iter([0.9, observed]) + api = SimpleNamespace(inspect_job=Mock()) + assert ( + prepared_batch._status( + {"job_id": "job"}, + {"image": cloud.WORKER_IMAGE}, + "owner", + 1, + api, + SECRETS, + lambda: next(clock), + ) + == "TIMEOUT" + ) + api.inspect_job.assert_not_called() + + +@pytest.mark.parametrize( + "url", + [ + "https://huggingface.co/jobs//job-0", + "https://foreign.invalid/jobs/owner/job-0", + "https://huggingface.co/jobs/owner/different-job", + ], +) +def test_malformed_controller_owner_url_never_bypasses_identity(cohort, url): + submit = cohort.launch.side_effect + + def malformed(*args, **kwargs): + receipt = submit(*args, **kwargs) + receipt["job_url"] = url + return receipt + + cohort.launch.side_effect = malformed + with pytest.raises(RuntimeError, match="during submit"): + prepared_batch.run(**cohort.arguments, submit=True) + cohort.api.inspect_job.assert_not_called() + assert cohort.launch.call_count == 1 + + +@pytest.mark.parametrize( + "stage", ["COMPLETED", "ERROR", "CANCELED", "DELETED", "RUNNING"] +) +def test_status_preserves_terminal_observation_at_deadline(stage): + job = SimpleNamespace( + id="job", + docker_image=cloud.WORKER_IMAGE, + owner=SimpleNamespace(name="owner"), + status=SimpleNamespace(stage=stage), + ) + clock = iter([0.0, 0.1, 1.1]) + result = prepared_batch._status( + {"job_id": "job"}, + {"image": cloud.WORKER_IMAGE}, + "owner", + 1, + SimpleNamespace(inspect_job=lambda **kwargs: job), + SECRETS, + lambda: next(clock), + ) + assert result == ("TIMEOUT" if stage == "RUNNING" else stage) diff --git a/examples/hub_validation_pipeline/test_real_tasks.py b/examples/hub_validation_pipeline/test_real_tasks.py new file mode 100644 index 000000000..f9fc93f28 --- /dev/null +++ b/examples/hub_validation_pipeline/test_real_tasks.py @@ -0,0 +1,633 @@ +"""CPU-only operator-panel, acquisition and SDK transfer-contract checks.""" + +import hashlib +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from examples.coding_audit.hf import HFTaskBackend +from examples.hub_validation_pipeline import cloud, real_tasks +from huggingface_hub import RepoFile, RepoFolder +from huggingface_hub.errors import SandboxError +from openenv.validation.runner import source_digest + + +TOY_NAMES = ("toy-alpha", "toy-beta") +TOY_SOURCE = {"kind": "hf", "repo_id": "example/toy-tasks", "revision": "b" * 40} + + +def toy_config(name=TOY_NAMES[0]): + config = cloud._loads( + (Path(__file__).parents[1] / "coding_audit/config.json").read_bytes() + ) + config["execution"].update( + agent_image="example/toy@sha256:" + "c" * 64, + verifier_image="example/toy@sha256:" + "c" * 64, + submission_paths=[f"/app/{name}.txt"], + timeout_seconds=60, + max_tool_output_bytes=1048576, + ) + config["negative_cases"] = [ + {"id": "toy-empty", "script": f": > /app/{name}.txt"}, + {"id": "toy-wrong", "script": f"printf pear > /app/{name}.txt"}, + ] + config["validation"].update( + required_negative_cases=[case["id"] for case in config["negative_cases"]], + target_attempts=2, + reference_repeats=2, + ) + config["seeds"] = [17, 29] + config["generation"]["candidate_count"] = 1 + config["target"].update(max_steps=3, max_tokens=64, max_total_tokens=256) + config["generator"] = dict(config["target"], max_steps=1) + config["cloud"].update(results_repo="example/results", timeout_seconds=60) + return cloud._config(config)[0] + + +def write_panel(directory, names=TOY_NAMES): + tasks = [] + for name in names: + config = toy_config(name) + tasks.append( + { + "name": name, + "task_path": f"fixtures/{name}", + "image": config["execution"]["agent_image"], + "submission_path": config["execution"]["submission_paths"][0], + "config": config, + "negative_counterexamples": [{"id": "toy-wrong", "expected": "apple"}], + "annotations": {"caveat": "Synthetic fixture only"}, + } + ) + path = directory / "panel.json" + path.write_bytes( + cloud._json_bytes( + { + "schema": "openenv-native-evaluation-panel-v1", + "evaluation_only": True, + "source": TOY_SOURCE, + "tasks": tasks, + } + ) + ) + return path + + +def source_files(name): + return { + "task.toml": ( + '[task]\nname = "toy/' + name + '"\n' + "[agent]\ntimeout_sec = 900.0\n[verifier]\ntimeout_sec = 900.0\n" + '[environment]\ndocker_image = "example/' + name + ':fixture"\n' + "cpus = 1\nmemory_mb = 2048\nstorage_mb = 10240\ngpus = 0\n" + ).encode(), + "instruction.md": b"Public task instruction\n", + "solution/solve.sh": b"#!/bin/bash\necho native-reference\n", + "tests/test.sh": b"#!/bin/bash\necho 0 > /logs/verifier/reward.txt\n", + "tests/test_outputs.py": b"# native verifier asset\n", + "environment/fixture/task.toml": b"nested asset, never a separate task\n", + } + + +def hf_fakes(files=None): + files = ( + files + if files is not None + else { + f"fixtures/{name}/{path}": data + for name in TOY_NAMES + for path, data in source_files(name).items() + } + ) + api = Mock() + api.dataset_info.return_value = SimpleNamespace( + sha=TOY_SOURCE["revision"], gated=False, disabled=False + ) + + def listing(repo_id, **kwargs): + prefix = kwargs["path_in_repo"] + "/" + yield RepoFolder(path=prefix + "tests", oid="0" * 40) + for path, data in files.items(): + if path.startswith(prefix): + yield RepoFile( + path=path, + size=len(data), + oid=hashlib.sha1(f"blob {len(data)}\0".encode() + data).hexdigest(), + ) + + api.list_repo_tree.side_effect = listing + + def download(repo_id, filename, **kwargs): + destination = kwargs["local_dir"] / filename + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(files[filename]) + return str(destination) + + return api, Mock(side_effect=download) + + +@pytest.fixture(autouse=True) +def no_live_network(monkeypatch): + import socket + + monkeypatch.setattr( + socket.socket, "connect", Mock(side_effect=AssertionError("network forbidden")) + ) + monkeypatch.setattr( + cloud, "submit", Mock(side_effect=AssertionError("submission forbidden")) + ) + for name in cloud.SECRET_NAMES: + monkeypatch.delenv(name, raising=False) + + +def test_prepare_pinned_selected_native_trees_and_consumable_manifest(tmp_path): + api, download = hf_fakes() + panel_path = write_panel(tmp_path) + panel = cloud._loads(panel_path.read_bytes()) + output = tmp_path / "recipes" + manifest = real_tasks.prepare( + output, panel_path=panel_path, api=api, download=download + ) + assert manifest == cloud._loads((output / "manifest.json").read_bytes()) + assert manifest["evaluation_only"] is True and manifest["status"] == "prepared" + assert [row["name"] for row in manifest["tasks"]] == list(TOY_NAMES) + panel_sha = hashlib.sha256(panel_path.read_bytes()).hexdigest() + assert manifest["evaluation_panel"] == { + "path": str(panel_path), + "sha256": panel_sha, + "bytes": panel_path.stat().st_size, + } + for row, task in zip(manifest["tasks"], panel["tasks"]): + root = Path(row["task_root"]) + assert row["source"] == { + **TOY_SOURCE, + "task_path": task["task_path"], + "evaluation_panel_sha256": panel_sha, + "task_file_hashes": { + path: record["sha256"] for path, record in row["files"].items() + }, + } + assert row["source_digest"] == source_digest(root) + assert row["source_image"].endswith(":fixture") + assert row["evaluation_only"] is True + for path, data in source_files(row["name"]).items(): + assert (root / path).read_bytes() == data + assert not (root / path).is_symlink() + assert row["files"][path]["sha256"] == hashlib.sha256(data).hexdigest() + config = cloud._loads(Path(row["config_path"]).read_bytes()) + assert config == task["config"] + assert row["budget"] == cloud._config(config)[1] + assert row["negative_counterexamples"] == task["negative_counterexamples"] + assert row["annotations"] == task["annotations"] + assert "negative_counterexamples" not in config + for call in api.list_repo_tree.call_args_list: + assert call.args[0] == TOY_SOURCE["repo_id"] + assert call.kwargs["revision"] == TOY_SOURCE["revision"] + assert call.kwargs["path_in_repo"] in {"fixtures/" + name for name in TOY_NAMES} + assert call.kwargs["recursive"] is True and call.kwargs["token"] is False + for call in download.call_args_list: + assert call.kwargs["revision"] == TOY_SOURCE["revision"] + assert call.kwargs["repo_type"] == "dataset" and call.kwargs["token"] is False + with pytest.raises(FileExistsError): + real_tasks.prepare(output, panel_path=panel_path, api=api, download=download) + + +def test_bundle_and_cli_pass_exact_preparation_contract(tmp_path, monkeypatch, capsys): + api, download = hf_fakes() + panel_path = write_panel(tmp_path) + + def write_plan(task, config, directory, **kwargs): + directory.mkdir(parents=True) + (directory / "plan.json").write_bytes(cloud._json_bytes({"fixture": True})) + + prepare = Mock(side_effect=write_plan) + monkeypatch.setattr(cloud, "prepare", prepare) + monkeypatch.setattr(real_tasks, "HfApi", Mock(return_value=api)) + monkeypatch.setattr(real_tasks, "hf_hub_download", download) + output = tmp_path / "cli" + real_tasks.main( + [ + "--panel", + str(panel_path), + "--output", + str(output), + "--task", + "toy-alpha", + "--bundle", + ] + ) + row = json.loads((output / "manifest.json").read_text())["tasks"][0] + call = prepare.call_args + assert call.args[0] == Path(row["task_root"]) + assert call.args[1]["cloud"]["results_repo"] == "example/results" + assert call.kwargs["source"] == row["source"] + assert row["bundle_plan"] == str(call.args[2] / "plan.json") + assert row["plan_path"] == "bundles/toy-alpha/plan.json" + assert ( + row["plan_sha256"] + == hashlib.sha256(Path(row["bundle_plan"]).read_bytes()).hexdigest() + ) + assert json.loads(capsys.readouterr().out)["status"] == "prepared" + + +def test_real_cloud_bundle_roundtrip_preserves_source_and_native_files(tmp_path): + api, download = hf_fakes() + panel_path = write_panel(tmp_path) + manifest = real_tasks.prepare( + tmp_path / "bundled", + panel_path=panel_path, + task_names=["toy-alpha"], + bundle=True, + api=api, + download=download, + ) + row = manifest["tasks"][0] + plan, config, data, bundle_manifest = cloud._read_plan(row["bundle_plan"]) + _, files = cloud._checked_archive(data, plan["bundle_sha256"]) + assert plan["task_source"] == bundle_manifest["task_source"] == row["source"] + assert config["execution"]["agent_image"] == row["execution_image"] + for path, data in source_files("toy-alpha").items(): + assert files["task/" + path] == data + + +def test_bundle_rejects_native_file_change_and_restore_after_acquisition( + tmp_path, monkeypatch +): + api, download = hf_fakes() + panel_path = write_panel(tmp_path) + original_tree = cloud._tree + + def changed_verifier(root, **kwargs): + root = Path(root) + if root.name != "toy-alpha": + return original_tree(root, **kwargs) + verifier = root / "tests/test.sh" + original = verifier.read_bytes() + verifier.write_text("#!/bin/bash\necho 1 > /logs/verifier/reward.txt\n") + try: + return original_tree(root, **kwargs) + finally: + verifier.write_bytes(original) + + monkeypatch.setattr(cloud, "_tree", changed_verifier) + output = tmp_path / "recipes" + with pytest.raises(ValueError, match="acquisition"): + real_tasks.prepare( + output, + panel_path=panel_path, + task_names=["toy-alpha"], + bundle=True, + api=api, + download=download, + ) + assert not (output / "bundles/toy-alpha/plan.json").exists() + + +@pytest.mark.parametrize( + "case", + [ + "revision", + "gated", + "escape", + "environment", + "file-cap", + "total-cap", + "hash", + "symlink", + "directory", + "missing", + ], +) +def test_acquisition_rejects_bad_source_before_materializing( + tmp_path, monkeypatch, case +): + name = "toy-alpha" + api, download = hf_fakes() + panel_path = write_panel(tmp_path) + if case == "revision": + api.dataset_info.return_value.sha = "main" + elif case == "gated": + api.dataset_info.return_value.gated = "manual" + elif case in ("escape", "environment"): + path = ( + "fixtures/elsewhere/task.toml" + if case == "escape" + else f"fixtures/{name}/.env" + ) + api.list_repo_tree.side_effect = None + api.list_repo_tree.return_value = [RepoFile(path=path, size=0, oid="0" * 40)] + elif case in ("file-cap", "total-cap"): + monkeypatch.setattr( + real_tasks, "MAX_FILE_BYTES" if case == "file-cap" else "MAX_TOTAL_BYTES", 1 + ) + elif case == "missing": + api.list_repo_tree.side_effect = None + api.list_repo_tree.return_value = [] + else: + original = download.side_effect + + def changed(*args, **kwargs): + path = Path(original(*args, **kwargs)) + if case == "hash": + path.write_bytes(b"x" * path.stat().st_size) + elif case == "symlink": + target = tmp_path / "outside" + target.write_text("outside") + path.unlink() + path.symlink_to(target) + else: + path.unlink() + path.mkdir() + return path + + download.side_effect = changed + destination = tmp_path / "task" + with pytest.raises(ValueError): + panel, _ = real_tasks._load_panel(panel_path) + real_tasks.acquire_task( + panel["tasks"][0], + destination, + source=panel["source"], + api=api, + download=download, + ) + assert not destination.exists() + if case in ( + "revision", + "gated", + "escape", + "environment", + "file-cap", + "total-cap", + "missing", + ): + download.assert_not_called() + + +@pytest.mark.parametrize("name", TOY_NAMES) +def test_same_image_fresh_solver_grader_transfer_with_sdk_fakes(tmp_path, name): + api, download = hf_fakes() + panel_path = write_panel(tmp_path) + root = tmp_path / "native" + panel, _ = real_tasks._load_panel(panel_path) + task = next(task for task in panel["tasks"] if task["name"] == name) + real_tasks.acquire_task( + task, root, source=panel["source"], api=api, download=download + ) + original_digest = source_digest(root) + config = toy_config(name) + execution = config["execution"] + exported = task["submission_path"] + submission_bytes = b"candidate file bytes\n" + verifier_output = "APT: fetching native verifier dependencies 雪\n" * 2048 + assert 16384 < len(verifier_output.encode()) < 1048576 + jobs, created = {}, [] + jobs_api = Mock() + + def inspect_job(*, job_id, **kwargs): + return SimpleNamespace( + id=job_id, + docker_image=execution["agent_image"], + status=SimpleNamespace(stage=jobs[job_id]), + owner=SimpleNamespace(name="owner"), + ) + + jobs_api.inspect_job.side_effect = inspect_job + jobs_api.cancel_job.side_effect = lambda *, job_id, **kwargs: jobs.update( + {job_id: "CANCELED"} + ) + + def factory(**kwargs): + assert kwargs["image"] == execution["agent_image"] + assert kwargs["forward_hf_token"] is False + assert ( + "env" not in kwargs and "secrets" not in kwargs and "volumes" not in kwargs + ) + identifier = f"fake-{len(created)}" + jobs[identifier] = "RUNNING" + data = {} + directories = {"/", "/tmp"} + + def stat_file(path): + if path in directories: + return SimpleNamespace(type="dir", size=0) + if path not in data: + raise SandboxError("missing", status_code=404) + return SimpleNamespace(type="file", size=len(data[path])) + + files = SimpleNamespace( + MAX_READ_BYTES=1048576, + data=data, + stat=stat_file, + exists=lambda path: path in data or path in directories, + read=lambda path: data[path], + write=lambda path, raw, **kwargs: data.update( + {path: raw.encode() if isinstance(raw, str) else raw} + ), + mkdir=directories.add, + delete=lambda path: data.pop(path), + ) + + def run(argv, **kwargs): + assert kwargs["cwd"] == "/app" + assert 0 < kwargs["timeout"] <= 60 + if argv == ["/bin/bash", "/tests/test.sh"]: + assert data[exported] == submission_bytes + assert data["/tests/test.sh"] == source_files(name)["tests/test.sh"] + assert not any(path.startswith("/solution") for path in data) + kwargs["on_stdout"](verifier_output) + kwargs["on_stderr"]("Native verifier completed.\n") + data["/logs/verifier/reward.txt"] = b"0\n" + else: + assert not any( + path.startswith(("/tests", "/solution")) for path in data + ) + data[exported] = submission_bytes + data["/app/not-exported.txt"] = b"not transferred" + return SimpleNamespace(exit_code=0, timed_out=False) + + sandbox = SimpleNamespace(id=identifier, files=files, run=run) + created.append(sandbox) + return sandbox + + backend = HFTaskBackend( + root, execution, token="execution-only", api=jobs_api, sandbox_factory=factory + ) + submission = backend.execute_script( + config["negative_cases"][0]["script"], "negative" + ) + grade = backend.grade(submission) + assert submission["error"] is None and grade["error"] is None + assert submission["files"] == {exported: submission_bytes} + assert grade["reward"] == 0 and grade["rawreward"] == b"0\n" + assert grade["transcript"] == [ + { + "exit_code": 0, + "output": verifier_output + "Native verifier completed.\n", + "output_truncated": False, + } + ] + assert submission["cleanup_verified"] and grade["cleanup_verified"] + assert len(created) == 2 and created[0].id != created[1].id + assert "/app/not-exported.txt" not in created[1].files.data + assert source_digest(root) == original_digest + + +def test_recipe_requires_explicit_panel_before_acquisition(tmp_path, monkeypatch): + acquisition = Mock(side_effect=AssertionError("No acquisition")) + monkeypatch.setattr(real_tasks, "HfApi", acquisition) + with pytest.raises(TypeError, match="panel_path"): + real_tasks.prepare(tmp_path / "recipes") + with pytest.raises(SystemExit): + real_tasks.main(["--output", str(tmp_path / "recipes")]) + acquisition.assert_not_called() + assert not (tmp_path / "recipes").exists() + + +@pytest.mark.parametrize( + "case", + [ + "schema", + "evaluation", + "source", + "repo", + "revision", + "empty", + "duplicate", + "name", + "path", + "image", + "submission", + "counterexamples", + "annotations", + "credentials", + "duplicate-json", + ], +) +def test_invalid_operator_panel_fails_before_acquisition(tmp_path, monkeypatch, case): + panel_path = write_panel(tmp_path) + panel = cloud._loads(panel_path.read_bytes()) + task = panel["tasks"][0] + if case == "schema": + panel["schema"] = "unknown" + elif case == "evaluation": + panel["evaluation_only"] = False + elif case == "source": + panel["source"]["kind"] = "local" + elif case == "repo": + panel["source"]["repo_id"] = "../escape/name" + elif case == "revision": + panel["source"]["revision"] = "main" + elif case == "empty": + panel["tasks"] = [] + elif case == "duplicate": + panel["tasks"].append(task) + elif case == "name": + task["name"] = "../escape" + elif case == "path": + task["task_path"] = "fixtures/.env-private" + elif case == "image": + task["image"] = "example/other@sha256:" + "d" * 64 + elif case == "submission": + task["submission_path"] = "/app/other.txt" + elif case == "counterexamples": + task["negative_counterexamples"] = "invalid" + elif case == "annotations": + task["annotations"] = [] + elif case == "credentials": + task["annotations"] = {"credential": "hf_" + "Z" * 32} + panel_path.write_bytes(cloud._json_bytes(panel)) + if case == "duplicate-json": + panel_path.write_bytes( + panel_path.read_bytes().replace( + b'"evaluation_only":true', + b'"evaluation_only":true,"evaluation_only":true', + ) + ) + acquisition = Mock(side_effect=AssertionError("No acquisition")) + monkeypatch.setattr(real_tasks, "HfApi", acquisition) + with pytest.raises(ValueError): + real_tasks.prepare(tmp_path / "recipes", panel_path=panel_path) + acquisition.assert_not_called() + assert not (tmp_path / "recipes").exists() + + +def test_recipe_hashes_exact_panel_bytes_and_loads_only_once(tmp_path, monkeypatch): + panel_path = write_panel(tmp_path) + # A noncanonical encoding must be hashed as supplied, rather than reserialized. + original_bytes = b" " + panel_path.read_bytes() + b"\n\n" + panel_path.write_bytes(original_bytes) + read_file = cloud._text_file + reads = [] + + def replace_after_read(path): + result = read_file(path) + if path == panel_path: + reads.append(path) + panel_path.write_text('{"replacement": true}') + return result + + monkeypatch.setattr(cloud, "_text_file", replace_after_read) + api, download = hf_fakes() + manifest = real_tasks.prepare( + tmp_path / "recipes", panel_path=panel_path, api=api, download=download + ) + expected = hashlib.sha256(original_bytes).hexdigest() + assert reads == [panel_path] + assert manifest["evaluation_panel"]["sha256"] == expected + assert manifest["evaluation_panel"]["bytes"] == len(original_bytes) + assert all( + row["source"]["evaluation_panel_sha256"] == expected + for row in manifest["tasks"] + ) + assert Path(manifest["tasks"][0]["config_path"]).read_bytes() == cloud._json_bytes( + toy_config() + ) + + +def test_dockerfile_only_recipe_preserves_native_manifest(tmp_path): + name = TOY_NAMES[0] + files = source_files(name) + files["task.toml"] = b'version = "1.0"\n[environment]\ncpus = 1\n' + api, download = hf_fakes({f"fixtures/{name}/{p}": raw for p, raw in files.items()}) + result = real_tasks.prepare( + tmp_path / "recipe", + panel_path=write_panel(tmp_path, [name]), + api=api, + download=download, + ) + row = result["tasks"][0] + assert row["source_image"] is None + assert row["execution_image"] == toy_config(name)["execution"]["agent_image"] + assert (Path(row["task_root"]) / "task.toml").read_bytes() == files["task.toml"] + + +def test_root_task_panel_acquires_and_bundles_exact_root_files(tmp_path): + name = TOY_NAMES[0] + files = source_files(name) + api, download = hf_fakes(files) + + def listing(repo_id, **kwargs): + assert kwargs["path_in_repo"] is None + for path, data in files.items(): + yield RepoFile( + path=path, + size=len(data), + oid=hashlib.sha1(f"blob {len(data)}\0".encode() + data).hexdigest(), + ) + + api.list_repo_tree.side_effect = listing + panel = write_panel(tmp_path, [name]) + value = cloud._loads(panel.read_bytes()) + value["tasks"][0]["task_path"] = "." + panel.write_bytes(cloud._json_bytes(value)) + result = real_tasks.prepare( + tmp_path / "recipe", panel_path=panel, api=api, download=download + ) + row = result["tasks"][0] + assert row["source"]["task_path"] == "." + assert set(row["files"]) == set(files) + for path, content in files.items(): + assert (Path(row["task_root"]) / path).read_bytes() == content diff --git a/examples/hub_validation_pipeline/test_sources.py b/examples/hub_validation_pipeline/test_sources.py new file mode 100644 index 000000000..ad997135e --- /dev/null +++ b/examples/hub_validation_pipeline/test_sources.py @@ -0,0 +1,617 @@ +"""Network-free checks for discovery, immutable locators, and task assets.""" + +import asyncio +import hashlib +import io +import json +import tarfile +from fnmatch import fnmatchcase +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, Mock + +import pytest +from examples.hub_validation_pipeline import real_tasks, sources +from huggingface_hub import RepoFile + +SHA = "a" * 40 +DIGEST = "sha256:" + "b" * 64 +TASK_DIGEST = "sha256:" + "c" * 64 + + +def hf_candidate(**metadata): + return {"source": "hf", "id": "org/bundle", "revision": SHA, "metadata": metadata} + + +def fake_hf(monkeypatch, files): + api = Mock() + api.dataset_info.return_value = SimpleNamespace( + sha=SHA, gated=False, disabled=False + ) + api.list_repo_files.return_value = list(files) + api.list_repo_tree.return_value = [ + RepoFile( + path=name, + size=len(content.encode()), + oid=hashlib.sha1( + f"blob {len(content.encode())}\0".encode() + content.encode() + ).hexdigest(), + ) + for name, content in files.items() + ] + factory = Mock(return_value=api) + monkeypatch.setattr(sources, "HfApi", factory) + + def download(repo_id, **kwargs): + root = kwargs["local_dir"] + for name, content in files.items(): + if any(fnmatchcase(name, pattern) for pattern in kwargs["allow_patterns"]): + destination = root / name + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(content) + return str(root) + + downloader = Mock(side_effect=download) + monkeypatch.setattr(sources, "snapshot_download", downloader) + return api, factory, downloader + + +def test_hf_exact_conjunction_trending_and_exhausted_pagination(monkeypatch): + api = Mock() + consumed = [] + + def pages(): + for number, tags in enumerate( + [ + ["harbor", "rl-environment"], + ["library:harbor"], + ["library:harbor", "rl-environment"], + ["library:harbor", "rl-environment"], + ] + ): + consumed.append(number) + yield SimpleNamespace( + id=f"org/{number}", sha=SHA, tags=tags, gated=False, disabled=False + ) + + api.list_datasets.return_value = pages() + monkeypatch.setattr(sources, "HfApi", Mock(return_value=api)) + candidates = sources.discover_hf(1) + assert [candidate["id"] for candidate in candidates] == ["org/2"] + assert candidates[0]["revision"] == SHA + assert consumed == [0, 1, 2, 3] + api.list_datasets.assert_called_once_with( + filter=("library:harbor", "rl-environment"), + sort="trending_score", + expand=["sha", "tags", "gated", "disabled"], + limit=None, + ) + + +def test_hf_bounded_assets_pinned_and_fresh(monkeypatch, tmp_path): + files = {"registry.json": "{}", "shared/data[train].txt": "shared"} + for number in range(5000): + prefix = f"environment/tasks/{number:04d}/" + files[prefix + "task.toml"] = "version = '1.0'" + files[prefix + "instruction.md"] = "do a task" + files[prefix + "environment/assets/data.txt"] = "actual asset" + api, factory, downloader = fake_hf(monkeypatch, files) + first = sources.acquire(hf_candidate(), tmp_path, max_tasks=1)[0] + second = sources.acquire(hf_candidate(), tmp_path, max_tasks=1)[0] + assert first["task_id"] == f"hf:org/bundle@{SHA}:environment/tasks/0000" + assert first["task_id"] == second["task_id"] + assert first["path"] != second["path"] + path = Path(first["path"]) + assert path.is_absolute() and path == path.resolve() + assert (path / "environment/assets/data.txt").read_text() == "actual asset" + assert not (path.parent / "0001").exists() + root = path.parents[2] + assert (root / "shared/data[train].txt").read_text() == "shared" + assert len(first["download_patterns"]) == 5 + assert first["revision"] == SHA and first["source_id"] == "org/bundle" + assert SHA in path.parts + factory.assert_called_with(token=False) + api.dataset_info.assert_called_with("org/bundle", revision=SHA) + api.list_repo_tree.assert_called_with( + "org/bundle", repo_type="dataset", revision=SHA, recursive=True, token=False + ) + assert downloader.call_args.kwargs["revision"] == SHA + assert downloader.call_args.kwargs["repo_type"] == "dataset" + assert downloader.call_args.kwargs["force_download"] is True + + +@pytest.mark.parametrize("prefix", ["", "environment/tasks/solo/"]) +def test_hf_individual_and_nested_tasks(monkeypatch, tmp_path, prefix): + fake_hf(monkeypatch, {prefix + "task.toml": "", prefix + "instruction.md": ""}) + [record] = sources.acquire(hf_candidate(), tmp_path, 1) + assert record["task_path"] == (prefix.rstrip("/") or ".") + assert (Path(record["path"]) / "instruction.md").is_file() + + +def test_hf_same_basename_has_distinct_identity(monkeypatch, tmp_path): + fake_hf(monkeypatch, {"a/same/task.toml": "", "b/same/task.toml": ""}) + records = sources.acquire(hf_candidate(), tmp_path, 2) + assert len({record["task_id"] for record in records}) == 2 + assert {record["task_path"] for record in records} == {"a/same", "b/same"} + + +def test_hf_gated_malformed_and_floating_sources_fail(monkeypatch, tmp_path): + api, factory, download = fake_hf(monkeypatch, {"README.md": "no tasks"}) + with pytest.raises(PermissionError, match="gated"): + sources.acquire(hf_candidate(gated="manual"), tmp_path, 1) + factory.assert_not_called() + with pytest.raises(ValueError, match="No task.toml"): + sources.acquire(hf_candidate(), tmp_path, 1) + download.assert_not_called() + candidate = {**hf_candidate(), "revision": "main"} + with pytest.raises(ValueError, match="immutable"): + sources.acquire(candidate, tmp_path, 1) + api.dataset_info.return_value.sha = "d" * 40 + with pytest.raises(ValueError, match="disagrees"): + sources.acquire(hf_candidate(), tmp_path, 1) + + +def test_unsafe_source_paths_and_symlinks_fail(monkeypatch, tmp_path): + fake_hf(monkeypatch, {"../escape/task.toml": ""}) + with pytest.raises(ValueError, match="Unsafe source path"): + sources.acquire(hf_candidate(), tmp_path, 1) + outside = tmp_path / "outside" + outside.write_text("original") + _, _, download = fake_hf(monkeypatch, {"task.toml": "original"}) + + def link(repo, **kwargs): + (kwargs["local_dir"] / "task.toml").symlink_to(outside) + + download.side_effect = link + with pytest.raises(ValueError, match="contained regular file"): + sources.acquire(hf_candidate(), tmp_path / "cache", 1) + assert outside.read_text() == "original" + + +def test_hf_acquisition_rejects_contained_links(monkeypatch, tmp_path): + _, _, download = fake_hf(monkeypatch, {"task.toml": "original"}) + + def link(repo, **kwargs): + root = kwargs["local_dir"] + (root / "actual").write_text("original") + (root / "task.toml").symlink_to(root / "actual") + + download.side_effect = link + with pytest.raises(ValueError, match="contained regular file"): + sources.acquire(hf_candidate(), tmp_path, 1) + + +class PackageTaskId: + def __init__(self, org, name, ref): + self.org, self.name, self.ref = org, name, ref + + +def fake_harbor(monkeypatch): + tasks = [ + PackageTaskId("a", "same", TASK_DIGEST), + PackageTaskId("b", "same", "sha256:" + "d" * 64), + ] + registry = SimpleNamespace( + list_datasets=AsyncMock( + return_value=[ + SimpleNamespace( + name="org/dataset", + version=DIGEST, + description="bundle", + task_count=2, + ) + ] + ), + get_dataset_metadata=AsyncMock( + return_value=SimpleNamespace(version=DIGEST, task_ids=tasks, files=[]) + ), + download_dataset_files=AsyncMock(return_value={}), + ) + + payloads = { + "task.toml": b"", + "instruction.md": b"task", + "environment/assets/data": b"asset", + } + + def rows(): + return [ + dict( + path=name, + size_bytes=len(data), + content_hash=hashlib.sha256(data).hexdigest(), + ) + for name, data in client.payloads.items() + ] + + async def resolve(org, name, ref): + return SimpleNamespace( + id=org + "/" + name, + content_hash=ref.removeprefix("sha256:"), + archive_path=org + "/" + name, + ) + + async def bounded_download(remote_path, destination, byte_limit): + with tarfile.open(destination, "w:gz") as archive: + for name, data in client.payloads.items(): + member = tarfile.TarInfo(name) + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + for member, data in client.extra_members: + archive.addfile(member, io.BytesIO(data)) + return destination.stat().st_size + + client = SimpleNamespace( + payloads=payloads, + extra_members=[], + download_tasks=AsyncMock( + side_effect=AssertionError("Unbounded task download forbidden") + ), + resolve_task_version=AsyncMock(side_effect=resolve), + get_task_version_files=AsyncMock(side_effect=lambda *args: rows()), + get_dataset_version_files=AsyncMock(return_value=[]), + download=AsyncMock(side_effect=bounded_download), + ) + monkeypatch.setattr( + sources, "_download_harbor_file", client.download, raising=False + ) + monkeypatch.setattr( + sources, + "_harbor_api", + lambda: (Mock(return_value=registry), PackageTaskId, Mock(return_value=client)), + ) + return registry, client + + +def test_harbor_discovery_and_bounded_digest_acquisition(monkeypatch, tmp_path): + registry, client = fake_harbor(monkeypatch) + [candidate] = sources.discover_harbor(1) + assert candidate["revision"] == DIGEST + [record] = sources.acquire(candidate, tmp_path, 1) + registry.get_dataset_metadata.assert_awaited_once_with(f"org/dataset@{DIGEST}") + client.resolve_task_version.assert_awaited_once_with("a", "same", TASK_DIGEST) + client.download.assert_awaited_once() + client.download_tasks.assert_not_awaited() + assert record["revision"] == TASK_DIGEST + assert record["task_id"] == f"harbor:a/same@{TASK_DIGEST}:." + assert record["metadata"]["candidate_id"] == "org/dataset" + assert record["metadata"]["candidate_revision"] == DIGEST + assert (Path(record["path"]) / "environment/assets/data").read_text() == "asset" + + +@pytest.mark.parametrize("missing_root", [False, True]) +def test_harbor_nested_fixture_cannot_replace_package_root( + monkeypatch, tmp_path, missing_root +): + _, client = fake_harbor(monkeypatch) + client.payloads["environment/fixture/task.toml"] = b"fixture" + if missing_root: + del client.payloads["task.toml"] + [candidate] = sources.discover_harbor(1) + if missing_root: + with pytest.raises(ValueError, match="no root task.toml"): + sources.acquire(candidate, tmp_path, 1) + else: + [record] = sources.acquire(candidate, tmp_path, 1) + assert record["task_path"] == "." + assert (Path(record["path"]) / "environment/fixture/task.toml").is_file() + + +def test_hf_nested_fixture_remains_parent_task_asset(monkeypatch, tmp_path): + fake_hf( + monkeypatch, + { + "tasks/parent/task.toml": "parent", + "tasks/parent/environment/fixture/task.toml": "fixture", + "tasks/sibling/task.toml": "sibling", + }, + ) + records = sources.acquire(hf_candidate(), tmp_path, 2) + assert [row["task_path"] for row in records] == ["tasks/parent", "tasks/sibling"] + assert ( + Path(records[0]["path"]) / "environment/fixture/task.toml" + ).read_text() == "fixture" + + +def test_harbor_single_task_and_cross_org_identity(monkeypatch, tmp_path): + registry, _ = fake_harbor(monkeypatch) + candidate = { + "source": "harbor", + "id": "a/same", + "revision": TASK_DIGEST, + "metadata": {"kind": "task"}, + } + [record] = sources.acquire(candidate, tmp_path, 1) + registry.get_dataset_metadata.assert_not_awaited() + assert record["source_id"] == "a/same" and record["revision"] == TASK_DIGEST + [dataset] = sources.discover_harbor(1) + records = sources.acquire(dataset, tmp_path, 2) + assert len({record["task_id"] for record in records}) == 2 + assert {record["source_id"] for record in records} == {"a/same", "b/same"} + + +def test_harbor_inaccessible_or_floating_tasks_fail(monkeypatch, tmp_path): + registry, client = fake_harbor(monkeypatch) + [candidate] = sources.discover_harbor(1) + registry.get_dataset_metadata.side_effect = PermissionError("unavailable task") + with pytest.raises(PermissionError, match="unavailable"): + sources.acquire(candidate, tmp_path, 1) + client.download_tasks.assert_not_awaited() + registry.get_dataset_metadata.side_effect = None + registry.get_dataset_metadata.return_value.task_ids[0].ref = "latest" + with pytest.raises(ValueError, match="immutable"): + sources.acquire(candidate, tmp_path, 1) + client.download_tasks.assert_not_awaited() + + +def test_harbor_wheel_mismatch_reports_pinned_install(monkeypatch): + # The PyPI 0.25.0 wheel shares the version string but lacks native listing. + monkeypatch.setattr( + sources, + "distribution", + lambda name: SimpleNamespace(version="0.25.0", read_text=lambda name: None), + ) + with pytest.raises(RuntimeError, match="source capability mismatch") as error: + sources.discover_harbor(1) + assert sources.HARBOR_INSTALL in str(error.value) + assert sources.HARBOR_GIT_REVISION in str(error.value) + + +def test_harbor_unsupported_listing_reports_install(monkeypatch): + registry, _ = fake_harbor(monkeypatch) + registry.list_datasets.side_effect = NotImplementedError("not yet supported") + with pytest.raises(RuntimeError, match="listing is unsupported") as error: + sources.discover_harbor(1) + assert sources.HARBOR_INSTALL in str(error.value) + + +def test_harbor_native_cli_json_and_model_contract(monkeypatch, capsys): + # Exercise the real Git-pinned API/CLI conversion, mocking only its DB read. + pytest.importorskip("harbor") + from harbor.cli.datasets import list_datasets + from harbor.db.client import RegistryDB + + native_rows = [ + { + "content_hash": "b" * 64, + "description": "bundle", + "package": {"org": {"name": "org"}, "name": "dataset"}, + "dataset_version_task": [{"count": 2}], + }, + { + "content_hash": "d" * 64, + "description": None, + "package": {"org": {"name": "z"}, "name": "files-only"}, + "dataset_version_task": [], + }, + ] + # A wheel without this method must fail this contract, rather than pass mocks. + monkeypatch.setattr( + RegistryDB, "list_dataset_versions", AsyncMock(return_value=native_rows) + ) + list_datasets(as_json=True) + cli_rows = json.loads(capsys.readouterr().out) + assert cli_rows == [ + { + "name": "org/dataset", + "version": DIGEST, + "description": "bundle", + "task_count": 2, + }, + { + "name": "z/files-only", + "version": "sha256:" + "d" * 64, + "description": "", + "task_count": 0, + }, + ] + [candidate] = sources.discover_harbor(1) + assert candidate["id"] == cli_rows[0]["name"] + assert candidate["revision"] == cli_rows[0]["version"] + assert candidate["metadata"]["task_count"] == cli_rows[0]["task_count"] + + +def test_harbor_native_acquisition_models(monkeypatch, tmp_path): + pytest.importorskip("harbor") + from harbor.db.client import RegistryDB, ResolvedTaskVersion + from harbor.models.registry import DatasetMetadata + from harbor.models.task.id import PackageTaskId as NativeTaskId + from harbor.registry.client.package import PackageDatasetClient + from harbor.tasks.client import TaskClient + + task = NativeTaskId(org="a", name="same", ref=TASK_DIGEST) + bundle = DatasetMetadata( + name="org/dataset", version=DIGEST, description="", task_ids=[task] + ) + monkeypatch.setattr( + PackageDatasetClient, "get_dataset_metadata", AsyncMock(return_value=bundle) + ) + monkeypatch.setattr( + RegistryDB, + "resolve_task_version", + AsyncMock( + return_value=ResolvedTaskVersion( + id="version-id", + archive_path="archive", + content_hash=TASK_DIGEST.removeprefix("sha256:"), + ) + ), + ) + payloads = {"task.toml": b"", "instruction.md": b"native task"} + monkeypatch.setattr( + RegistryDB, + "get_task_version_files", + AsyncMock( + return_value=[ + dict( + path=name, + size_bytes=len(data), + content_hash=hashlib.sha256(data).hexdigest(), + storage_path=None, + ) + for name, data in payloads.items() + ] + ), + ) + unbounded = AsyncMock( + side_effect=AssertionError("Unbounded archive downloader forbidden") + ) + monkeypatch.setattr(TaskClient, "download_tasks", unbounded) + + async def download(remote_path, destination, byte_limit): + assert remote_path == "archive" and byte_limit == real_tasks.MAX_TOTAL_BYTES + with tarfile.open(destination, "w:gz") as archive: + for name, data in payloads.items(): + member = tarfile.TarInfo(name) + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + return destination.stat().st_size + + monkeypatch.setattr(sources, "_download_harbor_file", download) + [record] = sources.acquire( + {"source": "harbor", "id": "org/dataset", "revision": DIGEST}, tmp_path, 1 + ) + assert record["revision"] == TASK_DIGEST + assert (Path(record["path"]) / "instruction.md").read_text() == "native task" + assert record["task_file_hashes"] == { + name: hashlib.sha256(data).hexdigest() for name, data in payloads.items() + } + unbounded.assert_not_awaited() + + +@pytest.mark.parametrize( + "case", ["file_bytes", "total_bytes", "file_count", "unknown_size"] +) +def test_hf_acquisition_caps_shared_assets_before_download(monkeypatch, tmp_path, case): + files = {"task/task.toml": "", "shared/large": "x"} + if case == "file_count": + files.update({f"shared/{index}": "" for index in range(real_tasks.MAX_FILES)}) + if case == "total_bytes": + files.update({f"shared/{index}": "" for index in range(5)}) + api, _, download = fake_hf(monkeypatch, files) + if case == "file_bytes": + api.list_repo_tree.return_value[1].size = 1024**3 + elif case == "total_bytes": + for item in api.list_repo_tree.return_value[1:]: + item.size = real_tasks.MAX_FILE_BYTES + elif case == "unknown_size": + api.list_repo_tree.return_value[1].size = None + with pytest.raises(ValueError, match="cap"): + sources.acquire(hf_candidate(), tmp_path, 1) + download.assert_not_called() + + +def test_hf_acquisition_requires_pinned_blob_bytes(monkeypatch, tmp_path): + api, _, download = fake_hf(monkeypatch, {"task/task.toml": "original"}) + api.list_repo_tree.return_value[0].blob_id = "0" * 40 + with pytest.raises(ValueError, match="blob"): + sources.acquire(hf_candidate(), tmp_path, 1) + download.assert_called_once() + + +@pytest.mark.parametrize( + "case", ["file_bytes", "total_bytes", "file_count", "unknown_size"] +) +@pytest.mark.parametrize("scope", ["task", "shared"]) +def test_harbor_caps_inventory_before_download(monkeypatch, tmp_path, case, scope): + registry, client = fake_harbor(monkeypatch) + rows = [dict(path="task.toml", size_bytes=0, content_hash="a" * 64)] + if case == "file_bytes": + rows[0]["size_bytes"] = 1024**3 + elif case == "total_bytes": + rows = [ + dict(rows[0], path=f"file-{index}", size_bytes=real_tasks.MAX_FILE_BYTES) + for index in range(5) + ] + elif case == "file_count": + rows = [ + dict(rows[0], path=f"file-{index}") + for index in range(real_tasks.MAX_FILES + 1) + ] + else: + rows[0]["size_bytes"] = None + if scope == "task": + client.get_task_version_files.side_effect = None + client.get_task_version_files.return_value = rows + else: + registry.get_dataset_metadata.return_value.files = [ + SimpleNamespace(path="shared") + ] + registry.get_dataset_metadata.return_value.dataset_version_id = "dataset-id" + client.get_dataset_version_files.return_value = rows + [candidate] = sources.discover_harbor(1) + with pytest.raises(ValueError, match="cap"): + sources.acquire(candidate, tmp_path, 1) + client.download.assert_not_awaited() + client.download_tasks.assert_not_awaited() + registry.download_dataset_files.assert_not_awaited() + + +@pytest.mark.parametrize( + "case", ["extra", "link", "oversized", "hash", "missing", "expanded"] +) +def test_harbor_archive_must_match_bounded_inventory(monkeypatch, tmp_path, case): + _, client = fake_harbor(monkeypatch) + if case == "expanded": + member = tarfile.TarInfo("padding") + member.size = real_tasks.MAX_TOTAL_BYTES + 1 + client.extra_members = [(member, b"x" * member.size)] + elif case in {"extra", "link"}: + member = tarfile.TarInfo("unexpected") + if case == "link": + member.type = tarfile.SYMTYPE + member.linkname = "/outside" + client.extra_members = [(member, b"")] + else: + original = client.get_task_version_files.side_effect + rows = original("version") + if case == "oversized": + client.payloads["instruction.md"] = b"x" * (real_tasks.MAX_FILE_BYTES + 1) + elif case == "hash": + client.payloads["instruction.md"] = b"evil" + else: + del client.payloads["instruction.md"] + client.get_task_version_files.side_effect = None + client.get_task_version_files.return_value = rows + [candidate] = sources.discover_harbor(1) + with pytest.raises(ValueError, match="Harbor archive"): + sources.acquire(candidate, tmp_path, 1) + assert not list(tmp_path.rglob("instruction.md")) + + +@pytest.mark.parametrize("declared", [True, False]) +def test_harbor_download_caps_headers_and_stream(monkeypatch, tmp_path, declared): + import httpx + from harbor.auth import client as auth + + storage = SimpleNamespace( + create_signed_url=AsyncMock( + return_value={"signedURL": "https://example.invalid/archive"} + ) + ) + hub = SimpleNamespace(storage=SimpleNamespace(from_=lambda bucket: storage)) + monkeypatch.setattr( + auth, "create_authenticated_client", AsyncMock(return_value=hub) + ) + response = MagicMock() + response.__aenter__.return_value = response + response.headers = {"content-length": "11"} if declared else {} + chunks = [] + + async def stream(**kwargs): + for data in [b"123456", b"78901"]: + chunks.append(data) + yield data + + response.aiter_bytes = stream + http = MagicMock() + http.__aenter__.return_value = http + http.stream.return_value = response + monkeypatch.setattr(httpx, "AsyncClient", lambda **kwargs: http) + target = tmp_path / "archive" + with pytest.raises(ValueError, match="byte cap"): + asyncio.run(sources._download_harbor_file("archive-path", target, 10)) + if declared: + assert chunks == [] and not target.exists() + else: + assert target.read_bytes() == b"123456" diff --git a/examples/hub_validation_pipeline/test_spend.py b/examples/hub_validation_pipeline/test_spend.py new file mode 100644 index 000000000..34f70c3f2 --- /dev/null +++ b/examples/hub_validation_pipeline/test_spend.py @@ -0,0 +1,714 @@ +"""Offline spend accounting, durability and real process/thread coordination.""" + +import json +import multiprocessing +import stat +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from decimal import Decimal, localcontext +from fractions import Fraction + +import pytest +from examples.hub_validation_pipeline import spend + + +@pytest.mark.parametrize("operation", ["reserve", "set_limit"]) +def test_idempotent_retry_must_confirm_directory_durability( + ledger, monkeypatch, operation +): + real_fsync = spend.os.fsync + calls = [] + + def fail_directory(fd): + directory = stat.S_ISDIR(spend.os.fstat(fd).st_mode) + calls.append("directory" if directory else "file") + if directory: + raise OSError("injected directory fsync failure") + return real_fsync(fd) + + monkeypatch.setattr(spend.os, "fsync", fail_directory) + + def retry(): + if operation == "reserve": + return spend.reserve(ledger, "durability", "1", "work") + return spend.set_limit(ledger, "5000", "reviewed example budget increase") + + with pytest.raises(OSError): + retry() + assert calls == ["file", "directory"] + before = ledger.read_bytes() + calls.clear() + with pytest.raises(OSError): + retry() + assert calls == ["directory"] + assert ledger.read_bytes() == before + monkeypatch.setattr(spend.os, "fsync", real_fsync) + report = retry() + assert ledger.read_bytes() == before + if operation == "reserve": + assert report["charged_usd_exact"] == "1" + else: + assert report["limit_usd_exact"] == "5000" + assert len(json.loads(ledger.read_text())["limit_changes"]) == 1 + + +@pytest.mark.parametrize("operation", ["summary", "reserve", "settle", "set_limit"]) +def test_saved_settlement_without_evidence_cannot_release_capacity(ledger, operation): + spend.reserve(ledger, "unfinished", "1", "unknown cost") + data = json.loads(ledger.read_text()) + data["reservations"]["unfinished"].update(status="settled", spent_usd="0") + ledger.write_text(json.dumps(data)) + original = ledger.read_bytes() + with pytest.raises(ValueError): + if operation == "summary": + spend.summary(ledger) + elif operation == "reserve": + spend.reserve(ledger, "new", "1", "work") + elif operation == "set_limit": + spend.set_limit(ledger, "5000", "reviewed example budget increase") + else: + spend.settle(ledger, "unfinished", "0", "new receipt") + assert ledger.read_bytes() == original + + +def _reserve_worker(path, barrier, results, amount, attempts, limit=None): + barrier.wait(timeout=10) + if limit is not None: + spend.set_limit(path, limit, "reviewed example budget increase") + successes = 0 + for index in range(attempts): + try: + spend.reserve( + path, + f"{multiprocessing.current_process().name}-{index}", + amount, + "race", + ) + except ValueError: + continue + successes += 1 + results.put(successes) + + +@pytest.fixture +def ledger(tmp_path): + path = tmp_path / "ledger.json" + spend.init(path, "10") + return path + + +def test_existing_schema_allocations_and_unknown_metadata_are_preserved(tmp_path): + path = tmp_path / "ledger.json" + data = { + "schema": "openenv-shared-spend-v1", + "limit_usd": "500", + "created_utc": "original creation", + "updated_utc": "original update", + "checked_utc": "keep this timestamp", + "metadata": {"nested": [1, "kept", None]}, + "reservations": { + identifier: dict( + reserved_usd=amount, + spent_usd=None, + status="reserved", + purpose="existing allocation", + evidence=None, + reserved_utc="original reservation", + metadata={"preserve": True}, + ) + for identifier, amount in ( + ("cohort-a", "48.123456"), + ("cohort-b", "150"), + ("image-probe", "5"), + ("generation-cohort", "100"), + ) + }, + } + path.write_text(json.dumps(data)) + before = path.read_bytes() + report = spend.summary(path) + assert Fraction(report["charged_usd_exact"]) == Fraction("303.123456") + assert Fraction(report["remaining_usd_exact"]) == Fraction("196.876544") + assert report["remaining_usd_rounded_up"] == "196.876544" + assert spend.init(path, "500.000") == report + assert path.read_bytes() == before + with pytest.raises(ValueError, match="limit differs"): + spend.init(path, "501") + assert path.read_bytes() == before + spend.reserve(path, "last", "196.876544", "remaining allocation") + before = path.read_bytes() + with pytest.raises(ValueError): + spend.reserve(path, "too-much", "1e-300", "must fail exactly") + assert path.read_bytes() == before + reservations = json.loads(before)["reservations"] + report = spend.set_limit(path, "5000", "reviewed example budget increase") + assert report["remaining_usd_exact"] == "4500" + raised = json.loads(path.read_text()) + assert raised["reservations"] == reservations + assert raised["reservations"]["generation-cohort"]["reserved_usd"] == "100" + # The existing writer's reservation serialization is unchanged by a limit update. + reservation_bytes = json.dumps({"reservations": reservations}, indent=2)[1:-2] + assert reservation_bytes.encode() in before + assert reservation_bytes.encode() in path.read_bytes() + history = raised["limit_changes"] + assert len(history) == 1 + assert history[0] == { + "previous_limit_usd": "500", + "limit_usd": "5000", + "evidence": "reviewed example budget increase", + "changed_utc": history[0]["changed_utc"], + } + assert datetime.fromisoformat(history[0]["changed_utc"]).tzinfo == timezone.utc + spend.reserve(path, "additional", "4500", "newly approved capacity") + before = path.read_bytes() + with pytest.raises(ValueError): + spend.reserve(path, "still-too-much", "1e-300", "must fail exactly") + assert path.read_bytes() == before + spend.settle(path, "cohort-a", "40", "receipt") + saved = json.loads(path.read_text()) + assert saved["limit_usd"] == "5000" + assert saved["limit_changes"] == history + for key in ("schema", "created_utc", "checked_utc", "metadata"): + assert saved[key] == data[key] + assert saved["updated_utc"] != data["updated_utc"] + for identifier, original in data["reservations"].items(): + row = saved["reservations"][identifier] + if identifier != "cohort-a": + assert row == original + else: + assert row["reserved_utc"] == original["reserved_utc"] + assert row["metadata"] == original["metadata"] + assert row["reserved_usd"] == original["reserved_usd"] + + +def test_reservation_and_settlement_are_idempotent_without_reusing_ids(ledger): + first = spend.reserve(ledger, "run", "1.0", "purpose", "plan") + before = ledger.read_bytes() + assert spend.reserve(ledger, "run", Decimal("1.000"), "retry", "other") == first + assert ledger.read_bytes() == before + with pytest.raises(ValueError, match="fresh ID"): + spend.reserve(ledger, "run", "2", "purpose") + assert ledger.read_bytes() == before + settled = spend.settle(ledger, "run", "0.7", "receipt") + before = ledger.read_bytes() + assert spend.settle(ledger, "run", "0.700", "retry receipt") == settled + assert ledger.read_bytes() == before + with pytest.raises(ValueError, match="manual reconciliation"): + spend.settle(ledger, "run", "0.8", "different receipt") + with pytest.raises(ValueError, match="fresh ID"): + spend.reserve(ledger, "run", "1.0", "reused") + assert ledger.read_bytes() == before + + +def test_actual_overspend_is_recorded_and_blocks_new_reservations(ledger): + spend.reserve(ledger, "a", "6", "first") + spend.reserve(ledger, "b", "4", "second") + report = spend.settle(ledger, "a", "12.000000000000000000000000000001", "receipt") + actual = Fraction("12.000000000000000000000000000001") + assert Fraction(report["charged_usd_exact"]) == actual + 4 + assert Fraction(report["remaining_usd_exact"]) == 6 - actual + assert report["over_budget"] is True + saved = json.loads(ledger.read_text()) + assert saved["over_budget"] is True + assert Fraction(saved["reservations"]["a"]["spent_usd"]) == actual + before = ledger.read_bytes() + with pytest.raises(ValueError, match="blocked"): + spend.reserve(ledger, "c", "0", "blocked even at zero") + assert ledger.read_bytes() == before + # Other real costs can still be settled after the cap is exceeded. + report = spend.settle(ledger, "b", "0", "cancelled before billing") + assert Fraction(report["charged_usd_exact"]) == actual + assert report["statuses"] == {"a": "settled", "b": "settled"} + for key in ("limit", "reserved", "spent", "charged", "remaining"): + exact = Fraction(report[f"{key}_usd_exact"]) + displayed = Fraction(report[f"{key}_usd_rounded_up"]) + assert exact <= displayed < exact + Fraction(1, 1000000) + + +def test_reservation_overrun_below_shared_cap_also_freezes_new_work(ledger): + spend.reserve(ledger, "a", "1", "first") + report = spend.settle(ledger, "a", "2", "receipt") + assert report["over_budget"] is True + assert report["remaining_usd_exact"] == "8" + before = json.loads(ledger.read_text()) + report = spend.set_limit(ledger, "5000", "reviewed example budget increase") + assert report["over_budget"] is True + assert report["remaining_usd_exact"] == "4998" + saved = json.loads(ledger.read_text()) + assert saved["over_budget"] is True + assert saved["reservations"] == before["reservations"] + with pytest.raises(ValueError): + spend.reserve(ledger, "b", "1", "blocked") + + +def test_normal_settlement_releases_unused_reservation(ledger): + spend.reserve(ledger, "a", "10", "first") + assert spend.settle(ledger, "a", "2", "receipt")["over_budget"] is False + report = spend.reserve(ledger, "b", "8", "released capacity") + assert report["charged_usd_exact"] == "10" + assert report["reserved_usd_exact"] == "8" + assert report["spent_usd_exact"] == "2" + + +@pytest.mark.parametrize( + "bad", + [ + True, + False, + 0.1, + float("nan"), + float("inf"), + Decimal("NaN"), + Decimal("sNaN"), + Decimal("Infinity"), + "NaN", + "Infinity", + "-1", + "-0", + "1/2", + "1_0", + " 1 ", + "", + "malformed", + None, + [], + {}, + ], +) +def test_invalid_money_is_rejected_without_partial_writes(ledger, tmp_path, bad): + missing = tmp_path / "never-created.json" + with pytest.raises(ValueError): + spend.init(missing, bad) + assert not missing.exists() + spend.reserve(ledger, "a", "1", "valid") + before = ledger.read_bytes() + with pytest.raises(ValueError): + spend.reserve(ledger, "bad", bad, "invalid") + with pytest.raises(ValueError): + spend.settle(ledger, "a", bad, "receipt") + with pytest.raises(ValueError): + spend.set_limit(ledger, bad, "reviewed example budget increase") + assert ledger.read_bytes() == before + + +def test_tiny_values_large_integers_and_decimal_context_do_not_round(tmp_path): + path = tmp_path / "precise.json" + with localcontext() as context: + context.prec = 2 + spend.init(path, Decimal("1.0000000000000000000000000000000000000001")) + spend.reserve(path, "whole", "1", "exact") + spend.reserve(path, "remainder", "1e-40", "last exact digit") + before = path.read_bytes() + with pytest.raises(ValueError): + spend.reserve(path, "tiny-over", "1e-300", "cannot round down") + assert path.read_bytes() == before + tiny = tmp_path / "tiny.json" + spend.init(tiny, "1e-300") + spend.reserve(tiny, "one", "1e-301", "tiny") + report = spend.reserve(tiny, "nine", "9e-301", "tiny") + assert report["remaining_usd_exact"] == "0" + assert Fraction(report["charged_usd_exact"]) == Fraction(Decimal("1e-300")) + assert report["charged_usd_rounded_up"] == "0.000001" + large = tmp_path / "large.json" + spend.init(large, 9007199254740993) + report = spend.reserve(large, "large", 9007199254740992, "exact integer") + assert report["remaining_usd_exact"] == "1" + + +def test_limit_changes_are_exact_append_only_and_idempotent(ledger, tmp_path): + missing = tmp_path / "missing.json" + with pytest.raises(ValueError, match="initialized"): + spend.set_limit(missing, "5000", "reviewed example budget increase") + assert not missing.exists() + with localcontext() as context: + context.prec = 2 + exact = Decimal("5000.0000000000000000000000000000000000000001") + report = spend.set_limit(ledger, exact, "reviewed example budget increase") + assert Fraction(report["limit_usd_exact"]) == Fraction(exact) + before = ledger.read_bytes() + assert spend.set_limit(ledger, str(exact), "retry evidence") == report + assert ledger.read_bytes() == before + first = json.loads(before)["limit_changes"][0] + report = spend.set_limit(ledger, "1e-300", "another decision") + assert Fraction(report["limit_usd_exact"]) == Fraction("1e-300") + changes = json.loads(ledger.read_text())["limit_changes"] + assert len(changes) == 2 + assert changes[0] == first + assert changes[1]["previous_limit_usd"] == str(exact) + report = spend.set_limit(ledger, 9007199254740993, "large exact decision") + assert report["limit_usd_exact"] == "9007199254740993" + + +def test_limit_reduction_latches_over_budget_and_raise_cannot_clear_it(ledger): + spend.reserve(ledger, "held", "5", "pending billing") + assert spend.set_limit(ledger, "4", "reduced ceiling")["over_budget"] is True + assert spend.set_limit(ledger, "5000", "reviewed example budget increase")[ + "over_budget" + ] + with pytest.raises(ValueError, match="blocked"): + spend.reserve(ledger, "blocked", "0", "still blocked") + + +@pytest.mark.parametrize("saved_flag", [False, True]) +def test_saved_over_budget_condition_survives_limit_increase(ledger, saved_flag): + spend.reserve(ledger, "held", "10", "existing hold") + data = json.loads(ledger.read_text()) + data["limit_usd"] = "9" if not saved_flag else "10" + if saved_flag: + data["over_budget"] = True + ledger.write_text(json.dumps(data)) + assert spend.set_limit(ledger, "5000", "reviewed example budget increase")[ + "over_budget" + ] + assert json.loads(ledger.read_text())["over_budget"] is True + + +@pytest.mark.parametrize( + "history", + [ + None, + {}, + "invalid", + [None], + [{}], + [{"previous_limit_usd": "500", "limit_usd": "10", "evidence": "decision"}], + ], +) +def test_malformed_limit_history_fails_closed(ledger, history): + data = json.loads(ledger.read_text()) + data["limit_changes"] = history + ledger.write_text(json.dumps(data)) + before = ledger.read_bytes() + with pytest.raises(ValueError): + spend.set_limit(ledger, "5000", "reviewed example budget increase") + with pytest.raises(ValueError): + spend.summary(ledger) + assert ledger.read_bytes() == before + + +@pytest.mark.parametrize( + "corruption", + [ + {"limit_usd": 10.0}, + {"previous_limit_usd": "-1"}, + {"evidence": " "}, + {"changed_utc": "bad timestamp"}, + {"changed_utc": "2026-11-10T00:00:00"}, + {"changed_utc": "2026-11-10T00:00:00+01:00"}, + {"limit_usd": "9"}, + {"previous_limit_usd": "10"}, + ], +) +def test_corrupt_limit_change_row_fails_closed(ledger, corruption): + change = dict( + previous_limit_usd="500", + limit_usd="10", + evidence="decision", + changed_utc="2026-11-10T00:00:00+00:00", + ) + test_malformed_limit_history_fails_closed(ledger, [change | corruption]) + + +def test_discontinuous_limit_history_fails_closed(ledger): + change = dict( + previous_limit_usd="500", + limit_usd="9", + evidence="decision", + changed_utc="2026-11-10T00:00:00+00:00", + ) + test_malformed_limit_history_fails_closed( + ledger, [change, change | {"previous_limit_usd": "8", "limit_usd": "10"}] + ) + + +@pytest.mark.parametrize( + "indent,newline,ensure_ascii", [(None, "\n", True), (4, "\r\n", False)] +) +def test_limit_change_preserves_original_reservation_bytes( + tmp_path, indent, newline, ensure_ascii +): + path = tmp_path / "ledger.json" + reservations = json.dumps( + { + "tasksmith": dict( + reserved_usd="100.0000", + spent_usd=None, + status="reserved", + purpose='original "reservations": note', + evidence="original évidence", + metadata={"keep": 1.25}, + ) + }, + indent=indent, + ensure_ascii=ensure_ascii, + ) + reservations = reservations.replace("\n", newline).replace( + "1.25", "1.25000000000000000000000000001" + ) + path.write_text( + '{"schema":"openenv-shared-spend-v1","limit_usd":"500",' + '"metadata":{"reservations":"unrelated"},"reservations":' + reservations + "}" + ) + spend.set_limit(path, "5000", "reviewed example budget increase") + assert reservations.encode() in path.read_bytes() + assert spend.summary(path)["reserved_usd_exact"] == "100" + + +@pytest.mark.parametrize("field", ["limit_usd", "reserved_usd"]) +def test_float_ledger_amounts_cannot_be_rewritten_by_limit_change(ledger, field): + spend.reserve(ledger, "held", "1", "existing hold") + data = json.loads(ledger.read_text()) + if field == "limit_usd": + data[field] = 10.0 + else: + data["reservations"]["held"][field] = 1.0 + ledger.write_text(json.dumps(data)) + before = ledger.read_bytes() + with pytest.raises(ValueError): + spend.set_limit(ledger, "5000", "reviewed example budget increase") + assert ledger.read_bytes() == before + + +@pytest.mark.parametrize("amount,attempts,expected", [("0.6", 1, 1), ("0.03", 20, 33)]) +def test_process_budget_race_and_no_lost_updates(tmp_path, amount, attempts, expected): + path = tmp_path / "shared.json" + spend.init(path, "1") + context = multiprocessing.get_context("spawn") + barrier, results = context.Barrier(2), context.Queue() + processes = [ + context.Process( + target=_reserve_worker, args=(path, barrier, results, amount, attempts) + ) + for _ in range(2) + ] + try: + for process in processes: + process.start() + for process in processes: + process.join(timeout=15) + assert process.exitcode == 0 + successes = sum(results.get(timeout=2) for _ in processes) + report = spend.summary(path) + assert successes == expected == len(report["statuses"]) + assert Fraction(report["charged_usd_exact"]) == expected * Fraction(amount) <= 1 + finally: + for process in processes: + if process.is_alive(): + process.terminate() + process.join(timeout=2) + results.close() + results.join_thread() + + +def test_threads_share_the_same_budget(ledger): + def reserve(index): + try: + spend.reserve(ledger, f"thread-{index}", "1", "thread race") + return 1 + except ValueError: + return 0 + + with ThreadPoolExecutor(max_workers=8) as executor: + successes = sum(executor.map(reserve, range(32))) + report = spend.summary(ledger) + assert successes == 10 == len(report["statuses"]) + assert report["charged_usd_exact"] == "10" + + +def test_concurrent_limit_retry_and_reservations_share_the_lock(tmp_path): + path = tmp_path / "shared.json" + spend.init(path, "500") + spend.reserve(path, "tasksmith", "100", "existing allocation") + context = multiprocessing.get_context("spawn") + barrier, results = context.Barrier(2), context.Queue() + processes = [ + context.Process( + target=_reserve_worker, + args=(path, barrier, results, "3000", 1, "5000"), + ) + for _ in range(2) + ] + try: + for process in processes: + process.start() + for process in processes: + process.join(timeout=15) + assert process.exitcode == 0 + assert sum(results.get(timeout=2) for _ in processes) == 1 + report = spend.summary(path) + assert report["charged_usd_exact"] == "3100" + saved = json.loads(path.read_text()) + assert len(saved["limit_changes"]) == 1 + assert saved["reservations"]["tasksmith"]["reserved_usd"] == "100" + finally: + for process in processes: + if process.is_alive(): + process.terminate() + process.join(timeout=2) + results.close() + results.join_thread() + + +def test_concurrent_different_limit_changes_keep_all_history(ledger): + with ThreadPoolExecutor(max_workers=8) as executor: + list( + executor.map( + lambda limit: spend.set_limit(ledger, limit, "reviewed decision"), + range(11, 27), + ) + ) + data = json.loads(ledger.read_text()) + changes = data["limit_changes"] + assert len(changes) == 16 + assert {row["limit_usd"] for row in changes} == {str(n) for n in range(11, 27)} + previous = "10" + for row in changes: + assert row["previous_limit_usd"] == previous + previous = row["limit_usd"] + assert data["limit_usd"] == previous + + +@pytest.mark.parametrize("evidence", [None, "", " ", True, {}]) +def test_settlement_requires_evidence_and_existing_id(ledger, evidence): + spend.reserve(ledger, "a", "1", "purpose") + before = ledger.read_bytes() + with pytest.raises(ValueError): + spend.settle(ledger, "a", "1", evidence) + with pytest.raises(ValueError, match="Unknown reservation"): + spend.settle(ledger, "missing", "1", "receipt") + with pytest.raises(ValueError): + spend.set_limit(ledger, "5000", evidence) + with pytest.raises(ValueError): + spend.set_limit(ledger, "10", evidence) + assert ledger.read_bytes() == before + + +@pytest.mark.parametrize("operation", ["reserve", "set_limit"]) +def test_failed_atomic_replace_keeps_original_and_cleans_temporary_file( + ledger, monkeypatch, operation +): + before, files = ledger.read_bytes(), set(ledger.parent.iterdir()) + + def fail_replace(*args): + raise OSError("private diagnostic payload") + + monkeypatch.setattr(spend.os, "replace", fail_replace) + with pytest.raises(OSError): + if operation == "reserve": + spend.reserve(ledger, "a", "1", "purpose") + else: + spend.set_limit(ledger, "5000", "reviewed example budget increase") + assert ledger.read_bytes() == before + assert set(ledger.parent.iterdir()) == files + + +def test_duplicate_keys_and_corrupt_costs_fail_closed(ledger): + ledger.write_text( + '{"schema":"openenv-shared-spend-v1","limit_usd":"10","limit_usd":"100","reservations":{}}' + ) + before = ledger.read_bytes() + with pytest.raises(ValueError, match="Duplicate"): + spend.reserve(ledger, "a", "1", "purpose") + assert ledger.read_bytes() == before + ledger.write_text( + json.dumps( + dict( + schema="openenv-shared-spend-v1", + limit_usd="10", + reservations={ + "bad": dict(status="reserved", reserved_usd=True, spent_usd=None) + }, + ) + ) + ) + before = ledger.read_bytes() + with pytest.raises(ValueError): + spend.reserve(ledger, "a", "1", "purpose") + assert ledger.read_bytes() == before + + +def test_module_cli_outputs_totals_and_statuses_without_freeform_evidence( + ledger, capsys +): + command = [ + sys.executable, + "-B", + "-m", + "examples.hub_validation_pipeline.spend", + "--ledger", + str(ledger), + ] + result = subprocess.run( + command + + [ + "reserve", + "--id", + "cli", + "--usd", + "3", + "--purpose", + "purpose", + "--evidence", + "private evidence", + ], + check=True, + text=True, + capture_output=True, + ) + assert json.loads(result.stdout)["charged_usd_exact"] == "3" + assert "private evidence" not in result.stdout + assert not result.stderr + result = subprocess.run( + command + + ["settle", "--id", "cli", "--usd", "2", "--evidence", "private receipt"], + check=True, + text=True, + capture_output=True, + ) + assert json.loads(result.stdout)["statuses"] == {"cli": "settled"} + result = subprocess.run( + command + ["summary"], check=True, text=True, capture_output=True + ) + assert json.loads(result.stdout)["spent_usd_exact"] == "2" + result = subprocess.run( + command + + [ + "set-limit", + "--usd", + "5000", + "--evidence", + "reviewed example budget increase", + ], + check=True, + text=True, + capture_output=True, + ) + assert json.loads(result.stdout)["limit_usd_exact"] == "5000" + assert json.loads(result.stdout)["spent_usd_exact"] == "2" + assert "user decision" not in result.stdout + before = ledger.read_bytes() + result = subprocess.run( + command + ["set-limit", "--usd", "6000"], text=True, capture_output=True + ) + assert result.returncode == 2 + assert ledger.read_bytes() == before + with pytest.raises(SystemExit) as error: + spend.main( + [ + "--ledger", + str(ledger), + "reserve", + "--id", + "bad", + "--usd", + "private invalid amount", + "--purpose", + "purpose", + ] + ) + assert error.value.code == 2 + output = capsys.readouterr() + assert not output.out + assert "private invalid amount" not in output.err