diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..70ace00 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Preserve the hash-verified upstream evaluator byte-for-byte. +benchmarks/blenderbench_direct/ref_based_eval.py whitespace=-blank-at-eol diff --git a/.github/workflows/blenderbench-portable.yml b/.github/workflows/blenderbench-portable.yml new file mode 100644 index 0000000..16e23a8 --- /dev/null +++ b/.github/workflows/blenderbench-portable.yml @@ -0,0 +1,44 @@ +name: BlenderBench portable unit tests + +on: + push: + paths: + - "benchmarks/blenderbench_direct/**" + - "tools/direct_*.py" + - "tests/test_blenderbench_publication.py" + - "tests/test_direct_*.py" + - ".github/workflows/blenderbench-portable.yml" + pull_request: + paths: + - "benchmarks/blenderbench_direct/**" + - "tools/direct_*.py" + - "tests/test_blenderbench_publication.py" + - "tests/test_direct_*.py" + - ".github/workflows/blenderbench-portable.yml" + +permissions: + contents: read + +jobs: + unit: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.13" + check-latest: false + - name: Install minimal hash-locked fixture environment + run: >- + python -m pip install --disable-pip-version-check + --require-hashes -r benchmarks/blenderbench_direct/requirements-fixture.lock + - name: Run CPU-only fixture tests + run: >- + python -m unittest -v + tests.test_blenderbench_publication + tests.test_direct_codex + tests.test_direct_recovery + tests.test_direct_saved_cleanup + tests.test_direct_suite + tests.test_direct_unrestricted diff --git a/.gitignore b/.gitignore index 08d11b1..26ad8f5 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ htmlcov/ /.env.* !/.env.example !/.env.template + +# BlenderBench data, model evidence, and publication artifacts stay external. +/blenderbench-data/ +/blenderbench-results/ diff --git a/benchmarks/blenderbench_direct/README.md b/benchmarks/blenderbench_direct/README.md new file mode 100644 index 0000000..635fe39 --- /dev/null +++ b/benchmarks/blenderbench_direct/README.md @@ -0,0 +1,223 @@ +# Portable full-27 BlenderBench runner + +This package runs the 27 public BlenderBench tasks with a model selected at the +CLI. It is an external sequential dispatcher, not a reasoning controller. One +direct Codex session per task receives exactly four Blender MCP tools: + +- `execute_blender_code_for_cli` +- `get_render_as_image_for_cli` +- `get_runtime_python_api_docs_for_cli` +- `search_api_docs` + +`get_render_as_image_for_cli` independently reopens a saved checkpoint, renders +the exact on-disk bytes with CUDA-only Cycles in the configured CLI backend, and +returns actual PNG image content plus the source SHA-256 through MCP for each +required visual-inspection round. Each edit uses `expected_output_blend`; the MCP +server rejects pre-existing or unchanged output and reports server-computed +source/output hashes. Codex's built-in `view_image` remains disabled. + +The frozen publication protocol requests exactly 10 sequential, meaningful, +saved edit/render/visual-inspection rounds per task. Generation has no shell +scene operations, goal/solution code or goal-scene access, VLM judge, CLIP or +photometric feedback, or evaluator feedback. Saved scenes are opened, audited, +GPU-rendered, and scored only after generation exits. A stopped/failed run is +never regenerated or resumed automatically. + +## External provenance and attribution + +The input manifest pins the [BlenderBench dataset](https://huggingface.co/datasets/DietCoke4671/BlenderBench) +by DietCoke4671 and contributors at revision +`203e4d325e9438ca55b29bdfc4f6a90842d74e68`, under **CC BY 4.0**. Preserve +upstream and per-asset attribution when redistributing downloaded assets. The +repository stores only manifests; downloaded `.blend` files and images belong +outside Git. + +The metric wrapper extracts `clip_similarity` and `photometric_loss` from +[VIGA release](https://github.com/Fugtemypt123/VIGA-release) revision +`69cb8ef0651bf68124815682df4a0f8e8b57c141`. The hash-verified upstream +`ref_based_eval.py` is included under its MIT license; see +[`VIGA-LICENSE.txt`](VIGA-LICENSE.txt). The wrapper refuses evaluator bytes unless +SHA-256 is +`9ee08818aa858dccfff3efbb36cb9a98e8887f14e6168ad1a23731351cdeb3b5`. +CLIP is `openai/clip-vit-base-patch32` at revision +`3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268`. Scoring reports CLIP **image +embedding cosine similarity**, N-CLIP (`1 - cosine`), and photometric loss. + +## Public setup + +1. Install exactly `codex-cli 0.154.0` (upstream source revision + `6b9826e3aa83b1a5947db50f4332cb9c65f1b340`). The launcher ignores user + configuration, uses a read-only Codex sandbox, and disables the 0.154.0 + built-in shell, image, dynamic/code-mode, app/plugin, browser, computer-use, + and image-generation feature paths. MCP tools are forced onto the direct + surface (`omit_tools_from = ["deferred", "code_mode"]`). Exactly the four + listed Blender tools remain available; ordinary model reasoning is unchanged. +2. Install the MCP server and a GPU-capable standalone `bpy` Python. Record + `bpy.app.version_string`. The runtime requires Cycles CUDA, enables CUDA + devices only, disables CPU, and has no CPU-render fallback. +3. Copy `tools/direct_codex.example.toml` outside the repository and replace all + absolute placeholders. It must enable exactly the four tools above and name + the same bpy interpreter passed to the runner. +4. Create a separate exact CPython **3.13.15 x86_64 Linux / CUDA 12.8** scoring + environment and install the complete hash-locked dependency set with + `python -m pip install --require-hashes -r + benchmarks/blenderbench_direct/requirements-scoring.lock`. The lock pins the + `torch 2.8.0+cu128` CPython-3.13 wheel by immutable URL and hash and is the + normative reproduction environment. Enter this venv and invoke every `run`, + `recover`, and `score` command with that venv's `python`; external scorer + executable/script options are intentionally not accepted. Before generation + or scoring, an isolated child verifies lexical `sys.executable`, CPython and + venv identity, x86_64/64-bit/SOABI, exact imported runtime versions, CUDA, + and the exact 41-distribution lock (apart from allowed `pip`). + + Download the eight required CLIP files at the pinned revision into a dedicated + snapshot directory. `clip-snapshot-manifest.json` is the exact path/size/SHA-256 + allowlist. `--clip-snapshot` is verified before and after each scorer process; + offline/local-only loading rejects extras, missing files, byte changes, and + symlinks outside a standard Hugging Face model cache root. + + **Security boundary:** the historical lock intentionally retains + `setuptools 78.1.0` and `transformers 4.55.4` for exact environment fidelity. + A 2026-09-11 `pip-audit` reports 16 findings across those two distributions; + fixed versions would change the locked environment. The wrapper therefore + accepts only the pinned CLIP revision from an offline cache, requests + `weights_only=True`, and must not be used with untrusted model caches or + checkpoints. Any dependency upgrade requires a separately hashed evaluation + condition and numerical-equivalence validation. +5. Use external dataset and output directories. Keep credentials out of MCP TOML; + public manifests omit MCP `args`, `env`, and `variables` and redact recognized + secret-bearing keys and values recursively, but raw Codex evidence can still + contain sensitive model output and must be reviewed before publication. + +Download all 27 pinned tasks (or pass an explicit `--tasks` subset): + +```sh +python -m benchmarks.blenderbench_direct.runner download \ + --dataset-root /external/blenderbench-data +``` + +Run all 27 tasks by omitting `--tasks`: + +```sh +python -m benchmarks.blenderbench_direct.runner run \ + --dataset-root /external/blenderbench-data \ + --mcp-toml /external/blender-mcp.toml \ + --codex-binary /absolute/bin/codex \ + --codex-version 'EXACT CODEX --version OUTPUT' \ + --bpy-python /absolute/bpy-venv/bin/python \ + --bpy-version 'EXACT BPY VERSION PREFIX' \ + --clip-snapshot /external/clip-vit-base-patch32-snapshot \ + --output-root /external/fresh-run \ + --model 'PROVIDER/MODEL-ID' \ + --model-revision 'PROVIDER-REVISION-IF-KNOWN' \ + --rounds 10 --approve-blender-tools +``` + +An explicit subset uses the same fresh-run command plus, for example: + +```sh +--tasks level1/camera1 level2/attribute1 +``` + +`--rounds` defaults to 10 and accepts only 10. Every runtime path, requested +model, Codex pin, bpy pin, and CLIP snapshot is explicit. The output root must not +exist. Existing task output, checkpoint files, or run roots are never treated as +authorization to regenerate. + +## Recovery and standalone scoring + +Recovery is an explicit audit-and-scoring operation. It never calls Codex or +rewrites retained generation artifacts; it writes reopened-scene audit renders +and scores to a separate fresh output root. Before doing so it revalidates the +retained generation JSONL, attested checkpoint hashes, and ordered visual-image +evidence: + +```sh +python -m benchmarks.blenderbench_direct.runner recover \ + --source-root /external/retained-run --output-root /external/fresh-recovery \ + --dataset-root /external/blenderbench-data \ + --bpy-python /absolute/bpy-venv/bin/python --bpy-version 'EXACT BPY VERSION PREFIX' \ + --clip-snapshot /external/clip-vit-base-patch32-snapshot +``` + +Standalone scoring also never generates. It admits only the +`auditNN/verified.png` images produced by independent saved-scene reopen and GPU +rendering whose `audit-admission.json` records successful meaningful-change +validation and whose source, render, and full audit hashes match the retained +audit metadata. Model-written checkpoint PNGs are not scoring inputs: + +```sh +python -m benchmarks.blenderbench_direct.runner score \ + --run-root /external/retained-run \ + --dataset-root /external/blenderbench-data \ + --clip-snapshot /external/clip-vit-base-patch32-snapshot \ + --output /external/retained-run/scoring +``` + +`aggregate.json` and `aggregate.csv` include complete, failed, and missing tasks. +The JSON aggregate records the pinned dataset, evaluator, CLIP, and canonical +scorer digest. Every target is hash-verified before standalone scoring starts, +and every completed scorer output must repeat the exact evaluator and CLIP identity. +Best round is minimum N-CLIP; an exact tie selects the earliest round. Paired +photometric loss comes from that same round. Failures are not dropped from the +denominator. + +## Canonical protocol and artifacts + +`run-manifest.json` records resolved Codex/bpy/scorer versions, scorer SHA-256, +redacted effective config, exact tool list, dataset/evaluator/model revisions, +render and generation rules, initialization-manifest digest, implementation-file +digests, full Git revision and dirty state, and host/CPU/GPU/runtime provenance. Its +`protocol_sha256` is SHA-256 over canonical sorted UTF-8 JSON of the `protocol` +object; volatile hardware provenance is recorded but excluded from that hash. +The Git state and source digests are included: identical clean checkouts reproduce +the identity, while a dirty checkout is explicitly distinct and cannot claim clean +source provenance. Generation JSONL is checked after Codex exits and before any +audit or scoring; shell/file-change/builtin-tool operations, non-allowlisted MCP +calls, goal/solution access, and score/evaluator feedback fail the task closed. + +Large datasets, raw model captures, `.blend` files, renders, scores, and videos +stay outside Git. Publish immutable artifacts separately and add only verified +metadata to a report, for example: + +```json +{"url":"https://YOUR-ARTIFACT-HOST/path","sha256":"MEASURED-64-HEX-DIGEST"} +``` + +Do not replace placeholders with invented URLs or digests. Hash the uploaded +bytes, download by the public URL, hash again, and require equality before +claiming publication. + +## Compilation video + +Create a JSON spec with task-local `duration_seconds`, target/initial images, +model, optional final `task_tokens`, and round rows containing `round`, +`elapsed_seconds`, `image`, and `clip_image_cosine_similarity`. Then run: + +```sh +python -m benchmarks.blenderbench_direct.runner video \ + --spec /external/video-spec.json --output /external/compilation.mp4 \ + --ffmpeg /absolute/bin/ffmpeg +``` + +Overlays say “CLIP image cosine similarity” exactly. Token displays say +“Task-level tokens (linear interpolation)” explicitly; they are not event-level +usage and remain unavailable when task usage is unavailable. A sidecar JSON +records every rendered frame. + +## Limitations + +- Codex's built-in `view_image` path remains disabled. Visual inspection is + instead restricted to `get_render_as_image_for_cli`; post-run JSONL validation + requires exactly one successful PNG image response for each sequential round + checkpoint. +- MCP Python runs with server OS privileges; Codex's read-only sandbox does not + sandbox the MCP server. Use a disposable isolated account/runtime. +- Version pins establish identity, not equivalence across hardware or providers. +- GPU admission and full runs require Blender/bpy, CUDA, model access, and CLIP + weights; CI tests exercise only pure/fixture orchestration. +- Meaningful-change auditing detects repeated/nonfinite complete scene state, but + it is not a semantic judge. +- The direct 10-round protocol is not demonstrated to have the same budget, + interaction pattern, or judging procedure as VIGA. Do **not** describe results + as a fair same-budget VIGA comparison. diff --git a/benchmarks/blenderbench_direct/TDD_EVIDENCE.md b/benchmarks/blenderbench_direct/TDD_EVIDENCE.md new file mode 100644 index 0000000..822c0ef --- /dev/null +++ b/benchmarks/blenderbench_direct/TDD_EVIDENCE.md @@ -0,0 +1,64 @@ +# Publication-readiness TDD evidence + +All commands ran from the repository root. Goal/solution code and benchmark generation were not executed. + +| Behavior | RED (before production patch) | GREEN (after production patch) | +|---|---|---| +| Protocol source identity | `python -m unittest -v tests.test_blenderbench_publication.ProtocolManifestTests.test_protocol_hash_binds_initialization_implementation_and_git_state` → `TypeError: build_manifest() got an unexpected keyword argument 'source_identity'` | same command → `Ran 1 test ... OK` | +| Scoring-lock protocol binding | `...ProtocolManifestTests.test_source_identity_binds_hash_locked_scoring_environment` → `KeyError: 'scoring_lock_sha256'` | same command → `Ran 1 test ... OK` | +| Evaluator/CLIP output identity | `...ScoringTests.test_score_pair_rejects_wrong_evaluator_or_clip_identity` → `AssertionError: ValueError not raised` | same command → `Ran 1 test ... OK` | +| Standalone target verification | `...ScoringTests.test_standalone_scoring_verifies_dataset_before_reading_renders` → `AssertionError: ValueError not raised` | same command → `Ran 1 test ... OK` | +| Aggregate provenance | `...ScoringTests.test_standalone_aggregate_records_dataset_evaluator_and_scorer_provenance` → `KeyError: 'provenance'` | same command → `Ran 1 test ... OK` | +| CSV aggregate provenance | `...ScoringTests.test_json_and_csv_outputs_are_machine_readable` → `KeyError: 'dataset_revision'` | same command → `Ran 1 test ... OK` | +| Generation event enforcement | `python -m unittest -v tests.test_direct_codex.DirectCodexTests.test_generation_stream_fails_closed_on_nonapproved_operations_and_feedback` → `AttributeError: ... no attribute 'verify_generation_events'` | same command → `Ran 1 test ... OK` | +| Evaluator URL | `...ProtocolManifestTests.test_evaluator_url_resolves_to_verified_release_repository` → obsolete repository URL assertion failure | corrected test after eliminating an over-structural concatenated-literal assertion → `Ran 1 test ... OK` | +| Vendored evaluator bytes | `...ProtocolManifestTests.test_pinned_evaluator_bytes_are_present_and_bound_into_source_identity` → `AssertionError: False is not true` because `ref_based_eval.py` was absent | same command after adding the hash-verified upstream file and binding it into source identity → `Ran 1 test ... OK` | +| Recursive public redaction | `...ProtocolManifestTests.test_public_config_omits_nested_args_env_variables_and_bearer_urls` → synthetic bearer value remained in serialized output | same command → `Ran 1 test ... OK` | +| Dataset task-path symlink | `...DatasetManifestTests.test_task_path_components_may_not_be_symlinks` → verifier reached an asset through the symlink instead of rejecting the path component | same command → `Ran 1 test ... OK` | +| Broken legacy hooks | `python -m unittest -v tests.test_direct_recovery.RecoveryTests.test_broken_legacy_recovery_hooks_are_removed` → `hasattr(..., 'score_checkpoints')` / later `hasattr(..., 'run_task')` was true | same command → `Ran 1 test ... OK` | +| Immutable CI/lock | `...SuiteProtocolTests.test_ci_uses_immutable_actions_python_and_hashed_scoring_lock` → mutable action refs; after first patch the required lock command was split differently than asserted | exact full-SHA actions, CPython 3.13 CI selection, fixed runner image, and hash lock → `Ran 1 test ... OK` | +| Lexical scorer executable | `...ScoringTests.test_fake_path_executable_is_never_used_and_argv_is_canonical` → a PATH-selected executable remained possible | same command → `Ran 1 test ... OK` | +| Exact scorer runtime and distribution lock | `...ScoringTests.test_runtime_preflight_rejects_wrong_python_and_extra_distribution` → mismatched Python and extra distributions were not rejected | same command → `Ran 1 test ... OK` | +| Immutable CLIP snapshot | `...ScoringTests.test_clip_snapshot_manifest_rejects_extra_or_mutated_bytes` → extra and modified snapshot files were accepted | same command → `Ran 1 test ... OK` | +| Publication manifests | `...SuiteProtocolTests.test_publication_result_manifests_are_small_complete_and_path_free` → assertions failed against the older copied manifest bytes | same command → `Ran 1 test ... OK` | +| Root README result link | `...SuiteProtocolTests.test_root_readme_pins_latest_video_and_metric_meaning` → the clickable immutable video link and metric clarification were absent | same command → `Ran 1 test ... OK` | +| Package-safe runtime import | `...DirectCodexTests.test_generation_workdir_accepts_canonical_worker_when_imported_as_tools_module` → `ModuleNotFoundError: No module named 'direct_runtime'` | same command → `Ran 1 test ... OK` | +| Frozen manifest validation | `...ProtocolManifestTests.test_run_manifest_validation_rejects_rehashed_protocol_rule_drift` → five subtests accepted rehashed drift in tools, CLIP, rendering, generation, and Codex version | same command → `Ran 1 test ... OK` | +| Resumed download cleanup | `...DatasetManifestTests.test_successful_resumed_download_removes_superseded_partial` → retained partial still existed after successful installation | same command → `Ran 1 test ... OK` | +| Standalone audited-render scoring | `...ScoringTests.test_standalone_scoring_uses_independent_audit_render_not_model_png` → scorer received `work/iteration01.png` instead of `audit01/verified.png` | same command → `Ran 1 test ... OK` | +| Standalone audit admission | `...ScoringTests.test_standalone_scoring_rejects_unverified_audit_image` → an unverified `audit01/verified.png` produced a completed score | same command → `Ran 1 test ... OK` | +| Recovery renderer identity | `...RunnerConfigTests.test_recovery_rejects_renderer_different_from_generation_manifest` → recovery accepted a different bpy version when the new CLI pin matched it | same command → `Ran 1 test ... OK` | +| Audit metadata binding | `...ScoringTests.test_standalone_scoring_rejects_admission_without_audit_metadata` → a render with a hand-written admission but no retained audit metadata produced a completed score | same command after requiring matching source/render/full-audit hashes and explicit meaningful-change admission → `Ran 1 test ... OK` | +| MCP image inspection | `...TestRunBlenderCLI.test_cli_render_tool_returns_image_content` → `get_render_as_image_for_cli` was absent | same command after adding the PNG-returning saved-scene render tool → `Ran 1 test ... OK` | +| Per-round visual evidence | `...DirectCodexTests.test_generation_stream_requires_one_successful_image_inspection_per_round` → `verify_generation_events()` did not accept or enforce a round count; `...test_generation_stream_binds_visual_inspection_to_each_round_checkpoint` then accepted the wrong checkpoint sequence | both commands after requiring one successful PNG response for each ordered `iterationNN.blend` → `Ran 1 test ... OK` | +| Visual-tool source binding | `...ProtocolManifestTests.test_visual_inspection_tool_bytes_are_bound_into_source_identity` → the new MCP tool implementation was absent from `implementation_sha256` | same command after binding the MCP tool and CLI backend bytes → `Ran 1 test ... OK` | +| Blender resolution fields | `...DirectCodexTests.test_generation_stream_allows_blender_resolution_properties` → the `solution` filter falsely rejected `resolution_x` | same command after matching `solution` only as a complete prohibited word/file stem → `Ran 1 test ... OK` | +| Visual checkpoint byte binding | `...DirectCodexTests.test_generation_stream_rejects_visual_inspection_from_wrong_directory` → validation accepted an identically named checkpoint from another directory | same command after requiring exact ordered paths, source SHA-256 metadata, retained-byte equality, and edit/inspection alternation → `Ran 1 test ... OK` | +| Audit runtime admission | `...ScoringTests.test_standalone_scoring_rejects_failed_or_non_cuda_audit_metadata` → matching source/render/audit hashes could pass despite failed non-CUDA runtime evidence | same command after requiring successful process, exact generation renderer identity, CUDA-only devices, and GPU scene evidence → `Ran 1 test ... OK` | +| Recovery generation evidence | `...RunnerConfigTests.test_recovery_revalidates_retained_generation_events` → retained generation JSONL had no recovery validation entrypoint | same command after recovery revalidates the regular retained event stream, visual evidence, exact checkpoint paths, and source hashes before audit/scoring → `Ran 1 test ... OK` | +| Exact CUDA visual render | Additional assertions in `...TestRunBlenderCLI.test_cli_render_tool_returns_image_content` → the tool rendered a synchronized unsaved copy and lacked CUDA-only setup | same command after rendering the exact hashed checkpoint bytes with explicit CUDA-only Cycles setup → `Ran 1 test ... OK` | +| Fresh checkpoint attestation | `...TestRunBlenderCLI.test_execute_code_cli_attests_fresh_checkpoint_creation` → `expected_output_blend` was unsupported; `...DirectCodexTests.test_generation_stream_rejects_unattested_precomputed_checkpoint` then accepted precomputed bytes plus a no-op | both commands after server-side fresh-output/source/output hashing and event-level attestation validation → `Ran 1 test ... OK` | +| Cycles audit admission | The EEVEE case in `...ScoringTests.test_standalone_scoring_rejects_failed_or_non_cuda_audit_metadata` → matching hashes and CUDA preference fields were accepted without checking the engine | same command after requiring `engine == "CYCLES"` → `Ran 1 test ... OK` | +| Edit/inspection dispatch ordering | `...DirectCodexTests.test_generation_stream_rejects_next_edit_dispatched_before_prior_inspection` → an interleaved second edit's `item.started` event was accepted before the first edit had completed and been visually inspected | same command after tracking the active edit from dispatch through completion and rejecting any overlapping or pre-inspection edit → `Ran 1 test ... OK` | + +Final direct/publication gate: 76 tests, all passed. The related MCP CLI/tool-listing gate: 41 tests, all passed. + +## Verification notes + +- Static parse: all 22 changed Python files parsed; all 6 benchmark JSON files decoded. +- Manifest validation: 27 tasks, 108 pinned assets, 11 implementation hashes, deterministic repeated dirty-tree identity. +- The final source identity reported revision `4a9f5cc6e616080185252bb919fa3d1e730760b7` and `git_dirty: true`, as required for this uncommitted verification worktree. +- Scoring lock TDD: the focused CI-lock test first failed because the ARM-generated lock omitted CUDA dependencies, then failed again while it still selected stale HTTP packages. It passed after switching to the historical hash-pinned `torch 2.8.0+cu128` x86_64 wheel, pinning the observed transitive versions, and regenerating the lock for CPython 3.13 / x86_64 manylinux. +- Scoring lock verification: `uv pip install --dry-run --system --python-platform x86_64-manylinux_2_28 --python-version 3.13 --require-hashes` resolved all 41 locked packages, including CUDA 12.8 dependencies, successfully. A fresh `pip-audit` reports 16 findings across fidelity-pinned `setuptools 78.1.0` and `transformers 4.55.4`; the README records the offline pinned-model trust boundary and treats any dependency upgrade as a separately hashed condition. +- Full repository suite with `mcp/.venv/bin/python -m unittest discover -s tests -p 'test*.py' -q`: 435 tests ran; 8 errors and 23 skips. The exact environment-only errors were the three missing-Blender `setUpClass` failures in `TestBackgroundServer`, `TestForegroundServer`, and `TestInteractiveServer`, plus missing-Pillow failures in `CameraTests.test_video_calls_tokens`, three `DirectCodexTests` checkpoint tests, and `test_token_efficiency_video_evidence`. The known baseline was 403 tests with 1 failure, 7 errors, and 23 skips: the same three missing-Blender blockers plus five Pillow-dependent outcomes (four errors and one dependent assertion failure). There were no current assertion failures and no focused publication failure. +- The root README poster and immutable Hugging Face URL were fetched successfully. The downloaded video was H.264, 848×464, 134.900 seconds, 10,161,884 bytes, and SHA-256 `04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17`. + +## Review state + +The prior `REQUEST_CHANGES` findings were addressed without running benchmark +generation: visual inspection now has a hash-bound MCP image-returning path with +ordered per-round evidence validation, standalone scoring validates retained +audit metadata and admission hashes, and recovery requires the renderer version +recorded by the generation manifest. The final independent re-review returned +`APPROVED`: overlapping edit dispatch and edits pending checkpoint inspection +are rejected, and no Critical or Important regressions were found. diff --git a/benchmarks/blenderbench_direct/VIGA-LICENSE.txt b/benchmarks/blenderbench_direct/VIGA-LICENSE.txt new file mode 100644 index 0000000..e9abec7 --- /dev/null +++ b/benchmarks/blenderbench_direct/VIGA-LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Fugtemypt123 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/benchmarks/blenderbench_direct/__init__.py b/benchmarks/blenderbench_direct/__init__.py new file mode 100644 index 0000000..690cf48 --- /dev/null +++ b/benchmarks/blenderbench_direct/__init__.py @@ -0,0 +1,3 @@ +"""Publication BlenderBench runner components.""" + +__all__ = ["dataset", "protocol", "runner", "scoring", "video"] diff --git a/benchmarks/blenderbench_direct/assets-manifest.json b/benchmarks/blenderbench_direct/assets-manifest.json new file mode 100644 index 0000000..ee84013 --- /dev/null +++ b/benchmarks/blenderbench_direct/assets-manifest.json @@ -0,0 +1,1037 @@ +{ + "schema_version": 1, + "repository": "DietCoke4671/BlenderBench", + "revision": "203e4d325e9438ca55b29bdfc4f6a90842d74e68", + "tree_pages": 1, + "tasks": { + "level1/camera1": { + "task.txt": { + "path": "data/blenderbench/level1/camera1/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera1/start.py", + "size": 146, + "sha256": "8680900dd5a85c2467f86a7ec0c47e3868569e387948eb7e45aab65794981c06", + "git_blob_oid": "01a913bc51a3cbc16124c5e967396a5c0e25fd22", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera1/blender_file.blend", + "size": 11654045, + "sha256": "777b62ebe45894d18fa056c6c8c4d97d8cfc36115246e5fad46a803b52200415", + "git_blob_oid": "29514b1060cbc66d19ec8b19478aaebfef5358ec", + "lfs": { + "oid": "777b62ebe45894d18fa056c6c8c4d97d8cfc36115246e5fad46a803b52200415", + "size": 11654045, + "pointerSize": 133 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera1/renders/goal/render1.png", + "size": 280875, + "sha256": "a1f17f06448cd899c5912323f9b9dd07ca6ea30dc0aebba11f28cb85afb29d2c", + "git_blob_oid": "a74b55578860c62d8f82a488959df660a60ff4a9", + "lfs": { + "oid": "a1f17f06448cd899c5912323f9b9dd07ca6ea30dc0aebba11f28cb85afb29d2c", + "size": 280875, + "pointerSize": 131 + } + } + }, + "level1/camera2": { + "task.txt": { + "path": "data/blenderbench/level1/camera2/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera2/start.py", + "size": 146, + "sha256": "41cae2396027226fb32fd06a6e0a6e913bd895811bdee383416ef1bd1598d186", + "git_blob_oid": "ac190d95666978eb2bca17ad5f2d79dd088f8c36", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera2/blender_file.blend", + "size": 21287404, + "sha256": "9e53b677deab9a576b1d311f716c311def91591db41a33518a0433bc4cef8d3b", + "git_blob_oid": "09ff4f35c567f062437a47651d499108fea0a03a", + "lfs": { + "oid": "9e53b677deab9a576b1d311f716c311def91591db41a33518a0433bc4cef8d3b", + "size": 21287404, + "pointerSize": 133 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera2/renders/goal/render1.png", + "size": 256033, + "sha256": "ce58bcf86a6866d992e9d115391cf4a411b65395ffd612ccb7d28dc7ba5e5b58", + "git_blob_oid": "369f1d0a773fe19cb7bc5882d34d9db05bb00a61", + "lfs": { + "oid": "ce58bcf86a6866d992e9d115391cf4a411b65395ffd612ccb7d28dc7ba5e5b58", + "size": 256033, + "pointerSize": 131 + } + } + }, + "level1/camera3": { + "task.txt": { + "path": "data/blenderbench/level1/camera3/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera3/start.py", + "size": 190, + "sha256": "f5acd7609ff23e01e3d435173d46ea45907fc2b3771dea91f5da0007e51500f2", + "git_blob_oid": "b1ce78b0c679749e66cc1fa7a8972695af81cce2", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera3/blender_file.blend", + "size": 19714277, + "sha256": "2d8bb6bc5b8614ab26d6319249c4cf915e5e8107f7859ec768dfb0e02b875403", + "git_blob_oid": "0f5bce886d59f7decf761e932c5f8c287dc69273", + "lfs": { + "oid": "2d8bb6bc5b8614ab26d6319249c4cf915e5e8107f7859ec768dfb0e02b875403", + "size": 19714277, + "pointerSize": 133 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera3/renders/goal/render1.png", + "size": 323359, + "sha256": "3cde2e268b0fb73a25bc96180bbe96818f464bbe71fc8fe9bc1dff8a3d782d18", + "git_blob_oid": "5df097446778f579549300ea050a818c8b35056c", + "lfs": { + "oid": "3cde2e268b0fb73a25bc96180bbe96818f464bbe71fc8fe9bc1dff8a3d782d18", + "size": 323359, + "pointerSize": 131 + } + } + }, + "level1/camera4": { + "task.txt": { + "path": "data/blenderbench/level1/camera4/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera4/start.py", + "size": 145, + "sha256": "a994ae0f9c936a8a4a469b5245bc5cc438e7a112ec55288b7c2473998872d657", + "git_blob_oid": "e033c342fb7af3269574d25707e0060b06b774a6", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera4/blender_file.blend", + "size": 1092706714, + "sha256": "e99ccd5e43e80d8dbdd4d840ea4290f1dbc6b4e06d21be07eeed7d39fecafd0a", + "git_blob_oid": "b66db22091cd21021b2630ca2fc01040cf35b27f", + "lfs": { + "oid": "e99ccd5e43e80d8dbdd4d840ea4290f1dbc6b4e06d21be07eeed7d39fecafd0a", + "size": 1092706714, + "pointerSize": 135 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera4/renders/goal/render1.png", + "size": 434190, + "sha256": "a6cad78bdebb96be3147176d6c4c460df4782c51430fcd268dfddd87f499aa93", + "git_blob_oid": "2e09e3634d5c1c7cd3a0bd3d30ac333922d3570e", + "lfs": { + "oid": "a6cad78bdebb96be3147176d6c4c460df4782c51430fcd268dfddd87f499aa93", + "size": 434190, + "pointerSize": 131 + } + } + }, + "level1/camera5": { + "task.txt": { + "path": "data/blenderbench/level1/camera5/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera5/start.py", + "size": 148, + "sha256": "2d09f0ea2b51ace93663882d86aa2fa4201f394710879c12c6774173793a8971", + "git_blob_oid": "5761047b75c43463abb73e2c967c5c64c9de8c51", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera5/blender_file.blend", + "size": 1092786654, + "sha256": "06f30a3ccd0d81be515f1892605fd575f6933052e15b9b297b7d96a48aa664d2", + "git_blob_oid": "f54d89e9d4c3f0d32b34e51ac2cd5d51743bb4b0", + "lfs": { + "oid": "06f30a3ccd0d81be515f1892605fd575f6933052e15b9b297b7d96a48aa664d2", + "size": 1092786654, + "pointerSize": 135 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera5/renders/goal/render1.png", + "size": 442674, + "sha256": "0124cf59482f3950b2c62eddf8d734e752e45e5300fbf30ce472f74b05e30f9d", + "git_blob_oid": "d411365abfd0112bcefbd1c253b631cd59f48e6f", + "lfs": { + "oid": "0124cf59482f3950b2c62eddf8d734e752e45e5300fbf30ce472f74b05e30f9d", + "size": 442674, + "pointerSize": 131 + } + } + }, + "level1/camera6": { + "task.txt": { + "path": "data/blenderbench/level1/camera6/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera6/start.py", + "size": 146, + "sha256": "1539bdf69d7df4cb2da3770a59f2a33ed28d3d20aec3c3e2677351b297850f9c", + "git_blob_oid": "a0449cfce1acdf566251c2634b2b38469006af82", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera6/blender_file.blend", + "size": 5718592, + "sha256": "4e0fb03e37b9072c9ca65bab2b6594deb4254d21b3e13c08b8eeaf1fce76e45f", + "git_blob_oid": "2073a629bfaa3343eebd0b66a0a7475d005aadc4", + "lfs": { + "oid": "4e0fb03e37b9072c9ca65bab2b6594deb4254d21b3e13c08b8eeaf1fce76e45f", + "size": 5718592, + "pointerSize": 132 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera6/renders/goal/render1.png", + "size": 264383, + "sha256": "e8ef88036bdd3e417383c349da3f531e22698d765f34b60851ec132d143f99d3", + "git_blob_oid": "cb03a3a9e48c23b0900716f00cd510d252856940", + "lfs": { + "oid": "e8ef88036bdd3e417383c349da3f531e22698d765f34b60851ec132d143f99d3", + "size": 264383, + "pointerSize": 131 + } + } + }, + "level1/camera7": { + "task.txt": { + "path": "data/blenderbench/level1/camera7/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera7/start.py", + "size": 146, + "sha256": "5471be8faa7f4bd3ba2963a6716298a007d8a004a11df38cb87b9f026d9f4bbc", + "git_blob_oid": "9c0fa4d13f79519e9950d330ececa5e5563334fa", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera7/blender_file.blend", + "size": 198756884, + "sha256": "6106fe112e0bfdf4e7aab26ff4e901ad8f942a9e6493839a9d795806cf97ab60", + "git_blob_oid": "8a09b0908159af7cb9d32370e8b7d87c58590b4f", + "lfs": { + "oid": "6106fe112e0bfdf4e7aab26ff4e901ad8f942a9e6493839a9d795806cf97ab60", + "size": 198756884, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera7/renders/goal/render1.png", + "size": 527080, + "sha256": "7f5efeca8b9586450cbb5a019bc03f2b0616f76d29fecfc699cca0059150a8fd", + "git_blob_oid": "7ac2abc00d8b688de48c4278a56bc8bf3bf517fb", + "lfs": { + "oid": "7f5efeca8b9586450cbb5a019bc03f2b0616f76d29fecfc699cca0059150a8fd", + "size": 527080, + "pointerSize": 131 + } + } + }, + "level1/camera8": { + "task.txt": { + "path": "data/blenderbench/level1/camera8/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera8/start.py", + "size": 146, + "sha256": "1dd7b06ac2d5949e0453f0e0c8ae857a417e986d9cc3b6fad748d26abf1b8151", + "git_blob_oid": "5b670d67b5a328d591968b16afe9be57b7735c17", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera8/blender_file.blend", + "size": 2303572, + "sha256": "64124da51512f689ff1c2d0a6475ccf377bb94a87a3bcc9fd14e0d38de7a4e00", + "git_blob_oid": "2a8b648f8eef7daf70287a587ce7a490c8ec31bb", + "lfs": { + "oid": "64124da51512f689ff1c2d0a6475ccf377bb94a87a3bcc9fd14e0d38de7a4e00", + "size": 2303572, + "pointerSize": 132 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera8/renders/goal/render1.png", + "size": 198412, + "sha256": "b4e026fea8211aaf7f1149cd3e20a2ac6346882484eee5804ea6e19dc43b6bc4", + "git_blob_oid": "2ce8adacf714c8fbc432e30ff94b3989b2ce2991", + "lfs": { + "oid": "b4e026fea8211aaf7f1149cd3e20a2ac6346882484eee5804ea6e19dc43b6bc4", + "size": 198412, + "pointerSize": 131 + } + } + }, + "level1/camera9": { + "task.txt": { + "path": "data/blenderbench/level1/camera9/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera9/start.py", + "size": 136, + "sha256": "6c104adc69f6e714e7b51d3e834b8bd7e6836813899ea84c972c6d2208ac40b6", + "git_blob_oid": "786bf8646e259f12b3b2960baf3c40358fe89663", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera9/blender_file.blend", + "size": 4107448, + "sha256": "af9b2132fbffb66a1a4832c500c82bc00ee19ac90a40c2b1f96e95bfe74b295d", + "git_blob_oid": "0f1887978fb6c72a5b7c1d1fb3a51d26ca498069", + "lfs": { + "oid": "af9b2132fbffb66a1a4832c500c82bc00ee19ac90a40c2b1f96e95bfe74b295d", + "size": 4107448, + "pointerSize": 132 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera9/renders/goal/render1.png", + "size": 311165, + "sha256": "df51a5786e84b1d47c2df08c95d6adb3f7594dec20fa17c47424bb08b16145b6", + "git_blob_oid": "615154f4ffee4b912add08312db65fd507688af7", + "lfs": { + "oid": "df51a5786e84b1d47c2df08c95d6adb3f7594dec20fa17c47424bb08b16145b6", + "size": 311165, + "pointerSize": 131 + } + } + }, + "level2/attribute1": { + "task.txt": { + "path": "data/blenderbench/level2/attribute1/task.txt", + "size": 119, + "sha256": "e2f9834ab4a6888041e4e65c01806c816067cbd68931f49d181a938347186caa", + "git_blob_oid": "c55b5774e43199c6638deb8ff5990a3395a4ee3f", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute1/start.py", + "size": 1240, + "sha256": "57448a36ff834acc820d071fc99b26285d8968d4fece18300a5c512e18ac2f18", + "git_blob_oid": "86c038455c8a485c251ed84a64285bf10dc6896f", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute1/blender_file.blend", + "size": 950771988, + "sha256": "d5eca6c92d3cee2ce8a4ea9986a644e4d61a1d66c123c81050616026fa12f5f8", + "git_blob_oid": "bfbc64a723927a21ec431856ee3b0bcb17971736", + "lfs": { + "oid": "d5eca6c92d3cee2ce8a4ea9986a644e4d61a1d66c123c81050616026fa12f5f8", + "size": 950771988, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute1/renders/goal/render1.png", + "size": 435882, + "sha256": "a269058785099fd3f273334e70fa147de3ee3556a500ded8d9b064213fcd8981", + "git_blob_oid": "ec260b02febe6c7dbaea38a531f584ed828dd677", + "lfs": { + "oid": "a269058785099fd3f273334e70fa147de3ee3556a500ded8d9b064213fcd8981", + "size": 435882, + "pointerSize": 131 + } + } + }, + "level2/attribute2": { + "task.txt": { + "path": "data/blenderbench/level2/attribute2/task.txt", + "size": 119, + "sha256": "e2f9834ab4a6888041e4e65c01806c816067cbd68931f49d181a938347186caa", + "git_blob_oid": "c55b5774e43199c6638deb8ff5990a3395a4ee3f", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute2/start.py", + "size": 1180, + "sha256": "83702d094c0fa4d6e4a29711dfa024f587339353b1f9ddd6aff5e17b3ab426a8", + "git_blob_oid": "1c1f9de3379324f7e099bb90596f2ff70382ce8e", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute2/blender_file.blend", + "size": 950771988, + "sha256": "d5eca6c92d3cee2ce8a4ea9986a644e4d61a1d66c123c81050616026fa12f5f8", + "git_blob_oid": "bfbc64a723927a21ec431856ee3b0bcb17971736", + "lfs": { + "oid": "d5eca6c92d3cee2ce8a4ea9986a644e4d61a1d66c123c81050616026fa12f5f8", + "size": 950771988, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute2/renders/goal/render1.png", + "size": 435876, + "sha256": "0744a471d3a07fe9c4a38402f9be09310e5e26dc4abbc62f6b634d20b1f6c23f", + "git_blob_oid": "c40abc765e184b1851ed8ac5469bad8045e5f144", + "lfs": { + "oid": "0744a471d3a07fe9c4a38402f9be09310e5e26dc4abbc62f6b634d20b1f6c23f", + "size": 435876, + "pointerSize": 131 + } + } + }, + "level2/attribute3": { + "task.txt": { + "path": "data/blenderbench/level2/attribute3/task.txt", + "size": 119, + "sha256": "e2f9834ab4a6888041e4e65c01806c816067cbd68931f49d181a938347186caa", + "git_blob_oid": "c55b5774e43199c6638deb8ff5990a3395a4ee3f", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute3/start.py", + "size": 1178, + "sha256": "0bd075a70511e2952ff888d6a7d72499311a82764c7ab8c7bb7ae659ba210706", + "git_blob_oid": "eb069eba41ab9147184bce60690f6b5355b6c959", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute3/blender_file.blend", + "size": 950771988, + "sha256": "d5eca6c92d3cee2ce8a4ea9986a644e4d61a1d66c123c81050616026fa12f5f8", + "git_blob_oid": "bfbc64a723927a21ec431856ee3b0bcb17971736", + "lfs": { + "oid": "d5eca6c92d3cee2ce8a4ea9986a644e4d61a1d66c123c81050616026fa12f5f8", + "size": 950771988, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute3/renders/goal/render1.png", + "size": 435892, + "sha256": "3886f82c111b6e7dada41bcd6129e9c782f543a11e02860c693a5e3c8a336da7", + "git_blob_oid": "a28d2858477a08df993ec088e53805f8c16d9df6", + "lfs": { + "oid": "3886f82c111b6e7dada41bcd6129e9c782f543a11e02860c693a5e3c8a336da7", + "size": 435892, + "pointerSize": 131 + } + } + }, + "level2/attribute4": { + "task.txt": { + "path": "data/blenderbench/level2/attribute4/task.txt", + "size": 130, + "sha256": "fba1c19366ae11259cee9886681c0c8688f65cccc033e45ce82bd1a8cf1d7afc", + "git_blob_oid": "e30cdd33cdd13d3059c5e9861224baaac06af74c", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute4/start.py", + "size": 703, + "sha256": "1991b217fd4e044f7ef0966bbb38e8e417bb1fa967effba61bbfc880f7e36543", + "git_blob_oid": "e2a610617d76c2c223303277742b9484f9ec1e7c", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute4/blender_file.blend", + "size": 947346540, + "sha256": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "git_blob_oid": "651e1029bc10ca62d380be72c08e122720b5a7c2", + "lfs": { + "oid": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "size": 947346540, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute4/renders/goal/render1.png", + "size": 433064, + "sha256": "dd049697d61750d34d9f0587a832d7b8ad322197e63b4319b1938675b64eefcb", + "git_blob_oid": "1c8fb0f28aec077a44f6a96ce1a3c6e852b04378", + "lfs": { + "oid": "dd049697d61750d34d9f0587a832d7b8ad322197e63b4319b1938675b64eefcb", + "size": 433064, + "pointerSize": 131 + } + } + }, + "level2/attribute5": { + "task.txt": { + "path": "data/blenderbench/level2/attribute5/task.txt", + "size": 130, + "sha256": "fba1c19366ae11259cee9886681c0c8688f65cccc033e45ce82bd1a8cf1d7afc", + "git_blob_oid": "e30cdd33cdd13d3059c5e9861224baaac06af74c", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute5/start.py", + "size": 644, + "sha256": "d80af70f035011ae1d6bf7001c9cad008e3cb63a6442bfbd87b23e0d4e09fb0a", + "git_blob_oid": "7d91d84c63e898aa443debdcf83e46bc00498586", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute5/blender_file.blend", + "size": 947346540, + "sha256": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "git_blob_oid": "651e1029bc10ca62d380be72c08e122720b5a7c2", + "lfs": { + "oid": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "size": 947346540, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute5/renders/goal/render1.png", + "size": 433064, + "sha256": "0b33dfa853ce71b3e0a91ae9813e90dba0544f489251d8a811c810c1491fce55", + "git_blob_oid": "3824f88b86b5b559a493a42a983a78ecb68adb99", + "lfs": { + "oid": "0b33dfa853ce71b3e0a91ae9813e90dba0544f489251d8a811c810c1491fce55", + "size": 433064, + "pointerSize": 131 + } + } + }, + "level2/attribute6": { + "task.txt": { + "path": "data/blenderbench/level2/attribute6/task.txt", + "size": 130, + "sha256": "fba1c19366ae11259cee9886681c0c8688f65cccc033e45ce82bd1a8cf1d7afc", + "git_blob_oid": "e30cdd33cdd13d3059c5e9861224baaac06af74c", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute6/start.py", + "size": 641, + "sha256": "aadf4e84cd6b0e03c47f45cc1e1b6f12a436b6273ffd7f81cd3f2d28a4c072bf", + "git_blob_oid": "8c4abaa13956c250a59146898251b52928d5138f", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute6/blender_file.blend", + "size": 947346540, + "sha256": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "git_blob_oid": "651e1029bc10ca62d380be72c08e122720b5a7c2", + "lfs": { + "oid": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "size": 947346540, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute6/renders/goal/render1.png", + "size": 433064, + "sha256": "1eda79fb1fd8773714467694b24cbbf71f049549b0016cac79254724e5c5b5cf", + "git_blob_oid": "518e7143a5ee90a2bb6d8e73b972c7b5d4d1cca8", + "lfs": { + "oid": "1eda79fb1fd8773714467694b24cbbf71f049549b0016cac79254724e5c5b5cf", + "size": 433064, + "pointerSize": 131 + } + } + }, + "level2/attribute7": { + "task.txt": { + "path": "data/blenderbench/level2/attribute7/task.txt", + "size": 157, + "sha256": "f389ae85ca7ee5cb03d3ab75aa37722c9c5c00475e378cdc6d40a92b59fcc328", + "git_blob_oid": "9a3d8216657a654dc6be9b8404e878c259c5ed02", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute7/start.py", + "size": 1346, + "sha256": "abd953c4d36252658613f4c385787dbe6baf4beedc346789ea0a6d210c49775b", + "git_blob_oid": "3031bcfffd0d61e2efe8d5aa43ad7b7ed79246c1", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute7/blender_file.blend", + "size": 768637773, + "sha256": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "git_blob_oid": "05fcbfc615f12ce6600305c52710ec2d70080a5b", + "lfs": { + "oid": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "size": 768637773, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute7/renders/goal/render1.png", + "size": 496461, + "sha256": "4e0dcf670648bba309e19853047bb294bbf588ffb8928ed9f9a45bef729f9f32", + "git_blob_oid": "cfed756b5128bc758a055def9ebef92b21b0479b", + "lfs": { + "oid": "4e0dcf670648bba309e19853047bb294bbf588ffb8928ed9f9a45bef729f9f32", + "size": 496461, + "pointerSize": 131 + } + } + }, + "level2/attribute8": { + "task.txt": { + "path": "data/blenderbench/level2/attribute8/task.txt", + "size": 157, + "sha256": "f389ae85ca7ee5cb03d3ab75aa37722c9c5c00475e378cdc6d40a92b59fcc328", + "git_blob_oid": "9a3d8216657a654dc6be9b8404e878c259c5ed02", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute8/start.py", + "size": 1346, + "sha256": "06cc98b4dae682711a3b06f82ca6cd5a967c30915722b2d3ca57a445f6ae83db", + "git_blob_oid": "b37dd16de190575b85f359fa99e72ce5593411ed", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute8/blender_file.blend", + "size": 768637773, + "sha256": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "git_blob_oid": "05fcbfc615f12ce6600305c52710ec2d70080a5b", + "lfs": { + "oid": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "size": 768637773, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute8/renders/goal/render1.png", + "size": 496462, + "sha256": "2177e15ad99cb6559092b37de04aa778b13445c89598736ab67ab4f54cfa4ec3", + "git_blob_oid": "3fb1988fe9e2dbd5e711a2b9c253726fee099256", + "lfs": { + "oid": "2177e15ad99cb6559092b37de04aa778b13445c89598736ab67ab4f54cfa4ec3", + "size": 496462, + "pointerSize": 131 + } + } + }, + "level2/attribute9": { + "task.txt": { + "path": "data/blenderbench/level2/attribute9/task.txt", + "size": 157, + "sha256": "f389ae85ca7ee5cb03d3ab75aa37722c9c5c00475e378cdc6d40a92b59fcc328", + "git_blob_oid": "9a3d8216657a654dc6be9b8404e878c259c5ed02", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute9/start.py", + "size": 1346, + "sha256": "956f2d67b54cda24edfeba5d5bfc766002efe018a4a77c0d505d4cf391410bf1", + "git_blob_oid": "133230ae8b18e1fb4c6be19880194deb6fadb6c2", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute9/blender_file.blend", + "size": 768637773, + "sha256": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "git_blob_oid": "05fcbfc615f12ce6600305c52710ec2d70080a5b", + "lfs": { + "oid": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "size": 768637773, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute9/renders/goal/render1.png", + "size": 496462, + "sha256": "057562178eba0d2cf993227378f6f93c2689eda870a060cc836edce85f892eb0", + "git_blob_oid": "3448027c64be93eee308939ef1f9793ac04bd5bb", + "lfs": { + "oid": "057562178eba0d2cf993227378f6f93c2689eda870a060cc836edce85f892eb0", + "size": 496462, + "pointerSize": 131 + } + } + }, + "level3/attribute1": { + "task.txt": { + "path": "data/blenderbench/level3/attribute1/task.txt", + "size": 206, + "sha256": "a682f48631e343a9c378a222e0651a7f8dd6f82b39ca8d0ce26305f42bfa422d", + "git_blob_oid": "df0d929a1e1d9cdcc321642c98c5d60a6bc90bb9", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute1/start.py", + "size": 1434, + "sha256": "e9d1a0d7c912f9806f5879117403938995c1155d13c33af9a0c0c25068909769", + "git_blob_oid": "7d5b416c2be2ddc0f55b8b96482c0536cec22496", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute1/blender_file.blend", + "size": 1103281707, + "sha256": "58bd9031ded0eb4c092951573e7e16f2704b29b1a2eb239331c88b5ba74d3aea", + "git_blob_oid": "598a2385913f422503103677c28e70a16ba5134b", + "lfs": { + "oid": "58bd9031ded0eb4c092951573e7e16f2704b29b1a2eb239331c88b5ba74d3aea", + "size": 1103281707, + "pointerSize": 135 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute1/renders/goal/render1.png", + "size": 424605, + "sha256": "203bc0d5b3cd6bff06e19baa56c56370b5a8da9155b5b0ee3e8598a0886318f0", + "git_blob_oid": "d9f070b573c1cfe18b9d06fd2777c97155cdb154", + "lfs": { + "oid": "203bc0d5b3cd6bff06e19baa56c56370b5a8da9155b5b0ee3e8598a0886318f0", + "size": 424605, + "pointerSize": 131 + } + } + }, + "level3/attribute2": { + "task.txt": { + "path": "data/blenderbench/level3/attribute2/task.txt", + "size": 206, + "sha256": "a682f48631e343a9c378a222e0651a7f8dd6f82b39ca8d0ce26305f42bfa422d", + "git_blob_oid": "df0d929a1e1d9cdcc321642c98c5d60a6bc90bb9", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute2/start.py", + "size": 1374, + "sha256": "0c115cf6fdbf43f677ebb5244e6afc9d75dd9635a9f4c6de06abe28a3f68f191", + "git_blob_oid": "df1d330fdd5658db8875f63713eeb753bfcce23c", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute2/blender_file.blend", + "size": 1104010951, + "sha256": "8dd5b6c18ae4d7f77a94370a9c0203f6ea06911e590c43d4b563c2f0195854be", + "git_blob_oid": "3c8b7beeb7aa513fa1c66d186039151f9d38077f", + "lfs": { + "oid": "8dd5b6c18ae4d7f77a94370a9c0203f6ea06911e590c43d4b563c2f0195854be", + "size": 1104010951, + "pointerSize": 135 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute2/renders/goal/render1.png", + "size": 424611, + "sha256": "b5b1526c9fde7aecd35f842598627b63172338284d69aaa36d6a40649200ca30", + "git_blob_oid": "fec9d6a34397b5ab37a6e14e207f7e232da8a860", + "lfs": { + "oid": "b5b1526c9fde7aecd35f842598627b63172338284d69aaa36d6a40649200ca30", + "size": 424611, + "pointerSize": 131 + } + } + }, + "level3/attribute3": { + "task.txt": { + "path": "data/blenderbench/level3/attribute3/task.txt", + "size": 206, + "sha256": "a682f48631e343a9c378a222e0651a7f8dd6f82b39ca8d0ce26305f42bfa422d", + "git_blob_oid": "df0d929a1e1d9cdcc321642c98c5d60a6bc90bb9", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute3/start.py", + "size": 1372, + "sha256": "de2d9d5cba1b04d58b1d698726c29968d8fbfed225c2b2b24f32f5f765a532ba", + "git_blob_oid": "c4d5bbd48128bb87ef8f8186a1cd68439edbfdb2", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute3/blender_file.blend", + "size": 1104010951, + "sha256": "7c1a17de951be40e203fc96dde62afc0056392394897e3478de322cf84ef4272", + "git_blob_oid": "c0450d126230e49f06bc0afc80710ff278778e73", + "lfs": { + "oid": "7c1a17de951be40e203fc96dde62afc0056392394897e3478de322cf84ef4272", + "size": 1104010951, + "pointerSize": 135 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute3/renders/goal/render1.png", + "size": 424612, + "sha256": "741983290631f850e8e7a011679328f9e6da5545a671c23d73c17beb88cae346", + "git_blob_oid": "70f334823a5eb76be195cadf3158929b755f9267", + "lfs": { + "oid": "741983290631f850e8e7a011679328f9e6da5545a671c23d73c17beb88cae346", + "size": 424612, + "pointerSize": 131 + } + } + }, + "level3/attribute4": { + "task.txt": { + "path": "data/blenderbench/level3/attribute4/task.txt", + "size": 217, + "sha256": "f614d97a13bbae1dedef105ca932a0acfdaf7162e9d19cd4ec98ce3c26e3c647", + "git_blob_oid": "0f4f7411e78fbfdf51b265988ee7ba26933de194", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute4/start.py", + "size": 897, + "sha256": "bfce3b4e6919cdfa62ddb5893d42d64493d2357505ae83ece52d87a424ff0d10", + "git_blob_oid": "c8cf8717af01beef3fa61cc2fdb182a9132bd1ce", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute4/blender_file.blend", + "size": 1100061468, + "sha256": "250865ed23f0d6dcb03c8ee9302ae5e55473fb60d35c6b487533fd3f8bd3e29d", + "git_blob_oid": "78c59f7e8ca4edbbb336542d964fff9b238fa96d", + "lfs": { + "oid": "250865ed23f0d6dcb03c8ee9302ae5e55473fb60d35c6b487533fd3f8bd3e29d", + "size": 1100061468, + "pointerSize": 135 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute4/renders/goal/render1.png", + "size": 424822, + "sha256": "2dfaea1568f9cfb0400600c874e214dc9b15578708571d1d9897605330ffc928", + "git_blob_oid": "1e0bf1cecf36801c6aef9cdbd5f335f6a577897a", + "lfs": { + "oid": "2dfaea1568f9cfb0400600c874e214dc9b15578708571d1d9897605330ffc928", + "size": 424822, + "pointerSize": 131 + } + } + }, + "level3/attribute5": { + "task.txt": { + "path": "data/blenderbench/level3/attribute5/task.txt", + "size": 217, + "sha256": "f614d97a13bbae1dedef105ca932a0acfdaf7162e9d19cd4ec98ce3c26e3c647", + "git_blob_oid": "0f4f7411e78fbfdf51b265988ee7ba26933de194", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute5/start.py", + "size": 838, + "sha256": "8fb7657a111ad845d72eaa3c1599f46f0ee1bb4c7b9b8e97b2858d7c2a71c215", + "git_blob_oid": "59d3e7ce0a94c4c34a243de6f0266e80084ddb04", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute5/blender_file.blend", + "size": 947346540, + "sha256": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "git_blob_oid": "651e1029bc10ca62d380be72c08e122720b5a7c2", + "lfs": { + "oid": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "size": 947346540, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute5/renders/goal/render1.png", + "size": 432648, + "sha256": "08f94e6af9f1bd74f52a7302f7673da3ebfb318437d4a986780158967944635b", + "git_blob_oid": "e9aef85c8c070db78c4d9f85a0ae46eeef6b3b92", + "lfs": { + "oid": "08f94e6af9f1bd74f52a7302f7673da3ebfb318437d4a986780158967944635b", + "size": 432648, + "pointerSize": 131 + } + } + }, + "level3/attribute6": { + "task.txt": { + "path": "data/blenderbench/level3/attribute6/task.txt", + "size": 217, + "sha256": "f614d97a13bbae1dedef105ca932a0acfdaf7162e9d19cd4ec98ce3c26e3c647", + "git_blob_oid": "0f4f7411e78fbfdf51b265988ee7ba26933de194", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute6/start.py", + "size": 835, + "sha256": "6e1bed8235bb346af995c6f281b05f2d482c3f36dd0432a9df9ce62674dd7dbd", + "git_blob_oid": "c6ebc619f78ff7952522a61c9c1d7eb3dad40bdb", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute6/blender_file.blend", + "size": 947346540, + "sha256": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "git_blob_oid": "651e1029bc10ca62d380be72c08e122720b5a7c2", + "lfs": { + "oid": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "size": 947346540, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute6/renders/goal/render1.png", + "size": 432652, + "sha256": "86b764ac22fe162549f3d5a2995e0e8160e836cf6abac763c120f9de283a8089", + "git_blob_oid": "10d2d97575e32eeb841547b7270f74cf7fabd6ff", + "lfs": { + "oid": "86b764ac22fe162549f3d5a2995e0e8160e836cf6abac763c120f9de283a8089", + "size": 432652, + "pointerSize": 131 + } + } + }, + "level3/attribute7": { + "task.txt": { + "path": "data/blenderbench/level3/attribute7/task.txt", + "size": 244, + "sha256": "9f6dd788a109d7888acd415b2dec5b86441aaf2c9c31bd898657b3acd769afb1", + "git_blob_oid": "20dc920fca0ea87f55f7a39e4be480283ca1b6e3", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute7/start.py", + "size": 1539, + "sha256": "88c03b7b328cb7251425acdde7959aa4cb9bea9add652c68faa5d442460fa4f9", + "git_blob_oid": "aeda4ddbd4422cd29ba466dd3577f9f05a9ea7e4", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute7/blender_file.blend", + "size": 766325689, + "sha256": "c4efab1f4cfc624e047491836f5516a0d2bd0e4f803926bca3e94e23a0ada149", + "git_blob_oid": "ee8f5fb6f9a868345bb02a9cac6deee6e68e3f45", + "lfs": { + "oid": "c4efab1f4cfc624e047491836f5516a0d2bd0e4f803926bca3e94e23a0ada149", + "size": 766325689, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute7/renders/goal/render1.png", + "size": 496540, + "sha256": "0ada6e89b53c28b9036418def514b728dd0eeb664eb3f7a1ca9496a5e8ef93f1", + "git_blob_oid": "066f39046e01e72f80fb53c8606b7deaf2811ab5", + "lfs": { + "oid": "0ada6e89b53c28b9036418def514b728dd0eeb664eb3f7a1ca9496a5e8ef93f1", + "size": 496540, + "pointerSize": 131 + } + } + }, + "level3/attribute8": { + "task.txt": { + "path": "data/blenderbench/level3/attribute8/task.txt", + "size": 244, + "sha256": "9f6dd788a109d7888acd415b2dec5b86441aaf2c9c31bd898657b3acd769afb1", + "git_blob_oid": "20dc920fca0ea87f55f7a39e4be480283ca1b6e3", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute8/start.py", + "size": 1539, + "sha256": "6fdf229c8df32bdfd1700e90cd1c135e1eba67038effd3c766753a386e523c2a", + "git_blob_oid": "37c534a03c87c0689ca42cedbc8cb572edace78b", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute8/blender_file.blend", + "size": 768637773, + "sha256": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "git_blob_oid": "05fcbfc615f12ce6600305c52710ec2d70080a5b", + "lfs": { + "oid": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "size": 768637773, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute8/renders/goal/render1.png", + "size": 496540, + "sha256": "33e602e64c9b3f6825472277653bd168de39645a61cde8d8112efe0ae0cde1ad", + "git_blob_oid": "8d6dff62f9fc08e27eafe9ec5a0cd6b90c4de73f", + "lfs": { + "oid": "33e602e64c9b3f6825472277653bd168de39645a61cde8d8112efe0ae0cde1ad", + "size": 496540, + "pointerSize": 131 + } + } + }, + "level3/attribute9": { + "task.txt": { + "path": "data/blenderbench/level3/attribute9/task.txt", + "size": 244, + "sha256": "9f6dd788a109d7888acd415b2dec5b86441aaf2c9c31bd898657b3acd769afb1", + "git_blob_oid": "20dc920fca0ea87f55f7a39e4be480283ca1b6e3", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute9/start.py", + "size": 1539, + "sha256": "bd47ca26dfd4b224a33766dcd40101cbf3f8b168773541d7444d6ea1c4039a6b", + "git_blob_oid": "94fe4672c7fe453266a24537cfd7e0df0556ce1f", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute9/blender_file.blend", + "size": 768637773, + "sha256": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "git_blob_oid": "05fcbfc615f12ce6600305c52710ec2d70080a5b", + "lfs": { + "oid": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "size": 768637773, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute9/renders/goal/render1.png", + "size": 496540, + "sha256": "27653bfcb7c78fe51239c3905060ce112e65c8875966b2d81f3d0ae33fa322ee", + "git_blob_oid": "5baa5bd208c5f160405d1fa96895eddd9ea9887d", + "lfs": { + "oid": "27653bfcb7c78fe51239c3905060ce112e65c8875966b2d81f3d0ae33fa322ee", + "size": 496540, + "pointerSize": 131 + } + } + } + }, + "source_url": "https://huggingface.co/datasets/DietCoke4671/BlenderBench", + "attribution": "BlenderBench dataset by DietCoke4671 and contributors", + "license": "CC BY 4.0" +} diff --git a/benchmarks/blenderbench_direct/clip-snapshot-manifest.json b/benchmarks/blenderbench_direct/clip-snapshot-manifest.json new file mode 100644 index 0000000..e70cc84 --- /dev/null +++ b/benchmarks/blenderbench_direct/clip-snapshot-manifest.json @@ -0,0 +1,57 @@ +{ + "files": [ + { + "path": "config.json", + "sha256": "b575ef3c36f2a057fa19e221650105052d61cc9c1a972ec15019c6261ec98770", + "size": 4186, + "symlink": true + }, + { + "path": "merges.txt", + "sha256": "f526393189112391ce6f9795d4695f704121ce452c3aad1f5335cc41337eba85", + "size": 524657, + "symlink": true + }, + { + "path": "preprocessor_config.json", + "sha256": "910e70b3956ac9879ebc90b22fb3bc8a75b6a0677814500101a4c072bd7857bd", + "size": 316, + "symlink": true + }, + { + "path": "pytorch_model.bin", + "sha256": "a63082132ba4f97a80bea76823f544493bffa8082296d62d71581a4feff1576f", + "size": 605247071, + "symlink": true + }, + { + "path": "special_tokens_map.json", + "sha256": "f8c0d6c39aee3f8431078ef6646567b0aba7f2246e9c54b8b99d55c22b707cbf", + "size": 389, + "symlink": true + }, + { + "path": "tokenizer.json", + "sha256": "b556ac8c99757ffb677208af34bc8c6721572114111a6e0aaf5fa69ff0b8d842", + "size": 2224041, + "symlink": true + }, + { + "path": "tokenizer_config.json", + "sha256": "34b7336e4bee12e0a9730eaf5189f582ef3c3eea5027f65730e5717256755aad", + "size": 592, + "symlink": true + }, + { + "path": "vocab.json", + "sha256": "5047b556ce86ccaf6aa22b3ffccfc52d391ea4accdab9c2f2407da5b742d4363", + "size": 862328, + "symlink": true + } + ], + "python_version": "3.13.15 (main, Sep 1 2026, 14:18:08) [Clang 22.1.3 ]", + "repository": "openai/clip-vit-base-patch32", + "revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", + "schema_version": 1, + "snapshot_exists": true +} diff --git a/benchmarks/blenderbench_direct/dataset.py b/benchmarks/blenderbench_direct/dataset.py new file mode 100644 index 0000000..e937311 --- /dev/null +++ b/benchmarks/blenderbench_direct/dataset.py @@ -0,0 +1,209 @@ +"""Hash-pinned, input-only 27-task BlenderBench dataset downloader. + +Only task text, start script, initial scene, and target image are staged. Upstream +Python is data: it is hashed and never imported, compiled, or executed here. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import shutil +import urllib.parse +import urllib.request +import uuid + +HERE = Path(__file__).resolve().parent +DATASET_REPOSITORY = "DietCoke4671/BlenderBench" +DATASET_SOURCE_URL = "https://huggingface.co/datasets/DietCoke4671/BlenderBench" +DATASET_ATTRIBUTION = "BlenderBench dataset by DietCoke4671 and contributors" +DATASET_LICENSE = "CC BY 4.0" +DATASET_REVISION = "203e4d325e9438ca55b29bdfc4f6a90842d74e68" +_ASSET_NAMES = {"task.txt", "start.py", "scene.blend", "target.png"} + + +def _read(name: str) -> dict: + return json.loads((HERE / name).read_text(encoding="utf-8")) + + +def load_manifest() -> dict: + assets = _read("assets-manifest.json") + initialization = _read("initialization-manifest.json") + if assets["revision"] != DATASET_REVISION or initialization["revision"] != DATASET_REVISION: + raise ValueError("dataset manifest revision drift") + if set(assets["tasks"]) != set(initialization["tasks"]): + raise ValueError("asset and initialization task sets differ") + tasks = {} + for task in sorted(assets["tasks"]): + tasks[task] = {**initialization["tasks"][task], "assets": assets["tasks"][task]} + if len(tasks) != 27: + raise ValueError("pinned manifest must contain exactly 27 tasks") + return {"schema_version": 1, "repository": assets["repository"], + "source_url": DATASET_SOURCE_URL, "attribution": DATASET_ATTRIBUTION, + "license": DATASET_LICENSE, "revision": DATASET_REVISION, "tasks": tasks} + + +def task_ids() -> list[str]: + return sorted(load_manifest()["tasks"]) + + +def select_tasks(selected: list[str] | None) -> list[str]: + canonical = task_ids() + if selected is None: + return canonical + if not selected or len(selected) != len(set(selected)) or set(selected) - set(canonical): + raise ValueError("--tasks must be a nonempty, unique subset of the 27 canonical task IDs") + return list(selected) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _safe_root(root: Path) -> Path: + root = root.expanduser().absolute() + for parent in (root, *root.parents): + if parent.is_symlink(): + raise ValueError(f"symlink forbidden in dataset path: {parent}") + return root + + +def _safe_task_root(root: Path, task: str) -> Path: + current = _safe_root(root) + for component in Path(task).parts: + current = current / component + if current.is_symlink(): + raise ValueError(f"symlink forbidden in dataset path: {current}") + return current + + +def _verify(path: Path, record: dict) -> dict: + if path.is_symlink() or not path.is_file(): + raise ValueError(f"asset must be a regular non-symlink file: {path}") + if path.stat().st_size != record["size"] or sha256(path) != record["sha256"]: + raise ValueError(f"asset size/hash mismatch: {path}") + return {"size": record["size"], "sha256": record["sha256"]} + + +def verify(task: str, root: Path) -> dict: + manifest = load_manifest() + verify_dataset_surface(root, manifest) + record = manifest["tasks"][task] + directory = _safe_task_root(Path(root), task) + allowed = set(record["assets"]) | {"checkpoint.json"} + try: + entries = {entry.name for entry in directory.iterdir()} + except OSError as exc: + raise ValueError(f"cannot enumerate dataset task root: {directory}") from exc + unexpected = sorted(entries - allowed) + if unexpected: + raise ValueError(f"unexpected dataset entries outside the input-only manifest: {unexpected}") + files = {name: _verify(directory / name, asset) for name, asset in record["assets"].items()} + return {"task": task, "dataset_revision": DATASET_REVISION, "files": files} + + +def verify_dataset_surface(root: Path, manifest: dict | None = None) -> None: + """Reject every path outside the declared input-only dataset surface.""" + root = Path(root).expanduser().absolute() + manifest = load_manifest() if manifest is None else manifest + allowed_files: set[Path] = set() + allowed_dirs: set[Path] = set() + for task, record in manifest["tasks"].items(): + task_path = Path(task) + allowed_dirs.update(task_path.parents) + allowed_dirs.add(task_path) + allowed_files.update(task_path / name for name in record["assets"]) + allowed_files.add(task_path / "checkpoint.json") + allowed_dirs.discard(Path(".")) + if root.is_symlink() or not root.is_dir(): + raise ValueError("dataset root must be a regular directory") + for path in root.rglob("*"): + relative = path.relative_to(root) + if path.is_symlink(): + raise ValueError(f"symlink forbidden in dataset root: {relative}") + if path.is_dir(): + allowed = relative in allowed_dirs + else: + allowed = path.is_file() and relative in allowed_files + if not allowed: + raise ValueError(f"unexpected path in input-only dataset root: {relative}") + + +def _download(url: str, destination: Path, record: dict) -> None: + partial = destination.with_name(destination.name + f".{uuid.uuid4().hex}.partial") + older = sorted(destination.parent.glob(destination.name + ".*.partial"), + key=lambda item: item.stat().st_size, reverse=True) + offset = 0 + try: + with partial.open("xb") as output: + if older: + if older[0].is_symlink() or older[0].stat().st_size > record["size"]: + raise ValueError("invalid retained partial download") + offset = older[0].stat().st_size + with older[0].open("rb") as source: + shutil.copyfileobj(source, output) + request = urllib.request.Request(url, headers={"Range": f"bytes={offset}-"}) if offset else url + if offset < record["size"]: + with urllib.request.urlopen(request) as response: + if offset: + expected = f"bytes {offset}-{record['size'] - 1}/{record['size']}" + if response.status != 206 or response.headers.get("Content-Range") != expected: + raise ValueError("server did not honor exact resume range") + while block := response.read(8 * 1024 * 1024): + output.write(block) + if output.tell() > record["size"]: + raise ValueError("download exceeds pinned size") + output.flush() + os.fsync(output.fileno()) + _verify(partial, record) + partial.replace(destination) + for retained in older: + retained.unlink() + except BaseException: + # Partial bytes are retained for explicit later resume and diagnosis. + raise + + +def stage(task: str, root: Path) -> Path: + manifest = load_manifest() + record = manifest["tasks"][task] + destination = _safe_task_root(Path(root), task) + destination.mkdir(parents=True, exist_ok=True) + state_path = destination / "checkpoint.json" + state = {"task": task, "dataset_revision": DATASET_REVISION, "status": "staging", "files": {}} + + def save() -> None: + temporary = state_path.with_name(state_path.name + f".{uuid.uuid4().hex}.tmp") + temporary.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(state_path) + + save() + try: + for name, asset in record["assets"].items(): + if name not in _ASSET_NAMES: + raise ValueError(f"unapproved asset name: {name}") + path = destination / name + if path.exists(): + details = _verify(path, asset) + details["cached"] = True + else: + url = (f"https://huggingface.co/datasets/{DATASET_REPOSITORY}/resolve/" + f"{DATASET_REVISION}/{urllib.parse.quote(asset['path'], safe='/')}") + _download(url, path, asset) + details = {**_verify(path, asset), "cached": False, "url": url} + state["files"][name] = details + save() + state["status"] = "verified" + state["verification"] = verify(task, root) + save() + except BaseException as exc: + state["status"] = "failed" + state["error"] = f"{type(exc).__name__}: {exc}" + save() + raise + return destination diff --git a/benchmarks/blenderbench_direct/initialization-manifest.json b/benchmarks/blenderbench_direct/initialization-manifest.json new file mode 100644 index 0000000..a99c9c0 --- /dev/null +++ b/benchmarks/blenderbench_direct/initialization-manifest.json @@ -0,0 +1,1944 @@ +{ + "schema_version": 1, + "revision": "203e4d325e9438ca55b29bdfc4f6a90842d74e68", + "tasks": { + "level1/camera1": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "8680900dd5a85c2467f86a7ec0c47e3868569e387948eb7e45aab65794981c06", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 1.0765, + -1.2848, + 1.4343 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 1.5701, + 0.0, + 0.6753 + ] + } + ] + }, + "level1/camera2": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "41cae2396027226fb32fd06a6e0a6e913bd895811bdee383416ef1bd1598d186", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -3.4813, + 4.5001, + 0.5115 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 1.6678, + 0.0, + 3.8693 + ] + } + ] + }, + "level1/camera3": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "f5acd7609ff23e01e3d435173d46ea45907fc2b3771dea91f5da0007e51500f2", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 6.7303, + -4.87, + 3.7385 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 1.2811, + -0.0, + 0.9355 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "data.lens", + "value": 50 + } + ] + }, + "level1/camera4": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "a994ae0f9c936a8a4a469b5245bc5cc438e7a112ec55288b7c2473998872d657", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 1.14, + 0.85, + 0.53 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 2.4435, + 1.117, + 2.6354 + ] + } + ] + }, + "level1/camera5": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "2d09f0ea2b51ace93663882d86aa2fa4201f394710879c12c6774173793a8971", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.225, + 0.71, + -2.83 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.4363, + 3.3161, + 0.0349 + ] + } + ] + }, + "level1/camera6": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "1539bdf69d7df4cb2da3770a59f2a33ed28d3d20aec3c3e2677351b297850f9c", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 2.4689, + -2.6258, + 1.3083 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 1.406, + 0.0, + 0.7626 + ] + } + ] + }, + "level1/camera7": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "5471be8faa7f4bd3ba2963a6716298a007d8a004a11df38cb87b9f026d9f4bbc", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 1.3353, + 1.474, + 1.5357 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.7679, + 0.2094, + 0.0 + ] + } + ] + }, + "level1/camera8": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "1dd7b06ac2d5949e0453f0e0c8ae857a417e986d9cc3b6fad748d26abf1b8151", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 7.3589, + -6.9258, + 4.9583 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 1.2664, + 0.0, + 0.78 + ] + } + ] + }, + "level1/camera9": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "6c104adc69f6e714e7b51d3e834b8bd7e6836813899ea84c972c6d2208ac40b6", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 0.1104, + -2.6639, + 9.8843 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 0.3142, + 0, + 0 + ] + } + ] + }, + "level2/attribute1": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "57448a36ff834acc820d071fc99b26285d8968d4fece18300a5c512e18ac2f18", + "task_description": "First adjust the room brightness, then adjust the size of the character's belly so that it looks like the target image.", + "initialization": [ + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 1.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 1.2 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellySag", + "property": "value", + "value": 5 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellyShrink", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ShoulderWideness", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BackTaper", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestEnlarge", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestArea", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "MoreChin", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "Abs", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "eyelids", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "nose", + "property": "value", + "value": 0 + } + ] + }, + "level2/attribute2": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "83702d094c0fa4d6e4a29711dfa024f587339353b1f9ddd6aff5e17b3ab426a8", + "task_description": "First adjust the room brightness, then adjust the size of the character's belly so that it looks like the target image.", + "initialization": [ + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 0.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 400 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 0.8, + 0.9, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.5, + 0.7, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0, + 0 + ] + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellySag", + "property": "value", + "value": 5 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellyShrink", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ShoulderWideness", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BackTaper", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestEnlarge", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestArea", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "MoreChin", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "Abs", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "eyelids", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "nose", + "property": "value", + "value": 0 + } + ] + }, + "level2/attribute3": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "0bd075a70511e2952ff888d6a7d72499311a82764c7ab8c7bb7ae659ba210706", + "task_description": "First adjust the room brightness, then adjust the size of the character's belly so that it looks like the target image.", + "initialization": [ + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 57 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 0.85, + 0 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.2, + 0.6, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0.95, + 0.6 + ] + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellySag", + "property": "value", + "value": 5 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellyShrink", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ShoulderWideness", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BackTaper", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestEnlarge", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestArea", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "MoreChin", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "Abs", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "eyelids", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "nose", + "property": "value", + "value": 0 + } + ] + }, + "level2/attribute4": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "1991b217fd4e044f7ef0966bbb38e8e417bb1fa967effba61bbfc880f7e36543", + "task_description": "First adjust the brightness of the room, then adjust the position of the basketball so that it looks the same as the target image.", + "initialization": [ + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 1.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 1.2 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "basketball", + "property": "location", + "value": [ + 13.4023, + 3.7485, + 0.1584 + ] + } + ] + }, + "level2/attribute5": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "d80af70f035011ae1d6bf7001c9cad008e3cb63a6442bfbd87b23e0d4e09fb0a", + "task_description": "First adjust the brightness of the room, then adjust the position of the basketball so that it looks the same as the target image.", + "initialization": [ + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 0.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 400 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 0.8, + 0.9, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.5, + 0.7, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0, + 0 + ] + }, + { + "kind": "object", + "object": "basketball", + "property": "location", + "value": [ + 14.4023, + 2.7485, + 0.1584 + ] + } + ] + }, + "level2/attribute6": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "aadf4e84cd6b0e03c47f45cc1e1b6f12a436b6273ffd7f81cd3f2d28a4c072bf", + "task_description": "First adjust the brightness of the room, then adjust the position of the basketball so that it looks the same as the target image.", + "initialization": [ + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 57 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 0.85, + 0 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.2, + 0.6, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0.95, + 0.6 + ] + }, + { + "kind": "object", + "object": "basketball", + "property": "location", + "value": [ + 13.9023, + 3.0485, + 0.1584 + ] + } + ] + }, + "level2/attribute7": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "abd953c4d36252658613f4c385787dbe6baf4beedc346789ea0a6d210c49775b", + "task_description": "First move the cabinet to the correct position, then move the plant to the correct position. You need to move the cabinet to observe the plant in the mirror.", + "initialization": [ + { + "kind": "object", + "object": "Cabinet", + "property": "location", + "value": [ + 2.4661, + 14.201, + 0.1277 + ] + }, + { + "kind": "object", + "object": "Plant", + "property": "location", + "value": [ + 3.2168, + 14.327, + 0.57311 + ] + }, + { + "kind": "object", + "object": "Mirror", + "property": "location", + "value": [ + 0.1374, + 13.5573, + 1.396 + ] + }, + { + "kind": "object", + "object": "Sink", + "property": "location", + "value": [ + 1.7123, + 12.7161, + 0.8306 + ] + }, + { + "kind": "object", + "object": "Toilet", + "property": "location", + "value": [ + 1.9395, + 12.5101, + 0.4867 + ] + } + ] + }, + "level2/attribute8": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "06cc98b4dae682711a3b06f82ca6cd5a967c30915722b2d3ca57a445f6ae83db", + "task_description": "First move the cabinet to the correct position, then move the plant to the correct position. You need to move the cabinet to observe the plant in the mirror.", + "initialization": [ + { + "kind": "object", + "object": "Cabinet", + "property": "location", + "value": [ + 2.4661, + 14.201, + 0.1277 + ] + }, + { + "kind": "object", + "object": "Plant", + "property": "location", + "value": [ + 3.2168, + 14.327, + 0.57311 + ] + }, + { + "kind": "object", + "object": "Mirror", + "property": "location", + "value": [ + 0.1374, + 13.5573, + 1.396 + ] + }, + { + "kind": "object", + "object": "Sink", + "property": "location", + "value": [ + 1.7123, + 12.7161, + 0.8306 + ] + }, + { + "kind": "object", + "object": "Toilet", + "property": "location", + "value": [ + 1.9395, + 14.5101, + 0.4867 + ] + } + ] + }, + "level2/attribute9": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "956f2d67b54cda24edfeba5d5bfc766002efe018a4a77c0d505d4cf391410bf1", + "task_description": "First move the cabinet to the correct position, then move the plant to the correct position. You need to move the cabinet to observe the plant in the mirror.", + "initialization": [ + { + "kind": "object", + "object": "Cabinet", + "property": "location", + "value": [ + 2.4661, + 14.201, + 0.1277 + ] + }, + { + "kind": "object", + "object": "Plant", + "property": "location", + "value": [ + 3.2168, + 14.327, + 0.57311 + ] + }, + { + "kind": "object", + "object": "Mirror", + "property": "location", + "value": [ + 0.1374, + 13.5573, + 1.396 + ] + }, + { + "kind": "object", + "object": "Sink", + "property": "location", + "value": [ + 1.7123, + 14.7161, + 0.8306 + ] + }, + { + "kind": "object", + "object": "Toilet", + "property": "location", + "value": [ + 1.9395, + 14.5101, + 0.4867 + ] + } + ] + }, + "level3/attribute1": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "e9d1a0d7c912f9806f5879117403938995c1155d13c33af9a0c0c25068909769", + "task_description": "First adjust the room brightness, then adjust the size of the character's belly so that it looks like the target image. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.1222, + -0.3259, + 4.4354 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0873, + 0.5934, + 0.2094 + ] + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 1.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 1.2 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellySag", + "property": "value", + "value": 5 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellyShrink", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ShoulderWideness", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BackTaper", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestEnlarge", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestArea", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "MoreChin", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "Abs", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "eyelids", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "nose", + "property": "value", + "value": 0 + } + ] + }, + "level3/attribute2": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "0c115cf6fdbf43f677ebb5244e6afc9d75dd9635a9f4c6de06abe28a3f68f191", + "task_description": "First adjust the room brightness, then adjust the size of the character's belly so that it looks like the target image. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.1222, + -0.3259, + 4.4354 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0873, + 0.5934, + 0.2094 + ] + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 0.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 400 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 0.8, + 0.9, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.5, + 0.7, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0, + 0 + ] + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellySag", + "property": "value", + "value": 5 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellyShrink", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ShoulderWideness", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BackTaper", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestEnlarge", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestArea", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "MoreChin", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "Abs", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "eyelids", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "nose", + "property": "value", + "value": 0 + } + ] + }, + "level3/attribute3": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "de2d9d5cba1b04d58b1d698726c29968d8fbfed225c2b2b24f32f5f765a532ba", + "task_description": "First adjust the room brightness, then adjust the size of the character's belly so that it looks like the target image. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.1222, + -0.3259, + 4.4354 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0873, + 0.5934, + 0.2094 + ] + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 57 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 0.85, + 0 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.2, + 0.6, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0.95, + 0.6 + ] + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellySag", + "property": "value", + "value": 5 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellyShrink", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ShoulderWideness", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BackTaper", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestEnlarge", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestArea", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "MoreChin", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "Abs", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "eyelids", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "nose", + "property": "value", + "value": 0 + } + ] + }, + "level3/attribute4": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "bfce3b4e6919cdfa62ddb5893d42d64493d2357505ae83ece52d87a424ff0d10", + "task_description": "First adjust the brightness of the room, then adjust the position of the basketball so that it looks the same as the target image. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.1222, + -0.3259, + 4.4354 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0873, + 0.5934, + 0.2094 + ] + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 1.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 1.2 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "basketball", + "property": "location", + "value": [ + 13.4023, + 3.7485, + 0.1584 + ] + } + ] + }, + "level3/attribute5": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "8fb7657a111ad845d72eaa3c1599f46f0ee1bb4c7b9b8e97b2858d7c2a71c215", + "task_description": "First adjust the brightness of the room, then adjust the position of the basketball so that it looks the same as the target image. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.1222, + -0.3259, + 4.4354 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0873, + 0.5934, + 0.2094 + ] + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 0.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 400 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 0.8, + 0.9, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.5, + 0.7, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0, + 0 + ] + }, + { + "kind": "object", + "object": "basketball", + "property": "location", + "value": [ + 14.4023, + 2.7485, + 0.1584 + ] + } + ] + }, + "level3/attribute6": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "6e1bed8235bb346af995c6f281b05f2d482c3f36dd0432a9df9ce62674dd7dbd", + "task_description": "First adjust the brightness of the room, then adjust the position of the basketball so that it looks the same as the target image. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.1222, + -0.3259, + 4.4354 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0873, + 0.5934, + 0.2094 + ] + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 57 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 0.85, + 0 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.2, + 0.6, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0.95, + 0.6 + ] + }, + { + "kind": "object", + "object": "basketball", + "property": "location", + "value": [ + 13.9023, + 3.0485, + 0.1584 + ] + } + ] + }, + "level3/attribute7": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "88c03b7b328cb7251425acdde7959aa4cb9bea9add652c68faa5d442460fa4f9", + "task_description": "First move the cabinet to the correct position, then move the plant to the correct position. You need to move the cabinet to observe the plant in the mirror. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 0.2645, + 0.2215, + -2.9188 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0698, + 2.426, + 0.0 + ] + }, + { + "kind": "object", + "object": "Cabinet", + "property": "location", + "value": [ + 2.4661, + 14.201, + 0.1277 + ] + }, + { + "kind": "object", + "object": "Plant", + "property": "location", + "value": [ + 3.2168, + 14.327, + 0.57311 + ] + }, + { + "kind": "object", + "object": "Mirror", + "property": "location", + "value": [ + 0.1374, + 13.5573, + 1.396 + ] + }, + { + "kind": "object", + "object": "Sink", + "property": "location", + "value": [ + 1.7123, + 12.7161, + 0.8306 + ] + }, + { + "kind": "object", + "object": "Toilet", + "property": "location", + "value": [ + 1.9395, + 12.5101, + 0.4867 + ] + } + ] + }, + "level3/attribute8": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "6fdf229c8df32bdfd1700e90cd1c135e1eba67038effd3c766753a386e523c2a", + "task_description": "First move the cabinet to the correct position, then move the plant to the correct position. You need to move the cabinet to observe the plant in the mirror. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 0.2645, + 0.2215, + -2.9188 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0698, + 2.426, + 0.0 + ] + }, + { + "kind": "object", + "object": "Cabinet", + "property": "location", + "value": [ + 2.4661, + 14.201, + 0.1277 + ] + }, + { + "kind": "object", + "object": "Plant", + "property": "location", + "value": [ + 3.2168, + 14.327, + 0.57311 + ] + }, + { + "kind": "object", + "object": "Mirror", + "property": "location", + "value": [ + 0.1374, + 13.5573, + 1.396 + ] + }, + { + "kind": "object", + "object": "Sink", + "property": "location", + "value": [ + 1.7123, + 12.7161, + 0.8306 + ] + }, + { + "kind": "object", + "object": "Toilet", + "property": "location", + "value": [ + 1.9395, + 14.5101, + 0.4867 + ] + } + ] + }, + "level3/attribute9": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "bd47ca26dfd4b224a33766dcd40101cbf3f8b168773541d7444d6ea1c4039a6b", + "task_description": "First move the cabinet to the correct position, then move the plant to the correct position. You need to move the cabinet to observe the plant in the mirror. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 0.2645, + 0.2215, + -2.9188 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0698, + 2.426, + 0.0 + ] + }, + { + "kind": "object", + "object": "Cabinet", + "property": "location", + "value": [ + 2.4661, + 14.201, + 0.1277 + ] + }, + { + "kind": "object", + "object": "Plant", + "property": "location", + "value": [ + 3.2168, + 14.327, + 0.57311 + ] + }, + { + "kind": "object", + "object": "Mirror", + "property": "location", + "value": [ + 0.1374, + 13.5573, + 1.396 + ] + }, + { + "kind": "object", + "object": "Sink", + "property": "location", + "value": [ + 1.7123, + 14.7161, + 0.8306 + ] + }, + { + "kind": "object", + "object": "Toilet", + "property": "location", + "value": [ + 1.9395, + 14.5101, + 0.4867 + ] + } + ] + } + } +} diff --git a/benchmarks/blenderbench_direct/protocol.py b/benchmarks/blenderbench_direct/protocol.py new file mode 100644 index 0000000..2790f63 --- /dev/null +++ b/benchmarks/blenderbench_direct/protocol.py @@ -0,0 +1,262 @@ +"""Canonical BlenderBench protocol manifests and provenance.""" +from __future__ import annotations + +import hashlib +import json +import os +import platform +from pathlib import Path +import re +import subprocess +from typing import Any + +TOOLS = ( + "execute_blender_code_for_cli", + "get_render_as_image_for_cli", + "get_runtime_python_api_docs_for_cli", + "search_api_docs", +) +PROTOCOL_VERSION = "blenderbench-direct-v1" +EVALUATOR_REPOSITORY = "https://github.com/Fugtemypt123/VIGA-release" +EVALUATOR_REVISION = "69cb8ef0651bf68124815682df4a0f8e8b57c141" +EVALUATOR_SHA256 = "9ee08818aa858dccfff3efbb36cb9a98e8887f14e6168ad1a23731351cdeb3b5" +CLIP_MODEL = "openai/clip-vit-base-patch32" +CLIP_REVISION = "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268" +CODEX_VERSION = "codex-cli 0.154.0" +CODEX_SOURCE_REVISION = "6b9826e3aa83b1a5947db50f4332cb9c65f1b340" +REPOSITORY = Path(__file__).resolve().parents[2] +IMPLEMENTATION_PATHS = ( + "benchmarks/blenderbench_direct/dataset.py", + "benchmarks/blenderbench_direct/protocol.py", + "benchmarks/blenderbench_direct/runner.py", + "benchmarks/blenderbench_direct/ref_based_eval.py", + "benchmarks/blenderbench_direct/score_pair.py", + "benchmarks/blenderbench_direct/scoring.py", + "benchmarks/blenderbench_direct/suite.py", + "mcp/blmcp/tools/execute_blender_code.py", + "mcp/blmcp/tools_helpers/blender_cli.py", + "tools/direct_codex.py", + "tools/direct_runtime.py", +) + + +def canonical_bytes(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=False, allow_nan=False).encode("utf-8") + + +def digest(value: Any) -> str: + return hashlib.sha256(canonical_bytes(value)).hexdigest() + + +def redact(value: Any, key: str = "") -> Any: + normalized = re.sub(r"[^a-z0-9]+", "_", key.lower()).strip("_") + if normalized in {"args", "env", "variables"}: + return "" + sensitive = re.compile( + r"(^|_)(token|secret|password|passwd|pwd|credential|authorization|auth|api_?key|pat|cookie|session)($|_)") + if sensitive.search(normalized): + return "" + if isinstance(value, dict): + return {name: redact(child, str(name)) for name, child in value.items()} + if isinstance(value, list): + return [redact(child, key) for child in value] + if isinstance(value, str) and (re.search(r"(?i)\bbearer\s+\S+", value) + or re.search(r"(?i)https?://[^/@\s:]+:[^@\s]+@", value) + or re.search(r"(?i)[?&](?:token|key|secret|signature|auth)=[^&\s]+", value)): + return "" + return value + + +def read_mcp_config(path: Path) -> dict: + import tomllib + return tomllib.loads(Path(path).read_text(encoding="utf-8")) + + +def resolved_version(binary: Path, *arguments: str) -> str: + return subprocess.check_output([str(binary), *arguments], text=True, + stderr=subprocess.STDOUT).strip() + + +def runtime_provenance() -> dict: + cpu = platform.processor() or platform.machine() + provenance = { + "platform": platform.platform(), + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "cpu": cpu, + "python": platform.python_version(), + "python_implementation": platform.python_implementation(), + "logical_cpu_count": os.cpu_count(), + } + try: + provenance["gpu"] = resolved_version(Path("nvidia-smi"), + "--query-gpu=name,driver_version,memory.total", "--format=csv,noheader") + except (OSError, subprocess.SubprocessError): + provenance["gpu"] = None + return provenance + + +def collect_source_identity(repository: Path = REPOSITORY) -> dict: + """Bind the protocol to source bytes and Git state without host-specific data.""" + repository = Path(repository).resolve() + initialization = repository / "benchmarks/blenderbench_direct/initialization-manifest.json" + assets = repository / "benchmarks/blenderbench_direct/assets-manifest.json" + clip_manifest_path = repository / "benchmarks/blenderbench_direct/clip-snapshot-manifest.json" + scoring_lock = repository / "benchmarks/blenderbench_direct/requirements-scoring.lock" + implementation = {} + for relative in IMPLEMENTATION_PATHS: + path = repository / relative + if path.is_symlink() or not path.is_file(): + raise ValueError(f"protocol implementation is missing or a symlink: {relative}") + implementation[relative] = hashlib.sha256(path.read_bytes()).hexdigest() + try: + revision = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=repository, text=True, + stderr=subprocess.STDOUT).strip() + status = subprocess.check_output( + ["git", "status", "--porcelain=v1", "--untracked-files=all"], + cwd=repository, text=True, stderr=subprocess.STDOUT) + except (OSError, subprocess.SubprocessError) as exc: + raise ValueError("protocol source must be inside a readable Git checkout") from exc + if len(revision) != 40: + raise ValueError("Git revision must be a full 40-hex commit") + assets_bytes = assets.read_bytes() + clip_manifest_bytes = clip_manifest_path.read_bytes() + try: + assets_manifest = json.loads(assets_bytes) + clip_manifest = json.loads(clip_manifest_bytes) + except json.JSONDecodeError as exc: + raise ValueError("assets manifest is not valid JSON") from exc + return { + "assets_manifest": assets_manifest, + "assets_manifest_sha256": hashlib.sha256(assets_bytes).hexdigest(), + "clip_snapshot_manifest": clip_manifest, + "clip_snapshot_manifest_sha256": hashlib.sha256(clip_manifest_bytes).hexdigest(), + "initialization_manifest_sha256": hashlib.sha256(initialization.read_bytes()).hexdigest(), + "scoring_lock_sha256": hashlib.sha256(scoring_lock.read_bytes()).hexdigest(), + "implementation_sha256": implementation, + "git_revision": revision, + "git_dirty": bool(status), + } + + +def build_manifest(config: dict, *, dataset_revision: str, evaluator_revision: str, + model_revision: str | None, versions: dict, + provenance: dict | None = None, source_identity: dict | None = None) -> dict: + public_config = redact(config) + protocol = { + "protocol_version": PROTOCOL_VERSION, + "dataset_revision": dataset_revision, + "evaluator_revision": evaluator_revision, + "evaluator_sha256": EVALUATOR_SHA256, + "clip_model": CLIP_MODEL, + "clip_revision": CLIP_REVISION, + "model": config.get("model"), + "model_revision": model_revision, + "rounds": config.get("rounds"), + "tools": list(TOOLS), + "generation": { + "reasoning": "direct model reasoning", + "sequential_meaningful_saved_rounds": config.get("rounds"), + "shell_scene_operations": False, + "goal_code_or_solution_access": False, + "vlm_judge": False, + "score_feedback_to_generation": False, + "saved_scene_audit_timing": "after generation", + }, + "rendering": {"engine": "Cycles", "device": "GPU", "cpu_fallback": False}, + "versions": versions, + "config": public_config, + "source_identity": (collect_source_identity() if source_identity is None + else source_identity), + } + return { + "schema_version": 1, + "protocol": protocol, + "protocol_sha256": digest(protocol), + "config": public_config, + "versions": versions, + "provenance": provenance if provenance is not None else runtime_provenance(), + } + + +def validate_run_manifest(manifest: dict, *, tasks: list[str], rounds: int) -> dict: + """Validate a generation manifest before recovery or standalone scoring.""" + if not isinstance(manifest, dict) or manifest.get("schema_version") != 1: + raise ValueError("run manifest schema is invalid") + body = manifest.get("protocol") + if not isinstance(body, dict) or manifest.get("protocol_sha256") != digest(body): + raise ValueError("run manifest protocol hash is invalid") + if (body.get("protocol_version") != PROTOCOL_VERSION + or body.get("dataset_revision") != __import__( + "benchmarks.blenderbench_direct.dataset", fromlist=["DATASET_REVISION"]).DATASET_REVISION + or body.get("evaluator_revision") != EVALUATOR_REVISION + or body.get("evaluator_sha256") != EVALUATOR_SHA256 + or body.get("rounds") != rounds + or body.get("config", {}).get("tasks") != tasks): + raise ValueError("run manifest does not match the requested frozen protocol") + expected_generation = { + "reasoning": "direct model reasoning", + "sequential_meaningful_saved_rounds": rounds, + "shell_scene_operations": False, + "goal_code_or_solution_access": False, + "vlm_judge": False, + "score_feedback_to_generation": False, + "saved_scene_audit_timing": "after generation", + } + config = body.get("config", {}) + versions = body.get("versions") + frozen_rules_match = ( + body.get("clip_model") == CLIP_MODEL + and body.get("clip_revision") == CLIP_REVISION + and body.get("tools") == list(TOOLS) + and body.get("generation") == expected_generation + and body.get("rendering") == {"engine": "Cycles", "device": "GPU", "cpu_fallback": False} + and body.get("model") == config.get("model") + and body.get("model_revision") == config.get("model_revision") + and isinstance(versions, dict) + and versions.get("codex") == CODEX_VERSION + and versions.get("scorer_python") == "Python 3.13.15" + and isinstance(versions.get("bpy"), str) + and versions["bpy"].startswith(str(config.get("bpy_version", ""))) + ) + if not frozen_rules_match: + raise ValueError("run manifest does not match the requested frozen protocol") + source = body.get("source_identity") + current = collect_source_identity() + required = {"assets_manifest", "assets_manifest_sha256", "clip_snapshot_manifest", + "clip_snapshot_manifest_sha256", "initialization_manifest_sha256", + "scoring_lock_sha256", "implementation_sha256", "git_revision", "git_dirty"} + if not isinstance(source, dict) or not required.issubset(source): + raise ValueError("run manifest source identity is incomplete") + if source["git_dirty"] is not False: + raise ValueError("publication run manifest requires a clean Git source identity") + for key in ("assets_manifest", "assets_manifest_sha256", "clip_snapshot_manifest", + "clip_snapshot_manifest_sha256", "initialization_manifest_sha256", + "scoring_lock_sha256", "implementation_sha256", "git_revision"): + if source.get(key) != current[key]: + raise ValueError(f"run manifest {key} differs from local scoring protocol") + return manifest + + +def load_run_manifest(run_root: Path, *, tasks: list[str], rounds: int) -> dict: + path = Path(run_root) / "run-manifest.json" + if path.is_symlink() or not path.is_file(): + raise ValueError("standalone scoring requires a regular run-manifest.json") + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError("run-manifest.json is unreadable") from exc + return validate_run_manifest(manifest, tasks=tasks, rounds=rounds) + + +def write_manifest(path: Path, manifest: dict) -> Path: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8") + temporary.replace(path) + return path diff --git a/benchmarks/blenderbench_direct/publication/artifact-manifest.json b/benchmarks/blenderbench_direct/publication/artifact-manifest.json new file mode 100644 index 0000000..3780646 --- /dev/null +++ b/benchmarks/blenderbench_direct/publication/artifact-manifest.json @@ -0,0 +1,3943 @@ +{ + "compilation_video": { + "bytes": 10161884, + "codec": "h264", + "duration_seconds": 134.9, + "height": 464, + "publication_status": "published", + "sample_aspect_ratio": "1:1", + "sha256": "04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17", + "url": "https://huggingface.co/datasets/michaelgold/blenderbench-direct-results/resolve/main/video/blenderbench-complete-compilation.mp4", + "width": 848 + }, + "run_records": { + "camera1-cuda13-02": { + "process.json": { + "bytes": 97, + "sha256": "dc8bfbf1fc0acd387e3a2026a083ebead76a406c6969af75aaaedaec86d670f8" + } + }, + "remaining-direct-03": { + "events.jsonl": { + "bytes": 12900, + "sha256": "a57e0c82694eb6acb01f3bb9f44ae9ac7058e1f9a6eb256bf95f91db184c1a9b" + }, + "process.json": { + "bytes": 850, + "sha256": "4a88a364b25b8b949879516b519fd85c359450aac333e923ce82183c9dbc12c3" + }, + "status.json": { + "bytes": 4244, + "sha256": "0cc076a1213c40de4bb0894e266ca79c779d9cc14151a4b35692f11e5b345942" + } + }, + "unfinished-live-edit-00ba68b-01": { + "dispatcher.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "events.jsonl": { + "bytes": 4894, + "sha256": "509313c4de6f1d959a03daeac9af1b0e4440c5597c31a5997362fc53165ba568" + }, + "launcher.json": { + "bytes": 1068, + "sha256": "0c221f3d5f44259ad4ac7d34e39b4ecfabea5d5ec704dd873d58bcdbd0d67ed3" + }, + "process.json": { + "bytes": 1217, + "sha256": "0ce3469195af0fc9be38486b9a7e9d4560e2b1038c9f8d5ed42a1a69f298cdc1" + }, + "restart-manifest.json": { + "bytes": 41505, + "sha256": "9311e25520fe8c65495cdb0bdc73f9b38ec4ac97ab97fd63069f41bc521125df" + }, + "status.json": { + "bytes": 1521, + "sha256": "5a919bb870086aa315e3e3849626baeffe169e987653c4670010afce9bca3a6d" + } + }, + "unrestricted-4a9f5cc-01": { + "dispatcher.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "events.jsonl": { + "bytes": 8470, + "sha256": "06ffca3000afa683551cea4628eff4f2eb8f89ca21fdf934cb6e61d71ac41be6" + }, + "launcher.json": { + "bytes": 1070, + "sha256": "9c4637dd2bb1ff0c5486cf05eff9fa7343031beb09a2f95354ea9747830db867" + }, + "process.json": { + "bytes": 1260, + "sha256": "4d575179fef242cdc8975a0ba8723db56db7c9336bfa72c3ff0d865847b184ec" + }, + "restart-manifest.json": { + "bytes": 53623, + "sha256": "f5057c3b1eec35b9873655025869165a769a8f3acdcce85584b5dd9a1b0e658c" + }, + "status.json": { + "bytes": 2433, + "sha256": "665fb60beceeab413e328f5008f67aa33b35146bc6b56e78a35bb176ef008d4b" + } + } + }, + "schema_version": 1, + "tasks": { + "level1/camera1": { + "evidence": [ + { + "bytes": 7326712, + "kind": "codex.jsonl", + "sha256": "5d7bd4bba7ebc24ecf08cd17f98e3855045b0742a50014f4c7df3038da5a701f" + } + ], + "rounds": [ + { + "blend_bytes": 12361923, + "blend_sha256": "d9beb3448fc8832615dfb919635ef56a0312fb95e852bf31655889527ebc3b6a", + "clip_similarity": 0.9618514776229858, + "nclip": 0.03814852237701416, + "photometric_loss": 0.05728062987327576, + "render": { + "bytes": 266969, + "sha256": "d3cb9813561664b36f44db06b374c06c013e527d13f5a49b8e0b33580f22b93e" + }, + "round": 1 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "814b2814076b89bcf332b3eb04f9052721fca718976a7d3f909f410a5739cef8", + "clip_similarity": 0.9618485569953918, + "nclip": 0.038151443004608154, + "photometric_loss": 0.022054409608244896, + "render": { + "bytes": 263304, + "sha256": "ffa63c8e500a8f6db3dcce94da9bcca07eeeabab4795d51b90f130033213abde" + }, + "round": 2 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "9a2fdbb8d6d0f03f25c64fc79ad4d725adcf87854e749d0c1ed2e7e50bc65584", + "clip_similarity": 0.9832024574279785, + "nclip": 0.016797542572021484, + "photometric_loss": 0.025472132489085197, + "render": { + "bytes": 273660, + "sha256": "2b96552a076cc29c4af3aaa5038f468287362279e23635fc8e9202ea90720111" + }, + "round": 3 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "68bba553d0c2253e854ab3c99c8192924ecc460643f33078460d95ca7049d390", + "clip_similarity": 0.9919335246086121, + "nclip": 0.00806647539138794, + "photometric_loss": 0.011989844031631947, + "render": { + "bytes": 270480, + "sha256": "7d98214714d6116a040636cce4448a69f4132f78cba01478965193f65ef4f574" + }, + "round": 4 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "e5676ad17577a29ac43ff68d61ec072fc70315e9f740dad58dc9dc85f068c707", + "clip_similarity": 0.9911091923713684, + "nclip": 0.008890807628631592, + "photometric_loss": 0.015071357600390911, + "render": { + "bytes": 277688, + "sha256": "47ea8dde2c78614077998a0e05a200baf974a7fc0bd7ed26b89fd91f5e128041" + }, + "round": 5 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "662efa0ec1caa119a053bc1328e9b6fec04942107a0e206622857ea5e7bad2cf", + "clip_similarity": 0.99659264087677, + "nclip": 0.0034073591232299805, + "photometric_loss": 0.00529375160112977, + "render": { + "bytes": 272724, + "sha256": "814eb8959c84da949affa35db6c042981966f3d5486195bc09bfce8ffd685bba" + }, + "round": 6 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "e0117ca89cbf48fe0bf89e22800244e1a7d0adf7d11d1c4bea486ed7daa3dcaf", + "clip_similarity": 0.9975831508636475, + "nclip": 0.002416849136352539, + "photometric_loss": 0.003328760154545307, + "render": { + "bytes": 273059, + "sha256": "299bf354c236162575d35f68b73bf1bb802ef94bbd4bdb391fc20df830f39b3c" + }, + "round": 7 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "c9c465783a06a6224f8c0b3e2690c5982823866bdc922d6133e64ccac29a5c07", + "clip_similarity": 0.9963770508766174, + "nclip": 0.0036229491233825684, + "photometric_loss": 0.004889530595391989, + "render": { + "bytes": 274419, + "sha256": "8e3d6d3cbd57a9b72db566e55c087b0b311ab946ab9e97f9c09c0b081f25d801" + }, + "round": 8 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "2f7c02277025caf472dca101022f4b663e520e8a2138e43e82e7417e7922ce83", + "clip_similarity": 0.9973839521408081, + "nclip": 0.0026160478591918945, + "photometric_loss": 0.0037112270947545767, + "render": { + "bytes": 274317, + "sha256": "12dffa0deaa846e21873a0f760e4114d16bcd208eef72844503470fe6d9dbbba" + }, + "round": 9 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "0c08683a1a8ba385b6a7dfcb59429cf2385dbffdd9a5af1073a0f2a1d7a6510e", + "clip_similarity": 0.9968132972717285, + "nclip": 0.0031867027282714844, + "photometric_loss": 0.0055841426365077496, + "render": { + "bytes": 274254, + "sha256": "dee437f3d4ba9a224f5ed1a161e996226d8e26376f55d5335c0bc7ed4d454552" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7843, + "sha256": "bf7181ba8c0cab76c7a4e17d4154ebc54dee9bf1d28c12140b89792d705f2543" + }, + "target": { + "bytes": 280875, + "sha256": "a1f17f06448cd899c5912323f9b9dd07ca6ea30dc0aebba11f28cb85afb29d2c" + } + }, + "level1/camera2": { + "evidence": [ + { + "bytes": 571409, + "kind": "events.jsonl", + "sha256": "126562bb77f9149e61809366231ae28cd4a211b62a185e5644791652351fd329" + }, + { + "bytes": 6694522, + "kind": "run.jsonl", + "sha256": "d5b59e3fbc5bfed853de242d35145b8c4f4dbb64673a5e48e0be2701b196c622" + } + ], + "rounds": [ + { + "blend_bytes": 20761447, + "blend_sha256": "233c85503fe77a330874cb0c5f90ff1c81a3467db56c04c99bb2c881bc8456b1", + "clip_similarity": 0.9183803200721741, + "nclip": 0.08161967992782593, + "photometric_loss": 0.10068005323410034, + "render": { + "bytes": 248901, + "sha256": "5a492d0b1f63de8d5a8849e8039e8b5764312edc1d2b3d3d45d231ae5770cd8e" + }, + "round": 1 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "2e775e2359dd6992f506d62249e465e5fcfc9867582c7a5fa0fbab88e38109c1", + "clip_similarity": 0.9619098901748657, + "nclip": 0.03809010982513428, + "photometric_loss": 0.09336703270673752, + "render": { + "bytes": 236872, + "sha256": "b5b8d5334b404b9b7c42937e28bf589c07180b3d53b858624bf43d4d932a7ce3" + }, + "round": 2 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "2a67d0599e5338d9a3920a35bb67a3762af6d4076b704674a90d61c418d14897", + "clip_similarity": 0.9870508909225464, + "nclip": 0.012949109077453613, + "photometric_loss": 0.04738546535372734, + "render": { + "bytes": 248710, + "sha256": "4d3e9c637018e98f695b50445e5e7784d98cab5f0438ece258ef7076baa28d2e" + }, + "round": 3 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "23bec1ec272c76a8c4fe080e537a83f8b781264ddc23c6ef9b2b8f10f2946fcc", + "clip_similarity": 0.9788923263549805, + "nclip": 0.02110767364501953, + "photometric_loss": 0.04208802059292793, + "render": { + "bytes": 245617, + "sha256": "1c57bbb4037dd996ca84286bdb4c3b8272eab9e2d5f63f05141de0b047a187d6" + }, + "round": 4 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "6e2fc6d97fc65aae2d4d3acb1369d5cb99459d5c4d996e7609c8aa4a8a64a448", + "clip_similarity": 0.9878735542297363, + "nclip": 0.012126445770263672, + "photometric_loss": 0.02797190099954605, + "render": { + "bytes": 244309, + "sha256": "2bf2caead17e5ea78291fc7538d508d130dedce461f6dbfe9dcfb645db4e64c3" + }, + "round": 5 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "760a579c3771d74877f95d008fd6002a9fff03a076222bf4fb38feb745ba93b7", + "clip_similarity": 0.9910882711410522, + "nclip": 0.008911728858947754, + "photometric_loss": 0.018511703237891197, + "render": { + "bytes": 246201, + "sha256": "10cf415f645a87e053e95cb3acd0b47d70970e1c719acb83fee85cf90fbeaa07" + }, + "round": 6 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "d43438a5de61c90c9d925ee2412ac9264331f64189810bda425e6b2321b9a77d", + "clip_similarity": 0.9913892149925232, + "nclip": 0.008610785007476807, + "photometric_loss": 0.01795218326151371, + "render": { + "bytes": 246916, + "sha256": "419f20a91a86f603e1966321c7eefba55cb957c833af728ce646865df757823c" + }, + "round": 7 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "54c37f532740417bc72731db98e5fa5b49099b0abe2c22bee11676a518dc9031", + "clip_similarity": 0.9955885410308838, + "nclip": 0.004411458969116211, + "photometric_loss": 0.0033672682475298643, + "render": { + "bytes": 247144, + "sha256": "a57c3b551000f7cccf675746cb78034235b6d1e77510e44f2f8e5a9e22c70508" + }, + "round": 8 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "8cc4984cfdf2eec05e57d9a39d572226057f5641828a3be8f9e84c5fb7efa827", + "clip_similarity": 0.9925321340560913, + "nclip": 0.007467865943908691, + "photometric_loss": 0.005620954558253288, + "render": { + "bytes": 246638, + "sha256": "2e4b88adb5021caf87301a34b17866f939a5f6f3366caf8510932ea45793b905" + }, + "round": 9 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "42ce90379cc6fa4e0ab21510835936b3ecf49ebb9fbd242891fbd5ce7cd6f32e", + "clip_similarity": 0.9927474856376648, + "nclip": 0.007252514362335205, + "photometric_loss": 0.0010773908579722047, + "render": { + "bytes": 245908, + "sha256": "66c52097be62dd3d74c6ad9cb05fa1abb7f8d6146345cdd9627779b12ed6b907" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7416, + "sha256": "f27fbc7dba4c20af2ff3b3c94e8fc5bf0b337595cb9e23fcdd38b75cabe5a976" + }, + "target": { + "bytes": 256033, + "sha256": "ce58bcf86a6866d992e9d115391cf4a411b65395ffd612ccb7d28dc7ba5e5b58" + } + }, + "level1/camera3": { + "evidence": [ + { + "bytes": 364327, + "kind": "events.jsonl", + "sha256": "e55b23f45382aa3e4c9770a077fabbc69298fa0ffa178855e931ee4fe2c840f7" + }, + { + "bytes": 8656277, + "kind": "run.jsonl", + "sha256": "bba451a2de8fa3399f149f6dd2de2569b18feac595db8551a87624a273c8600d" + } + ], + "rounds": [ + { + "blend_bytes": 20490390, + "blend_sha256": "37eafd16726748a941daf10a131817b36c637d03458a73012d8898b94fd8075f", + "clip_similarity": 0.8642843961715698, + "nclip": 0.13571560382843018, + "photometric_loss": 0.060653116554021835, + "render": { + "bytes": 301782, + "sha256": "4fd322c531a97268f17a16fae1bd753b47f33abe61c583a953efd104e5701f52" + }, + "round": 1 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "bd8179e20bad9fd4b9f3387f41a8e97d5aa3a8d6459180dae0ede85407b05862", + "clip_similarity": 0.8662276268005371, + "nclip": 0.1337723731994629, + "photometric_loss": 0.10488191992044449, + "render": { + "bytes": 335255, + "sha256": "5bc345dfd77bc9044aa7c0ef39bf4c1047f6c96182404387e7fa0f99d522f9c1" + }, + "round": 2 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "2ce73c817bf030b1e65deaf9dbd7db52ad6cfe13fa52a60bb7febb82d6aa0593", + "clip_similarity": 0.9695447683334351, + "nclip": 0.03045523166656494, + "photometric_loss": 0.05925656855106354, + "render": { + "bytes": 318052, + "sha256": "cb7054895001715c9fafd5861f149abb2fa63b9c550b54909723fdbe8f2d1dae" + }, + "round": 3 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "75f53a1dc1e1c827e0958124904bc1092398c6c12c51b79e9e5134f137d24677", + "clip_similarity": 0.9561019539833069, + "nclip": 0.043898046016693115, + "photometric_loss": 0.054005783051252365, + "render": { + "bytes": 336085, + "sha256": "9dc252a3911c815b6a89feea0a208b334e70ed6223b76004fd4d231963e7c33d" + }, + "round": 4 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "45240ebea9fea5d92a19911fbbc7a224d77988eedee37c19575226efddab419e", + "clip_similarity": 0.9670233726501465, + "nclip": 0.032976627349853516, + "photometric_loss": 0.03750957176089287, + "render": { + "bytes": 323442, + "sha256": "8197f7c2f9c642e364f451c565ed25afcdbaf0861addc74d3a47682a86943d8e" + }, + "round": 5 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "644fe7869944f8b513646c76b47a74db69ea7e2eb183dd4a60bd89ba0b823548", + "clip_similarity": 0.9768242835998535, + "nclip": 0.023175716400146484, + "photometric_loss": 0.03000887669622898, + "render": { + "bytes": 323758, + "sha256": "7906c731fd7c00d796e20f7e94dad6ea7d2adf2af167c109d4f7223ffcd63cfa" + }, + "round": 6 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "d4c9d0261b3c8a5336e647d7be9e95506d8fcd917d4326c8f312247fd9f63192", + "clip_similarity": 0.9761712551116943, + "nclip": 0.023828744888305664, + "photometric_loss": 0.03189892694354057, + "render": { + "bytes": 319815, + "sha256": "c2655f30a27023df0a0a12f6825b217bc53c832966ff77471193af99f07bf8d1" + }, + "round": 7 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "05905d9f2b955e2b35eaec28d935c308b7e88fc40bd2336f5acc183abc303ee3", + "clip_similarity": 0.9916093945503235, + "nclip": 0.008390605449676514, + "photometric_loss": 0.0070646158419549465, + "render": { + "bytes": 318353, + "sha256": "f5fe718a162ce67da40123ee1e527621c70611381b89f887e3b479cbfb0d6135" + }, + "round": 8 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "4d63215e8d4d45c66d93c04fcd504850c6d9116fb97dcb64baccfc9c4b7a2da2", + "clip_similarity": 0.9977397918701172, + "nclip": 0.0022602081298828125, + "photometric_loss": 0.001466981484554708, + "render": { + "bytes": 320827, + "sha256": "75f08bba423e0d425b706de3f17c642813356ebf7d065c4396f1c944d26451c9" + }, + "round": 9 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "121032ad79be1286e40cf0de7fc299d2a4e160c4e82b54e4c6c82b7af3824240", + "clip_similarity": 0.9840778708457947, + "nclip": 0.015922129154205322, + "photometric_loss": 0.016791703179478645, + "render": { + "bytes": 318141, + "sha256": "cb83d983a80f1468917d40db1b925e7155f4702ba2df8b98d708fa64064351b2" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7656, + "sha256": "f8bbf09c1ef219f1ce5cd6adea15ab29fb63c3e2b41c2ed2b140c118f656a0b9" + }, + "target": { + "bytes": 323359, + "sha256": "3cde2e268b0fb73a25bc96180bbe96818f464bbe71fc8fe9bc1dff8a3d782d18" + } + }, + "level1/camera4": { + "evidence": [ + { + "bytes": 1628407, + "kind": "events.jsonl", + "sha256": "b76e78da802fff805d6e18f9b53e7bb307406e2e34fecb8e6939008ebaf0c595" + }, + { + "bytes": 10628475, + "kind": "run.jsonl", + "sha256": "c01d49aa6fa22afbd0a5ca59d6a89bb04b7f5b226eae72230c27426f340966cd" + } + ], + "rounds": [ + { + "blend_bytes": 932877522, + "blend_sha256": "749fd7e76f7977f37367f675d475ab6468e1100c5b90baacd4b47b34ed850bf7", + "clip_similarity": 0.9709898233413696, + "nclip": 0.02901017665863037, + "photometric_loss": 0.04666049778461456, + "render": { + "bytes": 435845, + "sha256": "b806926ab498c58bc1a7acf4644dfb2ee410b059902c57f6c62d398c630a8bee" + }, + "round": 1 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "7a17a766b3d72e9ee7de55a474f534bd6ff01b49c87a8b7df32a6088376c1ec1", + "clip_similarity": 0.9746537208557129, + "nclip": 0.02534627914428711, + "photometric_loss": 0.04516137018799782, + "render": { + "bytes": 439810, + "sha256": "cada208ea5c020ba1f30c66e8b28b90118a907d3ac5e416de821904a6a167f8e" + }, + "round": 2 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "ca0abf496ea19efbd477f9d14713cf055991ace54cd79555f1dbd2dd51fe53d1", + "clip_similarity": 0.937958836555481, + "nclip": 0.06204116344451904, + "photometric_loss": 0.02753789909183979, + "render": { + "bytes": 435640, + "sha256": "d7bb56d780f7bb38a974e278e5d136c506b4375e0f4ab9d125d0da87ea3451a0" + }, + "round": 3 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "e2a73943cd5af94c3768e12468d57a1a02b8fcf75ffe530c5a2ba299f298bf1a", + "clip_similarity": 0.9316172003746033, + "nclip": 0.06838279962539673, + "photometric_loss": 0.06786150485277176, + "render": { + "bytes": 390435, + "sha256": "dc36306bc2dabd0f38bee9cf6f5b4c3dfaa50a983c25a97697493210dece28ae" + }, + "round": 4 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "ca8e4646c0431f1fa4611316eb82be51f9b89721ae65f60f1ffcaacf3f7b9a2a", + "clip_similarity": 0.9833697080612183, + "nclip": 0.01663029193878174, + "photometric_loss": 0.013681826181709766, + "render": { + "bytes": 437084, + "sha256": "3840170f9ec9783f34029d3cf9bb7274156b07f2f5bdcdce3326820cf6cc4fb0" + }, + "round": 5 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "0d29ebecc3f26a5c7c853803b9c441fa7027fd7f1308069c87becd97846dc6b1", + "clip_similarity": 0.9912315011024475, + "nclip": 0.00876849889755249, + "photometric_loss": 0.007649693172425032, + "render": { + "bytes": 436351, + "sha256": "c9aef7b4746f2cdec26b5a36d1758f9aecbb46cc2a222207db1f8fae4d610aab" + }, + "round": 6 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "394265801ba272228c7f2c27cf5ab36cef1a1a02b5bb47dbbe23d02399113262", + "clip_similarity": 0.9908938407897949, + "nclip": 0.009106159210205078, + "photometric_loss": 0.008667061105370522, + "render": { + "bytes": 436440, + "sha256": "303a87a6593a0ba8b22d74a432c334ece1c47f068bd03a33bf296276a166a550" + }, + "round": 7 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "2925f0d270302a5d54aebe89aefeaebeef511a0937bb92603580e0c5d0e0f220", + "clip_similarity": 0.986689567565918, + "nclip": 0.013310432434082031, + "photometric_loss": 0.008767171762883663, + "render": { + "bytes": 435621, + "sha256": "6a0e99de10e14533dc5a67568619371a4a45bd62fb1cd16b9aec855a70907ccb" + }, + "round": 8 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "59b552ec0b8ab59c766fddf506a76666da68a8d244d1389d82c42e9471c195ab", + "clip_similarity": 0.9909461736679077, + "nclip": 0.009053826332092285, + "photometric_loss": 0.006298090796917677, + "render": { + "bytes": 436021, + "sha256": "8c17604a3ada95989614ae464e7363863c2d8b3dd828e997766c7ec2f7b1bc16" + }, + "round": 9 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "4a54852b6b216c35c8eee7c55e8eddda5a3e42a7cfb1730f20c056e11eb922e6", + "clip_similarity": 0.9899377226829529, + "nclip": 0.01006227731704712, + "photometric_loss": 0.004026280716061592, + "render": { + "bytes": 435960, + "sha256": "e964e2e9323e46c01bee2082c3712dfb9283e100c8f430ac32c5c51184980f52" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7410, + "sha256": "7337a36ef1cabf7dcece460aa8a704597e579c2c65ec1f8e8393ae3c1932541e" + }, + "target": { + "bytes": 434190, + "sha256": "a6cad78bdebb96be3147176d6c4c460df4782c51430fcd268dfddd87f499aa93" + } + }, + "level1/camera5": { + "evidence": [ + { + "bytes": 1630375, + "kind": "events.jsonl", + "sha256": "3fea0213ce7070dc06fca06a24ae52636119854441eceb007d2397b9d4544c8f" + }, + { + "bytes": 10575004, + "kind": "run.jsonl", + "sha256": "214d476156d3c42aa1a9e78b3ac219bdfd32f2d68e40a5a84b80d8a1c25b84ee" + } + ], + "rounds": [ + { + "blend_bytes": 932941898, + "blend_sha256": "2b0c597fa140a618225d3b200ec26b37e44965f6c24ed6ad0774e00a5aa3a8fc", + "clip_similarity": 0.9789350628852844, + "nclip": 0.021064937114715576, + "photometric_loss": 0.028528744354844093, + "render": { + "bytes": 436832, + "sha256": "108cd9f5188e9adb662e7cdaa1344a781c1bbabfd29ed49bcd52797b9cc4c638" + }, + "round": 1 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "ebcdde440e336aa41bcd25ab206defcc6738979d3998ead9490f9c1674f3c899", + "clip_similarity": 0.939391016960144, + "nclip": 0.06060898303985596, + "photometric_loss": 0.04119781777262688, + "render": { + "bytes": 434722, + "sha256": "afed7cba4fa673e8bf4845a4d0ff66e6974237db46d23ffebed51ce124a8e28b" + }, + "round": 2 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "5ef8264c090b163664cb384cd6e43336aaaf52819728ae79987e448723148fa3", + "clip_similarity": 0.9926620721817017, + "nclip": 0.00733792781829834, + "photometric_loss": 0.009993723593652248, + "render": { + "bytes": 439444, + "sha256": "3b4234fda467c48ce1095444456b389e673156c8c8665a3f291a7857c93b99f6" + }, + "round": 3 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "b29982ccf623c233dcbb4607c9778d2f89fdad0f4171e54f3ff0771bb40101fa", + "clip_similarity": 0.9879449605941772, + "nclip": 0.012055039405822754, + "photometric_loss": 0.018624017015099525, + "render": { + "bytes": 436569, + "sha256": "cf0d68190043b3a4556a87dff17739a58417c50aa3c7079bee10dcaf1beed174" + }, + "round": 4 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "e206c5721dd1a982f74dafe3a6ca2eb22838e01478a04fad37b95d510d545224", + "clip_similarity": 0.9861524701118469, + "nclip": 0.013847529888153076, + "photometric_loss": 0.02692614309489727, + "render": { + "bytes": 435229, + "sha256": "4e7e141b9ed42c93a683fa702a140e55f67e3e93ba10aea57becd07c30fddd91" + }, + "round": 5 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "2af838b0a633a89ad3c74003365cc88a91bb227cd4bdc975a9a7be18b9f24de6", + "clip_similarity": 0.9947119355201721, + "nclip": 0.005288064479827881, + "photometric_loss": 0.007714033592492342, + "render": { + "bytes": 439693, + "sha256": "5582441b5c5c3b9bbb73067cec358a6514af493f5c5e3e7c60c5846c1729847c" + }, + "round": 6 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "172cf43f6e63f9f114be8704aa7c23c4b210a89387ae49be9a270c59e2bff95d", + "clip_similarity": 0.9964922666549683, + "nclip": 0.0035077333450317383, + "photometric_loss": 0.0044174473732709885, + "render": { + "bytes": 439085, + "sha256": "0c6337703e66bb0bfdf3e86b8292d025cc800840c77600693fb27c2a102c268d" + }, + "round": 7 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "ed2b2fa49a6d3810f08f440d2899d00bd385aad33dba82e4f1c2a76612b56efc", + "clip_similarity": 0.9942107796669006, + "nclip": 0.005789220333099365, + "photometric_loss": 0.005532005336135626, + "render": { + "bytes": 438157, + "sha256": "21f7b04e48bc4c8c6cd2237eefa8c3a060dcd7f511a5665864d19681e66b805b" + }, + "round": 8 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "05f61a09a01442fe071397d7a340ea102142ae441905290c13688b23c9ce57d4", + "clip_similarity": 0.9913418292999268, + "nclip": 0.008658170700073242, + "photometric_loss": 0.005203785840421915, + "render": { + "bytes": 438674, + "sha256": "bbd10e8100352574a9799fa4e74537cd7ec5f58cd3114a8632c3927a97a0b765" + }, + "round": 9 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "b45fcb9f30b942a2445d7a395a14290d9b703954ebe21d102c005504da2d72d9", + "clip_similarity": 0.9959232807159424, + "nclip": 0.004076719284057617, + "photometric_loss": 0.004915030673146248, + "render": { + "bytes": 439034, + "sha256": "d98978a1590adc51272afa61af6870eddafee04494812bae254134a31de798e3" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7420, + "sha256": "361e1bccec2c68db93e75db1ce65452e7435a4ccd7c218be2a955dad23a55f3b" + }, + "target": { + "bytes": 442674, + "sha256": "0124cf59482f3950b2c62eddf8d734e752e45e5300fbf30ce472f74b05e30f9d" + } + }, + "level1/camera6": { + "evidence": [ + { + "bytes": 179993, + "kind": "events.jsonl", + "sha256": "f8b4b655c2e2a762e8ca465138348e98b24c6f24d6e7d5b1e51871570dfd4cee" + }, + { + "bytes": 7037515, + "kind": "run.jsonl", + "sha256": "916816ae71b951c73744141a4e6f6fad8772e8f85247f25609623d60de5b5984" + } + ], + "rounds": [ + { + "blend_bytes": 5890950, + "blend_sha256": "837b65d0e9179115e146b98e6e42bfd69da77ace0c751869439e1c9a1d64fec9", + "clip_similarity": 0.9859825372695923, + "nclip": 0.014017462730407715, + "photometric_loss": 0.07624026387929916, + "render": { + "bytes": 257520, + "sha256": "79e84c3406a4985f43d5d58009bc431f43ef84a743b17b002e088832d3da54b2" + }, + "round": 1 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "afedfe5bb34ddd058b8318b0d3358534eea83ec96fcfd0fa62f41ab83663a93b", + "clip_similarity": 0.9614191055297852, + "nclip": 0.038580894470214844, + "photometric_loss": 0.08258999139070511, + "render": { + "bytes": 251889, + "sha256": "e5db685b08ac0ea59b924b1b16eb018d4f7a8dae3fc3b0116d36f3505cefd999" + }, + "round": 2 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "7e3eacca442df727cf985922ecd9ef88d477067f3c43275fd38511ad50f4ac5c", + "clip_similarity": 0.9827094078063965, + "nclip": 0.017290592193603516, + "photometric_loss": 0.08534161001443863, + "render": { + "bytes": 256184, + "sha256": "8a163f1b54d1bfc08169cfb34ec361e4a3fc9006b205df4f5e0653576f8a7beb" + }, + "round": 3 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "bef1e757450f4e8c3345c871157d35746182532b79b903147f2d08be8c6a56b4", + "clip_similarity": 0.970423698425293, + "nclip": 0.02957630157470703, + "photometric_loss": 0.07037719339132309, + "render": { + "bytes": 255201, + "sha256": "bd0a1f18b68fb49055bd5e426d72a3f8890042f6ebd10253e458a207871256f5" + }, + "round": 4 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "15c904a636098f0a518295ba03a17bca6c3f29918405bb672e2dffdcd159288c", + "clip_similarity": 0.9938302040100098, + "nclip": 0.006169795989990234, + "photometric_loss": 0.030057286843657494, + "render": { + "bytes": 267350, + "sha256": "ccdf07db4ff58b6d9d6aead1afebe935ffaa167e9735a2da86e451d15c0aeb0a" + }, + "round": 5 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "4e2da15f5aa5b41a42c2417f8653c7c4247c4821da0940735d2d75b066438125", + "clip_similarity": 0.9984504580497742, + "nclip": 0.00154954195022583, + "photometric_loss": 0.006648885551840067, + "render": { + "bytes": 265589, + "sha256": "5ff2ab9661c9b3a97736eacadd364e63eec19a7047190eef09c856fb7497d5e6" + }, + "round": 6 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "7d4dea19fd4e3cd06d8b1fcfef3c1cc2b5e8dc64b87d950426bc927b1de3153f", + "clip_similarity": 0.9985253214836121, + "nclip": 0.0014746785163879395, + "photometric_loss": 0.002669217064976692, + "render": { + "bytes": 265059, + "sha256": "8656838b23fdad6d778bec04afdf29944d6a51eb56f6c98da324c88606ecd7e5" + }, + "round": 7 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "60b5cc3da7c27674a8fb6de85af5e6e7486e4b39f8ac4afa9371609cfff89480", + "clip_similarity": 0.9963637590408325, + "nclip": 0.0036362409591674805, + "photometric_loss": 0.010073143988847733, + "render": { + "bytes": 265308, + "sha256": "979a0bc8bf5987f2e33c0a4097d8f629074045754e32bbdaede48a6e9baa2f3e" + }, + "round": 8 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "929c7d7f173d447bf1937c70804fa8b56f25d7c65f0f5bb79a39ba937ce65620", + "clip_similarity": 0.9996985793113708, + "nclip": 0.0003014206886291504, + "photometric_loss": 0.0003250166482757777, + "render": { + "bytes": 265119, + "sha256": "8c8b3d6acf8b2ab27366edacb451d640e0244b773bf27e4b0fd8bf9b9f51f678" + }, + "round": 9 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "67113e710df6c0ad81f5567a49f0022490382b20c0095fb69b2d319cc8cc2b91", + "clip_similarity": 0.9997468590736389, + "nclip": 0.000253140926361084, + "photometric_loss": 3.515696880640462e-05, + "render": { + "bytes": 265335, + "sha256": "61209025de8e51ac923b56fe9817442c24b3fc1a86b666870263f8e40b05acfc" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7421, + "sha256": "af99de7e7eb5485915e4d1266159e9b030c7046f62ad1a92b4ad530bd3f51da7" + }, + "target": { + "bytes": 264383, + "sha256": "e8ef88036bdd3e417383c349da3f531e22698d765f34b60851ec132d143f99d3" + } + }, + "level1/camera7": { + "evidence": [ + { + "bytes": 399124, + "kind": "events.jsonl", + "sha256": "be294d9361496bb5cae4110e5c22760512df5cb1bfd6ff106ef4cb18ff72246f" + }, + { + "bytes": 10612546, + "kind": "run.jsonl", + "sha256": "dd134d649599d327b39e2f327e30844a2f32f437e940157f413b3afde0756121" + } + ], + "rounds": [ + { + "blend_bytes": 171090997, + "blend_sha256": "5adbf52d81f66ca57b499869629bccf5a1c9ca06b135e10365fa3109c8bf7189", + "clip_similarity": 0.9739203453063965, + "nclip": 0.026079654693603516, + "photometric_loss": 0.0196079071611166, + "render": { + "bytes": 532216, + "sha256": "77f410052c5deeb134ecf2aeb58ade3d5d86c51418df50b1f21ec66358aa344d" + }, + "round": 1 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "ce57e6779a3b556141064b23ae884643f98f31243b187118ff38b840b2e9d7ab", + "clip_similarity": 0.9848811626434326, + "nclip": 0.015118837356567383, + "photometric_loss": 0.014287490397691727, + "render": { + "bytes": 530203, + "sha256": "bb2360f5cfa887f9ad5a404fab9665238f3cf883d1c8148db842bef7dd4f20a7" + }, + "round": 2 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "cb4668d3f764e534008932bc2d0df7e444780223a91ead7aa6bcff05580301d8", + "clip_similarity": 0.9873546361923218, + "nclip": 0.012645363807678223, + "photometric_loss": 0.013220968656241894, + "render": { + "bytes": 525743, + "sha256": "c1ab43b471ff642b2812cc651cff47a8726a7da16bc17548035bfa41b8229055" + }, + "round": 3 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "778f8427b917e38ace2451bcc8ca4e24aa2df930ff7a271a0bf2b20550b7e870", + "clip_similarity": 0.9847162961959839, + "nclip": 0.015283703804016113, + "photometric_loss": 0.013996700756251812, + "render": { + "bytes": 534249, + "sha256": "57ceaf32769573ce03046e41653a1db5d25ff5c97ebbeafa08864b9862842310" + }, + "round": 4 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "236c196bfe34535678bd9475d161a7174ba34cfb9970274a4ef71b6a0f5bf4e7", + "clip_similarity": 0.9858249425888062, + "nclip": 0.014175057411193848, + "photometric_loss": 0.010731506161391735, + "render": { + "bytes": 529124, + "sha256": "81d610d6be6fa30933c6d1312e0ce85350fac6373ccc144d8e42791d153968b8" + }, + "round": 5 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "93d552ea0d03a53383602e1ef72fd6fcaf3294edd22c0b49ce79e228a448b31f", + "clip_similarity": 0.9875136017799377, + "nclip": 0.012486398220062256, + "photometric_loss": 0.007920750416815281, + "render": { + "bytes": 527588, + "sha256": "30dcdb3fbc3999ac4c631900eeb1bdd001a30ebdd4828ec5b507f97fc57d371f" + }, + "round": 6 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "d035b80ed088f03efdfa3288a989d8857942d24ffc7e29c6575350198cadcf54", + "clip_similarity": 0.9810191988945007, + "nclip": 0.018980801105499268, + "photometric_loss": 0.01195294689387083, + "render": { + "bytes": 526823, + "sha256": "c07a04726e0186540c3172f5fd1f00af52ac5b0946fa95a85e6bdf535c501fdf" + }, + "round": 7 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "abc43b0128394173f3a69a451a710841d47bc7668d8b0e516a3a55ca9dfe0ddd", + "clip_similarity": 0.983600378036499, + "nclip": 0.016399621963500977, + "photometric_loss": 0.012078434228897095, + "render": { + "bytes": 529833, + "sha256": "3e8bbe5ccd4fd3e3b77eda42c3a9049f796e8179b5f2a2ab62617d484f8e002b" + }, + "round": 8 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "9dda9bbaec94af40da8cee7c32bcee18c77f7763d6c2f4e0a09f677d28b45b82", + "clip_similarity": 0.989441454410553, + "nclip": 0.010558545589447021, + "photometric_loss": 0.006557795684784651, + "render": { + "bytes": 527344, + "sha256": "acde947545f37451eb6dd0c226d1ebe475d8d6c6cf1a564a7645fb5383223f74" + }, + "round": 9 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "d5a658674a9c649d7be4e19d28811e6d72b4964caef0632cff2cf28501c32463", + "clip_similarity": 0.9878263473510742, + "nclip": 0.012173652648925781, + "photometric_loss": 0.005300699267536402, + "render": { + "bytes": 527243, + "sha256": "335ec63b3073cd4b839fef1791a977c886e5d27a65d395d968b17c8e80b11a5a" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7418, + "sha256": "601a0e0911833525ce64681125358026fea4a56dd6b9de6488817dcd628ad357" + }, + "target": { + "bytes": 527080, + "sha256": "7f5efeca8b9586450cbb5a019bc03f2b0616f76d29fecfc699cca0059150a8fd" + } + }, + "level1/camera8": { + "evidence": [ + { + "bytes": 107429, + "kind": "events.jsonl", + "sha256": "496246090490adfdd14506f3857f91449cb895423e3d5d7aecf1fd5287f53475" + }, + { + "bytes": 6001678, + "kind": "run.jsonl", + "sha256": "0729c0c58830240f23fa9a7297b8d3dce4aa596f1c0b9dcb5598922f9261f401" + } + ], + "rounds": [ + { + "blend_bytes": 2899917, + "blend_sha256": "774875b72c48876175acd617560d0d0c09993ee959085c6c8ded1b53c7b1fded", + "clip_similarity": 0.9928888082504272, + "nclip": 0.007111191749572754, + "photometric_loss": 0.0036105772014707327, + "render": { + "bytes": 180919, + "sha256": "7923a314c10af011edcbe5ae6014df56bd7ded35c97b1a4e72b5475f418d5334" + }, + "round": 1 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "6c0a8d49e31e6160499eca2c442a809fa784dfb91e03aa9594e0ac7023e55192", + "clip_similarity": 0.9933868050575256, + "nclip": 0.006613194942474365, + "photometric_loss": 0.01031858753412962, + "render": { + "bytes": 182767, + "sha256": "145a478c74e4232ecfa7276438af1235fc8b545fc5490ad579704107509aa3bf" + }, + "round": 2 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "1102bf2532f961715030d59b40af2083aea33d9e1bfe773d49a60e0fe5bfdeb9", + "clip_similarity": 0.9976049661636353, + "nclip": 0.002395033836364746, + "photometric_loss": 0.0006101542385295033, + "render": { + "bytes": 183557, + "sha256": "8c297db8eb308a8b04f796e34f009827d9a47a8580c746e6cf5e2e5606501030" + }, + "round": 3 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "78f181c3c3d6d1902a2ba41f9df4001cd440d72a0a50b15163c8ecfaf9edd872", + "clip_similarity": 0.9950434565544128, + "nclip": 0.004956543445587158, + "photometric_loss": 0.002057823585346341, + "render": { + "bytes": 182767, + "sha256": "3af0c3c8df22593db7b2a55ca4c04749fc3dc5800ca024b4efa324008f6e898b" + }, + "round": 4 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "02096f7833faea44fba015d12b75a09033202d86839dd06f31e382348458d5e0", + "clip_similarity": 0.9986875653266907, + "nclip": 0.0013124346733093262, + "photometric_loss": 0.00032354166614823043, + "render": { + "bytes": 183671, + "sha256": "3f8c669e45792eaeb07726cb85f7f6014e3bf371a3bcc53c6a69e2044c69ddad" + }, + "round": 5 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "03788353c806e4f44ba79e69d51ed8958c4b95073aa775fde3aac723a7bceee6", + "clip_similarity": 0.9983348846435547, + "nclip": 0.0016651153564453125, + "photometric_loss": 0.0001591976615600288, + "render": { + "bytes": 183536, + "sha256": "af1081c8ca55ca2ffa98de9754564e3d8f34a2413cec32612a0850b4c27564b1" + }, + "round": 6 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "02daa1ff5f27448d5b62e56cf3632b126971c502dc43881c20b277c644e0eeb1", + "clip_similarity": 0.9977761507034302, + "nclip": 0.0022238492965698242, + "photometric_loss": 7.834874122636393e-05, + "render": { + "bytes": 183767, + "sha256": "5bb008ab1b3b9f15b24e59f37990e00d66622dc62fcaff95f4315e885d3fbc0a" + }, + "round": 7 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "13babf3fc14653839393f825c008120ddda3f4e768bdaa43fe4f6b6ba08eeae9", + "clip_similarity": 0.9982494115829468, + "nclip": 0.0017505884170532227, + "photometric_loss": 0.0008940323605202138, + "render": { + "bytes": 183924, + "sha256": "31a32c007d60aab2ba74b92fed99825fd14a8cf19b098da4ee2179659c5558ea" + }, + "round": 8 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "6509cf3a02255a8bb38312bd97d1b4236e9df4bc50446f98a5c3ffd528866a7a", + "clip_similarity": 0.9981532096862793, + "nclip": 0.0018467903137207031, + "photometric_loss": 0.00019516983593348414, + "render": { + "bytes": 183923, + "sha256": "daffa380cf437d8a93d2b05efeed8a36f7fcc4f56bfeaf5c3362bcbdcfb907d0" + }, + "round": 9 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "01b47f25b8fc8d07dc0b7d55be7f1b4a2d3c64c7a0a6e58f8aec60f0c2229ee0", + "clip_similarity": 0.9990020990371704, + "nclip": 0.0009979009628295898, + "photometric_loss": 3.855396425933577e-05, + "render": { + "bytes": 183802, + "sha256": "c21eb8b7c4b70722d09701dcb9f6b2e08f2728e1dd2189cfa642fbea6c4042df" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7438, + "sha256": "84594d2a0446aa1ef4df470428bf2675084c23607d76277dd23f589ad491fca2" + }, + "target": { + "bytes": 198412, + "sha256": "b4e026fea8211aaf7f1149cd3e20a2ac6346882484eee5804ea6e19dc43b6bc4" + } + }, + "level1/camera9": { + "evidence": [ + { + "bytes": 94851, + "kind": "events.jsonl", + "sha256": "21054d0d1aaeae67cf557c62e55b2e9944c102dcf209cbb9ab6f7257bd2e3ecf" + }, + { + "bytes": 8372868, + "kind": "run.jsonl", + "sha256": "57480b23105573257199c701efb0d2564cf0bc9714e280c5e63a0d60f53c02da" + } + ], + "rounds": [ + { + "blend_bytes": 4463030, + "blend_sha256": "f9c990172cb7ca08e6ae20b10107275fc589bac8de8207f14a3b6c7439680e1a", + "clip_similarity": 0.9513208270072937, + "nclip": 0.0486791729927063, + "photometric_loss": 0.025137677788734436, + "render": { + "bytes": 311637, + "sha256": "f6ad8fe8d14b4b511deaac9501fef945ee717ad583885b111ecbb66c808b8aa4" + }, + "round": 1 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "89bd0aac9aba21594b5e012f46f65a259d09116fb1a3bdce6d38a61f6035a722", + "clip_similarity": 0.9853408336639404, + "nclip": 0.01465916633605957, + "photometric_loss": 0.009238298051059246, + "render": { + "bytes": 310500, + "sha256": "f31dd0f0b603d3193893e2238be1047dcd3afcd72c0828ae0f42e54f2626b196" + }, + "round": 2 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "7e1fc1123217617bca75de1662fd2f19afcfbe5f9cbd962d5ac31195ff7ae964", + "clip_similarity": 0.9905526041984558, + "nclip": 0.00944739580154419, + "photometric_loss": 0.008991035632789135, + "render": { + "bytes": 310542, + "sha256": "d1db10e8a159649c805f0251c13dc3fa9fc4f74089a0f9add06c2b4014a51675" + }, + "round": 3 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "6da14002e390ee0e90c8fb9b6e55a6fcad3847b14dc0dbb6be4a5a7421ad3ebc", + "clip_similarity": 0.9939846992492676, + "nclip": 0.006015300750732422, + "photometric_loss": 0.0064002289436757565, + "render": { + "bytes": 312715, + "sha256": "4ed539e5b05aaf55463e514cb698382149c61e27f51ed0bba2ee298a5903eaa4" + }, + "round": 4 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "3e4e696b865d6e4939966e5060894a83c83fe370aeae6a43c9e7c52a4fa68a01", + "clip_similarity": 0.9969817399978638, + "nclip": 0.0030182600021362305, + "photometric_loss": 0.005433480720967054, + "render": { + "bytes": 310915, + "sha256": "d0b79acb66100b0a05758c009f23ace29d1efd74959cdd6d3f86c316f26b6ed3" + }, + "round": 5 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "1382ab3f47b34a06d56e667ef07db43e70d92b6e651c65633f4d27f919944b55", + "clip_similarity": 0.9983714818954468, + "nclip": 0.0016285181045532227, + "photometric_loss": 0.0012882169103249907, + "render": { + "bytes": 310652, + "sha256": "c2d47d2fd35531117523147bd97dff3d0b511b85c1eed4f51cea4755b0fd2aa8" + }, + "round": 6 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "19094fd64181d4780cfaa023fa09a25b8474fd3fdc3dee5fbbe96df81c4a3064", + "clip_similarity": 0.9987772703170776, + "nclip": 0.0012227296829223633, + "photometric_loss": 0.0006784373545087874, + "render": { + "bytes": 309989, + "sha256": "4fad5e79b83e974156f810a10078dcc70eebfbdbef4c7b4e4bff60e85eb21a01" + }, + "round": 7 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "b2b86f2d82e7276ec57cc1252fc3612a1f70ee1fd788bba2ecee03f782f45047", + "clip_similarity": 0.9967776536941528, + "nclip": 0.003222346305847168, + "photometric_loss": 0.0009337430819869041, + "render": { + "bytes": 310239, + "sha256": "05007cb82eefc7220bead065a48c5ca2b0c328978163a224cbdf4e9a8c448c26" + }, + "round": 8 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "8f811dc81224704430d54e9be29fc9a586ffa0e4582de5cd4c2e44038a8bc9df", + "clip_similarity": 0.9995280504226685, + "nclip": 0.00047194957733154297, + "photometric_loss": 0.0004399387107696384, + "render": { + "bytes": 310348, + "sha256": "47367201f173f475d5487ef0d2a858baf5c9ff595477f9531ab11791fec8a7b6" + }, + "round": 9 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "be155dbf927bd5e410d847be2378a45f00d3c441079c4dd7b6cd192c5ce90487", + "clip_similarity": 0.9991905093193054, + "nclip": 0.0008094906806945801, + "photometric_loss": 0.0003071240207646042, + "render": { + "bytes": 310456, + "sha256": "14a3b1b89d5268da101af3c5ebc26c29461a520fab59fd40a959579dc796d51b" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7431, + "sha256": "21735e12136fac3b57d6d1eeea8a05747592402cd8df629ca120ae6b9525eff9" + }, + "target": { + "bytes": 311165, + "sha256": "df51a5786e84b1d47c2df08c95d6adb3f7594dec20fa17c47424bb08b16145b6" + } + }, + "level2/attribute1": { + "evidence": [ + { + "bytes": 1489445, + "kind": "events.jsonl", + "sha256": "6d4d138469bfc8c4c56677f947cd8e045b54dbefe5964a7dea5a8f0ffbce54cc" + } + ], + "rounds": [ + { + "blend_bytes": 949424441, + "blend_sha256": "1e565bff9cbe7b822d7403e55d3bc1dcfeb0a39aa92276de4ecc4ba4dba77c02", + "clip_similarity": 0.9616481065750122, + "nclip": 0.03835189342498779, + "photometric_loss": 0.03206987306475639, + "render": { + "bytes": 3035159, + "sha256": "aa4807b88f9bde1d77bb5e37629684b2115f22b61c6f17c1ca73287375a34b21" + }, + "round": 1 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "d85a246f1e7f754e216860de4e01fc660d84b5b1d2065b5d3168183bdd168b6e", + "clip_similarity": 0.9758520126342773, + "nclip": 0.024147987365722656, + "photometric_loss": 0.020144661888480186, + "render": { + "bytes": 3045247, + "sha256": "cfc963a03a0788090fbdf42b9e2bbb89be0ee87ae9b22a75e919c41de2653172" + }, + "round": 2 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "eb6422915c3ce4af1073b71f37d21338474f48a1d06f806f5d3c096f7f13abfc", + "clip_similarity": 0.9763191342353821, + "nclip": 0.02368086576461792, + "photometric_loss": 0.015242948196828365, + "render": { + "bytes": 3040790, + "sha256": "3dcffb98b4cdbe65ae33fb01032761d83d490d7fe4c93a833750a49103704677" + }, + "round": 3 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "8c33a3132cc32270b96c771c7ca978a23745441c1a9762c8451dc8f4a3d114f3", + "clip_similarity": 0.9755369424819946, + "nclip": 0.02446305751800537, + "photometric_loss": 0.015236021019518375, + "render": { + "bytes": 3040243, + "sha256": "dba49561195b08488ae1a0d53b546de734e742c2b6eb7e5149d8958fc727d816" + }, + "round": 4 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "220c2c3dd4fc28a8919ed5f5e6217ac5de2ec3ca16d477a9daec63b902a9d561", + "clip_similarity": 0.9777381420135498, + "nclip": 0.022261857986450195, + "photometric_loss": 0.01512475311756134, + "render": { + "bytes": 3035300, + "sha256": "d1c78833cbc456ae14b1ce041ed3bbd06bf3e006e79c5cc6d0cb8d5c11765d16" + }, + "round": 5 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "068276672a643ebabe71486efae1ce3acb8f56b804e3a58a2fcb9c6fee7a3f45", + "clip_similarity": 0.9761902093887329, + "nclip": 0.02380979061126709, + "photometric_loss": 0.0151130435988307, + "render": { + "bytes": 3033925, + "sha256": "3c279d92a2ec3e92630bbe098c619df69163d458ae8105090fdb24f3dbe3be10" + }, + "round": 6 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "c39265f69d222f4e97114112a96d4b56d16f3a71e521a9e5729a17c78be04d7e", + "clip_similarity": 0.976814866065979, + "nclip": 0.023185133934020996, + "photometric_loss": 0.01512062270194292, + "render": { + "bytes": 3034742, + "sha256": "8a330f4d3d0fad7c7031769030eed277f7720484d15562e563af4d2c04b91539" + }, + "round": 7 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "a704a8942747fc3371bbdd6231f1ba35e7ddbe5e4ba3cd07021834cbc6af4401", + "clip_similarity": 0.976791262626648, + "nclip": 0.02320873737335205, + "photometric_loss": 0.015117769129574299, + "render": { + "bytes": 3034761, + "sha256": "1ecb25364beb1f83f08f1d8e073a00a171d54f9c8e9225ff1f3fb2f5beb75828" + }, + "round": 8 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "6fdd0bdf5249063684ef18645217bb03c374272874cce8929f7bbd0d1660590b", + "clip_similarity": 0.9776802062988281, + "nclip": 0.022319793701171875, + "photometric_loss": 0.015120291151106358, + "render": { + "bytes": 3035015, + "sha256": "5d94a5b5418b2703c85eed3f908156a67dd4675cb5098dfd28c74bc5ef4a64e5" + }, + "round": 9 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "e33accc273b2480f031bba55e353df90802d86e345e11fbd6b9acadd44a27cdb", + "clip_similarity": 0.9777544736862183, + "nclip": 0.02224552631378174, + "photometric_loss": 0.015119296498596668, + "render": { + "bytes": 3034786, + "sha256": "fa47897f60a5d9046001285695df81c97735f67921d27534b29257abd305a09f" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7660, + "sha256": "f1921da48be6fb80affc40803c16939438152148f5c40171c7673ff8788ad7d2" + }, + "target": { + "bytes": 435882, + "sha256": "a269058785099fd3f273334e70fa147de3ee3556a500ded8d9b064213fcd8981" + } + }, + "level2/attribute2": { + "evidence": [ + { + "bytes": 1489512, + "kind": "events.jsonl", + "sha256": "a93beb1267ee745dd27e7498964c1ae787820ea167f12f727e94e234e38876ba" + } + ], + "rounds": [ + { + "blend_bytes": 949424441, + "blend_sha256": "882312ac90f9ab4e4e50e48eaa6bb4170807704168ce1dccdcdfef9e9225d48f", + "clip_similarity": 0.9783761501312256, + "nclip": 0.021623849868774414, + "photometric_loss": 0.023375628516077995, + "render": { + "bytes": 2999030, + "sha256": "6beda2b96f488fdff61fa71718d217393f2296c45a24c20eea41b2d200e51d4b" + }, + "round": 1 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "733b4df9380626f5cdc7d54837fa564de14d5ba1091a31933bdbd1c18141b72b", + "clip_similarity": 0.9806861281394958, + "nclip": 0.01931387186050415, + "photometric_loss": 0.01821855455636978, + "render": { + "bytes": 3025233, + "sha256": "c03d37b649e7f613a806494aa835f8bfd8e9c33c8165d95dc7125dd51985a0f5" + }, + "round": 2 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "ae2390ddf2bf6f9ee6d5d473185a936fcf68edb9df7f4c90d50d0638bd403fb4", + "clip_similarity": 0.9808710217475891, + "nclip": 0.01912897825241089, + "photometric_loss": 0.016760846599936485, + "render": { + "bytes": 3045603, + "sha256": "ce52f84d25c9a879c3bf5f98fc9875231d57f3a16c05fbedfed0786d1e7d693d" + }, + "round": 3 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "91efbb33590c2e1fe5b608d9924dcd2c98609c0f585bad35d6e2a7acfa53b08b", + "clip_similarity": 0.9811261892318726, + "nclip": 0.01887381076812744, + "photometric_loss": 0.016651509329676628, + "render": { + "bytes": 3040739, + "sha256": "b0431d4d18bbcbc3d286f708c981cf756e86ec4f8f0f9740a9ed2a1a454a4501" + }, + "round": 4 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "5c8bd94c0314ebd4bae377b6a5f7193b4c81dd368256c0b0381f405d993435a0", + "clip_similarity": 0.9820566177368164, + "nclip": 0.017943382263183594, + "photometric_loss": 0.016636313870549202, + "render": { + "bytes": 3039345, + "sha256": "68cfffac21da51743520aeeaffc73149d8b886d347531ec504e173852b2b04a4" + }, + "round": 5 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "25d4ce76f9f63630249e3790de94e8a3145003de017efad1f27b551b8d5d6484", + "clip_similarity": 0.9820483922958374, + "nclip": 0.017951607704162598, + "photometric_loss": 0.016636930406093597, + "render": { + "bytes": 3039332, + "sha256": "99ee4ab8308811d8155e13b327e2c47973db63b9794bb114123550aa89442eae" + }, + "round": 6 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "dde7ab314504a3ef4c7de24a4fe846029af74825546d82f29c078640cdf02a6d", + "clip_similarity": 0.9812586307525635, + "nclip": 0.018741369247436523, + "photometric_loss": 0.016626911237835884, + "render": { + "bytes": 3038499, + "sha256": "ee64a9d46874b58a0215d941e00b6b75b884e6d41bfc78ff31e3a6e99963e389" + }, + "round": 7 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "e98ed32620e7ce642df4b07ebf26f07ecfd6e97a0f7d8b59a8ee2345178aa1f8", + "clip_similarity": 0.9812836647033691, + "nclip": 0.01871633529663086, + "photometric_loss": 0.01662970520555973, + "render": { + "bytes": 3038401, + "sha256": "59c58887845be9ded30c6eaf9367984dbc30017f9e0214eede342b5486b8c85e" + }, + "round": 8 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "15593ecf6a309580011a203d68b2519029f18b8dad0936a1a824de813b49bc0e", + "clip_similarity": 0.9816768169403076, + "nclip": 0.018323183059692383, + "photometric_loss": 0.01663150079548359, + "render": { + "bytes": 3038593, + "sha256": "c084d4aa7e80ddcbead7d82b1f35202bc539d1ca34afe8d91d62c9a5e71e9ca7" + }, + "round": 9 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "7d82c31ceca869e85ac81e5efc745f6420ec2dec216484457fc9f86c78a6b06b", + "clip_similarity": 0.9813920855522156, + "nclip": 0.018607914447784424, + "photometric_loss": 0.016634559258818626, + "render": { + "bytes": 3038754, + "sha256": "8c59abcd322c2a4bec6a4f94eaa63a84ab9aa7f55a97b32c84c7d6588a1b68e3" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7666, + "sha256": "5aa86a85646b9760945346c21a6f6657e63f6ca46950b679862ee5d34184deaf" + }, + "target": { + "bytes": 435876, + "sha256": "0744a471d3a07fe9c4a38402f9be09310e5e26dc4abbc62f6b634d20b1f6c23f" + } + }, + "level2/attribute3": { + "evidence": [ + { + "bytes": 1489205, + "kind": "events.jsonl", + "sha256": "036d8d1b59caac459e9ea36ca3f80bb37ff940007e9efde052fab9bb53ea0e74" + } + ], + "rounds": [ + { + "blend_bytes": 949424441, + "blend_sha256": "113bf72a065d22b3cf48448ad5657b7236449a0bb89c072bf3cc30b8e77efeef", + "clip_similarity": 0.9737488627433777, + "nclip": 0.026251137256622314, + "photometric_loss": 0.01580880396068096, + "render": { + "bytes": 3038266, + "sha256": "c3f3b42a13a3328705aaf935239f44faa998260b649de2f412a2740caf429804" + }, + "round": 1 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "d6a2131d15eb29e00ac8a0dceb8f82e08a003c2b9aa756e646630116e847ad3c", + "clip_similarity": 0.9718049764633179, + "nclip": 0.02819502353668213, + "photometric_loss": 0.02121717669069767, + "render": { + "bytes": 3000697, + "sha256": "c12c7f92ca10a3aa2b4f812fef2c78209645dd80454d28e95b16bbc0be59effb" + }, + "round": 2 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "5c40e4f3457556b354fcf91731e5d71e320e2812c8384b8eb5596209841e4545", + "clip_similarity": 0.9792160987854004, + "nclip": 0.02078390121459961, + "photometric_loss": 0.021017536520957947, + "render": { + "bytes": 3007735, + "sha256": "d5932cf757b1f60d11f852ca1ec5c1eca0a8154f1733b2db7edbe0d82ac4983a" + }, + "round": 3 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "79ed406718e54bd39d292dddee197eb9d68c5540d0b327e91ba7524f2eb007a8", + "clip_similarity": 0.9793153405189514, + "nclip": 0.020684659481048584, + "photometric_loss": 0.01562858745455742, + "render": { + "bytes": 3038284, + "sha256": "be7f2216b96479b9ef5e4d74aa6ce89052c11a87e5467320c7f90c512062489e" + }, + "round": 4 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "b260539226ad62f49d3387934a66f03db3b87e627c8046e8971835dccecb85b6", + "clip_similarity": 0.9762895703315735, + "nclip": 0.023710429668426514, + "photometric_loss": 0.015242993831634521, + "render": { + "bytes": 3040810, + "sha256": "e8e04bf11387a85ceafe9b47fdc30a8c37523a489acf4936ad4b61f871581016" + }, + "round": 5 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "01172623305c08bbbf865395f27d4a82e80920f980c68f0658577d121cad41f4", + "clip_similarity": 0.9765477180480957, + "nclip": 0.023452281951904297, + "photometric_loss": 0.015135708265006542, + "render": { + "bytes": 3036082, + "sha256": "d4e42e55b310bbb2ddeec61532e92584c3e0f4b249adedbb1719812103f30b81" + }, + "round": 6 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "486618df98b0386d6fdb371b6c4170a62efb5833b6899aa902953389bb605ed6", + "clip_similarity": 0.9764472842216492, + "nclip": 0.02355271577835083, + "photometric_loss": 0.015115156769752502, + "render": { + "bytes": 3033885, + "sha256": "1695d085e9d2c270ca07bec83cbc34fc3215d046192e1846a91b7259d317157e" + }, + "round": 7 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "ba9fcf5d28d5362182c60992df40a76b4886c6a3db6b3d0715a21bbec2cc386b", + "clip_similarity": 0.9759871959686279, + "nclip": 0.02401280403137207, + "photometric_loss": 0.015113621018826962, + "render": { + "bytes": 3033841, + "sha256": "07ce05b6c3f379eaa5e77f3846901b276549a31ef34beb977a5686e01dea0862" + }, + "round": 8 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "6ec9362f5a8e3474f74e9a0da7073ddb56c5d842f5161023fa95a37cefcbba4f", + "clip_similarity": 0.9774898290634155, + "nclip": 0.022510170936584473, + "photometric_loss": 0.015127800405025482, + "render": { + "bytes": 3035273, + "sha256": "312a041a3b8ea8c7889c7e79a8b13fad11fbbf73dab93c5f2c2807cd182d7432" + }, + "round": 9 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "e3ca3d9290cf3702e8c306ba8388a34053af0427e10efbfc3524cebda811ec4a", + "clip_similarity": 0.9767491221427917, + "nclip": 0.023250877857208252, + "photometric_loss": 0.015117752365767956, + "render": { + "bytes": 3034771, + "sha256": "2d2544dd2e2c25607833c112f7b04c91433fa4f00b3a72b0871476912e1e27e3" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7666, + "sha256": "0564fb08e82de849edaa0d8bdeda4d3c8a8b6ac249868b974c88f77c4e52ee8d" + }, + "target": { + "bytes": 435892, + "sha256": "3886f82c111b6e7dada41bcd6129e9c782f543a11e02860c693a5e3c8a336da7" + } + }, + "level2/attribute4": { + "evidence": [ + { + "bytes": 1323480, + "kind": "events.jsonl", + "sha256": "c3cac4087cc1a1fe5c3f5f34768b4f0ea9eae647e25ce137d886187dc35f3880" + } + ], + "rounds": [ + { + "blend_bytes": 940184532, + "blend_sha256": "8310e08dafd21a5225517efc091c53f14acc08ee8e65702d4f421f3598a93afb", + "clip_similarity": 0.9745771288871765, + "nclip": 0.025422871112823486, + "photometric_loss": 0.02019927091896534, + "render": { + "bytes": 2955115, + "sha256": "16d8eb7c9d20f8d0a9573cba78473abb9867339b730b3a2443a7f5c692a18449" + }, + "round": 1 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "b01225040e1d12bad96ff6d554bf81f12970eaace058d4b76ec3ea5b88ad3f09", + "clip_similarity": 0.9726730585098267, + "nclip": 0.02732694149017334, + "photometric_loss": 0.01562739722430706, + "render": { + "bytes": 2980024, + "sha256": "381daf79ed2269fece3d52f0d0c3901462a72c436344f2703b36630d19c9b15a" + }, + "round": 2 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "d9e102721b30721f7b4cd8953f308046e120ae4a4b7c6d14913d4e0071d91678", + "clip_similarity": 0.9584736227989197, + "nclip": 0.04152637720108032, + "photometric_loss": 0.024826040491461754, + "render": { + "bytes": 2982224, + "sha256": "6d11b84059359c8c973070dbc98c3e67af510c38ed5bcd4aa7611f57b551d174" + }, + "round": 3 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "02c5e07bb556542c8d890181ae6b9643a39bd0d689f5b481629cb426086637ed", + "clip_similarity": 0.9532421231269836, + "nclip": 0.04675787687301636, + "photometric_loss": 0.031184980645775795, + "render": { + "bytes": 2974963, + "sha256": "c42fbf82e09c6ff8d99c0e27d444db902d65b191830abd49aef4651b8a3ab062" + }, + "round": 4 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "e3906bf524eee568eec3059099ca8d0502bc2bec87173d0a46a3d84548559a3d", + "clip_similarity": 0.9654848575592041, + "nclip": 0.0345151424407959, + "photometric_loss": 0.03093068115413189, + "render": { + "bytes": 2979217, + "sha256": "8e3322d079e97b16a9784a04256a7834634c9dae9a468c76361f1ebcbc94744f" + }, + "round": 5 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "365a0bcbd1ffb6a6f90392bb11bb412c77940d10e395923c8c22d322268356eb", + "clip_similarity": 0.9615705013275146, + "nclip": 0.03842949867248535, + "photometric_loss": 0.030709154903888702, + "render": { + "bytes": 2977789, + "sha256": "dd745856e3155649fb989a6f94a684138910fc33c1a252f27f14e0da5555af3c" + }, + "round": 6 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "5f406cd1b169a821ddd049c96d5aa4f7bc3e641f9b2e958319b8e45c44a8c338", + "clip_similarity": 0.9655020236968994, + "nclip": 0.034497976303100586, + "photometric_loss": 0.030589738860726357, + "render": { + "bytes": 2979565, + "sha256": "e709c4f2421b6ce5cfa9739d04be1b35ceef0b552be719a6ce811ab12e9c2c31" + }, + "round": 7 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "2f7d2f07d9369084d7932d49933942bfe90c54a11ab1261a97d4295d9f2d03c4", + "clip_similarity": 0.9670642018318176, + "nclip": 0.03293579816818237, + "photometric_loss": 0.030599204823374748, + "render": { + "bytes": 2979677, + "sha256": "c0aacf4f381698ebd546346afd806ca833f39f58a8bfc6c518c8463432f27ed4" + }, + "round": 8 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "1c85fd6849a1f66b69e16de8b892eda578e3b35a37fc6cc1d34a3d1732aefdb3", + "clip_similarity": 0.9656827449798584, + "nclip": 0.0343172550201416, + "photometric_loss": 0.030554519966244698, + "render": { + "bytes": 2978938, + "sha256": "f7be7217e49eb1857a4f6176295bc1ad1712486523b474dbb9c860dc31bcf478" + }, + "round": 9 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "ed19f7f771bae56f25269b2e49f67c124aac86deebb113bc66d7f0bd4eb33243", + "clip_similarity": 0.9664871692657471, + "nclip": 0.03351283073425293, + "photometric_loss": 0.030548766255378723, + "render": { + "bytes": 2979101, + "sha256": "50c0626ac1782484efab69c95c57eb2dfa6007d14e38db12c6bea2b618ed9175" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7660, + "sha256": "8ef915fae4dfdf98b9e6f86563b1d54d7b9ca1d3ae35b2dd9fd69865572c8b87" + }, + "target": { + "bytes": 433064, + "sha256": "dd049697d61750d34d9f0587a832d7b8ad322197e63b4319b1938675b64eefcb" + } + }, + "level2/attribute5": { + "evidence": [ + { + "bytes": 1319636, + "kind": "events.jsonl", + "sha256": "5e70e46af62448ad2bf771b77a9738cfe6affa873abb9012046ee36cc36b59f7" + } + ], + "rounds": [ + { + "blend_bytes": 940184532, + "blend_sha256": "67dacaa47c9c5c46a9861f579af526e16fa2791a7274380303725d264da0c4d8", + "clip_similarity": 0.9714045524597168, + "nclip": 0.028595447540283203, + "photometric_loss": 0.016888050362467766, + "render": { + "bytes": 2992284, + "sha256": "c827bf49aa0800d0bc9c17d305965d2236d0bc12fe8300d2264a439d4b668d25" + }, + "round": 1 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "b1e3ed0fc24095d5eb13c351255ff1f1aad81891e725de60e4ca1fa6cdd9de1b", + "clip_similarity": 0.9766486883163452, + "nclip": 0.023351311683654785, + "photometric_loss": 0.01576325297355652, + "render": { + "bytes": 2987400, + "sha256": "358fe6b8b86c342a7b5b895ebd541f6b27aebaf80b20efd236631e93dfd3fed4" + }, + "round": 2 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "83887082e62043daaa6da4757bcc86750e675da952b8dd4ea9e6662cbf112fdb", + "clip_similarity": 0.969855010509491, + "nclip": 0.030144989490509033, + "photometric_loss": 0.018261505290865898, + "render": { + "bytes": 2950422, + "sha256": "169f6a286c9f91879743fab781200842574cdd70d0a0f233b1e64f8d51de302e" + }, + "round": 3 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "457e1e80ff383eab109d5668f5c254593bcb0bb2221d56c1c965d0ce7b276b49", + "clip_similarity": 0.9712028503417969, + "nclip": 0.028797149658203125, + "photometric_loss": 0.014592328108847141, + "render": { + "bytes": 2982542, + "sha256": "28e803bfc2568a447a82108a709cd24f42c169b64a6e5193c6794fa2b15524a1" + }, + "round": 4 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "b8530aed42d9c3319116bd90ea39b3827e564d30117cb7081aaf181b34752af5", + "clip_similarity": 0.9800482988357544, + "nclip": 0.019951701164245605, + "photometric_loss": 0.014472146518528461, + "render": { + "bytes": 2985411, + "sha256": "b9fce9fe928c2d2705792f0a60a858fe2733bcc15f88701fb7bca84b33aaade2" + }, + "round": 5 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "174b3025903e0f92ee71ae866535324c43aea637a7ce68fc024a6a46fca24766", + "clip_similarity": 0.981130838394165, + "nclip": 0.01886916160583496, + "photometric_loss": 0.014343224465847015, + "render": { + "bytes": 2986237, + "sha256": "10cb1babed5aaa8c75bbaa038935714499cbac0ca72b87278f8d668b2e924b25" + }, + "round": 6 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "7b855e24f2f09010746bf2edb9da75da7219db81c8f3c8717df79062ef29507f", + "clip_similarity": 0.9813302159309387, + "nclip": 0.01866978406906128, + "photometric_loss": 0.01404707133769989, + "render": { + "bytes": 2985549, + "sha256": "dd5b0b9ce272271592cbf2f1a31be43c58aca29dec32e45d64e9f46df9a0696a" + }, + "round": 7 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "08d72b29fc92a0179b4fe890a88a073d1f7001fd7d81c566aee247709a150a6c", + "clip_similarity": 0.9820970296859741, + "nclip": 0.01790297031402588, + "photometric_loss": 0.014138872735202312, + "render": { + "bytes": 2986075, + "sha256": "af57de90dc47297013bba3627d345b9e1cf2057eeab4be71863989e736dbc209" + }, + "round": 8 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "1a569a78e74a4694a0ce5c8dab9771ae18cedd1752747baf5a00f46a5c9e9162", + "clip_similarity": 0.9830480813980103, + "nclip": 0.016951918601989746, + "photometric_loss": 0.013992843218147755, + "render": { + "bytes": 2985804, + "sha256": "90b7e6775020cc48c4f481dfcf1b27bb91e549bf303e59242f1169e361436ec7" + }, + "round": 9 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "901160ecfc9885902e711c2942bf5071c3f4c6e5ad5d1ebe9a3337e464da052b", + "clip_similarity": 0.9829906225204468, + "nclip": 0.017009377479553223, + "photometric_loss": 0.013999431394040585, + "render": { + "bytes": 2985487, + "sha256": "3fad542dc6dbf6d6112c9b5ca2eb66dd7030206e42249aaa55cf43cefb9b140d" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7666, + "sha256": "7a38668115b17d5017f471704b600bf17466d5130316b5505674441a95e4ff83" + }, + "target": { + "bytes": 433064, + "sha256": "0b33dfa853ce71b3e0a91ae9813e90dba0544f489251d8a811c810c1491fce55" + } + }, + "level2/attribute6": { + "evidence": [ + { + "bytes": 1323085, + "kind": "events.jsonl", + "sha256": "62fc68529fdd17fd274c87349f2564aef95ee63c8079617ce897c5a31357c4b7" + } + ], + "rounds": [ + { + "blend_bytes": 940184532, + "blend_sha256": "a6cbccd0441951188d66304b2da639a956b01bdbe0428380ec0bcae05f58af60", + "clip_similarity": 0.9696398973464966, + "nclip": 0.030360102653503418, + "photometric_loss": 0.019036224111914635, + "render": { + "bytes": 2981721, + "sha256": "13c1b3e9c4698293e9894ccc943f8152dc91f5bf3df6033572819d414299edb8" + }, + "round": 1 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "d2ae6344ba60b0a1d3f31ef11e1e65f7fbef403fad235601933cb3b5bcc7a2c3", + "clip_similarity": 0.9568657279014587, + "nclip": 0.04313427209854126, + "photometric_loss": 0.018575983121991158, + "render": { + "bytes": 2990428, + "sha256": "56f3a20fc48604323b6f108e479f7904c2a71578b635ad7a5cd8d49caca32dcd" + }, + "round": 2 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "b2671ff7b3ef79c23e07858b7b9bc21c11a46ddbb806fc0b62fbbb698aff72e3", + "clip_similarity": 0.9651858806610107, + "nclip": 0.03481411933898926, + "photometric_loss": 0.01557935494929552, + "render": { + "bytes": 2986658, + "sha256": "fb61bda5ce7a38a421406aa318d9caebb86533a2eb202925e1a3f7265250ce47" + }, + "round": 3 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "701eacbd69c21a88a9412bfbb351fab8b6e12ee4d2b5d25cd930915a0657f83e", + "clip_similarity": 0.9659790992736816, + "nclip": 0.03402090072631836, + "photometric_loss": 0.01718028448522091, + "render": { + "bytes": 2981347, + "sha256": "e9649f3cc48cef0455f4f645ad3e38122ec10d154ed12b6b4082015a871f2298" + }, + "round": 4 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "a3e84f5be3ab210a64da01b4f00c0c65dea9a1b5c065da9134c040770bc84de3", + "clip_similarity": 0.9705029129981995, + "nclip": 0.029497087001800537, + "photometric_loss": 0.01845850981771946, + "render": { + "bytes": 2966621, + "sha256": "82559f9b41fb40ae33ddb502e7f333a3b2a5e0cb46f0f07296d3d28dba58420f" + }, + "round": 5 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "5999b5cfb151d2efba3eb72a76a8f73b3d8f5c08e884f2558b5943d83edb8dc0", + "clip_similarity": 0.9824894666671753, + "nclip": 0.017510533332824707, + "photometric_loss": 0.017971405759453773, + "render": { + "bytes": 2969069, + "sha256": "0b5573b05a83becc253ea611c733baa78aeb2da7bba228e756fa1ee625855e57" + }, + "round": 6 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "d58130d5ead2d4f5341127ceaaae428af480f68c79ab072854e8f45a6e63856c", + "clip_similarity": 0.983490526676178, + "nclip": 0.01650947332382202, + "photometric_loss": 0.017553135752677917, + "render": { + "bytes": 2967827, + "sha256": "9965efe5c3bf14db8579fe468aeb7ca23037dec11004961994b7da573517e402" + }, + "round": 7 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "dc3ab7a8c2b733c4f7e0fd90ef4d08a2f69bdce801c3336ad2903b252242b127", + "clip_similarity": 0.9839648604393005, + "nclip": 0.016035139560699463, + "photometric_loss": 0.01759149320423603, + "render": { + "bytes": 2968407, + "sha256": "989b5a838529141c027bb6e85459cec713338f4ffaaec405f4c17cadb6aee1f8" + }, + "round": 8 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "6a84d386171ccbc67abd38de70ed228349f8b567a865b07ffbde0382beefb3d4", + "clip_similarity": 0.9842134714126587, + "nclip": 0.01578652858734131, + "photometric_loss": 0.01759195886552334, + "render": { + "bytes": 2968179, + "sha256": "8be63127a6c56c9676e78f6922a4ae1bd2bf126b1ca56326b6f2b2f4cc3675dc" + }, + "round": 9 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "4fca13e1573704f948453c191cbcd3e0ee8f0c87218c958ad417a0f3a02c4c75", + "clip_similarity": 0.9838805198669434, + "nclip": 0.01611948013305664, + "photometric_loss": 0.0175719503313303, + "render": { + "bytes": 2968119, + "sha256": "6bedbf2ce1a6cf48ffa559a42ebccb6852ff4a04bc608b1b89255b5a91432140" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7659, + "sha256": "b599c3a4ae983dec616c337046b09b494852dd359d8163fe1588e042f425a647" + }, + "target": { + "bytes": 433064, + "sha256": "1eda79fb1fd8773714467694b24cbbf71f049549b0016cac79254724e5c5b5cf" + } + }, + "level2/attribute7": { + "evidence": [ + { + "bytes": 1268392, + "kind": "events.jsonl", + "sha256": "dce01e63512a9e3bafbdf6c3c96205ede419b550dc77f1c19c7493a6c3f9fea1" + }, + { + "bytes": 11651770, + "kind": "run.jsonl", + "sha256": "e35870804b513a9a37189edaf38510b90bcd9e44f40dd3380aa8e74059079ff2" + } + ], + "rounds": [ + { + "blend_bytes": 646096642, + "blend_sha256": "f010800f8cc8419c6d4ad399bbd1e5ad84c75b7c83412e6b3187b5a496164cf8", + "clip_similarity": 0.9875383377075195, + "nclip": 0.012461662292480469, + "photometric_loss": 0.005757831037044525, + "render": { + "bytes": 576705, + "sha256": "e43166183b876109daafa9c5b2880a63a8b4f0e068ca5dd6e73c3e38a1a11047" + }, + "round": 1 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "5dcc84073191c4099f17fb6ff4b524e41df6d1b9636bdc666eee43337f634a19", + "clip_similarity": 0.985919177532196, + "nclip": 0.014080822467803955, + "photometric_loss": 0.004302607383579016, + "render": { + "bytes": 576399, + "sha256": "366f4dd96fd968dc5ee35c4628fdf3a75b8b21a628d349ff2d2c1af08a1d28b2" + }, + "round": 2 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "dab28a97ceeaca0c4650c70cfdb74e71c3d3e3e06eb8212283461f375077fe1c", + "clip_similarity": 0.987267255783081, + "nclip": 0.012732744216918945, + "photometric_loss": 0.0038804830983281136, + "render": { + "bytes": 577059, + "sha256": "5864f5ca23351baed3de29ba097e60b86d0eaa766d704a9fbabd1a4aa043c9d8" + }, + "round": 3 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "f6fa92b9046bc84c8a169fbebde44301a0d9c7e7665521768ab6870085519eda", + "clip_similarity": 0.9858017563819885, + "nclip": 0.014198243618011475, + "photometric_loss": 0.003929381724447012, + "render": { + "bytes": 576889, + "sha256": "4f1ae0e453f929996644a0895051c19626f729e867831964fa29671cfaca2225" + }, + "round": 4 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "2698fe51ec0a9aca2afe883e25788a60ec38b4e47741efcdf5459715bf2824a5", + "clip_similarity": 0.9856680631637573, + "nclip": 0.014331936836242676, + "photometric_loss": 0.0038459450006484985, + "render": { + "bytes": 577057, + "sha256": "4185453069c3174730f35ff1f9ccc42d9eeaa80ad23cffd245efa3dab2bd207d" + }, + "round": 5 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "145bbba903be3edacd0d3783c98984b3b4f9f827847542a5e38b763ba61cab8a", + "clip_similarity": 0.9873605370521545, + "nclip": 0.012639462947845459, + "photometric_loss": 0.0036604765336960554, + "render": { + "bytes": 576927, + "sha256": "93089d36a5a0fc40f5c8dc923f3d74a61888e02ddf42444c27991ede736a19c2" + }, + "round": 6 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "51e6fed894faf60848cc0a17b078c2faad9b3c27245a9b9730aeadfa7a3d11a9", + "clip_similarity": 0.9880251884460449, + "nclip": 0.011974811553955078, + "photometric_loss": 0.0034597732592374086, + "render": { + "bytes": 576869, + "sha256": "e1a6d696a802e899a40dbf57b2226b4b0a4ecb39aabd788c90dfa5f6ec0f7148" + }, + "round": 7 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "b2d34676246a2065c9ef4c58f7b6b51ab38698208d23d2296623b8fd6ba7c527", + "clip_similarity": 0.9892903566360474, + "nclip": 0.010709643363952637, + "photometric_loss": 0.003371603088453412, + "render": { + "bytes": 577019, + "sha256": "b200a48eb287105d7e8b5003cfcfadd0d29662db5a8bd388f56775ba9a9e2caf" + }, + "round": 8 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "e6acf86451a2043c5db9d0bf51bd8e8fbe165d0e3f25d6da194a19df03afb511", + "clip_similarity": 0.9894779324531555, + "nclip": 0.010522067546844482, + "photometric_loss": 0.0033791407477110624, + "render": { + "bytes": 576818, + "sha256": "b8950d0069b25e584932e146deddf9fc18ffbc8d0ea738cb31dcb201e14904f3" + }, + "round": 9 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "22edd288b13f3b569418ef40c632825667ea2ca0f638d42bcc89526660842366", + "clip_similarity": 0.9892759323120117, + "nclip": 0.010724067687988281, + "photometric_loss": 0.0033672337885946035, + "render": { + "bytes": 576970, + "sha256": "1fcc64dd437079e420dac180cf3976845ed83d01de3368b5a2ba306890964a81" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7597, + "sha256": "d600bb2bbef13fb508547ef7c1330fde7c2f5de94535b187099ca7877445822d" + }, + "target": { + "bytes": 496461, + "sha256": "4e0dcf670648bba309e19853047bb294bbf588ffb8928ed9f9a45bef729f9f32" + } + }, + "level2/attribute8": { + "evidence": [ + { + "bytes": 1269817, + "kind": "events.jsonl", + "sha256": "b7dcb3440ba2f433f9141688aeb7a39b07d6255d9beac967c0b9db57cf6eae13" + }, + { + "bytes": 11669509, + "kind": "run.jsonl", + "sha256": "539448b867d21f6820fed02cacd420b43bae2cfcde0906ad58d9be0bcb3da685" + } + ], + "rounds": [ + { + "blend_bytes": 646096642, + "blend_sha256": "73de396f058a5e66af2eeb10d27e205b16196049c886e5e74917c2aac930ace7", + "clip_similarity": 0.9375687837600708, + "nclip": 0.0624312162399292, + "photometric_loss": 0.0443587563931942, + "render": { + "bytes": 573444, + "sha256": "400a007df0d28672ae0036c9bf53b52ccd4d69b03e0374facc29f40def0159ce" + }, + "round": 1 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "dcf1b9d54ed87e7fad2403a3719d28d3fc5c73686c7bdceedfdd9f480ebb802b", + "clip_similarity": 0.9542206525802612, + "nclip": 0.04577934741973877, + "photometric_loss": 0.01470732968300581, + "render": { + "bytes": 572568, + "sha256": "4d5bc9e81e0a901e67a14c2eda5882b2d0d2c0d7ef943fcf75d2c1966d006584" + }, + "round": 2 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "81856857f53fa28665f1c14fe7ac40407d84b4e893021e5dfacaa820def088d0", + "clip_similarity": 0.9696425199508667, + "nclip": 0.0303574800491333, + "photometric_loss": 0.008994323201477528, + "render": { + "bytes": 571842, + "sha256": "1ff4475afd98b45d8c8bcbde7f75b968df79ca1a71ff00a5e53cdf8d11adb64a" + }, + "round": 3 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "f1a86bc0f5e93a1a541bc7d828412799ae6b3b5755ed13b174c95705e701da67", + "clip_similarity": 0.970038890838623, + "nclip": 0.029961109161376953, + "photometric_loss": 0.009061017073690891, + "render": { + "bytes": 572324, + "sha256": "0d676318d42ce7dfe0d44f6857f247d4e3d27f21e0ad6ff2331fca36a85f448b" + }, + "round": 4 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "9ff249023bdab3ec1932eec1d3a0da91c831e67152369f682a93c6d8e26ffeb3", + "clip_similarity": 0.9676366448402405, + "nclip": 0.03236335515975952, + "photometric_loss": 0.007559685502201319, + "render": { + "bytes": 572659, + "sha256": "cc90ec970976b3176cda8a3e8eb0b598c2eb816c5d7cfa7096c55c5ab75a2c67" + }, + "round": 5 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "f1878d7c91d284ec41da0c8531fbe16b98e4c203c9cf682dd1a5da61e1c3a3de", + "clip_similarity": 0.9871110916137695, + "nclip": 0.012888908386230469, + "photometric_loss": 0.004122924525290728, + "render": { + "bytes": 577132, + "sha256": "9df09fa70f4d21a6916c84b9062dba64c7a847a5ffee44e7007fde26f3abe36f" + }, + "round": 6 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "384dc16e511781ab3ec8c830d9d2045672ba32ea1ab32efcc17e4da1b7cba162", + "clip_similarity": 0.987730860710144, + "nclip": 0.012269139289855957, + "photometric_loss": 0.003989724908024073, + "render": { + "bytes": 577326, + "sha256": "5d6115de3cda15c3c5edb77fd7842345b7748426add554fa18fe55b4588ee261" + }, + "round": 7 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "d0318cf8cfa1034e925bf24e28103f692446066b29ea40e29516472bbf7334a7", + "clip_similarity": 0.9878823757171631, + "nclip": 0.012117624282836914, + "photometric_loss": 0.0037759870756417513, + "render": { + "bytes": 576810, + "sha256": "6810f1ee430eb2b63c7694ec495fe2b52c0ff043e3e6c465c704b9e4783386a4" + }, + "round": 8 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "bdb4d0b187717c514a8481db1c855046055ef1cc18f6f1e4ebd90b1e2b81dc98", + "clip_similarity": 0.9888083934783936, + "nclip": 0.011191606521606445, + "photometric_loss": 0.0037378640845417976, + "render": { + "bytes": 576794, + "sha256": "6e5a6d132ac90ca4f581afaa6ad495434e9e8067c3ad4f6bc6cbd1d1db497647" + }, + "round": 9 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "40ff077ce3c637351b247c5b1902cef94dcfeb58c348038827fc2d737bcad1d9", + "clip_similarity": 0.987591028213501, + "nclip": 0.012408971786499023, + "photometric_loss": 0.00383252021856606, + "render": { + "bytes": 576676, + "sha256": "214d003df0c9c9708a976176766bfe1e34d13ce1a2d0fa37278180551a19f6c4" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7582, + "sha256": "b5348ae5fb98fb5110a75bb829540b2c5b0c00b45d81957e8a4c4d27d24a02c3" + }, + "target": { + "bytes": 496462, + "sha256": "2177e15ad99cb6559092b37de04aa778b13445c89598736ab67ab4f54cfa4ec3" + } + }, + "level2/attribute9": { + "evidence": [ + { + "bytes": 1276347, + "kind": "events.jsonl", + "sha256": "38e78ac3ba9131429b4ba1e7991fffae24fec67fc38f1562cfa6d55340c87618" + }, + { + "bytes": 12677793, + "kind": "run.jsonl", + "sha256": "388b97b7af39d9fcf422acad433cddf9804d69fe5d9746a17b1b14e036edb657" + } + ], + "rounds": [ + { + "blend_bytes": 646096642, + "blend_sha256": "65eb87dbb60382acde0d1755328380612a8d56d7358d1dcd8bedeaa9de220f44", + "clip_similarity": 0.9311380982398987, + "nclip": 0.06886190176010132, + "photometric_loss": 0.04209288954734802, + "render": { + "bytes": 574086, + "sha256": "10e204a80bd71a7a98f72f274c75c09a041cb5e077ce0fd1e50af31710267a6a" + }, + "round": 1 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "1a8a7ab0d907850fc59f6c454ae27cca90a9213ffcc37b09c075b32f426b7ebd", + "clip_similarity": 0.9439935684204102, + "nclip": 0.056006431579589844, + "photometric_loss": 0.009841673076152802, + "render": { + "bytes": 571044, + "sha256": "1669dd4298dccfaa757ce39fc7f7fb66843b172505215f6562072d0c365abeca" + }, + "round": 2 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "493243da91270ef58f53094a9ee876537fc6e8495b10d6624341d7e069917b51", + "clip_similarity": 0.9462493658065796, + "nclip": 0.05375063419342041, + "photometric_loss": 0.008510514162480831, + "render": { + "bytes": 570889, + "sha256": "c6b64498610d811209b02e1cd1724444ac9c2d5a4af03620af5eaa3cc9d37242" + }, + "round": 3 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "a0325a906e9c9b292df4a4de13512170efe8498e3795d8ca6660db8f661598cd", + "clip_similarity": 0.9438660144805908, + "nclip": 0.05613398551940918, + "photometric_loss": 0.008608759380877018, + "render": { + "bytes": 570975, + "sha256": "c5257707ea03978ad6f75f38255d29507b4b9846b3496897dfab85696dfc5eec" + }, + "round": 4 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "008b0c185b35c47692c3937ddf89edfac927776c9f63158519760c7f84cdf616", + "clip_similarity": 0.9444617033004761, + "nclip": 0.055538296699523926, + "photometric_loss": 0.008243502117693424, + "render": { + "bytes": 570731, + "sha256": "8b566bea54659004bd45ba2d8f50679d73218ee178e6a64f7b63dca1f872762a" + }, + "round": 5 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "93b8ece3bc1c9a7f868497a3c1192facf04da985ae485c60aead9ffb136bb89e", + "clip_similarity": 0.9668443202972412, + "nclip": 0.03315567970275879, + "photometric_loss": 0.007134303450584412, + "render": { + "bytes": 572242, + "sha256": "94a6fed6d0aee4f0cb136952aae7d279e7780784adf79afe64ec3e023313fc0a" + }, + "round": 6 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "74715f61e1e06946832481bfc09418287d26930ee9fa0d1ef97dcce44750b455", + "clip_similarity": 0.9873204827308655, + "nclip": 0.012679517269134521, + "photometric_loss": 0.0035751655232161283, + "render": { + "bytes": 576926, + "sha256": "7596968060f472fe018c3e6f988012182737ba85cb1ea77630660c30cf521cc8" + }, + "round": 7 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "83b0147bb39cf23b0b303085db533eba80450afdc5f63495536f376f842a7791", + "clip_similarity": 0.9876059293746948, + "nclip": 0.012394070625305176, + "photometric_loss": 0.0035959018860012293, + "render": { + "bytes": 576865, + "sha256": "61ed38e1bd458ef76a09a872abc0a41166c5e0fda121a09ba2a3659ba676016b" + }, + "round": 8 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "01eddafba8a08b92cea3d25023d646ed144c61f16067accc24429cb2d138e322", + "clip_similarity": 0.9885677099227905, + "nclip": 0.011432290077209473, + "photometric_loss": 0.0034589776769280434, + "render": { + "bytes": 576751, + "sha256": "7e43417fb69bd19e1c42fa77859a41c8a2cb3816814277dbc3dccb5e7e6db016" + }, + "round": 9 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "b65a33473c91499cee0a7fd21040d8254e02769eae479ce4a4de586136cb1f1e", + "clip_similarity": 0.996465265750885, + "nclip": 0.0035347342491149902, + "photometric_loss": 0.0007936670444905758, + "render": { + "bytes": 495582, + "sha256": "ab971bde432f76467214da31dc9e30f7cc8df2c07757d1ecac6fc41cb15bf4c9" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7592, + "sha256": "072b3a428c33b2c6746f69c8708efca58a82f7d66d311c0b6a517e2f11e62503" + }, + "target": { + "bytes": 496462, + "sha256": "057562178eba0d2cf993227378f6f93c2689eda870a060cc836edce85f892eb0" + } + }, + "level3/attribute1": { + "evidence": [ + { + "bytes": 1924773, + "kind": "events.jsonl", + "sha256": "7cdd1d853afc32fab164046ad650f136846d170f0bbdda839986cf8a2acd37ba" + }, + { + "bytes": 12456131, + "kind": "run.jsonl", + "sha256": "649267b8bd1af6d394b76f78e1a949c8b967d571cf3785532873067b5bcb6a5c" + } + ], + "rounds": [ + { + "blend_bytes": 943257481, + "blend_sha256": "d196ab227eeb0d9bbf8aeb5a3fc21d2c3fc9160706f86b5ec83725bb71563e9e", + "clip_similarity": 0.9098252058029175, + "nclip": 0.09017479419708252, + "photometric_loss": 0.07946688681840897, + "render": { + "bytes": 437856, + "sha256": "cf768dc71d60754b6b47878bc3083d7ee4fb3a134a47b7308f2555911bc8612d" + }, + "round": 1 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "68d96abfe206c419a9b0f5ac0502934cfcbebe752a21bcdb1622e831f0a734f0", + "clip_similarity": 0.9217170476913452, + "nclip": 0.07828295230865479, + "photometric_loss": 0.07681163400411606, + "render": { + "bytes": 445514, + "sha256": "90222ee64633245062ec6900dffcc4a661528c569a4be1553c857749917d774a" + }, + "round": 2 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "d7f754f89eb59ce922735307b77d0b3f3f2f75b1c86b73e210e96ae022cb5de9", + "clip_similarity": 0.6607498526573181, + "nclip": 0.3392501473426819, + "photometric_loss": 0.3766263425350189, + "render": { + "bytes": 316640, + "sha256": "719cb3b1a6c9fa887e9acd06dff160b6ee7dc236e0babab9485db4fefda268af" + }, + "round": 3 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "9daa282a29611c0727e3079e84d9647230e11c910663fa98fbec755cb53f3c59", + "clip_similarity": 0.9430404901504517, + "nclip": 0.05695950984954834, + "photometric_loss": 0.05672503635287285, + "render": { + "bytes": 440654, + "sha256": "00c1aa47d6e554d689a6747d23871be8e903c0e9135c2a1b5272cfce3417f9ab" + }, + "round": 4 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "85aea62426e1edb97eac76a62c93485e15bfc0a7aca1303dceb7f0ec16069363", + "clip_similarity": 0.9597296714782715, + "nclip": 0.040270328521728516, + "photometric_loss": 0.04194614291191101, + "render": { + "bytes": 441536, + "sha256": "6cf37b6dac65ed8075ef15ee2c1918cde72bbfe506b2d806494eb1a33fcc7f90" + }, + "round": 5 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "cc2dfa99a5a9935af1116993ddc9a075b53e8dbe3f33ed6e8c4b0028a63538b2", + "clip_similarity": 0.9801073670387268, + "nclip": 0.019892632961273193, + "photometric_loss": 0.027477452531456947, + "render": { + "bytes": 439684, + "sha256": "83f1a404ed94adcda1685a3056d0684b98e0edc4cb1fa0576fcdc8780ff98c30" + }, + "round": 6 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "a96d54d0a9081d8e217e96d05ea79e7684589ec80d83960c428045802c771bbf", + "clip_similarity": 0.9802656173706055, + "nclip": 0.01973438262939453, + "photometric_loss": 0.02749941311776638, + "render": { + "bytes": 439822, + "sha256": "2b962ad059b2d2267fd57a258eef56a1ae0ef55c87d2dac8dd05e57ff723b473" + }, + "round": 7 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "d8e0cda6fa2d2c2fe3b7083ef94459d7ce48d87d1d07a0786419b517c277cdc2", + "clip_similarity": 0.9807744026184082, + "nclip": 0.019225597381591797, + "photometric_loss": 0.020207971334457397, + "render": { + "bytes": 436411, + "sha256": "058865231de1aac153469994e240deb6bebeaf37a9f556236c136ccd5506fe3b" + }, + "round": 8 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "0afd50da6af25a3b0a7a0ed2bfbbb164a7d6327260601a5cfbde615fcb6b46cb", + "clip_similarity": 0.9810705780982971, + "nclip": 0.01892942190170288, + "photometric_loss": 0.02021031640470028, + "render": { + "bytes": 436315, + "sha256": "adacdbbf9d6138369df8e09b1c6cd6ff71221d7b2c64d6126c2796e4471db7ee" + }, + "round": 9 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "62f5babbc4d88c7ae4c623ce1d2662d63f347ea71ec83cbd7488a952b0f6b882", + "clip_similarity": 0.9806057810783386, + "nclip": 0.019394218921661377, + "photometric_loss": 0.02021102048456669, + "render": { + "bytes": 436030, + "sha256": "2687228629f47deba7af61617bc8dbcd6bb33fba1336cbb6a65ecf085ae68d66" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7497, + "sha256": "4aff82f8cd193a86a1f54a03287777c24817cb4bc52e0e7ee3103279f5fc1c3d" + }, + "target": { + "bytes": 424605, + "sha256": "203bc0d5b3cd6bff06e19baa56c56370b5a8da9155b5b0ee3e8598a0886318f0" + } + }, + "level3/attribute2": { + "evidence": [ + { + "bytes": 1648840, + "kind": "events.jsonl", + "sha256": "1796cf588ab2454593ce4d759892963c85228b9512a5bd8abaa1161b9129471d" + }, + { + "bytes": 11476592, + "kind": "run.jsonl", + "sha256": "30a6423455572a7fe343aabb62794392093eb06b2101372fdd2a7fe7edcd2610" + } + ], + "rounds": [ + { + "blend_bytes": 943257481, + "blend_sha256": "f64441abd6439ac1d2a105519fcdaf483db5e2618b8052d2ce54d5a34ff5bce5", + "clip_similarity": 0.8939650654792786, + "nclip": 0.10603493452072144, + "photometric_loss": 0.07529605180025101, + "render": { + "bytes": 511612, + "sha256": "a6c4d0d9d2448febf5f155cb60ac241f0811eed9cf6a415ae031d4a911d7697d" + }, + "round": 1 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "24c0a33d4b340a2fd3391612525871605a352abc4f5f4f5a2afd123cbaeeae81", + "clip_similarity": 0.9033817052841187, + "nclip": 0.09661829471588135, + "photometric_loss": 0.06520041078329086, + "render": { + "bytes": 512949, + "sha256": "6528a4a2567cf905f829616137cc7bc0c783c9a4091615814045aa17df523d54" + }, + "round": 2 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "bd30a0600b2595792d3456e96ab55bfc94f1f85c2f196a95c58c5677be88ef74", + "clip_similarity": 0.9098818302154541, + "nclip": 0.0901181697845459, + "photometric_loss": 0.07429742068052292, + "render": { + "bytes": 503387, + "sha256": "f19bec2f145aadc6c83b64f02671186d267d4f682e4ca839295d3d519e23188e" + }, + "round": 3 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "6da5e0d68be1c841f0e9537c1d685d8874639191e62b5788ac4d13a92b724a13", + "clip_similarity": 0.9416781663894653, + "nclip": 0.05832183361053467, + "photometric_loss": 0.05184561386704445, + "render": { + "bytes": 511333, + "sha256": "1f61013aff2ec53764f84271a4cbfa5546c636093c7f12b7feed20ca83f6b457" + }, + "round": 4 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "6a6953527030927b5051b0844cf92c544215fbb57536195ce7efdfb198a1080a", + "clip_similarity": 0.9510102272033691, + "nclip": 0.04898977279663086, + "photometric_loss": 0.03256066143512726, + "render": { + "bytes": 507821, + "sha256": "f865c47e2403be7539d29d041ce5bf8034b6a6de233bd243a54e11abbd5508c5" + }, + "round": 5 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "ccebe27c27c0d8694e818b5fea0212a9d676575fb181758e5820a94b272cde5a", + "clip_similarity": 0.8367416858673096, + "nclip": 0.16325831413269043, + "photometric_loss": 0.19988812506198883, + "render": { + "bytes": 296806, + "sha256": "cb06bca484dbcd440dee336127c540a0f93954da6feabcb504771aa7425fb523" + }, + "round": 6 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "d2b09aea2bd718d2c40697217d2cdde50fd038890477e2faabfc28fd2cf7850f", + "clip_similarity": 0.9569584131240845, + "nclip": 0.04304158687591553, + "photometric_loss": 0.03064643405377865, + "render": { + "bytes": 504544, + "sha256": "32a966af73f2e7399fef60937c27651683e0ba13668cf194d513c9f61dacc506" + }, + "round": 7 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "211ac3fddfb7a4282969cfef6ca9ede8acbc177ca73160ea0b8c7fb11c7d1e13", + "clip_similarity": 0.9623216390609741, + "nclip": 0.03767836093902588, + "photometric_loss": 0.039537735283374786, + "render": { + "bytes": 510452, + "sha256": "af1cdcf0edf152c31ad1983be1f0e9b858ab66307ae827820d63170e3e466d40" + }, + "round": 8 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "0a32a779c25755655e37fa85d78e4e232b5e8e5553a942a968e34c0ad1a688ee", + "clip_similarity": 0.9504269361495972, + "nclip": 0.04957306385040283, + "photometric_loss": 0.038483042269945145, + "render": { + "bytes": 508909, + "sha256": "ee29f6f931c0849431ba91817002b21646b80c9b99c58b86430aa64aaee33513" + }, + "round": 9 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "287a36e751701e2ed584fb60b2521dd41e9199d8ed0a2db2c0cec09b1dbc22dd", + "clip_similarity": 0.9837775230407715, + "nclip": 0.016222476959228516, + "photometric_loss": 0.012915946543216705, + "render": { + "bytes": 497559, + "sha256": "913b594d0a51def59197e77da626dfcf6c1d8d3a63884fde437992d6e44e0cc3" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7576, + "sha256": "15d6c6bba6b98581aebffe05bba7d8d2527675719507cea1edf87bcf3ffdfe30" + }, + "target": { + "bytes": 424611, + "sha256": "b5b1526c9fde7aecd35f842598627b63172338284d69aaa36d6a40649200ca30" + } + }, + "level3/attribute3": { + "evidence": [ + { + "bytes": 1644760, + "kind": "events.jsonl", + "sha256": "2276cbb35c03fe4e10c8fc0c2ec46307a73489d0b6e3a77ac374347bd9391bbd" + }, + { + "bytes": 10571760, + "kind": "run.jsonl", + "sha256": "32c1768926f7f9c6eb0b093daad72ffaa28cfc0edc5abf05989ec091853623d1" + } + ], + "rounds": [ + { + "blend_bytes": 943257481, + "blend_sha256": "905d034f447fa149ba400181b0b0eb11eef8eed7c31f9915e600919c7ab825f0", + "clip_similarity": 0.8677472472190857, + "nclip": 0.1322527527809143, + "photometric_loss": 0.09606408327817917, + "render": { + "bytes": 531126, + "sha256": "8ec0f8b94bb97d48e9c3af48113a182cdb61caa45d3a0f689e02fce1f087b2e9" + }, + "round": 1 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "a8cb2f5caf6ceabad2e6efaefe58406c8af74faa6a3584c5b5683fd47762fca9", + "clip_similarity": 0.8839718103408813, + "nclip": 0.11602818965911865, + "photometric_loss": 0.09201572090387344, + "render": { + "bytes": 506872, + "sha256": "db26a334e5aefac3b30123caf3084e721940a51b7900db2505e11e73bc942ee3" + }, + "round": 2 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "e1647875e6179b6bf709f808044c22c5e94ba0beffb9eadd1e349525fb3a975e", + "clip_similarity": 0.9038984775543213, + "nclip": 0.09610152244567871, + "photometric_loss": 0.056030403822660446, + "render": { + "bytes": 517855, + "sha256": "d534c627ffd56ccaa3722914c59a5b50d069fad68da9e9227f0d35c18daa31fa" + }, + "round": 3 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "72d6a98fbd8f7346ea2662591e3422a99c59bb7a51fbf8f32584c5fa55f6b8c3", + "clip_similarity": 0.9551177024841309, + "nclip": 0.04488229751586914, + "photometric_loss": 0.04528190195560455, + "render": { + "bytes": 554013, + "sha256": "afaf64992f3693dde1ea564a7ffa53e0ec6d54601c3ce3e7ff2711fa3c6842f1" + }, + "round": 4 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "89eb345c12ed2c5f55ab43bcbccfae3f09661f9862c90fc1f52c258d977dca40", + "clip_similarity": 0.9572995901107788, + "nclip": 0.04270040988922119, + "photometric_loss": 0.024362504482269287, + "render": { + "bytes": 550803, + "sha256": "8fb0524191c1fa0ab09a05fc94292bbed3dfa330d37ae6aae0e5781f2417096c" + }, + "round": 5 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "a648d1134f65df7a2c7a86ff449c27667035115f6ae41daa02d8a2d17a0c67f8", + "clip_similarity": 0.9733030796051025, + "nclip": 0.02669692039489746, + "photometric_loss": 0.013502489775419235, + "render": { + "bytes": 531895, + "sha256": "5b565a0695486f1c49762938d3a0c6b53fd608b3c73897bc9aaf5628f026687e" + }, + "round": 6 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "6a8ed76a3c9777d077bf60bfe84dcb5be3342a98bd1788558f7237c91387db3e", + "clip_similarity": 0.9710009098052979, + "nclip": 0.02899909019470215, + "photometric_loss": 0.015630224719643593, + "render": { + "bytes": 532191, + "sha256": "4b996e79216f6a6a7c4593b95168ebe1e8d89533838987c364a677be813d6126" + }, + "round": 7 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "f02bc9c2cd08e19be10ae26b8c295d4f492539669ba17284c8316714fad84c87", + "clip_similarity": 0.9763938188552856, + "nclip": 0.023606181144714355, + "photometric_loss": 0.0294965710490942, + "render": { + "bytes": 543864, + "sha256": "82f2ad0e099d4d62fa6bd6050cf4096883ebb3848aa096a5d6dc54f62081f942" + }, + "round": 8 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "ebf489166ca86013939c9563ae1621a7574bef760e81a73ead41e3d71490e7fa", + "clip_similarity": 0.9807580709457397, + "nclip": 0.019241929054260254, + "photometric_loss": 0.01762007176876068, + "render": { + "bytes": 493951, + "sha256": "15543161db39db67314ef96bc5cdbc62dec33d0b7f0aadb0532914f2c6c654fc" + }, + "round": 9 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "2bed5b84e5832e3b1f77c5f708610020e17fc1c99abfa551d26af761ef5eff5b", + "clip_similarity": 0.9846137762069702, + "nclip": 0.015386223793029785, + "photometric_loss": 0.014947629533708096, + "render": { + "bytes": 452720, + "sha256": "8792f280a74d5241396fd7258a97a4c3764a27ae2fac012057ce47c0b041c0dc" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7579, + "sha256": "6b06718714b446687104cd3fedaaf1013858fbd898226eac493653e184a89e8c" + }, + "target": { + "bytes": 424612, + "sha256": "741983290631f850e8e7a011679328f9e6da5545a671c23d73c17beb88cae346" + } + }, + "level3/attribute4": { + "evidence": [ + { + "bytes": 1478020, + "kind": "events.jsonl", + "sha256": "f36c9bc8498f565c429033c4a5bdf60a965eed0c49b1a980c3c4e3487c362541" + }, + { + "bytes": 10564972, + "kind": "run.jsonl", + "sha256": "67faba27e77874d461fc6688d2d315a5f0925c5111b6c93ef3d1451b95cf41fb" + } + ], + "rounds": [ + { + "blend_bytes": 934082247, + "blend_sha256": "5131d3c803ea6d5620d173cb3659b1b33bb51e0fd6bf6e9af218031f8f0495ee", + "clip_similarity": 0.9420864582061768, + "nclip": 0.05791354179382324, + "photometric_loss": 0.0750909373164177, + "render": { + "bytes": 518774, + "sha256": "56af8f831a16e0affc7ce9696c9c4fb253573530b80ed1b2540951245d30e4e5" + }, + "round": 1 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "b59d30f43b0d5ef05a4f3e4fe7a1ca6564b731180f5ce5e2dd3474e68eb1a872", + "clip_similarity": 0.9318283200263977, + "nclip": 0.0681716799736023, + "photometric_loss": 0.0782821848988533, + "render": { + "bytes": 541032, + "sha256": "2d4b4e7b495c075015e7f8f72db9f0bf109b0ba9acd92e58870a1d9a383e6068" + }, + "round": 2 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "4183228afb42c0a971a413e1ee40e9a7fb35e36834375b1fbc5fc3e798590e90", + "clip_similarity": 0.9207460284233093, + "nclip": 0.07925397157669067, + "photometric_loss": 0.06497704237699509, + "render": { + "bytes": 521347, + "sha256": "db83000464d25db6c197fb7fd496f2a4038006152f2da01e09d7152c713a613a" + }, + "round": 3 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "7926b755715f0652bd793937d8d2abb4c6ec64842d13d3a5222b9e5a90560686", + "clip_similarity": 0.9671326875686646, + "nclip": 0.03286731243133545, + "photometric_loss": 0.05156431719660759, + "render": { + "bytes": 524447, + "sha256": "01a129ff6fd46e1324553977743f41ddacb44575e4e7262970d5f92967282cbe" + }, + "round": 4 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "a5bb81a44cd89e75af3107de310cde23a9312bb9af08278e4c18fce24a002850", + "clip_similarity": 0.9705365300178528, + "nclip": 0.029463469982147217, + "photometric_loss": 0.0347987599670887, + "render": { + "bytes": 533853, + "sha256": "9147bfc573837e397ccd76efbd4f8d5882405659c76ce9ba49c6579e4c233d57" + }, + "round": 5 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "aeeb801dcbbb6c982c0074e4344fd4a7380c9aee5ddc7709bbd495be51f92f8d", + "clip_similarity": 0.9707145094871521, + "nclip": 0.0292854905128479, + "photometric_loss": 0.034984197467565536, + "render": { + "bytes": 534942, + "sha256": "fc0ded05e44e3a0e9f4dba8fb4d6c103ac224824a00336f018d29a6873bc5388" + }, + "round": 6 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "be5a3763f00350a309a71ac620098d52dc50dab22b3963c0e823f4661569d376", + "clip_similarity": 0.9824424982070923, + "nclip": 0.017557501792907715, + "photometric_loss": 0.013661672361195087, + "render": { + "bytes": 530968, + "sha256": "8b33343c64def130b047f44a14aa705238158894437019a1c8f965daab8a01bd" + }, + "round": 7 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "038ba840c2453f5af9fd5949791f5186394b00af55fc2e0509cb2862ecd4f164", + "clip_similarity": 0.9853713512420654, + "nclip": 0.01462864875793457, + "photometric_loss": 0.01268050167709589, + "render": { + "bytes": 529286, + "sha256": "d07faae2bc18fcf89132a62e53f65ba07a9330dbf901fd627c125269c575ac54" + }, + "round": 8 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "466923ada6fc70c35d2bcd255b148e48ee0391bf1f8431812a6e4c5c9bbed192", + "clip_similarity": 0.9894673824310303, + "nclip": 0.010532617568969727, + "photometric_loss": 0.009694959037005901, + "render": { + "bytes": 467846, + "sha256": "3dda0ce6e66793f067744c83fe968cd89e8ae99b135944bd14570583a073ab0c" + }, + "round": 9 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "7d59841ce102985dd971eb78b784333422990ae7363f7b4af056f5396faa6327", + "clip_similarity": 0.9926517009735107, + "nclip": 0.007348299026489258, + "photometric_loss": 0.009027284570038319, + "render": { + "bytes": 419004, + "sha256": "bb42dd9c8331b8da388256afd9d09bf9991ab02474f7ac55b719a4d6f2fba3ce" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7576, + "sha256": "0d6920e9f4a598cc948be29abfcea38d9a160c2c8a1d77039f81b47005f9609b" + }, + "target": { + "bytes": 424822, + "sha256": "2dfaea1568f9cfb0400600c874e214dc9b15578708571d1d9897605330ffc928" + } + }, + "level3/attribute5": { + "evidence": [ + { + "bytes": 1479032, + "kind": "events.jsonl", + "sha256": "b15c7851740791b688efcb909bb256af0780fe0b5633af4f7b7e9d8ae3b8c91f" + }, + { + "bytes": 10176930, + "kind": "run.jsonl", + "sha256": "90735fcf49a82c05fe534f3abf446b9bdfa9e61ab50fe60d1f951136cb135d6b" + } + ], + "rounds": [ + { + "blend_bytes": 940184532, + "blend_sha256": "ae784d052246c405391e7e71604c7927d2cfe04f5155568dc003d3894b635d93", + "clip_similarity": 0.8690069317817688, + "nclip": 0.1309930682182312, + "photometric_loss": 0.0782204195857048, + "render": { + "bytes": 725607, + "sha256": "544270131b52b81beaa2e57be047b15a1db9678a809c4712edc32cf42901a1b4" + }, + "round": 1 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "436800aaa9c10a414dd635758d90906e0d6ff20d33247dea3fbd66fa931caa3d", + "clip_similarity": 0.9351325631141663, + "nclip": 0.06486743688583374, + "photometric_loss": 0.07839251309633255, + "render": { + "bytes": 611342, + "sha256": "590769f6693e689a84f807c169c5a298b43d39875f4f5e56dbe6740600938cd1" + }, + "round": 2 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "86d3053b64c349574ca2d291e19d7a8fedd27cb152eb5a5a5b66907d0c6ecbfe", + "clip_similarity": 0.9289098978042603, + "nclip": 0.07109010219573975, + "photometric_loss": 0.07732649147510529, + "render": { + "bytes": 599245, + "sha256": "b564d25de21faa10b32c848145a308241cdf9a54934bf1914a65e35fa9f67a0f" + }, + "round": 3 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "682440f54704db53d5a91caa8e74ae6669f53f4a08665df4583b945f31383a2f", + "clip_similarity": 0.9625266790390015, + "nclip": 0.037473320960998535, + "photometric_loss": 0.0576217956840992, + "render": { + "bytes": 594935, + "sha256": "8bf4ee786c6a1eb1e440d449aa606b584c466ead3484c2c69e8793b1c5d31f93" + }, + "round": 4 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "2d68ae4a509946e865a2cba319a8ee268dcb5cad6c00c4591b8e64be6ef222f3", + "clip_similarity": 0.9493187665939331, + "nclip": 0.050681233406066895, + "photometric_loss": 0.032019589096307755, + "render": { + "bytes": 610052, + "sha256": "a9dd9a766aa0d43a5851ce285b046507dbca81a34fe107c7b943c2a45d35bf65" + }, + "round": 5 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "b5a579ead8aa971a546b0a709b91800a3936b2bf9336289a7f0807255f574deb", + "clip_similarity": 0.6391876339912415, + "nclip": 0.36081236600875854, + "photometric_loss": 0.3678968846797943, + "render": { + "bytes": 235433, + "sha256": "0401ab41438e8a708fa67a9b89dff04305d5bbc8d58979885dbaf552f107fb7e" + }, + "round": 6 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "2217799d1aa027f4e4720dfc8532f61ee3f5619638bb5d0fa8838ebe574c7cc4", + "clip_similarity": 0.9608088731765747, + "nclip": 0.03919112682342529, + "photometric_loss": 0.041936907917261124, + "render": { + "bytes": 605091, + "sha256": "1de3519a3b0fdc4ba96da6c1df9844220789c1c91475526aa9bb00c53561e0de" + }, + "round": 7 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "1a8dc07690a76e19413bb86a3e2cec684f426688f276f7a61b3517486e286ace", + "clip_similarity": 0.9751001000404358, + "nclip": 0.02489989995956421, + "photometric_loss": 0.024898789823055267, + "render": { + "bytes": 602765, + "sha256": "5b9bb6f16b0fb1f2d9abaffc4c52ff83b1c2f4aa321676b874323215a2c5472e" + }, + "round": 8 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "cb960c4f119e78b50f373c26297d7c4c75f033e2b483f6cde99874f07f744cbf", + "clip_similarity": 0.974507749080658, + "nclip": 0.02549225091934204, + "photometric_loss": 0.020625248551368713, + "render": { + "bytes": 603952, + "sha256": "b3e352a5da9f0e21b129867c2465c884f9019cb4a8e8ee2c5ecb7ac66ac7a311" + }, + "round": 9 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "08aeae231f2be37bdd5058a86a8f4ffe44636b34d33122ca0ecbe9ed100cb22a", + "clip_similarity": 0.9755886793136597, + "nclip": 0.024411320686340332, + "photometric_loss": 0.027274271473288536, + "render": { + "bytes": 596760, + "sha256": "dbed381c401543c62e4f26cddbe2e50ea08e1330019ffe940a771b7a4602c996" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7576, + "sha256": "cdf73038198bc88a3d5dcced62db3f1375a6bf0ba242800ee40708b3abb8ccb7" + }, + "target": { + "bytes": 432648, + "sha256": "08f94e6af9f1bd74f52a7302f7673da3ebfb318437d4a986780158967944635b" + } + }, + "level3/attribute6": { + "evidence": [ + { + "bytes": 1479058, + "kind": "events.jsonl", + "sha256": "ce20e19c7ddf51a6b2eb085a1da7a1b83458dab12bd8ef86db33c85d81071ed2" + }, + { + "bytes": 11625245, + "kind": "run.jsonl", + "sha256": "94a1575055e8e26390a7b84caee27fc9edb22d235738eb63680e7f3e59f80f0f" + } + ], + "rounds": [ + { + "blend_bytes": 940184532, + "blend_sha256": "2e10801c0b2e77211b33afd7492f5638fe12da4bdcbff624512d941e399becb2", + "clip_similarity": 0.9185791611671448, + "nclip": 0.08142083883285522, + "photometric_loss": 0.09303504228591919, + "render": { + "bytes": 596763, + "sha256": "e3355b29953875aa82ba482b63eff6674fd9a763dd457a5033f755cf8d455cc7" + }, + "round": 1 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "f39c0020040f111c6b53e1c3d3f076cac0bb9bfd5a14ad7f4f78fd9874f3b8d3", + "clip_similarity": 0.897698700428009, + "nclip": 0.10230129957199097, + "photometric_loss": 0.08139199763536453, + "render": { + "bytes": 586908, + "sha256": "a09c0d7bc6d9d63406d10c19182531621b730dfd11bcc5ee044686110ddfeed9" + }, + "round": 2 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "ea5434425a94e0f42114ecb237b2c2b2671c57ff7c8e08f88fcfa16a79c2a30d", + "clip_similarity": 0.8258225321769714, + "nclip": 0.17417746782302856, + "photometric_loss": 0.05298319831490517, + "render": { + "bytes": 577033, + "sha256": "5e08f00ec3785c7ca030cc223613ef4036a17baacd5eb948803fada83323064e" + }, + "round": 3 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "bed818ddacbf1a2964f2a99c6a194a94fbeee9408e9e568a9ed444b0dda2366d", + "clip_similarity": 0.9696477651596069, + "nclip": 0.030352234840393066, + "photometric_loss": 0.04301448538899422, + "render": { + "bytes": 620402, + "sha256": "21b69c56fcece4b31aa6fc34627614cc3161c5af4e307db0af2dfb18bd940118" + }, + "round": 4 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "f724b2370bccdc87fe1f4b5e563f92fef051a55082fb0542b115acf86998d407", + "clip_similarity": 0.9585986137390137, + "nclip": 0.04140138626098633, + "photometric_loss": 0.03180944547057152, + "render": { + "bytes": 617964, + "sha256": "2274d4e087ca73732458ec1e23069748cc87676a1a29ac464375d0e89b0e7d60" + }, + "round": 5 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "ccd2b0edaea8df1f981e3b1606a0290f52c08fdee96689c36044e61dbadbc82c", + "clip_similarity": 0.9793316125869751, + "nclip": 0.020668387413024902, + "photometric_loss": 0.0314413346350193, + "render": { + "bytes": 618033, + "sha256": "61cef4e090e4f8c09db87e51befbe0cecbd94d9d433d7cf5ca2ba35cbc8642b0" + }, + "round": 6 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "3952c0bb2e00da4421838cbbca61b902fecee0c146e344b4871de27ec26a9ef9", + "clip_similarity": 0.9866102933883667, + "nclip": 0.0133897066116333, + "photometric_loss": 0.025586744770407677, + "render": { + "bytes": 609871, + "sha256": "7927b12414c1d691efcd7c95b554b75c14ea12aadc7b184f173f240685f50841" + }, + "round": 7 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "eae5c2753a35d03c775057a9845b22d65a5fdbd8e0bb2a679a60c60c4a0d5ea8", + "clip_similarity": 0.978234589099884, + "nclip": 0.021765410900115967, + "photometric_loss": 0.015896320343017578, + "render": { + "bytes": 615623, + "sha256": "a179f352e23d8d71a1367c6a9794356ce1898634880515da3548060be3a54687" + }, + "round": 8 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "6a0e7a867c3065fb3d3e389da6e7eb6d04b5026e512a5f67bebde3b89a89866c", + "clip_similarity": 0.9787060618400574, + "nclip": 0.021293938159942627, + "photometric_loss": 0.016108734533190727, + "render": { + "bytes": 619480, + "sha256": "72c5cd08fd24387ade745db035260bfbfaa1cdd95e5c73bfe57706c9b74037ea" + }, + "round": 9 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "c86bf1322ded94e74006940577614957b3245386a858ea762fe8e4bc1dd6b48b", + "clip_similarity": 0.9853477478027344, + "nclip": 0.014652252197265625, + "photometric_loss": 0.016418581828475, + "render": { + "bytes": 544474, + "sha256": "3ff9197ce859982919bbf227f0ca2ac8753bec23b0388b094d3ace6a1d32be1b" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7575, + "sha256": "398c569a73d340ea5ad60eae617166b1e1091af91d2d945450ca8ff6a9137ebe" + }, + "target": { + "bytes": 432652, + "sha256": "86b764ac22fe162549f3d5a2995e0e8160e836cf6abac763c120f9de283a8089" + } + }, + "level3/attribute7": { + "evidence": [ + { + "bytes": 1269312, + "kind": "events.jsonl", + "sha256": "dfd33829af7f97efc7fcd47ed5bf3ec656e93917a5564666a08acc4f20977b38" + }, + { + "bytes": 11695178, + "kind": "run.jsonl", + "sha256": "1f31b785d089e8b7ca48f8ce7c70038405f4ed2f9c7356534255ee91a682beb2" + } + ], + "rounds": [ + { + "blend_bytes": 646096642, + "blend_sha256": "c5b234ea7203c93af5042c5c878d5f6f7beae407cbab25871d636db8fa6be022", + "clip_similarity": 0.9302181005477905, + "nclip": 0.06978189945220947, + "photometric_loss": 0.031431227922439575, + "render": { + "bytes": 561924, + "sha256": "1ab8f237ca9e93dcefa0e8ddcc08787b56c9b8df04b92336c074eebb72e80ab7" + }, + "round": 1 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "c55d8ea788b92ddf19b01aeec67e00867290760b332ae07c9f2e91ba71189edc", + "clip_similarity": 0.9107853174209595, + "nclip": 0.08921468257904053, + "photometric_loss": 0.041224125772714615, + "render": { + "bytes": 573209, + "sha256": "0957b14f39708d867c8530fc7d6563e88eec7ac50edb856ad915ee919b335c72" + }, + "round": 2 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "729f7895832f179b9c0d9e2c7aa9b62f79bc86e2103f59cf69dcfc1be05964fe", + "clip_similarity": 0.9154404997825623, + "nclip": 0.08455950021743774, + "photometric_loss": 0.04107638821005821, + "render": { + "bytes": 573156, + "sha256": "f242e39a8aa5c30cb5bd40230ed9feb7b67d2beca41b6fbd9259381e02867e9d" + }, + "round": 3 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "717589dee89c45554a3207385aa30280e85ee135cc543fc55b48c57513203a1e", + "clip_similarity": 0.9029605388641357, + "nclip": 0.09703946113586426, + "photometric_loss": 0.04625360295176506, + "render": { + "bytes": 576957, + "sha256": "3c69929f5d050ab8ebec71a9f38ccb912ee71048e3fa6f9b11a29a144a78a9e4" + }, + "round": 4 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "07df53ab9336913bfbf3885699a14b3ef31556e8bcf357481c8da501c06a7920", + "clip_similarity": 0.93320631980896, + "nclip": 0.06679368019104004, + "photometric_loss": 0.03137661889195442, + "render": { + "bytes": 562350, + "sha256": "6d3fdb319961b2d09d518e82ad8e7b40339d922980015ab024a1b8c85f147a60" + }, + "round": 5 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "db8a3071796bcad90abc7146f43a36cf3cf8f43d225a4dc807a873f764de51e5", + "clip_similarity": 0.9286768436431885, + "nclip": 0.07132315635681152, + "photometric_loss": 0.031444914638996124, + "render": { + "bytes": 562255, + "sha256": "1e0eaa0c90d0a816acc9dbab9d3521ee332078c553f3a5e5caaa212e066d9caf" + }, + "round": 6 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "c2bcd628bd2c5e9b8e01c93c5dd247f78aa311810fdaced8ef973b6184108f8a", + "clip_similarity": 0.9289407134056091, + "nclip": 0.07105928659439087, + "photometric_loss": 0.03151537477970123, + "render": { + "bytes": 561958, + "sha256": "d82708fe11deef1285da6936f2ce160c076b39e8c4299d94417dd03e6b2f016e" + }, + "round": 7 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "b9fe704692f78541017edeef8e0ea7f8cbc8a8087a98ba93a87b13b16ebfc36c", + "clip_similarity": 0.9298222064971924, + "nclip": 0.07017779350280762, + "photometric_loss": 0.031483396887779236, + "render": { + "bytes": 561981, + "sha256": "1493e7f369bc2e3ee7542a741af34542f712324f2d8b6ac3496fad6d9ae574a0" + }, + "round": 8 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "6188d265a46217c1ec37e5585757e8e65d5ae68f99bbb036f10c7f75affac16a", + "clip_similarity": 0.9289346933364868, + "nclip": 0.07106530666351318, + "photometric_loss": 0.0314057394862175, + "render": { + "bytes": 561811, + "sha256": "a6289e687902b5c24afb3e7940bba8e8934b69fd0256e3f6eac1f14e3286f0e9" + }, + "round": 9 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "d0232cb99e6958c0492b7989bbaee634e5e5c16d19a4b1b4740e1af70a656205", + "clip_similarity": 0.9050087332725525, + "nclip": 0.09499126672744751, + "photometric_loss": 0.06344757229089737, + "render": { + "bytes": 562445, + "sha256": "93087024fa850a1d787f49e40b31525a5492de66254bfdb500cc64860590a017" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7574, + "sha256": "4af396ec7985842a179dee04858173ec9464fe80e3d55b672b83198f1a2cb56d" + }, + "target": { + "bytes": 496540, + "sha256": "0ada6e89b53c28b9036418def514b728dd0eeb664eb3f7a1ca9496a5e8ef93f1" + } + }, + "level3/attribute8": { + "evidence": [ + { + "bytes": 1271443, + "kind": "events.jsonl", + "sha256": "87fd893f7271ec4a898d328045bc20437c3b30a1a6a4e750347e98348abf7d4f" + }, + { + "bytes": 10611931, + "kind": "run.jsonl", + "sha256": "3b826dca03ab0600527cbfe35e138db4de25905b229fdf76e70f3abe9e000c68" + } + ], + "rounds": [ + { + "blend_bytes": 646096642, + "blend_sha256": "4fe0b6bba33e7fa419a69d12128a98ee1994479d899de320e9dea834b8afc3f2", + "clip_similarity": 0.9057605266571045, + "nclip": 0.09423947334289551, + "photometric_loss": 0.032523829489946365, + "render": { + "bytes": 589609, + "sha256": "eb3568f0195d2343b44612b6d1d2ea0f53864ca5c269a3074f63348f8cbd9cd5" + }, + "round": 1 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "f388c0a9e5d698394ecc073345a16242af2a7dc52abf41a8501c80ac65773eac", + "clip_similarity": 0.9555370807647705, + "nclip": 0.04446291923522949, + "photometric_loss": 0.042556773871183395, + "render": { + "bytes": 602319, + "sha256": "c55bf8a9e8728cfae6786857be7fe9f1aa1318f12366a3a095897e4dcaceb325" + }, + "round": 2 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "bba62674cd514c9c9030cdef5b93fe48e32bd548a8a391b8164024609e4cd45e", + "clip_similarity": 0.9614916443824768, + "nclip": 0.03850835561752319, + "photometric_loss": 0.030976975336670876, + "render": { + "bytes": 592945, + "sha256": "b934e7692abfe450fe9db1292e75e412e8166fd73374bf36cc603a286c3cadb0" + }, + "round": 3 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "2c8e2743e6080b9929ec51fb092b2e3cdbcc07a2c02ee352410f0f8d24d36a09", + "clip_similarity": 0.9753873944282532, + "nclip": 0.024612605571746826, + "photometric_loss": 0.025764750316739082, + "render": { + "bytes": 588589, + "sha256": "fdff2c4b91ef2532c3573eafff6063fb2427e769f08c1f822b4a8a1ff0ca20bd" + }, + "round": 4 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "567284551301b9fa61ea4b8db4d7f93d6a4a943efffb4e61ab40bc30878dffca", + "clip_similarity": 0.9746690988540649, + "nclip": 0.02533090114593506, + "photometric_loss": 0.025471189990639687, + "render": { + "bytes": 588321, + "sha256": "d93ad3f29d6e8e7a686070ab057a3fc2e7e3d6dd2b44df035b4fad8e474d569f" + }, + "round": 5 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "5bb2c52e3f078187e2b55240c63b5bbe2de840d2a3af15a37c7675a0ecc07b3e", + "clip_similarity": 0.974574089050293, + "nclip": 0.02542591094970703, + "photometric_loss": 0.029058828949928284, + "render": { + "bytes": 595137, + "sha256": "5c5ef3bb74740025f31a5883a541d2260407a37306c77e68d1d168f4b5b93bb3" + }, + "round": 6 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "b6d6fbd09f215277e8d97a93a0eba4f08440e59dc9e638d430d85ad79d383a61", + "clip_similarity": 0.9609359502792358, + "nclip": 0.03906404972076416, + "photometric_loss": 0.026920897886157036, + "render": { + "bytes": 606714, + "sha256": "08fbbd37baff628a60804a95f68ecffbea5096c6372a48fac3abbc06cdc6a2f0" + }, + "round": 7 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "9fff4cefd40e0d79663e822dedc7bab3f3e6a5d6a1c2913c53f43a57728c6191", + "clip_similarity": 0.9664453268051147, + "nclip": 0.033554673194885254, + "photometric_loss": 0.026461705565452576, + "render": { + "bytes": 606232, + "sha256": "5e2a970939dda2d7030160af7f79253e4af30b073dcd1636c87ce60cc06e0bcd" + }, + "round": 8 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "cdc02ad711e2fd50b7170621962cff7d96345c2610e983b8b65fb7d1f34c710f", + "clip_similarity": 0.9682265520095825, + "nclip": 0.03177344799041748, + "photometric_loss": 0.02800014615058899, + "render": { + "bytes": 606885, + "sha256": "a7b7ac1a158dea6f39fc1d7b0cd5884250ba2cb27309ec29997e37e0a2a41118" + }, + "round": 9 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "c6f44ec534001ceeda5b0d9e07381ba186251c499b9aea54960312ed95586d1e", + "clip_similarity": 0.9691956043243408, + "nclip": 0.03080439567565918, + "photometric_loss": 0.02725779078900814, + "render": { + "bytes": 607436, + "sha256": "1982957f7ad0b9975bcd87ceb046c25bcddb78026c398123fdeb23fea049238a" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7582, + "sha256": "9eccb85a612eac71a304af9e7c176d10970c312209874855a582e45c2ba8c9cb" + }, + "target": { + "bytes": 496540, + "sha256": "33e602e64c9b3f6825472277653bd168de39645a61cde8d8112efe0ae0cde1ad" + } + }, + "level3/attribute9": { + "evidence": [ + { + "bytes": 1270349, + "kind": "events.jsonl", + "sha256": "a93cc37fe14d3aca99bd278ab90e3e661b65bb796dbad8364c11dc3f47383897" + }, + { + "bytes": 11661650, + "kind": "run.jsonl", + "sha256": "2d807050858e48bb0509e8d3f09ede122c6731867165a98f9ffa535a2ec001ed" + } + ], + "rounds": [ + { + "blend_bytes": 646096642, + "blend_sha256": "18f474d748e3ef4359b88afac1f9d10b9e9a2aa3eed7a06e83172b9861a22068", + "clip_similarity": 0.9318258762359619, + "nclip": 0.06817412376403809, + "photometric_loss": 0.03224402293562889, + "render": { + "bytes": 573727, + "sha256": "3fd07c72310618b51861582efb145e8c71ef85ad84f445f2d37827085e37da6f" + }, + "round": 1 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "943a64ce40a167a414ff2562825a04c1952db250f0eb24a8e593b087115c2058", + "clip_similarity": 0.9383044242858887, + "nclip": 0.06169557571411133, + "photometric_loss": 0.043293412774801254, + "render": { + "bytes": 587693, + "sha256": "e0b55e00c3a3049a5026cd199048d583552d8094fb87545234667f7455ac91b7" + }, + "round": 2 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "7cfcd87a53034480700482853c48735978831d9ce4a73745576bc1c8eaca28f8", + "clip_similarity": 0.9413596391677856, + "nclip": 0.058640360832214355, + "photometric_loss": 0.04311363771557808, + "render": { + "bytes": 588713, + "sha256": "4875611c309e1901e728857aa1ac4a89558b73706da6e66385735ed58436b10e" + }, + "round": 3 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "698e536768b30834e4e16014254d4f6cc9e87de6e49db4d4c1442d3c38086611", + "clip_similarity": 0.9701520204544067, + "nclip": 0.02984797954559326, + "photometric_loss": 0.03251175582408905, + "render": { + "bytes": 590002, + "sha256": "24c27d1725ef16ba6d43de27d54297da4fa3a1e6304a020645ae3f4be7b56a2b" + }, + "round": 4 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "317b61159b933592bfe63c64a27caf74850f78986fef1209ebbafe1151b3563b", + "clip_similarity": 0.973321259021759, + "nclip": 0.026678740978240967, + "photometric_loss": 0.019820889458060265, + "render": { + "bytes": 586266, + "sha256": "f643c5a20584f10577cb75f1456f52f1f41c90b242515758400b7f0706cbb123" + }, + "round": 5 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "4f16bb04ee9e3ab6827a410ad472cf737e853e3550705a030b4dc0909ec85f52", + "clip_similarity": 0.9795242547988892, + "nclip": 0.02047574520111084, + "photometric_loss": 0.018832268193364143, + "render": { + "bytes": 586112, + "sha256": "bcc6c5a1a8ed5bf572a681c20ff2af9860a91fcf6003a393b7076b6128d5ea34" + }, + "round": 6 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "fb42311040247181df6d0c6254b4421551e7db0d5400d2eda43b347b3fbe116e", + "clip_similarity": 0.9769859910011292, + "nclip": 0.02301400899887085, + "photometric_loss": 0.0188379418104887, + "render": { + "bytes": 586516, + "sha256": "6906d54f5184bf20e6104a54ea2e62bdbaa46a0310dceea8fd1f9dd5b965ee7b" + }, + "round": 7 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "07e318a9c1903b7fbd4e01bc60d38f9bc9a4b82356984a2e2255ca1267dfb966", + "clip_similarity": 0.9763661623001099, + "nclip": 0.023633837699890137, + "photometric_loss": 0.018755586817860603, + "render": { + "bytes": 585929, + "sha256": "a09afd9eb5a15bca1319c675fc4c3b6ac38b8e2321c7b811c03fd6c7687be867" + }, + "round": 8 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "f2c0cba334f3ea4df53698b96ecbceea4e70e7ca9e5ea7d7b38ab8aa02c3a06b", + "clip_similarity": 0.9719067811965942, + "nclip": 0.02809321880340576, + "photometric_loss": 0.021445848047733307, + "render": { + "bytes": 593094, + "sha256": "ff4b064e9db0d87ddd0bafb5001b7f5ccd719ebfcae7f55a9a78be215c3e4bd6" + }, + "round": 9 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "060a3f3c25941539236446f405d8c9b1e2321a50deee184cddd95683da4989cd", + "clip_similarity": 0.9817102551460266, + "nclip": 0.01828974485397339, + "photometric_loss": 0.010348494164645672, + "render": { + "bytes": 548629, + "sha256": "77cc1d6ad1bbcec9edfbd043a3713dad9eab53e179ec516dcf42f67ce354df7f" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7580, + "sha256": "7e3cb13f2ae3c0de3032c58d4d4a9237681adcf41e49cf89c01add14481eb38c" + }, + "target": { + "bytes": 496540, + "sha256": "27653bfcb7c78fe51239c3905060ce112e65c8875966b2d81f3d0ae33fa322ee" + } + } + } +} diff --git a/benchmarks/blenderbench_direct/publication/reference-results.json b/benchmarks/blenderbench_direct/publication/reference-results.json new file mode 100644 index 0000000..1cb19c1 --- /dev/null +++ b/benchmarks/blenderbench_direct/publication/reference-results.json @@ -0,0 +1,347 @@ +{ + "categories": { + "level1": { + "mean_best_clip_similarity": 0.9962615238295661, + "mean_best_nclip_x100": 0.3738476170433892, + "mean_best_photometric_loss": 0.003033510640105103, + "tasks": 9 + }, + "level2": { + "mean_best_clip_similarity": 0.983968522813585, + "mean_best_nclip_x100": 1.603147718641493, + "mean_best_photometric_loss": 0.011897660300342573, + "tasks": 9 + }, + "level3": { + "mean_best_clip_similarity": 0.9771796133783128, + "mean_best_nclip_x100": 2.2820386621687145, + "mean_best_photometric_loss": 0.019716895185410976, + "tasks": 9 + } + }, + "dataset": { + "attribution": "DietCoke4671 and contributors", + "license": "CC BY 4.0", + "repository": "DietCoke4671/BlenderBench", + "revision": "203e4d325e9438ca55b29bdfc4f6a90842d74e68" + }, + "evaluator": { + "best_round_rule": "minimum N-CLIP; paired photometric loss from same round; earliest tie", + "clip_model": "openai/clip-vit-base-patch32", + "clip_model_revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", + "metric": "CLIP image-embedding cosine similarity; N-CLIP = 1 - similarity; not literal task accuracy", + "repository": "Fugtemypt123/VIGA-release", + "revision": "69cb8ef0651bf68124815682df4a0f8e8b57c141", + "source_sha256": "9ee08818aa858dccfff3efbb36cb9a98e8887f14e6168ad1a23731351cdeb3b5" + }, + "experiment": { + "codex_cli": "0.154.0", + "generation_model": "gpt-6-astra", + "generation_model_revision": null, + "implementation_commit": "4a9f5cc6e616080185252bb919fa3d1e730760b7", + "rounds_per_task": 10, + "scored_rounds": 270, + "tasks": 27, + "trajectory_policy": "one sequential ten-round trajectory per task; no CLIP or evaluator feedback to generation" + }, + "limitations": [ + "Public dataset; contamination risk is not measurable.", + "No hidden test set, hosted submission endpoint, or official leaderboard verification.", + "Model and inference budget differ from VIGA paper conditions.", + "One trajectory per task; no variance estimate.", + "No VLM judge by design." + ], + "results_source": { + "bytes": 6447, + "sha256": "073be2c44f1a763b66dea5138eda1b8959e8553fef6ce288f1d78ee92807d6e8" + }, + "rows": [ + { + "best_round": 7, + "clip_similarity": 0.9975831508636475, + "raw_nclip": 0.002416849136352539, + "raw_pl": 0.003328760154545307, + "rounds": 10, + "task": "level1/camera1" + }, + { + "best_round": 8, + "clip_similarity": 0.9955885410308838, + "raw_nclip": 0.004411458969116211, + "raw_pl": 0.0033672682475298643, + "rounds": 10, + "task": "level1/camera2" + }, + { + "best_round": 9, + "clip_similarity": 0.9977397918701172, + "raw_nclip": 0.0022602081298828125, + "raw_pl": 0.001466981484554708, + "rounds": 10, + "task": "level1/camera3" + }, + { + "best_round": 6, + "clip_similarity": 0.9912315011024475, + "raw_nclip": 0.00876849889755249, + "raw_pl": 0.007649693172425032, + "rounds": 10, + "task": "level1/camera4" + }, + { + "best_round": 7, + "clip_similarity": 0.9964922666549683, + "raw_nclip": 0.0035077333450317383, + "raw_pl": 0.0044174473732709885, + "rounds": 10, + "task": "level1/camera5" + }, + { + "best_round": 10, + "clip_similarity": 0.9997468590736389, + "raw_nclip": 0.000253140926361084, + "raw_pl": 3.515696880640462e-05, + "rounds": 10, + "task": "level1/camera6" + }, + { + "best_round": 9, + "clip_similarity": 0.989441454410553, + "raw_nclip": 0.010558545589447021, + "raw_pl": 0.006557795684784651, + "rounds": 10, + "task": "level1/camera7" + }, + { + "best_round": 10, + "clip_similarity": 0.9990020990371704, + "raw_nclip": 0.0009979009628295898, + "raw_pl": 3.855396425933577e-05, + "rounds": 10, + "task": "level1/camera8" + }, + { + "best_round": 9, + "clip_similarity": 0.9995280504226685, + "raw_nclip": 0.00047194957733154297, + "raw_pl": 0.0004399387107696384, + "rounds": 10, + "task": "level1/camera9" + }, + { + "best_round": 10, + "clip_similarity": 0.9777544736862183, + "raw_nclip": 0.02224552631378174, + "raw_pl": 0.015119296498596668, + "rounds": 10, + "task": "level2/attribute1" + }, + { + "best_round": 5, + "clip_similarity": 0.9820566177368164, + "raw_nclip": 0.017943382263183594, + "raw_pl": 0.016636313870549202, + "rounds": 10, + "task": "level2/attribute2" + }, + { + "best_round": 4, + "clip_similarity": 0.9793153405189514, + "raw_nclip": 0.020684659481048584, + "raw_pl": 0.01562858745455742, + "rounds": 10, + "task": "level2/attribute3" + }, + { + "best_round": 1, + "clip_similarity": 0.9745771288871765, + "raw_nclip": 0.025422871112823486, + "raw_pl": 0.02019927091896534, + "rounds": 10, + "task": "level2/attribute4" + }, + { + "best_round": 9, + "clip_similarity": 0.9830480813980103, + "raw_nclip": 0.016951918601989746, + "raw_pl": 0.013992843218147755, + "rounds": 10, + "task": "level2/attribute5" + }, + { + "best_round": 9, + "clip_similarity": 0.9842134714126587, + "raw_nclip": 0.01578652858734131, + "raw_pl": 0.01759195886552334, + "rounds": 10, + "task": "level2/attribute6" + }, + { + "best_round": 9, + "clip_similarity": 0.9894779324531555, + "raw_nclip": 0.010522067546844482, + "raw_pl": 0.0033791407477110624, + "rounds": 10, + "task": "level2/attribute7" + }, + { + "best_round": 9, + "clip_similarity": 0.9888083934783936, + "raw_nclip": 0.011191606521606445, + "raw_pl": 0.0037378640845417976, + "rounds": 10, + "task": "level2/attribute8" + }, + { + "best_round": 10, + "clip_similarity": 0.996465265750885, + "raw_nclip": 0.0035347342491149902, + "raw_pl": 0.0007936670444905758, + "rounds": 10, + "task": "level2/attribute9" + }, + { + "best_round": 9, + "clip_similarity": 0.9810705780982971, + "raw_nclip": 0.01892942190170288, + "raw_pl": 0.02021031640470028, + "rounds": 10, + "task": "level3/attribute1" + }, + { + "best_round": 10, + "clip_similarity": 0.9837775230407715, + "raw_nclip": 0.016222476959228516, + "raw_pl": 0.012915946543216705, + "rounds": 10, + "task": "level3/attribute2" + }, + { + "best_round": 10, + "clip_similarity": 0.9846137762069702, + "raw_nclip": 0.015386223793029785, + "raw_pl": 0.014947629533708096, + "rounds": 10, + "task": "level3/attribute3" + }, + { + "best_round": 10, + "clip_similarity": 0.9926517009735107, + "raw_nclip": 0.007348299026489258, + "raw_pl": 0.009027284570038319, + "rounds": 10, + "task": "level3/attribute4" + }, + { + "best_round": 10, + "clip_similarity": 0.9755886793136597, + "raw_nclip": 0.024411320686340332, + "raw_pl": 0.027274271473288536, + "rounds": 10, + "task": "level3/attribute5" + }, + { + "best_round": 7, + "clip_similarity": 0.9866102933883667, + "raw_nclip": 0.0133897066116333, + "raw_pl": 0.025586744770407677, + "rounds": 10, + "task": "level3/attribute6" + }, + { + "best_round": 5, + "clip_similarity": 0.93320631980896, + "raw_nclip": 0.06679368019104004, + "raw_pl": 0.03137661889195442, + "rounds": 10, + "task": "level3/attribute7" + }, + { + "best_round": 4, + "clip_similarity": 0.9753873944282532, + "raw_nclip": 0.024612605571746826, + "raw_pl": 0.025764750316739082, + "rounds": 10, + "task": "level3/attribute8" + }, + { + "best_round": 10, + "clip_similarity": 0.9817102551460266, + "raw_nclip": 0.01828974485397339, + "raw_pl": 0.010348494164645672, + "rounds": 10, + "task": "level3/attribute9" + } + ], + "schema_version": 1, + "scoring_runtime": { + "gpu": "NVIDIA GeForce RTX 5090", + "packages": { + "certifi": "2026.7.22", + "charset-normalizer": "3.5.1", + "filelock": "3.32.3", + "fsspec": "2026.7.0", + "hf-xet": "1.6.0", + "huggingface-hub": "0.36.2", + "idna": "3.19", + "jinja2": "3.1.6", + "markupsafe": "3.0.3", + "mpmath": "1.3.0", + "networkx": "3.6.1", + "numpy": "2.4.6", + "nvidia-cublas-cu12": "12.8.4.1", + "nvidia-cuda-cupti-cu12": "12.8.90", + "nvidia-cuda-nvrtc-cu12": "12.8.93", + "nvidia-cuda-runtime-cu12": "12.8.90", + "nvidia-cudnn-cu12": "9.10.2.21", + "nvidia-cufft-cu12": "11.3.3.83", + "nvidia-cufile-cu12": "1.13.1.3", + "nvidia-curand-cu12": "10.3.9.90", + "nvidia-cusolver-cu12": "11.7.3.90", + "nvidia-cusparse-cu12": "12.5.8.93", + "nvidia-cusparselt-cu12": "0.7.1", + "nvidia-nccl-cu12": "2.27.3", + "nvidia-nvjitlink-cu12": "12.8.93", + "nvidia-nvtx-cu12": "12.8.90", + "packaging": "26.3", + "pillow": "12.3.0", + "pyyaml": "6.0.3", + "regex": "2026.9.10", + "requests": "2.34.2", + "safetensors": "0.8.0", + "setuptools": "78.1.0", + "sympy": "1.14.0", + "tokenizers": "0.21.4", + "torch": "2.8.0+cu128", + "tqdm": "4.70.0", + "transformers": "4.55.4", + "triton": "3.4.0", + "typing-extensions": "4.16.0", + "urllib3": "2.7.0" + }, + "python": { + "architecture": "x86_64", + "bits": 64, + "cache_tag": "cpython-313", + "compiler": "Clang 22.1.3", + "executable_sha256": "93a93f4c8af5ae6f127fb0310f923d5fab89eb7257a0b801eec2e44b3f199840", + "implementation": "CPython", + "note": "Executable digest is historical provenance, not remote or hostile-host attestation.", + "soabi": "cpython-313-x86_64-linux-gnu", + "version": "3.13.15" + }, + "torch_cuda": "12.8" + }, + "status": "unofficial_public-dataset_evaluation; not an official BlenderBench leaderboard submission", + "video": { + "bytes": 10161884, + "codec": "h264", + "duration_seconds": 134.9, + "height": 464, + "publication_status": "published", + "sample_aspect_ratio": "1:1", + "sha256": "04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17", + "url": "https://huggingface.co/datasets/michaelgold/blenderbench-direct-results/resolve/main/video/blenderbench-complete-compilation.mp4", + "width": 848 + } +} diff --git a/benchmarks/blenderbench_direct/publication/results-summary.json b/benchmarks/blenderbench_direct/publication/results-summary.json new file mode 100644 index 0000000..326f744 --- /dev/null +++ b/benchmarks/blenderbench_direct/publication/results-summary.json @@ -0,0 +1,242 @@ +{ + "task_count": 27, + "round_count": 270, + "categories": { + "level1": { + "tasks": 9, + "mean_best_clip_similarity": 0.9962615238295661, + "mean_best_nclip_x100": 0.3738476170433892, + "mean_best_photometric_loss": 0.003033510640105103 + }, + "level2": { + "tasks": 9, + "mean_best_clip_similarity": 0.983968522813585, + "mean_best_nclip_x100": 1.603147718641493, + "mean_best_photometric_loss": 0.011897660300342573 + }, + "level3": { + "tasks": 9, + "mean_best_clip_similarity": 0.9771796133783128, + "mean_best_nclip_x100": 2.2820386621687145, + "mean_best_photometric_loss": 0.019716895185410976 + } + }, + "rows": [ + { + "task": "level1/camera1", + "rounds": 10, + "best_round": 7, + "clip_similarity": 0.9975831508636475, + "raw_nclip": 0.002416849136352539, + "raw_pl": 0.003328760154545307 + }, + { + "task": "level1/camera2", + "rounds": 10, + "best_round": 8, + "clip_similarity": 0.9955885410308838, + "raw_nclip": 0.004411458969116211, + "raw_pl": 0.0033672682475298643 + }, + { + "task": "level1/camera3", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9977397918701172, + "raw_nclip": 0.0022602081298828125, + "raw_pl": 0.001466981484554708 + }, + { + "task": "level1/camera4", + "rounds": 10, + "best_round": 6, + "clip_similarity": 0.9912315011024475, + "raw_nclip": 0.00876849889755249, + "raw_pl": 0.007649693172425032 + }, + { + "task": "level1/camera5", + "rounds": 10, + "best_round": 7, + "clip_similarity": 0.9964922666549683, + "raw_nclip": 0.0035077333450317383, + "raw_pl": 0.0044174473732709885 + }, + { + "task": "level1/camera6", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9997468590736389, + "raw_nclip": 0.000253140926361084, + "raw_pl": 3.515696880640462e-05 + }, + { + "task": "level1/camera7", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.989441454410553, + "raw_nclip": 0.010558545589447021, + "raw_pl": 0.006557795684784651 + }, + { + "task": "level1/camera8", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9990020990371704, + "raw_nclip": 0.0009979009628295898, + "raw_pl": 3.855396425933577e-05 + }, + { + "task": "level1/camera9", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9995280504226685, + "raw_nclip": 0.00047194957733154297, + "raw_pl": 0.0004399387107696384 + }, + { + "task": "level2/attribute1", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9777544736862183, + "raw_nclip": 0.02224552631378174, + "raw_pl": 0.015119296498596668 + }, + { + "task": "level2/attribute2", + "rounds": 10, + "best_round": 5, + "clip_similarity": 0.9820566177368164, + "raw_nclip": 0.017943382263183594, + "raw_pl": 0.016636313870549202 + }, + { + "task": "level2/attribute3", + "rounds": 10, + "best_round": 4, + "clip_similarity": 0.9793153405189514, + "raw_nclip": 0.020684659481048584, + "raw_pl": 0.01562858745455742 + }, + { + "task": "level2/attribute4", + "rounds": 10, + "best_round": 1, + "clip_similarity": 0.9745771288871765, + "raw_nclip": 0.025422871112823486, + "raw_pl": 0.02019927091896534 + }, + { + "task": "level2/attribute5", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9830480813980103, + "raw_nclip": 0.016951918601989746, + "raw_pl": 0.013992843218147755 + }, + { + "task": "level2/attribute6", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9842134714126587, + "raw_nclip": 0.01578652858734131, + "raw_pl": 0.01759195886552334 + }, + { + "task": "level2/attribute7", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9894779324531555, + "raw_nclip": 0.010522067546844482, + "raw_pl": 0.0033791407477110624 + }, + { + "task": "level2/attribute8", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9888083934783936, + "raw_nclip": 0.011191606521606445, + "raw_pl": 0.0037378640845417976 + }, + { + "task": "level2/attribute9", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.996465265750885, + "raw_nclip": 0.0035347342491149902, + "raw_pl": 0.0007936670444905758 + }, + { + "task": "level3/attribute1", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9810705780982971, + "raw_nclip": 0.01892942190170288, + "raw_pl": 0.02021031640470028 + }, + { + "task": "level3/attribute2", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9837775230407715, + "raw_nclip": 0.016222476959228516, + "raw_pl": 0.012915946543216705 + }, + { + "task": "level3/attribute3", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9846137762069702, + "raw_nclip": 0.015386223793029785, + "raw_pl": 0.014947629533708096 + }, + { + "task": "level3/attribute4", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9926517009735107, + "raw_nclip": 0.007348299026489258, + "raw_pl": 0.009027284570038319 + }, + { + "task": "level3/attribute5", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9755886793136597, + "raw_nclip": 0.024411320686340332, + "raw_pl": 0.027274271473288536 + }, + { + "task": "level3/attribute6", + "rounds": 10, + "best_round": 7, + "clip_similarity": 0.9866102933883667, + "raw_nclip": 0.0133897066116333, + "raw_pl": 0.025586744770407677 + }, + { + "task": "level3/attribute7", + "rounds": 10, + "best_round": 5, + "clip_similarity": 0.93320631980896, + "raw_nclip": 0.06679368019104004, + "raw_pl": 0.03137661889195442 + }, + { + "task": "level3/attribute8", + "rounds": 10, + "best_round": 4, + "clip_similarity": 0.9753873944282532, + "raw_nclip": 0.024612605571746826, + "raw_pl": 0.025764750316739082 + }, + { + "task": "level3/attribute9", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9817102551460266, + "raw_nclip": 0.01828974485397339, + "raw_pl": 0.010348494164645672 + } + ] +} diff --git a/benchmarks/blenderbench_direct/ref_based_eval.py b/benchmarks/blenderbench_direct/ref_based_eval.py new file mode 100644 index 0000000..e5f68cc --- /dev/null +++ b/benchmarks/blenderbench_direct/ref_based_eval.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python3 +""" +Evaluation script for AgenticVerifier blendergym results. +Adapted from the original evaluate.py to work with our codebase structure. +""" + +import os +import sys +import argparse +import json +from PIL import Image +from tqdm import tqdm +import numpy as np +import torch +from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize +from transformers import CLIPProcessor, CLIPModel +from concurrent.futures import ThreadPoolExecutor, as_completed + +# Task instance counts for different task types +TASK_INSTANCE_COUNT_DICT = { + 'level1': 9, + 'level2': 9, + 'level3': 9, +} + +# Global CLIP model/processor to share across threads +GLOBAL_CLIP_MODEL = None +GLOBAL_CLIP_PROCESSOR = None + + +def ensure_clip_loaded(): + """ + Lazily load the global CLIP model and processor once per process. + """ + global GLOBAL_CLIP_MODEL, GLOBAL_CLIP_PROCESSOR + if GLOBAL_CLIP_MODEL is None or GLOBAL_CLIP_PROCESSOR is None: + GLOBAL_CLIP_MODEL = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") + GLOBAL_CLIP_PROCESSOR = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") + + +def clip_similarity(image1, image2): + """ + Compute the CLIP similarity between two PIL images. + + Args: + image1 (PIL.Image): The first input image. + image2 (PIL.Image): The second input image. + + Returns: + float: The CLIP similarity between the two images. + """ + if image1.size != image2.size: + image2 = image2.resize(image1.size) + + # Ensure global model is initialized + ensure_clip_loaded() + + # Preprocess the images + images = [image1, image2] + inputs = GLOBAL_CLIP_PROCESSOR(images=images, return_tensors="pt") + + # Compute the features for the images + with torch.no_grad(): + features = GLOBAL_CLIP_MODEL.get_image_features(**inputs) + + # Compute the cosine similarity between the image features + sim = torch.nn.functional.cosine_similarity(features[0], features[1], dim=-1) + + return sim.item() + +def photometric_loss(image1: Image.Image, image2: Image.Image) -> float: + """ + Compute the photometric loss between two PIL images. + + Args: + image1 (PIL.Image): The first input image. + image2 (PIL.Image): The second input image. + + Returns: + float: The photometric loss between the two images. + """ + if image1.size != image2.size: + image2 = image2.resize(image1.size) + + # Convert images to numpy arrays + img1_array = np.array(image1)[:, :, :3] + img2_array = np.array(image2)[:, :, :3] + + # Normalize images to [0, 1] + img1_norm = img1_array.astype(np.float32) / 255.0 + img2_norm = img2_array.astype(np.float32) / 255.0 + + # Compute the squared difference between the normalized images + diff = np.square(img1_norm - img2_norm) + + # Compute the mean squared error + mse = np.mean(diff) + return mse + + +def process_task_instance(output_base_dir: str, task_dir: str): + """ + Process a single task instance directory and compute metrics across rounds. + + Returns: + tuple: (task_dir, task_instance_scores, best_n_clip, best_pl) + where best_* can be None if no valid rounds. + """ + task_instance_dir = os.path.join(output_base_dir, task_dir) + renders_dir = os.path.join(task_instance_dir, "renders") + + if not os.path.exists(renders_dir): + return task_dir, {}, None, None + + gt_renders_dir = f"data/blenderbench/{task_dir}/renders/goal" + if not os.path.exists(gt_renders_dir): + return task_dir, {}, None, None + + task_instance_scores = {} + + # Get all round directories (1, 2, 3, 4, etc.) + round_dirs = [d for d in os.listdir(renders_dir) + if os.path.isdir(os.path.join(renders_dir, d))] + try: + round_dirs.sort(key=lambda x: int(x)) + except Exception: + # Fallback to lexical sort if non-numeric dirs exist + round_dirs.sort() + + if not round_dirs: + return task_dir, {}, None, None + + for round_dir in round_dirs: + round_path = os.path.join(renders_dir, round_dir) + task_instance_scores[round_dir] = {} + + n_clip_views = [] + pl_views = [] + + # render1 + render1_path = os.path.join(round_path, "render1.png") + gt_render1_path = os.path.join(gt_renders_dir, "render1.png") + if os.path.exists(render1_path) and os.path.exists(gt_render1_path): + try: + proposal_render = Image.open(render1_path) + gt_render = Image.open(gt_render1_path) + n_clip = float(1 - clip_similarity(proposal_render, gt_render)) + pl = float(photometric_loss(proposal_render, gt_render)) + n_clip_views.append(n_clip) + pl_views.append(pl) + task_instance_scores[round_dir]['render1'] = {'n_clip': n_clip, 'pl': pl} + except Exception: + pass + + # render2 + render2_path = os.path.join(round_path, "render2.png") + gt_render2_path = os.path.join(gt_renders_dir, "render2.png") + if os.path.exists(render2_path) and os.path.exists(gt_render2_path): + try: + proposal_render2 = Image.open(render2_path) + gt_render2 = Image.open(gt_render2_path) + n_clip2 = float(1 - clip_similarity(proposal_render2, gt_render2)) + pl2 = float(photometric_loss(proposal_render2, gt_render2)) + n_clip_views.append(n_clip2) + pl_views.append(pl2) + task_instance_scores[round_dir]['render2'] = {'n_clip': n_clip2, 'pl': pl2} + except Exception: + pass + + if n_clip_views: + avg_n_clip = sum(n_clip_views) / len(n_clip_views) + avg_pl = sum(pl_views) / len(pl_views) + task_instance_scores[round_dir]['avg_n_clip'] = avg_n_clip + task_instance_scores[round_dir]['avg_pl'] = avg_pl + + # Determine best rounds + valid_rounds = {k: v for k, v in task_instance_scores.items() if 'avg_n_clip' in v and 'avg_pl' in v} + best_n_clip = None + best_pl = None + if valid_rounds: + best_n_clip_round = min(valid_rounds.keys(), key=lambda r: valid_rounds[r]['avg_n_clip']) + best_pl_round = min(valid_rounds.keys(), key=lambda r: valid_rounds[r]['avg_pl']) + best_n_clip = valid_rounds[best_n_clip_round]['avg_n_clip'] + best_pl = valid_rounds[best_n_clip_round]['avg_pl'] if best_n_clip_round in valid_rounds else None + + # Save individual instance scores + instance_scores_path = os.path.join(task_instance_dir, 'scores.json') + try: + with open(instance_scores_path, 'w') as f: + json.dump(task_instance_scores, f, indent=4) + except Exception: + pass + + return task_dir, task_instance_scores, best_n_clip, best_pl + +def extract_task_type_and_number(task_dir_name): + """ + Extract task type and number from directory name like 'placement1', 'material5', etc. + + Args: + task_dir_name (str): Directory name like 'placement1', 'material5' + + Returns: + tuple: (task_type, task_number) or (None, None) if invalid + """ + level_dir_name = task_dir_name.split("/")[0] + task_name = task_dir_name.split("/")[1] + for task_type in TASK_INSTANCE_COUNT_DICT.keys(): + if level_dir_name == task_type: + try: + task_number = int(task_name[-1]) + return task_type, task_number + except ValueError: + continue + return None, None + +def main() -> None: + """Run reference-based evaluation (CLIP + photometric loss) for BlenderBench results.""" + parser = argparse.ArgumentParser(description='Evaluate AgenticVerifier blendergym-hard results') + parser.add_argument('test_id', type=str, help='Test ID (e.g., 20250815_150016)') + parser.add_argument('--output_dir', type=str, default=None, + help='Output directory for evaluation results (default: output/blenderbench/{test_id}/)_evaluation)') + parser.add_argument('--missing_round_penalty_max', type=float, default=2.0, + help='Max penalty factor for earliest rounds.') + parser.add_argument('--missing_round_penalty_min', type=float, default=1.0, + help='Min penalty factor for latest rounds.') + + args = parser.parse_args() + test_id = args.test_id + penalty_max = float(args.missing_round_penalty_max) + penalty_min = float(args.missing_round_penalty_min) + MAX_ROUNDS = 10 + + # Set up paths + output_base_dir = f"output/blenderbench/{test_id}" + if not os.path.exists(output_base_dir): + raise ValueError(f"Output directory {output_base_dir} does not exist.") + + if args.output_dir: + eval_output_dir = args.output_dir + else: + eval_output_dir = os.path.join(output_base_dir, "_evaluation") + + os.makedirs(eval_output_dir, exist_ok=True) + + # Get all task directories + task_dirs = [] + for level_dir in os.listdir(output_base_dir): + if os.path.isdir(os.path.join(output_base_dir, level_dir)) and level_dir != "_evaluation": + for task_dir in os.listdir(os.path.join(output_base_dir, level_dir)): + if os.path.isdir(os.path.join(output_base_dir, level_dir, task_dir)): + task_dirs.append(os.path.join(level_dir, task_dir)) + + print(f"Found {len(task_dirs)} task directories in {output_base_dir}") + + # Group tasks by type + tasks_by_type = {} + for task_dir in task_dirs: + task_type, task_number = extract_task_type_and_number(task_dir) + if task_type and task_number: + if task_type not in tasks_by_type: + tasks_by_type[task_type] = [] + tasks_by_type[task_type].append((task_dir, task_number)) + + print(f"Grouped tasks by type: {list(tasks_by_type.keys())}") + + scores_across_tasks = {} + intermediates = {} + + # Ensure CLIP is loaded once (shared by threads) + ensure_clip_loaded() + + for task_type, task_instances in tasks_by_type.items(): + print(f"\nProcessing task type: {task_type}") + + # Sort by task number + task_instances.sort(key=lambda x: x[1]) + + scores_across_instances = { + 'best_n_clip': [], + 'best_pl': [], + 'instance_details': {} + } + + # Run per-instance processing in parallel threads + max_workers = min(8, (os.cpu_count() or 4)) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [ + executor.submit(process_task_instance, output_base_dir, task_dir) + for task_dir, _ in task_instances + ] + + for future in tqdm(as_completed(futures), total=len(futures), desc=f"Processing {task_type}"): + try: + task_dir, task_instance_scores, best_n_clip, best_pl = future.result() + scores_across_instances['instance_details'][task_dir] = task_instance_scores + if best_n_clip is not None and best_pl is not None: + scores_across_instances['best_n_clip'].append(best_n_clip) + scores_across_instances['best_pl'].append(best_pl) + print(f" {task_dir}: Best n_clip={best_n_clip:.4f}, Best pl={best_pl:.4f}") + else: + print(f" {task_dir}: No valid scores") + except Exception as e: + print(f" Error processing {task_type} instance: {e}") + + # Aggregate per-round averages across all instances (rounds 1..9) + per_round_values = {str(i): {'n_clip': [], 'pl': [], 'penalized_count': 0} for i in range(1, 11)} + for instance_scores in scores_across_instances['instance_details'].values(): + # Collect available round indices for this instance + available_rounds = sorted( + [int(r) for r, v in instance_scores.items() if isinstance(v, dict) and 'avg_n_clip' in v and 'avg_pl' in v] + ) + if not available_rounds: + continue + max_available_round = max(available_rounds) + + for round_idx in range(1, 11): + key = str(round_idx) + # Case 1: round exists normally + if key in instance_scores and 'avg_n_clip' in instance_scores[key] and 'avg_pl' in instance_scores[key]: + per_round_values[key]['n_clip'].append(instance_scores[key]['avg_n_clip']) + per_round_values[key]['pl'].append(instance_scores[key]['avg_pl']) + continue + + # Case 2: earlier round missing but later rounds exist -> penalize + if round_idx < max_available_round: + # Find the next available later round to base the penalty on + later_rounds = [r for r in available_rounds if r > round_idx] + if not later_rounds: + continue + next_round = min(later_rounds) + next_key = str(next_round) + base_n = instance_scores[next_key]['avg_n_clip'] + base_pl = instance_scores[next_key]['avg_pl'] + # Decaying penalty: higher for earlier rounds, lower for later rounds + if MAX_ROUNDS > 1: + t = (round_idx - 1) / (MAX_ROUNDS - 1) + else: + t = 0.0 + penalty_factor_round = penalty_max - t * (penalty_max - penalty_min) + per_round_values[key]['n_clip'].append(base_n * penalty_factor_round) + per_round_values[key]['pl'].append(base_pl * penalty_factor_round) + per_round_values[key]['penalized_count'] += 1 + continue + # Case 3: missing because process ended (no later rounds) -> ignore + + per_round_summary = {} + for key, vals in per_round_values.items(): + if vals['n_clip'] and vals['pl']: + per_round_summary[key] = { + 'avg_n_clip': sum(vals['n_clip']) / len(vals['n_clip']), + 'avg_pl': sum(vals['pl']) / len(vals['pl']), + 'num_instances': len(vals['n_clip']), + 'num_penalized': int(vals['penalized_count']) + } + + # Store per-round aggregation in intermediates structure too + scores_across_instances['per_round'] = per_round_summary + + # Aggregate results for this task type + if scores_across_instances['best_n_clip']: + scores_across_tasks[task_type] = { + 'best_n_clip': sum(scores_across_instances['best_n_clip']) / len(scores_across_instances['best_n_clip']), + 'best_pl': sum(scores_across_instances['best_pl']) / len(scores_across_instances['best_pl']), + 'num_instances': len(scores_across_instances['best_n_clip']), + 'per_round': per_round_summary + } + + print(f" Task {task_type} overall scores:") + print(f" Average best n_clip: {scores_across_tasks[task_type]['best_n_clip']:.4f}") + print(f" Average best pl: {scores_across_tasks[task_type]['best_pl']:.4f}") + print(f" Number of instances: {scores_across_tasks[task_type]['num_instances']}") + else: + print(f" No valid scores for task type {task_type}") + scores_across_tasks[task_type] = {} + + intermediates[task_type] = scores_across_instances + + # Save overall results + overall_scores_path = os.path.join(eval_output_dir, 'overall_scores.json') + with open(overall_scores_path, 'w') as f: + json.dump(scores_across_tasks, f, indent=4) + + intermediate_scores_path = os.path.join(eval_output_dir, 'intermediate_scores.json') + with open(intermediate_scores_path, 'w') as f: + json.dump(intermediates, f, indent=4) + + # Print summary + print(f"\n=== Evaluation Summary ===") + print(f"Test ID: {test_id}") + print(f"Output directory: {eval_output_dir}") + print(f"Results saved to: {overall_scores_path}") + + for task_type, scores in scores_across_tasks.items(): + if scores: + print(f"\n{task_type.upper()}:") + print(f" Average best n_clip: {scores['best_n_clip']:.4f}") + print(f" Average best pl: {scores['best_pl']:.4f}") + print(f" Instances evaluated: {scores['num_instances']}") + else: + print(f"\n{task_type.upper()}: No valid scores") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmarks/blenderbench_direct/requirements-fixture.in b/benchmarks/blenderbench_direct/requirements-fixture.in new file mode 100644 index 0000000..bc3b3db --- /dev/null +++ b/benchmarks/blenderbench_direct/requirements-fixture.in @@ -0,0 +1,5 @@ +# Minimal CPU-only dependencies for publication fixture tests. +# uv pip compile --python-version 3.11 --python-platform x86_64-manylinux_2_28 \ +# --index-url https://pypi.org/simple --generate-hashes \ +# -o requirements-fixture.lock requirements-fixture.in +pillow==12.3.0 diff --git a/benchmarks/blenderbench_direct/requirements-fixture.lock b/benchmarks/blenderbench_direct/requirements-fixture.lock new file mode 100644 index 0000000..38781b1 --- /dev/null +++ b/benchmarks/blenderbench_direct/requirements-fixture.lock @@ -0,0 +1,91 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --python-version 3.11 --python-platform x86_64-manylinux_2_28 --generate-hashes -o benchmarks/blenderbench_direct/requirements-fixture.lock benchmarks/blenderbench_direct/requirements-fixture.in +pillow==12.3.0 \ + --hash=sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756 \ + --hash=sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a \ + --hash=sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59 \ + --hash=sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45 \ + --hash=sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3 \ + --hash=sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df \ + --hash=sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139 \ + --hash=sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b \ + --hash=sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39 \ + --hash=sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e \ + --hash=sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8 \ + --hash=sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1 \ + --hash=sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8 \ + --hash=sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89 \ + --hash=sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5 \ + --hash=sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130 \ + --hash=sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd \ + --hash=sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d \ + --hash=sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b \ + --hash=sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed \ + --hash=sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace \ + --hash=sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb \ + --hash=sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931 \ + --hash=sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510 \ + --hash=sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6 \ + --hash=sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1 \ + --hash=sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce \ + --hash=sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385 \ + --hash=sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e \ + --hash=sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c \ + --hash=sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7 \ + --hash=sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace \ + --hash=sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c \ + --hash=sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f \ + --hash=sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64 \ + --hash=sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f \ + --hash=sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a \ + --hash=sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827 \ + --hash=sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17 \ + --hash=sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4 \ + --hash=sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a \ + --hash=sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701 \ + --hash=sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e \ + --hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 \ + --hash=sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66 \ + --hash=sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468 \ + --hash=sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217 \ + --hash=sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658 \ + --hash=sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418 \ + --hash=sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a \ + --hash=sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c \ + --hash=sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330 \ + --hash=sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402 \ + --hash=sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09 \ + --hash=sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930 \ + --hash=sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f \ + --hash=sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec \ + --hash=sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a \ + --hash=sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94 \ + --hash=sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468 \ + --hash=sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b \ + --hash=sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965 \ + --hash=sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8 \ + --hash=sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd \ + --hash=sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7 \ + --hash=sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c \ + --hash=sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777 \ + --hash=sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35 \ + --hash=sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9 \ + --hash=sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f \ + --hash=sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f \ + --hash=sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0 \ + --hash=sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c \ + --hash=sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71 \ + --hash=sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3 \ + --hash=sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838 \ + --hash=sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf \ + --hash=sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321 \ + --hash=sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26 \ + --hash=sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec \ + --hash=sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9 \ + --hash=sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65 \ + --hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \ + --hash=sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e \ + --hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \ + --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \ + --hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7 + # via -r benchmarks/blenderbench_direct/requirements-fixture.in diff --git a/benchmarks/blenderbench_direct/requirements-scoring.in b/benchmarks/blenderbench_direct/requirements-scoring.in new file mode 100644 index 0000000..3e38b5b --- /dev/null +++ b/benchmarks/blenderbench_direct/requirements-scoring.in @@ -0,0 +1,47 @@ +# Historical BlenderBench scoring environment recovered from the publication host. +# Target: exact historical CPython 3.13.15, x86_64 manylinux_2_28, CUDA 12.8. +# Every distribution is pinned to the observed version; regenerate with uv as below. +# uv pip compile --python-version 3.13 --python-platform x86_64-manylinux_2_28 \ +# --index-url https://pypi.org/simple --generate-hashes \ +# -o requirements-scoring.lock requirements-scoring.in +certifi==2026.7.22 +charset-normalizer==3.5.1 +filelock==3.32.3 +fsspec==2026.7.0 +hf-xet==1.6.0 +huggingface-hub==0.36.2 +idna==3.19 +jinja2==3.1.6 +markupsafe==3.0.3 +mpmath==1.3.0 +networkx==3.6.1 +numpy==2.4.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-nccl-cu12==2.27.3 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvtx-cu12==12.8.90 +packaging==26.3 +pillow==12.3.0 +pyyaml==6.0.3 +regex==2026.9.10 +requests==2.34.2 +safetensors==0.8.0 +setuptools==78.1.0 +sympy==1.14.0 +tokenizers==0.21.4 +torch @ https://download.pytorch.org/whl/cu128/torch-2.8.0%2Bcu128-cp313-cp313-manylinux_2_28_x86_64.whl +tqdm==4.70.0 +transformers==4.55.4 +triton==3.4.0 +typing-extensions==4.16.0 +urllib3==2.7.0 diff --git a/benchmarks/blenderbench_direct/requirements-scoring.lock b/benchmarks/blenderbench_direct/requirements-scoring.lock new file mode 100644 index 0000000..fc4f5bd --- /dev/null +++ b/benchmarks/blenderbench_direct/requirements-scoring.lock @@ -0,0 +1,933 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --python-version 3.13 --python-platform x86_64-manylinux_2_28 --generate-hashes -o benchmarks/blenderbench_direct/requirements-scoring.lock benchmarks/blenderbench_direct/requirements-scoring.in +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # requests +charset-normalizer==3.5.1 \ + --hash=sha256:00668ebb0609751758682eb0b5857e7c35b9f00e84dfdef062e103244ec94d45 \ + --hash=sha256:012a22b88a77ca2e59b98ac5889b0deb604147666032f45e6d6e217634d2550d \ + --hash=sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5 \ + --hash=sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b \ + --hash=sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f \ + --hash=sha256:07ffd07412fc5d5e84cd8952acf9ff7e4ed7a708e69d1bada19d8ba91711353f \ + --hash=sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5 \ + --hash=sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22 \ + --hash=sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5 \ + --hash=sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac \ + --hash=sha256:13e3afe97712e8887cd516e960c63f0b93122971e5b5e4b2622fe7701771e838 \ + --hash=sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90 \ + --hash=sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626 \ + --hash=sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4 \ + --hash=sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369 \ + --hash=sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b \ + --hash=sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e \ + --hash=sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee \ + --hash=sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1 \ + --hash=sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102 \ + --hash=sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8 \ + --hash=sha256:29880d17a8eb0b5cfdfd8944b468322928059aa35f1f5fa8ff22b149ec0b42f8 \ + --hash=sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9 \ + --hash=sha256:2e9cf9253119d8e5d111f05d71626786fd3d6193817316eab1ca088cdb8593cf \ + --hash=sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0 \ + --hash=sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031 \ + --hash=sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e \ + --hash=sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235 \ + --hash=sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072 \ + --hash=sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb \ + --hash=sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c \ + --hash=sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950 \ + --hash=sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2 \ + --hash=sha256:366ec70f5547c640d3ce1985722490f23faf4eb5216a7eeba78277490e78dacb \ + --hash=sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e \ + --hash=sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6 \ + --hash=sha256:3e5e1224c0a6a90e05843e07adfec669edebec17801c67072f51e59561d63c0b \ + --hash=sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2 \ + --hash=sha256:433c5a81eade63b47e522303bad236f59dba55ea6951746f5558355eeed8c75d \ + --hash=sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa \ + --hash=sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2 \ + --hash=sha256:494b70049a4d69aec6e8137c13af4cf8db8c9f9820a1392ac293b0dd2987a818 \ + --hash=sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032 \ + --hash=sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71 \ + --hash=sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96 \ + --hash=sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687 \ + --hash=sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8 \ + --hash=sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3 \ + --hash=sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61 \ + --hash=sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9 \ + --hash=sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1 \ + --hash=sha256:55261ac0d2941c42f196dd576f543d87a8ee03cd6f5e30dfb4d807b2e3b9121a \ + --hash=sha256:56490c595a28b1bb27dfc583e816152a9767721ef58b2c03b13f954d2f707420 \ + --hash=sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4 \ + --hash=sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65 \ + --hash=sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663 \ + --hash=sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f \ + --hash=sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591 \ + --hash=sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a \ + --hash=sha256:5ca0555312ae2fe82715cada7fac375530c2f3349e1eaa1bcb33d0283ac79a18 \ + --hash=sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e \ + --hash=sha256:5e2d0e146dcb57034f8b97dc58d2d512cb90aba253960ce449f695fec6a82c6f \ + --hash=sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7 \ + --hash=sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3 \ + --hash=sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c \ + --hash=sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3 \ + --hash=sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7 \ + --hash=sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96 \ + --hash=sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486 \ + --hash=sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3 \ + --hash=sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6 \ + --hash=sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b \ + --hash=sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731 \ + --hash=sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959 \ + --hash=sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9 \ + --hash=sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf \ + --hash=sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8 \ + --hash=sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e \ + --hash=sha256:789b8982559ae28dad2356519f841655756cdcd96616410590ae0b17454ee64f \ + --hash=sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885 \ + --hash=sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0 \ + --hash=sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506 \ + --hash=sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2 \ + --hash=sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0 \ + --hash=sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e \ + --hash=sha256:85de3134b5379856e323ba37c19c9256d39425f7b76a63af52b09fb4664c2e8f \ + --hash=sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e \ + --hash=sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491 \ + --hash=sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a \ + --hash=sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20 \ + --hash=sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449 \ + --hash=sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af \ + --hash=sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c \ + --hash=sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712 \ + --hash=sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7 \ + --hash=sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a \ + --hash=sha256:94fbf1c0c6cc0d3d5e50f9a9313a8cdca90dd696d34b381cd1704f8c9e939f20 \ + --hash=sha256:950f23cb393f85543777b0433f082cddd25b51ab398eac7971146495679efe5f \ + --hash=sha256:96eefc178f8636b9c760c5829345307fd81cfae9ab1e80997dbddeb0f54ee9a3 \ + --hash=sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9 \ + --hash=sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e \ + --hash=sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5 \ + --hash=sha256:994e883d17c559cdfd38c84003c8b27d25424a1077272a17e7cd27bfe0bf57b2 \ + --hash=sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36 \ + --hash=sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263 \ + --hash=sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4 \ + --hash=sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11 \ + --hash=sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a \ + --hash=sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3 \ + --hash=sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375 \ + --hash=sha256:a545775cfe815855ea32d7c27731d79da358ef2055b4a25830231b1622dd18aa \ + --hash=sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d \ + --hash=sha256:a6d095662e73e74f0a49988e0593373e243e3a52e27bfeea0a859e88acf4a0f5 \ + --hash=sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99 \ + --hash=sha256:a951ad59cad9145664a730d3036b40b844e74d2d3683da40111463cd3a83845d \ + --hash=sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c \ + --hash=sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488 \ + --hash=sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6 \ + --hash=sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc \ + --hash=sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b \ + --hash=sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f \ + --hash=sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00 \ + --hash=sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10 \ + --hash=sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598 \ + --hash=sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6 \ + --hash=sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962 \ + --hash=sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c \ + --hash=sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08 \ + --hash=sha256:ba2f37ee79e6338845261a3c5b1784e5d1acdff2c0785b284f1b633033d136ab \ + --hash=sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573 \ + --hash=sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90 \ + --hash=sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5 \ + --hash=sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18 \ + --hash=sha256:be47f99644b208bff7766314013f9acf57b056b04191d570d68ad14022cf5b1d \ + --hash=sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af \ + --hash=sha256:c1dcc36dcb96abc02236e182d17e0f71430152a6c2c7447421da2d2dc144edea \ + --hash=sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c \ + --hash=sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b \ + --hash=sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6 \ + --hash=sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8 \ + --hash=sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774 \ + --hash=sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004 \ + --hash=sha256:ce854f5f478050ade5a238731c4ca985a7d3b3cb53ff600a9b5c3b689b5f0a7a \ + --hash=sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a \ + --hash=sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2 \ + --hash=sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2 \ + --hash=sha256:d1ee1e296209fdce05b81b663250eefa02213a2da7b41bf26f7829b8ba3545aa \ + --hash=sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe \ + --hash=sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3 \ + --hash=sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc \ + --hash=sha256:e06efa066f7dbadbc84ebc126a97c452a6451dfcf589d89d788484949e1cf795 \ + --hash=sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d \ + --hash=sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc \ + --hash=sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893 \ + --hash=sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef \ + --hash=sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d \ + --hash=sha256:e9fbdce1e47394b09bc9f26ab117dfc8d6491977a11d86f592bb42c779db2fda \ + --hash=sha256:eb12fb2ba69ffa05f8695f61c69e591dc4b4a12ac3757ac8af8adb259bf56d17 \ + --hash=sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30 \ + --hash=sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7 \ + --hash=sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5 \ + --hash=sha256:f5542f9b941279d82d41eb0aa9f98eba36fe4df5c7086c651df7944935b37182 \ + --hash=sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f \ + --hash=sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9 \ + --hash=sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada \ + --hash=sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876 \ + --hash=sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a \ + --hash=sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348 \ + --hash=sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3 \ + --hash=sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f \ + --hash=sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0 \ + --hash=sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # requests +filelock==3.32.3 \ + --hash=sha256:0ffa185a3540854c95caa7fa76b76cb219d907415e2c5dc9af25fd970563487f \ + --hash=sha256:7f0ca4bcc0e181c60dbbd8aa9ab5b120ebb99e4e064e83636340056f833a1f09 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # torch + # transformers +fsspec==2026.7.0 \ + --hash=sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279 \ + --hash=sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # torch +hf-xet==1.6.0 \ + --hash=sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7 \ + --hash=sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675 \ + --hash=sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef \ + --hash=sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9 \ + --hash=sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3 \ + --hash=sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb \ + --hash=sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e \ + --hash=sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338 \ + --hash=sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d \ + --hash=sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d \ + --hash=sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765 \ + --hash=sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c \ + --hash=sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522 \ + --hash=sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f \ + --hash=sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a \ + --hash=sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b \ + --hash=sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub +huggingface-hub==0.36.2 \ + --hash=sha256:1934304d2fb224f8afa3b87007d58501acfda9215b334eed53072dd5e815ff7a \ + --hash=sha256:48f0c8eac16145dfce371e9d2d7772854a4f591bcb56c9cf548accf531d54270 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # tokenizers + # transformers +idna==3.19 \ + --hash=sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15 \ + --hash=sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # requests +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # jinja2 +mpmath==1.3.0 \ + --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ + --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # sympy +networkx==3.6.1 \ + --hash=sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509 \ + --hash=sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +numpy==2.4.6 \ + --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ + --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ + --hash=sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f \ + --hash=sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079 \ + --hash=sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096 \ + --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 \ + --hash=sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66 \ + --hash=sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d \ + --hash=sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1 \ + --hash=sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e \ + --hash=sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147 \ + --hash=sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd \ + --hash=sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75 \ + --hash=sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063 \ + --hash=sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73 \ + --hash=sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab \ + --hash=sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4 \ + --hash=sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41 \ + --hash=sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402 \ + --hash=sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698 \ + --hash=sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7 \ + --hash=sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8 \ + --hash=sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b \ + --hash=sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8 \ + --hash=sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0 \ + --hash=sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662 \ + --hash=sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91 \ + --hash=sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0 \ + --hash=sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f \ + --hash=sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3 \ + --hash=sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f \ + --hash=sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67 \ + --hash=sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6 \ + --hash=sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997 \ + --hash=sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b \ + --hash=sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e \ + --hash=sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538 \ + --hash=sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627 \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ + --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ + --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ + --hash=sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c \ + --hash=sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43 \ + --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ + --hash=sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8 \ + --hash=sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089 \ + --hash=sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778 \ + --hash=sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1 \ + --hash=sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb \ + --hash=sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261 \ + --hash=sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb \ + --hash=sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a \ + --hash=sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8 \ + --hash=sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359 \ + --hash=sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5 \ + --hash=sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7 \ + --hash=sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751 \ + --hash=sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8 \ + --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ + --hash=sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e \ + --hash=sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45 \ + --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ + --hash=sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895 \ + --hash=sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe \ + --hash=sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb \ + --hash=sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a \ + --hash=sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577 \ + --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ + --hash=sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a \ + --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ + --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \ + --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # transformers +nvidia-cublas-cu12==12.8.4.1 \ + --hash=sha256:47e9b82132fa8d2b4944e708049229601448aaad7e6f296f630f2d1a32de35af \ + --hash=sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142 \ + --hash=sha256:b86f6dd8935884615a0683b663891d43781b819ac4f2ba2b0c9604676af346d0 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # nvidia-cudnn-cu12 + # nvidia-cusolver-cu12 + # torch +nvidia-cuda-cupti-cu12==12.8.90 \ + --hash=sha256:4412396548808ddfed3f17a467b104ba7751e6b58678a4b840675c56d21cf7ed \ + --hash=sha256:bb479dcdf7e6d4f8b0b01b115260399bf34154a1a2e9fe11c85c517d87efd98e \ + --hash=sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cuda-nvrtc-cu12==12.8.93 \ + --hash=sha256:7a4b6b2904850fe78e0bd179c4b655c404d4bb799ef03ddc60804247099ae909 \ + --hash=sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994 \ + --hash=sha256:fc1fec1e1637854b4c0a65fb9a8346b51dd9ee69e61ebaccc82058441f15bce8 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cuda-runtime-cu12==12.8.90 \ + --hash=sha256:52bf7bbee900262ffefe5e9d5a2a69a30d97e2bc5bb6cc866688caa976966e3d \ + --hash=sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90 \ + --hash=sha256:c0c6027f01505bfed6c3b21ec546f69c687689aad5f1a377554bc6ca4aa993a8 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cudnn-cu12==9.10.2.21 \ + --hash=sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8 \ + --hash=sha256:c6288de7d63e6cf62988f0923f96dc339cea362decb1bf5b3141883392a7d65e \ + --hash=sha256:c9132cc3f8958447b4910a1720036d9eff5928cc3179b0a51fb6d167c6cc87d8 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cufft-cu12==11.3.3.83 \ + --hash=sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74 \ + --hash=sha256:7a64a98ef2a7c47f905aaf8931b69a3a43f27c55530c698bb2ed7c75c0b42cb7 \ + --hash=sha256:848ef7224d6305cdb2a4df928759dca7b1201874787083b6e7550dd6765ce69a + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cufile-cu12==1.13.1.3 \ + --hash=sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc \ + --hash=sha256:4beb6d4cce47c1a0f1013d72e02b0994730359e17801d395bdcbf20cfb3bb00a + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-curand-cu12==10.3.9.90 \ + --hash=sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9 \ + --hash=sha256:dfab99248034673b779bc6decafdc3404a8a6f502462201f2f31f11354204acd \ + --hash=sha256:f149a8ca457277da854f89cf282d6ef43176861926c7ac85b2a0fbd237c587ec + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cusolver-cu12==11.7.3.90 \ + --hash=sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450 \ + --hash=sha256:4a550db115fcabc4d495eb7d39ac8b58d4ab5d8e63274d3754df1c0ad6a22d34 \ + --hash=sha256:db9ed69dbef9715071232caa9b69c52ac7de3a95773c2db65bdba85916e4e5c0 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cusparse-cu12==12.5.8.93 \ + --hash=sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b \ + --hash=sha256:9a33604331cb2cac199f2e7f5104dfbb8a5a898c367a53dfda9ff2acb6b6b4dd \ + --hash=sha256:9b6c161cb130be1a07a27ea6923df8141f3c295852f4b260c65f18f3e0a091dc + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # nvidia-cusolver-cu12 + # torch +nvidia-cusparselt-cu12==0.7.1 \ + --hash=sha256:8878dce784d0fac90131b6817b607e803c36e629ba34dc5b433471382196b6a5 \ + --hash=sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623 \ + --hash=sha256:f67fbb5831940ec829c9117b7f33807db9f9678dc2a617fbe781cac17b4e1075 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-nccl-cu12==2.27.3 \ + --hash=sha256:9ddf1a245abc36c550870f26d537a9b6087fb2e2e3d6e0ef03374c6fd19d984f \ + --hash=sha256:adf27ccf4238253e0b826bce3ff5fa532d65fc42322c8bfdfaf28024c0fbe039 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-nvjitlink-cu12==12.8.93 \ + --hash=sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88 \ + --hash=sha256:adccd7161ace7261e01bb91e44e88da350895c270d23f744f0820c818b7229e7 \ + --hash=sha256:bd93fbeeee850917903583587f4fc3a4eafa022e34572251368238ab5e6bd67f + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # nvidia-cufft-cu12 + # nvidia-cusolver-cu12 + # nvidia-cusparse-cu12 + # torch +nvidia-nvtx-cu12==12.8.90 \ + --hash=sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f \ + --hash=sha256:619c8304aedc69f02ea82dd244541a83c3d9d40993381b3b590f1adaed3db41e \ + --hash=sha256:d7ad891da111ebafbf7e015d34879f7112832fc239ff0d7d776b6cb685274615 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # transformers +pillow==12.3.0 \ + --hash=sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756 \ + --hash=sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a \ + --hash=sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59 \ + --hash=sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45 \ + --hash=sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3 \ + --hash=sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df \ + --hash=sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139 \ + --hash=sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b \ + --hash=sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39 \ + --hash=sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e \ + --hash=sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8 \ + --hash=sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1 \ + --hash=sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8 \ + --hash=sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89 \ + --hash=sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5 \ + --hash=sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130 \ + --hash=sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd \ + --hash=sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d \ + --hash=sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b \ + --hash=sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed \ + --hash=sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace \ + --hash=sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb \ + --hash=sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931 \ + --hash=sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510 \ + --hash=sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6 \ + --hash=sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1 \ + --hash=sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce \ + --hash=sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385 \ + --hash=sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e \ + --hash=sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c \ + --hash=sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7 \ + --hash=sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace \ + --hash=sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c \ + --hash=sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f \ + --hash=sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64 \ + --hash=sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f \ + --hash=sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a \ + --hash=sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827 \ + --hash=sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17 \ + --hash=sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4 \ + --hash=sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a \ + --hash=sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701 \ + --hash=sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e \ + --hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 \ + --hash=sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66 \ + --hash=sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468 \ + --hash=sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217 \ + --hash=sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658 \ + --hash=sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418 \ + --hash=sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a \ + --hash=sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c \ + --hash=sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330 \ + --hash=sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402 \ + --hash=sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09 \ + --hash=sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930 \ + --hash=sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f \ + --hash=sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec \ + --hash=sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a \ + --hash=sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94 \ + --hash=sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468 \ + --hash=sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b \ + --hash=sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965 \ + --hash=sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8 \ + --hash=sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd \ + --hash=sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7 \ + --hash=sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c \ + --hash=sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777 \ + --hash=sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35 \ + --hash=sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9 \ + --hash=sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f \ + --hash=sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f \ + --hash=sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0 \ + --hash=sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c \ + --hash=sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71 \ + --hash=sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3 \ + --hash=sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838 \ + --hash=sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf \ + --hash=sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321 \ + --hash=sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26 \ + --hash=sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec \ + --hash=sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9 \ + --hash=sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65 \ + --hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \ + --hash=sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e \ + --hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \ + --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \ + --hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7 + # via -r benchmarks/blenderbench_direct/requirements-scoring.in +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # transformers +regex==2026.9.10 \ + --hash=sha256:030fa9e23624e39b3b94e46b90a5abd1a1678eb2f58fcdd3fd6c27526bf91c7e \ + --hash=sha256:032da15431c890d376f53547f0a6219f4f4cd19f3e4f11bdc321453b5bd207e4 \ + --hash=sha256:044bd4639b6bb409ec9e5d8b7accd57e02b4c4a4e2eafde916f8ae8006b3e40b \ + --hash=sha256:048a89ee797db10160bd2bd519286577a6b43a100279bd4b7d8456a3d69c80a0 \ + --hash=sha256:05fb018cfe7144585fc83882405906ff84994a2d154afc2509ecc7752c51f864 \ + --hash=sha256:07b45ba5c94b8fcb30cb6c56a11f715c57533a3017964504322ea52690a27b72 \ + --hash=sha256:0aa7589394230e0f0a422ab6b90841ff12c87e855e7aaf75d192a54a5f124548 \ + --hash=sha256:0acee94b480dd853e39434aa9a575f95385b1b4b8fa3feae56db363ca5cad782 \ + --hash=sha256:0b9ba3b2765cdfe18f0f561a69f78a69701f2896654a81c711108d35d14e5099 \ + --hash=sha256:0c32480f3371b75068decaf9e5da72c224e953830dd71e36e06cf80e30ea39d8 \ + --hash=sha256:1270cdec69248592bbe38a0b263ed58d907b891bd2b93703e225c317e421bda1 \ + --hash=sha256:13c52fc377792675f604a207a2ae5958c080f6854f7698d40d9ff034d95b1e76 \ + --hash=sha256:14caa05ce39ec70437af5aac8814c50ee6628f4a90353871c059692f448a164f \ + --hash=sha256:1562aabd9d4eb09bd88a62ad97ed06800094b529ac43419e43020b9cefec79b0 \ + --hash=sha256:175cf49ce7a994c88b8f15e3cb17cdb66a48ebb2d36de736b8205033db950f89 \ + --hash=sha256:1aa309ab7ba89a62d6cf70dbd38d4176440bce3c7001ab86256704cf4c18c6eb \ + --hash=sha256:1ad10a135fa0b4e4a462a61d07c6654d7518cfdb5cb8da08f9ff7d61384af1fe \ + --hash=sha256:1b891f77554bff991804cee24b78b40789f7d5993a24c7907bc7025fd2a70c8d \ + --hash=sha256:1e321e2c84f0e52c457f5ea5944f796d6e8e09cb99738ea98dcc1bfe402a128d \ + --hash=sha256:1e954e246466d5a1a78f563ce8364b5d7cb19e7adb0ccdec8f9c9610083187bc \ + --hash=sha256:1f0a8b4928823bc8b217a1ab7bf3d90598909dec9a70fbbfe9a52cc4eca55990 \ + --hash=sha256:1fbc8314436353e097c050e11b01a6c11433579437ed0579730157676ef59e2f \ + --hash=sha256:20e8bfb07ad79a282f8b95b56fe67f9750b1b7f775724e4ba1f23cb296115ce4 \ + --hash=sha256:217e98ba5fc8908ed8ffd4ebac04753a0c831067cbfb495b9821b94cc61eaa76 \ + --hash=sha256:239620b0e0681669367c0e218c8eb2551d9f8fe3b9fccfc8d0003377804e8348 \ + --hash=sha256:23ac9a28180f274d7dd7651fa131ad5b02d343b75df4b040737f0356223895dd \ + --hash=sha256:2479171edccced52ef02b899558f88ab2c235fe05b93180fdcae1670aacd89e1 \ + --hash=sha256:24d12a625a37c89c2b09303402a06942f55f071b95a7916a49c17034c3d47cd5 \ + --hash=sha256:2dd9286093c71afc8f55ef035c5b9d2776641fd72c6535f1febc92d0b0be9666 \ + --hash=sha256:2e67f8843f0e4b931f1fa860bf3bbe4134b714c0155cc5c7c0d7ea450230aae0 \ + --hash=sha256:31e4df2b11d48f61d511019bc1ee9b477055f17c352b68fe72db7a98b14d603c \ + --hash=sha256:3264132d576847ab5f88bb83e7debe67854bf165b3ea613bd467312b6099536a \ + --hash=sha256:3540734dbe241ebb3b87d5713781f6749a3e4d45480f506aa5fb5cbb0c37d249 \ + --hash=sha256:35ba3bab0c45079735f55ac61526774de1d84bc4a0333cc554e1a4ab74913924 \ + --hash=sha256:3a66e40a1a20de96a2fee00ed67e11012b62d85b277688258677fd19997addb7 \ + --hash=sha256:3bdeed3318a8eb2bbadc9c56347e0ff651639e934a47e168d05a3b12929fd0e7 \ + --hash=sha256:3fb4ae8cf83ef4e9addd43b2da31a9f45be816a8036fae8af59c8998b72718e2 \ + --hash=sha256:4971776b4f2bd7fd9a83eceb2cb2592cbe2924f639fe8045e6a9de5ba4bfcf25 \ + --hash=sha256:4a761ea45f2ad74c575ef5850ea514cef97302a552d3c7c9d1a1a870d4661d6c \ + --hash=sha256:4c66d54042a14a503907d81861b8a5235e6d1f03d4fbc1d8767f652eaf957ac1 \ + --hash=sha256:4db7d00c4afbfbb55b8e17b1e371da11418ea9389b030acec63c1fa4c7ad4b86 \ + --hash=sha256:4f0407474ffac8e5e89d93ca41d60891e29f0ab8423eb66ff292d850a86a0843 \ + --hash=sha256:53e182b6b04d0011909b47d51a2d72d908de07c7b1c7f16b3adda2204d723bc1 \ + --hash=sha256:5847e22bbf959764d776937d791d034cc2d19b787e361c88d97e859e8dc68502 \ + --hash=sha256:58c01f7b81079cf0817ba831ff4d9eff5d28be4a3ac76c353e6f09bd63f4c386 \ + --hash=sha256:58da726d3e766c0b3f5a3997dfaf0275898a1107b8191cdd6b0437fe45fd817d \ + --hash=sha256:5bef622850cf760154719d4e0d74b0a855962432995168e250069899ae12fe8f \ + --hash=sha256:5ccd139b2061132e7b265cfb4b4721baeb9f8928b81415304abf1ec7e3181c26 \ + --hash=sha256:5cef9f3d14796500ea834c41dbe688f1f6b23c7024dc23e8a794d7ebaf5d71d0 \ + --hash=sha256:63bb62cf62217dc38c8a6b2b61b165b0e4eb8fa93b0aba12139251c0986a8fa3 \ + --hash=sha256:681ed38664b64c6617d3c3c332018d1948c77e139c5ea667c1886efa671e426f \ + --hash=sha256:6888065672b341e5246f391ec16dc258a29218ac784172fd67c30d941544755b \ + --hash=sha256:6aebdd9a946de328b3f6f61dbf48dd064a36eb6dddf96e34ae6651d37f6e9383 \ + --hash=sha256:6afcad14310f1311d077553ed374b42a5e538f85a8c884b4e38e52de091c8077 \ + --hash=sha256:6b34a778c695d24e77c140e3b4c95da69282e34f2f6b02b55656aa4a0379f643 \ + --hash=sha256:6fd555fc9abef50c530869690b2daca054c8811a7aff632d11f9a7b2590b2742 \ + --hash=sha256:71879292c9c7ac67b1680345b16daba1be937cb027362cfa04e68f65db2dcfdd \ + --hash=sha256:75242f44a3e283106077be4ab717bc535e4701c9d54ad69e195945c22f137a1d \ + --hash=sha256:75aa39d3f4f1650eea84e46b0d8cefe77dd5478c10e3d0aaf0b0f00493475a7a \ + --hash=sha256:75f9297b16fcb588a1f8d8a55dabef3c0c20b0c7bac43c87ceaaaf1a825c12f4 \ + --hash=sha256:79e9432995e14c749d34209413de5e621ec8e67789bf4f46dbfabea9d06a2406 \ + --hash=sha256:7abb38b8c40f3a235235a44da452c64b7b5c1d650ec6351027db0e090804f2e5 \ + --hash=sha256:7dcad477c49c4c626a6c4fcd71b39a971aa217060cc40a6569fd24edcc0fa509 \ + --hash=sha256:7e6c0b5ec6ddee4032247585dc491b0fa58627745b66a705728703a3f0331231 \ + --hash=sha256:7f8f10015866608fe4c043cec2e4fe4c39a94bb50e45091de4cdf4004b9ae4b0 \ + --hash=sha256:866de9f98df0611d7b62b3a8729d3284a64c0cc6edd90bb95a533e443a4939cb \ + --hash=sha256:87f5f75c109f08f5c602d68e1af54cead8165189c727b6ac946b30b9833a3ba4 \ + --hash=sha256:880ac684c27176464c00c3fdc456116364f5ebc70da07aad0c2d4a7ba45e98db \ + --hash=sha256:88b02aa8d0ec9b6189fe933d425775882271c23700ac11fd26d1779b0f56fde3 \ + --hash=sha256:8ba1f78bd4fef2d8f84b894ec28ac3481afe6cc07aaa253ad4717ef7b3fe6bcb \ + --hash=sha256:8c07021a4faa3f092869adbd1f35cdc7a592276c807aeebc3ceb8ff1a638f0b4 \ + --hash=sha256:8d5c4518235a2ec1611e57af85fa488d529c1106aacff12adadcedf8687012cd \ + --hash=sha256:8e127d9a80cbf1c3276bb465c6d047e8705e97b58c2b8f2f0c0a69c336b44b37 \ + --hash=sha256:94c5ce3bc41d226b4eb89ca3f842b2e28c031487fb1f34eb2153d98235831325 \ + --hash=sha256:94d096369b7cd96d15343fef5257fe39eff9d0e8758b92a0e15e358b92cdb2fc \ + --hash=sha256:968c1e33edd9a104d1bf24c8d476c72de7e3839ae7f894b37e9e4f4739fdeeca \ + --hash=sha256:990797e765d89a423880052c68b61c31afe701de94a8c060f61c40605ca6c727 \ + --hash=sha256:9ce239acb15843ab03976626af810a4424b0409689ec2bbc52088ab5479ab487 \ + --hash=sha256:9d772586951d7d6a5d162d48f414065e483b1c81ab38fd8ed97c78b05883421a \ + --hash=sha256:9fbd2e5d8002dc49a6129fb321ec51c57a025e752ed525ddce0ba9223c4350a7 \ + --hash=sha256:a41693eb3fc4b92e6127d113813c6c395237f7edd3224abf67609af48c690d11 \ + --hash=sha256:abbfc1c33bf8efddcc43844aba61e036d74a918680dc3ce8ce2538b004eda0f9 \ + --hash=sha256:b298cdc33c5cc6969ff07f0fba19cc73e0fd8576373c50935feadaca2f6b4405 \ + --hash=sha256:b43456de605c8ee77eb75f07bc1ee44ba27f9cee22207deb77d495e954b7d953 \ + --hash=sha256:b71649169a9fcf30b395ee01047fa7ad6654a4c900ca75b23c04dedcce6a1f8c \ + --hash=sha256:b91c37551bf39d75116c02b146956f65b9aa0337a4a652f4ae186983789d4001 \ + --hash=sha256:b9d36b03dc362aa40ffaaec9d9bd75e87763529563ec008c43b0e07782f5be7a \ + --hash=sha256:bafa41b0dd63669e5c0f8adf3d24819efeb73c847f492eb011212eb352e69041 \ + --hash=sha256:bb7774924f8cd69f49cba0b3c2d679a6326f777e0e67d130ad5203e4df53f0d3 \ + --hash=sha256:bf29611e5376fec8f795879bb5c6153a76c3a292573d173c26784042b01eb840 \ + --hash=sha256:c014641157e9049b0603b8daa5343bd408d9b757b709aaa0f373cd3fab2d7944 \ + --hash=sha256:c103b3b14e011774af4fb7e4617ad4d72b9171905cd3b231a70a4efd76e477d7 \ + --hash=sha256:c22df8dd6373bbe3898e77429ffc85594300e39d752fd0e68a31e59d37899376 \ + --hash=sha256:c25a754bb81a2edcfc3b65eda50f017d736f818112ed43e8aafd595cb00678ae \ + --hash=sha256:c32818b28bcd153b25b63038348a9fe9b9fbcddb60df43f204c3ab55eeb57f77 \ + --hash=sha256:c37fa93bf18bf4f90b01c0fa9f11ea567ee4b7dd8bf96e63663e5edc37aa38cf \ + --hash=sha256:c3d95d7d9538b5b726dd6fcd7b6117a71e6565202f6d64f5845fb4d8f203f533 \ + --hash=sha256:c8fbd9cb30c68c1686b94029b9ef845d5870d3d65baf66cb126b676849b9d72b \ + --hash=sha256:cb76a9c4e07a6a47849726af0ed14c41741a182f097f134a8cf29c1bc0f4dde8 \ + --hash=sha256:ce7c118cb102975f974585688357a717ffbf9dddd64ab0bb1bc93eb5b367cf95 \ + --hash=sha256:cf377960d2ac37d987394a9dbaa75e91338c41a46d41e1d25e90125e7b3ee2dc \ + --hash=sha256:d278ad30ec83b6b9202685b0f80b741a51ea3ca7f0595ebda96e7628b6398876 \ + --hash=sha256:d2d377fd1cad611b806cdd732d86b65f536c768209890cb442556548daa65a23 \ + --hash=sha256:d414c411c06fe0009eac33488fb1591c66b5c2673e342e452e7bb2fe63da8194 \ + --hash=sha256:d8c668af8f7bdb1d18739c27d30cd9f4b371495a883f75a002fb7a39d740fecd \ + --hash=sha256:dce932f8e3ba936475ea3d0d8b59f7b050a9e206e994f53f8fd80299871e87da \ + --hash=sha256:debc629e98b95abaea1cf3057ca296151f348c697c9b8a59d18013adb302c0dd \ + --hash=sha256:e0dc78251154b66dc60211563fc115345da332eaa881e4e2523fb1edae3772f4 \ + --hash=sha256:e5e4a6e0734a685d13b9685622bb503bdbb2927f8b0df025a5085f0ea067475b \ + --hash=sha256:e6b99181d184d0f5c7b36b8d12b94d1e9499cce6246594331f9edc5d2ea9fceb \ + --hash=sha256:e7327795089ddb44912dce1434e1d7244be2e9fb48fcc2d6782936af7a3062db \ + --hash=sha256:ebb2ba68e4641a994061f70bf44ed448fba0b9b1d18c94ffb9efc1cca805b39b \ + --hash=sha256:ec8855f08c17895a26fbf5f19ed829722e19b34a96629e49a43c92974924026b \ + --hash=sha256:ecb2e7acb18f8cc4a67f0ad986c0af291ea4dd385d0614ba9bc09d7f8bbb478c \ + --hash=sha256:ef4c0a9dfdc90581b90b1b95a8c3d1557f8ff8f5a2a53536d26314de699d1468 \ + --hash=sha256:ef4ce69ff97fbb44b46751cfea5e859ad0b66d1a50abf34954f0645f51e81671 \ + --hash=sha256:ef5a059ea1c6ee5d1c7e99a2484e628608d010921efe876c6f0e2029d2f35eca \ + --hash=sha256:f0e2e5d23448b660d60a6ed85c46cc03b4b48bd276b8f4041d4a5fe2a4a0626b \ + --hash=sha256:f2374c27deb189b282ec7e16106752c22ad39b056bbd8018960b1e4cc95d67a1 \ + --hash=sha256:f2f43bf4e47ff7ce9e585558706d698c6204d0f80bf2207766382ed817c8e9f4 \ + --hash=sha256:f5c629df03adec31ee505dda3c8988f106c9390e4cbd343600036eb8b3d6724f \ + --hash=sha256:f70b9f0e39c2dba1d9da6bf7ef7c377cad7277f8440e9a69be05ede529ff024c \ + --hash=sha256:f7d4656e17ab736e9415a6442a345bfc97bb8b7dcce47884bb74a37f70f08d0c \ + --hash=sha256:f8bdec659a8fa7af51a32b224b3b7c02bc415d54ffd35187b1d224176b17d607 \ + --hash=sha256:faa911fbbcf8ac90bda0e0657d60768e3390954ef0588211d63a22add1cb1cd1 \ + --hash=sha256:fbc4e2f3cb7ce8436154e6483079e7d35eeb321a952fa936e180300630d8b873 \ + --hash=sha256:fd6bd89b9fc06018d35851cab0240adb7dd84d51941b19f6574ac90cd54e3ae5 \ + --hash=sha256:ff4d7b14ea19e50c8d9d6d83f45bd9b45cbb624c07ac1fa54db0a019049abed7 \ + --hash=sha256:ff6b3267318661dfddf6b3628663e00e5946bd0a5c8fa678537a1401f0388f91 \ + --hash=sha256:ffc2da104e43db716ce30cef9f28049a1faa6aca385dd8771b033268d0730b07 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # transformers +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # transformers +safetensors==0.8.0 \ + --hash=sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358 \ + --hash=sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f \ + --hash=sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d \ + --hash=sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d \ + --hash=sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0 \ + --hash=sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc \ + --hash=sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235 \ + --hash=sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98 \ + --hash=sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4 \ + --hash=sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846 \ + --hash=sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca \ + --hash=sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0 \ + --hash=sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25 \ + --hash=sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452 \ + --hash=sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d \ + --hash=sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78 \ + --hash=sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # transformers +setuptools==78.1.0 \ + --hash=sha256:18fd474d4a82a5f83dac888df697af65afa82dec7323d09c3e37d1f14288da54 \ + --hash=sha256:3e386e96793c8702ae83d17b853fb93d3e09ef82ec62722e61da5cd22376dcd8 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch + # triton +sympy==1.14.0 \ + --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ + --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +tokenizers==0.21.4 \ + --hash=sha256:1340ff877ceedfa937544b7d79f5b7becf33a4cfb58f89b3b49927004ef66f78 \ + --hash=sha256:2107ad649e2cda4488d41dfd031469e9da3fcbfd6183e74e4958fa729ffbf9c6 \ + --hash=sha256:2ccc10a7c3bcefe0f242867dc914fc1226ee44321eb618cfe3019b5df3400133 \ + --hash=sha256:39b376f5a1aee67b4d29032ee85511bbd1b99007ec735f7f35c8a2eb104eade5 \ + --hash=sha256:3c1f4317576e465ac9ef0d165b247825a2a4078bcd01cba6b54b867bdf9fdd8b \ + --hash=sha256:3c73012da95afafdf235ba80047699df4384fdc481527448a078ffd00e45a7d9 \ + --hash=sha256:475d807a5c3eb72c59ad9b5fcdb254f6e17f53dfcbb9903233b0dfa9c943b597 \ + --hash=sha256:51b7eabb104f46c1c50b486520555715457ae833d5aee9ff6ae853d1130506ff \ + --hash=sha256:5e2f601a8e0cd5be5cc7506b20a79112370b9b3e9cb5f13f68ab11acd6ca7d60 \ + --hash=sha256:6c42a930bc5f4c47f4ea775c91de47d27910881902b0f20e4990ebe045a415d0 \ + --hash=sha256:714b05b2e1af1288bd1bc56ce496c4cebb64a20d158ee802887757791191e6e2 \ + --hash=sha256:c212aa4e45ec0bb5274b16b6f31dd3f1c41944025c2358faaa5782c754e84c24 \ + --hash=sha256:cc88bb34e23a54cc42713d6d98af5f1bf79c07653d24fe984d2d695ba2c922a2 \ + --hash=sha256:f23186c40395fc390d27f519679a58023f368a0aad234af145e0f39ad1212732 \ + --hash=sha256:fa23f85fbc9a02ec5c6978da172cdcbac23498c3ca9f3645c5c68740ac007880 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # transformers +torch @ https://download.pytorch.org/whl/cu128/torch-2.8.0%2Bcu128-cp313-cp313-manylinux_2_28_x86_64.whl \ + --hash=sha256:3a852369a38dec343d45ecd0bc3660f79b88a23e0c878d18707f7c13bf49538f + # via -r benchmarks/blenderbench_direct/requirements-scoring.in +tqdm==4.70.0 \ + --hash=sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220 \ + --hash=sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # transformers +transformers==4.55.4 \ + --hash=sha256:574a30559bc273c7a4585599ff28ab6b676e96dc56ffd2025ecfce2fd0ab915d \ + --hash=sha256:df28f3849665faba4af5106f0db4510323277c4bb595055340544f7e59d06458 + # via -r benchmarks/blenderbench_direct/requirements-scoring.in +triton==3.4.0 \ + --hash=sha256:00ea4a1c15f079bbf49ff02c312210fdb8ff05b1503ea30812610e2a3b171f3b \ + --hash=sha256:0e582bfd8147afd0e17f410b39ee161df933a2aca9f653e1178daae98f87f601 \ + --hash=sha256:1b1389a284a8c5f29749f643e3b1fc7513e3d162ca6d50f4e3d658de7dba631b \ + --hash=sha256:3d5658665a16be28d74ee5775408e4fd391bcbb0162ad5a3503b4f0c99778522 \ + --hash=sha256:40b5d803a83b5e70895424a51c3a1a23dcc0f158a79f682aaa2c940903e747b9 \ + --hash=sha256:5acb528f01dc14362c4a88fa4dcfa227d132f6e61f272a6874e84d6485037ca4 \ + --hash=sha256:5f2f15de16fefacac008378919687c0ada6a867dcc0fe70a5d5a11240ae8b814 \ + --hash=sha256:6b17588c59cbe1d987133ea7cd1cc2e5229e0544a06a9908bfaf68c61f56495a \ + --hash=sha256:8550672b1184f05187f4140db32e33e61b592046fd3e1eb907e3b7db5321b750 \ + --hash=sha256:86eb0cb7634328cd577fa51b59338f1af4199cf9ab62385ac42a82783d9116a4 \ + --hash=sha256:dd8d32d337f58124fe254043d282a4bc133c2aca7989bc6358b2c546d2121513 \ + --hash=sha256:e2b0afe420d202d96f50b847d744a487b780567975455e56f64b061152ee9554 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # torch +urllib3==2.7.0 \ + --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ + --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # requests diff --git a/benchmarks/blenderbench_direct/runner.py b/benchmarks/blenderbench_direct/runner.py new file mode 100644 index 0000000..e32aeac --- /dev/null +++ b/benchmarks/blenderbench_direct/runner.py @@ -0,0 +1,159 @@ +"""Publication BlenderBench CLI and explicit portable configuration.""" +from __future__ import annotations + +import argparse +from dataclasses import asdict, dataclass +import json +from pathlib import Path +import sys +from typing import Sequence + +from . import dataset, protocol, scoring, video + + +@dataclass(frozen=True) +class RunConfig: + dataset_root: Path + mcp_toml: Path + codex_binary: Path + codex_version: str + bpy_python: Path + bpy_version: str + clip_snapshot: Path + output_root: Path + model: str + model_revision: str | None + rounds: int + tasks: tuple[str, ...] + + +def _path(value: str) -> Path: + return Path(value).expanduser().absolute() + + +def _external(path: Path) -> Path: + repository = Path(__file__).resolve().parents[2] + candidate = path.expanduser().resolve() + if candidate == repository or repository in candidate.parents: + raise ValueError(f"dataset/output roots must stay outside this repository: {candidate}") + return candidate + + +def _run_options(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--dataset-root", type=_path, required=True) + parser.add_argument("--mcp-toml", type=_path, required=True) + parser.add_argument("--codex-binary", type=_path, required=True) + parser.add_argument("--codex-version", required=True, help="Exact output expected from CODEx_BINARY --version") + parser.add_argument("--bpy-python", type=_path, required=True) + parser.add_argument("--bpy-version", required=True, help="Exact bpy.app.version_string prefix required") + parser.add_argument("--clip-snapshot", type=_path, required=True, + help="Offline CLIP snapshot matching clip-snapshot-manifest.json") + parser.add_argument("--output-root", type=_path, required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--model-revision") + parser.add_argument("--rounds", type=int, choices=[10], default=10, + help="Frozen publication protocol: exactly 10 rounds") + parser.add_argument("--tasks", nargs="+", help="Explicit unique subset; omit for all 27 tasks") + parser.add_argument("--approve-blender-tools", action="store_true") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + download = commands.add_parser("download", help="Download and verify the pinned 27-task dataset") + download.add_argument("--dataset-root", type=_path, required=True) + download.add_argument("--tasks", nargs="+") + run = commands.add_parser("run", help="Fresh generation; never resumes or regenerates existing output") + _run_options(run) + recover = commands.add_parser("recover", help="Audit/score saved output only; never generate") + recover.add_argument("--source-root", type=_path, required=True) + recover.add_argument("--output-root", type=_path, required=True, help="Fresh recovery evidence root") + recover.add_argument("--dataset-root", type=_path, required=True) + recover.add_argument("--bpy-python", type=_path, required=True) + recover.add_argument("--bpy-version", required=True) + recover.add_argument("--clip-snapshot", type=_path, required=True) + recover.add_argument("--tasks", nargs="+") + recover.add_argument("--rounds", type=int, choices=[10], default=10) + score = commands.add_parser("score", help="Score saved renders only; never generate") + score.add_argument("--run-root", type=_path, required=True) + score.add_argument("--dataset-root", type=_path, required=True) + score.add_argument("--clip-snapshot", type=_path, required=True) + score.add_argument("--output", type=_path) + score.add_argument("--tasks", nargs="+") + score.add_argument("--rounds", type=int, choices=[10], default=10) + movie = commands.add_parser("video", help="Generate a labelled compilation video from a JSON spec") + movie.add_argument("--spec", type=_path, required=True) + movie.add_argument("--output", type=_path, required=True) + movie.add_argument("--ffmpeg") + movie.add_argument("--fps", type=int, default=24) + movie.add_argument("--seconds-per-task", type=float, default=3.0) + return parser + + +def resolve_config(args: argparse.Namespace, *, check_files: bool = True) -> RunConfig: + selected = tuple(dataset.select_tasks(args.tasks)) + args.dataset_root = _external(args.dataset_root) + args.output_root = _external(args.output_root) + config = RunConfig(dataset_root=args.dataset_root, mcp_toml=args.mcp_toml, + codex_binary=args.codex_binary, codex_version=args.codex_version, + bpy_python=args.bpy_python, bpy_version=args.bpy_version, + clip_snapshot=args.clip_snapshot, + output_root=args.output_root, model=args.model, model_revision=args.model_revision, + rounds=args.rounds, tasks=selected) + if (not config.model.strip() or config.codex_version != protocol.CODEX_VERSION + or not config.bpy_version.strip()): + raise ValueError("model and exact Codex 0.154.0/bpy pins are required") + if check_files: + for name in ("mcp_toml", "codex_binary", "bpy_python"): + if not getattr(config, name).is_file(): + raise FileNotFoundError(f"--{name.replace('_', '-')} is not a file: {getattr(config, name)}") + scoring.verify_clip_snapshot(config.clip_snapshot) + scoring.preflight_scorer_runtime() + for task in config.tasks: + dataset.verify(task, config.dataset_root) + return config + + +def _score(args: argparse.Namespace) -> int: + args.run_root = _external(args.run_root) + args.dataset_root = _external(args.dataset_root) + if args.output is not None: + args.output = _external(args.output) + tasks = dataset.select_tasks(args.tasks) + result = scoring.score_saved_run(args.run_root, args.dataset_root, tasks, args.rounds, + args.clip_snapshot) + json_path, csv_path = scoring.write_outputs(result, args.output or args.run_root / "scoring") + print(json.dumps({"json": str(json_path), "csv": str(csv_path), **result["summary"]})) + return int(result["summary"]["failed_tasks"] != 0) + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.command == "download": + args.dataset_root = _external(args.dataset_root) + for task in dataset.select_tasks(args.tasks): + print(dataset.stage(task, args.dataset_root)) + return 0 + if args.command == "score": + return _score(args) + if args.command == "video": + args.output = _external(args.output) + video.generate(args.spec, args.output, fps=args.fps, + seconds_per_task=args.seconds_per_task, ffmpeg=args.ffmpeg) + print(args.output) + return 0 + if args.command == "recover": + args.source_root = _external(args.source_root) + args.output_root = _external(args.output_root) + args.dataset_root = _external(args.dataset_root) + from .suite import recover_suite + return recover_suite(args) + if args.command == "run": + config = resolve_config(args) + from .suite import run_suite + return run_suite(config, approved=args.approve_blender_tools) + raise AssertionError(args.command) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/blenderbench_direct/score_pair.py b/benchmarks/blenderbench_direct/score_pair.py new file mode 100644 index 0000000..459630a --- /dev/null +++ b/benchmarks/blenderbench_direct/score_pair.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Pinned GPU image-pair scorer using VIGA's exact metric functions. + +Place the pinned VIGA ``ref_based_eval.py`` beside this script. This wrapper +hash-verifies it, extracts only three metric functions, and never runs VIGA's +agent or judge. Model weights must exist in the byte-verified offline snapshot supplied by the +parent runner. No Hub lookup is performed. +""" +from __future__ import annotations + +import ast +import hashlib +import json +import os +from pathlib import Path +import sys + +os.environ.setdefault("HF_HUB_OFFLINE", "1") + +import numpy as np +from PIL import Image +import torch +from transformers import CLIPModel, CLIPProcessor + +VIGA_REVISION = "69cb8ef0651bf68124815682df4a0f8e8b57c141" +EVALUATOR_SHA256 = "9ee08818aa858dccfff3efbb36cb9a98e8887f14e6168ad1a23731351cdeb3b5" +CLIP_REVISION = "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268" +EVALUATOR_SOURCE_URL = ("https://github.com/Fugtemypt123/VIGA-release/blob/" + + VIGA_REVISION + "/evaluators/blenderbench/ref_based_eval.py") + + +def main(argv: list[str] | None = None) -> int: + args = sys.argv[1:] if argv is None else argv + if len(args) != 2: + raise SystemExit("usage: score_pair.py IMAGE TARGET") + source = Path(__file__).with_name("ref_based_eval.py") + if hashlib.sha256(source.read_bytes()).hexdigest() != EVALUATOR_SHA256: + raise ValueError(f"evaluator bytes do not match pinned VIGA source: {EVALUATOR_SOURCE_URL}") + if not torch.cuda.is_available(): + raise RuntimeError("CUDA unavailable; publication scorer has no CPU fallback") + snapshot = os.environ.get("BLENDERBENCH_CLIP_SNAPSHOT") + if not snapshot or not Path(snapshot).is_dir(): + raise ValueError("verified BLENDERBENCH_CLIP_SNAPSHOT is required") + model = CLIPModel.from_pretrained(snapshot, local_files_only=True, + use_safetensors=False, weights_only=True).to("cuda").eval() + processor = CLIPProcessor.from_pretrained(snapshot, local_files_only=True, use_fast=False) + + def cuda_processor(*positional, **keywords): + return processor(*positional, **keywords).to("cuda") + + namespace = {"np": np, "torch": torch, "Image": Image, "GLOBAL_CLIP_MODEL": model, + "GLOBAL_CLIP_PROCESSOR": cuda_processor} + tree = ast.parse(source.read_text(encoding="utf-8")) + for name in ("ensure_clip_loaded", "clip_similarity", "photometric_loss"): + node = next(item for item in tree.body if isinstance(item, ast.FunctionDef) and item.name == name) + exec(compile(ast.Module(body=[node], type_ignores=[]), str(source), "exec"), namespace) + with Image.open(args[0]) as image: + candidate = image.convert("RGB"); candidate.load() + with Image.open(args[1]) as image: + target = image.convert("RGB"); target.load() + cosine = float(namespace["clip_similarity"](candidate, target)) + loss = float(namespace["photometric_loss"](candidate, target)) + if not np.isfinite(cosine) or not np.isfinite(loss): + raise ValueError("nonfinite evaluator output") + print(json.dumps({"clip_image_cosine_similarity": cosine, "nclip": 1.0 - cosine, + "photometric_loss": loss, "clip_model": "openai/clip-vit-base-patch32", + "clip_revision": CLIP_REVISION, "viga_revision": VIGA_REVISION, + "evaluator_sha256": EVALUATOR_SHA256, "device": str(next(model.parameters()).device), + "gpu": torch.cuda.get_device_name(0), "torch_version": torch.__version__, + "transformers_version": __import__("transformers").__version__, + "numpy_version": np.__version__, "pillow_version": __import__("PIL").__version__}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/blenderbench_direct/scoring.py b/benchmarks/blenderbench_direct/scoring.py new file mode 100644 index 0000000..44bc1e8 --- /dev/null +++ b/benchmarks/blenderbench_direct/scoring.py @@ -0,0 +1,436 @@ +"""Portable post-generation BlenderBench scoring and aggregation.""" +from __future__ import annotations + +from collections import defaultdict +import csv +import hashlib +import json +import math +import os +from pathlib import Path +import re +import subprocess +from typing import Iterable + +from . import dataset, protocol + + +CANONICAL_SCORER = Path(__file__).with_name("score_pair.py") +SCORING_LOCK = Path(__file__).with_name("requirements-scoring.lock") +CLIP_SNAPSHOT_MANIFEST = Path(__file__).with_name("clip-snapshot-manifest.json") +_ALLOWED_BOOTSTRAP_DISTRIBUTIONS = {"pip"} +_SCORER_ENV_KEYS = {"CUDA_VISIBLE_DEVICES", "HOME"} +_EXPECTED_RUNTIME = { + "python_version": "3.13.15", "python_compiler": "Clang 22.1.3", + "implementation": "CPython", "cache_tag": "cpython-313", + "soabi": "cpython-313-x86_64-linux-gnu", "system": "Linux", "machine": "x86_64", + "bits": 64, "torch_version": "2.8.0+cu128", "torch_cuda": "12.8", + "transformers_version": "4.55.4", "numpy_version": "2.4.6", "pillow_version": "12.3.0", +} + + +def _normalized_distribution(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def locked_distributions(lock: Path = SCORING_LOCK) -> dict[str, str]: + """Read the exact distributions from the generated hash lock.""" + expected: dict[str, str] = {} + for line in Path(lock).read_text(encoding="utf-8").splitlines(): + if not line or line[0].isspace() or line.startswith("#"): + continue + requirement = line.removesuffix(" \\") + if "==" in requirement: + name, version = requirement.split("==", 1) + elif " @ " in requirement: + name, url = requirement.split(" @ ", 1) + filename = __import__("urllib.parse", fromlist=["unquote"]).unquote(url).rsplit("/", 1)[-1] + match = re.search(r"-([0-9][A-Za-z0-9.+]*)-cp\d", filename) + if match is None: + raise ValueError(f"cannot derive locked version for {name}") + version = match.group(1).replace("%2B", "+") + else: + raise ValueError(f"unsupported scoring lock requirement: {requirement}") + normalized = _normalized_distribution(name.strip()) + if normalized in expected: + raise ValueError(f"duplicate scoring lock distribution: {normalized}") + expected[normalized] = version.strip() + if not expected: + raise ValueError("scoring lock has no distributions") + return expected + + +def _isolated_env(clip_snapshot: Path | None = None) -> dict[str, str]: + result = {key: value for key, value in os.environ.items() if key in _SCORER_ENV_KEYS} + result.update(PYTHONNOUSERSITE="1", PYTHONSAFEPATH="1", PYTHONDONTWRITEBYTECODE="1", + HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1") + if clip_snapshot is not None: + result["BLENDERBENCH_CLIP_SNAPSHOT"] = str(Path(clip_snapshot).absolute()) + return result + + +def verify_clip_snapshot(snapshot: Path) -> dict: + """Verify the complete offline snapshot allowlist, sizes, and SHA-256 bytes.""" + root = Path(snapshot).expanduser().absolute() + manifest_bytes = CLIP_SNAPSHOT_MANIFEST.read_bytes() + manifest = json.loads(manifest_bytes) + if manifest.get("revision") != protocol.CLIP_REVISION or manifest.get("schema_version") != 1: + raise ValueError("CLIP snapshot manifest identity is invalid") + expected = {row["path"]: row for row in manifest.get("files", [])} + if len(expected) != 8 or any(Path(name).name != name for name in expected): + raise ValueError("CLIP snapshot manifest allowlist is invalid") + if not root.is_dir(): + raise ValueError("offline CLIP snapshot directory is missing") + actual = {entry.name for entry in root.iterdir()} + if actual != set(expected): + raise ValueError(f"CLIP snapshot entries differ from exact allowlist: {sorted(actual ^ set(expected))}") + cache_root = root.parent.parent if root.parent.name == "snapshots" else None + verified = [] + for name in sorted(expected): + path = root / name + if not path.is_file(): + raise ValueError(f"CLIP snapshot file is missing: {name}") + if path.is_symlink(): + resolved = path.resolve(strict=True) + if cache_root is None or cache_root.resolve() not in resolved.parents: + raise ValueError(f"CLIP snapshot symlink escapes declared Hugging Face cache: {name}") + row = expected[name] + size, sha256 = path.stat().st_size, dataset.sha256(path) + if size != row["size"] or sha256 != row["sha256"]: + raise ValueError(f"CLIP snapshot bytes differ from manifest: {name}") + verified.append({"path": name, "size": size, "sha256": sha256}) + return {"repository": manifest["repository"], "revision": manifest["revision"], + "manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(), "files": verified} + + +def _runtime_probe() -> str: + return ( + "import importlib.metadata as m,json,os,platform,struct,sys,sysconfig;" + "import ctypes,numpy,PIL,torch,transformers;" + "c=ctypes.CDLL('libcuda.so.1');v=ctypes.c_int();" + "assert c.cuInit(0)==0 and c.cuDriverGetVersion(ctypes.byref(v))==0;" + "d=[(x.metadata.get('Name'),x.version) for x in m.distributions()];" + "f=sys.flags;print(json.dumps({'implementation':platform.python_implementation()," + "'python_version':platform.python_version(),'python_compiler':platform.python_compiler().strip()," + "'python_build':platform.python_build()," + "'cache_tag':sys.implementation.cache_tag,'soabi':sysconfig.get_config_var('SOABI')," + "'system':platform.system(),'machine':platform.machine(),'bits':struct.calcsize('P')*8," + "'lexical_executable':sys.executable,'resolved_executable':os.path.realpath(sys.executable)," + "'prefix':sys.prefix,'base_prefix':sys.base_prefix,'isolated':f.isolated," + "'ignore_environment':f.ignore_environment,'no_user_site':f.no_user_site," + "'safe_path':f.safe_path,'distributions':d,'torch_version':torch.__version__," + "'torch_cuda':torch.version.cuda,'cuda_available':torch.cuda.is_available()," + "'gpu':torch.cuda.get_device_name(0),'cuda_driver_api_version':v.value," + "'transformers_version':transformers.__version__,'numpy_version':numpy.__version__," + "'pillow_version':PIL.__version__},sort_keys=True))" + ) + + +def preflight_scorer_runtime(lock: Path = SCORING_LOCK) -> dict: + """Local reproducibility preflight for the lexical runner venv; not host attestation.""" + lexical = Path(__import__("sys").executable).expanduser().absolute() + process = subprocess.run([str(lexical), "-I", "-c", _runtime_probe()], capture_output=True, + text=True, cwd=CANONICAL_SCORER.parent, env=_isolated_env()) + if process.returncode: + raise ValueError(f"scorer runtime preflight failed: {process.stderr.strip()}") + try: + identity = json.loads(process.stdout) + except json.JSONDecodeError as exc: + raise ValueError("scorer runtime preflight was not valid JSON") from exc + if not isinstance(identity, dict) or any(identity.get(key) != value + for key, value in _EXPECTED_RUNTIME.items()): + raise ValueError("runner is not the exact historical CPython 3.13.15 scoring runtime") + prefix = Path(identity.get("prefix", "")).absolute() + if (identity.get("base_prefix") == identity.get("prefix") + or lexical.parent != prefix / "bin" + or Path(identity.get("lexical_executable", "")).absolute() != lexical + or any(identity.get(key) != expected for key, expected in + {"isolated": 1, "ignore_environment": 1, "no_user_site": 1, + "safe_path": True, "cuda_available": True}.items())): + raise ValueError("scorer must run from the lexical isolated CUDA-enabled publication venv") + installed: dict[str, str] = {} + for entry in identity.get("distributions", []): + if not isinstance(entry, list) or len(entry) != 2 or not all(isinstance(x, str) for x in entry): + raise ValueError("scorer distribution preflight is malformed") + name = _normalized_distribution(entry[0]) + if name in installed: + raise ValueError(f"duplicate installed scoring distribution: {name}") + installed[name] = entry[1] + expected = locked_distributions(lock) + mismatches = {name: {"expected": version, "installed": installed.get(name)} + for name, version in expected.items() if installed.get(name) != version} + extras = sorted(set(installed) - set(expected) - _ALLOWED_BOOTSTRAP_DISTRIBUTIONS) + if mismatches or extras: + raise ValueError(f"scorer distributions do not exactly match scoring lock: " + f"mismatches={mismatches}, extras={extras}") + return {"trust_boundary": "local reproducibility preflight; not remote or hostile-host attestation", + "remote_attestation": False, **{key: identity[key] for key in identity if key != "distributions"}, + "executable_sha256": dataset.sha256(lexical), + "scoring_lock_sha256": dataset.sha256(Path(lock)), "locked_distributions": expected, + "excluded_bootstrap_distributions": {name: installed[name] for name in sorted(installed) + if name in _ALLOWED_BOOTSTRAP_DISTRIBUTIONS}, + "invocation": {"argv_prefix": [str(lexical), "-I"], "isolated": True}} + + +def scorer_sha256() -> str: + if CANONICAL_SCORER.is_symlink() or not CANONICAL_SCORER.is_file(): + raise ValueError("canonical score_pair.py must be a regular repository file") + return dataset.sha256(CANONICAL_SCORER) + + +def scoring_provenance(runtime_preflight: dict, clip_snapshot: dict) -> dict: + return {"dataset_revision": dataset.DATASET_REVISION, + "evaluator_repository": protocol.EVALUATOR_REPOSITORY, + "evaluator_revision": protocol.EVALUATOR_REVISION, + "evaluator_sha256": protocol.EVALUATOR_SHA256, + "clip_model": protocol.CLIP_MODEL, "clip_revision": protocol.CLIP_REVISION, + "clip_snapshot": clip_snapshot, "scorer_sha256": scorer_sha256(), + "scorer_runtime": runtime_preflight, + "identity_validation": "outputs checked after local runtime and byte-manifest preflight"} + + +def verified_audit_images(task_root: Path, rounds: int, bpy_version: str) -> dict[int, Path]: + """Admit only renders recorded after successful independent scene validation.""" + task_root = Path(task_root).absolute() + admission_path = task_root / "audit-admission.json" + if admission_path.is_symlink() or not admission_path.is_file(): + raise ValueError("standalone scoring requires audit-admission.json") + try: + rows = json.loads(admission_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError("audit admission is unreadable") from exc + if (not isinstance(rows, list) or [row.get("round") for row in rows] + != list(range(1, rounds + 1))): + raise ValueError("audit admission rounds are incomplete") + images = {} + for number, row in enumerate(rows, 1): + image = task_root / f"audit{number:02}" / "verified.png" + metadata_path = image.parent / "metadata.json" + blend = task_root / "work" / f"iteration{number:02}.blend" + if metadata_path.is_symlink() or not metadata_path.is_file(): + raise ValueError(f"audit metadata is missing for round {number}") + try: + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"audit metadata is unreadable for round {number}") from exc + runtime = metadata.get("runtime", {}) + evidence = metadata.get("evidence", {}) + devices = evidence.get("devices", []) if isinstance(evidence, dict) else [] + if (image.is_symlink() or blend.is_symlink() or not image.is_file() or not blend.is_file() + or row.get("meaningful_change_validated") is not True + or row.get("image") != str(image) or row.get("source") != str(blend) + or row.get("image_sha256") != dataset.sha256(image) + or row.get("source_sha256") != dataset.sha256(blend) + or metadata.get("png") != str(image) or metadata.get("source") != str(blend) + or metadata.get("png_sha256") != row.get("image_sha256") + or metadata.get("source_sha256") != row.get("source_sha256") + or not isinstance(metadata.get("audit"), dict) + or row.get("audit_sha256") != protocol.digest(metadata["audit"]) + or not isinstance(runtime, dict) or runtime.get("returncode") != 0 + or not isinstance(evidence, dict) or evidence.get("blender_version") != bpy_version + or evidence.get("compute_device_type") != "CUDA" + or evidence.get("scene_device") != "GPU" + or evidence.get("engine") != "CYCLES" + or not any(isinstance(device, dict) and device.get("type") == "CUDA" + and device.get("use") is True for device in devices) + or any(isinstance(device, dict) and device.get("use") is True + and device.get("type") != "CUDA" for device in devices)): + raise ValueError(f"audit admission bytes are invalid for round {number}") + images[number] = image + return images + + +def _normalized(row: dict) -> dict: + result = dict(row) + if result.get("status") == "complete": + cosine = float(result["clip_image_cosine_similarity"]) + loss = float(result["photometric_loss"]) + if not math.isfinite(cosine) or not math.isfinite(loss): + raise ValueError("score metrics must be finite") + result["clip_image_cosine_similarity"] = cosine + result["nclip"] = 1.0 - cosine + result["photometric_loss"] = loss + return result + + +def aggregate(rows: Iterable[dict], expected_tasks: Iterable[str], expected_rounds: int | None = None, + provenance: dict | None = None) -> dict: + grouped: dict[str, list[dict]] = defaultdict(list) + for raw in rows: + row = _normalized(raw) + grouped[row["task"]].append(row) + tasks = [] + for task in expected_tasks: + task_rows = sorted(grouped.get(task, []), key=lambda row: row.get("round", 0)) + successful = [row for row in task_rows if row.get("status") == "complete"] + failed = [row for row in task_rows if row.get("status") != "complete"] + if not task_rows: + status = "missing" + elif expected_rounds is not None and len(successful) != expected_rounds: + status = "failed" + failed.append({"task": task, "status": "failed", + "error": f"expected {expected_rounds} completed rounds; found {len(successful)}"}) + elif failed: + status = "failed" + else: + status = "complete" + # Explicit round secondary key makes equal N-CLIP choose the earliest round. + best = min(successful, key=lambda row: (row["nclip"], row["round"])) if successful else None + tasks.append({"task": task, "status": status, "completed_rounds": len(successful), + "failed_rounds": len(failed), "best_round": best, "rounds": task_rows, + "failures": failed}) + completed = sum(task["status"] == "complete" for task in tasks) + completed_best = [task["best_round"] for task in tasks + if task["status"] == "complete" and task["best_round"] is not None] + mean_nclip = (sum(row["nclip"] for row in completed_best) / len(completed_best) + if completed_best else None) + mean_pl = (sum(row["photometric_loss"] for row in completed_best) / len(completed_best) + if completed_best else None) + result = { + "schema_version": 1, + "metric_definition": { + "clip_image_cosine_similarity": "cosine similarity between CLIP image embeddings", + "nclip": "1 - CLIP image cosine similarity", + "photometric_loss": "pinned VIGA evaluator photometric_loss output", + "best_round": "minimum N-CLIP; earliest round wins exact ties", + }, + "tasks": tasks, + "summary": {"expected_tasks": len(tasks), "completed_tasks": completed, + "failed_tasks": len(tasks) - completed, + "completion_rate": completed / len(tasks) if tasks else 0.0, + "mean_best_nclip_completed_tasks": mean_nclip, + "mean_best_photometric_loss_completed_tasks": mean_pl}, + } + if provenance is not None: + result["provenance"] = provenance + return result + + +def write_outputs(result: dict, output: Path) -> tuple[Path, Path]: + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + json_path = output / "aggregate.json" + csv_path = output / "aggregate.csv" + temporary = json_path.with_suffix(".json.tmp") + temporary.write_text(json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8") + temporary.replace(json_path) + fields = ["task", "status", "completed_rounds", "failed_rounds", "best_round", + "best_clip_image_cosine_similarity", "best_nclip", "best_photometric_loss", "error", + "dataset_revision", "evaluator_repository", "evaluator_revision", + "evaluator_sha256", "clip_model", "clip_revision", "scorer_sha256", + "identity_validation", "protocol_sha256", "git_revision", + "scoring_lock_sha256", "model", "model_revision", "codex_version", + "bpy_version", "scorer_python_version", "scorer_python_implementation", + "scorer_python_sha256"] + with csv_path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=fields) + writer.writeheader() + provenance = result.get("provenance", {}) + source = provenance.get("source_identity", {}) + versions = provenance.get("versions", {}) + scorer_runtime = provenance.get("scorer_runtime", {}) + flat_provenance = {key: provenance.get(key, "") for key in fields[9:]} + flat_provenance.update({ + "git_revision": source.get("git_revision", ""), + "scoring_lock_sha256": source.get("scoring_lock_sha256", ""), + "codex_version": versions.get("codex", ""), + "bpy_version": versions.get("bpy", ""), + "scorer_python_version": scorer_runtime.get("python_version", ""), + "scorer_python_implementation": scorer_runtime.get("implementation", ""), + "scorer_python_sha256": scorer_runtime.get("executable_sha256", ""), + }) + for task in result["tasks"]: + best = task["best_round"] or {} + errors = "; ".join(str(row.get("error", "unknown failure")) for row in task["failures"]) + writer.writerow({"task": task["task"], "status": task["status"], + "completed_rounds": task["completed_rounds"], "failed_rounds": task["failed_rounds"], + "best_round": best.get("round", ""), + "best_clip_image_cosine_similarity": best.get("clip_image_cosine_similarity", ""), + "best_nclip": best.get("nclip", ""), + "best_photometric_loss": best.get("photometric_loss", ""), "error": errors, + **flat_provenance}) + return json_path, csv_path + + +def score_pair(image: Path, target: Path, clip_snapshot: Path, + runtime_preflight: dict | None = None) -> dict: + """Score exact image bytes with the canonical scorer under this lexical venv.""" + runtime_preflight = runtime_preflight or preflight_scorer_runtime() + snapshot_before = verify_clip_snapshot(clip_snapshot) + image, target = Path(image).absolute(), Path(target).absolute() + image_sha256, target_sha256 = dataset.sha256(image), dataset.sha256(target) + lexical = str(Path(__import__("sys").executable).expanduser().absolute()) + argv = [lexical, "-I", str(CANONICAL_SCORER), str(image), str(target)] + process = subprocess.run(argv, capture_output=True, text=True, cwd=CANONICAL_SCORER.parent, + env=_isolated_env(clip_snapshot)) + snapshot_after = verify_clip_snapshot(clip_snapshot) + if snapshot_after != snapshot_before: + raise ValueError("CLIP snapshot changed during scoring") + if process.returncode: + raise RuntimeError(f"scorer exited {process.returncode}: {process.stderr.strip()}") + try: + payload = json.loads(process.stdout) + except json.JSONDecodeError as exc: + raise ValueError("scorer must emit one JSON object") from exc + identity = {"clip_model": protocol.CLIP_MODEL, "clip_revision": protocol.CLIP_REVISION, + "viga_revision": protocol.EVALUATOR_REVISION, + "evaluator_sha256": protocol.EVALUATOR_SHA256, + "torch_version": _EXPECTED_RUNTIME["torch_version"], + "transformers_version": _EXPECTED_RUNTIME["transformers_version"], + "numpy_version": _EXPECTED_RUNTIME["numpy_version"], + "pillow_version": _EXPECTED_RUNTIME["pillow_version"]} + if not isinstance(payload, dict) or any(payload.get(key) != value for key, value in identity.items()): + raise ValueError("scorer evaluator/model/runtime identity does not match pinned protocol") + if not str(payload.get("device", "")).startswith("cuda"): + raise ValueError("scorer identity must report a CUDA device") + try: + cosine = payload.get("clip_image_cosine_similarity", payload.get("clip_similarity")) + loss = payload.get("photometric_loss", payload.get("raw_pl")) + return {"clip_image_cosine_similarity": float(cosine), "photometric_loss": float(loss), + "candidate_image_sha256": image_sha256, "target_image_sha256": target_sha256, + "scorer_payload": payload} + except (ValueError, TypeError) as exc: + raise ValueError("scorer must emit JSON with CLIP image cosine similarity and photometric loss") from exc + + +def score_saved_run(run_root: Path, dataset_root: Path, tasks: Iterable[str], rounds: int, + clip_snapshot: Path) -> dict: + tasks = list(tasks) + manifest = protocol.load_run_manifest(Path(run_root), tasks=tasks, rounds=rounds) + runtime_preflight = preflight_scorer_runtime() + snapshot = verify_clip_snapshot(clip_snapshot) + for task in tasks: + dataset.verify(task, Path(dataset_root)) + rows = [] + for task in tasks: + task_root = Path(run_root) / task.replace("/", "-") + target = Path(dataset_root) / task / "target.png" + try: + admitted_images = verified_audit_images( + task_root, rounds, manifest["protocol"]["versions"]["bpy"]) + except Exception as exc: + rows.extend({"task": task, "round": number, "status": "failed", + "error": f"{type(exc).__name__}: {exc}"} + for number in range(1, rounds + 1)) + continue + for number in range(1, rounds + 1): + # Only score the independently reopened/GPU-rendered audit image. The + # model-written checkpoint PNG is generation evidence, not an admitted + # standalone publication score input. + image = admitted_images[number] + row = {"task": task, "round": number} + try: + if not image.is_file(): + raise FileNotFoundError(image) + row.update(score_pair(image, target, clip_snapshot, runtime_preflight), + status="complete", image=str(image)) + except Exception as exc: + row.update(status="failed", error=f"{type(exc).__name__}: {exc}", image=str(image)) + rows.append(row) + generation = manifest["protocol"] + provenance = scoring_provenance(runtime_preflight, snapshot) + provenance.update({"protocol_sha256": manifest["protocol_sha256"], + "source_identity": generation["source_identity"], "model": generation["model"], + "model_revision": generation["model_revision"], "versions": generation["versions"]}) + return aggregate(rows, tasks, expected_rounds=rounds, provenance=provenance) diff --git a/benchmarks/blenderbench_direct/suite.py b/benchmarks/blenderbench_direct/suite.py new file mode 100644 index 0000000..51e222b --- /dev/null +++ b/benchmarks/blenderbench_direct/suite.py @@ -0,0 +1,303 @@ +"""Sequential direct-model generation followed by independent audit and scoring.""" +from __future__ import annotations + +from dataclasses import asdict +import json +import os +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace +import uuid + +from . import dataset, protocol, scoring +from tools import direct_codex as direct +from tools import direct_runtime as runtime + + +def generation_prompt(task_description: str, work: Path, rounds: int) -> str: + tools = ", ".join(protocol.TOOLS) + return f"""Solve this BlenderBench task using direct model reasoning and only the four Blender MCP tools. +Task: {task_description} +Initial scene: {work / 'initial.blend'} +All scene properties may be edited, including geometry, materials, lights, objects, and cameras. +Perform exactly {rounds} meaningful edit, render, and visual-inspection rounds sequentially. Each round must +change the saved scene meaningfully and continue from the preceding saved checkpoint. Inspect generated +images only through observable Blender MCP responses. Call get_render_as_image_for_cli after each saved round +and visually inspect the returned image content; Codex's built-in view_image is disabled. No CLIP/scoring, +score feedback, VLM judge, benchmark controller, goal_code, +solution code, goal scene, or oracle access. Do not read evaluator, audit, or post-generation result files. +Use only {tools} for scene operations; no shell scene operations. The standalone bpy backend opens the +specified blend file afresh for each call, so save each edit before the call returns. Use GPU-only Cycles: +enable CUDA devices, disable CPU, and never silently fall back to CPU. Save the scene before transient PNG +filepath/encoding changes, and do not save those output overrides. Never modify initial.blend, dataset +files, or target.png. Save absolute paths {work / 'iteration01.blend'} and {work / 'iteration01.png'} +through {work / f'iteration{rounds:02}.blend'} and {work / f'iteration{rounds:02}.png'}. +For each edit call, set blend_file to initial.blend for round 1 or the preceding iteration blend thereafter, +and set expected_output_blend to that round's new iterationNN.blend so the MCP server attests fresh changed bytes. +Append rounds.jsonl via MCP with one JSON object per completed round containing round, absolute blend, +absolute png, and a nonempty rationale describing the visual hypothesis and actual edit. Stop only after +all {rounds} sequential saved rounds and ledger rows exist.""" + + +def _json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8") + temporary.replace(path) + + +def _bpy_version(config) -> str: + code = "import bpy; print(bpy.app.version_string)" + return subprocess.check_output([str(config.bpy_python), "-c", code], text=True, + stderr=subprocess.STDOUT).strip().splitlines()[-1] + + +def _versions(config) -> dict: + codex = protocol.resolved_version(config.codex_binary, "--version") + if codex != protocol.CODEX_VERSION or config.codex_version != protocol.CODEX_VERSION: + raise ValueError(f"Codex version {codex!r} does not equal frozen {protocol.CODEX_VERSION!r}") + bpy = _bpy_version(config) + if not bpy.startswith(config.bpy_version): + raise ValueError(f"bpy version {bpy!r} does not match pin {config.bpy_version!r}") + scorer_runtime = scoring.preflight_scorer_runtime() + clip_snapshot = scoring.verify_clip_snapshot(config.clip_snapshot) + return {"codex": codex, "codex_source_revision": protocol.CODEX_SOURCE_REVISION, + "bpy": bpy, "scorer_python": "Python " + scorer_runtime["python_version"], + "scorer_script_sha256": scoring.scorer_sha256(), + "scorer_runtime": scorer_runtime, "clip_snapshot": clip_snapshot} + + +def _preflight(config, mcp_config: dict, work: Path, evidence: Path) -> None: + args = SimpleNamespace(codex=str(config.codex_binary), model=config.model, workdir=work) + nonce = uuid.uuid4().hex + code = ("import bpy,sys; result={'preflight':" + repr(nonce) + + ", 'python':sys.executable, 'version':bpy.app.version_string}") + prompt = (f"Call only execute_blender_code_for_cli once with blend_file={work / 'initial.blend'} " + f"and code={code}. Do not mutate/save. No shell or other tools.") + events = direct.invoke(args, mcp_config, evidence, "preflight", prompt) + direct.verify_preflight(events, nonce, str(config.bpy_python), config.bpy_version) + + +def verify_retained_generation(task_root: Path, work: Path, rounds: int) -> None: + """Revalidate the retained generation event stream before recovery admission.""" + path = Path(task_root) / "codex" / "run.jsonl" + if path.is_symlink() or not path.is_file(): + raise ValueError("recovery requires regular retained codex/run.jsonl evidence") + try: + events = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + except (OSError, json.JSONDecodeError) as exc: + raise ValueError("retained generation event evidence is unreadable") from exc + direct.verify_generation_events(events, rounds=rounds, workdir=work) + + +def _task(config, task: str, mcp_config: dict, scorer_runtime: dict) -> list[dict]: + record = dataset.load_manifest()["tasks"][task] + source = config.dataset_root / task + verification = dataset.verify(task, config.dataset_root) + task_root = config.output_root / task.replace("/", "-") + task_root.mkdir(parents=True, exist_ok=False) + _json(task_root / "input-verification.json", verification) + events = task_root / "events.jsonl" + policy = {key: record[key] for key in ("reviewed", "review_basis", "start_sha256", + "task_description", "initialization")} + policy.update(task=task, pin_verified=True, stop_file=str(config.output_root / "STOP")) + runtime.emit(events, "initializing", task=task) + initial = runtime.initialize(source, policy, config.bpy_python, config.bpy_version, + task_root / "work", events) + work = task_root / "work" + initialized = Path(initial["blend"]) + baseline_path = work / "initial.blend" + initialized.rename(baseline_path) + initial_hash = runtime.sha(baseline_path) + evidence = task_root / "codex" + evidence.mkdir(mode=0o700) + _json(evidence / "mcp-config.redacted.json", protocol.redact(mcp_config)) + direct.verify_generation_workdir(work) + _preflight(config, mcp_config, work, evidence) + runtime.emit(events, "generation_started", task=task) + args = SimpleNamespace(codex=str(config.codex_binary), model=config.model, workdir=work) + generation_events = direct.invoke(args, mcp_config, evidence, "run", + generation_prompt(record["task_description"], work, config.rounds), source / "target.png") + direct.verify_generation_events(generation_events, rounds=config.rounds, workdir=work) + runtime.emit(events, "generation_finished", task=task) + if runtime.sha(baseline_path) != initial_hash: + raise ValueError("generation modified initial.blend") + dataset.verify(task, config.dataset_root) + artifacts = direct.verify_checkpoints(work, config.rounds, policy=policy) + _json(task_root / "artifact-check.json", artifacts) + + previous = initial + seen = {json.dumps(initial["audit"], sort_keys=True, allow_nan=False)} + rows = [] + admissions = [] + for number in range(1, config.rounds + 1): + audit = runtime.inspect_render(work / f"iteration{number:02}.blend", policy, + config.bpy_python, config.bpy_version, task_root / f"audit{number:02}", events) + direct.validate_edit(previous, audit, policy) + state = json.dumps(audit["audit"], sort_keys=True, allow_nan=False) + if state in seen: + raise ValueError(f"round {number} repeats an earlier complete scene state") + seen.add(state); previous = audit + admissions.append({"round": number, "source": audit["source"], + "source_sha256": audit["source_sha256"], "image": audit["png"], + "image_sha256": audit["png_sha256"], "audit_sha256": protocol.digest(audit["audit"]), + "meaningful_change_validated": True}) + _json(task_root / "audit-admission.json", admissions) + try: + metrics = scoring.score_pair(Path(audit["png"]), source / "target.png", + config.clip_snapshot, scorer_runtime) + row = {"task": task, "round": number, "status": "complete", **metrics, + "blend": audit["blend"], "blend_sha256": audit["blend_sha256"], + "image": audit["png"], "image_sha256": audit["png_sha256"]} + except Exception as exc: + row = {"task": task, "round": number, "status": "failed", + "error": f"{type(exc).__name__}: {exc}", "blend": audit["blend"], + "image": audit["png"]} + rows.append(row) + _json(task_root / "scores.json", rows) + return rows + + +def recover_suite(args) -> int: + """Reopen, audit, render, and score retained rounds; never call generation.""" + tasks = dataset.select_tasks(args.tasks) + for name in ("bpy_python",): + if not getattr(args, name).is_file(): + raise FileNotFoundError(getattr(args, name)) + manifest = protocol.load_run_manifest(args.source_root, tasks=tasks, rounds=args.rounds) + scorer_runtime = scoring.preflight_scorer_runtime() + clip_snapshot = scoring.verify_clip_snapshot(args.clip_snapshot) + score_provenance = scoring.scoring_provenance(scorer_runtime, clip_snapshot) + score_provenance.update({"protocol_sha256": manifest["protocol_sha256"], + "source_identity": manifest["protocol"]["source_identity"], + "model": manifest["protocol"]["model"], + "model_revision": manifest["protocol"]["model_revision"], + "versions": manifest["protocol"]["versions"]}) + actual_bpy = _bpy_version(args) + generation_bpy = manifest["protocol"]["versions"].get("bpy") + if actual_bpy != generation_bpy: + raise ValueError(f"recovery bpy version {actual_bpy!r} differs from generation manifest {generation_bpy!r}") + if not actual_bpy.startswith(args.bpy_version): + raise ValueError(f"bpy version {actual_bpy!r} does not match pin {args.bpy_version!r}") + if args.output_root.exists(): + raise FileExistsError("fresh --output-root required for recovery evidence") + args.output_root.mkdir(parents=True, exist_ok=False) + all_rows = [] + for task in tasks: + record = dataset.load_manifest()["tasks"][task] + dataset.verify(task, args.dataset_root) + source_task = args.source_root / task.replace("/", "-") + work = source_task / "work" + policy = {key: record[key] for key in ("reviewed", "review_basis", "start_sha256", + "task_description", "initialization")} + policy.update(task=task, pin_verified=True, stop_file=str(args.output_root / "STOP")) + verify_retained_generation(source_task, work, args.rounds) + direct.verify_checkpoints(work, args.rounds, policy=policy) + baseline = json.loads((work / "metadata.json").read_text(encoding="utf-8")) + initial_path = work / "initial.blend" + initial_hash = runtime.sha(initial_path) + if initial_hash != baseline["blend_sha256"]: + raise ValueError(f"retained initial hash mismatch for {task}") + previous = baseline + seen = {json.dumps(baseline["audit"], sort_keys=True, allow_nan=False)} + rows = [] + admissions = [] + task_out = args.output_root / task.replace("/", "-") + task_out.mkdir(parents=True, exist_ok=False) + for number in range(1, args.rounds + 1): + audit = runtime.inspect_render(work / f"iteration{number:02}.blend", policy, + args.bpy_python, args.bpy_version, task_out / f"audit{number:02}", + task_out / "events.jsonl") + direct.validate_edit(previous, audit, policy) + state = json.dumps(audit["audit"], sort_keys=True, allow_nan=False) + if state in seen: + raise ValueError(f"round {number} repeats an earlier scene state") + seen.add(state); previous = audit + admissions.append({"round": number, "source": audit["source"], + "source_sha256": audit["source_sha256"], "image": audit["png"], + "image_sha256": audit["png_sha256"], "audit_sha256": protocol.digest(audit["audit"]), + "meaningful_change_validated": True}) + _json(task_out / "audit-admission.json", admissions) + try: + metrics = scoring.score_pair(Path(audit["png"]), + args.dataset_root / task / "target.png", args.clip_snapshot, scorer_runtime) + row = {"task": task, "round": number, "status": "complete", **metrics, + "blend": audit["blend"], "blend_sha256": audit["blend_sha256"], + "image": audit["png"], "image_sha256": audit["png_sha256"]} + except Exception as exc: + row = {"task": task, "round": number, "status": "failed", + "error": f"{type(exc).__name__}: {exc}"} + rows.append(row); all_rows.append(row) + _json(task_out / "scores.json", rows) + if runtime.sha(initial_path) != initial_hash: + raise ValueError(f"recovery modified retained initial scene for {task}") + result = scoring.aggregate(all_rows, tasks, expected_rounds=args.rounds, + provenance=score_provenance) + scoring.write_outputs(result, args.output_root / "scoring") + _json(args.output_root / "recovery-manifest.json", {"schema_version": 1, + "source_root": str(args.source_root), "regenerated": False, "tasks": tasks, + "bpy_version": actual_bpy, **score_provenance, + "dataset_revision": dataset.DATASET_REVISION, + "evaluator_revision": protocol.EVALUATOR_REVISION, + "evaluator_sha256": protocol.EVALUATOR_SHA256}) + return int(result["summary"]["failed_tasks"] != 0) + + +def run_suite(config, *, approved: bool) -> int: + if not approved: + raise ValueError("--approve-blender-tools is required for arbitrary MCP Python execution") + if config.output_root.exists(): + raise FileExistsError("fresh --output-root required; automatic resume/regeneration is forbidden") + source_identity = protocol.collect_source_identity() + if source_identity["git_dirty"]: + raise ValueError("publication generation requires a clean Git source tree") + raw_mcp = protocol.read_mcp_config(config.mcp_toml) + mcp_config = direct.prepare_config(raw_mcp, approved=True) + configured_bpy = mcp_config["mcp_servers"]["blender"]["env"]["BLENDER_MCP_BPY_PYTHON"] + if Path(configured_bpy).absolute() != config.bpy_python: + raise ValueError("MCP TOML bpy interpreter differs from --bpy-python") + versions = _versions(config) + config.output_root.mkdir(parents=True, exist_ok=False) + public_config = asdict(config) + public_config = {key: ([str(item) for item in value] if key == "tasks" else str(value) + if isinstance(value, Path) else value) for key, value in public_config.items()} + public_config["mcp"] = raw_mcp + manifest = protocol.build_manifest(public_config, dataset_revision=dataset.DATASET_REVISION, + evaluator_revision=protocol.EVALUATOR_REVISION, model_revision=config.model_revision, + versions=versions, source_identity=source_identity) + manifest["dataset"] = {"source_url": dataset.DATASET_SOURCE_URL, + "attribution": dataset.DATASET_ATTRIBUTION, "license": dataset.DATASET_LICENSE} + manifest["evaluator"] = {"source_url": protocol.EVALUATOR_REPOSITORY, + "revision": protocol.EVALUATOR_REVISION, "sha256": protocol.EVALUATOR_SHA256} + protocol.write_manifest(config.output_root / "run-manifest.json", manifest) + _json(config.output_root / "process.json", {"pid": os.getpid(), "argv": sys.argv, + "protocol_sha256": manifest["protocol_sha256"]}) + all_rows = [] + states = {} + runtime.emit(config.output_root / "events.jsonl", "dispatcher_started", tasks=list(config.tasks)) + for task in config.tasks: + if (config.output_root / "STOP").exists(): + states[task] = {"status": "cancelled"}; break + try: + rows = _task(config, task, mcp_config, versions["scorer_runtime"]) + all_rows.extend(rows) + state = "complete" if all(row["status"] == "complete" for row in rows) else "failed" + states[task] = {"status": state} + except Exception as exc: + states[task] = {"status": "failed", "error": f"{type(exc).__name__}: {exc}"} + _json(config.output_root / "status.json", states) + # Every selected task is represented, including cancellation/not-started cases. + score_provenance = scoring.scoring_provenance( + versions["scorer_runtime"], versions["clip_snapshot"]) + score_provenance.update({"protocol_sha256": manifest["protocol_sha256"], + "source_identity": manifest["protocol"]["source_identity"], + "model": manifest["protocol"]["model"], + "model_revision": manifest["protocol"]["model_revision"], + "versions": versions}) + result = scoring.aggregate(all_rows, config.tasks, expected_rounds=config.rounds, + provenance=score_provenance) + scoring.write_outputs(result, config.output_root / "scoring") + runtime.emit(config.output_root / "events.jsonl", "dispatcher_finished", states=states) + return int(result["summary"]["failed_tasks"] != 0) diff --git a/benchmarks/blenderbench_direct/video.py b/benchmarks/blenderbench_direct/video.py new file mode 100644 index 0000000..8f59515 --- /dev/null +++ b/benchmarks/blenderbench_direct/video.py @@ -0,0 +1,107 @@ +"""Portable BlenderBench compilation video with metric-accurate overlays.""" +from __future__ import annotations + +import json +import math +from pathlib import Path +import shutil +import subprocess +from typing import Iterable + + +def timeline_state(elapsed: float, *, duration: float, rounds: Iterable[dict], + task_tokens: int | None) -> dict: + if not math.isfinite(elapsed) or not math.isfinite(duration) or duration <= 0: + raise ValueError("finite nonnegative elapsed and positive duration required") + selected = None + for row in sorted(rounds, key=lambda item: (item["elapsed_seconds"], item["round"])): + if row["elapsed_seconds"] <= elapsed: + selected = row + else: + break + tokens = None if task_tokens is None else round(task_tokens * min(1.0, max(0.0, elapsed / duration))) + return { + "round": selected["round"] if selected else 0, + "image": selected.get("image") if selected else None, + "clip_image_cosine_similarity": selected.get("clip_image_cosine_similarity") if selected else None, + "task_tokens_interpolated": tokens, + "elapsed_seconds": elapsed, + } + + +def overlay_labels(model: str, task: str, state: dict, rounds: int) -> str: + similarity = state.get("clip_image_cosine_similarity") + metric = ("CLIP image cosine similarity: unavailable" if similarity is None else + f"CLIP image cosine similarity: {similarity:.6f}") + tokens = state.get("task_tokens_interpolated") + token_label = ("Task-level tokens (linear interpolation): unavailable" if tokens is None else + f"Task-level tokens (linear interpolation): ~{tokens:,}") + return "\n".join((f"{model} | {task} | Round {state['round']}/{rounds}", metric, token_label)) + + +def generate(spec_path: Path, output: Path, *, fps: int = 24, seconds_per_task: float = 3.0, + ffmpeg: str | None = None) -> dict: + """Generate an H.264 compilation from an explicit, machine-readable task spec. + + Each task record requires target, initial, duration_seconds, rounds, model and + optional task_tokens. Round elapsed times are task-local. Token counts shown in + frames are explicitly linear task-level interpolation, never event-level usage. + """ + from PIL import Image, ImageDraw + + spec = json.loads(Path(spec_path).read_text(encoding="utf-8")) + tasks = spec["tasks"] + if not tasks: + raise ValueError("video spec contains no tasks") + if fps < 1 or seconds_per_task <= 0: + raise ValueError("positive fps and seconds-per-task required") + encoder = ffmpeg or shutil.which("ffmpeg") + if not encoder: + raise RuntimeError("ffmpeg not found; pass --ffmpeg") + output = Path(output) + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists(): + raise FileExistsError(output) + width, height = 1024, 600 + process = subprocess.Popen([encoder, "-hide_banner", "-loglevel", "error", "-n", + "-f", "rawvideo", "-pix_fmt", "rgb24", "-s", f"{width}x{height}", "-r", str(fps), + "-i", "-", "-an", "-c:v", "libx264", "-pix_fmt", "yuv420p", str(output)], + stdin=subprocess.PIPE) + report = {"schema_version": 1, "output": str(output), "fps": fps, + "seconds_per_task": seconds_per_task, "frames": [], + "metric_label": "CLIP image cosine similarity", + "token_label": "task-level linear interpolation"} + try: + frame_count = round(fps * seconds_per_task) + for task in tasks: + with Image.open(task["target"]) as image: + target = image.convert("RGB").resize((512, 512)) + initial = Path(task["initial"]) + for index in range(frame_count): + elapsed = task["duration_seconds"] * index / max(1, frame_count - 1) + state = timeline_state(elapsed, duration=task["duration_seconds"], + rounds=task["rounds"], task_tokens=task.get("task_tokens")) + source = Path(state["image"]) if state["image"] else initial + with Image.open(source) as image: + candidate = image.convert("RGB").resize((512, 512)) + frame = Image.new("RGB", (width, height), "black") + frame.paste(target, (0, 88)); frame.paste(candidate, (512, 88)) + draw = ImageDraw.Draw(frame) + draw.text((12, 12), "TARGET (static)", fill="white") + labels = overlay_labels(task["model"], task["task"], state, len(task["rounds"])) + draw.multiline_text((524, 8), labels, fill="white", spacing=3) + if process.stdin is None: + raise RuntimeError("ffmpeg stdin unavailable") + process.stdin.write(frame.tobytes()) + report["frames"].append({"task": task["task"], "task_frame": index, **state}) + if process.stdin is not None: + process.stdin.close() + if process.wait() != 0: + raise RuntimeError(f"ffmpeg exited {process.returncode}") + except BaseException: + process.kill(); process.wait() + raise + report_path = output.with_suffix(output.suffix + ".json") + report_path.write_text(json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8") + return report diff --git a/benchmarks/token_efficiency/codex_runner.py b/benchmarks/token_efficiency/codex_runner.py index 6152fa8..22e9b14 100644 --- a/benchmarks/token_efficiency/codex_runner.py +++ b/benchmarks/token_efficiency/codex_runner.py @@ -458,7 +458,7 @@ def main(argv=None): parser.add_argument("--condition", choices=CONDITIONS, default="enhanced_bpy") parser.add_argument("--server-python", default="/REQUIRES_REVIEW/server-python") parser.add_argument("--runtime", default="/REQUIRES_REVIEW/runtime") - parser.add_argument("--model", default="REQUIRES_PARENT_MODEL_SELECTION") + parser.add_argument("--model", required=True) parser.add_argument("--fixture-stream") args = parser.parse_args(argv) if args.mode == "dry-run": diff --git a/benchmarks/token_efficiency/creative_smoke.py b/benchmarks/token_efficiency/creative_smoke.py index e50c31d..162e550 100644 --- a/benchmarks/token_efficiency/creative_smoke.py +++ b/benchmarks/token_efficiency/creative_smoke.py @@ -264,7 +264,7 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", default="/tmp/voxel-video-mcp-trial") parser.add_argument("--condition", choices=["original", "enhanced", "both"], default="both") - parser.add_argument("--model", default="gpt-6-astra") + parser.add_argument("--model", required=True) parser.add_argument("--seconds", type=int, default=600) parser.add_argument("--output-bytes", type=int, default=2 * 1024 * 1024) parser.add_argument("--max-tools", type=int, default=80) diff --git a/mcp/blmcp/tools/execute_blender_code.py b/mcp/blmcp/tools/execute_blender_code.py index 1a4fd83..379022c 100644 --- a/mcp/blmcp/tools/execute_blender_code.py +++ b/mcp/blmcp/tools/execute_blender_code.py @@ -6,10 +6,15 @@ __all__ = ("register",) +import base64 +import hashlib +from pathlib import Path +import tempfile + from blmcp.tools_helpers.blender_cli import run_blender_cli, synced_blend_for_cli from blmcp.tools_helpers.connection import send_code -from mcp.server.fastmcp import FastMCP # pylint: disable=import-error,no-name-in-module -from mcp.types import ToolAnnotations # pylint: disable=import-error,no-name-in-module +from mcp.server.fastmcp import FastMCP, Image # pylint: disable=import-error,no-name-in-module +from mcp.types import ImageContent, ToolAnnotations # pylint: disable=import-error,no-name-in-module def register(mcp: FastMCP) -> None: @@ -38,18 +43,114 @@ def execute_blender_code(code: str) -> dict[str, object]: destructiveHint=True, ) ) - def execute_blender_code_for_cli(blend_file: str, code: str) -> dict[str, object]: + def execute_blender_code_for_cli(blend_file: str, code: str, + expected_output_blend: str | None = None) -> dict[str, object]: """ Execute Python code in a background Blender process. Opens *blend_file* with ``blender --background`` and runs *code*. - Assign a dict to ``result`` to return data. + Assign a dict to ``result`` to return data. When *expected_output_blend* + is supplied, that path must be newly created with bytes different from + the source; the result includes server-computed checkpoint hashes. """ + output = Path(expected_output_blend).absolute() if expected_output_blend else None + source = (Path(blend_file).resolve(strict=True) if output is not None + else Path(blend_file)) + if output is not None and (output == source or output.exists() or output.is_symlink()): + raise ValueError("expected output checkpoint must be a fresh distinct path") + source_sha256 = None + if output is not None: + with source.open("rb") as stream: + source_sha256 = hashlib.file_digest(stream, "sha256").hexdigest() # LLM-generated code may return non-JSON-serializable values # (e.g. Blender objects), handled by `run_blender_cli` via `default=repr`. - with synced_blend_for_cli(blend_file) as synced_path: - value = run_blender_cli(synced_path, code, arbitrary_code=True) - assert isinstance(value, dict), ( - "Expected dict from `run_blender_cli`, got {!r}".format(type(value)) + if output is None: + with synced_blend_for_cli(str(source)) as synced_path: + value = run_blender_cli(synced_path, code, arbitrary_code=True) + else: + value = run_blender_cli(str(source), code, arbitrary_code=True) + assert isinstance(value, dict), ( + "Expected dict from `run_blender_cli`, got {!r}".format(type(value)) + ) + if "_checkpoint" in value: + raise ValueError("result uses reserved _checkpoint field") + if output is not None: + if output.is_symlink() or not output.is_file(): + raise ValueError("expected output checkpoint was not created as a regular file") + with output.open("rb") as stream: + output_sha256 = hashlib.file_digest(stream, "sha256").hexdigest() + if output_sha256 == source_sha256: + raise ValueError("expected output checkpoint does not differ from source") + if "_checkpoint" in value: + raise ValueError("result uses reserved _checkpoint field") + value["_checkpoint"] = {"path": str(output), "source": str(source), + "source_sha256": source_sha256, "output_sha256": output_sha256, + "existed_before": False} + return value + + @mcp.tool( + annotations=ToolAnnotations( + title="Render Scene as Image", + readOnlyHint=True, + ) + ) + def get_render_as_image() -> Image: + """Render the connected Blender scene and return PNG image content without saving overrides.""" + with tempfile.TemporaryDirectory(prefix="blmcp-render-") as temporary: + render_path = Path(temporary) / "render.png" + code = ( + "import bpy\n" + "_scene = bpy.context.scene\n" + "_old = (_scene.render.filepath, _scene.render.image_settings.file_format, " + "_scene.render.use_file_extension)\n" + "try:\n" + f" _scene.render.filepath = {str(render_path)!r}\n" + " _scene.render.image_settings.file_format = 'PNG'\n" + " _scene.render.use_file_extension = True\n" + " bpy.ops.render.render(write_still=True)\n" + " result = {'rendered': True}\n" + "finally:\n" + " (_scene.render.filepath, _scene.render.image_settings.file_format, " + "_scene.render.use_file_extension) = _old" + ) + response = send_code(code, strict_json=True) + if response.get("status") != "ok" or not render_path.is_file(): + raise RuntimeError("Blender did not produce the requested PNG render") + return Image(data=render_path.read_bytes(), format="png") + + @mcp.tool( + annotations=ToolAnnotations( + title="Render Saved Scene as Image for Command-Line", + readOnlyHint=True, + ) + ) + def get_render_as_image_for_cli(blend_file: str) -> ImageContent: + """Open a saved blend in a background process, render it, and return PNG image content.""" + source = Path(blend_file).resolve(strict=True) + with source.open("rb") as stream: + source_sha256 = hashlib.file_digest(stream, "sha256").hexdigest() + with tempfile.TemporaryDirectory(prefix="blmcp-render-") as temporary: + render_path = Path(temporary) / "render.png" + code = ( + "import bpy\n" + "prefs=bpy.context.preferences.addons['cycles'].preferences\n" + "prefs.compute_device_type='CUDA'\n" + "prefs.refresh_devices()\n" + "assert any(d.type=='CUDA' for d in prefs.devices)\n" + "for d in prefs.devices: d.use=(d.type=='CUDA')\n" + "s=bpy.context.scene\n" + "assert s.render.engine=='CYCLES'\n" + "s.cycles.device='GPU'\n" + f"s.render.filepath = {str(render_path)!r}\n" + "s.render.image_settings.file_format = 'PNG'\n" + "s.render.use_file_extension = True\n" + "bpy.ops.render.render(write_still=True)\n" + "result = {'rendered': True}" ) - return value + run_blender_cli(str(source), code, arbitrary_code=False) + if not render_path.is_file(): + raise RuntimeError("Blender CLI did not produce the requested PNG render") + return ImageContent(type="image", + data=base64.b64encode(render_path.read_bytes()).decode("ascii"), + mimeType="image/png", _meta={"source": str(source), + "source_sha256": source_sha256}) diff --git a/readme.md b/readme.md index e3eee26..cd24dcd 100644 --- a/readme.md +++ b/readme.md @@ -6,13 +6,22 @@ This branch packages an enhanced Blender MCP workflow for agentic Blender work. This is a modified distribution of Blender Lab's Blender MCP project. See [NOTICE.md](NOTICE.md) for provenance and [SECURITY.md](SECURITY.md) for the execution-risk model. Public `bpy.dev` website assets will live in a separate repository. +### BlenderBench result + +[![BlenderBench result](https://bpy.dev/assets/blenderbench-result.webp)](https://huggingface.co/datasets/michaelgold/blenderbench-direct-results/resolve/7c2c43be4517aee4d76d2b445578988de186970c/video/blenderbench-complete-compilation.mp4) + +The compilation covers **27 tasks / 270 rounds**. Its on-video “CLIP accuracy” label means +**CLIP image-embedding cosine similarity, not literal accuracy**. The linked immutable +H.264 video is 848×464, 134.9 seconds, and 10,161,884 bytes; SHA-256: +`04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17`. + ### What is enhanced - **Saved-file tools:** `_for_cli` tools operate on `.blend` files in fresh subprocesses without requiring a live Blender UI session. - **Backend selection:** use a Blender executable backend or a standalone `bpy` Python backend with `BLENDER_MCP_CLI_BACKEND`. - **Runtime API lookup:** agents can query exact signatures, enums, defaults, and availability from the active Blender/`bpy` runtime instead of relying only on bundled docs. - **Process isolation:** command-line executions are bounded by explicit timeouts, output caps, compact JSON framing, and source-file overwrite checks. This is crash/process-state isolation, not an OS sandbox. -- **Benchmark tooling:** deterministic token-efficiency tasks and exploratory creative comparison harnesses support repeatable original-vs-enhanced analysis. +- **Benchmark tooling:** deterministic token-efficiency tasks, exploratory creative comparison harnesses, and a [portable model-neutral full-27 BlenderBench runner](benchmarks/blenderbench_direct/README.md) with pinned inputs and post-generation scoring. ### Current publication status diff --git a/tests/test_blender_cli.py b/tests/test_blender_cli.py index dabc194..c1b76e9 100644 --- a/tests/test_blender_cli.py +++ b/tests/test_blender_cli.py @@ -280,6 +280,92 @@ def decorator(function: Callable[..., object]) -> object: arbitrary_code=True, ) + def test_cli_render_tool_returns_image_content(self) -> None: + registered: dict[str, object] = {} + + class FakeMCP: + def tool(self, **_kwargs: object) -> Callable[[Callable[..., object]], object]: + def decorator(function: Callable[..., object]) -> object: + registered[function.__name__] = function + return function + return decorator + + execute_blender_code.register(FakeMCP()) # type: ignore[arg-type] + self.assertIn("get_render_as_image_for_cli", registered) + tool = registered["get_render_as_image_for_cli"] + import tempfile + with tempfile.TemporaryDirectory() as tmp: + source = os.path.join(tmp, "scene.blend") + with open(source, "wb") as stream: + stream.write(b"blend-bytes") + render = os.path.join(tmp, "render.png") + def produce(_blend: str, _code: str, *, arbitrary_code: bool) -> dict[str, object]: + self.assertFalse(arbitrary_code) + with open(render, "wb") as stream: + stream.write(b"png-bytes") + return {"rendered": True} + temporary = mock.MagicMock() + temporary.__enter__.return_value = tmp + temporary.__exit__.return_value = False + with mock.patch.object(execute_blender_code.tempfile, "TemporaryDirectory", + return_value=temporary), \ + mock.patch.object(execute_blender_code, "synced_blend_for_cli", + return_value=contextlib.nullcontext("synced.blend")), \ + mock.patch.object(execute_blender_code, "run_blender_cli", + side_effect=produce) as run: + image = tool(source) # type: ignore[operator] + self.assertEqual(image.mimeType, "image/png") + self.assertEqual(image.data, "cG5nLWJ5dGVz") + self.assertEqual(run.call_args.args[0], source) + code = run.call_args.args[1] + self.assertIn("bpy.ops.render.render(write_still=True)", code) + self.assertIn("compute_device_type='CUDA'", code) + self.assertIn("d.use=(d.type=='CUDA')", code) + + def test_execute_code_cli_attests_fresh_checkpoint_creation(self) -> None: + registered: dict[str, object] = {} + class FakeMCP: + def tool(self, **_kwargs: object) -> Callable[[Callable[..., object]], object]: + def decorator(function: Callable[..., object]) -> object: + registered[function.__name__] = function + return function + return decorator + execute_blender_code.register(FakeMCP()) # type: ignore[arg-type] + import tempfile + with tempfile.TemporaryDirectory() as tmp: + source = os.path.join(tmp, "initial.blend") + output = os.path.join(tmp, "iteration01.blend") + with open(source, "wb") as stream: stream.write(b"initial") + def create(*_args: object, **_kwargs: object) -> dict[str, object]: + with open(output, "wb") as stream: stream.write(b"changed") + return {"saved": True} + with mock.patch.object(execute_blender_code, "synced_blend_for_cli", + return_value=contextlib.nullcontext(source)) as sync, \ + mock.patch.object(execute_blender_code, "run_blender_cli", side_effect=create): + result = registered["execute_blender_code_for_cli"]( + source, "save", expected_output_blend=output) # type: ignore[operator] + self.assertEqual(result["_checkpoint"]["path"], output) + self.assertFalse(result["_checkpoint"]["existed_before"]) + self.assertNotEqual(result["_checkpoint"]["source_sha256"], + result["_checkpoint"]["output_sha256"]) + sync.assert_not_called() + + def test_execute_code_cli_rejects_forged_checkpoint_without_attestation_argument(self) -> None: + registered: dict[str, object] = {} + class FakeMCP: + def tool(self, **_kwargs: object) -> Callable[[Callable[..., object]], object]: + def decorator(function: Callable[..., object]) -> object: + registered[function.__name__] = function + return function + return decorator + execute_blender_code.register(FakeMCP()) # type: ignore[arg-type] + with mock.patch.object(execute_blender_code, "synced_blend_for_cli", + return_value=contextlib.nullcontext("scene.blend")), \ + mock.patch.object(execute_blender_code, "run_blender_cli", + return_value={"_checkpoint": {"output_sha256": "forged"}}): + with self.assertRaisesRegex(ValueError, "reserved _checkpoint"): + registered["execute_blender_code_for_cli"]("scene.blend", "forge") # type: ignore[operator] + def test_explicit_error_marker_raises(self) -> None: with self.assertRaisesRegex(RuntimeError, "Blender error: broken"): self._run_with_completed_process( diff --git a/tests/test_blenderbench_publication.py b/tests/test_blenderbench_publication.py new file mode 100644 index 0000000..4fb0d10 --- /dev/null +++ b/tests/test_blenderbench_publication.py @@ -0,0 +1,746 @@ +"""CI-safe publication runner tests; no Blender, GPU, model, or network.""" +from __future__ import annotations + +import csv +import json +from pathlib import Path +import tempfile +import unittest + +from benchmarks.blenderbench_direct import dataset, protocol, scoring, video +from benchmarks.blenderbench_direct.runner import build_parser, resolve_config + + +class DatasetManifestTests(unittest.TestCase): + def test_task_path_components_may_not_be_symlinks(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + outside = root / "outside" + outside.mkdir() + data = root / "data" + data.mkdir() + (data / "level1").symlink_to(outside, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "^symlink forbidden in dataset"): + dataset.verify("level1/camera1", data) + + def test_verified_dataset_task_contains_only_input_assets_and_checkpoint(self): + from unittest.mock import patch + record = {name: {"size": 1, "sha256": "a" * 64} + for name in ("task.txt", "start.py", "scene.blend", "target.png")} + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + task = root / "level1/camera1" + task.mkdir(parents=True) + for name in record: + (task / name).write_bytes(b"x") + (task / "disguised_payload.py").write_text("pass") + manifest = {"tasks": {"level1/camera1": {"assets": record}}} + with patch.object(dataset, "load_manifest", return_value=manifest), \ + patch.object(dataset, "_verify", return_value={"size": 1, "sha256": "a" * 64}): + with self.assertRaisesRegex(ValueError, "unexpected path"): + dataset.verify("level1/camera1", root) + + def test_verified_dataset_rejects_contamination_outside_selected_task(self): + from unittest.mock import patch + record = {name: {"size": 1, "sha256": "a" * 64} + for name in ("task.txt", "start.py", "scene.blend", "target.png")} + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); task = root / "level1/camera1"; task.mkdir(parents=True) + for name in record: (task / name).write_bytes(b"x") + (root / "obfuscated-payload.bin").write_bytes(b"hidden code") + manifest = {"tasks": {"level1/camera1": {"assets": record}}} + with patch.object(dataset, "load_manifest", return_value=manifest), \ + patch.object(dataset, "_verify", return_value={}): + with self.assertRaisesRegex(ValueError, "dataset root"): + dataset.verify("level1/camera1", root) + + def test_successful_resumed_download_removes_superseded_partial(self): + import hashlib + import io + from unittest.mock import patch + payload = b"complete-payload" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + destination = root / "asset.bin" + retained = root / "asset.bin.retained.partial" + retained.write_bytes(payload[:5]) + class Response(io.BytesIO): + status = 206 + headers = {"Content-Range": f"bytes 5-{len(payload) - 1}/{len(payload)}"} + record = {"size": len(payload), "sha256": hashlib.sha256(payload).hexdigest()} + with patch.object(dataset.urllib.request, "urlopen", + return_value=Response(payload[5:])): + dataset._download("https://example.invalid/asset", destination, record) + self.assertEqual(destination.read_bytes(), payload) + self.assertFalse(retained.exists()) + + def test_manifest_is_pinned_complete_and_has_no_edit_policy_fields(self): + manifest = dataset.load_manifest() + self.assertEqual(manifest["revision"], dataset.DATASET_REVISION) + self.assertEqual(len(manifest["tasks"]), 27) + self.assertEqual(dataset.task_ids(), sorted(manifest["tasks"])) + forbidden = {"allowed_transforms", "allowed_camera_data", "allowed_light_data", + "allowed_shape_keys", "location_bounds", "bounds", "unsupported_blocker"} + def keys(value): + if isinstance(value, dict): + yield from value + for key, child in value.items(): + yield key + yield from keys(child) + elif isinstance(value, list): + for child in value: + yield from keys(child) + self.assertFalse(forbidden.intersection(keys(manifest))) + for task, record in manifest["tasks"].items(): + self.assertEqual(set(record["assets"]), {"task.txt", "start.py", "scene.blend", "target.png"}) + self.assertTrue(record["task_description"]) + self.assertEqual(len(record["start_sha256"]), 64) + self.assertIsInstance(record["initialization"], list) + self.assertNotIn("goal.py", json.dumps(record)) + + def test_subset_selection_defaults_to_all_and_rejects_duplicates(self): + all_tasks = dataset.task_ids() + self.assertEqual(dataset.select_tasks(None), all_tasks) + self.assertEqual(dataset.select_tasks([all_tasks[2], all_tasks[0]]), [all_tasks[2], all_tasks[0]]) + for invalid in [[], [all_tasks[0], all_tasks[0]], ["unknown/task"]]: + with self.assertRaises(ValueError): + dataset.select_tasks(invalid) + + +class RunnerConfigTests(unittest.TestCase): + def test_external_scorer_options_are_not_accepted(self): + parser = build_parser() + with self.assertRaises(SystemExit): + parser.parse_args(["score", "--run-root", "/run", "--dataset-root", "/data", + "--clip-snapshot", "/clip", "--scorer-python", "/fake", + "--scorer-python-sha256", "a" * 64, "--scorer-script", "/fake.py"]) + + def required_argv(self, root: Path) -> list[str]: + return ["run", "--dataset-root", str(root / "data"), "--mcp-toml", str(root / "mcp.toml"), + "--codex-binary", str(root / "codex"), "--codex-version", "codex-cli 0.154.0", + "--bpy-python", str(root / "bpy-python"), "--bpy-version", "5.2.1", + "--clip-snapshot", str(root / "clip-snapshot"), + "--output-root", str(root / "output"), "--model", "provider/model"] + + def test_all_runtime_paths_and_version_pins_are_explicit_and_rounds_frozen(self): + parser = build_parser() + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + args = parser.parse_args(self.required_argv(root)) + config = resolve_config(args, check_files=False) + self.assertEqual(config.rounds, 10) + self.assertEqual(config.model, "provider/model") + self.assertEqual(config.codex_version, "codex-cli 0.154.0") + self.assertEqual(config.bpy_version, "5.2.1") + for field in ("dataset_root", "mcp_toml", "codex_binary", "bpy_python", + "clip_snapshot", "output_root"): + self.assertTrue(getattr(config, field).is_absolute(), field) + with self.assertRaises(SystemExit): + parser.parse_args(self.required_argv(root) + ["--rounds", "9"]) + + def test_generated_roots_must_stay_outside_git_repository(self): + parser = build_parser() + root = Path(__file__).resolve().parents[1] + args = parser.parse_args(self.required_argv(root)) + with self.assertRaisesRegex(ValueError, "outside"): + resolve_config(args, check_files=False) + + def test_recover_and_score_are_explicit_and_never_fall_back_to_generation(self): + parser = build_parser() + recover = parser.parse_args(["recover", "--source-root", "/old", "--output-root", "/recovery", + "--dataset-root", "/data", "--bpy-python", "/bpy", "--bpy-version", "5.2.1", + "--clip-snapshot", "/clip"]) + self.assertEqual(recover.command, "recover") + self.assertEqual(recover.output_root, Path("/recovery")) + self.assertEqual(parser.parse_args(["score", "--run-root", "/run", "--dataset-root", "/data", + "--clip-snapshot", "/clip"]).command, + "score") + + def test_recovery_rejects_renderer_different_from_generation_manifest(self): + from types import SimpleNamespace + from unittest.mock import patch + from benchmarks.blenderbench_direct import suite + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bpy = root / "bpy"; bpy.write_bytes(b"binary") + args = SimpleNamespace(tasks=None, bpy_python=bpy, bpy_version="5.3.0", + source_root=root / "source", output_root=root / "recovery", + dataset_root=root / "dataset", clip_snapshot=root / "clip", rounds=10) + manifest = {"protocol_sha256": "a" * 64, "protocol": { + "source_identity": {}, "model": "m", "model_revision": None, + "versions": {"bpy": "5.2.1"}}} + with patch.object(dataset, "select_tasks", return_value=[]), \ + patch.object(protocol, "load_run_manifest", return_value=manifest), \ + patch.object(scoring, "preflight_scorer_runtime", return_value={}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={}), \ + patch.object(suite, "_bpy_version", return_value="5.3.0"): + with self.assertRaisesRegex(ValueError, "generation manifest"): + suite.recover_suite(args) + def test_recovery_revalidates_retained_generation_events(self): + from benchmarks.blenderbench_direct import suite + with tempfile.TemporaryDirectory() as tmp: + task = Path(tmp) / "level1-camera1" + work = task / "work"; work.mkdir(parents=True) + (work / "iteration01.blend").write_bytes(b"checkpoint") + codex = task / "codex"; codex.mkdir() + (codex / "run.jsonl").write_text("\n".join(map(json.dumps, [ + {"type": "item.completed", "item": {"id": "edit", "type": "mcp_tool_call", + "server": "blender", "tool": "execute_blender_code_for_cli", + "status": "completed", "error": None, "result": {}}}, + {"type": "turn.completed"}, + ]))) + with self.assertRaisesRegex(ValueError, "visual inspection"): + suite.verify_retained_generation(task, work, 1) + + +class ProtocolManifestTests(unittest.TestCase): + def test_evaluator_url_resolves_to_verified_release_repository(self): + self.assertEqual(protocol.EVALUATOR_REPOSITORY, + "https://github.com/Fugtemypt123/VIGA-release") + source = (Path(__file__).resolve().parents[1] / + "benchmarks/blenderbench_direct/score_pair.py").read_text() + self.assertIn("https://github.com/Fugtemypt123/VIGA-release/blob/", source) + self.assertIn('VIGA_REVISION = "' + protocol.EVALUATOR_REVISION + '"', source) + + def test_pinned_evaluator_bytes_are_present_and_bound_into_source_identity(self): + root = Path(__file__).resolve().parents[1] + relative = "benchmarks/blenderbench_direct/ref_based_eval.py" + evaluator = root / relative + self.assertTrue(evaluator.is_file()) + self.assertFalse(evaluator.is_symlink()) + self.assertEqual(dataset.sha256(evaluator), protocol.EVALUATOR_SHA256) + identity = protocol.collect_source_identity() + self.assertEqual(identity["implementation_sha256"][relative], protocol.EVALUATOR_SHA256) + + def test_visual_inspection_tool_bytes_are_bound_into_source_identity(self): + relative = "mcp/blmcp/tools/execute_blender_code.py" + identity = protocol.collect_source_identity() + self.assertIn(relative, identity["implementation_sha256"]) + self.assertEqual(identity["implementation_sha256"][relative], + dataset.sha256(Path(__file__).resolve().parents[1] / relative)) + + def test_protocol_hash_is_deterministic_and_runtime_provenance_is_not_hashed(self): + base = {"model": "provider/model", "rounds": 10, "mcp": {"token": "secret", "enabled_tools": list(protocol.TOOLS)}} + first = protocol.build_manifest(base, dataset_revision="dataset-rev", evaluator_revision="eval-rev", + model_revision="model-rev", versions={"codex": "1", "bpy": "2"}, + provenance={"host": "one"}) + second = protocol.build_manifest(base, dataset_revision="dataset-rev", evaluator_revision="eval-rev", + model_revision="model-rev", versions={"codex": "1", "bpy": "2"}, + provenance={"host": "two"}) + self.assertEqual(first["protocol_sha256"], second["protocol_sha256"]) + self.assertEqual(first["protocol"]["tools"], list(protocol.TOOLS)) + self.assertEqual(first["config"]["mcp"]["token"], "") + self.assertNotIn("protocol_sha256", first["protocol"]) + + def test_protocol_hash_binds_initialization_implementation_and_git_state(self): + base = {"model": "provider/model", "rounds": 10} + source = {"initialization_manifest_sha256": "a" * 64, + "implementation_sha256": {"suite.py": "b" * 64}, + "git_revision": "c" * 40, "git_dirty": False} + first = protocol.build_manifest(base, dataset_revision="dataset", evaluator_revision="eval", + model_revision=None, versions={}, source_identity=source, provenance={}) + self.assertEqual(first["protocol"]["source_identity"], source) + dirty = protocol.build_manifest(base, dataset_revision="dataset", evaluator_revision="eval", + model_revision=None, versions={}, source_identity={**source, "git_dirty": True}, provenance={}) + changed = protocol.build_manifest(base, dataset_revision="dataset", evaluator_revision="eval", + model_revision=None, versions={}, source_identity={**source, + "implementation_sha256": {"suite.py": "d" * 64}}, provenance={}) + self.assertNotEqual(first["protocol_sha256"], dirty["protocol_sha256"]) + self.assertNotEqual(first["protocol_sha256"], changed["protocol_sha256"]) + + def test_run_manifest_validation_rejects_rehashed_protocol_rule_drift(self): + import copy + source = protocol.collect_source_identity() + source["git_dirty"] = False + config = {"model": "provider/model", "model_revision": None, "rounds": 10, + "tasks": ["level1/camera1"], "bpy_version": "5.2.1"} + versions = {"codex": protocol.CODEX_VERSION, + "codex_source_revision": protocol.CODEX_SOURCE_REVISION, + "bpy": "5.2.1", "scorer_python": "Python 3.13.15"} + manifest = protocol.build_manifest(config, dataset_revision=dataset.DATASET_REVISION, + evaluator_revision=protocol.EVALUATOR_REVISION, model_revision=None, + versions=versions, source_identity=source, provenance={}) + protocol.validate_run_manifest(manifest, tasks=config["tasks"], rounds=10) + mutations = [ + ("tools", ["shell"]), + ("clip_revision", "wrong"), + ("rendering", {"engine": "Cycles", "device": "CPU", "cpu_fallback": True}), + ("generation", {**manifest["protocol"]["generation"], + "score_feedback_to_generation": True}), + ("versions", {**versions, "codex": "codex-cli 999"}), + ] + for key, value in mutations: + changed = copy.deepcopy(manifest) + changed["protocol"][key] = value + changed["protocol_sha256"] = protocol.digest(changed["protocol"]) + with self.subTest(key=key), self.assertRaisesRegex(ValueError, "frozen protocol"): + protocol.validate_run_manifest(changed, tasks=config["tasks"], rounds=10) + + def test_source_identity_binds_hash_locked_scoring_environment(self): + identity = protocol.collect_source_identity() + root = Path(__file__).resolve().parents[1] / "benchmarks/blenderbench_direct" + self.assertEqual(identity["scoring_lock_sha256"], dataset.sha256(root / "requirements-scoring.lock")) + self.assertEqual(identity["assets_manifest_sha256"], dataset.sha256(root / "assets-manifest.json")) + self.assertEqual(identity["assets_manifest"], + json.loads((root / "assets-manifest.json").read_text(encoding="utf-8"))) + self.assertEqual(identity["clip_snapshot_manifest_sha256"], + "90813610730d94e7b677552a4488a23ce3c2e2cd2ddfdd332756ea8e70db4ebb") + self.assertEqual(identity["clip_snapshot_manifest"]["revision"], protocol.CLIP_REVISION) + + def test_public_config_omits_nested_args_env_variables_and_bearer_urls(self): + config = {"mcp": {"mcp_servers": {"blender": { + "args": ["--header", "Bearer SYNTHETIC-NOT-A-SECRET"], + "env": {"GITHUB_PAT": "SYNTHETIC-PAT", "SAFE": [{"nested": "SYNTHETIC"}]}, + "variables": [{"CUSTOM_LOGIN": "https://user:SYNTHETIC@example.invalid/path"}], + "command": "/public/bin/server"}}}} + public = protocol.redact(config) + encoded = json.dumps(public) + for secret in ("SYNTHETIC-NOT-A-SECRET", "SYNTHETIC-PAT", "SYNTHETIC@example", "SYNTHETIC"): + self.assertNotIn(secret, encoded) + server = public["mcp"]["mcp_servers"]["blender"] + self.assertEqual(server["command"], "/public/bin/server") + self.assertEqual(server["args"], "") + + +class ScoringTests(unittest.TestCase): + def write_run_manifest(self, root: Path, tasks=("level1/camera1",), rounds=10) -> dict: + config = {"model": "gpt-6-astra", "model_revision": None, + "rounds": rounds, "tasks": list(tasks)} + source = protocol.collect_source_identity() + source["git_dirty"] = False + manifest = protocol.build_manifest(config, dataset_revision=dataset.DATASET_REVISION, + evaluator_revision=protocol.EVALUATOR_REVISION, model_revision=None, + versions={"codex": "codex-cli 0.154.0", "bpy": "5.2.1", + "scorer_python": "Python 3.13.15"}, provenance={}, source_identity=source) + protocol.write_manifest(root / "run-manifest.json", manifest) + return manifest + + def scorer_payload(self, **changes): + payload = {"clip_image_cosine_similarity": .5, "photometric_loss": .25, + "clip_model": protocol.CLIP_MODEL, "clip_revision": protocol.CLIP_REVISION, + "viga_revision": protocol.EVALUATOR_REVISION, + "evaluator_sha256": protocol.EVALUATOR_SHA256, "device": "cuda:0", + "torch_version": "2.8.0+cu128", "transformers_version": "4.55.4", + "numpy_version": "2.4.6", "pillow_version": "12.3.0"} + payload.update(changes) + return payload + + def test_standalone_scoring_rejects_tampered_run_manifest(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manifest = self.write_run_manifest(root) + manifest["protocol_sha256"] = "0" * 64 + protocol.write_manifest(root / "run-manifest.json", manifest) + with self.assertRaisesRegex(ValueError, "protocol hash"): + scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 10, + root / "clip") + + def test_fake_path_executable_is_never_used_and_argv_is_canonical(self): + import os, sys + from types import SimpleNamespace + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + fake = root / "python" + sentinel = root / "executed" + fake.write_text("#!/bin/sh\ntouch " + str(sentinel) + "\n") + fake.chmod(0o755) + image, target = root / "image.png", root / "target.png" + image.write_bytes(b"candidate"); target.write_bytes(b"target") + completed = SimpleNamespace(returncode=0, stdout=json.dumps(self.scorer_payload()), stderr="") + with patch.dict(os.environ, {"PATH": str(root)}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={"verified": True}), \ + patch.object(scoring.subprocess, "run", return_value=completed) as run: + result = scoring.score_pair(image, target, root / "clip", {"preflight": True}) + argv = run.call_args.args[0] + self.assertEqual(argv[:3], [str(Path(sys.executable).absolute()), "-I", + str(scoring.CANONICAL_SCORER)]) + self.assertFalse(sentinel.exists()) + self.assertEqual(result["candidate_image_sha256"], dataset.sha256(image)) + self.assertEqual(result["target_image_sha256"], dataset.sha256(target)) + + def test_clip_snapshot_manifest_rejects_extra_or_mutated_bytes(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp); root = base / "snapshot"; root.mkdir() + names = ["config.json", "merges.txt", "preprocessor_config.json", "pytorch_model.bin", + "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json", "vocab.json"] + rows = [] + for name in names: + payload = ("bytes-" + name).encode() + (root / name).write_bytes(payload) + rows.append({"path": name, "size": len(payload), + "sha256": __import__("hashlib").sha256(payload).hexdigest(), "symlink": False}) + manifest = base / "manifest.json" + manifest.write_text(json.dumps({"schema_version": 1, "repository": protocol.CLIP_MODEL, + "revision": protocol.CLIP_REVISION, "files": rows})) + with patch.object(scoring, "CLIP_SNAPSHOT_MANIFEST", manifest): + self.assertEqual(len(scoring.verify_clip_snapshot(root)["files"]), 8) + (root / "goal.py").write_text("pass") + with self.assertRaisesRegex(ValueError, "allowlist"): + scoring.verify_clip_snapshot(root) + (root / "goal.py").unlink() + (root / "tokenizer.json").write_bytes(b"mutated") + with self.assertRaisesRegex(ValueError, "bytes"): + scoring.verify_clip_snapshot(root) + + def test_score_pair_rejects_wrong_evaluator_or_clip_identity(self): + from types import SimpleNamespace + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); image = root / "image"; target = root / "target" + image.write_bytes(b"i"); target.write_bytes(b"t") + completed = SimpleNamespace(returncode=0, + stdout=json.dumps(self.scorer_payload(clip_revision="wrong")), stderr="") + with patch.object(scoring, "verify_clip_snapshot", return_value={"verified": True}), \ + patch.object(scoring.subprocess, "run", return_value=completed): + with self.assertRaisesRegex(ValueError, "identity"): + scoring.score_pair(image, target, root / "clip", {"preflight": True}) + + def test_runtime_preflight_rejects_wrong_python_and_extra_distribution(self): + import sys + from types import SimpleNamespace + from unittest.mock import patch + expected = dict(scoring._EXPECTED_RUNTIME) + expected.update({"python_build": ["main", "Clang 22.1.3"], + "lexical_executable": str(Path(sys.executable).absolute()), + "resolved_executable": str(Path(sys.executable).resolve()), + "prefix": str(Path(sys.executable).absolute().parent.parent), "base_prefix": "/base", + "isolated": 1, "ignore_environment": 1, "no_user_site": 1, "safe_path": True, + "cuda_available": True, + "distributions": [[name, version] for name, version in scoring.locked_distributions().items()]}) + wrong = {**expected, "python_version": "3.13.14"} + with patch.object(scoring.subprocess, "run", return_value=SimpleNamespace( + returncode=0, stdout=json.dumps(wrong), stderr="")): + with self.assertRaisesRegex(ValueError, "3.13.15"): + scoring.preflight_scorer_runtime() + expected["distributions"].append(["unexpected", "1"]) + with patch.object(scoring.subprocess, "run", return_value=SimpleNamespace( + returncode=0, stdout=json.dumps(expected), stderr="")): + with self.assertRaisesRegex(ValueError, "extras"): + scoring.preflight_scorer_runtime() + + def test_standalone_scoring_verifies_dataset_before_reading_renders(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.write_run_manifest(root) + with patch.object(scoring, "preflight_scorer_runtime", return_value={}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={}), \ + patch.object(dataset, "verify", side_effect=ValueError("target hash mismatch")) as verify: + with self.assertRaisesRegex(ValueError, "target hash mismatch"): + scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 10, + root / "clip") + verify.assert_called_once_with("level1/camera1", root / "dataset") + + def test_standalone_scoring_uses_independent_audit_render_not_model_png(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.write_run_manifest(root, rounds=1) + task = root / "level1-camera1" + (task / "work").mkdir(parents=True) + (task / "work/iteration01.png").write_bytes(b"model-written") + blend = task / "work/iteration01.blend" + blend.write_bytes(b"independently-audited-blend") + (task / "audit01").mkdir() + audited = task / "audit01/verified.png" + audited.write_bytes(b"independent-audit") + audit_record = {"objects": {"Cube": {"location": [1, 0, 0]}}} + (task / "audit01/metadata.json").write_text(json.dumps({ + "source": str(blend), "source_sha256": dataset.sha256(blend), + "png": str(audited), "png_sha256": dataset.sha256(audited), + "audit": audit_record, "runtime": {"returncode": 0}, + "evidence": {"blender_version": "5.2.1", "compute_device_type": "CUDA", + "scene_device": "GPU", "engine": "CYCLES", + "devices": [{"type": "CUDA", "use": True}]}})) + (task / "audit-admission.json").write_text(json.dumps([{ + "round": 1, "source": str(blend), "source_sha256": dataset.sha256(blend), + "image": str(audited), "image_sha256": dataset.sha256(audited), + "audit_sha256": protocol.digest(audit_record), + "meaningful_change_validated": True}])) + target = root / "dataset/level1/camera1/target.png" + target.parent.mkdir(parents=True); target.write_bytes(b"target") + with patch.object(scoring, "preflight_scorer_runtime", return_value={}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={}), \ + patch.object(dataset, "verify", return_value={"verified": True}), \ + patch.object(scoring, "score_pair", return_value={ + "clip_image_cosine_similarity": .5, "photometric_loss": .25}) as score: + result = scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 1, + root / "clip") + self.assertEqual(result["summary"]["completed_tasks"], 1) + self.assertEqual(score.call_args.args[0], audited) + + def test_standalone_scoring_rejects_unverified_audit_image(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.write_run_manifest(root, rounds=1) + audit = root / "level1-camera1/audit01" + audit.mkdir(parents=True) + (audit / "verified.png").write_bytes(b"unverified") + with patch.object(scoring, "preflight_scorer_runtime", return_value={}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={}), \ + patch.object(dataset, "verify", return_value={"verified": True}), \ + patch.object(scoring, "score_pair", return_value={ + "clip_image_cosine_similarity": .5, "photometric_loss": .25}) as score: + result = scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 1, + root / "clip") + self.assertEqual(result["summary"]["completed_tasks"], 0) + score.assert_not_called() + + def test_standalone_scoring_rejects_admission_without_audit_metadata(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.write_run_manifest(root, rounds=1) + task = root / "level1-camera1" + blend = task / "work/iteration01.blend" + blend.parent.mkdir(parents=True); blend.write_bytes(b"blend") + audit = task / "audit01"; audit.mkdir() + image = audit / "verified.png"; image.write_bytes(b"render") + (task / "audit-admission.json").write_text(json.dumps([{ + "round": 1, "source": str(blend), "source_sha256": dataset.sha256(blend), + "image": str(image), "image_sha256": dataset.sha256(image), + "audit_sha256": "a" * 64, "meaningful_change_validated": True}])) + target = root / "dataset/level1/camera1/target.png" + target.parent.mkdir(parents=True); target.write_bytes(b"target") + with patch.object(scoring, "preflight_scorer_runtime", return_value={}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={}), \ + patch.object(dataset, "verify", return_value={"verified": True}), \ + patch.object(scoring, "score_pair", return_value={ + "clip_image_cosine_similarity": .5, "photometric_loss": .25}) as score: + result = scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 1, + root / "clip") + self.assertEqual(result["summary"]["completed_tasks"], 0) + score.assert_not_called() + + def test_standalone_scoring_rejects_failed_or_non_cuda_audit_metadata(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.write_run_manifest(root, rounds=1) + task = root / "level1-camera1" + blend = task / "work/iteration01.blend" + blend.parent.mkdir(parents=True); blend.write_bytes(b"blend") + audit = task / "audit01"; audit.mkdir() + image = audit / "verified.png"; image.write_bytes(b"render") + audit_record = {"objects": {"Cube": {"location": [1, 0, 0]}}} + (audit / "metadata.json").write_text(json.dumps({ + "source": str(blend), "source_sha256": dataset.sha256(blend), + "png": str(image), "png_sha256": dataset.sha256(image), "audit": audit_record, + "runtime": {"returncode": 0}, "evidence": {"blender_version": "5.2.1", + "compute_device_type": "CUDA", "scene_device": "GPU", "engine": "BLENDER_EEVEE_NEXT", + "devices": [{"type": "CUDA", "use": True}]}})) + (task / "audit-admission.json").write_text(json.dumps([{ + "round": 1, "source": str(blend), "source_sha256": dataset.sha256(blend), + "image": str(image), "image_sha256": dataset.sha256(image), + "audit_sha256": protocol.digest(audit_record), + "meaningful_change_validated": True}])) + target = root / "dataset/level1/camera1/target.png" + target.parent.mkdir(parents=True); target.write_bytes(b"target") + with patch.object(scoring, "preflight_scorer_runtime", return_value={}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={}), \ + patch.object(dataset, "verify", return_value={"verified": True}), \ + patch.object(scoring, "score_pair", return_value={ + "clip_image_cosine_similarity": .5, "photometric_loss": .25}) as score: + result = scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 1, + root / "clip") + self.assertEqual(result["summary"]["completed_tasks"], 0) + score.assert_not_called() + + def test_standalone_aggregate_records_dataset_evaluator_and_scorer_provenance(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); manifest = self.write_run_manifest(root, rounds=1) + runtime = {"executable_sha256": "a" * 64, "scoring_lock_sha256": + manifest["protocol"]["source_identity"]["scoring_lock_sha256"]} + snapshot = {"manifest_sha256": "b" * 64} + with patch.object(scoring, "preflight_scorer_runtime", return_value=runtime), \ + patch.object(scoring, "verify_clip_snapshot", return_value=snapshot), \ + patch.object(dataset, "verify", return_value={"verified": True}): + result = scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 1, + root / "clip") + provenance = result["provenance"] + self.assertEqual(provenance["dataset_revision"], dataset.DATASET_REVISION) + self.assertEqual(provenance["evaluator_revision"], protocol.EVALUATOR_REVISION) + self.assertEqual(provenance["evaluator_sha256"], protocol.EVALUATOR_SHA256) + self.assertEqual(provenance["clip_revision"], protocol.CLIP_REVISION) + self.assertEqual(provenance["clip_snapshot"], snapshot) + self.assertEqual(provenance["scorer_sha256"], dataset.sha256(scoring.CANONICAL_SCORER)) + self.assertEqual(provenance["protocol_sha256"], manifest["protocol_sha256"]) + self.assertEqual(provenance["source_identity"], manifest["protocol"]["source_identity"]) + self.assertEqual(provenance["model"], "gpt-6-astra") + self.assertEqual(provenance["versions"]["codex"], "codex-cli 0.154.0") + self.assertEqual(provenance["scorer_runtime"], runtime) + + def test_aggregate_uses_nclip_and_earliest_round_ties_and_records_failures(self): + rows = [ + {"task": "level1/camera1", "round": 1, "status": "complete", "clip_image_cosine_similarity": .8, "photometric_loss": .4}, + {"task": "level1/camera1", "round": 2, "status": "complete", "clip_image_cosine_similarity": .8, "photometric_loss": .2}, + {"task": "level1/camera2", "round": 1, "status": "failed", "error": "render failed"}, + ] + result = scoring.aggregate(rows, expected_tasks=["level1/camera1", "level1/camera2", "level1/camera3"]) + best = result["tasks"][0]["best_round"] + self.assertEqual(best["round"], 1) + self.assertAlmostEqual(best["nclip"], .2) + self.assertEqual(result["summary"]["completed_tasks"], 1) + self.assertEqual(result["summary"]["failed_tasks"], 2) + self.assertAlmostEqual(result["summary"]["mean_best_nclip_completed_tasks"], .2) + self.assertAlmostEqual(result["summary"]["mean_best_photometric_loss_completed_tasks"], .4) + self.assertEqual(result["tasks"][2]["status"], "missing") + + def test_csv_flattens_generation_and_scorer_runtime_provenance(self): + provenance = {"protocol_sha256": "a" * 64, "model": "gpt-6-astra", + "model_revision": None, "source_identity": {"git_revision": "b" * 40, + "scoring_lock_sha256": "c" * 64}, + "versions": {"codex": "codex-cli 0.154.0", "bpy": "5.2.1"}, + "scorer_runtime": {"executable_sha256": "d" * 64, + "implementation": "CPython", "python_version": "3.11.16"}} + result = scoring.aggregate([], ["t"], provenance=provenance) + with tempfile.TemporaryDirectory() as tmp: + _, csv_path = scoring.write_outputs(result, Path(tmp)) + with csv_path.open(newline="") as stream: + row = next(csv.DictReader(stream)) + self.assertEqual(row["protocol_sha256"], "a" * 64) + self.assertEqual(row["scoring_lock_sha256"], "c" * 64) + self.assertEqual(row["scorer_python_sha256"], "d" * 64) + self.assertEqual(row["codex_version"], "codex-cli 0.154.0") + self.assertEqual(row["bpy_version"], "5.2.1") + + def test_json_and_csv_outputs_are_machine_readable(self): + provenance = {"dataset_revision": "dataset", "evaluator_revision": "evaluator", + "evaluator_sha256": "a" * 64, "clip_revision": "clip", + "scorer_sha256": "b" * 64} + result = scoring.aggregate([{"task": "t", "round": 1, "status": "complete", + "clip_image_cosine_similarity": .75, "photometric_loss": .5}], + ["t"], provenance=provenance) + with tempfile.TemporaryDirectory() as tmp: + json_path, csv_path = scoring.write_outputs(result, Path(tmp)) + self.assertEqual(json.loads(json_path.read_text())["summary"]["completed_tasks"], 1) + with csv_path.open(newline="") as stream: + rows = list(csv.DictReader(stream)) + self.assertEqual(rows[0]["best_round"], "1") + self.assertEqual(rows[0]["best_nclip"], "0.25") + for key, value in provenance.items(): + self.assertEqual(rows[0][key], value) + + +class SuiteProtocolTests(unittest.TestCase): + def test_ci_uses_immutable_actions_python_and_hashed_scoring_lock(self): + import re + root = Path(__file__).resolve().parents[1] + workflow = (root / ".github/workflows/blenderbench-portable.yml").read_text() + refs = re.findall(r"uses:\s*[^@\s]+@([^\s#]+)", workflow) + self.assertTrue(refs) + self.assertTrue(all(re.fullmatch(r"[0-9a-f]{40}", ref) for ref in refs)) + self.assertIn('runs-on: ubuntu-24.04', workflow) + self.assertIn('python-version: "3.13"', workflow) + self.assertIn('--require-hashes -r benchmarks/blenderbench_direct/requirements-fixture.lock', workflow) + self.assertNotIn('requirements-scoring.lock', workflow) + source = (root / "benchmarks/blenderbench_direct/requirements-scoring.in").read_text() + self.assertNotIn("--extra-index-url", source) + self.assertIn( + "torch @ https://download.pytorch.org/whl/cu128/torch-2.8.0%2Bcu128-cp313-cp313-manylinux_2_28_x86_64.whl", + source, + ) + lock = (root / "benchmarks/blenderbench_direct/requirements-scoring.lock").read_text() + for package in ("numpy", "pillow", "transformers", + "nvidia-cuda-nvrtc-cu12", "nvidia-cudnn-cu12", "triton"): + self.assertRegex(lock, rf"(?m)^{package}==[^\s]+.*\\\n(?:\s+--hash=sha256:[0-9a-f]{{64}})") + self.assertRegex(lock, r"(?m)^torch @ https://download\.pytorch\.org/[^\s]+.*\\\n(?:\s+--hash=sha256:[0-9a-f]{64})") + self.assertIn("requests==2.34.2", lock) + self.assertIn("setuptools==78.1.0", lock) + self.assertIn("numpy==2.4.6", lock) + self.assertIn("transformers==4.55.4", lock) + self.assertIn("tqdm==4.70.0", lock) + self.assertIn("urllib3==2.7.0", lock) + + def test_publication_result_manifests_are_small_complete_and_path_free(self): + root = Path(__file__).resolve().parents[1] / "benchmarks/blenderbench_direct/publication" + expected = { + "results-summary.json": "073be2c44f1a763b66dea5138eda1b8959e8553fef6ce288f1d78ee92807d6e8", + "reference-results.json": "1a50fc5ab89acf56bed4db49ec435a6bf1be866ba0213f1869a68891ba24cc0e", + "artifact-manifest.json": "087f17b1c953373d80aedb6287a1ba79363e9ebf4b232c374bed5fb11a29c978", + } + for name, digest in expected.items(): + self.assertEqual(dataset.sha256(root / name), digest) + text = (root / name).read_text(encoding="utf-8") + self.assertNotIn("/home/", text) + self.assertNotRegex(text, r"(?:token|password|secret|credential)\s*[:=]") + summary = json.loads((root / "results-summary.json").read_text()) + reference = json.loads((root / "reference-results.json").read_text()) + artifacts = json.loads((root / "artifact-manifest.json").read_text()) + self.assertEqual((summary["task_count"], summary["round_count"]), (27, 270)) + self.assertEqual(len(reference["rows"]), 27) + self.assertEqual(reference["experiment"]["scored_rounds"], 270) + self.assertEqual(len(reference["scoring_runtime"]["packages"]), 41) + self.assertEqual(reference["video"]["sha256"], + "04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17") + self.assertEqual(len(artifacts["tasks"]), 27) + self.assertEqual(sum(len(task["rounds"]) for task in artifacts["tasks"].values()), 270) + self.assertTrue(all(round_["blend_bytes"] > 0 for task in artifacts["tasks"].values() + for round_ in task["rounds"])) + self.assertEqual(artifacts["compilation_video"]["sha256"], + "04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17") + + def test_root_readme_pins_latest_video_and_metric_meaning(self): + text = (Path(__file__).resolve().parents[1] / "readme.md").read_text(encoding="utf-8") + immutable = ("https://huggingface.co/datasets/michaelgold/blenderbench-direct-results/resolve/" + "7c2c43be4517aee4d76d2b445578988de186970c/video/" + "blenderbench-complete-compilation.mp4") + self.assertIn(f"[![BlenderBench result](https://bpy.dev/assets/blenderbench-result.webp)]({immutable})", text) + self.assertIn("27 tasks / 270 rounds", text) + self.assertIn("CLIP image-embedding cosine similarity, not literal accuracy", text) + self.assertIn("04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17", text) + + def test_generation_prompt_freezes_public_protocol_without_edit_bounds(self): + from benchmarks.blenderbench_direct.suite import generation_prompt + prompt = generation_prompt("task text", Path("/work").absolute(), 10) + self.assertIn("exactly 10 meaningful", prompt) + self.assertIn("Call get_render_as_image_for_cli after each saved round", prompt) + self.assertIn("expected_output_blend", prompt) + self.assertIn("No CLIP/scoring", prompt) + self.assertIn("goal_code", prompt) + for tool in protocol.TOOLS: + self.assertIn(tool, prompt) + for obsolete in ("allowed_transforms", "location_bounds", "allowed edit"): + self.assertNotIn(obsolete, prompt) + + def test_committed_benchmark_runtime_and_docs_have_no_private_home_or_model_default(self): + import subprocess + root = Path(__file__).resolve().parents[1] + tracked = subprocess.check_output(["git", "ls-files", "benchmarks", "tools"], + cwd=root, text=True).splitlines() + untracked = subprocess.check_output( + ["git", "ls-files", "--others", "--exclude-standard", "benchmarks", "tools"], + cwd=root, text=True).splitlines() + files = sorted(set(tracked + untracked)) + offenders = {} + for relative in files: + path = root / relative + if (relative.startswith("benchmarks/blenderbench_direct/publication/") + or not relative.endswith((".py", ".md", ".rst", ".toml", ".json")) + or not path.exists()): + continue + text = path.read_text(encoding="utf-8") + found = [needle for needle in ("/home/mg", "gpt-6-astra", "configured-model", + "configured-codex", "configured-bpy", "configured-version") + if needle in text] + if found: + offenders[relative] = found + self.assertEqual(offenders, {}) + + +class VideoTests(unittest.TestCase): + def test_overlay_labels_metric_and_task_level_token_interpolation_explicitly(self): + state = video.timeline_state(5.0, duration=10.0, rounds=[ + {"round": 1, "elapsed_seconds": 2.0, "image": "one.png", "clip_image_cosine_similarity": .5}, + {"round": 2, "elapsed_seconds": 8.0, "image": "two.png", "clip_image_cosine_similarity": .8}, + ], task_tokens=1000) + self.assertEqual(state["round"], 1) + self.assertEqual(state["task_tokens_interpolated"], 500) + labels = video.overlay_labels("provider/model", "level1/camera1", state, 10) + self.assertIn("CLIP image cosine similarity: 0.500000", labels) + self.assertIn("Task-level tokens (linear interpolation): ~500", labels) + unavailable = video.timeline_state(1, duration=10, rounds=[], task_tokens=None) + self.assertIn("unavailable", video.overlay_labels("m", "t", unavailable, 10)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_direct_codex.py b/tests/test_direct_codex.py new file mode 100644 index 0000000..674f9e4 --- /dev/null +++ b/tests/test_direct_codex.py @@ -0,0 +1,397 @@ +"""Direct launcher tests; fixtures are synthetic, not benchmark evidence.""" +import importlib.util +from pathlib import Path +import unittest + +SCRIPT = Path(__file__).resolve().parents[1] / 'tools' / 'direct_codex.py' + +class DirectCodexTests(unittest.TestCase): + def load(self): + self.assertTrue(SCRIPT.is_file(), 'repo-backed launcher is missing') + spec = importlib.util.spec_from_file_location('direct_codex', SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def test_generation_exposes_visual_inspection_with_exactly_four_tools(self): + m = self.load() + self.assertEqual(len(m.TOOLS), 4) + self.assertIn('get_render_as_image_for_cli', m.TOOLS) + + def test_explicit_scoped_approval_required(self): + m = self.load() + config = {'mcp_servers': {'blender': {'command': '/server/bin/blender-mcp', + 'enabled_tools': list(m.TOOLS), 'env': {'BLENDER_MCP_CLI_BACKEND': 'bpy', + 'BLENDER_MCP_BPY_PYTHON': '/runtime/bin/python'}}}} + with self.assertRaisesRegex(ValueError, 'explicit approval'): + m.prepare_config(config, approved=False) + result = m.prepare_config(config, approved=True) + server = result['mcp_servers']['blender'] + self.assertEqual(server['tools'], {t: {'approval_mode': 'approve'} for t in m.TOOLS}) + self.assertNotIn('default_tools_approval_mode', server) + self.assertNotIn('tools', config['mcp_servers']['blender']) + self.assertTrue(server['required']) + self.assertEqual(server['omit_tools_from'], ['deferred', 'code_mode']) + + def test_rejects_extra_tools_and_conflicting_policy(self): + m = self.load() + base = {'mcp_servers': {'blender': {'command': '/server', + 'enabled_tools': list(m.TOOLS), 'env': {'BLENDER_MCP_CLI_BACKEND': 'bpy', + 'BLENDER_MCP_BPY_PYTHON': '/venv/bin/python'}}}} + import copy + for key, value in [('enabled_tools', list(m.TOOLS) + ['execute_blender_code']), + ('default_tools_approval_mode', 'auto'), + ('tools', {m.TOOLS[0]: {'approval_mode': 'prompt'}}), + ('enabled', False)]: + config = copy.deepcopy(base) + config['mcp_servers']['blender'][key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + m.prepare_config(config, approved=True) + for extra in [{'approval_policy': 'never'}, {'mcp_servers': {'other': {}}}]: + config = copy.deepcopy(base) + config.update(extra) + with self.assertRaises(ValueError): + m.prepare_config(config, approved=True) + + def test_generation_stream_allows_blender_resolution_properties(self): + m = self.load() + events = [ + {'type': 'item.completed', 'item': {'id': 'edit', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'execute_blender_code_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'code': 'bpy.context.scene.render.resolution_x = 512'}, + 'result': {'structured_content': {'saved': True}}}}, + {'type': 'turn.completed'}, + ] + m.verify_generation_events(events) + + def test_generation_stream_requires_one_successful_image_inspection_per_round(self): + m = self.load() + events = [ + {'type': 'item.completed', 'item': {'id': 'edit', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'execute_blender_code_for_cli', 'status': 'completed', + 'error': None, 'result': {'structured_content': {'saved': True}}}}, + {'type': 'turn.completed'}, + ] + with self.assertRaisesRegex(ValueError, 'visual inspection'): + m.verify_generation_events(events, rounds=1) + events.insert(1, {'type': 'item.completed', 'item': {'id': 'inspect', + 'type': 'mcp_tool_call', 'server': 'blender', 'tool': 'get_render_as_image_for_cli', + 'status': 'completed', 'error': None, + 'arguments': {'blend_file': '/work/iteration01.blend'}, + 'result': {'content': [{'type': 'image', 'mimeType': 'image/png', 'data': 'AA=='}]}}}) + m.verify_generation_events(events, rounds=1) + + def test_generation_stream_binds_visual_inspection_to_each_round_checkpoint(self): + m = self.load() + events = [ + {'type': 'item.completed', 'item': {'id': 'edit', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'execute_blender_code_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'code': 'edit'}, 'result': {'structured_content': {}}}}, + {'type': 'item.completed', 'item': {'id': 'inspect', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'get_render_as_image_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'blend_file': '/work/iteration02.blend'}, + 'result': {'content': [{'type': 'image', 'mimeType': 'image/png', 'data': 'AA=='}]}}}, + {'type': 'turn.completed'}, + ] + with self.assertRaisesRegex(ValueError, 'checkpoint sequence'): + m.verify_generation_events(events, rounds=1) + + def test_generation_stream_rejects_visual_inspection_from_wrong_directory(self): + import hashlib, tempfile + m = self.load() + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / 'work'; work.mkdir() + checkpoint = work / 'iteration01.blend'; checkpoint.write_bytes(b'checkpoint') + wrong = Path(tmp) / 'wrong' / checkpoint.name + events = [ + {'type': 'item.completed', 'item': {'id': 'edit', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'execute_blender_code_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'code': 'edit'}, 'result': {'structured_content': {}}}}, + {'type': 'item.completed', 'item': {'id': 'inspect', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'get_render_as_image_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'blend_file': str(wrong)}, + 'result': {'content': [{'type': 'image', 'mimeType': 'image/png', 'data': 'AA==', + '_meta': {'source_sha256': hashlib.sha256(b'checkpoint').hexdigest()}}]}}}, + {'type': 'turn.completed'}, + ] + with self.assertRaisesRegex(ValueError, 'checkpoint path'): + m.verify_generation_events(events, rounds=1, workdir=work) + + def test_generation_stream_rejects_unattested_precomputed_checkpoint(self): + import hashlib, tempfile + m = self.load() + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp); initial = work / 'initial.blend'; initial.write_bytes(b'initial') + checkpoint = work / 'iteration01.blend' + checkpoint.write_bytes(b'precomputed') + digest = hashlib.sha256(b'precomputed').hexdigest() + events = [ + {'type': 'item.completed', 'item': {'id': 'noop', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'execute_blender_code_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'blend_file': str(work / 'initial.blend'), 'code': 'result={}'}, + 'result': {'structured_content': {}}}}, + {'type': 'item.completed', 'item': {'id': 'inspect', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'get_render_as_image_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'blend_file': str(checkpoint)}, + 'result': {'content': [{'type': 'image', 'mimeType': 'image/png', 'data': 'AA==', + '_meta': {'source': str(checkpoint), 'source_sha256': digest}}]}}}, + {'type': 'turn.completed'}, + ] + with self.assertRaisesRegex(ValueError, 'attested checkpoint'): + m.verify_generation_events(events, rounds=1, workdir=work) + events[0]['item']['result']['structured_content'] = {'_checkpoint': { + 'path': str(checkpoint), 'source': str(initial), + 'source_sha256': hashlib.sha256(b'initial').hexdigest(), + 'output_sha256': digest, 'existed_before': False}} + m.verify_generation_events(events, rounds=1, workdir=work) + + def test_generation_stream_rejects_next_edit_dispatched_before_prior_inspection(self): + m = self.load() + checkpoint = lambda n: {'path': f'/work/iteration{n:02}.blend'} + image = lambda n: {'type': 'item.completed', 'item': {'id': f'i{n}', + 'type': 'mcp_tool_call', 'server': 'blender', 'tool': 'get_render_as_image_for_cli', + 'status': 'completed', 'error': None, + 'arguments': {'blend_file': f'/work/iteration{n:02}.blend'}, + 'result': {'content': [{'type': 'image', 'mimeType': 'image/png', 'data': 'AA=='}]}}} + edit = lambda n, kind='item.completed': {'type': kind, 'item': {'id': f'e{n}', + 'type': 'mcp_tool_call', 'server': 'blender', 'tool': 'execute_blender_code_for_cli', + 'status': 'completed', 'error': None, + 'arguments': {'expected_output_blend': f'/work/iteration{n:02}.blend'}, + 'result': {'structured_content': {'_checkpoint': checkpoint(n)}}}} + events = [edit(1, 'item.started'), edit(2, 'item.started'), edit(1), image(1), + edit(2), image(2), {'type': 'turn.completed'}] + with self.assertRaisesRegex(ValueError, 'dispatch'): + m.verify_generation_events(events, rounds=2) + + def test_preflight_requires_successful_mcp_result_not_exit_zero(self): + m = self.load() + self.assertTrue(hasattr(m, 'verify_preflight'), 'result verification missing') + import copy + event = {'type': 'item.completed', 'item': {'type': 'mcp_tool_call', + 'server': 'blender', 'tool': m.TOOLS[0], 'status': 'completed', 'error': None, + 'result': {'structured_content': {'preflight': 'nonce', + 'python': '/venv/bin/python', 'version': '5.2.1 LTS'}}}} + m.verify_preflight([event], 'nonce', '/venv/bin/python') + for events in [[], [{'type': 'turn.completed'}]]: + with self.assertRaises(ValueError): + m.verify_preflight(events, 'nonce', '/venv/bin/python') + for field, value in [('error', {'message': 'MCP tool call requires approval, but approval policy is never'}), + ('status', 'failed'), ('result', {'isError': True}), + ('result', {'structured_content': {'preflight': 'nonce', 'python': '/wrong'}})]: + broken = copy.deepcopy(event) + broken['item'][field] = value + with self.assertRaises(ValueError): + m.verify_preflight([broken], 'nonce', '/venv/bin/python') + + def test_checkpoints_required_and_pngs_decoded(self): + m = self.load() + self.assertTrue(hasattr(m, 'verify_checkpoints'), 'checkpoint verifier missing') + import tempfile, json + from PIL import Image + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + with self.assertRaises(ValueError): + m.verify_checkpoints(root, 2) + rows = [] + for n in (1, 2): + blend, png = root / f'iteration{n:02}.blend', root / f'iteration{n:02}.png' + blend.write_bytes(b'BLENDER' + bytes([n])) + Image.new('RGB', (2, 2)).save(png) + rows.append({'round': n, 'pose': {'location': [n, 0, 0], 'rotation': [0, 0, 0]}, + 'blend': str(blend), 'png': str(png)}) + (root / 'rounds.jsonl').write_text('\n'.join(map(json.dumps, rows))) + self.assertEqual(len(m.verify_checkpoints(root, 2)), 2) + (root / 'iteration02.png').write_bytes(b'not a PNG') + with self.assertRaises(ValueError): + m.verify_checkpoints(root, 2) + + def test_checkpoint_ancestors_may_not_be_symlinks(self): + m = self.load() + import tempfile + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + real = base / 'real' + real.mkdir() + linked = base / 'linked' + linked.symlink_to(real, target_is_directory=True) + with self.assertRaisesRegex(ValueError, 'symlink'): + m.verify_checkpoints(linked, 1) + + def test_generation_workdir_is_an_exact_noncontaminated_surface(self): + m = self.load() + import tempfile + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); (root / 'initial.blend').write_bytes(b'blend') + m.verify_generation_workdir(root) + (root / 'obfuscated.bin').write_bytes(b'hidden payload') + with self.assertRaisesRegex(ValueError, 'runtime root'): + m.verify_generation_workdir(root) + + def test_generation_workdir_accepts_canonical_worker_when_imported_as_tools_module(self): + m = self.load() + from tools import direct_runtime + import tempfile + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / 'initial.blend').write_bytes(b'blend') + (root / 'worker.py').write_text(direct_runtime.WORKER) + m.verify_generation_workdir(root) + + def test_cli_denied_preflight_stops_run_even_with_exit_zero(self): + m = self.load() + self.assertTrue(hasattr(m, 'main'), 'CLI missing') + import tempfile, json, subprocess, sys + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + work = root / 'work' + work.mkdir() + fake = root / 'codex' + fake.write_text('#!' + sys.executable + '\nimport sys\n' + 'print("codex-cli 0.154.0" if "--version" in sys.argv else ' + '\'{"type":"turn.completed"}\')\n') + fake.chmod(0o755) + (work / 'initial.blend').write_bytes(b'fixture') + (root / 'prompt.txt').write_text('fixture generation task') + (root / 'target.png').write_bytes(b'fixture') + config = root / 'server.toml' + config.write_text('[mcp_servers.blender]\ncommand="/server"\n' + 'enabled_tools=' + json.dumps(list(m.TOOLS)) + '\n' + '[mcp_servers.blender.env]\nBLENDER_MCP_CLI_BACKEND="bpy"\n' + 'BLENDER_MCP_BPY_PYTHON="/venv/bin/python"\n') + args = [sys.executable, str(SCRIPT), 'run', '--config', str(config), + '--codex', str(fake), '--codex-version', 'codex-cli 0.154.0', + '--bpy-version', '5.2.1', '--workdir', str(work), + '--evidence', str(root / 'logs'), + '--model', 'test-model', '--approve-blender-tools', + '--prompt', str(root / 'prompt.txt'), '--target', str(root / 'target.png')] + proc = subprocess.run(args, capture_output=True, text=True) + self.assertEqual(proc.returncode, 1, proc.stderr) + self.assertIn('no successful', proc.stderr) + self.assertFalse((root / 'logs' / 'run.jsonl').exists()) + argv = json.loads((root / 'logs' / 'preflight.argv.json').read_text()) + self.assertIn('--strict-config', argv) + self.assertIn('read-only', argv) + for feature in m.DISABLED_FEATURES: + self.assertIn(feature, argv) + self.assertIn('--disable', argv) + self.assertTrue(any('experimental_request_user_input=false' in a for a in argv)) + self.assertNotIn('--dangerously-bypass-approvals-and-sandbox', argv) + self.assertTrue(any('approval_mode' in a and 'approve' in a for a in argv)) + # A successful preflight followed by exit 0 and zero rounds still fails. + fake.write_text('#!' + sys.executable + '\nimport sys,json,re\n' + 'if "--version" in sys.argv:\n print("codex-cli 0.154.0"); sys.exit()\n' + 'prompt=sys.stdin.read()\n' + 'nonce=re.search(r"[a-f0-9]{32}", prompt)\n' + 'payload={"preflight":nonce.group() if nonce else "",' + '"python":"/venv/bin/python","version":"5.2.1 LTS"}\n' + 'print(json.dumps({"type":"item.completed","item":' + '{"id":"fixture-call","type":"mcp_tool_call","server":"blender","tool":"execute_blender_code_for_cli",' + '"status":"completed","error":None,"result":{"structured_content":payload}}}))\n') + args[args.index('--evidence') + 1] = str(root / 'logs2') + proc = subprocess.run(args, capture_output=True, text=True) + self.assertEqual(proc.returncode, 1, proc.stderr) + self.assertIn('generation event stream is empty or lacks', proc.stderr) + self.assertTrue((root / 'logs2' / 'run.jsonl').is_file()) + + def test_task_policy_is_ignored_but_noop_is_rejected(self): + import copy + m = self.load() + self.assertTrue(hasattr(m, 'validate_edit'), 'task policy validator missing') + before = {'objects': {'Cube': {'type': 'MESH', 'location': [0, 0, 0], + 'scale': [1, 1, 1]}}, 'preserve': {'mesh': 'fixed'}} + after = copy.deepcopy(before) + after['objects']['Cube']['location'] = [1, 0, 0] + policy = {'allowed_transforms': {'Cube': ['location']}, + 'location_bounds': {'Cube': [[-2, 2]] * 3}} + m.validate_edit(before, after, policy) + with self.assertRaises(ValueError): + m.validate_edit(before, before, policy) + m.validate_edit(before, {**after, 'preserve': {'mesh': 'changed'}}, policy) + after['objects']['Cube']['location'] = [3, 0, 0] + m.validate_edit(before, after, policy) + + def test_all_property_families_and_geometry_are_editable(self): + import copy + m = self.load() + before = {'objects': {'Cam': {'type': 'CAMERA', 'lens': 50}, + 'Lamp': {'type': 'LIGHT', 'light_data': {'energy': 20, 'color': [1, 1, 1]}}}, + 'shape_keys': {'Keys': {'Smile': 0, 'Basis': 0}}, 'preserve': {'mesh': 'fixed'}} + for policy, mutate in [ + ({'allowed_camera_data': {'Cam': ['lens']}}, lambda x: x['objects']['Cam'].update(lens=60)), + ({'allowed_light_data': {'Lamp': ['energy', 'color']}}, lambda x: x['objects']['Lamp']['light_data'].update(energy=30, color=[1, 0, 1])), + ({'allowed_shape_keys': {'Keys': ['Smile']}}, lambda x: x['shape_keys']['Keys'].update(Smile=0.5))]: + after = copy.deepcopy(before) + mutate(after) + m.validate_edit(before, after, policy) + after['preserve']['mesh'] = 'changed' + m.validate_edit(before, after, policy) + after = copy.deepcopy(before) + after['objects']['Cam']['lens'] = float('nan') + with self.assertRaises(ValueError): + m.validate_edit(before, after, {'allowed_camera_data': {'Cam': ['lens']}}) + with self.assertRaises(ValueError): + m.validate_edit(before, before, {'allowed_camera_data': {'Cam': ['sensor_width']}}) + with self.assertRaises(ValueError): + m.validate_edit(before, before, {'allowed_light_data': {'Lamp': ['shadow_soft_size']}}) + + def test_non_camera_ledger_needs_no_fake_pose(self): + import tempfile, json + from PIL import Image + m = self.load() + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + blend, png = root / 'iteration01.blend', root / 'iteration01.png' + blend.write_bytes(b'BLENDERfixture') + Image.new('RGB', (2, 2)).save(png) + (root / 'rounds.jsonl').write_text(json.dumps({'round': 1, + 'blend': str(blend), 'png': str(png), 'rationale': 'move cube'})) + self.assertEqual(len(m.verify_checkpoints(root, 1, policy={'allowed_transforms': {'Cube': ['location']}})), 1) + + def test_toml_roundtrip_preserves_keys_booleans_and_paths(self): + m = self.load() + import tomllib + value = {'literal.dot': {'env': {'PATH': '/space dir/bin'}, 'required': True, + 'tools': list(m.TOOLS)}} + self.assertEqual(tomllib.loads('mcp_servers=' + m.toml_value(value))['mcp_servers'], value) + + def test_generation_stream_rejects_empty_or_incomplete_turns(self): + m = self.load() + for events in ([], [ + {'type': 'item.completed', 'item': {'id': 'call', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': m.TOOLS[0], 'status': 'completed', + 'arguments': {}, 'result': {'structured_content': {'ok': True}}}}, + ], [{'type': 'turn.completed'}]): + with self.subTest(events=events), self.assertRaises(ValueError): + m.verify_generation_events(events) + + def test_generation_stream_fails_closed_on_nonapproved_operations_and_feedback(self): + m = self.load() + allowed = [ + {'type': 'item.completed', 'item': {'id': 'reason', 'type': 'reasoning'}}, + {'type': 'item.completed', 'item': {'id': 'call', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': m.TOOLS[0], 'status': 'completed', + 'arguments': {'code': "import bpy; bpy.context.scene.render.filepath='iteration01.png'"}, + 'result': {'structured_content': {'ok': True}}}}, + {'type': 'turn.completed'}, + ] + m.verify_generation_events(allowed) + prohibited = [ + {'type': 'item.completed', 'item': {'id': 'shell', 'type': 'command_execution', + 'command': 'python edit_scene.py'}}, + {'type': 'item.completed', 'item': {'id': 'write', 'type': 'file_change', + 'path': 'scene.blend'}}, + {'type': 'item.completed', 'item': {'id': 'goal', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': m.TOOLS[0], 'status': 'completed', + 'arguments': {'code': "open('../goal.py').read()"}}}, + {'type': 'item.completed', 'item': {'id': 'score', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': m.TOOLS[0], 'status': 'completed', + 'arguments': {'code': "json.load(open('scoring/aggregate.json'))"}}}, + ] + for event in prohibited: + with self.subTest(item=event['item']['id']), self.assertRaises(ValueError): + m.verify_generation_events([event]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_direct_recovery.py b/tests/test_direct_recovery.py new file mode 100644 index 0000000..1f2ac1f --- /dev/null +++ b/tests/test_direct_recovery.py @@ -0,0 +1,21 @@ +"""Compatibility entrypoint must not expose broken legacy recovery hooks.""" +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools")) +import direct_suite as suite + + +class RecoveryTests(unittest.TestCase): + def test_broken_legacy_recovery_hooks_are_removed(self): + self.assertFalse(hasattr(suite, "score_checkpoints")) + self.assertFalse(hasattr(suite, "recover_task")) + self.assertFalse(hasattr(suite, "run_task")) + self.assertFalse(hasattr(suite, "task_plan")) + self.assertTrue(callable(suite.main)) + self.assertTrue(callable(suite.run_suite)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_direct_saved_cleanup.py b/tests/test_direct_saved_cleanup.py new file mode 100644 index 0000000..3d79c38 --- /dev/null +++ b/tests/test_direct_saved_cleanup.py @@ -0,0 +1,87 @@ +"""Unrestricted saved edits; orphan cleanup alone is not a round.""" +import copy +import sys +import unittest +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools')) +import direct_codex as direct + + +class SavedCleanupTests(unittest.TestCase): + def fixture(self): + before = {'audit': {'objects': {'Camera': {'type': 'CAMERA', 'lens': 50}}, + 'preserve': {'materials': {'orphan': 'orphan-hash', 'used': 'used-hash'}, + 'meshes': {'mesh': 'geometry-hash'}, + 'scenes': {'Scene': 'lighting-hash'}}}, + 'material_lifecycle': {'orphan': {'users': 0, 'use_fake_user': False, + 'use_extra_user': False, 'library': None}, + 'used': {'users': 1, 'use_fake_user': False, + 'use_extra_user': False, 'library': None}}} + after = copy.deepcopy(before) + after['audit']['objects']['Camera']['lens'] = 55 + del after['audit']['preserve']['materials']['orphan'] + del after['material_lifecycle']['orphan'] + return before, after, {'allowed_camera_data': {'Camera': ['lens']}} + + def test_worker_records_material_lifecycle_without_mutating(self): + import ast + from types import SimpleNamespace as NS + import direct_runtime + functions = [n for n in ast.parse(direct_runtime.WORKER).body + if isinstance(n, ast.FunctionDef) and n.name == 'material_lifecycle'] + self.assertEqual(len(functions), 1) + material = NS(name='orphan', users=0, use_fake_user=False, + use_extra_user=False, library=None) + ns = {'bpy': NS(data=NS(materials=[material]))} + exec(compile(ast.Module(body=functions, type_ignores=[]), 'worker', 'exec'), ns) + self.assertEqual(ns['material_lifecycle']()['orphan'], + self.fixture()[0]['material_lifecycle']['orphan']) + self.assertFalse(material.use_fake_user) + + + def test_zero_user_unused_material_loss_is_accepted(self): + before, after, policy = self.fixture() + original = copy.deepcopy(before) + direct.validate_edit(before, after, policy) + self.assertEqual(before, original) + + def test_used_material_change_or_loss_is_accepted(self): + for value in ('changed-hash', None): + before, after, policy = self.fixture() + if value is None: + del after['audit']['preserve']['materials']['used'] + else: + after['audit']['preserve']['materials']['used'] = value + direct.validate_edit(before, after, policy) + + def test_material_edits_need_no_lifecycle_permission(self): + for field, value in [('users', 1), ('use_fake_user', True), + ('use_extra_user', True), ('library', '//linked.blend'), + ('users', None)]: + before, after, policy = self.fixture() + before['material_lifecycle']['orphan'][field] = value + direct.validate_edit(before, after, policy) + before, after, policy = self.fixture() + del before['material_lifecycle'] + direct.validate_edit(before, after, policy) + + def test_geometry_lighting_and_material_additions_are_accepted(self): + for collection, name in [('meshes', 'mesh'), ('scenes', 'Scene'), ('materials', 'new')]: + before, after, policy = self.fixture() + after['audit']['preserve'][collection][name] = 'changed' + direct.validate_edit(before, after, policy) + + def test_cleanup_alone_is_not_a_meaningful_round(self): + before, after, policy = self.fixture() + after['audit']['objects']['Camera']['lens'] = 50 + with self.assertRaisesRegex(ValueError, 'no meaningful scene change'): + direct.validate_edit(before, after, policy) + + def test_present_orphan_content_may_change(self): + before, after, policy = self.fixture() + after['audit']['preserve']['materials']['orphan'] = 'changed' + direct.validate_edit(before, after, policy) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_direct_suite.py b/tests/test_direct_suite.py new file mode 100644 index 0000000..42b07c8 --- /dev/null +++ b/tests/test_direct_suite.py @@ -0,0 +1,92 @@ +"""Synthetic orchestration contracts, never benchmark evidence.""" +import importlib.util +import sys +import unittest +from pathlib import Path + +TOOLS = Path(__file__).resolve().parents[1] / 'tools' +sys.path.insert(0, str(TOOLS)) + +class SuiteTests(unittest.TestCase): + def gpu_setup(self, active_engine='CYCLES'): + from types import SimpleNamespace as NS + import direct_runtime + active = NS(render=NS(engine=active_engine), cycles=NS(device='CPU')) + inactive = NS(render=NS(engine='BLENDER_EEVEE'), cycles=NS(device='CPU')) + devices = [NS(type='CUDA', use=False), NS(type='CPU', use=True)] + prefs = NS(compute_device_type='NONE', refresh_devices=lambda: None, devices=devices) + bpy = NS(context=NS(scene=active, preferences=NS(addons={'cycles': NS(preferences=prefs)})), + data=NS(scenes=[inactive, active])) + # Exercise the actual worker setup without importing native bpy locally. + setup = direct_runtime.WORKER.split('# GPU setup', 1)[1].split("assert s.camera", 1)[0] + exec('# GPU setup' + setup, {'bpy': bpy}) + return active, inactive, devices + + def test_inactive_eevee_scene_is_preserved_with_active_cycles(self): + active, inactive, devices = self.gpu_setup() + self.assertEqual(active.render.engine, 'CYCLES') + self.assertEqual(active.cycles.device, 'GPU') + self.assertEqual(inactive.render.engine, 'BLENDER_EEVEE') + self.assertEqual(inactive.cycles.device, 'CPU') + self.assertEqual([d.use for d in devices], [True, False]) + + def test_active_eevee_still_fails_closed_without_conversion(self): + with self.assertRaisesRegex(AssertionError, 'unsupported non-Cycles'): + self.gpu_setup('BLENDER_EEVEE') + + def test_rna_shared_graph_is_linear_and_preserves_leaves(self): + import ast + from types import SimpleNamespace + import direct_runtime + tree = ast.parse(direct_runtime.WORKER) + names = {'scalar', 'ref', 'collection_props', 'props'} + funcs = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names] + class ID: pass + class ShapeKey: pass + ns = {'bpy': SimpleNamespace(types=SimpleNamespace(ID=ID, ShapeKey=ShapeKey)), 'SKIP': set()} + exec(compile(ast.Module(body=funcs, type_ignores=[]), 'worker-test', 'exec'), ns) + pointer = lambda key: SimpleNamespace(identifier=key, type='POINTER', is_readonly=False) + value = SimpleNamespace(identifier='value', type='INT', is_readonly=False) + class Node: + calls = 0 + bl_rna = SimpleNamespace(identifier='Node', properties=[pointer('left'), pointer('right'), value]) + def __init__(self, child=None): self.left = self.right = child; self.value = 1 + def as_pointer(self): + Node.calls += 1 + return id(self) + leaf = Node() + node = leaf + for _ in range(12): node = Node(node) + a = ns['props'](node) + self.assertLess(Node.calls, 40, 'shared graph expands exponentially') + leaf.value = 2 + self.assertNotEqual(a, ns['props'](node), 'leaf content must remain audited') + + def test_heterogeneous_rna_collection_uses_each_schema(self): + import ast + from types import SimpleNamespace + import direct_runtime + tree = ast.parse(direct_runtime.WORKER) + funcs = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'collection_props'] + class ID: pass + class ShapeKey: pass + ns = {'bpy': SimpleNamespace(types=SimpleNamespace(ID=ID, ShapeKey=ShapeKey)), + 'SKIP': set(), 'props': lambda x, **kw: {'type': x.bl_rna.identifier} if x else None} + exec(compile(ast.Module(body=funcs, type_ignores=[]), 'worker-test', 'exec'), ns) + field = SimpleNamespace(identifier='text', type='POINTER', is_readonly=False) + a = SimpleNamespace(bl_rna=SimpleNamespace(identifier='NodeA', properties=[field]), text=None) + b = SimpleNamespace(bl_rna=SimpleNamespace(identifier='NodeB', properties=[])) + class Values(list): + def foreach_get(self, *args): pass + got = ns['collection_props'](Values([a, b] * 70), ()) + self.assertEqual(len(got), 140) + self.assertEqual(got[1], {'type': 'NodeB'}) + + def test_nonfunctional_legacy_retry_and_stale_validation_docs_are_removed(self): + self.assertFalse((TOOLS / 'direct_camera_retry.py').exists()) + self.assertFalse((TOOLS / 'direct_camera_retry.md').exists()) + self.assertFalse((TOOLS / 'direct_saved_validation.md').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_direct_unrestricted.py b/tests/test_direct_unrestricted.py new file mode 100644 index 0000000..437d243 --- /dev/null +++ b/tests/test_direct_unrestricted.py @@ -0,0 +1,36 @@ +"""User correction: edit validity is not a task allowlist.""" +import copy +import sys +import unittest +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools')) +import direct_codex as direct + +class UnrestrictedTests(unittest.TestCase): + def test_all_scene_edits_ignore_local_policy(self): + before = {'objects': {'Cube': {'location': [0, 0, 0]}}, + 'shape_keys': {'Key': {'Smile': 0}}, + 'preserve': {'materials': {'used': 'old'}, 'meshes': {'mesh': 'old'}}} + mutations = [ + lambda a: a['objects'].update(New={'type': 'LIGHT', 'energy': 300}), + lambda a: a['objects'].pop('Cube'), + lambda a: a['objects']['Cube'].update(location=[999, 0, 0]), + lambda a: a['objects'].update(Camera={'lens': 60}), + lambda a: a['shape_keys']['Key'].update(Smile=1), + lambda a: a['preserve']['materials'].update(used='new'), + lambda a: a['preserve']['meshes'].update(mesh='new'), + ] + for mutate in mutations: + after = copy.deepcopy(before) + mutate(after) + direct.validate_edit(before, after, {'unsupported_blocker': True, + 'location_bounds': {'Cube': [[-1, 1]] * 3}}) + + def test_nonfinite_and_noop_still_invalid(self): + before = {'objects': {'Cube': {'location': [0, 0, 0]}}} + with self.assertRaisesRegex(ValueError, 'no meaningful scene change'): + direct.validate_edit(before, before, {}) + after = copy.deepcopy(before) + after['objects']['Cube']['location'][0] = float('nan') + with self.assertRaisesRegex(ValueError, 'finite'): + direct.validate_edit(before, after, {}) diff --git a/tests/test_tool_listing.py b/tests/test_tool_listing.py index 7077f1e..e6826f0 100644 --- a/tests/test_tool_listing.py +++ b/tests/test_tool_listing.py @@ -28,12 +28,12 @@ { "name": "execute_blender_code", "description": "\n" - "Execute Python code in the connected Blender instance.\n" - "\n" - "The code runs in Blender's Python environment with full access to ``bpy``.\n" - "To return data, assign a JSON-serialisable dict to a variable named ``result``.\n" - "Deferred completion via ``check_is_finished`` is only supported by the\n" - "interactive addon server, and is rejected in background mode.\n", +"Execute Python code in the connected Blender instance.\n" +"\n" +"The code runs in Blender's Python environment with full access to ``bpy``.\n" +"To return data, assign a JSON-serialisable dict to a variable named ``result``.\n" +"Deferred completion via ``check_is_finished`` is only supported by the\n" +"interactive addon server, and is rejected in background mode.\n", "inputSchema": { "properties": { "code": { @@ -51,10 +51,12 @@ { "name": "execute_blender_code_for_cli", "description": "\n" - "Execute Python code in a background Blender process.\n" - "\n" - "Opens *blend_file* with ``blender --background`` and runs *code*.\n" - "Assign a dict to ``result`` to return data.\n", +"Execute Python code in a background Blender process.\n" +"\n" +"Opens *blend_file* with ``blender --background`` and runs *code*.\n" +"Assign a dict to ``result`` to return data. When *expected_output_blend*\n" +"is supplied, that path must be newly created with bytes different from\n" +"the source; the result includes server-computed checkpoint hashes.\n", "inputSchema": { "properties": { "blend_file": { @@ -64,6 +66,18 @@ "code": { "title": "Code", "type": "string" + }, + "expected_output_blend": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": None, + "title": "Expected Output Blend" } }, "required": [ @@ -74,10 +88,36 @@ "type": "object" } }, + { + "name": "get_render_as_image", + "description": "Render the connected Blender scene and return PNG image content without saving overrides.", + "inputSchema": { + "properties": {}, + "title": "get_render_as_imageArguments", + "type": "object" + } + }, + { + "name": "get_render_as_image_for_cli", + "description": "Open a saved blend in a background process, render it, and return PNG image content.", + "inputSchema": { + "properties": { + "blend_file": { + "title": "Blend File", + "type": "string" + } + }, + "required": [ + "blend_file" + ], + "title": "get_render_as_image_for_cliArguments", + "type": "object" + } + }, { "name": "get_blendfile_summary_datablocks", "description": "\n" - "Return a summary of the blend file: data-block counts, active workspace, and render engine.\n", +"Return a summary of the blend file: data-block counts, active workspace, and render engine.\n", "inputSchema": { "properties": {}, "title": "get_blendfile_summary_datablocksArguments", @@ -87,7 +127,7 @@ { "name": "get_blendfile_summary_datablocks_for_cli", "description": "\n" - "Return a data-block summary by opening *blend_file* in background Blender.\n", +"Return a data-block summary by opening *blend_file* in background Blender.\n", "inputSchema": { "properties": { "blend_file": { @@ -105,8 +145,8 @@ { "name": "get_blendfile_summary_missing_files", "description": "\n" - "Report external file references that are missing from disk\n" - "(images, libraries, fonts, sounds, movie clips, caches, sequences).\n", +"Report external file references that are missing from disk\n" +"(images, libraries, fonts, sounds, movie clips, caches, sequences).\n", "inputSchema": { "properties": {}, "title": "get_blendfile_summary_missing_filesArguments", @@ -116,7 +156,7 @@ { "name": "get_blendfile_summary_missing_files_for_cli", "description": "\n" - "Report missing file references by opening *blend_file* in background Blender.\n", +"Report missing file references by opening *blend_file* in background Blender.\n", "inputSchema": { "properties": { "blend_file": { @@ -134,7 +174,7 @@ { "name": "get_blendfile_summary_of_linked_libraries", "description": "\n" - "Return a tree of directly and indirectly linked library files.\n", +"Return a tree of directly and indirectly linked library files.\n", "inputSchema": { "properties": {}, "title": "get_blendfile_summary_of_linked_librariesArguments", @@ -144,7 +184,7 @@ { "name": "get_blendfile_summary_of_linked_libraries_for_cli", "description": "\n" - "Return linked-library info by opening *blend_file* in background Blender.\n", +"Return linked-library info by opening *blend_file* in background Blender.\n", "inputSchema": { "properties": { "blend_file": { @@ -162,7 +202,7 @@ { "name": "get_blendfile_summary_path_info", "description": "\n" - "Simple/fast access to the blend file's path, save status, age, and backups.\n", +"Simple/fast access to the blend file's path, save status, age, and backups.\n", "inputSchema": { "properties": {}, "title": "get_blendfile_summary_path_infoArguments", @@ -172,7 +212,7 @@ { "name": "get_blendfile_summary_path_info_for_cli", "description": "\n" - "Return path info by opening *blend_file* in background Blender.\n", +"Return path info by opening *blend_file* in background Blender.\n", "inputSchema": { "properties": { "blend_file": { @@ -190,7 +230,7 @@ { "name": "get_blendfile_summary_usage_guess", "description": "\n" - "Guess the primary use-cases of the current blend file (scored 0-100 with certainty).\n", +"Guess the primary use-cases of the current blend file (scored 0-100 with certainty).\n", "inputSchema": { "properties": {}, "title": "get_blendfile_summary_usage_guessArguments", @@ -200,7 +240,7 @@ { "name": "get_blendfile_summary_usage_guess_for_cli", "description": "\n" - "Guess use-cases by opening *blend_file* in background Blender.\n", +"Guess use-cases by opening *blend_file* in background Blender.\n", "inputSchema": { "properties": { "blend_file": { @@ -218,10 +258,10 @@ { "name": "get_object_detail_summary", "description": "\n" - "Return a structured summary of the object identified by *name*.\n" - "\n" - "Includes type, transforms, parent, children, modifiers, constraints,\n" - "materials, visibility, data-block name, and collections.\n", +"Return a structured summary of the object identified by *name*.\n" +"\n" +"Includes type, transforms, parent, children, modifiers, constraints,\n" +"materials, visibility, data-block name, and collections.\n", "inputSchema": { "properties": { "name": { @@ -239,10 +279,10 @@ { "name": "get_objects_summary", "description": "\n" - "Return the scene's collection hierarchy and their objects.\n" - "\n" - "Each collection lists its objects (name, type, parent, data name,\n" - "selection, visibility) and nested child collections.\n", +"Return the scene's collection hierarchy and their objects.\n" +"\n" +"Each collection lists its objects (name, type, parent, data name,\n" +"selection, visibility) and nested child collections.\n", "inputSchema": { "properties": {}, "title": "get_objects_summaryArguments", @@ -252,55 +292,55 @@ { "name": "get_python_api_docs", "description": "\n" - "Return the Blender Python API docs for *identifier*, or list\n" - "modules matching a trailing-``*`` discovery pattern.\n" - "\n" - "*identifier* should be a fully-qualified Python name (e.g.\n" - "``bpy.app`` or ``bpy.types.Scene.frame_current``).\n" - "The trailing-``*`` forms are supported as discovery entry-points:\n" - "\n" - "- ``*`` enumerates the top-level modules (``bpy``, ``bmesh``,\n" - " ``mathutils``, ``gpu``, ...).\n" - "- ``X.*`` enumerates the direct-child identifiers under the\n" - " *X* namespace (``bpy.*`` -> ``bpy.app``, ``bpy.context``, ...).\n" - "\n" - "Both return a ``namespace`` response even when ``X.rst`` would\n" - "otherwise resolve to ``exact``; the ``.*`` form lets an agent\n" - "force the child listing.\n" - "\n" - "The response always carries ``kind``, ``found``, and ``identifier``.\n" - "The remaining keys depend on ``kind``:\n" - "\n" - "- ``\"exact\"`` (``found=True``): ``.rst`` was read.\n" - " Extra keys: ``content`` (RST text), ``examples``. When the\n" - " file exceeds 32 KB, ``content`` is replaced with a dot-point\n" - " summary of the file's top-level definitions (prefixed by a\n" - " header noting the truncation) and ``examples`` is empty -\n" - " re-query individual members for their rendered blocks.\n" - "- ``\"namespace\"`` (``found=True``):\n" - " no ``.rst`` but ``..rst`` siblings exist.\n" - " Extra key: ``submodules`` (list of child identifiers).\n" - "- ``\"definition\"`` (``found=True``):\n" - " *identifier* is defined inside a parent RST\n" - " (e.g. ``bpy.props.IntProperty`` lives in ``bpy.props.rst``).\n" - " Extra keys: ``content`` (rendered block), ``examples``.\n" - "- ``\"partial\"`` (``found=False``):\n" - " the parent RST was located but the trailing component isn't defined in it.\n" - " Extra keys:\n" - " - ``parent`` the identifier whose RST was loaded.\n" - " - ``available`` top-level definitions in that RST.\n" - " - ``submodules`` sibling identifiers ``.`` with their own RSTs,\n" - " filtered to those whose last component contains every character of the missing tail.\n" - "\n" - " For a toctree landing page like ``bpy.types`` ``available`` is empty and ``submodules``\n" - " is the near-miss list; for a self-contained module like ``bpy.props`` it's the reverse.\n" - "- ``\"suggestions\"`` (``found=False``):\n" - " no direct match, but *identifier* appears as a component of other files.\n" - " Extra key: ``suggestions`` (list of full identifiers).\n" - "- ``\"missing\"`` (``found=False``): nothing matched.\n" - "\n" - "``examples`` (present on the ``exact`` and ``definition`` kinds)\n" - "is a list of ``{path, content}`` entries referenced from this documentation.\n", +"Return the Blender Python API docs for *identifier*, or list\n" +"modules matching a trailing-``*`` discovery pattern.\n" +"\n" +"*identifier* should be a fully-qualified Python name (e.g.\n" +"``bpy.app`` or ``bpy.types.Scene.frame_current``).\n" +"The trailing-``*`` forms are supported as discovery entry-points:\n" +"\n" +"- ``*`` enumerates the top-level modules (``bpy``, ``bmesh``,\n" +" ``mathutils``, ``gpu``, ...).\n" +"- ``X.*`` enumerates the direct-child identifiers under the\n" +" *X* namespace (``bpy.*`` -> ``bpy.app``, ``bpy.context``, ...).\n" +"\n" +"Both return a ``namespace`` response even when ``X.rst`` would\n" +"otherwise resolve to ``exact``; the ``.*`` form lets an agent\n" +"force the child listing.\n" +"\n" +"The response always carries ``kind``, ``found``, and ``identifier``.\n" +"The remaining keys depend on ``kind``:\n" +"\n" +"- ``\"exact\"`` (``found=True``): ``.rst`` was read.\n" +" Extra keys: ``content`` (RST text), ``examples``. When the\n" +" file exceeds 32 KB, ``content`` is replaced with a dot-point\n" +" summary of the file's top-level definitions (prefixed by a\n" +" header noting the truncation) and ``examples`` is empty -\n" +" re-query individual members for their rendered blocks.\n" +"- ``\"namespace\"`` (``found=True``):\n" +" no ``.rst`` but ``..rst`` siblings exist.\n" +" Extra key: ``submodules`` (list of child identifiers).\n" +"- ``\"definition\"`` (``found=True``):\n" +" *identifier* is defined inside a parent RST\n" +" (e.g. ``bpy.props.IntProperty`` lives in ``bpy.props.rst``).\n" +" Extra keys: ``content`` (rendered block), ``examples``.\n" +"- ``\"partial\"`` (``found=False``):\n" +" the parent RST was located but the trailing component isn't defined in it.\n" +" Extra keys:\n" +" - ``parent`` the identifier whose RST was loaded.\n" +" - ``available`` top-level definitions in that RST.\n" +" - ``submodules`` sibling identifiers ``.`` with their own RSTs,\n" +" filtered to those whose last component contains every character of the missing tail.\n" +"\n" +" For a toctree landing page like ``bpy.types`` ``available`` is empty and ``submodules``\n" +" is the near-miss list; for a self-contained module like ``bpy.props`` it's the reverse.\n" +"- ``\"suggestions\"`` (``found=False``):\n" +" no direct match, but *identifier* appears as a component of other files.\n" +" Extra key: ``suggestions`` (list of full identifiers).\n" +"- ``\"missing\"`` (``found=False``): nothing matched.\n" +"\n" +"``examples`` (present on the ``exact`` and ``definition`` kinds)\n" +"is a list of ``{path, content}`` entries referenced from this documentation.\n", "inputSchema": { "properties": { "identifier": { @@ -318,10 +358,10 @@ { "name": "get_runtime_python_api_docs", "description": "\n" - "Return compact docs for an exact Python API *identifier* from the connected Blender runtime.\n" - "\n" - "Use a fully-qualified identifier such as ``bpy.ops.mesh.primitive_cube_add``\n" - "or ``bpy.types.Object.location``. Use ``search_api_docs`` to discover identifiers.\n", +"Return compact docs for an exact Python API *identifier* from the connected Blender runtime.\n" +"\n" +"Use a fully-qualified identifier such as ``bpy.ops.mesh.primitive_cube_add``\n" +"or ``bpy.types.Object.location``. Use ``search_api_docs`` to discover identifiers.\n", "inputSchema": { "properties": { "identifier": { @@ -339,11 +379,11 @@ { "name": "get_runtime_python_api_docs_for_cli", "description": "\n" - "Return compact docs for an exact Python API *identifier* from a background runtime.\n" - "\n" - "Opens *blend_file* with the selected Blender or ``bpy`` CLI backend. Use a\n" - "fully-qualified identifier such as ``bpy.ops.mesh.primitive_cube_add`` or\n" - "``bpy.types.Object.location``. Use ``search_api_docs`` to discover identifiers.\n", +"Return compact docs for an exact Python API *identifier* from a background runtime.\n" +"\n" +"Opens *blend_file* with the selected Blender or ``bpy`` CLI backend. Use a\n" +"fully-qualified identifier such as ``bpy.ops.mesh.primitive_cube_add`` or\n" +"``bpy.types.Object.location``. Use ``search_api_docs`` to discover identifiers.\n", "inputSchema": { "properties": { "blend_file": { @@ -366,12 +406,12 @@ { "name": "get_screenshot_of_area_as_image", "description": "\n" - "Take a screenshot of a single Blender area and return it as a PNG image.\n" - "\n" - "*area_ui_type* matches the area's ``ui_type``.\n" - "\n" - "*size_limit_in_bytes* caps the image size in bytes.\n" - "Zero (the default) uses the MCP message size limit.\n", +"Take a screenshot of a single Blender area and return it as a PNG image.\n" +"\n" +"*area_ui_type* matches the area's ``ui_type``.\n" +"\n" +"*size_limit_in_bytes* caps the image size in bytes.\n" +"Zero (the default) uses the MCP message size limit.\n", "inputSchema": { "properties": { "area_ui_type": { @@ -418,10 +458,10 @@ { "name": "get_screenshot_of_window_as_image", "description": "\n" - "Take a screenshot of the entire Blender window and return it as a PNG image.\n" - "\n" - "*size_limit_in_bytes* caps the image size in bytes.\n" - "Zero (the default) uses the MCP message size limit.\n", +"Take a screenshot of the entire Blender window and return it as a PNG image.\n" +"\n" +"*size_limit_in_bytes* caps the image size in bytes.\n" +"Zero (the default) uses the MCP message size limit.\n", "inputSchema": { "properties": { "size_limit_in_bytes": { @@ -437,7 +477,7 @@ { "name": "get_screenshot_of_window_as_json", "description": "\n" - "Return a JSON description of the Blender window layout, areas, active object, and selection.\n", +"Return a JSON description of the Blender window layout, areas, active object, and selection.\n", "inputSchema": { "properties": {}, "title": "get_screenshot_of_window_as_jsonArguments", @@ -447,7 +487,7 @@ { "name": "jump_to_tab_by_name", "description": "\n" - "Switch the active workspace tab to *name*.\n", +"Switch the active workspace tab to *name*.\n", "inputSchema": { "properties": { "name": { @@ -465,10 +505,10 @@ { "name": "jump_to_tab_by_space_type", "description": "\n" - "Switch to a workspace whose main area matches *space_type*.\n" - "\n" - "If *allow_edits* is True and no matching workspace exists, a new one\n" - "is created by duplicating the current workspace.\n", +"Switch to a workspace whose main area matches *space_type*.\n" +"\n" +"If *allow_edits* is True and no matching workspace exists, a new one\n" +"is created by duplicating the current workspace.\n", "inputSchema": { "properties": { "space_type": { @@ -491,10 +531,10 @@ { "name": "jump_to_view3d_object_by_name", "description": "\n" - "Move the 3D viewport to focus on an object by *name*.\n" - "\n" - "If *allow_edits* is True the object may be un-hidden and its\n" - "collections enabled to make it visible.\n", +"Move the 3D viewport to focus on an object by *name*.\n" +"\n" +"If *allow_edits* is True the object may be un-hidden and its\n" +"collections enabled to make it visible.\n", "inputSchema": { "properties": { "name": { @@ -517,10 +557,10 @@ { "name": "jump_to_view3d_object_data_by_name", "description": "\n" - "Move the 3D viewport to the object whose data block matches *name*.\n" - "\n" - "If *allow_edits* is True the object may be un-hidden and its\n" - "collections enabled to make it visible.\n", +"Move the 3D viewport to the object whose data block matches *name*.\n" +"\n" +"If *allow_edits* is True the object may be un-hidden and its\n" +"collections enabled to make it visible.\n", "inputSchema": { "properties": { "name": { @@ -543,7 +583,7 @@ { "name": "render_thumbnail_to_path", "description": "\n" - "Render a small, low-quality thumbnail to *output_path* (temporarily overrides settings).\n", +"Render a small, low-quality thumbnail to *output_path* (temporarily overrides settings).\n", "inputSchema": { "properties": { "output_path": { @@ -561,7 +601,7 @@ { "name": "render_viewport_to_path", "description": "\n" - "Render the current scene to *output_path* using current render settings.\n", +"Render the current scene to *output_path* using current render settings.\n", "inputSchema": { "properties": { "output_path": { @@ -579,32 +619,32 @@ { "name": "search_api_docs", "description": "\n" - "Full-text search over the bundled Blender Python API reference.\n" - "\n" - "Returns a ranked list of hits. Each hit has:\n" - "\n" - "- ``path``: file path relative to the bundled docs.\n" - "- ``text``: the matching paragraph plus ``context``\n" - " paragraphs on either side.\n" - "- ``breadcrumb``: the section path containing the hit\n" - " (``Section > Sub-section > ...``).\n" - "- ``index``: the hit's position in the result list.\n" - "- ``score``: a relevance score; higher is better.\n" - "\n" - "The query is tokenised on whitespace and matched\n" - "case-insensitively. Every token must appear somewhere in\n" - "the paragraph body, the file path, or an enclosing section\n" - "title - in any order. Common English stop-words (``the``,\n" - "``a``, ``how``, ``to``, ...) are dropped, so natural\n" - "phrasings like ``\"how to bake\"`` work as expected. Regular\n" - "expressions are not supported.\n" - "\n" - "Use ``context`` to pull more surrounding paragraphs into\n" - "each hit (symmetric, default 0). Use ``index`` with the\n" - "position of a previous hit (same query) to get that hit\n" - "alone with its text widened to its enclosing section.\n" - "\n" - "Read-only; consults bundled RST files only.\n", +"Full-text search over the bundled Blender Python API reference.\n" +"\n" +"Returns a ranked list of hits. Each hit has:\n" +"\n" +"- ``path``: file path relative to the bundled docs.\n" +"- ``text``: the matching paragraph plus ``context``\n" +" paragraphs on either side.\n" +"- ``breadcrumb``: the section path containing the hit\n" +" (``Section > Sub-section > ...``).\n" +"- ``index``: the hit's position in the result list.\n" +"- ``score``: a relevance score; higher is better.\n" +"\n" +"The query is tokenised on whitespace and matched\n" +"case-insensitively. Every token must appear somewhere in\n" +"the paragraph body, the file path, or an enclosing section\n" +"title - in any order. Common English stop-words (``the``,\n" +"``a``, ``how``, ``to``, ...) are dropped, so natural\n" +"phrasings like ``\"how to bake\"`` work as expected. Regular\n" +"expressions are not supported.\n" +"\n" +"Use ``context`` to pull more surrounding paragraphs into\n" +"each hit (symmetric, default 0). Use ``index`` with the\n" +"position of a previous hit (same query) to get that hit\n" +"alone with its text widened to its enclosing section.\n" +"\n" +"Read-only; consults bundled RST files only.\n", "inputSchema": { "properties": { "query": { @@ -644,32 +684,32 @@ { "name": "search_manual_docs", "description": "\n" - "Full-text search over the bundled Blender user manual.\n" - "\n" - "Returns a ranked list of hits. Each hit has:\n" - "\n" - "- ``path``: file path relative to the bundled docs.\n" - "- ``text``: the matching paragraph plus ``context``\n" - " paragraphs on either side.\n" - "- ``breadcrumb``: the section path containing the hit\n" - " (``Section > Sub-section > ...``).\n" - "- ``index``: the hit's position in the result list.\n" - "- ``score``: a relevance score; higher is better.\n" - "\n" - "The query is tokenised on whitespace and matched\n" - "case-insensitively. Every token must appear somewhere in\n" - "the paragraph body, the file path, or an enclosing section\n" - "title - in any order. Common English stop-words (``the``,\n" - "``a``, ``how``, ``to``, ...) are dropped, so natural\n" - "phrasings like ``\"how to bake\"`` work as expected. Regular\n" - "expressions are not supported.\n" - "\n" - "Use ``context`` to pull more surrounding paragraphs into\n" - "each hit (symmetric, default 0). Use ``index`` with the\n" - "position of a previous hit (same query) to get that hit\n" - "alone with its text widened to its enclosing section.\n" - "\n" - "Read-only; consults bundled RST files only.\n", +"Full-text search over the bundled Blender user manual.\n" +"\n" +"Returns a ranked list of hits. Each hit has:\n" +"\n" +"- ``path``: file path relative to the bundled docs.\n" +"- ``text``: the matching paragraph plus ``context``\n" +" paragraphs on either side.\n" +"- ``breadcrumb``: the section path containing the hit\n" +" (``Section > Sub-section > ...``).\n" +"- ``index``: the hit's position in the result list.\n" +"- ``score``: a relevance score; higher is better.\n" +"\n" +"The query is tokenised on whitespace and matched\n" +"case-insensitively. Every token must appear somewhere in\n" +"the paragraph body, the file path, or an enclosing section\n" +"title - in any order. Common English stop-words (``the``,\n" +"``a``, ``how``, ``to``, ...) are dropped, so natural\n" +"phrasings like ``\"how to bake\"`` work as expected. Regular\n" +"expressions are not supported.\n" +"\n" +"Use ``context`` to pull more surrounding paragraphs into\n" +"each hit (symmetric, default 0). Use ``index`` with the\n" +"position of a previous hit (same query) to get that hit\n" +"alone with its text widened to its enclosing section.\n" +"\n" +"Read-only; consults bundled RST files only.\n", "inputSchema": { "properties": { "query": { diff --git a/tools/DIRECT_SUITE.md b/tools/DIRECT_SUITE.md new file mode 100644 index 0000000..2161387 --- /dev/null +++ b/tools/DIRECT_SUITE.md @@ -0,0 +1,14 @@ +# BlenderBench direct suite + +The former host-specific dispatcher has been replaced by the model-neutral, +full-27 runner in [`benchmarks/blenderbench_direct`](../benchmarks/blenderbench_direct/README.md). + +Use: + +```sh +python -m benchmarks.blenderbench_direct.runner --help +``` + +All dataset, MCP, Codex, bpy, scorer, output, model, and version inputs are +explicit. New generation always requires a fresh output root. Recovery and +scoring are separate commands and never regenerate missing rounds. diff --git a/tools/direct_codex.example.toml b/tools/direct_codex.example.toml new file mode 100644 index 0000000..74cf49c --- /dev/null +++ b/tools/direct_codex.example.toml @@ -0,0 +1,13 @@ +# Reviewed server config only; the launcher adds exact per-tool approvals +# ONLY with --approve-blender-tools. Keep machine paths outside the repo. +[mcp_servers.blender] +command = "/absolute/server-venv/bin/blender-mcp" +cwd = "/absolute/blender-mcp/mcp" +enabled_tools = ["execute_blender_code_for_cli", "get_render_as_image_for_cli", "get_runtime_python_api_docs_for_cli", "search_api_docs"] +required = true +# Explicit client deadline, NOT a backend timeout override or no-deadline promise. +tool_timeout_sec = 86400 + +[mcp_servers.blender.env] +BLENDER_MCP_CLI_BACKEND = "bpy" +BLENDER_MCP_BPY_PYTHON = "/absolute/bpy-venv/bin/python" diff --git a/tools/direct_codex.md b/tools/direct_codex.md new file mode 100644 index 0000000..e450a6a --- /dev/null +++ b/tools/direct_codex.md @@ -0,0 +1,45 @@ +# Low-level direct Codex launcher + +`direct_codex.py` is the low-level one-task launcher used by the portable +[full BlenderBench runner](../benchmarks/blenderbench_direct/README.md). Prefer the +full runner for publication runs because it owns pinned data, initialization, +post-generation saved-scene audit, scoring, aggregation, and provenance. + +The launcher admits exactly these MCP tools: + +- `execute_blender_code_for_cli` +- `get_render_as_image_for_cli` +- `get_runtime_python_api_docs_for_cli` +- `search_api_docs` + +It requires explicit scoped preapproval with `--approve-blender-tools`, retains +Codex `workspace-write` plus `approval_policy="never"`, rejects extra MCP servers +or tools, and verifies a real successful preflight MCP result. MCP execution has +the server process's OS privileges; this is not a security sandbox. + +For benchmark rounds, pass `expected_output_blend` to +`execute_blender_code_for_cli`. The server requires a fresh changed checkpoint +and returns source/output SHA-256 evidence. The following +`get_render_as_image_for_cli` call renders those exact bytes with CUDA-only +Cycles and returns PNG content plus its source hash. + +Every path, model, and runtime version pin is explicit: + +```sh +python tools/direct_codex.py run \ + --config /external/blender-mcp.toml \ + --codex /absolute/bin/codex \ + --codex-version 'EXACT CODEX --version OUTPUT' \ + --bpy-version 'EXACT BPY VERSION PREFIX' \ + --workdir /external/fresh-task-work \ + --evidence /external/fresh-private-evidence \ + --model 'PROVIDER/MODEL-ID' \ + --prompt /external/task-prompt.txt \ + --target /external/target.png \ + --rounds 10 --approve-blender-tools +``` + +`tools/direct_codex.example.toml` documents the MCP schema. Keep credentials out +of TOML and inspect retained raw argv/JSONL/stderr before publication. Existing +iteration files are rejected. This launcher does not resume, regenerate, score, +or establish full BlenderBench completion by itself. diff --git a/tools/direct_codex.py b/tools/direct_codex.py new file mode 100644 index 0000000..b92f742 --- /dev/null +++ b/tools/direct_codex.py @@ -0,0 +1,408 @@ +"""Direct Codex launch/config preflight; no benchmark controller or scoring.""" +import copy + +TOOLS = ('execute_blender_code_for_cli', 'get_render_as_image_for_cli', + 'get_runtime_python_api_docs_for_cli', 'search_api_docs') +CODEX_VERSION = 'codex-cli 0.154.0' +CODEX_SOURCE_REVISION = '6b9826e3aa83b1a5947db50f4332cb9c65f1b340' +CODEX_FEATURE_REGISTRY_URL = ( + 'https://github.com/openai/codex/blob/' + CODEX_SOURCE_REVISION + + '/codex-rs/features/src/lib.rs') +DISABLED_FEATURES = ( + 'shell_tool', 'unified_exec', 'unified_exec_tty', 'shell_snapshot', 'view_image', + 'sleep_tool', 'code_mode', 'code_mode_host', 'code_mode_prewarm', + 'request_permissions_tool', 'multi_agent', 'multi_agent_v2', 'apps', 'enable_mcp_apps', + 'tool_suggest', 'plugins', 'hooks', 'in_app_browser', 'in_app_local_automation', + 'browser_use', 'browser_use_full_cdp_access', 'browser_use_external', 'computer_use', + 'remote_plugin', 'plugin_sharing', 'image_generation', 'standalone_web_search', + 'web_search_request', 'web_search_cached') + + +def prepare_config(config, *, approved): + if not approved: + raise ValueError('explicit approval required: --approve-blender-tools') + if set(config) != {'mcp_servers'} or set(config['mcp_servers']) != {'blender'}: + raise ValueError('config must contain only mcp_servers.blender') + result = copy.deepcopy(config) + server = result['mcp_servers']['blender'] + allowed = {'command', 'args', 'cwd', 'env', 'enabled_tools', 'required', + 'startup_timeout_sec', 'tool_timeout_sec'} + if set(server) - allowed: + raise ValueError('unsupported or conflicting MCP settings') + if sorted(server.get('enabled_tools', [])) != sorted(TOOLS): + raise ValueError('enabled_tools must be exactly the four standalone tools') + env = server.get('env', {}) + from pathlib import Path + if (env.get('BLENDER_MCP_CLI_BACKEND') != 'bpy' + or not Path(env.get('BLENDER_MCP_BPY_PYTHON', '')).is_absolute() + or not Path(server.get('command', '')).is_absolute()): + raise ValueError('absolute server and standalone bpy interpreter required') + server['tools'] = {tool: {'approval_mode': 'approve'} for tool in TOOLS} + server['required'] = True + # Codex 0.154 otherwise defers MCP tools through tool_search, whose dynamic + # calls are not represented by exec JSONL. Force the four tools direct. + server['omit_tools_from'] = ['deferred', 'code_mode'] + return result + + +def verify_generation_workdir(root): + """Require the mounted generation root to contain only initialized runtime inputs.""" + import hashlib + from pathlib import Path + root = Path(root).absolute() + allowed = {'initial.blend', 'initial_verified.png', 'audit.json', 'metadata.json', + 'runtime.log', 'worker.py', 'worker.json'} + if root.is_symlink() or not root.is_dir(): + raise ValueError('generation runtime root must be a regular directory') + entries = list(root.iterdir()) + unexpected = sorted(entry.name for entry in entries if entry.name not in allowed) + if unexpected or any(entry.is_symlink() or not entry.is_file() for entry in entries): + raise ValueError(f'generation runtime root contains undeclared paths: {unexpected}') + initial = root / 'initial.blend' + if not initial.is_file(): + raise ValueError('generation runtime root requires initial.blend') + worker = root / 'worker.py' + if worker.exists(): + try: + from tools import direct_runtime + except ModuleNotFoundError: # direct script invocation adds tools/ to sys.path + import direct_runtime + expected = hashlib.sha256(direct_runtime.WORKER.encode()).hexdigest() + if hashlib.sha256(worker.read_bytes()).hexdigest() != expected: + raise ValueError('generation runtime worker.py differs from canonical runtime bytes') + + +def verify_preflight(events, nonce, python, bpy_version='5.2.1'): + """Require a completed real MCP item, never trust the final assistant text.""" + import json + for event in events: + item = event.get('item', {}) + if (event.get('type') != 'item.completed' or item.get('type') != 'mcp_tool_call' + or item.get('server') != 'blender' or item.get('tool') != TOOLS[0]): + continue + result = item.get('result') or {} + if item.get('status') != 'completed' or item.get('error') or result.get('isError'): + continue + payload = result.get('structured_content') + if payload is None: + try: + payload = json.loads(''.join(c.get('text', '') for c in result.get('content', []))) + except (ValueError, TypeError): + continue + if (isinstance(payload, dict) and payload.get('preflight') == nonce + and payload.get('python') == python + and str(payload.get('version', '')).startswith(bpy_version)): + return + raise ValueError(f'no successful standalone bpy {bpy_version} MCP preflight result') + + +def verify_generation_events(events, rounds=None, workdir=None): + """Reject any generation operation outside the four approved MCP tools.""" + import json + import re + forbidden = re.compile( + r'(?i)(goal(?:_code)?\.py|\bsolution(?:\.py)?\b|oracle|scores?\.json|scoring[/\\]|' + r'aggregate\.json|ref_based_eval|evaluator|clipmodel|photometric_loss)') + seen = {} + completed = set() + visual_checkpoints = [] + pending_checkpoint = None + active_edit = None + turn_completed = 0 + for index, event in enumerate(events): + if event.get('type') in ('error', 'turn.failed'): + raise ValueError('generation event stream reports failure') + if event.get('type') == 'turn.completed': + turn_completed += 1 + if index != len(events) - 1: + raise ValueError('generation turn completion must terminate the event stream') + continue + if event.get('type') not in ('item.started', 'item.updated', 'item.completed'): + continue + item = event.get('item') + if not isinstance(item, dict) or not isinstance(item.get('id'), str): + raise ValueError('malformed generation item event') + ident, kind = item['id'], item.get('type') + signature = (kind, item.get('server'), item.get('tool')) + if ident in seen and seen[ident] != signature: + raise ValueError('generation item identity changed in event stream') + seen[ident] = signature + if kind in ('reasoning', 'agent_message'): + continue + if kind != 'mcp_tool_call': + raise ValueError(f'prohibited Codex generation operation: {kind!r}') + if item.get('server') != 'blender' or item.get('tool') not in TOOLS: + raise ValueError('generation used a tool outside the four approved Blender MCP tools') + arguments = json.dumps(item.get('arguments', {}), sort_keys=True, default=str) + if forbidden.search(arguments): + raise ValueError('generation MCP arguments access prohibited goal/solution or score feedback') + if event.get('type') == 'item.started' and item.get('tool') == 'execute_blender_code_for_cli': + if active_edit is not None or pending_checkpoint is not None: + raise ValueError('next edit dispatch preceded visual inspection of prior checkpoint') + active_edit = ident + if event.get('type') == 'item.completed': + if ident in completed: + raise ValueError('duplicate generation item completion') + completed.add(ident) + result = item.get('result') or {} + if (item.get('status') != 'completed' or item.get('error') + or (isinstance(result, dict) and result.get('isError'))): + raise ValueError('generation MCP tool call failed') + if item.get('tool') == 'execute_blender_code_for_cli': + if pending_checkpoint is not None or (active_edit is not None and active_edit != ident): + raise ValueError('next edit dispatch preceded visual inspection of prior checkpoint') + structured = result.get('structured_content') if isinstance(result, dict) else None + if isinstance(structured, dict) and isinstance(structured.get('_checkpoint'), dict): + pending_checkpoint = structured['_checkpoint'] + active_edit = None + if item.get('tool') == 'get_render_as_image_for_cli': + content = result.get('content', []) if isinstance(result, dict) else [] + if not any(isinstance(value, dict) and value.get('type') == 'image' + and value.get('mimeType') == 'image/png' and value.get('data') + for value in content): + raise ValueError('visual inspection tool did not return PNG image content') + arguments = item.get('arguments') + if not isinstance(arguments, dict) or not isinstance(arguments.get('blend_file'), str): + raise ValueError('visual inspection requires a saved checkpoint path') + path = __import__('pathlib').Path(arguments['blend_file']).absolute() + metadata = next((value.get('_meta', value.get('meta')) for value in content + if isinstance(value, dict) and value.get('type') == 'image'), None) + if workdir is not None: + expected_path = (__import__('pathlib').Path(workdir).resolve() + / f'iteration{len(visual_checkpoints)+1:02}.blend') + if path != expected_path: + raise ValueError('visual inspection checkpoint path or source hash is invalid') + path = path.resolve(strict=True) + import hashlib + with path.open('rb') as stream: + actual_hash = hashlib.file_digest(stream, 'sha256').hexdigest() + source_path = (__import__('pathlib').Path(workdir).resolve() / 'initial.blend' + if not visual_checkpoints else + __import__('pathlib').Path(workdir).resolve() + / f'iteration{len(visual_checkpoints):02}.blend') + if not isinstance(pending_checkpoint, dict): + raise ValueError('visual inspection requires an attested checkpoint creation') + with source_path.open('rb') as stream: + source_hash = hashlib.file_digest(stream, 'sha256').hexdigest() + if (not isinstance(metadata, dict) + or metadata.get('source') != str(path) + or metadata.get('source_sha256') != actual_hash + or pending_checkpoint.get('path') != str(path) + or pending_checkpoint.get('source') != str(source_path) + or pending_checkpoint.get('source_sha256') != source_hash + or pending_checkpoint.get('output_sha256') != actual_hash + or pending_checkpoint.get('existed_before') is not False): + raise ValueError('visual inspection checkpoint path or source hash is invalid') + visual_checkpoints.append(path.name) + pending_checkpoint = None + if any(ident not in completed and signature[0] == 'mcp_tool_call' + for ident, signature in seen.items()): + raise ValueError('generation event stream contains unfinished MCP tool call') + if turn_completed != 1 or not completed: + raise ValueError('generation event stream is empty or lacks a completed turn/tool call') + if rounds is not None: + expected = [f'iteration{number:02}.blend' for number in range(1, rounds + 1)] + if visual_checkpoints != expected: + raise ValueError(f'visual inspection checkpoint sequence must be exactly {expected}') + if pending_checkpoint is not None: + raise ValueError('generation created an uninspected checkpoint') + + +def validate_edit(before, after, policy): + """Require finite, meaningful saved changes; local edit policy is ignored.""" + import math + left = copy.deepcopy(before.get('audit', before)) + right = copy.deepcopy(after.get('audit', after)) + # Evidenced orphan cleanup alone is not a meaningful editing round. + materials = left.get('preserve', {}).get('materials', {}) + saved_materials = right.get('preserve', {}).get('materials', {}) + for name in set(materials) - set(saved_materials): + lifecycle = before.get('material_lifecycle', {}).get(name, {}) + if (type(lifecycle.get('users')) is int and lifecycle['users'] == 0 + and lifecycle.get('use_fake_user') is False + and lifecycle.get('use_extra_user') is False + and 'library' in lifecycle and lifecycle['library'] is None): + del materials[name] + def finite(value): + if isinstance(value, dict): + return all(finite(v) for v in value.values()) + if isinstance(value, (list, tuple)): + return all(finite(v) for v in value) + return not isinstance(value, float) or math.isfinite(value) + + if not finite(left) or not finite(right): + raise ValueError('nonfinite scene property') + if left == right: + raise ValueError('no meaningful scene change') + + +def verify_checkpoints(root, rounds, policy=None): + """Artifact gate only: independent scene/GPU/invariant audit still required.""" + import hashlib + import json + import math + from pathlib import Path + from PIL import Image + root = Path(root).expanduser().absolute() + for component in (root, *root.parents): + if component.is_symlink(): + raise ValueError(f'symlink forbidden in checkpoint path: {component}') + try: + rows = [json.loads(line) for line in (root / 'rounds.jsonl').read_text().splitlines()] + if rounds < 1 or [r['round'] for r in rows] != list(range(1, rounds + 1)): + raise ValueError('missing, duplicate, or unordered rounds') + hashes, poses, evidence = set(), set(), [] + for n, row in enumerate(rows, 1): + blend, png = root / f'iteration{n:02}.blend', root / f'iteration{n:02}.png' + if row['blend'] != str(blend) or row['png'] != str(png): + raise ValueError('checkpoint paths must match expected absolute paths') + if blend.is_symlink() or png.is_symlink() or not blend.stat().st_size: + raise ValueError('checkpoint must be a nonempty regular file, not a symlink') + with blend.open('rb') as stream: + digest = hashlib.file_digest(stream, 'sha256').hexdigest() + if policy is None: + pose = tuple(row['pose']['location'] + row['pose']['rotation']) + if len(pose) != 6 or not all(type(v) in (int, float) and math.isfinite(v) for v in pose): + raise ValueError('pose must contain six finite numbers') + if pose in poses: + raise ValueError('duplicate reported pose') + poses.add(pose) + elif not isinstance(row.get('rationale'), str) or not row['rationale'].strip(): + raise ValueError('task checkpoint requires rationale') + if digest in hashes: + raise ValueError('duplicate checkpoint bytes') + hashes.add(digest) + with Image.open(png) as image: + if image.format != 'PNG': + raise ValueError('checkpoint image must be PNG') + image.load() + dimensions = list(image.size) + evidence.append({'round': n, 'blend_sha256': digest, 'dimensions': dimensions}) + return evidence + except (OSError, KeyError, TypeError) as exc: + raise ValueError(f'incomplete checkpoint artifacts: {exc}') from exc + + +def toml_value(value): + """Encode nested inline TOML, preserving literal dotted keys and booleans.""" + import json + if isinstance(value, dict): + return '{' + ', '.join(json.dumps(k) + '=' + toml_value(v) for k, v in value.items()) + '}' + if isinstance(value, list): + return '[' + ', '.join(toml_value(v) for v in value) + ']' + return json.dumps(value, allow_nan=False) + + +def invoke(args, config, evidence, label, prompt, target=None): + import json + import subprocess + command = [args.codex] + for feature in DISABLED_FEATURES: + command += ['--disable', feature] + command += ['exec', '--ignore-user-config', '--strict-config', + '--skip-git-repo-check', '--sandbox', 'read-only', '--model', args.model, + '--json', '-C', str(args.workdir), '-c', 'approval_policy="never"', + '-c', 'web_search="disabled"', + '-c', 'tools.experimental_request_user_input=false', + '-c', 'tools.update_plan=false', + '-c', 'mcp_servers=' + toml_value(config['mcp_servers']), + '-o', str(evidence / (label + '.final.txt'))] + if target: + import hashlib + target = target.resolve(strict=True) + (evidence / (label + '.visible-input.json')).write_text(json.dumps({ + 'mechanism': 'Codex exec -i bootstrap attachment; Codex 0.154 JSONL does not emit ImageView', + 'path': str(target), 'sha256': hashlib.sha256(target.read_bytes()).hexdigest()})) + command += ['-i', str(target)] + command += ['-'] + (evidence / (label + '.argv.json')).write_text(json.dumps(command, indent=2)) + (evidence / (label + '.prompt.txt')).write_text(prompt) + with (evidence / (label + '.jsonl')).open('x') as out, (evidence / (label + '.stderr')).open('x') as err: + proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=out, stderr=err, cwd=args.workdir) + (evidence / (label + '.process.json')).write_text(json.dumps({'pid': proc.pid})) + proc.communicate(prompt.encode()) + (evidence / (label + '.exit-code.txt')).write_text(str(proc.returncode)) + if proc.returncode: + raise ValueError(f'{label}: Codex exited {proc.returncode}; inspect retained logs') + return [json.loads(line) for line in (evidence / (label + '.jsonl')).read_text().splitlines()] + + +def main(argv=None): + import argparse + import json + import subprocess + import sys + import tomllib + import uuid + from pathlib import Path + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=['preflight', 'run']) + parser.add_argument('--config', type=Path, required=True) + parser.add_argument('--codex', required=True) + parser.add_argument('--codex-version', required=True) + parser.add_argument('--bpy-version', required=True) + parser.add_argument('--workdir', type=Path, required=True) + parser.add_argument('--evidence', type=Path, required=True, help='fresh private evidence directory') + parser.add_argument('--model', required=True) + parser.add_argument('--approve-blender-tools', action='store_true', + help='explicitly preapprove arbitrary code and docs on the configured MCP server') + parser.add_argument('--prompt', type=Path) + parser.add_argument('--target', type=Path) + parser.add_argument('--rounds', type=int, default=10) + args = parser.parse_args(argv) + prompt = '' + try: + config = prepare_config(tomllib.loads(args.config.read_text()), approved=args.approve_blender_tools) + args.workdir = args.workdir.resolve(strict=True) + args.evidence = args.evidence.absolute() + if not (args.workdir / 'initial.blend').is_file(): + raise ValueError('workdir requires initial.blend') + verify_generation_workdir(args.workdir) + if args.mode == 'run': + if not args.prompt or not args.target or args.rounds < 1: + raise ValueError('run requires --prompt, --target, and positive --rounds') + prompt = args.prompt.read_text() + args.target = args.target.resolve(strict=True) + if (args.workdir / 'rounds.jsonl').exists() or any(args.workdir.glob('iteration*')): + raise ValueError('run requires fresh checkpoint paths; resume is unsupported') + version = subprocess.check_output([args.codex, '--version'], text=True).strip() + if args.codex_version != CODEX_VERSION or version != CODEX_VERSION: + raise ValueError(f'publication protocol requires exactly {CODEX_VERSION!r}; got {version!r}') + args.evidence.mkdir(parents=True, exist_ok=False, mode=0o700) + (args.evidence / 'config.json').write_text(json.dumps(config, indent=2)) + (args.evidence / 'codex-version.txt').write_text(version) + nonce = uuid.uuid4().hex + code = ('import bpy,sys; result={"preflight":' + repr(nonce) + + ',"python":sys.executable,"version":bpy.app.version_string}') + preflight = ('Call only execute_blender_code_for_cli once with blend_file=' + + str(args.workdir / 'initial.blend') + ' and code=' + code + + '. Do not mutate or save the scene. Do not use shell or other tools.') + events = invoke(args, config, args.evidence, 'preflight', preflight) + verify_preflight(events, nonce, config['mcp_servers']['blender']['env']['BLENDER_MCP_BPY_PYTHON'], args.bpy_version) + if args.mode == 'run': + contract = ('\nUse only these MCP tools for scene operations: ' + ', '.join(TOOLS) + + '. No shell scene operations, benchmark controller, VLM judge, scoring, or publication. ' + 'After every saved round call get_render_as_image_for_cli on that round checkpoint and ' + 'visually inspect its returned PNG content; built-in view_image is disabled. Write rounds.jsonl via MCP with one row per ' + 'completed round: round (integer), pose={location:[x,y,z],rotation:[x,y,z]}, ' + 'blend and png (absolute paths), rationale. Save iteration01.blend/iteration01.png ' + f'through iteration{args.rounds:02}.blend/iteration{args.rounds:02}.png. ' + 'For each edit set execute_blender_code_for_cli blend_file to initial.blend or the preceding ' + 'iteration and expected_output_blend to the new iteration blend. ' + 'Inspection/no-op calls are not rounds. Never alter initial.blend.') + events = invoke(args, config, args.evidence, 'run', prompt + contract, args.target) + verify_generation_events(events, rounds=args.rounds, workdir=args.workdir) + checkpoints = verify_checkpoints(args.workdir, args.rounds) + (args.evidence / 'artifact-check.json').write_text(json.dumps({ + 'status': 'artifacts_complete_not_scene_audited', 'checkpoints': checkpoints}, indent=2)) + print('Checkpoint artifacts complete; independent scene/GPU/invariant audit and scoring remain required.') + else: + print('Standalone bpy 5.2.1 MCP preflight passed; generation not run.') + return 0 + except (ValueError, OSError, subprocess.SubprocessError) as exc: + print(str(exc), file=sys.stderr) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/direct_runtime.py b/tools/direct_runtime.py new file mode 100644 index 0000000..1f6b188 --- /dev/null +++ b/tools/direct_runtime.py @@ -0,0 +1,389 @@ +"""Independent custom-bpy 5.2.1 audits and CUDA renders (never Blender CLI). + +Initialization trust contract: parent verifies the dataset pin and reviews start.py, +then supplies policy pin_verified=True, start_sha256, and reviewed literal +initialization operations. Upstream source is hashed but NEVER executed. +Output directories must not exist. STOP in the output, or policy stop_file, cancels. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess +import time + + +def sha(path): + h = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + h.update(block) + return h.hexdigest() + + +def emit(path, event, **fields): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + row = {'event': event, 'time_unix': time.time(), **fields} + with path.open('a') as stream: + stream.write(json.dumps(row, sort_keys=True, allow_nan=False) + '\n') + stream.flush() + os.fsync(stream.fileno()) + return row + + +def _disk(directory, policy): + floor = int(policy.get('disk_floor_bytes', 1024**3)) + if floor < 0: + raise ValueError('negative disk floor') + free = shutil.disk_usage(directory).free + if free < floor: + raise OSError(f'disk floor safety: {free} < {floor}') + return free + + +def run_process(argv, log, event_path, policy): + """No deadline: retain native stdout/stderr; report liveness, not fake progress.""" + log = Path(log) + _disk(log.parent, policy) + stop = Path(policy.get('stop_file', log.parent / 'STOP')) + if stop.exists(): + raise InterruptedError('manual stop file: ' + str(stop)) + start = time.monotonic() + with log.open('xb') as stream: + process = subprocess.Popen(list(map(str, argv)), stdout=stream, stderr=subprocess.STDOUT, + start_new_session=True) + common = {'pid': process.pid, 'parent_pid': os.getpid(), 'task': policy.get('task'), + 'argv': list(map(str, argv)), 'log': str(log.absolute())} + last_size = 0 + last_activity = start + last_slow = start + try: + emit(event_path, 'process_started', **common) + while process.poll() is None: + if stop.exists(): + raise InterruptedError('manual stop file: ' + str(stop)) + free = _disk(log.parent, policy) + size = log.stat().st_size + now = time.monotonic() + activity = {} + try: + activity['proc_stat'] = Path(f'/proc/{process.pid}/stat').read_text() + activity['proc_io'] = Path(f'/proc/{process.pid}/io').read_text() + except (FileNotFoundError, PermissionError): + pass + if size != last_size: + last_activity = now + emit(event_path, 'heartbeat', **common, elapsed_seconds=now-start, + stdout_bytes=size, new_stdout_bytes=size-last_size, free_bytes=free, + seconds_since_stdout=now-last_activity, activity=activity) + if now-last_slow >= float(policy.get('slow_event_seconds', 60)): + emit(event_path, 'slow_operation', **common, elapsed_seconds=now-start, + stdout_bytes=size, activity=activity, action='continue_without_timeout') + last_slow = now + last_size = size + time.sleep(float(policy.get('heartbeat_seconds', 2))) + except BaseException as exc: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + emit(event_path, 'process_cancelled', **common, reason=str(exc), returncode=process.returncode) + raise + stream.flush() + os.fsync(stream.fileno()) + result = {**common, 'returncode': process.returncode, 'elapsed_seconds': time.monotonic()-start} + emit(event_path, 'process_exited', **result) + if process.returncode: + raise RuntimeError(f'custom bpy process exited {process.returncode}; retained log: {log}') + return result + + +# This source is executed exclusively by the explicitly configured bpy Python. +WORKER = r''' +import bpy, hashlib, json, sys, os +from pathlib import Path +cfg=json.loads(Path(sys.argv[1]).read_text()) +assert bpy.app.version_string.startswith(cfg['bpy_version']), ('configured bpy version required', bpy.app.version_string) +bpy.context.preferences.filepaths.use_scripts_auto_execute=False +bpy.ops.wm.open_mainfile(filepath=cfg['source'], use_scripts=False) +assert not bpy.app.autoexec_fail, 'automatic script dependency rejected' +for op in cfg.get('initialization', []): + if op['kind']=='object': + obj=bpy.data.objects[op['object']] + field=op['property'] + if field in ('location','rotation_euler','scale'): + setattr(obj,field,op['value']) + elif field in ('data.lens','data.energy','data.color'): + setattr(obj.data,field.split('.')[1],op['value']) + else: raise ValueError('unreviewed initialization property: '+field) + elif op['kind']=='shape_key' and op['property']=='value': + bpy.data.shape_keys[op['datablock']].key_blocks[op['key']].value=op['value'] + else: raise ValueError('unreviewed initialization operation') + +# ID references are stable names, never memory-address reprs. All editable scalar +# RNA properties are included; nested embedded structs and collections recurse. +# Unsupported cyclic embedded structures fail closed instead of disappearing. +SKIP={'rna_type','is_updated','is_updated_data','is_updated_transform','tag','users', + 'use_fake_user','use_extra_user','is_embedded_data','is_runtime_data', + 'session_uid','original','id_data','library','override_library','preview','depsgraph', + 'animation_data_clear','name_full'} +def scalar(v): + if isinstance(v,(str,bool,int,float,type(None))): return v + if isinstance(v,bpy.types.ID): return ref(v) + if hasattr(v,'items'): return {k:scalar(value) for k,value in v.items()} + if isinstance(v,set): return sorted(v) + return [scalar(i) for i in v] +def ref(v): + return {'id_type':v.bl_rna.identifier,'name':v.name, + 'library':v.library.filepath if v.library else None} +def collection_props(values, stack): + # RNA foreach_get avoids millions of Python per-vertex/per-component calls. + # Every writable primitive field is retained; variable/non-numeric fields + # and nested pointers still use the general serializer. + if len(values)<128 or not hasattr(values,'foreach_get'): + return [ref(i) if isinstance(i,bpy.types.ID) else props(i,stack=stack) for i in values] + from array import array + first=values[0] + if isinstance(first,bpy.types.ID): return [ref(i) for i in values] + if any(i.bl_rna.identifier != first.bl_rna.identifier for i in values): + return [ref(i) if isinstance(i,bpy.types.ID) else props(i,stack=stack) for i in values] + result={'count':len(values),'rna_type':first.bl_rna.identifier,'fields':{}} + for p in first.bl_rna.properties: + k=p.identifier + if k in SKIP: continue + if p.type not in ('POINTER','COLLECTION') and p.is_readonly: continue + if isinstance(first,bpy.types.ShapeKey) and k=='value': continue + if p.type in ('FLOAT','INT','BOOLEAN') and (not p.is_array or p.array_length): + width=p.array_length if p.is_array else 1 + typecode={'FLOAT':'f','INT':'i','BOOLEAN':'b'}[p.type] + data=array(typecode,[0])*(len(values)*width) + try: + values.foreach_get(k,data) + result['fields'][k]={'width':width,'type':typecode,'sha256':hashlib.sha256(data.tobytes()).hexdigest()} + continue + except (AttributeError,TypeError,RuntimeError): + # Some legacy UV collections expose a different indexed RNA + # subtype from their iterator. Never reuse that field schema. + return [props(i,stack=stack) for i in values] + if p.type=='POINTER': + result['fields'][k]=[ref(v) if isinstance(v,bpy.types.ID) else props(v,stack=stack) for v in (getattr(i,k) for i in values)] + elif p.type=='COLLECTION': + result['fields'][k]=[collection_props(getattr(i,k),stack) for i in values] + else: + result['fields'][k]=[scalar(getattr(i,k)) for i in values] + return result + +def props(x, omit=(), stack=()): + # An RNA graph is not a tree (armature children/parents form dense graphs). + # Serialize each embedded node once per root with deterministic traversal + # references, retaining ALL node content without exponential duplication. + global _rna_seen + if not stack: _rna_seen={} + if x is None: return None + address=(x.bl_rna.identifier,x.as_pointer()) + if address in _rna_seen: return {'rna_ref':_rna_seen[address]} + _rna_seen[address]=len(_rna_seen) + stack=stack+(address,) + out={} + for p in x.bl_rna.properties: + k=p.identifier + if k in SKIP or k in omit or (isinstance(x,bpy.types.ShapeKey) and k=='value'): continue + if p.type not in ('POINTER','COLLECTION') and p.is_readonly: continue + v=getattr(x,k) + if p.type=='POINTER': + out[k]=ref(v) if isinstance(v,bpy.types.ID) else props(v,stack=stack) + elif p.type=='COLLECTION': + out[k]=collection_props(v,stack) + else: + if p.type=='STRING' and (p.subtype in ('FILE_PATH','DIR_PATH') or k=='filepath') and v.startswith('//') and v!='//': + v=os.path.normpath(bpy.path.abspath(v)) + out[k]=scalar(v) + if isinstance(x,bpy.types.ID): + out['custom_properties']={k:scalar(v.to_list() if hasattr(v,'to_list') else v) for k,v in x.items() if k!='_RNA_UI'} + return out + +def digest(value): + return hashlib.sha256(json.dumps(value,sort_keys=True,allow_nan=False).encode()).hexdigest() +def tree(t): + if t is None: return None + return {'properties':props(t), 'nodes':[{ 'properties':props(n), + 'inputs':[props(i) for i in n.inputs], 'outputs':[props(i) for i in n.outputs]} for n in t.nodes], + 'links':[(l.from_node.name,l.from_socket.identifier,l.to_node.name,l.to_socket.identifier) for l in t.links]} +def material_lifecycle(): + # Evidence for normal save cleanup, never add fake users just for auditing. + return {m.name: {'users':m.users, 'use_fake_user':bool(m.use_fake_user), + 'use_extra_user':bool(m.use_extra_user), + 'library':m.library.filepath if m.library else None} + for m in bpy.data.materials} + +def audit(): + objects={} + for o in bpy.data.objects: + print('__FULL_AUDIT_OBJECT__'+o.name,flush=True) + obj={'type':o.type,'location':list(o.location),'rotation_euler':list(o.rotation_euler),'scale':list(o.scale), + 'properties':props(o,('location','rotation_euler','scale','matrix_world','matrix_local','matrix_basis','dimensions')), + 'visibility':{s.name:{vl.name:o.hide_get(view_layer=vl) for vl in s.view_layers if o.name in vl.objects} for s in bpy.data.scenes}} + if o.type=='CAMERA': obj['lens']=o.data.lens + if o.type=='LIGHT': obj['light_data']={'energy':o.data.energy,'color':list(o.data.color)} + objects[o.name]=obj + preserve={} + for collection in ('meshes','curves','metaballs','armatures','lattices','materials','lights','worlds','cameras','node_groups','actions','collections','scenes','particles','textures','speakers','volumes','pointclouds','grease_pencils'): + items=getattr(bpy.data,collection,()) + values={} + for item in items: + print('__FULL_AUDIT_ITEM__'+json.dumps({'collection':collection,'name':item.name}),flush=True) + bound=any(o.data==item for o in bpy.data.objects) if collection in ('cameras','lights') else False + omissions=('lens','angle','angle_x','angle_y') if collection=='cameras' and bound else (('energy','color') if collection=='lights' and bound else ()) + value=props(item,omissions) + if hasattr(item,'node_tree'): value['node_tree_contents']=tree(item.node_tree) + if collection=='node_groups': value['contents']=tree(item) + values[item.name]=digest(value) + preserve[collection]=values + images={} + for image in bpy.data.images: + if image.type in ('RENDER_RESULT','COMPOSITING'): continue + data={'properties':props(image)} + if image.packed_file: data['bytes']=hashlib.sha256(image.packed_file.data).hexdigest() + elif image.source=='FILE': + path=Path(bpy.path.abspath(image.filepath,library=image.library)) + if not path.is_file(): raise RuntimeError('unavailable image dependency: '+str(path)) + data['bytes']=hashlib.sha256(path.read_bytes()).hexdigest() + images[image.name]=digest(data) + preserve['images']=images + for lib in bpy.data.libraries: + path=Path(bpy.path.abspath(lib.filepath)) + preserve.setdefault('linked_libraries',{})[lib.name]=hashlib.sha256(path.read_bytes()).hexdigest() + preserve['shape_keys']={key.name:digest(props(key)) for key in bpy.data.shape_keys} + shape_keys={key.name:{block.name:block.value for block in key.key_blocks} for key in bpy.data.shape_keys} + return {'objects':objects,'shape_keys':shape_keys,'preserve':preserve} + +# GPU setup is the only runtime-local device override. Engine, samples and +# resolution are never altered. Audit after setup so source CPU flags normalize. +prefs=bpy.context.preferences.addons['cycles'].preferences +prefs.compute_device_type='CUDA' +prefs.refresh_devices() +assert any(d.type=='CUDA' for d in prefs.devices), 'CUDA unavailable; CPU fallback forbidden' +for d in prefs.devices: d.use=(d.type=='CUDA') +# Only the active scene is rendered. Inactive scenes can use different engines; +# leave their engine AND device settings intact, retaining them in full audits. +s=bpy.context.scene +assert s.render.engine=='CYCLES', 'unsupported non-Cycles engine; not silently changed' +s.cycles.device='GPU' +assert s.camera is not None, 'scene has no active camera' +evidence={'compute_device_type':prefs.compute_device_type, + 'devices':[{'name':d.name,'type':d.type,'use':bool(d.use),'id':d.id} for d in prefs.devices], + 'scene_device':s.cycles.device,'engine':s.render.engine,'samples':s.cycles.samples, + 'resolution':[s.render.resolution_x,s.render.resolution_y,s.render.resolution_percentage], + 'blender_version':bpy.app.version_string,'blender_version_tuple':list(bpy.app.version), + 'build_hash':bpy.app.build_hash.decode(errors='replace'),'python_executable':sys.executable, + 'pid':os.getpid()} +if cfg.get('save'): + bpy.context.preferences.filepaths.save_version=0 + bpy.ops.wm.save_as_mainfile(filepath=cfg['save'], relative_remap=True) +record={'audit':audit(),'evidence':evidence,'material_lifecycle':material_lifecycle(), + 'settings':{'engine':s.render.engine,'samples':s.cycles.samples, + 'resolution_x':s.render.resolution_x,'resolution_y':s.render.resolution_y, + 'resolution_percentage':s.render.resolution_percentage}} +Path(cfg['report']).write_text(json.dumps(record,sort_keys=True,allow_nan=False)) +print('__FULL_AUDIT_READY__',flush=True) +if cfg.get('png'): + # Output encoding and destination are transient and are never saved. + s.render.filepath=cfg['png'] + s.render.image_settings.file_format='PNG' + s.render.use_file_extension=True + bpy.ops.render.render(write_still=True) + print('__FULL_RENDER_FINISHED__',flush=True) +''' + + +def _execute(source, policy, bpy_python, bpy_version, output, event_path, render, start=None): + # Preserve the venv executable path: resolving its symlink loses venv imports. + python = Path(bpy_python).absolute() + if not python.is_file(): + raise FileNotFoundError(python) + if not bpy_version: + raise ValueError('explicit bpy version pin required') + source = Path(source).resolve(strict=True) + if source.suffix != '.blend': + raise ValueError('source must be a saved .blend') + output = Path(output).absolute() + output.mkdir(parents=True, exist_ok=False) + _disk(output, policy) + original = sha(source) + png = output / ('initial_verified.png' if start else 'verified.png') if render else None + saved = output / 'start_initialized.blend' if start else None + config = {'source': str(source), 'report': str(output / 'audit.json'), 'bpy_version': bpy_version, + 'save': str(saved) if saved else None, 'png': str(png) if png else None} + if start: + config['initialization'] = policy['initialization'] + if sha(start) != policy['start_sha256']: + raise ValueError('verified start.py hash drift') + script = output / 'worker.py' + script.write_text(WORKER) + config_path = output / 'worker.json' + config_path.write_text(json.dumps(config)) + try: + process = run_process([str(python), '-u', str(script), str(config_path)], output / 'runtime.log', event_path, policy) + finally: + if sha(source) != original: + emit(event_path, 'source_changed', source=str(source), task=policy.get('task')) + raise ValueError('source blend changed during independent verification') + result = json.loads(Path(config['report']).read_text()) + evidence = result['evidence'] + if (not evidence['blender_version'].startswith(bpy_version) or evidence['compute_device_type'] != 'CUDA' + or evidence['scene_device'] != 'GPU' or not any(d['type']=='CUDA' and d['use'] for d in evidence['devices']) + or any(d['use'] and d['type']!='CUDA' for d in evidence['devices'])): + raise ValueError('custom bpy CUDA evidence failed') + result.update(task=policy.get('task'), source=str(source), source_sha256=original, + blend=str(saved or source), blend_sha256=sha(saved or source), + runtime=process, output=str(output), audit_path=config['report'], + png=str(png) if png else None) + if png: + from PIL import Image + with Image.open(png) as image: + if image.format != 'PNG': raise ValueError('render is not PNG') + image.load() + dimensions = list(image.size) + settings = result['settings'] + expected = [max(1, settings[k]*settings['resolution_percentage']//100) for k in ('resolution_x','resolution_y')] + if dimensions != expected: + raise ValueError(f'decoded PNG dimensions {dimensions} != task {expected}') + result.update(png_dimensions=dimensions, png_sha256=sha(png)) + emit(event_path, 'render_completed', task=policy.get('task'), png=str(png), + png_sha256=result['png_sha256'], png_dimensions=dimensions, + blend=result['blend'], blend_sha256=result['blend_sha256'], + runtime=process, evidence=evidence, source_sha256=original) + (output / 'metadata.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + return result + + +def inspect_render(blend, task_policy, bpy_python, bpy_version, output, event_path, render=True): + """Independently open/audit/render an unchanged saved blend via custom bpy.""" + return _execute(blend, task_policy, bpy_python, bpy_version, output, event_path, render) + + +def initialize(inp, task_policy, bpy_python, bpy_version, output, event_path): + """Apply parent-reviewed literal initialization; save and render the state. + + inp is a directory with exactly one top-level .blend and start.py, or a mapping + with explicit blend/start_py paths. No goal/oracle files are inspected. + """ + if task_policy.get('pin_verified') is not True or not task_policy.get('start_sha256') or not isinstance(task_policy.get('initialization'), list): + raise ValueError('initialization requires parent-verified pin/policy, start_sha256 and literal initialization') + if task_policy.get('reviewed') is False: + raise ValueError('unsupported or unreviewed initialization policy') + if isinstance(inp, dict): + source, start = Path(inp['blend']), Path(inp['start_py']) + else: + inp = Path(inp) + sources = list(inp.glob('*.blend')) + if len(sources) != 1: + raise ValueError('expected exactly one safe top-level input blend') + source, start = sources[0], inp / 'start.py' + if start.name != 'start.py' or sha(start) != task_policy['start_sha256']: + raise ValueError('parent-verified start.py hash mismatch') + return _execute(source, task_policy, bpy_python, bpy_version, output, event_path, True, start.resolve()) diff --git a/tools/direct_suite.py b/tools/direct_suite.py new file mode 100644 index 0000000..e45edc3 --- /dev/null +++ b/tools/direct_suite.py @@ -0,0 +1,15 @@ +"""Compatibility entry point for the portable BlenderBench runner. + +Use ``python -m benchmarks.blenderbench_direct.runner`` for new automation. +""" +from __future__ import annotations + +from pathlib import Path + +from benchmarks.blenderbench_direct.runner import main +from benchmarks.blenderbench_direct.suite import run_suite +ROOT = Path.cwd() + + +if __name__ == "__main__": + raise SystemExit(main())