From be56d7d327773a592ae64aed56408c5994776188 Mon Sep 17 00:00:00 2001 From: Michael Gold Date: Thu, 10 Sep 2026 12:20:03 -0400 Subject: [PATCH 1/6] fix: add explicitly approved direct Codex bpy launcher --- tests/test_direct_codex.py | 150 ++++++++++++++++++++++++ tools/direct_codex.example.toml | 13 +++ tools/direct_codex.md | 123 ++++++++++++++++++++ tools/direct_codex.py | 197 ++++++++++++++++++++++++++++++++ 4 files changed, 483 insertions(+) create mode 100644 tests/test_direct_codex.py create mode 100644 tools/direct_codex.example.toml create mode 100644 tools/direct_codex.md create mode 100644 tools/direct_codex.py diff --git a/tests/test_direct_codex.py b/tests/test_direct_codex.py new file mode 100644 index 0000000..29aff89 --- /dev/null +++ b/tests/test_direct_codex.py @@ -0,0 +1,150 @@ +"""Direct launcher tests; fixtures are synthetic, not benchmark evidence.""" +import importlib.util +from pathlib import Path +import unittest + +SCRIPT = Path(__file__).resolve().parents[1] / 'tools' / 'direct_codex.py' + +class DirectCodexTests(unittest.TestCase): + def load(self): + self.assertTrue(SCRIPT.is_file(), 'repo-backed launcher is missing') + spec = importlib.util.spec_from_file_location('direct_codex', SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def test_explicit_scoped_approval_required(self): + m = self.load() + config = {'mcp_servers': {'blender': {'command': '/server/bin/blender-mcp', + 'enabled_tools': list(m.TOOLS), 'env': {'BLENDER_MCP_CLI_BACKEND': 'bpy', + 'BLENDER_MCP_BPY_PYTHON': '/runtime/bin/python'}}}} + with self.assertRaisesRegex(ValueError, 'explicit approval'): + m.prepare_config(config, approved=False) + result = m.prepare_config(config, approved=True) + server = result['mcp_servers']['blender'] + self.assertEqual(server['tools'], {t: {'approval_mode': 'approve'} for t in m.TOOLS}) + self.assertNotIn('default_tools_approval_mode', server) + self.assertNotIn('tools', config['mcp_servers']['blender']) + self.assertTrue(server['required']) + + def test_rejects_extra_tools_and_conflicting_policy(self): + m = self.load() + base = {'mcp_servers': {'blender': {'command': '/server', + 'enabled_tools': list(m.TOOLS), 'env': {'BLENDER_MCP_CLI_BACKEND': 'bpy', + 'BLENDER_MCP_BPY_PYTHON': '/venv/bin/python'}}}} + import copy + for key, value in [('enabled_tools', list(m.TOOLS) + ['execute_blender_code']), + ('default_tools_approval_mode', 'auto'), + ('tools', {m.TOOLS[0]: {'approval_mode': 'prompt'}}), + ('enabled', False)]: + config = copy.deepcopy(base) + config['mcp_servers']['blender'][key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + m.prepare_config(config, approved=True) + for extra in [{'approval_policy': 'never'}, {'mcp_servers': {'other': {}}}]: + config = copy.deepcopy(base) + config.update(extra) + with self.assertRaises(ValueError): + m.prepare_config(config, approved=True) + + def test_preflight_requires_successful_mcp_result_not_exit_zero(self): + m = self.load() + self.assertTrue(hasattr(m, 'verify_preflight'), 'result verification missing') + import copy + event = {'type': 'item.completed', 'item': {'type': 'mcp_tool_call', + 'server': 'blender', 'tool': m.TOOLS[0], 'status': 'completed', 'error': None, + 'result': {'structured_content': {'preflight': 'nonce', + 'python': '/venv/bin/python', 'version': '5.2.1 LTS'}}}} + m.verify_preflight([event], 'nonce', '/venv/bin/python') + for events in [[], [{'type': 'turn.completed'}]]: + with self.assertRaises(ValueError): + m.verify_preflight(events, 'nonce', '/venv/bin/python') + for field, value in [('error', {'message': 'MCP tool call requires approval, but approval policy is never'}), + ('status', 'failed'), ('result', {'isError': True}), + ('result', {'structured_content': {'preflight': 'nonce', 'python': '/wrong'}})]: + broken = copy.deepcopy(event) + broken['item'][field] = value + with self.assertRaises(ValueError): + m.verify_preflight([broken], 'nonce', '/venv/bin/python') + + def test_checkpoints_required_and_pngs_decoded(self): + m = self.load() + self.assertTrue(hasattr(m, 'verify_checkpoints'), 'checkpoint verifier missing') + import tempfile, json + from PIL import Image + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + with self.assertRaises(ValueError): + m.verify_checkpoints(root, 2) + rows = [] + for n in (1, 2): + blend, png = root / f'iteration{n:02}.blend', root / f'iteration{n:02}.png' + blend.write_bytes(b'BLENDER' + bytes([n])) + Image.new('RGB', (2, 2)).save(png) + rows.append({'round': n, 'pose': {'location': [n, 0, 0], 'rotation': [0, 0, 0]}, + 'blend': str(blend), 'png': str(png)}) + (root / 'rounds.jsonl').write_text('\n'.join(map(json.dumps, rows))) + self.assertEqual(len(m.verify_checkpoints(root, 2)), 2) + (root / 'iteration02.png').write_bytes(b'not a PNG') + with self.assertRaises(ValueError): + m.verify_checkpoints(root, 2) + + def test_cli_denied_preflight_stops_run_even_with_exit_zero(self): + m = self.load() + self.assertTrue(hasattr(m, 'main'), 'CLI missing') + import tempfile, json, subprocess, sys + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + fake = root / 'codex' + fake.write_text('#!' + sys.executable + '\nimport sys\n' + 'print("codex-cli 0.154.0" if "--version" in sys.argv else ' + '\'{"type":"turn.completed"}\')\n') + fake.chmod(0o755) + (root / 'initial.blend').write_bytes(b'fixture') + (root / 'prompt.txt').write_text('fixture generation task') + (root / 'target.png').write_bytes(b'fixture') + config = root / 'server.toml' + config.write_text('[mcp_servers.blender]\ncommand="/server"\n' + 'enabled_tools=' + json.dumps(list(m.TOOLS)) + '\n' + '[mcp_servers.blender.env]\nBLENDER_MCP_CLI_BACKEND="bpy"\n' + 'BLENDER_MCP_BPY_PYTHON="/venv/bin/python"\n') + args = [sys.executable, str(SCRIPT), 'run', '--config', str(config), + '--codex', str(fake), '--workdir', str(root), '--evidence', str(root / 'logs'), + '--model', 'test-model', '--approve-blender-tools', + '--prompt', str(root / 'prompt.txt'), '--target', str(root / 'target.png')] + proc = subprocess.run(args, capture_output=True, text=True) + self.assertEqual(proc.returncode, 1, proc.stderr) + self.assertIn('no successful', proc.stderr) + self.assertFalse((root / 'logs' / 'run.jsonl').exists()) + argv = json.loads((root / 'logs' / 'preflight.argv.json').read_text()) + self.assertIn('--strict-config', argv) + self.assertIn('workspace-write', argv) + self.assertNotIn('--dangerously-bypass-approvals-and-sandbox', argv) + self.assertTrue(any('approval_mode' in a and 'approve' in a for a in argv)) + # A successful preflight followed by exit 0 and zero rounds still fails. + fake.write_text('#!' + sys.executable + '\nimport sys,json,re\n' + 'if "--version" in sys.argv:\n print("codex-cli 0.154.0"); sys.exit()\n' + 'prompt=sys.stdin.read()\n' + 'nonce=re.search(r"[a-f0-9]{32}", prompt)\n' + 'payload={"preflight":nonce.group() if nonce else "",' + '"python":"/venv/bin/python","version":"5.2.1 LTS"}\n' + 'print(json.dumps({"type":"item.completed","item":' + '{"type":"mcp_tool_call","server":"blender","tool":"execute_blender_code_for_cli",' + '"status":"completed","error":None,"result":{"structured_content":payload}}}))\n') + args[args.index('--evidence') + 1] = str(root / 'logs2') + proc = subprocess.run(args, capture_output=True, text=True) + self.assertEqual(proc.returncode, 1, proc.stderr) + self.assertIn('incomplete checkpoint artifacts', proc.stderr) + self.assertTrue((root / 'logs2' / 'run.jsonl').is_file()) + + def test_toml_roundtrip_preserves_keys_booleans_and_paths(self): + m = self.load() + import tomllib + value = {'literal.dot': {'env': {'PATH': '/space dir/bin'}, 'required': True, + 'tools': list(m.TOOLS)}} + self.assertEqual(tomllib.loads('mcp_servers=' + m.toml_value(value))['mcp_servers'], value) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/direct_codex.example.toml b/tools/direct_codex.example.toml new file mode 100644 index 0000000..63e099d --- /dev/null +++ b/tools/direct_codex.example.toml @@ -0,0 +1,13 @@ +# Reviewed server config only; the launcher adds exact per-tool approvals +# ONLY with --approve-blender-tools. Keep machine paths outside the repo. +[mcp_servers.blender] +command = "/absolute/server-venv/bin/blender-mcp" +cwd = "/absolute/blender-mcp/mcp" +enabled_tools = ["execute_blender_code_for_cli", "get_runtime_python_api_docs_for_cli", "search_api_docs", "get_python_api_docs"] +required = true +# Explicit client deadline, NOT a backend timeout override or no-deadline promise. +tool_timeout_sec = 86400 + +[mcp_servers.blender.env] +BLENDER_MCP_CLI_BACKEND = "bpy" +BLENDER_MCP_BPY_PYTHON = "/absolute/bpy-venv/bin/python" diff --git a/tools/direct_codex.md b/tools/direct_codex.md new file mode 100644 index 0000000..7a9d985 --- /dev/null +++ b/tools/direct_codex.md @@ -0,0 +1,123 @@ +# Direct Codex + standalone bpy + +`direct_codex.py` is a direct-process launcher and artifact gate, **not a benchmark +controller**. It runs one read-only MCP preflight and, in `run` mode, one continuous +Codex generation session. It does not initialize data, orchestrate refinement +rounds, run a judge, score checkpoints, or publish anything. Camera1 is permitted; +it does not use the legacy camera controller's camera2–camera9 task registry. + +## Approval fix and scope + +Supported/tested config contract: **Codex CLI 0.154.0**, standalone **bpy 5.2.1**. +The launcher fails closed on other Codex versions pending a schema review. + +The observed failure was `MCP tool call requires approval, but approval policy is +never`. Exit 0 and final assistant text did not prove that any round ran. That is +an MCP approval denial, not evidence of a shell sandbox failure. + +Official references (version-pinned source is authoritative for this launcher): + +- [Config reference](https://developers.openai.com/codex/config-reference) +- [Live schema](https://developers.openai.com/codex/config-schema.json) +- [0.154.0 schema](https://github.com/openai/codex/blob/rust-v0.154.0/codex-rs/core/config.schema.json): + `RawMcpServerConfig.tools`, `McpServerToolConfig.approval_mode`, `AppToolApproval`. +- [0.154.0 approval implementation](https://github.com/openai/codex/blob/rust-v0.154.0/codex-rs/core/src/mcp_tool_call.rs#L2345-L2356): + `auto` consults tool annotations; `prompt` requires approval; `writes` requires + approval unless read-only; **`approve` does not request tool approval**. + `auto` is NOT synonymous with unconditional preapproval. + +Only after the user approves arbitrary Python execution by this particular server, +pass `--approve-blender-tools`. The launcher adds +`mcp_servers.blender.tools..approval_mode="approve"` to exactly: + +- `execute_blender_code_for_cli` +- `get_runtime_python_api_docs_for_cli` +- `search_api_docs` +- `get_python_api_docs` + +It retains `--sandbox workspace-write`, explicitly sets `approval_policy="never"`, +and uses `--ignore-user-config --strict-config`. It never enables a global approval +bypass, changes user config, or silently replaces a conflicting approval setting. +Server defaults, per-tool policy in input, extra servers/tools, or unrelated config +keys are rejected. The complete MCP table is supplied as a TOML inline CLI override, +not lossy recursive dotted-key flattening. Runtime paths preserve venv spelling. + +**This is real preauthorization, not a sandbox.** MCP Python has the invoking +server's OS privileges, potentially beyond Codex workspace-write. Review +[SECURITY.md](../SECURITY.md) and [backend docs](../readme_bpy_backend.rst). +Use an isolated disposable account/runtime and an external workspace without +project Codex config/plugins. Built-in Codex tools are not an MCP allowlist; +the prompt prohibits shell scene operations and permits `view_image` inspection. +Managed policies may still deny tools. No attempt is made to override them. + +## Usage + +Run from the repository root. Copy `tools/direct_codex.example.toml` outside the +repo and fill in verified absolute server/interpreter paths. Use an installed MCP +1.x server environment. Keep credentials out of TOML: effective config and argv +are deliberately retained in the evidence directory. Check `codex login status` +without reading credential files. + +Prepare a fresh external workspace with reviewed `initial.blend`; do not give the +model solution scripts/oracles. Independently admit the GPU runtime with real +renders before generation. The preflight below only checks tool execution, +`sys.executable`, and bpy version, **not GPU rendering**. Both modes make paid model +calls and require authorization for the selected exact model. + +```sh +uv run --frozen --project benchmarks/blenderbench_camera python tools/direct_codex.py preflight \ + --config /external/server.toml --codex /absolute/bin/codex \ + --workdir /external/camera1 --evidence /external/preflight-evidence \ + --model YOUR_EXACT_MODEL --approve-blender-tools + +uv run --frozen --project benchmarks/blenderbench_camera python tools/direct_codex.py run \ + --config /external/server.toml --codex /absolute/bin/codex \ + --workdir /external/camera1 --evidence /external/generation-evidence \ + --model YOUR_EXACT_MODEL --approve-blender-tools \ + --prompt /external/camera1-prompt.txt --target /external/target.png --rounds 10 +``` + +Using this locked Python environment provides Pillow; **no camera controller code +is imported**. The prompt is user-owned: specify Camera1-only changes, fixed lens, +render settings and scene invariants, CUDA GPU enabled/CPU disabled, exact ten +meaningful rounds, source preservation, and no scoring until generation ends. +The launcher appends only the tool/output contract. Each round must save +`iteration01.blend`/`iteration01.png` through `iteration10.*`, plus one JSONL row: + +```json +{"round":1,"pose":{"location":[1,2,3],"rotation":[0,0,0]},"blend":"/external/camera1/iteration01.blend","png":"/external/camera1/iteration01.png","rationale":"camera adjustment"} +``` + +The preflight requires an actual completed successful execution-tool JSONL result +containing a fresh nonce, the exact configured Python path, and version 5.2.1. +A final assistant claim, denied/failed tool, empty transcript, or exit 0 alone +cannot pass. Generation is not launched when preflight fails. Raw JSONL, stderr, +argv, prompt, child PID, and exit code are retained in a fresh private directory. +Existing iteration paths/ledger are refused; resume is unsupported. + +## Completion and limitations + +After generation, the artifact gate requires exact sequential ledger rounds, +expected absolute checkpoint paths, nonempty distinct checkpoint bytes, distinct +finite reported poses, and fully decoded PNGs. It records SHA-256 and dimensions. +A run that exits 0 with zero rounds fails. Success is explicitly labelled +`artifacts_complete_not_scene_audited`: this gate does **not** reopen `.blend` files +or prove pose truth, scene preservation, meaningful edits, GPU selection, image +correspondence, or source immutability. Independent checkpoint audits remain +mandatory before claiming ten valid benchmark rounds. Report missing/invalid +rounds honestly. Post-generation GPU CLIP scoring/best checkpoint selection is +separate; no scores or VLM judge are fed into generation. + +No launcher wall-clock timeout is imposed. **Stock backend execution still has a +120-second timeout**, and the example's MCP client timeout is **86400 seconds**. +Neither is unlimited and this launcher does not patch either. Long render runtime +changes require separate review; do not describe this setup as no-hard-timeouts. +Cancellation/supervision and disk-floor checks belong to the operator; interrupted +runs retain logs and must use fresh destinations. No full live ten-round run or +new GPU admission is claimed by the tests here. + +```sh +uv run --frozen --project benchmarks/blenderbench_camera python -m unittest tests.test_direct_codex -v +``` + +Tests use synthetic local subprocess fixtures, not model calls or Blender results. diff --git a/tools/direct_codex.py b/tools/direct_codex.py new file mode 100644 index 0000000..e411b8a --- /dev/null +++ b/tools/direct_codex.py @@ -0,0 +1,197 @@ +"""Direct Codex launch/config preflight; no benchmark controller or scoring.""" +import copy + +TOOLS = ('execute_blender_code_for_cli', 'get_runtime_python_api_docs_for_cli', + 'search_api_docs', 'get_python_api_docs') + + +def prepare_config(config, *, approved): + if not approved: + raise ValueError('explicit approval required: --approve-blender-tools') + if set(config) != {'mcp_servers'} or set(config['mcp_servers']) != {'blender'}: + raise ValueError('config must contain only mcp_servers.blender') + result = copy.deepcopy(config) + server = result['mcp_servers']['blender'] + allowed = {'command', 'args', 'cwd', 'env', 'enabled_tools', 'required', + 'startup_timeout_sec', 'tool_timeout_sec'} + if set(server) - allowed: + raise ValueError('unsupported or conflicting MCP settings') + if sorted(server.get('enabled_tools', [])) != sorted(TOOLS): + raise ValueError('enabled_tools must be exactly the four standalone tools') + env = server.get('env', {}) + from pathlib import Path + if (env.get('BLENDER_MCP_CLI_BACKEND') != 'bpy' + or not Path(env.get('BLENDER_MCP_BPY_PYTHON', '')).is_absolute() + or not Path(server.get('command', '')).is_absolute()): + raise ValueError('absolute server and standalone bpy interpreter required') + server['tools'] = {tool: {'approval_mode': 'approve'} for tool in TOOLS} + server['required'] = True + return result + + +def verify_preflight(events, nonce, python): + """Require a completed real MCP item, never trust the final assistant text.""" + import json + for event in events: + item = event.get('item', {}) + if (event.get('type') != 'item.completed' or item.get('type') != 'mcp_tool_call' + or item.get('server') != 'blender' or item.get('tool') != TOOLS[0]): + continue + result = item.get('result') or {} + if item.get('status') != 'completed' or item.get('error') or result.get('isError'): + continue + payload = result.get('structured_content') + if payload is None: + try: + payload = json.loads(''.join(c.get('text', '') for c in result.get('content', []))) + except (ValueError, TypeError): + continue + if (isinstance(payload, dict) and payload.get('preflight') == nonce + and payload.get('python') == python + and str(payload.get('version', '')).split()[:1] == ['5.2.1']): + return + raise ValueError('no successful standalone bpy 5.2.1 MCP preflight result') + + +def verify_checkpoints(root, rounds): + """Artifact gate only: independent scene/GPU/invariant audit still required.""" + import hashlib + import json + import math + from PIL import Image + try: + rows = [json.loads(line) for line in (root / 'rounds.jsonl').read_text().splitlines()] + if rounds < 1 or [r['round'] for r in rows] != list(range(1, rounds + 1)): + raise ValueError('missing, duplicate, or unordered rounds') + hashes, poses, evidence = set(), set(), [] + for n, row in enumerate(rows, 1): + blend, png = root / f'iteration{n:02}.blend', root / f'iteration{n:02}.png' + if row['blend'] != str(blend) or row['png'] != str(png): + raise ValueError('checkpoint paths must match expected absolute paths') + if blend.is_symlink() or png.is_symlink() or not blend.stat().st_size: + raise ValueError('checkpoint must be a nonempty regular file, not a symlink') + with blend.open('rb') as stream: + digest = hashlib.file_digest(stream, 'sha256').hexdigest() + pose = tuple(row['pose']['location'] + row['pose']['rotation']) + if len(pose) != 6 or not all(type(v) in (int, float) and math.isfinite(v) for v in pose): + raise ValueError('pose must contain six finite numbers') + if digest in hashes or pose in poses: + raise ValueError('duplicate checkpoint bytes or reported pose') + hashes.add(digest) + poses.add(pose) + with Image.open(png) as image: + if image.format != 'PNG': + raise ValueError('checkpoint image must be PNG') + image.load() + dimensions = list(image.size) + evidence.append({'round': n, 'blend_sha256': digest, 'dimensions': dimensions}) + return evidence + except (OSError, KeyError, TypeError) as exc: + raise ValueError(f'incomplete checkpoint artifacts: {exc}') from exc + + +def toml_value(value): + """Encode nested inline TOML, preserving literal dotted keys and booleans.""" + import json + if isinstance(value, dict): + return '{' + ', '.join(json.dumps(k) + '=' + toml_value(v) for k, v in value.items()) + '}' + if isinstance(value, list): + return '[' + ', '.join(toml_value(v) for v in value) + ']' + return json.dumps(value, allow_nan=False) + + +def invoke(args, config, evidence, label, prompt, target=None): + import json + import subprocess + command = [args.codex, 'exec', '--ignore-user-config', '--strict-config', + '--skip-git-repo-check', '--sandbox', 'workspace-write', '--model', args.model, + '--json', '-C', str(args.workdir), '-c', 'approval_policy="never"', + '-c', 'mcp_servers=' + toml_value(config['mcp_servers']), + '-o', str(evidence / (label + '.final.txt'))] + if target: + command += ['-i', str(target)] + command += ['-'] + (evidence / (label + '.argv.json')).write_text(json.dumps(command, indent=2)) + (evidence / (label + '.prompt.txt')).write_text(prompt) + with (evidence / (label + '.jsonl')).open('x') as out, (evidence / (label + '.stderr')).open('x') as err: + proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=out, stderr=err, cwd=args.workdir) + (evidence / (label + '.process.json')).write_text(json.dumps({'pid': proc.pid})) + proc.communicate(prompt.encode()) + (evidence / (label + '.exit-code.txt')).write_text(str(proc.returncode)) + if proc.returncode: + raise ValueError(f'{label}: Codex exited {proc.returncode}; inspect retained logs') + return [json.loads(line) for line in (evidence / (label + '.jsonl')).read_text().splitlines()] + + +def main(argv=None): + import argparse + import json + import subprocess + import sys + import tomllib + import uuid + from pathlib import Path + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=['preflight', 'run']) + parser.add_argument('--config', type=Path, required=True) + parser.add_argument('--codex', default='codex') + parser.add_argument('--workdir', type=Path, required=True) + parser.add_argument('--evidence', type=Path, required=True, help='fresh private evidence directory') + parser.add_argument('--model', required=True) + parser.add_argument('--approve-blender-tools', action='store_true', + help='explicitly preapprove arbitrary code and docs on the configured MCP server') + parser.add_argument('--prompt', type=Path) + parser.add_argument('--target', type=Path) + parser.add_argument('--rounds', type=int, default=10) + args = parser.parse_args(argv) + prompt = '' + try: + config = prepare_config(tomllib.loads(args.config.read_text()), approved=args.approve_blender_tools) + args.workdir = args.workdir.resolve(strict=True) + args.evidence = args.evidence.absolute() + if not (args.workdir / 'initial.blend').is_file(): + raise ValueError('workdir requires initial.blend') + if args.mode == 'run': + if not args.prompt or not args.target or args.rounds < 1: + raise ValueError('run requires --prompt, --target, and positive --rounds') + prompt = args.prompt.read_text() + args.target = args.target.resolve(strict=True) + if (args.workdir / 'rounds.jsonl').exists() or any(args.workdir.glob('iteration*')): + raise ValueError('run requires fresh checkpoint paths; resume is unsupported') + version = subprocess.check_output([args.codex, '--version'], text=True).strip() + if version != 'codex-cli 0.154.0': + raise ValueError(f'unsupported Codex version {version!r}; review schema before updating pin') + args.evidence.mkdir(parents=True, exist_ok=False, mode=0o700) + (args.evidence / 'config.json').write_text(json.dumps(config, indent=2)) + (args.evidence / 'codex-version.txt').write_text(version) + nonce = uuid.uuid4().hex + code = ('import bpy,sys; result={"preflight":' + repr(nonce) + + ',"python":sys.executable,"version":bpy.app.version_string}') + preflight = ('Call only execute_blender_code_for_cli once with blend_file=' + + str(args.workdir / 'initial.blend') + ' and code=' + code + + '. Do not mutate or save the scene. Do not use shell or other tools.') + events = invoke(args, config, args.evidence, 'preflight', preflight) + verify_preflight(events, nonce, config['mcp_servers']['blender']['env']['BLENDER_MCP_BPY_PYTHON']) + if args.mode == 'run': + contract = ('\nUse only these MCP tools for scene operations: ' + ', '.join(TOOLS) + + '. No shell scene operations, benchmark controller, VLM judge, scoring, or publication. ' + 'Use view_image for image inspection. Write rounds.jsonl via MCP with one row per ' + 'completed round: round (integer), pose={location:[x,y,z],rotation:[x,y,z]}, ' + 'blend and png (absolute paths), rationale. Save iteration01.blend/iteration01.png ' + f'through iteration{args.rounds:02}.blend/iteration{args.rounds:02}.png. ' + 'Inspection/no-op calls are not rounds. Never alter initial.blend.') + invoke(args, config, args.evidence, 'run', prompt + contract, args.target) + checkpoints = verify_checkpoints(args.workdir, args.rounds) + (args.evidence / 'artifact-check.json').write_text(json.dumps({ + 'status': 'artifacts_complete_not_scene_audited', 'checkpoints': checkpoints}, indent=2)) + print('Checkpoint artifacts complete; independent scene/GPU/invariant audit and scoring remain required.') + else: + print('Standalone bpy 5.2.1 MCP preflight passed; generation not run.') + return 0 + except (ValueError, OSError, subprocess.SubprocessError) as exc: + print(str(exc), file=sys.stderr) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) From e1771dcc38dafc6f5b0793fa6b7e7b217c37a8f5 Mon Sep 17 00:00:00 2001 From: Michael Gold Date: Thu, 10 Sep 2026 13:06:04 -0400 Subject: [PATCH 2/6] feat: run full BlenderBench tasks through direct Codex with exact audits --- tests/test_direct_codex.py | 56 ++++++ tests/test_direct_suite.py | 74 +++++++ tools/DIRECT_SUITE.md | 24 +++ tools/direct_codex.py | 79 +++++++- tools/direct_runtime.py | 383 +++++++++++++++++++++++++++++++++++++ tools/direct_suite.py | 184 ++++++++++++++++++ 6 files changed, 793 insertions(+), 7 deletions(-) create mode 100644 tests/test_direct_suite.py create mode 100644 tools/DIRECT_SUITE.md create mode 100644 tools/direct_runtime.py create mode 100644 tools/direct_suite.py diff --git a/tests/test_direct_codex.py b/tests/test_direct_codex.py index 29aff89..96d19c6 100644 --- a/tests/test_direct_codex.py +++ b/tests/test_direct_codex.py @@ -138,6 +138,62 @@ def test_cli_denied_preflight_stops_run_even_with_exit_zero(self): self.assertIn('incomplete checkpoint artifacts', proc.stderr) self.assertTrue((root / 'logs2' / 'run.jsonl').is_file()) + def test_task_policy_supports_non_camera_and_rejects_forbidden(self): + import copy + m = self.load() + self.assertTrue(hasattr(m, 'validate_edit'), 'task policy validator missing') + before = {'objects': {'Cube': {'type': 'MESH', 'location': [0, 0, 0], + 'scale': [1, 1, 1]}}, 'preserve': {'mesh': 'fixed'}} + after = copy.deepcopy(before) + after['objects']['Cube']['location'] = [1, 0, 0] + policy = {'allowed_transforms': {'Cube': ['location']}, + 'location_bounds': {'Cube': [[-2, 2]] * 3}} + m.validate_edit(before, after, policy) + for bad in (before, {**after, 'preserve': {'mesh': 'changed'}}): + with self.assertRaises(ValueError): + m.validate_edit(before, bad, policy) + after['objects']['Cube']['location'] = [3, 0, 0] + with self.assertRaises(ValueError): + m.validate_edit(before, after, policy) + + def test_all_exact_property_families(self): + import copy + m = self.load() + before = {'objects': {'Cam': {'type': 'CAMERA', 'lens': 50}, + 'Lamp': {'type': 'LIGHT', 'light_data': {'energy': 20, 'color': [1, 1, 1]}}}, + 'shape_keys': {'Keys': {'Smile': 0, 'Basis': 0}}, 'preserve': {'mesh': 'fixed'}} + for policy, mutate in [ + ({'allowed_camera_data': {'Cam': ['lens']}}, lambda x: x['objects']['Cam'].update(lens=60)), + ({'allowed_light_data': {'Lamp': ['energy', 'color']}}, lambda x: x['objects']['Lamp']['light_data'].update(energy=30, color=[1, 0, 1])), + ({'allowed_shape_keys': {'Keys': ['Smile']}}, lambda x: x['shape_keys']['Keys'].update(Smile=0.5))]: + after = copy.deepcopy(before) + mutate(after) + m.validate_edit(before, after, policy) + after['preserve']['mesh'] = 'changed' + with self.assertRaises(ValueError): + m.validate_edit(before, after, policy) + after = copy.deepcopy(before) + after['objects']['Cam']['lens'] = float('nan') + with self.assertRaises(ValueError): + m.validate_edit(before, after, {'allowed_camera_data': {'Cam': ['lens']}}) + with self.assertRaises(ValueError): + m.validate_edit(before, before, {'allowed_camera_data': {'Cam': ['sensor_width']}}) + with self.assertRaises(ValueError): + m.validate_edit(before, before, {'allowed_light_data': {'Lamp': ['shadow_soft_size']}}) + + def test_non_camera_ledger_needs_no_fake_pose(self): + import tempfile, json + from PIL import Image + m = self.load() + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + blend, png = root / 'iteration01.blend', root / 'iteration01.png' + blend.write_bytes(b'BLENDERfixture') + Image.new('RGB', (2, 2)).save(png) + (root / 'rounds.jsonl').write_text(json.dumps({'round': 1, + 'blend': str(blend), 'png': str(png), 'rationale': 'move cube'})) + self.assertEqual(len(m.verify_checkpoints(root, 1, policy={'allowed_transforms': {'Cube': ['location']}})), 1) + def test_toml_roundtrip_preserves_keys_booleans_and_paths(self): m = self.load() import tomllib diff --git a/tests/test_direct_suite.py b/tests/test_direct_suite.py new file mode 100644 index 0000000..e33df4a --- /dev/null +++ b/tests/test_direct_suite.py @@ -0,0 +1,74 @@ +"""Synthetic orchestration contracts, never benchmark evidence.""" +import importlib.util +import sys +import unittest +from pathlib import Path + +TOOLS = Path(__file__).resolve().parents[1] / 'tools' +sys.path.insert(0, str(TOOLS)) + +class SuiteTests(unittest.TestCase): + def test_rna_shared_graph_is_linear_and_preserves_leaves(self): + import ast + from types import SimpleNamespace + import direct_runtime + tree = ast.parse(direct_runtime.WORKER) + names = {'scalar', 'ref', 'collection_props', 'props'} + funcs = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names] + class ID: pass + class ShapeKey: pass + ns = {'bpy': SimpleNamespace(types=SimpleNamespace(ID=ID, ShapeKey=ShapeKey)), 'SKIP': set()} + exec(compile(ast.Module(body=funcs, type_ignores=[]), 'worker-test', 'exec'), ns) + pointer = lambda key: SimpleNamespace(identifier=key, type='POINTER', is_readonly=False) + value = SimpleNamespace(identifier='value', type='INT', is_readonly=False) + class Node: + calls = 0 + bl_rna = SimpleNamespace(identifier='Node', properties=[pointer('left'), pointer('right'), value]) + def __init__(self, child=None): self.left = self.right = child; self.value = 1 + def as_pointer(self): + Node.calls += 1 + return id(self) + leaf = Node() + node = leaf + for _ in range(12): node = Node(node) + a = ns['props'](node) + self.assertLess(Node.calls, 40, 'shared graph expands exponentially') + leaf.value = 2 + self.assertNotEqual(a, ns['props'](node), 'leaf content must remain audited') + + def test_heterogeneous_rna_collection_uses_each_schema(self): + import ast + from types import SimpleNamespace + import direct_runtime + tree = ast.parse(direct_runtime.WORKER) + funcs = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'collection_props'] + class ID: pass + class ShapeKey: pass + ns = {'bpy': SimpleNamespace(types=SimpleNamespace(ID=ID, ShapeKey=ShapeKey)), + 'SKIP': set(), 'props': lambda x, **kw: {'type': x.bl_rna.identifier} if x else None} + exec(compile(ast.Module(body=funcs, type_ignores=[]), 'worker-test', 'exec'), ns) + field = SimpleNamespace(identifier='text', type='POINTER', is_readonly=False) + a = SimpleNamespace(bl_rna=SimpleNamespace(identifier='NodeA', properties=[field]), text=None) + b = SimpleNamespace(bl_rna=SimpleNamespace(identifier='NodeB', properties=[])) + class Values(list): + def foreach_get(self, *args): pass + got = ns['collection_props'](Values([a, b] * 70), ()) + self.assertEqual(len(got), 140) + self.assertEqual(got[1], {'type': 'NodeB'}) + + def test_plan_excludes_only_completed_camera1(self): + script = TOOLS / 'direct_suite.py' + self.assertTrue(script.exists(), 'suite dispatcher missing') + spec = importlib.util.spec_from_file_location('suite', script) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + tasks = {'level1/camera1': {}} + tasks.update({f'level2/task{i}': {} for i in range(26)}) + plan = module.task_plan(tasks) + self.assertEqual(len(plan), 26) + self.assertNotIn('level1/camera1', plan) + with self.assertRaises(ValueError): + module.task_plan({'level1/camera1': {}}) + +if __name__ == '__main__': + unittest.main() diff --git a/tools/DIRECT_SUITE.md b/tools/DIRECT_SUITE.md new file mode 100644 index 0000000..9bfd024 --- /dev/null +++ b/tools/DIRECT_SUITE.md @@ -0,0 +1,24 @@ +# Direct full-suite execution on mghavn + +`tools/direct_suite.py` is an external sequential dispatcher, not a model reasoning controller. It verifies the pinned input-only manifest, applies reviewed literal initialization using standalone bpy 5.2.1, independently renders/reopens the saved baseline, executes a genuine four-tool Codex preflight, and invokes one direct Codex generation session per task with ten requested meaningful rounds. No evaluator scores or VLM judge enter generation. + +Exactly 26 policy tasks are selected by excluding completed `level1/camera1` from the 27 canonical tasks. Existing completed output is not modified. Active staging is observed, never duplicated. Task work directories must be fresh; there is no automatic generation restart or hidden budget reset. + +Post-generation, artifacts are decoded/hashed, each scene is independently audited/rendered on CUDA, exact task-policy preservation and meaningful edits are checked against both baseline and preceding checkpoint, and repeated complete scene states are rejected. The offline pinned GPU CLIP pair scorer runs after the generation process exits. `scores.json` retains paired NCLIP/PL and `result.json` selects minimum raw NCLIP, earliest-round tie. + +Runtime provenance: `direct_runtime.py` was copied from the reviewed full-suite runtime in the original worktree without editing the original. Its admitted interpreter is the authorized mghavn custom-bpy venv. Embedded RNA graphs are serialized with deterministic repeated-node references to avoid exponential armature expansion; heterogeneous collections retain each element's RNA schema. Neither fix drops audited fields. + +## Invocation + +```sh +nohup /home/mg/blenderbench-direct/enhanced-mcp/mcp/.venv/bin/python -u \ + /home/mg/blenderbench-direct/direct_suite.py \ + --output /home/mg/blenderbench-direct/remaining-direct-03 \ + > /home/mg/blenderbench-direct/remaining-direct-03.log 2>&1 < /dev/null & +``` + +The process holds a host-wide advisory lock. `process.json`, `status.json`, dispatcher `events.jsonl`, per-task native heartbeat logs, and per-task `codex/{preflight,run}.{jsonl,stderr,process.json,argv.json,prompt.txt}` are retained. The output-root `STOP` cancels native workers and prevents the next task; **during an active direct Codex invocation it is only observed after that invocation returns**, not immediate cancellation. Kill that exact Codex process if immediate interruption is needed. No claims of isolation beyond workspace-write and explicit per-tool approval are made. + +**Deadlines remain backend 120 seconds and MCP client 86400 seconds.** They are not unlimited and have not been changed. Native verification has manual cancellation/disk-floor safety and no operation deadline. + +Initialization/audit failures and generation/scoring failures are retained as failed tasks; later tasks continue sequentially. A started suite is not a completed benchmark. No merge, push, publication, or automatic repair/retry is performed. diff --git a/tools/direct_codex.py b/tools/direct_codex.py index e411b8a..83ffa8f 100644 --- a/tools/direct_codex.py +++ b/tools/direct_codex.py @@ -53,7 +53,67 @@ def verify_preflight(events, nonce, python): raise ValueError('no successful standalone bpy 5.2.1 MCP preflight result') -def verify_checkpoints(root, rounds): +def validate_edit(before, after, policy): + """Compare independent RNA audits, masking only exact task allowances.""" + import math + left = copy.deepcopy(before.get('audit', before)) + right = copy.deepcopy(after.get('audit', after)) + changed = False + if policy.get('unsupported_blocker'): + raise ValueError('unsupported task policy') + + def remove(a, b, field, width=None): + nonlocal changed + av, bv = a[field], b[field] + for value in (av, bv): + values = value if isinstance(value, list) else [value] + if width is not None and len(values) != width: + raise ValueError('invalid property dimensions') + if not all(type(v) in (int, float) and math.isfinite(v) for v in values): + raise ValueError('nonfinite or nonnumeric allowed property') + changed |= av != bv + del a[field] + del b[field] + + try: + for name, fields in policy.get('allowed_transforms', {}).items(): + if not fields or len(set(fields)) != len(fields) or set(fields) - {'location', 'rotation_euler', 'scale'}: + raise ValueError('forbidden transform policy') + for field in fields: + remove(left['objects'][name], right['objects'][name], field, 3) + for name, fields in policy.get('allowed_camera_data', {}).items(): + if fields != ['lens']: + raise ValueError('forbidden camera data policy') + a, b = left['objects'][name], right['objects'][name] + if a['type'] != 'CAMERA' or b['type'] != 'CAMERA': + raise ValueError('lens allowance requires camera') + remove(a, b, 'lens', 1) + for name, fields in policy.get('allowed_light_data', {}).items(): + if not fields or len(set(fields)) != len(fields) or set(fields) - {'energy', 'color'}: + raise ValueError('forbidden light policy') + a, b = left['objects'][name], right['objects'][name] + if a['type'] != 'LIGHT' or b['type'] != 'LIGHT': + raise ValueError('light allowance requires light') + for field in fields: + remove(a['light_data'], b['light_data'], field, 3 if field == 'color' else 1) + for name, fields in policy.get('allowed_shape_keys', {}).items(): + if not fields or len(set(fields)) != len(fields): + raise ValueError('invalid shape key policy') + for field in fields: + remove(left['shape_keys'][name], right['shape_keys'][name], field, 1) + for name, bounds in policy.get('location_bounds', {}).items(): + location = after.get('audit', after)['objects'][name]['location'] + if len(location) != 3 or len(bounds) != 3 or any(not lo <= x <= hi for x, (lo, hi) in zip(location, bounds)): + raise ValueError('location outside bounds') + except (KeyError, TypeError) as exc: + raise ValueError('missing or malformed policy property') from exc + if left != right: + raise ValueError('forbidden scene edits outside task policy') + if not changed: + raise ValueError('no allowed scene change') + + +def verify_checkpoints(root, rounds, policy=None): """Artifact gate only: independent scene/GPU/invariant audit still required.""" import hashlib import json @@ -72,13 +132,18 @@ def verify_checkpoints(root, rounds): raise ValueError('checkpoint must be a nonempty regular file, not a symlink') with blend.open('rb') as stream: digest = hashlib.file_digest(stream, 'sha256').hexdigest() - pose = tuple(row['pose']['location'] + row['pose']['rotation']) - if len(pose) != 6 or not all(type(v) in (int, float) and math.isfinite(v) for v in pose): - raise ValueError('pose must contain six finite numbers') - if digest in hashes or pose in poses: - raise ValueError('duplicate checkpoint bytes or reported pose') + if policy is None: + pose = tuple(row['pose']['location'] + row['pose']['rotation']) + if len(pose) != 6 or not all(type(v) in (int, float) and math.isfinite(v) for v in pose): + raise ValueError('pose must contain six finite numbers') + if pose in poses: + raise ValueError('duplicate reported pose') + poses.add(pose) + elif not isinstance(row.get('rationale'), str) or not row['rationale'].strip(): + raise ValueError('task checkpoint requires rationale') + if digest in hashes: + raise ValueError('duplicate checkpoint bytes') hashes.add(digest) - poses.add(pose) with Image.open(png) as image: if image.format != 'PNG': raise ValueError('checkpoint image must be PNG') diff --git a/tools/direct_runtime.py b/tools/direct_runtime.py new file mode 100644 index 0000000..f66b747 --- /dev/null +++ b/tools/direct_runtime.py @@ -0,0 +1,383 @@ +"""Independent custom-bpy 5.2.1 audits and CUDA renders (never Blender CLI). + +Initialization trust contract: parent verifies the dataset pin and reviews start.py, +then supplies policy pin_verified=True, start_sha256, and reviewed literal +initialization operations. Upstream source is hashed but NEVER executed. +Output directories must not exist. STOP in the output, or policy stop_file, cancels. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess +import time + +CUSTOM_BPY_PYTHON = '/home/mg/blenderbench-direct/enhanced-mcp/mcp/.venv/bin/python' + + +def sha(path): + h = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + h.update(block) + return h.hexdigest() + + +def emit(path, event, **fields): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + row = {'event': event, 'time_unix': time.time(), **fields} + with path.open('a') as stream: + stream.write(json.dumps(row, sort_keys=True, allow_nan=False) + '\n') + stream.flush() + os.fsync(stream.fileno()) + return row + + +def _disk(directory, policy): + floor = int(policy.get('disk_floor_bytes', 1024**3)) + if floor < 0: + raise ValueError('negative disk floor') + free = shutil.disk_usage(directory).free + if free < floor: + raise OSError(f'disk floor safety: {free} < {floor}') + return free + + +def run_process(argv, log, event_path, policy): + """No deadline: retain native stdout/stderr; report liveness, not fake progress.""" + log = Path(log) + _disk(log.parent, policy) + stop = Path(policy.get('stop_file', log.parent / 'STOP')) + if stop.exists(): + raise InterruptedError('manual stop file: ' + str(stop)) + start = time.monotonic() + with log.open('xb') as stream: + process = subprocess.Popen(list(map(str, argv)), stdout=stream, stderr=subprocess.STDOUT, + start_new_session=True) + common = {'pid': process.pid, 'parent_pid': os.getpid(), 'task': policy.get('task'), + 'argv': list(map(str, argv)), 'log': str(log.absolute())} + last_size = 0 + last_activity = start + last_slow = start + try: + emit(event_path, 'process_started', **common) + while process.poll() is None: + if stop.exists(): + raise InterruptedError('manual stop file: ' + str(stop)) + free = _disk(log.parent, policy) + size = log.stat().st_size + now = time.monotonic() + activity = {} + try: + activity['proc_stat'] = Path(f'/proc/{process.pid}/stat').read_text() + activity['proc_io'] = Path(f'/proc/{process.pid}/io').read_text() + except (FileNotFoundError, PermissionError): + pass + if size != last_size: + last_activity = now + emit(event_path, 'heartbeat', **common, elapsed_seconds=now-start, + stdout_bytes=size, new_stdout_bytes=size-last_size, free_bytes=free, + seconds_since_stdout=now-last_activity, activity=activity) + if now-last_slow >= float(policy.get('slow_event_seconds', 60)): + emit(event_path, 'slow_operation', **common, elapsed_seconds=now-start, + stdout_bytes=size, activity=activity, action='continue_without_timeout') + last_slow = now + last_size = size + time.sleep(float(policy.get('heartbeat_seconds', 2))) + except BaseException as exc: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + emit(event_path, 'process_cancelled', **common, reason=str(exc), returncode=process.returncode) + raise + stream.flush() + os.fsync(stream.fileno()) + result = {**common, 'returncode': process.returncode, 'elapsed_seconds': time.monotonic()-start} + emit(event_path, 'process_exited', **result) + if process.returncode: + raise RuntimeError(f'custom bpy process exited {process.returncode}; retained log: {log}') + return result + + +# This source is executed exclusively by CUSTOM_BPY_PYTHON. Never imports camera.rounds. +WORKER = r''' +import bpy, hashlib, json, sys, os +from pathlib import Path +cfg=json.loads(Path(sys.argv[1]).read_text()) +assert tuple(bpy.app.version)==(5,2,1), ('custom bpy version required', bpy.app.version_string) +bpy.context.preferences.filepaths.use_scripts_auto_execute=False +bpy.ops.wm.open_mainfile(filepath=cfg['source'], use_scripts=False) +assert not bpy.app.autoexec_fail, 'automatic script dependency rejected' +for op in cfg.get('initialization', []): + if op['kind']=='object': + obj=bpy.data.objects[op['object']] + field=op['property'] + if field in ('location','rotation_euler','scale'): + setattr(obj,field,op['value']) + elif field in ('data.lens','data.energy','data.color'): + setattr(obj.data,field.split('.')[1],op['value']) + else: raise ValueError('unreviewed initialization property: '+field) + elif op['kind']=='shape_key' and op['property']=='value': + bpy.data.shape_keys[op['datablock']].key_blocks[op['key']].value=op['value'] + else: raise ValueError('unreviewed initialization operation') + +# ID references are stable names, never memory-address reprs. All editable scalar +# RNA properties are included; nested embedded structs and collections recurse. +# Unsupported cyclic embedded structures fail closed instead of disappearing. +SKIP={'rna_type','is_updated','is_updated_data','is_updated_transform','tag','users', + 'use_fake_user','use_extra_user','is_embedded_data','is_runtime_data', + 'session_uid','original','id_data','library','override_library','preview','depsgraph', + 'animation_data_clear','name_full'} +def scalar(v): + if isinstance(v,(str,bool,int,float,type(None))): return v + if isinstance(v,bpy.types.ID): return ref(v) + if hasattr(v,'items'): return {k:scalar(value) for k,value in v.items()} + if isinstance(v,set): return sorted(v) + return [scalar(i) for i in v] +def ref(v): + return {'id_type':v.bl_rna.identifier,'name':v.name, + 'library':v.library.filepath if v.library else None} +def collection_props(values, stack): + # RNA foreach_get avoids millions of Python per-vertex/per-component calls. + # Every writable primitive field is retained; variable/non-numeric fields + # and nested pointers still use the general serializer. + if len(values)<128 or not hasattr(values,'foreach_get'): + return [ref(i) if isinstance(i,bpy.types.ID) else props(i,stack=stack) for i in values] + from array import array + first=values[0] + if isinstance(first,bpy.types.ID): return [ref(i) for i in values] + if any(i.bl_rna.identifier != first.bl_rna.identifier for i in values): + return [ref(i) if isinstance(i,bpy.types.ID) else props(i,stack=stack) for i in values] + result={'count':len(values),'rna_type':first.bl_rna.identifier,'fields':{}} + for p in first.bl_rna.properties: + k=p.identifier + if k in SKIP: continue + if p.type not in ('POINTER','COLLECTION') and p.is_readonly: continue + if isinstance(first,bpy.types.ShapeKey) and k=='value': continue + if p.type in ('FLOAT','INT','BOOLEAN') and (not p.is_array or p.array_length): + width=p.array_length if p.is_array else 1 + typecode={'FLOAT':'f','INT':'i','BOOLEAN':'b'}[p.type] + data=array(typecode,[0])*(len(values)*width) + try: + values.foreach_get(k,data) + result['fields'][k]={'width':width,'type':typecode,'sha256':hashlib.sha256(data.tobytes()).hexdigest()} + continue + except (AttributeError,TypeError,RuntimeError): + # Some legacy UV collections expose a different indexed RNA + # subtype from their iterator. Never reuse that field schema. + return [props(i,stack=stack) for i in values] + if p.type=='POINTER': + result['fields'][k]=[ref(v) if isinstance(v,bpy.types.ID) else props(v,stack=stack) for v in (getattr(i,k) for i in values)] + elif p.type=='COLLECTION': + result['fields'][k]=[collection_props(getattr(i,k),stack) for i in values] + else: + result['fields'][k]=[scalar(getattr(i,k)) for i in values] + return result + +def props(x, omit=(), stack=()): + # An RNA graph is not a tree (armature children/parents form dense graphs). + # Serialize each embedded node once per root with deterministic traversal + # references, retaining ALL node content without exponential duplication. + global _rna_seen + if not stack: _rna_seen={} + if x is None: return None + address=(x.bl_rna.identifier,x.as_pointer()) + if address in _rna_seen: return {'rna_ref':_rna_seen[address]} + _rna_seen[address]=len(_rna_seen) + stack=stack+(address,) + out={} + for p in x.bl_rna.properties: + k=p.identifier + if k in SKIP or k in omit or (isinstance(x,bpy.types.ShapeKey) and k=='value'): continue + if p.type not in ('POINTER','COLLECTION') and p.is_readonly: continue + v=getattr(x,k) + if p.type=='POINTER': + out[k]=ref(v) if isinstance(v,bpy.types.ID) else props(v,stack=stack) + elif p.type=='COLLECTION': + out[k]=collection_props(v,stack) + else: + if p.type=='STRING' and (p.subtype in ('FILE_PATH','DIR_PATH') or k=='filepath') and v.startswith('//') and v!='//': + v=os.path.normpath(bpy.path.abspath(v)) + out[k]=scalar(v) + if isinstance(x,bpy.types.ID): + out['custom_properties']={k:scalar(v.to_list() if hasattr(v,'to_list') else v) for k,v in x.items() if k!='_RNA_UI'} + return out + +def digest(value): + return hashlib.sha256(json.dumps(value,sort_keys=True,allow_nan=False).encode()).hexdigest() +def tree(t): + if t is None: return None + return {'properties':props(t), 'nodes':[{ 'properties':props(n), + 'inputs':[props(i) for i in n.inputs], 'outputs':[props(i) for i in n.outputs]} for n in t.nodes], + 'links':[(l.from_node.name,l.from_socket.identifier,l.to_node.name,l.to_socket.identifier) for l in t.links]} +def audit(): + objects={} + for o in bpy.data.objects: + print('__FULL_AUDIT_OBJECT__'+o.name,flush=True) + obj={'type':o.type,'location':list(o.location),'rotation_euler':list(o.rotation_euler),'scale':list(o.scale), + 'properties':props(o,('location','rotation_euler','scale','matrix_world','matrix_local','matrix_basis','dimensions')), + 'visibility':{s.name:{vl.name:o.hide_get(view_layer=vl) for vl in s.view_layers if o.name in vl.objects} for s in bpy.data.scenes}} + if o.type=='CAMERA': obj['lens']=o.data.lens + if o.type=='LIGHT': obj['light_data']={'energy':o.data.energy,'color':list(o.data.color)} + objects[o.name]=obj + preserve={} + for collection in ('meshes','curves','metaballs','armatures','lattices','materials','lights','worlds','cameras','node_groups','actions','collections','scenes','particles','textures','speakers','volumes','pointclouds','grease_pencils'): + items=getattr(bpy.data,collection,()) + values={} + for item in items: + print('__FULL_AUDIT_ITEM__'+json.dumps({'collection':collection,'name':item.name}),flush=True) + bound=any(o.data==item for o in bpy.data.objects) if collection in ('cameras','lights') else False + omissions=('lens','angle','angle_x','angle_y') if collection=='cameras' and bound else (('energy','color') if collection=='lights' and bound else ()) + value=props(item,omissions) + if hasattr(item,'node_tree'): value['node_tree_contents']=tree(item.node_tree) + if collection=='node_groups': value['contents']=tree(item) + values[item.name]=digest(value) + preserve[collection]=values + images={} + for image in bpy.data.images: + if image.type in ('RENDER_RESULT','COMPOSITING'): continue + data={'properties':props(image)} + if image.packed_file: data['bytes']=hashlib.sha256(image.packed_file.data).hexdigest() + elif image.source=='FILE': + path=Path(bpy.path.abspath(image.filepath,library=image.library)) + if not path.is_file(): raise RuntimeError('unavailable image dependency: '+str(path)) + data['bytes']=hashlib.sha256(path.read_bytes()).hexdigest() + images[image.name]=digest(data) + preserve['images']=images + for lib in bpy.data.libraries: + path=Path(bpy.path.abspath(lib.filepath)) + preserve.setdefault('linked_libraries',{})[lib.name]=hashlib.sha256(path.read_bytes()).hexdigest() + preserve['shape_keys']={key.name:digest(props(key)) for key in bpy.data.shape_keys} + shape_keys={key.name:{block.name:block.value for block in key.key_blocks} for key in bpy.data.shape_keys} + return {'objects':objects,'shape_keys':shape_keys,'preserve':preserve} + +# GPU setup is the only runtime-local device override. Engine, samples and +# resolution are never altered. Audit after setup so source CPU flags normalize. +prefs=bpy.context.preferences.addons['cycles'].preferences +prefs.compute_device_type='CUDA' +prefs.refresh_devices() +assert any(d.type=='CUDA' for d in prefs.devices), 'CUDA unavailable; CPU fallback forbidden' +for d in prefs.devices: d.use=(d.type=='CUDA') +for scene in bpy.data.scenes: + assert scene.render.engine=='CYCLES', 'unsupported non-Cycles engine; not silently changed' + scene.cycles.device='GPU' +s=bpy.context.scene +assert s.camera is not None, 'scene has no active camera' +evidence={'compute_device_type':prefs.compute_device_type, + 'devices':[{'name':d.name,'type':d.type,'use':bool(d.use),'id':d.id} for d in prefs.devices], + 'scene_device':s.cycles.device,'engine':s.render.engine,'samples':s.cycles.samples, + 'resolution':[s.render.resolution_x,s.render.resolution_y,s.render.resolution_percentage], + 'blender_version':bpy.app.version_string,'blender_version_tuple':list(bpy.app.version), + 'build_hash':bpy.app.build_hash.decode(errors='replace'),'python_executable':sys.executable, + 'pid':os.getpid()} +if cfg.get('save'): + bpy.context.preferences.filepaths.save_version=0 + bpy.ops.wm.save_as_mainfile(filepath=cfg['save'], relative_remap=True) +record={'audit':audit(),'evidence':evidence, + 'settings':{'engine':s.render.engine,'samples':s.cycles.samples, + 'resolution_x':s.render.resolution_x,'resolution_y':s.render.resolution_y, + 'resolution_percentage':s.render.resolution_percentage}} +Path(cfg['report']).write_text(json.dumps(record,sort_keys=True,allow_nan=False)) +print('__FULL_AUDIT_READY__',flush=True) +if cfg.get('png'): + # Output encoding and destination are transient and are never saved. + s.render.filepath=cfg['png'] + s.render.image_settings.file_format='PNG' + s.render.use_file_extension=True + bpy.ops.render.render(write_still=True) + print('__FULL_RENDER_FINISHED__',flush=True) +''' + + +def _execute(source, policy, bpy_python, output, event_path, render, start=None): + # Preserve the venv executable path: resolving its symlink loses venv imports. + python = Path(bpy_python).absolute() + if str(python) != CUSTOM_BPY_PYTHON: + raise ValueError('only installed custom bpy Python is admitted: ' + CUSTOM_BPY_PYTHON) + if not python.is_file(): + raise FileNotFoundError(python) + source = Path(source).resolve(strict=True) + if source.suffix != '.blend': + raise ValueError('source must be a saved .blend') + output = Path(output).absolute() + output.mkdir(parents=True, exist_ok=False) + _disk(output, policy) + original = sha(source) + png = output / ('initial_verified.png' if start else 'verified.png') if render else None + saved = output / 'start_initialized.blend' if start else None + config = {'source': str(source), 'report': str(output / 'audit.json'), + 'save': str(saved) if saved else None, 'png': str(png) if png else None} + if start: + config['initialization'] = policy['initialization'] + if sha(start) != policy['start_sha256']: + raise ValueError('verified start.py hash drift') + script = output / 'worker.py' + script.write_text(WORKER) + config_path = output / 'worker.json' + config_path.write_text(json.dumps(config)) + try: + process = run_process([str(python), '-u', str(script), str(config_path)], output / 'runtime.log', event_path, policy) + finally: + if sha(source) != original: + emit(event_path, 'source_changed', source=str(source), task=policy.get('task')) + raise ValueError('source blend changed during independent verification') + result = json.loads(Path(config['report']).read_text()) + evidence = result['evidence'] + if (evidence['blender_version_tuple'] != [5, 2, 1] or evidence['compute_device_type'] != 'CUDA' + or evidence['scene_device'] != 'GPU' or not any(d['type']=='CUDA' and d['use'] for d in evidence['devices']) + or any(d['use'] and d['type']!='CUDA' for d in evidence['devices'])): + raise ValueError('custom bpy CUDA evidence failed') + result.update(task=policy.get('task'), source=str(source), source_sha256=original, + blend=str(saved or source), blend_sha256=sha(saved or source), + runtime=process, output=str(output), audit_path=config['report'], + png=str(png) if png else None) + if png: + from PIL import Image + with Image.open(png) as image: + if image.format != 'PNG': raise ValueError('render is not PNG') + image.load() + dimensions = list(image.size) + settings = result['settings'] + expected = [max(1, settings[k]*settings['resolution_percentage']//100) for k in ('resolution_x','resolution_y')] + if dimensions != expected: + raise ValueError(f'decoded PNG dimensions {dimensions} != task {expected}') + result.update(png_dimensions=dimensions, png_sha256=sha(png)) + emit(event_path, 'render_completed', task=policy.get('task'), png=str(png), + png_sha256=result['png_sha256'], png_dimensions=dimensions, + blend=result['blend'], blend_sha256=result['blend_sha256'], + runtime=process, evidence=evidence, source_sha256=original) + (output / 'metadata.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + return result + + +def inspect_render(blend, task_policy, bpy_python, output, event_path, render=True): + """Independently open/audit/render an unchanged saved blend via custom bpy.""" + return _execute(blend, task_policy, bpy_python, output, event_path, render) + + +def initialize(inp, task_policy, bpy_python, output, event_path): + """Apply parent-reviewed literal initialization; save and render the state. + + inp is a directory with exactly one top-level .blend and start.py, or a mapping + with explicit blend/start_py paths. No goal/oracle files are inspected. + """ + if task_policy.get('pin_verified') is not True or not task_policy.get('start_sha256') or not isinstance(task_policy.get('initialization'), list): + raise ValueError('initialization requires parent-verified pin/policy, start_sha256 and literal initialization') + if task_policy.get('unsupported_blocker') or task_policy.get('reviewed') is False: + raise ValueError('unsupported or unreviewed initialization policy') + if isinstance(inp, dict): + source, start = Path(inp['blend']), Path(inp['start_py']) + else: + inp = Path(inp) + sources = list(inp.glob('*.blend')) + if len(sources) != 1: + raise ValueError('expected exactly one safe top-level input blend') + source, start = sources[0], inp / 'start.py' + if start.name != 'start.py' or sha(start) != task_policy['start_sha256']: + raise ValueError('parent-verified start.py hash mismatch') + return _execute(source, task_policy, bpy_python, output, event_path, True, start.resolve()) diff --git a/tools/direct_suite.py b/tools/direct_suite.py new file mode 100644 index 0000000..d155cd5 --- /dev/null +++ b/tools/direct_suite.py @@ -0,0 +1,184 @@ +"""External sequential dispatcher: native admission, direct Codex, post-run audit/score. +No model reasoning loop, score feedback, VLM judge, or upstream script execution. +""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import tomllib +import uuid +from types import SimpleNamespace + +import direct_codex as direct +import direct_runtime as runtime + +ROOT = Path('/home/mg/blenderbench-direct') +FIELDS = ('allowed_transforms', 'allowed_camera_data', 'allowed_light_data', + 'allowed_shape_keys', 'location_bounds') + + +def task_plan(tasks): + plan = [t for t in tasks if t != 'level1/camera1'] + if len(tasks) != 27 or len(plan) != 26: + raise ValueError('expected 27 canonical tasks, 26 remaining') + return plan + + +def run_task(task, policy, out, config, dataset): + inp = ROOT / 'dataset' / task + verification = dataset.verify(task, inp) + policy = dict(policy, task=task, pin_verified=True, stop_file=str(out.parent / 'STOP')) + if any(field not in policy for field in FIELDS): + raise ValueError('missing exact task policy fields') + out.mkdir(parents=True, exist_ok=False) + (out / 'input-verification.json').write_text(json.dumps(verification, indent=2)) + events = out / 'events.jsonl' + runtime.emit(events, 'initializing', task=task) + initial = runtime.initialize(inp, policy, runtime.CUSTOM_BPY_PYTHON, out / 'work', events) + work = out / 'work' + blend = work / 'initial.blend' + Path(initial['blend']).rename(blend) + baseline = runtime.inspect_render(blend, policy, runtime.CUSTOM_BPY_PYTHON, out / 'admission', events) + reopened = runtime.inspect_render(blend, policy, runtime.CUSTOM_BPY_PYTHON, out / 'reopen', events, render=False) + if initial['audit'] != baseline['audit'] or baseline['audit'] != reopened['audit']: + raise ValueError('initialization/reopen independent invariant mismatch') + initial_hash = runtime.sha(blend) + runtime.emit(events, 'native_admission_passed', task=task, source_sha256=initial_hash) + args = SimpleNamespace(codex='/home/mg/.local/bin/codex', model='gpt-6-astra', workdir=work) + evidence = out / 'codex' + evidence.mkdir(mode=0o700) + (evidence / 'config.json').write_text(json.dumps(config, indent=2)) + nonce = uuid.uuid4().hex + code = 'import bpy,sys; result=' + repr({'preflight': nonce}) + code += '; result.update(python=sys.executable,version=bpy.app.version_string)' + prompt = ('Call only execute_blender_code_for_cli once with blend_file=' + str(blend) + + ' and code=' + code + '. Do not mutate/save. No shell or other tools.') + preflight = direct.invoke(args, config, evidence, 'preflight', prompt) + direct.verify_preflight(preflight, nonce, runtime.CUSTOM_BPY_PYTHON) + runtime.emit(events, 'preflight_passed', task=task) + contract = ('Solve this BlenderBench task by direct use of the four Blender MCP tools.\n' + + policy['task_description'] + '\nTarget image is attached. Initial scene: ' + str(blend) + + '\nExact mutable-property policy (all unlisted scene data must remain unchanged):\n' + + json.dumps({f: policy[f] for f in FIELDS}, indent=2) + + '\nPerform exactly 10 meaningful edit/render/visual-inspection rounds sequentially. ' + 'Inspect initial scene and image, reason yourself about edits, render and inspect each checkpoint ' + 'using view_image. Do not delegate reasoning to scripts/controllers. No CLIP/scoring, VLM judge, ' + 'oracle/goal .blend access, or score feedback. Do not read audit or evaluation files. ' + 'Use only execute_blender_code_for_cli, get_runtime_python_api_docs_for_cli, search_api_docs, ' + 'get_python_api_docs for scene work; no shell scene operations. ' + 'The backend code-call deadline remains 120 seconds; the MCP client deadline remains 86400 seconds. ' + 'Use CUDA rendering, enable CUDA devices only and disable CPU; retain engine, samples, resolution ' + 'and all other saved scene settings. Save scene BEFORE transient PNG output setting changes; ' + 'never save the render filepath/encoding overrides. Disable backup save_version. ' + 'Never alter initial.blend. Do not create/delete objects or alter unlisted properties. ' + 'Every round must be a distinct allowed edit, not inspection/no-op or a repeat of any earlier scene state. ' + 'Save absolute paths ' + str(work / 'iteration01.blend') + ' and ' + + str(work / 'iteration01.png') + ' through iteration10.blend and iteration10.png. ' + 'Append rounds.jsonl in this work directory via MCP, one JSON object per completed round: ' + 'round integer 1..10, blend absolute path, png absolute path, rationale explaining the visual ' + 'hypothesis and actual edit. Do not invent camera poses for non-camera tasks. ' + 'Each checkpoint must remain directly openable with dependencies intact. ' + 'Stop only after all ten rounds and their images and ledger rows are written.') + runtime.emit(events, 'generation_started', task=task) + direct.invoke(args, config, evidence, 'run', contract, inp / 'target.png') + if runtime.sha(blend) != initial_hash: + raise ValueError('Codex modified initial.blend') + artifacts = direct.verify_checkpoints(work, 10, policy=policy) + (out / 'artifact-check.json').write_text(json.dumps(artifacts, indent=2)) + runtime.emit(events, 'generation_finished', task=task) + # All external scoring occurs only AFTER the full direct generation session. + previous = baseline + seen = {json.dumps(baseline['audit'], sort_keys=True)} + rows = [] + for n in range(1, 11): + audit = runtime.inspect_render(work / f'iteration{n:02}.blend', policy, + runtime.CUSTOM_BPY_PYTHON, out / f'audit{n:02}', events) + direct.validate_edit(baseline, audit, policy) + direct.validate_edit(previous, audit, policy) + state = json.dumps(audit['audit'], sort_keys=True) + if state in seen: + raise ValueError('repeated allowed scene state') + seen.add(state) + previous = audit + score_dir = out / f'score{n:02}' + score_dir.mkdir() + runtime.run_process([ROOT / 'score-venv/bin/python', ROOT / 'blenderbench-score-pair.py', + audit['png'], inp / 'target.png'], score_dir / 'score.json', events, policy) + score = json.loads((score_dir / 'score.json').read_text()) + rows.append(dict(round=n, **score, blend=audit['blend'], blend_sha256=audit['blend_sha256'])) + (out / 'scores.json').write_text(json.dumps(rows, indent=2)) + best = min(rows, key=lambda row: (row['raw_nclip'], row['round'])) + (out / 'result.json').write_text(json.dumps({'status': 'complete', 'task': task, + 'rounds': rows, 'best': best, 'feedback_to_generation': False, 'vlm_judge': False}, indent=2)) + runtime.emit(events, 'task_complete', task=task, rounds=len(rows)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + out = args.output.absolute() + out.mkdir(parents=True, exist_ok=True) + with (ROOT / 'direct-suite.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + sys.path.insert(0, str(ROOT / 'input-tools')) + import dataset + tasks = json.loads((ROOT / 'input-tools/task-policies.json').read_text())['tasks'] + plan = task_plan(tasks) + if set(tasks) != set(dataset.task_ids()): + raise ValueError('canonical policy/manifest task mismatch') + config = direct.prepare_config(tomllib.loads((ROOT / 'mghavn-custom-bpy.toml').read_text()), approved=True) + version = subprocess.check_output(['/home/mg/.local/bin/codex', '--version'], text=True).strip() + if version != 'codex-cli 0.154.0': + raise ValueError('unreviewed Codex version') + runtime.emit(out / 'events.jsonl', 'dispatcher_started', pid=os.getpid(), tasks=plan, + backend_deadline_seconds=120, client_deadline_seconds=86400) + (out / 'process.json').write_text(json.dumps({'pid': os.getpid(), 'argv': sys.argv, + 'cwd': os.getcwd(), 'tasks': plan, 'model': 'gpt-6-astra'}, indent=2)) + states = {} + def status(task, state, **extra): + states[task] = dict(status=state, **extra) + temp = out / 'status.tmp' + temp.write_text(json.dumps(states, indent=2)) + temp.replace(out / 'status.json') + runtime.emit(out / 'events.jsonl', state, task=task, **extra) + for task in plan: + if (out / 'STOP').exists(): + break + task_out = out / task.replace('/', '-') + if task_out.exists(): + raise ValueError('fresh task output required; automatic restart forbidden') + # Do not duplicate the active downloader. Wait for its verified checkpoint. + checkpoint = ROOT / 'dataset' / task / 'checkpoint.json' + while True: + if (out / 'STOP').exists(): + raise InterruptedError('manual STOP') + try: + staging = json.loads(checkpoint.read_text()) + except (FileNotFoundError, json.JSONDecodeError): + staging = {} + if staging.get('status') == 'verified': + break + if staging.get('status') == 'failed': + status(task, 'input_failed', error=staging.get('error')) + break + status(task, 'waiting_for_staging') + time.sleep(10) + if staging.get('status') != 'verified': + continue + status(task, 'running', output=str(task_out)) + try: + run_task(task, tasks[task], task_out, config, dataset) + except Exception as exc: + status(task, 'failed', error=repr(exc), output=str(task_out)) + else: + status(task, 'complete', output=str(task_out)) + runtime.emit(out / 'events.jsonl', 'dispatcher_finished', states=states) + + +if __name__ == '__main__': + main() From 25254ebfe01bb7d857597dc9cd162519f411f00f Mon Sep 17 00:00:00 2001 From: Michael Gold Date: Thu, 10 Sep 2026 14:24:59 -0400 Subject: [PATCH 3/6] fix: preserve inactive scene engines in Camera3 CUDA admission --- tests/test_direct_suite.py | 83 +++++++++++++++++++++++++++++++ tools/direct_camera_retry.md | 43 +++++++++++++++++ tools/direct_camera_retry.py | 94 ++++++++++++++++++++++++++++++++++++ tools/direct_runtime.py | 7 +-- 4 files changed, 224 insertions(+), 3 deletions(-) create mode 100644 tools/direct_camera_retry.md create mode 100644 tools/direct_camera_retry.py diff --git a/tests/test_direct_suite.py b/tests/test_direct_suite.py index e33df4a..14777d2 100644 --- a/tests/test_direct_suite.py +++ b/tests/test_direct_suite.py @@ -8,6 +8,32 @@ sys.path.insert(0, str(TOOLS)) class SuiteTests(unittest.TestCase): + def gpu_setup(self, active_engine='CYCLES'): + from types import SimpleNamespace as NS + import direct_runtime + active = NS(render=NS(engine=active_engine), cycles=NS(device='CPU')) + inactive = NS(render=NS(engine='BLENDER_EEVEE'), cycles=NS(device='CPU')) + devices = [NS(type='CUDA', use=False), NS(type='CPU', use=True)] + prefs = NS(compute_device_type='NONE', refresh_devices=lambda: None, devices=devices) + bpy = NS(context=NS(scene=active, preferences=NS(addons={'cycles': NS(preferences=prefs)})), + data=NS(scenes=[inactive, active])) + # Exercise the actual worker setup without importing native bpy locally. + setup = direct_runtime.WORKER.split('# GPU setup', 1)[1].split("assert s.camera", 1)[0] + exec('# GPU setup' + setup, {'bpy': bpy}) + return active, inactive, devices + + def test_inactive_eevee_scene_is_preserved_with_active_cycles(self): + active, inactive, devices = self.gpu_setup() + self.assertEqual(active.render.engine, 'CYCLES') + self.assertEqual(active.cycles.device, 'GPU') + self.assertEqual(inactive.render.engine, 'BLENDER_EEVEE') + self.assertEqual(inactive.cycles.device, 'CPU') + self.assertEqual([d.use for d in devices], [True, False]) + + def test_active_eevee_still_fails_closed_without_conversion(self): + with self.assertRaisesRegex(AssertionError, 'unsupported non-Cycles'): + self.gpu_setup('BLENDER_EEVEE') + def test_rna_shared_graph_is_linear_and_preserves_leaves(self): import ast from types import SimpleNamespace @@ -56,6 +82,63 @@ def foreach_get(self, *args): pass self.assertEqual(len(got), 140) self.assertEqual(got[1], {'type': 'NodeB'}) + def test_retry_requires_finished_suite_and_same_global_lock(self): + import tempfile + import fcntl + import direct_camera_retry as retry + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + suite = root / 'suite' + suite.mkdir() + self.assertFalse(retry.suite_finished(suite)) + (suite / 'events.jsonl').write_text('{"event":"dispatcher_started"}\n') + self.assertFalse(retry.suite_finished(suite)) + (suite / 'events.jsonl').write_text('{"event":"dispatcher_finished"}\n') + self.assertTrue(retry.suite_finished(suite)) + with (root / 'direct-suite.lock').open('a') as held: + fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) + with (root / 'direct-suite.lock').open('a') as other: + self.assertFalse(retry.try_lock(other)) + fcntl.flock(held, fcntl.LOCK_UN) + with (root / 'direct-suite.lock').open('a') as other: + self.assertTrue(retry.try_lock(other)) + (suite / 'STOP').touch() + with self.assertRaises(InterruptedError): + retry.suite_finished(suite) + + def test_retry_process_waits_without_work_and_honors_stop(self): + import json + import subprocess + import tempfile + import time + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + predecessor = root / 'suite' + predecessor.mkdir() + out = root / 'retry' + bootstrap = ('import sys; sys.path.insert(0,' + repr(str(TOOLS)) + '); ' + 'import direct_camera_retry as r; from pathlib import Path; ' + 'r.suite.ROOT=Path(' + repr(tmp) + '); r.main()') + process = subprocess.Popen([sys.executable, '-c', bootstrap, '--output', str(out), + '--after-suite', str(predecessor)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 5 + while not (out / 'status.json').exists() and time.monotonic() < deadline: + time.sleep(0.02) + row = json.loads((out / 'status.json').read_text()) + self.assertEqual(row['status'], 'waiting_for_suite') + self.assertFalse((out / 'level1-camera3').exists()) + (out / 'STOP').touch() + process.communicate(timeout=15) + self.assertNotEqual(process.returncode, 0) + self.assertEqual(json.loads((out / 'status.json').read_text())['status'], 'failed') + self.assertFalse((out / 'level1-camera3').exists()) + finally: + if process.poll() is None: + process.kill() + process.communicate() + def test_plan_excludes_only_completed_camera1(self): script = TOOLS / 'direct_suite.py' self.assertTrue(script.exists(), 'suite dispatcher missing') diff --git a/tools/direct_camera_retry.md b/tools/direct_camera_retry.md new file mode 100644 index 0000000..4856ddf --- /dev/null +++ b/tools/direct_camera_retry.md @@ -0,0 +1,43 @@ +# Camera3 mixed-scene compatibility and safe retry + +Camera3's pinned input opens `Yoga studio scene`, whose engine is already CYCLES, +with Camera1, 512 samples, and 512x512 at 100%. A second inactive scene named +`Scene` uses BLENDER_EEVEE. The old runtime rejected that inactive scene before +rendering the active Cycles scene. Only the active scene now receives the CUDA +runtime-local GPU device override. Neither scene's engine changes, and inactive +scene device settings remain untouched. Full RNA audits still cover both scenes. +Active non-Cycles rendering remains unsupported and fails closed; this is not +an EEVEE/EGL implementation or a Cycles conversion. + +Use an isolated copy of direct_runtime.py, direct_codex.py, direct_suite.py and +direct_camera_retry.py. Do not overwrite files imported by an active dispatcher. +Launch the retry script durably (detached session, retained stdout/stderr): + +``` +/path/to/custom-bpy/python -u /isolated/direct_camera_retry.py \ + --output /fresh/camera3-retry \ + --after-suite /existing/remaining-suite +``` + +The wrapper requires a normally finished predecessor event AND acquisition of +`/home/mg/blenderbench-direct/direct-suite.lock`. It holds that lock throughout +admission, ten direct Codex rounds, full independent audits and post-generation +GPU CLIP scoring. No VLM judge or feedback into generation is added. The exact +four approved MCP tools and existing deadlines/config checks are unchanged. +The original failed Camera3 directory remains evidence, never a resume target. + +A predecessor crash does not authorize concurrent retry: the wrapper keeps +waiting without launching work. A STOP file in either the predecessor output or +retry output cancels the queue. After generation starts the existing suite's +STOP limitation applies: direct Codex notices cancellation after invoke returns. + +Monitor `process.json`, `status.json`, `events.jsonl`, and retained process stdout. +Actual native admission evidence is in `level1-camera3/{work,admission,reopen}`: +initialization and independent render must produce parent-decoded PNGs; all three +full audits must agree and source/saved-scene hashes must remain unchanged. +A queue status of `waiting_for_suite` is NOT native GPU admission or completion. + +Tests: `python -m unittest discover -s tests -p 'test_direct*.py' -v`. +The mixed-scene regression was observed failing at the old engine assertion +before the fix. Queue tests exercise real flock contention and a real waiting +subprocess with STOP cancellation. Synthetic tests are not GPU render evidence. diff --git a/tools/direct_camera_retry.py b/tools/direct_camera_retry.py new file mode 100644 index 0000000..2946939 --- /dev/null +++ b/tools/direct_camera_retry.py @@ -0,0 +1,94 @@ +"""One fresh Camera3 retry after a normally finished suite; never preempt it. + +Launch with nohup/setsid and retain stdout. A STOP in either output cancels the +queue. An interrupted/crashed predecessor is NOT equivalent to a finished suite. +The existing global flock is held through native admission, generation and score. +""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import tomllib + +import direct_suite as suite +import direct_runtime as runtime + + +def suite_finished(output): + if (output / 'STOP').exists(): + raise InterruptedError('predecessor STOP: retry cancelled') + try: + with (output / 'events.jsonl').open() as stream: + last = None + for line in stream: + if line.strip(): + last = json.loads(line) + return bool(last and last.get('event') == 'dispatcher_finished') + except (FileNotFoundError, json.JSONDecodeError): + return False + + +def try_lock(lock): + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return False + return True + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--after-suite', type=Path, required=True) + args = parser.parse_args() + out = args.output.absolute() + predecessor = args.after_suite.resolve(strict=True) + out.mkdir(parents=True, exist_ok=False) + events = out / 'events.jsonl' + process = {'pid': os.getpid(), 'argv': sys.argv, 'cwd': os.getcwd(), + 'after_suite': str(predecessor), 'task': 'level1/camera3'} + (out / 'process.json').write_text(json.dumps(process, indent=2)) + def status(state, **extra): + row = dict(process, status=state, **extra) + temp = out / 'status.tmp' + temp.write_text(json.dumps(row, indent=2)) + temp.replace(out / 'status.json') + runtime.emit(events, state, **process, **extra) + try: + with (suite.ROOT / 'direct-suite.lock').open('a') as lock: + while True: + if (out / 'STOP').exists(): + raise InterruptedError('retry STOP') + finished = suite_finished(predecessor) + if finished and try_lock(lock): + break + status('waiting_for_suite', predecessor_finished=finished) + time.sleep(10) + # Recheck under the lock before any native GPU or model work. + if not suite_finished(predecessor) or (out / 'STOP').exists(): + raise InterruptedError('admission cancelled') + sys.path.insert(0, str(suite.ROOT / 'input-tools')) + import dataset + tasks = json.loads((suite.ROOT / 'input-tools/task-policies.json').read_text())['tasks'] + if set(tasks) != set(dataset.task_ids()): + raise ValueError('canonical policy/manifest task mismatch') + config = suite.direct.prepare_config(tomllib.loads( + (suite.ROOT / 'mghavn-custom-bpy.toml').read_text()), approved=True) + version = subprocess.check_output(['/home/mg/.local/bin/codex', '--version'], text=True).strip() + if version != 'codex-cli 0.154.0': + raise ValueError('unreviewed Codex version') + status('running') + suite.run_task('level1/camera3', tasks['level1/camera3'], + out / 'level1-camera3', config, dataset) + status('complete') + except BaseException as exc: + status('failed', error=repr(exc)) + raise + + +if __name__ == '__main__': + main() diff --git a/tools/direct_runtime.py b/tools/direct_runtime.py index f66b747..43f298b 100644 --- a/tools/direct_runtime.py +++ b/tools/direct_runtime.py @@ -263,10 +263,11 @@ def audit(): prefs.refresh_devices() assert any(d.type=='CUDA' for d in prefs.devices), 'CUDA unavailable; CPU fallback forbidden' for d in prefs.devices: d.use=(d.type=='CUDA') -for scene in bpy.data.scenes: - assert scene.render.engine=='CYCLES', 'unsupported non-Cycles engine; not silently changed' - scene.cycles.device='GPU' +# Only the active scene is rendered. Inactive scenes can use different engines; +# leave their engine AND device settings intact, retaining them in full audits. s=bpy.context.scene +assert s.render.engine=='CYCLES', 'unsupported non-Cycles engine; not silently changed' +s.cycles.device='GPU' assert s.camera is not None, 'scene has no active camera' evidence={'compute_device_type':prefs.compute_device_type, 'devices':[{'name':d.name,'type':d.type,'use':bool(d.use),'id':d.id} for d in prefs.devices], From 00ba68b636f3b3eadbfc0f7cbe60ea8fd52ad119 Mon Sep 17 00:00:00 2001 From: Michael Gold Date: Thu, 10 Sep 2026 19:28:07 -0400 Subject: [PATCH 4/6] fix: defer saved-scene audits and allow evidenced orphan material cleanup --- tests/test_direct_saved_cleanup.py | 120 +++++++++++++++++++++++++++++ tests/test_direct_suite.py | 10 +++ tools/direct_codex.py | 12 +++ tools/direct_runtime.py | 9 ++- tools/direct_saved_validation.md | 47 +++++++++++ tools/direct_suite.py | 21 +++-- 6 files changed, 212 insertions(+), 7 deletions(-) create mode 100644 tests/test_direct_saved_cleanup.py create mode 100644 tools/direct_saved_validation.md diff --git a/tests/test_direct_saved_cleanup.py b/tests/test_direct_saved_cleanup.py new file mode 100644 index 0000000..a236865 --- /dev/null +++ b/tests/test_direct_saved_cleanup.py @@ -0,0 +1,120 @@ +"""Saved delivery audits tolerate only evidenced orphan material removal.""" +import copy +import sys +import unittest +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools')) +import direct_codex as direct + + +class SavedCleanupTests(unittest.TestCase): + def fixture(self): + before = {'audit': {'objects': {'Camera': {'type': 'CAMERA', 'lens': 50}}, + 'preserve': {'materials': {'orphan': 'orphan-hash', 'used': 'used-hash'}, + 'meshes': {'mesh': 'geometry-hash'}, + 'scenes': {'Scene': 'lighting-hash'}}}, + 'material_lifecycle': {'orphan': {'users': 0, 'use_fake_user': False, + 'use_extra_user': False, 'library': None}, + 'used': {'users': 1, 'use_fake_user': False, + 'use_extra_user': False, 'library': None}}} + after = copy.deepcopy(before) + after['audit']['objects']['Camera']['lens'] = 55 + del after['audit']['preserve']['materials']['orphan'] + del after['material_lifecycle']['orphan'] + return before, after, {'allowed_camera_data': {'Camera': ['lens']}} + + def test_worker_records_material_lifecycle_without_mutating(self): + import ast + from types import SimpleNamespace as NS + import direct_runtime + functions = [n for n in ast.parse(direct_runtime.WORKER).body + if isinstance(n, ast.FunctionDef) and n.name == 'material_lifecycle'] + self.assertEqual(len(functions), 1) + material = NS(name='orphan', users=0, use_fake_user=False, + use_extra_user=False, library=None) + ns = {'bpy': NS(data=NS(materials=[material]))} + exec(compile(ast.Module(body=functions, type_ignores=[]), 'worker', 'exec'), ns) + self.assertEqual(ns['material_lifecycle']()['orphan'], + self.fixture()[0]['material_lifecycle']['orphan']) + self.assertFalse(material.use_fake_user) + + def test_generation_starts_without_independent_reopen_gate(self): + import tempfile + from types import SimpleNamespace as NS + from unittest.mock import patch + import direct_suite as suite + class GenerationReached(Exception): pass + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + def initialize(inp, policy, python, output, events): + output.mkdir() + blend = output / 'start_initialized.blend' + blend.write_bytes(b'fixture') + return {'blend': str(blend), **self.fixture()[0]} + def invoke(args, config, evidence, label, prompt, target=None): + if label == 'run': + self.assertIn('previous saved checkpoint', prompt) + raise GenerationReached() + return [] + policy = {f: {} for f in suite.FIELDS} + policy['task_description'] = 'test' + with patch.object(suite, 'ROOT', root), \ + patch.object(suite.runtime, 'initialize', initialize), \ + patch.object(suite.runtime, 'inspect_render', side_effect=AssertionError('pre-generation reopen')), \ + patch.object(suite.direct, 'invoke', invoke), \ + patch.object(suite.direct, 'verify_preflight'): + with self.assertRaises(GenerationReached): + suite.run_task('level2/attribute1', policy, root / 'task', {}, + NS(verify=lambda *a: {})) + + def test_zero_user_unused_material_loss_is_accepted(self): + before, after, policy = self.fixture() + original = copy.deepcopy(before) + direct.validate_edit(before, after, policy) + self.assertEqual(before, original) + + def test_used_material_change_or_loss_is_rejected(self): + for value in ('changed-hash', None): + before, after, policy = self.fixture() + if value is None: + del after['audit']['preserve']['materials']['used'] + else: + after['audit']['preserve']['materials']['used'] = value + with self.assertRaisesRegex(ValueError, 'forbidden scene edits'): + direct.validate_edit(before, after, policy) + + def test_missing_lifecycle_and_protected_orphans_fail_closed(self): + for field, value in [('users', 1), ('use_fake_user', True), + ('use_extra_user', True), ('library', '//linked.blend'), + ('users', None)]: + before, after, policy = self.fixture() + before['material_lifecycle']['orphan'][field] = value + with self.assertRaisesRegex(ValueError, 'forbidden scene edits'): + direct.validate_edit(before, after, policy) + before, after, policy = self.fixture() + del before['material_lifecycle'] + with self.assertRaisesRegex(ValueError, 'forbidden scene edits'): + direct.validate_edit(before, after, policy) + + def test_geometry_lighting_and_material_additions_still_fail(self): + for collection, name in [('meshes', 'mesh'), ('scenes', 'Scene'), ('materials', 'new')]: + before, after, policy = self.fixture() + after['audit']['preserve'][collection][name] = 'changed' + with self.assertRaisesRegex(ValueError, 'forbidden scene edits'): + direct.validate_edit(before, after, policy) + + def test_cleanup_alone_is_not_a_meaningful_round(self): + before, after, policy = self.fixture() + after['audit']['objects']['Camera']['lens'] = 50 + with self.assertRaisesRegex(ValueError, 'no allowed scene change'): + direct.validate_edit(before, after, policy) + + def test_present_orphan_content_remains_audited(self): + before, after, policy = self.fixture() + after['audit']['preserve']['materials']['orphan'] = 'changed' + with self.assertRaisesRegex(ValueError, 'forbidden scene edits'): + direct.validate_edit(before, after, policy) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_direct_suite.py b/tests/test_direct_suite.py index 14777d2..e74d0c2 100644 --- a/tests/test_direct_suite.py +++ b/tests/test_direct_suite.py @@ -139,6 +139,16 @@ def test_retry_process_waits_without_work_and_honors_stop(self): process.kill() process.communicate() + def test_restart_selection_is_unique_canonical_and_excludes_camera1(self): + import direct_suite as suite + tasks = {'level1/camera1': {}} + tasks.update({f'level2/task{i}': {} for i in range(26)}) + selected = ['level2/task3', 'level2/task4'] + self.assertEqual(suite.task_plan(tasks, selected), selected) + for invalid in [['unknown'], ['level1/camera1'], [selected[0]] * 2, []]: + with self.assertRaises(ValueError): + suite.task_plan(tasks, invalid) + def test_plan_excludes_only_completed_camera1(self): script = TOOLS / 'direct_suite.py' self.assertTrue(script.exists(), 'suite dispatcher missing') diff --git a/tools/direct_codex.py b/tools/direct_codex.py index 83ffa8f..2e1c1ed 100644 --- a/tools/direct_codex.py +++ b/tools/direct_codex.py @@ -58,6 +58,18 @@ def validate_edit(before, after, policy): import math left = copy.deepcopy(before.get('audit', before)) right = copy.deepcopy(after.get('audit', after)) + # Only disappearance of an independently evidenced, local, zero-user + # material is normal save/reopen cleanup. Keep present material contents, + # all additions and every object/slot/geometry/light/camera audit exact. + materials = left.get('preserve', {}).get('materials', {}) + saved_materials = right.get('preserve', {}).get('materials', {}) + for name in set(materials) - set(saved_materials): + lifecycle = before.get('material_lifecycle', {}).get(name, {}) + if (type(lifecycle.get('users')) is int and lifecycle['users'] == 0 + and lifecycle.get('use_fake_user') is False + and lifecycle.get('use_extra_user') is False + and 'library' in lifecycle and lifecycle['library'] is None): + del materials[name] changed = False if policy.get('unsupported_blocker'): raise ValueError('unsupported task policy') diff --git a/tools/direct_runtime.py b/tools/direct_runtime.py index 43f298b..ed2a071 100644 --- a/tools/direct_runtime.py +++ b/tools/direct_runtime.py @@ -215,6 +215,13 @@ def tree(t): return {'properties':props(t), 'nodes':[{ 'properties':props(n), 'inputs':[props(i) for i in n.inputs], 'outputs':[props(i) for i in n.outputs]} for n in t.nodes], 'links':[(l.from_node.name,l.from_socket.identifier,l.to_node.name,l.to_socket.identifier) for l in t.links]} +def material_lifecycle(): + # Evidence for normal save cleanup, never add fake users just for auditing. + return {m.name: {'users':m.users, 'use_fake_user':bool(m.use_fake_user), + 'use_extra_user':bool(m.use_extra_user), + 'library':m.library.filepath if m.library else None} + for m in bpy.data.materials} + def audit(): objects={} for o in bpy.data.objects: @@ -279,7 +286,7 @@ def audit(): if cfg.get('save'): bpy.context.preferences.filepaths.save_version=0 bpy.ops.wm.save_as_mainfile(filepath=cfg['save'], relative_remap=True) -record={'audit':audit(),'evidence':evidence, +record={'audit':audit(),'evidence':evidence,'material_lifecycle':material_lifecycle(), 'settings':{'engine':s.render.engine,'samples':s.cycles.samples, 'resolution_x':s.render.resolution_x,'resolution_y':s.render.resolution_y, 'resolution_percentage':s.render.resolution_percentage}} diff --git a/tools/direct_saved_validation.md b/tools/direct_saved_validation.md new file mode 100644 index 0000000..6162346 --- /dev/null +++ b/tools/direct_saved_validation.md @@ -0,0 +1,47 @@ +# Direct suite: live edits and saved-deliverable validation + +User-approved workflow supersedes the former independent reopen admission gate. +Initialization still verifies pinned inputs, applies reviewed literal task initialization, +saves a protected initial blend, records the full native RNA baseline, and renders a +CUDA-only PNG which the parent decodes. Generation starts without independently +reopening and comparing that initialized file first. + +The deployed standalone MCP backend is **not a persistent Blender session**: +`blender_cli.py` launches `bpy_cli_runner.py` as a subprocess per request, and its +`main()` opens `blend_file` before executing code. Direct Codex continues edits from +the previous saved checkpoint across calls and edits the open scene within each call. +Unsaved memory cannot persist across these calls. This change does not introduce a +persistent server, replace the backend, or change the stock 120-second code deadline. + +After all ten direct meaningful rounds, the parent independently reopens/renders +saved deliverables, validates each against the initialization baseline and prior +checkpoint, checks distinct allowed state, and then performs GPU CLIP scoring. +No oracle/goal scripts or score feedback are given to generation. The exact four +MCP tools and all five task policy families remain unchanged. + +## Narrow cleanup exception + +The native audit records material lifecycle evidence separately from content hashes. +Only a material **missing from a later saved audit** can be excused, and only when +the earlier independent evidence records integer `users == 0`, `use_fake_user == false`, +`use_extra_user == false`, and `library == null`. No fake users are added for auditing. +Missing evidence fails closed. Present orphan material contents, new materials, +assigned materials, all object/material slot references, geometry, cameras, lights, +worlds, scenes and other datablocks remain audited exactly. Cleanup alone never +counts as a meaningful edit. No other datablock class is exempted without evidence. + +`tests/test_direct_saved_cleanup.py` exercises accepted orphan loss, rejected assigned +material changes/deletions, protected/missing lifecycle evidence, other forbidden +changes, no-op rejection, the actual worker lifecycle serializer, and generation +reaching Codex without any independent pre-generation reopen call. + +## Authorized restarts + +`direct_suite.py --output FRESH_ROOT --tasks TASK_ID ...` permits an explicit unique +canonical subset, never camera1. Operators must verify the canonical manifest, +completed results and artifacts, stop the preceding dispatcher and retry through +STOP, verify both processes and descendants exited, then construct the unfinished +subset. Retain previous roots including failures. The unchanged shared +`direct-suite.lock` serializes the new dispatcher through initialization, generation, +validation and scoring. Keep process.json, events.jsonl, status.json and launcher +stdout/stderr as durable handles. A running task is not a completed result. diff --git a/tools/direct_suite.py b/tools/direct_suite.py index d155cd5..7a80b74 100644 --- a/tools/direct_suite.py +++ b/tools/direct_suite.py @@ -21,10 +21,14 @@ 'allowed_shape_keys', 'location_bounds') -def task_plan(tasks): +def task_plan(tasks, selected=None): plan = [t for t in tasks if t != 'level1/camera1'] if len(tasks) != 27 or len(plan) != 26: raise ValueError('expected 27 canonical tasks, 26 remaining') + if selected is not None: + if not selected or len(set(selected)) != len(selected) or set(selected) - set(plan): + raise ValueError('restart tasks must be unique canonical unfinished task IDs, excluding camera1') + return list(selected) return plan @@ -42,10 +46,10 @@ def run_task(task, policy, out, config, dataset): work = out / 'work' blend = work / 'initial.blend' Path(initial['blend']).rename(blend) - baseline = runtime.inspect_render(blend, policy, runtime.CUSTOM_BPY_PYTHON, out / 'admission', events) - reopened = runtime.inspect_render(blend, policy, runtime.CUSTOM_BPY_PYTHON, out / 'reopen', events, render=False) - if initial['audit'] != baseline['audit'] or baseline['audit'] != reopened['audit']: - raise ValueError('initialization/reopen independent invariant mismatch') + # Generation follows the initialized scene, without a separate reopen gate. + # Saved checkpoints are independently audited after generation against this + # full in-memory baseline; only evidenced orphan material loss is allowed. + baseline = initial initial_hash = runtime.sha(blend) runtime.emit(events, 'native_admission_passed', task=task, source_sha256=initial_hash) args = SimpleNamespace(codex='/home/mg/.local/bin/codex', model='gpt-6-astra', workdir=work) @@ -71,6 +75,10 @@ def run_task(task, policy, out, config, dataset): 'Use only execute_blender_code_for_cli, get_runtime_python_api_docs_for_cli, search_api_docs, ' 'get_python_api_docs for scene work; no shell scene operations. ' 'The backend code-call deadline remains 120 seconds; the MCP client deadline remains 86400 seconds. ' + 'This standalone backend starts a new bpy subprocess and opens blend_file on every call; ' + 'unsaved memory does NOT persist across calls. Continue edits from the previous saved checkpoint ' + '(initial.blend only for the first edit); save intermediate work before a call returns. ' + 'Within a call, edit the open scene continuously without unnecessary reloads. ' 'Use CUDA rendering, enable CUDA devices only and disable CPU; retain engine, samples, resolution ' 'and all other saved scene settings. Save scene BEFORE transient PNG output setting changes; ' 'never save the render filepath/encoding overrides. Disable backup save_version. ' @@ -120,6 +128,7 @@ def run_task(task, policy, out, config, dataset): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--tasks', nargs='+', help='explicit user-authorized fresh restart subset') args = parser.parse_args() out = args.output.absolute() out.mkdir(parents=True, exist_ok=True) @@ -128,7 +137,7 @@ def main(): sys.path.insert(0, str(ROOT / 'input-tools')) import dataset tasks = json.loads((ROOT / 'input-tools/task-policies.json').read_text())['tasks'] - plan = task_plan(tasks) + plan = task_plan(tasks, args.tasks) if set(tasks) != set(dataset.task_ids()): raise ValueError('canonical policy/manifest task mismatch') config = direct.prepare_config(tomllib.loads((ROOT / 'mghavn-custom-bpy.toml').read_text()), approved=True) From 4a9f5cc6e616080185252bb919fa3d1e730760b7 Mon Sep 17 00:00:00 2001 From: Michael Gold Date: Thu, 10 Sep 2026 21:07:53 -0400 Subject: [PATCH 5/6] Remove local edit restrictions and recover saved direct-suite checkpoints --- tests/test_direct_codex.py | 16 +++---- tests/test_direct_recovery.py | 37 ++++++++++++++++ tests/test_direct_saved_cleanup.py | 33 +++++++------- tests/test_direct_unrestricted.py | 36 +++++++++++++++ tools/direct_codex.py | 70 ++++++------------------------ tools/direct_suite.py | 52 +++++++++++++++------- 6 files changed, 146 insertions(+), 98 deletions(-) create mode 100644 tests/test_direct_recovery.py create mode 100644 tests/test_direct_unrestricted.py diff --git a/tests/test_direct_codex.py b/tests/test_direct_codex.py index 96d19c6..461986f 100644 --- a/tests/test_direct_codex.py +++ b/tests/test_direct_codex.py @@ -138,7 +138,7 @@ def test_cli_denied_preflight_stops_run_even_with_exit_zero(self): self.assertIn('incomplete checkpoint artifacts', proc.stderr) self.assertTrue((root / 'logs2' / 'run.jsonl').is_file()) - def test_task_policy_supports_non_camera_and_rejects_forbidden(self): + def test_task_policy_is_ignored_but_noop_is_rejected(self): import copy m = self.load() self.assertTrue(hasattr(m, 'validate_edit'), 'task policy validator missing') @@ -149,14 +149,13 @@ def test_task_policy_supports_non_camera_and_rejects_forbidden(self): policy = {'allowed_transforms': {'Cube': ['location']}, 'location_bounds': {'Cube': [[-2, 2]] * 3}} m.validate_edit(before, after, policy) - for bad in (before, {**after, 'preserve': {'mesh': 'changed'}}): - with self.assertRaises(ValueError): - m.validate_edit(before, bad, policy) - after['objects']['Cube']['location'] = [3, 0, 0] with self.assertRaises(ValueError): - m.validate_edit(before, after, policy) + m.validate_edit(before, before, policy) + m.validate_edit(before, {**after, 'preserve': {'mesh': 'changed'}}, policy) + after['objects']['Cube']['location'] = [3, 0, 0] + m.validate_edit(before, after, policy) - def test_all_exact_property_families(self): + def test_all_property_families_and_geometry_are_editable(self): import copy m = self.load() before = {'objects': {'Cam': {'type': 'CAMERA', 'lens': 50}, @@ -170,8 +169,7 @@ def test_all_exact_property_families(self): mutate(after) m.validate_edit(before, after, policy) after['preserve']['mesh'] = 'changed' - with self.assertRaises(ValueError): - m.validate_edit(before, after, policy) + m.validate_edit(before, after, policy) after = copy.deepcopy(before) after['objects']['Cam']['lens'] = float('nan') with self.assertRaises(ValueError): diff --git a/tests/test_direct_recovery.py b/tests/test_direct_recovery.py new file mode 100644 index 0000000..00fd895 --- /dev/null +++ b/tests/test_direct_recovery.py @@ -0,0 +1,37 @@ +import json +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools')) +import direct_suite as suite + +class RecoveryTests(unittest.TestCase): + def test_recovery_scores_existing_work_without_generation(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + old = root / 'old' + work = old / 'work' + work.mkdir(parents=True) + (work / 'initial.blend').write_bytes(b'original') + baseline = {'blend_sha256': suite.runtime.sha(work / 'initial.blend'), 'audit': {}} + (work / 'metadata.json').write_text(json.dumps(baseline)) + dataset = SimpleNamespace(verify=lambda *a: {'verified': True}) + with patch.object(suite, 'ROOT', root), patch.object(suite.direct, 'invoke', side_effect=AssertionError('regeneration')), patch.object(suite, 'score_checkpoints') as score: + suite.recover_task('level2/attribute1', {}, root / 'new', old, dataset) + self.assertEqual(score.call_args.args[2], work) + self.assertTrue((root / 'new/recovery.json').is_file()) + + def test_recovery_rejects_modified_initial(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + work = root / 'old/work' + work.mkdir(parents=True) + (work / 'initial.blend').write_bytes(b'changed') + (work / 'metadata.json').write_text(json.dumps({'blend_sha256': 'original'})) + with self.assertRaisesRegex(ValueError, 'initial.blend'): + suite.recover_task('level2/attribute1', {}, root / 'new', work.parent, SimpleNamespace(verify=lambda *a: {})) + +if __name__ == '__main__': unittest.main() diff --git a/tests/test_direct_saved_cleanup.py b/tests/test_direct_saved_cleanup.py index a236865..d47c001 100644 --- a/tests/test_direct_saved_cleanup.py +++ b/tests/test_direct_saved_cleanup.py @@ -1,4 +1,4 @@ -"""Saved delivery audits tolerate only evidenced orphan material removal.""" +"""Unrestricted saved edits; orphan cleanup alone is not a round.""" import copy import sys import unittest @@ -54,9 +54,13 @@ def initialize(inp, policy, python, output, events): def invoke(args, config, evidence, label, prompt, target=None): if label == 'run': self.assertIn('previous saved checkpoint', prompt) + self.assertIn('All scene properties may be edited', prompt) + self.assertNotIn('unlisted', prompt) + self.assertNotIn('allowed edit', prompt) + self.assertIn('goal code', prompt) raise GenerationReached() return [] - policy = {f: {} for f in suite.FIELDS} + policy = {} policy['task_description'] = 'test' with patch.object(suite, 'ROOT', root), \ patch.object(suite.runtime, 'initialize', initialize), \ @@ -73,47 +77,42 @@ def test_zero_user_unused_material_loss_is_accepted(self): direct.validate_edit(before, after, policy) self.assertEqual(before, original) - def test_used_material_change_or_loss_is_rejected(self): + def test_used_material_change_or_loss_is_accepted(self): for value in ('changed-hash', None): before, after, policy = self.fixture() if value is None: del after['audit']['preserve']['materials']['used'] else: after['audit']['preserve']['materials']['used'] = value - with self.assertRaisesRegex(ValueError, 'forbidden scene edits'): - direct.validate_edit(before, after, policy) + direct.validate_edit(before, after, policy) - def test_missing_lifecycle_and_protected_orphans_fail_closed(self): + def test_material_edits_need_no_lifecycle_permission(self): for field, value in [('users', 1), ('use_fake_user', True), ('use_extra_user', True), ('library', '//linked.blend'), ('users', None)]: before, after, policy = self.fixture() before['material_lifecycle']['orphan'][field] = value - with self.assertRaisesRegex(ValueError, 'forbidden scene edits'): - direct.validate_edit(before, after, policy) + direct.validate_edit(before, after, policy) before, after, policy = self.fixture() del before['material_lifecycle'] - with self.assertRaisesRegex(ValueError, 'forbidden scene edits'): - direct.validate_edit(before, after, policy) + direct.validate_edit(before, after, policy) - def test_geometry_lighting_and_material_additions_still_fail(self): + def test_geometry_lighting_and_material_additions_are_accepted(self): for collection, name in [('meshes', 'mesh'), ('scenes', 'Scene'), ('materials', 'new')]: before, after, policy = self.fixture() after['audit']['preserve'][collection][name] = 'changed' - with self.assertRaisesRegex(ValueError, 'forbidden scene edits'): - direct.validate_edit(before, after, policy) + direct.validate_edit(before, after, policy) def test_cleanup_alone_is_not_a_meaningful_round(self): before, after, policy = self.fixture() after['audit']['objects']['Camera']['lens'] = 50 - with self.assertRaisesRegex(ValueError, 'no allowed scene change'): + with self.assertRaisesRegex(ValueError, 'no meaningful scene change'): direct.validate_edit(before, after, policy) - def test_present_orphan_content_remains_audited(self): + def test_present_orphan_content_may_change(self): before, after, policy = self.fixture() after['audit']['preserve']['materials']['orphan'] = 'changed' - with self.assertRaisesRegex(ValueError, 'forbidden scene edits'): - direct.validate_edit(before, after, policy) + direct.validate_edit(before, after, policy) if __name__ == '__main__': diff --git a/tests/test_direct_unrestricted.py b/tests/test_direct_unrestricted.py new file mode 100644 index 0000000..437d243 --- /dev/null +++ b/tests/test_direct_unrestricted.py @@ -0,0 +1,36 @@ +"""User correction: edit validity is not a task allowlist.""" +import copy +import sys +import unittest +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools')) +import direct_codex as direct + +class UnrestrictedTests(unittest.TestCase): + def test_all_scene_edits_ignore_local_policy(self): + before = {'objects': {'Cube': {'location': [0, 0, 0]}}, + 'shape_keys': {'Key': {'Smile': 0}}, + 'preserve': {'materials': {'used': 'old'}, 'meshes': {'mesh': 'old'}}} + mutations = [ + lambda a: a['objects'].update(New={'type': 'LIGHT', 'energy': 300}), + lambda a: a['objects'].pop('Cube'), + lambda a: a['objects']['Cube'].update(location=[999, 0, 0]), + lambda a: a['objects'].update(Camera={'lens': 60}), + lambda a: a['shape_keys']['Key'].update(Smile=1), + lambda a: a['preserve']['materials'].update(used='new'), + lambda a: a['preserve']['meshes'].update(mesh='new'), + ] + for mutate in mutations: + after = copy.deepcopy(before) + mutate(after) + direct.validate_edit(before, after, {'unsupported_blocker': True, + 'location_bounds': {'Cube': [[-1, 1]] * 3}}) + + def test_nonfinite_and_noop_still_invalid(self): + before = {'objects': {'Cube': {'location': [0, 0, 0]}}} + with self.assertRaisesRegex(ValueError, 'no meaningful scene change'): + direct.validate_edit(before, before, {}) + after = copy.deepcopy(before) + after['objects']['Cube']['location'][0] = float('nan') + with self.assertRaisesRegex(ValueError, 'finite'): + direct.validate_edit(before, after, {}) diff --git a/tools/direct_codex.py b/tools/direct_codex.py index 2e1c1ed..e13b7cf 100644 --- a/tools/direct_codex.py +++ b/tools/direct_codex.py @@ -54,13 +54,11 @@ def verify_preflight(events, nonce, python): def validate_edit(before, after, policy): - """Compare independent RNA audits, masking only exact task allowances.""" + """Require finite, meaningful saved changes; local edit policy is ignored.""" import math left = copy.deepcopy(before.get('audit', before)) right = copy.deepcopy(after.get('audit', after)) - # Only disappearance of an independently evidenced, local, zero-user - # material is normal save/reopen cleanup. Keep present material contents, - # all additions and every object/slot/geometry/light/camera audit exact. + # Evidenced orphan cleanup alone is not a meaningful editing round. materials = left.get('preserve', {}).get('materials', {}) saved_materials = right.get('preserve', {}).get('materials', {}) for name in set(materials) - set(saved_materials): @@ -70,59 +68,17 @@ def validate_edit(before, after, policy): and lifecycle.get('use_extra_user') is False and 'library' in lifecycle and lifecycle['library'] is None): del materials[name] - changed = False - if policy.get('unsupported_blocker'): - raise ValueError('unsupported task policy') - - def remove(a, b, field, width=None): - nonlocal changed - av, bv = a[field], b[field] - for value in (av, bv): - values = value if isinstance(value, list) else [value] - if width is not None and len(values) != width: - raise ValueError('invalid property dimensions') - if not all(type(v) in (int, float) and math.isfinite(v) for v in values): - raise ValueError('nonfinite or nonnumeric allowed property') - changed |= av != bv - del a[field] - del b[field] - - try: - for name, fields in policy.get('allowed_transforms', {}).items(): - if not fields or len(set(fields)) != len(fields) or set(fields) - {'location', 'rotation_euler', 'scale'}: - raise ValueError('forbidden transform policy') - for field in fields: - remove(left['objects'][name], right['objects'][name], field, 3) - for name, fields in policy.get('allowed_camera_data', {}).items(): - if fields != ['lens']: - raise ValueError('forbidden camera data policy') - a, b = left['objects'][name], right['objects'][name] - if a['type'] != 'CAMERA' or b['type'] != 'CAMERA': - raise ValueError('lens allowance requires camera') - remove(a, b, 'lens', 1) - for name, fields in policy.get('allowed_light_data', {}).items(): - if not fields or len(set(fields)) != len(fields) or set(fields) - {'energy', 'color'}: - raise ValueError('forbidden light policy') - a, b = left['objects'][name], right['objects'][name] - if a['type'] != 'LIGHT' or b['type'] != 'LIGHT': - raise ValueError('light allowance requires light') - for field in fields: - remove(a['light_data'], b['light_data'], field, 3 if field == 'color' else 1) - for name, fields in policy.get('allowed_shape_keys', {}).items(): - if not fields or len(set(fields)) != len(fields): - raise ValueError('invalid shape key policy') - for field in fields: - remove(left['shape_keys'][name], right['shape_keys'][name], field, 1) - for name, bounds in policy.get('location_bounds', {}).items(): - location = after.get('audit', after)['objects'][name]['location'] - if len(location) != 3 or len(bounds) != 3 or any(not lo <= x <= hi for x, (lo, hi) in zip(location, bounds)): - raise ValueError('location outside bounds') - except (KeyError, TypeError) as exc: - raise ValueError('missing or malformed policy property') from exc - if left != right: - raise ValueError('forbidden scene edits outside task policy') - if not changed: - raise ValueError('no allowed scene change') + def finite(value): + if isinstance(value, dict): + return all(finite(v) for v in value.values()) + if isinstance(value, (list, tuple)): + return all(finite(v) for v in value) + return not isinstance(value, float) or math.isfinite(value) + + if not finite(left) or not finite(right): + raise ValueError('nonfinite scene property') + if left == right: + raise ValueError('no meaningful scene change') def verify_checkpoints(root, rounds, policy=None): diff --git a/tools/direct_suite.py b/tools/direct_suite.py index 7a80b74..1a1eac2 100644 --- a/tools/direct_suite.py +++ b/tools/direct_suite.py @@ -17,8 +17,7 @@ import direct_runtime as runtime ROOT = Path('/home/mg/blenderbench-direct') -FIELDS = ('allowed_transforms', 'allowed_camera_data', 'allowed_light_data', - 'allowed_shape_keys', 'location_bounds') + def task_plan(tasks, selected=None): @@ -36,8 +35,7 @@ def run_task(task, policy, out, config, dataset): inp = ROOT / 'dataset' / task verification = dataset.verify(task, inp) policy = dict(policy, task=task, pin_verified=True, stop_file=str(out.parent / 'STOP')) - if any(field not in policy for field in FIELDS): - raise ValueError('missing exact task policy fields') + out.mkdir(parents=True, exist_ok=False) (out / 'input-verification.json').write_text(json.dumps(verification, indent=2)) events = out / 'events.jsonl' @@ -47,8 +45,7 @@ def run_task(task, policy, out, config, dataset): blend = work / 'initial.blend' Path(initial['blend']).rename(blend) # Generation follows the initialized scene, without a separate reopen gate. - # Saved checkpoints are independently audited after generation against this - # full in-memory baseline; only evidenced orphan material loss is allowed. + # Saved checkpoints are independently reopened after generation for integrity. baseline = initial initial_hash = runtime.sha(blend) runtime.emit(events, 'native_admission_passed', task=task, source_sha256=initial_hash) @@ -66,12 +63,11 @@ def run_task(task, policy, out, config, dataset): runtime.emit(events, 'preflight_passed', task=task) contract = ('Solve this BlenderBench task by direct use of the four Blender MCP tools.\n' + policy['task_description'] + '\nTarget image is attached. Initial scene: ' + str(blend) - + '\nExact mutable-property policy (all unlisted scene data must remain unchanged):\n' - + json.dumps({f: policy[f] for f in FIELDS}, indent=2) + + '\nAll scene properties may be edited, including objects, geometry, materials, lights and cameras.\n' + '\nPerform exactly 10 meaningful edit/render/visual-inspection rounds sequentially. ' 'Inspect initial scene and image, reason yourself about edits, render and inspect each checkpoint ' 'using view_image. Do not delegate reasoning to scripts/controllers. No CLIP/scoring, VLM judge, ' - 'oracle/goal .blend access, or score feedback. Do not read audit or evaluation files. ' + 'oracle/goal code or goal .blend access, or score feedback. Do not read audit or evaluation files. ' 'Use only execute_blender_code_for_cli, get_runtime_python_api_docs_for_cli, search_api_docs, ' 'get_python_api_docs for scene work; no shell scene operations. ' 'The backend code-call deadline remains 120 seconds; the MCP client deadline remains 86400 seconds. ' @@ -79,11 +75,10 @@ def run_task(task, policy, out, config, dataset): 'unsaved memory does NOT persist across calls. Continue edits from the previous saved checkpoint ' '(initial.blend only for the first edit); save intermediate work before a call returns. ' 'Within a call, edit the open scene continuously without unnecessary reloads. ' - 'Use CUDA rendering, enable CUDA devices only and disable CPU; retain engine, samples, resolution ' - 'and all other saved scene settings. Save scene BEFORE transient PNG output setting changes; ' + 'Use Cycles CUDA rendering, enable CUDA devices only and disable CPU. Save scene BEFORE transient PNG output setting changes; ' 'never save the render filepath/encoding overrides. Disable backup save_version. ' - 'Never alter initial.blend. Do not create/delete objects or alter unlisted properties. ' - 'Every round must be a distinct allowed edit, not inspection/no-op or a repeat of any earlier scene state. ' + 'Never alter initial.blend, source dataset files or target image. ' + 'Every round must be a distinct meaningful edit, not inspection/no-op or a repeat of any earlier scene state. ' 'Save absolute paths ' + str(work / 'iteration01.blend') + ' and ' + str(work / 'iteration01.png') + ' through iteration10.blend and iteration10.png. ' 'Append rounds.jsonl in this work directory via MCP, one JSON object per completed round: ' @@ -95,6 +90,28 @@ def run_task(task, policy, out, config, dataset): direct.invoke(args, config, evidence, 'run', contract, inp / 'target.png') if runtime.sha(blend) != initial_hash: raise ValueError('Codex modified initial.blend') + dataset.verify(task, inp) + score_checkpoints(task, policy, work, out, baseline) + + +def recover_task(task, policy, out, old, dataset): + """Score retained ten-round delivery in place; never regenerate or rewrite it.""" + work = old / 'work' + baseline = json.loads((work / 'metadata.json').read_text()) + if runtime.sha(work / 'initial.blend') != baseline['blend_sha256']: + raise ValueError('modified initial.blend in recovery') + verification = dataset.verify(task, ROOT / 'dataset' / task) + out.mkdir(parents=True, exist_ok=False) + (out / 'input-verification.json').write_text(json.dumps(verification, indent=2)) + (out / 'recovery.json').write_text(json.dumps({'source': str(old), 'regenerated': False})) + policy = dict(policy, task=task, stop_file=str(out.parent / 'STOP')) + score_checkpoints(task, policy, work, out, baseline) + + +def score_checkpoints(task, policy, work, out, baseline): + """Validate and score saved artifacts only after generation has ended.""" + events = out / 'events.jsonl' + inp = ROOT / 'dataset' / task artifacts = direct.verify_checkpoints(work, 10, policy=policy) (out / 'artifact-check.json').write_text(json.dumps(artifacts, indent=2)) runtime.emit(events, 'generation_finished', task=task) @@ -109,7 +126,7 @@ def run_task(task, policy, out, config, dataset): direct.validate_edit(previous, audit, policy) state = json.dumps(audit['audit'], sort_keys=True) if state in seen: - raise ValueError('repeated allowed scene state') + raise ValueError('repeated scene state') seen.add(state) previous = audit score_dir = out / f'score{n:02}' @@ -129,6 +146,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--tasks', nargs='+', help='explicit user-authorized fresh restart subset') + parser.add_argument('--recover-root', type=Path, help='reuse complete ledgers from a stopped predecessor') args = parser.parse_args() out = args.output.absolute() out.mkdir(parents=True, exist_ok=True) @@ -181,7 +199,11 @@ def status(task, state, **extra): continue status(task, 'running', output=str(task_out)) try: - run_task(task, tasks[task], task_out, config, dataset) + old = args.recover_root / task.replace('/', '-') if args.recover_root else None + if old is not None and (old / 'work/rounds.jsonl').is_file(): + recover_task(task, tasks[task], task_out, old, dataset) + else: + run_task(task, tasks[task], task_out, config, dataset) except Exception as exc: status(task, 'failed', error=repr(exc), output=str(task_out)) else: From 582c4b6849f353bdf84b35df2d2129d4af3bad6b Mon Sep 17 00:00:00 2001 From: Michael Gold Date: Fri, 11 Sep 2026 16:06:29 -0400 Subject: [PATCH 6/6] [verified] feat: publish reproducible BlenderBench suite --- .gitattributes | 2 + .github/workflows/blenderbench-portable.yml | 44 + .gitignore | 4 + benchmarks/blenderbench_direct/README.md | 223 + .../blenderbench_direct/TDD_EVIDENCE.md | 64 + .../blenderbench_direct/VIGA-LICENSE.txt | 21 + benchmarks/blenderbench_direct/__init__.py | 3 + .../blenderbench_direct/assets-manifest.json | 1037 +++++ .../clip-snapshot-manifest.json | 57 + benchmarks/blenderbench_direct/dataset.py | 209 + .../initialization-manifest.json | 1944 ++++++++ benchmarks/blenderbench_direct/protocol.py | 262 ++ .../publication/artifact-manifest.json | 3943 +++++++++++++++++ .../publication/reference-results.json | 347 ++ .../publication/results-summary.json | 242 + .../blenderbench_direct/ref_based_eval.py | 404 ++ .../requirements-fixture.in | 5 + .../requirements-fixture.lock | 91 + .../requirements-scoring.in | 47 + .../requirements-scoring.lock | 933 ++++ benchmarks/blenderbench_direct/runner.py | 159 + benchmarks/blenderbench_direct/score_pair.py | 76 + benchmarks/blenderbench_direct/scoring.py | 436 ++ benchmarks/blenderbench_direct/suite.py | 303 ++ benchmarks/blenderbench_direct/video.py | 107 + benchmarks/token_efficiency/codex_runner.py | 2 +- benchmarks/token_efficiency/creative_smoke.py | 2 +- mcp/blmcp/tools/execute_blender_code.py | 119 +- readme.md | 11 +- tests/test_blender_cli.py | 86 + tests/test_blenderbench_publication.py | 746 ++++ tests/test_direct_codex.py | 203 +- tests/test_direct_recovery.py | 42 +- tests/test_direct_saved_cleanup.py | 32 - tests/test_direct_suite.py | 83 +- tests/test_tool_listing.py | 370 +- tools/DIRECT_SUITE.md | 26 +- tools/direct_camera_retry.md | 43 - tools/direct_camera_retry.py | 94 - tools/direct_codex.example.toml | 2 +- tools/direct_codex.md | 142 +- tools/direct_codex.py | 204 +- tools/direct_runtime.py | 26 +- tools/direct_saved_validation.md | 47 - tools/direct_suite.py | 220 +- 45 files changed, 12591 insertions(+), 872 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/blenderbench-portable.yml create mode 100644 benchmarks/blenderbench_direct/README.md create mode 100644 benchmarks/blenderbench_direct/TDD_EVIDENCE.md create mode 100644 benchmarks/blenderbench_direct/VIGA-LICENSE.txt create mode 100644 benchmarks/blenderbench_direct/__init__.py create mode 100644 benchmarks/blenderbench_direct/assets-manifest.json create mode 100644 benchmarks/blenderbench_direct/clip-snapshot-manifest.json create mode 100644 benchmarks/blenderbench_direct/dataset.py create mode 100644 benchmarks/blenderbench_direct/initialization-manifest.json create mode 100644 benchmarks/blenderbench_direct/protocol.py create mode 100644 benchmarks/blenderbench_direct/publication/artifact-manifest.json create mode 100644 benchmarks/blenderbench_direct/publication/reference-results.json create mode 100644 benchmarks/blenderbench_direct/publication/results-summary.json create mode 100644 benchmarks/blenderbench_direct/ref_based_eval.py create mode 100644 benchmarks/blenderbench_direct/requirements-fixture.in create mode 100644 benchmarks/blenderbench_direct/requirements-fixture.lock create mode 100644 benchmarks/blenderbench_direct/requirements-scoring.in create mode 100644 benchmarks/blenderbench_direct/requirements-scoring.lock create mode 100644 benchmarks/blenderbench_direct/runner.py create mode 100644 benchmarks/blenderbench_direct/score_pair.py create mode 100644 benchmarks/blenderbench_direct/scoring.py create mode 100644 benchmarks/blenderbench_direct/suite.py create mode 100644 benchmarks/blenderbench_direct/video.py create mode 100644 tests/test_blenderbench_publication.py delete mode 100644 tools/direct_camera_retry.md delete mode 100644 tools/direct_camera_retry.py delete mode 100644 tools/direct_saved_validation.md diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..70ace00 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Preserve the hash-verified upstream evaluator byte-for-byte. +benchmarks/blenderbench_direct/ref_based_eval.py whitespace=-blank-at-eol diff --git a/.github/workflows/blenderbench-portable.yml b/.github/workflows/blenderbench-portable.yml new file mode 100644 index 0000000..16e23a8 --- /dev/null +++ b/.github/workflows/blenderbench-portable.yml @@ -0,0 +1,44 @@ +name: BlenderBench portable unit tests + +on: + push: + paths: + - "benchmarks/blenderbench_direct/**" + - "tools/direct_*.py" + - "tests/test_blenderbench_publication.py" + - "tests/test_direct_*.py" + - ".github/workflows/blenderbench-portable.yml" + pull_request: + paths: + - "benchmarks/blenderbench_direct/**" + - "tools/direct_*.py" + - "tests/test_blenderbench_publication.py" + - "tests/test_direct_*.py" + - ".github/workflows/blenderbench-portable.yml" + +permissions: + contents: read + +jobs: + unit: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.13" + check-latest: false + - name: Install minimal hash-locked fixture environment + run: >- + python -m pip install --disable-pip-version-check + --require-hashes -r benchmarks/blenderbench_direct/requirements-fixture.lock + - name: Run CPU-only fixture tests + run: >- + python -m unittest -v + tests.test_blenderbench_publication + tests.test_direct_codex + tests.test_direct_recovery + tests.test_direct_saved_cleanup + tests.test_direct_suite + tests.test_direct_unrestricted diff --git a/.gitignore b/.gitignore index 08d11b1..26ad8f5 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ htmlcov/ /.env.* !/.env.example !/.env.template + +# BlenderBench data, model evidence, and publication artifacts stay external. +/blenderbench-data/ +/blenderbench-results/ diff --git a/benchmarks/blenderbench_direct/README.md b/benchmarks/blenderbench_direct/README.md new file mode 100644 index 0000000..635fe39 --- /dev/null +++ b/benchmarks/blenderbench_direct/README.md @@ -0,0 +1,223 @@ +# Portable full-27 BlenderBench runner + +This package runs the 27 public BlenderBench tasks with a model selected at the +CLI. It is an external sequential dispatcher, not a reasoning controller. One +direct Codex session per task receives exactly four Blender MCP tools: + +- `execute_blender_code_for_cli` +- `get_render_as_image_for_cli` +- `get_runtime_python_api_docs_for_cli` +- `search_api_docs` + +`get_render_as_image_for_cli` independently reopens a saved checkpoint, renders +the exact on-disk bytes with CUDA-only Cycles in the configured CLI backend, and +returns actual PNG image content plus the source SHA-256 through MCP for each +required visual-inspection round. Each edit uses `expected_output_blend`; the MCP +server rejects pre-existing or unchanged output and reports server-computed +source/output hashes. Codex's built-in `view_image` remains disabled. + +The frozen publication protocol requests exactly 10 sequential, meaningful, +saved edit/render/visual-inspection rounds per task. Generation has no shell +scene operations, goal/solution code or goal-scene access, VLM judge, CLIP or +photometric feedback, or evaluator feedback. Saved scenes are opened, audited, +GPU-rendered, and scored only after generation exits. A stopped/failed run is +never regenerated or resumed automatically. + +## External provenance and attribution + +The input manifest pins the [BlenderBench dataset](https://huggingface.co/datasets/DietCoke4671/BlenderBench) +by DietCoke4671 and contributors at revision +`203e4d325e9438ca55b29bdfc4f6a90842d74e68`, under **CC BY 4.0**. Preserve +upstream and per-asset attribution when redistributing downloaded assets. The +repository stores only manifests; downloaded `.blend` files and images belong +outside Git. + +The metric wrapper extracts `clip_similarity` and `photometric_loss` from +[VIGA release](https://github.com/Fugtemypt123/VIGA-release) revision +`69cb8ef0651bf68124815682df4a0f8e8b57c141`. The hash-verified upstream +`ref_based_eval.py` is included under its MIT license; see +[`VIGA-LICENSE.txt`](VIGA-LICENSE.txt). The wrapper refuses evaluator bytes unless +SHA-256 is +`9ee08818aa858dccfff3efbb36cb9a98e8887f14e6168ad1a23731351cdeb3b5`. +CLIP is `openai/clip-vit-base-patch32` at revision +`3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268`. Scoring reports CLIP **image +embedding cosine similarity**, N-CLIP (`1 - cosine`), and photometric loss. + +## Public setup + +1. Install exactly `codex-cli 0.154.0` (upstream source revision + `6b9826e3aa83b1a5947db50f4332cb9c65f1b340`). The launcher ignores user + configuration, uses a read-only Codex sandbox, and disables the 0.154.0 + built-in shell, image, dynamic/code-mode, app/plugin, browser, computer-use, + and image-generation feature paths. MCP tools are forced onto the direct + surface (`omit_tools_from = ["deferred", "code_mode"]`). Exactly the four + listed Blender tools remain available; ordinary model reasoning is unchanged. +2. Install the MCP server and a GPU-capable standalone `bpy` Python. Record + `bpy.app.version_string`. The runtime requires Cycles CUDA, enables CUDA + devices only, disables CPU, and has no CPU-render fallback. +3. Copy `tools/direct_codex.example.toml` outside the repository and replace all + absolute placeholders. It must enable exactly the four tools above and name + the same bpy interpreter passed to the runner. +4. Create a separate exact CPython **3.13.15 x86_64 Linux / CUDA 12.8** scoring + environment and install the complete hash-locked dependency set with + `python -m pip install --require-hashes -r + benchmarks/blenderbench_direct/requirements-scoring.lock`. The lock pins the + `torch 2.8.0+cu128` CPython-3.13 wheel by immutable URL and hash and is the + normative reproduction environment. Enter this venv and invoke every `run`, + `recover`, and `score` command with that venv's `python`; external scorer + executable/script options are intentionally not accepted. Before generation + or scoring, an isolated child verifies lexical `sys.executable`, CPython and + venv identity, x86_64/64-bit/SOABI, exact imported runtime versions, CUDA, + and the exact 41-distribution lock (apart from allowed `pip`). + + Download the eight required CLIP files at the pinned revision into a dedicated + snapshot directory. `clip-snapshot-manifest.json` is the exact path/size/SHA-256 + allowlist. `--clip-snapshot` is verified before and after each scorer process; + offline/local-only loading rejects extras, missing files, byte changes, and + symlinks outside a standard Hugging Face model cache root. + + **Security boundary:** the historical lock intentionally retains + `setuptools 78.1.0` and `transformers 4.55.4` for exact environment fidelity. + A 2026-09-11 `pip-audit` reports 16 findings across those two distributions; + fixed versions would change the locked environment. The wrapper therefore + accepts only the pinned CLIP revision from an offline cache, requests + `weights_only=True`, and must not be used with untrusted model caches or + checkpoints. Any dependency upgrade requires a separately hashed evaluation + condition and numerical-equivalence validation. +5. Use external dataset and output directories. Keep credentials out of MCP TOML; + public manifests omit MCP `args`, `env`, and `variables` and redact recognized + secret-bearing keys and values recursively, but raw Codex evidence can still + contain sensitive model output and must be reviewed before publication. + +Download all 27 pinned tasks (or pass an explicit `--tasks` subset): + +```sh +python -m benchmarks.blenderbench_direct.runner download \ + --dataset-root /external/blenderbench-data +``` + +Run all 27 tasks by omitting `--tasks`: + +```sh +python -m benchmarks.blenderbench_direct.runner run \ + --dataset-root /external/blenderbench-data \ + --mcp-toml /external/blender-mcp.toml \ + --codex-binary /absolute/bin/codex \ + --codex-version 'EXACT CODEX --version OUTPUT' \ + --bpy-python /absolute/bpy-venv/bin/python \ + --bpy-version 'EXACT BPY VERSION PREFIX' \ + --clip-snapshot /external/clip-vit-base-patch32-snapshot \ + --output-root /external/fresh-run \ + --model 'PROVIDER/MODEL-ID' \ + --model-revision 'PROVIDER-REVISION-IF-KNOWN' \ + --rounds 10 --approve-blender-tools +``` + +An explicit subset uses the same fresh-run command plus, for example: + +```sh +--tasks level1/camera1 level2/attribute1 +``` + +`--rounds` defaults to 10 and accepts only 10. Every runtime path, requested +model, Codex pin, bpy pin, and CLIP snapshot is explicit. The output root must not +exist. Existing task output, checkpoint files, or run roots are never treated as +authorization to regenerate. + +## Recovery and standalone scoring + +Recovery is an explicit audit-and-scoring operation. It never calls Codex or +rewrites retained generation artifacts; it writes reopened-scene audit renders +and scores to a separate fresh output root. Before doing so it revalidates the +retained generation JSONL, attested checkpoint hashes, and ordered visual-image +evidence: + +```sh +python -m benchmarks.blenderbench_direct.runner recover \ + --source-root /external/retained-run --output-root /external/fresh-recovery \ + --dataset-root /external/blenderbench-data \ + --bpy-python /absolute/bpy-venv/bin/python --bpy-version 'EXACT BPY VERSION PREFIX' \ + --clip-snapshot /external/clip-vit-base-patch32-snapshot +``` + +Standalone scoring also never generates. It admits only the +`auditNN/verified.png` images produced by independent saved-scene reopen and GPU +rendering whose `audit-admission.json` records successful meaningful-change +validation and whose source, render, and full audit hashes match the retained +audit metadata. Model-written checkpoint PNGs are not scoring inputs: + +```sh +python -m benchmarks.blenderbench_direct.runner score \ + --run-root /external/retained-run \ + --dataset-root /external/blenderbench-data \ + --clip-snapshot /external/clip-vit-base-patch32-snapshot \ + --output /external/retained-run/scoring +``` + +`aggregate.json` and `aggregate.csv` include complete, failed, and missing tasks. +The JSON aggregate records the pinned dataset, evaluator, CLIP, and canonical +scorer digest. Every target is hash-verified before standalone scoring starts, +and every completed scorer output must repeat the exact evaluator and CLIP identity. +Best round is minimum N-CLIP; an exact tie selects the earliest round. Paired +photometric loss comes from that same round. Failures are not dropped from the +denominator. + +## Canonical protocol and artifacts + +`run-manifest.json` records resolved Codex/bpy/scorer versions, scorer SHA-256, +redacted effective config, exact tool list, dataset/evaluator/model revisions, +render and generation rules, initialization-manifest digest, implementation-file +digests, full Git revision and dirty state, and host/CPU/GPU/runtime provenance. Its +`protocol_sha256` is SHA-256 over canonical sorted UTF-8 JSON of the `protocol` +object; volatile hardware provenance is recorded but excluded from that hash. +The Git state and source digests are included: identical clean checkouts reproduce +the identity, while a dirty checkout is explicitly distinct and cannot claim clean +source provenance. Generation JSONL is checked after Codex exits and before any +audit or scoring; shell/file-change/builtin-tool operations, non-allowlisted MCP +calls, goal/solution access, and score/evaluator feedback fail the task closed. + +Large datasets, raw model captures, `.blend` files, renders, scores, and videos +stay outside Git. Publish immutable artifacts separately and add only verified +metadata to a report, for example: + +```json +{"url":"https://YOUR-ARTIFACT-HOST/path","sha256":"MEASURED-64-HEX-DIGEST"} +``` + +Do not replace placeholders with invented URLs or digests. Hash the uploaded +bytes, download by the public URL, hash again, and require equality before +claiming publication. + +## Compilation video + +Create a JSON spec with task-local `duration_seconds`, target/initial images, +model, optional final `task_tokens`, and round rows containing `round`, +`elapsed_seconds`, `image`, and `clip_image_cosine_similarity`. Then run: + +```sh +python -m benchmarks.blenderbench_direct.runner video \ + --spec /external/video-spec.json --output /external/compilation.mp4 \ + --ffmpeg /absolute/bin/ffmpeg +``` + +Overlays say “CLIP image cosine similarity” exactly. Token displays say +“Task-level tokens (linear interpolation)” explicitly; they are not event-level +usage and remain unavailable when task usage is unavailable. A sidecar JSON +records every rendered frame. + +## Limitations + +- Codex's built-in `view_image` path remains disabled. Visual inspection is + instead restricted to `get_render_as_image_for_cli`; post-run JSONL validation + requires exactly one successful PNG image response for each sequential round + checkpoint. +- MCP Python runs with server OS privileges; Codex's read-only sandbox does not + sandbox the MCP server. Use a disposable isolated account/runtime. +- Version pins establish identity, not equivalence across hardware or providers. +- GPU admission and full runs require Blender/bpy, CUDA, model access, and CLIP + weights; CI tests exercise only pure/fixture orchestration. +- Meaningful-change auditing detects repeated/nonfinite complete scene state, but + it is not a semantic judge. +- The direct 10-round protocol is not demonstrated to have the same budget, + interaction pattern, or judging procedure as VIGA. Do **not** describe results + as a fair same-budget VIGA comparison. diff --git a/benchmarks/blenderbench_direct/TDD_EVIDENCE.md b/benchmarks/blenderbench_direct/TDD_EVIDENCE.md new file mode 100644 index 0000000..822c0ef --- /dev/null +++ b/benchmarks/blenderbench_direct/TDD_EVIDENCE.md @@ -0,0 +1,64 @@ +# Publication-readiness TDD evidence + +All commands ran from the repository root. Goal/solution code and benchmark generation were not executed. + +| Behavior | RED (before production patch) | GREEN (after production patch) | +|---|---|---| +| Protocol source identity | `python -m unittest -v tests.test_blenderbench_publication.ProtocolManifestTests.test_protocol_hash_binds_initialization_implementation_and_git_state` → `TypeError: build_manifest() got an unexpected keyword argument 'source_identity'` | same command → `Ran 1 test ... OK` | +| Scoring-lock protocol binding | `...ProtocolManifestTests.test_source_identity_binds_hash_locked_scoring_environment` → `KeyError: 'scoring_lock_sha256'` | same command → `Ran 1 test ... OK` | +| Evaluator/CLIP output identity | `...ScoringTests.test_score_pair_rejects_wrong_evaluator_or_clip_identity` → `AssertionError: ValueError not raised` | same command → `Ran 1 test ... OK` | +| Standalone target verification | `...ScoringTests.test_standalone_scoring_verifies_dataset_before_reading_renders` → `AssertionError: ValueError not raised` | same command → `Ran 1 test ... OK` | +| Aggregate provenance | `...ScoringTests.test_standalone_aggregate_records_dataset_evaluator_and_scorer_provenance` → `KeyError: 'provenance'` | same command → `Ran 1 test ... OK` | +| CSV aggregate provenance | `...ScoringTests.test_json_and_csv_outputs_are_machine_readable` → `KeyError: 'dataset_revision'` | same command → `Ran 1 test ... OK` | +| Generation event enforcement | `python -m unittest -v tests.test_direct_codex.DirectCodexTests.test_generation_stream_fails_closed_on_nonapproved_operations_and_feedback` → `AttributeError: ... no attribute 'verify_generation_events'` | same command → `Ran 1 test ... OK` | +| Evaluator URL | `...ProtocolManifestTests.test_evaluator_url_resolves_to_verified_release_repository` → obsolete repository URL assertion failure | corrected test after eliminating an over-structural concatenated-literal assertion → `Ran 1 test ... OK` | +| Vendored evaluator bytes | `...ProtocolManifestTests.test_pinned_evaluator_bytes_are_present_and_bound_into_source_identity` → `AssertionError: False is not true` because `ref_based_eval.py` was absent | same command after adding the hash-verified upstream file and binding it into source identity → `Ran 1 test ... OK` | +| Recursive public redaction | `...ProtocolManifestTests.test_public_config_omits_nested_args_env_variables_and_bearer_urls` → synthetic bearer value remained in serialized output | same command → `Ran 1 test ... OK` | +| Dataset task-path symlink | `...DatasetManifestTests.test_task_path_components_may_not_be_symlinks` → verifier reached an asset through the symlink instead of rejecting the path component | same command → `Ran 1 test ... OK` | +| Broken legacy hooks | `python -m unittest -v tests.test_direct_recovery.RecoveryTests.test_broken_legacy_recovery_hooks_are_removed` → `hasattr(..., 'score_checkpoints')` / later `hasattr(..., 'run_task')` was true | same command → `Ran 1 test ... OK` | +| Immutable CI/lock | `...SuiteProtocolTests.test_ci_uses_immutable_actions_python_and_hashed_scoring_lock` → mutable action refs; after first patch the required lock command was split differently than asserted | exact full-SHA actions, CPython 3.13 CI selection, fixed runner image, and hash lock → `Ran 1 test ... OK` | +| Lexical scorer executable | `...ScoringTests.test_fake_path_executable_is_never_used_and_argv_is_canonical` → a PATH-selected executable remained possible | same command → `Ran 1 test ... OK` | +| Exact scorer runtime and distribution lock | `...ScoringTests.test_runtime_preflight_rejects_wrong_python_and_extra_distribution` → mismatched Python and extra distributions were not rejected | same command → `Ran 1 test ... OK` | +| Immutable CLIP snapshot | `...ScoringTests.test_clip_snapshot_manifest_rejects_extra_or_mutated_bytes` → extra and modified snapshot files were accepted | same command → `Ran 1 test ... OK` | +| Publication manifests | `...SuiteProtocolTests.test_publication_result_manifests_are_small_complete_and_path_free` → assertions failed against the older copied manifest bytes | same command → `Ran 1 test ... OK` | +| Root README result link | `...SuiteProtocolTests.test_root_readme_pins_latest_video_and_metric_meaning` → the clickable immutable video link and metric clarification were absent | same command → `Ran 1 test ... OK` | +| Package-safe runtime import | `...DirectCodexTests.test_generation_workdir_accepts_canonical_worker_when_imported_as_tools_module` → `ModuleNotFoundError: No module named 'direct_runtime'` | same command → `Ran 1 test ... OK` | +| Frozen manifest validation | `...ProtocolManifestTests.test_run_manifest_validation_rejects_rehashed_protocol_rule_drift` → five subtests accepted rehashed drift in tools, CLIP, rendering, generation, and Codex version | same command → `Ran 1 test ... OK` | +| Resumed download cleanup | `...DatasetManifestTests.test_successful_resumed_download_removes_superseded_partial` → retained partial still existed after successful installation | same command → `Ran 1 test ... OK` | +| Standalone audited-render scoring | `...ScoringTests.test_standalone_scoring_uses_independent_audit_render_not_model_png` → scorer received `work/iteration01.png` instead of `audit01/verified.png` | same command → `Ran 1 test ... OK` | +| Standalone audit admission | `...ScoringTests.test_standalone_scoring_rejects_unverified_audit_image` → an unverified `audit01/verified.png` produced a completed score | same command → `Ran 1 test ... OK` | +| Recovery renderer identity | `...RunnerConfigTests.test_recovery_rejects_renderer_different_from_generation_manifest` → recovery accepted a different bpy version when the new CLI pin matched it | same command → `Ran 1 test ... OK` | +| Audit metadata binding | `...ScoringTests.test_standalone_scoring_rejects_admission_without_audit_metadata` → a render with a hand-written admission but no retained audit metadata produced a completed score | same command after requiring matching source/render/full-audit hashes and explicit meaningful-change admission → `Ran 1 test ... OK` | +| MCP image inspection | `...TestRunBlenderCLI.test_cli_render_tool_returns_image_content` → `get_render_as_image_for_cli` was absent | same command after adding the PNG-returning saved-scene render tool → `Ran 1 test ... OK` | +| Per-round visual evidence | `...DirectCodexTests.test_generation_stream_requires_one_successful_image_inspection_per_round` → `verify_generation_events()` did not accept or enforce a round count; `...test_generation_stream_binds_visual_inspection_to_each_round_checkpoint` then accepted the wrong checkpoint sequence | both commands after requiring one successful PNG response for each ordered `iterationNN.blend` → `Ran 1 test ... OK` | +| Visual-tool source binding | `...ProtocolManifestTests.test_visual_inspection_tool_bytes_are_bound_into_source_identity` → the new MCP tool implementation was absent from `implementation_sha256` | same command after binding the MCP tool and CLI backend bytes → `Ran 1 test ... OK` | +| Blender resolution fields | `...DirectCodexTests.test_generation_stream_allows_blender_resolution_properties` → the `solution` filter falsely rejected `resolution_x` | same command after matching `solution` only as a complete prohibited word/file stem → `Ran 1 test ... OK` | +| Visual checkpoint byte binding | `...DirectCodexTests.test_generation_stream_rejects_visual_inspection_from_wrong_directory` → validation accepted an identically named checkpoint from another directory | same command after requiring exact ordered paths, source SHA-256 metadata, retained-byte equality, and edit/inspection alternation → `Ran 1 test ... OK` | +| Audit runtime admission | `...ScoringTests.test_standalone_scoring_rejects_failed_or_non_cuda_audit_metadata` → matching source/render/audit hashes could pass despite failed non-CUDA runtime evidence | same command after requiring successful process, exact generation renderer identity, CUDA-only devices, and GPU scene evidence → `Ran 1 test ... OK` | +| Recovery generation evidence | `...RunnerConfigTests.test_recovery_revalidates_retained_generation_events` → retained generation JSONL had no recovery validation entrypoint | same command after recovery revalidates the regular retained event stream, visual evidence, exact checkpoint paths, and source hashes before audit/scoring → `Ran 1 test ... OK` | +| Exact CUDA visual render | Additional assertions in `...TestRunBlenderCLI.test_cli_render_tool_returns_image_content` → the tool rendered a synchronized unsaved copy and lacked CUDA-only setup | same command after rendering the exact hashed checkpoint bytes with explicit CUDA-only Cycles setup → `Ran 1 test ... OK` | +| Fresh checkpoint attestation | `...TestRunBlenderCLI.test_execute_code_cli_attests_fresh_checkpoint_creation` → `expected_output_blend` was unsupported; `...DirectCodexTests.test_generation_stream_rejects_unattested_precomputed_checkpoint` then accepted precomputed bytes plus a no-op | both commands after server-side fresh-output/source/output hashing and event-level attestation validation → `Ran 1 test ... OK` | +| Cycles audit admission | The EEVEE case in `...ScoringTests.test_standalone_scoring_rejects_failed_or_non_cuda_audit_metadata` → matching hashes and CUDA preference fields were accepted without checking the engine | same command after requiring `engine == "CYCLES"` → `Ran 1 test ... OK` | +| Edit/inspection dispatch ordering | `...DirectCodexTests.test_generation_stream_rejects_next_edit_dispatched_before_prior_inspection` → an interleaved second edit's `item.started` event was accepted before the first edit had completed and been visually inspected | same command after tracking the active edit from dispatch through completion and rejecting any overlapping or pre-inspection edit → `Ran 1 test ... OK` | + +Final direct/publication gate: 76 tests, all passed. The related MCP CLI/tool-listing gate: 41 tests, all passed. + +## Verification notes + +- Static parse: all 22 changed Python files parsed; all 6 benchmark JSON files decoded. +- Manifest validation: 27 tasks, 108 pinned assets, 11 implementation hashes, deterministic repeated dirty-tree identity. +- The final source identity reported revision `4a9f5cc6e616080185252bb919fa3d1e730760b7` and `git_dirty: true`, as required for this uncommitted verification worktree. +- Scoring lock TDD: the focused CI-lock test first failed because the ARM-generated lock omitted CUDA dependencies, then failed again while it still selected stale HTTP packages. It passed after switching to the historical hash-pinned `torch 2.8.0+cu128` x86_64 wheel, pinning the observed transitive versions, and regenerating the lock for CPython 3.13 / x86_64 manylinux. +- Scoring lock verification: `uv pip install --dry-run --system --python-platform x86_64-manylinux_2_28 --python-version 3.13 --require-hashes` resolved all 41 locked packages, including CUDA 12.8 dependencies, successfully. A fresh `pip-audit` reports 16 findings across fidelity-pinned `setuptools 78.1.0` and `transformers 4.55.4`; the README records the offline pinned-model trust boundary and treats any dependency upgrade as a separately hashed condition. +- Full repository suite with `mcp/.venv/bin/python -m unittest discover -s tests -p 'test*.py' -q`: 435 tests ran; 8 errors and 23 skips. The exact environment-only errors were the three missing-Blender `setUpClass` failures in `TestBackgroundServer`, `TestForegroundServer`, and `TestInteractiveServer`, plus missing-Pillow failures in `CameraTests.test_video_calls_tokens`, three `DirectCodexTests` checkpoint tests, and `test_token_efficiency_video_evidence`. The known baseline was 403 tests with 1 failure, 7 errors, and 23 skips: the same three missing-Blender blockers plus five Pillow-dependent outcomes (four errors and one dependent assertion failure). There were no current assertion failures and no focused publication failure. +- The root README poster and immutable Hugging Face URL were fetched successfully. The downloaded video was H.264, 848×464, 134.900 seconds, 10,161,884 bytes, and SHA-256 `04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17`. + +## Review state + +The prior `REQUEST_CHANGES` findings were addressed without running benchmark +generation: visual inspection now has a hash-bound MCP image-returning path with +ordered per-round evidence validation, standalone scoring validates retained +audit metadata and admission hashes, and recovery requires the renderer version +recorded by the generation manifest. The final independent re-review returned +`APPROVED`: overlapping edit dispatch and edits pending checkpoint inspection +are rejected, and no Critical or Important regressions were found. diff --git a/benchmarks/blenderbench_direct/VIGA-LICENSE.txt b/benchmarks/blenderbench_direct/VIGA-LICENSE.txt new file mode 100644 index 0000000..e9abec7 --- /dev/null +++ b/benchmarks/blenderbench_direct/VIGA-LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Fugtemypt123 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/benchmarks/blenderbench_direct/__init__.py b/benchmarks/blenderbench_direct/__init__.py new file mode 100644 index 0000000..690cf48 --- /dev/null +++ b/benchmarks/blenderbench_direct/__init__.py @@ -0,0 +1,3 @@ +"""Publication BlenderBench runner components.""" + +__all__ = ["dataset", "protocol", "runner", "scoring", "video"] diff --git a/benchmarks/blenderbench_direct/assets-manifest.json b/benchmarks/blenderbench_direct/assets-manifest.json new file mode 100644 index 0000000..ee84013 --- /dev/null +++ b/benchmarks/blenderbench_direct/assets-manifest.json @@ -0,0 +1,1037 @@ +{ + "schema_version": 1, + "repository": "DietCoke4671/BlenderBench", + "revision": "203e4d325e9438ca55b29bdfc4f6a90842d74e68", + "tree_pages": 1, + "tasks": { + "level1/camera1": { + "task.txt": { + "path": "data/blenderbench/level1/camera1/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera1/start.py", + "size": 146, + "sha256": "8680900dd5a85c2467f86a7ec0c47e3868569e387948eb7e45aab65794981c06", + "git_blob_oid": "01a913bc51a3cbc16124c5e967396a5c0e25fd22", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera1/blender_file.blend", + "size": 11654045, + "sha256": "777b62ebe45894d18fa056c6c8c4d97d8cfc36115246e5fad46a803b52200415", + "git_blob_oid": "29514b1060cbc66d19ec8b19478aaebfef5358ec", + "lfs": { + "oid": "777b62ebe45894d18fa056c6c8c4d97d8cfc36115246e5fad46a803b52200415", + "size": 11654045, + "pointerSize": 133 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera1/renders/goal/render1.png", + "size": 280875, + "sha256": "a1f17f06448cd899c5912323f9b9dd07ca6ea30dc0aebba11f28cb85afb29d2c", + "git_blob_oid": "a74b55578860c62d8f82a488959df660a60ff4a9", + "lfs": { + "oid": "a1f17f06448cd899c5912323f9b9dd07ca6ea30dc0aebba11f28cb85afb29d2c", + "size": 280875, + "pointerSize": 131 + } + } + }, + "level1/camera2": { + "task.txt": { + "path": "data/blenderbench/level1/camera2/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera2/start.py", + "size": 146, + "sha256": "41cae2396027226fb32fd06a6e0a6e913bd895811bdee383416ef1bd1598d186", + "git_blob_oid": "ac190d95666978eb2bca17ad5f2d79dd088f8c36", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera2/blender_file.blend", + "size": 21287404, + "sha256": "9e53b677deab9a576b1d311f716c311def91591db41a33518a0433bc4cef8d3b", + "git_blob_oid": "09ff4f35c567f062437a47651d499108fea0a03a", + "lfs": { + "oid": "9e53b677deab9a576b1d311f716c311def91591db41a33518a0433bc4cef8d3b", + "size": 21287404, + "pointerSize": 133 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera2/renders/goal/render1.png", + "size": 256033, + "sha256": "ce58bcf86a6866d992e9d115391cf4a411b65395ffd612ccb7d28dc7ba5e5b58", + "git_blob_oid": "369f1d0a773fe19cb7bc5882d34d9db05bb00a61", + "lfs": { + "oid": "ce58bcf86a6866d992e9d115391cf4a411b65395ffd612ccb7d28dc7ba5e5b58", + "size": 256033, + "pointerSize": 131 + } + } + }, + "level1/camera3": { + "task.txt": { + "path": "data/blenderbench/level1/camera3/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera3/start.py", + "size": 190, + "sha256": "f5acd7609ff23e01e3d435173d46ea45907fc2b3771dea91f5da0007e51500f2", + "git_blob_oid": "b1ce78b0c679749e66cc1fa7a8972695af81cce2", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera3/blender_file.blend", + "size": 19714277, + "sha256": "2d8bb6bc5b8614ab26d6319249c4cf915e5e8107f7859ec768dfb0e02b875403", + "git_blob_oid": "0f5bce886d59f7decf761e932c5f8c287dc69273", + "lfs": { + "oid": "2d8bb6bc5b8614ab26d6319249c4cf915e5e8107f7859ec768dfb0e02b875403", + "size": 19714277, + "pointerSize": 133 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera3/renders/goal/render1.png", + "size": 323359, + "sha256": "3cde2e268b0fb73a25bc96180bbe96818f464bbe71fc8fe9bc1dff8a3d782d18", + "git_blob_oid": "5df097446778f579549300ea050a818c8b35056c", + "lfs": { + "oid": "3cde2e268b0fb73a25bc96180bbe96818f464bbe71fc8fe9bc1dff8a3d782d18", + "size": 323359, + "pointerSize": 131 + } + } + }, + "level1/camera4": { + "task.txt": { + "path": "data/blenderbench/level1/camera4/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera4/start.py", + "size": 145, + "sha256": "a994ae0f9c936a8a4a469b5245bc5cc438e7a112ec55288b7c2473998872d657", + "git_blob_oid": "e033c342fb7af3269574d25707e0060b06b774a6", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera4/blender_file.blend", + "size": 1092706714, + "sha256": "e99ccd5e43e80d8dbdd4d840ea4290f1dbc6b4e06d21be07eeed7d39fecafd0a", + "git_blob_oid": "b66db22091cd21021b2630ca2fc01040cf35b27f", + "lfs": { + "oid": "e99ccd5e43e80d8dbdd4d840ea4290f1dbc6b4e06d21be07eeed7d39fecafd0a", + "size": 1092706714, + "pointerSize": 135 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera4/renders/goal/render1.png", + "size": 434190, + "sha256": "a6cad78bdebb96be3147176d6c4c460df4782c51430fcd268dfddd87f499aa93", + "git_blob_oid": "2e09e3634d5c1c7cd3a0bd3d30ac333922d3570e", + "lfs": { + "oid": "a6cad78bdebb96be3147176d6c4c460df4782c51430fcd268dfddd87f499aa93", + "size": 434190, + "pointerSize": 131 + } + } + }, + "level1/camera5": { + "task.txt": { + "path": "data/blenderbench/level1/camera5/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera5/start.py", + "size": 148, + "sha256": "2d09f0ea2b51ace93663882d86aa2fa4201f394710879c12c6774173793a8971", + "git_blob_oid": "5761047b75c43463abb73e2c967c5c64c9de8c51", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera5/blender_file.blend", + "size": 1092786654, + "sha256": "06f30a3ccd0d81be515f1892605fd575f6933052e15b9b297b7d96a48aa664d2", + "git_blob_oid": "f54d89e9d4c3f0d32b34e51ac2cd5d51743bb4b0", + "lfs": { + "oid": "06f30a3ccd0d81be515f1892605fd575f6933052e15b9b297b7d96a48aa664d2", + "size": 1092786654, + "pointerSize": 135 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera5/renders/goal/render1.png", + "size": 442674, + "sha256": "0124cf59482f3950b2c62eddf8d734e752e45e5300fbf30ce472f74b05e30f9d", + "git_blob_oid": "d411365abfd0112bcefbd1c253b631cd59f48e6f", + "lfs": { + "oid": "0124cf59482f3950b2c62eddf8d734e752e45e5300fbf30ce472f74b05e30f9d", + "size": 442674, + "pointerSize": 131 + } + } + }, + "level1/camera6": { + "task.txt": { + "path": "data/blenderbench/level1/camera6/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera6/start.py", + "size": 146, + "sha256": "1539bdf69d7df4cb2da3770a59f2a33ed28d3d20aec3c3e2677351b297850f9c", + "git_blob_oid": "a0449cfce1acdf566251c2634b2b38469006af82", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera6/blender_file.blend", + "size": 5718592, + "sha256": "4e0fb03e37b9072c9ca65bab2b6594deb4254d21b3e13c08b8eeaf1fce76e45f", + "git_blob_oid": "2073a629bfaa3343eebd0b66a0a7475d005aadc4", + "lfs": { + "oid": "4e0fb03e37b9072c9ca65bab2b6594deb4254d21b3e13c08b8eeaf1fce76e45f", + "size": 5718592, + "pointerSize": 132 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera6/renders/goal/render1.png", + "size": 264383, + "sha256": "e8ef88036bdd3e417383c349da3f531e22698d765f34b60851ec132d143f99d3", + "git_blob_oid": "cb03a3a9e48c23b0900716f00cd510d252856940", + "lfs": { + "oid": "e8ef88036bdd3e417383c349da3f531e22698d765f34b60851ec132d143f99d3", + "size": 264383, + "pointerSize": 131 + } + } + }, + "level1/camera7": { + "task.txt": { + "path": "data/blenderbench/level1/camera7/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera7/start.py", + "size": 146, + "sha256": "5471be8faa7f4bd3ba2963a6716298a007d8a004a11df38cb87b9f026d9f4bbc", + "git_blob_oid": "9c0fa4d13f79519e9950d330ececa5e5563334fa", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera7/blender_file.blend", + "size": 198756884, + "sha256": "6106fe112e0bfdf4e7aab26ff4e901ad8f942a9e6493839a9d795806cf97ab60", + "git_blob_oid": "8a09b0908159af7cb9d32370e8b7d87c58590b4f", + "lfs": { + "oid": "6106fe112e0bfdf4e7aab26ff4e901ad8f942a9e6493839a9d795806cf97ab60", + "size": 198756884, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera7/renders/goal/render1.png", + "size": 527080, + "sha256": "7f5efeca8b9586450cbb5a019bc03f2b0616f76d29fecfc699cca0059150a8fd", + "git_blob_oid": "7ac2abc00d8b688de48c4278a56bc8bf3bf517fb", + "lfs": { + "oid": "7f5efeca8b9586450cbb5a019bc03f2b0616f76d29fecfc699cca0059150a8fd", + "size": 527080, + "pointerSize": 131 + } + } + }, + "level1/camera8": { + "task.txt": { + "path": "data/blenderbench/level1/camera8/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera8/start.py", + "size": 146, + "sha256": "1dd7b06ac2d5949e0453f0e0c8ae857a417e986d9cc3b6fad748d26abf1b8151", + "git_blob_oid": "5b670d67b5a328d591968b16afe9be57b7735c17", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera8/blender_file.blend", + "size": 2303572, + "sha256": "64124da51512f689ff1c2d0a6475ccf377bb94a87a3bcc9fd14e0d38de7a4e00", + "git_blob_oid": "2a8b648f8eef7daf70287a587ce7a490c8ec31bb", + "lfs": { + "oid": "64124da51512f689ff1c2d0a6475ccf377bb94a87a3bcc9fd14e0d38de7a4e00", + "size": 2303572, + "pointerSize": 132 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera8/renders/goal/render1.png", + "size": 198412, + "sha256": "b4e026fea8211aaf7f1149cd3e20a2ac6346882484eee5804ea6e19dc43b6bc4", + "git_blob_oid": "2ce8adacf714c8fbc432e30ff94b3989b2ce2991", + "lfs": { + "oid": "b4e026fea8211aaf7f1149cd3e20a2ac6346882484eee5804ea6e19dc43b6bc4", + "size": 198412, + "pointerSize": 131 + } + } + }, + "level1/camera9": { + "task.txt": { + "path": "data/blenderbench/level1/camera9/task.txt", + "size": 88, + "sha256": "a9023cd8a8838793a24ad4d7dac5ef9face61e2702cb49fafbff271dc510f613", + "git_blob_oid": "9961c9617df77c716b0b8502dce2718f52fe1764", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level1/camera9/start.py", + "size": 136, + "sha256": "6c104adc69f6e714e7b51d3e834b8bd7e6836813899ea84c972c6d2208ac40b6", + "git_blob_oid": "786bf8646e259f12b3b2960baf3c40358fe89663", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level1/camera9/blender_file.blend", + "size": 4107448, + "sha256": "af9b2132fbffb66a1a4832c500c82bc00ee19ac90a40c2b1f96e95bfe74b295d", + "git_blob_oid": "0f1887978fb6c72a5b7c1d1fb3a51d26ca498069", + "lfs": { + "oid": "af9b2132fbffb66a1a4832c500c82bc00ee19ac90a40c2b1f96e95bfe74b295d", + "size": 4107448, + "pointerSize": 132 + } + }, + "target.png": { + "path": "data/blenderbench/level1/camera9/renders/goal/render1.png", + "size": 311165, + "sha256": "df51a5786e84b1d47c2df08c95d6adb3f7594dec20fa17c47424bb08b16145b6", + "git_blob_oid": "615154f4ffee4b912add08312db65fd507688af7", + "lfs": { + "oid": "df51a5786e84b1d47c2df08c95d6adb3f7594dec20fa17c47424bb08b16145b6", + "size": 311165, + "pointerSize": 131 + } + } + }, + "level2/attribute1": { + "task.txt": { + "path": "data/blenderbench/level2/attribute1/task.txt", + "size": 119, + "sha256": "e2f9834ab4a6888041e4e65c01806c816067cbd68931f49d181a938347186caa", + "git_blob_oid": "c55b5774e43199c6638deb8ff5990a3395a4ee3f", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute1/start.py", + "size": 1240, + "sha256": "57448a36ff834acc820d071fc99b26285d8968d4fece18300a5c512e18ac2f18", + "git_blob_oid": "86c038455c8a485c251ed84a64285bf10dc6896f", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute1/blender_file.blend", + "size": 950771988, + "sha256": "d5eca6c92d3cee2ce8a4ea9986a644e4d61a1d66c123c81050616026fa12f5f8", + "git_blob_oid": "bfbc64a723927a21ec431856ee3b0bcb17971736", + "lfs": { + "oid": "d5eca6c92d3cee2ce8a4ea9986a644e4d61a1d66c123c81050616026fa12f5f8", + "size": 950771988, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute1/renders/goal/render1.png", + "size": 435882, + "sha256": "a269058785099fd3f273334e70fa147de3ee3556a500ded8d9b064213fcd8981", + "git_blob_oid": "ec260b02febe6c7dbaea38a531f584ed828dd677", + "lfs": { + "oid": "a269058785099fd3f273334e70fa147de3ee3556a500ded8d9b064213fcd8981", + "size": 435882, + "pointerSize": 131 + } + } + }, + "level2/attribute2": { + "task.txt": { + "path": "data/blenderbench/level2/attribute2/task.txt", + "size": 119, + "sha256": "e2f9834ab4a6888041e4e65c01806c816067cbd68931f49d181a938347186caa", + "git_blob_oid": "c55b5774e43199c6638deb8ff5990a3395a4ee3f", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute2/start.py", + "size": 1180, + "sha256": "83702d094c0fa4d6e4a29711dfa024f587339353b1f9ddd6aff5e17b3ab426a8", + "git_blob_oid": "1c1f9de3379324f7e099bb90596f2ff70382ce8e", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute2/blender_file.blend", + "size": 950771988, + "sha256": "d5eca6c92d3cee2ce8a4ea9986a644e4d61a1d66c123c81050616026fa12f5f8", + "git_blob_oid": "bfbc64a723927a21ec431856ee3b0bcb17971736", + "lfs": { + "oid": "d5eca6c92d3cee2ce8a4ea9986a644e4d61a1d66c123c81050616026fa12f5f8", + "size": 950771988, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute2/renders/goal/render1.png", + "size": 435876, + "sha256": "0744a471d3a07fe9c4a38402f9be09310e5e26dc4abbc62f6b634d20b1f6c23f", + "git_blob_oid": "c40abc765e184b1851ed8ac5469bad8045e5f144", + "lfs": { + "oid": "0744a471d3a07fe9c4a38402f9be09310e5e26dc4abbc62f6b634d20b1f6c23f", + "size": 435876, + "pointerSize": 131 + } + } + }, + "level2/attribute3": { + "task.txt": { + "path": "data/blenderbench/level2/attribute3/task.txt", + "size": 119, + "sha256": "e2f9834ab4a6888041e4e65c01806c816067cbd68931f49d181a938347186caa", + "git_blob_oid": "c55b5774e43199c6638deb8ff5990a3395a4ee3f", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute3/start.py", + "size": 1178, + "sha256": "0bd075a70511e2952ff888d6a7d72499311a82764c7ab8c7bb7ae659ba210706", + "git_blob_oid": "eb069eba41ab9147184bce60690f6b5355b6c959", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute3/blender_file.blend", + "size": 950771988, + "sha256": "d5eca6c92d3cee2ce8a4ea9986a644e4d61a1d66c123c81050616026fa12f5f8", + "git_blob_oid": "bfbc64a723927a21ec431856ee3b0bcb17971736", + "lfs": { + "oid": "d5eca6c92d3cee2ce8a4ea9986a644e4d61a1d66c123c81050616026fa12f5f8", + "size": 950771988, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute3/renders/goal/render1.png", + "size": 435892, + "sha256": "3886f82c111b6e7dada41bcd6129e9c782f543a11e02860c693a5e3c8a336da7", + "git_blob_oid": "a28d2858477a08df993ec088e53805f8c16d9df6", + "lfs": { + "oid": "3886f82c111b6e7dada41bcd6129e9c782f543a11e02860c693a5e3c8a336da7", + "size": 435892, + "pointerSize": 131 + } + } + }, + "level2/attribute4": { + "task.txt": { + "path": "data/blenderbench/level2/attribute4/task.txt", + "size": 130, + "sha256": "fba1c19366ae11259cee9886681c0c8688f65cccc033e45ce82bd1a8cf1d7afc", + "git_blob_oid": "e30cdd33cdd13d3059c5e9861224baaac06af74c", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute4/start.py", + "size": 703, + "sha256": "1991b217fd4e044f7ef0966bbb38e8e417bb1fa967effba61bbfc880f7e36543", + "git_blob_oid": "e2a610617d76c2c223303277742b9484f9ec1e7c", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute4/blender_file.blend", + "size": 947346540, + "sha256": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "git_blob_oid": "651e1029bc10ca62d380be72c08e122720b5a7c2", + "lfs": { + "oid": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "size": 947346540, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute4/renders/goal/render1.png", + "size": 433064, + "sha256": "dd049697d61750d34d9f0587a832d7b8ad322197e63b4319b1938675b64eefcb", + "git_blob_oid": "1c8fb0f28aec077a44f6a96ce1a3c6e852b04378", + "lfs": { + "oid": "dd049697d61750d34d9f0587a832d7b8ad322197e63b4319b1938675b64eefcb", + "size": 433064, + "pointerSize": 131 + } + } + }, + "level2/attribute5": { + "task.txt": { + "path": "data/blenderbench/level2/attribute5/task.txt", + "size": 130, + "sha256": "fba1c19366ae11259cee9886681c0c8688f65cccc033e45ce82bd1a8cf1d7afc", + "git_blob_oid": "e30cdd33cdd13d3059c5e9861224baaac06af74c", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute5/start.py", + "size": 644, + "sha256": "d80af70f035011ae1d6bf7001c9cad008e3cb63a6442bfbd87b23e0d4e09fb0a", + "git_blob_oid": "7d91d84c63e898aa443debdcf83e46bc00498586", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute5/blender_file.blend", + "size": 947346540, + "sha256": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "git_blob_oid": "651e1029bc10ca62d380be72c08e122720b5a7c2", + "lfs": { + "oid": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "size": 947346540, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute5/renders/goal/render1.png", + "size": 433064, + "sha256": "0b33dfa853ce71b3e0a91ae9813e90dba0544f489251d8a811c810c1491fce55", + "git_blob_oid": "3824f88b86b5b559a493a42a983a78ecb68adb99", + "lfs": { + "oid": "0b33dfa853ce71b3e0a91ae9813e90dba0544f489251d8a811c810c1491fce55", + "size": 433064, + "pointerSize": 131 + } + } + }, + "level2/attribute6": { + "task.txt": { + "path": "data/blenderbench/level2/attribute6/task.txt", + "size": 130, + "sha256": "fba1c19366ae11259cee9886681c0c8688f65cccc033e45ce82bd1a8cf1d7afc", + "git_blob_oid": "e30cdd33cdd13d3059c5e9861224baaac06af74c", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute6/start.py", + "size": 641, + "sha256": "aadf4e84cd6b0e03c47f45cc1e1b6f12a436b6273ffd7f81cd3f2d28a4c072bf", + "git_blob_oid": "8c4abaa13956c250a59146898251b52928d5138f", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute6/blender_file.blend", + "size": 947346540, + "sha256": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "git_blob_oid": "651e1029bc10ca62d380be72c08e122720b5a7c2", + "lfs": { + "oid": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "size": 947346540, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute6/renders/goal/render1.png", + "size": 433064, + "sha256": "1eda79fb1fd8773714467694b24cbbf71f049549b0016cac79254724e5c5b5cf", + "git_blob_oid": "518e7143a5ee90a2bb6d8e73b972c7b5d4d1cca8", + "lfs": { + "oid": "1eda79fb1fd8773714467694b24cbbf71f049549b0016cac79254724e5c5b5cf", + "size": 433064, + "pointerSize": 131 + } + } + }, + "level2/attribute7": { + "task.txt": { + "path": "data/blenderbench/level2/attribute7/task.txt", + "size": 157, + "sha256": "f389ae85ca7ee5cb03d3ab75aa37722c9c5c00475e378cdc6d40a92b59fcc328", + "git_blob_oid": "9a3d8216657a654dc6be9b8404e878c259c5ed02", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute7/start.py", + "size": 1346, + "sha256": "abd953c4d36252658613f4c385787dbe6baf4beedc346789ea0a6d210c49775b", + "git_blob_oid": "3031bcfffd0d61e2efe8d5aa43ad7b7ed79246c1", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute7/blender_file.blend", + "size": 768637773, + "sha256": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "git_blob_oid": "05fcbfc615f12ce6600305c52710ec2d70080a5b", + "lfs": { + "oid": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "size": 768637773, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute7/renders/goal/render1.png", + "size": 496461, + "sha256": "4e0dcf670648bba309e19853047bb294bbf588ffb8928ed9f9a45bef729f9f32", + "git_blob_oid": "cfed756b5128bc758a055def9ebef92b21b0479b", + "lfs": { + "oid": "4e0dcf670648bba309e19853047bb294bbf588ffb8928ed9f9a45bef729f9f32", + "size": 496461, + "pointerSize": 131 + } + } + }, + "level2/attribute8": { + "task.txt": { + "path": "data/blenderbench/level2/attribute8/task.txt", + "size": 157, + "sha256": "f389ae85ca7ee5cb03d3ab75aa37722c9c5c00475e378cdc6d40a92b59fcc328", + "git_blob_oid": "9a3d8216657a654dc6be9b8404e878c259c5ed02", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute8/start.py", + "size": 1346, + "sha256": "06cc98b4dae682711a3b06f82ca6cd5a967c30915722b2d3ca57a445f6ae83db", + "git_blob_oid": "b37dd16de190575b85f359fa99e72ce5593411ed", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute8/blender_file.blend", + "size": 768637773, + "sha256": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "git_blob_oid": "05fcbfc615f12ce6600305c52710ec2d70080a5b", + "lfs": { + "oid": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "size": 768637773, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute8/renders/goal/render1.png", + "size": 496462, + "sha256": "2177e15ad99cb6559092b37de04aa778b13445c89598736ab67ab4f54cfa4ec3", + "git_blob_oid": "3fb1988fe9e2dbd5e711a2b9c253726fee099256", + "lfs": { + "oid": "2177e15ad99cb6559092b37de04aa778b13445c89598736ab67ab4f54cfa4ec3", + "size": 496462, + "pointerSize": 131 + } + } + }, + "level2/attribute9": { + "task.txt": { + "path": "data/blenderbench/level2/attribute9/task.txt", + "size": 157, + "sha256": "f389ae85ca7ee5cb03d3ab75aa37722c9c5c00475e378cdc6d40a92b59fcc328", + "git_blob_oid": "9a3d8216657a654dc6be9b8404e878c259c5ed02", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level2/attribute9/start.py", + "size": 1346, + "sha256": "956f2d67b54cda24edfeba5d5bfc766002efe018a4a77c0d505d4cf391410bf1", + "git_blob_oid": "133230ae8b18e1fb4c6be19880194deb6fadb6c2", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level2/attribute9/blender_file.blend", + "size": 768637773, + "sha256": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "git_blob_oid": "05fcbfc615f12ce6600305c52710ec2d70080a5b", + "lfs": { + "oid": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "size": 768637773, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level2/attribute9/renders/goal/render1.png", + "size": 496462, + "sha256": "057562178eba0d2cf993227378f6f93c2689eda870a060cc836edce85f892eb0", + "git_blob_oid": "3448027c64be93eee308939ef1f9793ac04bd5bb", + "lfs": { + "oid": "057562178eba0d2cf993227378f6f93c2689eda870a060cc836edce85f892eb0", + "size": 496462, + "pointerSize": 131 + } + } + }, + "level3/attribute1": { + "task.txt": { + "path": "data/blenderbench/level3/attribute1/task.txt", + "size": 206, + "sha256": "a682f48631e343a9c378a222e0651a7f8dd6f82b39ca8d0ce26305f42bfa422d", + "git_blob_oid": "df0d929a1e1d9cdcc321642c98c5d60a6bc90bb9", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute1/start.py", + "size": 1434, + "sha256": "e9d1a0d7c912f9806f5879117403938995c1155d13c33af9a0c0c25068909769", + "git_blob_oid": "7d5b416c2be2ddc0f55b8b96482c0536cec22496", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute1/blender_file.blend", + "size": 1103281707, + "sha256": "58bd9031ded0eb4c092951573e7e16f2704b29b1a2eb239331c88b5ba74d3aea", + "git_blob_oid": "598a2385913f422503103677c28e70a16ba5134b", + "lfs": { + "oid": "58bd9031ded0eb4c092951573e7e16f2704b29b1a2eb239331c88b5ba74d3aea", + "size": 1103281707, + "pointerSize": 135 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute1/renders/goal/render1.png", + "size": 424605, + "sha256": "203bc0d5b3cd6bff06e19baa56c56370b5a8da9155b5b0ee3e8598a0886318f0", + "git_blob_oid": "d9f070b573c1cfe18b9d06fd2777c97155cdb154", + "lfs": { + "oid": "203bc0d5b3cd6bff06e19baa56c56370b5a8da9155b5b0ee3e8598a0886318f0", + "size": 424605, + "pointerSize": 131 + } + } + }, + "level3/attribute2": { + "task.txt": { + "path": "data/blenderbench/level3/attribute2/task.txt", + "size": 206, + "sha256": "a682f48631e343a9c378a222e0651a7f8dd6f82b39ca8d0ce26305f42bfa422d", + "git_blob_oid": "df0d929a1e1d9cdcc321642c98c5d60a6bc90bb9", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute2/start.py", + "size": 1374, + "sha256": "0c115cf6fdbf43f677ebb5244e6afc9d75dd9635a9f4c6de06abe28a3f68f191", + "git_blob_oid": "df1d330fdd5658db8875f63713eeb753bfcce23c", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute2/blender_file.blend", + "size": 1104010951, + "sha256": "8dd5b6c18ae4d7f77a94370a9c0203f6ea06911e590c43d4b563c2f0195854be", + "git_blob_oid": "3c8b7beeb7aa513fa1c66d186039151f9d38077f", + "lfs": { + "oid": "8dd5b6c18ae4d7f77a94370a9c0203f6ea06911e590c43d4b563c2f0195854be", + "size": 1104010951, + "pointerSize": 135 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute2/renders/goal/render1.png", + "size": 424611, + "sha256": "b5b1526c9fde7aecd35f842598627b63172338284d69aaa36d6a40649200ca30", + "git_blob_oid": "fec9d6a34397b5ab37a6e14e207f7e232da8a860", + "lfs": { + "oid": "b5b1526c9fde7aecd35f842598627b63172338284d69aaa36d6a40649200ca30", + "size": 424611, + "pointerSize": 131 + } + } + }, + "level3/attribute3": { + "task.txt": { + "path": "data/blenderbench/level3/attribute3/task.txt", + "size": 206, + "sha256": "a682f48631e343a9c378a222e0651a7f8dd6f82b39ca8d0ce26305f42bfa422d", + "git_blob_oid": "df0d929a1e1d9cdcc321642c98c5d60a6bc90bb9", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute3/start.py", + "size": 1372, + "sha256": "de2d9d5cba1b04d58b1d698726c29968d8fbfed225c2b2b24f32f5f765a532ba", + "git_blob_oid": "c4d5bbd48128bb87ef8f8186a1cd68439edbfdb2", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute3/blender_file.blend", + "size": 1104010951, + "sha256": "7c1a17de951be40e203fc96dde62afc0056392394897e3478de322cf84ef4272", + "git_blob_oid": "c0450d126230e49f06bc0afc80710ff278778e73", + "lfs": { + "oid": "7c1a17de951be40e203fc96dde62afc0056392394897e3478de322cf84ef4272", + "size": 1104010951, + "pointerSize": 135 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute3/renders/goal/render1.png", + "size": 424612, + "sha256": "741983290631f850e8e7a011679328f9e6da5545a671c23d73c17beb88cae346", + "git_blob_oid": "70f334823a5eb76be195cadf3158929b755f9267", + "lfs": { + "oid": "741983290631f850e8e7a011679328f9e6da5545a671c23d73c17beb88cae346", + "size": 424612, + "pointerSize": 131 + } + } + }, + "level3/attribute4": { + "task.txt": { + "path": "data/blenderbench/level3/attribute4/task.txt", + "size": 217, + "sha256": "f614d97a13bbae1dedef105ca932a0acfdaf7162e9d19cd4ec98ce3c26e3c647", + "git_blob_oid": "0f4f7411e78fbfdf51b265988ee7ba26933de194", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute4/start.py", + "size": 897, + "sha256": "bfce3b4e6919cdfa62ddb5893d42d64493d2357505ae83ece52d87a424ff0d10", + "git_blob_oid": "c8cf8717af01beef3fa61cc2fdb182a9132bd1ce", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute4/blender_file.blend", + "size": 1100061468, + "sha256": "250865ed23f0d6dcb03c8ee9302ae5e55473fb60d35c6b487533fd3f8bd3e29d", + "git_blob_oid": "78c59f7e8ca4edbbb336542d964fff9b238fa96d", + "lfs": { + "oid": "250865ed23f0d6dcb03c8ee9302ae5e55473fb60d35c6b487533fd3f8bd3e29d", + "size": 1100061468, + "pointerSize": 135 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute4/renders/goal/render1.png", + "size": 424822, + "sha256": "2dfaea1568f9cfb0400600c874e214dc9b15578708571d1d9897605330ffc928", + "git_blob_oid": "1e0bf1cecf36801c6aef9cdbd5f335f6a577897a", + "lfs": { + "oid": "2dfaea1568f9cfb0400600c874e214dc9b15578708571d1d9897605330ffc928", + "size": 424822, + "pointerSize": 131 + } + } + }, + "level3/attribute5": { + "task.txt": { + "path": "data/blenderbench/level3/attribute5/task.txt", + "size": 217, + "sha256": "f614d97a13bbae1dedef105ca932a0acfdaf7162e9d19cd4ec98ce3c26e3c647", + "git_blob_oid": "0f4f7411e78fbfdf51b265988ee7ba26933de194", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute5/start.py", + "size": 838, + "sha256": "8fb7657a111ad845d72eaa3c1599f46f0ee1bb4c7b9b8e97b2858d7c2a71c215", + "git_blob_oid": "59d3e7ce0a94c4c34a243de6f0266e80084ddb04", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute5/blender_file.blend", + "size": 947346540, + "sha256": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "git_blob_oid": "651e1029bc10ca62d380be72c08e122720b5a7c2", + "lfs": { + "oid": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "size": 947346540, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute5/renders/goal/render1.png", + "size": 432648, + "sha256": "08f94e6af9f1bd74f52a7302f7673da3ebfb318437d4a986780158967944635b", + "git_blob_oid": "e9aef85c8c070db78c4d9f85a0ae46eeef6b3b92", + "lfs": { + "oid": "08f94e6af9f1bd74f52a7302f7673da3ebfb318437d4a986780158967944635b", + "size": 432648, + "pointerSize": 131 + } + } + }, + "level3/attribute6": { + "task.txt": { + "path": "data/blenderbench/level3/attribute6/task.txt", + "size": 217, + "sha256": "f614d97a13bbae1dedef105ca932a0acfdaf7162e9d19cd4ec98ce3c26e3c647", + "git_blob_oid": "0f4f7411e78fbfdf51b265988ee7ba26933de194", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute6/start.py", + "size": 835, + "sha256": "6e1bed8235bb346af995c6f281b05f2d482c3f36dd0432a9df9ce62674dd7dbd", + "git_blob_oid": "c6ebc619f78ff7952522a61c9c1d7eb3dad40bdb", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute6/blender_file.blend", + "size": 947346540, + "sha256": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "git_blob_oid": "651e1029bc10ca62d380be72c08e122720b5a7c2", + "lfs": { + "oid": "0d1205f27f8f2493620645554db00c24d74d88b4ae0310fa846e30c3449aa84b", + "size": 947346540, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute6/renders/goal/render1.png", + "size": 432652, + "sha256": "86b764ac22fe162549f3d5a2995e0e8160e836cf6abac763c120f9de283a8089", + "git_blob_oid": "10d2d97575e32eeb841547b7270f74cf7fabd6ff", + "lfs": { + "oid": "86b764ac22fe162549f3d5a2995e0e8160e836cf6abac763c120f9de283a8089", + "size": 432652, + "pointerSize": 131 + } + } + }, + "level3/attribute7": { + "task.txt": { + "path": "data/blenderbench/level3/attribute7/task.txt", + "size": 244, + "sha256": "9f6dd788a109d7888acd415b2dec5b86441aaf2c9c31bd898657b3acd769afb1", + "git_blob_oid": "20dc920fca0ea87f55f7a39e4be480283ca1b6e3", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute7/start.py", + "size": 1539, + "sha256": "88c03b7b328cb7251425acdde7959aa4cb9bea9add652c68faa5d442460fa4f9", + "git_blob_oid": "aeda4ddbd4422cd29ba466dd3577f9f05a9ea7e4", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute7/blender_file.blend", + "size": 766325689, + "sha256": "c4efab1f4cfc624e047491836f5516a0d2bd0e4f803926bca3e94e23a0ada149", + "git_blob_oid": "ee8f5fb6f9a868345bb02a9cac6deee6e68e3f45", + "lfs": { + "oid": "c4efab1f4cfc624e047491836f5516a0d2bd0e4f803926bca3e94e23a0ada149", + "size": 766325689, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute7/renders/goal/render1.png", + "size": 496540, + "sha256": "0ada6e89b53c28b9036418def514b728dd0eeb664eb3f7a1ca9496a5e8ef93f1", + "git_blob_oid": "066f39046e01e72f80fb53c8606b7deaf2811ab5", + "lfs": { + "oid": "0ada6e89b53c28b9036418def514b728dd0eeb664eb3f7a1ca9496a5e8ef93f1", + "size": 496540, + "pointerSize": 131 + } + } + }, + "level3/attribute8": { + "task.txt": { + "path": "data/blenderbench/level3/attribute8/task.txt", + "size": 244, + "sha256": "9f6dd788a109d7888acd415b2dec5b86441aaf2c9c31bd898657b3acd769afb1", + "git_blob_oid": "20dc920fca0ea87f55f7a39e4be480283ca1b6e3", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute8/start.py", + "size": 1539, + "sha256": "6fdf229c8df32bdfd1700e90cd1c135e1eba67038effd3c766753a386e523c2a", + "git_blob_oid": "37c534a03c87c0689ca42cedbc8cb572edace78b", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute8/blender_file.blend", + "size": 768637773, + "sha256": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "git_blob_oid": "05fcbfc615f12ce6600305c52710ec2d70080a5b", + "lfs": { + "oid": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "size": 768637773, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute8/renders/goal/render1.png", + "size": 496540, + "sha256": "33e602e64c9b3f6825472277653bd168de39645a61cde8d8112efe0ae0cde1ad", + "git_blob_oid": "8d6dff62f9fc08e27eafe9ec5a0cd6b90c4de73f", + "lfs": { + "oid": "33e602e64c9b3f6825472277653bd168de39645a61cde8d8112efe0ae0cde1ad", + "size": 496540, + "pointerSize": 131 + } + } + }, + "level3/attribute9": { + "task.txt": { + "path": "data/blenderbench/level3/attribute9/task.txt", + "size": 244, + "sha256": "9f6dd788a109d7888acd415b2dec5b86441aaf2c9c31bd898657b3acd769afb1", + "git_blob_oid": "20dc920fca0ea87f55f7a39e4be480283ca1b6e3", + "lfs": null + }, + "start.py": { + "path": "data/blenderbench/level3/attribute9/start.py", + "size": 1539, + "sha256": "bd47ca26dfd4b224a33766dcd40101cbf3f8b168773541d7444d6ea1c4039a6b", + "git_blob_oid": "94fe4672c7fe453266a24537cfd7e0df0556ce1f", + "lfs": null + }, + "scene.blend": { + "path": "data/blenderbench/level3/attribute9/blender_file.blend", + "size": 768637773, + "sha256": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "git_blob_oid": "05fcbfc615f12ce6600305c52710ec2d70080a5b", + "lfs": { + "oid": "f49ef4d8b76eddfe71f73646d92ab3f7e6aa791580b5c7b093335f831386863c", + "size": 768637773, + "pointerSize": 134 + } + }, + "target.png": { + "path": "data/blenderbench/level3/attribute9/renders/goal/render1.png", + "size": 496540, + "sha256": "27653bfcb7c78fe51239c3905060ce112e65c8875966b2d81f3d0ae33fa322ee", + "git_blob_oid": "5baa5bd208c5f160405d1fa96895eddd9ea9887d", + "lfs": { + "oid": "27653bfcb7c78fe51239c3905060ce112e65c8875966b2d81f3d0ae33fa322ee", + "size": 496540, + "pointerSize": 131 + } + } + } + }, + "source_url": "https://huggingface.co/datasets/DietCoke4671/BlenderBench", + "attribution": "BlenderBench dataset by DietCoke4671 and contributors", + "license": "CC BY 4.0" +} diff --git a/benchmarks/blenderbench_direct/clip-snapshot-manifest.json b/benchmarks/blenderbench_direct/clip-snapshot-manifest.json new file mode 100644 index 0000000..e70cc84 --- /dev/null +++ b/benchmarks/blenderbench_direct/clip-snapshot-manifest.json @@ -0,0 +1,57 @@ +{ + "files": [ + { + "path": "config.json", + "sha256": "b575ef3c36f2a057fa19e221650105052d61cc9c1a972ec15019c6261ec98770", + "size": 4186, + "symlink": true + }, + { + "path": "merges.txt", + "sha256": "f526393189112391ce6f9795d4695f704121ce452c3aad1f5335cc41337eba85", + "size": 524657, + "symlink": true + }, + { + "path": "preprocessor_config.json", + "sha256": "910e70b3956ac9879ebc90b22fb3bc8a75b6a0677814500101a4c072bd7857bd", + "size": 316, + "symlink": true + }, + { + "path": "pytorch_model.bin", + "sha256": "a63082132ba4f97a80bea76823f544493bffa8082296d62d71581a4feff1576f", + "size": 605247071, + "symlink": true + }, + { + "path": "special_tokens_map.json", + "sha256": "f8c0d6c39aee3f8431078ef6646567b0aba7f2246e9c54b8b99d55c22b707cbf", + "size": 389, + "symlink": true + }, + { + "path": "tokenizer.json", + "sha256": "b556ac8c99757ffb677208af34bc8c6721572114111a6e0aaf5fa69ff0b8d842", + "size": 2224041, + "symlink": true + }, + { + "path": "tokenizer_config.json", + "sha256": "34b7336e4bee12e0a9730eaf5189f582ef3c3eea5027f65730e5717256755aad", + "size": 592, + "symlink": true + }, + { + "path": "vocab.json", + "sha256": "5047b556ce86ccaf6aa22b3ffccfc52d391ea4accdab9c2f2407da5b742d4363", + "size": 862328, + "symlink": true + } + ], + "python_version": "3.13.15 (main, Sep 1 2026, 14:18:08) [Clang 22.1.3 ]", + "repository": "openai/clip-vit-base-patch32", + "revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", + "schema_version": 1, + "snapshot_exists": true +} diff --git a/benchmarks/blenderbench_direct/dataset.py b/benchmarks/blenderbench_direct/dataset.py new file mode 100644 index 0000000..e937311 --- /dev/null +++ b/benchmarks/blenderbench_direct/dataset.py @@ -0,0 +1,209 @@ +"""Hash-pinned, input-only 27-task BlenderBench dataset downloader. + +Only task text, start script, initial scene, and target image are staged. Upstream +Python is data: it is hashed and never imported, compiled, or executed here. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import shutil +import urllib.parse +import urllib.request +import uuid + +HERE = Path(__file__).resolve().parent +DATASET_REPOSITORY = "DietCoke4671/BlenderBench" +DATASET_SOURCE_URL = "https://huggingface.co/datasets/DietCoke4671/BlenderBench" +DATASET_ATTRIBUTION = "BlenderBench dataset by DietCoke4671 and contributors" +DATASET_LICENSE = "CC BY 4.0" +DATASET_REVISION = "203e4d325e9438ca55b29bdfc4f6a90842d74e68" +_ASSET_NAMES = {"task.txt", "start.py", "scene.blend", "target.png"} + + +def _read(name: str) -> dict: + return json.loads((HERE / name).read_text(encoding="utf-8")) + + +def load_manifest() -> dict: + assets = _read("assets-manifest.json") + initialization = _read("initialization-manifest.json") + if assets["revision"] != DATASET_REVISION or initialization["revision"] != DATASET_REVISION: + raise ValueError("dataset manifest revision drift") + if set(assets["tasks"]) != set(initialization["tasks"]): + raise ValueError("asset and initialization task sets differ") + tasks = {} + for task in sorted(assets["tasks"]): + tasks[task] = {**initialization["tasks"][task], "assets": assets["tasks"][task]} + if len(tasks) != 27: + raise ValueError("pinned manifest must contain exactly 27 tasks") + return {"schema_version": 1, "repository": assets["repository"], + "source_url": DATASET_SOURCE_URL, "attribution": DATASET_ATTRIBUTION, + "license": DATASET_LICENSE, "revision": DATASET_REVISION, "tasks": tasks} + + +def task_ids() -> list[str]: + return sorted(load_manifest()["tasks"]) + + +def select_tasks(selected: list[str] | None) -> list[str]: + canonical = task_ids() + if selected is None: + return canonical + if not selected or len(selected) != len(set(selected)) or set(selected) - set(canonical): + raise ValueError("--tasks must be a nonempty, unique subset of the 27 canonical task IDs") + return list(selected) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _safe_root(root: Path) -> Path: + root = root.expanduser().absolute() + for parent in (root, *root.parents): + if parent.is_symlink(): + raise ValueError(f"symlink forbidden in dataset path: {parent}") + return root + + +def _safe_task_root(root: Path, task: str) -> Path: + current = _safe_root(root) + for component in Path(task).parts: + current = current / component + if current.is_symlink(): + raise ValueError(f"symlink forbidden in dataset path: {current}") + return current + + +def _verify(path: Path, record: dict) -> dict: + if path.is_symlink() or not path.is_file(): + raise ValueError(f"asset must be a regular non-symlink file: {path}") + if path.stat().st_size != record["size"] or sha256(path) != record["sha256"]: + raise ValueError(f"asset size/hash mismatch: {path}") + return {"size": record["size"], "sha256": record["sha256"]} + + +def verify(task: str, root: Path) -> dict: + manifest = load_manifest() + verify_dataset_surface(root, manifest) + record = manifest["tasks"][task] + directory = _safe_task_root(Path(root), task) + allowed = set(record["assets"]) | {"checkpoint.json"} + try: + entries = {entry.name for entry in directory.iterdir()} + except OSError as exc: + raise ValueError(f"cannot enumerate dataset task root: {directory}") from exc + unexpected = sorted(entries - allowed) + if unexpected: + raise ValueError(f"unexpected dataset entries outside the input-only manifest: {unexpected}") + files = {name: _verify(directory / name, asset) for name, asset in record["assets"].items()} + return {"task": task, "dataset_revision": DATASET_REVISION, "files": files} + + +def verify_dataset_surface(root: Path, manifest: dict | None = None) -> None: + """Reject every path outside the declared input-only dataset surface.""" + root = Path(root).expanduser().absolute() + manifest = load_manifest() if manifest is None else manifest + allowed_files: set[Path] = set() + allowed_dirs: set[Path] = set() + for task, record in manifest["tasks"].items(): + task_path = Path(task) + allowed_dirs.update(task_path.parents) + allowed_dirs.add(task_path) + allowed_files.update(task_path / name for name in record["assets"]) + allowed_files.add(task_path / "checkpoint.json") + allowed_dirs.discard(Path(".")) + if root.is_symlink() or not root.is_dir(): + raise ValueError("dataset root must be a regular directory") + for path in root.rglob("*"): + relative = path.relative_to(root) + if path.is_symlink(): + raise ValueError(f"symlink forbidden in dataset root: {relative}") + if path.is_dir(): + allowed = relative in allowed_dirs + else: + allowed = path.is_file() and relative in allowed_files + if not allowed: + raise ValueError(f"unexpected path in input-only dataset root: {relative}") + + +def _download(url: str, destination: Path, record: dict) -> None: + partial = destination.with_name(destination.name + f".{uuid.uuid4().hex}.partial") + older = sorted(destination.parent.glob(destination.name + ".*.partial"), + key=lambda item: item.stat().st_size, reverse=True) + offset = 0 + try: + with partial.open("xb") as output: + if older: + if older[0].is_symlink() or older[0].stat().st_size > record["size"]: + raise ValueError("invalid retained partial download") + offset = older[0].stat().st_size + with older[0].open("rb") as source: + shutil.copyfileobj(source, output) + request = urllib.request.Request(url, headers={"Range": f"bytes={offset}-"}) if offset else url + if offset < record["size"]: + with urllib.request.urlopen(request) as response: + if offset: + expected = f"bytes {offset}-{record['size'] - 1}/{record['size']}" + if response.status != 206 or response.headers.get("Content-Range") != expected: + raise ValueError("server did not honor exact resume range") + while block := response.read(8 * 1024 * 1024): + output.write(block) + if output.tell() > record["size"]: + raise ValueError("download exceeds pinned size") + output.flush() + os.fsync(output.fileno()) + _verify(partial, record) + partial.replace(destination) + for retained in older: + retained.unlink() + except BaseException: + # Partial bytes are retained for explicit later resume and diagnosis. + raise + + +def stage(task: str, root: Path) -> Path: + manifest = load_manifest() + record = manifest["tasks"][task] + destination = _safe_task_root(Path(root), task) + destination.mkdir(parents=True, exist_ok=True) + state_path = destination / "checkpoint.json" + state = {"task": task, "dataset_revision": DATASET_REVISION, "status": "staging", "files": {}} + + def save() -> None: + temporary = state_path.with_name(state_path.name + f".{uuid.uuid4().hex}.tmp") + temporary.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(state_path) + + save() + try: + for name, asset in record["assets"].items(): + if name not in _ASSET_NAMES: + raise ValueError(f"unapproved asset name: {name}") + path = destination / name + if path.exists(): + details = _verify(path, asset) + details["cached"] = True + else: + url = (f"https://huggingface.co/datasets/{DATASET_REPOSITORY}/resolve/" + f"{DATASET_REVISION}/{urllib.parse.quote(asset['path'], safe='/')}") + _download(url, path, asset) + details = {**_verify(path, asset), "cached": False, "url": url} + state["files"][name] = details + save() + state["status"] = "verified" + state["verification"] = verify(task, root) + save() + except BaseException as exc: + state["status"] = "failed" + state["error"] = f"{type(exc).__name__}: {exc}" + save() + raise + return destination diff --git a/benchmarks/blenderbench_direct/initialization-manifest.json b/benchmarks/blenderbench_direct/initialization-manifest.json new file mode 100644 index 0000000..a99c9c0 --- /dev/null +++ b/benchmarks/blenderbench_direct/initialization-manifest.json @@ -0,0 +1,1944 @@ +{ + "schema_version": 1, + "revision": "203e4d325e9438ca55b29bdfc4f6a90842d74e68", + "tasks": { + "level1/camera1": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "8680900dd5a85c2467f86a7ec0c47e3868569e387948eb7e45aab65794981c06", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 1.0765, + -1.2848, + 1.4343 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 1.5701, + 0.0, + 0.6753 + ] + } + ] + }, + "level1/camera2": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "41cae2396027226fb32fd06a6e0a6e913bd895811bdee383416ef1bd1598d186", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -3.4813, + 4.5001, + 0.5115 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 1.6678, + 0.0, + 3.8693 + ] + } + ] + }, + "level1/camera3": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "f5acd7609ff23e01e3d435173d46ea45907fc2b3771dea91f5da0007e51500f2", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 6.7303, + -4.87, + 3.7385 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 1.2811, + -0.0, + 0.9355 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "data.lens", + "value": 50 + } + ] + }, + "level1/camera4": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "a994ae0f9c936a8a4a469b5245bc5cc438e7a112ec55288b7c2473998872d657", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 1.14, + 0.85, + 0.53 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 2.4435, + 1.117, + 2.6354 + ] + } + ] + }, + "level1/camera5": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "2d09f0ea2b51ace93663882d86aa2fa4201f394710879c12c6774173793a8971", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.225, + 0.71, + -2.83 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.4363, + 3.3161, + 0.0349 + ] + } + ] + }, + "level1/camera6": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "1539bdf69d7df4cb2da3770a59f2a33ed28d3d20aec3c3e2677351b297850f9c", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 2.4689, + -2.6258, + 1.3083 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 1.406, + 0.0, + 0.7626 + ] + } + ] + }, + "level1/camera7": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "5471be8faa7f4bd3ba2963a6716298a007d8a004a11df38cb87b9f026d9f4bbc", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 1.3353, + 1.474, + 1.5357 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.7679, + 0.2094, + 0.0 + ] + } + ] + }, + "level1/camera8": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "1dd7b06ac2d5949e0453f0e0c8ae857a417e986d9cc3b6fad748d26abf1b8151", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 7.3589, + -6.9258, + 4.9583 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 1.2664, + 0.0, + 0.78 + ] + } + ] + }, + "level1/camera9": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "6c104adc69f6e714e7b51d3e834b8bd7e6836813899ea84c972c6d2208ac40b6", + "task_description": "Adjust the camera position so that the viewing angle is consistent with the target image", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 0.1104, + -2.6639, + 9.8843 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + 0.3142, + 0, + 0 + ] + } + ] + }, + "level2/attribute1": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "57448a36ff834acc820d071fc99b26285d8968d4fece18300a5c512e18ac2f18", + "task_description": "First adjust the room brightness, then adjust the size of the character's belly so that it looks like the target image.", + "initialization": [ + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 1.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 1.2 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellySag", + "property": "value", + "value": 5 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellyShrink", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ShoulderWideness", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BackTaper", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestEnlarge", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestArea", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "MoreChin", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "Abs", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "eyelids", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "nose", + "property": "value", + "value": 0 + } + ] + }, + "level2/attribute2": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "83702d094c0fa4d6e4a29711dfa024f587339353b1f9ddd6aff5e17b3ab426a8", + "task_description": "First adjust the room brightness, then adjust the size of the character's belly so that it looks like the target image.", + "initialization": [ + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 0.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 400 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 0.8, + 0.9, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.5, + 0.7, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0, + 0 + ] + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellySag", + "property": "value", + "value": 5 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellyShrink", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ShoulderWideness", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BackTaper", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestEnlarge", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestArea", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "MoreChin", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "Abs", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "eyelids", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "nose", + "property": "value", + "value": 0 + } + ] + }, + "level2/attribute3": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "0bd075a70511e2952ff888d6a7d72499311a82764c7ab8c7bb7ae659ba210706", + "task_description": "First adjust the room brightness, then adjust the size of the character's belly so that it looks like the target image.", + "initialization": [ + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 57 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 0.85, + 0 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.2, + 0.6, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0.95, + 0.6 + ] + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellySag", + "property": "value", + "value": 5 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellyShrink", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ShoulderWideness", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BackTaper", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestEnlarge", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestArea", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "MoreChin", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "Abs", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "eyelids", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "nose", + "property": "value", + "value": 0 + } + ] + }, + "level2/attribute4": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "1991b217fd4e044f7ef0966bbb38e8e417bb1fa967effba61bbfc880f7e36543", + "task_description": "First adjust the brightness of the room, then adjust the position of the basketball so that it looks the same as the target image.", + "initialization": [ + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 1.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 1.2 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "basketball", + "property": "location", + "value": [ + 13.4023, + 3.7485, + 0.1584 + ] + } + ] + }, + "level2/attribute5": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "d80af70f035011ae1d6bf7001c9cad008e3cb63a6442bfbd87b23e0d4e09fb0a", + "task_description": "First adjust the brightness of the room, then adjust the position of the basketball so that it looks the same as the target image.", + "initialization": [ + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 0.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 400 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 0.8, + 0.9, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.5, + 0.7, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0, + 0 + ] + }, + { + "kind": "object", + "object": "basketball", + "property": "location", + "value": [ + 14.4023, + 2.7485, + 0.1584 + ] + } + ] + }, + "level2/attribute6": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "aadf4e84cd6b0e03c47f45cc1e1b6f12a436b6273ffd7f81cd3f2d28a4c072bf", + "task_description": "First adjust the brightness of the room, then adjust the position of the basketball so that it looks the same as the target image.", + "initialization": [ + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 57 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 0.85, + 0 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.2, + 0.6, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0.95, + 0.6 + ] + }, + { + "kind": "object", + "object": "basketball", + "property": "location", + "value": [ + 13.9023, + 3.0485, + 0.1584 + ] + } + ] + }, + "level2/attribute7": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "abd953c4d36252658613f4c385787dbe6baf4beedc346789ea0a6d210c49775b", + "task_description": "First move the cabinet to the correct position, then move the plant to the correct position. You need to move the cabinet to observe the plant in the mirror.", + "initialization": [ + { + "kind": "object", + "object": "Cabinet", + "property": "location", + "value": [ + 2.4661, + 14.201, + 0.1277 + ] + }, + { + "kind": "object", + "object": "Plant", + "property": "location", + "value": [ + 3.2168, + 14.327, + 0.57311 + ] + }, + { + "kind": "object", + "object": "Mirror", + "property": "location", + "value": [ + 0.1374, + 13.5573, + 1.396 + ] + }, + { + "kind": "object", + "object": "Sink", + "property": "location", + "value": [ + 1.7123, + 12.7161, + 0.8306 + ] + }, + { + "kind": "object", + "object": "Toilet", + "property": "location", + "value": [ + 1.9395, + 12.5101, + 0.4867 + ] + } + ] + }, + "level2/attribute8": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "06cc98b4dae682711a3b06f82ca6cd5a967c30915722b2d3ca57a445f6ae83db", + "task_description": "First move the cabinet to the correct position, then move the plant to the correct position. You need to move the cabinet to observe the plant in the mirror.", + "initialization": [ + { + "kind": "object", + "object": "Cabinet", + "property": "location", + "value": [ + 2.4661, + 14.201, + 0.1277 + ] + }, + { + "kind": "object", + "object": "Plant", + "property": "location", + "value": [ + 3.2168, + 14.327, + 0.57311 + ] + }, + { + "kind": "object", + "object": "Mirror", + "property": "location", + "value": [ + 0.1374, + 13.5573, + 1.396 + ] + }, + { + "kind": "object", + "object": "Sink", + "property": "location", + "value": [ + 1.7123, + 12.7161, + 0.8306 + ] + }, + { + "kind": "object", + "object": "Toilet", + "property": "location", + "value": [ + 1.9395, + 14.5101, + 0.4867 + ] + } + ] + }, + "level2/attribute9": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "956f2d67b54cda24edfeba5d5bfc766002efe018a4a77c0d505d4cf391410bf1", + "task_description": "First move the cabinet to the correct position, then move the plant to the correct position. You need to move the cabinet to observe the plant in the mirror.", + "initialization": [ + { + "kind": "object", + "object": "Cabinet", + "property": "location", + "value": [ + 2.4661, + 14.201, + 0.1277 + ] + }, + { + "kind": "object", + "object": "Plant", + "property": "location", + "value": [ + 3.2168, + 14.327, + 0.57311 + ] + }, + { + "kind": "object", + "object": "Mirror", + "property": "location", + "value": [ + 0.1374, + 13.5573, + 1.396 + ] + }, + { + "kind": "object", + "object": "Sink", + "property": "location", + "value": [ + 1.7123, + 14.7161, + 0.8306 + ] + }, + { + "kind": "object", + "object": "Toilet", + "property": "location", + "value": [ + 1.9395, + 14.5101, + 0.4867 + ] + } + ] + }, + "level3/attribute1": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "e9d1a0d7c912f9806f5879117403938995c1155d13c33af9a0c0c25068909769", + "task_description": "First adjust the room brightness, then adjust the size of the character's belly so that it looks like the target image. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.1222, + -0.3259, + 4.4354 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0873, + 0.5934, + 0.2094 + ] + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 1.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 1.2 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellySag", + "property": "value", + "value": 5 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellyShrink", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ShoulderWideness", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BackTaper", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestEnlarge", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestArea", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "MoreChin", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "Abs", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "eyelids", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "nose", + "property": "value", + "value": 0 + } + ] + }, + "level3/attribute2": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "0c115cf6fdbf43f677ebb5244e6afc9d75dd9635a9f4c6de06abe28a3f68f191", + "task_description": "First adjust the room brightness, then adjust the size of the character's belly so that it looks like the target image. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.1222, + -0.3259, + 4.4354 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0873, + 0.5934, + 0.2094 + ] + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 0.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 400 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 0.8, + 0.9, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.5, + 0.7, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0, + 0 + ] + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellySag", + "property": "value", + "value": 5 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellyShrink", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ShoulderWideness", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BackTaper", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestEnlarge", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestArea", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "MoreChin", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "Abs", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "eyelids", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "nose", + "property": "value", + "value": 0 + } + ] + }, + "level3/attribute3": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "de2d9d5cba1b04d58b1d698726c29968d8fbfed225c2b2b24f32f5f765a532ba", + "task_description": "First adjust the room brightness, then adjust the size of the character's belly so that it looks like the target image. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.1222, + -0.3259, + 4.4354 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0873, + 0.5934, + 0.2094 + ] + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 57 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 0.85, + 0 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.2, + 0.6, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0.95, + 0.6 + ] + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellySag", + "property": "value", + "value": 5 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BellyShrink", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ShoulderWideness", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "BackTaper", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestEnlarge", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "ChestArea", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "MoreChin", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "Abs", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "eyelids", + "property": "value", + "value": 0 + }, + { + "kind": "shape_key", + "datablock": "Key", + "key": "nose", + "property": "value", + "value": 0 + } + ] + }, + "level3/attribute4": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "bfce3b4e6919cdfa62ddb5893d42d64493d2357505ae83ece52d87a424ff0d10", + "task_description": "First adjust the brightness of the room, then adjust the position of the basketball so that it looks the same as the target image. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.1222, + -0.3259, + 4.4354 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0873, + 0.5934, + 0.2094 + ] + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 1.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 1.2 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 1, + 1 + ] + }, + { + "kind": "object", + "object": "basketball", + "property": "location", + "value": [ + 13.4023, + 3.7485, + 0.1584 + ] + } + ] + }, + "level3/attribute5": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "8fb7657a111ad845d72eaa3c1599f46f0ee1bb4c7b9b8e97b2858d7c2a71c215", + "task_description": "First adjust the brightness of the room, then adjust the position of the basketball so that it looks the same as the target image. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.1222, + -0.3259, + 4.4354 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0873, + 0.5934, + 0.2094 + ] + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 0.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 400 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 0.8, + 0.9, + 1 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.5, + 0.7, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0, + 0 + ] + }, + { + "kind": "object", + "object": "basketball", + "property": "location", + "value": [ + 14.4023, + 2.7485, + 0.1584 + ] + } + ] + }, + "level3/attribute6": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "6e1bed8235bb346af995c6f281b05f2d482c3f36dd0432a9df9ce62674dd7dbd", + "task_description": "First adjust the brightness of the room, then adjust the position of the basketball so that it looks the same as the target image. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + -1.1222, + -0.3259, + 4.4354 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0873, + 0.5934, + 0.2094 + ] + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.energy", + "value": 2.5 + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.energy", + "value": 57 + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.energy", + "value": 0 + }, + { + "kind": "object", + "object": "lamp_on_the_cupboard", + "property": "data.color", + "value": [ + 1, + 0.85, + 0 + ] + }, + { + "kind": "object", + "object": "lamp_on_shelf", + "property": "data.color", + "value": [ + 0.2, + 0.6, + 1 + ] + }, + { + "kind": "object", + "object": "light_on_ceilling", + "property": "data.color", + "value": [ + 1, + 0.95, + 0.6 + ] + }, + { + "kind": "object", + "object": "basketball", + "property": "location", + "value": [ + 13.9023, + 3.0485, + 0.1584 + ] + } + ] + }, + "level3/attribute7": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "88c03b7b328cb7251425acdde7959aa4cb9bea9add652c68faa5d442460fa4f9", + "task_description": "First move the cabinet to the correct position, then move the plant to the correct position. You need to move the cabinet to observe the plant in the mirror. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 0.2645, + 0.2215, + -2.9188 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0698, + 2.426, + 0.0 + ] + }, + { + "kind": "object", + "object": "Cabinet", + "property": "location", + "value": [ + 2.4661, + 14.201, + 0.1277 + ] + }, + { + "kind": "object", + "object": "Plant", + "property": "location", + "value": [ + 3.2168, + 14.327, + 0.57311 + ] + }, + { + "kind": "object", + "object": "Mirror", + "property": "location", + "value": [ + 0.1374, + 13.5573, + 1.396 + ] + }, + { + "kind": "object", + "object": "Sink", + "property": "location", + "value": [ + 1.7123, + 12.7161, + 0.8306 + ] + }, + { + "kind": "object", + "object": "Toilet", + "property": "location", + "value": [ + 1.9395, + 12.5101, + 0.4867 + ] + } + ] + }, + "level3/attribute8": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "6fdf229c8df32bdfd1700e90cd1c135e1eba67038effd3c766753a386e523c2a", + "task_description": "First move the cabinet to the correct position, then move the plant to the correct position. You need to move the cabinet to observe the plant in the mirror. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 0.2645, + 0.2215, + -2.9188 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0698, + 2.426, + 0.0 + ] + }, + { + "kind": "object", + "object": "Cabinet", + "property": "location", + "value": [ + 2.4661, + 14.201, + 0.1277 + ] + }, + { + "kind": "object", + "object": "Plant", + "property": "location", + "value": [ + 3.2168, + 14.327, + 0.57311 + ] + }, + { + "kind": "object", + "object": "Mirror", + "property": "location", + "value": [ + 0.1374, + 13.5573, + 1.396 + ] + }, + { + "kind": "object", + "object": "Sink", + "property": "location", + "value": [ + 1.7123, + 12.7161, + 0.8306 + ] + }, + { + "kind": "object", + "object": "Toilet", + "property": "location", + "value": [ + 1.9395, + 14.5101, + 0.4867 + ] + } + ] + }, + "level3/attribute9": { + "reviewed": true, + "review_basis": "Pinned task.txt and start.py only; imports and unused helper definitions discarded; literal assignments reviewed. No upstream code execution.", + "start_sha256": "bd47ca26dfd4b224a33766dcd40101cbf3f8b168773541d7444d6ea1c4039a6b", + "task_description": "First move the cabinet to the correct position, then move the plant to the correct position. You need to move the cabinet to observe the plant in the mirror. You need to adjust the camera angle so that you can see the object you want to modify.", + "initialization": [ + { + "kind": "object", + "object": "Camera1", + "property": "location", + "value": [ + 0.2645, + 0.2215, + -2.9188 + ] + }, + { + "kind": "object", + "object": "Camera1", + "property": "rotation_euler", + "value": [ + -0.0698, + 2.426, + 0.0 + ] + }, + { + "kind": "object", + "object": "Cabinet", + "property": "location", + "value": [ + 2.4661, + 14.201, + 0.1277 + ] + }, + { + "kind": "object", + "object": "Plant", + "property": "location", + "value": [ + 3.2168, + 14.327, + 0.57311 + ] + }, + { + "kind": "object", + "object": "Mirror", + "property": "location", + "value": [ + 0.1374, + 13.5573, + 1.396 + ] + }, + { + "kind": "object", + "object": "Sink", + "property": "location", + "value": [ + 1.7123, + 14.7161, + 0.8306 + ] + }, + { + "kind": "object", + "object": "Toilet", + "property": "location", + "value": [ + 1.9395, + 14.5101, + 0.4867 + ] + } + ] + } + } +} diff --git a/benchmarks/blenderbench_direct/protocol.py b/benchmarks/blenderbench_direct/protocol.py new file mode 100644 index 0000000..2790f63 --- /dev/null +++ b/benchmarks/blenderbench_direct/protocol.py @@ -0,0 +1,262 @@ +"""Canonical BlenderBench protocol manifests and provenance.""" +from __future__ import annotations + +import hashlib +import json +import os +import platform +from pathlib import Path +import re +import subprocess +from typing import Any + +TOOLS = ( + "execute_blender_code_for_cli", + "get_render_as_image_for_cli", + "get_runtime_python_api_docs_for_cli", + "search_api_docs", +) +PROTOCOL_VERSION = "blenderbench-direct-v1" +EVALUATOR_REPOSITORY = "https://github.com/Fugtemypt123/VIGA-release" +EVALUATOR_REVISION = "69cb8ef0651bf68124815682df4a0f8e8b57c141" +EVALUATOR_SHA256 = "9ee08818aa858dccfff3efbb36cb9a98e8887f14e6168ad1a23731351cdeb3b5" +CLIP_MODEL = "openai/clip-vit-base-patch32" +CLIP_REVISION = "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268" +CODEX_VERSION = "codex-cli 0.154.0" +CODEX_SOURCE_REVISION = "6b9826e3aa83b1a5947db50f4332cb9c65f1b340" +REPOSITORY = Path(__file__).resolve().parents[2] +IMPLEMENTATION_PATHS = ( + "benchmarks/blenderbench_direct/dataset.py", + "benchmarks/blenderbench_direct/protocol.py", + "benchmarks/blenderbench_direct/runner.py", + "benchmarks/blenderbench_direct/ref_based_eval.py", + "benchmarks/blenderbench_direct/score_pair.py", + "benchmarks/blenderbench_direct/scoring.py", + "benchmarks/blenderbench_direct/suite.py", + "mcp/blmcp/tools/execute_blender_code.py", + "mcp/blmcp/tools_helpers/blender_cli.py", + "tools/direct_codex.py", + "tools/direct_runtime.py", +) + + +def canonical_bytes(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=False, allow_nan=False).encode("utf-8") + + +def digest(value: Any) -> str: + return hashlib.sha256(canonical_bytes(value)).hexdigest() + + +def redact(value: Any, key: str = "") -> Any: + normalized = re.sub(r"[^a-z0-9]+", "_", key.lower()).strip("_") + if normalized in {"args", "env", "variables"}: + return "" + sensitive = re.compile( + r"(^|_)(token|secret|password|passwd|pwd|credential|authorization|auth|api_?key|pat|cookie|session)($|_)") + if sensitive.search(normalized): + return "" + if isinstance(value, dict): + return {name: redact(child, str(name)) for name, child in value.items()} + if isinstance(value, list): + return [redact(child, key) for child in value] + if isinstance(value, str) and (re.search(r"(?i)\bbearer\s+\S+", value) + or re.search(r"(?i)https?://[^/@\s:]+:[^@\s]+@", value) + or re.search(r"(?i)[?&](?:token|key|secret|signature|auth)=[^&\s]+", value)): + return "" + return value + + +def read_mcp_config(path: Path) -> dict: + import tomllib + return tomllib.loads(Path(path).read_text(encoding="utf-8")) + + +def resolved_version(binary: Path, *arguments: str) -> str: + return subprocess.check_output([str(binary), *arguments], text=True, + stderr=subprocess.STDOUT).strip() + + +def runtime_provenance() -> dict: + cpu = platform.processor() or platform.machine() + provenance = { + "platform": platform.platform(), + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "cpu": cpu, + "python": platform.python_version(), + "python_implementation": platform.python_implementation(), + "logical_cpu_count": os.cpu_count(), + } + try: + provenance["gpu"] = resolved_version(Path("nvidia-smi"), + "--query-gpu=name,driver_version,memory.total", "--format=csv,noheader") + except (OSError, subprocess.SubprocessError): + provenance["gpu"] = None + return provenance + + +def collect_source_identity(repository: Path = REPOSITORY) -> dict: + """Bind the protocol to source bytes and Git state without host-specific data.""" + repository = Path(repository).resolve() + initialization = repository / "benchmarks/blenderbench_direct/initialization-manifest.json" + assets = repository / "benchmarks/blenderbench_direct/assets-manifest.json" + clip_manifest_path = repository / "benchmarks/blenderbench_direct/clip-snapshot-manifest.json" + scoring_lock = repository / "benchmarks/blenderbench_direct/requirements-scoring.lock" + implementation = {} + for relative in IMPLEMENTATION_PATHS: + path = repository / relative + if path.is_symlink() or not path.is_file(): + raise ValueError(f"protocol implementation is missing or a symlink: {relative}") + implementation[relative] = hashlib.sha256(path.read_bytes()).hexdigest() + try: + revision = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=repository, text=True, + stderr=subprocess.STDOUT).strip() + status = subprocess.check_output( + ["git", "status", "--porcelain=v1", "--untracked-files=all"], + cwd=repository, text=True, stderr=subprocess.STDOUT) + except (OSError, subprocess.SubprocessError) as exc: + raise ValueError("protocol source must be inside a readable Git checkout") from exc + if len(revision) != 40: + raise ValueError("Git revision must be a full 40-hex commit") + assets_bytes = assets.read_bytes() + clip_manifest_bytes = clip_manifest_path.read_bytes() + try: + assets_manifest = json.loads(assets_bytes) + clip_manifest = json.loads(clip_manifest_bytes) + except json.JSONDecodeError as exc: + raise ValueError("assets manifest is not valid JSON") from exc + return { + "assets_manifest": assets_manifest, + "assets_manifest_sha256": hashlib.sha256(assets_bytes).hexdigest(), + "clip_snapshot_manifest": clip_manifest, + "clip_snapshot_manifest_sha256": hashlib.sha256(clip_manifest_bytes).hexdigest(), + "initialization_manifest_sha256": hashlib.sha256(initialization.read_bytes()).hexdigest(), + "scoring_lock_sha256": hashlib.sha256(scoring_lock.read_bytes()).hexdigest(), + "implementation_sha256": implementation, + "git_revision": revision, + "git_dirty": bool(status), + } + + +def build_manifest(config: dict, *, dataset_revision: str, evaluator_revision: str, + model_revision: str | None, versions: dict, + provenance: dict | None = None, source_identity: dict | None = None) -> dict: + public_config = redact(config) + protocol = { + "protocol_version": PROTOCOL_VERSION, + "dataset_revision": dataset_revision, + "evaluator_revision": evaluator_revision, + "evaluator_sha256": EVALUATOR_SHA256, + "clip_model": CLIP_MODEL, + "clip_revision": CLIP_REVISION, + "model": config.get("model"), + "model_revision": model_revision, + "rounds": config.get("rounds"), + "tools": list(TOOLS), + "generation": { + "reasoning": "direct model reasoning", + "sequential_meaningful_saved_rounds": config.get("rounds"), + "shell_scene_operations": False, + "goal_code_or_solution_access": False, + "vlm_judge": False, + "score_feedback_to_generation": False, + "saved_scene_audit_timing": "after generation", + }, + "rendering": {"engine": "Cycles", "device": "GPU", "cpu_fallback": False}, + "versions": versions, + "config": public_config, + "source_identity": (collect_source_identity() if source_identity is None + else source_identity), + } + return { + "schema_version": 1, + "protocol": protocol, + "protocol_sha256": digest(protocol), + "config": public_config, + "versions": versions, + "provenance": provenance if provenance is not None else runtime_provenance(), + } + + +def validate_run_manifest(manifest: dict, *, tasks: list[str], rounds: int) -> dict: + """Validate a generation manifest before recovery or standalone scoring.""" + if not isinstance(manifest, dict) or manifest.get("schema_version") != 1: + raise ValueError("run manifest schema is invalid") + body = manifest.get("protocol") + if not isinstance(body, dict) or manifest.get("protocol_sha256") != digest(body): + raise ValueError("run manifest protocol hash is invalid") + if (body.get("protocol_version") != PROTOCOL_VERSION + or body.get("dataset_revision") != __import__( + "benchmarks.blenderbench_direct.dataset", fromlist=["DATASET_REVISION"]).DATASET_REVISION + or body.get("evaluator_revision") != EVALUATOR_REVISION + or body.get("evaluator_sha256") != EVALUATOR_SHA256 + or body.get("rounds") != rounds + or body.get("config", {}).get("tasks") != tasks): + raise ValueError("run manifest does not match the requested frozen protocol") + expected_generation = { + "reasoning": "direct model reasoning", + "sequential_meaningful_saved_rounds": rounds, + "shell_scene_operations": False, + "goal_code_or_solution_access": False, + "vlm_judge": False, + "score_feedback_to_generation": False, + "saved_scene_audit_timing": "after generation", + } + config = body.get("config", {}) + versions = body.get("versions") + frozen_rules_match = ( + body.get("clip_model") == CLIP_MODEL + and body.get("clip_revision") == CLIP_REVISION + and body.get("tools") == list(TOOLS) + and body.get("generation") == expected_generation + and body.get("rendering") == {"engine": "Cycles", "device": "GPU", "cpu_fallback": False} + and body.get("model") == config.get("model") + and body.get("model_revision") == config.get("model_revision") + and isinstance(versions, dict) + and versions.get("codex") == CODEX_VERSION + and versions.get("scorer_python") == "Python 3.13.15" + and isinstance(versions.get("bpy"), str) + and versions["bpy"].startswith(str(config.get("bpy_version", ""))) + ) + if not frozen_rules_match: + raise ValueError("run manifest does not match the requested frozen protocol") + source = body.get("source_identity") + current = collect_source_identity() + required = {"assets_manifest", "assets_manifest_sha256", "clip_snapshot_manifest", + "clip_snapshot_manifest_sha256", "initialization_manifest_sha256", + "scoring_lock_sha256", "implementation_sha256", "git_revision", "git_dirty"} + if not isinstance(source, dict) or not required.issubset(source): + raise ValueError("run manifest source identity is incomplete") + if source["git_dirty"] is not False: + raise ValueError("publication run manifest requires a clean Git source identity") + for key in ("assets_manifest", "assets_manifest_sha256", "clip_snapshot_manifest", + "clip_snapshot_manifest_sha256", "initialization_manifest_sha256", + "scoring_lock_sha256", "implementation_sha256", "git_revision"): + if source.get(key) != current[key]: + raise ValueError(f"run manifest {key} differs from local scoring protocol") + return manifest + + +def load_run_manifest(run_root: Path, *, tasks: list[str], rounds: int) -> dict: + path = Path(run_root) / "run-manifest.json" + if path.is_symlink() or not path.is_file(): + raise ValueError("standalone scoring requires a regular run-manifest.json") + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError("run-manifest.json is unreadable") from exc + return validate_run_manifest(manifest, tasks=tasks, rounds=rounds) + + +def write_manifest(path: Path, manifest: dict) -> Path: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8") + temporary.replace(path) + return path diff --git a/benchmarks/blenderbench_direct/publication/artifact-manifest.json b/benchmarks/blenderbench_direct/publication/artifact-manifest.json new file mode 100644 index 0000000..3780646 --- /dev/null +++ b/benchmarks/blenderbench_direct/publication/artifact-manifest.json @@ -0,0 +1,3943 @@ +{ + "compilation_video": { + "bytes": 10161884, + "codec": "h264", + "duration_seconds": 134.9, + "height": 464, + "publication_status": "published", + "sample_aspect_ratio": "1:1", + "sha256": "04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17", + "url": "https://huggingface.co/datasets/michaelgold/blenderbench-direct-results/resolve/main/video/blenderbench-complete-compilation.mp4", + "width": 848 + }, + "run_records": { + "camera1-cuda13-02": { + "process.json": { + "bytes": 97, + "sha256": "dc8bfbf1fc0acd387e3a2026a083ebead76a406c6969af75aaaedaec86d670f8" + } + }, + "remaining-direct-03": { + "events.jsonl": { + "bytes": 12900, + "sha256": "a57e0c82694eb6acb01f3bb9f44ae9ac7058e1f9a6eb256bf95f91db184c1a9b" + }, + "process.json": { + "bytes": 850, + "sha256": "4a88a364b25b8b949879516b519fd85c359450aac333e923ce82183c9dbc12c3" + }, + "status.json": { + "bytes": 4244, + "sha256": "0cc076a1213c40de4bb0894e266ca79c779d9cc14151a4b35692f11e5b345942" + } + }, + "unfinished-live-edit-00ba68b-01": { + "dispatcher.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "events.jsonl": { + "bytes": 4894, + "sha256": "509313c4de6f1d959a03daeac9af1b0e4440c5597c31a5997362fc53165ba568" + }, + "launcher.json": { + "bytes": 1068, + "sha256": "0c221f3d5f44259ad4ac7d34e39b4ecfabea5d5ec704dd873d58bcdbd0d67ed3" + }, + "process.json": { + "bytes": 1217, + "sha256": "0ce3469195af0fc9be38486b9a7e9d4560e2b1038c9f8d5ed42a1a69f298cdc1" + }, + "restart-manifest.json": { + "bytes": 41505, + "sha256": "9311e25520fe8c65495cdb0bdc73f9b38ec4ac97ab97fd63069f41bc521125df" + }, + "status.json": { + "bytes": 1521, + "sha256": "5a919bb870086aa315e3e3849626baeffe169e987653c4670010afce9bca3a6d" + } + }, + "unrestricted-4a9f5cc-01": { + "dispatcher.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "events.jsonl": { + "bytes": 8470, + "sha256": "06ffca3000afa683551cea4628eff4f2eb8f89ca21fdf934cb6e61d71ac41be6" + }, + "launcher.json": { + "bytes": 1070, + "sha256": "9c4637dd2bb1ff0c5486cf05eff9fa7343031beb09a2f95354ea9747830db867" + }, + "process.json": { + "bytes": 1260, + "sha256": "4d575179fef242cdc8975a0ba8723db56db7c9336bfa72c3ff0d865847b184ec" + }, + "restart-manifest.json": { + "bytes": 53623, + "sha256": "f5057c3b1eec35b9873655025869165a769a8f3acdcce85584b5dd9a1b0e658c" + }, + "status.json": { + "bytes": 2433, + "sha256": "665fb60beceeab413e328f5008f67aa33b35146bc6b56e78a35bb176ef008d4b" + } + } + }, + "schema_version": 1, + "tasks": { + "level1/camera1": { + "evidence": [ + { + "bytes": 7326712, + "kind": "codex.jsonl", + "sha256": "5d7bd4bba7ebc24ecf08cd17f98e3855045b0742a50014f4c7df3038da5a701f" + } + ], + "rounds": [ + { + "blend_bytes": 12361923, + "blend_sha256": "d9beb3448fc8832615dfb919635ef56a0312fb95e852bf31655889527ebc3b6a", + "clip_similarity": 0.9618514776229858, + "nclip": 0.03814852237701416, + "photometric_loss": 0.05728062987327576, + "render": { + "bytes": 266969, + "sha256": "d3cb9813561664b36f44db06b374c06c013e527d13f5a49b8e0b33580f22b93e" + }, + "round": 1 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "814b2814076b89bcf332b3eb04f9052721fca718976a7d3f909f410a5739cef8", + "clip_similarity": 0.9618485569953918, + "nclip": 0.038151443004608154, + "photometric_loss": 0.022054409608244896, + "render": { + "bytes": 263304, + "sha256": "ffa63c8e500a8f6db3dcce94da9bcca07eeeabab4795d51b90f130033213abde" + }, + "round": 2 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "9a2fdbb8d6d0f03f25c64fc79ad4d725adcf87854e749d0c1ed2e7e50bc65584", + "clip_similarity": 0.9832024574279785, + "nclip": 0.016797542572021484, + "photometric_loss": 0.025472132489085197, + "render": { + "bytes": 273660, + "sha256": "2b96552a076cc29c4af3aaa5038f468287362279e23635fc8e9202ea90720111" + }, + "round": 3 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "68bba553d0c2253e854ab3c99c8192924ecc460643f33078460d95ca7049d390", + "clip_similarity": 0.9919335246086121, + "nclip": 0.00806647539138794, + "photometric_loss": 0.011989844031631947, + "render": { + "bytes": 270480, + "sha256": "7d98214714d6116a040636cce4448a69f4132f78cba01478965193f65ef4f574" + }, + "round": 4 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "e5676ad17577a29ac43ff68d61ec072fc70315e9f740dad58dc9dc85f068c707", + "clip_similarity": 0.9911091923713684, + "nclip": 0.008890807628631592, + "photometric_loss": 0.015071357600390911, + "render": { + "bytes": 277688, + "sha256": "47ea8dde2c78614077998a0e05a200baf974a7fc0bd7ed26b89fd91f5e128041" + }, + "round": 5 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "662efa0ec1caa119a053bc1328e9b6fec04942107a0e206622857ea5e7bad2cf", + "clip_similarity": 0.99659264087677, + "nclip": 0.0034073591232299805, + "photometric_loss": 0.00529375160112977, + "render": { + "bytes": 272724, + "sha256": "814eb8959c84da949affa35db6c042981966f3d5486195bc09bfce8ffd685bba" + }, + "round": 6 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "e0117ca89cbf48fe0bf89e22800244e1a7d0adf7d11d1c4bea486ed7daa3dcaf", + "clip_similarity": 0.9975831508636475, + "nclip": 0.002416849136352539, + "photometric_loss": 0.003328760154545307, + "render": { + "bytes": 273059, + "sha256": "299bf354c236162575d35f68b73bf1bb802ef94bbd4bdb391fc20df830f39b3c" + }, + "round": 7 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "c9c465783a06a6224f8c0b3e2690c5982823866bdc922d6133e64ccac29a5c07", + "clip_similarity": 0.9963770508766174, + "nclip": 0.0036229491233825684, + "photometric_loss": 0.004889530595391989, + "render": { + "bytes": 274419, + "sha256": "8e3d6d3cbd57a9b72db566e55c087b0b311ab946ab9e97f9c09c0b081f25d801" + }, + "round": 8 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "2f7c02277025caf472dca101022f4b663e520e8a2138e43e82e7417e7922ce83", + "clip_similarity": 0.9973839521408081, + "nclip": 0.0026160478591918945, + "photometric_loss": 0.0037112270947545767, + "render": { + "bytes": 274317, + "sha256": "12dffa0deaa846e21873a0f760e4114d16bcd208eef72844503470fe6d9dbbba" + }, + "round": 9 + }, + { + "blend_bytes": 12361923, + "blend_sha256": "0c08683a1a8ba385b6a7dfcb59429cf2385dbffdd9a5af1073a0f2a1d7a6510e", + "clip_similarity": 0.9968132972717285, + "nclip": 0.0031867027282714844, + "photometric_loss": 0.0055841426365077496, + "render": { + "bytes": 274254, + "sha256": "dee437f3d4ba9a224f5ed1a161e996226d8e26376f55d5335c0bc7ed4d454552" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7843, + "sha256": "bf7181ba8c0cab76c7a4e17d4154ebc54dee9bf1d28c12140b89792d705f2543" + }, + "target": { + "bytes": 280875, + "sha256": "a1f17f06448cd899c5912323f9b9dd07ca6ea30dc0aebba11f28cb85afb29d2c" + } + }, + "level1/camera2": { + "evidence": [ + { + "bytes": 571409, + "kind": "events.jsonl", + "sha256": "126562bb77f9149e61809366231ae28cd4a211b62a185e5644791652351fd329" + }, + { + "bytes": 6694522, + "kind": "run.jsonl", + "sha256": "d5b59e3fbc5bfed853de242d35145b8c4f4dbb64673a5e48e0be2701b196c622" + } + ], + "rounds": [ + { + "blend_bytes": 20761447, + "blend_sha256": "233c85503fe77a330874cb0c5f90ff1c81a3467db56c04c99bb2c881bc8456b1", + "clip_similarity": 0.9183803200721741, + "nclip": 0.08161967992782593, + "photometric_loss": 0.10068005323410034, + "render": { + "bytes": 248901, + "sha256": "5a492d0b1f63de8d5a8849e8039e8b5764312edc1d2b3d3d45d231ae5770cd8e" + }, + "round": 1 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "2e775e2359dd6992f506d62249e465e5fcfc9867582c7a5fa0fbab88e38109c1", + "clip_similarity": 0.9619098901748657, + "nclip": 0.03809010982513428, + "photometric_loss": 0.09336703270673752, + "render": { + "bytes": 236872, + "sha256": "b5b8d5334b404b9b7c42937e28bf589c07180b3d53b858624bf43d4d932a7ce3" + }, + "round": 2 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "2a67d0599e5338d9a3920a35bb67a3762af6d4076b704674a90d61c418d14897", + "clip_similarity": 0.9870508909225464, + "nclip": 0.012949109077453613, + "photometric_loss": 0.04738546535372734, + "render": { + "bytes": 248710, + "sha256": "4d3e9c637018e98f695b50445e5e7784d98cab5f0438ece258ef7076baa28d2e" + }, + "round": 3 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "23bec1ec272c76a8c4fe080e537a83f8b781264ddc23c6ef9b2b8f10f2946fcc", + "clip_similarity": 0.9788923263549805, + "nclip": 0.02110767364501953, + "photometric_loss": 0.04208802059292793, + "render": { + "bytes": 245617, + "sha256": "1c57bbb4037dd996ca84286bdb4c3b8272eab9e2d5f63f05141de0b047a187d6" + }, + "round": 4 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "6e2fc6d97fc65aae2d4d3acb1369d5cb99459d5c4d996e7609c8aa4a8a64a448", + "clip_similarity": 0.9878735542297363, + "nclip": 0.012126445770263672, + "photometric_loss": 0.02797190099954605, + "render": { + "bytes": 244309, + "sha256": "2bf2caead17e5ea78291fc7538d508d130dedce461f6dbfe9dcfb645db4e64c3" + }, + "round": 5 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "760a579c3771d74877f95d008fd6002a9fff03a076222bf4fb38feb745ba93b7", + "clip_similarity": 0.9910882711410522, + "nclip": 0.008911728858947754, + "photometric_loss": 0.018511703237891197, + "render": { + "bytes": 246201, + "sha256": "10cf415f645a87e053e95cb3acd0b47d70970e1c719acb83fee85cf90fbeaa07" + }, + "round": 6 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "d43438a5de61c90c9d925ee2412ac9264331f64189810bda425e6b2321b9a77d", + "clip_similarity": 0.9913892149925232, + "nclip": 0.008610785007476807, + "photometric_loss": 0.01795218326151371, + "render": { + "bytes": 246916, + "sha256": "419f20a91a86f603e1966321c7eefba55cb957c833af728ce646865df757823c" + }, + "round": 7 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "54c37f532740417bc72731db98e5fa5b49099b0abe2c22bee11676a518dc9031", + "clip_similarity": 0.9955885410308838, + "nclip": 0.004411458969116211, + "photometric_loss": 0.0033672682475298643, + "render": { + "bytes": 247144, + "sha256": "a57c3b551000f7cccf675746cb78034235b6d1e77510e44f2f8e5a9e22c70508" + }, + "round": 8 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "8cc4984cfdf2eec05e57d9a39d572226057f5641828a3be8f9e84c5fb7efa827", + "clip_similarity": 0.9925321340560913, + "nclip": 0.007467865943908691, + "photometric_loss": 0.005620954558253288, + "render": { + "bytes": 246638, + "sha256": "2e4b88adb5021caf87301a34b17866f939a5f6f3366caf8510932ea45793b905" + }, + "round": 9 + }, + { + "blend_bytes": 20761447, + "blend_sha256": "42ce90379cc6fa4e0ab21510835936b3ecf49ebb9fbd242891fbd5ce7cd6f32e", + "clip_similarity": 0.9927474856376648, + "nclip": 0.007252514362335205, + "photometric_loss": 0.0010773908579722047, + "render": { + "bytes": 245908, + "sha256": "66c52097be62dd3d74c6ad9cb05fa1abb7f8d6146345cdd9627779b12ed6b907" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7416, + "sha256": "f27fbc7dba4c20af2ff3b3c94e8fc5bf0b337595cb9e23fcdd38b75cabe5a976" + }, + "target": { + "bytes": 256033, + "sha256": "ce58bcf86a6866d992e9d115391cf4a411b65395ffd612ccb7d28dc7ba5e5b58" + } + }, + "level1/camera3": { + "evidence": [ + { + "bytes": 364327, + "kind": "events.jsonl", + "sha256": "e55b23f45382aa3e4c9770a077fabbc69298fa0ffa178855e931ee4fe2c840f7" + }, + { + "bytes": 8656277, + "kind": "run.jsonl", + "sha256": "bba451a2de8fa3399f149f6dd2de2569b18feac595db8551a87624a273c8600d" + } + ], + "rounds": [ + { + "blend_bytes": 20490390, + "blend_sha256": "37eafd16726748a941daf10a131817b36c637d03458a73012d8898b94fd8075f", + "clip_similarity": 0.8642843961715698, + "nclip": 0.13571560382843018, + "photometric_loss": 0.060653116554021835, + "render": { + "bytes": 301782, + "sha256": "4fd322c531a97268f17a16fae1bd753b47f33abe61c583a953efd104e5701f52" + }, + "round": 1 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "bd8179e20bad9fd4b9f3387f41a8e97d5aa3a8d6459180dae0ede85407b05862", + "clip_similarity": 0.8662276268005371, + "nclip": 0.1337723731994629, + "photometric_loss": 0.10488191992044449, + "render": { + "bytes": 335255, + "sha256": "5bc345dfd77bc9044aa7c0ef39bf4c1047f6c96182404387e7fa0f99d522f9c1" + }, + "round": 2 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "2ce73c817bf030b1e65deaf9dbd7db52ad6cfe13fa52a60bb7febb82d6aa0593", + "clip_similarity": 0.9695447683334351, + "nclip": 0.03045523166656494, + "photometric_loss": 0.05925656855106354, + "render": { + "bytes": 318052, + "sha256": "cb7054895001715c9fafd5861f149abb2fa63b9c550b54909723fdbe8f2d1dae" + }, + "round": 3 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "75f53a1dc1e1c827e0958124904bc1092398c6c12c51b79e9e5134f137d24677", + "clip_similarity": 0.9561019539833069, + "nclip": 0.043898046016693115, + "photometric_loss": 0.054005783051252365, + "render": { + "bytes": 336085, + "sha256": "9dc252a3911c815b6a89feea0a208b334e70ed6223b76004fd4d231963e7c33d" + }, + "round": 4 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "45240ebea9fea5d92a19911fbbc7a224d77988eedee37c19575226efddab419e", + "clip_similarity": 0.9670233726501465, + "nclip": 0.032976627349853516, + "photometric_loss": 0.03750957176089287, + "render": { + "bytes": 323442, + "sha256": "8197f7c2f9c642e364f451c565ed25afcdbaf0861addc74d3a47682a86943d8e" + }, + "round": 5 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "644fe7869944f8b513646c76b47a74db69ea7e2eb183dd4a60bd89ba0b823548", + "clip_similarity": 0.9768242835998535, + "nclip": 0.023175716400146484, + "photometric_loss": 0.03000887669622898, + "render": { + "bytes": 323758, + "sha256": "7906c731fd7c00d796e20f7e94dad6ea7d2adf2af167c109d4f7223ffcd63cfa" + }, + "round": 6 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "d4c9d0261b3c8a5336e647d7be9e95506d8fcd917d4326c8f312247fd9f63192", + "clip_similarity": 0.9761712551116943, + "nclip": 0.023828744888305664, + "photometric_loss": 0.03189892694354057, + "render": { + "bytes": 319815, + "sha256": "c2655f30a27023df0a0a12f6825b217bc53c832966ff77471193af99f07bf8d1" + }, + "round": 7 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "05905d9f2b955e2b35eaec28d935c308b7e88fc40bd2336f5acc183abc303ee3", + "clip_similarity": 0.9916093945503235, + "nclip": 0.008390605449676514, + "photometric_loss": 0.0070646158419549465, + "render": { + "bytes": 318353, + "sha256": "f5fe718a162ce67da40123ee1e527621c70611381b89f887e3b479cbfb0d6135" + }, + "round": 8 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "4d63215e8d4d45c66d93c04fcd504850c6d9116fb97dcb64baccfc9c4b7a2da2", + "clip_similarity": 0.9977397918701172, + "nclip": 0.0022602081298828125, + "photometric_loss": 0.001466981484554708, + "render": { + "bytes": 320827, + "sha256": "75f08bba423e0d425b706de3f17c642813356ebf7d065c4396f1c944d26451c9" + }, + "round": 9 + }, + { + "blend_bytes": 20490390, + "blend_sha256": "121032ad79be1286e40cf0de7fc299d2a4e160c4e82b54e4c6c82b7af3824240", + "clip_similarity": 0.9840778708457947, + "nclip": 0.015922129154205322, + "photometric_loss": 0.016791703179478645, + "render": { + "bytes": 318141, + "sha256": "cb83d983a80f1468917d40db1b925e7155f4702ba2df8b98d708fa64064351b2" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7656, + "sha256": "f8bbf09c1ef219f1ce5cd6adea15ab29fb63c3e2b41c2ed2b140c118f656a0b9" + }, + "target": { + "bytes": 323359, + "sha256": "3cde2e268b0fb73a25bc96180bbe96818f464bbe71fc8fe9bc1dff8a3d782d18" + } + }, + "level1/camera4": { + "evidence": [ + { + "bytes": 1628407, + "kind": "events.jsonl", + "sha256": "b76e78da802fff805d6e18f9b53e7bb307406e2e34fecb8e6939008ebaf0c595" + }, + { + "bytes": 10628475, + "kind": "run.jsonl", + "sha256": "c01d49aa6fa22afbd0a5ca59d6a89bb04b7f5b226eae72230c27426f340966cd" + } + ], + "rounds": [ + { + "blend_bytes": 932877522, + "blend_sha256": "749fd7e76f7977f37367f675d475ab6468e1100c5b90baacd4b47b34ed850bf7", + "clip_similarity": 0.9709898233413696, + "nclip": 0.02901017665863037, + "photometric_loss": 0.04666049778461456, + "render": { + "bytes": 435845, + "sha256": "b806926ab498c58bc1a7acf4644dfb2ee410b059902c57f6c62d398c630a8bee" + }, + "round": 1 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "7a17a766b3d72e9ee7de55a474f534bd6ff01b49c87a8b7df32a6088376c1ec1", + "clip_similarity": 0.9746537208557129, + "nclip": 0.02534627914428711, + "photometric_loss": 0.04516137018799782, + "render": { + "bytes": 439810, + "sha256": "cada208ea5c020ba1f30c66e8b28b90118a907d3ac5e416de821904a6a167f8e" + }, + "round": 2 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "ca0abf496ea19efbd477f9d14713cf055991ace54cd79555f1dbd2dd51fe53d1", + "clip_similarity": 0.937958836555481, + "nclip": 0.06204116344451904, + "photometric_loss": 0.02753789909183979, + "render": { + "bytes": 435640, + "sha256": "d7bb56d780f7bb38a974e278e5d136c506b4375e0f4ab9d125d0da87ea3451a0" + }, + "round": 3 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "e2a73943cd5af94c3768e12468d57a1a02b8fcf75ffe530c5a2ba299f298bf1a", + "clip_similarity": 0.9316172003746033, + "nclip": 0.06838279962539673, + "photometric_loss": 0.06786150485277176, + "render": { + "bytes": 390435, + "sha256": "dc36306bc2dabd0f38bee9cf6f5b4c3dfaa50a983c25a97697493210dece28ae" + }, + "round": 4 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "ca8e4646c0431f1fa4611316eb82be51f9b89721ae65f60f1ffcaacf3f7b9a2a", + "clip_similarity": 0.9833697080612183, + "nclip": 0.01663029193878174, + "photometric_loss": 0.013681826181709766, + "render": { + "bytes": 437084, + "sha256": "3840170f9ec9783f34029d3cf9bb7274156b07f2f5bdcdce3326820cf6cc4fb0" + }, + "round": 5 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "0d29ebecc3f26a5c7c853803b9c441fa7027fd7f1308069c87becd97846dc6b1", + "clip_similarity": 0.9912315011024475, + "nclip": 0.00876849889755249, + "photometric_loss": 0.007649693172425032, + "render": { + "bytes": 436351, + "sha256": "c9aef7b4746f2cdec26b5a36d1758f9aecbb46cc2a222207db1f8fae4d610aab" + }, + "round": 6 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "394265801ba272228c7f2c27cf5ab36cef1a1a02b5bb47dbbe23d02399113262", + "clip_similarity": 0.9908938407897949, + "nclip": 0.009106159210205078, + "photometric_loss": 0.008667061105370522, + "render": { + "bytes": 436440, + "sha256": "303a87a6593a0ba8b22d74a432c334ece1c47f068bd03a33bf296276a166a550" + }, + "round": 7 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "2925f0d270302a5d54aebe89aefeaebeef511a0937bb92603580e0c5d0e0f220", + "clip_similarity": 0.986689567565918, + "nclip": 0.013310432434082031, + "photometric_loss": 0.008767171762883663, + "render": { + "bytes": 435621, + "sha256": "6a0e99de10e14533dc5a67568619371a4a45bd62fb1cd16b9aec855a70907ccb" + }, + "round": 8 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "59b552ec0b8ab59c766fddf506a76666da68a8d244d1389d82c42e9471c195ab", + "clip_similarity": 0.9909461736679077, + "nclip": 0.009053826332092285, + "photometric_loss": 0.006298090796917677, + "render": { + "bytes": 436021, + "sha256": "8c17604a3ada95989614ae464e7363863c2d8b3dd828e997766c7ec2f7b1bc16" + }, + "round": 9 + }, + { + "blend_bytes": 932877522, + "blend_sha256": "4a54852b6b216c35c8eee7c55e8eddda5a3e42a7cfb1730f20c056e11eb922e6", + "clip_similarity": 0.9899377226829529, + "nclip": 0.01006227731704712, + "photometric_loss": 0.004026280716061592, + "render": { + "bytes": 435960, + "sha256": "e964e2e9323e46c01bee2082c3712dfb9283e100c8f430ac32c5c51184980f52" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7410, + "sha256": "7337a36ef1cabf7dcece460aa8a704597e579c2c65ec1f8e8393ae3c1932541e" + }, + "target": { + "bytes": 434190, + "sha256": "a6cad78bdebb96be3147176d6c4c460df4782c51430fcd268dfddd87f499aa93" + } + }, + "level1/camera5": { + "evidence": [ + { + "bytes": 1630375, + "kind": "events.jsonl", + "sha256": "3fea0213ce7070dc06fca06a24ae52636119854441eceb007d2397b9d4544c8f" + }, + { + "bytes": 10575004, + "kind": "run.jsonl", + "sha256": "214d476156d3c42aa1a9e78b3ac219bdfd32f2d68e40a5a84b80d8a1c25b84ee" + } + ], + "rounds": [ + { + "blend_bytes": 932941898, + "blend_sha256": "2b0c597fa140a618225d3b200ec26b37e44965f6c24ed6ad0774e00a5aa3a8fc", + "clip_similarity": 0.9789350628852844, + "nclip": 0.021064937114715576, + "photometric_loss": 0.028528744354844093, + "render": { + "bytes": 436832, + "sha256": "108cd9f5188e9adb662e7cdaa1344a781c1bbabfd29ed49bcd52797b9cc4c638" + }, + "round": 1 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "ebcdde440e336aa41bcd25ab206defcc6738979d3998ead9490f9c1674f3c899", + "clip_similarity": 0.939391016960144, + "nclip": 0.06060898303985596, + "photometric_loss": 0.04119781777262688, + "render": { + "bytes": 434722, + "sha256": "afed7cba4fa673e8bf4845a4d0ff66e6974237db46d23ffebed51ce124a8e28b" + }, + "round": 2 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "5ef8264c090b163664cb384cd6e43336aaaf52819728ae79987e448723148fa3", + "clip_similarity": 0.9926620721817017, + "nclip": 0.00733792781829834, + "photometric_loss": 0.009993723593652248, + "render": { + "bytes": 439444, + "sha256": "3b4234fda467c48ce1095444456b389e673156c8c8665a3f291a7857c93b99f6" + }, + "round": 3 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "b29982ccf623c233dcbb4607c9778d2f89fdad0f4171e54f3ff0771bb40101fa", + "clip_similarity": 0.9879449605941772, + "nclip": 0.012055039405822754, + "photometric_loss": 0.018624017015099525, + "render": { + "bytes": 436569, + "sha256": "cf0d68190043b3a4556a87dff17739a58417c50aa3c7079bee10dcaf1beed174" + }, + "round": 4 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "e206c5721dd1a982f74dafe3a6ca2eb22838e01478a04fad37b95d510d545224", + "clip_similarity": 0.9861524701118469, + "nclip": 0.013847529888153076, + "photometric_loss": 0.02692614309489727, + "render": { + "bytes": 435229, + "sha256": "4e7e141b9ed42c93a683fa702a140e55f67e3e93ba10aea57becd07c30fddd91" + }, + "round": 5 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "2af838b0a633a89ad3c74003365cc88a91bb227cd4bdc975a9a7be18b9f24de6", + "clip_similarity": 0.9947119355201721, + "nclip": 0.005288064479827881, + "photometric_loss": 0.007714033592492342, + "render": { + "bytes": 439693, + "sha256": "5582441b5c5c3b9bbb73067cec358a6514af493f5c5e3e7c60c5846c1729847c" + }, + "round": 6 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "172cf43f6e63f9f114be8704aa7c23c4b210a89387ae49be9a270c59e2bff95d", + "clip_similarity": 0.9964922666549683, + "nclip": 0.0035077333450317383, + "photometric_loss": 0.0044174473732709885, + "render": { + "bytes": 439085, + "sha256": "0c6337703e66bb0bfdf3e86b8292d025cc800840c77600693fb27c2a102c268d" + }, + "round": 7 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "ed2b2fa49a6d3810f08f440d2899d00bd385aad33dba82e4f1c2a76612b56efc", + "clip_similarity": 0.9942107796669006, + "nclip": 0.005789220333099365, + "photometric_loss": 0.005532005336135626, + "render": { + "bytes": 438157, + "sha256": "21f7b04e48bc4c8c6cd2237eefa8c3a060dcd7f511a5665864d19681e66b805b" + }, + "round": 8 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "05f61a09a01442fe071397d7a340ea102142ae441905290c13688b23c9ce57d4", + "clip_similarity": 0.9913418292999268, + "nclip": 0.008658170700073242, + "photometric_loss": 0.005203785840421915, + "render": { + "bytes": 438674, + "sha256": "bbd10e8100352574a9799fa4e74537cd7ec5f58cd3114a8632c3927a97a0b765" + }, + "round": 9 + }, + { + "blend_bytes": 932941898, + "blend_sha256": "b45fcb9f30b942a2445d7a395a14290d9b703954ebe21d102c005504da2d72d9", + "clip_similarity": 0.9959232807159424, + "nclip": 0.004076719284057617, + "photometric_loss": 0.004915030673146248, + "render": { + "bytes": 439034, + "sha256": "d98978a1590adc51272afa61af6870eddafee04494812bae254134a31de798e3" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7420, + "sha256": "361e1bccec2c68db93e75db1ce65452e7435a4ccd7c218be2a955dad23a55f3b" + }, + "target": { + "bytes": 442674, + "sha256": "0124cf59482f3950b2c62eddf8d734e752e45e5300fbf30ce472f74b05e30f9d" + } + }, + "level1/camera6": { + "evidence": [ + { + "bytes": 179993, + "kind": "events.jsonl", + "sha256": "f8b4b655c2e2a762e8ca465138348e98b24c6f24d6e7d5b1e51871570dfd4cee" + }, + { + "bytes": 7037515, + "kind": "run.jsonl", + "sha256": "916816ae71b951c73744141a4e6f6fad8772e8f85247f25609623d60de5b5984" + } + ], + "rounds": [ + { + "blend_bytes": 5890950, + "blend_sha256": "837b65d0e9179115e146b98e6e42bfd69da77ace0c751869439e1c9a1d64fec9", + "clip_similarity": 0.9859825372695923, + "nclip": 0.014017462730407715, + "photometric_loss": 0.07624026387929916, + "render": { + "bytes": 257520, + "sha256": "79e84c3406a4985f43d5d58009bc431f43ef84a743b17b002e088832d3da54b2" + }, + "round": 1 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "afedfe5bb34ddd058b8318b0d3358534eea83ec96fcfd0fa62f41ab83663a93b", + "clip_similarity": 0.9614191055297852, + "nclip": 0.038580894470214844, + "photometric_loss": 0.08258999139070511, + "render": { + "bytes": 251889, + "sha256": "e5db685b08ac0ea59b924b1b16eb018d4f7a8dae3fc3b0116d36f3505cefd999" + }, + "round": 2 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "7e3eacca442df727cf985922ecd9ef88d477067f3c43275fd38511ad50f4ac5c", + "clip_similarity": 0.9827094078063965, + "nclip": 0.017290592193603516, + "photometric_loss": 0.08534161001443863, + "render": { + "bytes": 256184, + "sha256": "8a163f1b54d1bfc08169cfb34ec361e4a3fc9006b205df4f5e0653576f8a7beb" + }, + "round": 3 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "bef1e757450f4e8c3345c871157d35746182532b79b903147f2d08be8c6a56b4", + "clip_similarity": 0.970423698425293, + "nclip": 0.02957630157470703, + "photometric_loss": 0.07037719339132309, + "render": { + "bytes": 255201, + "sha256": "bd0a1f18b68fb49055bd5e426d72a3f8890042f6ebd10253e458a207871256f5" + }, + "round": 4 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "15c904a636098f0a518295ba03a17bca6c3f29918405bb672e2dffdcd159288c", + "clip_similarity": 0.9938302040100098, + "nclip": 0.006169795989990234, + "photometric_loss": 0.030057286843657494, + "render": { + "bytes": 267350, + "sha256": "ccdf07db4ff58b6d9d6aead1afebe935ffaa167e9735a2da86e451d15c0aeb0a" + }, + "round": 5 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "4e2da15f5aa5b41a42c2417f8653c7c4247c4821da0940735d2d75b066438125", + "clip_similarity": 0.9984504580497742, + "nclip": 0.00154954195022583, + "photometric_loss": 0.006648885551840067, + "render": { + "bytes": 265589, + "sha256": "5ff2ab9661c9b3a97736eacadd364e63eec19a7047190eef09c856fb7497d5e6" + }, + "round": 6 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "7d4dea19fd4e3cd06d8b1fcfef3c1cc2b5e8dc64b87d950426bc927b1de3153f", + "clip_similarity": 0.9985253214836121, + "nclip": 0.0014746785163879395, + "photometric_loss": 0.002669217064976692, + "render": { + "bytes": 265059, + "sha256": "8656838b23fdad6d778bec04afdf29944d6a51eb56f6c98da324c88606ecd7e5" + }, + "round": 7 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "60b5cc3da7c27674a8fb6de85af5e6e7486e4b39f8ac4afa9371609cfff89480", + "clip_similarity": 0.9963637590408325, + "nclip": 0.0036362409591674805, + "photometric_loss": 0.010073143988847733, + "render": { + "bytes": 265308, + "sha256": "979a0bc8bf5987f2e33c0a4097d8f629074045754e32bbdaede48a6e9baa2f3e" + }, + "round": 8 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "929c7d7f173d447bf1937c70804fa8b56f25d7c65f0f5bb79a39ba937ce65620", + "clip_similarity": 0.9996985793113708, + "nclip": 0.0003014206886291504, + "photometric_loss": 0.0003250166482757777, + "render": { + "bytes": 265119, + "sha256": "8c8b3d6acf8b2ab27366edacb451d640e0244b773bf27e4b0fd8bf9b9f51f678" + }, + "round": 9 + }, + { + "blend_bytes": 5890950, + "blend_sha256": "67113e710df6c0ad81f5567a49f0022490382b20c0095fb69b2d319cc8cc2b91", + "clip_similarity": 0.9997468590736389, + "nclip": 0.000253140926361084, + "photometric_loss": 3.515696880640462e-05, + "render": { + "bytes": 265335, + "sha256": "61209025de8e51ac923b56fe9817442c24b3fc1a86b666870263f8e40b05acfc" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7421, + "sha256": "af99de7e7eb5485915e4d1266159e9b030c7046f62ad1a92b4ad530bd3f51da7" + }, + "target": { + "bytes": 264383, + "sha256": "e8ef88036bdd3e417383c349da3f531e22698d765f34b60851ec132d143f99d3" + } + }, + "level1/camera7": { + "evidence": [ + { + "bytes": 399124, + "kind": "events.jsonl", + "sha256": "be294d9361496bb5cae4110e5c22760512df5cb1bfd6ff106ef4cb18ff72246f" + }, + { + "bytes": 10612546, + "kind": "run.jsonl", + "sha256": "dd134d649599d327b39e2f327e30844a2f32f437e940157f413b3afde0756121" + } + ], + "rounds": [ + { + "blend_bytes": 171090997, + "blend_sha256": "5adbf52d81f66ca57b499869629bccf5a1c9ca06b135e10365fa3109c8bf7189", + "clip_similarity": 0.9739203453063965, + "nclip": 0.026079654693603516, + "photometric_loss": 0.0196079071611166, + "render": { + "bytes": 532216, + "sha256": "77f410052c5deeb134ecf2aeb58ade3d5d86c51418df50b1f21ec66358aa344d" + }, + "round": 1 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "ce57e6779a3b556141064b23ae884643f98f31243b187118ff38b840b2e9d7ab", + "clip_similarity": 0.9848811626434326, + "nclip": 0.015118837356567383, + "photometric_loss": 0.014287490397691727, + "render": { + "bytes": 530203, + "sha256": "bb2360f5cfa887f9ad5a404fab9665238f3cf883d1c8148db842bef7dd4f20a7" + }, + "round": 2 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "cb4668d3f764e534008932bc2d0df7e444780223a91ead7aa6bcff05580301d8", + "clip_similarity": 0.9873546361923218, + "nclip": 0.012645363807678223, + "photometric_loss": 0.013220968656241894, + "render": { + "bytes": 525743, + "sha256": "c1ab43b471ff642b2812cc651cff47a8726a7da16bc17548035bfa41b8229055" + }, + "round": 3 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "778f8427b917e38ace2451bcc8ca4e24aa2df930ff7a271a0bf2b20550b7e870", + "clip_similarity": 0.9847162961959839, + "nclip": 0.015283703804016113, + "photometric_loss": 0.013996700756251812, + "render": { + "bytes": 534249, + "sha256": "57ceaf32769573ce03046e41653a1db5d25ff5c97ebbeafa08864b9862842310" + }, + "round": 4 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "236c196bfe34535678bd9475d161a7174ba34cfb9970274a4ef71b6a0f5bf4e7", + "clip_similarity": 0.9858249425888062, + "nclip": 0.014175057411193848, + "photometric_loss": 0.010731506161391735, + "render": { + "bytes": 529124, + "sha256": "81d610d6be6fa30933c6d1312e0ce85350fac6373ccc144d8e42791d153968b8" + }, + "round": 5 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "93d552ea0d03a53383602e1ef72fd6fcaf3294edd22c0b49ce79e228a448b31f", + "clip_similarity": 0.9875136017799377, + "nclip": 0.012486398220062256, + "photometric_loss": 0.007920750416815281, + "render": { + "bytes": 527588, + "sha256": "30dcdb3fbc3999ac4c631900eeb1bdd001a30ebdd4828ec5b507f97fc57d371f" + }, + "round": 6 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "d035b80ed088f03efdfa3288a989d8857942d24ffc7e29c6575350198cadcf54", + "clip_similarity": 0.9810191988945007, + "nclip": 0.018980801105499268, + "photometric_loss": 0.01195294689387083, + "render": { + "bytes": 526823, + "sha256": "c07a04726e0186540c3172f5fd1f00af52ac5b0946fa95a85e6bdf535c501fdf" + }, + "round": 7 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "abc43b0128394173f3a69a451a710841d47bc7668d8b0e516a3a55ca9dfe0ddd", + "clip_similarity": 0.983600378036499, + "nclip": 0.016399621963500977, + "photometric_loss": 0.012078434228897095, + "render": { + "bytes": 529833, + "sha256": "3e8bbe5ccd4fd3e3b77eda42c3a9049f796e8179b5f2a2ab62617d484f8e002b" + }, + "round": 8 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "9dda9bbaec94af40da8cee7c32bcee18c77f7763d6c2f4e0a09f677d28b45b82", + "clip_similarity": 0.989441454410553, + "nclip": 0.010558545589447021, + "photometric_loss": 0.006557795684784651, + "render": { + "bytes": 527344, + "sha256": "acde947545f37451eb6dd0c226d1ebe475d8d6c6cf1a564a7645fb5383223f74" + }, + "round": 9 + }, + { + "blend_bytes": 171090997, + "blend_sha256": "d5a658674a9c649d7be4e19d28811e6d72b4964caef0632cff2cf28501c32463", + "clip_similarity": 0.9878263473510742, + "nclip": 0.012173652648925781, + "photometric_loss": 0.005300699267536402, + "render": { + "bytes": 527243, + "sha256": "335ec63b3073cd4b839fef1791a977c886e5d27a65d395d968b17c8e80b11a5a" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7418, + "sha256": "601a0e0911833525ce64681125358026fea4a56dd6b9de6488817dcd628ad357" + }, + "target": { + "bytes": 527080, + "sha256": "7f5efeca8b9586450cbb5a019bc03f2b0616f76d29fecfc699cca0059150a8fd" + } + }, + "level1/camera8": { + "evidence": [ + { + "bytes": 107429, + "kind": "events.jsonl", + "sha256": "496246090490adfdd14506f3857f91449cb895423e3d5d7aecf1fd5287f53475" + }, + { + "bytes": 6001678, + "kind": "run.jsonl", + "sha256": "0729c0c58830240f23fa9a7297b8d3dce4aa596f1c0b9dcb5598922f9261f401" + } + ], + "rounds": [ + { + "blend_bytes": 2899917, + "blend_sha256": "774875b72c48876175acd617560d0d0c09993ee959085c6c8ded1b53c7b1fded", + "clip_similarity": 0.9928888082504272, + "nclip": 0.007111191749572754, + "photometric_loss": 0.0036105772014707327, + "render": { + "bytes": 180919, + "sha256": "7923a314c10af011edcbe5ae6014df56bd7ded35c97b1a4e72b5475f418d5334" + }, + "round": 1 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "6c0a8d49e31e6160499eca2c442a809fa784dfb91e03aa9594e0ac7023e55192", + "clip_similarity": 0.9933868050575256, + "nclip": 0.006613194942474365, + "photometric_loss": 0.01031858753412962, + "render": { + "bytes": 182767, + "sha256": "145a478c74e4232ecfa7276438af1235fc8b545fc5490ad579704107509aa3bf" + }, + "round": 2 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "1102bf2532f961715030d59b40af2083aea33d9e1bfe773d49a60e0fe5bfdeb9", + "clip_similarity": 0.9976049661636353, + "nclip": 0.002395033836364746, + "photometric_loss": 0.0006101542385295033, + "render": { + "bytes": 183557, + "sha256": "8c297db8eb308a8b04f796e34f009827d9a47a8580c746e6cf5e2e5606501030" + }, + "round": 3 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "78f181c3c3d6d1902a2ba41f9df4001cd440d72a0a50b15163c8ecfaf9edd872", + "clip_similarity": 0.9950434565544128, + "nclip": 0.004956543445587158, + "photometric_loss": 0.002057823585346341, + "render": { + "bytes": 182767, + "sha256": "3af0c3c8df22593db7b2a55ca4c04749fc3dc5800ca024b4efa324008f6e898b" + }, + "round": 4 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "02096f7833faea44fba015d12b75a09033202d86839dd06f31e382348458d5e0", + "clip_similarity": 0.9986875653266907, + "nclip": 0.0013124346733093262, + "photometric_loss": 0.00032354166614823043, + "render": { + "bytes": 183671, + "sha256": "3f8c669e45792eaeb07726cb85f7f6014e3bf371a3bcc53c6a69e2044c69ddad" + }, + "round": 5 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "03788353c806e4f44ba79e69d51ed8958c4b95073aa775fde3aac723a7bceee6", + "clip_similarity": 0.9983348846435547, + "nclip": 0.0016651153564453125, + "photometric_loss": 0.0001591976615600288, + "render": { + "bytes": 183536, + "sha256": "af1081c8ca55ca2ffa98de9754564e3d8f34a2413cec32612a0850b4c27564b1" + }, + "round": 6 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "02daa1ff5f27448d5b62e56cf3632b126971c502dc43881c20b277c644e0eeb1", + "clip_similarity": 0.9977761507034302, + "nclip": 0.0022238492965698242, + "photometric_loss": 7.834874122636393e-05, + "render": { + "bytes": 183767, + "sha256": "5bb008ab1b3b9f15b24e59f37990e00d66622dc62fcaff95f4315e885d3fbc0a" + }, + "round": 7 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "13babf3fc14653839393f825c008120ddda3f4e768bdaa43fe4f6b6ba08eeae9", + "clip_similarity": 0.9982494115829468, + "nclip": 0.0017505884170532227, + "photometric_loss": 0.0008940323605202138, + "render": { + "bytes": 183924, + "sha256": "31a32c007d60aab2ba74b92fed99825fd14a8cf19b098da4ee2179659c5558ea" + }, + "round": 8 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "6509cf3a02255a8bb38312bd97d1b4236e9df4bc50446f98a5c3ffd528866a7a", + "clip_similarity": 0.9981532096862793, + "nclip": 0.0018467903137207031, + "photometric_loss": 0.00019516983593348414, + "render": { + "bytes": 183923, + "sha256": "daffa380cf437d8a93d2b05efeed8a36f7fcc4f56bfeaf5c3362bcbdcfb907d0" + }, + "round": 9 + }, + { + "blend_bytes": 2899917, + "blend_sha256": "01b47f25b8fc8d07dc0b7d55be7f1b4a2d3c64c7a0a6e58f8aec60f0c2229ee0", + "clip_similarity": 0.9990020990371704, + "nclip": 0.0009979009628295898, + "photometric_loss": 3.855396425933577e-05, + "render": { + "bytes": 183802, + "sha256": "c21eb8b7c4b70722d09701dcb9f6b2e08f2728e1dd2189cfa642fbea6c4042df" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7438, + "sha256": "84594d2a0446aa1ef4df470428bf2675084c23607d76277dd23f589ad491fca2" + }, + "target": { + "bytes": 198412, + "sha256": "b4e026fea8211aaf7f1149cd3e20a2ac6346882484eee5804ea6e19dc43b6bc4" + } + }, + "level1/camera9": { + "evidence": [ + { + "bytes": 94851, + "kind": "events.jsonl", + "sha256": "21054d0d1aaeae67cf557c62e55b2e9944c102dcf209cbb9ab6f7257bd2e3ecf" + }, + { + "bytes": 8372868, + "kind": "run.jsonl", + "sha256": "57480b23105573257199c701efb0d2564cf0bc9714e280c5e63a0d60f53c02da" + } + ], + "rounds": [ + { + "blend_bytes": 4463030, + "blend_sha256": "f9c990172cb7ca08e6ae20b10107275fc589bac8de8207f14a3b6c7439680e1a", + "clip_similarity": 0.9513208270072937, + "nclip": 0.0486791729927063, + "photometric_loss": 0.025137677788734436, + "render": { + "bytes": 311637, + "sha256": "f6ad8fe8d14b4b511deaac9501fef945ee717ad583885b111ecbb66c808b8aa4" + }, + "round": 1 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "89bd0aac9aba21594b5e012f46f65a259d09116fb1a3bdce6d38a61f6035a722", + "clip_similarity": 0.9853408336639404, + "nclip": 0.01465916633605957, + "photometric_loss": 0.009238298051059246, + "render": { + "bytes": 310500, + "sha256": "f31dd0f0b603d3193893e2238be1047dcd3afcd72c0828ae0f42e54f2626b196" + }, + "round": 2 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "7e1fc1123217617bca75de1662fd2f19afcfbe5f9cbd962d5ac31195ff7ae964", + "clip_similarity": 0.9905526041984558, + "nclip": 0.00944739580154419, + "photometric_loss": 0.008991035632789135, + "render": { + "bytes": 310542, + "sha256": "d1db10e8a159649c805f0251c13dc3fa9fc4f74089a0f9add06c2b4014a51675" + }, + "round": 3 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "6da14002e390ee0e90c8fb9b6e55a6fcad3847b14dc0dbb6be4a5a7421ad3ebc", + "clip_similarity": 0.9939846992492676, + "nclip": 0.006015300750732422, + "photometric_loss": 0.0064002289436757565, + "render": { + "bytes": 312715, + "sha256": "4ed539e5b05aaf55463e514cb698382149c61e27f51ed0bba2ee298a5903eaa4" + }, + "round": 4 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "3e4e696b865d6e4939966e5060894a83c83fe370aeae6a43c9e7c52a4fa68a01", + "clip_similarity": 0.9969817399978638, + "nclip": 0.0030182600021362305, + "photometric_loss": 0.005433480720967054, + "render": { + "bytes": 310915, + "sha256": "d0b79acb66100b0a05758c009f23ace29d1efd74959cdd6d3f86c316f26b6ed3" + }, + "round": 5 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "1382ab3f47b34a06d56e667ef07db43e70d92b6e651c65633f4d27f919944b55", + "clip_similarity": 0.9983714818954468, + "nclip": 0.0016285181045532227, + "photometric_loss": 0.0012882169103249907, + "render": { + "bytes": 310652, + "sha256": "c2d47d2fd35531117523147bd97dff3d0b511b85c1eed4f51cea4755b0fd2aa8" + }, + "round": 6 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "19094fd64181d4780cfaa023fa09a25b8474fd3fdc3dee5fbbe96df81c4a3064", + "clip_similarity": 0.9987772703170776, + "nclip": 0.0012227296829223633, + "photometric_loss": 0.0006784373545087874, + "render": { + "bytes": 309989, + "sha256": "4fad5e79b83e974156f810a10078dcc70eebfbdbef4c7b4e4bff60e85eb21a01" + }, + "round": 7 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "b2b86f2d82e7276ec57cc1252fc3612a1f70ee1fd788bba2ecee03f782f45047", + "clip_similarity": 0.9967776536941528, + "nclip": 0.003222346305847168, + "photometric_loss": 0.0009337430819869041, + "render": { + "bytes": 310239, + "sha256": "05007cb82eefc7220bead065a48c5ca2b0c328978163a224cbdf4e9a8c448c26" + }, + "round": 8 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "8f811dc81224704430d54e9be29fc9a586ffa0e4582de5cd4c2e44038a8bc9df", + "clip_similarity": 0.9995280504226685, + "nclip": 0.00047194957733154297, + "photometric_loss": 0.0004399387107696384, + "render": { + "bytes": 310348, + "sha256": "47367201f173f475d5487ef0d2a858baf5c9ff595477f9531ab11791fec8a7b6" + }, + "round": 9 + }, + { + "blend_bytes": 4463030, + "blend_sha256": "be155dbf927bd5e410d847be2378a45f00d3c441079c4dd7b6cd192c5ce90487", + "clip_similarity": 0.9991905093193054, + "nclip": 0.0008094906806945801, + "photometric_loss": 0.0003071240207646042, + "render": { + "bytes": 310456, + "sha256": "14a3b1b89d5268da101af3c5ebc26c29461a520fab59fd40a959579dc796d51b" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7431, + "sha256": "21735e12136fac3b57d6d1eeea8a05747592402cd8df629ca120ae6b9525eff9" + }, + "target": { + "bytes": 311165, + "sha256": "df51a5786e84b1d47c2df08c95d6adb3f7594dec20fa17c47424bb08b16145b6" + } + }, + "level2/attribute1": { + "evidence": [ + { + "bytes": 1489445, + "kind": "events.jsonl", + "sha256": "6d4d138469bfc8c4c56677f947cd8e045b54dbefe5964a7dea5a8f0ffbce54cc" + } + ], + "rounds": [ + { + "blend_bytes": 949424441, + "blend_sha256": "1e565bff9cbe7b822d7403e55d3bc1dcfeb0a39aa92276de4ecc4ba4dba77c02", + "clip_similarity": 0.9616481065750122, + "nclip": 0.03835189342498779, + "photometric_loss": 0.03206987306475639, + "render": { + "bytes": 3035159, + "sha256": "aa4807b88f9bde1d77bb5e37629684b2115f22b61c6f17c1ca73287375a34b21" + }, + "round": 1 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "d85a246f1e7f754e216860de4e01fc660d84b5b1d2065b5d3168183bdd168b6e", + "clip_similarity": 0.9758520126342773, + "nclip": 0.024147987365722656, + "photometric_loss": 0.020144661888480186, + "render": { + "bytes": 3045247, + "sha256": "cfc963a03a0788090fbdf42b9e2bbb89be0ee87ae9b22a75e919c41de2653172" + }, + "round": 2 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "eb6422915c3ce4af1073b71f37d21338474f48a1d06f806f5d3c096f7f13abfc", + "clip_similarity": 0.9763191342353821, + "nclip": 0.02368086576461792, + "photometric_loss": 0.015242948196828365, + "render": { + "bytes": 3040790, + "sha256": "3dcffb98b4cdbe65ae33fb01032761d83d490d7fe4c93a833750a49103704677" + }, + "round": 3 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "8c33a3132cc32270b96c771c7ca978a23745441c1a9762c8451dc8f4a3d114f3", + "clip_similarity": 0.9755369424819946, + "nclip": 0.02446305751800537, + "photometric_loss": 0.015236021019518375, + "render": { + "bytes": 3040243, + "sha256": "dba49561195b08488ae1a0d53b546de734e742c2b6eb7e5149d8958fc727d816" + }, + "round": 4 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "220c2c3dd4fc28a8919ed5f5e6217ac5de2ec3ca16d477a9daec63b902a9d561", + "clip_similarity": 0.9777381420135498, + "nclip": 0.022261857986450195, + "photometric_loss": 0.01512475311756134, + "render": { + "bytes": 3035300, + "sha256": "d1c78833cbc456ae14b1ce041ed3bbd06bf3e006e79c5cc6d0cb8d5c11765d16" + }, + "round": 5 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "068276672a643ebabe71486efae1ce3acb8f56b804e3a58a2fcb9c6fee7a3f45", + "clip_similarity": 0.9761902093887329, + "nclip": 0.02380979061126709, + "photometric_loss": 0.0151130435988307, + "render": { + "bytes": 3033925, + "sha256": "3c279d92a2ec3e92630bbe098c619df69163d458ae8105090fdb24f3dbe3be10" + }, + "round": 6 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "c39265f69d222f4e97114112a96d4b56d16f3a71e521a9e5729a17c78be04d7e", + "clip_similarity": 0.976814866065979, + "nclip": 0.023185133934020996, + "photometric_loss": 0.01512062270194292, + "render": { + "bytes": 3034742, + "sha256": "8a330f4d3d0fad7c7031769030eed277f7720484d15562e563af4d2c04b91539" + }, + "round": 7 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "a704a8942747fc3371bbdd6231f1ba35e7ddbe5e4ba3cd07021834cbc6af4401", + "clip_similarity": 0.976791262626648, + "nclip": 0.02320873737335205, + "photometric_loss": 0.015117769129574299, + "render": { + "bytes": 3034761, + "sha256": "1ecb25364beb1f83f08f1d8e073a00a171d54f9c8e9225ff1f3fb2f5beb75828" + }, + "round": 8 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "6fdd0bdf5249063684ef18645217bb03c374272874cce8929f7bbd0d1660590b", + "clip_similarity": 0.9776802062988281, + "nclip": 0.022319793701171875, + "photometric_loss": 0.015120291151106358, + "render": { + "bytes": 3035015, + "sha256": "5d94a5b5418b2703c85eed3f908156a67dd4675cb5098dfd28c74bc5ef4a64e5" + }, + "round": 9 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "e33accc273b2480f031bba55e353df90802d86e345e11fbd6b9acadd44a27cdb", + "clip_similarity": 0.9777544736862183, + "nclip": 0.02224552631378174, + "photometric_loss": 0.015119296498596668, + "render": { + "bytes": 3034786, + "sha256": "fa47897f60a5d9046001285695df81c97735f67921d27534b29257abd305a09f" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7660, + "sha256": "f1921da48be6fb80affc40803c16939438152148f5c40171c7673ff8788ad7d2" + }, + "target": { + "bytes": 435882, + "sha256": "a269058785099fd3f273334e70fa147de3ee3556a500ded8d9b064213fcd8981" + } + }, + "level2/attribute2": { + "evidence": [ + { + "bytes": 1489512, + "kind": "events.jsonl", + "sha256": "a93beb1267ee745dd27e7498964c1ae787820ea167f12f727e94e234e38876ba" + } + ], + "rounds": [ + { + "blend_bytes": 949424441, + "blend_sha256": "882312ac90f9ab4e4e50e48eaa6bb4170807704168ce1dccdcdfef9e9225d48f", + "clip_similarity": 0.9783761501312256, + "nclip": 0.021623849868774414, + "photometric_loss": 0.023375628516077995, + "render": { + "bytes": 2999030, + "sha256": "6beda2b96f488fdff61fa71718d217393f2296c45a24c20eea41b2d200e51d4b" + }, + "round": 1 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "733b4df9380626f5cdc7d54837fa564de14d5ba1091a31933bdbd1c18141b72b", + "clip_similarity": 0.9806861281394958, + "nclip": 0.01931387186050415, + "photometric_loss": 0.01821855455636978, + "render": { + "bytes": 3025233, + "sha256": "c03d37b649e7f613a806494aa835f8bfd8e9c33c8165d95dc7125dd51985a0f5" + }, + "round": 2 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "ae2390ddf2bf6f9ee6d5d473185a936fcf68edb9df7f4c90d50d0638bd403fb4", + "clip_similarity": 0.9808710217475891, + "nclip": 0.01912897825241089, + "photometric_loss": 0.016760846599936485, + "render": { + "bytes": 3045603, + "sha256": "ce52f84d25c9a879c3bf5f98fc9875231d57f3a16c05fbedfed0786d1e7d693d" + }, + "round": 3 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "91efbb33590c2e1fe5b608d9924dcd2c98609c0f585bad35d6e2a7acfa53b08b", + "clip_similarity": 0.9811261892318726, + "nclip": 0.01887381076812744, + "photometric_loss": 0.016651509329676628, + "render": { + "bytes": 3040739, + "sha256": "b0431d4d18bbcbc3d286f708c981cf756e86ec4f8f0f9740a9ed2a1a454a4501" + }, + "round": 4 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "5c8bd94c0314ebd4bae377b6a5f7193b4c81dd368256c0b0381f405d993435a0", + "clip_similarity": 0.9820566177368164, + "nclip": 0.017943382263183594, + "photometric_loss": 0.016636313870549202, + "render": { + "bytes": 3039345, + "sha256": "68cfffac21da51743520aeeaffc73149d8b886d347531ec504e173852b2b04a4" + }, + "round": 5 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "25d4ce76f9f63630249e3790de94e8a3145003de017efad1f27b551b8d5d6484", + "clip_similarity": 0.9820483922958374, + "nclip": 0.017951607704162598, + "photometric_loss": 0.016636930406093597, + "render": { + "bytes": 3039332, + "sha256": "99ee4ab8308811d8155e13b327e2c47973db63b9794bb114123550aa89442eae" + }, + "round": 6 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "dde7ab314504a3ef4c7de24a4fe846029af74825546d82f29c078640cdf02a6d", + "clip_similarity": 0.9812586307525635, + "nclip": 0.018741369247436523, + "photometric_loss": 0.016626911237835884, + "render": { + "bytes": 3038499, + "sha256": "ee64a9d46874b58a0215d941e00b6b75b884e6d41bfc78ff31e3a6e99963e389" + }, + "round": 7 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "e98ed32620e7ce642df4b07ebf26f07ecfd6e97a0f7d8b59a8ee2345178aa1f8", + "clip_similarity": 0.9812836647033691, + "nclip": 0.01871633529663086, + "photometric_loss": 0.01662970520555973, + "render": { + "bytes": 3038401, + "sha256": "59c58887845be9ded30c6eaf9367984dbc30017f9e0214eede342b5486b8c85e" + }, + "round": 8 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "15593ecf6a309580011a203d68b2519029f18b8dad0936a1a824de813b49bc0e", + "clip_similarity": 0.9816768169403076, + "nclip": 0.018323183059692383, + "photometric_loss": 0.01663150079548359, + "render": { + "bytes": 3038593, + "sha256": "c084d4aa7e80ddcbead7d82b1f35202bc539d1ca34afe8d91d62c9a5e71e9ca7" + }, + "round": 9 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "7d82c31ceca869e85ac81e5efc745f6420ec2dec216484457fc9f86c78a6b06b", + "clip_similarity": 0.9813920855522156, + "nclip": 0.018607914447784424, + "photometric_loss": 0.016634559258818626, + "render": { + "bytes": 3038754, + "sha256": "8c59abcd322c2a4bec6a4f94eaa63a84ab9aa7f55a97b32c84c7d6588a1b68e3" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7666, + "sha256": "5aa86a85646b9760945346c21a6f6657e63f6ca46950b679862ee5d34184deaf" + }, + "target": { + "bytes": 435876, + "sha256": "0744a471d3a07fe9c4a38402f9be09310e5e26dc4abbc62f6b634d20b1f6c23f" + } + }, + "level2/attribute3": { + "evidence": [ + { + "bytes": 1489205, + "kind": "events.jsonl", + "sha256": "036d8d1b59caac459e9ea36ca3f80bb37ff940007e9efde052fab9bb53ea0e74" + } + ], + "rounds": [ + { + "blend_bytes": 949424441, + "blend_sha256": "113bf72a065d22b3cf48448ad5657b7236449a0bb89c072bf3cc30b8e77efeef", + "clip_similarity": 0.9737488627433777, + "nclip": 0.026251137256622314, + "photometric_loss": 0.01580880396068096, + "render": { + "bytes": 3038266, + "sha256": "c3f3b42a13a3328705aaf935239f44faa998260b649de2f412a2740caf429804" + }, + "round": 1 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "d6a2131d15eb29e00ac8a0dceb8f82e08a003c2b9aa756e646630116e847ad3c", + "clip_similarity": 0.9718049764633179, + "nclip": 0.02819502353668213, + "photometric_loss": 0.02121717669069767, + "render": { + "bytes": 3000697, + "sha256": "c12c7f92ca10a3aa2b4f812fef2c78209645dd80454d28e95b16bbc0be59effb" + }, + "round": 2 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "5c40e4f3457556b354fcf91731e5d71e320e2812c8384b8eb5596209841e4545", + "clip_similarity": 0.9792160987854004, + "nclip": 0.02078390121459961, + "photometric_loss": 0.021017536520957947, + "render": { + "bytes": 3007735, + "sha256": "d5932cf757b1f60d11f852ca1ec5c1eca0a8154f1733b2db7edbe0d82ac4983a" + }, + "round": 3 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "79ed406718e54bd39d292dddee197eb9d68c5540d0b327e91ba7524f2eb007a8", + "clip_similarity": 0.9793153405189514, + "nclip": 0.020684659481048584, + "photometric_loss": 0.01562858745455742, + "render": { + "bytes": 3038284, + "sha256": "be7f2216b96479b9ef5e4d74aa6ce89052c11a87e5467320c7f90c512062489e" + }, + "round": 4 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "b260539226ad62f49d3387934a66f03db3b87e627c8046e8971835dccecb85b6", + "clip_similarity": 0.9762895703315735, + "nclip": 0.023710429668426514, + "photometric_loss": 0.015242993831634521, + "render": { + "bytes": 3040810, + "sha256": "e8e04bf11387a85ceafe9b47fdc30a8c37523a489acf4936ad4b61f871581016" + }, + "round": 5 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "01172623305c08bbbf865395f27d4a82e80920f980c68f0658577d121cad41f4", + "clip_similarity": 0.9765477180480957, + "nclip": 0.023452281951904297, + "photometric_loss": 0.015135708265006542, + "render": { + "bytes": 3036082, + "sha256": "d4e42e55b310bbb2ddeec61532e92584c3e0f4b249adedbb1719812103f30b81" + }, + "round": 6 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "486618df98b0386d6fdb371b6c4170a62efb5833b6899aa902953389bb605ed6", + "clip_similarity": 0.9764472842216492, + "nclip": 0.02355271577835083, + "photometric_loss": 0.015115156769752502, + "render": { + "bytes": 3033885, + "sha256": "1695d085e9d2c270ca07bec83cbc34fc3215d046192e1846a91b7259d317157e" + }, + "round": 7 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "ba9fcf5d28d5362182c60992df40a76b4886c6a3db6b3d0715a21bbec2cc386b", + "clip_similarity": 0.9759871959686279, + "nclip": 0.02401280403137207, + "photometric_loss": 0.015113621018826962, + "render": { + "bytes": 3033841, + "sha256": "07ce05b6c3f379eaa5e77f3846901b276549a31ef34beb977a5686e01dea0862" + }, + "round": 8 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "6ec9362f5a8e3474f74e9a0da7073ddb56c5d842f5161023fa95a37cefcbba4f", + "clip_similarity": 0.9774898290634155, + "nclip": 0.022510170936584473, + "photometric_loss": 0.015127800405025482, + "render": { + "bytes": 3035273, + "sha256": "312a041a3b8ea8c7889c7e79a8b13fad11fbbf73dab93c5f2c2807cd182d7432" + }, + "round": 9 + }, + { + "blend_bytes": 949424441, + "blend_sha256": "e3ca3d9290cf3702e8c306ba8388a34053af0427e10efbfc3524cebda811ec4a", + "clip_similarity": 0.9767491221427917, + "nclip": 0.023250877857208252, + "photometric_loss": 0.015117752365767956, + "render": { + "bytes": 3034771, + "sha256": "2d2544dd2e2c25607833c112f7b04c91433fa4f00b3a72b0871476912e1e27e3" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7666, + "sha256": "0564fb08e82de849edaa0d8bdeda4d3c8a8b6ac249868b974c88f77c4e52ee8d" + }, + "target": { + "bytes": 435892, + "sha256": "3886f82c111b6e7dada41bcd6129e9c782f543a11e02860c693a5e3c8a336da7" + } + }, + "level2/attribute4": { + "evidence": [ + { + "bytes": 1323480, + "kind": "events.jsonl", + "sha256": "c3cac4087cc1a1fe5c3f5f34768b4f0ea9eae647e25ce137d886187dc35f3880" + } + ], + "rounds": [ + { + "blend_bytes": 940184532, + "blend_sha256": "8310e08dafd21a5225517efc091c53f14acc08ee8e65702d4f421f3598a93afb", + "clip_similarity": 0.9745771288871765, + "nclip": 0.025422871112823486, + "photometric_loss": 0.02019927091896534, + "render": { + "bytes": 2955115, + "sha256": "16d8eb7c9d20f8d0a9573cba78473abb9867339b730b3a2443a7f5c692a18449" + }, + "round": 1 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "b01225040e1d12bad96ff6d554bf81f12970eaace058d4b76ec3ea5b88ad3f09", + "clip_similarity": 0.9726730585098267, + "nclip": 0.02732694149017334, + "photometric_loss": 0.01562739722430706, + "render": { + "bytes": 2980024, + "sha256": "381daf79ed2269fece3d52f0d0c3901462a72c436344f2703b36630d19c9b15a" + }, + "round": 2 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "d9e102721b30721f7b4cd8953f308046e120ae4a4b7c6d14913d4e0071d91678", + "clip_similarity": 0.9584736227989197, + "nclip": 0.04152637720108032, + "photometric_loss": 0.024826040491461754, + "render": { + "bytes": 2982224, + "sha256": "6d11b84059359c8c973070dbc98c3e67af510c38ed5bcd4aa7611f57b551d174" + }, + "round": 3 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "02c5e07bb556542c8d890181ae6b9643a39bd0d689f5b481629cb426086637ed", + "clip_similarity": 0.9532421231269836, + "nclip": 0.04675787687301636, + "photometric_loss": 0.031184980645775795, + "render": { + "bytes": 2974963, + "sha256": "c42fbf82e09c6ff8d99c0e27d444db902d65b191830abd49aef4651b8a3ab062" + }, + "round": 4 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "e3906bf524eee568eec3059099ca8d0502bc2bec87173d0a46a3d84548559a3d", + "clip_similarity": 0.9654848575592041, + "nclip": 0.0345151424407959, + "photometric_loss": 0.03093068115413189, + "render": { + "bytes": 2979217, + "sha256": "8e3322d079e97b16a9784a04256a7834634c9dae9a468c76361f1ebcbc94744f" + }, + "round": 5 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "365a0bcbd1ffb6a6f90392bb11bb412c77940d10e395923c8c22d322268356eb", + "clip_similarity": 0.9615705013275146, + "nclip": 0.03842949867248535, + "photometric_loss": 0.030709154903888702, + "render": { + "bytes": 2977789, + "sha256": "dd745856e3155649fb989a6f94a684138910fc33c1a252f27f14e0da5555af3c" + }, + "round": 6 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "5f406cd1b169a821ddd049c96d5aa4f7bc3e641f9b2e958319b8e45c44a8c338", + "clip_similarity": 0.9655020236968994, + "nclip": 0.034497976303100586, + "photometric_loss": 0.030589738860726357, + "render": { + "bytes": 2979565, + "sha256": "e709c4f2421b6ce5cfa9739d04be1b35ceef0b552be719a6ce811ab12e9c2c31" + }, + "round": 7 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "2f7d2f07d9369084d7932d49933942bfe90c54a11ab1261a97d4295d9f2d03c4", + "clip_similarity": 0.9670642018318176, + "nclip": 0.03293579816818237, + "photometric_loss": 0.030599204823374748, + "render": { + "bytes": 2979677, + "sha256": "c0aacf4f381698ebd546346afd806ca833f39f58a8bfc6c518c8463432f27ed4" + }, + "round": 8 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "1c85fd6849a1f66b69e16de8b892eda578e3b35a37fc6cc1d34a3d1732aefdb3", + "clip_similarity": 0.9656827449798584, + "nclip": 0.0343172550201416, + "photometric_loss": 0.030554519966244698, + "render": { + "bytes": 2978938, + "sha256": "f7be7217e49eb1857a4f6176295bc1ad1712486523b474dbb9c860dc31bcf478" + }, + "round": 9 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "ed19f7f771bae56f25269b2e49f67c124aac86deebb113bc66d7f0bd4eb33243", + "clip_similarity": 0.9664871692657471, + "nclip": 0.03351283073425293, + "photometric_loss": 0.030548766255378723, + "render": { + "bytes": 2979101, + "sha256": "50c0626ac1782484efab69c95c57eb2dfa6007d14e38db12c6bea2b618ed9175" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7660, + "sha256": "8ef915fae4dfdf98b9e6f86563b1d54d7b9ca1d3ae35b2dd9fd69865572c8b87" + }, + "target": { + "bytes": 433064, + "sha256": "dd049697d61750d34d9f0587a832d7b8ad322197e63b4319b1938675b64eefcb" + } + }, + "level2/attribute5": { + "evidence": [ + { + "bytes": 1319636, + "kind": "events.jsonl", + "sha256": "5e70e46af62448ad2bf771b77a9738cfe6affa873abb9012046ee36cc36b59f7" + } + ], + "rounds": [ + { + "blend_bytes": 940184532, + "blend_sha256": "67dacaa47c9c5c46a9861f579af526e16fa2791a7274380303725d264da0c4d8", + "clip_similarity": 0.9714045524597168, + "nclip": 0.028595447540283203, + "photometric_loss": 0.016888050362467766, + "render": { + "bytes": 2992284, + "sha256": "c827bf49aa0800d0bc9c17d305965d2236d0bc12fe8300d2264a439d4b668d25" + }, + "round": 1 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "b1e3ed0fc24095d5eb13c351255ff1f1aad81891e725de60e4ca1fa6cdd9de1b", + "clip_similarity": 0.9766486883163452, + "nclip": 0.023351311683654785, + "photometric_loss": 0.01576325297355652, + "render": { + "bytes": 2987400, + "sha256": "358fe6b8b86c342a7b5b895ebd541f6b27aebaf80b20efd236631e93dfd3fed4" + }, + "round": 2 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "83887082e62043daaa6da4757bcc86750e675da952b8dd4ea9e6662cbf112fdb", + "clip_similarity": 0.969855010509491, + "nclip": 0.030144989490509033, + "photometric_loss": 0.018261505290865898, + "render": { + "bytes": 2950422, + "sha256": "169f6a286c9f91879743fab781200842574cdd70d0a0f233b1e64f8d51de302e" + }, + "round": 3 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "457e1e80ff383eab109d5668f5c254593bcb0bb2221d56c1c965d0ce7b276b49", + "clip_similarity": 0.9712028503417969, + "nclip": 0.028797149658203125, + "photometric_loss": 0.014592328108847141, + "render": { + "bytes": 2982542, + "sha256": "28e803bfc2568a447a82108a709cd24f42c169b64a6e5193c6794fa2b15524a1" + }, + "round": 4 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "b8530aed42d9c3319116bd90ea39b3827e564d30117cb7081aaf181b34752af5", + "clip_similarity": 0.9800482988357544, + "nclip": 0.019951701164245605, + "photometric_loss": 0.014472146518528461, + "render": { + "bytes": 2985411, + "sha256": "b9fce9fe928c2d2705792f0a60a858fe2733bcc15f88701fb7bca84b33aaade2" + }, + "round": 5 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "174b3025903e0f92ee71ae866535324c43aea637a7ce68fc024a6a46fca24766", + "clip_similarity": 0.981130838394165, + "nclip": 0.01886916160583496, + "photometric_loss": 0.014343224465847015, + "render": { + "bytes": 2986237, + "sha256": "10cb1babed5aaa8c75bbaa038935714499cbac0ca72b87278f8d668b2e924b25" + }, + "round": 6 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "7b855e24f2f09010746bf2edb9da75da7219db81c8f3c8717df79062ef29507f", + "clip_similarity": 0.9813302159309387, + "nclip": 0.01866978406906128, + "photometric_loss": 0.01404707133769989, + "render": { + "bytes": 2985549, + "sha256": "dd5b0b9ce272271592cbf2f1a31be43c58aca29dec32e45d64e9f46df9a0696a" + }, + "round": 7 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "08d72b29fc92a0179b4fe890a88a073d1f7001fd7d81c566aee247709a150a6c", + "clip_similarity": 0.9820970296859741, + "nclip": 0.01790297031402588, + "photometric_loss": 0.014138872735202312, + "render": { + "bytes": 2986075, + "sha256": "af57de90dc47297013bba3627d345b9e1cf2057eeab4be71863989e736dbc209" + }, + "round": 8 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "1a569a78e74a4694a0ce5c8dab9771ae18cedd1752747baf5a00f46a5c9e9162", + "clip_similarity": 0.9830480813980103, + "nclip": 0.016951918601989746, + "photometric_loss": 0.013992843218147755, + "render": { + "bytes": 2985804, + "sha256": "90b7e6775020cc48c4f481dfcf1b27bb91e549bf303e59242f1169e361436ec7" + }, + "round": 9 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "901160ecfc9885902e711c2942bf5071c3f4c6e5ad5d1ebe9a3337e464da052b", + "clip_similarity": 0.9829906225204468, + "nclip": 0.017009377479553223, + "photometric_loss": 0.013999431394040585, + "render": { + "bytes": 2985487, + "sha256": "3fad542dc6dbf6d6112c9b5ca2eb66dd7030206e42249aaa55cf43cefb9b140d" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7666, + "sha256": "7a38668115b17d5017f471704b600bf17466d5130316b5505674441a95e4ff83" + }, + "target": { + "bytes": 433064, + "sha256": "0b33dfa853ce71b3e0a91ae9813e90dba0544f489251d8a811c810c1491fce55" + } + }, + "level2/attribute6": { + "evidence": [ + { + "bytes": 1323085, + "kind": "events.jsonl", + "sha256": "62fc68529fdd17fd274c87349f2564aef95ee63c8079617ce897c5a31357c4b7" + } + ], + "rounds": [ + { + "blend_bytes": 940184532, + "blend_sha256": "a6cbccd0441951188d66304b2da639a956b01bdbe0428380ec0bcae05f58af60", + "clip_similarity": 0.9696398973464966, + "nclip": 0.030360102653503418, + "photometric_loss": 0.019036224111914635, + "render": { + "bytes": 2981721, + "sha256": "13c1b3e9c4698293e9894ccc943f8152dc91f5bf3df6033572819d414299edb8" + }, + "round": 1 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "d2ae6344ba60b0a1d3f31ef11e1e65f7fbef403fad235601933cb3b5bcc7a2c3", + "clip_similarity": 0.9568657279014587, + "nclip": 0.04313427209854126, + "photometric_loss": 0.018575983121991158, + "render": { + "bytes": 2990428, + "sha256": "56f3a20fc48604323b6f108e479f7904c2a71578b635ad7a5cd8d49caca32dcd" + }, + "round": 2 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "b2671ff7b3ef79c23e07858b7b9bc21c11a46ddbb806fc0b62fbbb698aff72e3", + "clip_similarity": 0.9651858806610107, + "nclip": 0.03481411933898926, + "photometric_loss": 0.01557935494929552, + "render": { + "bytes": 2986658, + "sha256": "fb61bda5ce7a38a421406aa318d9caebb86533a2eb202925e1a3f7265250ce47" + }, + "round": 3 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "701eacbd69c21a88a9412bfbb351fab8b6e12ee4d2b5d25cd930915a0657f83e", + "clip_similarity": 0.9659790992736816, + "nclip": 0.03402090072631836, + "photometric_loss": 0.01718028448522091, + "render": { + "bytes": 2981347, + "sha256": "e9649f3cc48cef0455f4f645ad3e38122ec10d154ed12b6b4082015a871f2298" + }, + "round": 4 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "a3e84f5be3ab210a64da01b4f00c0c65dea9a1b5c065da9134c040770bc84de3", + "clip_similarity": 0.9705029129981995, + "nclip": 0.029497087001800537, + "photometric_loss": 0.01845850981771946, + "render": { + "bytes": 2966621, + "sha256": "82559f9b41fb40ae33ddb502e7f333a3b2a5e0cb46f0f07296d3d28dba58420f" + }, + "round": 5 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "5999b5cfb151d2efba3eb72a76a8f73b3d8f5c08e884f2558b5943d83edb8dc0", + "clip_similarity": 0.9824894666671753, + "nclip": 0.017510533332824707, + "photometric_loss": 0.017971405759453773, + "render": { + "bytes": 2969069, + "sha256": "0b5573b05a83becc253ea611c733baa78aeb2da7bba228e756fa1ee625855e57" + }, + "round": 6 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "d58130d5ead2d4f5341127ceaaae428af480f68c79ab072854e8f45a6e63856c", + "clip_similarity": 0.983490526676178, + "nclip": 0.01650947332382202, + "photometric_loss": 0.017553135752677917, + "render": { + "bytes": 2967827, + "sha256": "9965efe5c3bf14db8579fe468aeb7ca23037dec11004961994b7da573517e402" + }, + "round": 7 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "dc3ab7a8c2b733c4f7e0fd90ef4d08a2f69bdce801c3336ad2903b252242b127", + "clip_similarity": 0.9839648604393005, + "nclip": 0.016035139560699463, + "photometric_loss": 0.01759149320423603, + "render": { + "bytes": 2968407, + "sha256": "989b5a838529141c027bb6e85459cec713338f4ffaaec405f4c17cadb6aee1f8" + }, + "round": 8 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "6a84d386171ccbc67abd38de70ed228349f8b567a865b07ffbde0382beefb3d4", + "clip_similarity": 0.9842134714126587, + "nclip": 0.01578652858734131, + "photometric_loss": 0.01759195886552334, + "render": { + "bytes": 2968179, + "sha256": "8be63127a6c56c9676e78f6922a4ae1bd2bf126b1ca56326b6f2b2f4cc3675dc" + }, + "round": 9 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "4fca13e1573704f948453c191cbcd3e0ee8f0c87218c958ad417a0f3a02c4c75", + "clip_similarity": 0.9838805198669434, + "nclip": 0.01611948013305664, + "photometric_loss": 0.0175719503313303, + "render": { + "bytes": 2968119, + "sha256": "6bedbf2ce1a6cf48ffa559a42ebccb6852ff4a04bc608b1b89255b5a91432140" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7659, + "sha256": "b599c3a4ae983dec616c337046b09b494852dd359d8163fe1588e042f425a647" + }, + "target": { + "bytes": 433064, + "sha256": "1eda79fb1fd8773714467694b24cbbf71f049549b0016cac79254724e5c5b5cf" + } + }, + "level2/attribute7": { + "evidence": [ + { + "bytes": 1268392, + "kind": "events.jsonl", + "sha256": "dce01e63512a9e3bafbdf6c3c96205ede419b550dc77f1c19c7493a6c3f9fea1" + }, + { + "bytes": 11651770, + "kind": "run.jsonl", + "sha256": "e35870804b513a9a37189edaf38510b90bcd9e44f40dd3380aa8e74059079ff2" + } + ], + "rounds": [ + { + "blend_bytes": 646096642, + "blend_sha256": "f010800f8cc8419c6d4ad399bbd1e5ad84c75b7c83412e6b3187b5a496164cf8", + "clip_similarity": 0.9875383377075195, + "nclip": 0.012461662292480469, + "photometric_loss": 0.005757831037044525, + "render": { + "bytes": 576705, + "sha256": "e43166183b876109daafa9c5b2880a63a8b4f0e068ca5dd6e73c3e38a1a11047" + }, + "round": 1 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "5dcc84073191c4099f17fb6ff4b524e41df6d1b9636bdc666eee43337f634a19", + "clip_similarity": 0.985919177532196, + "nclip": 0.014080822467803955, + "photometric_loss": 0.004302607383579016, + "render": { + "bytes": 576399, + "sha256": "366f4dd96fd968dc5ee35c4628fdf3a75b8b21a628d349ff2d2c1af08a1d28b2" + }, + "round": 2 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "dab28a97ceeaca0c4650c70cfdb74e71c3d3e3e06eb8212283461f375077fe1c", + "clip_similarity": 0.987267255783081, + "nclip": 0.012732744216918945, + "photometric_loss": 0.0038804830983281136, + "render": { + "bytes": 577059, + "sha256": "5864f5ca23351baed3de29ba097e60b86d0eaa766d704a9fbabd1a4aa043c9d8" + }, + "round": 3 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "f6fa92b9046bc84c8a169fbebde44301a0d9c7e7665521768ab6870085519eda", + "clip_similarity": 0.9858017563819885, + "nclip": 0.014198243618011475, + "photometric_loss": 0.003929381724447012, + "render": { + "bytes": 576889, + "sha256": "4f1ae0e453f929996644a0895051c19626f729e867831964fa29671cfaca2225" + }, + "round": 4 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "2698fe51ec0a9aca2afe883e25788a60ec38b4e47741efcdf5459715bf2824a5", + "clip_similarity": 0.9856680631637573, + "nclip": 0.014331936836242676, + "photometric_loss": 0.0038459450006484985, + "render": { + "bytes": 577057, + "sha256": "4185453069c3174730f35ff1f9ccc42d9eeaa80ad23cffd245efa3dab2bd207d" + }, + "round": 5 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "145bbba903be3edacd0d3783c98984b3b4f9f827847542a5e38b763ba61cab8a", + "clip_similarity": 0.9873605370521545, + "nclip": 0.012639462947845459, + "photometric_loss": 0.0036604765336960554, + "render": { + "bytes": 576927, + "sha256": "93089d36a5a0fc40f5c8dc923f3d74a61888e02ddf42444c27991ede736a19c2" + }, + "round": 6 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "51e6fed894faf60848cc0a17b078c2faad9b3c27245a9b9730aeadfa7a3d11a9", + "clip_similarity": 0.9880251884460449, + "nclip": 0.011974811553955078, + "photometric_loss": 0.0034597732592374086, + "render": { + "bytes": 576869, + "sha256": "e1a6d696a802e899a40dbf57b2226b4b0a4ecb39aabd788c90dfa5f6ec0f7148" + }, + "round": 7 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "b2d34676246a2065c9ef4c58f7b6b51ab38698208d23d2296623b8fd6ba7c527", + "clip_similarity": 0.9892903566360474, + "nclip": 0.010709643363952637, + "photometric_loss": 0.003371603088453412, + "render": { + "bytes": 577019, + "sha256": "b200a48eb287105d7e8b5003cfcfadd0d29662db5a8bd388f56775ba9a9e2caf" + }, + "round": 8 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "e6acf86451a2043c5db9d0bf51bd8e8fbe165d0e3f25d6da194a19df03afb511", + "clip_similarity": 0.9894779324531555, + "nclip": 0.010522067546844482, + "photometric_loss": 0.0033791407477110624, + "render": { + "bytes": 576818, + "sha256": "b8950d0069b25e584932e146deddf9fc18ffbc8d0ea738cb31dcb201e14904f3" + }, + "round": 9 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "22edd288b13f3b569418ef40c632825667ea2ca0f638d42bcc89526660842366", + "clip_similarity": 0.9892759323120117, + "nclip": 0.010724067687988281, + "photometric_loss": 0.0033672337885946035, + "render": { + "bytes": 576970, + "sha256": "1fcc64dd437079e420dac180cf3976845ed83d01de3368b5a2ba306890964a81" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7597, + "sha256": "d600bb2bbef13fb508547ef7c1330fde7c2f5de94535b187099ca7877445822d" + }, + "target": { + "bytes": 496461, + "sha256": "4e0dcf670648bba309e19853047bb294bbf588ffb8928ed9f9a45bef729f9f32" + } + }, + "level2/attribute8": { + "evidence": [ + { + "bytes": 1269817, + "kind": "events.jsonl", + "sha256": "b7dcb3440ba2f433f9141688aeb7a39b07d6255d9beac967c0b9db57cf6eae13" + }, + { + "bytes": 11669509, + "kind": "run.jsonl", + "sha256": "539448b867d21f6820fed02cacd420b43bae2cfcde0906ad58d9be0bcb3da685" + } + ], + "rounds": [ + { + "blend_bytes": 646096642, + "blend_sha256": "73de396f058a5e66af2eeb10d27e205b16196049c886e5e74917c2aac930ace7", + "clip_similarity": 0.9375687837600708, + "nclip": 0.0624312162399292, + "photometric_loss": 0.0443587563931942, + "render": { + "bytes": 573444, + "sha256": "400a007df0d28672ae0036c9bf53b52ccd4d69b03e0374facc29f40def0159ce" + }, + "round": 1 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "dcf1b9d54ed87e7fad2403a3719d28d3fc5c73686c7bdceedfdd9f480ebb802b", + "clip_similarity": 0.9542206525802612, + "nclip": 0.04577934741973877, + "photometric_loss": 0.01470732968300581, + "render": { + "bytes": 572568, + "sha256": "4d5bc9e81e0a901e67a14c2eda5882b2d0d2c0d7ef943fcf75d2c1966d006584" + }, + "round": 2 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "81856857f53fa28665f1c14fe7ac40407d84b4e893021e5dfacaa820def088d0", + "clip_similarity": 0.9696425199508667, + "nclip": 0.0303574800491333, + "photometric_loss": 0.008994323201477528, + "render": { + "bytes": 571842, + "sha256": "1ff4475afd98b45d8c8bcbde7f75b968df79ca1a71ff00a5e53cdf8d11adb64a" + }, + "round": 3 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "f1a86bc0f5e93a1a541bc7d828412799ae6b3b5755ed13b174c95705e701da67", + "clip_similarity": 0.970038890838623, + "nclip": 0.029961109161376953, + "photometric_loss": 0.009061017073690891, + "render": { + "bytes": 572324, + "sha256": "0d676318d42ce7dfe0d44f6857f247d4e3d27f21e0ad6ff2331fca36a85f448b" + }, + "round": 4 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "9ff249023bdab3ec1932eec1d3a0da91c831e67152369f682a93c6d8e26ffeb3", + "clip_similarity": 0.9676366448402405, + "nclip": 0.03236335515975952, + "photometric_loss": 0.007559685502201319, + "render": { + "bytes": 572659, + "sha256": "cc90ec970976b3176cda8a3e8eb0b598c2eb816c5d7cfa7096c55c5ab75a2c67" + }, + "round": 5 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "f1878d7c91d284ec41da0c8531fbe16b98e4c203c9cf682dd1a5da61e1c3a3de", + "clip_similarity": 0.9871110916137695, + "nclip": 0.012888908386230469, + "photometric_loss": 0.004122924525290728, + "render": { + "bytes": 577132, + "sha256": "9df09fa70f4d21a6916c84b9062dba64c7a847a5ffee44e7007fde26f3abe36f" + }, + "round": 6 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "384dc16e511781ab3ec8c830d9d2045672ba32ea1ab32efcc17e4da1b7cba162", + "clip_similarity": 0.987730860710144, + "nclip": 0.012269139289855957, + "photometric_loss": 0.003989724908024073, + "render": { + "bytes": 577326, + "sha256": "5d6115de3cda15c3c5edb77fd7842345b7748426add554fa18fe55b4588ee261" + }, + "round": 7 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "d0318cf8cfa1034e925bf24e28103f692446066b29ea40e29516472bbf7334a7", + "clip_similarity": 0.9878823757171631, + "nclip": 0.012117624282836914, + "photometric_loss": 0.0037759870756417513, + "render": { + "bytes": 576810, + "sha256": "6810f1ee430eb2b63c7694ec495fe2b52c0ff043e3e6c465c704b9e4783386a4" + }, + "round": 8 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "bdb4d0b187717c514a8481db1c855046055ef1cc18f6f1e4ebd90b1e2b81dc98", + "clip_similarity": 0.9888083934783936, + "nclip": 0.011191606521606445, + "photometric_loss": 0.0037378640845417976, + "render": { + "bytes": 576794, + "sha256": "6e5a6d132ac90ca4f581afaa6ad495434e9e8067c3ad4f6bc6cbd1d1db497647" + }, + "round": 9 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "40ff077ce3c637351b247c5b1902cef94dcfeb58c348038827fc2d737bcad1d9", + "clip_similarity": 0.987591028213501, + "nclip": 0.012408971786499023, + "photometric_loss": 0.00383252021856606, + "render": { + "bytes": 576676, + "sha256": "214d003df0c9c9708a976176766bfe1e34d13ce1a2d0fa37278180551a19f6c4" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7582, + "sha256": "b5348ae5fb98fb5110a75bb829540b2c5b0c00b45d81957e8a4c4d27d24a02c3" + }, + "target": { + "bytes": 496462, + "sha256": "2177e15ad99cb6559092b37de04aa778b13445c89598736ab67ab4f54cfa4ec3" + } + }, + "level2/attribute9": { + "evidence": [ + { + "bytes": 1276347, + "kind": "events.jsonl", + "sha256": "38e78ac3ba9131429b4ba1e7991fffae24fec67fc38f1562cfa6d55340c87618" + }, + { + "bytes": 12677793, + "kind": "run.jsonl", + "sha256": "388b97b7af39d9fcf422acad433cddf9804d69fe5d9746a17b1b14e036edb657" + } + ], + "rounds": [ + { + "blend_bytes": 646096642, + "blend_sha256": "65eb87dbb60382acde0d1755328380612a8d56d7358d1dcd8bedeaa9de220f44", + "clip_similarity": 0.9311380982398987, + "nclip": 0.06886190176010132, + "photometric_loss": 0.04209288954734802, + "render": { + "bytes": 574086, + "sha256": "10e204a80bd71a7a98f72f274c75c09a041cb5e077ce0fd1e50af31710267a6a" + }, + "round": 1 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "1a8a7ab0d907850fc59f6c454ae27cca90a9213ffcc37b09c075b32f426b7ebd", + "clip_similarity": 0.9439935684204102, + "nclip": 0.056006431579589844, + "photometric_loss": 0.009841673076152802, + "render": { + "bytes": 571044, + "sha256": "1669dd4298dccfaa757ce39fc7f7fb66843b172505215f6562072d0c365abeca" + }, + "round": 2 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "493243da91270ef58f53094a9ee876537fc6e8495b10d6624341d7e069917b51", + "clip_similarity": 0.9462493658065796, + "nclip": 0.05375063419342041, + "photometric_loss": 0.008510514162480831, + "render": { + "bytes": 570889, + "sha256": "c6b64498610d811209b02e1cd1724444ac9c2d5a4af03620af5eaa3cc9d37242" + }, + "round": 3 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "a0325a906e9c9b292df4a4de13512170efe8498e3795d8ca6660db8f661598cd", + "clip_similarity": 0.9438660144805908, + "nclip": 0.05613398551940918, + "photometric_loss": 0.008608759380877018, + "render": { + "bytes": 570975, + "sha256": "c5257707ea03978ad6f75f38255d29507b4b9846b3496897dfab85696dfc5eec" + }, + "round": 4 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "008b0c185b35c47692c3937ddf89edfac927776c9f63158519760c7f84cdf616", + "clip_similarity": 0.9444617033004761, + "nclip": 0.055538296699523926, + "photometric_loss": 0.008243502117693424, + "render": { + "bytes": 570731, + "sha256": "8b566bea54659004bd45ba2d8f50679d73218ee178e6a64f7b63dca1f872762a" + }, + "round": 5 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "93b8ece3bc1c9a7f868497a3c1192facf04da985ae485c60aead9ffb136bb89e", + "clip_similarity": 0.9668443202972412, + "nclip": 0.03315567970275879, + "photometric_loss": 0.007134303450584412, + "render": { + "bytes": 572242, + "sha256": "94a6fed6d0aee4f0cb136952aae7d279e7780784adf79afe64ec3e023313fc0a" + }, + "round": 6 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "74715f61e1e06946832481bfc09418287d26930ee9fa0d1ef97dcce44750b455", + "clip_similarity": 0.9873204827308655, + "nclip": 0.012679517269134521, + "photometric_loss": 0.0035751655232161283, + "render": { + "bytes": 576926, + "sha256": "7596968060f472fe018c3e6f988012182737ba85cb1ea77630660c30cf521cc8" + }, + "round": 7 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "83b0147bb39cf23b0b303085db533eba80450afdc5f63495536f376f842a7791", + "clip_similarity": 0.9876059293746948, + "nclip": 0.012394070625305176, + "photometric_loss": 0.0035959018860012293, + "render": { + "bytes": 576865, + "sha256": "61ed38e1bd458ef76a09a872abc0a41166c5e0fda121a09ba2a3659ba676016b" + }, + "round": 8 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "01eddafba8a08b92cea3d25023d646ed144c61f16067accc24429cb2d138e322", + "clip_similarity": 0.9885677099227905, + "nclip": 0.011432290077209473, + "photometric_loss": 0.0034589776769280434, + "render": { + "bytes": 576751, + "sha256": "7e43417fb69bd19e1c42fa77859a41c8a2cb3816814277dbc3dccb5e7e6db016" + }, + "round": 9 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "b65a33473c91499cee0a7fd21040d8254e02769eae479ce4a4de586136cb1f1e", + "clip_similarity": 0.996465265750885, + "nclip": 0.0035347342491149902, + "photometric_loss": 0.0007936670444905758, + "render": { + "bytes": 495582, + "sha256": "ab971bde432f76467214da31dc9e30f7cc8df2c07757d1ecac6fc41cb15bf4c9" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7592, + "sha256": "072b3a428c33b2c6746f69c8708efca58a82f7d66d311c0b6a517e2f11e62503" + }, + "target": { + "bytes": 496462, + "sha256": "057562178eba0d2cf993227378f6f93c2689eda870a060cc836edce85f892eb0" + } + }, + "level3/attribute1": { + "evidence": [ + { + "bytes": 1924773, + "kind": "events.jsonl", + "sha256": "7cdd1d853afc32fab164046ad650f136846d170f0bbdda839986cf8a2acd37ba" + }, + { + "bytes": 12456131, + "kind": "run.jsonl", + "sha256": "649267b8bd1af6d394b76f78e1a949c8b967d571cf3785532873067b5bcb6a5c" + } + ], + "rounds": [ + { + "blend_bytes": 943257481, + "blend_sha256": "d196ab227eeb0d9bbf8aeb5a3fc21d2c3fc9160706f86b5ec83725bb71563e9e", + "clip_similarity": 0.9098252058029175, + "nclip": 0.09017479419708252, + "photometric_loss": 0.07946688681840897, + "render": { + "bytes": 437856, + "sha256": "cf768dc71d60754b6b47878bc3083d7ee4fb3a134a47b7308f2555911bc8612d" + }, + "round": 1 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "68d96abfe206c419a9b0f5ac0502934cfcbebe752a21bcdb1622e831f0a734f0", + "clip_similarity": 0.9217170476913452, + "nclip": 0.07828295230865479, + "photometric_loss": 0.07681163400411606, + "render": { + "bytes": 445514, + "sha256": "90222ee64633245062ec6900dffcc4a661528c569a4be1553c857749917d774a" + }, + "round": 2 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "d7f754f89eb59ce922735307b77d0b3f3f2f75b1c86b73e210e96ae022cb5de9", + "clip_similarity": 0.6607498526573181, + "nclip": 0.3392501473426819, + "photometric_loss": 0.3766263425350189, + "render": { + "bytes": 316640, + "sha256": "719cb3b1a6c9fa887e9acd06dff160b6ee7dc236e0babab9485db4fefda268af" + }, + "round": 3 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "9daa282a29611c0727e3079e84d9647230e11c910663fa98fbec755cb53f3c59", + "clip_similarity": 0.9430404901504517, + "nclip": 0.05695950984954834, + "photometric_loss": 0.05672503635287285, + "render": { + "bytes": 440654, + "sha256": "00c1aa47d6e554d689a6747d23871be8e903c0e9135c2a1b5272cfce3417f9ab" + }, + "round": 4 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "85aea62426e1edb97eac76a62c93485e15bfc0a7aca1303dceb7f0ec16069363", + "clip_similarity": 0.9597296714782715, + "nclip": 0.040270328521728516, + "photometric_loss": 0.04194614291191101, + "render": { + "bytes": 441536, + "sha256": "6cf37b6dac65ed8075ef15ee2c1918cde72bbfe506b2d806494eb1a33fcc7f90" + }, + "round": 5 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "cc2dfa99a5a9935af1116993ddc9a075b53e8dbe3f33ed6e8c4b0028a63538b2", + "clip_similarity": 0.9801073670387268, + "nclip": 0.019892632961273193, + "photometric_loss": 0.027477452531456947, + "render": { + "bytes": 439684, + "sha256": "83f1a404ed94adcda1685a3056d0684b98e0edc4cb1fa0576fcdc8780ff98c30" + }, + "round": 6 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "a96d54d0a9081d8e217e96d05ea79e7684589ec80d83960c428045802c771bbf", + "clip_similarity": 0.9802656173706055, + "nclip": 0.01973438262939453, + "photometric_loss": 0.02749941311776638, + "render": { + "bytes": 439822, + "sha256": "2b962ad059b2d2267fd57a258eef56a1ae0ef55c87d2dac8dd05e57ff723b473" + }, + "round": 7 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "d8e0cda6fa2d2c2fe3b7083ef94459d7ce48d87d1d07a0786419b517c277cdc2", + "clip_similarity": 0.9807744026184082, + "nclip": 0.019225597381591797, + "photometric_loss": 0.020207971334457397, + "render": { + "bytes": 436411, + "sha256": "058865231de1aac153469994e240deb6bebeaf37a9f556236c136ccd5506fe3b" + }, + "round": 8 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "0afd50da6af25a3b0a7a0ed2bfbbb164a7d6327260601a5cfbde615fcb6b46cb", + "clip_similarity": 0.9810705780982971, + "nclip": 0.01892942190170288, + "photometric_loss": 0.02021031640470028, + "render": { + "bytes": 436315, + "sha256": "adacdbbf9d6138369df8e09b1c6cd6ff71221d7b2c64d6126c2796e4471db7ee" + }, + "round": 9 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "62f5babbc4d88c7ae4c623ce1d2662d63f347ea71ec83cbd7488a952b0f6b882", + "clip_similarity": 0.9806057810783386, + "nclip": 0.019394218921661377, + "photometric_loss": 0.02021102048456669, + "render": { + "bytes": 436030, + "sha256": "2687228629f47deba7af61617bc8dbcd6bb33fba1336cbb6a65ecf085ae68d66" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7497, + "sha256": "4aff82f8cd193a86a1f54a03287777c24817cb4bc52e0e7ee3103279f5fc1c3d" + }, + "target": { + "bytes": 424605, + "sha256": "203bc0d5b3cd6bff06e19baa56c56370b5a8da9155b5b0ee3e8598a0886318f0" + } + }, + "level3/attribute2": { + "evidence": [ + { + "bytes": 1648840, + "kind": "events.jsonl", + "sha256": "1796cf588ab2454593ce4d759892963c85228b9512a5bd8abaa1161b9129471d" + }, + { + "bytes": 11476592, + "kind": "run.jsonl", + "sha256": "30a6423455572a7fe343aabb62794392093eb06b2101372fdd2a7fe7edcd2610" + } + ], + "rounds": [ + { + "blend_bytes": 943257481, + "blend_sha256": "f64441abd6439ac1d2a105519fcdaf483db5e2618b8052d2ce54d5a34ff5bce5", + "clip_similarity": 0.8939650654792786, + "nclip": 0.10603493452072144, + "photometric_loss": 0.07529605180025101, + "render": { + "bytes": 511612, + "sha256": "a6c4d0d9d2448febf5f155cb60ac241f0811eed9cf6a415ae031d4a911d7697d" + }, + "round": 1 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "24c0a33d4b340a2fd3391612525871605a352abc4f5f4f5a2afd123cbaeeae81", + "clip_similarity": 0.9033817052841187, + "nclip": 0.09661829471588135, + "photometric_loss": 0.06520041078329086, + "render": { + "bytes": 512949, + "sha256": "6528a4a2567cf905f829616137cc7bc0c783c9a4091615814045aa17df523d54" + }, + "round": 2 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "bd30a0600b2595792d3456e96ab55bfc94f1f85c2f196a95c58c5677be88ef74", + "clip_similarity": 0.9098818302154541, + "nclip": 0.0901181697845459, + "photometric_loss": 0.07429742068052292, + "render": { + "bytes": 503387, + "sha256": "f19bec2f145aadc6c83b64f02671186d267d4f682e4ca839295d3d519e23188e" + }, + "round": 3 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "6da5e0d68be1c841f0e9537c1d685d8874639191e62b5788ac4d13a92b724a13", + "clip_similarity": 0.9416781663894653, + "nclip": 0.05832183361053467, + "photometric_loss": 0.05184561386704445, + "render": { + "bytes": 511333, + "sha256": "1f61013aff2ec53764f84271a4cbfa5546c636093c7f12b7feed20ca83f6b457" + }, + "round": 4 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "6a6953527030927b5051b0844cf92c544215fbb57536195ce7efdfb198a1080a", + "clip_similarity": 0.9510102272033691, + "nclip": 0.04898977279663086, + "photometric_loss": 0.03256066143512726, + "render": { + "bytes": 507821, + "sha256": "f865c47e2403be7539d29d041ce5bf8034b6a6de233bd243a54e11abbd5508c5" + }, + "round": 5 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "ccebe27c27c0d8694e818b5fea0212a9d676575fb181758e5820a94b272cde5a", + "clip_similarity": 0.8367416858673096, + "nclip": 0.16325831413269043, + "photometric_loss": 0.19988812506198883, + "render": { + "bytes": 296806, + "sha256": "cb06bca484dbcd440dee336127c540a0f93954da6feabcb504771aa7425fb523" + }, + "round": 6 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "d2b09aea2bd718d2c40697217d2cdde50fd038890477e2faabfc28fd2cf7850f", + "clip_similarity": 0.9569584131240845, + "nclip": 0.04304158687591553, + "photometric_loss": 0.03064643405377865, + "render": { + "bytes": 504544, + "sha256": "32a966af73f2e7399fef60937c27651683e0ba13668cf194d513c9f61dacc506" + }, + "round": 7 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "211ac3fddfb7a4282969cfef6ca9ede8acbc177ca73160ea0b8c7fb11c7d1e13", + "clip_similarity": 0.9623216390609741, + "nclip": 0.03767836093902588, + "photometric_loss": 0.039537735283374786, + "render": { + "bytes": 510452, + "sha256": "af1cdcf0edf152c31ad1983be1f0e9b858ab66307ae827820d63170e3e466d40" + }, + "round": 8 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "0a32a779c25755655e37fa85d78e4e232b5e8e5553a942a968e34c0ad1a688ee", + "clip_similarity": 0.9504269361495972, + "nclip": 0.04957306385040283, + "photometric_loss": 0.038483042269945145, + "render": { + "bytes": 508909, + "sha256": "ee29f6f931c0849431ba91817002b21646b80c9b99c58b86430aa64aaee33513" + }, + "round": 9 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "287a36e751701e2ed584fb60b2521dd41e9199d8ed0a2db2c0cec09b1dbc22dd", + "clip_similarity": 0.9837775230407715, + "nclip": 0.016222476959228516, + "photometric_loss": 0.012915946543216705, + "render": { + "bytes": 497559, + "sha256": "913b594d0a51def59197e77da626dfcf6c1d8d3a63884fde437992d6e44e0cc3" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7576, + "sha256": "15d6c6bba6b98581aebffe05bba7d8d2527675719507cea1edf87bcf3ffdfe30" + }, + "target": { + "bytes": 424611, + "sha256": "b5b1526c9fde7aecd35f842598627b63172338284d69aaa36d6a40649200ca30" + } + }, + "level3/attribute3": { + "evidence": [ + { + "bytes": 1644760, + "kind": "events.jsonl", + "sha256": "2276cbb35c03fe4e10c8fc0c2ec46307a73489d0b6e3a77ac374347bd9391bbd" + }, + { + "bytes": 10571760, + "kind": "run.jsonl", + "sha256": "32c1768926f7f9c6eb0b093daad72ffaa28cfc0edc5abf05989ec091853623d1" + } + ], + "rounds": [ + { + "blend_bytes": 943257481, + "blend_sha256": "905d034f447fa149ba400181b0b0eb11eef8eed7c31f9915e600919c7ab825f0", + "clip_similarity": 0.8677472472190857, + "nclip": 0.1322527527809143, + "photometric_loss": 0.09606408327817917, + "render": { + "bytes": 531126, + "sha256": "8ec0f8b94bb97d48e9c3af48113a182cdb61caa45d3a0f689e02fce1f087b2e9" + }, + "round": 1 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "a8cb2f5caf6ceabad2e6efaefe58406c8af74faa6a3584c5b5683fd47762fca9", + "clip_similarity": 0.8839718103408813, + "nclip": 0.11602818965911865, + "photometric_loss": 0.09201572090387344, + "render": { + "bytes": 506872, + "sha256": "db26a334e5aefac3b30123caf3084e721940a51b7900db2505e11e73bc942ee3" + }, + "round": 2 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "e1647875e6179b6bf709f808044c22c5e94ba0beffb9eadd1e349525fb3a975e", + "clip_similarity": 0.9038984775543213, + "nclip": 0.09610152244567871, + "photometric_loss": 0.056030403822660446, + "render": { + "bytes": 517855, + "sha256": "d534c627ffd56ccaa3722914c59a5b50d069fad68da9e9227f0d35c18daa31fa" + }, + "round": 3 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "72d6a98fbd8f7346ea2662591e3422a99c59bb7a51fbf8f32584c5fa55f6b8c3", + "clip_similarity": 0.9551177024841309, + "nclip": 0.04488229751586914, + "photometric_loss": 0.04528190195560455, + "render": { + "bytes": 554013, + "sha256": "afaf64992f3693dde1ea564a7ffa53e0ec6d54601c3ce3e7ff2711fa3c6842f1" + }, + "round": 4 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "89eb345c12ed2c5f55ab43bcbccfae3f09661f9862c90fc1f52c258d977dca40", + "clip_similarity": 0.9572995901107788, + "nclip": 0.04270040988922119, + "photometric_loss": 0.024362504482269287, + "render": { + "bytes": 550803, + "sha256": "8fb0524191c1fa0ab09a05fc94292bbed3dfa330d37ae6aae0e5781f2417096c" + }, + "round": 5 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "a648d1134f65df7a2c7a86ff449c27667035115f6ae41daa02d8a2d17a0c67f8", + "clip_similarity": 0.9733030796051025, + "nclip": 0.02669692039489746, + "photometric_loss": 0.013502489775419235, + "render": { + "bytes": 531895, + "sha256": "5b565a0695486f1c49762938d3a0c6b53fd608b3c73897bc9aaf5628f026687e" + }, + "round": 6 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "6a8ed76a3c9777d077bf60bfe84dcb5be3342a98bd1788558f7237c91387db3e", + "clip_similarity": 0.9710009098052979, + "nclip": 0.02899909019470215, + "photometric_loss": 0.015630224719643593, + "render": { + "bytes": 532191, + "sha256": "4b996e79216f6a6a7c4593b95168ebe1e8d89533838987c364a677be813d6126" + }, + "round": 7 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "f02bc9c2cd08e19be10ae26b8c295d4f492539669ba17284c8316714fad84c87", + "clip_similarity": 0.9763938188552856, + "nclip": 0.023606181144714355, + "photometric_loss": 0.0294965710490942, + "render": { + "bytes": 543864, + "sha256": "82f2ad0e099d4d62fa6bd6050cf4096883ebb3848aa096a5d6dc54f62081f942" + }, + "round": 8 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "ebf489166ca86013939c9563ae1621a7574bef760e81a73ead41e3d71490e7fa", + "clip_similarity": 0.9807580709457397, + "nclip": 0.019241929054260254, + "photometric_loss": 0.01762007176876068, + "render": { + "bytes": 493951, + "sha256": "15543161db39db67314ef96bc5cdbc62dec33d0b7f0aadb0532914f2c6c654fc" + }, + "round": 9 + }, + { + "blend_bytes": 943257481, + "blend_sha256": "2bed5b84e5832e3b1f77c5f708610020e17fc1c99abfa551d26af761ef5eff5b", + "clip_similarity": 0.9846137762069702, + "nclip": 0.015386223793029785, + "photometric_loss": 0.014947629533708096, + "render": { + "bytes": 452720, + "sha256": "8792f280a74d5241396fd7258a97a4c3764a27ae2fac012057ce47c0b041c0dc" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7579, + "sha256": "6b06718714b446687104cd3fedaaf1013858fbd898226eac493653e184a89e8c" + }, + "target": { + "bytes": 424612, + "sha256": "741983290631f850e8e7a011679328f9e6da5545a671c23d73c17beb88cae346" + } + }, + "level3/attribute4": { + "evidence": [ + { + "bytes": 1478020, + "kind": "events.jsonl", + "sha256": "f36c9bc8498f565c429033c4a5bdf60a965eed0c49b1a980c3c4e3487c362541" + }, + { + "bytes": 10564972, + "kind": "run.jsonl", + "sha256": "67faba27e77874d461fc6688d2d315a5f0925c5111b6c93ef3d1451b95cf41fb" + } + ], + "rounds": [ + { + "blend_bytes": 934082247, + "blend_sha256": "5131d3c803ea6d5620d173cb3659b1b33bb51e0fd6bf6e9af218031f8f0495ee", + "clip_similarity": 0.9420864582061768, + "nclip": 0.05791354179382324, + "photometric_loss": 0.0750909373164177, + "render": { + "bytes": 518774, + "sha256": "56af8f831a16e0affc7ce9696c9c4fb253573530b80ed1b2540951245d30e4e5" + }, + "round": 1 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "b59d30f43b0d5ef05a4f3e4fe7a1ca6564b731180f5ce5e2dd3474e68eb1a872", + "clip_similarity": 0.9318283200263977, + "nclip": 0.0681716799736023, + "photometric_loss": 0.0782821848988533, + "render": { + "bytes": 541032, + "sha256": "2d4b4e7b495c075015e7f8f72db9f0bf109b0ba9acd92e58870a1d9a383e6068" + }, + "round": 2 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "4183228afb42c0a971a413e1ee40e9a7fb35e36834375b1fbc5fc3e798590e90", + "clip_similarity": 0.9207460284233093, + "nclip": 0.07925397157669067, + "photometric_loss": 0.06497704237699509, + "render": { + "bytes": 521347, + "sha256": "db83000464d25db6c197fb7fd496f2a4038006152f2da01e09d7152c713a613a" + }, + "round": 3 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "7926b755715f0652bd793937d8d2abb4c6ec64842d13d3a5222b9e5a90560686", + "clip_similarity": 0.9671326875686646, + "nclip": 0.03286731243133545, + "photometric_loss": 0.05156431719660759, + "render": { + "bytes": 524447, + "sha256": "01a129ff6fd46e1324553977743f41ddacb44575e4e7262970d5f92967282cbe" + }, + "round": 4 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "a5bb81a44cd89e75af3107de310cde23a9312bb9af08278e4c18fce24a002850", + "clip_similarity": 0.9705365300178528, + "nclip": 0.029463469982147217, + "photometric_loss": 0.0347987599670887, + "render": { + "bytes": 533853, + "sha256": "9147bfc573837e397ccd76efbd4f8d5882405659c76ce9ba49c6579e4c233d57" + }, + "round": 5 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "aeeb801dcbbb6c982c0074e4344fd4a7380c9aee5ddc7709bbd495be51f92f8d", + "clip_similarity": 0.9707145094871521, + "nclip": 0.0292854905128479, + "photometric_loss": 0.034984197467565536, + "render": { + "bytes": 534942, + "sha256": "fc0ded05e44e3a0e9f4dba8fb4d6c103ac224824a00336f018d29a6873bc5388" + }, + "round": 6 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "be5a3763f00350a309a71ac620098d52dc50dab22b3963c0e823f4661569d376", + "clip_similarity": 0.9824424982070923, + "nclip": 0.017557501792907715, + "photometric_loss": 0.013661672361195087, + "render": { + "bytes": 530968, + "sha256": "8b33343c64def130b047f44a14aa705238158894437019a1c8f965daab8a01bd" + }, + "round": 7 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "038ba840c2453f5af9fd5949791f5186394b00af55fc2e0509cb2862ecd4f164", + "clip_similarity": 0.9853713512420654, + "nclip": 0.01462864875793457, + "photometric_loss": 0.01268050167709589, + "render": { + "bytes": 529286, + "sha256": "d07faae2bc18fcf89132a62e53f65ba07a9330dbf901fd627c125269c575ac54" + }, + "round": 8 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "466923ada6fc70c35d2bcd255b148e48ee0391bf1f8431812a6e4c5c9bbed192", + "clip_similarity": 0.9894673824310303, + "nclip": 0.010532617568969727, + "photometric_loss": 0.009694959037005901, + "render": { + "bytes": 467846, + "sha256": "3dda0ce6e66793f067744c83fe968cd89e8ae99b135944bd14570583a073ab0c" + }, + "round": 9 + }, + { + "blend_bytes": 934082247, + "blend_sha256": "7d59841ce102985dd971eb78b784333422990ae7363f7b4af056f5396faa6327", + "clip_similarity": 0.9926517009735107, + "nclip": 0.007348299026489258, + "photometric_loss": 0.009027284570038319, + "render": { + "bytes": 419004, + "sha256": "bb42dd9c8331b8da388256afd9d09bf9991ab02474f7ac55b719a4d6f2fba3ce" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7576, + "sha256": "0d6920e9f4a598cc948be29abfcea38d9a160c2c8a1d77039f81b47005f9609b" + }, + "target": { + "bytes": 424822, + "sha256": "2dfaea1568f9cfb0400600c874e214dc9b15578708571d1d9897605330ffc928" + } + }, + "level3/attribute5": { + "evidence": [ + { + "bytes": 1479032, + "kind": "events.jsonl", + "sha256": "b15c7851740791b688efcb909bb256af0780fe0b5633af4f7b7e9d8ae3b8c91f" + }, + { + "bytes": 10176930, + "kind": "run.jsonl", + "sha256": "90735fcf49a82c05fe534f3abf446b9bdfa9e61ab50fe60d1f951136cb135d6b" + } + ], + "rounds": [ + { + "blend_bytes": 940184532, + "blend_sha256": "ae784d052246c405391e7e71604c7927d2cfe04f5155568dc003d3894b635d93", + "clip_similarity": 0.8690069317817688, + "nclip": 0.1309930682182312, + "photometric_loss": 0.0782204195857048, + "render": { + "bytes": 725607, + "sha256": "544270131b52b81beaa2e57be047b15a1db9678a809c4712edc32cf42901a1b4" + }, + "round": 1 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "436800aaa9c10a414dd635758d90906e0d6ff20d33247dea3fbd66fa931caa3d", + "clip_similarity": 0.9351325631141663, + "nclip": 0.06486743688583374, + "photometric_loss": 0.07839251309633255, + "render": { + "bytes": 611342, + "sha256": "590769f6693e689a84f807c169c5a298b43d39875f4f5e56dbe6740600938cd1" + }, + "round": 2 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "86d3053b64c349574ca2d291e19d7a8fedd27cb152eb5a5a5b66907d0c6ecbfe", + "clip_similarity": 0.9289098978042603, + "nclip": 0.07109010219573975, + "photometric_loss": 0.07732649147510529, + "render": { + "bytes": 599245, + "sha256": "b564d25de21faa10b32c848145a308241cdf9a54934bf1914a65e35fa9f67a0f" + }, + "round": 3 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "682440f54704db53d5a91caa8e74ae6669f53f4a08665df4583b945f31383a2f", + "clip_similarity": 0.9625266790390015, + "nclip": 0.037473320960998535, + "photometric_loss": 0.0576217956840992, + "render": { + "bytes": 594935, + "sha256": "8bf4ee786c6a1eb1e440d449aa606b584c466ead3484c2c69e8793b1c5d31f93" + }, + "round": 4 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "2d68ae4a509946e865a2cba319a8ee268dcb5cad6c00c4591b8e64be6ef222f3", + "clip_similarity": 0.9493187665939331, + "nclip": 0.050681233406066895, + "photometric_loss": 0.032019589096307755, + "render": { + "bytes": 610052, + "sha256": "a9dd9a766aa0d43a5851ce285b046507dbca81a34fe107c7b943c2a45d35bf65" + }, + "round": 5 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "b5a579ead8aa971a546b0a709b91800a3936b2bf9336289a7f0807255f574deb", + "clip_similarity": 0.6391876339912415, + "nclip": 0.36081236600875854, + "photometric_loss": 0.3678968846797943, + "render": { + "bytes": 235433, + "sha256": "0401ab41438e8a708fa67a9b89dff04305d5bbc8d58979885dbaf552f107fb7e" + }, + "round": 6 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "2217799d1aa027f4e4720dfc8532f61ee3f5619638bb5d0fa8838ebe574c7cc4", + "clip_similarity": 0.9608088731765747, + "nclip": 0.03919112682342529, + "photometric_loss": 0.041936907917261124, + "render": { + "bytes": 605091, + "sha256": "1de3519a3b0fdc4ba96da6c1df9844220789c1c91475526aa9bb00c53561e0de" + }, + "round": 7 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "1a8dc07690a76e19413bb86a3e2cec684f426688f276f7a61b3517486e286ace", + "clip_similarity": 0.9751001000404358, + "nclip": 0.02489989995956421, + "photometric_loss": 0.024898789823055267, + "render": { + "bytes": 602765, + "sha256": "5b9bb6f16b0fb1f2d9abaffc4c52ff83b1c2f4aa321676b874323215a2c5472e" + }, + "round": 8 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "cb960c4f119e78b50f373c26297d7c4c75f033e2b483f6cde99874f07f744cbf", + "clip_similarity": 0.974507749080658, + "nclip": 0.02549225091934204, + "photometric_loss": 0.020625248551368713, + "render": { + "bytes": 603952, + "sha256": "b3e352a5da9f0e21b129867c2465c884f9019cb4a8e8ee2c5ecb7ac66ac7a311" + }, + "round": 9 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "08aeae231f2be37bdd5058a86a8f4ffe44636b34d33122ca0ecbe9ed100cb22a", + "clip_similarity": 0.9755886793136597, + "nclip": 0.024411320686340332, + "photometric_loss": 0.027274271473288536, + "render": { + "bytes": 596760, + "sha256": "dbed381c401543c62e4f26cddbe2e50ea08e1330019ffe940a771b7a4602c996" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7576, + "sha256": "cdf73038198bc88a3d5dcced62db3f1375a6bf0ba242800ee40708b3abb8ccb7" + }, + "target": { + "bytes": 432648, + "sha256": "08f94e6af9f1bd74f52a7302f7673da3ebfb318437d4a986780158967944635b" + } + }, + "level3/attribute6": { + "evidence": [ + { + "bytes": 1479058, + "kind": "events.jsonl", + "sha256": "ce20e19c7ddf51a6b2eb085a1da7a1b83458dab12bd8ef86db33c85d81071ed2" + }, + { + "bytes": 11625245, + "kind": "run.jsonl", + "sha256": "94a1575055e8e26390a7b84caee27fc9edb22d235738eb63680e7f3e59f80f0f" + } + ], + "rounds": [ + { + "blend_bytes": 940184532, + "blend_sha256": "2e10801c0b2e77211b33afd7492f5638fe12da4bdcbff624512d941e399becb2", + "clip_similarity": 0.9185791611671448, + "nclip": 0.08142083883285522, + "photometric_loss": 0.09303504228591919, + "render": { + "bytes": 596763, + "sha256": "e3355b29953875aa82ba482b63eff6674fd9a763dd457a5033f755cf8d455cc7" + }, + "round": 1 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "f39c0020040f111c6b53e1c3d3f076cac0bb9bfd5a14ad7f4f78fd9874f3b8d3", + "clip_similarity": 0.897698700428009, + "nclip": 0.10230129957199097, + "photometric_loss": 0.08139199763536453, + "render": { + "bytes": 586908, + "sha256": "a09c0d7bc6d9d63406d10c19182531621b730dfd11bcc5ee044686110ddfeed9" + }, + "round": 2 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "ea5434425a94e0f42114ecb237b2c2b2671c57ff7c8e08f88fcfa16a79c2a30d", + "clip_similarity": 0.8258225321769714, + "nclip": 0.17417746782302856, + "photometric_loss": 0.05298319831490517, + "render": { + "bytes": 577033, + "sha256": "5e08f00ec3785c7ca030cc223613ef4036a17baacd5eb948803fada83323064e" + }, + "round": 3 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "bed818ddacbf1a2964f2a99c6a194a94fbeee9408e9e568a9ed444b0dda2366d", + "clip_similarity": 0.9696477651596069, + "nclip": 0.030352234840393066, + "photometric_loss": 0.04301448538899422, + "render": { + "bytes": 620402, + "sha256": "21b69c56fcece4b31aa6fc34627614cc3161c5af4e307db0af2dfb18bd940118" + }, + "round": 4 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "f724b2370bccdc87fe1f4b5e563f92fef051a55082fb0542b115acf86998d407", + "clip_similarity": 0.9585986137390137, + "nclip": 0.04140138626098633, + "photometric_loss": 0.03180944547057152, + "render": { + "bytes": 617964, + "sha256": "2274d4e087ca73732458ec1e23069748cc87676a1a29ac464375d0e89b0e7d60" + }, + "round": 5 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "ccd2b0edaea8df1f981e3b1606a0290f52c08fdee96689c36044e61dbadbc82c", + "clip_similarity": 0.9793316125869751, + "nclip": 0.020668387413024902, + "photometric_loss": 0.0314413346350193, + "render": { + "bytes": 618033, + "sha256": "61cef4e090e4f8c09db87e51befbe0cecbd94d9d433d7cf5ca2ba35cbc8642b0" + }, + "round": 6 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "3952c0bb2e00da4421838cbbca61b902fecee0c146e344b4871de27ec26a9ef9", + "clip_similarity": 0.9866102933883667, + "nclip": 0.0133897066116333, + "photometric_loss": 0.025586744770407677, + "render": { + "bytes": 609871, + "sha256": "7927b12414c1d691efcd7c95b554b75c14ea12aadc7b184f173f240685f50841" + }, + "round": 7 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "eae5c2753a35d03c775057a9845b22d65a5fdbd8e0bb2a679a60c60c4a0d5ea8", + "clip_similarity": 0.978234589099884, + "nclip": 0.021765410900115967, + "photometric_loss": 0.015896320343017578, + "render": { + "bytes": 615623, + "sha256": "a179f352e23d8d71a1367c6a9794356ce1898634880515da3548060be3a54687" + }, + "round": 8 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "6a0e7a867c3065fb3d3e389da6e7eb6d04b5026e512a5f67bebde3b89a89866c", + "clip_similarity": 0.9787060618400574, + "nclip": 0.021293938159942627, + "photometric_loss": 0.016108734533190727, + "render": { + "bytes": 619480, + "sha256": "72c5cd08fd24387ade745db035260bfbfaa1cdd95e5c73bfe57706c9b74037ea" + }, + "round": 9 + }, + { + "blend_bytes": 940184532, + "blend_sha256": "c86bf1322ded94e74006940577614957b3245386a858ea762fe8e4bc1dd6b48b", + "clip_similarity": 0.9853477478027344, + "nclip": 0.014652252197265625, + "photometric_loss": 0.016418581828475, + "render": { + "bytes": 544474, + "sha256": "3ff9197ce859982919bbf227f0ca2ac8753bec23b0388b094d3ace6a1d32be1b" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7575, + "sha256": "398c569a73d340ea5ad60eae617166b1e1091af91d2d945450ca8ff6a9137ebe" + }, + "target": { + "bytes": 432652, + "sha256": "86b764ac22fe162549f3d5a2995e0e8160e836cf6abac763c120f9de283a8089" + } + }, + "level3/attribute7": { + "evidence": [ + { + "bytes": 1269312, + "kind": "events.jsonl", + "sha256": "dfd33829af7f97efc7fcd47ed5bf3ec656e93917a5564666a08acc4f20977b38" + }, + { + "bytes": 11695178, + "kind": "run.jsonl", + "sha256": "1f31b785d089e8b7ca48f8ce7c70038405f4ed2f9c7356534255ee91a682beb2" + } + ], + "rounds": [ + { + "blend_bytes": 646096642, + "blend_sha256": "c5b234ea7203c93af5042c5c878d5f6f7beae407cbab25871d636db8fa6be022", + "clip_similarity": 0.9302181005477905, + "nclip": 0.06978189945220947, + "photometric_loss": 0.031431227922439575, + "render": { + "bytes": 561924, + "sha256": "1ab8f237ca9e93dcefa0e8ddcc08787b56c9b8df04b92336c074eebb72e80ab7" + }, + "round": 1 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "c55d8ea788b92ddf19b01aeec67e00867290760b332ae07c9f2e91ba71189edc", + "clip_similarity": 0.9107853174209595, + "nclip": 0.08921468257904053, + "photometric_loss": 0.041224125772714615, + "render": { + "bytes": 573209, + "sha256": "0957b14f39708d867c8530fc7d6563e88eec7ac50edb856ad915ee919b335c72" + }, + "round": 2 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "729f7895832f179b9c0d9e2c7aa9b62f79bc86e2103f59cf69dcfc1be05964fe", + "clip_similarity": 0.9154404997825623, + "nclip": 0.08455950021743774, + "photometric_loss": 0.04107638821005821, + "render": { + "bytes": 573156, + "sha256": "f242e39a8aa5c30cb5bd40230ed9feb7b67d2beca41b6fbd9259381e02867e9d" + }, + "round": 3 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "717589dee89c45554a3207385aa30280e85ee135cc543fc55b48c57513203a1e", + "clip_similarity": 0.9029605388641357, + "nclip": 0.09703946113586426, + "photometric_loss": 0.04625360295176506, + "render": { + "bytes": 576957, + "sha256": "3c69929f5d050ab8ebec71a9f38ccb912ee71048e3fa6f9b11a29a144a78a9e4" + }, + "round": 4 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "07df53ab9336913bfbf3885699a14b3ef31556e8bcf357481c8da501c06a7920", + "clip_similarity": 0.93320631980896, + "nclip": 0.06679368019104004, + "photometric_loss": 0.03137661889195442, + "render": { + "bytes": 562350, + "sha256": "6d3fdb319961b2d09d518e82ad8e7b40339d922980015ab024a1b8c85f147a60" + }, + "round": 5 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "db8a3071796bcad90abc7146f43a36cf3cf8f43d225a4dc807a873f764de51e5", + "clip_similarity": 0.9286768436431885, + "nclip": 0.07132315635681152, + "photometric_loss": 0.031444914638996124, + "render": { + "bytes": 562255, + "sha256": "1e0eaa0c90d0a816acc9dbab9d3521ee332078c553f3a5e5caaa212e066d9caf" + }, + "round": 6 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "c2bcd628bd2c5e9b8e01c93c5dd247f78aa311810fdaced8ef973b6184108f8a", + "clip_similarity": 0.9289407134056091, + "nclip": 0.07105928659439087, + "photometric_loss": 0.03151537477970123, + "render": { + "bytes": 561958, + "sha256": "d82708fe11deef1285da6936f2ce160c076b39e8c4299d94417dd03e6b2f016e" + }, + "round": 7 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "b9fe704692f78541017edeef8e0ea7f8cbc8a8087a98ba93a87b13b16ebfc36c", + "clip_similarity": 0.9298222064971924, + "nclip": 0.07017779350280762, + "photometric_loss": 0.031483396887779236, + "render": { + "bytes": 561981, + "sha256": "1493e7f369bc2e3ee7542a741af34542f712324f2d8b6ac3496fad6d9ae574a0" + }, + "round": 8 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "6188d265a46217c1ec37e5585757e8e65d5ae68f99bbb036f10c7f75affac16a", + "clip_similarity": 0.9289346933364868, + "nclip": 0.07106530666351318, + "photometric_loss": 0.0314057394862175, + "render": { + "bytes": 561811, + "sha256": "a6289e687902b5c24afb3e7940bba8e8934b69fd0256e3f6eac1f14e3286f0e9" + }, + "round": 9 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "d0232cb99e6958c0492b7989bbaee634e5e5c16d19a4b1b4740e1af70a656205", + "clip_similarity": 0.9050087332725525, + "nclip": 0.09499126672744751, + "photometric_loss": 0.06344757229089737, + "render": { + "bytes": 562445, + "sha256": "93087024fa850a1d787f49e40b31525a5492de66254bfdb500cc64860590a017" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7574, + "sha256": "4af396ec7985842a179dee04858173ec9464fe80e3d55b672b83198f1a2cb56d" + }, + "target": { + "bytes": 496540, + "sha256": "0ada6e89b53c28b9036418def514b728dd0eeb664eb3f7a1ca9496a5e8ef93f1" + } + }, + "level3/attribute8": { + "evidence": [ + { + "bytes": 1271443, + "kind": "events.jsonl", + "sha256": "87fd893f7271ec4a898d328045bc20437c3b30a1a6a4e750347e98348abf7d4f" + }, + { + "bytes": 10611931, + "kind": "run.jsonl", + "sha256": "3b826dca03ab0600527cbfe35e138db4de25905b229fdf76e70f3abe9e000c68" + } + ], + "rounds": [ + { + "blend_bytes": 646096642, + "blend_sha256": "4fe0b6bba33e7fa419a69d12128a98ee1994479d899de320e9dea834b8afc3f2", + "clip_similarity": 0.9057605266571045, + "nclip": 0.09423947334289551, + "photometric_loss": 0.032523829489946365, + "render": { + "bytes": 589609, + "sha256": "eb3568f0195d2343b44612b6d1d2ea0f53864ca5c269a3074f63348f8cbd9cd5" + }, + "round": 1 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "f388c0a9e5d698394ecc073345a16242af2a7dc52abf41a8501c80ac65773eac", + "clip_similarity": 0.9555370807647705, + "nclip": 0.04446291923522949, + "photometric_loss": 0.042556773871183395, + "render": { + "bytes": 602319, + "sha256": "c55bf8a9e8728cfae6786857be7fe9f1aa1318f12366a3a095897e4dcaceb325" + }, + "round": 2 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "bba62674cd514c9c9030cdef5b93fe48e32bd548a8a391b8164024609e4cd45e", + "clip_similarity": 0.9614916443824768, + "nclip": 0.03850835561752319, + "photometric_loss": 0.030976975336670876, + "render": { + "bytes": 592945, + "sha256": "b934e7692abfe450fe9db1292e75e412e8166fd73374bf36cc603a286c3cadb0" + }, + "round": 3 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "2c8e2743e6080b9929ec51fb092b2e3cdbcc07a2c02ee352410f0f8d24d36a09", + "clip_similarity": 0.9753873944282532, + "nclip": 0.024612605571746826, + "photometric_loss": 0.025764750316739082, + "render": { + "bytes": 588589, + "sha256": "fdff2c4b91ef2532c3573eafff6063fb2427e769f08c1f822b4a8a1ff0ca20bd" + }, + "round": 4 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "567284551301b9fa61ea4b8db4d7f93d6a4a943efffb4e61ab40bc30878dffca", + "clip_similarity": 0.9746690988540649, + "nclip": 0.02533090114593506, + "photometric_loss": 0.025471189990639687, + "render": { + "bytes": 588321, + "sha256": "d93ad3f29d6e8e7a686070ab057a3fc2e7e3d6dd2b44df035b4fad8e474d569f" + }, + "round": 5 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "5bb2c52e3f078187e2b55240c63b5bbe2de840d2a3af15a37c7675a0ecc07b3e", + "clip_similarity": 0.974574089050293, + "nclip": 0.02542591094970703, + "photometric_loss": 0.029058828949928284, + "render": { + "bytes": 595137, + "sha256": "5c5ef3bb74740025f31a5883a541d2260407a37306c77e68d1d168f4b5b93bb3" + }, + "round": 6 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "b6d6fbd09f215277e8d97a93a0eba4f08440e59dc9e638d430d85ad79d383a61", + "clip_similarity": 0.9609359502792358, + "nclip": 0.03906404972076416, + "photometric_loss": 0.026920897886157036, + "render": { + "bytes": 606714, + "sha256": "08fbbd37baff628a60804a95f68ecffbea5096c6372a48fac3abbc06cdc6a2f0" + }, + "round": 7 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "9fff4cefd40e0d79663e822dedc7bab3f3e6a5d6a1c2913c53f43a57728c6191", + "clip_similarity": 0.9664453268051147, + "nclip": 0.033554673194885254, + "photometric_loss": 0.026461705565452576, + "render": { + "bytes": 606232, + "sha256": "5e2a970939dda2d7030160af7f79253e4af30b073dcd1636c87ce60cc06e0bcd" + }, + "round": 8 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "cdc02ad711e2fd50b7170621962cff7d96345c2610e983b8b65fb7d1f34c710f", + "clip_similarity": 0.9682265520095825, + "nclip": 0.03177344799041748, + "photometric_loss": 0.02800014615058899, + "render": { + "bytes": 606885, + "sha256": "a7b7ac1a158dea6f39fc1d7b0cd5884250ba2cb27309ec29997e37e0a2a41118" + }, + "round": 9 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "c6f44ec534001ceeda5b0d9e07381ba186251c499b9aea54960312ed95586d1e", + "clip_similarity": 0.9691956043243408, + "nclip": 0.03080439567565918, + "photometric_loss": 0.02725779078900814, + "render": { + "bytes": 607436, + "sha256": "1982957f7ad0b9975bcd87ceb046c25bcddb78026c398123fdeb23fea049238a" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7582, + "sha256": "9eccb85a612eac71a304af9e7c176d10970c312209874855a582e45c2ba8c9cb" + }, + "target": { + "bytes": 496540, + "sha256": "33e602e64c9b3f6825472277653bd168de39645a61cde8d8112efe0ae0cde1ad" + } + }, + "level3/attribute9": { + "evidence": [ + { + "bytes": 1270349, + "kind": "events.jsonl", + "sha256": "a93cc37fe14d3aca99bd278ab90e3e661b65bb796dbad8364c11dc3f47383897" + }, + { + "bytes": 11661650, + "kind": "run.jsonl", + "sha256": "2d807050858e48bb0509e8d3f09ede122c6731867165a98f9ffa535a2ec001ed" + } + ], + "rounds": [ + { + "blend_bytes": 646096642, + "blend_sha256": "18f474d748e3ef4359b88afac1f9d10b9e9a2aa3eed7a06e83172b9861a22068", + "clip_similarity": 0.9318258762359619, + "nclip": 0.06817412376403809, + "photometric_loss": 0.03224402293562889, + "render": { + "bytes": 573727, + "sha256": "3fd07c72310618b51861582efb145e8c71ef85ad84f445f2d37827085e37da6f" + }, + "round": 1 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "943a64ce40a167a414ff2562825a04c1952db250f0eb24a8e593b087115c2058", + "clip_similarity": 0.9383044242858887, + "nclip": 0.06169557571411133, + "photometric_loss": 0.043293412774801254, + "render": { + "bytes": 587693, + "sha256": "e0b55e00c3a3049a5026cd199048d583552d8094fb87545234667f7455ac91b7" + }, + "round": 2 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "7cfcd87a53034480700482853c48735978831d9ce4a73745576bc1c8eaca28f8", + "clip_similarity": 0.9413596391677856, + "nclip": 0.058640360832214355, + "photometric_loss": 0.04311363771557808, + "render": { + "bytes": 588713, + "sha256": "4875611c309e1901e728857aa1ac4a89558b73706da6e66385735ed58436b10e" + }, + "round": 3 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "698e536768b30834e4e16014254d4f6cc9e87de6e49db4d4c1442d3c38086611", + "clip_similarity": 0.9701520204544067, + "nclip": 0.02984797954559326, + "photometric_loss": 0.03251175582408905, + "render": { + "bytes": 590002, + "sha256": "24c27d1725ef16ba6d43de27d54297da4fa3a1e6304a020645ae3f4be7b56a2b" + }, + "round": 4 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "317b61159b933592bfe63c64a27caf74850f78986fef1209ebbafe1151b3563b", + "clip_similarity": 0.973321259021759, + "nclip": 0.026678740978240967, + "photometric_loss": 0.019820889458060265, + "render": { + "bytes": 586266, + "sha256": "f643c5a20584f10577cb75f1456f52f1f41c90b242515758400b7f0706cbb123" + }, + "round": 5 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "4f16bb04ee9e3ab6827a410ad472cf737e853e3550705a030b4dc0909ec85f52", + "clip_similarity": 0.9795242547988892, + "nclip": 0.02047574520111084, + "photometric_loss": 0.018832268193364143, + "render": { + "bytes": 586112, + "sha256": "bcc6c5a1a8ed5bf572a681c20ff2af9860a91fcf6003a393b7076b6128d5ea34" + }, + "round": 6 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "fb42311040247181df6d0c6254b4421551e7db0d5400d2eda43b347b3fbe116e", + "clip_similarity": 0.9769859910011292, + "nclip": 0.02301400899887085, + "photometric_loss": 0.0188379418104887, + "render": { + "bytes": 586516, + "sha256": "6906d54f5184bf20e6104a54ea2e62bdbaa46a0310dceea8fd1f9dd5b965ee7b" + }, + "round": 7 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "07e318a9c1903b7fbd4e01bc60d38f9bc9a4b82356984a2e2255ca1267dfb966", + "clip_similarity": 0.9763661623001099, + "nclip": 0.023633837699890137, + "photometric_loss": 0.018755586817860603, + "render": { + "bytes": 585929, + "sha256": "a09afd9eb5a15bca1319c675fc4c3b6ac38b8e2321c7b811c03fd6c7687be867" + }, + "round": 8 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "f2c0cba334f3ea4df53698b96ecbceea4e70e7ca9e5ea7d7b38ab8aa02c3a06b", + "clip_similarity": 0.9719067811965942, + "nclip": 0.02809321880340576, + "photometric_loss": 0.021445848047733307, + "render": { + "bytes": 593094, + "sha256": "ff4b064e9db0d87ddd0bafb5001b7f5ccd719ebfcae7f55a9a78be215c3e4bd6" + }, + "round": 9 + }, + { + "blend_bytes": 646096642, + "blend_sha256": "060a3f3c25941539236446f405d8c9b1e2321a50deee184cddd95683da4989cd", + "clip_similarity": 0.9817102551460266, + "nclip": 0.01828974485397339, + "photometric_loss": 0.010348494164645672, + "render": { + "bytes": 548629, + "sha256": "77cc1d6ad1bbcec9edfbd043a3713dad9eab53e179ec516dcf42f67ce354df7f" + }, + "round": 10 + } + ], + "scores": { + "bytes": 7580, + "sha256": "7e3cb13f2ae3c0de3032c58d4d4a9237681adcf41e49cf89c01add14481eb38c" + }, + "target": { + "bytes": 496540, + "sha256": "27653bfcb7c78fe51239c3905060ce112e65c8875966b2d81f3d0ae33fa322ee" + } + } + } +} diff --git a/benchmarks/blenderbench_direct/publication/reference-results.json b/benchmarks/blenderbench_direct/publication/reference-results.json new file mode 100644 index 0000000..1cb19c1 --- /dev/null +++ b/benchmarks/blenderbench_direct/publication/reference-results.json @@ -0,0 +1,347 @@ +{ + "categories": { + "level1": { + "mean_best_clip_similarity": 0.9962615238295661, + "mean_best_nclip_x100": 0.3738476170433892, + "mean_best_photometric_loss": 0.003033510640105103, + "tasks": 9 + }, + "level2": { + "mean_best_clip_similarity": 0.983968522813585, + "mean_best_nclip_x100": 1.603147718641493, + "mean_best_photometric_loss": 0.011897660300342573, + "tasks": 9 + }, + "level3": { + "mean_best_clip_similarity": 0.9771796133783128, + "mean_best_nclip_x100": 2.2820386621687145, + "mean_best_photometric_loss": 0.019716895185410976, + "tasks": 9 + } + }, + "dataset": { + "attribution": "DietCoke4671 and contributors", + "license": "CC BY 4.0", + "repository": "DietCoke4671/BlenderBench", + "revision": "203e4d325e9438ca55b29bdfc4f6a90842d74e68" + }, + "evaluator": { + "best_round_rule": "minimum N-CLIP; paired photometric loss from same round; earliest tie", + "clip_model": "openai/clip-vit-base-patch32", + "clip_model_revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", + "metric": "CLIP image-embedding cosine similarity; N-CLIP = 1 - similarity; not literal task accuracy", + "repository": "Fugtemypt123/VIGA-release", + "revision": "69cb8ef0651bf68124815682df4a0f8e8b57c141", + "source_sha256": "9ee08818aa858dccfff3efbb36cb9a98e8887f14e6168ad1a23731351cdeb3b5" + }, + "experiment": { + "codex_cli": "0.154.0", + "generation_model": "gpt-6-astra", + "generation_model_revision": null, + "implementation_commit": "4a9f5cc6e616080185252bb919fa3d1e730760b7", + "rounds_per_task": 10, + "scored_rounds": 270, + "tasks": 27, + "trajectory_policy": "one sequential ten-round trajectory per task; no CLIP or evaluator feedback to generation" + }, + "limitations": [ + "Public dataset; contamination risk is not measurable.", + "No hidden test set, hosted submission endpoint, or official leaderboard verification.", + "Model and inference budget differ from VIGA paper conditions.", + "One trajectory per task; no variance estimate.", + "No VLM judge by design." + ], + "results_source": { + "bytes": 6447, + "sha256": "073be2c44f1a763b66dea5138eda1b8959e8553fef6ce288f1d78ee92807d6e8" + }, + "rows": [ + { + "best_round": 7, + "clip_similarity": 0.9975831508636475, + "raw_nclip": 0.002416849136352539, + "raw_pl": 0.003328760154545307, + "rounds": 10, + "task": "level1/camera1" + }, + { + "best_round": 8, + "clip_similarity": 0.9955885410308838, + "raw_nclip": 0.004411458969116211, + "raw_pl": 0.0033672682475298643, + "rounds": 10, + "task": "level1/camera2" + }, + { + "best_round": 9, + "clip_similarity": 0.9977397918701172, + "raw_nclip": 0.0022602081298828125, + "raw_pl": 0.001466981484554708, + "rounds": 10, + "task": "level1/camera3" + }, + { + "best_round": 6, + "clip_similarity": 0.9912315011024475, + "raw_nclip": 0.00876849889755249, + "raw_pl": 0.007649693172425032, + "rounds": 10, + "task": "level1/camera4" + }, + { + "best_round": 7, + "clip_similarity": 0.9964922666549683, + "raw_nclip": 0.0035077333450317383, + "raw_pl": 0.0044174473732709885, + "rounds": 10, + "task": "level1/camera5" + }, + { + "best_round": 10, + "clip_similarity": 0.9997468590736389, + "raw_nclip": 0.000253140926361084, + "raw_pl": 3.515696880640462e-05, + "rounds": 10, + "task": "level1/camera6" + }, + { + "best_round": 9, + "clip_similarity": 0.989441454410553, + "raw_nclip": 0.010558545589447021, + "raw_pl": 0.006557795684784651, + "rounds": 10, + "task": "level1/camera7" + }, + { + "best_round": 10, + "clip_similarity": 0.9990020990371704, + "raw_nclip": 0.0009979009628295898, + "raw_pl": 3.855396425933577e-05, + "rounds": 10, + "task": "level1/camera8" + }, + { + "best_round": 9, + "clip_similarity": 0.9995280504226685, + "raw_nclip": 0.00047194957733154297, + "raw_pl": 0.0004399387107696384, + "rounds": 10, + "task": "level1/camera9" + }, + { + "best_round": 10, + "clip_similarity": 0.9777544736862183, + "raw_nclip": 0.02224552631378174, + "raw_pl": 0.015119296498596668, + "rounds": 10, + "task": "level2/attribute1" + }, + { + "best_round": 5, + "clip_similarity": 0.9820566177368164, + "raw_nclip": 0.017943382263183594, + "raw_pl": 0.016636313870549202, + "rounds": 10, + "task": "level2/attribute2" + }, + { + "best_round": 4, + "clip_similarity": 0.9793153405189514, + "raw_nclip": 0.020684659481048584, + "raw_pl": 0.01562858745455742, + "rounds": 10, + "task": "level2/attribute3" + }, + { + "best_round": 1, + "clip_similarity": 0.9745771288871765, + "raw_nclip": 0.025422871112823486, + "raw_pl": 0.02019927091896534, + "rounds": 10, + "task": "level2/attribute4" + }, + { + "best_round": 9, + "clip_similarity": 0.9830480813980103, + "raw_nclip": 0.016951918601989746, + "raw_pl": 0.013992843218147755, + "rounds": 10, + "task": "level2/attribute5" + }, + { + "best_round": 9, + "clip_similarity": 0.9842134714126587, + "raw_nclip": 0.01578652858734131, + "raw_pl": 0.01759195886552334, + "rounds": 10, + "task": "level2/attribute6" + }, + { + "best_round": 9, + "clip_similarity": 0.9894779324531555, + "raw_nclip": 0.010522067546844482, + "raw_pl": 0.0033791407477110624, + "rounds": 10, + "task": "level2/attribute7" + }, + { + "best_round": 9, + "clip_similarity": 0.9888083934783936, + "raw_nclip": 0.011191606521606445, + "raw_pl": 0.0037378640845417976, + "rounds": 10, + "task": "level2/attribute8" + }, + { + "best_round": 10, + "clip_similarity": 0.996465265750885, + "raw_nclip": 0.0035347342491149902, + "raw_pl": 0.0007936670444905758, + "rounds": 10, + "task": "level2/attribute9" + }, + { + "best_round": 9, + "clip_similarity": 0.9810705780982971, + "raw_nclip": 0.01892942190170288, + "raw_pl": 0.02021031640470028, + "rounds": 10, + "task": "level3/attribute1" + }, + { + "best_round": 10, + "clip_similarity": 0.9837775230407715, + "raw_nclip": 0.016222476959228516, + "raw_pl": 0.012915946543216705, + "rounds": 10, + "task": "level3/attribute2" + }, + { + "best_round": 10, + "clip_similarity": 0.9846137762069702, + "raw_nclip": 0.015386223793029785, + "raw_pl": 0.014947629533708096, + "rounds": 10, + "task": "level3/attribute3" + }, + { + "best_round": 10, + "clip_similarity": 0.9926517009735107, + "raw_nclip": 0.007348299026489258, + "raw_pl": 0.009027284570038319, + "rounds": 10, + "task": "level3/attribute4" + }, + { + "best_round": 10, + "clip_similarity": 0.9755886793136597, + "raw_nclip": 0.024411320686340332, + "raw_pl": 0.027274271473288536, + "rounds": 10, + "task": "level3/attribute5" + }, + { + "best_round": 7, + "clip_similarity": 0.9866102933883667, + "raw_nclip": 0.0133897066116333, + "raw_pl": 0.025586744770407677, + "rounds": 10, + "task": "level3/attribute6" + }, + { + "best_round": 5, + "clip_similarity": 0.93320631980896, + "raw_nclip": 0.06679368019104004, + "raw_pl": 0.03137661889195442, + "rounds": 10, + "task": "level3/attribute7" + }, + { + "best_round": 4, + "clip_similarity": 0.9753873944282532, + "raw_nclip": 0.024612605571746826, + "raw_pl": 0.025764750316739082, + "rounds": 10, + "task": "level3/attribute8" + }, + { + "best_round": 10, + "clip_similarity": 0.9817102551460266, + "raw_nclip": 0.01828974485397339, + "raw_pl": 0.010348494164645672, + "rounds": 10, + "task": "level3/attribute9" + } + ], + "schema_version": 1, + "scoring_runtime": { + "gpu": "NVIDIA GeForce RTX 5090", + "packages": { + "certifi": "2026.7.22", + "charset-normalizer": "3.5.1", + "filelock": "3.32.3", + "fsspec": "2026.7.0", + "hf-xet": "1.6.0", + "huggingface-hub": "0.36.2", + "idna": "3.19", + "jinja2": "3.1.6", + "markupsafe": "3.0.3", + "mpmath": "1.3.0", + "networkx": "3.6.1", + "numpy": "2.4.6", + "nvidia-cublas-cu12": "12.8.4.1", + "nvidia-cuda-cupti-cu12": "12.8.90", + "nvidia-cuda-nvrtc-cu12": "12.8.93", + "nvidia-cuda-runtime-cu12": "12.8.90", + "nvidia-cudnn-cu12": "9.10.2.21", + "nvidia-cufft-cu12": "11.3.3.83", + "nvidia-cufile-cu12": "1.13.1.3", + "nvidia-curand-cu12": "10.3.9.90", + "nvidia-cusolver-cu12": "11.7.3.90", + "nvidia-cusparse-cu12": "12.5.8.93", + "nvidia-cusparselt-cu12": "0.7.1", + "nvidia-nccl-cu12": "2.27.3", + "nvidia-nvjitlink-cu12": "12.8.93", + "nvidia-nvtx-cu12": "12.8.90", + "packaging": "26.3", + "pillow": "12.3.0", + "pyyaml": "6.0.3", + "regex": "2026.9.10", + "requests": "2.34.2", + "safetensors": "0.8.0", + "setuptools": "78.1.0", + "sympy": "1.14.0", + "tokenizers": "0.21.4", + "torch": "2.8.0+cu128", + "tqdm": "4.70.0", + "transformers": "4.55.4", + "triton": "3.4.0", + "typing-extensions": "4.16.0", + "urllib3": "2.7.0" + }, + "python": { + "architecture": "x86_64", + "bits": 64, + "cache_tag": "cpython-313", + "compiler": "Clang 22.1.3", + "executable_sha256": "93a93f4c8af5ae6f127fb0310f923d5fab89eb7257a0b801eec2e44b3f199840", + "implementation": "CPython", + "note": "Executable digest is historical provenance, not remote or hostile-host attestation.", + "soabi": "cpython-313-x86_64-linux-gnu", + "version": "3.13.15" + }, + "torch_cuda": "12.8" + }, + "status": "unofficial_public-dataset_evaluation; not an official BlenderBench leaderboard submission", + "video": { + "bytes": 10161884, + "codec": "h264", + "duration_seconds": 134.9, + "height": 464, + "publication_status": "published", + "sample_aspect_ratio": "1:1", + "sha256": "04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17", + "url": "https://huggingface.co/datasets/michaelgold/blenderbench-direct-results/resolve/main/video/blenderbench-complete-compilation.mp4", + "width": 848 + } +} diff --git a/benchmarks/blenderbench_direct/publication/results-summary.json b/benchmarks/blenderbench_direct/publication/results-summary.json new file mode 100644 index 0000000..326f744 --- /dev/null +++ b/benchmarks/blenderbench_direct/publication/results-summary.json @@ -0,0 +1,242 @@ +{ + "task_count": 27, + "round_count": 270, + "categories": { + "level1": { + "tasks": 9, + "mean_best_clip_similarity": 0.9962615238295661, + "mean_best_nclip_x100": 0.3738476170433892, + "mean_best_photometric_loss": 0.003033510640105103 + }, + "level2": { + "tasks": 9, + "mean_best_clip_similarity": 0.983968522813585, + "mean_best_nclip_x100": 1.603147718641493, + "mean_best_photometric_loss": 0.011897660300342573 + }, + "level3": { + "tasks": 9, + "mean_best_clip_similarity": 0.9771796133783128, + "mean_best_nclip_x100": 2.2820386621687145, + "mean_best_photometric_loss": 0.019716895185410976 + } + }, + "rows": [ + { + "task": "level1/camera1", + "rounds": 10, + "best_round": 7, + "clip_similarity": 0.9975831508636475, + "raw_nclip": 0.002416849136352539, + "raw_pl": 0.003328760154545307 + }, + { + "task": "level1/camera2", + "rounds": 10, + "best_round": 8, + "clip_similarity": 0.9955885410308838, + "raw_nclip": 0.004411458969116211, + "raw_pl": 0.0033672682475298643 + }, + { + "task": "level1/camera3", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9977397918701172, + "raw_nclip": 0.0022602081298828125, + "raw_pl": 0.001466981484554708 + }, + { + "task": "level1/camera4", + "rounds": 10, + "best_round": 6, + "clip_similarity": 0.9912315011024475, + "raw_nclip": 0.00876849889755249, + "raw_pl": 0.007649693172425032 + }, + { + "task": "level1/camera5", + "rounds": 10, + "best_round": 7, + "clip_similarity": 0.9964922666549683, + "raw_nclip": 0.0035077333450317383, + "raw_pl": 0.0044174473732709885 + }, + { + "task": "level1/camera6", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9997468590736389, + "raw_nclip": 0.000253140926361084, + "raw_pl": 3.515696880640462e-05 + }, + { + "task": "level1/camera7", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.989441454410553, + "raw_nclip": 0.010558545589447021, + "raw_pl": 0.006557795684784651 + }, + { + "task": "level1/camera8", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9990020990371704, + "raw_nclip": 0.0009979009628295898, + "raw_pl": 3.855396425933577e-05 + }, + { + "task": "level1/camera9", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9995280504226685, + "raw_nclip": 0.00047194957733154297, + "raw_pl": 0.0004399387107696384 + }, + { + "task": "level2/attribute1", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9777544736862183, + "raw_nclip": 0.02224552631378174, + "raw_pl": 0.015119296498596668 + }, + { + "task": "level2/attribute2", + "rounds": 10, + "best_round": 5, + "clip_similarity": 0.9820566177368164, + "raw_nclip": 0.017943382263183594, + "raw_pl": 0.016636313870549202 + }, + { + "task": "level2/attribute3", + "rounds": 10, + "best_round": 4, + "clip_similarity": 0.9793153405189514, + "raw_nclip": 0.020684659481048584, + "raw_pl": 0.01562858745455742 + }, + { + "task": "level2/attribute4", + "rounds": 10, + "best_round": 1, + "clip_similarity": 0.9745771288871765, + "raw_nclip": 0.025422871112823486, + "raw_pl": 0.02019927091896534 + }, + { + "task": "level2/attribute5", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9830480813980103, + "raw_nclip": 0.016951918601989746, + "raw_pl": 0.013992843218147755 + }, + { + "task": "level2/attribute6", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9842134714126587, + "raw_nclip": 0.01578652858734131, + "raw_pl": 0.01759195886552334 + }, + { + "task": "level2/attribute7", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9894779324531555, + "raw_nclip": 0.010522067546844482, + "raw_pl": 0.0033791407477110624 + }, + { + "task": "level2/attribute8", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9888083934783936, + "raw_nclip": 0.011191606521606445, + "raw_pl": 0.0037378640845417976 + }, + { + "task": "level2/attribute9", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.996465265750885, + "raw_nclip": 0.0035347342491149902, + "raw_pl": 0.0007936670444905758 + }, + { + "task": "level3/attribute1", + "rounds": 10, + "best_round": 9, + "clip_similarity": 0.9810705780982971, + "raw_nclip": 0.01892942190170288, + "raw_pl": 0.02021031640470028 + }, + { + "task": "level3/attribute2", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9837775230407715, + "raw_nclip": 0.016222476959228516, + "raw_pl": 0.012915946543216705 + }, + { + "task": "level3/attribute3", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9846137762069702, + "raw_nclip": 0.015386223793029785, + "raw_pl": 0.014947629533708096 + }, + { + "task": "level3/attribute4", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9926517009735107, + "raw_nclip": 0.007348299026489258, + "raw_pl": 0.009027284570038319 + }, + { + "task": "level3/attribute5", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9755886793136597, + "raw_nclip": 0.024411320686340332, + "raw_pl": 0.027274271473288536 + }, + { + "task": "level3/attribute6", + "rounds": 10, + "best_round": 7, + "clip_similarity": 0.9866102933883667, + "raw_nclip": 0.0133897066116333, + "raw_pl": 0.025586744770407677 + }, + { + "task": "level3/attribute7", + "rounds": 10, + "best_round": 5, + "clip_similarity": 0.93320631980896, + "raw_nclip": 0.06679368019104004, + "raw_pl": 0.03137661889195442 + }, + { + "task": "level3/attribute8", + "rounds": 10, + "best_round": 4, + "clip_similarity": 0.9753873944282532, + "raw_nclip": 0.024612605571746826, + "raw_pl": 0.025764750316739082 + }, + { + "task": "level3/attribute9", + "rounds": 10, + "best_round": 10, + "clip_similarity": 0.9817102551460266, + "raw_nclip": 0.01828974485397339, + "raw_pl": 0.010348494164645672 + } + ] +} diff --git a/benchmarks/blenderbench_direct/ref_based_eval.py b/benchmarks/blenderbench_direct/ref_based_eval.py new file mode 100644 index 0000000..e5f68cc --- /dev/null +++ b/benchmarks/blenderbench_direct/ref_based_eval.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python3 +""" +Evaluation script for AgenticVerifier blendergym results. +Adapted from the original evaluate.py to work with our codebase structure. +""" + +import os +import sys +import argparse +import json +from PIL import Image +from tqdm import tqdm +import numpy as np +import torch +from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize +from transformers import CLIPProcessor, CLIPModel +from concurrent.futures import ThreadPoolExecutor, as_completed + +# Task instance counts for different task types +TASK_INSTANCE_COUNT_DICT = { + 'level1': 9, + 'level2': 9, + 'level3': 9, +} + +# Global CLIP model/processor to share across threads +GLOBAL_CLIP_MODEL = None +GLOBAL_CLIP_PROCESSOR = None + + +def ensure_clip_loaded(): + """ + Lazily load the global CLIP model and processor once per process. + """ + global GLOBAL_CLIP_MODEL, GLOBAL_CLIP_PROCESSOR + if GLOBAL_CLIP_MODEL is None or GLOBAL_CLIP_PROCESSOR is None: + GLOBAL_CLIP_MODEL = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") + GLOBAL_CLIP_PROCESSOR = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") + + +def clip_similarity(image1, image2): + """ + Compute the CLIP similarity between two PIL images. + + Args: + image1 (PIL.Image): The first input image. + image2 (PIL.Image): The second input image. + + Returns: + float: The CLIP similarity between the two images. + """ + if image1.size != image2.size: + image2 = image2.resize(image1.size) + + # Ensure global model is initialized + ensure_clip_loaded() + + # Preprocess the images + images = [image1, image2] + inputs = GLOBAL_CLIP_PROCESSOR(images=images, return_tensors="pt") + + # Compute the features for the images + with torch.no_grad(): + features = GLOBAL_CLIP_MODEL.get_image_features(**inputs) + + # Compute the cosine similarity between the image features + sim = torch.nn.functional.cosine_similarity(features[0], features[1], dim=-1) + + return sim.item() + +def photometric_loss(image1: Image.Image, image2: Image.Image) -> float: + """ + Compute the photometric loss between two PIL images. + + Args: + image1 (PIL.Image): The first input image. + image2 (PIL.Image): The second input image. + + Returns: + float: The photometric loss between the two images. + """ + if image1.size != image2.size: + image2 = image2.resize(image1.size) + + # Convert images to numpy arrays + img1_array = np.array(image1)[:, :, :3] + img2_array = np.array(image2)[:, :, :3] + + # Normalize images to [0, 1] + img1_norm = img1_array.astype(np.float32) / 255.0 + img2_norm = img2_array.astype(np.float32) / 255.0 + + # Compute the squared difference between the normalized images + diff = np.square(img1_norm - img2_norm) + + # Compute the mean squared error + mse = np.mean(diff) + return mse + + +def process_task_instance(output_base_dir: str, task_dir: str): + """ + Process a single task instance directory and compute metrics across rounds. + + Returns: + tuple: (task_dir, task_instance_scores, best_n_clip, best_pl) + where best_* can be None if no valid rounds. + """ + task_instance_dir = os.path.join(output_base_dir, task_dir) + renders_dir = os.path.join(task_instance_dir, "renders") + + if not os.path.exists(renders_dir): + return task_dir, {}, None, None + + gt_renders_dir = f"data/blenderbench/{task_dir}/renders/goal" + if not os.path.exists(gt_renders_dir): + return task_dir, {}, None, None + + task_instance_scores = {} + + # Get all round directories (1, 2, 3, 4, etc.) + round_dirs = [d for d in os.listdir(renders_dir) + if os.path.isdir(os.path.join(renders_dir, d))] + try: + round_dirs.sort(key=lambda x: int(x)) + except Exception: + # Fallback to lexical sort if non-numeric dirs exist + round_dirs.sort() + + if not round_dirs: + return task_dir, {}, None, None + + for round_dir in round_dirs: + round_path = os.path.join(renders_dir, round_dir) + task_instance_scores[round_dir] = {} + + n_clip_views = [] + pl_views = [] + + # render1 + render1_path = os.path.join(round_path, "render1.png") + gt_render1_path = os.path.join(gt_renders_dir, "render1.png") + if os.path.exists(render1_path) and os.path.exists(gt_render1_path): + try: + proposal_render = Image.open(render1_path) + gt_render = Image.open(gt_render1_path) + n_clip = float(1 - clip_similarity(proposal_render, gt_render)) + pl = float(photometric_loss(proposal_render, gt_render)) + n_clip_views.append(n_clip) + pl_views.append(pl) + task_instance_scores[round_dir]['render1'] = {'n_clip': n_clip, 'pl': pl} + except Exception: + pass + + # render2 + render2_path = os.path.join(round_path, "render2.png") + gt_render2_path = os.path.join(gt_renders_dir, "render2.png") + if os.path.exists(render2_path) and os.path.exists(gt_render2_path): + try: + proposal_render2 = Image.open(render2_path) + gt_render2 = Image.open(gt_render2_path) + n_clip2 = float(1 - clip_similarity(proposal_render2, gt_render2)) + pl2 = float(photometric_loss(proposal_render2, gt_render2)) + n_clip_views.append(n_clip2) + pl_views.append(pl2) + task_instance_scores[round_dir]['render2'] = {'n_clip': n_clip2, 'pl': pl2} + except Exception: + pass + + if n_clip_views: + avg_n_clip = sum(n_clip_views) / len(n_clip_views) + avg_pl = sum(pl_views) / len(pl_views) + task_instance_scores[round_dir]['avg_n_clip'] = avg_n_clip + task_instance_scores[round_dir]['avg_pl'] = avg_pl + + # Determine best rounds + valid_rounds = {k: v for k, v in task_instance_scores.items() if 'avg_n_clip' in v and 'avg_pl' in v} + best_n_clip = None + best_pl = None + if valid_rounds: + best_n_clip_round = min(valid_rounds.keys(), key=lambda r: valid_rounds[r]['avg_n_clip']) + best_pl_round = min(valid_rounds.keys(), key=lambda r: valid_rounds[r]['avg_pl']) + best_n_clip = valid_rounds[best_n_clip_round]['avg_n_clip'] + best_pl = valid_rounds[best_n_clip_round]['avg_pl'] if best_n_clip_round in valid_rounds else None + + # Save individual instance scores + instance_scores_path = os.path.join(task_instance_dir, 'scores.json') + try: + with open(instance_scores_path, 'w') as f: + json.dump(task_instance_scores, f, indent=4) + except Exception: + pass + + return task_dir, task_instance_scores, best_n_clip, best_pl + +def extract_task_type_and_number(task_dir_name): + """ + Extract task type and number from directory name like 'placement1', 'material5', etc. + + Args: + task_dir_name (str): Directory name like 'placement1', 'material5' + + Returns: + tuple: (task_type, task_number) or (None, None) if invalid + """ + level_dir_name = task_dir_name.split("/")[0] + task_name = task_dir_name.split("/")[1] + for task_type in TASK_INSTANCE_COUNT_DICT.keys(): + if level_dir_name == task_type: + try: + task_number = int(task_name[-1]) + return task_type, task_number + except ValueError: + continue + return None, None + +def main() -> None: + """Run reference-based evaluation (CLIP + photometric loss) for BlenderBench results.""" + parser = argparse.ArgumentParser(description='Evaluate AgenticVerifier blendergym-hard results') + parser.add_argument('test_id', type=str, help='Test ID (e.g., 20250815_150016)') + parser.add_argument('--output_dir', type=str, default=None, + help='Output directory for evaluation results (default: output/blenderbench/{test_id}/)_evaluation)') + parser.add_argument('--missing_round_penalty_max', type=float, default=2.0, + help='Max penalty factor for earliest rounds.') + parser.add_argument('--missing_round_penalty_min', type=float, default=1.0, + help='Min penalty factor for latest rounds.') + + args = parser.parse_args() + test_id = args.test_id + penalty_max = float(args.missing_round_penalty_max) + penalty_min = float(args.missing_round_penalty_min) + MAX_ROUNDS = 10 + + # Set up paths + output_base_dir = f"output/blenderbench/{test_id}" + if not os.path.exists(output_base_dir): + raise ValueError(f"Output directory {output_base_dir} does not exist.") + + if args.output_dir: + eval_output_dir = args.output_dir + else: + eval_output_dir = os.path.join(output_base_dir, "_evaluation") + + os.makedirs(eval_output_dir, exist_ok=True) + + # Get all task directories + task_dirs = [] + for level_dir in os.listdir(output_base_dir): + if os.path.isdir(os.path.join(output_base_dir, level_dir)) and level_dir != "_evaluation": + for task_dir in os.listdir(os.path.join(output_base_dir, level_dir)): + if os.path.isdir(os.path.join(output_base_dir, level_dir, task_dir)): + task_dirs.append(os.path.join(level_dir, task_dir)) + + print(f"Found {len(task_dirs)} task directories in {output_base_dir}") + + # Group tasks by type + tasks_by_type = {} + for task_dir in task_dirs: + task_type, task_number = extract_task_type_and_number(task_dir) + if task_type and task_number: + if task_type not in tasks_by_type: + tasks_by_type[task_type] = [] + tasks_by_type[task_type].append((task_dir, task_number)) + + print(f"Grouped tasks by type: {list(tasks_by_type.keys())}") + + scores_across_tasks = {} + intermediates = {} + + # Ensure CLIP is loaded once (shared by threads) + ensure_clip_loaded() + + for task_type, task_instances in tasks_by_type.items(): + print(f"\nProcessing task type: {task_type}") + + # Sort by task number + task_instances.sort(key=lambda x: x[1]) + + scores_across_instances = { + 'best_n_clip': [], + 'best_pl': [], + 'instance_details': {} + } + + # Run per-instance processing in parallel threads + max_workers = min(8, (os.cpu_count() or 4)) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [ + executor.submit(process_task_instance, output_base_dir, task_dir) + for task_dir, _ in task_instances + ] + + for future in tqdm(as_completed(futures), total=len(futures), desc=f"Processing {task_type}"): + try: + task_dir, task_instance_scores, best_n_clip, best_pl = future.result() + scores_across_instances['instance_details'][task_dir] = task_instance_scores + if best_n_clip is not None and best_pl is not None: + scores_across_instances['best_n_clip'].append(best_n_clip) + scores_across_instances['best_pl'].append(best_pl) + print(f" {task_dir}: Best n_clip={best_n_clip:.4f}, Best pl={best_pl:.4f}") + else: + print(f" {task_dir}: No valid scores") + except Exception as e: + print(f" Error processing {task_type} instance: {e}") + + # Aggregate per-round averages across all instances (rounds 1..9) + per_round_values = {str(i): {'n_clip': [], 'pl': [], 'penalized_count': 0} for i in range(1, 11)} + for instance_scores in scores_across_instances['instance_details'].values(): + # Collect available round indices for this instance + available_rounds = sorted( + [int(r) for r, v in instance_scores.items() if isinstance(v, dict) and 'avg_n_clip' in v and 'avg_pl' in v] + ) + if not available_rounds: + continue + max_available_round = max(available_rounds) + + for round_idx in range(1, 11): + key = str(round_idx) + # Case 1: round exists normally + if key in instance_scores and 'avg_n_clip' in instance_scores[key] and 'avg_pl' in instance_scores[key]: + per_round_values[key]['n_clip'].append(instance_scores[key]['avg_n_clip']) + per_round_values[key]['pl'].append(instance_scores[key]['avg_pl']) + continue + + # Case 2: earlier round missing but later rounds exist -> penalize + if round_idx < max_available_round: + # Find the next available later round to base the penalty on + later_rounds = [r for r in available_rounds if r > round_idx] + if not later_rounds: + continue + next_round = min(later_rounds) + next_key = str(next_round) + base_n = instance_scores[next_key]['avg_n_clip'] + base_pl = instance_scores[next_key]['avg_pl'] + # Decaying penalty: higher for earlier rounds, lower for later rounds + if MAX_ROUNDS > 1: + t = (round_idx - 1) / (MAX_ROUNDS - 1) + else: + t = 0.0 + penalty_factor_round = penalty_max - t * (penalty_max - penalty_min) + per_round_values[key]['n_clip'].append(base_n * penalty_factor_round) + per_round_values[key]['pl'].append(base_pl * penalty_factor_round) + per_round_values[key]['penalized_count'] += 1 + continue + # Case 3: missing because process ended (no later rounds) -> ignore + + per_round_summary = {} + for key, vals in per_round_values.items(): + if vals['n_clip'] and vals['pl']: + per_round_summary[key] = { + 'avg_n_clip': sum(vals['n_clip']) / len(vals['n_clip']), + 'avg_pl': sum(vals['pl']) / len(vals['pl']), + 'num_instances': len(vals['n_clip']), + 'num_penalized': int(vals['penalized_count']) + } + + # Store per-round aggregation in intermediates structure too + scores_across_instances['per_round'] = per_round_summary + + # Aggregate results for this task type + if scores_across_instances['best_n_clip']: + scores_across_tasks[task_type] = { + 'best_n_clip': sum(scores_across_instances['best_n_clip']) / len(scores_across_instances['best_n_clip']), + 'best_pl': sum(scores_across_instances['best_pl']) / len(scores_across_instances['best_pl']), + 'num_instances': len(scores_across_instances['best_n_clip']), + 'per_round': per_round_summary + } + + print(f" Task {task_type} overall scores:") + print(f" Average best n_clip: {scores_across_tasks[task_type]['best_n_clip']:.4f}") + print(f" Average best pl: {scores_across_tasks[task_type]['best_pl']:.4f}") + print(f" Number of instances: {scores_across_tasks[task_type]['num_instances']}") + else: + print(f" No valid scores for task type {task_type}") + scores_across_tasks[task_type] = {} + + intermediates[task_type] = scores_across_instances + + # Save overall results + overall_scores_path = os.path.join(eval_output_dir, 'overall_scores.json') + with open(overall_scores_path, 'w') as f: + json.dump(scores_across_tasks, f, indent=4) + + intermediate_scores_path = os.path.join(eval_output_dir, 'intermediate_scores.json') + with open(intermediate_scores_path, 'w') as f: + json.dump(intermediates, f, indent=4) + + # Print summary + print(f"\n=== Evaluation Summary ===") + print(f"Test ID: {test_id}") + print(f"Output directory: {eval_output_dir}") + print(f"Results saved to: {overall_scores_path}") + + for task_type, scores in scores_across_tasks.items(): + if scores: + print(f"\n{task_type.upper()}:") + print(f" Average best n_clip: {scores['best_n_clip']:.4f}") + print(f" Average best pl: {scores['best_pl']:.4f}") + print(f" Instances evaluated: {scores['num_instances']}") + else: + print(f"\n{task_type.upper()}: No valid scores") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmarks/blenderbench_direct/requirements-fixture.in b/benchmarks/blenderbench_direct/requirements-fixture.in new file mode 100644 index 0000000..bc3b3db --- /dev/null +++ b/benchmarks/blenderbench_direct/requirements-fixture.in @@ -0,0 +1,5 @@ +# Minimal CPU-only dependencies for publication fixture tests. +# uv pip compile --python-version 3.11 --python-platform x86_64-manylinux_2_28 \ +# --index-url https://pypi.org/simple --generate-hashes \ +# -o requirements-fixture.lock requirements-fixture.in +pillow==12.3.0 diff --git a/benchmarks/blenderbench_direct/requirements-fixture.lock b/benchmarks/blenderbench_direct/requirements-fixture.lock new file mode 100644 index 0000000..38781b1 --- /dev/null +++ b/benchmarks/blenderbench_direct/requirements-fixture.lock @@ -0,0 +1,91 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --python-version 3.11 --python-platform x86_64-manylinux_2_28 --generate-hashes -o benchmarks/blenderbench_direct/requirements-fixture.lock benchmarks/blenderbench_direct/requirements-fixture.in +pillow==12.3.0 \ + --hash=sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756 \ + --hash=sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a \ + --hash=sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59 \ + --hash=sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45 \ + --hash=sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3 \ + --hash=sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df \ + --hash=sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139 \ + --hash=sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b \ + --hash=sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39 \ + --hash=sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e \ + --hash=sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8 \ + --hash=sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1 \ + --hash=sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8 \ + --hash=sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89 \ + --hash=sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5 \ + --hash=sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130 \ + --hash=sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd \ + --hash=sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d \ + --hash=sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b \ + --hash=sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed \ + --hash=sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace \ + --hash=sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb \ + --hash=sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931 \ + --hash=sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510 \ + --hash=sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6 \ + --hash=sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1 \ + --hash=sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce \ + --hash=sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385 \ + --hash=sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e \ + --hash=sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c \ + --hash=sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7 \ + --hash=sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace \ + --hash=sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c \ + --hash=sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f \ + --hash=sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64 \ + --hash=sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f \ + --hash=sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a \ + --hash=sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827 \ + --hash=sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17 \ + --hash=sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4 \ + --hash=sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a \ + --hash=sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701 \ + --hash=sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e \ + --hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 \ + --hash=sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66 \ + --hash=sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468 \ + --hash=sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217 \ + --hash=sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658 \ + --hash=sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418 \ + --hash=sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a \ + --hash=sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c \ + --hash=sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330 \ + --hash=sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402 \ + --hash=sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09 \ + --hash=sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930 \ + --hash=sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f \ + --hash=sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec \ + --hash=sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a \ + --hash=sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94 \ + --hash=sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468 \ + --hash=sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b \ + --hash=sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965 \ + --hash=sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8 \ + --hash=sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd \ + --hash=sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7 \ + --hash=sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c \ + --hash=sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777 \ + --hash=sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35 \ + --hash=sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9 \ + --hash=sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f \ + --hash=sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f \ + --hash=sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0 \ + --hash=sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c \ + --hash=sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71 \ + --hash=sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3 \ + --hash=sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838 \ + --hash=sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf \ + --hash=sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321 \ + --hash=sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26 \ + --hash=sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec \ + --hash=sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9 \ + --hash=sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65 \ + --hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \ + --hash=sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e \ + --hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \ + --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \ + --hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7 + # via -r benchmarks/blenderbench_direct/requirements-fixture.in diff --git a/benchmarks/blenderbench_direct/requirements-scoring.in b/benchmarks/blenderbench_direct/requirements-scoring.in new file mode 100644 index 0000000..3e38b5b --- /dev/null +++ b/benchmarks/blenderbench_direct/requirements-scoring.in @@ -0,0 +1,47 @@ +# Historical BlenderBench scoring environment recovered from the publication host. +# Target: exact historical CPython 3.13.15, x86_64 manylinux_2_28, CUDA 12.8. +# Every distribution is pinned to the observed version; regenerate with uv as below. +# uv pip compile --python-version 3.13 --python-platform x86_64-manylinux_2_28 \ +# --index-url https://pypi.org/simple --generate-hashes \ +# -o requirements-scoring.lock requirements-scoring.in +certifi==2026.7.22 +charset-normalizer==3.5.1 +filelock==3.32.3 +fsspec==2026.7.0 +hf-xet==1.6.0 +huggingface-hub==0.36.2 +idna==3.19 +jinja2==3.1.6 +markupsafe==3.0.3 +mpmath==1.3.0 +networkx==3.6.1 +numpy==2.4.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-nccl-cu12==2.27.3 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvtx-cu12==12.8.90 +packaging==26.3 +pillow==12.3.0 +pyyaml==6.0.3 +regex==2026.9.10 +requests==2.34.2 +safetensors==0.8.0 +setuptools==78.1.0 +sympy==1.14.0 +tokenizers==0.21.4 +torch @ https://download.pytorch.org/whl/cu128/torch-2.8.0%2Bcu128-cp313-cp313-manylinux_2_28_x86_64.whl +tqdm==4.70.0 +transformers==4.55.4 +triton==3.4.0 +typing-extensions==4.16.0 +urllib3==2.7.0 diff --git a/benchmarks/blenderbench_direct/requirements-scoring.lock b/benchmarks/blenderbench_direct/requirements-scoring.lock new file mode 100644 index 0000000..fc4f5bd --- /dev/null +++ b/benchmarks/blenderbench_direct/requirements-scoring.lock @@ -0,0 +1,933 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --python-version 3.13 --python-platform x86_64-manylinux_2_28 --generate-hashes -o benchmarks/blenderbench_direct/requirements-scoring.lock benchmarks/blenderbench_direct/requirements-scoring.in +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # requests +charset-normalizer==3.5.1 \ + --hash=sha256:00668ebb0609751758682eb0b5857e7c35b9f00e84dfdef062e103244ec94d45 \ + --hash=sha256:012a22b88a77ca2e59b98ac5889b0deb604147666032f45e6d6e217634d2550d \ + --hash=sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5 \ + --hash=sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b \ + --hash=sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f \ + --hash=sha256:07ffd07412fc5d5e84cd8952acf9ff7e4ed7a708e69d1bada19d8ba91711353f \ + --hash=sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5 \ + --hash=sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22 \ + --hash=sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5 \ + --hash=sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac \ + --hash=sha256:13e3afe97712e8887cd516e960c63f0b93122971e5b5e4b2622fe7701771e838 \ + --hash=sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90 \ + --hash=sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626 \ + --hash=sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4 \ + --hash=sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369 \ + --hash=sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b \ + --hash=sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e \ + --hash=sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee \ + --hash=sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1 \ + --hash=sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102 \ + --hash=sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8 \ + --hash=sha256:29880d17a8eb0b5cfdfd8944b468322928059aa35f1f5fa8ff22b149ec0b42f8 \ + --hash=sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9 \ + --hash=sha256:2e9cf9253119d8e5d111f05d71626786fd3d6193817316eab1ca088cdb8593cf \ + --hash=sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0 \ + --hash=sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031 \ + --hash=sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e \ + --hash=sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235 \ + --hash=sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072 \ + --hash=sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb \ + --hash=sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c \ + --hash=sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950 \ + --hash=sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2 \ + --hash=sha256:366ec70f5547c640d3ce1985722490f23faf4eb5216a7eeba78277490e78dacb \ + --hash=sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e \ + --hash=sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6 \ + --hash=sha256:3e5e1224c0a6a90e05843e07adfec669edebec17801c67072f51e59561d63c0b \ + --hash=sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2 \ + --hash=sha256:433c5a81eade63b47e522303bad236f59dba55ea6951746f5558355eeed8c75d \ + --hash=sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa \ + --hash=sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2 \ + --hash=sha256:494b70049a4d69aec6e8137c13af4cf8db8c9f9820a1392ac293b0dd2987a818 \ + --hash=sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032 \ + --hash=sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71 \ + --hash=sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96 \ + --hash=sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687 \ + --hash=sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8 \ + --hash=sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3 \ + --hash=sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61 \ + --hash=sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9 \ + --hash=sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1 \ + --hash=sha256:55261ac0d2941c42f196dd576f543d87a8ee03cd6f5e30dfb4d807b2e3b9121a \ + --hash=sha256:56490c595a28b1bb27dfc583e816152a9767721ef58b2c03b13f954d2f707420 \ + --hash=sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4 \ + --hash=sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65 \ + --hash=sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663 \ + --hash=sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f \ + --hash=sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591 \ + --hash=sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a \ + --hash=sha256:5ca0555312ae2fe82715cada7fac375530c2f3349e1eaa1bcb33d0283ac79a18 \ + --hash=sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e \ + --hash=sha256:5e2d0e146dcb57034f8b97dc58d2d512cb90aba253960ce449f695fec6a82c6f \ + --hash=sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7 \ + --hash=sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3 \ + --hash=sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c \ + --hash=sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3 \ + --hash=sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7 \ + --hash=sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96 \ + --hash=sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486 \ + --hash=sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3 \ + --hash=sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6 \ + --hash=sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b \ + --hash=sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731 \ + --hash=sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959 \ + --hash=sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9 \ + --hash=sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf \ + --hash=sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8 \ + --hash=sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e \ + --hash=sha256:789b8982559ae28dad2356519f841655756cdcd96616410590ae0b17454ee64f \ + --hash=sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885 \ + --hash=sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0 \ + --hash=sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506 \ + --hash=sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2 \ + --hash=sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0 \ + --hash=sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e \ + --hash=sha256:85de3134b5379856e323ba37c19c9256d39425f7b76a63af52b09fb4664c2e8f \ + --hash=sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e \ + --hash=sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491 \ + --hash=sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a \ + --hash=sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20 \ + --hash=sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449 \ + --hash=sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af \ + --hash=sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c \ + --hash=sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712 \ + --hash=sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7 \ + --hash=sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a \ + --hash=sha256:94fbf1c0c6cc0d3d5e50f9a9313a8cdca90dd696d34b381cd1704f8c9e939f20 \ + --hash=sha256:950f23cb393f85543777b0433f082cddd25b51ab398eac7971146495679efe5f \ + --hash=sha256:96eefc178f8636b9c760c5829345307fd81cfae9ab1e80997dbddeb0f54ee9a3 \ + --hash=sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9 \ + --hash=sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e \ + --hash=sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5 \ + --hash=sha256:994e883d17c559cdfd38c84003c8b27d25424a1077272a17e7cd27bfe0bf57b2 \ + --hash=sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36 \ + --hash=sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263 \ + --hash=sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4 \ + --hash=sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11 \ + --hash=sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a \ + --hash=sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3 \ + --hash=sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375 \ + --hash=sha256:a545775cfe815855ea32d7c27731d79da358ef2055b4a25830231b1622dd18aa \ + --hash=sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d \ + --hash=sha256:a6d095662e73e74f0a49988e0593373e243e3a52e27bfeea0a859e88acf4a0f5 \ + --hash=sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99 \ + --hash=sha256:a951ad59cad9145664a730d3036b40b844e74d2d3683da40111463cd3a83845d \ + --hash=sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c \ + --hash=sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488 \ + --hash=sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6 \ + --hash=sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc \ + --hash=sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b \ + --hash=sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f \ + --hash=sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00 \ + --hash=sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10 \ + --hash=sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598 \ + --hash=sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6 \ + --hash=sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962 \ + --hash=sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c \ + --hash=sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08 \ + --hash=sha256:ba2f37ee79e6338845261a3c5b1784e5d1acdff2c0785b284f1b633033d136ab \ + --hash=sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573 \ + --hash=sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90 \ + --hash=sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5 \ + --hash=sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18 \ + --hash=sha256:be47f99644b208bff7766314013f9acf57b056b04191d570d68ad14022cf5b1d \ + --hash=sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af \ + --hash=sha256:c1dcc36dcb96abc02236e182d17e0f71430152a6c2c7447421da2d2dc144edea \ + --hash=sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c \ + --hash=sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b \ + --hash=sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6 \ + --hash=sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8 \ + --hash=sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774 \ + --hash=sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004 \ + --hash=sha256:ce854f5f478050ade5a238731c4ca985a7d3b3cb53ff600a9b5c3b689b5f0a7a \ + --hash=sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a \ + --hash=sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2 \ + --hash=sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2 \ + --hash=sha256:d1ee1e296209fdce05b81b663250eefa02213a2da7b41bf26f7829b8ba3545aa \ + --hash=sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe \ + --hash=sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3 \ + --hash=sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc \ + --hash=sha256:e06efa066f7dbadbc84ebc126a97c452a6451dfcf589d89d788484949e1cf795 \ + --hash=sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d \ + --hash=sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc \ + --hash=sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893 \ + --hash=sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef \ + --hash=sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d \ + --hash=sha256:e9fbdce1e47394b09bc9f26ab117dfc8d6491977a11d86f592bb42c779db2fda \ + --hash=sha256:eb12fb2ba69ffa05f8695f61c69e591dc4b4a12ac3757ac8af8adb259bf56d17 \ + --hash=sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30 \ + --hash=sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7 \ + --hash=sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5 \ + --hash=sha256:f5542f9b941279d82d41eb0aa9f98eba36fe4df5c7086c651df7944935b37182 \ + --hash=sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f \ + --hash=sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9 \ + --hash=sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada \ + --hash=sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876 \ + --hash=sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a \ + --hash=sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348 \ + --hash=sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3 \ + --hash=sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f \ + --hash=sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0 \ + --hash=sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # requests +filelock==3.32.3 \ + --hash=sha256:0ffa185a3540854c95caa7fa76b76cb219d907415e2c5dc9af25fd970563487f \ + --hash=sha256:7f0ca4bcc0e181c60dbbd8aa9ab5b120ebb99e4e064e83636340056f833a1f09 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # torch + # transformers +fsspec==2026.7.0 \ + --hash=sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279 \ + --hash=sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # torch +hf-xet==1.6.0 \ + --hash=sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7 \ + --hash=sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675 \ + --hash=sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef \ + --hash=sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9 \ + --hash=sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3 \ + --hash=sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb \ + --hash=sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e \ + --hash=sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338 \ + --hash=sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d \ + --hash=sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d \ + --hash=sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765 \ + --hash=sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c \ + --hash=sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522 \ + --hash=sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f \ + --hash=sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a \ + --hash=sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b \ + --hash=sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub +huggingface-hub==0.36.2 \ + --hash=sha256:1934304d2fb224f8afa3b87007d58501acfda9215b334eed53072dd5e815ff7a \ + --hash=sha256:48f0c8eac16145dfce371e9d2d7772854a4f591bcb56c9cf548accf531d54270 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # tokenizers + # transformers +idna==3.19 \ + --hash=sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15 \ + --hash=sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # requests +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # jinja2 +mpmath==1.3.0 \ + --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ + --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # sympy +networkx==3.6.1 \ + --hash=sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509 \ + --hash=sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +numpy==2.4.6 \ + --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ + --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ + --hash=sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f \ + --hash=sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079 \ + --hash=sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096 \ + --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 \ + --hash=sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66 \ + --hash=sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d \ + --hash=sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1 \ + --hash=sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e \ + --hash=sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147 \ + --hash=sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd \ + --hash=sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75 \ + --hash=sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063 \ + --hash=sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73 \ + --hash=sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab \ + --hash=sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4 \ + --hash=sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41 \ + --hash=sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402 \ + --hash=sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698 \ + --hash=sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7 \ + --hash=sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8 \ + --hash=sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b \ + --hash=sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8 \ + --hash=sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0 \ + --hash=sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662 \ + --hash=sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91 \ + --hash=sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0 \ + --hash=sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f \ + --hash=sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3 \ + --hash=sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f \ + --hash=sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67 \ + --hash=sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6 \ + --hash=sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997 \ + --hash=sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b \ + --hash=sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e \ + --hash=sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538 \ + --hash=sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627 \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ + --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ + --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ + --hash=sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c \ + --hash=sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43 \ + --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ + --hash=sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8 \ + --hash=sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089 \ + --hash=sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778 \ + --hash=sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1 \ + --hash=sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb \ + --hash=sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261 \ + --hash=sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb \ + --hash=sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a \ + --hash=sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8 \ + --hash=sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359 \ + --hash=sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5 \ + --hash=sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7 \ + --hash=sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751 \ + --hash=sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8 \ + --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ + --hash=sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e \ + --hash=sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45 \ + --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ + --hash=sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895 \ + --hash=sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe \ + --hash=sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb \ + --hash=sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a \ + --hash=sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577 \ + --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ + --hash=sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a \ + --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ + --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \ + --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # transformers +nvidia-cublas-cu12==12.8.4.1 \ + --hash=sha256:47e9b82132fa8d2b4944e708049229601448aaad7e6f296f630f2d1a32de35af \ + --hash=sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142 \ + --hash=sha256:b86f6dd8935884615a0683b663891d43781b819ac4f2ba2b0c9604676af346d0 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # nvidia-cudnn-cu12 + # nvidia-cusolver-cu12 + # torch +nvidia-cuda-cupti-cu12==12.8.90 \ + --hash=sha256:4412396548808ddfed3f17a467b104ba7751e6b58678a4b840675c56d21cf7ed \ + --hash=sha256:bb479dcdf7e6d4f8b0b01b115260399bf34154a1a2e9fe11c85c517d87efd98e \ + --hash=sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cuda-nvrtc-cu12==12.8.93 \ + --hash=sha256:7a4b6b2904850fe78e0bd179c4b655c404d4bb799ef03ddc60804247099ae909 \ + --hash=sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994 \ + --hash=sha256:fc1fec1e1637854b4c0a65fb9a8346b51dd9ee69e61ebaccc82058441f15bce8 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cuda-runtime-cu12==12.8.90 \ + --hash=sha256:52bf7bbee900262ffefe5e9d5a2a69a30d97e2bc5bb6cc866688caa976966e3d \ + --hash=sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90 \ + --hash=sha256:c0c6027f01505bfed6c3b21ec546f69c687689aad5f1a377554bc6ca4aa993a8 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cudnn-cu12==9.10.2.21 \ + --hash=sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8 \ + --hash=sha256:c6288de7d63e6cf62988f0923f96dc339cea362decb1bf5b3141883392a7d65e \ + --hash=sha256:c9132cc3f8958447b4910a1720036d9eff5928cc3179b0a51fb6d167c6cc87d8 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cufft-cu12==11.3.3.83 \ + --hash=sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74 \ + --hash=sha256:7a64a98ef2a7c47f905aaf8931b69a3a43f27c55530c698bb2ed7c75c0b42cb7 \ + --hash=sha256:848ef7224d6305cdb2a4df928759dca7b1201874787083b6e7550dd6765ce69a + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cufile-cu12==1.13.1.3 \ + --hash=sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc \ + --hash=sha256:4beb6d4cce47c1a0f1013d72e02b0994730359e17801d395bdcbf20cfb3bb00a + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-curand-cu12==10.3.9.90 \ + --hash=sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9 \ + --hash=sha256:dfab99248034673b779bc6decafdc3404a8a6f502462201f2f31f11354204acd \ + --hash=sha256:f149a8ca457277da854f89cf282d6ef43176861926c7ac85b2a0fbd237c587ec + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cusolver-cu12==11.7.3.90 \ + --hash=sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450 \ + --hash=sha256:4a550db115fcabc4d495eb7d39ac8b58d4ab5d8e63274d3754df1c0ad6a22d34 \ + --hash=sha256:db9ed69dbef9715071232caa9b69c52ac7de3a95773c2db65bdba85916e4e5c0 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-cusparse-cu12==12.5.8.93 \ + --hash=sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b \ + --hash=sha256:9a33604331cb2cac199f2e7f5104dfbb8a5a898c367a53dfda9ff2acb6b6b4dd \ + --hash=sha256:9b6c161cb130be1a07a27ea6923df8141f3c295852f4b260c65f18f3e0a091dc + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # nvidia-cusolver-cu12 + # torch +nvidia-cusparselt-cu12==0.7.1 \ + --hash=sha256:8878dce784d0fac90131b6817b607e803c36e629ba34dc5b433471382196b6a5 \ + --hash=sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623 \ + --hash=sha256:f67fbb5831940ec829c9117b7f33807db9f9678dc2a617fbe781cac17b4e1075 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-nccl-cu12==2.27.3 \ + --hash=sha256:9ddf1a245abc36c550870f26d537a9b6087fb2e2e3d6e0ef03374c6fd19d984f \ + --hash=sha256:adf27ccf4238253e0b826bce3ff5fa532d65fc42322c8bfdfaf28024c0fbe039 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +nvidia-nvjitlink-cu12==12.8.93 \ + --hash=sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88 \ + --hash=sha256:adccd7161ace7261e01bb91e44e88da350895c270d23f744f0820c818b7229e7 \ + --hash=sha256:bd93fbeeee850917903583587f4fc3a4eafa022e34572251368238ab5e6bd67f + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # nvidia-cufft-cu12 + # nvidia-cusolver-cu12 + # nvidia-cusparse-cu12 + # torch +nvidia-nvtx-cu12==12.8.90 \ + --hash=sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f \ + --hash=sha256:619c8304aedc69f02ea82dd244541a83c3d9d40993381b3b590f1adaed3db41e \ + --hash=sha256:d7ad891da111ebafbf7e015d34879f7112832fc239ff0d7d776b6cb685274615 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # transformers +pillow==12.3.0 \ + --hash=sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756 \ + --hash=sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a \ + --hash=sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59 \ + --hash=sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45 \ + --hash=sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3 \ + --hash=sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df \ + --hash=sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139 \ + --hash=sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b \ + --hash=sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39 \ + --hash=sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e \ + --hash=sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8 \ + --hash=sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1 \ + --hash=sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8 \ + --hash=sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89 \ + --hash=sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5 \ + --hash=sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130 \ + --hash=sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd \ + --hash=sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d \ + --hash=sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b \ + --hash=sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed \ + --hash=sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace \ + --hash=sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb \ + --hash=sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931 \ + --hash=sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510 \ + --hash=sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6 \ + --hash=sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1 \ + --hash=sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce \ + --hash=sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385 \ + --hash=sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e \ + --hash=sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c \ + --hash=sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7 \ + --hash=sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace \ + --hash=sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c \ + --hash=sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f \ + --hash=sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64 \ + --hash=sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f \ + --hash=sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a \ + --hash=sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827 \ + --hash=sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17 \ + --hash=sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4 \ + --hash=sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a \ + --hash=sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701 \ + --hash=sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e \ + --hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 \ + --hash=sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66 \ + --hash=sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468 \ + --hash=sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217 \ + --hash=sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658 \ + --hash=sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418 \ + --hash=sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a \ + --hash=sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c \ + --hash=sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330 \ + --hash=sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402 \ + --hash=sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09 \ + --hash=sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930 \ + --hash=sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f \ + --hash=sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec \ + --hash=sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a \ + --hash=sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94 \ + --hash=sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468 \ + --hash=sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b \ + --hash=sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965 \ + --hash=sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8 \ + --hash=sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd \ + --hash=sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7 \ + --hash=sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c \ + --hash=sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777 \ + --hash=sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35 \ + --hash=sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9 \ + --hash=sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f \ + --hash=sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f \ + --hash=sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0 \ + --hash=sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c \ + --hash=sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71 \ + --hash=sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3 \ + --hash=sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838 \ + --hash=sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf \ + --hash=sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321 \ + --hash=sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26 \ + --hash=sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec \ + --hash=sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9 \ + --hash=sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65 \ + --hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \ + --hash=sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e \ + --hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \ + --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \ + --hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7 + # via -r benchmarks/blenderbench_direct/requirements-scoring.in +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # transformers +regex==2026.9.10 \ + --hash=sha256:030fa9e23624e39b3b94e46b90a5abd1a1678eb2f58fcdd3fd6c27526bf91c7e \ + --hash=sha256:032da15431c890d376f53547f0a6219f4f4cd19f3e4f11bdc321453b5bd207e4 \ + --hash=sha256:044bd4639b6bb409ec9e5d8b7accd57e02b4c4a4e2eafde916f8ae8006b3e40b \ + --hash=sha256:048a89ee797db10160bd2bd519286577a6b43a100279bd4b7d8456a3d69c80a0 \ + --hash=sha256:05fb018cfe7144585fc83882405906ff84994a2d154afc2509ecc7752c51f864 \ + --hash=sha256:07b45ba5c94b8fcb30cb6c56a11f715c57533a3017964504322ea52690a27b72 \ + --hash=sha256:0aa7589394230e0f0a422ab6b90841ff12c87e855e7aaf75d192a54a5f124548 \ + --hash=sha256:0acee94b480dd853e39434aa9a575f95385b1b4b8fa3feae56db363ca5cad782 \ + --hash=sha256:0b9ba3b2765cdfe18f0f561a69f78a69701f2896654a81c711108d35d14e5099 \ + --hash=sha256:0c32480f3371b75068decaf9e5da72c224e953830dd71e36e06cf80e30ea39d8 \ + --hash=sha256:1270cdec69248592bbe38a0b263ed58d907b891bd2b93703e225c317e421bda1 \ + --hash=sha256:13c52fc377792675f604a207a2ae5958c080f6854f7698d40d9ff034d95b1e76 \ + --hash=sha256:14caa05ce39ec70437af5aac8814c50ee6628f4a90353871c059692f448a164f \ + --hash=sha256:1562aabd9d4eb09bd88a62ad97ed06800094b529ac43419e43020b9cefec79b0 \ + --hash=sha256:175cf49ce7a994c88b8f15e3cb17cdb66a48ebb2d36de736b8205033db950f89 \ + --hash=sha256:1aa309ab7ba89a62d6cf70dbd38d4176440bce3c7001ab86256704cf4c18c6eb \ + --hash=sha256:1ad10a135fa0b4e4a462a61d07c6654d7518cfdb5cb8da08f9ff7d61384af1fe \ + --hash=sha256:1b891f77554bff991804cee24b78b40789f7d5993a24c7907bc7025fd2a70c8d \ + --hash=sha256:1e321e2c84f0e52c457f5ea5944f796d6e8e09cb99738ea98dcc1bfe402a128d \ + --hash=sha256:1e954e246466d5a1a78f563ce8364b5d7cb19e7adb0ccdec8f9c9610083187bc \ + --hash=sha256:1f0a8b4928823bc8b217a1ab7bf3d90598909dec9a70fbbfe9a52cc4eca55990 \ + --hash=sha256:1fbc8314436353e097c050e11b01a6c11433579437ed0579730157676ef59e2f \ + --hash=sha256:20e8bfb07ad79a282f8b95b56fe67f9750b1b7f775724e4ba1f23cb296115ce4 \ + --hash=sha256:217e98ba5fc8908ed8ffd4ebac04753a0c831067cbfb495b9821b94cc61eaa76 \ + --hash=sha256:239620b0e0681669367c0e218c8eb2551d9f8fe3b9fccfc8d0003377804e8348 \ + --hash=sha256:23ac9a28180f274d7dd7651fa131ad5b02d343b75df4b040737f0356223895dd \ + --hash=sha256:2479171edccced52ef02b899558f88ab2c235fe05b93180fdcae1670aacd89e1 \ + --hash=sha256:24d12a625a37c89c2b09303402a06942f55f071b95a7916a49c17034c3d47cd5 \ + --hash=sha256:2dd9286093c71afc8f55ef035c5b9d2776641fd72c6535f1febc92d0b0be9666 \ + --hash=sha256:2e67f8843f0e4b931f1fa860bf3bbe4134b714c0155cc5c7c0d7ea450230aae0 \ + --hash=sha256:31e4df2b11d48f61d511019bc1ee9b477055f17c352b68fe72db7a98b14d603c \ + --hash=sha256:3264132d576847ab5f88bb83e7debe67854bf165b3ea613bd467312b6099536a \ + --hash=sha256:3540734dbe241ebb3b87d5713781f6749a3e4d45480f506aa5fb5cbb0c37d249 \ + --hash=sha256:35ba3bab0c45079735f55ac61526774de1d84bc4a0333cc554e1a4ab74913924 \ + --hash=sha256:3a66e40a1a20de96a2fee00ed67e11012b62d85b277688258677fd19997addb7 \ + --hash=sha256:3bdeed3318a8eb2bbadc9c56347e0ff651639e934a47e168d05a3b12929fd0e7 \ + --hash=sha256:3fb4ae8cf83ef4e9addd43b2da31a9f45be816a8036fae8af59c8998b72718e2 \ + --hash=sha256:4971776b4f2bd7fd9a83eceb2cb2592cbe2924f639fe8045e6a9de5ba4bfcf25 \ + --hash=sha256:4a761ea45f2ad74c575ef5850ea514cef97302a552d3c7c9d1a1a870d4661d6c \ + --hash=sha256:4c66d54042a14a503907d81861b8a5235e6d1f03d4fbc1d8767f652eaf957ac1 \ + --hash=sha256:4db7d00c4afbfbb55b8e17b1e371da11418ea9389b030acec63c1fa4c7ad4b86 \ + --hash=sha256:4f0407474ffac8e5e89d93ca41d60891e29f0ab8423eb66ff292d850a86a0843 \ + --hash=sha256:53e182b6b04d0011909b47d51a2d72d908de07c7b1c7f16b3adda2204d723bc1 \ + --hash=sha256:5847e22bbf959764d776937d791d034cc2d19b787e361c88d97e859e8dc68502 \ + --hash=sha256:58c01f7b81079cf0817ba831ff4d9eff5d28be4a3ac76c353e6f09bd63f4c386 \ + --hash=sha256:58da726d3e766c0b3f5a3997dfaf0275898a1107b8191cdd6b0437fe45fd817d \ + --hash=sha256:5bef622850cf760154719d4e0d74b0a855962432995168e250069899ae12fe8f \ + --hash=sha256:5ccd139b2061132e7b265cfb4b4721baeb9f8928b81415304abf1ec7e3181c26 \ + --hash=sha256:5cef9f3d14796500ea834c41dbe688f1f6b23c7024dc23e8a794d7ebaf5d71d0 \ + --hash=sha256:63bb62cf62217dc38c8a6b2b61b165b0e4eb8fa93b0aba12139251c0986a8fa3 \ + --hash=sha256:681ed38664b64c6617d3c3c332018d1948c77e139c5ea667c1886efa671e426f \ + --hash=sha256:6888065672b341e5246f391ec16dc258a29218ac784172fd67c30d941544755b \ + --hash=sha256:6aebdd9a946de328b3f6f61dbf48dd064a36eb6dddf96e34ae6651d37f6e9383 \ + --hash=sha256:6afcad14310f1311d077553ed374b42a5e538f85a8c884b4e38e52de091c8077 \ + --hash=sha256:6b34a778c695d24e77c140e3b4c95da69282e34f2f6b02b55656aa4a0379f643 \ + --hash=sha256:6fd555fc9abef50c530869690b2daca054c8811a7aff632d11f9a7b2590b2742 \ + --hash=sha256:71879292c9c7ac67b1680345b16daba1be937cb027362cfa04e68f65db2dcfdd \ + --hash=sha256:75242f44a3e283106077be4ab717bc535e4701c9d54ad69e195945c22f137a1d \ + --hash=sha256:75aa39d3f4f1650eea84e46b0d8cefe77dd5478c10e3d0aaf0b0f00493475a7a \ + --hash=sha256:75f9297b16fcb588a1f8d8a55dabef3c0c20b0c7bac43c87ceaaaf1a825c12f4 \ + --hash=sha256:79e9432995e14c749d34209413de5e621ec8e67789bf4f46dbfabea9d06a2406 \ + --hash=sha256:7abb38b8c40f3a235235a44da452c64b7b5c1d650ec6351027db0e090804f2e5 \ + --hash=sha256:7dcad477c49c4c626a6c4fcd71b39a971aa217060cc40a6569fd24edcc0fa509 \ + --hash=sha256:7e6c0b5ec6ddee4032247585dc491b0fa58627745b66a705728703a3f0331231 \ + --hash=sha256:7f8f10015866608fe4c043cec2e4fe4c39a94bb50e45091de4cdf4004b9ae4b0 \ + --hash=sha256:866de9f98df0611d7b62b3a8729d3284a64c0cc6edd90bb95a533e443a4939cb \ + --hash=sha256:87f5f75c109f08f5c602d68e1af54cead8165189c727b6ac946b30b9833a3ba4 \ + --hash=sha256:880ac684c27176464c00c3fdc456116364f5ebc70da07aad0c2d4a7ba45e98db \ + --hash=sha256:88b02aa8d0ec9b6189fe933d425775882271c23700ac11fd26d1779b0f56fde3 \ + --hash=sha256:8ba1f78bd4fef2d8f84b894ec28ac3481afe6cc07aaa253ad4717ef7b3fe6bcb \ + --hash=sha256:8c07021a4faa3f092869adbd1f35cdc7a592276c807aeebc3ceb8ff1a638f0b4 \ + --hash=sha256:8d5c4518235a2ec1611e57af85fa488d529c1106aacff12adadcedf8687012cd \ + --hash=sha256:8e127d9a80cbf1c3276bb465c6d047e8705e97b58c2b8f2f0c0a69c336b44b37 \ + --hash=sha256:94c5ce3bc41d226b4eb89ca3f842b2e28c031487fb1f34eb2153d98235831325 \ + --hash=sha256:94d096369b7cd96d15343fef5257fe39eff9d0e8758b92a0e15e358b92cdb2fc \ + --hash=sha256:968c1e33edd9a104d1bf24c8d476c72de7e3839ae7f894b37e9e4f4739fdeeca \ + --hash=sha256:990797e765d89a423880052c68b61c31afe701de94a8c060f61c40605ca6c727 \ + --hash=sha256:9ce239acb15843ab03976626af810a4424b0409689ec2bbc52088ab5479ab487 \ + --hash=sha256:9d772586951d7d6a5d162d48f414065e483b1c81ab38fd8ed97c78b05883421a \ + --hash=sha256:9fbd2e5d8002dc49a6129fb321ec51c57a025e752ed525ddce0ba9223c4350a7 \ + --hash=sha256:a41693eb3fc4b92e6127d113813c6c395237f7edd3224abf67609af48c690d11 \ + --hash=sha256:abbfc1c33bf8efddcc43844aba61e036d74a918680dc3ce8ce2538b004eda0f9 \ + --hash=sha256:b298cdc33c5cc6969ff07f0fba19cc73e0fd8576373c50935feadaca2f6b4405 \ + --hash=sha256:b43456de605c8ee77eb75f07bc1ee44ba27f9cee22207deb77d495e954b7d953 \ + --hash=sha256:b71649169a9fcf30b395ee01047fa7ad6654a4c900ca75b23c04dedcce6a1f8c \ + --hash=sha256:b91c37551bf39d75116c02b146956f65b9aa0337a4a652f4ae186983789d4001 \ + --hash=sha256:b9d36b03dc362aa40ffaaec9d9bd75e87763529563ec008c43b0e07782f5be7a \ + --hash=sha256:bafa41b0dd63669e5c0f8adf3d24819efeb73c847f492eb011212eb352e69041 \ + --hash=sha256:bb7774924f8cd69f49cba0b3c2d679a6326f777e0e67d130ad5203e4df53f0d3 \ + --hash=sha256:bf29611e5376fec8f795879bb5c6153a76c3a292573d173c26784042b01eb840 \ + --hash=sha256:c014641157e9049b0603b8daa5343bd408d9b757b709aaa0f373cd3fab2d7944 \ + --hash=sha256:c103b3b14e011774af4fb7e4617ad4d72b9171905cd3b231a70a4efd76e477d7 \ + --hash=sha256:c22df8dd6373bbe3898e77429ffc85594300e39d752fd0e68a31e59d37899376 \ + --hash=sha256:c25a754bb81a2edcfc3b65eda50f017d736f818112ed43e8aafd595cb00678ae \ + --hash=sha256:c32818b28bcd153b25b63038348a9fe9b9fbcddb60df43f204c3ab55eeb57f77 \ + --hash=sha256:c37fa93bf18bf4f90b01c0fa9f11ea567ee4b7dd8bf96e63663e5edc37aa38cf \ + --hash=sha256:c3d95d7d9538b5b726dd6fcd7b6117a71e6565202f6d64f5845fb4d8f203f533 \ + --hash=sha256:c8fbd9cb30c68c1686b94029b9ef845d5870d3d65baf66cb126b676849b9d72b \ + --hash=sha256:cb76a9c4e07a6a47849726af0ed14c41741a182f097f134a8cf29c1bc0f4dde8 \ + --hash=sha256:ce7c118cb102975f974585688357a717ffbf9dddd64ab0bb1bc93eb5b367cf95 \ + --hash=sha256:cf377960d2ac37d987394a9dbaa75e91338c41a46d41e1d25e90125e7b3ee2dc \ + --hash=sha256:d278ad30ec83b6b9202685b0f80b741a51ea3ca7f0595ebda96e7628b6398876 \ + --hash=sha256:d2d377fd1cad611b806cdd732d86b65f536c768209890cb442556548daa65a23 \ + --hash=sha256:d414c411c06fe0009eac33488fb1591c66b5c2673e342e452e7bb2fe63da8194 \ + --hash=sha256:d8c668af8f7bdb1d18739c27d30cd9f4b371495a883f75a002fb7a39d740fecd \ + --hash=sha256:dce932f8e3ba936475ea3d0d8b59f7b050a9e206e994f53f8fd80299871e87da \ + --hash=sha256:debc629e98b95abaea1cf3057ca296151f348c697c9b8a59d18013adb302c0dd \ + --hash=sha256:e0dc78251154b66dc60211563fc115345da332eaa881e4e2523fb1edae3772f4 \ + --hash=sha256:e5e4a6e0734a685d13b9685622bb503bdbb2927f8b0df025a5085f0ea067475b \ + --hash=sha256:e6b99181d184d0f5c7b36b8d12b94d1e9499cce6246594331f9edc5d2ea9fceb \ + --hash=sha256:e7327795089ddb44912dce1434e1d7244be2e9fb48fcc2d6782936af7a3062db \ + --hash=sha256:ebb2ba68e4641a994061f70bf44ed448fba0b9b1d18c94ffb9efc1cca805b39b \ + --hash=sha256:ec8855f08c17895a26fbf5f19ed829722e19b34a96629e49a43c92974924026b \ + --hash=sha256:ecb2e7acb18f8cc4a67f0ad986c0af291ea4dd385d0614ba9bc09d7f8bbb478c \ + --hash=sha256:ef4c0a9dfdc90581b90b1b95a8c3d1557f8ff8f5a2a53536d26314de699d1468 \ + --hash=sha256:ef4ce69ff97fbb44b46751cfea5e859ad0b66d1a50abf34954f0645f51e81671 \ + --hash=sha256:ef5a059ea1c6ee5d1c7e99a2484e628608d010921efe876c6f0e2029d2f35eca \ + --hash=sha256:f0e2e5d23448b660d60a6ed85c46cc03b4b48bd276b8f4041d4a5fe2a4a0626b \ + --hash=sha256:f2374c27deb189b282ec7e16106752c22ad39b056bbd8018960b1e4cc95d67a1 \ + --hash=sha256:f2f43bf4e47ff7ce9e585558706d698c6204d0f80bf2207766382ed817c8e9f4 \ + --hash=sha256:f5c629df03adec31ee505dda3c8988f106c9390e4cbd343600036eb8b3d6724f \ + --hash=sha256:f70b9f0e39c2dba1d9da6bf7ef7c377cad7277f8440e9a69be05ede529ff024c \ + --hash=sha256:f7d4656e17ab736e9415a6442a345bfc97bb8b7dcce47884bb74a37f70f08d0c \ + --hash=sha256:f8bdec659a8fa7af51a32b224b3b7c02bc415d54ffd35187b1d224176b17d607 \ + --hash=sha256:faa911fbbcf8ac90bda0e0657d60768e3390954ef0588211d63a22add1cb1cd1 \ + --hash=sha256:fbc4e2f3cb7ce8436154e6483079e7d35eeb321a952fa936e180300630d8b873 \ + --hash=sha256:fd6bd89b9fc06018d35851cab0240adb7dd84d51941b19f6574ac90cd54e3ae5 \ + --hash=sha256:ff4d7b14ea19e50c8d9d6d83f45bd9b45cbb624c07ac1fa54db0a019049abed7 \ + --hash=sha256:ff6b3267318661dfddf6b3628663e00e5946bd0a5c8fa678537a1401f0388f91 \ + --hash=sha256:ffc2da104e43db716ce30cef9f28049a1faa6aca385dd8771b033268d0730b07 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # transformers +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # transformers +safetensors==0.8.0 \ + --hash=sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358 \ + --hash=sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f \ + --hash=sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d \ + --hash=sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d \ + --hash=sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0 \ + --hash=sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc \ + --hash=sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235 \ + --hash=sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98 \ + --hash=sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4 \ + --hash=sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846 \ + --hash=sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca \ + --hash=sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0 \ + --hash=sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25 \ + --hash=sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452 \ + --hash=sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d \ + --hash=sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78 \ + --hash=sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # transformers +setuptools==78.1.0 \ + --hash=sha256:18fd474d4a82a5f83dac888df697af65afa82dec7323d09c3e37d1f14288da54 \ + --hash=sha256:3e386e96793c8702ae83d17b853fb93d3e09ef82ec62722e61da5cd22376dcd8 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch + # triton +sympy==1.14.0 \ + --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ + --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +tokenizers==0.21.4 \ + --hash=sha256:1340ff877ceedfa937544b7d79f5b7becf33a4cfb58f89b3b49927004ef66f78 \ + --hash=sha256:2107ad649e2cda4488d41dfd031469e9da3fcbfd6183e74e4958fa729ffbf9c6 \ + --hash=sha256:2ccc10a7c3bcefe0f242867dc914fc1226ee44321eb618cfe3019b5df3400133 \ + --hash=sha256:39b376f5a1aee67b4d29032ee85511bbd1b99007ec735f7f35c8a2eb104eade5 \ + --hash=sha256:3c1f4317576e465ac9ef0d165b247825a2a4078bcd01cba6b54b867bdf9fdd8b \ + --hash=sha256:3c73012da95afafdf235ba80047699df4384fdc481527448a078ffd00e45a7d9 \ + --hash=sha256:475d807a5c3eb72c59ad9b5fcdb254f6e17f53dfcbb9903233b0dfa9c943b597 \ + --hash=sha256:51b7eabb104f46c1c50b486520555715457ae833d5aee9ff6ae853d1130506ff \ + --hash=sha256:5e2f601a8e0cd5be5cc7506b20a79112370b9b3e9cb5f13f68ab11acd6ca7d60 \ + --hash=sha256:6c42a930bc5f4c47f4ea775c91de47d27910881902b0f20e4990ebe045a415d0 \ + --hash=sha256:714b05b2e1af1288bd1bc56ce496c4cebb64a20d158ee802887757791191e6e2 \ + --hash=sha256:c212aa4e45ec0bb5274b16b6f31dd3f1c41944025c2358faaa5782c754e84c24 \ + --hash=sha256:cc88bb34e23a54cc42713d6d98af5f1bf79c07653d24fe984d2d695ba2c922a2 \ + --hash=sha256:f23186c40395fc390d27f519679a58023f368a0aad234af145e0f39ad1212732 \ + --hash=sha256:fa23f85fbc9a02ec5c6978da172cdcbac23498c3ca9f3645c5c68740ac007880 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # transformers +torch @ https://download.pytorch.org/whl/cu128/torch-2.8.0%2Bcu128-cp313-cp313-manylinux_2_28_x86_64.whl \ + --hash=sha256:3a852369a38dec343d45ecd0bc3660f79b88a23e0c878d18707f7c13bf49538f + # via -r benchmarks/blenderbench_direct/requirements-scoring.in +tqdm==4.70.0 \ + --hash=sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220 \ + --hash=sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # transformers +transformers==4.55.4 \ + --hash=sha256:574a30559bc273c7a4585599ff28ab6b676e96dc56ffd2025ecfce2fd0ab915d \ + --hash=sha256:df28f3849665faba4af5106f0db4510323277c4bb595055340544f7e59d06458 + # via -r benchmarks/blenderbench_direct/requirements-scoring.in +triton==3.4.0 \ + --hash=sha256:00ea4a1c15f079bbf49ff02c312210fdb8ff05b1503ea30812610e2a3b171f3b \ + --hash=sha256:0e582bfd8147afd0e17f410b39ee161df933a2aca9f653e1178daae98f87f601 \ + --hash=sha256:1b1389a284a8c5f29749f643e3b1fc7513e3d162ca6d50f4e3d658de7dba631b \ + --hash=sha256:3d5658665a16be28d74ee5775408e4fd391bcbb0162ad5a3503b4f0c99778522 \ + --hash=sha256:40b5d803a83b5e70895424a51c3a1a23dcc0f158a79f682aaa2c940903e747b9 \ + --hash=sha256:5acb528f01dc14362c4a88fa4dcfa227d132f6e61f272a6874e84d6485037ca4 \ + --hash=sha256:5f2f15de16fefacac008378919687c0ada6a867dcc0fe70a5d5a11240ae8b814 \ + --hash=sha256:6b17588c59cbe1d987133ea7cd1cc2e5229e0544a06a9908bfaf68c61f56495a \ + --hash=sha256:8550672b1184f05187f4140db32e33e61b592046fd3e1eb907e3b7db5321b750 \ + --hash=sha256:86eb0cb7634328cd577fa51b59338f1af4199cf9ab62385ac42a82783d9116a4 \ + --hash=sha256:dd8d32d337f58124fe254043d282a4bc133c2aca7989bc6358b2c546d2121513 \ + --hash=sha256:e2b0afe420d202d96f50b847d744a487b780567975455e56f64b061152ee9554 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # torch +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # huggingface-hub + # torch +urllib3==2.7.0 \ + --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ + --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 + # via + # -r benchmarks/blenderbench_direct/requirements-scoring.in + # requests diff --git a/benchmarks/blenderbench_direct/runner.py b/benchmarks/blenderbench_direct/runner.py new file mode 100644 index 0000000..e32aeac --- /dev/null +++ b/benchmarks/blenderbench_direct/runner.py @@ -0,0 +1,159 @@ +"""Publication BlenderBench CLI and explicit portable configuration.""" +from __future__ import annotations + +import argparse +from dataclasses import asdict, dataclass +import json +from pathlib import Path +import sys +from typing import Sequence + +from . import dataset, protocol, scoring, video + + +@dataclass(frozen=True) +class RunConfig: + dataset_root: Path + mcp_toml: Path + codex_binary: Path + codex_version: str + bpy_python: Path + bpy_version: str + clip_snapshot: Path + output_root: Path + model: str + model_revision: str | None + rounds: int + tasks: tuple[str, ...] + + +def _path(value: str) -> Path: + return Path(value).expanduser().absolute() + + +def _external(path: Path) -> Path: + repository = Path(__file__).resolve().parents[2] + candidate = path.expanduser().resolve() + if candidate == repository or repository in candidate.parents: + raise ValueError(f"dataset/output roots must stay outside this repository: {candidate}") + return candidate + + +def _run_options(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--dataset-root", type=_path, required=True) + parser.add_argument("--mcp-toml", type=_path, required=True) + parser.add_argument("--codex-binary", type=_path, required=True) + parser.add_argument("--codex-version", required=True, help="Exact output expected from CODEx_BINARY --version") + parser.add_argument("--bpy-python", type=_path, required=True) + parser.add_argument("--bpy-version", required=True, help="Exact bpy.app.version_string prefix required") + parser.add_argument("--clip-snapshot", type=_path, required=True, + help="Offline CLIP snapshot matching clip-snapshot-manifest.json") + parser.add_argument("--output-root", type=_path, required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--model-revision") + parser.add_argument("--rounds", type=int, choices=[10], default=10, + help="Frozen publication protocol: exactly 10 rounds") + parser.add_argument("--tasks", nargs="+", help="Explicit unique subset; omit for all 27 tasks") + parser.add_argument("--approve-blender-tools", action="store_true") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + download = commands.add_parser("download", help="Download and verify the pinned 27-task dataset") + download.add_argument("--dataset-root", type=_path, required=True) + download.add_argument("--tasks", nargs="+") + run = commands.add_parser("run", help="Fresh generation; never resumes or regenerates existing output") + _run_options(run) + recover = commands.add_parser("recover", help="Audit/score saved output only; never generate") + recover.add_argument("--source-root", type=_path, required=True) + recover.add_argument("--output-root", type=_path, required=True, help="Fresh recovery evidence root") + recover.add_argument("--dataset-root", type=_path, required=True) + recover.add_argument("--bpy-python", type=_path, required=True) + recover.add_argument("--bpy-version", required=True) + recover.add_argument("--clip-snapshot", type=_path, required=True) + recover.add_argument("--tasks", nargs="+") + recover.add_argument("--rounds", type=int, choices=[10], default=10) + score = commands.add_parser("score", help="Score saved renders only; never generate") + score.add_argument("--run-root", type=_path, required=True) + score.add_argument("--dataset-root", type=_path, required=True) + score.add_argument("--clip-snapshot", type=_path, required=True) + score.add_argument("--output", type=_path) + score.add_argument("--tasks", nargs="+") + score.add_argument("--rounds", type=int, choices=[10], default=10) + movie = commands.add_parser("video", help="Generate a labelled compilation video from a JSON spec") + movie.add_argument("--spec", type=_path, required=True) + movie.add_argument("--output", type=_path, required=True) + movie.add_argument("--ffmpeg") + movie.add_argument("--fps", type=int, default=24) + movie.add_argument("--seconds-per-task", type=float, default=3.0) + return parser + + +def resolve_config(args: argparse.Namespace, *, check_files: bool = True) -> RunConfig: + selected = tuple(dataset.select_tasks(args.tasks)) + args.dataset_root = _external(args.dataset_root) + args.output_root = _external(args.output_root) + config = RunConfig(dataset_root=args.dataset_root, mcp_toml=args.mcp_toml, + codex_binary=args.codex_binary, codex_version=args.codex_version, + bpy_python=args.bpy_python, bpy_version=args.bpy_version, + clip_snapshot=args.clip_snapshot, + output_root=args.output_root, model=args.model, model_revision=args.model_revision, + rounds=args.rounds, tasks=selected) + if (not config.model.strip() or config.codex_version != protocol.CODEX_VERSION + or not config.bpy_version.strip()): + raise ValueError("model and exact Codex 0.154.0/bpy pins are required") + if check_files: + for name in ("mcp_toml", "codex_binary", "bpy_python"): + if not getattr(config, name).is_file(): + raise FileNotFoundError(f"--{name.replace('_', '-')} is not a file: {getattr(config, name)}") + scoring.verify_clip_snapshot(config.clip_snapshot) + scoring.preflight_scorer_runtime() + for task in config.tasks: + dataset.verify(task, config.dataset_root) + return config + + +def _score(args: argparse.Namespace) -> int: + args.run_root = _external(args.run_root) + args.dataset_root = _external(args.dataset_root) + if args.output is not None: + args.output = _external(args.output) + tasks = dataset.select_tasks(args.tasks) + result = scoring.score_saved_run(args.run_root, args.dataset_root, tasks, args.rounds, + args.clip_snapshot) + json_path, csv_path = scoring.write_outputs(result, args.output or args.run_root / "scoring") + print(json.dumps({"json": str(json_path), "csv": str(csv_path), **result["summary"]})) + return int(result["summary"]["failed_tasks"] != 0) + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.command == "download": + args.dataset_root = _external(args.dataset_root) + for task in dataset.select_tasks(args.tasks): + print(dataset.stage(task, args.dataset_root)) + return 0 + if args.command == "score": + return _score(args) + if args.command == "video": + args.output = _external(args.output) + video.generate(args.spec, args.output, fps=args.fps, + seconds_per_task=args.seconds_per_task, ffmpeg=args.ffmpeg) + print(args.output) + return 0 + if args.command == "recover": + args.source_root = _external(args.source_root) + args.output_root = _external(args.output_root) + args.dataset_root = _external(args.dataset_root) + from .suite import recover_suite + return recover_suite(args) + if args.command == "run": + config = resolve_config(args) + from .suite import run_suite + return run_suite(config, approved=args.approve_blender_tools) + raise AssertionError(args.command) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/blenderbench_direct/score_pair.py b/benchmarks/blenderbench_direct/score_pair.py new file mode 100644 index 0000000..459630a --- /dev/null +++ b/benchmarks/blenderbench_direct/score_pair.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Pinned GPU image-pair scorer using VIGA's exact metric functions. + +Place the pinned VIGA ``ref_based_eval.py`` beside this script. This wrapper +hash-verifies it, extracts only three metric functions, and never runs VIGA's +agent or judge. Model weights must exist in the byte-verified offline snapshot supplied by the +parent runner. No Hub lookup is performed. +""" +from __future__ import annotations + +import ast +import hashlib +import json +import os +from pathlib import Path +import sys + +os.environ.setdefault("HF_HUB_OFFLINE", "1") + +import numpy as np +from PIL import Image +import torch +from transformers import CLIPModel, CLIPProcessor + +VIGA_REVISION = "69cb8ef0651bf68124815682df4a0f8e8b57c141" +EVALUATOR_SHA256 = "9ee08818aa858dccfff3efbb36cb9a98e8887f14e6168ad1a23731351cdeb3b5" +CLIP_REVISION = "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268" +EVALUATOR_SOURCE_URL = ("https://github.com/Fugtemypt123/VIGA-release/blob/" + + VIGA_REVISION + "/evaluators/blenderbench/ref_based_eval.py") + + +def main(argv: list[str] | None = None) -> int: + args = sys.argv[1:] if argv is None else argv + if len(args) != 2: + raise SystemExit("usage: score_pair.py IMAGE TARGET") + source = Path(__file__).with_name("ref_based_eval.py") + if hashlib.sha256(source.read_bytes()).hexdigest() != EVALUATOR_SHA256: + raise ValueError(f"evaluator bytes do not match pinned VIGA source: {EVALUATOR_SOURCE_URL}") + if not torch.cuda.is_available(): + raise RuntimeError("CUDA unavailable; publication scorer has no CPU fallback") + snapshot = os.environ.get("BLENDERBENCH_CLIP_SNAPSHOT") + if not snapshot or not Path(snapshot).is_dir(): + raise ValueError("verified BLENDERBENCH_CLIP_SNAPSHOT is required") + model = CLIPModel.from_pretrained(snapshot, local_files_only=True, + use_safetensors=False, weights_only=True).to("cuda").eval() + processor = CLIPProcessor.from_pretrained(snapshot, local_files_only=True, use_fast=False) + + def cuda_processor(*positional, **keywords): + return processor(*positional, **keywords).to("cuda") + + namespace = {"np": np, "torch": torch, "Image": Image, "GLOBAL_CLIP_MODEL": model, + "GLOBAL_CLIP_PROCESSOR": cuda_processor} + tree = ast.parse(source.read_text(encoding="utf-8")) + for name in ("ensure_clip_loaded", "clip_similarity", "photometric_loss"): + node = next(item for item in tree.body if isinstance(item, ast.FunctionDef) and item.name == name) + exec(compile(ast.Module(body=[node], type_ignores=[]), str(source), "exec"), namespace) + with Image.open(args[0]) as image: + candidate = image.convert("RGB"); candidate.load() + with Image.open(args[1]) as image: + target = image.convert("RGB"); target.load() + cosine = float(namespace["clip_similarity"](candidate, target)) + loss = float(namespace["photometric_loss"](candidate, target)) + if not np.isfinite(cosine) or not np.isfinite(loss): + raise ValueError("nonfinite evaluator output") + print(json.dumps({"clip_image_cosine_similarity": cosine, "nclip": 1.0 - cosine, + "photometric_loss": loss, "clip_model": "openai/clip-vit-base-patch32", + "clip_revision": CLIP_REVISION, "viga_revision": VIGA_REVISION, + "evaluator_sha256": EVALUATOR_SHA256, "device": str(next(model.parameters()).device), + "gpu": torch.cuda.get_device_name(0), "torch_version": torch.__version__, + "transformers_version": __import__("transformers").__version__, + "numpy_version": np.__version__, "pillow_version": __import__("PIL").__version__}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/blenderbench_direct/scoring.py b/benchmarks/blenderbench_direct/scoring.py new file mode 100644 index 0000000..44bc1e8 --- /dev/null +++ b/benchmarks/blenderbench_direct/scoring.py @@ -0,0 +1,436 @@ +"""Portable post-generation BlenderBench scoring and aggregation.""" +from __future__ import annotations + +from collections import defaultdict +import csv +import hashlib +import json +import math +import os +from pathlib import Path +import re +import subprocess +from typing import Iterable + +from . import dataset, protocol + + +CANONICAL_SCORER = Path(__file__).with_name("score_pair.py") +SCORING_LOCK = Path(__file__).with_name("requirements-scoring.lock") +CLIP_SNAPSHOT_MANIFEST = Path(__file__).with_name("clip-snapshot-manifest.json") +_ALLOWED_BOOTSTRAP_DISTRIBUTIONS = {"pip"} +_SCORER_ENV_KEYS = {"CUDA_VISIBLE_DEVICES", "HOME"} +_EXPECTED_RUNTIME = { + "python_version": "3.13.15", "python_compiler": "Clang 22.1.3", + "implementation": "CPython", "cache_tag": "cpython-313", + "soabi": "cpython-313-x86_64-linux-gnu", "system": "Linux", "machine": "x86_64", + "bits": 64, "torch_version": "2.8.0+cu128", "torch_cuda": "12.8", + "transformers_version": "4.55.4", "numpy_version": "2.4.6", "pillow_version": "12.3.0", +} + + +def _normalized_distribution(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def locked_distributions(lock: Path = SCORING_LOCK) -> dict[str, str]: + """Read the exact distributions from the generated hash lock.""" + expected: dict[str, str] = {} + for line in Path(lock).read_text(encoding="utf-8").splitlines(): + if not line or line[0].isspace() or line.startswith("#"): + continue + requirement = line.removesuffix(" \\") + if "==" in requirement: + name, version = requirement.split("==", 1) + elif " @ " in requirement: + name, url = requirement.split(" @ ", 1) + filename = __import__("urllib.parse", fromlist=["unquote"]).unquote(url).rsplit("/", 1)[-1] + match = re.search(r"-([0-9][A-Za-z0-9.+]*)-cp\d", filename) + if match is None: + raise ValueError(f"cannot derive locked version for {name}") + version = match.group(1).replace("%2B", "+") + else: + raise ValueError(f"unsupported scoring lock requirement: {requirement}") + normalized = _normalized_distribution(name.strip()) + if normalized in expected: + raise ValueError(f"duplicate scoring lock distribution: {normalized}") + expected[normalized] = version.strip() + if not expected: + raise ValueError("scoring lock has no distributions") + return expected + + +def _isolated_env(clip_snapshot: Path | None = None) -> dict[str, str]: + result = {key: value for key, value in os.environ.items() if key in _SCORER_ENV_KEYS} + result.update(PYTHONNOUSERSITE="1", PYTHONSAFEPATH="1", PYTHONDONTWRITEBYTECODE="1", + HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1") + if clip_snapshot is not None: + result["BLENDERBENCH_CLIP_SNAPSHOT"] = str(Path(clip_snapshot).absolute()) + return result + + +def verify_clip_snapshot(snapshot: Path) -> dict: + """Verify the complete offline snapshot allowlist, sizes, and SHA-256 bytes.""" + root = Path(snapshot).expanduser().absolute() + manifest_bytes = CLIP_SNAPSHOT_MANIFEST.read_bytes() + manifest = json.loads(manifest_bytes) + if manifest.get("revision") != protocol.CLIP_REVISION or manifest.get("schema_version") != 1: + raise ValueError("CLIP snapshot manifest identity is invalid") + expected = {row["path"]: row for row in manifest.get("files", [])} + if len(expected) != 8 or any(Path(name).name != name for name in expected): + raise ValueError("CLIP snapshot manifest allowlist is invalid") + if not root.is_dir(): + raise ValueError("offline CLIP snapshot directory is missing") + actual = {entry.name for entry in root.iterdir()} + if actual != set(expected): + raise ValueError(f"CLIP snapshot entries differ from exact allowlist: {sorted(actual ^ set(expected))}") + cache_root = root.parent.parent if root.parent.name == "snapshots" else None + verified = [] + for name in sorted(expected): + path = root / name + if not path.is_file(): + raise ValueError(f"CLIP snapshot file is missing: {name}") + if path.is_symlink(): + resolved = path.resolve(strict=True) + if cache_root is None or cache_root.resolve() not in resolved.parents: + raise ValueError(f"CLIP snapshot symlink escapes declared Hugging Face cache: {name}") + row = expected[name] + size, sha256 = path.stat().st_size, dataset.sha256(path) + if size != row["size"] or sha256 != row["sha256"]: + raise ValueError(f"CLIP snapshot bytes differ from manifest: {name}") + verified.append({"path": name, "size": size, "sha256": sha256}) + return {"repository": manifest["repository"], "revision": manifest["revision"], + "manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(), "files": verified} + + +def _runtime_probe() -> str: + return ( + "import importlib.metadata as m,json,os,platform,struct,sys,sysconfig;" + "import ctypes,numpy,PIL,torch,transformers;" + "c=ctypes.CDLL('libcuda.so.1');v=ctypes.c_int();" + "assert c.cuInit(0)==0 and c.cuDriverGetVersion(ctypes.byref(v))==0;" + "d=[(x.metadata.get('Name'),x.version) for x in m.distributions()];" + "f=sys.flags;print(json.dumps({'implementation':platform.python_implementation()," + "'python_version':platform.python_version(),'python_compiler':platform.python_compiler().strip()," + "'python_build':platform.python_build()," + "'cache_tag':sys.implementation.cache_tag,'soabi':sysconfig.get_config_var('SOABI')," + "'system':platform.system(),'machine':platform.machine(),'bits':struct.calcsize('P')*8," + "'lexical_executable':sys.executable,'resolved_executable':os.path.realpath(sys.executable)," + "'prefix':sys.prefix,'base_prefix':sys.base_prefix,'isolated':f.isolated," + "'ignore_environment':f.ignore_environment,'no_user_site':f.no_user_site," + "'safe_path':f.safe_path,'distributions':d,'torch_version':torch.__version__," + "'torch_cuda':torch.version.cuda,'cuda_available':torch.cuda.is_available()," + "'gpu':torch.cuda.get_device_name(0),'cuda_driver_api_version':v.value," + "'transformers_version':transformers.__version__,'numpy_version':numpy.__version__," + "'pillow_version':PIL.__version__},sort_keys=True))" + ) + + +def preflight_scorer_runtime(lock: Path = SCORING_LOCK) -> dict: + """Local reproducibility preflight for the lexical runner venv; not host attestation.""" + lexical = Path(__import__("sys").executable).expanduser().absolute() + process = subprocess.run([str(lexical), "-I", "-c", _runtime_probe()], capture_output=True, + text=True, cwd=CANONICAL_SCORER.parent, env=_isolated_env()) + if process.returncode: + raise ValueError(f"scorer runtime preflight failed: {process.stderr.strip()}") + try: + identity = json.loads(process.stdout) + except json.JSONDecodeError as exc: + raise ValueError("scorer runtime preflight was not valid JSON") from exc + if not isinstance(identity, dict) or any(identity.get(key) != value + for key, value in _EXPECTED_RUNTIME.items()): + raise ValueError("runner is not the exact historical CPython 3.13.15 scoring runtime") + prefix = Path(identity.get("prefix", "")).absolute() + if (identity.get("base_prefix") == identity.get("prefix") + or lexical.parent != prefix / "bin" + or Path(identity.get("lexical_executable", "")).absolute() != lexical + or any(identity.get(key) != expected for key, expected in + {"isolated": 1, "ignore_environment": 1, "no_user_site": 1, + "safe_path": True, "cuda_available": True}.items())): + raise ValueError("scorer must run from the lexical isolated CUDA-enabled publication venv") + installed: dict[str, str] = {} + for entry in identity.get("distributions", []): + if not isinstance(entry, list) or len(entry) != 2 or not all(isinstance(x, str) for x in entry): + raise ValueError("scorer distribution preflight is malformed") + name = _normalized_distribution(entry[0]) + if name in installed: + raise ValueError(f"duplicate installed scoring distribution: {name}") + installed[name] = entry[1] + expected = locked_distributions(lock) + mismatches = {name: {"expected": version, "installed": installed.get(name)} + for name, version in expected.items() if installed.get(name) != version} + extras = sorted(set(installed) - set(expected) - _ALLOWED_BOOTSTRAP_DISTRIBUTIONS) + if mismatches or extras: + raise ValueError(f"scorer distributions do not exactly match scoring lock: " + f"mismatches={mismatches}, extras={extras}") + return {"trust_boundary": "local reproducibility preflight; not remote or hostile-host attestation", + "remote_attestation": False, **{key: identity[key] for key in identity if key != "distributions"}, + "executable_sha256": dataset.sha256(lexical), + "scoring_lock_sha256": dataset.sha256(Path(lock)), "locked_distributions": expected, + "excluded_bootstrap_distributions": {name: installed[name] for name in sorted(installed) + if name in _ALLOWED_BOOTSTRAP_DISTRIBUTIONS}, + "invocation": {"argv_prefix": [str(lexical), "-I"], "isolated": True}} + + +def scorer_sha256() -> str: + if CANONICAL_SCORER.is_symlink() or not CANONICAL_SCORER.is_file(): + raise ValueError("canonical score_pair.py must be a regular repository file") + return dataset.sha256(CANONICAL_SCORER) + + +def scoring_provenance(runtime_preflight: dict, clip_snapshot: dict) -> dict: + return {"dataset_revision": dataset.DATASET_REVISION, + "evaluator_repository": protocol.EVALUATOR_REPOSITORY, + "evaluator_revision": protocol.EVALUATOR_REVISION, + "evaluator_sha256": protocol.EVALUATOR_SHA256, + "clip_model": protocol.CLIP_MODEL, "clip_revision": protocol.CLIP_REVISION, + "clip_snapshot": clip_snapshot, "scorer_sha256": scorer_sha256(), + "scorer_runtime": runtime_preflight, + "identity_validation": "outputs checked after local runtime and byte-manifest preflight"} + + +def verified_audit_images(task_root: Path, rounds: int, bpy_version: str) -> dict[int, Path]: + """Admit only renders recorded after successful independent scene validation.""" + task_root = Path(task_root).absolute() + admission_path = task_root / "audit-admission.json" + if admission_path.is_symlink() or not admission_path.is_file(): + raise ValueError("standalone scoring requires audit-admission.json") + try: + rows = json.loads(admission_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError("audit admission is unreadable") from exc + if (not isinstance(rows, list) or [row.get("round") for row in rows] + != list(range(1, rounds + 1))): + raise ValueError("audit admission rounds are incomplete") + images = {} + for number, row in enumerate(rows, 1): + image = task_root / f"audit{number:02}" / "verified.png" + metadata_path = image.parent / "metadata.json" + blend = task_root / "work" / f"iteration{number:02}.blend" + if metadata_path.is_symlink() or not metadata_path.is_file(): + raise ValueError(f"audit metadata is missing for round {number}") + try: + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"audit metadata is unreadable for round {number}") from exc + runtime = metadata.get("runtime", {}) + evidence = metadata.get("evidence", {}) + devices = evidence.get("devices", []) if isinstance(evidence, dict) else [] + if (image.is_symlink() or blend.is_symlink() or not image.is_file() or not blend.is_file() + or row.get("meaningful_change_validated") is not True + or row.get("image") != str(image) or row.get("source") != str(blend) + or row.get("image_sha256") != dataset.sha256(image) + or row.get("source_sha256") != dataset.sha256(blend) + or metadata.get("png") != str(image) or metadata.get("source") != str(blend) + or metadata.get("png_sha256") != row.get("image_sha256") + or metadata.get("source_sha256") != row.get("source_sha256") + or not isinstance(metadata.get("audit"), dict) + or row.get("audit_sha256") != protocol.digest(metadata["audit"]) + or not isinstance(runtime, dict) or runtime.get("returncode") != 0 + or not isinstance(evidence, dict) or evidence.get("blender_version") != bpy_version + or evidence.get("compute_device_type") != "CUDA" + or evidence.get("scene_device") != "GPU" + or evidence.get("engine") != "CYCLES" + or not any(isinstance(device, dict) and device.get("type") == "CUDA" + and device.get("use") is True for device in devices) + or any(isinstance(device, dict) and device.get("use") is True + and device.get("type") != "CUDA" for device in devices)): + raise ValueError(f"audit admission bytes are invalid for round {number}") + images[number] = image + return images + + +def _normalized(row: dict) -> dict: + result = dict(row) + if result.get("status") == "complete": + cosine = float(result["clip_image_cosine_similarity"]) + loss = float(result["photometric_loss"]) + if not math.isfinite(cosine) or not math.isfinite(loss): + raise ValueError("score metrics must be finite") + result["clip_image_cosine_similarity"] = cosine + result["nclip"] = 1.0 - cosine + result["photometric_loss"] = loss + return result + + +def aggregate(rows: Iterable[dict], expected_tasks: Iterable[str], expected_rounds: int | None = None, + provenance: dict | None = None) -> dict: + grouped: dict[str, list[dict]] = defaultdict(list) + for raw in rows: + row = _normalized(raw) + grouped[row["task"]].append(row) + tasks = [] + for task in expected_tasks: + task_rows = sorted(grouped.get(task, []), key=lambda row: row.get("round", 0)) + successful = [row for row in task_rows if row.get("status") == "complete"] + failed = [row for row in task_rows if row.get("status") != "complete"] + if not task_rows: + status = "missing" + elif expected_rounds is not None and len(successful) != expected_rounds: + status = "failed" + failed.append({"task": task, "status": "failed", + "error": f"expected {expected_rounds} completed rounds; found {len(successful)}"}) + elif failed: + status = "failed" + else: + status = "complete" + # Explicit round secondary key makes equal N-CLIP choose the earliest round. + best = min(successful, key=lambda row: (row["nclip"], row["round"])) if successful else None + tasks.append({"task": task, "status": status, "completed_rounds": len(successful), + "failed_rounds": len(failed), "best_round": best, "rounds": task_rows, + "failures": failed}) + completed = sum(task["status"] == "complete" for task in tasks) + completed_best = [task["best_round"] for task in tasks + if task["status"] == "complete" and task["best_round"] is not None] + mean_nclip = (sum(row["nclip"] for row in completed_best) / len(completed_best) + if completed_best else None) + mean_pl = (sum(row["photometric_loss"] for row in completed_best) / len(completed_best) + if completed_best else None) + result = { + "schema_version": 1, + "metric_definition": { + "clip_image_cosine_similarity": "cosine similarity between CLIP image embeddings", + "nclip": "1 - CLIP image cosine similarity", + "photometric_loss": "pinned VIGA evaluator photometric_loss output", + "best_round": "minimum N-CLIP; earliest round wins exact ties", + }, + "tasks": tasks, + "summary": {"expected_tasks": len(tasks), "completed_tasks": completed, + "failed_tasks": len(tasks) - completed, + "completion_rate": completed / len(tasks) if tasks else 0.0, + "mean_best_nclip_completed_tasks": mean_nclip, + "mean_best_photometric_loss_completed_tasks": mean_pl}, + } + if provenance is not None: + result["provenance"] = provenance + return result + + +def write_outputs(result: dict, output: Path) -> tuple[Path, Path]: + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + json_path = output / "aggregate.json" + csv_path = output / "aggregate.csv" + temporary = json_path.with_suffix(".json.tmp") + temporary.write_text(json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8") + temporary.replace(json_path) + fields = ["task", "status", "completed_rounds", "failed_rounds", "best_round", + "best_clip_image_cosine_similarity", "best_nclip", "best_photometric_loss", "error", + "dataset_revision", "evaluator_repository", "evaluator_revision", + "evaluator_sha256", "clip_model", "clip_revision", "scorer_sha256", + "identity_validation", "protocol_sha256", "git_revision", + "scoring_lock_sha256", "model", "model_revision", "codex_version", + "bpy_version", "scorer_python_version", "scorer_python_implementation", + "scorer_python_sha256"] + with csv_path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=fields) + writer.writeheader() + provenance = result.get("provenance", {}) + source = provenance.get("source_identity", {}) + versions = provenance.get("versions", {}) + scorer_runtime = provenance.get("scorer_runtime", {}) + flat_provenance = {key: provenance.get(key, "") for key in fields[9:]} + flat_provenance.update({ + "git_revision": source.get("git_revision", ""), + "scoring_lock_sha256": source.get("scoring_lock_sha256", ""), + "codex_version": versions.get("codex", ""), + "bpy_version": versions.get("bpy", ""), + "scorer_python_version": scorer_runtime.get("python_version", ""), + "scorer_python_implementation": scorer_runtime.get("implementation", ""), + "scorer_python_sha256": scorer_runtime.get("executable_sha256", ""), + }) + for task in result["tasks"]: + best = task["best_round"] or {} + errors = "; ".join(str(row.get("error", "unknown failure")) for row in task["failures"]) + writer.writerow({"task": task["task"], "status": task["status"], + "completed_rounds": task["completed_rounds"], "failed_rounds": task["failed_rounds"], + "best_round": best.get("round", ""), + "best_clip_image_cosine_similarity": best.get("clip_image_cosine_similarity", ""), + "best_nclip": best.get("nclip", ""), + "best_photometric_loss": best.get("photometric_loss", ""), "error": errors, + **flat_provenance}) + return json_path, csv_path + + +def score_pair(image: Path, target: Path, clip_snapshot: Path, + runtime_preflight: dict | None = None) -> dict: + """Score exact image bytes with the canonical scorer under this lexical venv.""" + runtime_preflight = runtime_preflight or preflight_scorer_runtime() + snapshot_before = verify_clip_snapshot(clip_snapshot) + image, target = Path(image).absolute(), Path(target).absolute() + image_sha256, target_sha256 = dataset.sha256(image), dataset.sha256(target) + lexical = str(Path(__import__("sys").executable).expanduser().absolute()) + argv = [lexical, "-I", str(CANONICAL_SCORER), str(image), str(target)] + process = subprocess.run(argv, capture_output=True, text=True, cwd=CANONICAL_SCORER.parent, + env=_isolated_env(clip_snapshot)) + snapshot_after = verify_clip_snapshot(clip_snapshot) + if snapshot_after != snapshot_before: + raise ValueError("CLIP snapshot changed during scoring") + if process.returncode: + raise RuntimeError(f"scorer exited {process.returncode}: {process.stderr.strip()}") + try: + payload = json.loads(process.stdout) + except json.JSONDecodeError as exc: + raise ValueError("scorer must emit one JSON object") from exc + identity = {"clip_model": protocol.CLIP_MODEL, "clip_revision": protocol.CLIP_REVISION, + "viga_revision": protocol.EVALUATOR_REVISION, + "evaluator_sha256": protocol.EVALUATOR_SHA256, + "torch_version": _EXPECTED_RUNTIME["torch_version"], + "transformers_version": _EXPECTED_RUNTIME["transformers_version"], + "numpy_version": _EXPECTED_RUNTIME["numpy_version"], + "pillow_version": _EXPECTED_RUNTIME["pillow_version"]} + if not isinstance(payload, dict) or any(payload.get(key) != value for key, value in identity.items()): + raise ValueError("scorer evaluator/model/runtime identity does not match pinned protocol") + if not str(payload.get("device", "")).startswith("cuda"): + raise ValueError("scorer identity must report a CUDA device") + try: + cosine = payload.get("clip_image_cosine_similarity", payload.get("clip_similarity")) + loss = payload.get("photometric_loss", payload.get("raw_pl")) + return {"clip_image_cosine_similarity": float(cosine), "photometric_loss": float(loss), + "candidate_image_sha256": image_sha256, "target_image_sha256": target_sha256, + "scorer_payload": payload} + except (ValueError, TypeError) as exc: + raise ValueError("scorer must emit JSON with CLIP image cosine similarity and photometric loss") from exc + + +def score_saved_run(run_root: Path, dataset_root: Path, tasks: Iterable[str], rounds: int, + clip_snapshot: Path) -> dict: + tasks = list(tasks) + manifest = protocol.load_run_manifest(Path(run_root), tasks=tasks, rounds=rounds) + runtime_preflight = preflight_scorer_runtime() + snapshot = verify_clip_snapshot(clip_snapshot) + for task in tasks: + dataset.verify(task, Path(dataset_root)) + rows = [] + for task in tasks: + task_root = Path(run_root) / task.replace("/", "-") + target = Path(dataset_root) / task / "target.png" + try: + admitted_images = verified_audit_images( + task_root, rounds, manifest["protocol"]["versions"]["bpy"]) + except Exception as exc: + rows.extend({"task": task, "round": number, "status": "failed", + "error": f"{type(exc).__name__}: {exc}"} + for number in range(1, rounds + 1)) + continue + for number in range(1, rounds + 1): + # Only score the independently reopened/GPU-rendered audit image. The + # model-written checkpoint PNG is generation evidence, not an admitted + # standalone publication score input. + image = admitted_images[number] + row = {"task": task, "round": number} + try: + if not image.is_file(): + raise FileNotFoundError(image) + row.update(score_pair(image, target, clip_snapshot, runtime_preflight), + status="complete", image=str(image)) + except Exception as exc: + row.update(status="failed", error=f"{type(exc).__name__}: {exc}", image=str(image)) + rows.append(row) + generation = manifest["protocol"] + provenance = scoring_provenance(runtime_preflight, snapshot) + provenance.update({"protocol_sha256": manifest["protocol_sha256"], + "source_identity": generation["source_identity"], "model": generation["model"], + "model_revision": generation["model_revision"], "versions": generation["versions"]}) + return aggregate(rows, tasks, expected_rounds=rounds, provenance=provenance) diff --git a/benchmarks/blenderbench_direct/suite.py b/benchmarks/blenderbench_direct/suite.py new file mode 100644 index 0000000..51e222b --- /dev/null +++ b/benchmarks/blenderbench_direct/suite.py @@ -0,0 +1,303 @@ +"""Sequential direct-model generation followed by independent audit and scoring.""" +from __future__ import annotations + +from dataclasses import asdict +import json +import os +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace +import uuid + +from . import dataset, protocol, scoring +from tools import direct_codex as direct +from tools import direct_runtime as runtime + + +def generation_prompt(task_description: str, work: Path, rounds: int) -> str: + tools = ", ".join(protocol.TOOLS) + return f"""Solve this BlenderBench task using direct model reasoning and only the four Blender MCP tools. +Task: {task_description} +Initial scene: {work / 'initial.blend'} +All scene properties may be edited, including geometry, materials, lights, objects, and cameras. +Perform exactly {rounds} meaningful edit, render, and visual-inspection rounds sequentially. Each round must +change the saved scene meaningfully and continue from the preceding saved checkpoint. Inspect generated +images only through observable Blender MCP responses. Call get_render_as_image_for_cli after each saved round +and visually inspect the returned image content; Codex's built-in view_image is disabled. No CLIP/scoring, +score feedback, VLM judge, benchmark controller, goal_code, +solution code, goal scene, or oracle access. Do not read evaluator, audit, or post-generation result files. +Use only {tools} for scene operations; no shell scene operations. The standalone bpy backend opens the +specified blend file afresh for each call, so save each edit before the call returns. Use GPU-only Cycles: +enable CUDA devices, disable CPU, and never silently fall back to CPU. Save the scene before transient PNG +filepath/encoding changes, and do not save those output overrides. Never modify initial.blend, dataset +files, or target.png. Save absolute paths {work / 'iteration01.blend'} and {work / 'iteration01.png'} +through {work / f'iteration{rounds:02}.blend'} and {work / f'iteration{rounds:02}.png'}. +For each edit call, set blend_file to initial.blend for round 1 or the preceding iteration blend thereafter, +and set expected_output_blend to that round's new iterationNN.blend so the MCP server attests fresh changed bytes. +Append rounds.jsonl via MCP with one JSON object per completed round containing round, absolute blend, +absolute png, and a nonempty rationale describing the visual hypothesis and actual edit. Stop only after +all {rounds} sequential saved rounds and ledger rows exist.""" + + +def _json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8") + temporary.replace(path) + + +def _bpy_version(config) -> str: + code = "import bpy; print(bpy.app.version_string)" + return subprocess.check_output([str(config.bpy_python), "-c", code], text=True, + stderr=subprocess.STDOUT).strip().splitlines()[-1] + + +def _versions(config) -> dict: + codex = protocol.resolved_version(config.codex_binary, "--version") + if codex != protocol.CODEX_VERSION or config.codex_version != protocol.CODEX_VERSION: + raise ValueError(f"Codex version {codex!r} does not equal frozen {protocol.CODEX_VERSION!r}") + bpy = _bpy_version(config) + if not bpy.startswith(config.bpy_version): + raise ValueError(f"bpy version {bpy!r} does not match pin {config.bpy_version!r}") + scorer_runtime = scoring.preflight_scorer_runtime() + clip_snapshot = scoring.verify_clip_snapshot(config.clip_snapshot) + return {"codex": codex, "codex_source_revision": protocol.CODEX_SOURCE_REVISION, + "bpy": bpy, "scorer_python": "Python " + scorer_runtime["python_version"], + "scorer_script_sha256": scoring.scorer_sha256(), + "scorer_runtime": scorer_runtime, "clip_snapshot": clip_snapshot} + + +def _preflight(config, mcp_config: dict, work: Path, evidence: Path) -> None: + args = SimpleNamespace(codex=str(config.codex_binary), model=config.model, workdir=work) + nonce = uuid.uuid4().hex + code = ("import bpy,sys; result={'preflight':" + repr(nonce) + + ", 'python':sys.executable, 'version':bpy.app.version_string}") + prompt = (f"Call only execute_blender_code_for_cli once with blend_file={work / 'initial.blend'} " + f"and code={code}. Do not mutate/save. No shell or other tools.") + events = direct.invoke(args, mcp_config, evidence, "preflight", prompt) + direct.verify_preflight(events, nonce, str(config.bpy_python), config.bpy_version) + + +def verify_retained_generation(task_root: Path, work: Path, rounds: int) -> None: + """Revalidate the retained generation event stream before recovery admission.""" + path = Path(task_root) / "codex" / "run.jsonl" + if path.is_symlink() or not path.is_file(): + raise ValueError("recovery requires regular retained codex/run.jsonl evidence") + try: + events = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + except (OSError, json.JSONDecodeError) as exc: + raise ValueError("retained generation event evidence is unreadable") from exc + direct.verify_generation_events(events, rounds=rounds, workdir=work) + + +def _task(config, task: str, mcp_config: dict, scorer_runtime: dict) -> list[dict]: + record = dataset.load_manifest()["tasks"][task] + source = config.dataset_root / task + verification = dataset.verify(task, config.dataset_root) + task_root = config.output_root / task.replace("/", "-") + task_root.mkdir(parents=True, exist_ok=False) + _json(task_root / "input-verification.json", verification) + events = task_root / "events.jsonl" + policy = {key: record[key] for key in ("reviewed", "review_basis", "start_sha256", + "task_description", "initialization")} + policy.update(task=task, pin_verified=True, stop_file=str(config.output_root / "STOP")) + runtime.emit(events, "initializing", task=task) + initial = runtime.initialize(source, policy, config.bpy_python, config.bpy_version, + task_root / "work", events) + work = task_root / "work" + initialized = Path(initial["blend"]) + baseline_path = work / "initial.blend" + initialized.rename(baseline_path) + initial_hash = runtime.sha(baseline_path) + evidence = task_root / "codex" + evidence.mkdir(mode=0o700) + _json(evidence / "mcp-config.redacted.json", protocol.redact(mcp_config)) + direct.verify_generation_workdir(work) + _preflight(config, mcp_config, work, evidence) + runtime.emit(events, "generation_started", task=task) + args = SimpleNamespace(codex=str(config.codex_binary), model=config.model, workdir=work) + generation_events = direct.invoke(args, mcp_config, evidence, "run", + generation_prompt(record["task_description"], work, config.rounds), source / "target.png") + direct.verify_generation_events(generation_events, rounds=config.rounds, workdir=work) + runtime.emit(events, "generation_finished", task=task) + if runtime.sha(baseline_path) != initial_hash: + raise ValueError("generation modified initial.blend") + dataset.verify(task, config.dataset_root) + artifacts = direct.verify_checkpoints(work, config.rounds, policy=policy) + _json(task_root / "artifact-check.json", artifacts) + + previous = initial + seen = {json.dumps(initial["audit"], sort_keys=True, allow_nan=False)} + rows = [] + admissions = [] + for number in range(1, config.rounds + 1): + audit = runtime.inspect_render(work / f"iteration{number:02}.blend", policy, + config.bpy_python, config.bpy_version, task_root / f"audit{number:02}", events) + direct.validate_edit(previous, audit, policy) + state = json.dumps(audit["audit"], sort_keys=True, allow_nan=False) + if state in seen: + raise ValueError(f"round {number} repeats an earlier complete scene state") + seen.add(state); previous = audit + admissions.append({"round": number, "source": audit["source"], + "source_sha256": audit["source_sha256"], "image": audit["png"], + "image_sha256": audit["png_sha256"], "audit_sha256": protocol.digest(audit["audit"]), + "meaningful_change_validated": True}) + _json(task_root / "audit-admission.json", admissions) + try: + metrics = scoring.score_pair(Path(audit["png"]), source / "target.png", + config.clip_snapshot, scorer_runtime) + row = {"task": task, "round": number, "status": "complete", **metrics, + "blend": audit["blend"], "blend_sha256": audit["blend_sha256"], + "image": audit["png"], "image_sha256": audit["png_sha256"]} + except Exception as exc: + row = {"task": task, "round": number, "status": "failed", + "error": f"{type(exc).__name__}: {exc}", "blend": audit["blend"], + "image": audit["png"]} + rows.append(row) + _json(task_root / "scores.json", rows) + return rows + + +def recover_suite(args) -> int: + """Reopen, audit, render, and score retained rounds; never call generation.""" + tasks = dataset.select_tasks(args.tasks) + for name in ("bpy_python",): + if not getattr(args, name).is_file(): + raise FileNotFoundError(getattr(args, name)) + manifest = protocol.load_run_manifest(args.source_root, tasks=tasks, rounds=args.rounds) + scorer_runtime = scoring.preflight_scorer_runtime() + clip_snapshot = scoring.verify_clip_snapshot(args.clip_snapshot) + score_provenance = scoring.scoring_provenance(scorer_runtime, clip_snapshot) + score_provenance.update({"protocol_sha256": manifest["protocol_sha256"], + "source_identity": manifest["protocol"]["source_identity"], + "model": manifest["protocol"]["model"], + "model_revision": manifest["protocol"]["model_revision"], + "versions": manifest["protocol"]["versions"]}) + actual_bpy = _bpy_version(args) + generation_bpy = manifest["protocol"]["versions"].get("bpy") + if actual_bpy != generation_bpy: + raise ValueError(f"recovery bpy version {actual_bpy!r} differs from generation manifest {generation_bpy!r}") + if not actual_bpy.startswith(args.bpy_version): + raise ValueError(f"bpy version {actual_bpy!r} does not match pin {args.bpy_version!r}") + if args.output_root.exists(): + raise FileExistsError("fresh --output-root required for recovery evidence") + args.output_root.mkdir(parents=True, exist_ok=False) + all_rows = [] + for task in tasks: + record = dataset.load_manifest()["tasks"][task] + dataset.verify(task, args.dataset_root) + source_task = args.source_root / task.replace("/", "-") + work = source_task / "work" + policy = {key: record[key] for key in ("reviewed", "review_basis", "start_sha256", + "task_description", "initialization")} + policy.update(task=task, pin_verified=True, stop_file=str(args.output_root / "STOP")) + verify_retained_generation(source_task, work, args.rounds) + direct.verify_checkpoints(work, args.rounds, policy=policy) + baseline = json.loads((work / "metadata.json").read_text(encoding="utf-8")) + initial_path = work / "initial.blend" + initial_hash = runtime.sha(initial_path) + if initial_hash != baseline["blend_sha256"]: + raise ValueError(f"retained initial hash mismatch for {task}") + previous = baseline + seen = {json.dumps(baseline["audit"], sort_keys=True, allow_nan=False)} + rows = [] + admissions = [] + task_out = args.output_root / task.replace("/", "-") + task_out.mkdir(parents=True, exist_ok=False) + for number in range(1, args.rounds + 1): + audit = runtime.inspect_render(work / f"iteration{number:02}.blend", policy, + args.bpy_python, args.bpy_version, task_out / f"audit{number:02}", + task_out / "events.jsonl") + direct.validate_edit(previous, audit, policy) + state = json.dumps(audit["audit"], sort_keys=True, allow_nan=False) + if state in seen: + raise ValueError(f"round {number} repeats an earlier scene state") + seen.add(state); previous = audit + admissions.append({"round": number, "source": audit["source"], + "source_sha256": audit["source_sha256"], "image": audit["png"], + "image_sha256": audit["png_sha256"], "audit_sha256": protocol.digest(audit["audit"]), + "meaningful_change_validated": True}) + _json(task_out / "audit-admission.json", admissions) + try: + metrics = scoring.score_pair(Path(audit["png"]), + args.dataset_root / task / "target.png", args.clip_snapshot, scorer_runtime) + row = {"task": task, "round": number, "status": "complete", **metrics, + "blend": audit["blend"], "blend_sha256": audit["blend_sha256"], + "image": audit["png"], "image_sha256": audit["png_sha256"]} + except Exception as exc: + row = {"task": task, "round": number, "status": "failed", + "error": f"{type(exc).__name__}: {exc}"} + rows.append(row); all_rows.append(row) + _json(task_out / "scores.json", rows) + if runtime.sha(initial_path) != initial_hash: + raise ValueError(f"recovery modified retained initial scene for {task}") + result = scoring.aggregate(all_rows, tasks, expected_rounds=args.rounds, + provenance=score_provenance) + scoring.write_outputs(result, args.output_root / "scoring") + _json(args.output_root / "recovery-manifest.json", {"schema_version": 1, + "source_root": str(args.source_root), "regenerated": False, "tasks": tasks, + "bpy_version": actual_bpy, **score_provenance, + "dataset_revision": dataset.DATASET_REVISION, + "evaluator_revision": protocol.EVALUATOR_REVISION, + "evaluator_sha256": protocol.EVALUATOR_SHA256}) + return int(result["summary"]["failed_tasks"] != 0) + + +def run_suite(config, *, approved: bool) -> int: + if not approved: + raise ValueError("--approve-blender-tools is required for arbitrary MCP Python execution") + if config.output_root.exists(): + raise FileExistsError("fresh --output-root required; automatic resume/regeneration is forbidden") + source_identity = protocol.collect_source_identity() + if source_identity["git_dirty"]: + raise ValueError("publication generation requires a clean Git source tree") + raw_mcp = protocol.read_mcp_config(config.mcp_toml) + mcp_config = direct.prepare_config(raw_mcp, approved=True) + configured_bpy = mcp_config["mcp_servers"]["blender"]["env"]["BLENDER_MCP_BPY_PYTHON"] + if Path(configured_bpy).absolute() != config.bpy_python: + raise ValueError("MCP TOML bpy interpreter differs from --bpy-python") + versions = _versions(config) + config.output_root.mkdir(parents=True, exist_ok=False) + public_config = asdict(config) + public_config = {key: ([str(item) for item in value] if key == "tasks" else str(value) + if isinstance(value, Path) else value) for key, value in public_config.items()} + public_config["mcp"] = raw_mcp + manifest = protocol.build_manifest(public_config, dataset_revision=dataset.DATASET_REVISION, + evaluator_revision=protocol.EVALUATOR_REVISION, model_revision=config.model_revision, + versions=versions, source_identity=source_identity) + manifest["dataset"] = {"source_url": dataset.DATASET_SOURCE_URL, + "attribution": dataset.DATASET_ATTRIBUTION, "license": dataset.DATASET_LICENSE} + manifest["evaluator"] = {"source_url": protocol.EVALUATOR_REPOSITORY, + "revision": protocol.EVALUATOR_REVISION, "sha256": protocol.EVALUATOR_SHA256} + protocol.write_manifest(config.output_root / "run-manifest.json", manifest) + _json(config.output_root / "process.json", {"pid": os.getpid(), "argv": sys.argv, + "protocol_sha256": manifest["protocol_sha256"]}) + all_rows = [] + states = {} + runtime.emit(config.output_root / "events.jsonl", "dispatcher_started", tasks=list(config.tasks)) + for task in config.tasks: + if (config.output_root / "STOP").exists(): + states[task] = {"status": "cancelled"}; break + try: + rows = _task(config, task, mcp_config, versions["scorer_runtime"]) + all_rows.extend(rows) + state = "complete" if all(row["status"] == "complete" for row in rows) else "failed" + states[task] = {"status": state} + except Exception as exc: + states[task] = {"status": "failed", "error": f"{type(exc).__name__}: {exc}"} + _json(config.output_root / "status.json", states) + # Every selected task is represented, including cancellation/not-started cases. + score_provenance = scoring.scoring_provenance( + versions["scorer_runtime"], versions["clip_snapshot"]) + score_provenance.update({"protocol_sha256": manifest["protocol_sha256"], + "source_identity": manifest["protocol"]["source_identity"], + "model": manifest["protocol"]["model"], + "model_revision": manifest["protocol"]["model_revision"], + "versions": versions}) + result = scoring.aggregate(all_rows, config.tasks, expected_rounds=config.rounds, + provenance=score_provenance) + scoring.write_outputs(result, config.output_root / "scoring") + runtime.emit(config.output_root / "events.jsonl", "dispatcher_finished", states=states) + return int(result["summary"]["failed_tasks"] != 0) diff --git a/benchmarks/blenderbench_direct/video.py b/benchmarks/blenderbench_direct/video.py new file mode 100644 index 0000000..8f59515 --- /dev/null +++ b/benchmarks/blenderbench_direct/video.py @@ -0,0 +1,107 @@ +"""Portable BlenderBench compilation video with metric-accurate overlays.""" +from __future__ import annotations + +import json +import math +from pathlib import Path +import shutil +import subprocess +from typing import Iterable + + +def timeline_state(elapsed: float, *, duration: float, rounds: Iterable[dict], + task_tokens: int | None) -> dict: + if not math.isfinite(elapsed) or not math.isfinite(duration) or duration <= 0: + raise ValueError("finite nonnegative elapsed and positive duration required") + selected = None + for row in sorted(rounds, key=lambda item: (item["elapsed_seconds"], item["round"])): + if row["elapsed_seconds"] <= elapsed: + selected = row + else: + break + tokens = None if task_tokens is None else round(task_tokens * min(1.0, max(0.0, elapsed / duration))) + return { + "round": selected["round"] if selected else 0, + "image": selected.get("image") if selected else None, + "clip_image_cosine_similarity": selected.get("clip_image_cosine_similarity") if selected else None, + "task_tokens_interpolated": tokens, + "elapsed_seconds": elapsed, + } + + +def overlay_labels(model: str, task: str, state: dict, rounds: int) -> str: + similarity = state.get("clip_image_cosine_similarity") + metric = ("CLIP image cosine similarity: unavailable" if similarity is None else + f"CLIP image cosine similarity: {similarity:.6f}") + tokens = state.get("task_tokens_interpolated") + token_label = ("Task-level tokens (linear interpolation): unavailable" if tokens is None else + f"Task-level tokens (linear interpolation): ~{tokens:,}") + return "\n".join((f"{model} | {task} | Round {state['round']}/{rounds}", metric, token_label)) + + +def generate(spec_path: Path, output: Path, *, fps: int = 24, seconds_per_task: float = 3.0, + ffmpeg: str | None = None) -> dict: + """Generate an H.264 compilation from an explicit, machine-readable task spec. + + Each task record requires target, initial, duration_seconds, rounds, model and + optional task_tokens. Round elapsed times are task-local. Token counts shown in + frames are explicitly linear task-level interpolation, never event-level usage. + """ + from PIL import Image, ImageDraw + + spec = json.loads(Path(spec_path).read_text(encoding="utf-8")) + tasks = spec["tasks"] + if not tasks: + raise ValueError("video spec contains no tasks") + if fps < 1 or seconds_per_task <= 0: + raise ValueError("positive fps and seconds-per-task required") + encoder = ffmpeg or shutil.which("ffmpeg") + if not encoder: + raise RuntimeError("ffmpeg not found; pass --ffmpeg") + output = Path(output) + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists(): + raise FileExistsError(output) + width, height = 1024, 600 + process = subprocess.Popen([encoder, "-hide_banner", "-loglevel", "error", "-n", + "-f", "rawvideo", "-pix_fmt", "rgb24", "-s", f"{width}x{height}", "-r", str(fps), + "-i", "-", "-an", "-c:v", "libx264", "-pix_fmt", "yuv420p", str(output)], + stdin=subprocess.PIPE) + report = {"schema_version": 1, "output": str(output), "fps": fps, + "seconds_per_task": seconds_per_task, "frames": [], + "metric_label": "CLIP image cosine similarity", + "token_label": "task-level linear interpolation"} + try: + frame_count = round(fps * seconds_per_task) + for task in tasks: + with Image.open(task["target"]) as image: + target = image.convert("RGB").resize((512, 512)) + initial = Path(task["initial"]) + for index in range(frame_count): + elapsed = task["duration_seconds"] * index / max(1, frame_count - 1) + state = timeline_state(elapsed, duration=task["duration_seconds"], + rounds=task["rounds"], task_tokens=task.get("task_tokens")) + source = Path(state["image"]) if state["image"] else initial + with Image.open(source) as image: + candidate = image.convert("RGB").resize((512, 512)) + frame = Image.new("RGB", (width, height), "black") + frame.paste(target, (0, 88)); frame.paste(candidate, (512, 88)) + draw = ImageDraw.Draw(frame) + draw.text((12, 12), "TARGET (static)", fill="white") + labels = overlay_labels(task["model"], task["task"], state, len(task["rounds"])) + draw.multiline_text((524, 8), labels, fill="white", spacing=3) + if process.stdin is None: + raise RuntimeError("ffmpeg stdin unavailable") + process.stdin.write(frame.tobytes()) + report["frames"].append({"task": task["task"], "task_frame": index, **state}) + if process.stdin is not None: + process.stdin.close() + if process.wait() != 0: + raise RuntimeError(f"ffmpeg exited {process.returncode}") + except BaseException: + process.kill(); process.wait() + raise + report_path = output.with_suffix(output.suffix + ".json") + report_path.write_text(json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8") + return report diff --git a/benchmarks/token_efficiency/codex_runner.py b/benchmarks/token_efficiency/codex_runner.py index 6152fa8..22e9b14 100644 --- a/benchmarks/token_efficiency/codex_runner.py +++ b/benchmarks/token_efficiency/codex_runner.py @@ -458,7 +458,7 @@ def main(argv=None): parser.add_argument("--condition", choices=CONDITIONS, default="enhanced_bpy") parser.add_argument("--server-python", default="/REQUIRES_REVIEW/server-python") parser.add_argument("--runtime", default="/REQUIRES_REVIEW/runtime") - parser.add_argument("--model", default="REQUIRES_PARENT_MODEL_SELECTION") + parser.add_argument("--model", required=True) parser.add_argument("--fixture-stream") args = parser.parse_args(argv) if args.mode == "dry-run": diff --git a/benchmarks/token_efficiency/creative_smoke.py b/benchmarks/token_efficiency/creative_smoke.py index e50c31d..162e550 100644 --- a/benchmarks/token_efficiency/creative_smoke.py +++ b/benchmarks/token_efficiency/creative_smoke.py @@ -264,7 +264,7 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", default="/tmp/voxel-video-mcp-trial") parser.add_argument("--condition", choices=["original", "enhanced", "both"], default="both") - parser.add_argument("--model", default="gpt-6-astra") + parser.add_argument("--model", required=True) parser.add_argument("--seconds", type=int, default=600) parser.add_argument("--output-bytes", type=int, default=2 * 1024 * 1024) parser.add_argument("--max-tools", type=int, default=80) diff --git a/mcp/blmcp/tools/execute_blender_code.py b/mcp/blmcp/tools/execute_blender_code.py index 1a4fd83..379022c 100644 --- a/mcp/blmcp/tools/execute_blender_code.py +++ b/mcp/blmcp/tools/execute_blender_code.py @@ -6,10 +6,15 @@ __all__ = ("register",) +import base64 +import hashlib +from pathlib import Path +import tempfile + from blmcp.tools_helpers.blender_cli import run_blender_cli, synced_blend_for_cli from blmcp.tools_helpers.connection import send_code -from mcp.server.fastmcp import FastMCP # pylint: disable=import-error,no-name-in-module -from mcp.types import ToolAnnotations # pylint: disable=import-error,no-name-in-module +from mcp.server.fastmcp import FastMCP, Image # pylint: disable=import-error,no-name-in-module +from mcp.types import ImageContent, ToolAnnotations # pylint: disable=import-error,no-name-in-module def register(mcp: FastMCP) -> None: @@ -38,18 +43,114 @@ def execute_blender_code(code: str) -> dict[str, object]: destructiveHint=True, ) ) - def execute_blender_code_for_cli(blend_file: str, code: str) -> dict[str, object]: + def execute_blender_code_for_cli(blend_file: str, code: str, + expected_output_blend: str | None = None) -> dict[str, object]: """ Execute Python code in a background Blender process. Opens *blend_file* with ``blender --background`` and runs *code*. - Assign a dict to ``result`` to return data. + Assign a dict to ``result`` to return data. When *expected_output_blend* + is supplied, that path must be newly created with bytes different from + the source; the result includes server-computed checkpoint hashes. """ + output = Path(expected_output_blend).absolute() if expected_output_blend else None + source = (Path(blend_file).resolve(strict=True) if output is not None + else Path(blend_file)) + if output is not None and (output == source or output.exists() or output.is_symlink()): + raise ValueError("expected output checkpoint must be a fresh distinct path") + source_sha256 = None + if output is not None: + with source.open("rb") as stream: + source_sha256 = hashlib.file_digest(stream, "sha256").hexdigest() # LLM-generated code may return non-JSON-serializable values # (e.g. Blender objects), handled by `run_blender_cli` via `default=repr`. - with synced_blend_for_cli(blend_file) as synced_path: - value = run_blender_cli(synced_path, code, arbitrary_code=True) - assert isinstance(value, dict), ( - "Expected dict from `run_blender_cli`, got {!r}".format(type(value)) + if output is None: + with synced_blend_for_cli(str(source)) as synced_path: + value = run_blender_cli(synced_path, code, arbitrary_code=True) + else: + value = run_blender_cli(str(source), code, arbitrary_code=True) + assert isinstance(value, dict), ( + "Expected dict from `run_blender_cli`, got {!r}".format(type(value)) + ) + if "_checkpoint" in value: + raise ValueError("result uses reserved _checkpoint field") + if output is not None: + if output.is_symlink() or not output.is_file(): + raise ValueError("expected output checkpoint was not created as a regular file") + with output.open("rb") as stream: + output_sha256 = hashlib.file_digest(stream, "sha256").hexdigest() + if output_sha256 == source_sha256: + raise ValueError("expected output checkpoint does not differ from source") + if "_checkpoint" in value: + raise ValueError("result uses reserved _checkpoint field") + value["_checkpoint"] = {"path": str(output), "source": str(source), + "source_sha256": source_sha256, "output_sha256": output_sha256, + "existed_before": False} + return value + + @mcp.tool( + annotations=ToolAnnotations( + title="Render Scene as Image", + readOnlyHint=True, + ) + ) + def get_render_as_image() -> Image: + """Render the connected Blender scene and return PNG image content without saving overrides.""" + with tempfile.TemporaryDirectory(prefix="blmcp-render-") as temporary: + render_path = Path(temporary) / "render.png" + code = ( + "import bpy\n" + "_scene = bpy.context.scene\n" + "_old = (_scene.render.filepath, _scene.render.image_settings.file_format, " + "_scene.render.use_file_extension)\n" + "try:\n" + f" _scene.render.filepath = {str(render_path)!r}\n" + " _scene.render.image_settings.file_format = 'PNG'\n" + " _scene.render.use_file_extension = True\n" + " bpy.ops.render.render(write_still=True)\n" + " result = {'rendered': True}\n" + "finally:\n" + " (_scene.render.filepath, _scene.render.image_settings.file_format, " + "_scene.render.use_file_extension) = _old" + ) + response = send_code(code, strict_json=True) + if response.get("status") != "ok" or not render_path.is_file(): + raise RuntimeError("Blender did not produce the requested PNG render") + return Image(data=render_path.read_bytes(), format="png") + + @mcp.tool( + annotations=ToolAnnotations( + title="Render Saved Scene as Image for Command-Line", + readOnlyHint=True, + ) + ) + def get_render_as_image_for_cli(blend_file: str) -> ImageContent: + """Open a saved blend in a background process, render it, and return PNG image content.""" + source = Path(blend_file).resolve(strict=True) + with source.open("rb") as stream: + source_sha256 = hashlib.file_digest(stream, "sha256").hexdigest() + with tempfile.TemporaryDirectory(prefix="blmcp-render-") as temporary: + render_path = Path(temporary) / "render.png" + code = ( + "import bpy\n" + "prefs=bpy.context.preferences.addons['cycles'].preferences\n" + "prefs.compute_device_type='CUDA'\n" + "prefs.refresh_devices()\n" + "assert any(d.type=='CUDA' for d in prefs.devices)\n" + "for d in prefs.devices: d.use=(d.type=='CUDA')\n" + "s=bpy.context.scene\n" + "assert s.render.engine=='CYCLES'\n" + "s.cycles.device='GPU'\n" + f"s.render.filepath = {str(render_path)!r}\n" + "s.render.image_settings.file_format = 'PNG'\n" + "s.render.use_file_extension = True\n" + "bpy.ops.render.render(write_still=True)\n" + "result = {'rendered': True}" ) - return value + run_blender_cli(str(source), code, arbitrary_code=False) + if not render_path.is_file(): + raise RuntimeError("Blender CLI did not produce the requested PNG render") + return ImageContent(type="image", + data=base64.b64encode(render_path.read_bytes()).decode("ascii"), + mimeType="image/png", _meta={"source": str(source), + "source_sha256": source_sha256}) diff --git a/readme.md b/readme.md index e3eee26..cd24dcd 100644 --- a/readme.md +++ b/readme.md @@ -6,13 +6,22 @@ This branch packages an enhanced Blender MCP workflow for agentic Blender work. This is a modified distribution of Blender Lab's Blender MCP project. See [NOTICE.md](NOTICE.md) for provenance and [SECURITY.md](SECURITY.md) for the execution-risk model. Public `bpy.dev` website assets will live in a separate repository. +### BlenderBench result + +[![BlenderBench result](https://bpy.dev/assets/blenderbench-result.webp)](https://huggingface.co/datasets/michaelgold/blenderbench-direct-results/resolve/7c2c43be4517aee4d76d2b445578988de186970c/video/blenderbench-complete-compilation.mp4) + +The compilation covers **27 tasks / 270 rounds**. Its on-video “CLIP accuracy” label means +**CLIP image-embedding cosine similarity, not literal accuracy**. The linked immutable +H.264 video is 848×464, 134.9 seconds, and 10,161,884 bytes; SHA-256: +`04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17`. + ### What is enhanced - **Saved-file tools:** `_for_cli` tools operate on `.blend` files in fresh subprocesses without requiring a live Blender UI session. - **Backend selection:** use a Blender executable backend or a standalone `bpy` Python backend with `BLENDER_MCP_CLI_BACKEND`. - **Runtime API lookup:** agents can query exact signatures, enums, defaults, and availability from the active Blender/`bpy` runtime instead of relying only on bundled docs. - **Process isolation:** command-line executions are bounded by explicit timeouts, output caps, compact JSON framing, and source-file overwrite checks. This is crash/process-state isolation, not an OS sandbox. -- **Benchmark tooling:** deterministic token-efficiency tasks and exploratory creative comparison harnesses support repeatable original-vs-enhanced analysis. +- **Benchmark tooling:** deterministic token-efficiency tasks, exploratory creative comparison harnesses, and a [portable model-neutral full-27 BlenderBench runner](benchmarks/blenderbench_direct/README.md) with pinned inputs and post-generation scoring. ### Current publication status diff --git a/tests/test_blender_cli.py b/tests/test_blender_cli.py index dabc194..c1b76e9 100644 --- a/tests/test_blender_cli.py +++ b/tests/test_blender_cli.py @@ -280,6 +280,92 @@ def decorator(function: Callable[..., object]) -> object: arbitrary_code=True, ) + def test_cli_render_tool_returns_image_content(self) -> None: + registered: dict[str, object] = {} + + class FakeMCP: + def tool(self, **_kwargs: object) -> Callable[[Callable[..., object]], object]: + def decorator(function: Callable[..., object]) -> object: + registered[function.__name__] = function + return function + return decorator + + execute_blender_code.register(FakeMCP()) # type: ignore[arg-type] + self.assertIn("get_render_as_image_for_cli", registered) + tool = registered["get_render_as_image_for_cli"] + import tempfile + with tempfile.TemporaryDirectory() as tmp: + source = os.path.join(tmp, "scene.blend") + with open(source, "wb") as stream: + stream.write(b"blend-bytes") + render = os.path.join(tmp, "render.png") + def produce(_blend: str, _code: str, *, arbitrary_code: bool) -> dict[str, object]: + self.assertFalse(arbitrary_code) + with open(render, "wb") as stream: + stream.write(b"png-bytes") + return {"rendered": True} + temporary = mock.MagicMock() + temporary.__enter__.return_value = tmp + temporary.__exit__.return_value = False + with mock.patch.object(execute_blender_code.tempfile, "TemporaryDirectory", + return_value=temporary), \ + mock.patch.object(execute_blender_code, "synced_blend_for_cli", + return_value=contextlib.nullcontext("synced.blend")), \ + mock.patch.object(execute_blender_code, "run_blender_cli", + side_effect=produce) as run: + image = tool(source) # type: ignore[operator] + self.assertEqual(image.mimeType, "image/png") + self.assertEqual(image.data, "cG5nLWJ5dGVz") + self.assertEqual(run.call_args.args[0], source) + code = run.call_args.args[1] + self.assertIn("bpy.ops.render.render(write_still=True)", code) + self.assertIn("compute_device_type='CUDA'", code) + self.assertIn("d.use=(d.type=='CUDA')", code) + + def test_execute_code_cli_attests_fresh_checkpoint_creation(self) -> None: + registered: dict[str, object] = {} + class FakeMCP: + def tool(self, **_kwargs: object) -> Callable[[Callable[..., object]], object]: + def decorator(function: Callable[..., object]) -> object: + registered[function.__name__] = function + return function + return decorator + execute_blender_code.register(FakeMCP()) # type: ignore[arg-type] + import tempfile + with tempfile.TemporaryDirectory() as tmp: + source = os.path.join(tmp, "initial.blend") + output = os.path.join(tmp, "iteration01.blend") + with open(source, "wb") as stream: stream.write(b"initial") + def create(*_args: object, **_kwargs: object) -> dict[str, object]: + with open(output, "wb") as stream: stream.write(b"changed") + return {"saved": True} + with mock.patch.object(execute_blender_code, "synced_blend_for_cli", + return_value=contextlib.nullcontext(source)) as sync, \ + mock.patch.object(execute_blender_code, "run_blender_cli", side_effect=create): + result = registered["execute_blender_code_for_cli"]( + source, "save", expected_output_blend=output) # type: ignore[operator] + self.assertEqual(result["_checkpoint"]["path"], output) + self.assertFalse(result["_checkpoint"]["existed_before"]) + self.assertNotEqual(result["_checkpoint"]["source_sha256"], + result["_checkpoint"]["output_sha256"]) + sync.assert_not_called() + + def test_execute_code_cli_rejects_forged_checkpoint_without_attestation_argument(self) -> None: + registered: dict[str, object] = {} + class FakeMCP: + def tool(self, **_kwargs: object) -> Callable[[Callable[..., object]], object]: + def decorator(function: Callable[..., object]) -> object: + registered[function.__name__] = function + return function + return decorator + execute_blender_code.register(FakeMCP()) # type: ignore[arg-type] + with mock.patch.object(execute_blender_code, "synced_blend_for_cli", + return_value=contextlib.nullcontext("scene.blend")), \ + mock.patch.object(execute_blender_code, "run_blender_cli", + return_value={"_checkpoint": {"output_sha256": "forged"}}): + with self.assertRaisesRegex(ValueError, "reserved _checkpoint"): + registered["execute_blender_code_for_cli"]("scene.blend", "forge") # type: ignore[operator] + def test_explicit_error_marker_raises(self) -> None: with self.assertRaisesRegex(RuntimeError, "Blender error: broken"): self._run_with_completed_process( diff --git a/tests/test_blenderbench_publication.py b/tests/test_blenderbench_publication.py new file mode 100644 index 0000000..4fb0d10 --- /dev/null +++ b/tests/test_blenderbench_publication.py @@ -0,0 +1,746 @@ +"""CI-safe publication runner tests; no Blender, GPU, model, or network.""" +from __future__ import annotations + +import csv +import json +from pathlib import Path +import tempfile +import unittest + +from benchmarks.blenderbench_direct import dataset, protocol, scoring, video +from benchmarks.blenderbench_direct.runner import build_parser, resolve_config + + +class DatasetManifestTests(unittest.TestCase): + def test_task_path_components_may_not_be_symlinks(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + outside = root / "outside" + outside.mkdir() + data = root / "data" + data.mkdir() + (data / "level1").symlink_to(outside, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "^symlink forbidden in dataset"): + dataset.verify("level1/camera1", data) + + def test_verified_dataset_task_contains_only_input_assets_and_checkpoint(self): + from unittest.mock import patch + record = {name: {"size": 1, "sha256": "a" * 64} + for name in ("task.txt", "start.py", "scene.blend", "target.png")} + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + task = root / "level1/camera1" + task.mkdir(parents=True) + for name in record: + (task / name).write_bytes(b"x") + (task / "disguised_payload.py").write_text("pass") + manifest = {"tasks": {"level1/camera1": {"assets": record}}} + with patch.object(dataset, "load_manifest", return_value=manifest), \ + patch.object(dataset, "_verify", return_value={"size": 1, "sha256": "a" * 64}): + with self.assertRaisesRegex(ValueError, "unexpected path"): + dataset.verify("level1/camera1", root) + + def test_verified_dataset_rejects_contamination_outside_selected_task(self): + from unittest.mock import patch + record = {name: {"size": 1, "sha256": "a" * 64} + for name in ("task.txt", "start.py", "scene.blend", "target.png")} + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); task = root / "level1/camera1"; task.mkdir(parents=True) + for name in record: (task / name).write_bytes(b"x") + (root / "obfuscated-payload.bin").write_bytes(b"hidden code") + manifest = {"tasks": {"level1/camera1": {"assets": record}}} + with patch.object(dataset, "load_manifest", return_value=manifest), \ + patch.object(dataset, "_verify", return_value={}): + with self.assertRaisesRegex(ValueError, "dataset root"): + dataset.verify("level1/camera1", root) + + def test_successful_resumed_download_removes_superseded_partial(self): + import hashlib + import io + from unittest.mock import patch + payload = b"complete-payload" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + destination = root / "asset.bin" + retained = root / "asset.bin.retained.partial" + retained.write_bytes(payload[:5]) + class Response(io.BytesIO): + status = 206 + headers = {"Content-Range": f"bytes 5-{len(payload) - 1}/{len(payload)}"} + record = {"size": len(payload), "sha256": hashlib.sha256(payload).hexdigest()} + with patch.object(dataset.urllib.request, "urlopen", + return_value=Response(payload[5:])): + dataset._download("https://example.invalid/asset", destination, record) + self.assertEqual(destination.read_bytes(), payload) + self.assertFalse(retained.exists()) + + def test_manifest_is_pinned_complete_and_has_no_edit_policy_fields(self): + manifest = dataset.load_manifest() + self.assertEqual(manifest["revision"], dataset.DATASET_REVISION) + self.assertEqual(len(manifest["tasks"]), 27) + self.assertEqual(dataset.task_ids(), sorted(manifest["tasks"])) + forbidden = {"allowed_transforms", "allowed_camera_data", "allowed_light_data", + "allowed_shape_keys", "location_bounds", "bounds", "unsupported_blocker"} + def keys(value): + if isinstance(value, dict): + yield from value + for key, child in value.items(): + yield key + yield from keys(child) + elif isinstance(value, list): + for child in value: + yield from keys(child) + self.assertFalse(forbidden.intersection(keys(manifest))) + for task, record in manifest["tasks"].items(): + self.assertEqual(set(record["assets"]), {"task.txt", "start.py", "scene.blend", "target.png"}) + self.assertTrue(record["task_description"]) + self.assertEqual(len(record["start_sha256"]), 64) + self.assertIsInstance(record["initialization"], list) + self.assertNotIn("goal.py", json.dumps(record)) + + def test_subset_selection_defaults_to_all_and_rejects_duplicates(self): + all_tasks = dataset.task_ids() + self.assertEqual(dataset.select_tasks(None), all_tasks) + self.assertEqual(dataset.select_tasks([all_tasks[2], all_tasks[0]]), [all_tasks[2], all_tasks[0]]) + for invalid in [[], [all_tasks[0], all_tasks[0]], ["unknown/task"]]: + with self.assertRaises(ValueError): + dataset.select_tasks(invalid) + + +class RunnerConfigTests(unittest.TestCase): + def test_external_scorer_options_are_not_accepted(self): + parser = build_parser() + with self.assertRaises(SystemExit): + parser.parse_args(["score", "--run-root", "/run", "--dataset-root", "/data", + "--clip-snapshot", "/clip", "--scorer-python", "/fake", + "--scorer-python-sha256", "a" * 64, "--scorer-script", "/fake.py"]) + + def required_argv(self, root: Path) -> list[str]: + return ["run", "--dataset-root", str(root / "data"), "--mcp-toml", str(root / "mcp.toml"), + "--codex-binary", str(root / "codex"), "--codex-version", "codex-cli 0.154.0", + "--bpy-python", str(root / "bpy-python"), "--bpy-version", "5.2.1", + "--clip-snapshot", str(root / "clip-snapshot"), + "--output-root", str(root / "output"), "--model", "provider/model"] + + def test_all_runtime_paths_and_version_pins_are_explicit_and_rounds_frozen(self): + parser = build_parser() + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + args = parser.parse_args(self.required_argv(root)) + config = resolve_config(args, check_files=False) + self.assertEqual(config.rounds, 10) + self.assertEqual(config.model, "provider/model") + self.assertEqual(config.codex_version, "codex-cli 0.154.0") + self.assertEqual(config.bpy_version, "5.2.1") + for field in ("dataset_root", "mcp_toml", "codex_binary", "bpy_python", + "clip_snapshot", "output_root"): + self.assertTrue(getattr(config, field).is_absolute(), field) + with self.assertRaises(SystemExit): + parser.parse_args(self.required_argv(root) + ["--rounds", "9"]) + + def test_generated_roots_must_stay_outside_git_repository(self): + parser = build_parser() + root = Path(__file__).resolve().parents[1] + args = parser.parse_args(self.required_argv(root)) + with self.assertRaisesRegex(ValueError, "outside"): + resolve_config(args, check_files=False) + + def test_recover_and_score_are_explicit_and_never_fall_back_to_generation(self): + parser = build_parser() + recover = parser.parse_args(["recover", "--source-root", "/old", "--output-root", "/recovery", + "--dataset-root", "/data", "--bpy-python", "/bpy", "--bpy-version", "5.2.1", + "--clip-snapshot", "/clip"]) + self.assertEqual(recover.command, "recover") + self.assertEqual(recover.output_root, Path("/recovery")) + self.assertEqual(parser.parse_args(["score", "--run-root", "/run", "--dataset-root", "/data", + "--clip-snapshot", "/clip"]).command, + "score") + + def test_recovery_rejects_renderer_different_from_generation_manifest(self): + from types import SimpleNamespace + from unittest.mock import patch + from benchmarks.blenderbench_direct import suite + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bpy = root / "bpy"; bpy.write_bytes(b"binary") + args = SimpleNamespace(tasks=None, bpy_python=bpy, bpy_version="5.3.0", + source_root=root / "source", output_root=root / "recovery", + dataset_root=root / "dataset", clip_snapshot=root / "clip", rounds=10) + manifest = {"protocol_sha256": "a" * 64, "protocol": { + "source_identity": {}, "model": "m", "model_revision": None, + "versions": {"bpy": "5.2.1"}}} + with patch.object(dataset, "select_tasks", return_value=[]), \ + patch.object(protocol, "load_run_manifest", return_value=manifest), \ + patch.object(scoring, "preflight_scorer_runtime", return_value={}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={}), \ + patch.object(suite, "_bpy_version", return_value="5.3.0"): + with self.assertRaisesRegex(ValueError, "generation manifest"): + suite.recover_suite(args) + def test_recovery_revalidates_retained_generation_events(self): + from benchmarks.blenderbench_direct import suite + with tempfile.TemporaryDirectory() as tmp: + task = Path(tmp) / "level1-camera1" + work = task / "work"; work.mkdir(parents=True) + (work / "iteration01.blend").write_bytes(b"checkpoint") + codex = task / "codex"; codex.mkdir() + (codex / "run.jsonl").write_text("\n".join(map(json.dumps, [ + {"type": "item.completed", "item": {"id": "edit", "type": "mcp_tool_call", + "server": "blender", "tool": "execute_blender_code_for_cli", + "status": "completed", "error": None, "result": {}}}, + {"type": "turn.completed"}, + ]))) + with self.assertRaisesRegex(ValueError, "visual inspection"): + suite.verify_retained_generation(task, work, 1) + + +class ProtocolManifestTests(unittest.TestCase): + def test_evaluator_url_resolves_to_verified_release_repository(self): + self.assertEqual(protocol.EVALUATOR_REPOSITORY, + "https://github.com/Fugtemypt123/VIGA-release") + source = (Path(__file__).resolve().parents[1] / + "benchmarks/blenderbench_direct/score_pair.py").read_text() + self.assertIn("https://github.com/Fugtemypt123/VIGA-release/blob/", source) + self.assertIn('VIGA_REVISION = "' + protocol.EVALUATOR_REVISION + '"', source) + + def test_pinned_evaluator_bytes_are_present_and_bound_into_source_identity(self): + root = Path(__file__).resolve().parents[1] + relative = "benchmarks/blenderbench_direct/ref_based_eval.py" + evaluator = root / relative + self.assertTrue(evaluator.is_file()) + self.assertFalse(evaluator.is_symlink()) + self.assertEqual(dataset.sha256(evaluator), protocol.EVALUATOR_SHA256) + identity = protocol.collect_source_identity() + self.assertEqual(identity["implementation_sha256"][relative], protocol.EVALUATOR_SHA256) + + def test_visual_inspection_tool_bytes_are_bound_into_source_identity(self): + relative = "mcp/blmcp/tools/execute_blender_code.py" + identity = protocol.collect_source_identity() + self.assertIn(relative, identity["implementation_sha256"]) + self.assertEqual(identity["implementation_sha256"][relative], + dataset.sha256(Path(__file__).resolve().parents[1] / relative)) + + def test_protocol_hash_is_deterministic_and_runtime_provenance_is_not_hashed(self): + base = {"model": "provider/model", "rounds": 10, "mcp": {"token": "secret", "enabled_tools": list(protocol.TOOLS)}} + first = protocol.build_manifest(base, dataset_revision="dataset-rev", evaluator_revision="eval-rev", + model_revision="model-rev", versions={"codex": "1", "bpy": "2"}, + provenance={"host": "one"}) + second = protocol.build_manifest(base, dataset_revision="dataset-rev", evaluator_revision="eval-rev", + model_revision="model-rev", versions={"codex": "1", "bpy": "2"}, + provenance={"host": "two"}) + self.assertEqual(first["protocol_sha256"], second["protocol_sha256"]) + self.assertEqual(first["protocol"]["tools"], list(protocol.TOOLS)) + self.assertEqual(first["config"]["mcp"]["token"], "") + self.assertNotIn("protocol_sha256", first["protocol"]) + + def test_protocol_hash_binds_initialization_implementation_and_git_state(self): + base = {"model": "provider/model", "rounds": 10} + source = {"initialization_manifest_sha256": "a" * 64, + "implementation_sha256": {"suite.py": "b" * 64}, + "git_revision": "c" * 40, "git_dirty": False} + first = protocol.build_manifest(base, dataset_revision="dataset", evaluator_revision="eval", + model_revision=None, versions={}, source_identity=source, provenance={}) + self.assertEqual(first["protocol"]["source_identity"], source) + dirty = protocol.build_manifest(base, dataset_revision="dataset", evaluator_revision="eval", + model_revision=None, versions={}, source_identity={**source, "git_dirty": True}, provenance={}) + changed = protocol.build_manifest(base, dataset_revision="dataset", evaluator_revision="eval", + model_revision=None, versions={}, source_identity={**source, + "implementation_sha256": {"suite.py": "d" * 64}}, provenance={}) + self.assertNotEqual(first["protocol_sha256"], dirty["protocol_sha256"]) + self.assertNotEqual(first["protocol_sha256"], changed["protocol_sha256"]) + + def test_run_manifest_validation_rejects_rehashed_protocol_rule_drift(self): + import copy + source = protocol.collect_source_identity() + source["git_dirty"] = False + config = {"model": "provider/model", "model_revision": None, "rounds": 10, + "tasks": ["level1/camera1"], "bpy_version": "5.2.1"} + versions = {"codex": protocol.CODEX_VERSION, + "codex_source_revision": protocol.CODEX_SOURCE_REVISION, + "bpy": "5.2.1", "scorer_python": "Python 3.13.15"} + manifest = protocol.build_manifest(config, dataset_revision=dataset.DATASET_REVISION, + evaluator_revision=protocol.EVALUATOR_REVISION, model_revision=None, + versions=versions, source_identity=source, provenance={}) + protocol.validate_run_manifest(manifest, tasks=config["tasks"], rounds=10) + mutations = [ + ("tools", ["shell"]), + ("clip_revision", "wrong"), + ("rendering", {"engine": "Cycles", "device": "CPU", "cpu_fallback": True}), + ("generation", {**manifest["protocol"]["generation"], + "score_feedback_to_generation": True}), + ("versions", {**versions, "codex": "codex-cli 999"}), + ] + for key, value in mutations: + changed = copy.deepcopy(manifest) + changed["protocol"][key] = value + changed["protocol_sha256"] = protocol.digest(changed["protocol"]) + with self.subTest(key=key), self.assertRaisesRegex(ValueError, "frozen protocol"): + protocol.validate_run_manifest(changed, tasks=config["tasks"], rounds=10) + + def test_source_identity_binds_hash_locked_scoring_environment(self): + identity = protocol.collect_source_identity() + root = Path(__file__).resolve().parents[1] / "benchmarks/blenderbench_direct" + self.assertEqual(identity["scoring_lock_sha256"], dataset.sha256(root / "requirements-scoring.lock")) + self.assertEqual(identity["assets_manifest_sha256"], dataset.sha256(root / "assets-manifest.json")) + self.assertEqual(identity["assets_manifest"], + json.loads((root / "assets-manifest.json").read_text(encoding="utf-8"))) + self.assertEqual(identity["clip_snapshot_manifest_sha256"], + "90813610730d94e7b677552a4488a23ce3c2e2cd2ddfdd332756ea8e70db4ebb") + self.assertEqual(identity["clip_snapshot_manifest"]["revision"], protocol.CLIP_REVISION) + + def test_public_config_omits_nested_args_env_variables_and_bearer_urls(self): + config = {"mcp": {"mcp_servers": {"blender": { + "args": ["--header", "Bearer SYNTHETIC-NOT-A-SECRET"], + "env": {"GITHUB_PAT": "SYNTHETIC-PAT", "SAFE": [{"nested": "SYNTHETIC"}]}, + "variables": [{"CUSTOM_LOGIN": "https://user:SYNTHETIC@example.invalid/path"}], + "command": "/public/bin/server"}}}} + public = protocol.redact(config) + encoded = json.dumps(public) + for secret in ("SYNTHETIC-NOT-A-SECRET", "SYNTHETIC-PAT", "SYNTHETIC@example", "SYNTHETIC"): + self.assertNotIn(secret, encoded) + server = public["mcp"]["mcp_servers"]["blender"] + self.assertEqual(server["command"], "/public/bin/server") + self.assertEqual(server["args"], "") + + +class ScoringTests(unittest.TestCase): + def write_run_manifest(self, root: Path, tasks=("level1/camera1",), rounds=10) -> dict: + config = {"model": "gpt-6-astra", "model_revision": None, + "rounds": rounds, "tasks": list(tasks)} + source = protocol.collect_source_identity() + source["git_dirty"] = False + manifest = protocol.build_manifest(config, dataset_revision=dataset.DATASET_REVISION, + evaluator_revision=protocol.EVALUATOR_REVISION, model_revision=None, + versions={"codex": "codex-cli 0.154.0", "bpy": "5.2.1", + "scorer_python": "Python 3.13.15"}, provenance={}, source_identity=source) + protocol.write_manifest(root / "run-manifest.json", manifest) + return manifest + + def scorer_payload(self, **changes): + payload = {"clip_image_cosine_similarity": .5, "photometric_loss": .25, + "clip_model": protocol.CLIP_MODEL, "clip_revision": protocol.CLIP_REVISION, + "viga_revision": protocol.EVALUATOR_REVISION, + "evaluator_sha256": protocol.EVALUATOR_SHA256, "device": "cuda:0", + "torch_version": "2.8.0+cu128", "transformers_version": "4.55.4", + "numpy_version": "2.4.6", "pillow_version": "12.3.0"} + payload.update(changes) + return payload + + def test_standalone_scoring_rejects_tampered_run_manifest(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manifest = self.write_run_manifest(root) + manifest["protocol_sha256"] = "0" * 64 + protocol.write_manifest(root / "run-manifest.json", manifest) + with self.assertRaisesRegex(ValueError, "protocol hash"): + scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 10, + root / "clip") + + def test_fake_path_executable_is_never_used_and_argv_is_canonical(self): + import os, sys + from types import SimpleNamespace + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + fake = root / "python" + sentinel = root / "executed" + fake.write_text("#!/bin/sh\ntouch " + str(sentinel) + "\n") + fake.chmod(0o755) + image, target = root / "image.png", root / "target.png" + image.write_bytes(b"candidate"); target.write_bytes(b"target") + completed = SimpleNamespace(returncode=0, stdout=json.dumps(self.scorer_payload()), stderr="") + with patch.dict(os.environ, {"PATH": str(root)}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={"verified": True}), \ + patch.object(scoring.subprocess, "run", return_value=completed) as run: + result = scoring.score_pair(image, target, root / "clip", {"preflight": True}) + argv = run.call_args.args[0] + self.assertEqual(argv[:3], [str(Path(sys.executable).absolute()), "-I", + str(scoring.CANONICAL_SCORER)]) + self.assertFalse(sentinel.exists()) + self.assertEqual(result["candidate_image_sha256"], dataset.sha256(image)) + self.assertEqual(result["target_image_sha256"], dataset.sha256(target)) + + def test_clip_snapshot_manifest_rejects_extra_or_mutated_bytes(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp); root = base / "snapshot"; root.mkdir() + names = ["config.json", "merges.txt", "preprocessor_config.json", "pytorch_model.bin", + "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json", "vocab.json"] + rows = [] + for name in names: + payload = ("bytes-" + name).encode() + (root / name).write_bytes(payload) + rows.append({"path": name, "size": len(payload), + "sha256": __import__("hashlib").sha256(payload).hexdigest(), "symlink": False}) + manifest = base / "manifest.json" + manifest.write_text(json.dumps({"schema_version": 1, "repository": protocol.CLIP_MODEL, + "revision": protocol.CLIP_REVISION, "files": rows})) + with patch.object(scoring, "CLIP_SNAPSHOT_MANIFEST", manifest): + self.assertEqual(len(scoring.verify_clip_snapshot(root)["files"]), 8) + (root / "goal.py").write_text("pass") + with self.assertRaisesRegex(ValueError, "allowlist"): + scoring.verify_clip_snapshot(root) + (root / "goal.py").unlink() + (root / "tokenizer.json").write_bytes(b"mutated") + with self.assertRaisesRegex(ValueError, "bytes"): + scoring.verify_clip_snapshot(root) + + def test_score_pair_rejects_wrong_evaluator_or_clip_identity(self): + from types import SimpleNamespace + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); image = root / "image"; target = root / "target" + image.write_bytes(b"i"); target.write_bytes(b"t") + completed = SimpleNamespace(returncode=0, + stdout=json.dumps(self.scorer_payload(clip_revision="wrong")), stderr="") + with patch.object(scoring, "verify_clip_snapshot", return_value={"verified": True}), \ + patch.object(scoring.subprocess, "run", return_value=completed): + with self.assertRaisesRegex(ValueError, "identity"): + scoring.score_pair(image, target, root / "clip", {"preflight": True}) + + def test_runtime_preflight_rejects_wrong_python_and_extra_distribution(self): + import sys + from types import SimpleNamespace + from unittest.mock import patch + expected = dict(scoring._EXPECTED_RUNTIME) + expected.update({"python_build": ["main", "Clang 22.1.3"], + "lexical_executable": str(Path(sys.executable).absolute()), + "resolved_executable": str(Path(sys.executable).resolve()), + "prefix": str(Path(sys.executable).absolute().parent.parent), "base_prefix": "/base", + "isolated": 1, "ignore_environment": 1, "no_user_site": 1, "safe_path": True, + "cuda_available": True, + "distributions": [[name, version] for name, version in scoring.locked_distributions().items()]}) + wrong = {**expected, "python_version": "3.13.14"} + with patch.object(scoring.subprocess, "run", return_value=SimpleNamespace( + returncode=0, stdout=json.dumps(wrong), stderr="")): + with self.assertRaisesRegex(ValueError, "3.13.15"): + scoring.preflight_scorer_runtime() + expected["distributions"].append(["unexpected", "1"]) + with patch.object(scoring.subprocess, "run", return_value=SimpleNamespace( + returncode=0, stdout=json.dumps(expected), stderr="")): + with self.assertRaisesRegex(ValueError, "extras"): + scoring.preflight_scorer_runtime() + + def test_standalone_scoring_verifies_dataset_before_reading_renders(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.write_run_manifest(root) + with patch.object(scoring, "preflight_scorer_runtime", return_value={}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={}), \ + patch.object(dataset, "verify", side_effect=ValueError("target hash mismatch")) as verify: + with self.assertRaisesRegex(ValueError, "target hash mismatch"): + scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 10, + root / "clip") + verify.assert_called_once_with("level1/camera1", root / "dataset") + + def test_standalone_scoring_uses_independent_audit_render_not_model_png(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.write_run_manifest(root, rounds=1) + task = root / "level1-camera1" + (task / "work").mkdir(parents=True) + (task / "work/iteration01.png").write_bytes(b"model-written") + blend = task / "work/iteration01.blend" + blend.write_bytes(b"independently-audited-blend") + (task / "audit01").mkdir() + audited = task / "audit01/verified.png" + audited.write_bytes(b"independent-audit") + audit_record = {"objects": {"Cube": {"location": [1, 0, 0]}}} + (task / "audit01/metadata.json").write_text(json.dumps({ + "source": str(blend), "source_sha256": dataset.sha256(blend), + "png": str(audited), "png_sha256": dataset.sha256(audited), + "audit": audit_record, "runtime": {"returncode": 0}, + "evidence": {"blender_version": "5.2.1", "compute_device_type": "CUDA", + "scene_device": "GPU", "engine": "CYCLES", + "devices": [{"type": "CUDA", "use": True}]}})) + (task / "audit-admission.json").write_text(json.dumps([{ + "round": 1, "source": str(blend), "source_sha256": dataset.sha256(blend), + "image": str(audited), "image_sha256": dataset.sha256(audited), + "audit_sha256": protocol.digest(audit_record), + "meaningful_change_validated": True}])) + target = root / "dataset/level1/camera1/target.png" + target.parent.mkdir(parents=True); target.write_bytes(b"target") + with patch.object(scoring, "preflight_scorer_runtime", return_value={}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={}), \ + patch.object(dataset, "verify", return_value={"verified": True}), \ + patch.object(scoring, "score_pair", return_value={ + "clip_image_cosine_similarity": .5, "photometric_loss": .25}) as score: + result = scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 1, + root / "clip") + self.assertEqual(result["summary"]["completed_tasks"], 1) + self.assertEqual(score.call_args.args[0], audited) + + def test_standalone_scoring_rejects_unverified_audit_image(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.write_run_manifest(root, rounds=1) + audit = root / "level1-camera1/audit01" + audit.mkdir(parents=True) + (audit / "verified.png").write_bytes(b"unverified") + with patch.object(scoring, "preflight_scorer_runtime", return_value={}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={}), \ + patch.object(dataset, "verify", return_value={"verified": True}), \ + patch.object(scoring, "score_pair", return_value={ + "clip_image_cosine_similarity": .5, "photometric_loss": .25}) as score: + result = scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 1, + root / "clip") + self.assertEqual(result["summary"]["completed_tasks"], 0) + score.assert_not_called() + + def test_standalone_scoring_rejects_admission_without_audit_metadata(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.write_run_manifest(root, rounds=1) + task = root / "level1-camera1" + blend = task / "work/iteration01.blend" + blend.parent.mkdir(parents=True); blend.write_bytes(b"blend") + audit = task / "audit01"; audit.mkdir() + image = audit / "verified.png"; image.write_bytes(b"render") + (task / "audit-admission.json").write_text(json.dumps([{ + "round": 1, "source": str(blend), "source_sha256": dataset.sha256(blend), + "image": str(image), "image_sha256": dataset.sha256(image), + "audit_sha256": "a" * 64, "meaningful_change_validated": True}])) + target = root / "dataset/level1/camera1/target.png" + target.parent.mkdir(parents=True); target.write_bytes(b"target") + with patch.object(scoring, "preflight_scorer_runtime", return_value={}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={}), \ + patch.object(dataset, "verify", return_value={"verified": True}), \ + patch.object(scoring, "score_pair", return_value={ + "clip_image_cosine_similarity": .5, "photometric_loss": .25}) as score: + result = scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 1, + root / "clip") + self.assertEqual(result["summary"]["completed_tasks"], 0) + score.assert_not_called() + + def test_standalone_scoring_rejects_failed_or_non_cuda_audit_metadata(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.write_run_manifest(root, rounds=1) + task = root / "level1-camera1" + blend = task / "work/iteration01.blend" + blend.parent.mkdir(parents=True); blend.write_bytes(b"blend") + audit = task / "audit01"; audit.mkdir() + image = audit / "verified.png"; image.write_bytes(b"render") + audit_record = {"objects": {"Cube": {"location": [1, 0, 0]}}} + (audit / "metadata.json").write_text(json.dumps({ + "source": str(blend), "source_sha256": dataset.sha256(blend), + "png": str(image), "png_sha256": dataset.sha256(image), "audit": audit_record, + "runtime": {"returncode": 0}, "evidence": {"blender_version": "5.2.1", + "compute_device_type": "CUDA", "scene_device": "GPU", "engine": "BLENDER_EEVEE_NEXT", + "devices": [{"type": "CUDA", "use": True}]}})) + (task / "audit-admission.json").write_text(json.dumps([{ + "round": 1, "source": str(blend), "source_sha256": dataset.sha256(blend), + "image": str(image), "image_sha256": dataset.sha256(image), + "audit_sha256": protocol.digest(audit_record), + "meaningful_change_validated": True}])) + target = root / "dataset/level1/camera1/target.png" + target.parent.mkdir(parents=True); target.write_bytes(b"target") + with patch.object(scoring, "preflight_scorer_runtime", return_value={}), \ + patch.object(scoring, "verify_clip_snapshot", return_value={}), \ + patch.object(dataset, "verify", return_value={"verified": True}), \ + patch.object(scoring, "score_pair", return_value={ + "clip_image_cosine_similarity": .5, "photometric_loss": .25}) as score: + result = scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 1, + root / "clip") + self.assertEqual(result["summary"]["completed_tasks"], 0) + score.assert_not_called() + + def test_standalone_aggregate_records_dataset_evaluator_and_scorer_provenance(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); manifest = self.write_run_manifest(root, rounds=1) + runtime = {"executable_sha256": "a" * 64, "scoring_lock_sha256": + manifest["protocol"]["source_identity"]["scoring_lock_sha256"]} + snapshot = {"manifest_sha256": "b" * 64} + with patch.object(scoring, "preflight_scorer_runtime", return_value=runtime), \ + patch.object(scoring, "verify_clip_snapshot", return_value=snapshot), \ + patch.object(dataset, "verify", return_value={"verified": True}): + result = scoring.score_saved_run(root, root / "dataset", ["level1/camera1"], 1, + root / "clip") + provenance = result["provenance"] + self.assertEqual(provenance["dataset_revision"], dataset.DATASET_REVISION) + self.assertEqual(provenance["evaluator_revision"], protocol.EVALUATOR_REVISION) + self.assertEqual(provenance["evaluator_sha256"], protocol.EVALUATOR_SHA256) + self.assertEqual(provenance["clip_revision"], protocol.CLIP_REVISION) + self.assertEqual(provenance["clip_snapshot"], snapshot) + self.assertEqual(provenance["scorer_sha256"], dataset.sha256(scoring.CANONICAL_SCORER)) + self.assertEqual(provenance["protocol_sha256"], manifest["protocol_sha256"]) + self.assertEqual(provenance["source_identity"], manifest["protocol"]["source_identity"]) + self.assertEqual(provenance["model"], "gpt-6-astra") + self.assertEqual(provenance["versions"]["codex"], "codex-cli 0.154.0") + self.assertEqual(provenance["scorer_runtime"], runtime) + + def test_aggregate_uses_nclip_and_earliest_round_ties_and_records_failures(self): + rows = [ + {"task": "level1/camera1", "round": 1, "status": "complete", "clip_image_cosine_similarity": .8, "photometric_loss": .4}, + {"task": "level1/camera1", "round": 2, "status": "complete", "clip_image_cosine_similarity": .8, "photometric_loss": .2}, + {"task": "level1/camera2", "round": 1, "status": "failed", "error": "render failed"}, + ] + result = scoring.aggregate(rows, expected_tasks=["level1/camera1", "level1/camera2", "level1/camera3"]) + best = result["tasks"][0]["best_round"] + self.assertEqual(best["round"], 1) + self.assertAlmostEqual(best["nclip"], .2) + self.assertEqual(result["summary"]["completed_tasks"], 1) + self.assertEqual(result["summary"]["failed_tasks"], 2) + self.assertAlmostEqual(result["summary"]["mean_best_nclip_completed_tasks"], .2) + self.assertAlmostEqual(result["summary"]["mean_best_photometric_loss_completed_tasks"], .4) + self.assertEqual(result["tasks"][2]["status"], "missing") + + def test_csv_flattens_generation_and_scorer_runtime_provenance(self): + provenance = {"protocol_sha256": "a" * 64, "model": "gpt-6-astra", + "model_revision": None, "source_identity": {"git_revision": "b" * 40, + "scoring_lock_sha256": "c" * 64}, + "versions": {"codex": "codex-cli 0.154.0", "bpy": "5.2.1"}, + "scorer_runtime": {"executable_sha256": "d" * 64, + "implementation": "CPython", "python_version": "3.11.16"}} + result = scoring.aggregate([], ["t"], provenance=provenance) + with tempfile.TemporaryDirectory() as tmp: + _, csv_path = scoring.write_outputs(result, Path(tmp)) + with csv_path.open(newline="") as stream: + row = next(csv.DictReader(stream)) + self.assertEqual(row["protocol_sha256"], "a" * 64) + self.assertEqual(row["scoring_lock_sha256"], "c" * 64) + self.assertEqual(row["scorer_python_sha256"], "d" * 64) + self.assertEqual(row["codex_version"], "codex-cli 0.154.0") + self.assertEqual(row["bpy_version"], "5.2.1") + + def test_json_and_csv_outputs_are_machine_readable(self): + provenance = {"dataset_revision": "dataset", "evaluator_revision": "evaluator", + "evaluator_sha256": "a" * 64, "clip_revision": "clip", + "scorer_sha256": "b" * 64} + result = scoring.aggregate([{"task": "t", "round": 1, "status": "complete", + "clip_image_cosine_similarity": .75, "photometric_loss": .5}], + ["t"], provenance=provenance) + with tempfile.TemporaryDirectory() as tmp: + json_path, csv_path = scoring.write_outputs(result, Path(tmp)) + self.assertEqual(json.loads(json_path.read_text())["summary"]["completed_tasks"], 1) + with csv_path.open(newline="") as stream: + rows = list(csv.DictReader(stream)) + self.assertEqual(rows[0]["best_round"], "1") + self.assertEqual(rows[0]["best_nclip"], "0.25") + for key, value in provenance.items(): + self.assertEqual(rows[0][key], value) + + +class SuiteProtocolTests(unittest.TestCase): + def test_ci_uses_immutable_actions_python_and_hashed_scoring_lock(self): + import re + root = Path(__file__).resolve().parents[1] + workflow = (root / ".github/workflows/blenderbench-portable.yml").read_text() + refs = re.findall(r"uses:\s*[^@\s]+@([^\s#]+)", workflow) + self.assertTrue(refs) + self.assertTrue(all(re.fullmatch(r"[0-9a-f]{40}", ref) for ref in refs)) + self.assertIn('runs-on: ubuntu-24.04', workflow) + self.assertIn('python-version: "3.13"', workflow) + self.assertIn('--require-hashes -r benchmarks/blenderbench_direct/requirements-fixture.lock', workflow) + self.assertNotIn('requirements-scoring.lock', workflow) + source = (root / "benchmarks/blenderbench_direct/requirements-scoring.in").read_text() + self.assertNotIn("--extra-index-url", source) + self.assertIn( + "torch @ https://download.pytorch.org/whl/cu128/torch-2.8.0%2Bcu128-cp313-cp313-manylinux_2_28_x86_64.whl", + source, + ) + lock = (root / "benchmarks/blenderbench_direct/requirements-scoring.lock").read_text() + for package in ("numpy", "pillow", "transformers", + "nvidia-cuda-nvrtc-cu12", "nvidia-cudnn-cu12", "triton"): + self.assertRegex(lock, rf"(?m)^{package}==[^\s]+.*\\\n(?:\s+--hash=sha256:[0-9a-f]{{64}})") + self.assertRegex(lock, r"(?m)^torch @ https://download\.pytorch\.org/[^\s]+.*\\\n(?:\s+--hash=sha256:[0-9a-f]{64})") + self.assertIn("requests==2.34.2", lock) + self.assertIn("setuptools==78.1.0", lock) + self.assertIn("numpy==2.4.6", lock) + self.assertIn("transformers==4.55.4", lock) + self.assertIn("tqdm==4.70.0", lock) + self.assertIn("urllib3==2.7.0", lock) + + def test_publication_result_manifests_are_small_complete_and_path_free(self): + root = Path(__file__).resolve().parents[1] / "benchmarks/blenderbench_direct/publication" + expected = { + "results-summary.json": "073be2c44f1a763b66dea5138eda1b8959e8553fef6ce288f1d78ee92807d6e8", + "reference-results.json": "1a50fc5ab89acf56bed4db49ec435a6bf1be866ba0213f1869a68891ba24cc0e", + "artifact-manifest.json": "087f17b1c953373d80aedb6287a1ba79363e9ebf4b232c374bed5fb11a29c978", + } + for name, digest in expected.items(): + self.assertEqual(dataset.sha256(root / name), digest) + text = (root / name).read_text(encoding="utf-8") + self.assertNotIn("/home/", text) + self.assertNotRegex(text, r"(?:token|password|secret|credential)\s*[:=]") + summary = json.loads((root / "results-summary.json").read_text()) + reference = json.loads((root / "reference-results.json").read_text()) + artifacts = json.loads((root / "artifact-manifest.json").read_text()) + self.assertEqual((summary["task_count"], summary["round_count"]), (27, 270)) + self.assertEqual(len(reference["rows"]), 27) + self.assertEqual(reference["experiment"]["scored_rounds"], 270) + self.assertEqual(len(reference["scoring_runtime"]["packages"]), 41) + self.assertEqual(reference["video"]["sha256"], + "04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17") + self.assertEqual(len(artifacts["tasks"]), 27) + self.assertEqual(sum(len(task["rounds"]) for task in artifacts["tasks"].values()), 270) + self.assertTrue(all(round_["blend_bytes"] > 0 for task in artifacts["tasks"].values() + for round_ in task["rounds"])) + self.assertEqual(artifacts["compilation_video"]["sha256"], + "04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17") + + def test_root_readme_pins_latest_video_and_metric_meaning(self): + text = (Path(__file__).resolve().parents[1] / "readme.md").read_text(encoding="utf-8") + immutable = ("https://huggingface.co/datasets/michaelgold/blenderbench-direct-results/resolve/" + "7c2c43be4517aee4d76d2b445578988de186970c/video/" + "blenderbench-complete-compilation.mp4") + self.assertIn(f"[![BlenderBench result](https://bpy.dev/assets/blenderbench-result.webp)]({immutable})", text) + self.assertIn("27 tasks / 270 rounds", text) + self.assertIn("CLIP image-embedding cosine similarity, not literal accuracy", text) + self.assertIn("04966320de680203baed6c01c3e211059350454139dc8a802c557cce24815c17", text) + + def test_generation_prompt_freezes_public_protocol_without_edit_bounds(self): + from benchmarks.blenderbench_direct.suite import generation_prompt + prompt = generation_prompt("task text", Path("/work").absolute(), 10) + self.assertIn("exactly 10 meaningful", prompt) + self.assertIn("Call get_render_as_image_for_cli after each saved round", prompt) + self.assertIn("expected_output_blend", prompt) + self.assertIn("No CLIP/scoring", prompt) + self.assertIn("goal_code", prompt) + for tool in protocol.TOOLS: + self.assertIn(tool, prompt) + for obsolete in ("allowed_transforms", "location_bounds", "allowed edit"): + self.assertNotIn(obsolete, prompt) + + def test_committed_benchmark_runtime_and_docs_have_no_private_home_or_model_default(self): + import subprocess + root = Path(__file__).resolve().parents[1] + tracked = subprocess.check_output(["git", "ls-files", "benchmarks", "tools"], + cwd=root, text=True).splitlines() + untracked = subprocess.check_output( + ["git", "ls-files", "--others", "--exclude-standard", "benchmarks", "tools"], + cwd=root, text=True).splitlines() + files = sorted(set(tracked + untracked)) + offenders = {} + for relative in files: + path = root / relative + if (relative.startswith("benchmarks/blenderbench_direct/publication/") + or not relative.endswith((".py", ".md", ".rst", ".toml", ".json")) + or not path.exists()): + continue + text = path.read_text(encoding="utf-8") + found = [needle for needle in ("/home/mg", "gpt-6-astra", "configured-model", + "configured-codex", "configured-bpy", "configured-version") + if needle in text] + if found: + offenders[relative] = found + self.assertEqual(offenders, {}) + + +class VideoTests(unittest.TestCase): + def test_overlay_labels_metric_and_task_level_token_interpolation_explicitly(self): + state = video.timeline_state(5.0, duration=10.0, rounds=[ + {"round": 1, "elapsed_seconds": 2.0, "image": "one.png", "clip_image_cosine_similarity": .5}, + {"round": 2, "elapsed_seconds": 8.0, "image": "two.png", "clip_image_cosine_similarity": .8}, + ], task_tokens=1000) + self.assertEqual(state["round"], 1) + self.assertEqual(state["task_tokens_interpolated"], 500) + labels = video.overlay_labels("provider/model", "level1/camera1", state, 10) + self.assertIn("CLIP image cosine similarity: 0.500000", labels) + self.assertIn("Task-level tokens (linear interpolation): ~500", labels) + unavailable = video.timeline_state(1, duration=10, rounds=[], task_tokens=None) + self.assertIn("unavailable", video.overlay_labels("m", "t", unavailable, 10)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_direct_codex.py b/tests/test_direct_codex.py index 461986f..674f9e4 100644 --- a/tests/test_direct_codex.py +++ b/tests/test_direct_codex.py @@ -14,6 +14,11 @@ def load(self): spec.loader.exec_module(module) return module + def test_generation_exposes_visual_inspection_with_exactly_four_tools(self): + m = self.load() + self.assertEqual(len(m.TOOLS), 4) + self.assertIn('get_render_as_image_for_cli', m.TOOLS) + def test_explicit_scoped_approval_required(self): m = self.load() config = {'mcp_servers': {'blender': {'command': '/server/bin/blender-mcp', @@ -27,6 +32,7 @@ def test_explicit_scoped_approval_required(self): self.assertNotIn('default_tools_approval_mode', server) self.assertNotIn('tools', config['mcp_servers']['blender']) self.assertTrue(server['required']) + self.assertEqual(server['omit_tools_from'], ['deferred', 'code_mode']) def test_rejects_extra_tools_and_conflicting_policy(self): m = self.load() @@ -48,6 +54,116 @@ def test_rejects_extra_tools_and_conflicting_policy(self): with self.assertRaises(ValueError): m.prepare_config(config, approved=True) + def test_generation_stream_allows_blender_resolution_properties(self): + m = self.load() + events = [ + {'type': 'item.completed', 'item': {'id': 'edit', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'execute_blender_code_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'code': 'bpy.context.scene.render.resolution_x = 512'}, + 'result': {'structured_content': {'saved': True}}}}, + {'type': 'turn.completed'}, + ] + m.verify_generation_events(events) + + def test_generation_stream_requires_one_successful_image_inspection_per_round(self): + m = self.load() + events = [ + {'type': 'item.completed', 'item': {'id': 'edit', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'execute_blender_code_for_cli', 'status': 'completed', + 'error': None, 'result': {'structured_content': {'saved': True}}}}, + {'type': 'turn.completed'}, + ] + with self.assertRaisesRegex(ValueError, 'visual inspection'): + m.verify_generation_events(events, rounds=1) + events.insert(1, {'type': 'item.completed', 'item': {'id': 'inspect', + 'type': 'mcp_tool_call', 'server': 'blender', 'tool': 'get_render_as_image_for_cli', + 'status': 'completed', 'error': None, + 'arguments': {'blend_file': '/work/iteration01.blend'}, + 'result': {'content': [{'type': 'image', 'mimeType': 'image/png', 'data': 'AA=='}]}}}) + m.verify_generation_events(events, rounds=1) + + def test_generation_stream_binds_visual_inspection_to_each_round_checkpoint(self): + m = self.load() + events = [ + {'type': 'item.completed', 'item': {'id': 'edit', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'execute_blender_code_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'code': 'edit'}, 'result': {'structured_content': {}}}}, + {'type': 'item.completed', 'item': {'id': 'inspect', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'get_render_as_image_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'blend_file': '/work/iteration02.blend'}, + 'result': {'content': [{'type': 'image', 'mimeType': 'image/png', 'data': 'AA=='}]}}}, + {'type': 'turn.completed'}, + ] + with self.assertRaisesRegex(ValueError, 'checkpoint sequence'): + m.verify_generation_events(events, rounds=1) + + def test_generation_stream_rejects_visual_inspection_from_wrong_directory(self): + import hashlib, tempfile + m = self.load() + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / 'work'; work.mkdir() + checkpoint = work / 'iteration01.blend'; checkpoint.write_bytes(b'checkpoint') + wrong = Path(tmp) / 'wrong' / checkpoint.name + events = [ + {'type': 'item.completed', 'item': {'id': 'edit', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'execute_blender_code_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'code': 'edit'}, 'result': {'structured_content': {}}}}, + {'type': 'item.completed', 'item': {'id': 'inspect', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'get_render_as_image_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'blend_file': str(wrong)}, + 'result': {'content': [{'type': 'image', 'mimeType': 'image/png', 'data': 'AA==', + '_meta': {'source_sha256': hashlib.sha256(b'checkpoint').hexdigest()}}]}}}, + {'type': 'turn.completed'}, + ] + with self.assertRaisesRegex(ValueError, 'checkpoint path'): + m.verify_generation_events(events, rounds=1, workdir=work) + + def test_generation_stream_rejects_unattested_precomputed_checkpoint(self): + import hashlib, tempfile + m = self.load() + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp); initial = work / 'initial.blend'; initial.write_bytes(b'initial') + checkpoint = work / 'iteration01.blend' + checkpoint.write_bytes(b'precomputed') + digest = hashlib.sha256(b'precomputed').hexdigest() + events = [ + {'type': 'item.completed', 'item': {'id': 'noop', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'execute_blender_code_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'blend_file': str(work / 'initial.blend'), 'code': 'result={}'}, + 'result': {'structured_content': {}}}}, + {'type': 'item.completed', 'item': {'id': 'inspect', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': 'get_render_as_image_for_cli', 'status': 'completed', + 'error': None, 'arguments': {'blend_file': str(checkpoint)}, + 'result': {'content': [{'type': 'image', 'mimeType': 'image/png', 'data': 'AA==', + '_meta': {'source': str(checkpoint), 'source_sha256': digest}}]}}}, + {'type': 'turn.completed'}, + ] + with self.assertRaisesRegex(ValueError, 'attested checkpoint'): + m.verify_generation_events(events, rounds=1, workdir=work) + events[0]['item']['result']['structured_content'] = {'_checkpoint': { + 'path': str(checkpoint), 'source': str(initial), + 'source_sha256': hashlib.sha256(b'initial').hexdigest(), + 'output_sha256': digest, 'existed_before': False}} + m.verify_generation_events(events, rounds=1, workdir=work) + + def test_generation_stream_rejects_next_edit_dispatched_before_prior_inspection(self): + m = self.load() + checkpoint = lambda n: {'path': f'/work/iteration{n:02}.blend'} + image = lambda n: {'type': 'item.completed', 'item': {'id': f'i{n}', + 'type': 'mcp_tool_call', 'server': 'blender', 'tool': 'get_render_as_image_for_cli', + 'status': 'completed', 'error': None, + 'arguments': {'blend_file': f'/work/iteration{n:02}.blend'}, + 'result': {'content': [{'type': 'image', 'mimeType': 'image/png', 'data': 'AA=='}]}}} + edit = lambda n, kind='item.completed': {'type': kind, 'item': {'id': f'e{n}', + 'type': 'mcp_tool_call', 'server': 'blender', 'tool': 'execute_blender_code_for_cli', + 'status': 'completed', 'error': None, + 'arguments': {'expected_output_blend': f'/work/iteration{n:02}.blend'}, + 'result': {'structured_content': {'_checkpoint': checkpoint(n)}}}} + events = [edit(1, 'item.started'), edit(2, 'item.started'), edit(1), image(1), + edit(2), image(2), {'type': 'turn.completed'}] + with self.assertRaisesRegex(ValueError, 'dispatch'): + m.verify_generation_events(events, rounds=2) + def test_preflight_requires_successful_mcp_result_not_exit_zero(self): m = self.load() self.assertTrue(hasattr(m, 'verify_preflight'), 'result verification missing') @@ -90,18 +206,52 @@ def test_checkpoints_required_and_pngs_decoded(self): with self.assertRaises(ValueError): m.verify_checkpoints(root, 2) + def test_checkpoint_ancestors_may_not_be_symlinks(self): + m = self.load() + import tempfile + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + real = base / 'real' + real.mkdir() + linked = base / 'linked' + linked.symlink_to(real, target_is_directory=True) + with self.assertRaisesRegex(ValueError, 'symlink'): + m.verify_checkpoints(linked, 1) + + def test_generation_workdir_is_an_exact_noncontaminated_surface(self): + m = self.load() + import tempfile + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); (root / 'initial.blend').write_bytes(b'blend') + m.verify_generation_workdir(root) + (root / 'obfuscated.bin').write_bytes(b'hidden payload') + with self.assertRaisesRegex(ValueError, 'runtime root'): + m.verify_generation_workdir(root) + + def test_generation_workdir_accepts_canonical_worker_when_imported_as_tools_module(self): + m = self.load() + from tools import direct_runtime + import tempfile + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / 'initial.blend').write_bytes(b'blend') + (root / 'worker.py').write_text(direct_runtime.WORKER) + m.verify_generation_workdir(root) + def test_cli_denied_preflight_stops_run_even_with_exit_zero(self): m = self.load() self.assertTrue(hasattr(m, 'main'), 'CLI missing') import tempfile, json, subprocess, sys with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) + work = root / 'work' + work.mkdir() fake = root / 'codex' fake.write_text('#!' + sys.executable + '\nimport sys\n' 'print("codex-cli 0.154.0" if "--version" in sys.argv else ' '\'{"type":"turn.completed"}\')\n') fake.chmod(0o755) - (root / 'initial.blend').write_bytes(b'fixture') + (work / 'initial.blend').write_bytes(b'fixture') (root / 'prompt.txt').write_text('fixture generation task') (root / 'target.png').write_bytes(b'fixture') config = root / 'server.toml' @@ -110,7 +260,9 @@ def test_cli_denied_preflight_stops_run_even_with_exit_zero(self): '[mcp_servers.blender.env]\nBLENDER_MCP_CLI_BACKEND="bpy"\n' 'BLENDER_MCP_BPY_PYTHON="/venv/bin/python"\n') args = [sys.executable, str(SCRIPT), 'run', '--config', str(config), - '--codex', str(fake), '--workdir', str(root), '--evidence', str(root / 'logs'), + '--codex', str(fake), '--codex-version', 'codex-cli 0.154.0', + '--bpy-version', '5.2.1', '--workdir', str(work), + '--evidence', str(root / 'logs'), '--model', 'test-model', '--approve-blender-tools', '--prompt', str(root / 'prompt.txt'), '--target', str(root / 'target.png')] proc = subprocess.run(args, capture_output=True, text=True) @@ -119,7 +271,11 @@ def test_cli_denied_preflight_stops_run_even_with_exit_zero(self): self.assertFalse((root / 'logs' / 'run.jsonl').exists()) argv = json.loads((root / 'logs' / 'preflight.argv.json').read_text()) self.assertIn('--strict-config', argv) - self.assertIn('workspace-write', argv) + self.assertIn('read-only', argv) + for feature in m.DISABLED_FEATURES: + self.assertIn(feature, argv) + self.assertIn('--disable', argv) + self.assertTrue(any('experimental_request_user_input=false' in a for a in argv)) self.assertNotIn('--dangerously-bypass-approvals-and-sandbox', argv) self.assertTrue(any('approval_mode' in a and 'approve' in a for a in argv)) # A successful preflight followed by exit 0 and zero rounds still fails. @@ -130,12 +286,12 @@ def test_cli_denied_preflight_stops_run_even_with_exit_zero(self): 'payload={"preflight":nonce.group() if nonce else "",' '"python":"/venv/bin/python","version":"5.2.1 LTS"}\n' 'print(json.dumps({"type":"item.completed","item":' - '{"type":"mcp_tool_call","server":"blender","tool":"execute_blender_code_for_cli",' + '{"id":"fixture-call","type":"mcp_tool_call","server":"blender","tool":"execute_blender_code_for_cli",' '"status":"completed","error":None,"result":{"structured_content":payload}}}))\n') args[args.index('--evidence') + 1] = str(root / 'logs2') proc = subprocess.run(args, capture_output=True, text=True) self.assertEqual(proc.returncode, 1, proc.stderr) - self.assertIn('incomplete checkpoint artifacts', proc.stderr) + self.assertIn('generation event stream is empty or lacks', proc.stderr) self.assertTrue((root / 'logs2' / 'run.jsonl').is_file()) def test_task_policy_is_ignored_but_noop_is_rejected(self): @@ -199,6 +355,43 @@ def test_toml_roundtrip_preserves_keys_booleans_and_paths(self): 'tools': list(m.TOOLS)}} self.assertEqual(tomllib.loads('mcp_servers=' + m.toml_value(value))['mcp_servers'], value) + def test_generation_stream_rejects_empty_or_incomplete_turns(self): + m = self.load() + for events in ([], [ + {'type': 'item.completed', 'item': {'id': 'call', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': m.TOOLS[0], 'status': 'completed', + 'arguments': {}, 'result': {'structured_content': {'ok': True}}}}, + ], [{'type': 'turn.completed'}]): + with self.subTest(events=events), self.assertRaises(ValueError): + m.verify_generation_events(events) + + def test_generation_stream_fails_closed_on_nonapproved_operations_and_feedback(self): + m = self.load() + allowed = [ + {'type': 'item.completed', 'item': {'id': 'reason', 'type': 'reasoning'}}, + {'type': 'item.completed', 'item': {'id': 'call', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': m.TOOLS[0], 'status': 'completed', + 'arguments': {'code': "import bpy; bpy.context.scene.render.filepath='iteration01.png'"}, + 'result': {'structured_content': {'ok': True}}}}, + {'type': 'turn.completed'}, + ] + m.verify_generation_events(allowed) + prohibited = [ + {'type': 'item.completed', 'item': {'id': 'shell', 'type': 'command_execution', + 'command': 'python edit_scene.py'}}, + {'type': 'item.completed', 'item': {'id': 'write', 'type': 'file_change', + 'path': 'scene.blend'}}, + {'type': 'item.completed', 'item': {'id': 'goal', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': m.TOOLS[0], 'status': 'completed', + 'arguments': {'code': "open('../goal.py').read()"}}}, + {'type': 'item.completed', 'item': {'id': 'score', 'type': 'mcp_tool_call', + 'server': 'blender', 'tool': m.TOOLS[0], 'status': 'completed', + 'arguments': {'code': "json.load(open('scoring/aggregate.json'))"}}}, + ] + for event in prohibited: + with self.subTest(item=event['item']['id']), self.assertRaises(ValueError): + m.verify_generation_events([event]) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_direct_recovery.py b/tests/test_direct_recovery.py index 00fd895..1f2ac1f 100644 --- a/tests/test_direct_recovery.py +++ b/tests/test_direct_recovery.py @@ -1,37 +1,21 @@ -import json +"""Compatibility entrypoint must not expose broken legacy recovery hooks.""" import sys -import tempfile import unittest from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools')) + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools")) import direct_suite as suite + class RecoveryTests(unittest.TestCase): - def test_recovery_scores_existing_work_without_generation(self): - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - old = root / 'old' - work = old / 'work' - work.mkdir(parents=True) - (work / 'initial.blend').write_bytes(b'original') - baseline = {'blend_sha256': suite.runtime.sha(work / 'initial.blend'), 'audit': {}} - (work / 'metadata.json').write_text(json.dumps(baseline)) - dataset = SimpleNamespace(verify=lambda *a: {'verified': True}) - with patch.object(suite, 'ROOT', root), patch.object(suite.direct, 'invoke', side_effect=AssertionError('regeneration')), patch.object(suite, 'score_checkpoints') as score: - suite.recover_task('level2/attribute1', {}, root / 'new', old, dataset) - self.assertEqual(score.call_args.args[2], work) - self.assertTrue((root / 'new/recovery.json').is_file()) + def test_broken_legacy_recovery_hooks_are_removed(self): + self.assertFalse(hasattr(suite, "score_checkpoints")) + self.assertFalse(hasattr(suite, "recover_task")) + self.assertFalse(hasattr(suite, "run_task")) + self.assertFalse(hasattr(suite, "task_plan")) + self.assertTrue(callable(suite.main)) + self.assertTrue(callable(suite.run_suite)) - def test_recovery_rejects_modified_initial(self): - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - work = root / 'old/work' - work.mkdir(parents=True) - (work / 'initial.blend').write_bytes(b'changed') - (work / 'metadata.json').write_text(json.dumps({'blend_sha256': 'original'})) - with self.assertRaisesRegex(ValueError, 'initial.blend'): - suite.recover_task('level2/attribute1', {}, root / 'new', work.parent, SimpleNamespace(verify=lambda *a: {})) -if __name__ == '__main__': unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_direct_saved_cleanup.py b/tests/test_direct_saved_cleanup.py index d47c001..3d79c38 100644 --- a/tests/test_direct_saved_cleanup.py +++ b/tests/test_direct_saved_cleanup.py @@ -38,38 +38,6 @@ def test_worker_records_material_lifecycle_without_mutating(self): self.fixture()[0]['material_lifecycle']['orphan']) self.assertFalse(material.use_fake_user) - def test_generation_starts_without_independent_reopen_gate(self): - import tempfile - from types import SimpleNamespace as NS - from unittest.mock import patch - import direct_suite as suite - class GenerationReached(Exception): pass - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - def initialize(inp, policy, python, output, events): - output.mkdir() - blend = output / 'start_initialized.blend' - blend.write_bytes(b'fixture') - return {'blend': str(blend), **self.fixture()[0]} - def invoke(args, config, evidence, label, prompt, target=None): - if label == 'run': - self.assertIn('previous saved checkpoint', prompt) - self.assertIn('All scene properties may be edited', prompt) - self.assertNotIn('unlisted', prompt) - self.assertNotIn('allowed edit', prompt) - self.assertIn('goal code', prompt) - raise GenerationReached() - return [] - policy = {} - policy['task_description'] = 'test' - with patch.object(suite, 'ROOT', root), \ - patch.object(suite.runtime, 'initialize', initialize), \ - patch.object(suite.runtime, 'inspect_render', side_effect=AssertionError('pre-generation reopen')), \ - patch.object(suite.direct, 'invoke', invoke), \ - patch.object(suite.direct, 'verify_preflight'): - with self.assertRaises(GenerationReached): - suite.run_task('level2/attribute1', policy, root / 'task', {}, - NS(verify=lambda *a: {})) def test_zero_user_unused_material_loss_is_accepted(self): before, after, policy = self.fixture() diff --git a/tests/test_direct_suite.py b/tests/test_direct_suite.py index e74d0c2..42b07c8 100644 --- a/tests/test_direct_suite.py +++ b/tests/test_direct_suite.py @@ -82,86 +82,11 @@ def foreach_get(self, *args): pass self.assertEqual(len(got), 140) self.assertEqual(got[1], {'type': 'NodeB'}) - def test_retry_requires_finished_suite_and_same_global_lock(self): - import tempfile - import fcntl - import direct_camera_retry as retry - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - suite = root / 'suite' - suite.mkdir() - self.assertFalse(retry.suite_finished(suite)) - (suite / 'events.jsonl').write_text('{"event":"dispatcher_started"}\n') - self.assertFalse(retry.suite_finished(suite)) - (suite / 'events.jsonl').write_text('{"event":"dispatcher_finished"}\n') - self.assertTrue(retry.suite_finished(suite)) - with (root / 'direct-suite.lock').open('a') as held: - fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) - with (root / 'direct-suite.lock').open('a') as other: - self.assertFalse(retry.try_lock(other)) - fcntl.flock(held, fcntl.LOCK_UN) - with (root / 'direct-suite.lock').open('a') as other: - self.assertTrue(retry.try_lock(other)) - (suite / 'STOP').touch() - with self.assertRaises(InterruptedError): - retry.suite_finished(suite) + def test_nonfunctional_legacy_retry_and_stale_validation_docs_are_removed(self): + self.assertFalse((TOOLS / 'direct_camera_retry.py').exists()) + self.assertFalse((TOOLS / 'direct_camera_retry.md').exists()) + self.assertFalse((TOOLS / 'direct_saved_validation.md').exists()) - def test_retry_process_waits_without_work_and_honors_stop(self): - import json - import subprocess - import tempfile - import time - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - predecessor = root / 'suite' - predecessor.mkdir() - out = root / 'retry' - bootstrap = ('import sys; sys.path.insert(0,' + repr(str(TOOLS)) + '); ' - 'import direct_camera_retry as r; from pathlib import Path; ' - 'r.suite.ROOT=Path(' + repr(tmp) + '); r.main()') - process = subprocess.Popen([sys.executable, '-c', bootstrap, '--output', str(out), - '--after-suite', str(predecessor)], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - try: - deadline = time.monotonic() + 5 - while not (out / 'status.json').exists() and time.monotonic() < deadline: - time.sleep(0.02) - row = json.loads((out / 'status.json').read_text()) - self.assertEqual(row['status'], 'waiting_for_suite') - self.assertFalse((out / 'level1-camera3').exists()) - (out / 'STOP').touch() - process.communicate(timeout=15) - self.assertNotEqual(process.returncode, 0) - self.assertEqual(json.loads((out / 'status.json').read_text())['status'], 'failed') - self.assertFalse((out / 'level1-camera3').exists()) - finally: - if process.poll() is None: - process.kill() - process.communicate() - - def test_restart_selection_is_unique_canonical_and_excludes_camera1(self): - import direct_suite as suite - tasks = {'level1/camera1': {}} - tasks.update({f'level2/task{i}': {} for i in range(26)}) - selected = ['level2/task3', 'level2/task4'] - self.assertEqual(suite.task_plan(tasks, selected), selected) - for invalid in [['unknown'], ['level1/camera1'], [selected[0]] * 2, []]: - with self.assertRaises(ValueError): - suite.task_plan(tasks, invalid) - - def test_plan_excludes_only_completed_camera1(self): - script = TOOLS / 'direct_suite.py' - self.assertTrue(script.exists(), 'suite dispatcher missing') - spec = importlib.util.spec_from_file_location('suite', script) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - tasks = {'level1/camera1': {}} - tasks.update({f'level2/task{i}': {} for i in range(26)}) - plan = module.task_plan(tasks) - self.assertEqual(len(plan), 26) - self.assertNotIn('level1/camera1', plan) - with self.assertRaises(ValueError): - module.task_plan({'level1/camera1': {}}) if __name__ == '__main__': unittest.main() diff --git a/tests/test_tool_listing.py b/tests/test_tool_listing.py index 7077f1e..e6826f0 100644 --- a/tests/test_tool_listing.py +++ b/tests/test_tool_listing.py @@ -28,12 +28,12 @@ { "name": "execute_blender_code", "description": "\n" - "Execute Python code in the connected Blender instance.\n" - "\n" - "The code runs in Blender's Python environment with full access to ``bpy``.\n" - "To return data, assign a JSON-serialisable dict to a variable named ``result``.\n" - "Deferred completion via ``check_is_finished`` is only supported by the\n" - "interactive addon server, and is rejected in background mode.\n", +"Execute Python code in the connected Blender instance.\n" +"\n" +"The code runs in Blender's Python environment with full access to ``bpy``.\n" +"To return data, assign a JSON-serialisable dict to a variable named ``result``.\n" +"Deferred completion via ``check_is_finished`` is only supported by the\n" +"interactive addon server, and is rejected in background mode.\n", "inputSchema": { "properties": { "code": { @@ -51,10 +51,12 @@ { "name": "execute_blender_code_for_cli", "description": "\n" - "Execute Python code in a background Blender process.\n" - "\n" - "Opens *blend_file* with ``blender --background`` and runs *code*.\n" - "Assign a dict to ``result`` to return data.\n", +"Execute Python code in a background Blender process.\n" +"\n" +"Opens *blend_file* with ``blender --background`` and runs *code*.\n" +"Assign a dict to ``result`` to return data. When *expected_output_blend*\n" +"is supplied, that path must be newly created with bytes different from\n" +"the source; the result includes server-computed checkpoint hashes.\n", "inputSchema": { "properties": { "blend_file": { @@ -64,6 +66,18 @@ "code": { "title": "Code", "type": "string" + }, + "expected_output_blend": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": None, + "title": "Expected Output Blend" } }, "required": [ @@ -74,10 +88,36 @@ "type": "object" } }, + { + "name": "get_render_as_image", + "description": "Render the connected Blender scene and return PNG image content without saving overrides.", + "inputSchema": { + "properties": {}, + "title": "get_render_as_imageArguments", + "type": "object" + } + }, + { + "name": "get_render_as_image_for_cli", + "description": "Open a saved blend in a background process, render it, and return PNG image content.", + "inputSchema": { + "properties": { + "blend_file": { + "title": "Blend File", + "type": "string" + } + }, + "required": [ + "blend_file" + ], + "title": "get_render_as_image_for_cliArguments", + "type": "object" + } + }, { "name": "get_blendfile_summary_datablocks", "description": "\n" - "Return a summary of the blend file: data-block counts, active workspace, and render engine.\n", +"Return a summary of the blend file: data-block counts, active workspace, and render engine.\n", "inputSchema": { "properties": {}, "title": "get_blendfile_summary_datablocksArguments", @@ -87,7 +127,7 @@ { "name": "get_blendfile_summary_datablocks_for_cli", "description": "\n" - "Return a data-block summary by opening *blend_file* in background Blender.\n", +"Return a data-block summary by opening *blend_file* in background Blender.\n", "inputSchema": { "properties": { "blend_file": { @@ -105,8 +145,8 @@ { "name": "get_blendfile_summary_missing_files", "description": "\n" - "Report external file references that are missing from disk\n" - "(images, libraries, fonts, sounds, movie clips, caches, sequences).\n", +"Report external file references that are missing from disk\n" +"(images, libraries, fonts, sounds, movie clips, caches, sequences).\n", "inputSchema": { "properties": {}, "title": "get_blendfile_summary_missing_filesArguments", @@ -116,7 +156,7 @@ { "name": "get_blendfile_summary_missing_files_for_cli", "description": "\n" - "Report missing file references by opening *blend_file* in background Blender.\n", +"Report missing file references by opening *blend_file* in background Blender.\n", "inputSchema": { "properties": { "blend_file": { @@ -134,7 +174,7 @@ { "name": "get_blendfile_summary_of_linked_libraries", "description": "\n" - "Return a tree of directly and indirectly linked library files.\n", +"Return a tree of directly and indirectly linked library files.\n", "inputSchema": { "properties": {}, "title": "get_blendfile_summary_of_linked_librariesArguments", @@ -144,7 +184,7 @@ { "name": "get_blendfile_summary_of_linked_libraries_for_cli", "description": "\n" - "Return linked-library info by opening *blend_file* in background Blender.\n", +"Return linked-library info by opening *blend_file* in background Blender.\n", "inputSchema": { "properties": { "blend_file": { @@ -162,7 +202,7 @@ { "name": "get_blendfile_summary_path_info", "description": "\n" - "Simple/fast access to the blend file's path, save status, age, and backups.\n", +"Simple/fast access to the blend file's path, save status, age, and backups.\n", "inputSchema": { "properties": {}, "title": "get_blendfile_summary_path_infoArguments", @@ -172,7 +212,7 @@ { "name": "get_blendfile_summary_path_info_for_cli", "description": "\n" - "Return path info by opening *blend_file* in background Blender.\n", +"Return path info by opening *blend_file* in background Blender.\n", "inputSchema": { "properties": { "blend_file": { @@ -190,7 +230,7 @@ { "name": "get_blendfile_summary_usage_guess", "description": "\n" - "Guess the primary use-cases of the current blend file (scored 0-100 with certainty).\n", +"Guess the primary use-cases of the current blend file (scored 0-100 with certainty).\n", "inputSchema": { "properties": {}, "title": "get_blendfile_summary_usage_guessArguments", @@ -200,7 +240,7 @@ { "name": "get_blendfile_summary_usage_guess_for_cli", "description": "\n" - "Guess use-cases by opening *blend_file* in background Blender.\n", +"Guess use-cases by opening *blend_file* in background Blender.\n", "inputSchema": { "properties": { "blend_file": { @@ -218,10 +258,10 @@ { "name": "get_object_detail_summary", "description": "\n" - "Return a structured summary of the object identified by *name*.\n" - "\n" - "Includes type, transforms, parent, children, modifiers, constraints,\n" - "materials, visibility, data-block name, and collections.\n", +"Return a structured summary of the object identified by *name*.\n" +"\n" +"Includes type, transforms, parent, children, modifiers, constraints,\n" +"materials, visibility, data-block name, and collections.\n", "inputSchema": { "properties": { "name": { @@ -239,10 +279,10 @@ { "name": "get_objects_summary", "description": "\n" - "Return the scene's collection hierarchy and their objects.\n" - "\n" - "Each collection lists its objects (name, type, parent, data name,\n" - "selection, visibility) and nested child collections.\n", +"Return the scene's collection hierarchy and their objects.\n" +"\n" +"Each collection lists its objects (name, type, parent, data name,\n" +"selection, visibility) and nested child collections.\n", "inputSchema": { "properties": {}, "title": "get_objects_summaryArguments", @@ -252,55 +292,55 @@ { "name": "get_python_api_docs", "description": "\n" - "Return the Blender Python API docs for *identifier*, or list\n" - "modules matching a trailing-``*`` discovery pattern.\n" - "\n" - "*identifier* should be a fully-qualified Python name (e.g.\n" - "``bpy.app`` or ``bpy.types.Scene.frame_current``).\n" - "The trailing-``*`` forms are supported as discovery entry-points:\n" - "\n" - "- ``*`` enumerates the top-level modules (``bpy``, ``bmesh``,\n" - " ``mathutils``, ``gpu``, ...).\n" - "- ``X.*`` enumerates the direct-child identifiers under the\n" - " *X* namespace (``bpy.*`` -> ``bpy.app``, ``bpy.context``, ...).\n" - "\n" - "Both return a ``namespace`` response even when ``X.rst`` would\n" - "otherwise resolve to ``exact``; the ``.*`` form lets an agent\n" - "force the child listing.\n" - "\n" - "The response always carries ``kind``, ``found``, and ``identifier``.\n" - "The remaining keys depend on ``kind``:\n" - "\n" - "- ``\"exact\"`` (``found=True``): ``.rst`` was read.\n" - " Extra keys: ``content`` (RST text), ``examples``. When the\n" - " file exceeds 32 KB, ``content`` is replaced with a dot-point\n" - " summary of the file's top-level definitions (prefixed by a\n" - " header noting the truncation) and ``examples`` is empty -\n" - " re-query individual members for their rendered blocks.\n" - "- ``\"namespace\"`` (``found=True``):\n" - " no ``.rst`` but ``..rst`` siblings exist.\n" - " Extra key: ``submodules`` (list of child identifiers).\n" - "- ``\"definition\"`` (``found=True``):\n" - " *identifier* is defined inside a parent RST\n" - " (e.g. ``bpy.props.IntProperty`` lives in ``bpy.props.rst``).\n" - " Extra keys: ``content`` (rendered block), ``examples``.\n" - "- ``\"partial\"`` (``found=False``):\n" - " the parent RST was located but the trailing component isn't defined in it.\n" - " Extra keys:\n" - " - ``parent`` the identifier whose RST was loaded.\n" - " - ``available`` top-level definitions in that RST.\n" - " - ``submodules`` sibling identifiers ``.`` with their own RSTs,\n" - " filtered to those whose last component contains every character of the missing tail.\n" - "\n" - " For a toctree landing page like ``bpy.types`` ``available`` is empty and ``submodules``\n" - " is the near-miss list; for a self-contained module like ``bpy.props`` it's the reverse.\n" - "- ``\"suggestions\"`` (``found=False``):\n" - " no direct match, but *identifier* appears as a component of other files.\n" - " Extra key: ``suggestions`` (list of full identifiers).\n" - "- ``\"missing\"`` (``found=False``): nothing matched.\n" - "\n" - "``examples`` (present on the ``exact`` and ``definition`` kinds)\n" - "is a list of ``{path, content}`` entries referenced from this documentation.\n", +"Return the Blender Python API docs for *identifier*, or list\n" +"modules matching a trailing-``*`` discovery pattern.\n" +"\n" +"*identifier* should be a fully-qualified Python name (e.g.\n" +"``bpy.app`` or ``bpy.types.Scene.frame_current``).\n" +"The trailing-``*`` forms are supported as discovery entry-points:\n" +"\n" +"- ``*`` enumerates the top-level modules (``bpy``, ``bmesh``,\n" +" ``mathutils``, ``gpu``, ...).\n" +"- ``X.*`` enumerates the direct-child identifiers under the\n" +" *X* namespace (``bpy.*`` -> ``bpy.app``, ``bpy.context``, ...).\n" +"\n" +"Both return a ``namespace`` response even when ``X.rst`` would\n" +"otherwise resolve to ``exact``; the ``.*`` form lets an agent\n" +"force the child listing.\n" +"\n" +"The response always carries ``kind``, ``found``, and ``identifier``.\n" +"The remaining keys depend on ``kind``:\n" +"\n" +"- ``\"exact\"`` (``found=True``): ``.rst`` was read.\n" +" Extra keys: ``content`` (RST text), ``examples``. When the\n" +" file exceeds 32 KB, ``content`` is replaced with a dot-point\n" +" summary of the file's top-level definitions (prefixed by a\n" +" header noting the truncation) and ``examples`` is empty -\n" +" re-query individual members for their rendered blocks.\n" +"- ``\"namespace\"`` (``found=True``):\n" +" no ``.rst`` but ``..rst`` siblings exist.\n" +" Extra key: ``submodules`` (list of child identifiers).\n" +"- ``\"definition\"`` (``found=True``):\n" +" *identifier* is defined inside a parent RST\n" +" (e.g. ``bpy.props.IntProperty`` lives in ``bpy.props.rst``).\n" +" Extra keys: ``content`` (rendered block), ``examples``.\n" +"- ``\"partial\"`` (``found=False``):\n" +" the parent RST was located but the trailing component isn't defined in it.\n" +" Extra keys:\n" +" - ``parent`` the identifier whose RST was loaded.\n" +" - ``available`` top-level definitions in that RST.\n" +" - ``submodules`` sibling identifiers ``.`` with their own RSTs,\n" +" filtered to those whose last component contains every character of the missing tail.\n" +"\n" +" For a toctree landing page like ``bpy.types`` ``available`` is empty and ``submodules``\n" +" is the near-miss list; for a self-contained module like ``bpy.props`` it's the reverse.\n" +"- ``\"suggestions\"`` (``found=False``):\n" +" no direct match, but *identifier* appears as a component of other files.\n" +" Extra key: ``suggestions`` (list of full identifiers).\n" +"- ``\"missing\"`` (``found=False``): nothing matched.\n" +"\n" +"``examples`` (present on the ``exact`` and ``definition`` kinds)\n" +"is a list of ``{path, content}`` entries referenced from this documentation.\n", "inputSchema": { "properties": { "identifier": { @@ -318,10 +358,10 @@ { "name": "get_runtime_python_api_docs", "description": "\n" - "Return compact docs for an exact Python API *identifier* from the connected Blender runtime.\n" - "\n" - "Use a fully-qualified identifier such as ``bpy.ops.mesh.primitive_cube_add``\n" - "or ``bpy.types.Object.location``. Use ``search_api_docs`` to discover identifiers.\n", +"Return compact docs for an exact Python API *identifier* from the connected Blender runtime.\n" +"\n" +"Use a fully-qualified identifier such as ``bpy.ops.mesh.primitive_cube_add``\n" +"or ``bpy.types.Object.location``. Use ``search_api_docs`` to discover identifiers.\n", "inputSchema": { "properties": { "identifier": { @@ -339,11 +379,11 @@ { "name": "get_runtime_python_api_docs_for_cli", "description": "\n" - "Return compact docs for an exact Python API *identifier* from a background runtime.\n" - "\n" - "Opens *blend_file* with the selected Blender or ``bpy`` CLI backend. Use a\n" - "fully-qualified identifier such as ``bpy.ops.mesh.primitive_cube_add`` or\n" - "``bpy.types.Object.location``. Use ``search_api_docs`` to discover identifiers.\n", +"Return compact docs for an exact Python API *identifier* from a background runtime.\n" +"\n" +"Opens *blend_file* with the selected Blender or ``bpy`` CLI backend. Use a\n" +"fully-qualified identifier such as ``bpy.ops.mesh.primitive_cube_add`` or\n" +"``bpy.types.Object.location``. Use ``search_api_docs`` to discover identifiers.\n", "inputSchema": { "properties": { "blend_file": { @@ -366,12 +406,12 @@ { "name": "get_screenshot_of_area_as_image", "description": "\n" - "Take a screenshot of a single Blender area and return it as a PNG image.\n" - "\n" - "*area_ui_type* matches the area's ``ui_type``.\n" - "\n" - "*size_limit_in_bytes* caps the image size in bytes.\n" - "Zero (the default) uses the MCP message size limit.\n", +"Take a screenshot of a single Blender area and return it as a PNG image.\n" +"\n" +"*area_ui_type* matches the area's ``ui_type``.\n" +"\n" +"*size_limit_in_bytes* caps the image size in bytes.\n" +"Zero (the default) uses the MCP message size limit.\n", "inputSchema": { "properties": { "area_ui_type": { @@ -418,10 +458,10 @@ { "name": "get_screenshot_of_window_as_image", "description": "\n" - "Take a screenshot of the entire Blender window and return it as a PNG image.\n" - "\n" - "*size_limit_in_bytes* caps the image size in bytes.\n" - "Zero (the default) uses the MCP message size limit.\n", +"Take a screenshot of the entire Blender window and return it as a PNG image.\n" +"\n" +"*size_limit_in_bytes* caps the image size in bytes.\n" +"Zero (the default) uses the MCP message size limit.\n", "inputSchema": { "properties": { "size_limit_in_bytes": { @@ -437,7 +477,7 @@ { "name": "get_screenshot_of_window_as_json", "description": "\n" - "Return a JSON description of the Blender window layout, areas, active object, and selection.\n", +"Return a JSON description of the Blender window layout, areas, active object, and selection.\n", "inputSchema": { "properties": {}, "title": "get_screenshot_of_window_as_jsonArguments", @@ -447,7 +487,7 @@ { "name": "jump_to_tab_by_name", "description": "\n" - "Switch the active workspace tab to *name*.\n", +"Switch the active workspace tab to *name*.\n", "inputSchema": { "properties": { "name": { @@ -465,10 +505,10 @@ { "name": "jump_to_tab_by_space_type", "description": "\n" - "Switch to a workspace whose main area matches *space_type*.\n" - "\n" - "If *allow_edits* is True and no matching workspace exists, a new one\n" - "is created by duplicating the current workspace.\n", +"Switch to a workspace whose main area matches *space_type*.\n" +"\n" +"If *allow_edits* is True and no matching workspace exists, a new one\n" +"is created by duplicating the current workspace.\n", "inputSchema": { "properties": { "space_type": { @@ -491,10 +531,10 @@ { "name": "jump_to_view3d_object_by_name", "description": "\n" - "Move the 3D viewport to focus on an object by *name*.\n" - "\n" - "If *allow_edits* is True the object may be un-hidden and its\n" - "collections enabled to make it visible.\n", +"Move the 3D viewport to focus on an object by *name*.\n" +"\n" +"If *allow_edits* is True the object may be un-hidden and its\n" +"collections enabled to make it visible.\n", "inputSchema": { "properties": { "name": { @@ -517,10 +557,10 @@ { "name": "jump_to_view3d_object_data_by_name", "description": "\n" - "Move the 3D viewport to the object whose data block matches *name*.\n" - "\n" - "If *allow_edits* is True the object may be un-hidden and its\n" - "collections enabled to make it visible.\n", +"Move the 3D viewport to the object whose data block matches *name*.\n" +"\n" +"If *allow_edits* is True the object may be un-hidden and its\n" +"collections enabled to make it visible.\n", "inputSchema": { "properties": { "name": { @@ -543,7 +583,7 @@ { "name": "render_thumbnail_to_path", "description": "\n" - "Render a small, low-quality thumbnail to *output_path* (temporarily overrides settings).\n", +"Render a small, low-quality thumbnail to *output_path* (temporarily overrides settings).\n", "inputSchema": { "properties": { "output_path": { @@ -561,7 +601,7 @@ { "name": "render_viewport_to_path", "description": "\n" - "Render the current scene to *output_path* using current render settings.\n", +"Render the current scene to *output_path* using current render settings.\n", "inputSchema": { "properties": { "output_path": { @@ -579,32 +619,32 @@ { "name": "search_api_docs", "description": "\n" - "Full-text search over the bundled Blender Python API reference.\n" - "\n" - "Returns a ranked list of hits. Each hit has:\n" - "\n" - "- ``path``: file path relative to the bundled docs.\n" - "- ``text``: the matching paragraph plus ``context``\n" - " paragraphs on either side.\n" - "- ``breadcrumb``: the section path containing the hit\n" - " (``Section > Sub-section > ...``).\n" - "- ``index``: the hit's position in the result list.\n" - "- ``score``: a relevance score; higher is better.\n" - "\n" - "The query is tokenised on whitespace and matched\n" - "case-insensitively. Every token must appear somewhere in\n" - "the paragraph body, the file path, or an enclosing section\n" - "title - in any order. Common English stop-words (``the``,\n" - "``a``, ``how``, ``to``, ...) are dropped, so natural\n" - "phrasings like ``\"how to bake\"`` work as expected. Regular\n" - "expressions are not supported.\n" - "\n" - "Use ``context`` to pull more surrounding paragraphs into\n" - "each hit (symmetric, default 0). Use ``index`` with the\n" - "position of a previous hit (same query) to get that hit\n" - "alone with its text widened to its enclosing section.\n" - "\n" - "Read-only; consults bundled RST files only.\n", +"Full-text search over the bundled Blender Python API reference.\n" +"\n" +"Returns a ranked list of hits. Each hit has:\n" +"\n" +"- ``path``: file path relative to the bundled docs.\n" +"- ``text``: the matching paragraph plus ``context``\n" +" paragraphs on either side.\n" +"- ``breadcrumb``: the section path containing the hit\n" +" (``Section > Sub-section > ...``).\n" +"- ``index``: the hit's position in the result list.\n" +"- ``score``: a relevance score; higher is better.\n" +"\n" +"The query is tokenised on whitespace and matched\n" +"case-insensitively. Every token must appear somewhere in\n" +"the paragraph body, the file path, or an enclosing section\n" +"title - in any order. Common English stop-words (``the``,\n" +"``a``, ``how``, ``to``, ...) are dropped, so natural\n" +"phrasings like ``\"how to bake\"`` work as expected. Regular\n" +"expressions are not supported.\n" +"\n" +"Use ``context`` to pull more surrounding paragraphs into\n" +"each hit (symmetric, default 0). Use ``index`` with the\n" +"position of a previous hit (same query) to get that hit\n" +"alone with its text widened to its enclosing section.\n" +"\n" +"Read-only; consults bundled RST files only.\n", "inputSchema": { "properties": { "query": { @@ -644,32 +684,32 @@ { "name": "search_manual_docs", "description": "\n" - "Full-text search over the bundled Blender user manual.\n" - "\n" - "Returns a ranked list of hits. Each hit has:\n" - "\n" - "- ``path``: file path relative to the bundled docs.\n" - "- ``text``: the matching paragraph plus ``context``\n" - " paragraphs on either side.\n" - "- ``breadcrumb``: the section path containing the hit\n" - " (``Section > Sub-section > ...``).\n" - "- ``index``: the hit's position in the result list.\n" - "- ``score``: a relevance score; higher is better.\n" - "\n" - "The query is tokenised on whitespace and matched\n" - "case-insensitively. Every token must appear somewhere in\n" - "the paragraph body, the file path, or an enclosing section\n" - "title - in any order. Common English stop-words (``the``,\n" - "``a``, ``how``, ``to``, ...) are dropped, so natural\n" - "phrasings like ``\"how to bake\"`` work as expected. Regular\n" - "expressions are not supported.\n" - "\n" - "Use ``context`` to pull more surrounding paragraphs into\n" - "each hit (symmetric, default 0). Use ``index`` with the\n" - "position of a previous hit (same query) to get that hit\n" - "alone with its text widened to its enclosing section.\n" - "\n" - "Read-only; consults bundled RST files only.\n", +"Full-text search over the bundled Blender user manual.\n" +"\n" +"Returns a ranked list of hits. Each hit has:\n" +"\n" +"- ``path``: file path relative to the bundled docs.\n" +"- ``text``: the matching paragraph plus ``context``\n" +" paragraphs on either side.\n" +"- ``breadcrumb``: the section path containing the hit\n" +" (``Section > Sub-section > ...``).\n" +"- ``index``: the hit's position in the result list.\n" +"- ``score``: a relevance score; higher is better.\n" +"\n" +"The query is tokenised on whitespace and matched\n" +"case-insensitively. Every token must appear somewhere in\n" +"the paragraph body, the file path, or an enclosing section\n" +"title - in any order. Common English stop-words (``the``,\n" +"``a``, ``how``, ``to``, ...) are dropped, so natural\n" +"phrasings like ``\"how to bake\"`` work as expected. Regular\n" +"expressions are not supported.\n" +"\n" +"Use ``context`` to pull more surrounding paragraphs into\n" +"each hit (symmetric, default 0). Use ``index`` with the\n" +"position of a previous hit (same query) to get that hit\n" +"alone with its text widened to its enclosing section.\n" +"\n" +"Read-only; consults bundled RST files only.\n", "inputSchema": { "properties": { "query": { diff --git a/tools/DIRECT_SUITE.md b/tools/DIRECT_SUITE.md index 9bfd024..2161387 100644 --- a/tools/DIRECT_SUITE.md +++ b/tools/DIRECT_SUITE.md @@ -1,24 +1,14 @@ -# Direct full-suite execution on mghavn +# BlenderBench direct suite -`tools/direct_suite.py` is an external sequential dispatcher, not a model reasoning controller. It verifies the pinned input-only manifest, applies reviewed literal initialization using standalone bpy 5.2.1, independently renders/reopens the saved baseline, executes a genuine four-tool Codex preflight, and invokes one direct Codex generation session per task with ten requested meaningful rounds. No evaluator scores or VLM judge enter generation. +The former host-specific dispatcher has been replaced by the model-neutral, +full-27 runner in [`benchmarks/blenderbench_direct`](../benchmarks/blenderbench_direct/README.md). -Exactly 26 policy tasks are selected by excluding completed `level1/camera1` from the 27 canonical tasks. Existing completed output is not modified. Active staging is observed, never duplicated. Task work directories must be fresh; there is no automatic generation restart or hidden budget reset. - -Post-generation, artifacts are decoded/hashed, each scene is independently audited/rendered on CUDA, exact task-policy preservation and meaningful edits are checked against both baseline and preceding checkpoint, and repeated complete scene states are rejected. The offline pinned GPU CLIP pair scorer runs after the generation process exits. `scores.json` retains paired NCLIP/PL and `result.json` selects minimum raw NCLIP, earliest-round tie. - -Runtime provenance: `direct_runtime.py` was copied from the reviewed full-suite runtime in the original worktree without editing the original. Its admitted interpreter is the authorized mghavn custom-bpy venv. Embedded RNA graphs are serialized with deterministic repeated-node references to avoid exponential armature expansion; heterogeneous collections retain each element's RNA schema. Neither fix drops audited fields. - -## Invocation +Use: ```sh -nohup /home/mg/blenderbench-direct/enhanced-mcp/mcp/.venv/bin/python -u \ - /home/mg/blenderbench-direct/direct_suite.py \ - --output /home/mg/blenderbench-direct/remaining-direct-03 \ - > /home/mg/blenderbench-direct/remaining-direct-03.log 2>&1 < /dev/null & +python -m benchmarks.blenderbench_direct.runner --help ``` -The process holds a host-wide advisory lock. `process.json`, `status.json`, dispatcher `events.jsonl`, per-task native heartbeat logs, and per-task `codex/{preflight,run}.{jsonl,stderr,process.json,argv.json,prompt.txt}` are retained. The output-root `STOP` cancels native workers and prevents the next task; **during an active direct Codex invocation it is only observed after that invocation returns**, not immediate cancellation. Kill that exact Codex process if immediate interruption is needed. No claims of isolation beyond workspace-write and explicit per-tool approval are made. - -**Deadlines remain backend 120 seconds and MCP client 86400 seconds.** They are not unlimited and have not been changed. Native verification has manual cancellation/disk-floor safety and no operation deadline. - -Initialization/audit failures and generation/scoring failures are retained as failed tasks; later tasks continue sequentially. A started suite is not a completed benchmark. No merge, push, publication, or automatic repair/retry is performed. +All dataset, MCP, Codex, bpy, scorer, output, model, and version inputs are +explicit. New generation always requires a fresh output root. Recovery and +scoring are separate commands and never regenerate missing rounds. diff --git a/tools/direct_camera_retry.md b/tools/direct_camera_retry.md deleted file mode 100644 index 4856ddf..0000000 --- a/tools/direct_camera_retry.md +++ /dev/null @@ -1,43 +0,0 @@ -# Camera3 mixed-scene compatibility and safe retry - -Camera3's pinned input opens `Yoga studio scene`, whose engine is already CYCLES, -with Camera1, 512 samples, and 512x512 at 100%. A second inactive scene named -`Scene` uses BLENDER_EEVEE. The old runtime rejected that inactive scene before -rendering the active Cycles scene. Only the active scene now receives the CUDA -runtime-local GPU device override. Neither scene's engine changes, and inactive -scene device settings remain untouched. Full RNA audits still cover both scenes. -Active non-Cycles rendering remains unsupported and fails closed; this is not -an EEVEE/EGL implementation or a Cycles conversion. - -Use an isolated copy of direct_runtime.py, direct_codex.py, direct_suite.py and -direct_camera_retry.py. Do not overwrite files imported by an active dispatcher. -Launch the retry script durably (detached session, retained stdout/stderr): - -``` -/path/to/custom-bpy/python -u /isolated/direct_camera_retry.py \ - --output /fresh/camera3-retry \ - --after-suite /existing/remaining-suite -``` - -The wrapper requires a normally finished predecessor event AND acquisition of -`/home/mg/blenderbench-direct/direct-suite.lock`. It holds that lock throughout -admission, ten direct Codex rounds, full independent audits and post-generation -GPU CLIP scoring. No VLM judge or feedback into generation is added. The exact -four approved MCP tools and existing deadlines/config checks are unchanged. -The original failed Camera3 directory remains evidence, never a resume target. - -A predecessor crash does not authorize concurrent retry: the wrapper keeps -waiting without launching work. A STOP file in either the predecessor output or -retry output cancels the queue. After generation starts the existing suite's -STOP limitation applies: direct Codex notices cancellation after invoke returns. - -Monitor `process.json`, `status.json`, `events.jsonl`, and retained process stdout. -Actual native admission evidence is in `level1-camera3/{work,admission,reopen}`: -initialization and independent render must produce parent-decoded PNGs; all three -full audits must agree and source/saved-scene hashes must remain unchanged. -A queue status of `waiting_for_suite` is NOT native GPU admission or completion. - -Tests: `python -m unittest discover -s tests -p 'test_direct*.py' -v`. -The mixed-scene regression was observed failing at the old engine assertion -before the fix. Queue tests exercise real flock contention and a real waiting -subprocess with STOP cancellation. Synthetic tests are not GPU render evidence. diff --git a/tools/direct_camera_retry.py b/tools/direct_camera_retry.py deleted file mode 100644 index 2946939..0000000 --- a/tools/direct_camera_retry.py +++ /dev/null @@ -1,94 +0,0 @@ -"""One fresh Camera3 retry after a normally finished suite; never preempt it. - -Launch with nohup/setsid and retain stdout. A STOP in either output cancels the -queue. An interrupted/crashed predecessor is NOT equivalent to a finished suite. -The existing global flock is held through native admission, generation and score. -""" -import argparse -import fcntl -import json -import os -from pathlib import Path -import subprocess -import sys -import time -import tomllib - -import direct_suite as suite -import direct_runtime as runtime - - -def suite_finished(output): - if (output / 'STOP').exists(): - raise InterruptedError('predecessor STOP: retry cancelled') - try: - with (output / 'events.jsonl').open() as stream: - last = None - for line in stream: - if line.strip(): - last = json.loads(line) - return bool(last and last.get('event') == 'dispatcher_finished') - except (FileNotFoundError, json.JSONDecodeError): - return False - - -def try_lock(lock): - try: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError: - return False - return True - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--output', type=Path, required=True) - parser.add_argument('--after-suite', type=Path, required=True) - args = parser.parse_args() - out = args.output.absolute() - predecessor = args.after_suite.resolve(strict=True) - out.mkdir(parents=True, exist_ok=False) - events = out / 'events.jsonl' - process = {'pid': os.getpid(), 'argv': sys.argv, 'cwd': os.getcwd(), - 'after_suite': str(predecessor), 'task': 'level1/camera3'} - (out / 'process.json').write_text(json.dumps(process, indent=2)) - def status(state, **extra): - row = dict(process, status=state, **extra) - temp = out / 'status.tmp' - temp.write_text(json.dumps(row, indent=2)) - temp.replace(out / 'status.json') - runtime.emit(events, state, **process, **extra) - try: - with (suite.ROOT / 'direct-suite.lock').open('a') as lock: - while True: - if (out / 'STOP').exists(): - raise InterruptedError('retry STOP') - finished = suite_finished(predecessor) - if finished and try_lock(lock): - break - status('waiting_for_suite', predecessor_finished=finished) - time.sleep(10) - # Recheck under the lock before any native GPU or model work. - if not suite_finished(predecessor) or (out / 'STOP').exists(): - raise InterruptedError('admission cancelled') - sys.path.insert(0, str(suite.ROOT / 'input-tools')) - import dataset - tasks = json.loads((suite.ROOT / 'input-tools/task-policies.json').read_text())['tasks'] - if set(tasks) != set(dataset.task_ids()): - raise ValueError('canonical policy/manifest task mismatch') - config = suite.direct.prepare_config(tomllib.loads( - (suite.ROOT / 'mghavn-custom-bpy.toml').read_text()), approved=True) - version = subprocess.check_output(['/home/mg/.local/bin/codex', '--version'], text=True).strip() - if version != 'codex-cli 0.154.0': - raise ValueError('unreviewed Codex version') - status('running') - suite.run_task('level1/camera3', tasks['level1/camera3'], - out / 'level1-camera3', config, dataset) - status('complete') - except BaseException as exc: - status('failed', error=repr(exc)) - raise - - -if __name__ == '__main__': - main() diff --git a/tools/direct_codex.example.toml b/tools/direct_codex.example.toml index 63e099d..74cf49c 100644 --- a/tools/direct_codex.example.toml +++ b/tools/direct_codex.example.toml @@ -3,7 +3,7 @@ [mcp_servers.blender] command = "/absolute/server-venv/bin/blender-mcp" cwd = "/absolute/blender-mcp/mcp" -enabled_tools = ["execute_blender_code_for_cli", "get_runtime_python_api_docs_for_cli", "search_api_docs", "get_python_api_docs"] +enabled_tools = ["execute_blender_code_for_cli", "get_render_as_image_for_cli", "get_runtime_python_api_docs_for_cli", "search_api_docs"] required = true # Explicit client deadline, NOT a backend timeout override or no-deadline promise. tool_timeout_sec = 86400 diff --git a/tools/direct_codex.md b/tools/direct_codex.md index 7a9d985..e450a6a 100644 --- a/tools/direct_codex.md +++ b/tools/direct_codex.md @@ -1,123 +1,45 @@ -# Direct Codex + standalone bpy +# Low-level direct Codex launcher -`direct_codex.py` is a direct-process launcher and artifact gate, **not a benchmark -controller**. It runs one read-only MCP preflight and, in `run` mode, one continuous -Codex generation session. It does not initialize data, orchestrate refinement -rounds, run a judge, score checkpoints, or publish anything. Camera1 is permitted; -it does not use the legacy camera controller's camera2–camera9 task registry. +`direct_codex.py` is the low-level one-task launcher used by the portable +[full BlenderBench runner](../benchmarks/blenderbench_direct/README.md). Prefer the +full runner for publication runs because it owns pinned data, initialization, +post-generation saved-scene audit, scoring, aggregation, and provenance. -## Approval fix and scope - -Supported/tested config contract: **Codex CLI 0.154.0**, standalone **bpy 5.2.1**. -The launcher fails closed on other Codex versions pending a schema review. - -The observed failure was `MCP tool call requires approval, but approval policy is -never`. Exit 0 and final assistant text did not prove that any round ran. That is -an MCP approval denial, not evidence of a shell sandbox failure. - -Official references (version-pinned source is authoritative for this launcher): - -- [Config reference](https://developers.openai.com/codex/config-reference) -- [Live schema](https://developers.openai.com/codex/config-schema.json) -- [0.154.0 schema](https://github.com/openai/codex/blob/rust-v0.154.0/codex-rs/core/config.schema.json): - `RawMcpServerConfig.tools`, `McpServerToolConfig.approval_mode`, `AppToolApproval`. -- [0.154.0 approval implementation](https://github.com/openai/codex/blob/rust-v0.154.0/codex-rs/core/src/mcp_tool_call.rs#L2345-L2356): - `auto` consults tool annotations; `prompt` requires approval; `writes` requires - approval unless read-only; **`approve` does not request tool approval**. - `auto` is NOT synonymous with unconditional preapproval. - -Only after the user approves arbitrary Python execution by this particular server, -pass `--approve-blender-tools`. The launcher adds -`mcp_servers.blender.tools..approval_mode="approve"` to exactly: +The launcher admits exactly these MCP tools: - `execute_blender_code_for_cli` +- `get_render_as_image_for_cli` - `get_runtime_python_api_docs_for_cli` - `search_api_docs` -- `get_python_api_docs` - -It retains `--sandbox workspace-write`, explicitly sets `approval_policy="never"`, -and uses `--ignore-user-config --strict-config`. It never enables a global approval -bypass, changes user config, or silently replaces a conflicting approval setting. -Server defaults, per-tool policy in input, extra servers/tools, or unrelated config -keys are rejected. The complete MCP table is supplied as a TOML inline CLI override, -not lossy recursive dotted-key flattening. Runtime paths preserve venv spelling. - -**This is real preauthorization, not a sandbox.** MCP Python has the invoking -server's OS privileges, potentially beyond Codex workspace-write. Review -[SECURITY.md](../SECURITY.md) and [backend docs](../readme_bpy_backend.rst). -Use an isolated disposable account/runtime and an external workspace without -project Codex config/plugins. Built-in Codex tools are not an MCP allowlist; -the prompt prohibits shell scene operations and permits `view_image` inspection. -Managed policies may still deny tools. No attempt is made to override them. - -## Usage - -Run from the repository root. Copy `tools/direct_codex.example.toml` outside the -repo and fill in verified absolute server/interpreter paths. Use an installed MCP -1.x server environment. Keep credentials out of TOML: effective config and argv -are deliberately retained in the evidence directory. Check `codex login status` -without reading credential files. - -Prepare a fresh external workspace with reviewed `initial.blend`; do not give the -model solution scripts/oracles. Independently admit the GPU runtime with real -renders before generation. The preflight below only checks tool execution, -`sys.executable`, and bpy version, **not GPU rendering**. Both modes make paid model -calls and require authorization for the selected exact model. - -```sh -uv run --frozen --project benchmarks/blenderbench_camera python tools/direct_codex.py preflight \ - --config /external/server.toml --codex /absolute/bin/codex \ - --workdir /external/camera1 --evidence /external/preflight-evidence \ - --model YOUR_EXACT_MODEL --approve-blender-tools - -uv run --frozen --project benchmarks/blenderbench_camera python tools/direct_codex.py run \ - --config /external/server.toml --codex /absolute/bin/codex \ - --workdir /external/camera1 --evidence /external/generation-evidence \ - --model YOUR_EXACT_MODEL --approve-blender-tools \ - --prompt /external/camera1-prompt.txt --target /external/target.png --rounds 10 -``` - -Using this locked Python environment provides Pillow; **no camera controller code -is imported**. The prompt is user-owned: specify Camera1-only changes, fixed lens, -render settings and scene invariants, CUDA GPU enabled/CPU disabled, exact ten -meaningful rounds, source preservation, and no scoring until generation ends. -The launcher appends only the tool/output contract. Each round must save -`iteration01.blend`/`iteration01.png` through `iteration10.*`, plus one JSONL row: - -```json -{"round":1,"pose":{"location":[1,2,3],"rotation":[0,0,0]},"blend":"/external/camera1/iteration01.blend","png":"/external/camera1/iteration01.png","rationale":"camera adjustment"} -``` - -The preflight requires an actual completed successful execution-tool JSONL result -containing a fresh nonce, the exact configured Python path, and version 5.2.1. -A final assistant claim, denied/failed tool, empty transcript, or exit 0 alone -cannot pass. Generation is not launched when preflight fails. Raw JSONL, stderr, -argv, prompt, child PID, and exit code are retained in a fresh private directory. -Existing iteration paths/ledger are refused; resume is unsupported. -## Completion and limitations +It requires explicit scoped preapproval with `--approve-blender-tools`, retains +Codex `workspace-write` plus `approval_policy="never"`, rejects extra MCP servers +or tools, and verifies a real successful preflight MCP result. MCP execution has +the server process's OS privileges; this is not a security sandbox. -After generation, the artifact gate requires exact sequential ledger rounds, -expected absolute checkpoint paths, nonempty distinct checkpoint bytes, distinct -finite reported poses, and fully decoded PNGs. It records SHA-256 and dimensions. -A run that exits 0 with zero rounds fails. Success is explicitly labelled -`artifacts_complete_not_scene_audited`: this gate does **not** reopen `.blend` files -or prove pose truth, scene preservation, meaningful edits, GPU selection, image -correspondence, or source immutability. Independent checkpoint audits remain -mandatory before claiming ten valid benchmark rounds. Report missing/invalid -rounds honestly. Post-generation GPU CLIP scoring/best checkpoint selection is -separate; no scores or VLM judge are fed into generation. +For benchmark rounds, pass `expected_output_blend` to +`execute_blender_code_for_cli`. The server requires a fresh changed checkpoint +and returns source/output SHA-256 evidence. The following +`get_render_as_image_for_cli` call renders those exact bytes with CUDA-only +Cycles and returns PNG content plus its source hash. -No launcher wall-clock timeout is imposed. **Stock backend execution still has a -120-second timeout**, and the example's MCP client timeout is **86400 seconds**. -Neither is unlimited and this launcher does not patch either. Long render runtime -changes require separate review; do not describe this setup as no-hard-timeouts. -Cancellation/supervision and disk-floor checks belong to the operator; interrupted -runs retain logs and must use fresh destinations. No full live ten-round run or -new GPU admission is claimed by the tests here. +Every path, model, and runtime version pin is explicit: ```sh -uv run --frozen --project benchmarks/blenderbench_camera python -m unittest tests.test_direct_codex -v +python tools/direct_codex.py run \ + --config /external/blender-mcp.toml \ + --codex /absolute/bin/codex \ + --codex-version 'EXACT CODEX --version OUTPUT' \ + --bpy-version 'EXACT BPY VERSION PREFIX' \ + --workdir /external/fresh-task-work \ + --evidence /external/fresh-private-evidence \ + --model 'PROVIDER/MODEL-ID' \ + --prompt /external/task-prompt.txt \ + --target /external/target.png \ + --rounds 10 --approve-blender-tools ``` -Tests use synthetic local subprocess fixtures, not model calls or Blender results. +`tools/direct_codex.example.toml` documents the MCP schema. Keep credentials out +of TOML and inspect retained raw argv/JSONL/stderr before publication. Existing +iteration files are rejected. This launcher does not resume, regenerate, score, +or establish full BlenderBench completion by itself. diff --git a/tools/direct_codex.py b/tools/direct_codex.py index e13b7cf..b92f742 100644 --- a/tools/direct_codex.py +++ b/tools/direct_codex.py @@ -1,8 +1,21 @@ """Direct Codex launch/config preflight; no benchmark controller or scoring.""" import copy -TOOLS = ('execute_blender_code_for_cli', 'get_runtime_python_api_docs_for_cli', - 'search_api_docs', 'get_python_api_docs') +TOOLS = ('execute_blender_code_for_cli', 'get_render_as_image_for_cli', + 'get_runtime_python_api_docs_for_cli', 'search_api_docs') +CODEX_VERSION = 'codex-cli 0.154.0' +CODEX_SOURCE_REVISION = '6b9826e3aa83b1a5947db50f4332cb9c65f1b340' +CODEX_FEATURE_REGISTRY_URL = ( + 'https://github.com/openai/codex/blob/' + CODEX_SOURCE_REVISION + + '/codex-rs/features/src/lib.rs') +DISABLED_FEATURES = ( + 'shell_tool', 'unified_exec', 'unified_exec_tty', 'shell_snapshot', 'view_image', + 'sleep_tool', 'code_mode', 'code_mode_host', 'code_mode_prewarm', + 'request_permissions_tool', 'multi_agent', 'multi_agent_v2', 'apps', 'enable_mcp_apps', + 'tool_suggest', 'plugins', 'hooks', 'in_app_browser', 'in_app_local_automation', + 'browser_use', 'browser_use_full_cdp_access', 'browser_use_external', 'computer_use', + 'remote_plugin', 'plugin_sharing', 'image_generation', 'standalone_web_search', + 'web_search_request', 'web_search_cached') def prepare_config(config, *, approved): @@ -26,10 +39,40 @@ def prepare_config(config, *, approved): raise ValueError('absolute server and standalone bpy interpreter required') server['tools'] = {tool: {'approval_mode': 'approve'} for tool in TOOLS} server['required'] = True + # Codex 0.154 otherwise defers MCP tools through tool_search, whose dynamic + # calls are not represented by exec JSONL. Force the four tools direct. + server['omit_tools_from'] = ['deferred', 'code_mode'] return result -def verify_preflight(events, nonce, python): +def verify_generation_workdir(root): + """Require the mounted generation root to contain only initialized runtime inputs.""" + import hashlib + from pathlib import Path + root = Path(root).absolute() + allowed = {'initial.blend', 'initial_verified.png', 'audit.json', 'metadata.json', + 'runtime.log', 'worker.py', 'worker.json'} + if root.is_symlink() or not root.is_dir(): + raise ValueError('generation runtime root must be a regular directory') + entries = list(root.iterdir()) + unexpected = sorted(entry.name for entry in entries if entry.name not in allowed) + if unexpected or any(entry.is_symlink() or not entry.is_file() for entry in entries): + raise ValueError(f'generation runtime root contains undeclared paths: {unexpected}') + initial = root / 'initial.blend' + if not initial.is_file(): + raise ValueError('generation runtime root requires initial.blend') + worker = root / 'worker.py' + if worker.exists(): + try: + from tools import direct_runtime + except ModuleNotFoundError: # direct script invocation adds tools/ to sys.path + import direct_runtime + expected = hashlib.sha256(direct_runtime.WORKER.encode()).hexdigest() + if hashlib.sha256(worker.read_bytes()).hexdigest() != expected: + raise ValueError('generation runtime worker.py differs from canonical runtime bytes') + + +def verify_preflight(events, nonce, python, bpy_version='5.2.1'): """Require a completed real MCP item, never trust the final assistant text.""" import json for event in events: @@ -48,9 +91,121 @@ def verify_preflight(events, nonce, python): continue if (isinstance(payload, dict) and payload.get('preflight') == nonce and payload.get('python') == python - and str(payload.get('version', '')).split()[:1] == ['5.2.1']): + and str(payload.get('version', '')).startswith(bpy_version)): return - raise ValueError('no successful standalone bpy 5.2.1 MCP preflight result') + raise ValueError(f'no successful standalone bpy {bpy_version} MCP preflight result') + + +def verify_generation_events(events, rounds=None, workdir=None): + """Reject any generation operation outside the four approved MCP tools.""" + import json + import re + forbidden = re.compile( + r'(?i)(goal(?:_code)?\.py|\bsolution(?:\.py)?\b|oracle|scores?\.json|scoring[/\\]|' + r'aggregate\.json|ref_based_eval|evaluator|clipmodel|photometric_loss)') + seen = {} + completed = set() + visual_checkpoints = [] + pending_checkpoint = None + active_edit = None + turn_completed = 0 + for index, event in enumerate(events): + if event.get('type') in ('error', 'turn.failed'): + raise ValueError('generation event stream reports failure') + if event.get('type') == 'turn.completed': + turn_completed += 1 + if index != len(events) - 1: + raise ValueError('generation turn completion must terminate the event stream') + continue + if event.get('type') not in ('item.started', 'item.updated', 'item.completed'): + continue + item = event.get('item') + if not isinstance(item, dict) or not isinstance(item.get('id'), str): + raise ValueError('malformed generation item event') + ident, kind = item['id'], item.get('type') + signature = (kind, item.get('server'), item.get('tool')) + if ident in seen and seen[ident] != signature: + raise ValueError('generation item identity changed in event stream') + seen[ident] = signature + if kind in ('reasoning', 'agent_message'): + continue + if kind != 'mcp_tool_call': + raise ValueError(f'prohibited Codex generation operation: {kind!r}') + if item.get('server') != 'blender' or item.get('tool') not in TOOLS: + raise ValueError('generation used a tool outside the four approved Blender MCP tools') + arguments = json.dumps(item.get('arguments', {}), sort_keys=True, default=str) + if forbidden.search(arguments): + raise ValueError('generation MCP arguments access prohibited goal/solution or score feedback') + if event.get('type') == 'item.started' and item.get('tool') == 'execute_blender_code_for_cli': + if active_edit is not None or pending_checkpoint is not None: + raise ValueError('next edit dispatch preceded visual inspection of prior checkpoint') + active_edit = ident + if event.get('type') == 'item.completed': + if ident in completed: + raise ValueError('duplicate generation item completion') + completed.add(ident) + result = item.get('result') or {} + if (item.get('status') != 'completed' or item.get('error') + or (isinstance(result, dict) and result.get('isError'))): + raise ValueError('generation MCP tool call failed') + if item.get('tool') == 'execute_blender_code_for_cli': + if pending_checkpoint is not None or (active_edit is not None and active_edit != ident): + raise ValueError('next edit dispatch preceded visual inspection of prior checkpoint') + structured = result.get('structured_content') if isinstance(result, dict) else None + if isinstance(structured, dict) and isinstance(structured.get('_checkpoint'), dict): + pending_checkpoint = structured['_checkpoint'] + active_edit = None + if item.get('tool') == 'get_render_as_image_for_cli': + content = result.get('content', []) if isinstance(result, dict) else [] + if not any(isinstance(value, dict) and value.get('type') == 'image' + and value.get('mimeType') == 'image/png' and value.get('data') + for value in content): + raise ValueError('visual inspection tool did not return PNG image content') + arguments = item.get('arguments') + if not isinstance(arguments, dict) or not isinstance(arguments.get('blend_file'), str): + raise ValueError('visual inspection requires a saved checkpoint path') + path = __import__('pathlib').Path(arguments['blend_file']).absolute() + metadata = next((value.get('_meta', value.get('meta')) for value in content + if isinstance(value, dict) and value.get('type') == 'image'), None) + if workdir is not None: + expected_path = (__import__('pathlib').Path(workdir).resolve() + / f'iteration{len(visual_checkpoints)+1:02}.blend') + if path != expected_path: + raise ValueError('visual inspection checkpoint path or source hash is invalid') + path = path.resolve(strict=True) + import hashlib + with path.open('rb') as stream: + actual_hash = hashlib.file_digest(stream, 'sha256').hexdigest() + source_path = (__import__('pathlib').Path(workdir).resolve() / 'initial.blend' + if not visual_checkpoints else + __import__('pathlib').Path(workdir).resolve() + / f'iteration{len(visual_checkpoints):02}.blend') + if not isinstance(pending_checkpoint, dict): + raise ValueError('visual inspection requires an attested checkpoint creation') + with source_path.open('rb') as stream: + source_hash = hashlib.file_digest(stream, 'sha256').hexdigest() + if (not isinstance(metadata, dict) + or metadata.get('source') != str(path) + or metadata.get('source_sha256') != actual_hash + or pending_checkpoint.get('path') != str(path) + or pending_checkpoint.get('source') != str(source_path) + or pending_checkpoint.get('source_sha256') != source_hash + or pending_checkpoint.get('output_sha256') != actual_hash + or pending_checkpoint.get('existed_before') is not False): + raise ValueError('visual inspection checkpoint path or source hash is invalid') + visual_checkpoints.append(path.name) + pending_checkpoint = None + if any(ident not in completed and signature[0] == 'mcp_tool_call' + for ident, signature in seen.items()): + raise ValueError('generation event stream contains unfinished MCP tool call') + if turn_completed != 1 or not completed: + raise ValueError('generation event stream is empty or lacks a completed turn/tool call') + if rounds is not None: + expected = [f'iteration{number:02}.blend' for number in range(1, rounds + 1)] + if visual_checkpoints != expected: + raise ValueError(f'visual inspection checkpoint sequence must be exactly {expected}') + if pending_checkpoint is not None: + raise ValueError('generation created an uninspected checkpoint') def validate_edit(before, after, policy): @@ -86,7 +241,12 @@ def verify_checkpoints(root, rounds, policy=None): import hashlib import json import math + from pathlib import Path from PIL import Image + root = Path(root).expanduser().absolute() + for component in (root, *root.parents): + if component.is_symlink(): + raise ValueError(f'symlink forbidden in checkpoint path: {component}') try: rows = [json.loads(line) for line in (root / 'rounds.jsonl').read_text().splitlines()] if rounds < 1 or [r['round'] for r in rows] != list(range(1, rounds + 1)): @@ -136,12 +296,23 @@ def toml_value(value): def invoke(args, config, evidence, label, prompt, target=None): import json import subprocess - command = [args.codex, 'exec', '--ignore-user-config', '--strict-config', - '--skip-git-repo-check', '--sandbox', 'workspace-write', '--model', args.model, + command = [args.codex] + for feature in DISABLED_FEATURES: + command += ['--disable', feature] + command += ['exec', '--ignore-user-config', '--strict-config', + '--skip-git-repo-check', '--sandbox', 'read-only', '--model', args.model, '--json', '-C', str(args.workdir), '-c', 'approval_policy="never"', + '-c', 'web_search="disabled"', + '-c', 'tools.experimental_request_user_input=false', + '-c', 'tools.update_plan=false', '-c', 'mcp_servers=' + toml_value(config['mcp_servers']), '-o', str(evidence / (label + '.final.txt'))] if target: + import hashlib + target = target.resolve(strict=True) + (evidence / (label + '.visible-input.json')).write_text(json.dumps({ + 'mechanism': 'Codex exec -i bootstrap attachment; Codex 0.154 JSONL does not emit ImageView', + 'path': str(target), 'sha256': hashlib.sha256(target.read_bytes()).hexdigest()})) command += ['-i', str(target)] command += ['-'] (evidence / (label + '.argv.json')).write_text(json.dumps(command, indent=2)) @@ -167,7 +338,9 @@ def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('mode', choices=['preflight', 'run']) parser.add_argument('--config', type=Path, required=True) - parser.add_argument('--codex', default='codex') + parser.add_argument('--codex', required=True) + parser.add_argument('--codex-version', required=True) + parser.add_argument('--bpy-version', required=True) parser.add_argument('--workdir', type=Path, required=True) parser.add_argument('--evidence', type=Path, required=True, help='fresh private evidence directory') parser.add_argument('--model', required=True) @@ -184,6 +357,7 @@ def main(argv=None): args.evidence = args.evidence.absolute() if not (args.workdir / 'initial.blend').is_file(): raise ValueError('workdir requires initial.blend') + verify_generation_workdir(args.workdir) if args.mode == 'run': if not args.prompt or not args.target or args.rounds < 1: raise ValueError('run requires --prompt, --target, and positive --rounds') @@ -192,8 +366,8 @@ def main(argv=None): if (args.workdir / 'rounds.jsonl').exists() or any(args.workdir.glob('iteration*')): raise ValueError('run requires fresh checkpoint paths; resume is unsupported') version = subprocess.check_output([args.codex, '--version'], text=True).strip() - if version != 'codex-cli 0.154.0': - raise ValueError(f'unsupported Codex version {version!r}; review schema before updating pin') + if args.codex_version != CODEX_VERSION or version != CODEX_VERSION: + raise ValueError(f'publication protocol requires exactly {CODEX_VERSION!r}; got {version!r}') args.evidence.mkdir(parents=True, exist_ok=False, mode=0o700) (args.evidence / 'config.json').write_text(json.dumps(config, indent=2)) (args.evidence / 'codex-version.txt').write_text(version) @@ -204,16 +378,20 @@ def main(argv=None): + str(args.workdir / 'initial.blend') + ' and code=' + code + '. Do not mutate or save the scene. Do not use shell or other tools.') events = invoke(args, config, args.evidence, 'preflight', preflight) - verify_preflight(events, nonce, config['mcp_servers']['blender']['env']['BLENDER_MCP_BPY_PYTHON']) + verify_preflight(events, nonce, config['mcp_servers']['blender']['env']['BLENDER_MCP_BPY_PYTHON'], args.bpy_version) if args.mode == 'run': contract = ('\nUse only these MCP tools for scene operations: ' + ', '.join(TOOLS) + '. No shell scene operations, benchmark controller, VLM judge, scoring, or publication. ' - 'Use view_image for image inspection. Write rounds.jsonl via MCP with one row per ' + 'After every saved round call get_render_as_image_for_cli on that round checkpoint and ' + 'visually inspect its returned PNG content; built-in view_image is disabled. Write rounds.jsonl via MCP with one row per ' 'completed round: round (integer), pose={location:[x,y,z],rotation:[x,y,z]}, ' 'blend and png (absolute paths), rationale. Save iteration01.blend/iteration01.png ' f'through iteration{args.rounds:02}.blend/iteration{args.rounds:02}.png. ' + 'For each edit set execute_blender_code_for_cli blend_file to initial.blend or the preceding ' + 'iteration and expected_output_blend to the new iteration blend. ' 'Inspection/no-op calls are not rounds. Never alter initial.blend.') - invoke(args, config, args.evidence, 'run', prompt + contract, args.target) + events = invoke(args, config, args.evidence, 'run', prompt + contract, args.target) + verify_generation_events(events, rounds=args.rounds, workdir=args.workdir) checkpoints = verify_checkpoints(args.workdir, args.rounds) (args.evidence / 'artifact-check.json').write_text(json.dumps({ 'status': 'artifacts_complete_not_scene_audited', 'checkpoints': checkpoints}, indent=2)) diff --git a/tools/direct_runtime.py b/tools/direct_runtime.py index ed2a071..1f6b188 100644 --- a/tools/direct_runtime.py +++ b/tools/direct_runtime.py @@ -16,8 +16,6 @@ import subprocess import time -CUSTOM_BPY_PYTHON = '/home/mg/blenderbench-direct/enhanced-mcp/mcp/.venv/bin/python' - def sha(path): h = hashlib.sha256() @@ -104,12 +102,12 @@ def run_process(argv, log, event_path, policy): return result -# This source is executed exclusively by CUSTOM_BPY_PYTHON. Never imports camera.rounds. +# This source is executed exclusively by the explicitly configured bpy Python. WORKER = r''' import bpy, hashlib, json, sys, os from pathlib import Path cfg=json.loads(Path(sys.argv[1]).read_text()) -assert tuple(bpy.app.version)==(5,2,1), ('custom bpy version required', bpy.app.version_string) +assert bpy.app.version_string.startswith(cfg['bpy_version']), ('configured bpy version required', bpy.app.version_string) bpy.context.preferences.filepaths.use_scripts_auto_execute=False bpy.ops.wm.open_mainfile(filepath=cfg['source'], use_scripts=False) assert not bpy.app.autoexec_fail, 'automatic script dependency rejected' @@ -302,13 +300,13 @@ def audit(): ''' -def _execute(source, policy, bpy_python, output, event_path, render, start=None): +def _execute(source, policy, bpy_python, bpy_version, output, event_path, render, start=None): # Preserve the venv executable path: resolving its symlink loses venv imports. python = Path(bpy_python).absolute() - if str(python) != CUSTOM_BPY_PYTHON: - raise ValueError('only installed custom bpy Python is admitted: ' + CUSTOM_BPY_PYTHON) if not python.is_file(): raise FileNotFoundError(python) + if not bpy_version: + raise ValueError('explicit bpy version pin required') source = Path(source).resolve(strict=True) if source.suffix != '.blend': raise ValueError('source must be a saved .blend') @@ -318,7 +316,7 @@ def _execute(source, policy, bpy_python, output, event_path, render, start=None) original = sha(source) png = output / ('initial_verified.png' if start else 'verified.png') if render else None saved = output / 'start_initialized.blend' if start else None - config = {'source': str(source), 'report': str(output / 'audit.json'), + config = {'source': str(source), 'report': str(output / 'audit.json'), 'bpy_version': bpy_version, 'save': str(saved) if saved else None, 'png': str(png) if png else None} if start: config['initialization'] = policy['initialization'] @@ -336,7 +334,7 @@ def _execute(source, policy, bpy_python, output, event_path, render, start=None) raise ValueError('source blend changed during independent verification') result = json.loads(Path(config['report']).read_text()) evidence = result['evidence'] - if (evidence['blender_version_tuple'] != [5, 2, 1] or evidence['compute_device_type'] != 'CUDA' + if (not evidence['blender_version'].startswith(bpy_version) or evidence['compute_device_type'] != 'CUDA' or evidence['scene_device'] != 'GPU' or not any(d['type']=='CUDA' and d['use'] for d in evidence['devices']) or any(d['use'] and d['type']!='CUDA' for d in evidence['devices'])): raise ValueError('custom bpy CUDA evidence failed') @@ -363,12 +361,12 @@ def _execute(source, policy, bpy_python, output, event_path, render, start=None) return result -def inspect_render(blend, task_policy, bpy_python, output, event_path, render=True): +def inspect_render(blend, task_policy, bpy_python, bpy_version, output, event_path, render=True): """Independently open/audit/render an unchanged saved blend via custom bpy.""" - return _execute(blend, task_policy, bpy_python, output, event_path, render) + return _execute(blend, task_policy, bpy_python, bpy_version, output, event_path, render) -def initialize(inp, task_policy, bpy_python, output, event_path): +def initialize(inp, task_policy, bpy_python, bpy_version, output, event_path): """Apply parent-reviewed literal initialization; save and render the state. inp is a directory with exactly one top-level .blend and start.py, or a mapping @@ -376,7 +374,7 @@ def initialize(inp, task_policy, bpy_python, output, event_path): """ if task_policy.get('pin_verified') is not True or not task_policy.get('start_sha256') or not isinstance(task_policy.get('initialization'), list): raise ValueError('initialization requires parent-verified pin/policy, start_sha256 and literal initialization') - if task_policy.get('unsupported_blocker') or task_policy.get('reviewed') is False: + if task_policy.get('reviewed') is False: raise ValueError('unsupported or unreviewed initialization policy') if isinstance(inp, dict): source, start = Path(inp['blend']), Path(inp['start_py']) @@ -388,4 +386,4 @@ def initialize(inp, task_policy, bpy_python, output, event_path): source, start = sources[0], inp / 'start.py' if start.name != 'start.py' or sha(start) != task_policy['start_sha256']: raise ValueError('parent-verified start.py hash mismatch') - return _execute(source, task_policy, bpy_python, output, event_path, True, start.resolve()) + return _execute(source, task_policy, bpy_python, bpy_version, output, event_path, True, start.resolve()) diff --git a/tools/direct_saved_validation.md b/tools/direct_saved_validation.md deleted file mode 100644 index 6162346..0000000 --- a/tools/direct_saved_validation.md +++ /dev/null @@ -1,47 +0,0 @@ -# Direct suite: live edits and saved-deliverable validation - -User-approved workflow supersedes the former independent reopen admission gate. -Initialization still verifies pinned inputs, applies reviewed literal task initialization, -saves a protected initial blend, records the full native RNA baseline, and renders a -CUDA-only PNG which the parent decodes. Generation starts without independently -reopening and comparing that initialized file first. - -The deployed standalone MCP backend is **not a persistent Blender session**: -`blender_cli.py` launches `bpy_cli_runner.py` as a subprocess per request, and its -`main()` opens `blend_file` before executing code. Direct Codex continues edits from -the previous saved checkpoint across calls and edits the open scene within each call. -Unsaved memory cannot persist across these calls. This change does not introduce a -persistent server, replace the backend, or change the stock 120-second code deadline. - -After all ten direct meaningful rounds, the parent independently reopens/renders -saved deliverables, validates each against the initialization baseline and prior -checkpoint, checks distinct allowed state, and then performs GPU CLIP scoring. -No oracle/goal scripts or score feedback are given to generation. The exact four -MCP tools and all five task policy families remain unchanged. - -## Narrow cleanup exception - -The native audit records material lifecycle evidence separately from content hashes. -Only a material **missing from a later saved audit** can be excused, and only when -the earlier independent evidence records integer `users == 0`, `use_fake_user == false`, -`use_extra_user == false`, and `library == null`. No fake users are added for auditing. -Missing evidence fails closed. Present orphan material contents, new materials, -assigned materials, all object/material slot references, geometry, cameras, lights, -worlds, scenes and other datablocks remain audited exactly. Cleanup alone never -counts as a meaningful edit. No other datablock class is exempted without evidence. - -`tests/test_direct_saved_cleanup.py` exercises accepted orphan loss, rejected assigned -material changes/deletions, protected/missing lifecycle evidence, other forbidden -changes, no-op rejection, the actual worker lifecycle serializer, and generation -reaching Codex without any independent pre-generation reopen call. - -## Authorized restarts - -`direct_suite.py --output FRESH_ROOT --tasks TASK_ID ...` permits an explicit unique -canonical subset, never camera1. Operators must verify the canonical manifest, -completed results and artifacts, stop the preceding dispatcher and retry through -STOP, verify both processes and descendants exited, then construct the unfinished -subset. Retain previous roots including failures. The unchanged shared -`direct-suite.lock` serializes the new dispatcher through initialization, generation, -validation and scoring. Keep process.json, events.jsonl, status.json and launcher -stdout/stderr as durable handles. A running task is not a completed result. diff --git a/tools/direct_suite.py b/tools/direct_suite.py index 1a1eac2..e45edc3 100644 --- a/tools/direct_suite.py +++ b/tools/direct_suite.py @@ -1,215 +1,15 @@ -"""External sequential dispatcher: native admission, direct Codex, post-run audit/score. -No model reasoning loop, score feedback, VLM judge, or upstream script execution. -""" -import argparse -import fcntl -import json -import os -from pathlib import Path -import subprocess -import sys -import time -import tomllib -import uuid -from types import SimpleNamespace - -import direct_codex as direct -import direct_runtime as runtime - -ROOT = Path('/home/mg/blenderbench-direct') - - - -def task_plan(tasks, selected=None): - plan = [t for t in tasks if t != 'level1/camera1'] - if len(tasks) != 27 or len(plan) != 26: - raise ValueError('expected 27 canonical tasks, 26 remaining') - if selected is not None: - if not selected or len(set(selected)) != len(selected) or set(selected) - set(plan): - raise ValueError('restart tasks must be unique canonical unfinished task IDs, excluding camera1') - return list(selected) - return plan - +"""Compatibility entry point for the portable BlenderBench runner. -def run_task(task, policy, out, config, dataset): - inp = ROOT / 'dataset' / task - verification = dataset.verify(task, inp) - policy = dict(policy, task=task, pin_verified=True, stop_file=str(out.parent / 'STOP')) - - out.mkdir(parents=True, exist_ok=False) - (out / 'input-verification.json').write_text(json.dumps(verification, indent=2)) - events = out / 'events.jsonl' - runtime.emit(events, 'initializing', task=task) - initial = runtime.initialize(inp, policy, runtime.CUSTOM_BPY_PYTHON, out / 'work', events) - work = out / 'work' - blend = work / 'initial.blend' - Path(initial['blend']).rename(blend) - # Generation follows the initialized scene, without a separate reopen gate. - # Saved checkpoints are independently reopened after generation for integrity. - baseline = initial - initial_hash = runtime.sha(blend) - runtime.emit(events, 'native_admission_passed', task=task, source_sha256=initial_hash) - args = SimpleNamespace(codex='/home/mg/.local/bin/codex', model='gpt-6-astra', workdir=work) - evidence = out / 'codex' - evidence.mkdir(mode=0o700) - (evidence / 'config.json').write_text(json.dumps(config, indent=2)) - nonce = uuid.uuid4().hex - code = 'import bpy,sys; result=' + repr({'preflight': nonce}) - code += '; result.update(python=sys.executable,version=bpy.app.version_string)' - prompt = ('Call only execute_blender_code_for_cli once with blend_file=' + str(blend) - + ' and code=' + code + '. Do not mutate/save. No shell or other tools.') - preflight = direct.invoke(args, config, evidence, 'preflight', prompt) - direct.verify_preflight(preflight, nonce, runtime.CUSTOM_BPY_PYTHON) - runtime.emit(events, 'preflight_passed', task=task) - contract = ('Solve this BlenderBench task by direct use of the four Blender MCP tools.\n' - + policy['task_description'] + '\nTarget image is attached. Initial scene: ' + str(blend) - + '\nAll scene properties may be edited, including objects, geometry, materials, lights and cameras.\n' - + '\nPerform exactly 10 meaningful edit/render/visual-inspection rounds sequentially. ' - 'Inspect initial scene and image, reason yourself about edits, render and inspect each checkpoint ' - 'using view_image. Do not delegate reasoning to scripts/controllers. No CLIP/scoring, VLM judge, ' - 'oracle/goal code or goal .blend access, or score feedback. Do not read audit or evaluation files. ' - 'Use only execute_blender_code_for_cli, get_runtime_python_api_docs_for_cli, search_api_docs, ' - 'get_python_api_docs for scene work; no shell scene operations. ' - 'The backend code-call deadline remains 120 seconds; the MCP client deadline remains 86400 seconds. ' - 'This standalone backend starts a new bpy subprocess and opens blend_file on every call; ' - 'unsaved memory does NOT persist across calls. Continue edits from the previous saved checkpoint ' - '(initial.blend only for the first edit); save intermediate work before a call returns. ' - 'Within a call, edit the open scene continuously without unnecessary reloads. ' - 'Use Cycles CUDA rendering, enable CUDA devices only and disable CPU. Save scene BEFORE transient PNG output setting changes; ' - 'never save the render filepath/encoding overrides. Disable backup save_version. ' - 'Never alter initial.blend, source dataset files or target image. ' - 'Every round must be a distinct meaningful edit, not inspection/no-op or a repeat of any earlier scene state. ' - 'Save absolute paths ' + str(work / 'iteration01.blend') + ' and ' - + str(work / 'iteration01.png') + ' through iteration10.blend and iteration10.png. ' - 'Append rounds.jsonl in this work directory via MCP, one JSON object per completed round: ' - 'round integer 1..10, blend absolute path, png absolute path, rationale explaining the visual ' - 'hypothesis and actual edit. Do not invent camera poses for non-camera tasks. ' - 'Each checkpoint must remain directly openable with dependencies intact. ' - 'Stop only after all ten rounds and their images and ledger rows are written.') - runtime.emit(events, 'generation_started', task=task) - direct.invoke(args, config, evidence, 'run', contract, inp / 'target.png') - if runtime.sha(blend) != initial_hash: - raise ValueError('Codex modified initial.blend') - dataset.verify(task, inp) - score_checkpoints(task, policy, work, out, baseline) - - -def recover_task(task, policy, out, old, dataset): - """Score retained ten-round delivery in place; never regenerate or rewrite it.""" - work = old / 'work' - baseline = json.loads((work / 'metadata.json').read_text()) - if runtime.sha(work / 'initial.blend') != baseline['blend_sha256']: - raise ValueError('modified initial.blend in recovery') - verification = dataset.verify(task, ROOT / 'dataset' / task) - out.mkdir(parents=True, exist_ok=False) - (out / 'input-verification.json').write_text(json.dumps(verification, indent=2)) - (out / 'recovery.json').write_text(json.dumps({'source': str(old), 'regenerated': False})) - policy = dict(policy, task=task, stop_file=str(out.parent / 'STOP')) - score_checkpoints(task, policy, work, out, baseline) - - -def score_checkpoints(task, policy, work, out, baseline): - """Validate and score saved artifacts only after generation has ended.""" - events = out / 'events.jsonl' - inp = ROOT / 'dataset' / task - artifacts = direct.verify_checkpoints(work, 10, policy=policy) - (out / 'artifact-check.json').write_text(json.dumps(artifacts, indent=2)) - runtime.emit(events, 'generation_finished', task=task) - # All external scoring occurs only AFTER the full direct generation session. - previous = baseline - seen = {json.dumps(baseline['audit'], sort_keys=True)} - rows = [] - for n in range(1, 11): - audit = runtime.inspect_render(work / f'iteration{n:02}.blend', policy, - runtime.CUSTOM_BPY_PYTHON, out / f'audit{n:02}', events) - direct.validate_edit(baseline, audit, policy) - direct.validate_edit(previous, audit, policy) - state = json.dumps(audit['audit'], sort_keys=True) - if state in seen: - raise ValueError('repeated scene state') - seen.add(state) - previous = audit - score_dir = out / f'score{n:02}' - score_dir.mkdir() - runtime.run_process([ROOT / 'score-venv/bin/python', ROOT / 'blenderbench-score-pair.py', - audit['png'], inp / 'target.png'], score_dir / 'score.json', events, policy) - score = json.loads((score_dir / 'score.json').read_text()) - rows.append(dict(round=n, **score, blend=audit['blend'], blend_sha256=audit['blend_sha256'])) - (out / 'scores.json').write_text(json.dumps(rows, indent=2)) - best = min(rows, key=lambda row: (row['raw_nclip'], row['round'])) - (out / 'result.json').write_text(json.dumps({'status': 'complete', 'task': task, - 'rounds': rows, 'best': best, 'feedback_to_generation': False, 'vlm_judge': False}, indent=2)) - runtime.emit(events, 'task_complete', task=task, rounds=len(rows)) +Use ``python -m benchmarks.blenderbench_direct.runner`` for new automation. +""" +from __future__ import annotations +from pathlib import Path -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--output', type=Path, required=True) - parser.add_argument('--tasks', nargs='+', help='explicit user-authorized fresh restart subset') - parser.add_argument('--recover-root', type=Path, help='reuse complete ledgers from a stopped predecessor') - args = parser.parse_args() - out = args.output.absolute() - out.mkdir(parents=True, exist_ok=True) - with (ROOT / 'direct-suite.lock').open('a') as lock: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - sys.path.insert(0, str(ROOT / 'input-tools')) - import dataset - tasks = json.loads((ROOT / 'input-tools/task-policies.json').read_text())['tasks'] - plan = task_plan(tasks, args.tasks) - if set(tasks) != set(dataset.task_ids()): - raise ValueError('canonical policy/manifest task mismatch') - config = direct.prepare_config(tomllib.loads((ROOT / 'mghavn-custom-bpy.toml').read_text()), approved=True) - version = subprocess.check_output(['/home/mg/.local/bin/codex', '--version'], text=True).strip() - if version != 'codex-cli 0.154.0': - raise ValueError('unreviewed Codex version') - runtime.emit(out / 'events.jsonl', 'dispatcher_started', pid=os.getpid(), tasks=plan, - backend_deadline_seconds=120, client_deadline_seconds=86400) - (out / 'process.json').write_text(json.dumps({'pid': os.getpid(), 'argv': sys.argv, - 'cwd': os.getcwd(), 'tasks': plan, 'model': 'gpt-6-astra'}, indent=2)) - states = {} - def status(task, state, **extra): - states[task] = dict(status=state, **extra) - temp = out / 'status.tmp' - temp.write_text(json.dumps(states, indent=2)) - temp.replace(out / 'status.json') - runtime.emit(out / 'events.jsonl', state, task=task, **extra) - for task in plan: - if (out / 'STOP').exists(): - break - task_out = out / task.replace('/', '-') - if task_out.exists(): - raise ValueError('fresh task output required; automatic restart forbidden') - # Do not duplicate the active downloader. Wait for its verified checkpoint. - checkpoint = ROOT / 'dataset' / task / 'checkpoint.json' - while True: - if (out / 'STOP').exists(): - raise InterruptedError('manual STOP') - try: - staging = json.loads(checkpoint.read_text()) - except (FileNotFoundError, json.JSONDecodeError): - staging = {} - if staging.get('status') == 'verified': - break - if staging.get('status') == 'failed': - status(task, 'input_failed', error=staging.get('error')) - break - status(task, 'waiting_for_staging') - time.sleep(10) - if staging.get('status') != 'verified': - continue - status(task, 'running', output=str(task_out)) - try: - old = args.recover_root / task.replace('/', '-') if args.recover_root else None - if old is not None and (old / 'work/rounds.jsonl').is_file(): - recover_task(task, tasks[task], task_out, old, dataset) - else: - run_task(task, tasks[task], task_out, config, dataset) - except Exception as exc: - status(task, 'failed', error=repr(exc), output=str(task_out)) - else: - status(task, 'complete', output=str(task_out)) - runtime.emit(out / 'events.jsonl', 'dispatcher_finished', states=states) +from benchmarks.blenderbench_direct.runner import main +from benchmarks.blenderbench_direct.suite import run_suite +ROOT = Path.cwd() -if __name__ == '__main__': - main() +if __name__ == "__main__": + raise SystemExit(main())