From e264110cf0fae609ed8ca8c940156793a66d6ffd Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sat, 17 Oct 2026 17:22:56 -0400 Subject: [PATCH 01/14] fix: preserve tracked source frames through bounded cleaning recovery --- docs/known-limits.md | 1 + docs/paid-recovery.md | 72 +++- orchestrator/api/progress.py | 4 + orchestrator/pipeline.py | 3 + orchestrator/steps.py | 3 +- scripts/clean_candidates.py | 159 ++++++++ scripts/run_clip.py | 234 +++++++++++- scripts/stage_attempts.py | 134 ++++++- scripts/tests/test_clean_candidates.py | 361 ++++++++++++++++++ scripts/tests/test_marble_world.py | 5 + .../tests/test_orchestrator_api_submission.py | 33 ++ scripts/tests/test_pipeline_integration.py | 3 + scripts/tests/test_pipeline_plan.py | 18 + scripts/tests/test_run_clip_resume.py | 2 +- scripts/tests/test_select_world_mode.py | 10 +- scripts/tests/test_source_timing.py | 157 ++++++++ scripts/tests/test_stage_attempts.py | 279 +++++++++++++- scripts/tests/test_tracks_to_clean_masks.py | 322 ++++++++++++++++ scripts/tests/test_vlm_budget.py | 169 ++++++++ scripts/tracks_to_clean_masks.py | 273 +++++++++---- worker/modal_clean_video.py | 53 ++- worker/stages/vlm_budget.py | 116 ++++++ worker/stages/vlm_judge.py | 39 ++ worker/wander_worker/source_timing.py | 87 ++++- 24 files changed, 2419 insertions(+), 118 deletions(-) create mode 100644 scripts/clean_candidates.py create mode 100644 scripts/tests/test_clean_candidates.py create mode 100644 scripts/tests/test_tracks_to_clean_masks.py create mode 100644 scripts/tests/test_vlm_budget.py create mode 100644 worker/stages/vlm_budget.py diff --git a/docs/known-limits.md b/docs/known-limits.md index 31a0698..9b81c12 100644 --- a/docs/known-limits.md +++ b/docs/known-limits.md @@ -4,6 +4,7 @@ Approaches already measured to fail. Do not retry one without a new idea, and add a line here when you measure a new dead end. - Refining body pose against the footage makes it worse: a single camera leaves a null space. +- Semantic person masks can omit lower bodies against similar-colored floors: the fresh bottle-source clean left black and light trousers in several sampled frames despite a successful worker exit. Tracked video cleaning now unions the existing per-person instance masks and retains their exact source-frame selection; equal FPS and frame count do not imply the same pixels (the observed 30-to-12 FPS tracker and cleaner selected different source frames). Missing, empty, or incomplete track masks and mismatched source/timing metadata block submission. Full mask coverage is not an inpainting-quality pass; inspect the resulting clean for remnants and damaged surroundings. Failure evidence: `.context/evidence/bottle-source-rerun/clean-comparison.jpg`. - Faces below about 64 px of face height are unreadable; per-clip head refits do not help. - Smoke, fire, and water cannot be reconstructed from one moving camera. - Clips with too little parallax, or too dark, fail reconstruction. diff --git a/docs/paid-recovery.md b/docs/paid-recovery.md index e5f3fea..ce82772 100644 --- a/docs/paid-recovery.md +++ b/docs/paid-recovery.md @@ -8,6 +8,17 @@ This is not a universal spending cap: direct worker invocations, model-cache sta fine-tuning, OpenAI, and Marble generation are outside this ledger. Reported compute costs are estimates, not independently verified bills. +For a bounded run, `WANDER_VLM_BUDGET` can point to an existing private JSON file with +`schema: "wander.vlm-budget/1"`, positive integer `maxCalls` and `maxCompletionTokens`, an +`allowedModels` list, and an initially empty `calls` list. Calls through +`worker/stages/vlm_judge.py` reserve a request in that file before network access, include the +completion-token limit, and do not retry. Duplicate request bodies are refused across processes +and resumes, including after a failed or ambiguous response. Keep the same file for the whole +run; recreating it would lose its accounting. Recorded usage excludes prompts, images, keys, +and provider error text. This bounds requests and completion tokens for that helper, not input +tokens, monetary charges, or other OpenAI clients. Without the environment variable its existing +behavior is unchanged. + The allowance is three executions per original source SHA-256 and logical stage. The clean, first-frame clean, and multi-image clean operations share the inpainting allowance; individual LHM track suffixes have separate allowances. `--marble both` sequences its two clean operations @@ -16,7 +27,7 @@ allowance. A retry needs a new `--stage-hypothesis` and changed relevant paramet code. Each distinct clean operation can make its initial execution without a retry hypothesis, but all consume the same three-execution allowance. Pending or unknown attempts block further submissions for that source/logical stage until evidence is reconciled. Existing over-cap ledger -history remains readable and reconcilable; it never grants another execution. +history remains readable and reconcilable; history alone never grants another execution. The provider-agnostic `orchestrator/paid.py` policy likewise caps executions across run names by original source and logical stage. It persists retry hypotheses on the associated stage attempt @@ -37,11 +48,70 @@ existing source: a fresh ledger does not establish that no earlier jobs were pai Existing result paths are retained. LHM destinations with compatible recovery receipts use read-only recovery; legacy or incompatible destinations stop for inspection. Other paid stages need retained-output inspection and, when justified, a fresh candidate with the same ledger. +Full-video cleaning also supports the named same-run candidates described below. A successful subprocess is recorded as a completed execution, not a visual quality verdict. Additional selected shots share their original source's allowance and are treated as retries of the same stage. Automatic `--all-shots` processing can therefore stop after the first shot; review subsequent shots separately with distinct hypotheses instead of changing the source hash. +## Explicit one-run operator authorization + +After explicit operator approval, the direct runner accepts `--stage-authorization /absolute/permit.json` +(default `WANDER_STAGE_AUTHORIZATION`). The runner never creates a permit. A reviewed permit may +authorize unchanged parameters or an execution above the normal cap, without resetting history. +It does not bypass unresolved claims, retained-output checks, quality gates, or other provider budgets. +It applies only to this runner's supported paid Modal operations; `--orchestrated` rejects it. + +The JSON must contain exactly these fields: + +- `schema`: `wander.paid-stage-authorization/1`. +- `id`: a unique identifier (a UUID is suitable). +- `sourceSha256`: the canonical original source's lowercase SHA-256, even when submitting an alias. +- `candidate`: the exact runner candidate name. +- `reason`: the reviewed reason for this execution. +- `operatorApproval`: the operator's approval verbatim. +- `expiresAtEpoch`: a finite positive Unix timestamp, later than the claim time. +- `operations`: a nonempty object mapping concrete operation names to nonnegative integer counts + of **prior logical-stage executions** expected at submission. For example, `{"clean_first": 3, + "clean": 4}` authorizes those two operations in order against the shared inpainting count; + it does not authorize a retry of either one. Inspect the actual shared ledger when setting counts. + +Each claim reads and validates the permit under the ledger lock, records its exact file hash, +absolute path, and full contents, and consumes that ID's operation immediately. Failed and unknown +executions still consume it. Reconciliation never restores it. Reusing an ID with modified file +contents is rejected, including attempts to change counts or add operations. Keep the same permit +file unchanged while its distinct approved operations run. Candidate/source mismatches, expired or +malformed permits, unlisted operations, and stale counts stop submission; they never fall back to +the ordinary allowance. The shared ledger remains required across candidates, aliases, and worktrees. + +## Retain a corrected full-video clean within the same run + +Before any Marble submission for the run, `--clean-candidate ` writes a fresh full-video +clean under `clean-candidates//` in that run. The slug uses 1–64 letters, digits, underscores +or hyphens and starts with a letter or digit. Existing candidate directories are retained and +refused. This option supports `--marble video` or `none`; it is rejected with still-image modes, +`--orchestrated`, `--reuse-world`, or retained-world delivery. If `--only` is used, it must include +`clean`. Use `--force clean` when replacing a cached successful clean; changing the slug alone +cannot silently reuse that cached result. + +The fresh directory keeps the video, first frame, output masks, worker report, log, and any locally +generated tracking masks. Original outputs remain unchanged. The ordinary shared paid ledger and +retry/authorization rules still apply to the concrete `clean` operation and the same runner name; +a stage-candidate slug grants no new execution allowance. + +After a successful clean with a matching source/video report, the runner retains a candidate +`selection.json` and atomically writes `clean-selection.json` at the run root. The receipt pins +the actual source SHA-256 and each selected artifact's path and SHA-256. Subsequent review and +Marble commands use that selection without needing the candidate flag again. Missing, modified, +or escaping artifacts and inconsistent receipts fail closed. A failed candidate leaves the prior +selection intact, while the ordinary failed-stage/dependency state continues to block progression. +Normal reruns invalidate downstream results; successful cleaning still requires visual review. + +Recorded Marble requests, uncertain submissions, operations, logs, or generated worlds block +selection replacement before another clean is paid for. A run lock also serializes new Marble +submissions with candidate cleaning. Existing operation polling and explicit world recovery remain +read-only and do not require a new clean. This path cannot replace the input of an already-paid world. + ## Recover LHM outputs without inference The native LHM worker checkpoints generated files and a hash manifest in its Modal volume before diff --git a/orchestrator/api/progress.py b/orchestrator/api/progress.py index 5806df8..4490621 100644 --- a/orchestrator/api/progress.py +++ b/orchestrator/api/progress.py @@ -42,6 +42,10 @@ def previews(root: Path) -> list[dict]: source = Path(manifest.get("source") or root / "missing") candidates = [("Source clip", source, "video")] candidates += [("Cleaned clip", p, "video") for p in (root / "clips").glob("*-clean.mp4")] + candidates += [ + (f"Cleaning candidate: {p.parent.name}", p, "video") + for p in sorted(root.glob("clean-candidates/*/clean.mp4")) + ] for pattern in ( "first/*.png", "review/*.jpg", diff --git a/orchestrator/pipeline.py b/orchestrator/pipeline.py index 86dce9b..a9f6f15 100644 --- a/orchestrator/pipeline.py +++ b/orchestrator/pipeline.py @@ -98,6 +98,9 @@ def multiperson_graph(limit: int, marble: str, objects: bool = True, finetune: b stages, deps, world = world_half(marble) stages += ["pi3x", "frame_align", "tracks"] deps.update({"pi3x": [], "frame_align": ["pi3x"], "tracks": ["pi3x"]}) + if "clean" in deps: + # Video cleaning uses the complete tracked instance masks on their exact source grid. + deps["clean"].append("tracks") motions = [] for i in range(limit): prep, frozen, motion = f"person_prep_{i:02d}", f"lhm_frozen_{i:02d}", f"lhm_motion_{i:02d}" diff --git a/orchestrator/steps.py b/orchestrator/steps.py index 0fe1dc7..f97c1b8 100644 --- a/orchestrator/steps.py +++ b/orchestrator/steps.py @@ -354,7 +354,8 @@ class Step: caution=( "Look at the cleaned frames before spending on a world. Inpainting can take a " "railing or a flight of stairs with the person, and that is only visible to " - "someone looking." + "someone looking. Tracked-person video cleaning waits for tracks and reuses " + "their complete instance masks on the recorded source frame grid." ), ), Step( diff --git a/scripts/clean_candidates.py b/scripts/clean_candidates.py new file mode 100644 index 0000000..444e3bd --- /dev/null +++ b/scripts/clean_candidates.py @@ -0,0 +1,159 @@ +"""Retained full-video clean candidates and a hash-pinned downstream selection.""" + +import fcntl +import hashlib +import json +import os +import re +import tempfile +from contextlib import contextmanager +from pathlib import Path + +SCHEMA = "wander.clean-selection/1" +OUTPUTS = { + "video": "clean.mp4", + "frame": "clean-f0.png", + "masks": "masks.npz", + "report": "clean.json", + "log": "clean.log", +} +INPUTS = {"inputMasks": "track-clean-masks.npz", "inputReport": "track-clean-masks.json"} + + +def sha256(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def validate_slug(slug): + if not isinstance(slug, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}", slug): + raise ValueError( + "--clean-candidate must be a safe 1–64 character slug using letters, digits, _ or -" + ) + return slug + + +def candidate_directory(ctx, slug): + validate_slug(slug) + ctx = Path(ctx).resolve() + parent, directory = ctx / "clean-candidates", ctx / "clean-candidates" / slug + if parent.is_symlink() or directory.is_symlink() or not directory.resolve().is_relative_to(ctx): + raise ValueError("Clean candidate paths must stay inside the run without symlinks") + return directory + + +@contextmanager +def selection_lock(ctx): + """Coordinate clean selection with the runner's Marble submission.""" + path = Path(ctx) / "clean-selection.lock" + if path.is_symlink(): + raise ValueError("Clean selection lock cannot be a symlink") + with path.open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise ValueError("Another clean selection or Marble submission owns this run") from None + yield + + +def _write_json(path, document): + if path.is_symlink(): + raise ValueError("Clean selection receipts cannot be symlinks") + descriptor, temporary = tempfile.mkstemp(prefix=".clean-selection-", dir=path.parent) + try: + with os.fdopen(descriptor, "w") as stream: + json.dump(document, stream, indent=2, allow_nan=False) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + descriptor = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + finally: + Path(temporary).unlink(missing_ok=True) + + +def read_selection(ctx, clip, run_name): + """None means legacy paths; malformed or changed selections never fall back.""" + ctx = Path(ctx).resolve() + receipt = ctx / "clean-selection.json" + if receipt.is_symlink(): + raise ValueError("Clean selection receipt cannot be a symlink") + if not receipt.exists(): + if any(ctx.glob("clean-candidates/*/selection.json")): + raise ValueError( + "Clean selection receipt is missing despite a retained successful candidate" + ) + return None + try: + doc = json.loads(receipt.read_text()) + except (OSError, ValueError) as error: + raise ValueError("Clean selection receipt is unreadable") from error + if ( + not isinstance(doc, dict) + or set(doc) != {"schema", "candidate", "sourceSha256", "runName", "files"} + or doc["schema"] != SCHEMA + or doc["runName"] != run_name + or doc["sourceSha256"] != sha256(clip) + or not isinstance(doc["files"], dict) + or not set(OUTPUTS) <= set(doc["files"]) <= set(OUTPUTS) | set(INPUTS) + ): + raise ValueError("Clean selection receipt has invalid source, scope or files") + directory = candidate_directory(ctx, doc["candidate"]) + history = directory / "selection.json" + try: + if history.is_symlink() or json.loads(history.read_text()) != doc: + raise ValueError("Clean selection does not match its retained success receipt") + except (OSError, ValueError) as error: + raise ValueError("Clean selection does not match its retained success receipt") from error + for key, pin in doc["files"].items(): + expected = directory / (OUTPUTS | INPUTS)[key] + if ( + not isinstance(pin, dict) + or set(pin) != {"path", "sha256"} + or pin["path"] != expected.relative_to(ctx).as_posix() + or expected.is_symlink() + or not expected.is_file() + or expected.stat().st_size == 0 + or pin["sha256"] != sha256(expected) + ): + raise ValueError( + f"Selected clean candidate file changed or escaped its directory: {key}" + ) + return doc + + +def publish_selection(ctx, clip, run_name, slug): + """Call only after paid clean completes, while holding selection_lock.""" + ctx = Path(ctx).resolve() + directory = candidate_directory(ctx, slug) + document = { + "schema": SCHEMA, + "candidate": slug, + "sourceSha256": sha256(clip), + "runName": run_name, + "files": {}, + } + for key, name in (OUTPUTS | INPUTS).items(): + path = directory / name + if key in INPUTS and not path.exists() and not path.is_symlink(): + continue + if path.is_symlink() or not path.is_file() or path.stat().st_size == 0: + raise ValueError(f"Clean candidate is missing a complete retained file: {key}") + document["files"][key] = {"path": path.relative_to(ctx).as_posix(), "sha256": sha256(path)} + report = json.loads((directory / OUTPUTS["report"]).read_text()) + if ( + report.get("error") + or report.get("sourceSha256") != document["sourceSha256"] + or report.get("outputVideoSha256") != document["files"]["video"]["sha256"] + ): + raise ValueError("Clean candidate report does not confirm its source and output video") + # Retain a success receipt with each candidate before replacing the active selection. + history = directory / "selection.json" + if history.exists() or history.is_symlink(): + raise ValueError("Clean candidate already has a retained selection receipt") + _write_json(history, document) + _write_json(ctx / "clean-selection.json", document) + return document diff --git a/scripts/run_clip.py b/scripts/run_clip.py index 1fdb0b6..b39f853 100644 --- a/scripts/run_clip.py +++ b/scripts/run_clip.py @@ -33,6 +33,13 @@ from marble_world import submission_history from package_source_audio import source_audio_manifest from stage_attempts import StageAttempts, command_identity +from clean_candidates import ( + candidate_directory, + publish_selection, + read_selection, + selection_lock, + validate_slug, +) ROOT = Path(__file__).resolve().parent.parent if str(ROOT) not in sys.path: @@ -105,6 +112,24 @@ def validate_retained_scene(args): raise ValueError("--retained-scene requires --skip-finetune") +def validate_clean_candidate(args): + candidate = getattr(args, "clean_candidate", None) + if candidate is None: + return + validate_slug(candidate) + if args.orchestrated: + raise ValueError("--clean-candidate is supported by the direct runner only") + mode = "none" if args.skip_marble else args.marble + if mode not in {"video", "none"}: + raise ValueError("--clean-candidate requires full-video cleaning (--marble video or none)") + if getattr(args, "reuse_world", None) or getattr(args, "retained_scene", None): + raise ValueError("--clean-candidate cannot replace input to an existing or retained world") + if args.only and "clean" not in args.only.split(","): + raise ValueError( + "--clean-candidate requires clean in --only; omit the flag to use the saved selection" + ) + + def parse_object_spec(spec: str) -> dict: """`id:motion:prompt[:roi]` -> a dict for the objects stage. @@ -344,6 +369,11 @@ def __init__( self.clip = dest.resolve() self.video_url = "/" + quote(self.clip.relative_to(public).as_posix()) self.clean_mp4 = CLIPS / f"{self.name}-clean.mp4" + self.clean_report = self.ctx / "clean.json" + self.clean_masks_path = self.ctx / "masks.npz" + self.selected_clean_candidate = None + if not self.marble_recovery_only(): + self.load_clean_selection() self.first_png = self.ctx / "first" / "f_0000.png" self.key = os.environ.get(a.marble_key or "WLT_API_KEY", "") self.people_limit = a.people if a.people else (4 if a.all_people else 0) @@ -400,6 +430,7 @@ def paid_run(self, operation, cmd, log): candidate=getattr(self, "name", self.ctx.name), log=log, results=outputs, + authorization=getattr(self.a, "stage_authorization", None), ) except (OSError, ValueError, KeyError) as error: raise QualityStop("blocked", f"Paid {operation} was not submitted: {error}") from error @@ -529,6 +560,98 @@ def clean_first(self): ) def clean(self): + candidate = getattr(self.a, "clean_candidate", None) + if candidate is None: + if getattr(self, "selected_clean_candidate", None): + raise ValueError( + "A selected clean is retained; use a fresh --clean-candidate for another execution" + ) + return self.clean_to(self.ctx, self.clean_mp4) + with selection_lock(self.ctx): + self.load_clean_selection() + self.assert_clean_selection_replaceable() + directory = candidate_directory(self.ctx, candidate) + directory.parent.mkdir(exist_ok=True) + try: + directory.mkdir() + except FileExistsError: + raise ValueError( + "Clean candidate destination is retained; choose a fresh --clean-candidate" + ) from None + self.clean_to(directory, directory / "clean.mp4") + self.assert_clean_selection_replaceable() + publish_selection(self.ctx, self.clip, self.name, candidate) + self.load_clean_selection() + self.state.record( + "_clean_selection", + candidate=candidate, + receipt=str(self.ctx / "clean-selection.json"), + ) + + def load_clean_selection(self): + selected = read_selection(self.ctx, self.clip, self.name) + if selected: + self.selected_clean_candidate = selected["candidate"] + self.clean_mp4 = self.ctx / selected["files"]["video"]["path"] + self.clean_report = self.ctx / selected["files"]["report"]["path"] + self.clean_masks_path = self.ctx / selected["files"]["masks"]["path"] + + def marble_recovery_only(self): + suffix = {"marble_video": "clean", "marble_image": "image", "marble_multi": "multi"}.get( + getattr(self.a, "only", None) + ) + if suffix is None: + return False + if getattr(self.a, "reuse_world", None): + return True + kind, _ = submission_history(MARBLE_DIR, f"{self.name}-{suffix}") + return bool( + kind in {"world", "operation", "completed"} + or self.operation_id(self.ctx / f"marble_{suffix}.log") + or self.state.data["stages"].get("_marble", {}).get(suffix) + ) + + def assert_clean_selection_replaceable(self): + # Include uncertain/preparing receipts and legacy logs: none authorize replacing + # the input to a world that may already have been paid for. + saved = self.state.data["stages"] + for suffix in ("clean", "image", "multi"): + kind, _ = submission_history(MARBLE_DIR, f"{self.name}-{suffix}") + if ( + kind != "new" + or (self.ctx / f"marble_{suffix}.log").exists() + or (public_dir() / f"marble-{self.name}-{suffix}.spz").exists() + or saved.get("_marble", {}).get(suffix) + or saved.get(f"marble_{'video' if suffix == 'clean' else suffix}", {}).get("status") + in {"ok", "running"} + ): + raise ValueError( + "Cannot replace clean selection after a Marble submission or generated world" + ) + + def clean_to(self, directory, video): + flags = self.clean_flags() + if ( + getattr(self, "multi", False) + and not getattr(self.a, "clean_masks", None) + and not self.a.moved_mask + ): + from tracks_to_clean_masks import build_clean_masks + + masks = directory / "track-clean-masks.npz" + report = build_clean_masks( + self.ctx / "tracks", + clip=self.clip, + out=masks, + width=self.info["width"], + height=self.info["height"], + fps=self.a.fps, + dilate=self.a.dilate, + bottom_extra=self.a.bottom_extra, + ) + self.validate_clean_masks(masks) + (directory / "track-clean-masks.json").write_text(json.dumps(report, indent=2)) + flags += ["--masks-in", str(masks.resolve())] self.paid_run( "clean", [ @@ -538,16 +661,16 @@ def clean(self): "--clip", str(self.clip), "--out", - str(self.clean_mp4), + str(video), "--frame0", - str(self.ctx / "clean-f0.png"), + str(directory / "clean-f0.png"), "--masks", - str(self.ctx / "masks.npz"), + str(directory / "masks.npz"), "--report", - str(self.ctx / "clean.json"), - *self.clean_flags(), + str(directory / "clean.json"), + *flags, ], - self.ctx / "clean.log", + directory / "clean.log", ) def clean_flags(self): @@ -584,6 +707,14 @@ def validate_clean_masks(self, path: Path): with np.load(path, allow_pickle=False) as archive: packed = archive["masks"] declared = archive["shape"] if "shape" in archive.files else None + selection_fields = { + "source_indices", + "source_pts", + "source_time_base", + "source_sha256", + } + selected = selection_fields.intersection(archive.files) + selection = {key: archive[key] for key in selected} except (OSError, ValueError, KeyError) as error: raise ValueError(f"Invalid --clean-masks archive {path}: {error}") from error expected_height, expected_width = self.info["height"], self.info["width"] @@ -603,18 +734,55 @@ def validate_clean_masks(self, path: Path): "--clean-masks shape " f"{tuple(packed.shape)} does not match observed source sampling {expected_shape}" ) - if declared is not None and tuple(declared.tolist()) != ( - expected_shape[0], - expected_height, - expected_width, + if declared is not None and ( + declared.shape != (3,) + or tuple(declared.tolist()) != (expected_shape[0], expected_height, expected_width) ): raise ValueError( "--clean-masks shape metadata does not match the observed sampled frame count " "and source dimensions" ) + if selected: + from worker.wander_worker.source_timing import select_source_frames + + if getattr(self, "marble", getattr(self.a, "marble", None)) == "multi": + raise ValueError("Exact-source --clean-masks are not supported by --marble multi") + if ( + selected != selection_fields + or declared is None + or not np.issubdtype(declared.dtype, np.integer) + ): + raise ValueError( + "Explicit clean masks require complete source frame metadata and shape" + ) + indices, pts = selection["source_indices"], selection["source_pts"] + if any( + value.shape != (expected_shape[0],) or not np.issubdtype(value.dtype, np.integer) + for value in (indices, pts) + ): + raise ValueError( + "Explicit clean mask source indices and PTS must be integer frame arrays" + ) + if any( + selection[key].shape != () or selection[key].dtype.kind != "U" + for key in ("source_time_base", "source_sha256") + ): + raise ValueError( + "Explicit clean mask source hash and timebase must be text scalars" + ) + _, provenance = select_source_frames( + self.clip, self.a.fps, source_indices=indices.tolist() + ) + if ( + selection["source_sha256"].item() != provenance["sourceSha256"] + or selection["source_time_base"].item() != provenance["sourceTimeBase"] + or pts.tolist() != [frame["sourcePts"] for frame in provenance["frames"]] + ): + raise ValueError("Explicit clean masks do not match the selected source pixels") def review(self): """The credit gate. 1600 credits are about to be spent on whatever these frames show.""" + self.load_clean_selection() if self.a.reuse_world: say(" gate not applicable: --reuse-world generates nothing and spends nothing") return @@ -660,7 +828,7 @@ def review(self): shots.append(destination) if not shots: raise RuntimeError("nothing to review: the clean stage produced no frames") - report = self.ctx / "clean.json" + report = getattr(self, "clean_report", self.ctx / "clean.json") if report.exists(): r = json.loads(report.read_text()) say( @@ -720,6 +888,19 @@ def marble_submit(self, input_type, target, suffix, extra=None) -> str: if operation := self.marble_saved_operation(suffix): say(f" recovering recorded operation {operation}; no new generation") return operation + with selection_lock(self.ctx): + return self.marble_submit_locked(input_type, target, suffix, extra) + + def marble_submit_locked(self, input_type, target, suffix, extra=None): + if operation := self.marble_saved_operation(suffix): + say(f" recovering recorded operation {operation}; no new generation") + return operation + if input_type == "video" and suffix == "clean": + self.load_clean_selection() + if Path(target) != self.clean_mp4: + raise ValueError( + "Clean selection changed before Marble submission; inspect and resume" + ) log = self.ctx / f"marble_{suffix}.log" try: run( @@ -878,6 +1059,9 @@ def marble_image(self): self.marble_world("image", self.first_png, "image", extra) def marble_video(self): + if self.a.reuse_world or self.marble_saved_operation("clean"): + return self.marble_world("video", None, "clean") + self.load_clean_selection() extra = ["--prompt-file", str(self.prompt_json)] if self.prompt_json.exists() else [] self.marble_world("video", self.clean_mp4, "clean", extra) @@ -1759,7 +1943,10 @@ def objects_auto(self, world: Path): tracks, scale, masks = self.ctx / "tracks" / "tracks.json", world / "people.json", None else: tracks = world / "person" / "motion.json" - scale, masks = self.scale_file(world), self.ctx / "masks.npz" + scale, masks = ( + self.scale_file(world), + getattr(self, "clean_masks_path", self.ctx / "masks.npz"), + ) if scale is None: raise SkipStage( "no metre scale on disk for this world yet (place_fit has not run), " @@ -2817,6 +3004,16 @@ def summary(self, wall): # ---- scheduler ---------------------------------------------------------- def go(self): a = self.a + candidate = getattr(a, "clean_candidate", None) + if ( + candidate is not None + and self.state.done("clean") + and candidate != getattr(self, "selected_clean_candidate", None) + and "clean" not in (a.force or "").split(",") + ): + raise ValueError( + "Changing --clean-candidate requires --force clean when clean is cached" + ) named = (set(a.only.split(",")) if a.only else set()) | ( set(a.force.split(",")) if a.force else set() ) @@ -3231,6 +3428,11 @@ def main(): "--stage-hypothesis", help="New rationale for a paid retry; relevant parameters or code must also change", ) + ap.add_argument( + "--stage-authorization", + default=os.environ.get("WANDER_STAGE_AUTHORIZATION"), + help="Explicit one-run paid-stage authorization JSON; defaults to WANDER_STAGE_AUTHORIZATION", + ) ap.add_argument( "--marble-key", default="WLT_API_KEY", @@ -3325,6 +3527,11 @@ def main(): help="packed uint8 NPZ masks for the exact observed FFmpeg --fps sampling; skips the " "clean worker's segmentation, dilation, and contact-margin extension", ) + ap.add_argument( + "--clean-candidate", + metavar="SLUG", + help="write a fresh full-video clean under this run and select it after success; requires --force clean when cached", + ) ap.add_argument("--bottom-extra", type=int, default=40) ap.add_argument("--lama-px", type=int, default=960) ap.add_argument( @@ -3448,6 +3655,7 @@ def main(): a = ap.parse_args() try: validate_retained_scene(a) + validate_clean_candidate(a) validate_shared_scale( a.shared_scale, a.people or (4 if a.all_people else 0), orchestrated=a.orchestrated ) @@ -3464,6 +3672,8 @@ def main(): ap.error("--clean-masks cannot be combined with --moved-mask") if a.clean_masks and a.orchestrated: ap.error("--clean-masks is currently supported by the direct runner only") + if a.stage_authorization is not None and a.orchestrated: + ap.error("--stage-authorization is currently supported by the direct runner only") if a.orchestrated: from run_pipeline import main as run_orchestrated diff --git a/scripts/stage_attempts.py b/scripts/stage_attempts.py index 5632077..554da69 100644 --- a/scripts/stage_attempts.py +++ b/scripts/stage_attempts.py @@ -1,7 +1,8 @@ """Persistent allowance for runner-controlled paid Modal executions. -The three-execution ceiling is per original source SHA and logical stage, across candidates. +The default three-execution ceiling is per original source SHA and logical stage, across candidates. Inpainting operations share one allowance; individual LHM track suffixes have separate allowances. +An explicit, bounded operator authorization may grant a single next execution of each named operation. This does not count direct experiment scripts, SSH fine-tuning, OpenAI, or Marble generation. The orchestrator must account for those separately. Costs here are evidence, not dollar budgets. @@ -29,6 +30,17 @@ from pathlib import Path SCHEMA = "wander.pipeline-attempts/1" +AUTHORIZATION_SCHEMA = "wander.paid-stage-authorization/1" +AUTHORIZATION_FIELDS = { + "schema", + "id", + "sourceSha256", + "candidate", + "reason", + "operatorApproval", + "expiresAtEpoch", + "operations", +} MAX_EXECUTIONS = 3 INITIAL_HYPOTHESIS = "Initial paid execution using the recorded source, parameters, and code." STATUSES = {"pending", "unknown", "completed", "failed"} @@ -49,6 +61,65 @@ def logical_stage(operation): return "inpainting" if operation in {"clean", "clean_first", "clean_multi"} else operation +def _validate_authorization(permit): + """Validate scope without checking time, so expired claims remain reconcilable.""" + if not isinstance(permit, dict) or set(permit) != AUTHORIZATION_FIELDS: + raise ValueError("Malformed paid-stage authorization fields") + if ( + permit["schema"] != AUTHORIZATION_SCHEMA + or not isinstance(permit["id"], str) + or not re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", permit["id"]) + or not isinstance(permit["sourceSha256"], str) + or not re.fullmatch(r"[0-9a-f]{64}", permit["sourceSha256"]) + or any( + not isinstance(permit[key], str) or not permit[key].strip() + for key in ("candidate", "reason", "operatorApproval") + ) + ): + raise ValueError("Malformed paid-stage authorization identity or approval") + expiry = permit["expiresAtEpoch"] + try: + valid_expiry = type(expiry) in (int, float) and math.isfinite(expiry) and expiry > 0 + except OverflowError: + valid_expiry = False + if not valid_expiry: + raise ValueError("Paid-stage authorization expiry must be finite and positive") + operations = permit["operations"] + if ( + not isinstance(operations, dict) + or not operations + or any( + not isinstance(operation, str) + or not re.fullmatch(r"[A-Za-z0-9_.-]+", operation) + or operation == "inpainting" + or type(count) is not int + or count < 0 + for operation, count in operations.items() + ) + ): + raise ValueError("Paid-stage authorization requires concrete operations and integer counts") + + +def _unique_json_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("Duplicate paid-stage authorization JSON key") + result[key] = value + return result + + +def _read_authorization(path): + path = Path(path).resolve() + try: + raw = path.read_bytes() + permit = json.loads(raw, object_pairs_hook=_unique_json_keys) + except (OSError, ValueError): + raise ValueError("Paid-stage authorization is unreadable or malformed") from None + _validate_authorization(permit) + return {**permit, "path": str(path), "sha256": hashlib.sha256(raw).hexdigest()} + + CODE_FILES = { "worker/modal_clean_video.py": ("worker/wander_worker/masks.py",), "worker/modal_motion.py": ( @@ -272,6 +343,7 @@ def _validate(data): ): raise ValueError("Invalid paid-stage source aliases") identifiers, counts, direct_claim_sources = set(), {}, set() + authorization_hashes, authorized_operations = {}, set() for attempt in data["attempts"]: if not isinstance(attempt, dict): raise ValueError("Malformed saved paid-stage attempt") # noqa: TRY004 - malformed persisted data @@ -315,6 +387,29 @@ def _validate(data): # capped in begin(), but an existing fourth execution must never disappear. if claim.get("number") != counts[key]: raise ValueError("Invalid saved paid-stage execution count") + if "authorization" in claim: + permit = claim["authorization"] + if ( + not isinstance(permit, dict) + or set(permit) != AUTHORIZATION_FIELDS | {"path", "sha256"} + or not isinstance(permit["path"], str) + or not Path(permit["path"]).is_absolute() + or not isinstance(permit["sha256"], str) + or not re.fullmatch(r"[0-9a-f]{64}", permit["sha256"]) + ): + raise ValueError("Malformed saved paid-stage authorization") + _validate_authorization({key: permit[key] for key in AUTHORIZATION_FIELDS}) + permit_operation = permit["id"], claim.get("operation") + if ( + permit["sourceSha256"] != claim["sourceSha256"] + or permit["candidate"] != claim.get("candidate") + or permit["operations"].get(claim.get("operation")) != counts[key] - 1 + or authorization_hashes.get(permit["id"], permit["sha256"]) != permit["sha256"] + or permit_operation in authorized_operations + ): + raise ValueError("Invalid saved paid-stage authorization scope or reuse") + authorization_hashes[permit["id"]] = permit["sha256"] + authorized_operations.add(permit_operation) if direct_claim_sources.intersection(aliases): raise ValueError("Cannot alias a source hash that already owns paid-stage claims") @@ -329,6 +424,7 @@ def begin( candidate, log, results, + authorization=None, ): if not isinstance(source_sha256, str) or not re.fullmatch(r"[0-9a-f]{64}", source_sha256): raise ValueError("Original source SHA-256 is required for paid execution") @@ -359,14 +455,44 @@ def begin( raise ValueError( "Paid execution requires evidence reconciliation of pending/unknown claims" ) - if len(prior) >= MAX_EXECUTIONS: + permit = None + if authorization is not None: + # Read and validate under the ledger lock: a concurrent claim must consume + # the approved count before another caller can inspect it. + permit = _read_authorization(authorization) + if permit["expiresAtEpoch"] <= time.time(): + raise ValueError("Paid-stage authorization has expired") + if permit["sourceSha256"] != source_sha256: + raise ValueError( + "Paid-stage authorization source must match the canonical source" + ) + if permit["candidate"] != str(candidate): + raise ValueError("Paid-stage authorization candidate does not match") + if operation not in permit["operations"]: + raise ValueError("Paid-stage authorization does not include this operation") + for item in data["attempts"]: + used = item["claim"].get("authorization") + if used and used["id"] == permit["id"]: + if used["sha256"] != permit["sha256"]: + raise ValueError( + "Paid-stage authorization ID was already bound to another file hash" + ) + if item["claim"]["operation"] == operation: + raise ValueError( + "Paid-stage authorization operation is already consumed" + ) + if permit["operations"][operation] != len(prior): + raise ValueError( + "Paid-stage authorization prior execution count does not match" + ) + if permit is None and len(prior) >= MAX_EXECUTIONS: raise ValueError( "Paid-stage allowance exhausted: three total executions per source/stage" ) # Different requested workflows (such as --marble both) can each make their # initial execution, but consume the same logical-stage allowance. retries = [item for item in prior if item["claim"]["operation"] == operation] - if retries: + if retries and permit is None: if not explicit: raise ValueError("Paid retry requires an explicit new --stage-hypothesis") normalized = " ".join(hypothesis.casefold().split()) @@ -404,6 +530,8 @@ def begin( } ], } + if permit is not None: + attempt["claim"]["authorization"] = permit data["attempts"].append(attempt) return attempt diff --git a/scripts/tests/test_clean_candidates.py b/scripts/tests/test_clean_candidates.py new file mode 100644 index 0000000..e318b85 --- /dev/null +++ b/scripts/tests/test_clean_candidates.py @@ -0,0 +1,361 @@ +"""No-spend retained cleaning recovery, selection integrity and downstream contracts.""" + +import json +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import clean_candidates +import run_clip +from stage_attempts import StageAttempts + + +@pytest.fixture +def pipeline(tmp_path, monkeypatch): + p = object.__new__(run_clip.Pipeline) + p.ctx = tmp_path / "fixture" + p.ctx.mkdir() + p.name = "fixture" + p.clip = tmp_path / "source.mp4" + p.clip.write_bytes(b"original source") + p.info = {"width": 11, "height": 4} + p.multi = False + p.marble = "video" + p.clean_mp4 = tmp_path / "original-clean.mp4" + p.clean_report = p.ctx / "clean.json" + p.clean_masks_path = p.ctx / "masks.npz" + p.first_png = p.ctx / "first/f_0000.png" + p.selected_clean_candidate = None + p.a = SimpleNamespace( + clean_candidate="instance-masks-v1", + clean_masks=None, + moved_mask=False, + fps=12, + dilate=20, + bottom_extra=40, + lama_px=960, + only="clean", + force="clean", + skip_finetune=True, + reuse_world=None, + gate_pass=True, + stage_hypothesis="Use corrected instance masks", + stage_ledger=str(tmp_path / "ledger.json"), + ) + p.state = run_clip.State(p.ctx / "state.json") + p.state.record("clean", status="ok") + p.originals = {} + for path in ( + p.clean_mp4, + p.clean_report, + p.clean_masks_path, + p.ctx / "clean-f0.png", + p.ctx / "clean.log", + ): + path.write_bytes(b"retained original " + path.name.encode()) + p.originals[path] = path.read_bytes() + monkeypatch.setattr(run_clip, "MARBLE_DIR", tmp_path / "marble") + monkeypatch.setenv("WANDER_PUBLIC_DIR", str(tmp_path / "public")) + p.paid_run = Mock( + side_effect=lambda operation, command, log: write_clean_outputs(p, command, log) + ) + return p + + +def write_clean_outputs(p, command, log): + outputs = { + flag: Path(command[command.index(flag) + 1]) + for flag in ("--out", "--frame0", "--masks", "--report") + } + outputs["--out"].write_bytes(b"corrected clean video") + outputs["--frame0"].write_bytes(b"clean first frame") + outputs["--masks"].write_bytes(b"packed clean masks") + outputs["--report"].write_text( + json.dumps( + { + "sourceSha256": run_clip.file_sha256(p.clip), + "outputVideoSha256": run_clip.file_sha256(outputs["--out"]), + "error": None, + } + ) + ) + Path(log).write_text("completed provider output\n") + + +def assert_originals(p): + for path, content in p.originals.items(): + assert path.read_bytes() == content + + +def test_success_selects_fresh_outputs_and_preserves_originals(pipeline): + p = pipeline + p.clean() + p.paid_run.assert_called_once() + directory = p.ctx / "clean-candidates/instance-masks-v1" + assert p.clean_mp4 == directory / "clean.mp4" + assert p.clean_report == directory / "clean.json" + assert p.clean_masks_path == directory / "masks.npz" + receipt = json.loads((p.ctx / "clean-selection.json").read_text()) + assert receipt == json.loads((directory / "selection.json").read_text()) + assert receipt["sourceSha256"] == run_clip.file_sha256(p.clip) + assert set(receipt["files"]) == set(clean_candidates.OUTPUTS) + assert_originals(p) + # A fresh runner object loads the successful selection without needing the option again. + resumed = object.__new__(run_clip.Pipeline) + resumed.ctx, resumed.clip, resumed.name = p.ctx, p.clip, p.name + resumed.load_clean_selection() + assert resumed.clean_mp4 == p.clean_mp4 + + +def test_failed_candidate_keeps_previous_selection_and_all_artifacts(pipeline): + p = pipeline + p.clean() + old_receipt = (p.ctx / "clean-selection.json").read_bytes() + old_video = p.clean_mp4 + p.a.clean_candidate = "second-candidate" + p.paid_run.side_effect = RuntimeError("provider failed after launch") + with pytest.raises(RuntimeError, match="provider failed"): + p.clean() + assert p.clean_mp4 == old_video + assert (p.ctx / "clean-selection.json").read_bytes() == old_receipt + assert (p.ctx / "clean-candidates/second-candidate").is_dir() + assert_originals(p) + + +@pytest.mark.parametrize("collision", ["directory", "file", "symlink", "parent-symlink"]) +def test_destination_collision_stops_before_paid_call(pipeline, collision, tmp_path): + p = pipeline + directory = p.ctx / "clean-candidates/instance-masks-v1" + if collision == "parent-symlink": + (p.ctx / "clean-candidates").symlink_to(tmp_path, target_is_directory=True) + else: + directory.parent.mkdir() + if collision == "directory": + directory.mkdir() + elif collision == "file": + directory.write_bytes(b"existing file") + else: + directory.symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(ValueError): + p.clean() + p.paid_run.assert_not_called() + assert_originals(p) + + +@pytest.mark.parametrize( + "tamper", ["video", "source", "receipt", "path", "missing", "file-symlink", "candidate-symlink"] +) +def test_selected_hashes_and_path_scope_fail_closed(pipeline, tamper, tmp_path): + p = pipeline + p.clean() + receipt = p.ctx / "clean-selection.json" + directory = p.clean_mp4.parent + if tamper == "video": + p.clean_mp4.write_bytes(b"changed") + elif tamper == "source": + p.clip.write_bytes(b"different source") + elif tamper == "receipt": + receipt.write_text("{broken") + elif tamper == "path": + doc = json.loads(receipt.read_text()) + doc["files"]["video"]["path"] = "../outside.mp4" + receipt.write_text(json.dumps(doc)) + (directory / "selection.json").write_text(json.dumps(doc)) + elif tamper == "missing": + receipt.unlink() + elif tamper == "file-symlink": + external = tmp_path / "outside.mp4" + external.write_bytes(p.clean_mp4.read_bytes()) + p.clean_mp4.unlink() + p.clean_mp4.symlink_to(external) + else: + moved = tmp_path / "moved-candidate" + directory.rename(moved) + directory.symlink_to(moved, target_is_directory=True) + with pytest.raises(ValueError): + p.load_clean_selection() + + +@pytest.mark.parametrize("evidence", ["generation", "world", "log", "state", "spz"]) +def test_any_prior_world_submission_blocks_clean_replacement(pipeline, evidence): + p = pipeline + run_clip.MARBLE_DIR.mkdir() + if evidence == "generation": + (run_clip.MARBLE_DIR / "fixture-clean-generation.json").write_text('{"status":"preparing"}') + elif evidence == "world": + (run_clip.MARBLE_DIR / "fixture-image-world.json").write_text("{}") + elif evidence == "log": + (p.ctx / "marble_clean.log").write_text("outcome unknown") + elif evidence == "state": + p.state.record("_marble", clean="recorded-operation") + else: + run_clip.public_dir().mkdir() + (run_clip.public_dir() / "marble-fixture-clean.spz").write_bytes(b"paid world") + with pytest.raises(ValueError, match="Marble submission"): + p.clean() + p.paid_run.assert_not_called() + assert not (p.ctx / "clean-candidates").exists() + assert_originals(p) + + +def test_changed_candidate_requires_force_before_cached_clean_can_be_skipped(pipeline): + p = pipeline + p.a.force = None + with pytest.raises(ValueError, match="--force clean"): + p.go() + p.paid_run.assert_not_called() + + +def test_retained_candidate_does_not_reset_paid_allowance(pipeline, monkeypatch): + p = pipeline + ledger = StageAttempts(p.a.stage_ledger) + for index in range(3): + attempt = ledger.begin( + run_clip.file_sha256(p.clip), + "clean", + parameters={"version": index}, + code_version={"worker": "test"}, + hypothesis=f"test {index}", + candidate=p.name, + log=p.ctx / "old.log", + results=[], + ) + ledger.finish(attempt["id"], status="failed", evidence=[]) + p.paid_run = run_clip.Pipeline.paid_run.__get__(p) + provider = Mock() + monkeypatch.setattr(run_clip, "run", provider) + with pytest.raises(run_clip.QualityStop, match="three total"): + p.clean() + provider.assert_not_called() + assert len(json.loads(ledger.path.read_text())["attempts"]) == 3 + assert not (p.ctx / "clean-selection.json").exists() + assert_originals(p) + + +def test_review_and_marble_use_selected_success(pipeline, monkeypatch): + p = pipeline + p.clean() + video = p.clean_mp4 + monkeypatch.setattr(run_clip, "probe", lambda _: {"frames": 2, "duration": 1, "fps": 2}) + commands = [] + + def extract(command, **kwargs): + commands.append(command) + Path(command[-1]).write_bytes(b"review frame") + + monkeypatch.setattr(run_clip.subprocess, "run", extract) + p.review() + assert all(command[command.index("-i") + 1] == str(video) for command in commands) + p.marble_world = Mock() + p.marble_video() + assert p.marble_world.call_args.args[1] == video + + +def test_selection_lock_prevents_concurrent_submission_or_candidate(pipeline): + p = pipeline + with clean_candidates.selection_lock(p.ctx), pytest.raises(ValueError, match="owns this run"): + p.clean() + p.paid_run.assert_not_called() + + +@pytest.mark.parametrize("slug", ["../escape", "a/b", ".hidden", "", "has space", "x" * 65]) +def test_candidate_slug_is_bounded(slug): + with pytest.raises(ValueError): + clean_candidates.validate_slug(slug) + + +@pytest.mark.parametrize( + "changes", + [ + {"marble": "multi"}, + {"marble": "image"}, + {"marble": "both"}, + {"orchestrated": True}, + {"reuse_world": "world"}, + {"only": "review"}, + ], +) +def test_incompatible_candidate_options_rejected(changes): + args = SimpleNamespace( + clean_candidate="new", + orchestrated=False, + skip_marble=False, + marble="video", + only="clean", + reuse_world=None, + retained_scene=None, + ) + for key, value in changes.items(): + setattr(args, key, value) + with pytest.raises(ValueError): + run_clip.validate_clean_candidate(args) + + +@pytest.mark.parametrize("recovery", ["operation", "world", "explicit-world"]) +def test_real_constructor_allows_only_existing_marble_recovery_with_bad_clean_selection( + pipeline, monkeypatch, recovery +): + p = pipeline + p.clean() + p.clean_mp4.write_bytes(b"damaged old clean") + run_clip.MARBLE_DIR.mkdir() + if recovery == "operation": + (run_clip.MARBLE_DIR / "fixture-clean-generation.json").write_text( + '{"operation_id":"existing-op"}' + ) + elif recovery == "world": + (run_clip.MARBLE_DIR / "fixture-clean-world.json").write_text( + '{"world_id":"existing-world"}' + ) + # Construct the actual runner against the retained directory. Model/provider interfaces + # are never used; only the local video probe and certificate command are substituted. + monkeypatch.setattr(run_clip, "runs_dir", lambda: p.ctx.parent) + monkeypatch.setattr(run_clip, "probe", lambda _: p.info) + monkeypatch.setattr( + run_clip.subprocess, "run", lambda *a, **kw: SimpleNamespace(stdout="/local/certificates\n") + ) + args = SimpleNamespace( + clip=str(p.clip), + name=p.ctx.name, + fps=12, + marble_key="UNUSED_TEST_KEY", + people=0, + all_people=False, + skip_marble=False, + marble="video", + object=[], + no_objects=True, + skip_finetune=True, + only="marble_video", + reuse_world="existing-world" if recovery == "explicit-world" else None, + ) + constructed = run_clip.Pipeline(args, clip=p.clip, name=p.name) + assert constructed.selected_clean_candidate is None + if recovery == "operation": + constructed.marble_poll = Mock() + constructed.key = "synthetic-unused" + constructed.marble_video() + constructed.marble_poll.assert_called_once_with("video", "existing-op", "clean") + + +@pytest.mark.parametrize("only", ["marble_video", "review,marble_video"]) +def test_real_constructor_still_validates_selection_for_new_or_mixed_submission( + pipeline, monkeypatch, only +): + p = pipeline + p.clean() + p.clean_mp4.write_bytes(b"damaged old clean") + monkeypatch.setattr(run_clip, "runs_dir", lambda: p.ctx.parent) + monkeypatch.setattr(run_clip, "probe", lambda _: p.info) + args = SimpleNamespace(clip=str(p.clip), name=p.name, only=only, reuse_world=None) + if only != "marble_video": + run_clip.MARBLE_DIR.mkdir() + (run_clip.MARBLE_DIR / "fixture-clean-generation.json").write_text( + '{"operation_id":"existing-op"}' + ) + with pytest.raises(ValueError, match="changed"): + run_clip.Pipeline(args, clip=p.clip, name=p.name) diff --git a/scripts/tests/test_marble_world.py b/scripts/tests/test_marble_world.py index 7750620..ee81499 100644 --- a/scripts/tests/test_marble_world.py +++ b/scripts/tests/test_marble_world.py @@ -501,6 +501,11 @@ def test_pipeline_commands_keep_suffixes_and_recover_without_resubmitting(self): pipeline = object.__new__(run_clip.Pipeline) pipeline.ctx = self.root pipeline.name = "fixture" + pipeline.clip = self.root / "source.mp4" + pipeline.clip.write_bytes(b"original source fixture") + # The command-shape fixture uses the same synthetic input for image/video; + # a new video submission must still match the runner's selected clean path. + pipeline.clean_mp4 = self.png pipeline.key = "test-placeholder" pipeline.certs = "test-cert" pipeline.a = SimpleNamespace( diff --git a/scripts/tests/test_orchestrator_api_submission.py b/scripts/tests/test_orchestrator_api_submission.py index 359564a..fb50b6c 100644 --- a/scripts/tests/test_orchestrator_api_submission.py +++ b/scripts/tests/test_orchestrator_api_submission.py @@ -89,6 +89,39 @@ def test_monetary_cap_is_rejected_before_source_storage(self): self.assert_no_submission() self.assertFalse(list((self.root / "artifacts").rglob("*"))) + def test_clean_candidates_are_previewable_without_exposing_other_candidate_files(self): + response = self.upload("clean-recovery") + self.assertEqual(response.status_code, 201, response.text) + run = self.manifests()[0].parent + candidate = run / "clean-candidates" / "tracked-masks" + candidate.mkdir(parents=True) + (candidate / "clean.mp4").write_bytes(VIDEO) + (candidate / "clean.log").write_text("private provider diagnostics") + outside = self.root / "outside-video.mp4" + outside.write_bytes(VIDEO) + escaped = run / "clean-candidates" / "escape" + escaped.mkdir() + (escaped / "clean.mp4").symlink_to(outside) + + from orchestrator.api.progress import previews + + media = previews(run) + preview = next(p for p in media if p["path"].startswith("clean-candidates/")) + self.assertEqual(preview["label"], "Cleaning candidate: tracked-masks") + self.assertEqual(preview["path"], "clean-candidates/tracked-masks/clean.mp4") + self.assertEqual(len([p for p in media if p["path"].startswith("clean-candidates/")]), 1) + video = self.client.get( + "/api/pipeline/runs/clean-recovery/files/clean-candidates/tracked-masks/clean.mp4", + headers=AUTH, + ) + self.assertEqual(video.status_code, 200, video.text) + self.assertEqual(video.content, VIDEO) + for path in ("tracked-masks/clean.log", "escape/clean.mp4"): + denied = self.client.get( + f"/api/pipeline/runs/clean-recovery/files/clean-candidates/{path}", headers=AUTH + ) + self.assertEqual(denied.status_code, 404, denied.text) + def manifests(self): return list((self.root / "runs").glob("*/run.json")) diff --git a/scripts/tests/test_pipeline_integration.py b/scripts/tests/test_pipeline_integration.py index 1d45f55..2f3ba03 100644 --- a/scripts/tests/test_pipeline_integration.py +++ b/scripts/tests/test_pipeline_integration.py @@ -276,6 +276,9 @@ def test_multi_review_uses_hashed_cleaned_selection(tmp_path): runner = run_clip.Pipeline.__new__(run_clip.Pipeline) runner.ctx = tmp_path + runner.name = "fixture" + runner.clip = tmp_path / "source.mp4" + runner.clip.write_bytes(b"original source fixture") runner.state = run_clip.State(tmp_path / "state.json") runner.marble = "multi" runner.a = SimpleNamespace(reuse_world=None, gate_pass=False, no_gate=False) diff --git a/scripts/tests/test_pipeline_plan.py b/scripts/tests/test_pipeline_plan.py index 2290530..5e41e8c 100644 --- a/scripts/tests/test_pipeline_plan.py +++ b/scripts/tests/test_pipeline_plan.py @@ -85,6 +85,24 @@ def test_object_detection_and_review_wait_for_required_outputs(): assert "finetune" not in many["verify"] +@pytest.mark.parametrize("mode", ["video", "both", "none"]) +def test_tracked_video_clean_waits_for_camera_and_tracks(mode): + _, dependencies = multiperson_graph(2, mode) + order = list(TopologicalSorter(dependencies).static_order()) + assert "tracks" in dependencies["clean"] + assert order.index("pi3x") < order.index("tracks") < order.index("clean") + _, single = single_graph(mode) + assert "tracks" not in single["clean"] + + +@pytest.mark.parametrize("mode,operation", [("image", "clean_first"), ("multi", "clean_multi")]) +def test_still_image_cleaning_keeps_its_existing_prerequisites(mode, operation): + _, many = multiperson_graph(2, mode) + _, single = single_graph(mode) + assert many[operation] == single[operation] + assert "tracks" not in many[operation] + + @pytest.mark.parametrize("mode", MARBLE_MODES) @pytest.mark.parametrize("many,people", [(False, 1), (True, 1), (True, 3)]) @pytest.mark.parametrize("objects,finetune", itertools.product((False, True), (False, True))) diff --git a/scripts/tests/test_run_clip_resume.py b/scripts/tests/test_run_clip_resume.py index a8835b9..1bdc2b9 100644 --- a/scripts/tests/test_run_clip_resume.py +++ b/scripts/tests/test_run_clip_resume.py @@ -197,7 +197,7 @@ def execute(): assert calls == [] p.a.force = "tracks" assert p.go() == 0 - expected = set(p.stages) - {"clean", "pi3x", "frame_align"} + expected = set(p.stages) - {"pi3x", "frame_align"} assert set(calls) == expected assert all(p.state.done(stage) for stage in skipped) diff --git a/scripts/tests/test_select_world_mode.py b/scripts/tests/test_select_world_mode.py index fcf871e..8184fe4 100644 --- a/scripts/tests/test_select_world_mode.py +++ b/scripts/tests/test_select_world_mode.py @@ -14,6 +14,7 @@ import sys import tempfile import unittest +from types import SimpleNamespace from unittest.mock import patch import run_clip @@ -104,10 +105,17 @@ def test_runner_cli_defaults_to_video_without_launching_work(self): def test_pipeline_forwards_the_clean_video_and_generated_description(self): pipeline = object.__new__(run_clip.Pipeline) pipeline.ctx = self.root + pipeline.name = "fixture" + pipeline.clip = self.root / "source.mp4" + pipeline.clip.write_bytes(b"original source fixture") + pipeline.a = SimpleNamespace(reuse_world=None) pipeline.clean_mp4 = self.root / "clean.mp4" pipeline.clean_mp4.write_bytes(b"complete cleaned clip") pipeline.prompt_json.write_text('{"text_prompt": "Source description"}') - with patch.object(pipeline, "marble_world") as generate: + with ( + patch.object(run_clip, "MARBLE_DIR", self.root / "marble"), + patch.object(pipeline, "marble_world") as generate, + ): pipeline.marble_video() generate.assert_called_once_with( "video", pipeline.clean_mp4, "clean", ["--prompt-file", str(pipeline.prompt_json)] diff --git a/scripts/tests/test_source_timing.py b/scripts/tests/test_source_timing.py index 74914fb..46e6cee 100644 --- a/scripts/tests/test_source_timing.py +++ b/scripts/tests/test_source_timing.py @@ -389,6 +389,163 @@ def test_runner_existing_world_and_operation_recovery_need_no_clean_inputs(self) invoke.assert_not_called() self.assertEqual(list(root.iterdir()), []) + def test_explicit_selection_keeps_exact_pixels_on_vfr_nonzero_pts(self): + with tempfile.TemporaryDirectory() as directory: + clip = self.make_clip(Path(directory), "30", 18, "setpts=5/TB+N*N/(30*TB)") + selection = [0, 2, 5, 8, 10, 12, 17] + raw, provenance = source_timing.select_source_frames( + clip, source_indices=selection, width=16, height=16 + ) + pixels = np.frombuffer(raw, np.uint8).reshape(-1, 16, 16, 3) + self.assertEqual(pixels[:, 0, 0, 0].tolist(), selection) + self.assertEqual( + raw, + b"".join(bytes([i, 255 - i, i // 2]) * 16 * 16 for i in selection), + ) + probe = json.loads( + subprocess.run( + [ + "ffprobe", + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "frame=pts", + "-of", + "json", + str(clip), + ], + check=True, + capture_output=True, + timeout=30, + ).stdout + ) + self.assertEqual( + [f["sourcePts"] for f in provenance["frames"]], + [probe["frames"][i]["pts"] for i in selection], + ) + self.assertEqual([f["sourceIndex"] for f in provenance["frames"]], selection) + self.assertEqual([f["frameIndex"] for f in provenance["frames"]], list(range(7))) + self.assertGreater(provenance["sourceFirstPts"], 0) + self.assertEqual( + provenance["sourceSha256"], hashlib.sha256(clip.read_bytes()).hexdigest() + ) + _, mapping = source_timing.select_source_frames(clip, source_indices=selection) + self.assertEqual(mapping["frames"], provenance["frames"]) + for indices in (None, [], [1, 1], [2, 1], [-1], [18], [True], [1.0]): + with self.subTest(indices=indices), self.assertRaises(ValueError): + source_timing.select_source_frames(clip, source_indices=indices) + for options in ( + {"fps": 0}, + {"fps": float("nan")}, + {"width": 1}, + {"width": 1.5, "height": 2}, + ): + with self.subTest(options=options), self.assertRaises(ValueError): + source_timing.select_source_frames(clip, source_indices=[0], **options) + + def test_explicit_worker_metadata_validated_before_inpaint(self): + import modal_clean_video + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + clip = self.make_clip(root, "30", 18) + selection = [0, 2, 5, 8, 10, 12] + _, provenance = source_timing.select_source_frames(clip, source_indices=selection) + base = { + "masks": np.packbits(np.zeros((6, 16, 16), bool), axis=-1), + "shape": np.array([6, 16, 16]), + "source_indices": np.array(selection), + "source_pts": np.array([f["sourcePts"] for f in provenance["frames"]]), + "source_time_base": np.array(provenance["sourceTimeBase"]), + "source_sha256": np.array(provenance["sourceSha256"]), + } + cases = [ + {}, + {"source_indices": None}, + {"source_pts": np.array([0] * 6)}, + {"source_time_base": np.array("1/1")}, + {"source_sha256": np.array("0" * 64)}, + {"shape": np.array([6, 15, 16])}, + {"masks": np.zeros((5, 16, 2), np.uint8)}, + ] + for number, changes in enumerate(cases): + archive = dict(base) + archive.update(changes) + archive = {k: v for k, v in archive.items() if v is not None} + buf = io.BytesIO() + np.savez_compressed(buf, **archive) + worker_root = root / f"worker-{number}" + worker_root.mkdir() + load_lama = Mock(return_value=None) + modules = { + "cv2": SimpleNamespace(), + "torch": SimpleNamespace( + cuda=SimpleNamespace(is_available=lambda: True), device=str + ), + "simple_lama_inpainting": SimpleNamespace(SimpleLama=load_lama), + "wander_worker.masks": SimpleNamespace( + moved_content_masks=None, people_masks=None + ), + } + with ( + self.subTest(changes=list(changes)), + patch.dict(sys.modules, modules), + patch.object(modal_clean_video.cache, "commit"), + patch("tempfile.mkdtemp", return_value=str(worker_root)), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + result = modal_clean_video.clean.local( + clip.read_bytes(), + fps=12, + width=16, + height=16, + masks_npz=buf.getvalue(), + encode=False, + return_frames=True, + ) + if changes: + self.assertIsNotNone(result["report"]["error"]) + load_lama.assert_not_called() + else: + self.assertIsNone(result["report"]["error"]) + load_lama.assert_called_once() + self.assertEqual(result["report"]["indices"], selection) + with tarfile.open(fileobj=io.BytesIO(result["frames"])) as frames: + for i, source_index in enumerate(selection): + with frames.extractfile(f"f_{i:04d}.png") as image: + self.assertEqual( + Image.open(image).getpixel((0, 0))[0], source_index + ) + + def test_explicit_selection_rejects_changed_checksum_and_source(self): + with tempfile.TemporaryDirectory() as directory: + clip = self.make_clip(Path(directory), "30", 18) + run = source_timing._run + + def damaged(command): + result = run(command) + if command[0] == "ffmpeg" and command[-1] == "-": + result.stderr = re.sub( + rb"(\[showinfo@sampled[^\n]*? checksum:)[0-9A-F]+", + rb"\g<1>BAD00000", + result.stderr, + ) + return result + + with ( + patch.object(source_timing, "_run", side_effect=damaged), + self.assertRaisesRegex(ValueError, "emitted frame"), + ): + source_timing.select_source_frames(clip, source_indices=[0, 2]) + with ( + patch.object(source_timing, "_sha256", side_effect=["a", "b"]), + self.assertRaisesRegex(ValueError, "Source changed"), + ): + source_timing.select_source_frames(clip, source_indices=[0, 2]) + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_stage_attempts.py b/scripts/tests/test_stage_attempts.py index 5414f3b..de7f806 100644 --- a/scripts/tests/test_stage_attempts.py +++ b/scripts/tests/test_stage_attempts.py @@ -9,15 +9,17 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import json +import os import tempfile import threading +import time import unittest from pathlib import Path from types import SimpleNamespace from unittest.mock import patch import run_clip -from stage_attempts import CODE_FILES, StageAttempts, command_identity +from stage_attempts import AUTHORIZATION_SCHEMA, CODE_FILES, StageAttempts, command_identity SOURCE = "a" * 64 @@ -31,9 +33,17 @@ def setUp(self): self.log = self.root / "worker.log" self.log.write_text("saved provider evidence\n") - def begin(self, operation="pi3x", number=0, hypothesis=None, candidate="candidate-a"): + def begin( + self, + operation="pi3x", + number=0, + hypothesis=None, + candidate="candidate-a", + authorization=None, + source=SOURCE, + ): return self.ledger.begin( - SOURCE, + source, operation, parameters={"setting": number}, code_version={"worker.py": "code-v1"}, @@ -41,6 +51,7 @@ def begin(self, operation="pi3x", number=0, hypothesis=None, candidate="candidat candidate=candidate, log=self.log, results=[self.root / candidate / "result"], + authorization=authorization, ) def finish(self, attempt, status="failed"): @@ -269,6 +280,195 @@ def test_legacy_over_cap_history_can_be_reconciled_and_never_reset(self): self.assertEqual(retained[:3], saved["attempts"][:3]) self.assertTrue(retained[-1]["events"][-1]["reconciliation"]) + def permit(self, **changes): + permit = { + "schema": AUTHORIZATION_SCHEMA, + "id": "reviewed-rebuild-1", + "sourceSha256": SOURCE, + "candidate": "candidate-a", + "reason": "Reviewed full source rebuild", + "operatorApproval": "okay now start it end to end", + "expiresAtEpoch": time.time() + 3600, + "operations": {"pi3x": 0}, + } + permit.update(changes) + path = self.root / "authorization.json" + path.write_text(json.dumps(permit)) + return path + + def test_authorization_allows_one_unchanged_over_cap_execution_preserving_history(self): + for number in range(3): + attempt = self.begin(number=number, hypothesis=f"Reviewed variation {number}") + self.finish(attempt) + history = json.loads(self.ledger.path.read_text())["attempts"] + permit = self.permit(operations={"pi3x": 3}) + approved = self.begin(number=2, authorization=permit) + self.assertEqual(approved["claim"]["number"], 4) + saved = approved["claim"]["authorization"] + self.assertEqual(saved["id"], "reviewed-rebuild-1") + self.assertEqual(saved["sha256"], hashlib.sha256(permit.read_bytes()).hexdigest()) + self.assertEqual(saved["path"], str(permit.resolve())) + self.assertEqual(saved["operatorApproval"], "okay now start it end to end") + self.assertEqual(json.loads(self.ledger.path.read_text())["attempts"][:3], history) + self.finish(approved) + with self.assertRaisesRegex(ValueError, "consumed"): + self.begin(authorization=permit) + with self.assertRaisesRegex(ValueError, "three total"): + self.begin(number=4, hypothesis="Another variation") + + def test_authorization_is_required_for_unchanged_retry_below_cap(self): + self.finish(self.begin()) + with self.assertRaisesRegex(ValueError, "parameters or code"): + self.begin(hypothesis="Requested rebuild") + attempt = self.begin(authorization=self.permit(operations={"pi3x": 1})) + self.finish(attempt) + + def test_authorization_rejects_wrong_source_candidate_operation_and_count(self): + cases = ( + ({"sourceSha256": "b" * 64}, "canonical source"), + ({"candidate": "candidate-b"}, "candidate"), + ({"operations": {"tracks": 0}}, "operation"), + ({"operations": {"pi3x": 1}}, "count"), + ({"expiresAtEpoch": time.time() - 1}, "expired"), + ) + for changes, message in cases: + with self.subTest(changes=changes), self.assertRaisesRegex(ValueError, message): + self.begin(authorization=self.permit(**changes)) + self.assertFalse(self.ledger.path.exists()) + + def test_authorization_rejects_malformed_fields_and_duplicate_json_keys(self): + cases = ( + {"schema": "unknown"}, + {"id": ""}, + {"id": 7}, + {"candidate": " "}, + {"reason": ""}, + {"operatorApproval": None}, + {"expiresAtEpoch": True}, + {"expiresAtEpoch": float("nan")}, + {"expiresAtEpoch": float("inf")}, + {"expiresAtEpoch": 10**400}, + {"expiresAtEpoch": "tomorrow"}, + {"operations": {}}, + {"operations": []}, + {"operations": {"*": 0}}, + {"operations": {"inpainting": 0}}, + {"operations": {"pi3x": True}}, + {"operations": {"pi3x": -1}}, + {"operations": {"pi3x": 0.0}}, + {"operations": {"pi3x": 0, "tracks": "1"}}, + {"extra": "unrecognized"}, + ) + for changes in cases: + with self.subTest(changes=changes), self.assertRaises(ValueError): + self.begin(authorization=self.permit(**changes)) + path = self.permit() + for raw in ("null", "[1]", "{", '{"id":"first","id":"second"}'): + path.write_text(raw) + with self.subTest(raw=raw), self.assertRaises(ValueError): + self.begin(authorization=path) + path.unlink() + with self.assertRaisesRegex(ValueError, "unreadable"): + self.begin(authorization=path) + self.assertFalse(self.ledger.path.exists()) + + def test_authorization_shared_inpainting_operations_use_logical_counts(self): + permit = self.permit(operations={"clean_first": 0, "clean": 1}) + first = self.begin("clean_first", authorization=permit) + self.finish(first) + second = self.begin("clean", authorization=permit) + self.assertEqual(second["claim"]["number"], 2) + self.finish(second) + with self.assertRaisesRegex(ValueError, "consumed"): + self.begin("clean_first", authorization=permit) + + def test_authorization_changed_file_cannot_reuse_id_for_new_operation(self): + permit = self.permit() + self.finish(self.begin(authorization=permit)) + modified = self.permit(operations={"tracks": 0}) + with self.assertRaisesRegex(ValueError, "already bound"): + self.begin("tracks", authorization=modified) + + def test_authorization_must_name_canonical_source_even_for_alias_submission(self): + alias = "b" * 64 + self.ledger.path.write_text( + json.dumps( + { + "schema": "wander.pipeline-attempts/1", + "attempts": [], + "sourceAliases": {alias: SOURCE}, + } + ) + ) + with self.assertRaisesRegex(ValueError, "canonical source"): + self.begin(source=alias, authorization=self.permit(sourceSha256=alias)) + permit = self.permit() + attempt = self.begin(source=alias, authorization=permit) + self.assertEqual(attempt["claim"]["sourceSha256"], SOURCE) + self.assertEqual(attempt["claim"]["submittedSourceSha256"], alias) + self.finish(attempt) + with self.assertRaisesRegex(ValueError, "consumed"): + self.begin(authorization=permit) + + def test_authorization_never_bypasses_pending_or_unknown_claims(self): + pending = self.begin() + permit = self.permit(operations={"pi3x": 1}) + for status in ("pending", "unknown"): + if status == "unknown": + self.finish(pending, status=status) + with self.subTest(status=status), self.assertRaisesRegex(ValueError, "reconciliation"): + self.begin(authorization=permit) + + def test_authorized_unknown_execution_is_consumed_after_reconciliation(self): + permit = self.permit() + attempt = self.begin(authorization=permit) + self.finish(attempt, status="unknown") + with self.assertRaisesRegex(ValueError, "reconciliation"): + self.begin(authorization=permit) + self.ledger.reconcile( + attempt["id"], status="failed", evidence=self.log, reason="Observed failure" + ) + with self.assertRaisesRegex(ValueError, "consumed"): + self.begin(authorization=permit) + + def test_concurrent_authorization_claims_consume_exactly_once(self): + permit = self.permit() + barrier = threading.Barrier(2) + attempts, errors = [], [] + + def claim(): + barrier.wait(timeout=5) + try: + attempts.append(self.begin(authorization=permit)) + except ValueError as error: + errors.append(str(error)) + + workers = [threading.Thread(target=claim) for _ in range(2)] + for worker in workers: + worker.start() + for worker in workers: + worker.join(timeout=5) + self.assertFalse(worker.is_alive()) + self.assertEqual(len(attempts), 1) + self.assertEqual(len(errors), 1) + self.assertIn("reconciliation", errors[0]) + + def test_expired_saved_authorization_does_not_prevent_reconciliation(self): + attempt = self.begin(authorization=self.permit()) + with patch("stage_attempts.time.time", return_value=time.time() + 7200): + self.ledger.reconcile( + attempt["id"], status="failed", evidence=self.log, reason="Observed failure" + ) + + def test_malformed_saved_authorization_blocks_further_claims(self): + attempt = self.begin(authorization=self.permit()) + self.finish(attempt) + saved = json.loads(self.ledger.path.read_text()) + saved["attempts"][0]["claim"]["authorization"]["operations"]["pi3x"] = 1 + self.ledger.path.write_text(json.dumps(saved)) + with self.assertRaisesRegex(ValueError, "saved paid-stage authorization"): + self.begin("tracks") + class CommandIdentityTests(unittest.TestCase): def setUp(self): @@ -603,6 +803,71 @@ def test_pending_claim_blocks_before_provider_call(self): self.pipeline.paid_run("pi3x", command, self.root / "new.log") provider.assert_not_called() + def test_runner_passes_authorization_and_rejects_reuse_before_provider(self): + path = self.root / "authorization.json" + path.write_text( + json.dumps( + { + "schema": AUTHORIZATION_SCHEMA, + "id": "integration-approval", + "sourceSha256": SOURCE, + "candidate": self.pipeline.name, + "reason": "Reviewed full rebuild", + "operatorApproval": "start it end to end", + "expiresAtEpoch": time.time() + 3600, + "operations": {"pi3x": 0}, + } + ) + ) + self.pipeline.a.stage_authorization = str(path) + with patch.object(run_clip, "run") as provider: + self.pipeline.paid_run("pi3x", self.command(self.root / "output"), self.root / "log") + provider.assert_called_once() + saved = json.loads(Path(self.pipeline.a.stage_ledger).read_text())["attempts"] + self.assertEqual(saved[0]["claim"]["authorization"]["id"], "integration-approval") + with ( + patch.object(run_clip, "run") as provider, + self.assertRaisesRegex(run_clip.QualityStop, "consumed"), + ): + self.pipeline.paid_run( + "pi3x", self.command(self.root / "other-output"), self.root / "log2" + ) + provider.assert_not_called() + + def test_cli_authorization_environment_default_and_explicit_override(self): + for extra, expected in ( + ([], "/env-permit.json"), + (["--stage-authorization", "/cli-permit.json"], "/cli-permit.json"), + ): + with ( + self.subTest(extra=extra), + patch.dict(os.environ, {"WANDER_STAGE_AUTHORIZATION": "/env-permit.json"}), + patch.object( + sys, + "argv", + ["run_clip.py", "--clip", "unused", "--name", "test", "--no-publish", *extra], + ), + patch.object(run_clip, "resolve_shots", return_value=[]) as resolve, + self.assertRaises(SystemExit) as exited, + ): + run_clip.main() + self.assertEqual(exited.exception.code, 0) + self.assertEqual(resolve.call_args.args[0].stage_authorization, expected) + + def test_cli_rejects_authorization_for_orchestrated_submission(self): + with ( + patch.dict(os.environ, {"WANDER_STAGE_AUTHORIZATION": "/env-permit.json"}), + patch.object( + sys, "argv", ["run_clip.py", "--clip", "unused", "--name", "test", "--orchestrated"] + ), + patch.object(run_clip, "resolve_shots") as resolve, + patch("sys.stderr"), + self.assertRaises(SystemExit) as exited, + ): + run_clip.main() + self.assertEqual(exited.exception.code, 2) + resolve.assert_not_called() + def test_existing_output_is_preserved_and_blocks_before_provider_call(self): output = self.root / "paid-output" output.mkdir() @@ -662,13 +927,13 @@ def test_real_paid_stage_methods_preserve_existing_outputs(self): self.assertFalse(Path(self.pipeline.a.stage_ledger).exists()) def test_both_mode_sequences_shared_inpainting_allowance(self): - for graph in ( - lambda: run_clip.single_graph("both"), - lambda: run_clip.multiperson_graph(2, "both"), + for graph, clean_dependencies in ( + (lambda: run_clip.single_graph("both"), ["clean_first"]), + (lambda: run_clip.multiperson_graph(2, "both"), ["clean_first", "tracks"]), ): _stages, dependencies = graph() self.assertEqual(dependencies["clean_first"], []) - self.assertEqual(dependencies["clean"], ["clean_first"]) + self.assertEqual(dependencies["clean"], clean_dependencies) self.assertIn("clean", dependencies["review"]) def latest_event(self, operation): diff --git a/scripts/tests/test_tracks_to_clean_masks.py b/scripts/tests/test_tracks_to_clean_masks.py new file mode 100644 index 0000000..0e2597a --- /dev/null +++ b/scripts/tests/test_tracks_to_clean_masks.py @@ -0,0 +1,322 @@ +"""No-spend contracts for cleaning on a complete, exact tracked source grid.""" + +import hashlib +import json +import shutil +import subprocess +import sys +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import run_clip +import tracks_to_clean_masks + + +def track_fixture(root, clip, samples=5, width=11, height=4): + clip.write_bytes(b"source pixels") + source_sha = hashlib.sha256(clip.read_bytes()).hexdigest() + indices = list(range(0, 2 * samples, 2)) + provenance = { + "sourceSha256": source_sha, + "sourceTimeBase": "1/240", + "frames": [ + {"sourceIndex": index, "sourcePts": index * 10, "sourceRelativeTimeSeconds": index / 24} + for index in indices + ], + } + doc = { + "sourceSha256": source_sha, + "width": width, + "height": height, + "fps": 12.0, + "samples": samples, + "sourceIndices": indices, + "timestamps": [index / 24 for index in indices], + "trackCount": 2, + "tracks": [{"track": track, "samples": list(range(samples))} for track in range(2)], + } + root.mkdir(parents=True, exist_ok=True) + (root / "tracks.json").write_text(json.dumps(doc)) + payload = {"shape": np.array([height, width, 1.0])} + for track in range(2): + for sample in range(samples): + mask = np.zeros((height, width), dtype=bool) + mask[0, track + sample] = True + payload[f"t{track}_s{sample:03d}"] = np.packbits(mask) + np.savez_compressed(root / "masks.npz", **payload) + return doc, payload, provenance + + +@pytest.fixture +def tracking(tmp_path, monkeypatch): + clip, root, out = tmp_path / "clip.mp4", tmp_path / "tracks", tmp_path / "clean-masks.npz" + doc, payload, provenance = track_fixture(root, clip) + monkeypatch.setattr( + tracks_to_clean_masks, "resample_source", lambda *a, **kw: (b"", provenance) + ) + monkeypatch.setattr( + tracks_to_clean_masks, "select_source_frames", lambda *a, **kw: (b"", provenance) + ) + return root, clip, out, doc, payload, provenance + + +def build(tracking, **options): + root, clip, out, *_ = tracking + return tracks_to_clean_masks.build_clean_masks( + root, + clip=clip, + out=out, + width=11, + height=4, + fps=12, + dilate=options.get("dilate", 0), + bottom_extra=options.get("bottom_extra", 0), + ) + + +def test_unions_all_tracks_without_changing_source_grid(tracking): + _, _, out, doc, _, provenance = tracking + report = build(tracking) + assert report["framesCovered"] == 5 and report["samplesWithoutMask"] == 0 + with np.load(out, allow_pickle=False) as archive: + assert archive["source_indices"].tolist() == doc["sourceIndices"] + assert archive["source_pts"].tolist() == [ + frame["sourcePts"] for frame in provenance["frames"] + ] + assert archive["source_time_base"].item() == "1/240" + assert archive["source_sha256"].item() == doc["sourceSha256"] + masks = np.unpackbits(archive["masks"], axis=-1)[:, :, :11] + for sample, mask in enumerate(masks): + assert mask.sum() == 2 + assert mask[0, sample] and mask[0, sample + 1] + + +@pytest.mark.parametrize( + "change", ["source", "dimensions", "fps", "count", "timestamp", "coverage", "empty-tracks"] +) +def test_rejects_incompatible_track_metadata_without_writing_archive(tracking, change): + root, _, out, doc, *_ = tracking + if change == "source": + doc["sourceSha256"] = "0" * 64 + elif change == "dimensions": + doc["width"] = 12 + elif change == "fps": + doc["fps"] = 24 + elif change == "count": + doc["samples"] = 4 + elif change == "timestamp": + doc["timestamps"][2] += 0.01 + elif change == "coverage": + for track in doc["tracks"]: + track["samples"].pop() + else: + doc["tracks"], doc["trackCount"] = [], 0 + (root / "tracks.json").write_text(json.dumps(doc)) + with pytest.raises(ValueError): + build(tracking) + assert not out.exists() + + +@pytest.mark.parametrize( + "change", + [ + "missing-file", + "missing-mask", + "empty", + "wrong-shape", + "wrong-length", + "wrong-dtype", + "undeclared", + ], +) +def test_rejects_missing_or_incomplete_instance_masks(tracking, change): + root, _, out, _, payload, _ = tracking + if change == "missing-mask": + payload.pop("t0_s002") + elif change == "empty": + payload["t0_s002"][:] = 0 + elif change == "wrong-shape": + payload["shape"] = np.array([4, 12, 1]) + elif change == "wrong-length": + payload["t0_s002"] = np.array([255], np.uint8) + elif change == "wrong-dtype": + payload["t0_s002"] = payload["t0_s002"].astype(float) + elif change == "undeclared": + payload["t2_s002"] = payload["t0_s002"] + np.savez_compressed(root / "masks.npz", **payload) + if change == "missing-file": + (root / "masks.npz").unlink() + with pytest.raises(ValueError): + build(tracking) + assert not out.exists() + + +def test_dilation_and_downward_extension_are_applied_once(tracking): + _, _, out, *_ = tracking + build(tracking, dilate=1, bottom_extra=1) + with np.load(out) as archive: + mask = np.unpackbits(archive["masks"], axis=-1)[0, :, :11] + assert mask[:3, :3].all() + assert not mask[3].any() + assert not mask[:, 4:].any() + + +def test_exact_tracker_grid_is_preserved_when_fps_filter_selects_other_frames(tmp_path): + if not shutil.which("ffmpeg") or not shutil.which("ffprobe"): + pytest.skip("FFmpeg required for decoded source fixture") + clip, root, out = tmp_path / "clip.mp4", tmp_path / "tracks", tmp_path / "clean.npz" + _, payload, _ = track_fixture(root, clip, samples=2, width=16, height=12) + subprocess.run( + [ + "ffmpeg", + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=16x12:rate=6:duration=1", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + str(clip), + ], + check=True, + ) + _, sampled = tracks_to_clean_masks.resample_source(clip, 2) + indices = [0, 3] + _, exact = tracks_to_clean_masks.select_source_frames(clip, 2, source_indices=indices) + assert [frame["sourceIndex"] for frame in sampled["frames"]] != indices + doc = { + "sourceSha256": exact["sourceSha256"], + "width": 16, + "height": 12, + "fps": 2, + "samples": 2, + "sourceIndices": indices, + "timestamps": [frame["sourceRelativeTimeSeconds"] for frame in exact["frames"]], + "trackCount": 2, + "tracks": [{"track": i, "samples": [0, 1]} for i in range(2)], + } + (root / "tracks.json").write_text(json.dumps(doc)) + tracks_to_clean_masks.build_clean_masks( + root, clip=clip, out=out, width=16, height=12, fps=2, dilate=0, bottom_extra=0 + ) + with np.load(out) as archive: + assert archive["source_indices"].tolist() == indices + assert archive["source_pts"].tolist() == [frame["sourcePts"] for frame in exact["frames"]] + + +@pytest.fixture +def runner(tracking, monkeypatch): + from types import SimpleNamespace + from unittest.mock import Mock + from worker.wander_worker import source_timing + + root, clip, _, _, _, provenance = tracking + pipeline = object.__new__(run_clip.Pipeline) + pipeline.ctx = root.parent + pipeline.clip = clip + pipeline.clean_mp4 = root.parent / "clean.mp4" + pipeline.multi = True + pipeline.info = {"width": 11, "height": 4} + pipeline.a = SimpleNamespace( + clean_masks=None, + moved_mask=False, + fps=12, + dilate=0, + bottom_extra=0, + lama_px=960, + ) + pipeline.paid_run = Mock() + monkeypatch.setattr(run_clip, "clean_sample_count", lambda *a: 5) + monkeypatch.setattr(source_timing, "select_source_frames", lambda *a, **kw: (b"", provenance)) + return pipeline + + +def test_default_tracked_clean_forwards_generated_complete_masks(runner): + runner.clean() + runner.paid_run.assert_called_once() + command = runner.paid_run.call_args.args[1] + masks = Path(command[command.index("--masks-in") + 1]) + assert masks == runner.ctx / "track-clean-masks.npz" + with np.load(masks) as archive: + assert archive["source_indices"].tolist() == [0, 2, 4, 6, 8] + report = json.loads((runner.ctx / "track-clean-masks.json").read_text()) + assert report["framesCovered"] == 5 + + +@pytest.mark.parametrize("invalid", ["absent", "source", "mask", "timestamp"]) +def test_invalid_tracking_stops_before_paid_clean_without_semantic_fallback( + runner, tracking, invalid +): + root, _, _, doc, payload, _ = tracking + if invalid == "absent": + (root / "masks.npz").unlink() + elif invalid == "mask": + payload.pop("t1_s001") + np.savez_compressed(root / "masks.npz", **payload) + else: + if invalid == "source": + doc["sourceSha256"] = "0" * 64 + else: + doc["timestamps"][1] = 99 + (root / "tracks.json").write_text(json.dumps(doc)) + with pytest.raises(ValueError): + runner.clean() + runner.paid_run.assert_not_called() + + +@pytest.mark.parametrize("explicit", ["reviewed", "moved", "single"]) +def test_explicit_mask_choice_and_single_person_preserve_existing_path(runner, explicit): + if explicit == "reviewed": + reviewed = runner.ctx / "reviewed.npz" + np.savez_compressed(reviewed, masks=np.zeros((5, 4, 2), np.uint8)) + runner.a.clean_masks = str(reviewed) + elif explicit == "moved": + runner.a.moved_mask = True + else: + runner.multi = False + with patch.object(tracks_to_clean_masks, "build_clean_masks") as build_masks: + runner.clean() + build_masks.assert_not_called() + command = runner.paid_run.call_args.args[1] + assert ("--masks-in" in command) == (explicit == "reviewed") + assert ("--moved-mask" in command) == (explicit == "moved") + + +@pytest.mark.parametrize("corrupt", ["hash", "pts", "timebase", "partial", "shape-dtype"]) +def test_explicit_grid_metadata_is_validated_before_paid_submission(runner, tracking, corrupt): + build(tracking) + path = tracking[2] + with np.load(path) as archive: + payload = {key: archive[key] for key in archive.files} + if corrupt == "hash": + payload["source_sha256"] = np.array("0" * 64) + elif corrupt == "pts": + payload["source_pts"][0] += 1 + elif corrupt == "timebase": + payload["source_time_base"] = np.array("1/30") + elif corrupt == "partial": + payload.pop("source_indices") + else: + payload["shape"] = payload["shape"].astype(float) + np.savez_compressed(path, **payload) + runner.a.clean_masks = str(path) + with pytest.raises(ValueError): + runner.clean() + runner.paid_run.assert_not_called() + + +def test_exact_source_masks_rejected_before_paid_multi_image_clean(runner, tracking): + build(tracking) + runner.a.clean_masks = str(tracking[2]) + runner.marble = "multi" + with pytest.raises(ValueError, match="not supported by --marble multi"): + runner.clean_flags() + runner.paid_run.assert_not_called() diff --git a/scripts/tests/test_vlm_budget.py b/scripts/tests/test_vlm_budget.py new file mode 100644 index 0000000..b5875fc --- /dev/null +++ b/scripts/tests/test_vlm_budget.py @@ -0,0 +1,169 @@ +import io +import json +from pathlib import Path +import sys +from concurrent.futures import ThreadPoolExecutor +from unittest.mock import Mock + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "worker" / "stages")) +import vlm_judge + + +@pytest.fixture +def budget(tmp_path, monkeypatch): + path = tmp_path / "budget.json" + path.write_text( + json.dumps( + { + "schema": "wander.vlm-budget/1", + "maxCalls": 2, + "maxCompletionTokens": 4096, + "allowedModels": ["gpt-6-astra"], + "calls": [], + } + ) + ) + monkeypatch.setenv("WANDER_VLM_BUDGET", str(path)) + monkeypatch.setenv("OPENAI_API_KEY", "secret-test-key") + return path + + +def response(content='{"ok": true}'): + return io.BytesIO( + json.dumps( + { + "choices": [{"message": {"content": content}}], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, + "private": "secret-test-key", + }, + } + ).encode() + ) + + +def ask(prompt="private prompt", **kwargs): + return vlm_judge.ask_images("gpt-6-astra", [], prompt, **kwargs) + + +def test_success_quota_duplicate_and_token_bound(budget, monkeypatch): + network = Mock(side_effect=lambda *a, **kw: response()) + monkeypatch.setattr(vlm_judge.urllib.request, "urlopen", network) + assert ask() == {"ok": True} + assert "duplicate" in ask()["error"] + assert ask("second prompt") == {"ok": True} + assert "exhausted" in ask("third prompt")["error"] + assert network.call_count == 2 + body = json.loads(network.call_args.args[0].data) + assert body["max_completion_tokens"] == 4096 + data = json.loads(budget.read_text()) + assert data["calls"][0]["usage"] == { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, + } + assert all(c["status"] == "completed" for c in data["calls"]) + assert "secret-test-key" not in budget.read_text() + assert "private prompt" not in budget.read_text() + + +@pytest.mark.parametrize("failure", [TimeoutError, ConnectionError, ValueError]) +def test_failed_or_ambiguous_request_consumes_claim(budget, monkeypatch, failure): + def fail(*args, **kwargs): + assert json.loads(budget.read_text())["calls"][0]["status"] == "claimed" + raise failure("secret-test-key private prompt") + + network = Mock(side_effect=fail) + monkeypatch.setattr(vlm_judge.urllib.request, "urlopen", network) + assert "error" in ask(retries=20) + assert "duplicate" in ask()["error"] + assert network.call_count == 1 + assert json.loads(budget.read_text())["calls"][0]["status"] == "error" + assert "secret-test-key" not in budget.read_text() + + +@pytest.mark.parametrize( + "change", + [ + {"schema": "wrong"}, + {"maxCalls": 0}, + {"maxCalls": True}, + {"maxCompletionTokens": -1}, + {"allowedModels": ["other"]}, + {"calls": [{}]}, + ], +) +def test_bad_config_denies_network(budget, monkeypatch, change): + data = json.loads(budget.read_text()) + data.update(change) + budget.write_text(json.dumps(data)) + network = Mock() + monkeypatch.setattr(vlm_judge.urllib.request, "urlopen", network) + assert "error" in ask() + network.assert_not_called() + + +def test_missing_or_empty_config_denies_network(budget, monkeypatch): + budget.unlink() + network = Mock() + monkeypatch.setattr(vlm_judge.urllib.request, "urlopen", network) + assert "error" in ask() + monkeypatch.setenv("WANDER_VLM_BUDGET", "") + assert "error" in ask() + network.assert_not_called() + + +def test_parallel_calls_share_allowance(budget, monkeypatch): + network = Mock(side_effect=lambda *a, **kw: response()) + monkeypatch.setattr(vlm_judge.urllib.request, "urlopen", network) + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(lambda i: ask(str(i)), range(8))) + assert sum(r == {"ok": True} for r in results) == 2 + assert network.call_count == 2 + assert len(json.loads(budget.read_text())["calls"]) == 2 + + +def test_unconfigured_retains_retries(budget, monkeypatch): + monkeypatch.delenv("WANDER_VLM_BUDGET") + network = Mock(side_effect=[TimeoutError(), response()]) + monkeypatch.setattr(vlm_judge.urllib.request, "urlopen", network) + monkeypatch.setattr(vlm_judge.time, "sleep", lambda _: None) + assert ask() == {"ok": True} + assert network.call_count == 2 + assert "max_completion_tokens" not in json.loads(network.call_args.args[0].data) + + +def test_configured_bounds_and_model_are_used(budget, monkeypatch): + data = json.loads(budget.read_text()) + data.update(maxCalls=1, maxCompletionTokens=123, allowedModels=["configured-model"]) + budget.write_text(json.dumps(data)) + network = Mock(side_effect=lambda *a, **kw: response()) + monkeypatch.setattr(vlm_judge.urllib.request, "urlopen", network) + assert "not allowed" in ask()["error"] + assert vlm_judge.ask_images("configured-model", [], "prompt") == {"ok": True} + assert "exhausted" in vlm_judge.ask_images("configured-model", [], "other")["error"] + assert network.call_count == 1 + assert json.loads(network.call_args.args[0].data)["max_completion_tokens"] == 123 + + +def test_malformed_result_keeps_usage_and_consumes_claim(budget, monkeypatch): + network = Mock(side_effect=lambda *a, **kw: response("invalid json")) + monkeypatch.setattr(vlm_judge.urllib.request, "urlopen", network) + assert "error" in ask() + assert "duplicate" in ask()["error"] + call = json.loads(budget.read_text())["calls"][0] + assert call["status"] == "error" + assert call["usage"]["total_tokens"] == 30 + assert network.call_count == 1 + + +def test_claim_write_failure_prevents_network(budget, monkeypatch): + monkeypatch.setattr(vlm_judge.vlm_budget, "_save", Mock(side_effect=OSError("disk"))) + network = Mock() + monkeypatch.setattr(vlm_judge.urllib.request, "urlopen", network) + assert "error" in ask() + network.assert_not_called() diff --git a/scripts/tracks_to_clean_masks.py b/scripts/tracks_to_clean_masks.py index 68ef12e..fb1e1c7 100644 --- a/scripts/tracks_to_clean_masks.py +++ b/scripts/tracks_to_clean_masks.py @@ -1,104 +1,215 @@ #!/usr/bin/env python3 -"""Turn the tracking stage's per-track Mask R-CNN masks into a plate-cleaning mask archive. +"""Build a plate-cleaning archive from complete, source-aligned tracking instance masks. - uv run --locked --group inference python scripts/tracks_to_clean_masks.py .context/mp//tracks \ - --width 1920 --height 1080 --dilate 30 --bottom-extra 60 --out .context/mp//clean-masks.npz - run worker/modal_clean_video.py --clip ... --masks-in .context/mp//clean-masks.npz ... + uv run --locked python scripts/tracks_to_clean_masks.py RUN/tracks \ + --clip source.mp4 --fps 12 --width 1920 --height 1080 --out RUN/track-clean-masks.npz -The clean pass segments people with SegFormer, which is what the single-person pipeline has always -used and which is fine most of the time -- but on the elevator clip it leaves one man's light grey -trousers standing on the light floor in the last second, because SegFormer loses a low-contrast -lower body. The tracking stage has already run Mask R-CNN on every sampled frame for every track, -so the union of those instance masks is a better plate mask that costs nothing extra. +The semantic clean pass can miss low-contrast lower bodies. Tracking has already produced +per-person masks. Their exact recorded source frames are retained in the archive so cleaning +can select those same pixels, rather than applying masks to a nearby frame on the FPS grid. +Missing, empty, malformed, or mismatched masks stop before an archive is written. -`--masks-in` skips the clean pass's own dilation and downward extension, so both are applied here -with the same numbers and the same kernel anchoring as worker/modal_clean_video.py. +`--masks-in` skips the clean worker's own dilation and downward extension, so this helper +applies them with the same kernel anchoring as worker/modal_clean_video.py. """ -import argparse, json +import argparse +import hashlib +import json +import math +import re +import sys from pathlib import Path import cv2 import numpy as np +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +from worker.wander_worker.source_timing import resample_source, select_source_frames # noqa: E402 -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("tracks") - ap.add_argument("--out", required=True) - ap.add_argument("--width", type=int, required=True) - ap.add_argument("--height", type=int, required=True) - ap.add_argument("--dilate", type=int, default=20) - ap.add_argument("--bottom-extra", type=int, default=40) - a = ap.parse_args() - - root = Path(a.tracks) - doc = json.loads((root / "tracks.json").read_text()) - store = np.load(root / "masks.npz") - sh = store["shape"] - mh, mw = int(round(a.height * float(sh[2]))), int(round(a.width * float(sh[2]))) - n, H, W = doc["samples"], a.height, a.width - masks = np.zeros((n, H, W), bool) - per_sample = {} - for key in store.files: - if not key.startswith("t"): - continue - tid, _, srest = key[1:].partition("_s") - per_sample.setdefault(int(srest), []).append(key) - missing = 0 - for s in range(n): - keys = per_sample.get(s, []) - if not keys: - missing += 1 - continue - small = np.zeros((mh, mw), bool) - for key in keys: - m = np.unpackbits(store[key])[: mh * mw].reshape(mh, mw).astype(bool) - small |= m - masks[s] = cv2.resize(small.astype(np.uint8), (W, H), interpolation=cv2.INTER_NEAREST) > 0 - - if a.dilate > 0: - k = np.ones((2 * a.dilate + 1, 2 * a.dilate + 1), np.uint8) - masks = np.stack([cv2.dilate(m.astype(np.uint8), k).astype(bool) for m in masks]) - # same downward extension as worker/modal_clean_video.py: carry every occupied column to the - # bottom of the frame so the contact shadow goes with the person - for m in masks: - cols = np.where(m[int(0.6 * H) :].any(0))[0] - if len(cols): - low = np.argmax(m[::-1], axis=0) - for c in cols: - m[H - 1 - low[c] :, c] = True - if a.bottom_extra > 0: - k = np.ones((a.bottom_extra + 1, 2 * a.dilate + 1), np.uint8) - masks = np.stack( - [ - cv2.dilate(m.astype(np.uint8), k, anchor=(a.dilate, a.bottom_extra)).astype(bool) - for m in masks - ] +def build_clean_masks(tracks, *, clip, out, width, height, fps, dilate=20, bottom_extra=40): + """Validate all retained masks against observed clean inputs, then union and expand them.""" + if any(type(value) is not int or value <= 0 for value in (width, height)): + raise ValueError("Clean-mask dimensions must be positive integers") + if any(type(value) is not int or value < 0 for value in (dilate, bottom_extra)): + raise ValueError("Clean-mask margins must be nonnegative integers") + if not math.isfinite(fps) or fps <= 0: + raise ValueError("Clean-mask FPS must be positive and finite") + root, out = Path(tracks), Path(out) + document_path, masks_path = root / "tracks.json", root / "masks.npz" + try: + document_bytes = document_path.read_bytes() + doc = json.loads(document_bytes) + except (OSError, ValueError) as error: + raise ValueError( + f"Tracking metadata is unavailable or malformed: {document_path}" + ) from error + if not isinstance(doc, dict): + raise ValueError("Tracking metadata must be an object") + indices = doc.get("sourceIndices") + if ( + not isinstance(indices, list) + or not indices + or any(type(index) is not int or index < 0 for index in indices) + ): + raise ValueError("Tracking masks require an explicit source frame grid") + _, sampled = resample_source(clip, fps) + if len(indices) != len(sampled["frames"]): + raise ValueError("Tracking mask frame count does not match observed clean source sampling") + _, provenance = select_source_frames(clip, fps, source_indices=indices) + if provenance["sourceSha256"] != sampled["sourceSha256"]: + raise ValueError("Source changed while validating tracking mask sampling") + frames = provenance["frames"] + n = len(frames) + if doc.get("sourceSha256") != provenance["sourceSha256"]: + raise ValueError("Tracking masks do not belong to this source SHA-256") + if (doc.get("width"), doc.get("height")) != (width, height): + raise ValueError("Tracking masks do not match the source dimensions") + if ( + type(doc.get("samples")) is not int + or doc["samples"] != n + or not n + or type(doc.get("fps")) not in (int, float) + or not math.isfinite(doc["fps"]) + or not math.isclose(doc["fps"], fps, rel_tol=0, abs_tol=1e-9) + ): + raise ValueError("Tracking mask frame grid does not match observed clean source sampling") + timestamps = doc.get("timestamps") + if ( + not isinstance(timestamps, list) + or len(timestamps) != n + or any( + type(timestamp) not in (int, float) + or not math.isfinite(timestamp) + or abs(timestamp - frame["sourceRelativeTimeSeconds"]) > 1e-6 + for timestamp, frame in zip(timestamps, frames, strict=True) ) + ): + raise ValueError("Tracking mask timestamps do not match observed clean source sampling") + tracks = doc.get("tracks") + count = doc.get("trackCount") + if type(count) is not int or count <= 0 or not isinstance(tracks, list) or len(tracks) != count: + raise ValueError("Tracking masks require a nonempty complete track inventory") + expected, identifiers = set(), set() + for track in tracks: + if not isinstance(track, dict) or type(track.get("track")) is not int: + raise ValueError("Malformed track identity for cleaning masks") + track_id, samples = track["track"], track.get("samples") + if track_id in identifiers or not 0 <= track_id < count: + raise ValueError("Duplicate or out-of-range track identity for cleaning masks") + identifiers.add(track_id) + if ( + not isinstance(samples, list) + or not samples + or any(type(sample) is not int or not 0 <= sample < n for sample in samples) + or samples != sorted(set(samples)) + ): + raise ValueError("Malformed track sample inventory for cleaning masks") + expected.update((track_id, sample) for sample in samples) + if {sample for _, sample in expected} != set(range(n)): + raise ValueError("Tracking masks have incomplete frame coverage") + try: + store = np.load(masks_path, allow_pickle=False) + except (OSError, ValueError) as error: + raise ValueError( + f"Tracking mask archive is unavailable or malformed: {masks_path}" + ) from error + if not isinstance(store, np.lib.npyio.NpzFile): + raise ValueError("Tracking masks must be an NPZ archive") + masks = np.zeros((n, height, width), bool) + with store: + shape = store["shape"] if "shape" in store.files else None + if ( + shape is None + or shape.shape != (3,) + or not np.issubdtype(shape.dtype, np.number) + or not np.isfinite(shape).all() + or tuple(shape[:2]) != (height, width) + or not 0 < shape[2] <= 1 + ): + raise ValueError("Tracking mask archive has invalid source shape or mask scale") + mh, mw = round(height * float(shape[2])), round(width * float(shape[2])) + if min(mh, mw) <= 0: + raise ValueError("Tracking mask scale produces empty dimensions") + per_sample, actual = {}, set() + for key in store.files: + if key == "shape": + continue + match = re.fullmatch(r"t(\d+)_s(\d+)", key) + if match is None: + raise ValueError(f"Unrecognized tracking mask entry: {key}") + identity = tuple(map(int, match.groups())) + if identity in actual or identity not in expected: + raise ValueError("Tracking mask archive disagrees with the track sample inventory") + actual.add(identity) + packed = store[key] + if packed.dtype != np.uint8 or packed.shape != (math.ceil(mh * mw / 8),): + raise ValueError(f"Tracking mask has invalid packed shape or dtype: {key}") + mask = np.unpackbits(packed)[: mh * mw].reshape(mh, mw).astype(bool) + if not mask.any(): + raise ValueError(f"Tracking mask is empty: {key}") + per_sample.setdefault(identity[1], np.zeros((mh, mw), bool))[:] |= mask + if actual != expected: + raise ValueError("Tracking mask archive is missing declared track samples") + for sample, small in per_sample.items(): + masks[sample] = cv2.resize( + small.astype(np.uint8), (width, height), interpolation=cv2.INTER_NEAREST + ).astype(bool) - Path(a.out).parent.mkdir(parents=True, exist_ok=True) + if dilate > 0: + kernel = np.ones((2 * dilate + 1, 2 * dilate + 1), np.uint8) + for mask in masks: + mask[:] = cv2.dilate(mask.astype(np.uint8), kernel).astype(bool) + for mask in masks: + columns = np.where(mask[int(0.6 * height) :].any(0))[0] + low = np.argmax(mask[::-1], axis=0) + for column in columns: + mask[height - 1 - low[column] :, column] = True + if bottom_extra > 0: + kernel = np.ones((bottom_extra + 1, 2 * dilate + 1), np.uint8) + for mask in masks: + mask[:] = cv2.dilate( + mask.astype(np.uint8), kernel, anchor=(dilate, bottom_extra) + ).astype(bool) + out.parent.mkdir(parents=True, exist_ok=True) np.savez_compressed( - a.out, + out, masks=np.packbits(masks, axis=-1), - shape=np.array([n, H, W]), - source=np.array([str(root)]), + shape=np.array([n, height, width]), + source_sha256=np.array(provenance["sourceSha256"]), + source_indices=np.array(indices), + source_pts=np.array([frame["sourcePts"] for frame in frames]), + source_time_base=np.array(provenance["sourceTimeBase"]), + fps=np.array(fps), + tracksSha256=np.array(hashlib.sha256(document_bytes).hexdigest()), + source=np.array([str(root.resolve())]), ) - print( - json.dumps( - dict( - out=a.out, - samples=n, - samplesWithoutMask=missing, - tracks=doc["trackCount"], - maskFraction=float(masks.mean()), - framesCovered=int(masks.reshape(n, -1).any(1).sum()), - ), - indent=1, - ) + return dict( + out=str(out), + samples=n, + samplesWithoutMask=0, + tracks=count, + maskFraction=float(masks.mean()), + framesCovered=n, ) +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("tracks") + ap.add_argument("--clip", required=True) + ap.add_argument("--fps", type=float, default=12) + ap.add_argument("--out", required=True) + ap.add_argument("--width", type=int, required=True) + ap.add_argument("--height", type=int, required=True) + ap.add_argument("--dilate", type=int, default=20) + ap.add_argument("--bottom-extra", type=int, default=40) + args = ap.parse_args() + print(json.dumps(build_clean_masks(**vars(args)), indent=1)) + + if __name__ == "__main__": main() diff --git a/worker/modal_clean_video.py b/worker/modal_clean_video.py index 2d307f8..c543a2e 100644 --- a/worker/modal_clean_video.py +++ b/worker/modal_clean_video.py @@ -99,7 +99,7 @@ def clean( import cv2 import numpy as np from PIL import Image - from wander_worker.source_timing import resample_source, select_provenance + from wander_worker.source_timing import resample_source, select_provenance, select_source_frames t0 = time.time() root = Path(tempfile.mkdtemp(prefix="cleanvid-")) @@ -113,7 +113,51 @@ def clean( frames_tar = b"" try: t = time.time() - raw, provenance = resample_source(src, fps, width=W, height=H) + supplied = None + explicit = False + if masks_npz: + with np.load(io.BytesIO(masks_npz), allow_pickle=False) as archive: + supplied = {key: archive[key] for key in archive.files} + fields = {"source_indices", "source_pts", "source_time_base", "source_sha256"} + present = fields.intersection(supplied) + if present and present != fields: + raise ValueError("Supplied masks have incomplete explicit source metadata") + explicit = bool(present) + if explicit: + source_indices = supplied["source_indices"] + source_pts = supplied["source_pts"] + if any(a.ndim != 1 or a.dtype.kind not in "iu" for a in (source_indices, source_pts)): + raise ValueError("Supplied source indices and PTS must be integer vectors") + if any( + supplied[key].shape != () or supplied[key].dtype.kind != "U" + for key in ("source_time_base", "source_sha256") + ): + raise ValueError("Supplied source timebase and hash must be scalar strings") + if str(supplied["source_sha256"]) != hashlib.sha256(video_bytes).hexdigest(): + raise ValueError("Supplied masks source hash mismatch") + raw, provenance = select_source_frames( + src, fps, source_indices=source_indices.tolist(), width=W, height=H + ) + if ( + source_pts.tolist() != [f["sourcePts"] for f in provenance["frames"]] + or str(supplied["source_time_base"]) != provenance["sourceTimeBase"] + or str(supplied["source_sha256"]) != provenance["sourceSha256"] + ): + raise ValueError("Supplied masks source PTS/timebase/hash mismatch") + expected = [len(source_indices), H, W] + shape = supplied.get("shape") + packed = supplied.get("masks") + if ( + shape is None + or shape.dtype.kind not in "iu" + or shape.tolist() != expected + or packed is None + or packed.dtype != np.uint8 + or packed.shape != (len(source_indices), H, (W + 7) // 8) + ): + raise ValueError("Supplied masks shape does not match exact decoded grid") + else: + raw, provenance = resample_source(src, fps, width=W, height=H) imgs = np.frombuffer(raw, np.uint8).reshape(-1, H, W, 3) del raw timings["decode"] = time.time() - t @@ -146,9 +190,7 @@ def clean( elif masks_npz: # supplied masks (the quality gate feeds in the CPU reference's own) skip segmentation # and the extension below, so the comparison isolates LaMa - masks = np.unpackbits(np.load(io.BytesIO(masks_npz))["masks"], axis=-1)[ - :, :H, :W - ].astype(bool) + masks = np.unpackbits(supplied["masks"], axis=-1)[:, :H, :W].astype(bool) timings["masks"] = time.time() - t else: masks[need] = people_masks(imgs[need], dilate_px=dilate, batch_size=mask_batch) @@ -213,6 +255,7 @@ def clean( masks=np.packbits(masks, axis=-1), shape=np.array([n, H, W]), indices=np.array(indices), + source_indices=np.array(indices), source_pts=np.array([frame["sourcePts"] for frame in provenance["frames"]]), source_time_base=np.array(provenance["sourceTimeBase"]), source_sha256=np.array(provenance["sourceSha256"]), diff --git a/worker/stages/vlm_budget.py b/worker/stages/vlm_budget.py new file mode 100644 index 0000000..40d7168 --- /dev/null +++ b/worker/stages/vlm_budget.py @@ -0,0 +1,116 @@ +"""Opt-in, local, never-refunded request allowance for a bounded VLM run.""" + +from contextlib import contextmanager +import fcntl +import hashlib +import json +import os +from pathlib import Path +import tempfile +import time + + +class BudgetError(ValueError): + pass + + +def _validate(data): + if ( + not isinstance(data, dict) + or data.get("schema") != "wander.vlm-budget/1" + or type(data.get("maxCalls")) is not int + or data["maxCalls"] <= 0 + or type(data.get("maxCompletionTokens")) is not int + or data["maxCompletionTokens"] <= 0 + or not isinstance(data.get("allowedModels"), list) + or not data["allowedModels"] + or any(not isinstance(m, str) or not m.strip() for m in data["allowedModels"]) + or not isinstance(data.get("calls"), list) + ): + raise BudgetError("invalid VLM budget") + hashes = set() + for call in data["calls"]: + if ( + not isinstance(call, dict) + or not isinstance(call.get("bodyHash"), str) + or len(call["bodyHash"]) != 64 + or any(c not in "0123456789abcdef" for c in call["bodyHash"]) + or call["bodyHash"] in hashes + or call.get("status") not in ("claimed", "completed", "error") + ): + raise BudgetError("invalid VLM budget history") + hashes.add(call["bodyHash"]) + return data + + +@contextmanager +def _locked(path): + # Lock a stable sibling inode: replacing the JSON must not replace the lock. + with open(str(path) + ".lock", "a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + yield _validate(json.loads(path.read_text())) + + +def _save(path, data): + name = None + try: + with tempfile.NamedTemporaryFile(mode="w", dir=path.parent, delete=False) as out: + name = out.name + json.dump(data, out, indent=2) + out.flush() + os.fsync(out.fileno()) + os.replace(name, path) + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + if name and os.path.exists(name): + os.unlink(name) + + +def claim(path, body): + """Persist a claim before any network action; also apply the completion token cap.""" + path = Path(path).resolve(strict=True) + with _locked(path) as data: + if body["model"] not in data["allowedModels"]: + raise BudgetError("VLM model is not allowed") + body["max_completion_tokens"] = data["maxCompletionTokens"] + digest = hashlib.sha256( + json.dumps(body, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + if any(c["bodyHash"] == digest for c in data["calls"]): + raise BudgetError("duplicate VLM request denied") + if len(data["calls"]) >= data["maxCalls"]: + raise BudgetError("VLM request allowance exhausted") + data["calls"].append( + { + "bodyHash": digest, + "model": body["model"], + "status": "claimed", + "claimedAt": time.time(), + } + ) + _save(path, data) + return path, digest + + +def finish(receipt, status, usage=None, error=None): + path, digest = receipt + with _locked(path) as data: + call = next(c for c in data["calls"] if c["bodyHash"] == digest) + call["status"] = status + call["finishedAt"] = time.time() + # Allowlist numeric counters only: provider response/error text can contain inputs. + if isinstance(usage, dict): + call["usage"] = { + key: value + for key, value in usage.items() + if key in ("prompt_tokens", "completion_tokens", "total_tokens") + and type(value) is int + and value >= 0 + } + if error is not None: + call["errorType"] = type(error).__name__ + _save(path, data) diff --git a/worker/stages/vlm_judge.py b/worker/stages/vlm_judge.py index 02f7fb4..3cebb0c 100755 --- a/worker/stages/vlm_judge.py +++ b/worker/stages/vlm_judge.py @@ -17,6 +17,11 @@ import argparse, base64, json, os, ssl, time, urllib.error, urllib.request +try: + from . import vlm_budget +except ImportError: # Also supports direct script execution. + import vlm_budget + CTX = ssl.create_default_context() try: import certifi @@ -74,6 +79,8 @@ def ask_images(model, images, prompt, detail="low", retries=3, timeout=180): } ) body = {"model": model, "messages": [{"role": "user", "content": content}]} + if "WANDER_VLM_BUDGET" in os.environ: + return _ask_bounded(body, timeout) req = urllib.request.Request( "https://api.openai.com/v1/chat/completions", data=json.dumps(body).encode(), @@ -103,6 +110,38 @@ def ask_images(model, images, prompt, detail="low", retries=3, timeout=180): time.sleep(2**i) +def _ask_bounded(body, timeout): + receipt = None + usage = None + try: + # An empty path is configured-but-invalid, never permission to bypass the guard. + receipt = vlm_budget.claim(os.environ["WANDER_VLM_BUDGET"], body) + req = urllib.request.Request( + "https://api.openai.com/v1/chat/completions", + data=json.dumps(body).encode(), + headers={ + "Authorization": f"Bearer {os.environ['OPENAI_API_KEY']}", + "Content-Type": "application/json", + }, + ) + with urllib.request.urlopen(req, timeout=timeout, context=CTX) as response: + result = json.load(response) + usage = result.get("usage") + txt = result["choices"][0]["message"]["content"].strip() + txt = txt.removeprefix("```json").removeprefix("```").removesuffix("```").strip() + parsed = json.loads(txt) + vlm_budget.finish(receipt, "completed", usage=usage) + return parsed + except Exception as error: + if receipt is not None: + try: + vlm_budget.finish(receipt, "error", usage=usage, error=error) + except Exception: + pass # The durable claim still consumes its allowance. + message = str(error) if isinstance(error, vlm_budget.BudgetError) else type(error).__name__ + return {"error": f"bounded VLM request stopped: {message}"} + + def ask(model, img, prompt, retries=3): return ask_images(model, [img], prompt, retries=retries, timeout=120) diff --git a/worker/wander_worker/source_timing.py b/worker/wander_worker/source_timing.py index 48ec977..45f79bb 100644 --- a/worker/wander_worker/source_timing.py +++ b/worker/wander_worker/source_timing.py @@ -114,10 +114,11 @@ def _parse_selection(log: str, source_pts: list[int], time_base: Fraction) -> di } -def resample_source( +def _decode_source( source: str | Path, fps: float = 12.0, *, + source_indices: list[int] | None = None, width: int | None = None, height: int | None = None, ) -> tuple[bytes, dict]: @@ -130,9 +131,17 @@ def resample_source( the first decoded source frame, not container start_time or average FPS. """ source = Path(source) - if not math.isfinite(fps) or fps <= 0: + if ( + isinstance(fps, bool) + or not isinstance(fps, (int, float)) + or not math.isfinite(fps) + or fps <= 0 + ): raise ValueError("fps must be positive and finite") - if (width is None) != (height is None) or (width is not None and (width <= 0 or height <= 0)): + if (width is None) != (height is None) or ( + width is not None + and (type(width) is not int or type(height) is not int or width <= 0 or height <= 0) + ): raise ValueError("Specify both positive output dimensions or neither") source_sha = _sha256(source) probe = json.loads( @@ -158,7 +167,20 @@ def resample_source( raise ValueError("Source has missing decoded PTS") from error if not source_pts or any(b <= a for a, b in pairwise(source_pts)): raise ValueError("Source PTS must be present and strictly increasing") - filters = f"showinfo@source,fps@sample={fps},showinfo@sampled" + if source_indices is not None: + if ( + not isinstance(source_indices, list) + or not source_indices + or any(type(i) is not int or not 0 <= i < len(source_pts) for i in source_indices) + or any(b <= a for a, b in pairwise(source_indices)) + ): + raise ValueError( + "Explicit source indices must be strictly increasing integers in range" + ) + expression = "+".join(f"eq(n\\,{i})" for i in source_indices) + filters = f"showinfo@source,select={expression},showinfo@sampled" + else: + filters = f"showinfo@source,fps@sample={fps},showinfo@sampled" output = ["-f", "null", "-"] if width is not None: filters += f",scale={width}:{height}" @@ -181,7 +203,11 @@ def resample_source( ] decoded = _run(command) log = decoded.stderr.decode("utf-8", errors="replace") - provenance = _parse_selection(log, source_pts, time_base) + provenance = ( + _parse_explicit_selection(log, source_pts, time_base, source_indices) + if source_indices is not None + else _parse_selection(log, source_pts, time_base) + ) if width is not None and len(decoded.stdout) != len(provenance["frames"]) * width * height * 3: raise ValueError("RGB frame count does not match provenance") if _sha256(source) != source_sha: @@ -189,7 +215,9 @@ def resample_source( version = _run(["ffmpeg", "-version"]).stdout.decode().splitlines()[0] provenance.update( schema="wander.source-frame-provenance/1", - method="ffmpeg-fps-observed-selection", + method="ffmpeg-explicit-observed-selection" + if source_indices is not None + else "ffmpeg-fps-observed-selection", sourceSha256=source_sha, sourceTimeBase=str(time_base), sourceFirstPts=source_pts[0], @@ -204,6 +232,53 @@ def resample_source( return decoded.stdout, provenance +def _parse_explicit_selection(log, source_pts, time_base, selection): + input_base, inputs = _observed_frames(log, "source") + output_base, outputs = _observed_frames(log, "sampled") + if len(inputs) != len(source_pts) or len(outputs) != len(selection): + raise ValueError("Explicit selection decoded counts disagree") + offsets = { + pts * time_base - frame["pts"] * input_base + for pts, frame in zip(source_pts, inputs, strict=True) + } + if len(offsets) != 1: + raise ValueError("FFmpeg and FFprobe decoded source timestamps disagree") + frames = [] + for output, index in zip(outputs, selection, strict=True): + if ( + output["pts"] * output_base != inputs[index]["pts"] * input_base + or output["checksum"] != inputs[index]["checksum"] + ): + raise ValueError("Explicit selection does not match the emitted frame") + frames.append( + { + "frameIndex": output["index"], + "sourceIndex": index, + "sourcePts": source_pts[index], + "sourceTimeSeconds": float(source_pts[index] * time_base), + "sourceRelativeTimeSeconds": float((source_pts[index] - source_pts[0]) * time_base), + "resampledPts": output["pts"], + } + ) + return { + "sourceTimestampOffset": str(offsets.pop()), + "resampledTimeBase": str(output_base), + "frames": frames, + } + + +def resample_source(source, fps=12.0, *, width=None, height=None): + """Return RGB24 bytes and observed FPS-filter provenance (or mapping only).""" + return _decode_source(source, fps, width=width, height=height) + + +def select_source_frames(source, fps=12.0, *, source_indices, width=None, height=None): + """Decode exact source ordinals; fps labels the output, never changes its selection.""" + if source_indices is None: + raise ValueError("Explicit source indices are required") + return _decode_source(source, fps, source_indices=source_indices, width=width, height=height) + + def select_provenance(provenance: dict, selection: list[int]) -> list[dict]: """Retain exact source bindings in the caller's requested output order.""" frames = provenance["frames"] From 306c922b0ad90adcfa51eddfbd1e1343004c7b59 Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sun, 18 Oct 2026 00:56:00 -0400 Subject: [PATCH 02/14] fix: carry bounded authorization into web-triggered workers --- docs/pipeline-setup.md | 22 +++++++ orchestrator/agent/harness.py | 20 +++++++ orchestrator/session.py | 2 + scripts/tests/test_orchestrator_harness.py | 70 +++++++++++++++++++++- scripts/tests/test_orchestrator_session.py | 51 ++++++++++++++++ 5 files changed, 164 insertions(+), 1 deletion(-) diff --git a/docs/pipeline-setup.md b/docs/pipeline-setup.md index 011a7ee..4163e2e 100644 --- a/docs/pipeline-setup.md +++ b/docs/pipeline-setup.md @@ -37,6 +37,17 @@ runner also accepts the legacy `WANDER_RUNS_DIR` override. `WANDER_STAGE_LEDGER` shared paid-stage ledger; its default stays at repository `.context/pipeline-attempts.json` to retain previous accounting. The `wander` command and direct runner share that ledger. Keep the same absolute ledger across candidates and worktrees that share an allowance. + +For an explicitly authorized run, set `WANDER_STAGE_AUTHORIZATION` to the absolute path of +its bounded paid-stage authorization and `WANDER_VLM_BUDGET` to the absolute path of its VLM +request budget before starting the supervisor. These two settings pass through the explicit +environment allowlist to the reviewing harness and its stage subprocesses; their values are +not included in the generated prompt or brief. Restrict that supervisor to the selected run +and remove these settings before supervising unrelated runs. They do not create approvals: +the runner still validates the authorization's source, candidate, operation, execution count +and expiry against the same shared ledger, and the VLM budget separately limits requests. +See [paid recovery](paid-recovery.md) for the authorization and recovery procedure. + A differing `WANDER_PYTHON` is explicitly unverified. The default reviewing harness is Codex; `WANDER_REVIEW_AGENT` and `WANDER_REVIEW_COMMAND` select alternatives. Built-in Codex and Claude harnesses must pass a @@ -59,6 +70,17 @@ for each root; resumed sessions use the equivalent unchanged. The harness never infers shared roots or silently grants the whole repository; SQLite needs its directory writable for journal/WAL files as well as the database itself. +For authorized network-backed stages, set `WANDER_REVIEW_NETWORK=true` before starting the +supervisor. This explicitly enables network access for commands in the Codex `workspace-write` +sandbox, for both fresh and resumed sessions, without changing filesystem permissions. +The default is off, explicitly enforced in both fresh and resumed Codex commands so host +configuration or a previous session cannot implicitly enable it. +Values `true/false`, `1/0`, `yes/no`, and `on/off` are accepted without +case sensitivity; invalid values fail setup. Enabling it with another harness or sandbox +mode is rejected. This setting is translated to Codex configuration, not passed through as +an environment variable, and never triggers a fallback to a broader sandbox. Stage permits +and request budgets remain separate requirements. + Credentials come from the selected shared stage plan. World generation needs `WLT_API_KEY`, source-grounded world prompts and object descriptions need `OPENAI_API_KEY`, and Modal needs a complete token pair or the selected saved profile. The runner's default Modal profile is diff --git a/orchestrator/agent/harness.py b/orchestrator/agent/harness.py index f8021a8..7693e39 100644 --- a/orchestrator/agent/harness.py +++ b/orchestrator/agent/harness.py @@ -77,6 +77,8 @@ class HarnessPolicy: idle_nudges: int = 2 # Explicit operator grants for shared state outside the review workspace; never inferred. writable_roots: tuple[str, ...] = () + # Explicit network grant for commands inside Codex's workspace-write sandbox. + network_access: bool = False command: tuple[str, ...] = () # Only these host variables reach the harness. API credentials must be listed explicitly. passthrough: tuple[str, ...] = ("PATH", "HOME", "OPENAI_API_KEY") @@ -89,6 +91,10 @@ def __post_init__(self): raise ValueError("a command harness needs an explicit command") if self.sandbox not in {"read-only", "workspace-write", "danger-full-access"}: raise ValueError(f"unknown sandbox mode {self.sandbox!r}") + if not isinstance(self.network_access, bool): + raise ValueError("network access must be a boolean") + if self.network_access and (self.kind != "codex" or self.sandbox != "workspace-write"): + raise ValueError("network access requires the Codex workspace-write sandbox") if self.timeout_seconds < 1: raise ValueError("harness timeout must be positive") if self.idle_seconds < 0: @@ -115,6 +121,9 @@ def from_environment(cls, environ: dict[str, str] | None = None) -> HarnessPolic ) from None if not isinstance(writable_roots, list): raise ValueError("WANDER_REVIEW_WRITE_ROOTS must be a JSON list of directories") + network = environ.get("WANDER_REVIEW_NETWORK", "false").strip().lower() + if network not in {"true", "false", "1", "0", "yes", "no", "on", "off"}: + raise ValueError("WANDER_REVIEW_NETWORK must be true/false, 1/0, yes/no, or on/off") return cls( kind=kind, model=environ.get("WANDER_REVIEW_MODEL") or None, @@ -123,6 +132,7 @@ def from_environment(cls, environ: dict[str, str] | None = None) -> HarnessPolic idle_seconds=int(environ.get("WANDER_REVIEW_IDLE", "300")), idle_nudges=int(environ.get("WANDER_REVIEW_NUDGES", "2")), writable_roots=tuple(writable_roots), + network_access=network in {"true", "1", "yes", "on"}, command=command, ) @@ -152,6 +162,11 @@ def argv( "sandbox_workspace_write.writable_roots=" + json.dumps(self.writable_roots, ensure_ascii=False), ] + if self.sandbox == "workspace-write": + command += [ + "-c", + "sandbox_workspace_write.network_access=" + json.dumps(self.network_access), + ] return (*command, "-") if self.kind == "codex": # Prompt on stdin ("-"); edits are confined to the workspace by Codex's own sandbox. @@ -169,6 +184,11 @@ def argv( command += ["--model", self.model] for root in self.writable_roots: command += ["--add-dir", root] + if self.sandbox == "workspace-write": + command += [ + "-c", + "sandbox_workspace_write.network_access=" + json.dumps(self.network_access), + ] return (*command, "-") if self.kind == "claude": command = [ diff --git a/orchestrator/session.py b/orchestrator/session.py index d2e473f..ca80ffc 100644 --- a/orchestrator/session.py +++ b/orchestrator/session.py @@ -41,6 +41,8 @@ "WANDER_GPU_KEY", "WANDER_RUNS_ROOT", "WANDER_STAGE_LEDGER", + "WANDER_STAGE_AUTHORIZATION", + "WANDER_VLM_BUDGET", "WANDER_PYTHON", "WANDER_MODAL", ) diff --git a/scripts/tests/test_orchestrator_harness.py b/scripts/tests/test_orchestrator_harness.py index cb3a178..f0996a2 100644 --- a/scripts/tests/test_orchestrator_harness.py +++ b/scripts/tests/test_orchestrator_harness.py @@ -210,6 +210,70 @@ def test_the_prompt_reaches_the_agent_as_stdin(self): class PolicyTests(unittest.TestCase): + def test_network_access_is_explicit_for_fresh_and_resumed_codex(self): + for raw in ("true", "1", "yes", "on", " TRUE "): + policy = HarnessPolicy.from_environment({"WANDER_REVIEW_NETWORK": raw}) + self.assertTrue(policy.network_access) + self.assertNotIn("WANDER_REVIEW_NETWORK", policy.passthrough) + self.assertEqual(policy.extra_environment, {}) + for session in (None, "saved-session"): + with self.subTest(raw=raw, session=session): + argv = policy.argv(Path("/workspace"), Path("/workspace/TASK.md"), session) + setting = "sandbox_workspace_write.network_access=true" + self.assertEqual(argv.count(setting), 1) + self.assertEqual(argv[argv.index(setting) - 1], "-c") + self.assertNotIn("danger-full-access", " ".join(argv)) + if session: + self.assertIn('sandbox_mode="workspace-write"', argv) + else: + self.assertEqual(argv[argv.index("--sandbox") + 1], "workspace-write") + + def test_network_access_is_disabled_by_default_and_explicit_false(self): + for environ in ( + {}, + *({"WANDER_REVIEW_NETWORK": raw} for raw in ("false", "0", "no", "off")), + ): + policy = HarnessPolicy.from_environment(environ) + self.assertFalse(policy.network_access) + for session in (None, "saved-session"): + argv = policy.argv(Path("/workspace"), Path("/workspace/TASK.md"), session) + setting = "sandbox_workspace_write.network_access=false" + self.assertEqual(argv.count(setting), 1) + self.assertEqual(argv[argv.index(setting) - 1], "-c") + self.assertNotIn("sandbox_workspace_write.network_access=true", argv) + + def test_other_sandboxes_and_harnesses_receive_no_network_override(self): + for options in ( + {"sandbox": "read-only"}, + {"sandbox": "danger-full-access"}, + {"kind": "claude"}, + {"kind": "command", "command": ("custom-agent",)}, + ): + policy = HarnessPolicy(**options) + for session in (None, "saved-session"): + with self.subTest(options=options, session=session): + argv = policy.argv(Path("/workspace"), Path("/workspace/TASK.md"), session) + self.assertFalse(any("network_access" in arg for arg in argv)) + + def test_invalid_or_inapplicable_network_access_is_refused(self): + for raw in ("", "auto", "2", "enabled", "null"): + with self.subTest(raw=raw), self.assertRaisesRegex(ValueError, "WANDER_REVIEW_NETWORK"): + HarnessPolicy.from_environment({"WANDER_REVIEW_NETWORK": raw}) + for value in ("false", 1, None): + with self.subTest(value=value), self.assertRaisesRegex(ValueError, "boolean"): + HarnessPolicy(network_access=value) + for options in ( + {"kind": "claude"}, + {"kind": "command", "command": ("custom-agent",)}, + {"sandbox": "read-only"}, + {"sandbox": "danger-full-access"}, + ): + with ( + self.subTest(options=options), + self.assertRaisesRegex(ValueError, "Codex workspace-write"), + ): + HarnessPolicy(network_access=True, **options) + def test_the_clock_is_configured_from_the_environment(self): policy = HarnessPolicy.from_environment( {"WANDER_REVIEW_IDLE": "120", "WANDER_REVIEW_NUDGES": "1"} @@ -247,7 +311,11 @@ def test_new_codex_session_adds_only_explicit_writable_roots(self): roots = ("/shared state/database", "/shared state/ledger") policy = HarnessPolicy(writable_roots=roots) argv = policy.argv(Path("/ws"), Path("/ws/TASK.md")) - self.assertEqual(argv[-5:], ("--add-dir", roots[0], "--add-dir", roots[1], "-")) + directories = tuple( + argv[index + 1] for index, part in enumerate(argv) if part == "--add-dir" + ) + self.assertEqual(directories, roots) + self.assertEqual(argv[-1], "-") self.assertEqual(argv[argv.index("--sandbox") + 1], "workspace-write") self.assertEqual(argv[argv.index("--cd") + 1], "/ws") default = HarnessPolicy().argv(Path("/ws"), Path("/ws/TASK.md")) diff --git a/scripts/tests/test_orchestrator_session.py b/scripts/tests/test_orchestrator_session.py index eed72e8..80cdccc 100644 --- a/scripts/tests/test_orchestrator_session.py +++ b/scripts/tests/test_orchestrator_session.py @@ -390,6 +390,57 @@ def test_command_harness_passes_explicit_stage_executables_to_local_child(self): self.assertNotIn("WANDER_PYTHON", session.harness.policy.extra_environment) self.assertNotIn("WANDER_MODAL", session.harness.policy.extra_environment) + def test_paid_controls_reach_harness_and_stage_without_entering_prompt(self): + run_dir = self.open() + stage_repo = self.root / "stage-repository" + (stage_repo / "scripts").mkdir(parents=True) + names = ("WANDER_STAGE_AUTHORIZATION", "WANDER_VLM_BUDGET") + unrelated = "WANDER_UNRELATED_PRIVATE_SETTING" + observe = ( + "import json, os, pathlib\n" + f"values = {{key: os.environ.get(key) for key in {(*names, unrelated)!r}}}\n" + "pathlib.Path(os.environ['WANDER_RUN_DIR'], 'stage-environment.json')" + ".write_text(json.dumps(values))\n" + ) + (stage_repo / "scripts/run_clip.py").write_text(observe) + described = json.loads((run_dir / "run.json").read_text()) + described["repository"] = str(stage_repo) + (run_dir / "run.json").write_text(json.dumps(described)) + policy = self.agent( + "import json\n" + f"values = {{key: os.environ.get(key) for key in {(*names, unrelated)!r}}}\n" + "(brief.parent / 'harness-environment.json').write_text(json.dumps(values))\n" + "subprocess.run(['wander', 'step', '--wait', 'clean'], check=True)\n" + ) + controls = { + names[0]: str(self.root / "selected run authorization.json"), + names[1]: str(self.root / "selected run vlm budget.json"), + } + with patch.dict( + os.environ, + {**controls, unrelated: "must-stay-in-supervisor", "WANDER_REPOSITORY": str(ROOT)}, + ): + outcome = RunSession("run-1", run_dir, HarnessAgent(policy)).turn() + self.assertIsNone(outcome.error) + self.assertEqual(outcome.steps_run, 1) + for filename in ("harness-environment.json", "stage-environment.json"): + observed = json.loads((run_dir / filename).read_text()) + self.assertEqual(observed, {**controls, unrelated: None}) + for filename in ("BRIEF.md", "TASK.md", "prompt.txt"): + prompt = (run_dir / filename).read_text() + for value in (*controls, *controls.values(), "must-stay-in-supervisor"): + self.assertNotIn(value, prompt) + + def test_absent_paid_controls_are_not_added_to_harness_environment(self): + run_dir = self.open() + names = ("WANDER_STAGE_AUTHORIZATION", "WANDER_VLM_BUDGET") + with patch.dict(os.environ, {}, clear=False): + for name in names: + os.environ.pop(name, None) + session = RunSession("run-1", run_dir, HarnessAgent(self.agent("pass"))) + for name in names: + self.assertNotIn(name, session.harness.policy.extra_environment) + def test_a_turn_that_decides_nothing_stops_rather_than_repeating(self): run_dir = self.open() session = RunSession("run-1", run_dir, HarnessAgent(self.agent("pass"))) From 284fec214f6eb6d5fd48eba355c27d310f8454f1 Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sun, 18 Oct 2026 00:58:24 -0400 Subject: [PATCH 03/14] fix: retain bounded decoder error diagnostics --- scripts/tests/test_source_timing.py | 41 +++++++++++++++++++++++++++ worker/wander_worker/source_timing.py | 14 ++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/scripts/tests/test_source_timing.py b/scripts/tests/test_source_timing.py index 46e6cee..8e255d4 100644 --- a/scripts/tests/test_source_timing.py +++ b/scripts/tests/test_source_timing.py @@ -546,6 +546,47 @@ def damaged(command): ): source_timing.select_source_frames(clip, source_indices=[0, 2]) + def test_failed_decoder_reports_stderr_without_raw_rgb_stdout(self): + command = [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=size=16x16", + "-vf", + "missing_filter_diagnostic_fixture", + "-f", + "null", + "-", + ] + with self.assertRaises(RuntimeError) as caught: + source_timing._run(command) + message = str(caught.exception) + self.assertIn("ffmpeg exited with status", message) + self.assertIn("missing_filter_diagnostic_fixture", message) + self.assertIn("stderr", message) + + error = subprocess.CalledProcessError( + 7, + command, + output=b"RAW_RGB_MUST_NOT_APPEAR" * 10000, + stderr=b"DISCARDED_PREFIX" + b"x" * 20000 + b"fatal decoder detail at end", + ) + with patch.object(source_timing.subprocess, "run", side_effect=error) as run: + with self.assertRaises(RuntimeError) as caught: + source_timing._run(command) + run.assert_called_once_with(command, check=True, capture_output=True) + message = str(caught.exception) + self.assertIn("ffmpeg exited with status 7", message) + self.assertIn("truncated", message) + self.assertTrue(message.endswith("fatal decoder detail at end")) + self.assertNotIn("RAW_RGB_MUST_NOT_APPEAR", message) + self.assertNotIn("DISCARDED_PREFIX", message) + self.assertLess(len(message), 8300) + if __name__ == "__main__": unittest.main() diff --git a/worker/wander_worker/source_timing.py b/worker/wander_worker/source_timing.py index 45f79bb..30b7f87 100644 --- a/worker/wander_worker/source_timing.py +++ b/worker/wander_worker/source_timing.py @@ -24,7 +24,19 @@ def _sha256(path: Path) -> str: def _run(command: list[str]) -> subprocess.CompletedProcess: - return subprocess.run(command, check=True, capture_output=True) + try: + return subprocess.run(command, check=True, capture_output=True) + except subprocess.CalledProcessError as error: + # The decoder stdout is raw RGB; retain only bounded text diagnostics. + stderr = error.stderr or b"" + if isinstance(stderr, str): + stderr = stderr.encode("utf-8") + tail = stderr[-8192:].decode("utf-8", errors="replace") + truncated = " (tail; truncated)" if len(stderr) > 8192 else "" + raise RuntimeError( + f"{Path(command[0]).name} exited with status {error.returncode}; " + f"stderr{truncated}:\n{tail}" + ) from None def _observed_frames(log: str, name: str) -> tuple[Fraction, list[dict]]: From 39702702532c0a1ce072025254c2111eedb8194a Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sun, 18 Oct 2026 01:01:30 -0400 Subject: [PATCH 04/14] fix: balance exact-frame selection for worker ffmpeg --- docs/known-limits.md | 1 + scripts/tests/test_source_timing.py | 58 +++++++++++++++++++++++++++ worker/wander_worker/source_timing.py | 14 ++++++- 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/docs/known-limits.md b/docs/known-limits.md index 9b81c12..be0ccb5 100644 --- a/docs/known-limits.md +++ b/docs/known-limits.md @@ -5,6 +5,7 @@ Do not retry one without a new idea, and add a line here when you measure a new - Refining body pose against the footage makes it worse: a single camera leaves a null space. - Semantic person masks can omit lower bodies against similar-colored floors: the fresh bottle-source clean left black and light trousers in several sampled frames despite a successful worker exit. Tracked video cleaning now unions the existing per-person instance masks and retains their exact source-frame selection; equal FPS and frame count do not imply the same pixels (the observed 30-to-12 FPS tracker and cleaner selected different source frames). Missing, empty, or incomplete track masks and mismatched source/timing metadata block submission. Full mask coverage is not an inpainting-quality pass; inspect the resulting clean for remnants and damaged surroundings. Failure evidence: `.context/evidence/bottle-source-rerun/clean-comparison.jpg`. +- FFmpeg 5.1 rejects a long flat sum of exact-frame selection terms: the 120-frame clean correction failed before inpainting, and the parser failure reproduced in local Debian Bookworm while FFmpeg 8 passed. Balanced expressions preserve the same source indices and pass the full-resolution local Linux reproduction. Keep decoder stderr in failure reports; a successful local decode does not establish remote inpainting quality. Evidence: `.context/evidence/bottle-source-rerun/linux-decoder-check/`. - Faces below about 64 px of face height are unreadable; per-clip head refits do not help. - Smoke, fire, and water cannot be reconstructed from one moving camera. - Clips with too little parallax, or too dark, fail reconstruction. diff --git a/scripts/tests/test_source_timing.py b/scripts/tests/test_source_timing.py index 8e255d4..db1d276 100644 --- a/scripts/tests/test_source_timing.py +++ b/scripts/tests/test_source_timing.py @@ -587,6 +587,64 @@ def test_failed_decoder_reports_stderr_without_raw_rgb_stdout(self): self.assertNotIn("DISCARDED_PREFIX", message) self.assertLess(len(message), 8300) + def test_long_explicit_selection_preserves_every_exact_source_frame(self): + for count in (129, 1025): + with self.subTest(selected=count), tempfile.TemporaryDirectory() as directory: + clip = Path(directory) / "long.mkv" + indices = list(range(1, count * 2 + 1, 2)) + pixels = [ + bytes([i % 256, i // 256, (i * 17) % 256]) * 16 * 16 + for i in range(count * 2 + 3) + ] + subprocess.run( + [ + "ffmpeg", + "-v", + "error", + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + "-s", + "16x16", + "-r", + "30", + "-i", + "-", + "-c:v", + "ffv1", + str(clip), + ], + input=b"".join(pixels), + capture_output=True, + check=True, + timeout=30, + ) + raw, provenance = source_timing.select_source_frames( + clip, + source_indices=indices, + width=16, + height=16, + ) + self.assertEqual(raw, b"".join(pixels[i] for i in indices)) + self.assertEqual([f["sourceIndex"] for f in provenance["frames"]], indices) + self.assertEqual( + [f["frameIndex"] for f in provenance["frames"]], list(range(count)) + ) + expression = source_timing._selection_expression(indices) + self.assertIn(f"select={expression},", provenance["filter"]) + # Structural regression for legacy parsers, independently of the local FFmpeg version. + depth = maximum = 0 + for token in expression: + if token == "(": + depth += 1 + maximum = max(maximum, depth) + elif token == ")": + depth -= 1 + self.assertEqual(depth, 0) + self.assertEqual(expression.count("("), 2 * count - 1) + self.assertLessEqual(maximum, (count - 1).bit_length() + 1) + if __name__ == "__main__": unittest.main() diff --git a/worker/wander_worker/source_timing.py b/worker/wander_worker/source_timing.py index 30b7f87..d647795 100644 --- a/worker/wander_worker/source_timing.py +++ b/worker/wander_worker/source_timing.py @@ -126,6 +126,18 @@ def _parse_selection(log: str, source_pts: list[int], time_base: Fraction) -> di } +def _selection_expression(indices: list[int]) -> str: + # Older FFmpeg expression parsers exhaust their depth on a long flat sum. + # Pairing terms keeps parser depth logarithmic without changing the predicate. + terms = [f"eq(n\\,{index})" for index in indices] + while len(terms) > 1: + terms = [ + f"({terms[i]}+{terms[i + 1]})" if i + 1 < len(terms) else terms[i] + for i in range(0, len(terms), 2) + ] + return terms[0] + + def _decode_source( source: str | Path, fps: float = 12.0, @@ -189,7 +201,7 @@ def _decode_source( raise ValueError( "Explicit source indices must be strictly increasing integers in range" ) - expression = "+".join(f"eq(n\\,{i})" for i in source_indices) + expression = _selection_expression(source_indices) filters = f"showinfo@source,select={expression},showinfo@sampled" else: filters = f"showinfo@source,fps@sample={fps},showinfo@sampled" From 7833381302f17f0e6a7bae65b8e5beafb6d88aae Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sun, 18 Oct 2026 20:19:44 -0400 Subject: [PATCH 05/14] feat: stage pinned object weights with bounded CPU setup --- scripts/tests/test_stage_object_weights.py | 255 ++++++++++++++ worker/modal_stage_object_weights.py | 378 +++++++++++++++++++++ 2 files changed, 633 insertions(+) create mode 100644 scripts/tests/test_stage_object_weights.py create mode 100644 worker/modal_stage_object_weights.py diff --git a/scripts/tests/test_stage_object_weights.py b/scripts/tests/test_stage_object_weights.py new file mode 100644 index 0000000..002ba9d --- /dev/null +++ b/scripts/tests/test_stage_object_weights.py @@ -0,0 +1,255 @@ +"""CPU staging only; all download/provider interfaces are mocked.""" + +import hashlib +import io +import json +from pathlib import Path +import sys +import time +from unittest.mock import Mock, patch + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "worker")) +import modal_stage_object_weights as staging + + +@pytest.fixture +def manifest(): + return { + "schema": staging.SCHEMA, + "snapshots": [ + { + "repoId": staging.TRELLIS, + "revision": "a" * 40, + "files": ["pipeline.json", "ckpts/model.json", "ckpts/model.safetensors"], + }, + {"repoId": staging.SDXL, "revision": "b" * 40, "files": sorted(staging.SDXL_REQUIRED)}, + ], + "dino": { + "url": staging.DINO_URL, + "sha256": hashlib.sha256(b"dino").hexdigest(), + "bytes": 4, + }, + } + + +def downloader(**kwargs): + assert kwargs["token"] is False + path = ( + Path(kwargs["cache_dir"]) + / ("models--" + kwargs["repo_id"].replace("/", "--")) + / "snapshots" + / kwargs["revision"] + / kwargs["filename"] + ) + path.parent.mkdir(parents=True, exist_ok=True) + if kwargs["filename"] == "pipeline.json": + path.write_text(json.dumps({"args": {"models": {"decoder": "ckpts/model"}}})) + else: + path.write_bytes(b"test model bytes") + return str(path) + + +def run_stage(manifest, root, **kwargs): + return staging.stage_cache( + manifest, + time.time() + 60, + root=root, + commit=kwargs.pop("commit", Mock()), + downloader=kwargs.pop("downloader", downloader), + **kwargs, + ) + + +def test_success_pins_cache_layout_and_repeat_verifies_without_download(tmp_path, manifest): + receipt = tmp_path / "setup-receipts/object-weights-attempt.json" + commits = [] + + def commit(): + commits.append(json.loads(receipt.read_text())["status"]) + + def serve(*args, **kwargs): + assert commits == ["started"] + return io.BytesIO(b"dino") + + with patch.object(staging.urllib.request, "urlopen", side_effect=serve) as request: + result = run_stage(manifest, tmp_path, commit=commit) + assert result["status"] == "completed" + assert result["manifestSha256"] == staging.manifest_hash(manifest) + assert result["fileCount"] == sum(len(s["files"]) for s in manifest["snapshots"]) + 3 + assert "files" not in result + assert commits == ["started", "completed"] + request.assert_called_once_with(staging.DINO_URL, timeout=30) + assert (tmp_path / staging.DINO_PATH).read_bytes() == b"dino" + for snapshot in manifest["snapshots"]: + ref = ( + tmp_path + / "huggingface/hub" + / ("models--" + snapshot["repoId"].replace("/", "--")) + / "refs/main" + ) + assert ref.read_text() == snapshot["revision"] + forbidden = Mock(side_effect=AssertionError("no downloads on repeat")) + assert run_stage(manifest, tmp_path, downloader=forbidden) == result + forbidden.assert_not_called() + (tmp_path / staging.DINO_PATH).write_bytes(b"changed") + with pytest.raises(ValueError, match="changed"): + run_stage(manifest, tmp_path, downloader=forbidden) + + +@pytest.mark.parametrize( + "damage", + [ + "unpinned", + "extra_repo", + "missing_fp16", + "traversal", + "missing_trellis_pair", + "untrusted_dino", + "missing_hash", + ], +) +def test_invalid_manifest_never_starts_or_downloads(tmp_path, manifest, damage): + if damage == "unpinned": + manifest["snapshots"][0]["revision"] = "main" + elif damage == "extra_repo": + manifest["snapshots"][0]["repoId"] = "someone/model" + elif damage == "missing_fp16": + manifest["snapshots"][1]["files"].pop() + elif damage == "traversal": + manifest["snapshots"][0]["files"].append("../outside") + elif damage == "missing_trellis_pair": + manifest["snapshots"][0]["files"].pop() + elif damage == "untrusted_dino": + manifest["dino"]["url"] = "https://other.example/model" + else: + manifest["dino"].pop("sha256") + download, commit = Mock(), Mock() + with pytest.raises(ValueError): + run_stage(manifest, tmp_path, downloader=download, commit=commit) + download.assert_not_called() + commit.assert_not_called() + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize("remaining", [-1, 901, float("nan"), float("inf")]) +def test_invalid_deadline_prevents_work(tmp_path, manifest, remaining): + download = Mock() + with pytest.raises(ValueError, match="deadline"): + staging.stage_cache( + manifest, time.time() + remaining, root=tmp_path, commit=Mock(), downloader=download + ) + download.assert_not_called() + + +def test_failed_and_pending_claims_block_new_manifest_and_sanitize_receipts(tmp_path, manifest): + def fail(**kwargs): + receipt = json.loads((tmp_path / "setup-receipts/object-weights-attempt.json").read_text()) + assert receipt["status"] == "started" + raise RuntimeError("token=SECRET fixture") + + with pytest.raises(RuntimeError, match="failed") as error: + run_stage(manifest, tmp_path, downloader=fail) + receipt = tmp_path / "setup-receipts/object-weights-attempt.json" + assert "SECRET" not in receipt.read_text() + assert "SECRET" not in str(error.value) + saved = json.loads(receipt.read_text()) + assert saved["status"] == "failed" + manifest["snapshots"][0]["revision"] = "c" * 40 + for state in ("failed", "started"): + saved["status"] = state + receipt.write_text(json.dumps(saved)) + blocked = Mock() + with pytest.raises(RuntimeError, match="reconciliation"): + run_stage(manifest, tmp_path, downloader=blocked) + blocked.assert_not_called() + + +def test_dino_hash_failure_retains_failed_attempt_and_partial_file(tmp_path, manifest): + with patch.object(staging.urllib.request, "urlopen", return_value=io.BytesIO(b"oops")): + with pytest.raises(RuntimeError, match="failed"): + run_stage(manifest, tmp_path) + assert not (tmp_path / staging.DINO_PATH).exists() + assert (tmp_path / staging.DINO_PATH).with_suffix(".download").read_bytes() == b"oops" + assert ( + json.loads((tmp_path / "setup-receipts/object-weights-attempt.json").read_text())["status"] + == "failed" + ) + + +def test_existing_conflicting_ref_stops_before_download(tmp_path, manifest): + ref = tmp_path / "huggingface/hub/models--microsoft--TRELLIS-image-large/refs/main" + ref.parent.mkdir(parents=True) + ref.write_text("c" * 40) + download = Mock() + with pytest.raises(RuntimeError, match="failed"): + run_stage(manifest, tmp_path, downloader=download) + download.assert_not_called() + assert ref.read_text() == "c" * 40 + + +def test_pipeline_cannot_reference_unstaged_checkpoint(tmp_path, manifest): + def missing_model(**kwargs): + result = downloader(**kwargs) + if kwargs["filename"] == "pipeline.json": + Path(result).write_text('{"args":{"models":{"decoder":"ckpts/not-in-manifest"}}}') + return result + + with pytest.raises(RuntimeError, match="failed"): + run_stage(manifest, tmp_path, downloader=missing_model) + + +def test_local_submission_persists_before_call_and_blocks_second_call(tmp_path, manifest): + source, receipt = tmp_path / "manifest.json", tmp_path / "submission.json" + source.write_text(json.dumps(manifest)) + + def remote(*args): + assert json.loads(receipt.read_text())["status"] == "started" + raise TimeoutError("SECRET ambiguous provider result") + + with patch.object(staging.stage, "remote", side_effect=remote) as submit: + with pytest.raises(RuntimeError, match="ambiguous"): + staging.main(str(source), str(receipt), time.time() + 60) + with pytest.raises(RuntimeError, match="receipt exists"): + staging.main(str(source), str(receipt), time.time() + 60) + submit.assert_called_once() + assert "SECRET" not in receipt.read_text() + + +def test_no_gpu_image_or_inference_imports(): + import ast + + tree = ast.parse(Path(staging.__file__).read_text()) + imported = [node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom)] + imported += [ + alias.name + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + ] + assert not any( + name and name.split(".")[0] in {"torch", "diffusers", "trellis", "modal_image_to_3d"} + for name in imported + ) + functions = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "function" + ] + assert len(functions) == 1 + args = { + kw.arg: ast.literal_eval(kw.value) + for kw in functions[0].keywords + if kw.arg in {"cpu", "memory", "retries", "timeout", "max_containers"} + } + assert args == { + "cpu": (1, 2), + "memory": (1024, 4096), + "retries": 0, + "timeout": 900, + "max_containers": 1, + } + assert not any(kw.arg == "gpu" for kw in functions[0].keywords) diff --git a/worker/modal_stage_object_weights.py b/worker/modal_stage_object_weights.py new file mode 100644 index 0000000..8f0834a --- /dev/null +++ b/worker/modal_stage_object_weights.py @@ -0,0 +1,378 @@ +"""Explicit pinned, CPU-only object cache staging. No GPU image imports or inference. + +modal run worker/modal_stage_object_weights.py --manifest /absolute/manifest.json \ + --receipt /absolute/submission.json --deadline-epoch UNIX_SECONDS + +A failed or started receipt requires operator reconciliation; changing the manifest never +resets the remote attempt. Completed matching attempts only verify retained file hashes. +""" + +from contextlib import contextmanager +import hashlib +import json +import math +import os +from pathlib import Path, PurePosixPath +import re +import signal +import tempfile +import time +import urllib.request + +import modal + +SCHEMA = "wander.object-cache-manifest/1" +RECEIPT_SCHEMA = "wander.object-cache-attempt/1" +VOLUME = "wander-image-to-3d-cache" +TRELLIS = "microsoft/TRELLIS-image-large" +SDXL = "stabilityai/stable-diffusion-xl-base-1.0" +DINO_URL = "https://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_reg4_pretrain.pth" +DINO_PATH = "torch/hub/checkpoints/dinov2_vitl14_reg4_pretrain.pth" +SDXL_REQUIRED = { + "model_index.json", + "scheduler/scheduler_config.json", + "text_encoder/config.json", + "text_encoder/model.fp16.safetensors", + "text_encoder_2/config.json", + "text_encoder_2/model.fp16.safetensors", + "tokenizer/merges.txt", + "tokenizer/special_tokens_map.json", + "tokenizer/tokenizer_config.json", + "tokenizer/vocab.json", + "tokenizer_2/merges.txt", + "tokenizer_2/special_tokens_map.json", + "tokenizer_2/tokenizer_config.json", + "tokenizer_2/vocab.json", + "unet/config.json", + "unet/diffusion_pytorch_model.fp16.safetensors", + "vae/config.json", + "vae/diffusion_pytorch_model.fp16.safetensors", +} + +image = modal.Image.debian_slim(python_version="3.12").uv_pip_install("huggingface_hub==0.28.1") +app = modal.App("wander-object-cache-setup") +cache = modal.Volume.from_name(VOLUME) + + +def manifest_hash(manifest): + return hashlib.sha256( + json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +def validate_manifest(manifest): + if ( + not isinstance(manifest, dict) + or set(manifest) != {"schema", "snapshots", "dino"} + or manifest["schema"] != SCHEMA + ): + raise ValueError("Invalid object cache manifest") + snapshots = manifest["snapshots"] + if not isinstance(snapshots, list) or len(snapshots) != 2: + raise ValueError("Exactly TRELLIS and SDXL snapshots are required") + seen = set() + for snapshot in snapshots: + if not isinstance(snapshot, dict) or set(snapshot) != {"repoId", "revision", "files"}: + raise ValueError("Invalid pinned snapshot fields") + repo, revision, files = snapshot["repoId"], snapshot["revision"], snapshot["files"] + if ( + repo not in (TRELLIS, SDXL) + or repo in seen + or not isinstance(revision, str) + or not re.fullmatch(r"[0-9a-f]{40}", revision) + ): + raise ValueError("Snapshot must name an allowed repository and exact commit") + seen.add(repo) + if not isinstance(files, list) or not 1 <= len(files) <= 64: + raise ValueError("Snapshot requires a bounded explicit file list") + for name in files: + if ( + not isinstance(name, str) + or len(name) > 200 + or not re.fullmatch(r"[A-Za-z0-9_./-]+", name) + or PurePosixPath(name).is_absolute() + or ".." in PurePosixPath(name).parts + or str(PurePosixPath(name)) != name + ): + raise ValueError("Snapshot files must be exact relative paths without patterns") + if len(set(files)) != len(files): + raise ValueError("Duplicate snapshot file") + if repo == SDXL and set(files) != SDXL_REQUIRED: + raise ValueError("SDXL requires exactly the fp16 pipeline files") + if repo == TRELLIS: + if "pipeline.json" not in files or any( + name != "pipeline.json" + and not re.fullmatch(r"ckpts/[A-Za-z0-9_-]+\.(json|safetensors)", name) + for name in files + ): + raise ValueError("TRELLIS requires pipeline.json and explicit checkpoint files") + stems = { + name.removesuffix(".json") + for name in files + if name.startswith("ckpts/") and name.endswith(".json") + } + if not stems or set(files) != {"pipeline.json"} | { + stem + suffix for stem in stems for suffix in (".json", ".safetensors") + }: + raise ValueError("TRELLIS checkpoint config and weight pairs are required") + dino = manifest["dino"] + if ( + not isinstance(dino, dict) + or set(dino) != {"url", "sha256", "bytes"} + or dino["url"] != DINO_URL + or not isinstance(dino["sha256"], str) + or not re.fullmatch(r"[0-9a-f]{64}", dino["sha256"]) + or type(dino["bytes"]) is not int + or not 0 < dino["bytes"] <= 2 * 1024**3 + ): + raise ValueError("DINO requires the official URL, exact SHA256 and bounded byte count") + return manifest + + +def validate_deadline(deadline_epoch): + if ( + type(deadline_epoch) not in (int, float) + or not math.isfinite(deadline_epoch) + or not 0 < deadline_epoch - time.time() <= 900 + ): + raise ValueError("An absolute future deadline within 900 seconds is required") + + +def digest_file(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def write_receipt(path, document): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_symlink(): + raise ValueError("Receipt symlinks are not allowed") + descriptor, temporary = tempfile.mkstemp(dir=path.parent, prefix=".object-cache-") + try: + with os.fdopen(descriptor, "w") as stream: + json.dump(document, stream, indent=2, allow_nan=False) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + Path(temporary).unlink(missing_ok=True) + + +@contextmanager +def deadline_guard(deadline_epoch): + validate_deadline(deadline_epoch) + + def expired(*_): + raise TimeoutError("Object cache staging deadline exhausted") + + previous = signal.signal(signal.SIGALRM, expired) + signal.setitimer(signal.ITIMER_REAL, max(0.01, deadline_epoch - time.time() - 10)) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def file_record(root, path): + root, path = Path(root).resolve(), Path(path) + if not path.resolve().is_relative_to(root) or not path.is_file() or path.stat().st_size == 0: + raise ValueError("Missing or escaping cache file") + return { + "path": path.relative_to(root).as_posix(), + "bytes": path.stat().st_size, + "sha256": digest_file(path), + } + + +def download_dino(root, pin): + target = Path(root) / DINO_PATH + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() or target.is_symlink(): + if ( + target.is_symlink() + or target.stat().st_size != pin["bytes"] + or digest_file(target) != pin["sha256"] + ): + raise ValueError("Existing DINO checkpoint conflicts with the pinned manifest") + return target + pending = target.with_suffix(".download") + with pending.open("xb") as output: + with urllib.request.urlopen(pin["url"], timeout=30) as response: + total = 0 + while chunk := response.read(1024 * 1024): + total += len(chunk) + if total > pin["bytes"]: + raise ValueError("DINO download exceeds pinned byte count") + output.write(chunk) + if total != pin["bytes"] or digest_file(pending) != pin["sha256"]: + raise ValueError("DINO download does not match pinned size and SHA256") + pending.replace(target) + return target + + +def stage_cache(manifest, deadline_epoch, *, root, commit, downloader, call_id=None): + """Local-testable staging core. Caller serializes invocations for this volume.""" + validate_manifest(manifest) + validate_deadline(deadline_epoch) + root = Path(root).resolve() + receipt = root / "setup-receipts/object-weights-attempt.json" + pin = manifest_hash(manifest) + if receipt.exists(): + saved = json.loads(receipt.read_text()) + if saved.get("status") != "completed" or saved.get("manifestSha256") != pin: + raise RuntimeError("Prior cache attempt requires reconciliation; no automatic repeat") + for record in saved["files"]: + if file_record(root, root / record["path"]) != record: + raise ValueError("Completed object cache changed; reconciliation required") + return {key: value for key, value in saved.items() if key != "files"} + report = { + "schema": RECEIPT_SCHEMA, + "status": "started", + "manifestSha256": pin, + "volume": VOLUME, + "startedEpoch": time.time(), + "deadlineEpoch": deadline_epoch, + "functionCallId": call_id, + "cpuLimit": 2, + "memoryLimitMiB": 4096, + "gpu": None, + "files": [], + } + write_receipt(receipt, report) + commit() # Durable reservation before any download. + try: + with deadline_guard(deadline_epoch): + hub = root / "huggingface/hub" + for snapshot in manifest["snapshots"]: + repo, revision = snapshot["repoId"], snapshot["revision"] + model = hub / ("models--" + repo.replace("/", "--")) + ref = model / "refs/main" + if ref.exists() and ref.read_text().strip() != revision: + raise ValueError("Existing model main reference conflicts with pinned revision") + for name in snapshot["files"]: + path = Path( + downloader( + repo_id=repo, + revision=revision, + filename=name, + cache_dir=str(hub), + token=False, + ) + ) + if path != model / "snapshots" / revision / name: + raise ValueError("Downloader returned an unexpected cache path") + report["files"].append(file_record(root, path)) + if repo == TRELLIS: + pipeline = json.loads( + (model / "snapshots" / revision / "pipeline.json").read_text() + ) + models = pipeline.get("args", {}).get("models") + if ( + not isinstance(models, dict) + or not models + or any( + not isinstance(value, str) + or {value + ".json", value + ".safetensors"} - set(snapshot["files"]) + for value in models.values() + ) + ): + raise ValueError( + "TRELLIS pipeline references checkpoints absent from manifest" + ) + ref.parent.mkdir(parents=True, exist_ok=True) + ref.write_text(revision) + report["files"].append(file_record(root, ref)) + report["files"].append(file_record(root, download_dino(root, manifest["dino"]))) + report.update( + status="completed", + finishedEpoch=time.time(), + fileCount=len(report["files"]), + bytes=sum(item["bytes"] for item in report["files"]), + inventorySha256=manifest_hash(report["files"]), + snapshots=[ + { + "repoId": item["repoId"], + "revision": item["revision"], + "fileCount": len(item["files"]), + } + for item in manifest["snapshots"] + ], + ) + write_receipt(receipt, report) + commit() + except BaseException as error: + report.update(status="failed", finishedEpoch=time.time(), errorType=type(error).__name__) + write_receipt(receipt, report) + commit() + raise RuntimeError( + f"Object cache staging failed ({type(error).__name__}); inspect retained receipt" + ) from None + return {key: value for key, value in report.items() if key != "files"} + + +@app.function( + image=image, + cpu=(1, 2), + memory=(1024, 4096), + timeout=900, + retries=0, + max_containers=1, + scaledown_window=2, + volumes={"/cache": cache}, +) +def stage(manifest: dict, deadline_epoch: float): + from huggingface_hub import hf_hub_download + + cache.reload() + return stage_cache( + manifest, + deadline_epoch, + root="/cache", + commit=cache.commit, + downloader=hf_hub_download, + call_id=modal.current_function_call_id(), + ) + + +@app.local_entrypoint() +def main(manifest: str, receipt: str, deadline_epoch: float): + import fcntl + + manifest_path, receipt_path = Path(manifest), Path(receipt) + if not manifest_path.is_absolute() or not receipt_path.is_absolute(): + raise ValueError("Manifest and submission receipt paths must be absolute") + document = validate_manifest(json.loads(manifest_path.read_text())) + validate_deadline(deadline_epoch) + receipt_path.parent.mkdir(parents=True, exist_ok=True) + with Path(str(receipt_path) + ".lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if receipt_path.exists() or receipt_path.is_symlink(): + raise RuntimeError("Submission receipt exists; inspect/reconcile without resubmission") + record = { + "schema": RECEIPT_SCHEMA, + "status": "started", + "manifestSha256": manifest_hash(document), + "startedEpoch": time.time(), + "deadlineEpoch": deadline_epoch, + } + write_receipt(receipt_path, record) + try: + result = stage.remote(document, deadline_epoch) + except BaseException as error: + record.update( + status="failed", errorType=type(error).__name__, finishedEpoch=time.time() + ) + write_receipt(receipt_path, record) + raise RuntimeError( + "Object cache submission failed or is ambiguous; reconcile receipt before any retry" + ) from None + record.update(status="completed", result=result, finishedEpoch=time.time()) + write_receipt(receipt_path, record) + print(json.dumps(result, indent=2)) From 849b3fb1f01198aededca38f7477726d3558b07b Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sun, 18 Oct 2026 20:29:18 -0400 Subject: [PATCH 06/14] fix: preflight retained object models before GPU work --- docs/known-limits.md | 1 + scripts/tests/test_object_cache_offline.py | 357 +++++++++++++++++++++ worker/modal_image_to_3d.py | 226 ++++++++++++- worker/modal_stage_object_aux.py | 245 ++++++++++++++ 4 files changed, 812 insertions(+), 17 deletions(-) create mode 100644 scripts/tests/test_object_cache_offline.py create mode 100644 worker/modal_stage_object_aux.py diff --git a/docs/known-limits.md b/docs/known-limits.md index be0ccb5..e35ada7 100644 --- a/docs/known-limits.md +++ b/docs/known-limits.md @@ -6,6 +6,7 @@ Do not retry one without a new idea, and add a line here when you measure a new - Refining body pose against the footage makes it worse: a single camera leaves a null space. - Semantic person masks can omit lower bodies against similar-colored floors: the fresh bottle-source clean left black and light trousers in several sampled frames despite a successful worker exit. Tracked video cleaning now unions the existing per-person instance masks and retains their exact source-frame selection; equal FPS and frame count do not imply the same pixels (the observed 30-to-12 FPS tracker and cleaner selected different source frames). Missing, empty, or incomplete track masks and mismatched source/timing metadata block submission. Full mask coverage is not an inpainting-quality pass; inspect the resulting clean for remnants and damaged surroundings. Failure evidence: `.context/evidence/bottle-source-rerun/clean-comparison.jpg`. - FFmpeg 5.1 rejects a long flat sum of exact-frame selection terms: the 120-frame clean correction failed before inpainting, and the parser failure reproduced in local Debian Bookworm while FFmpeg 8 passed. Balanced expressions preserve the same source indices and pass the full-resolution local Linux reproduction. Keep decoder stderr in failure reports; a successful local decode does not establish remote inpainting quality. Evidence: `.context/evidence/bottle-source-rerun/linux-decoder-check/`. +- Complete tracked person masks can still omit a carried backpack. In the fresh bottle clean, input and output masks matched on all 120 frames, but four inspected backpack regions retained 17–42% unmasked source pixels. A local union with `moved_content_masks` covered those regions but also classified static stair posts and railings as people; its residual-area guard cannot reject semantic person false positives. Do not promote that union as a cleaning fix. Evidence: `.context/evidence/bottle-source-rerun/balanced-clean-review/`. - Faces below about 64 px of face height are unreadable; per-clip head refits do not help. - Smoke, fire, and water cannot be reconstructed from one moving camera. - Clips with too little parallax, or too dark, fail reconstruction. diff --git a/scripts/tests/test_object_cache_offline.py b/scripts/tests/test_object_cache_offline.py new file mode 100644 index 0000000..63cc713 --- /dev/null +++ b/scripts/tests/test_object_cache_offline.py @@ -0,0 +1,357 @@ +"""Pinned cache contracts and loader routing without GPU/model/provider calls.""" + +import hashlib +import io +import json +from pathlib import Path +import sys +import tarfile +import time +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "worker")) +import modal_image_to_3d as worker +import modal_stage_object_aux as aux +from modal_stage_object_weights import manifest_hash, file_record + + +@pytest.fixture +def retained(tmp_path): + inventory = [] + + def write(relative, content=b"fixture"): + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + return file_record(tmp_path, path) + + models = [] + for repo in (worker.MODEL_ID, worker.REFINER_ID): + relative = "huggingface/hub/models--" + repo.replace("/", "--") + "/snapshots/" + "a" * 40 + config = "pipeline.json" if repo == worker.MODEL_ID else "model_index.json" + inventory.append(write(relative + "/" + config)) + models.append({"repoId": repo, "revision": "a" * 40}) + inventory.append(write(worker.DINO_CHECKPOINT)) + primary = { + "schema": "wander.object-cache-attempt/1", + "status": "completed", + "files": inventory, + "inventorySha256": manifest_hash(inventory), + "snapshots": models, + } + code = "torch/hub/dinov2-" + "b" * 40 + inventory = [write(code + "/hubconf.py"), write("u2net/u2net.onnx")] + auxiliary = { + "schema": "wander.object-aux-attempt/1", + "status": "completed", + "files": inventory, + "inventorySha256": manifest_hash(inventory), + "dinoCodePath": code, + "dinoRevision": "b" * 40, + } + root = tmp_path / "setup-receipts" + root.mkdir() + for name, doc in zip(worker.CACHE_RECEIPTS, (primary, auxiliary), strict=True): + (root / name).write_text(json.dumps(doc)) + return tmp_path + + +def test_cpu_preflight_verifies_hashes_and_pins_snapshots(retained): + result = worker.checked_object_cache(retained, need_refiner=True) + assert result["models"][worker.MODEL_ID]["path"].endswith("/snapshots/" + "a" * 40) + assert ( + worker.checked_object_cache(retained, expected=result["receipts"], hash_files=False) + == result + ) + (retained / worker.DINO_CHECKPOINT).write_bytes(b"changed") + with pytest.raises(ValueError, match="checksum"): + worker.checked_object_cache(retained) + + +@pytest.mark.parametrize("damage", ["pending", "missing", "receipt_changed", "escape", "inventory"]) +def test_preflight_fails_closed_before_model_loading(retained, damage): + pins = worker.checked_object_cache(retained)["receipts"] + path = retained / "setup-receipts/object-aux-attempt.json" + doc = json.loads(path.read_text()) + if damage == "pending": + doc["status"] = "started" + elif damage == "missing": + (retained / "u2net/u2net.onnx").unlink() + elif damage == "receipt_changed": + doc["extra"] = "changed receipt" + elif damage == "escape": + doc["files"][0]["path"] = "../escape" + doc["inventorySha256"] = manifest_hash(doc["files"]) + else: + doc["inventorySha256"] = "0" * 64 + path.write_text(json.dumps(doc)) + with pytest.raises(ValueError): + worker.checked_object_cache(retained, expected=pins) + + +def test_trellis_uses_local_hub_source_and_exact_cached_checkpoint(retained): + result = worker.checked_object_cache(retained) + load_weights = Mock(return_value={"exact": "state"}) + torch = SimpleNamespace(load=load_weights) + + def original_hub(path, name, **kwargs): + assert path == result["dinoCodePath"] + assert name == "dinov2_vitl14_reg" + assert kwargs == {"source": "local", "pretrained": True} + return torch.hub.load_state_dict_from_url( + aux.U2NET_URL.replace( + "github.com/danielgatis/rembg/releases/download/v0.0.0/u2net.onnx", "invalid" + ) + ) + + torch.hub = SimpleNamespace( + load=original_hub, load_state_dict_from_url=Mock(side_effect=AssertionError("network")) + ) + hf = SimpleNamespace(hf_hub_download=Mock(side_effect=AssertionError("network"))) + + class Pipeline: + @staticmethod + def from_pretrained(path): + assert path == result["models"][worker.MODEL_ID]["path"] + with pytest.raises(ValueError, match="unstaged"): + hf.hf_hub_download("other", "weights") + with pytest.raises(ValueError, match="checkpoint"): + torch.hub.load("facebookresearch/dinov2", "dinov2_vitl14_reg", pretrained=True) + return torch.hub.load_state_dict_from_url( + "https://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_reg4_pretrain.pth", + map_location="cpu", + check_hash=True, + ) + + with patch.dict(sys.modules, {"torch": torch, "huggingface_hub": hf}): + assert worker.local_trellis_load(Pipeline, result) == {"exact": "state"} + load_weights.assert_called_once_with( + result["dinoWeightsPath"], map_location="cpu", weights_only=True + ) + assert torch.hub.load is original_hub + hf.hf_hub_download.assert_not_called() + + +def test_background_removal_model_uses_local_path(retained): + result = worker.checked_object_cache(retained) + + class Session: + download_models = Mock(side_effect=AssertionError("must not download")) + + pipeline = SimpleNamespace(preprocess_image=lambda image: Session.download_models()) + with patch.dict(sys.modules, {"rembg.sessions.u2net": SimpleNamespace(U2netSession=Session)}): + assert worker.local_preprocess(pipeline, object(), result) == result["u2netPath"] + Session.download_models.assert_not_called() + + +def test_main_stops_before_gpu_when_cpu_preflight_fails(tmp_path): + image = tmp_path / "crop.png" + image.write_bytes(b"no image decode required") + with ( + patch.object(worker.cache_preflight, "remote", side_effect=ValueError("missing cache")), + patch.object(worker.refine_crop, "remote") as refine, + patch.object(worker.image_to_3d, "remote") as shape, + ): + with pytest.raises(ValueError, match="missing cache"): + worker.main(image=str(image), out_dir=str(tmp_path / "out"), refine_first=True) + refine.assert_not_called() + shape.assert_not_called() + + +@pytest.fixture +def auxiliary_manifest(): + return { + "schema": aux.SCHEMA, + "dinoCode": {"revision": "c" * 40, "archiveSha256": "d" * 64, "bytes": 100}, + "u2net": {"url": aux.U2NET_URL, "md5": aux.U2NET_MD5, "bytes": 4}, + } + + +def code_archive(revision, dangerous=False): + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode="w:gz") as archive: + entry = tarfile.TarInfo( + "dinov2-" + revision + ("/../escape" if dangerous else "/hubconf.py") + ) + entry.size = 4 + archive.addfile(entry, io.BytesIO(b"code")) + return stream.getvalue() + + +def test_auxiliary_staging_has_separate_receipt_and_repeat_guard(tmp_path, auxiliary_manifest): + revision = auxiliary_manifest["dinoCode"]["revision"] + + def download(url, path, *args): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(code_archive(revision) if "codeload" in url else b"onnx") + + primary = tmp_path / "setup-receipts/object-weights-attempt.json" + primary.parent.mkdir() + primary.write_text("primary must remain unchanged") + with patch.object(aux, "download", side_effect=download) as fetch: + result = aux.stage_aux(auxiliary_manifest, time.time() + 60, root=tmp_path, commit=Mock()) + assert result["status"] == "completed" + assert result["u2netSha256"] == hashlib.sha256(b"onnx").hexdigest() + assert ( + aux.stage_aux(auxiliary_manifest, time.time() + 60, root=tmp_path, commit=Mock()) + == result + ) + assert fetch.call_count == 2 + assert primary.read_text() == "primary must remain unchanged" + receipt = tmp_path / "setup-receipts/object-aux-attempt.json" + saved = json.loads(receipt.read_text()) + saved["status"] = "failed" + receipt.write_text(json.dumps(saved)) + with ( + patch.object(aux, "download") as fetch, + pytest.raises(RuntimeError, match="reconciliation"), + ): + aux.stage_aux(auxiliary_manifest, time.time() + 60, root=tmp_path, commit=Mock()) + fetch.assert_not_called() + + +def test_auxiliary_download_checks_hash_and_size(tmp_path): + with patch.object(aux.urllib.request, "urlopen", return_value=io.BytesIO(b"onnx")) as fetch: + aux.download(aux.U2NET_URL, tmp_path / "model", 4, "md5", hashlib.md5(b"onnx").hexdigest()) + fetch.assert_called_once_with(aux.U2NET_URL, timeout=30) + with ( + patch.object(aux.urllib.request, "urlopen", return_value=io.BytesIO(b"bad")), + pytest.raises(ValueError, match="checksum/size"), + ): + aux.download(aux.U2NET_URL, tmp_path / "wrong", 4, "md5", aux.U2NET_MD5) + + +def test_auxiliary_archive_rejects_escaping_paths(tmp_path): + archive = tmp_path / "code.tar.gz" + archive.write_bytes(code_archive("c" * 40, dangerous=True)) + with pytest.raises(ValueError, match="member"): + aux.extract_code(archive, tmp_path / "out", "c" * 40) + assert not (tmp_path / "escape").exists() + + +@pytest.mark.parametrize("text_only", [False, True]) +def test_refiner_loads_only_pinned_local_snapshot(retained, text_only, monkeypatch): + from PIL import Image + + selected = worker.checked_object_cache(retained, need_refiner=True) + buffer = io.BytesIO() + image = Image.new("RGB", (8, 8)) + image.save(buffer, format="PNG") + pipeline = Mock(return_value=SimpleNamespace(images=[image])) + loader = Mock(return_value=pipeline) + diffusers = SimpleNamespace( + StableDiffusionXLPipeline=SimpleNamespace(from_pretrained=loader), + StableDiffusionXLImg2ImgPipeline=SimpleNamespace(from_pretrained=loader), + ) + torch = SimpleNamespace( + float16="float16", + cuda=SimpleNamespace(empty_cache=lambda: None), + Generator=lambda device: SimpleNamespace(manual_seed=lambda seed: seed), + ) + monkeypatch.setattr(worker, "checked_object_cache", lambda **kwargs: selected) + with ( + patch.dict(sys.modules, {"torch": torch, "diffusers": diffusers}), + patch.object(worker.cache, "commit"), + patch.object(worker.cache, "reload"), + patch.dict("os.environ", {}, clear=False), + ): + result = worker.refine_crop.local( + buffer.getvalue(), size=8, text_only=text_only, cache_receipts=selected["receipts"] + ) + loader.assert_called_once_with( + selected["models"][worker.REFINER_ID]["path"], + torch_dtype="float16", + variant="fp16", + use_safetensors=True, + local_files_only=True, + ) + assert result["meta"]["modelRevision"] == selected["models"][worker.REFINER_ID]["revision"] + + +def test_auxiliary_failure_receipt_is_durable_and_sanitized(tmp_path, auxiliary_manifest): + committed = [] + + def commit(): + committed.append( + json.loads((tmp_path / "setup-receipts/object-aux-attempt.json").read_text())["status"] + ) + + with ( + patch.object(aux, "download", side_effect=TimeoutError("secret header")), + pytest.raises(RuntimeError, match="reconcile"), + ): + aux.stage_aux(auxiliary_manifest, time.time() + 60, root=tmp_path, commit=commit) + assert committed == ["started", "failed"] + assert "secret header" not in (tmp_path / "setup-receipts/object-aux-attempt.json").read_text() + + +def test_auxiliary_resources_are_bounded_and_cpu_only(): + import ast + + tree = ast.parse(Path(aux.__file__).read_text()) + functions = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "function" + ] + assert len(functions) == 1 + limits = { + kw.arg: ast.literal_eval(kw.value) + for kw in functions[0].keywords + if kw.arg in {"cpu", "memory", "retries", "timeout", "max_containers"} + } + assert limits == { + "cpu": (1, 2), + "memory": (1024, 4096), + "retries": 0, + "timeout": 900, + "max_containers": 1, + } + assert all(kw.arg != "gpu" for kw in functions[0].keywords) + + +def test_auxiliary_entrypoint_packages_helper_and_imports_without_repository(tmp_path): + import ast + import shutil + import subprocess + + source = Path(aux.__file__) + tree = ast.parse(source.read_text()) + mounts = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "add_local_file" + ] + assert len(mounts) == 1 + mount = mounts[0] + assert ast.literal_eval(mount.args[1]) == "/root/modal_stage_object_weights.py" + assert {kw.arg: ast.literal_eval(kw.value) for kw in mount.keywords} == {"copy": True} + assert isinstance(mount.args[0], ast.Call) + assert mount.args[0].func.attr == "with_name" + assert ast.literal_eval(mount.args[0].args[0]) == "modal_stage_object_weights.py" + for filename in ("modal_stage_object_aux.py", "modal_stage_object_weights.py"): + shutil.copyfile(source.with_name(filename), tmp_path / filename) + # Reproduce the remote import namespace: only the entrypoint and its explicit helper, + # installed packages, and stdlib; no repository/worker package or PYTHONPATH fallback. + result = subprocess.run( + [ + sys.executable, + "-I", + "-c", + "import sys; sys.path.insert(0, sys.argv[1]); import modal_stage_object_aux as m; assert m.manifest_hash({'a':1}); print('standalone import passed')", + str(tmp_path), + ], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=30, + check=True, + ) + assert result.stdout.strip() == "standalone import passed" diff --git a/worker/modal_image_to_3d.py b/worker/modal_image_to_3d.py index 573d604..41dfac1 100644 --- a/worker/modal_image_to_3d.py +++ b/worker/modal_image_to_3d.py @@ -156,6 +156,180 @@ cache = modal.Volume.from_name("wander-image-to-3d-cache", create_if_missing=True) +CACHE_RECEIPTS = { + "object-weights-attempt.json": "wander.object-cache-attempt/1", + "object-aux-attempt.json": "wander.object-aux-attempt/1", +} +DINO_CHECKPOINT = "torch/hub/checkpoints/dinov2_vitl14_reg4_pretrain.pth" + + +def checked_object_cache( + root="/cache", *, need_refiner=False, need_rembg=True, expected=None, hash_files=True +): + """Validate staging receipts on CPU; GPU repeats scope/size and receipt-pin checks.""" + import hashlib + import json + import re + + root = Path(root).resolve() + documents, pins, records = {}, {}, {} + for filename, schema in CACHE_RECEIPTS.items(): + path = root / "setup-receipts" / filename + raw = path.read_bytes() + doc = json.loads(raw) + if doc.get("schema") != schema or doc.get("status") != "completed": + raise ValueError("Object cache setup is not complete") + files = doc.get("files") + if not isinstance(files, list) or not files or len(files) > 2500: + raise ValueError("Object cache inventory is missing or unbounded") + inventory_hash = hashlib.sha256( + json.dumps(files, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + if inventory_hash != doc.get("inventorySha256"): + raise ValueError("Object cache inventory changed") + for record in files: + relative = record.get("path") + if ( + not isinstance(relative, str) + or Path(relative).is_absolute() + or ".." in Path(relative).parts + or relative in records + ): + raise ValueError("Invalid object cache inventory path") + file = root / relative + if ( + not file.resolve().is_relative_to(root) + or not file.is_file() + or type(record.get("bytes")) is not int + or file.stat().st_size != record["bytes"] + ): + raise ValueError("Object cache file missing, changed or escaping") + if hash_files: + with file.open("rb") as stream: + if hashlib.file_digest(stream, "sha256").hexdigest() != record.get("sha256"): + raise ValueError("Object cache file checksum changed") + records[relative] = record + documents[filename] = doc + pins[filename] = hashlib.sha256(raw).hexdigest() + if expected is not None and pins != expected: + raise ValueError("Object cache receipt changed after CPU preflight") + primary, auxiliary = (documents[name] for name in CACHE_RECEIPTS) + models = {} + for model in primary.get("snapshots", []): + repo, revision = model.get("repoId"), model.get("revision") + if ( + repo not in (MODEL_ID, REFINER_ID) + or repo in models + or not isinstance(revision, str) + or not re.fullmatch(r"[0-9a-f]{40}", revision) + ): + raise ValueError("Invalid retained model revision") + relative = f"huggingface/hub/models--{repo.replace('/', '--')}/snapshots/{revision}" + filename = "pipeline.json" if repo == MODEL_ID else "model_index.json" + if f"{relative}/{filename}" not in records: + raise ValueError("Pinned model configuration is missing") + models[repo] = {"path": str(root / relative), "revision": revision} + if MODEL_ID not in models or (need_refiner and REFINER_ID not in models): + raise ValueError("Required object model is not staged") + code = auxiliary.get("dinoCodePath") + revision = auxiliary.get("dinoRevision") + if ( + not isinstance(revision, str) + or not re.fullmatch(r"[0-9a-f]{40}", revision) + or code != f"torch/hub/dinov2-{revision}" + or f"{code}/hubconf.py" not in records + or DINO_CHECKPOINT not in records + ): + raise ValueError("Pinned DINO code/checkpoint is missing") + if need_rembg and "u2net/u2net.onnx" not in records: + raise ValueError("Required background-removal model is not staged") + return { + "receipts": pins, + "models": models, + "dinoCodePath": str(root / code), + "dinoWeightsPath": str(root / DINO_CHECKPOINT), + "u2netPath": str(root / "u2net/u2net.onnx"), + "dinoRevision": revision, + } + + +# This preflight function uses only a CPU image and does not import or load models. +@app.function( + image=modal.Image.debian_slim(python_version="3.12"), + cpu=(1, 2), + memory=(1024, 4096), + timeout=900, + retries=0, + max_containers=1, + volumes={"/cache": cache}, + scaledown_window=2, +) +def cache_preflight(need_refiner: bool = False, need_rembg: bool = True): + cache.reload() + return checked_object_cache(need_refiner=need_refiner, need_rembg=need_rembg) + + +def offline_object_environment(): + import os + + os.environ["HF_HUB_OFFLINE"] = "1" + os.environ["TRANSFORMERS_OFFLINE"] = "1" + os.environ["HF_DATASETS_OFFLINE"] = "1" + + +def local_trellis_load(pipeline_class, retained): + """Route upstream torch.hub and rembg downloads to the verified retained files.""" + from unittest.mock import patch + import torch + import huggingface_hub + + original_load = torch.hub.load + + def load_hub(repo, name, *args, **kwargs): + if ( + repo != "facebookresearch/dinov2" + or name != "dinov2_vitl14_reg" + or args + or kwargs != {"pretrained": True} + ): + raise ValueError("Unexpected TRELLIS conditioning model request") + return original_load(retained["dinoCodePath"], name, source="local", pretrained=True) + + def weights(url, **kwargs): + if ( + url + != "https://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_reg4_pretrain.pth" + ): + raise ValueError("Unexpected DINO checkpoint request") + return torch.load( + retained["dinoWeightsPath"], + map_location=kwargs.get("map_location", "cpu"), + weights_only=True, + ) + + def forbidden_download(*args, **kwargs): + raise ValueError("TRELLIS attempted an unstaged Hugging Face checkpoint") + + with ( + patch.object(torch.hub, "load", load_hub), + patch.object(torch.hub, "load_state_dict_from_url", weights), + patch.object(huggingface_hub, "hf_hub_download", forbidden_download), + ): + return pipeline_class.from_pretrained(retained["models"][MODEL_ID]["path"]) + + +def local_preprocess(pipeline, image, retained): + from unittest.mock import patch + from rembg.sessions.u2net import U2netSession + + with patch.object( + U2netSession, + "download_models", + classmethod(lambda cls, *args, **kwargs: retained["u2netPath"]), + ): + return pipeline.preprocess_image(image) + + @app.function( image=image, cpu=4, memory=16384, timeout=900, volumes={"/cache": cache}, scaledown_window=2 ) @@ -204,6 +378,7 @@ def refine_crop( text_only: bool = False, candidates: int = 1, pick: int = 0, + cache_receipts: dict | None = None, ) -> dict: """SDXL on the crop, so the 3D stage gets something with edges in it. @@ -213,6 +388,11 @@ def refine_crop( whole object -- it exists because a crop that is only a label seen end-on has no bottle in it to recover, and meta records which of the two actually ran. """ + offline_object_environment() + cache.reload() + retained = checked_object_cache( + need_refiner=True, expected=cache_receipts, hash_files=cache_receipts is None + ) import hashlib, io, time import torch from PIL import Image @@ -230,14 +410,22 @@ def refine_crop( from diffusers import StableDiffusionXLPipeline pipe = StableDiffusionXLPipeline.from_pretrained( - REFINER_ID, torch_dtype=torch.float16, variant="fp16", use_safetensors=True + retained["models"][REFINER_ID]["path"], + torch_dtype=torch.float16, + variant="fp16", + use_safetensors=True, + local_files_only=True, ) common.update(height=size, width=size) else: from diffusers import StableDiffusionXLImg2ImgPipeline pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained( - REFINER_ID, torch_dtype=torch.float16, variant="fp16", use_safetensors=True + retained["models"][REFINER_ID]["path"], + torch_dtype=torch.float16, + variant="fp16", + use_safetensors=True, + local_files_only=True, ) flat = Image.new("RGB", src.size, (255, 255, 255)) flat.paste(src, (0, 0), src if src.mode == "RGBA" else None) @@ -271,12 +459,7 @@ def refine_crop( buf = io.BytesIO() out.save(buf, format="PNG") - try: - from huggingface_hub import HfApi - - revision = HfApi().model_info(REFINER_ID).sha - except Exception: - revision = None + revision = retained["models"][REFINER_ID]["revision"] meta = { "mode": "text-to-image (NO image conditioning)" if text_only @@ -284,6 +467,7 @@ def refine_crop( "imageConditioned": not text_only, "model": REFINER_ID, "modelRevision": revision, + "cacheReceipts": retained["receipts"], "licence": "CreativeML Open RAIL++-M (open weights, use-based restrictions; not OSI)", "prompt": prompt, "negativePrompt": negative_prompt, @@ -335,7 +519,13 @@ def image_to_3d( contact_px: int = 256, refiner: dict | None = None, note: str = "", + cache_receipts: dict | None = None, ) -> dict: + offline_object_environment() + cache.reload() + retained = checked_object_cache( + need_rembg=remove_bg, expected=cache_receipts, hash_files=cache_receipts is None + ) import hashlib, io, json, os, time, traceback import numpy as np from PIL import Image @@ -348,12 +538,12 @@ def image_to_3d( src = Image.open(io.BytesIO(png_bytes)) sha = hashlib.sha256(png_bytes).hexdigest() - pipeline = TrellisImageTo3DPipeline.from_pretrained(MODEL_ID) + pipeline = local_trellis_load(TrellisImageTo3DPipeline, retained) pipeline.cuda() cache.commit() if remove_bg: - cond = pipeline.preprocess_image(src) + cond = local_preprocess(pipeline, src, retained) else: # TRELLIS's own preprocess_image, minus the rembg call: alpha-crop (or whole-frame when the # crop is opaque), 1.2x margin, 518px, premultiply. Keep in step with upstream if it moves. @@ -439,16 +629,12 @@ def image_to_3d( preview_error = traceback.format_exc() print(preview_error, flush=True) - try: - from huggingface_hub import HfApi - - model_revision = HfApi().model_info(MODEL_ID).sha - except Exception: - model_revision = None - + model_revision = retained["models"][MODEL_ID]["revision"] meta = { "model": MODEL_ID, "modelRevision": model_revision, + "dinoCodeRevision": retained["dinoRevision"], + "cacheReceipts": retained["receipts"], "codeRepo": "https://github.com/microsoft/TRELLIS", "codeRevision": TRELLIS_REV, "licences": { @@ -581,6 +767,10 @@ def main( dest = Path(out_dir) dest.mkdir(parents=True, exist_ok=True) + retained = cache_preflight.remote( + need_refiner=refine_first or refine_text_only, + need_rembg=remove_bg or refine_first or refine_text_only, + ) refiner = None if refine_first or refine_text_only: r = refine_crop.remote( @@ -594,6 +784,7 @@ def main( text_only=refine_text_only, candidates=refine_candidates, pick=refine_pick, + cache_receipts=retained["receipts"], ) png, refiner = r["png"], r["meta"] (dest / "refined.png").write_bytes(png) @@ -621,6 +812,7 @@ def main( preview_video=preview_video, refiner=refiner, note=note, + cache_receipts=retained["receipts"], ) (dest / "object.ply").write_bytes(res["ply"]) if res["glb"]: diff --git a/worker/modal_stage_object_aux.py b/worker/modal_stage_object_aux.py new file mode 100644 index 0000000..322ad03 --- /dev/null +++ b/worker/modal_stage_object_aux.py @@ -0,0 +1,245 @@ +"""Pinned CPU-only DINO source/U2Net setup, separate from primary model staging.""" + +from pathlib import Path +import hashlib +import json +import re +import tarfile +import time +import urllib.request + +import modal + +try: + from worker.modal_stage_object_weights import ( + deadline_guard, + digest_file, + file_record, + manifest_hash, + validate_deadline, + write_receipt, + ) +except ModuleNotFoundError: + from modal_stage_object_weights import ( + deadline_guard, + digest_file, + file_record, + manifest_hash, + validate_deadline, + write_receipt, + ) + +SCHEMA = "wander.object-aux-manifest/1" +U2NET_URL = "https://github.com/danielgatis/rembg/releases/download/v0.0.0/u2net.onnx" +U2NET_MD5 = "60024c5c889badc19c04ad937298a77b" # rembg v2.0.63 sessions/u2net.py +# Modal mounts this entrypoint alone; make its CPU-only helper an explicit image input. +image = modal.Image.debian_slim(python_version="3.12").add_local_file( + Path(__file__).with_name("modal_stage_object_weights.py"), + "/root/modal_stage_object_weights.py", + copy=True, +) +app = modal.App("wander-object-aux-cache-setup") +cache = modal.Volume.from_name("wander-image-to-3d-cache") + + +def validate_manifest(doc): + if ( + not isinstance(doc, dict) + or set(doc) != {"schema", "dinoCode", "u2net"} + or doc["schema"] != SCHEMA + ): + raise ValueError("Invalid object auxiliary manifest") + code, model = doc["dinoCode"], doc["u2net"] + if ( + not isinstance(code, dict) + or set(code) != {"revision", "archiveSha256", "bytes"} + or not re.fullmatch(r"[0-9a-f]{40}", str(code["revision"])) + or not re.fullmatch(r"[0-9a-f]{64}", str(code["archiveSha256"])) + or type(code["bytes"]) is not int + or not 0 < code["bytes"] <= 20 * 1024**2 + ): + raise ValueError("DINO source requires a pinned commit, archive SHA256 and size") + if ( + not isinstance(model, dict) + or set(model) != {"url", "md5", "bytes"} + or model["url"] != U2NET_URL + or model["md5"] != U2NET_MD5 + or type(model["bytes"]) is not int + or not 0 < model["bytes"] <= 512 * 1024**2 + ): + raise ValueError("U2Net requires the official rembg URL/checksum and bounded size") + return doc + + +def download(url, path, expected_bytes, algorithm, expected_hash): + path.parent.mkdir(parents=True, exist_ok=True) + digest = hashlib.new(algorithm) + total = 0 + with path.open("xb") as output, urllib.request.urlopen(url, timeout=30) as response: + while chunk := response.read(1024 * 1024): + total += len(chunk) + if total > expected_bytes: + raise ValueError("Auxiliary download exceeds pinned size") + digest.update(chunk) + output.write(chunk) + if total != expected_bytes or digest.hexdigest() != expected_hash: + raise ValueError("Auxiliary download checksum/size mismatch") + + +def extract_code(archive_path, destination, revision): + destination.mkdir(parents=True, exist_ok=False) + prefix = "dinov2-" + revision + total = count = 0 + with tarfile.open(archive_path, "r:gz") as archive: + for member in archive: + parts = Path(member.name).parts + if ( + not parts + or parts[0] != prefix + or ".." in parts + or member.issym() + or member.islnk() + or not (member.isfile() or member.isdir()) + ): + raise ValueError("Unexpected DINO archive member") + total += member.size + count += 1 + if total > 100 * 1024**2 or count > 2000: + raise ValueError("DINO source archive exceeds extraction limit") + target = destination.joinpath(*parts[1:]) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + else: + target.parent.mkdir(parents=True, exist_ok=True) + with archive.extractfile(member) as source, target.open("xb") as output: + while chunk := source.read(1024 * 1024): + output.write(chunk) + if not (destination / "hubconf.py").is_file(): + raise ValueError("DINO source archive has no hub entrypoint") + + +def stage_aux(manifest, deadline_epoch, *, root, commit, call_id=None): + validate_manifest(manifest) + validate_deadline(deadline_epoch) + root = Path(root).resolve() + receipt = root / "setup-receipts/object-aux-attempt.json" + if receipt.exists(): + old = json.loads(receipt.read_text()) + if old.get("status") != "completed" or old.get("manifestSha256") != manifest_hash(manifest): + raise RuntimeError("Auxiliary setup requires reconciliation; no automatic repeat") + for record in old["files"]: + if file_record(root, root / record["path"]) != record: + raise ValueError("Retained auxiliary cache changed") + return {key: value for key, value in old.items() if key != "files"} + revision = manifest["dinoCode"]["revision"] + code = root / "torch/hub" / ("dinov2-" + revision) + report = { + "schema": "wander.object-aux-attempt/1", + "status": "started", + "manifestSha256": manifest_hash(manifest), + "dinoRevision": revision, + "dinoCodePath": str(code.relative_to(root)), + "startedEpoch": time.time(), + "deadlineEpoch": deadline_epoch, + "functionCallId": call_id, + "files": [], + } + write_receipt(receipt, report) + commit() + try: + with deadline_guard(deadline_epoch): + if code.exists() or code.is_symlink(): + raise ValueError("DINO source destination already exists; reconcile retained files") + archive_path = root / "setup-receipts/dinov2-source.tar.gz" + pin = manifest["dinoCode"] + download( + "https://codeload.github.com/facebookresearch/dinov2/tar.gz/" + revision, + archive_path, + pin["bytes"], + "sha256", + pin["archiveSha256"], + ) + extract_code(archive_path, code, revision) + model = root / "u2net/u2net.onnx" + pin = manifest["u2net"] + if model.exists() or model.is_symlink(): + raise ValueError("U2Net destination already exists; reconcile retained files") + pending = model.with_suffix(".download") + download(pin["url"], pending, pin["bytes"], "md5", pin["md5"]) + pending.replace(model) + report["files"] = [ + file_record(root, path) for path in sorted(code.rglob("*")) if path.is_file() + ] + report["files"].append(file_record(root, model)) + report.update( + status="completed", + finishedEpoch=time.time(), + fileCount=len(report["files"]), + inventorySha256=manifest_hash(report["files"]), + u2netSha256=digest_file(model), + ) + write_receipt(receipt, report) + commit() + except BaseException as error: + report.update(status="failed", finishedEpoch=time.time(), errorType=type(error).__name__) + write_receipt(receipt, report) + commit() + raise RuntimeError( + f"Auxiliary setup failed ({type(error).__name__}); reconcile receipt" + ) from None + return {key: value for key, value in report.items() if key != "files"} + + +@app.function( + image=image, + cpu=(1, 2), + memory=(1024, 4096), + timeout=900, + retries=0, + max_containers=1, + scaledown_window=2, + volumes={"/cache": cache}, +) +def stage(manifest: dict, deadline_epoch: float): + cache.reload() + return stage_aux( + manifest, + deadline_epoch, + root="/cache", + commit=cache.commit, + call_id=modal.current_function_call_id(), + ) + + +@app.local_entrypoint() +def main(manifest: str, receipt: str, deadline_epoch: float): + import fcntl + + manifest, receipt = Path(manifest), Path(receipt) + if not manifest.is_absolute() or not receipt.is_absolute(): + raise ValueError("Manifest and receipt must be absolute paths") + doc = validate_manifest(json.loads(manifest.read_text())) + validate_deadline(deadline_epoch) + receipt.parent.mkdir(parents=True, exist_ok=True) + with Path(str(receipt) + ".lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if receipt.exists() or receipt.is_symlink(): + raise RuntimeError("Existing submission receipt requires reconciliation") + record = { + "status": "started", + "manifestSha256": manifest_hash(doc), + "deadlineEpoch": deadline_epoch, + "startedEpoch": time.time(), + } + write_receipt(receipt, record) + try: + result = stage.remote(doc, deadline_epoch) + record.update(status="completed", result=result) + except BaseException as error: + record.update(status="failed", errorType=type(error).__name__) + write_receipt(receipt, record) + raise RuntimeError( + "Auxiliary submission failed or ambiguous; reconcile receipt" + ) from None + write_receipt(receipt, record) + print(json.dumps(result, indent=2)) From 292680e57dcdcc4532d12e866628e41987a1f5ae Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sun, 18 Oct 2026 20:39:16 -0400 Subject: [PATCH 07/14] fix: clarify unrecorded artifact links in run details --- admin/app/runs/[runId]/console.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/app/runs/[runId]/console.tsx b/admin/app/runs/[runId]/console.tsx index 75e4b94..cfbc38e 100644 --- a/admin/app/runs/[runId]/console.tsx +++ b/admin/app/runs/[runId]/console.tsx @@ -846,7 +846,7 @@ export default function RunConsole({ runId }: { runId: string }) { ? selected.freshInference === false ? 'Saved inputs were reused without creating a new attempt.' : 'This stage has not run yet, so there is nothing to look at.' - : 'This attempt produced no files.' + : 'No artifact links recorded for this attempt.' } /> From e069641626fa18f71b40a72db114d2348667ae44 Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sun, 18 Oct 2026 20:49:15 -0400 Subject: [PATCH 08/14] fix: support fresh person coordinates in object packaging --- docs/objects.md | 27 +- scripts/lift_object_3d.py | 27 +- scripts/object_coordinate_basis.py | 161 ++++++++++ scripts/package_objects.py | 30 +- scripts/tests/test_object_coordinate_basis.py | 277 ++++++++++++++++++ 5 files changed, 483 insertions(+), 39 deletions(-) create mode 100644 scripts/object_coordinate_basis.py create mode 100644 scripts/tests/test_object_coordinate_basis.py diff --git a/docs/objects.md b/docs/objects.md index f79ae39..d84c34d 100644 --- a/docs/objects.md +++ b/docs/objects.md @@ -32,6 +32,11 @@ The writer emits `schema: "wander.objects/2"`, plus `clip`, `fps`, `samples`, `s `timestamps`, `metresPerWorldUnit`, `coordinates`, `people: "people.json"`, and `objects`. The top-level sample grid comes from `people.json`; it differs from the baked object's frame grid. +New fits and packages also carry `coordinateBasis`, repeated in object `evidence`. It identifies +the scale source and fingerprints the person registration scales, vertical constants, and shared +drift on the source-frame grid. Packaging rejects a fit made against a different basis; re-lift +after changing those corrections. + | Object field | Meaning and current behavior | |---|---| | `id`, `label`, `prompt` | Stable object identifier, display name, and segmentation prompt. | @@ -122,7 +127,7 @@ Explicit `--object` requests still enable the stage. Without a named request, it 1. Runs `detect_object_flights.py` using camera poses, tracks, masks, and a metre scale. 2. Stops with an empty result if no flights pass. Single-person worlds can record detections, - but cannot package them through this lane without multiperson floor fits and wrist poses. + but this lane needs a `people.json` package and per-track wrist poses to package them. 3. Runs `lift_object_3d.py`, retaining flights that connect tracked wrists at both ends. 4. Runs `describe_object.py` on observed crops; without `OPENAI_API_KEY` it uses `--no-vlm`. 5. Optionally calls `worker/modal_image_to_3d.py`, then `object_appearance.py`. `--no-shape` @@ -145,10 +150,28 @@ uv run --locked --group inference scripts/package_objects.py \ --size-m 0.06,0.12,0.06 --shape-provenance 'Proxy; no reconstructed model' ``` -This requires `people.json` with `floorFit`, per-track `source-poses.pt`, and a reviewed +This requires `people.json`, per-track `source-poses.pt`, and a reviewed `wander.object-fit/2` fit. A `/1` fit additionally needs `--thrower` and `--catcher`. Optional `--model`, `--orientation`, and provenance flags add reviewed appearance/rotation. +Lifting and packaging share two supported coordinate bases. A fresh `package_multiperson.py` +manifest has identity person transforms and no `floorFit`: its scale is +`1.70 / sharedPlacement.bodyHeightUnits`, matching the detector's **avatar stature prior**. +Shared drift and person vertical constants are zero; no floor measurement or metric anchor is +invented. Its objects set `appliesSharedCameraDrift: false`. Sizes, gravity and wrist distances +expressed in metres remain conditional on that assumed stature. +The historical fit flag `driftCorrected: true` and segment space `driftCorrected` describe the +coordinate convention after applying the chosen basis corrections. For this fresh basis those +corrections are zero; these fields do not claim that camera drift was measured or removed. + +Legacy `floorFit` packages retain their metre scale, interpolated `sharedCameraDrift`, and +`C_` level `translationY` constants. A present but incomplete or nonfinite floor fit fails instead +of falling back to the stature prior. Person rotation, scale, and horizontal translation overrides +are unsupported; fresh packages also reject vertical translation overrides. Legacy vertical +translation is replaced by the existing level C constants. Fits without `coordinateBasis` remain +accepted only with legacy floor-fit packages; fresh packages require a newly lifted, tagged fit. +Reprojection, observation-count, wrist-contact, and source-clock gates apply unchanged in both bases. + The named `--object ID:MOTION:PROMPT[:ROI]` lane is manual/experimental: it segments an ROI, uses fixed thrower/catcher indices for flight fitting, and reuses a supplied model when present. Attached/worldDynamic requests currently omit the packager's required `--fit`; repeated named diff --git a/scripts/lift_object_3d.py b/scripts/lift_object_3d.py index c29b673..e33c977 100644 --- a/scripts/lift_object_3d.py +++ b/scripts/lift_object_3d.py @@ -27,6 +27,7 @@ from scipy.optimize import least_squares from scipy.spatial.transform import Rotation, Slerp from source_clock import camera_source_fps, resolve_source_fps +from object_coordinate_basis import resolve_object_basis def load_cameras(path): @@ -276,26 +277,13 @@ def main(): raise SystemExit(str(exc)) from exc with open(args.people) as handle: pj = json.load(handle) - mpu = pj["floorFit"]["metresPerWorldUnit"] + try: + basis = resolve_object_basis(pj) + except ValueError as exc: + raise SystemExit(str(exc)) from exc + mpu = basis.metres_per_world_unit g_units = 9.80665 / mpu - drift = pj["floorFit"]["sharedCameraDrift"] - doff = np.array(drift["offsetYUnits"], float) - src_of_sample = np.array(pj["sourceIndices"], float) - dsamp = src_of_sample[np.array(drift["samples"], int)] - lvC = pj["floorFit"]["levels"][[k for k in pj["floorFit"]["levels"] if k.startswith("C_")][0]] - ckey = [k for k in lvC if "translation" in k.lower()][0] - constC = lvC[ckey] - - by_track = {p["track"]: p for p in pj["people"]} - scale = { - t: by_track[t]["registrationScale"] - * by_track[t].get("sizeCorrection", {}).get("multiplier", 1.0) - for t in by_track - } - cconst = {t: constC[by_track[t]["id"]] for t in by_track} - - def drift_at(sf): - return float(np.interp(sf, dsamp, doff)) + scale, cconst, drift_at = basis.scale, basis.constants, basis.drift_at # ---- assemble the list of flights to fit spans = [] @@ -396,6 +384,7 @@ def nearest_track(sf, p): schema="wander.object-fit/2", clip=clip, metresPerWorldUnit=mpu, + coordinateBasis=basis.provenance, gravityUnitsPerS2=g_units, fps=fps, driftCorrected=True, diff --git a/scripts/object_coordinate_basis.py b/scripts/object_coordinate_basis.py new file mode 100644 index 0000000..4312fc1 --- /dev/null +++ b/scripts/object_coordinate_basis.py @@ -0,0 +1,161 @@ +"""Coordinate corrections shared by object lifting and packaging. + +Fresh people packages have no floor measurement. Their scale is the same 1.70 m +avatar stature prior used by the detector, with no invented floor or drift. +""" + +import hashlib +import json +from dataclasses import dataclass + +import numpy as np + + +def finite_number(value, name, positive=False): + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not np.isfinite(value) + or (positive and value <= 0) + ): + raise ValueError(f"{name} must be a finite {'positive ' if positive else ''}number") + return float(value) + + +def vector(value, name, length=None): + if not isinstance(value, list) or not value or (length and len(value) != length): + raise ValueError(f"{name} must be a nonempty numeric array") + return np.array([finite_number(v, name) for v in value]) + + +@dataclass +class ObjectCoordinateBasis: + metres_per_world_unit: float + scale: dict + constants: dict + drift_frames: np.ndarray + drift_offsets: np.ndarray + provenance: dict + + def drift_at(self, frame): + return float(np.interp(frame, self.drift_frames, self.drift_offsets)) + + def validate_fit(self, fit): + recorded = fit.get("coordinateBasis") + if recorded is None: + # Older fits were exclusively made against floorFit. Never reinterpret + # one in the new uncorrected frame merely because its scale is similar. + if self.provenance["mode"] != "floorFit": + raise ValueError("fit has no coordinateBasis; re-lift for the avatar stature prior") + elif recorded != self.provenance: + raise ValueError("fit coordinateBasis conflicts with people.json; re-lift the object") + if "metresPerWorldUnit" in fit: + mpu = finite_number(fit["metresPerWorldUnit"], "fit.metresPerWorldUnit", True) + if not np.isclose(mpu, self.metres_per_world_unit, rtol=1e-9, atol=0): + raise ValueError("fit metresPerWorldUnit conflicts with people.json") + + +def resolve_object_basis(people): + """Accept complete legacy floor corrections or fresh identity placement only.""" + try: + return _resolve(people) + except (KeyError, TypeError, AttributeError) as exc: + raise ValueError(f"incomplete object coordinate basis in people.json: {exc}") from exc + + +def _resolve(people): + source = vector(people["sourceIndices"], "sourceIndices") + if np.any(source < 0) or np.any(source != np.floor(source)) or np.any(np.diff(source) <= 0): + raise ValueError("sourceIndices must be increasing nonnegative integers") + rows = people["people"] + if not isinstance(rows, list) or not rows: + raise ValueError("people must be a nonempty array") + legacy = "floorFit" in people # Present-but-invalid is never a prior fallback. + if legacy: + floor = people["floorFit"] + mpu = finite_number(floor["metresPerWorldUnit"], "floorFit.metresPerWorldUnit", True) + drift = floor["sharedCameraDrift"] + samples = vector(drift["samples"], "floorFit.sharedCameraDrift.samples") + offsets = vector(drift["offsetYUnits"], "floorFit.sharedCameraDrift.offsetYUnits") + if ( + len(samples) != len(offsets) + or np.any(samples != np.floor(samples)) + or np.any(samples < 0) + or np.any(samples >= len(source)) + or np.any(np.diff(samples) <= 0) + ): + raise ValueError( + "floorFit drift needs matching offsets and increasing valid sample indices" + ) + drift_frames = source[samples.astype(int)] + levels = [v for k, v in floor["levels"].items() if k.startswith("C_")] + if len(levels) != 1: + raise ValueError("floorFit needs exactly one C_ correction level") + translations = levels[0]["translationY"] + provenance = dict(mode="floorFit", scaleSource="floorFit.metresPerWorldUnit") + else: + height = finite_number( + people["sharedPlacement"]["bodyHeightUnits"], "sharedPlacement.bodyHeightUnits", True + ) + mpu = finite_number(1.70 / height, "avatar prior metresPerWorldUnit", True) + drift_frames, offsets = source, np.zeros(len(source)) + translations = {p["id"]: 0.0 for p in rows} + provenance = dict( + mode="avatarStaturePrior", + scaleSource="1.70 m avatar stature prior / sharedPlacement.bodyHeightUnits", + staturePriorMetres=1.70, + bodyHeightUnits=height, + caveat="Assumed avatar stature; no measured metric anchor, floor fit or camera drift.", + ) + finite_number(9.80665 / mpu, "gravityUnitsPerS2", True) + scale, constants, ids = {}, {}, set() + for person in rows: + track, pid = person["track"], person["id"] + if isinstance(track, bool) or not isinstance(track, int) or track < 0 or track in scale: + raise ValueError("person tracks must be unique nonnegative integers") + if not isinstance(pid, str) or not pid or pid in ids: + raise ValueError("person ids must be unique nonempty strings") + ids.add(pid) + scale[track] = finite_number(person["registrationScale"], "registrationScale", True) + scale[track] *= finite_number( + person.get("sizeCorrection", {}).get("multiplier", 1.0), + "sizeCorrection.multiplier", + True, + ) + finite_number(scale[track], "combined registration scale", True) + constants[track] = finite_number(translations[pid], f"floorFit translationY[{pid}]") + transform = person.get("transform", {}) if legacy else person["transform"] + translation = vector(transform.get("translation", [0, 0, 0]), "transform.translation", 3) + quaternion = vector( + transform.get("quaternionXYZW", [0, 0, 0, 1]), "transform.quaternionXYZW", 4 + ) + transform_scale = finite_number(transform.get("scale", 1), "transform.scale", True) + # Legacy level C replaces the old per-person vertical translation in the + # viewer. Other rotations/scales/translations have never been supported. + if ( + np.any(translation[[0, 2]] != 0) + or (not legacy and translation[1] != 0) + or np.any(quaternion[:3] != 0) + or abs(quaternion[3]) != 1 + or transform_scale != 1 + ): + raise ValueError(f"unsupported nonidentity transform for person {pid}") + material = dict( + sourceIndices=source.tolist(), + people=[ + dict( + track=p["track"], + id=p["id"], + scale=scale[p["track"]], + constantY=constants[p["track"]], + ) + for p in rows + ], + driftSourceFrames=drift_frames.tolist(), + driftOffsetYUnits=offsets.tolist(), + ) + provenance.update( + metresPerWorldUnit=mpu, + correctionsSha256=hashlib.sha256(json.dumps(material, sort_keys=True).encode()).hexdigest(), + ) + return ObjectCoordinateBasis(mpu, scale, constants, drift_frames, offsets, provenance) diff --git a/scripts/package_objects.py b/scripts/package_objects.py index ae9702a..3c18634 100644 --- a/scripts/package_objects.py +++ b/scripts/package_objects.py @@ -31,6 +31,7 @@ import torch from scipy.spatial.transform import Rotation, Slerp from source_clock import camera_source_fps, resolve_source_fps +from object_coordinate_basis import resolve_object_basis CV = np.diag([1.0, -1.0, -1.0]) @@ -342,25 +343,16 @@ def main(): pj = json.load(handle) with open(a.fit) as handle: fit = json.load(handle) - ff = pj["floorFit"] - mpu = ff["metresPerWorldUnit"] + try: + basis = resolve_object_basis(pj) + basis.validate_fit(fit) + except ValueError as exc: + raise SystemExit(str(exc)) from exc + mpu = basis.metres_per_world_unit C = load_cameras(a.cameras) - src = np.array(pj["sourceIndices"], float) - drift = ff["sharedCameraDrift"] - doff = np.array(drift["offsetYUnits"], float) - dsrc = src[np.array(drift["samples"], int)] - lvC = ff["levels"][[k for k in ff["levels"] if k.startswith("C_")][0]]["translationY"] by_track = {p["track"]: p for p in pj["people"]} - scale = { - t: by_track[t]["registrationScale"] - * by_track[t].get("sizeCorrection", {}).get("multiplier", 1.0) - for t in by_track - } - cconst = {t: lvC[by_track[t]["id"]] for t in by_track} - - def drift_at(f): - return float(np.interp(f, dsrc, doff)) + scale, cconst, drift_at = basis.scale, basis.constants, basis.drift_at # ---- normalise /1 and /2 fits to one list of flights if fit.get("schema") == "wander.object-fit/2": @@ -636,8 +628,9 @@ def pos_corrected(f): segments=segments, ), transform=dict(translation=[0.0, obj_const, 0.0], quaternionXYZW=[0, 0, 0, 1], scale=1.0), - appliesSharedCameraDrift=True, + appliesSharedCameraDrift=basis.provenance["mode"] == "floorFit", evidence=dict( + coordinateBasis=basis.provenance, freeSpans=len(flights), tracked2dFrames=int(sum(len(fl.get("frames", [])) for fl in flights)), totalSourceFrames=int(nsrc), @@ -701,10 +694,11 @@ def pos_corrected(f): sourceIndices=pj["sourceIndices"], timestamps=pj["timestamps"], metresPerWorldUnit=mpu, + coordinateBasis=basis.provenance, coordinates=( "Raw SfM world, OpenGL, camera 0 = identity - the same frame the person " "PLYs are written in. Apply the shared scale, then transform.translation, " - "then floorFit.sharedCameraDrift, exactly as for a person." + "then floorFit.sharedCameraDrift when present, exactly as for a person." ), people="people.json", objects=[obj], diff --git a/scripts/tests/test_object_coordinate_basis.py b/scripts/tests/test_object_coordinate_basis.py new file mode 100644 index 0000000..dfe18f7 --- /dev/null +++ b/scripts/tests/test_object_coordinate_basis.py @@ -0,0 +1,277 @@ +"""Fresh package_multiperson coordinates and legacy floor-fit compatibility.""" + +import contextlib +import copy +import io +import json +import sys +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from object_coordinate_basis import resolve_object_basis + + +def fresh_people(): + # Same coordinate fields emitted by package_multiperson, including its + # camera-centred size correction and rounded 30 -> 12 fps sample grid. + return { + "schema": "wander.people/1", + "fps": 12, + "samples": 6, + "sourceIndices": [0, 2, 5, 8, 10, 12], + "timestamps": [f / 30 for f in [0, 2, 5, 8, 10, 12]], + "sharedPlacement": {"bodyHeightUnits": 0.85, "feetY": -0.76}, + "people": [ + { + "id": f"person{i}", + "track": i, + "registrationScale": 0.8, + "sizeCorrection": {"multiplier": 0.625, "appliedAboutSourceCamera": True}, + "transform": { + "translation": [0, 0, 0], + "quaternionXYZW": [0, 0, 0, 1], + "scale": 1, + }, + } + for i in range(2) + ], + } + + +def legacy_people(): + people = fresh_people() + people["floorFit"] = { + "metresPerWorldUnit": 1.8, + "sharedCameraDrift": {"samples": [0, 2, 5], "offsetYUnits": [0.1, 0.2, -0.1]}, + "levels": { + "C_sharedDriftPlusConstant": {"translationY": {"person0": -0.03, "person1": 0.04}} + }, + } + people["people"][0]["transform"]["translation"][1] = -0.08 # old level B + return people + + +def test_fresh_prior_has_zero_corrections_and_camera_centred_scales(): + basis = resolve_object_basis(fresh_people()) + assert basis.metres_per_world_unit == 2 + assert basis.scale == {0: 0.5, 1: 0.5} + assert basis.constants == {0: 0, 1: 0} + assert [basis.drift_at(f) for f in (-1, 0, 4, 12, 20)] == [0] * 5 + assert basis.provenance["mode"] == "avatarStaturePrior" + assert basis.provenance["staturePriorMetres"] == 1.70 + assert "no measured metric anchor" in basis.provenance["caveat"] + + +def test_legacy_corrections_remain_identical(): + basis = resolve_object_basis(legacy_people()) + assert basis.metres_per_world_unit == 1.8 + assert basis.constants == {0: -0.03, 1: 0.04} + np.testing.assert_allclose( + [basis.drift_at(f) for f in (0, 2.5, 5, 8.5, 12)], [0.1, 0.15, 0.2, 0.05, -0.1] + ) + basis.validate_fit({}) # Pre-provenance legacy fit remains supported. + + +@pytest.mark.parametrize("floor", [None, {}, {"metresPerWorldUnit": 2}]) +def test_present_incomplete_floor_fit_never_falls_back(floor): + people = fresh_people() + people["floorFit"] = floor + with pytest.raises(ValueError, match="incomplete"): + resolve_object_basis(people) + + +@pytest.mark.parametrize("value", [0, -1, float("nan"), float("inf"), True, "2"]) +def test_invalid_scales_fail(value): + for factory, path in [ + (fresh_people, ("sharedPlacement", "bodyHeightUnits")), + (legacy_people, ("floorFit", "metresPerWorldUnit")), + ]: + people = factory() + people[path[0]][path[1]] = value + with pytest.raises(ValueError, match="finite positive"): + resolve_object_basis(people) + + +@pytest.mark.parametrize( + "drift", + [ + {"samples": [], "offsetYUnits": []}, + {"samples": [0, 1], "offsetYUnits": [0]}, + {"samples": [0, 6], "offsetYUnits": [0, 1]}, + {"samples": [1, 0], "offsetYUnits": [0, 1]}, + {"samples": [0, 0], "offsetYUnits": [0, 1]}, + {"samples": [0, 0.5], "offsetYUnits": [0, 1]}, + {"samples": [0, 1], "offsetYUnits": [0, float("nan")]}, + ], +) +def test_invalid_legacy_drift_fails(drift): + people = legacy_people() + people["floorFit"]["sharedCameraDrift"] = drift + with pytest.raises(ValueError, match="floorFit"): + resolve_object_basis(people) + + +@pytest.mark.parametrize("translations", [{"person0": 0}, {"person0": 0, "person1": float("inf")}]) +def test_invalid_legacy_constants_fail(translations): + people = legacy_people() + people["floorFit"]["levels"]["C_sharedDriftPlusConstant"]["translationY"] = translations + with pytest.raises(ValueError): + resolve_object_basis(people) + + +@pytest.mark.parametrize( + "key,value", + [ + ("translation", [0, 1, 0]), + ("translation", [1, 0, 0]), + ("quaternionXYZW", [1, 0, 0, 0]), + ("scale", 2), + ], +) +def test_fresh_nonidentity_transforms_fail(key, value): + people = fresh_people() + people["people"][0]["transform"][key] = value + with pytest.raises(ValueError, match="unsupported nonidentity"): + resolve_object_basis(people) + + +def test_fit_basis_cannot_cross_modes_or_change_corrections(): + people = fresh_people() + basis = resolve_object_basis(people) + fit = {"coordinateBasis": basis.provenance, "metresPerWorldUnit": basis.metres_per_world_unit} + basis.validate_fit(fit) + with pytest.raises(ValueError, match="no coordinateBasis"): + basis.validate_fit({}) + with pytest.raises(ValueError, match="conflicts"): + resolve_object_basis(legacy_people()).validate_fit(fit) + for field in ("registrationScale", "id"): + altered = copy.deepcopy(people) + altered["people"][0][field] = 0.9 if field == "registrationScale" else "changed" + with pytest.raises(ValueError, match="conflicts"): + resolve_object_basis(altered).validate_fit(fit) + legacy = legacy_people() + fit["coordinateBasis"] = resolve_object_basis(legacy).provenance + legacy["floorFit"]["sharedCameraDrift"]["offsetYUnits"][0] += 0.01 + with pytest.raises(ValueError, match="conflicts"): + resolve_object_basis(legacy).validate_fit(fit) + + +def test_real_fresh_lift_and_package_preserve_basis_and_gates(tmp_path): + torch = pytest.importorskip("torch", reason="needs inference group") + import lift_object_3d + import package_objects + + people = fresh_people() + (tmp_path / "people.json").write_text(json.dumps(people)) + cameras = { + "cameras": [ + { + "sourceIndex": f, + "time": f / 30, + "camera_to_world": np.eye(4).tolist(), + "source_intrinsics": [[500, 0, 320], [0, 500, 240], [0, 0, 1]], + } + for f in people["sourceIndices"] + ] + } + (tmp_path / "cameras.json").write_text(json.dumps(cameras)) + frames = list(range(13)) + gravity = 9.80665 / 2 + positions = [ + package_objects.ballistic_position( + [0, 0, -2], [1, gravity * 0.2, 0], [0, -gravity, 0], f, 0, 30 + ) + for f in frames + ] + for track in range(2): + directory = tmp_path / "tracks" / f"track_{track:02d}" + directory.mkdir(parents=True) + # Wrists exactly meet the synthetic release/catch in the same registered frame. + joints = np.tile([track * 0.8, 0, 4], (22, 1)) + torch.save( + { + "sourceIndices": people["sourceIndices"], + "poses": [{"j3d": joints} for _ in people["sourceIndices"]], + }, + directory / "source-poses.pt", + ) + camera = lift_object_3d.load_cameras(tmp_path / "cameras.json") + pixels = [lift_object_3d.project(camera, f, p)[0].tolist() for f, p in zip(frames, positions)] + (tmp_path / "flights.json").write_text( + json.dumps({"fps": 30, "flights": [{"frames": frames, "px": pixels}]}) + ) + args = [ + "lift_object_3d.py", + "--people", + str(tmp_path / "people.json"), + "--cameras", + str(tmp_path / "cameras.json"), + "--tracks-dir", + str(tmp_path / "tracks"), + "--flights", + str(tmp_path / "flights.json"), + "--thrower", + "0", + "--catcher", + "1", + "--out", + str(tmp_path / "fit.json"), + ] + with patch.object(sys, "argv", args), contextlib.redirect_stdout(io.StringIO()): + lift_object_3d.main() + fit = json.loads((tmp_path / "fit.json").read_text()) + assert len(fit["flights"]) == 1 + assert fit["droppedFlights"] == [] + assert fit["gates"] == {"minObs": 10, "maxHandMetres": 1.3, "maxReprojPx": 8.0} + assert fit["constantY"] == 0 + assert fit["coordinateBasis"]["mode"] == "avatarStaturePrior" + args = [ + "package_objects.py", + "--world", + str(tmp_path), + "--fit", + str(tmp_path / "fit.json"), + "--cameras", + str(tmp_path / "cameras.json"), + "--tracks-dir", + str(tmp_path / "tracks"), + ] + with patch.object(sys, "argv", args), contextlib.redirect_stdout(io.StringIO()): + package_objects.main() + packaged = json.loads((tmp_path / "objects.json").read_text()) + obj = packaged["objects"][0] + assert ( + packaged["coordinateBasis"] == fit["coordinateBasis"] == obj["evidence"]["coordinateBasis"] + ) + assert obj["appliesSharedCameraDrift"] is False + assert obj["transform"]["translation"] == [0, 0, 0] + np.testing.assert_allclose(obj["bakedTrack"]["positions"], positions, atol=1e-5) + # A later floor correction cannot reuse this raw-frame fit or overwrite output. + (tmp_path / "people.json").write_text(json.dumps(legacy_people())) + previous = (tmp_path / "objects.json").read_text() + with patch.object(sys, "argv", args), pytest.raises(SystemExit, match="conflicts"): + package_objects.main() + assert (tmp_path / "objects.json").read_text() == previous + + +def test_legacy_packager_keeps_nonzero_floor_corrections(tmp_path): + pytest.importorskip("torch", reason="needs inference group") + import package_objects + from test_package_objects_timing import PackageObjectTimingTests + + args, output = PackageObjectTimingTests().package_fixture(tmp_path, 30) + path = tmp_path / "world/people.json" + people = json.loads(path.read_text()) + people["floorFit"]["sharedCameraDrift"]["offsetYUnits"] = [0.1, 0.2, -0.1] + people["floorFit"]["levels"]["C_test"]["translationY"] = {"person0": -0.03, "person1": 0.04} + path.write_text(json.dumps(people)) + with patch.object(sys, "argv", args), contextlib.redirect_stdout(io.StringIO()): + package_objects.main() + obj = json.loads(output.read_text())["objects"][0] + assert obj["appliesSharedCameraDrift"] is True + np.testing.assert_allclose(obj["transform"]["translation"], [0, 0.005, 0]) + np.testing.assert_allclose(obj["bakedTrack"]["positions"][30], [1, -1.108325, -2], atol=1e-5) From d09928e249958b86b7fc8943a7930f5c569bd75c Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sun, 18 Oct 2026 20:51:05 -0400 Subject: [PATCH 09/14] fix: advance workers after fast detached steps --- orchestrator/supervisor.py | 18 +++- scripts/tests/test_supervisor_wakeup.py | 129 ++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 4 deletions(-) diff --git a/orchestrator/supervisor.py b/orchestrator/supervisor.py index 174b94e..2ea1122 100644 --- a/orchestrator/supervisor.py +++ b/orchestrator/supervisor.py @@ -84,6 +84,7 @@ def due(run_dir: Path, pushed: dict[str, tuple[float, int]], push_after: float) class WorkOutcome: message: str observed_entries: int + resume_next_pass: bool = False def work_on(run_dir: Path, repository, policy: HarnessPolicy, turns: int) -> WorkOutcome: @@ -117,7 +118,11 @@ def work_on(run_dir: Path, repository, policy: HarnessPolicy, turns: int) -> Wor message = f"{run_id}: session ended without deciding — {outcome.error}" else: message = f"{run_id}: still going after {turns} turns" - return WorkOutcome(message, outcome.observed_entries) + # A fast detached step may finish before the harness's final message. If the + # turn budget ends there, its finish is already in observed_entries, but the + # next agent turn still needs to react to it. This is progress, not idle time. + resume_next_pass = outcome.wants_another_turn and outcome.steps_run > 0 and outcome.running == 0 + return WorkOutcome(message, outcome.observed_entries, resume_next_pass) def main(argv: list[str] | None = None) -> int: @@ -181,9 +186,14 @@ def main(argv: list[str] | None = None) -> int: continue outcome = work_on(run_dir, repository, policy, args.turns) print(outcome.message, flush=True) - # Keep the cursor the session actually observed. A reply can arrive between - # its final read and this assignment; reading again here would swallow it. - pushed[str(run_dir)] = (time.monotonic(), outcome.observed_entries) + if outcome.resume_next_pass: + # Continue useful work on the next pass, still bounded by --turns + # and the ordinary admission checks. Do not fabricate an unseen event. + pushed.pop(str(run_dir), None) + else: + # Keep the cursor the session actually observed. A reply can arrive between + # its final read and this assignment; reading again here would swallow it. + pushed[str(run_dir)] = (time.monotonic(), outcome.observed_entries) if single_pass: return 0 time.sleep(args.interval) diff --git a/scripts/tests/test_supervisor_wakeup.py b/scripts/tests/test_supervisor_wakeup.py index 00d7f99..6a0485d 100644 --- a/scripts/tests/test_supervisor_wakeup.py +++ b/scripts/tests/test_supervisor_wakeup.py @@ -83,3 +83,132 @@ def test_once_and_watch_are_mutually_exclusive(): with pytest.raises(SystemExit) as error: supervisor.main(["--once", "--watch"]) assert error.value.code == 2 + + +@pytest.mark.parametrize("turns", [1, 3]) +def test_fast_detached_finish_on_last_turn_resumes_on_next_pass(tmp_path, monkeypatch, turns): + from orchestrator.agent.contracts import AgentResult + from orchestrator.agent.harness import HarnessAgent, HarnessOutcome + + run = supervisor.open_run(tmp_path, run_id="fast-run", name="clip", source=None) + journal = Journal(run) + calls = [] + + def agent_turn(*args, **kwargs): + calls.append(len(journal.entries())) + if len(calls) <= turns: + attempt = f"attempt-{len(calls)}" + journal.append("step.started", step="review", attempt=attempt, number=len(calls)) + # The detached child finishes while the harness still prepares its final + # message, so RunSession.turn observes BOTH events before returning. + journal.append("step.finished", step="review", attempt=attempt, status="succeeded") + else: + assert journal.active_attempts() == {} + assert journal.last_status("review") == "succeeded" + journal.append("question", question="Review the next bounded action") + return HarnessOutcome(AgentResult(status="completed", transcript_path="inert")) + + passes = 0 + + class EndCheck(Exception): + pass + + def tick(_): + nonlocal passes + passes += 1 + if passes == 1: + # The fix must not exceed this pass's budget to evade the race. + assert len(calls) == turns + if passes == 3: + raise EndCheck + + monkeypatch.setattr(supervisor, "repository_from_environment", lambda: None) + monkeypatch.setattr( + HarnessPolicy, + "from_environment", + lambda: HarnessPolicy(kind="command", command=("unused",)), + ) + monkeypatch.setattr(HarnessAgent, "run", agent_turn) + monkeypatch.setattr(supervisor.time, "sleep", tick) + with pytest.raises(EndCheck): + supervisor.main( + [ + "--runs", + str(tmp_path), + "--run-id", + "fast-run", + "--watch", + "--turns", + str(turns), + "--push-after", + "3600", + ] + ) + assert len(calls) == turns + 1 + assert journal.unanswered() == "Review the next bounded action" + assert len([e for e in journal.entries() if e.kind == "step.started"]) == turns + + +@pytest.mark.parametrize("mode", ["active", "idle", "question", "finished", "paused", "error"]) +def test_fast_work_followup_does_not_bypass_waiting_or_backoff(tmp_path, monkeypatch, mode): + from orchestrator.agent.contracts import AgentResult + from orchestrator.agent.harness import HarnessAgent, HarnessOutcome + + run = supervisor.open_run(tmp_path, run_id="held-run", name="clip", source=None) + journal = Journal(run) + calls = [] + + def agent_turn(*args, **kwargs): + calls.append(1) + if mode != "idle": + journal.append("step.started", step="review", attempt="one", number=1) + if mode != "active": + journal.append("step.finished", step="review", attempt="one", status="succeeded") + if mode == "question": + journal.append("question", question="Operator decision required") + elif mode == "finished": + journal.append("run.finished", status="succeeded") + elif mode == "paused": + (run / "paused").touch() + result = AgentResult( + status="stalled" if mode == "error" else "completed", + transcript_path="inert", + error="silent" if mode == "error" else None, + ) + return HarnessOutcome(result) + + passes = 0 + + class EndCheck(Exception): + pass + + def tick(_): + nonlocal passes + passes += 1 + if passes == 3: + raise EndCheck + + monkeypatch.setattr(supervisor, "repository_from_environment", lambda: None) + monkeypatch.setattr( + HarnessPolicy, + "from_environment", + lambda: HarnessPolicy(kind="command", command=("unused",)), + ) + monkeypatch.setattr(HarnessAgent, "run", agent_turn) + monkeypatch.setattr(supervisor.time, "sleep", tick) + with pytest.raises(EndCheck): + supervisor.main( + [ + "--runs", + str(tmp_path), + "--run-id", + "held-run", + "--watch", + "--turns", + "1", + "--push-after", + "3600", + ] + ) + assert len(calls) == 1 + assert len(journal.active_attempts()) == (1 if mode == "active" else 0) From 70ed89bd85d6c50b75dad3c84815bb0c7dfa397b Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sun, 18 Oct 2026 20:53:15 -0400 Subject: [PATCH 10/14] fix: show active stages above retained run failures --- admin/components/run-overview.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/components/run-overview.tsx b/admin/components/run-overview.tsx index 1340776..511fb2d 100644 --- a/admin/components/run-overview.tsx +++ b/admin/components/run-overview.tsx @@ -11,8 +11,8 @@ function progressTitle(progress: RunProgress): string { if (progress.finished) return `Run finished: ${progress.finished}`; if (progress.paused) return 'Run paused'; if (progress.question) return 'Waiting for your answer'; - if (progress.stopped.length) return 'Processing stopped'; if (progress.currentStages.length) return `Running ${progress.currentStages.join(', ')}`; + if (progress.stopped.length) return 'Processing stopped'; return 'Waiting for the supervisor'; } From 80bb348925416a8ef9738ebf60b3dccef0be69eb Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sun, 18 Oct 2026 21:07:18 -0400 Subject: [PATCH 11/14] docs: record fresh room geometry failure --- docs/known-limits.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/known-limits.md b/docs/known-limits.md index e35ada7..50c4e76 100644 --- a/docs/known-limits.md +++ b/docs/known-limits.md @@ -10,6 +10,7 @@ Do not retry one without a new idea, and add a line here when you measure a new - Faces below about 64 px of face height are unreadable; per-clip head refits do not help. - Smoke, fire, and water cannot be reconstructed from one moving camera. - Clips with too little parallax, or too dark, fail reconstruction. +- A successful Marble video generation can invent major foreground structure despite an accurate prompt. The fresh bottle-source world added closed glazed doors across an open landing and changed the corridor beyond. Eleven local camera views did not recover the source layout; changing scale cannot repair the first-camera view because its translation is zero. Preserve this as a generated-world fidelity failure, not a successful placement or permission to regenerate. Evidence: `.context/evidence/fresh-room-origin-diagnostic-20261018/`. - GEN3C as a static fill generator lost to the FLUX panorama fill. - A dominant low horizontal plane can be a countertop rather than a walking floor. Kitchen source-labelled floor tiles lie about 0.61 body-heights below its selected counter plane; validate surface identity before using camera-height or walker-floor estimates. - A track with no usable complete-body height does not constrain body scale through vertical silhouette edges alone. In the retained hall-walk shot, head/floor fitting increased visible body area and reduced source overlap despite passing the camera-height check. Tracker-derived crop flags are also fallible: at six seconds its projected feet lie outside the image although the source boots are visible. A visible-width experiment improved overlap but still failed area and contact-noise checks. Preserve the failed verdict and investigate tracking disagreement; a size-check pass does not establish accurate placement. From e93fc597ec531f9803772f0f4c3f5e838f6bdcc9 Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sun, 18 Oct 2026 21:22:50 -0400 Subject: [PATCH 12/14] fix: retain scale-fit evidence when quality gates fail --- scripts/run_clip.py | 15 ++- scripts/tests/test_final_world_selection.py | 3 + scripts/tests/test_scale_fit_evidence.py | 121 ++++++++++++++++++++ 3 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 scripts/tests/test_scale_fit_evidence.py diff --git a/scripts/run_clip.py b/scripts/run_clip.py index b39f853..2c1da76 100644 --- a/scripts/run_clip.py +++ b/scripts/run_clip.py @@ -2418,6 +2418,8 @@ def diag(self, scale0: float, probe: int): f"{scale0:.6f}", "--tag", tag, + "--share", + str(SHARE), ], log, attempts=1, @@ -2430,7 +2432,10 @@ def diag(self, scale0: float, probe: int): ] if not rows: raise RuntimeError(f"the depth-ratio diagnostic printed no ratios; see {log}") - return rows, SHARE / f"{tag}-depth-ratio.png" + figure = SHARE / f"{tag}-depth-ratio.png" + if not figure.is_file() or figure.stat().st_size == 0: + raise RuntimeError(f"the depth-ratio diagnostic produced no figure; see {log}") + return rows, figure def person_ply(self) -> Path: p = public_dir() / "worlds" / f"{self.name}-4d" / "person" / "frame_000.ply" @@ -2477,8 +2482,7 @@ def scale_fit(self): for _, lo, hi in rows if not (SCALE_SPREAD_BAND[0] <= lo and hi <= SCALE_SPREAD_BAND[1]) ] - self.state.record( - "_scale", + report = dict( scale0=s, geomean=gm, ratios=ratios, @@ -2490,6 +2494,9 @@ def scale_fit(self): spreadOk=not wide, figure=str(fig), ) + # Keep the measurements available to the orchestrator even when a quality gate fails. + (self.ctx / "scale-fit.json").write_text(json.dumps(report, indent=2) + "\n") + self.state.record("_scale", **report) say( f" fitted scale0 {s:.4f}, ratios {[round(r, 3) for r in ratios]}, geomean {gm:.4f}, " f"p10 {min(p10):.3f}-{max(p10):.3f} p90 {min(p90):.3f}-{max(p90):.3f}" @@ -2510,7 +2517,7 @@ def scale_fit(self): f"{SCALE_RATIO_BAND[0]}-{SCALE_RATIO_BAND[1]} at any single scale (geomean fit {s:.4f} " f"gives {gm:.4f}). That is Marble geometry error or SfM drift, not a scale error -- it " f"means the avatar reads ~{abs(1 - min(ratios)) * 100:.0f}% wrong at one end of the " - f"clip. Read share/{self.name}-diag-*-depth-ratio.png, then either re-trim the clip to " + f"clip. Read {fig}, then either re-trim the clip to " f"the window that fits or pass --allow-scale-spread to ship the geometric-mean fit " f"with that caveat (gym, atrium and selfie did)." ) diff --git a/scripts/tests/test_final_world_selection.py b/scripts/tests/test_final_world_selection.py index 92a2d04..55a4f94 100644 --- a/scripts/tests/test_final_world_selection.py +++ b/scripts/tests/test_final_world_selection.py @@ -216,11 +216,14 @@ def test_scale_and_finetune_use_original_world_even_when_refinement_is_retained( p.state.record("finetune", status="ok") p.a.force = "finetune" commands = [] + monkeypatch.setattr(run_clip, "SHARE", p.ctx) def invoke(command, log, **kwargs): commands.append(command) if "--diag" in command: log.write_text("(ratio 1.000), p10 0.98 p90 1.02") + tag = command[command.index("--tag") + 1] + (p.ctx / f"{tag}-depth-ratio.png").write_bytes(b"diagnostic fixture") monkeypatch.setattr(run_clip, "run", invoke) p.diag(1, 0) diff --git a/scripts/tests/test_scale_fit_evidence.py b/scripts/tests/test_scale_fit_evidence.py new file mode 100644 index 0000000..cd69fec --- /dev/null +++ b/scripts/tests/test_scale_fit_evidence.py @@ -0,0 +1,121 @@ +"""Scale diagnostics stay with the run, including rejected measurements (no inference).""" + +import json +import math +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import run_clip + + +@pytest.fixture +def pipeline(tmp_path, monkeypatch): + p = object.__new__(run_clip.Pipeline) + p.ctx = tmp_path / "run" + p.ctx.mkdir() + p.name = "fixture" + p.clip = tmp_path / "source.mp4" + p.state = run_clip.State(p.ctx / "state.json") + p.a = SimpleNamespace( + scale0=None, + scale_probes=5, + allow_scale_spread=False, + only=None, + force=None, + skip_finetune=True, + reuse_world=None, + ) + p.stages = ["scale_fit"] + p.deps = {"scale_fit": []} + p.marble = "video" + p.multi = False + p.summary = Mock() + p.placement = Mock(return_value={"floorMatchedScale": 0.8}) + p.base_world_spz = Mock(return_value=p.ctx / "world.spz") + p.cameras = Mock(return_value=p.ctx / "cameras.json") + p.person_ply = Mock(return_value=p.ctx / "person.ply") + p.anchor_samples = Mock(return_value="0,1") + share = p.ctx / "share" + share.mkdir() + monkeypatch.setattr(run_clip, "SHARE", share) + return p + + +def diagnostic_worker(monkeypatch, rows_by_probe, *, figure=True): + commands = [] + + def invoke(command, log, **kwargs): + rows = rows_by_probe[len(commands)] + commands.append(command) + log.write_text( + "\n".join(f"(ratio {r:.3f}), p10 {lo:.3f} p90 {hi:.3f}" for r, lo, hi in rows) + ) + if figure: + output = Path(command[command.index("--share") + 1]) + tag = command[command.index("--tag") + 1] + (output / f"{tag}-depth-ratio.png").write_bytes(b"diagnostic fixture") + assert kwargs["attempts"] == 1 + + monkeypatch.setattr(run_clip, "run", invoke) + return commands + + +@pytest.mark.parametrize( + "rows,median_ok,spread_ok", + [ + ([(0.889, 0.088, 1.456), (1.125, 0.872, 1.115)], False, False), + ([(0.94, 0.92, 1.0), (1.064, 1.0, 1.08)], False, True), + ([(1.0, 0.8, 1.2), (1.0, 0.95, 1.05)], True, False), + ([(1.0, 0.95, 1.05), (1.0, 0.98, 1.02)], True, True), + ], +) +def test_fitted_measurements_and_figure_survive_quality_verdict( + pipeline, monkeypatch, rows, median_ok, spread_ok +): + p = pipeline + # The first probe changes the scale; the second converges but may still fail the gates. + commands = diagnostic_worker(monkeypatch, [[(0.9, 0.85, 1.0)], rows]) + accepted = median_ok and spread_ok + assert p.go() == (0 if accepted else 1) + assert len(commands) == 2 + stages = json.loads(p.state.path.read_text())["stages"] + assert stages["scale_fit"]["status"] == ("ok" if accepted else "failed") + report = json.loads((p.ctx / "scale-fit.json").read_text()) + assert report == stages["_scale"] + assert report["scale0"] == pytest.approx(0.8 * 0.9) + assert report["geomean"] == pytest.approx(math.sqrt(rows[0][0] * rows[1][0])) + assert report["ratios"] == [row[0] for row in rows] + assert report["p10"] == [row[1] for row in rows] + assert report["p90"] == [row[2] for row in rows] + assert report["gateOk"] is accepted + assert report["medianOk"] is median_ok + assert report["spreadOk"] is spread_ok + figure = Path(report["figure"]) + assert figure.parent == p.ctx / "share" + assert figure.is_file() + assert len(list(figure.parent.glob("*-depth-ratio.png"))) == 2 + if not accepted: + assert str(figure) in stages["scale_fit"]["error"] + + +@pytest.mark.parametrize("empty", [False, True]) +def test_missing_or_empty_figure_cannot_produce_a_dangling_evidence_reference( + pipeline, monkeypatch, empty +): + p = pipeline + p.a.scale0 = 1.0 + diagnostic_worker(monkeypatch, [[(1.0, 0.98, 1.02)]], figure=False) + if empty: + (p.ctx / "share/fixture-diag-s10000-depth-ratio.png").touch() + assert p.go() == 1 + stages = json.loads(p.state.path.read_text())["stages"] + assert stages["scale_fit"]["status"] == "failed" + assert "produced no figure" in stages["scale_fit"]["error"] + assert "_scale" not in stages + assert not (p.ctx / "scale-fit.json").exists() + assert (p.ctx / "diag-given.log").is_file() From 251aa7b7b21d42f1a1dc5f0a24066df3e80448d0 Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sun, 18 Oct 2026 21:22:50 -0400 Subject: [PATCH 13/14] fix: preserve source media in browser-compatible delivery --- docs/audio.md | 22 +- docs/known-limits.md | 2 + orchestrator/api/viewer.py | 11 + scripts/package_multiperson.py | 4 +- scripts/package_person_sequence.py | 4 +- scripts/package_source_audio.py | 269 ++++++++++++++++++++- scripts/run_clip.py | 13 +- scripts/tests/test_package_source_audio.py | 224 +++++++++++++++++ scripts/tests/test_pipeline_viewer.py | 110 +++++++++ 9 files changed, 646 insertions(+), 13 deletions(-) diff --git a/docs/audio.md b/docs/audio.md index 31094bf..1d189d5 100644 --- a/docs/audio.md +++ b/docs/audio.md @@ -25,10 +25,24 @@ The viewer discovers `audio.json` next to `people.json`, or in the world directo The timeline duration must match the viewer's people/sequence duration within 0.1 seconds. A track's sample zero occurs at `offsetSeconds` on the clip timeline, so sample time equals clip time minus offset. Trim original audio against the same original-video edit, retaining matching silence and verifying offsets. URLs resolve relative to `audio.json`. Additional provenance fields are allowed. Optional `provenance.attribution`, `provenance.license`, and `provenance.licenseUrl` show a visible audio excerpt credit while the original mix is selected; license links accept only HTTP(S) URLs. The recovered TOS package uses this credit for Blender Foundation's CC BY 3.0 source. Keep media and manifests in gitignored `public/worlds/-4d/` or `public/clips/audio/`; publish through the existing shared-assets snapshot only when authorized. PCM WAV and MP3 MIME entries already exist. No keys belong in manifests or client code. The attribution strip is currently shown only in original mode; it does not render per-stem credits or remain visible in spatial mode. The single-person and multiperson packagers probe the supplied local video with `ffprobe` and -write a minimal `audio.json` containing its embedded-audio availability, SHA-256 and the package -playback duration. Silent videos therefore show “No soundtrack available”; videos with an audio -stream keep their original embedded soundtrack. A failed or malformed probe stops packaging -before package files are written instead of guessing that the clip is silent. +write `audio.json` containing soundtrack availability, the submitted source SHA-256 and the +package playback duration. Silent videos therefore show “No soundtrack available”; ordinary +AAC inputs keep their embedded soundtrack. A failed or malformed probe stops packaging before +package files are written instead of guessing that the clip is silent. + +H.264 clips with original PCM audio use a separate browser delivery: `media/source.mp4` copies +only the encoded video, and `media/original.wav` copies the original PCM samples. The source +file remains unchanged. `source.hasAudio` is false for this silent delivery video, +`source.videoUrl` selects it, and `original.url` selects the authentic external mix. +`media/provenance.json` binds both output hashes to the submitted source. Packaging compares +every encoded video packet and timestamp, its time base, the PCM sample hash, sample count, +rate, channels and continuous audio timestamps before exposing the files. No encoding, +resampling, normalization or timeline changes occur. Currently this path accepts one +zero-origin H.264 video and one signed little-endian 16-, 24- or 32-bit PCM audio stream; +unsupported PCM layouts fail explicitly. Run previews and completion validate the delivery +hashes while still requiring the original exported source. The standalone printed viewer URL +also selects the delivery MP4. These local checks establish media preservation; browser +playback and audible synchronization still require actual viewer checks. An identical generated manifest can be reused. An existing manifest with different content, including authored external audio or reviewed stems, stops packaging before it changes the diff --git a/docs/known-limits.md b/docs/known-limits.md index 50c4e76..c94d52b 100644 --- a/docs/known-limits.md +++ b/docs/known-limits.md @@ -15,3 +15,5 @@ Do not retry one without a new idea, and add a line here when you measure a new - A dominant low horizontal plane can be a countertop rather than a walking floor. Kitchen source-labelled floor tiles lie about 0.61 body-heights below its selected counter plane; validate surface identity before using camera-height or walker-floor estimates. - A track with no usable complete-body height does not constrain body scale through vertical silhouette edges alone. In the retained hall-walk shot, head/floor fitting increased visible body area and reduced source overlap despite passing the camera-height check. Tracker-derived crop flags are also fallible: at six seconds its projected feet lie outside the image although the source boots are visible. A visible-width experiment improved overlap but still failed area and contact-noise checks. Preserve the failed verdict and investigate tracking disagreement; a size-check pass does not establish accurate placement. - Main-thread `toBlob` and `OffscreenCanvas.convertToBlob` took about four seconds per spectator image during native Quest XR sessions. Moving JPEG encoding to a dedicated worker reduced observed encoding latency to roughly 25–40 ms; simulated XR did not reproduce the main-thread stall. + +- A source MOV containing H.264 video and PCM audio can stall its browser video clock after Web Audio unlock, despite reporting `paused=false`, `readyState=4` and no media error. The same encoded video in a silent MP4 advanced under the same viewer settings. Packaging now stream-copies compatible video to MP4 and the original PCM to an external WAV; packet timestamps and samples must remain identical. This is a delivery compatibility failure, not an animation or inference failure. Source/media proof: `.context/evidence/bottle-source-rerun/media-compatibility/`. diff --git a/orchestrator/api/viewer.py b/orchestrator/api/viewer.py index 14065ea..cfe1978 100644 --- a/orchestrator/api/viewer.py +++ b/orchestrator/api/viewer.py @@ -13,6 +13,8 @@ from fastapi import Depends, HTTPException, Request from fastapi.responses import FileResponse, JSONResponse, Response +from scripts.package_source_audio import validate_source_media + WORLD_SUFFIXES = {"video": "clean", "both": "clean", "image": "image", "multi": "multi"} ASSET_SUFFIXES = { ".json", @@ -115,6 +117,7 @@ def asset_path(root: Path, relative: str, *, require_file: bool = True) -> Path: } allowed = ( relative == source_relative + or relative == f"worlds/{name}-4d/media/source.mp4" or relative in world_files or (relative.startswith(f"worlds/{name}-4d/") and path.suffix.lower() in ASSET_SUFFIXES) ) @@ -186,6 +189,14 @@ def scene_manifest(root: Path) -> dict | None: source = Path(described["source"]) video = f"clips/{name}/source{source.suffix.lower()}" asset_path(root, video) + audio_path = asset_path(root, base + "audio.json") + audio = read_json(audio_path) + delivered_video = validate_source_media( + audio_path.parent, audio, described.get("sourceSha256") + ) + if delivered_video: + video = base + delivered_video + asset_path(root, video) options = described.get("options") or {} mode = options.get("marble", "video") world = "none" diff --git a/scripts/package_multiperson.py b/scripts/package_multiperson.py index ad8c1a0..0a1e0be 100644 --- a/scripts/package_multiperson.py +++ b/scripts/package_multiperson.py @@ -37,7 +37,7 @@ quat_mul, ) from sfm_frame import camera0_reframe, describe as describe_frame # noqa: E402 -from package_source_audio import source_audio_manifest # noqa: E402 +from package_source_audio import prepare_source_media # noqa: E402 def body_stats(ply): @@ -154,7 +154,7 @@ def main(): out = Path(a.out) tracks_doc = json.loads((Path(a.tracks) / "tracks.json").read_text()) - audio = source_audio_manifest(a.clip, tracks_doc["duration"], out) + audio = prepare_source_media(a.clip, tracks_doc["duration"], out) out.mkdir(parents=True, exist_ok=True) quality = {t["track"]: t["quality"] for t in tracks_doc["tracks"]} diff --git a/scripts/package_person_sequence.py b/scripts/package_person_sequence.py index fbab4e3..b31fd30 100644 --- a/scripts/package_person_sequence.py +++ b/scripts/package_person_sequence.py @@ -20,7 +20,7 @@ import numpy as np import package_person_motion -from package_source_audio import source_audio_manifest +from package_source_audio import prepare_source_media from sfm_frame import camera0_reframe, describe as describe_frame from plyfile import PlyData, PlyElement @@ -109,7 +109,7 @@ def main(): person = out / "person" seq = json.loads((motion / "sequence.json").read_text()) duration = seq["duration"] if "duration" in seq else len(seq["frames"]) / seq.get("fps", 12) - audio = source_audio_manifest(a.clip, duration, out) + audio = prepare_source_media(a.clip, duration, out) person.mkdir(parents=True, exist_ok=True) cams = json.loads(Path(a.cameras).read_text()) # world -> the packaged frame: camera 0 at the origin, gravity on +y when the run's framealign.json diff --git a/scripts/package_source_audio.py b/scripts/package_source_audio.py index cdbba43..6a8f208 100644 --- a/scripts/package_source_audio.py +++ b/scripts/package_source_audio.py @@ -1,8 +1,9 @@ -"""Declare the supplied video's embedded soundtrack without modifying media.""" +"""Preserve source media and prepare lossless browser delivery for H.264/PCM clips.""" import hashlib import json import math +import tempfile from pathlib import Path import subprocess @@ -23,7 +24,7 @@ def source_audio_manifest(clip, duration, world): "-v", "error", "-show_entries", - "stream=codec_type", + "stream=codec_type,codec_name,start_time,time_base,sample_rate,channels", "-of", "json", str(clip), @@ -43,7 +44,9 @@ def source_audio_manifest(clip, duration, world): not isinstance(streams, list) or not streams or any( - not isinstance(stream, dict) or not isinstance(stream.get("codec_type"), str) + not isinstance(stream, dict) + or not isinstance(stream.get("codec_type"), str) + or not isinstance(stream.get("codec_name", ""), str) for stream in streams ) or not any(stream["codec_type"] == "video" for stream in streams) @@ -63,6 +66,24 @@ def source_audio_manifest(clip, duration, world): timeline={"durationSeconds": duration}, defaultMode="original", ) + audio_streams = [stream for stream in streams if stream["codec_type"] == "audio"] + if any(stream.get("codec_name", "").startswith("pcm_") for stream in audio_streams): + video_streams = [stream for stream in streams if stream["codec_type"] == "video"] + if ( + len(video_streams) != 1 + or video_streams[0].get("codec_name") != "h264" + or len(audio_streams) != 1 + or audio_streams[0].get("codec_name") not in {"pcm_s16le", "pcm_s24le", "pcm_s32le"} + or any(stream.get("start_time") != "0.000000" for stream in streams) + or not isinstance(video_streams[0].get("time_base"), str) + or not video_streams[0]["time_base"].startswith("1/") + ): + raise ValueError( + "Lossless PCM delivery requires one zero-origin H.264 video and one PCM audio stream" + ) + manifest["source"].update(hasAudio=False, videoUrl="media/source.mp4") + manifest["original"] = {"url": "media/original.wav", "offsetSeconds": 0} + manifest["provenance"] = {"delivery": "media/provenance.json"} # Old authored audio may belong to another source, even if the clip's stream types match. path = Path(world) / "audio.json" if path.exists(): @@ -76,4 +97,246 @@ def source_audio_manifest(clip, duration, world): raise ValueError(message) from error if previous != manifest: raise ValueError(message) + # Runner rebuilds remove the destination after this preflight. Equal metadata + # alone must never authorize deleting a missing or edited delivery asset. + if "videoUrl" in manifest["source"]: + try: + validate_source_media(world, manifest, source_hash) + except (OSError, ValueError) as error: + raise ValueError( + "Existing source-media delivery is missing or changed; use a fresh destination" + ) from error return manifest + + +def file_sha256(path): + with Path(path).open("rb") as source: + return hashlib.file_digest(source, "sha256").hexdigest() + + +def _run(command): + return subprocess.run(command, check=True, capture_output=True, text=True, timeout=120).stdout + + +def _video_packets(path): + return json.loads( + _run( + [ + "ffprobe", + "-v", + "error", + "-select_streams", + "v:0", + "-show_packets", + "-show_data_hash", + "sha256", + "-show_entries", + "packet=pts,dts,duration,data_hash:stream=time_base", + "-show_streams", + "-of", + "json", + str(path), + ] + ) + ) + + +def _audio_timing(path): + probe = json.loads( + _run( + [ + "ffprobe", + "-v", + "error", + "-select_streams", + "a:0", + "-show_packets", + "-show_entries", + "packet=pts,duration:stream=time_base,sample_rate,channels,codec_name", + "-of", + "json", + str(path), + ] + ) + ) + stream = probe["streams"][0] + if stream["time_base"] != "1/" + stream["sample_rate"]: + raise ValueError("PCM delivery requires sample-aligned timestamps") + sample_count = 0 + for packet in probe["packets"]: + if packet.get("pts") != sample_count or packet.get("duration", 0) <= 0: + raise ValueError("PCM delivery cannot remove audio timestamp gaps or overlaps") + sample_count += packet["duration"] + if not sample_count: + raise ValueError("PCM delivery requires recorded audio samples") + return {**stream, "sampleCount": sample_count} + + +def _pcm_hash(path): + return ( + _run( + [ + "ffmpeg", + "-v", + "error", + "-i", + str(path), + "-map", + "0:a:0", + "-c:a", + "copy", + "-f", + "hash", + "-hash", + "sha256", + "-", + ] + ) + .strip() + .removeprefix("SHA256=") + ) + + +def prepare_source_media(clip, duration, world): + """Materialize only a declared lossless delivery; never overwrite authored audio. + + A staging directory keeps failed stream checks from changing package assets. The + immutable submitted source remains authoritative; the MP4 only supplies the clock. + """ + world = Path(world) + manifest = source_audio_manifest(clip, duration, world) + if "videoUrl" not in manifest["source"]: + return manifest + world.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".source-media-", dir=world.parent) as temporary: + staged = Path(temporary) + video, audio = staged / "source.mp4", staged / "original.wav" + original_audio = _audio_timing(clip) + original_packets = _video_packets(clip) + time_base = original_packets["streams"][0]["time_base"] + _run( + [ + "ffmpeg", + "-v", + "error", + "-i", + str(clip), + "-map", + "0:v:0", + "-c:v", + "copy", + "-an", + "-video_track_timescale", + time_base.split("/")[1], + "-movflags", + "+faststart", + str(video), + ] + ) + _run( + [ + "ffmpeg", + "-v", + "error", + "-i", + str(clip), + "-map", + "0:a:0", + "-c:a", + "copy", + str(audio), + ] + ) + delivered_packets = _video_packets(video) + # Container metadata differs, but every encoded packet and its timestamp must match. + if ( + original_packets["packets"] != delivered_packets["packets"] + or time_base != delivered_packets["streams"][0]["time_base"] + ): + raise ValueError("Lossless delivery changed video packets or timing") + pcm_hash = _pcm_hash(clip) + if pcm_hash != _pcm_hash(audio) or original_audio != _audio_timing(audio): + raise ValueError("Lossless delivery changed original PCM samples") + receipt = { + "schema": "wander.source-media/1", + "sourceSha256": manifest["source"]["sha256"], + "video": {"url": "source.mp4", "sha256": file_sha256(video), "mode": "stream-copy"}, + "audio": {"url": "original.wav", "sha256": file_sha256(audio), "mode": "stream-copy"}, + "verification": { + "videoPacketsAndTimestampsIdentical": True, + "videoPacketCount": len(original_packets["packets"]), + "videoTimeBase": time_base, + "pcmSha256": pcm_hash, + "audio": original_audio, + }, + } + (staged / "provenance.json").write_text(json.dumps(receipt, indent=2) + "\n") + destination = world / "media" + if destination.exists(): + if ( + not destination.is_dir() + or destination.is_symlink() + or any( + not (destination / name).is_file() + or (destination / name).is_symlink() + or file_sha256(destination / name) != file_sha256(staged / name) + for name in ("source.mp4", "original.wav", "provenance.json") + ) + ): + raise ValueError("Existing source-media delivery differs; use a fresh destination") + else: + world.mkdir(parents=True, exist_ok=True) + staged.rename(destination) + return manifest + + +def validate_source_media(world, manifest, source_sha256): + """Validate declared package-local delivery files without running a media tool.""" + world = Path(world) + if not isinstance(manifest, dict): + raise ValueError("Invalid source-media manifest") + source = manifest.get("source", {}) + if not isinstance(source, dict): + raise ValueError("Invalid source-media source declaration") + if "videoUrl" not in source: + return None + provenance = manifest.get("provenance") + if not isinstance(provenance, dict): + raise ValueError("Invalid source-media provenance declaration") + if ( + not isinstance(source_sha256, str) + or len(source_sha256) != 64 + or any(character not in "0123456789abcdef" for character in source_sha256) + or manifest["source"].get("videoUrl") != "media/source.mp4" + or manifest["source"].get("hasAudio") is not False + or manifest["source"].get("sha256") != source_sha256 + or manifest.get("original") != {"url": "media/original.wav", "offsetSeconds": 0} + or provenance.get("delivery") != "media/provenance.json" + ): + raise ValueError("Invalid lossless source-media declaration") + for name in ("source.mp4", "original.wav", "provenance.json"): + path = world / "media" / name + if ( + not path.is_file() + or path.is_symlink() + or not path.resolve().is_relative_to(world.resolve()) + ): + raise ValueError("Missing or unsafe source-media delivery") + if (world / "media").is_symlink(): + raise ValueError("Unsafe source-media directory") + receipt = json.loads((world / "media/provenance.json").read_text()) + if ( + not isinstance(receipt, dict) + or receipt.get("schema") != "wander.source-media/1" + or receipt.get("sourceSha256") != source_sha256 + ): + raise ValueError("Source-media provenance names a different source") + for kind, name in (("video", "source.mp4"), ("audio", "original.wav")): + entry = receipt.get(kind) + if not isinstance(entry, dict) or entry != { + "url": name, + "sha256": file_sha256(world / "media" / name), + "mode": "stream-copy", + }: + raise ValueError("Source-media delivery bytes do not match provenance") + return "media/source.mp4" diff --git a/scripts/run_clip.py b/scripts/run_clip.py index 2c1da76..ada3d25 100644 --- a/scripts/run_clip.py +++ b/scripts/run_clip.py @@ -31,7 +31,7 @@ from urllib.parse import quote from pathlib import Path from marble_world import submission_history -from package_source_audio import source_audio_manifest +from package_source_audio import source_audio_manifest, validate_source_media from stage_attempts import StageAttempts, command_identity from clean_candidates import ( candidate_directory, @@ -2959,10 +2959,19 @@ def summary(self, wall): ) if doc["peopleCount"] > 1: people_arg = f"&people=/worlds/{self.name}-4d/people.json" + audio_path = manifest.parent / "audio.json" + video_url = self.video_url + if audio_path.is_file(): + audio = json.loads(audio_path.read_text()) + delivered_video = validate_source_media(manifest.parent, audio, file_sha256(self.clip)) + if delivered_video: + video_url = "/" + quote( + (manifest.parent / delivered_video).relative_to(public_dir()).as_posix() + ) base = ( f"\n http://127.0.0.1:5399/fourd.html?world=/{quote(world.relative_to(public_dir()).as_posix())}" f"&person=/worlds/{self.name}-4d/person/sequence.json{people_arg}" - f"&video={self.video_url}" + f"&video={video_url}" ) if getattr(self.a, "retained_scene", None) and (manifest.parent / "scene.json").is_file(): scene = f"/worlds/{self.name}-4d/scene.json" diff --git a/scripts/tests/test_package_source_audio.py b/scripts/tests/test_package_source_audio.py index bd30944..64fe6b8 100644 --- a/scripts/tests/test_package_source_audio.py +++ b/scripts/tests/test_package_source_audio.py @@ -37,6 +37,31 @@ def clips(tmp_path_factory): command += ["-c:a", "aac", "-shortest"] if has_audio else ["-an"] subprocess.run([*command, str(clip)], check=True, capture_output=True, timeout=30) outputs[has_audio] = clip + pcm = root / "source.mov" + subprocess.run( + [ + "ffmpeg", + "-v", + "error", + "-i", + str(outputs[False]), + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=48000:duration=1", + "-c:v", + "copy", + "-c:a", + "pcm_s16le", + "-ac", + "2", + str(pcm), + ], + check=True, + capture_output=True, + timeout=30, + ) + outputs["pcm"] = pcm return outputs @@ -276,3 +301,202 @@ def launched(command, log): child.side_effect = launched (pipeline.package_people if multiple else pipeline.package)() child.assert_called_once() + + +@pytest.mark.parametrize("multiple", [False, True]) +def test_pcm_source_is_delivered_losslessly(tmp_path, monkeypatch, clips, multiple): + import wave + + module, args = package_inputs(tmp_path, multiple) + out = tmp_path / "world" + clip = clips["pcm"] + before = clip.read_bytes() + run_package(monkeypatch, module, args, clip, out) + manifest = json.loads((out / "audio.json").read_text()) + assert manifest["source"] == { + "hasAudio": False, + "sha256": hashlib.sha256(before).hexdigest(), + "videoUrl": "media/source.mp4", + } + assert manifest["original"] == {"url": "media/original.wav", "offsetSeconds": 0} + receipt = json.loads((out / "media/provenance.json").read_text()) + assert receipt["sourceSha256"] == hashlib.sha256(before).hexdigest() + delivered = out / "media/source.mp4" + streams = json.loads( + subprocess.check_output( + [ + "ffprobe", + "-v", + "error", + "-show_streams", + "-of", + "json", + str(delivered), + ] + ) + )["streams"] + assert len(streams) == 1 and streams[0]["codec_type"] == "video" + + # Independently compare decoded frames+timestamps, beyond the helper's encoded-packet check. + def frames(path): + return subprocess.check_output( + [ + "ffmpeg", + "-v", + "error", + "-i", + str(path), + "-map", + "0:v:0", + "-f", + "framemd5", + "-", + ] + ) + + assert frames(clip) == frames(delivered) + with wave.open(str(out / "media/original.wav"), "rb") as audio: + assert ( + audio.getnchannels(), + audio.getframerate(), + audio.getsampwidth(), + audio.getnframes(), + ) == (2, 48000, 2, 48000) + samples = audio.readframes(audio.getnframes()) + original_samples = subprocess.check_output( + [ + "ffmpeg", + "-v", + "error", + "-i", + str(clip), + "-map", + "0:a:0", + "-c:a", + "copy", + "-f", + "s16le", + "-", + ] + ) + assert samples == original_samples + assert clip.read_bytes() == before + assert ( + package_source_audio.validate_source_media( + out, manifest, hashlib.sha256(before).hexdigest() + ) + == "media/source.mp4" + ) + run_package(monkeypatch, module, args, clip, out) + assert json.loads((out / "audio.json").read_text()) == manifest + + +def test_failed_stream_preservation_does_not_change_destination(tmp_path, monkeypatch, clips): + out = tmp_path / "world" + out.mkdir() + (out / "keep.txt").write_text("existing package") + original = package_source_audio._video_packets + + def changed_packets(path): + result = original(path) + if Path(path).name == "source.mp4": + result["packets"][0]["pts"] += 1 + return result + + monkeypatch.setattr(package_source_audio, "_video_packets", changed_packets) + with pytest.raises(ValueError, match="video packets or timing"): + package_source_audio.prepare_source_media(clips["pcm"], 1, out) + assert [path.name for path in out.iterdir()] == ["keep.txt"] + assert (out / "keep.txt").read_text() == "existing package" + + +def test_changed_delivery_is_never_overwritten(tmp_path, clips): + out = tmp_path / "world" + manifest = package_source_audio.prepare_source_media(clips["pcm"], 1, out) + (out / "audio.json").write_text(json.dumps(manifest)) + wav = out / "media/original.wav" + wav.write_bytes(b"authored replacement") + with pytest.raises(ValueError, match="fresh destination"): + package_source_audio.prepare_source_media(clips["pcm"], 1, out) + assert wav.read_bytes() == b"authored replacement" + + +@pytest.mark.parametrize( + "stream_change", ["multiple-audio", "nonzero-start", "other-video", "unsupported-pcm"] +) +def test_unsupported_pcm_delivery_is_explicit(tmp_path, monkeypatch, clips, stream_change): + streams = [ + { + "codec_type": "video", + "codec_name": "h264", + "start_time": "0.000000", + "time_base": "1/12800", + }, + {"codec_type": "audio", "codec_name": "pcm_s16le", "start_time": "0.000000"}, + ] + if stream_change == "multiple-audio": + streams.append(dict(streams[1])) + elif stream_change == "nonzero-start": + streams[1]["start_time"] = "0.100000" + elif stream_change == "other-video": + streams[0]["codec_name"] = "hevc" + else: + streams[1]["codec_name"] = "pcm_f32le" + monkeypatch.setattr( + package_source_audio.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(stdout=json.dumps({"streams": streams})), + ) + with pytest.raises(ValueError, match="zero-origin H.264"): + package_source_audio.source_audio_manifest(clips["pcm"], 1, tmp_path) + assert not (tmp_path / "audio.json").exists() + + +@pytest.mark.parametrize("pts", [0, 1, 3]) +def test_pcm_gaps_and_overlaps_are_rejected(monkeypatch, pts): + probe = { + "streams": [{"sample_rate": "48000", "time_base": "1/48000"}], + "packets": [{"pts": 0, "duration": 2}, {"pts": pts, "duration": 2}], + } + monkeypatch.setattr(package_source_audio, "_run", lambda command: json.dumps(probe)) + with pytest.raises(ValueError, match="gaps or overlaps"): + package_source_audio._audio_timing("fixture.mov") + + +def test_pcm_authored_manifest_is_preserved_before_media_creation(tmp_path, clips): + manifest = {"original": {"url": "reviewed.wav"}, "provenance": {"attribution": "Owner"}} + path = tmp_path / "audio.json" + path.write_text(json.dumps(manifest)) + before = path.read_bytes() + with pytest.raises(ValueError, match="fresh destination"): + package_source_audio.prepare_source_media(clips["pcm"], 1, tmp_path) + assert path.read_bytes() == before + assert not (tmp_path / "media").exists() + + +@pytest.mark.parametrize("multiple", [False, True]) +@pytest.mark.parametrize("asset", ["original.wav", "source.mp4", "provenance.json"]) +@pytest.mark.parametrize("failure", ["modified", "missing"]) +def test_runner_pcm_preflight_preserves_existing_delivery( + tmp_path, monkeypatch, clips, multiple, asset, failure +): + pipeline, out, child = runner_inputs(tmp_path, monkeypatch, clips["pcm"], multiple) + manifest = package_source_audio.prepare_source_media(pipeline.clip, 0.7, out) + (out / "audio.json").write_text(json.dumps(manifest)) + (out / "person").mkdir() + (out / "person/sequence.json").write_text("existing sequence") + target = out / "media" / asset + if failure == "modified": + target.write_bytes(b"modified existing asset") + else: + target.unlink() + before = {path.relative_to(out): path.read_bytes() for path in out.rglob("*") if path.is_file()} + deletion = Mock(side_effect=AssertionError("Preflight must fail before removing the package")) + monkeypatch.setattr(run_clip.shutil, "rmtree", deletion) + with pytest.raises(ValueError, match="Existing source-media delivery is missing or changed"): + (pipeline.package_people if multiple else pipeline.package)() + deletion.assert_not_called() + child.assert_not_called() + assert before == { + path.relative_to(out): path.read_bytes() for path in out.rglob("*") if path.is_file() + } diff --git a/scripts/tests/test_pipeline_viewer.py b/scripts/tests/test_pipeline_viewer.py index c153cba..7430471 100644 --- a/scripts/tests/test_pipeline_viewer.py +++ b/scripts/tests/test_pipeline_viewer.py @@ -329,3 +329,113 @@ def test_runner_prints_scoped_scene_link(packaged, monkeypatch, capsys): assert "fourd.html?scene=%2Fapi%2Fpipeline%2Fruns%2Fviewer-run%2Fviewer%2Fscene.json" in output assert "WANDER_API_TOKEN" in output assert "?world=/" not in output + + +def declare_source_delivery(root, world): + import hashlib + + source_sha = json.loads((root / "run.json").read_text())["sourceSha256"] + media = world / "media" + media.mkdir() + receipt = {"schema": "wander.source-media/1", "sourceSha256": source_sha} + for kind, name in (("video", "source.mp4"), ("audio", "original.wav")): + data = f"fixture {kind}".encode() + (media / name).write_bytes(data) + receipt[kind] = { + "url": name, + "sha256": hashlib.sha256(data).hexdigest(), + "mode": "stream-copy", + } + (media / "provenance.json").write_text(json.dumps(receipt)) + manifest = { + "schema": "wander.audio/1", + "source": {"hasAudio": False, "sha256": source_sha, "videoUrl": "media/source.mp4"}, + "original": {"url": "media/original.wav", "offsetSeconds": 0}, + "provenance": {"delivery": "media/provenance.json"}, + "timeline": {"durationSeconds": 1}, + } + (world / "audio.json").write_text(json.dumps(manifest)) + return manifest + + +def test_verified_lossless_video_selected_and_range_served(packaged): + case, root, world, source = packaged + declare_source_delivery(root, world) + params = scene_manifest(root)["params"] + assert params["video"].endswith("/media/source.mp4") + response = case.client.get(params["video"], headers={**AUTH, "Range": "bytes=0-6"}) + assert response.status_code == 206 and response.content == b"fixture" + assert response.headers["content-type"] == "video/mp4" + base = params["video"].rsplit("/", 1)[0] + assert case.client.get(base + "/original.wav", headers=AUTH).content == b"fixture audio" + # The immutable submitted/exported source is still required even with a derivative. + source.unlink() + assert scene_manifest(root) is None + + +@pytest.mark.parametrize( + "failure", + [ + "video", + "audio", + "receipt", + "source-sha", + "output-sha", + "traversal", + "cross-run", + "symlink", + "directory-symlink", + "source-shape", + "provenance-shape", + ], +) +def test_invalid_lossless_delivery_hides_preview(packaged, failure): + case, root, world, source = packaged + manifest = declare_source_delivery(root, world) + media = world / "media" + if failure in {"video", "audio", "receipt"}: + ( + media + / {"video": "source.mp4", "audio": "original.wav", "receipt": "provenance.json"}[ + failure + ] + ).unlink() + elif failure == "source-sha": + manifest["source"]["sha256"] = "0" * 64 + elif failure == "output-sha": + (media / "source.mp4").write_bytes(b"corrupt") + elif failure in {"traversal", "cross-run"}: + manifest["source"]["videoUrl"] = ( + "../source.mp4" if failure == "traversal" else "/worlds/other-4d/media/source.mp4" + ) + elif failure == "symlink": + video = media / "source.mp4" + video.unlink() + video.symlink_to(source) + elif failure == "directory-symlink": + moved = world / "other-media" + media.rename(moved) + media.symlink_to(moved, target_is_directory=True) + elif failure == "source-shape": + manifest["source"] = [] + else: + manifest["provenance"] = [] + (world / "audio.json").write_text(json.dumps(manifest)) + assert scene_manifest(root) is None + assert ( + case.client.get("/api/pipeline/runs/viewer-run/viewer/scene.json", headers=AUTH).status_code + == 404 + ) + + +def test_only_designated_package_video_is_allowlisted(packaged): + case, root, world, source = packaged + declare_source_delivery(root, world) + (world / "other.mp4").write_bytes(b"unrelated") + prefix = "/api/pipeline/runs/viewer-run/viewer/assets/" + assert ( + case.client.get( + prefix + (world / "other.mp4").relative_to(root / "public").as_posix(), headers=AUTH + ).status_code + == 404 + ) From 6d121510904a40abbf363e273f6443250b946664 Mon Sep 17 00:00:00 2001 From: Austin Jian Date: Sun, 18 Oct 2026 22:40:08 -0400 Subject: [PATCH 14/14] fix: compare visible surfaces in scale diagnostics --- docs/known-limits.md | 2 + scripts/bake_video_colours.py | 114 +++++++++++++++-- scripts/run_clip.py | 7 +- .../tests/test_depth_diagnostic_visibility.py | 117 ++++++++++++++++++ 4 files changed, 227 insertions(+), 13 deletions(-) create mode 100644 scripts/tests/test_depth_diagnostic_visibility.py diff --git a/docs/known-limits.md b/docs/known-limits.md index c94d52b..a3a32b6 100644 --- a/docs/known-limits.md +++ b/docs/known-limits.md @@ -11,6 +11,8 @@ Do not retry one without a new idea, and add a line here when you measure a new - Smoke, fire, and water cannot be reconstructed from one moving camera. - Clips with too little parallax, or too dark, fail reconstruction. - A successful Marble video generation can invent major foreground structure despite an accurate prompt. The fresh bottle-source world added closed glazed doors across an open landing and changed the corridor beyond. Eleven local camera views did not recover the source layout; changing scale cannot repair the first-camera view because its translation is zero. Preserve this as a generated-world fidelity failure, not a successful placement or permission to regenerate. Evidence: `.context/evidence/fresh-room-origin-diagnostic-20261018/`. +- An explicitly approved image-conditioned replacement restored that open landing, but source-matched beginning/middle/end views still showed a missing staircase through the right portal and an invented raised wedge at the central column. A good first view is not full-clip layout acceptance. Sparse, ambiguous source/render feature matches did not justify a rigid registration correction. Evidence: `.context/evidence/fresh-room-origin-diagnostic-20261018/image-repair-01/`. +- Counting every generated splat in a camera frustum against frontmost source depth mistakes hidden rear geometry for registration error; duplicating hidden splats can even move the fitted median. The scale diagnostic now compares one generated front-center depth per supported source texel, retains the old distribution separately, and reports overlap counts. Synthetic visible near/far errors still fail the unchanged bands. This center z-buffer is an approximation, not opacity rendering or proof of scene coverage; the image replacement's visible geometry still needs independent review. - GEN3C as a static fill generator lost to the FLUX panorama fill. - A dominant low horizontal plane can be a countertop rather than a walking floor. Kitchen source-labelled floor tiles lie about 0.61 body-heights below its selected counter plane; validate surface identity before using camera-height or walker-floor estimates. - A track with no usable complete-body height does not constrain body scale through vertical silhouette edges alone. In the retained hall-walk shot, head/floor fitting increased visible body area and reduced source overlap despite passing the camera-height check. Tracker-derived crop flags are also fallible: at six seconds its projected feet lie outside the image although the source boots are visible. A visible-width experiment improved overlap but still failed area and contact-noise checks. Preserve the failed verdict and investigate tracking disagreement; a size-check pass does not establish accurate placement. diff --git a/scripts/bake_video_colours.py b/scripts/bake_video_colours.py index 8a45372..9e91b86 100644 --- a/scripts/bake_video_colours.py +++ b/scripts/bake_video_colours.py @@ -315,6 +315,57 @@ def zbuffer_depth(points: np.ndarray, c2w: np.ndarray, K: np.ndarray, size, res, return d.astype(np.float32), sup +def diagnostic_depth_ratios(pixel_indices, depths, source_depth, source_support, unmasked): + """Compare one generated front-center depth per supported source texel. + + Inputs are live, in-frustum generated centers projected into the source depth grid. + Visibility comes only from the generated depths, never agreement with the reference. + This is a nearest-center z-buffer approximation, not Gaussian opacity rendering: it + does not account for projected splat extent, transparency or subpixel coverage. + Retain the old all-center ratios separately so changed measurements are explainable. + """ + pixels = np.asarray(pixel_indices) + depths = np.asarray(depths) + source = np.asarray(source_depth).reshape(-1) + support = np.asarray(source_support, dtype=bool).reshape(-1) + unmasked = np.asarray(unmasked, dtype=bool) + if ( + pixels.ndim != 1 + or depths.shape != pixels.shape + or unmasked.shape != pixels.shape + or support.shape != source.shape + or not np.issubdtype(pixels.dtype, np.integer) + or np.any(pixels < 0) + or np.any(pixels >= source.size) + ): + raise ValueError("Invalid projected depth-diagnostic inputs") + if np.any(~np.isfinite(depths) | (depths <= 0)): + raise ValueError("Nonfinite or nonpositive generated depth in diagnostic support") + front = np.full(source.size, np.inf, dtype=np.float64) + np.minimum.at(front, pixels, depths) + # Mask the front sample, without revealing a hidden center behind a masked sample. + visible_unmasked = np.zeros(source.size, dtype=bool) + np.logical_or.at(visible_unmasked, pixels, unmasked & (depths == front[pixels])) + compared = visible_unmasked & support + if not compared.any(): + raise ValueError("No supported, unmasked front-depth overlap for diagnostic") + legacy_samples = support[pixels] & unmasked + referenced = source[pixels[legacy_samples]] + if np.any(~np.isfinite(referenced) | (referenced <= 0)): + raise ValueError("Nonfinite or nonpositive source depth in diagnostic support") + ratios = front[compared] / source[compared] + legacy_ratios = depths[legacy_samples] / referenced + if any(np.any(~np.isfinite(values) | (values <= 0)) for values in (ratios, legacy_ratios)): + raise ValueError("Nonfinite or nonpositive depth ratio in diagnostic support") + counts = { + "sourceSupportedPixels": int(support.sum()), + "generatedFrontPixels": int(np.isfinite(front).sum()), + "comparedPixels": int(compared.sum()), + "legacySplats": int(legacy_samples.sum()), + } + return ratios, legacy_ratios, counts + + class JsonSource: def __init__( self, @@ -625,6 +676,14 @@ def main(): sum_w = torch.zeros(n, device=dev) count = torch.zeros(n, device=dev, dtype=torch.int32) ratio_log = [] + legacy_ratio_log = [] + if a.diag: + print( + "depth diagnostic metric: front-center-per-supported-texel/1 " + "(nearest-center z-buffer, not Gaussian opacity rendering); " + "legacy all-splat statistics are diagnostic only", + flush=True, + ) frames = src.frames[:: a.stride] fig_frames = ( [int(x) for x in a.fig_frames.split(",")] @@ -653,6 +712,8 @@ def main(): vis = live & (z > src.near * 0.5) & (u >= 0) & (u < SW) & (v >= 0) & (v < SH_) idx = torch.nonzero(vis).squeeze(1) if idx.numel() == 0: + if a.diag: + raise ValueError(f"frame {f}: no live generated centers in diagnostic frustum") continue uu, vv, zz = u[idx], v[idx], z[idx] dmap, dsup = src.depth_map(f) @@ -676,11 +737,32 @@ def main(): ] r = zz / torch.clamp(md, 1e-4) if a.diag: - rr = r[sp & (mk < 0.5)] - lr = torch.log(rr).cpu().numpy() + rr, legacy, coverage = diagnostic_depth_ratios( + flat.cpu().numpy(), + zz.cpu().numpy(), + dmap, + dsup, + (mk < 0.5).cpu().numpy(), + ) + lr = np.log(rr) ratio_log.append((f, lr)) + legacy_lr = np.log(legacy) + legacy_ratio_log.append((f, legacy_lr)) print( - f"frame {f}: {idx.numel()} splats in frustum, {sp.float().mean().item() * 100:.0f}% on supported depth, log(splat/map depth) median {np.median(lr):+.3f} (ratio {np.exp(np.median(lr)):.3f}), p10 {np.exp(np.percentile(lr, 10)):.3f} p90 {np.exp(np.percentile(lr, 90)):.3f}", + f"frame {f}: {idx.numel()} splats in frustum; " + f"{coverage['comparedPixels']} compared rays / " + f"{coverage['sourceSupportedPixels']} source-supported texels, " + f"{coverage['generatedFrontPixels']} generated front texels; " + f"log(front-center/source depth) median {np.median(lr):+.3f} " + f"(ratio {np.exp(np.median(lr)):.3f}), " + f"p10 {np.exp(np.percentile(lr, 10)):.3f} p90 {np.exp(np.percentile(lr, 90)):.3f}", + flush=True, + ) + print( + f"frame {f} legacy all-splat diagnostic only: {coverage['legacySplats']} samples, " + f"medianRatio={np.exp(np.median(legacy_lr)):.3f}, " + f"p10={np.exp(np.percentile(legacy_lr, 10)):.3f}, " + f"p90={np.exp(np.percentile(legacy_lr, 90)):.3f}", flush=True, ) continue @@ -717,20 +799,34 @@ def main(): flush=True, ) + if a.diag and not ratio_log: + raise ValueError("No sampled frames produced a depth diagnostic") + import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt if a.diag: - fig, ax = plt.subplots(figsize=(8, 4)) + fig, (ax, legacy_ax) = plt.subplots(1, 2, figsize=(12, 4)) for f, lr in ratio_log: ax.hist(lr, bins=120, range=(-1.5, 1.5), histtype="step", label=f"frame {f}") - ax.axvline(0, color="k", lw=0.8) - ax.set_xlabel("log(Marble splat depth / source depth) (0 = agree)") - ax.legend() - ax.set_title( - f"{a.world.name}, scale0 {scale0:.3f}: Marble geometry vs source depth along the same rays" + for f, lr in legacy_ratio_log: + legacy_ax.hist( + lr, + bins=120, + range=(-1.5, 1.5), + histtype="step", + label=f"frame {f}", + ) + for axes in (ax, legacy_ax): + axes.axvline(0, color="k", lw=0.8) + axes.set_xlabel("log(generated depth / source depth) (0 = agree)") + axes.legend() + ax.set_title("Front center per supported ray") + legacy_ax.set_title("Legacy all-splat: diagnostic only") + fig.suptitle( + f"{a.world.name}, scale0 {scale0:.3f}: nearest-center depth (not opacity rendered)" ) fig.tight_layout() fig.savefig(a.share / f"{a.tag}-depth-ratio.png", dpi=110) diff --git a/scripts/run_clip.py b/scripts/run_clip.py index ada3d25..abb9eeb 100644 --- a/scripts/run_clip.py +++ b/scripts/run_clip.py @@ -2503,11 +2503,10 @@ def scale_fit(self): ) if wide and not self.a.allow_scale_spread: raise RuntimeError( - f"the depth ratio is not ONE mode at 1.0: per-frame p10/p90 {wide} at scale0 {s:.4f} " + f"depth ratios exceed the required spread: per-frame p10/p90 {wide} at scale0 {s:.4f} " f"(want both inside {SCALE_SPREAD_BAND[0]}-{SCALE_SPREAD_BAND[1]}; medians " - f"{[round(r, 3) for r in ratios]}). A median near 1 over a spread this wide is two " - f"populations averaged, not a scale -- hpwide read p10 0.71 / p90 1.37 with medians " - f"0.98-1.02 while its frame was 15 deg off gravity. Read {fig}, check " + f"{[round(r, 3) for r in ratios]}). A fitted median near 1 does not establish " + f"consistent depth across the visible surfaces. Read {fig}, check " f"{self.ctx / 'frame_align.log'} and the world's geometry before passing " f"--allow-scale-spread." ) diff --git a/scripts/tests/test_depth_diagnostic_visibility.py b/scripts/tests/test_depth_diagnostic_visibility.py new file mode 100644 index 0000000..2d5b7ef --- /dev/null +++ b/scripts/tests/test_depth_diagnostic_visibility.py @@ -0,0 +1,117 @@ +"""Depth comparisons use visible generated centers, never agreement-based filtering.""" + +import sys +from pathlib import Path + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from bake_video_colours import diagnostic_depth_ratios +from run_clip import SCALE_RATIO_BAND, SCALE_SPREAD_BAND + + +def compare(pixels, depths, source=None, support=None, unmasked=None): + source = np.ones(8) if source is None else np.asarray(source) + support = np.ones(source.shape, dtype=bool) if support is None else support + unmasked = np.ones(len(pixels), dtype=bool) if unmasked is None else unmasked + return diagnostic_depth_ratios(np.asarray(pixels, dtype=int), depths, source, support, unmasked) + + +@pytest.mark.parametrize("copies", [1, 20]) +def test_hidden_rear_geometry_and_density_do_not_change_front_comparison(copies): + pixels = [*range(8), *([0, 1] * copies)] + depths = [1] * 8 + [1.5] * (copies * 2) + front, legacy, counts = compare(pixels, depths) + np.testing.assert_array_equal(front, np.ones(8)) + assert np.percentile(legacy, 90) == 1.5 + assert np.median(legacy) == (1 if copies == 1 else 1.5) + assert counts == { + "sourceSupportedPixels": 8, + "generatedFrontPixels": 8, + "comparedPixels": 8, + "legacySplats": 8 + 2 * copies, + } + + +def test_revealing_a_wrong_rear_surface_still_fails_upper_spread_band(): + # Remove its front occluders: the same rear samples now really are the visible depth. + ratios, _, _ = compare(range(8), [1.5, 1.5, 1, 1, 1, 1, 1, 1]) + assert np.median(ratios) == 1 + assert np.percentile(ratios, 90) == 1.5 > SCALE_SPREAD_BAND[1] + + +def test_wrong_foreground_is_not_filtered_out_for_disagreeing_with_source(): + ratios, _, _ = compare([*range(8), 0, 1], [1] * 8 + [0.75, 0.75]) + assert np.median(ratios) == 1 + assert np.percentile(ratios, 10) == 0.75 < SCALE_SPREAD_BAND[0] + + +def test_global_visible_scale_error_still_fails_median_band(): + ratios, _, _ = compare(range(8), [1.25] * 8) + assert np.median(ratios) == 1.25 > SCALE_RATIO_BAND[1] + + +def test_duplicate_front_centers_do_not_reweight_image_rays(): + pixels = [*range(8), *([0] * 100)] + ratios, _, counts = compare(pixels, [1, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5] + [1] * 100) + assert len(ratios) == counts["comparedPixels"] == 8 + assert np.median(ratios) == 1.5 + + +def test_unsupported_and_person_masked_rays_are_excluded_with_coverage_counts(): + ratios, legacy, counts = compare( + [0, 1, 2, 3], + [1, 5, 6, 2], + source=[1, np.nan, 1, 2], + support=[True, False, True, True], + unmasked=[True, True, False, True], + ) + np.testing.assert_array_equal(ratios, [1, 1]) + np.testing.assert_array_equal(legacy, [1, 1]) + assert counts["comparedPixels"] == 2 + assert counts["sourceSupportedPixels"] == 3 + assert counts["generatedFrontPixels"] == 4 + + +def test_masking_a_front_center_does_not_reveal_its_hidden_background(): + ratios, legacy, counts = compare([0, 0, 1], [1, 1.5, 1], unmasked=[False, True, True]) + np.testing.assert_array_equal(ratios, [1]) + np.testing.assert_array_equal(legacy, [1.5, 1]) + assert counts["comparedPixels"] == 1 + + +@pytest.mark.parametrize("case", ["empty-world", "unsupported", "masked", "disjoint"]) +def test_no_comparable_ray_fails_instead_of_producing_a_pass(case): + pixels, depths, support, unmasked = [0], [1], [True, False], [True] + if case == "empty-world": + pixels, depths, unmasked = [], [], [] + elif case == "unsupported": + support = [False, False] + elif case == "masked": + unmasked = [False] + else: + pixels = [1] + with pytest.raises(ValueError, match="No supported, unmasked front-depth overlap"): + compare(pixels, depths, source=[1, 1], support=support, unmasked=unmasked) + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), 0, -1]) +@pytest.mark.parametrize("side", ["source", "generated"]) +def test_invalid_compared_depth_fails_clearly(value, side): + with pytest.raises(ValueError, match=f"Nonfinite or nonpositive {side} depth"): + compare( + [0], [value if side == "generated" else 1], source=[value if side == "source" else 1] + ) + + +def test_report_counts_expose_partial_overlap_without_inventing_a_coverage_threshold(): + ratios, _, counts = compare([0], [1]) + np.testing.assert_array_equal(ratios, [1]) + assert counts["comparedPixels"] == 1 + assert counts["sourceSupportedPixels"] == 8 + + +def test_frozen_acceptance_bands_are_unchanged(): + assert SCALE_RATIO_BAND == (0.95, 1.05) + assert SCALE_SPREAD_BAND == (0.9, 1.1)