From 25fcefc320f57f7b20c87a11614adfbd24e0616d Mon Sep 17 00:00:00 2001 From: Moshu <1709219+DatMoshu@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:11:20 -0500 Subject: [PATCH 1/5] Bound generated storage and validate external checkout startup --- common/storage.py | 228 ++++++++++++++++++++++++ common/storage_protection.py | 161 +++++++++++++++++ common/workpaths.py | 21 ++- launchers/README.md | 4 +- launchers/_shared/common.bat | 10 +- launchers/_shared/common.sh | 36 ++-- launchers/_shared/config.bat | 5 + launchers/_shared/config.sh | 5 + launchers/dev/check-storage.bat | 6 + launchers/dev/check-storage.sh | 6 + launchers/editor/content-studio.ps1 | 12 +- launchers/editor/female-live-pose.ps1 | 9 +- launchers/pipeline/0-setup.bat | 8 +- launchers/pipeline/0-setup.sh | 10 +- tests/unit/test_client_staging.py | 2 +- tests/unit/test_fit_rebuild.py | 6 +- tests/unit/test_storage.py | 238 ++++++++++++++++++++++++++ tools/fit-lab/export_blender.py | 8 +- tools/godot/fetch.py | 25 ++- tools/storage/README.md | 66 +++++++ tools/storage/run.py | 46 +++++ tools/uo-content/client_import.py | 21 ++- tools/uo-content/equipment.py | 10 +- tools/uo-content/pipeline.py | 80 +++++++-- tools/uo-content/rebuild.py | 21 ++- tools/uo-content/studio.py | 11 +- tools/whole-outfit/run.py | 14 +- tools/worktree-venv/run.py | 25 ++- 28 files changed, 1017 insertions(+), 77 deletions(-) create mode 100644 common/storage.py create mode 100644 common/storage_protection.py create mode 100644 launchers/dev/check-storage.bat create mode 100755 launchers/dev/check-storage.sh create mode 100644 tests/unit/test_storage.py create mode 100644 tools/storage/README.md create mode 100644 tools/storage/run.py diff --git a/common/storage.py b/common/storage.py new file mode 100644 index 0000000..5e2e2dc --- /dev/null +++ b/common/storage.py @@ -0,0 +1,228 @@ +"""Bounded content-job storage; only explicitly regenerable derivatives expire.""" +from contextlib import contextmanager +import importlib.util +import json +import math +import os +from pathlib import Path +import re +import shutil +import subprocess +import time + +STORAGE_GUARD_VERSION = 1 +DERIVED = ('render', 'review', 'original-frames.json', 'item.vd', + 'import-package.zip', 'contact-sheet.png', 'inventory.png', + 'slot-preview.png', 'build.log') + + +def sibling(name): + spec = importlib.util.spec_from_file_location('_storage_' + name, Path(__file__).with_name(name + '.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +paths = sibling('workpaths') + + +class StorageLimit(ValueError): + pass + + +class Policy: + def __init__(self, max_bytes=None, days=None, keep=None): + gb = float(os.environ.get('SPRITEMOTION_MAX_WORK_GB', '5')) + self.max_bytes = max_bytes if max_bytes is not None else int(gb * 1_000_000_000) + self.days = days if days is not None else float(os.environ.get('SPRITEMOTION_OUTPUT_DAYS', '7')) + self.keep = keep if keep is not None else int(os.environ.get('SPRITEMOTION_KEEP_OUTPUT_JOBS', '10')) + if not math.isfinite(gb) or self.max_bytes <= 0 or not math.isfinite(self.days) or self.days <= 0 or self.keep < 1: + raise ValueError('Storage limits must be positive; keep at least one completed job.') + + +def usage(root): + """Logical bytes in the physical store; do not follow links or count hardlinks twice.""" + total = 0 + seen = set() + for parent, dirs, files in os.walk(root): + dirs[:] = [d for d in dirs if not linked(Path(parent, d))] + for name in files: + p = Path(parent, name) + if linked(p): + continue + s = p.stat() + key = (s.st_dev, s.st_ino) + if key not in seen: + total += s.st_size + seen.add(key) + return total + + +def linked(path): + return path.is_symlink() or getattr(path, 'is_junction', lambda: False)() + + +def protected_guard(candidates): + # Reuse fleet protection when installed; the identical portable fallback is + # shipped for users who do not have the fleet. Never bypass protected names. + fleet = Path(os.environ.get('ORG_ROOT', 'D:/.projects/.Org')) / 'tools/evidence/protected.py' + if fleet.is_file(): + spec = importlib.util.spec_from_file_location('_storage_fleet_protected', fleet) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + else: + module = sibling('storage_protection') + module.guard(candidates, tool='spritemotion-generated-retention') + + +def derivative_paths(job, initial_scene=False): + candidates = [job / name for name in DERIVED if (job / name).exists()] + if initial_scene and (job / 'item.blend').exists(): + candidates.append(job / 'item.blend') + for p in candidates: + if p.parent != job or linked(p): + raise ValueError('Linked generated output refused: ' + str(p)) + if p.is_dir(): + for parent, dirs, files in os.walk(p): + if any(linked(Path(parent, n)) for n in dirs + files): + raise ValueError('Linked generated output contents refused: ' + str(p)) + if any(Path(n).suffix.lower() not in {'.png', '.json', '.html'} for n in files): + raise ValueError('Unknown file in generated derivatives; retained for inspection: ' + str(p)) + return candidates + + +def delete_paths(candidates): + for p in candidates: + if p.is_dir(): + shutil.rmtree(p) + else: + p.unlink() + return [str(p) for p in candidates] + + +def remove_derived(job, initial_scene=False): + candidates = derivative_paths(job, initial_scene) + protected_guard(candidates) + return delete_paths(candidates) + + +@contextmanager +def pin(job): + """Keep an immutable rebuild source alive across patch/render/merge sessions.""" + marker = Path(job) / '.storage.pin' + try: + fd = os.open(marker, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except FileExistsError: + raise ValueError('This source job is already pinned by another rebuild.') + try: + os.write(fd, str(os.getpid()).encode()) + yield + finally: + os.close(fd) + marker.unlink() + + +class Store: + def __init__(self, home, policy=None): + self.home = paths.external_path(Path(home)) + self.policy = policy or Policy() + + def require(self, extra=0): + size = usage(self.home) + if size + extra > self.policy.max_bytes: + raise StorageLimit(f'Generated storage limit reached ({size + extra} > {self.policy.max_bytes} bytes). ' + 'Run tools/storage/run.py prune, remove obsolete derivatives, or select another external ' + 'SPRITEMOTION_CONTENT_HOME. Inputs and editable scenes are never expired. ' + 'SPRITEMOTION_MAX_WORK_GB sets the limit.') + return size + + def prune(self, protect=()): + protected = {Path(p).resolve() for p in protect} + jobs = [] + if linked(self.home / 'jobs'): + raise ValueError('Linked jobs directory refused; retention never follows links.') + for p in (self.home / 'jobs').glob('*/status.json'): + if not re.fullmatch('[a-f0-9]{12}', p.parent.name) or linked(p.parent) or linked(p) or p.parent.resolve() in protected or (p.parent / '.keep').exists() or (p.parent / '.storage.pin').exists(): + continue + try: + status = json.loads(p.read_text(encoding='utf-8')) + except (OSError, ValueError): + continue # Unknown or corrupt state is not proof of obsolescence. + if isinstance(status, dict) and status.get('state') in ('complete', 'failed'): + jobs.append((p.stat().st_mtime, p, status)) + plans = [] + for index, (stamp, p, status) in enumerate(sorted(jobs, reverse=True)): + if index < self.policy.keep or time.time() - stamp < self.policy.days * 86400: + continue + plans.append((p, status, derivative_paths(p.parent))) + protected_guard([q for p, status, candidates in plans for q in candidates]) + removed = [] + for p, status, candidates in plans: + deleted = delete_paths(candidates) + if deleted: + status.update(state='failed', error='Generated derivatives expired; retained inputs/settings/scene can rebuild.') + p.write_text(json.dumps(status, indent=2), encoding='utf-8') + removed.extend(deleted) + return removed + + @contextmanager + def session(self, protect=()): + self.home.mkdir(parents=True, exist_ok=True) + lock = self.home / '.storage.lock' + try: + fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except FileExistsError: + raise ValueError('Another content build/cleanup holds .storage.lock. Wait; after a crash verify no build is active before removing the lock.') + try: + os.write(fd, str(os.getpid()).encode()) + self.prune(protect) + self.require() + yield self + finally: + os.close(fd) + lock.unlink() + + def process(self, command, job, **kwargs): + """Stop the child on overflow; polling may allow one write interval of overshoot.""" + self.require() + child = subprocess.Popen(command, **kwargs) + try: + while child.poll() is None: + time.sleep(.25) + self.require() + self.require() + return child.returncode + except BaseException: + if child.poll() is None: + child.terminate() + try: + child.wait(timeout=5) + except subprocess.TimeoutExpired: + child.kill() + child.wait() + raise + + +def startup(checkout, python=None, paths_only=False): + checkout = Path(checkout).resolve() + module = checkout / 'common/storage.py' + if not module.is_file() or 'STORAGE_GUARD_VERSION = 1' not in module.read_text(encoding='utf-8'): + raise ValueError('Historical checkout lacks storage guards. Update/rebase it before resuming; no repository-local fallback is allowed.') + for name in ('common.bat', 'common.sh'): + text = (checkout / 'launchers/_shared' / name).read_text(encoding='utf-8').replace('\\', '/') + if 'tools/storage/run.py' not in text: + raise ValueError('Historical checkout launcher lacks storage startup validation: ' + name) + work, content = paths.work_root(), paths.content_home() + paths.external_path(paths.backend_home(), checkout) + paths.external_path(work, checkout) + paths.external_path(content, checkout) + if not paths_only: + if not python: + python = paths.checkout_venv(checkout) / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python') + exe = Path(python).resolve() + paths.external_path(exe, checkout) + result = subprocess.run([str(exe), '-c', 'import spritemotion; print(spritemotion.__file__)'], + cwd=checkout.parent, capture_output=True, text=True) + if result.returncode or Path(result.stdout.strip()).resolve().parent != checkout / 'common': + raise ValueError('Python does not import this checkout. Run its worktree-venv launcher or set SPRITEMOTION_PYTHON to its external environment.') + return dict(work_root=str(work), content_home=str(content), max_bytes=Policy().max_bytes) diff --git a/common/storage_protection.py b/common/storage_protection.py new file mode 100644 index 0000000..4b5b054 --- /dev/null +++ b/common/storage_protection.py @@ -0,0 +1,161 @@ +"""Protected-path guard for every purge or cleanup that touches D:/_agentShared (rule: evidence_central.md). + +A path is protected when any component, case-insensitive, contains one of PROTECTED_MARKERS, or when a file +starts with a PEM private-key or DPAPI blob header. Tools import this module instead of copying the list: + + import sys; sys.path.insert(0, 'D:/.projects/.Org/tools/evidence') + from protected import guard, safe_rmtree, safe_unlink, ProtectedPathError + +guard(paths) scans every planned path (and everything under planned folders) and raises ProtectedPathError +before anything is deleted, so a plan with one protected entry deletes nothing. Standard library only. +CLI: `python protected.py check PATH...` (exit 2 on a hit) and `python protected.py scan ROOT [--json]`. +""" +import argparse +import datetime as dt +import json +import os +from pathlib import Path +import shutil +import sys + +RULE_MARKERS = ('-private', 'private', 'credential', 'secret', 'token', '.key', 'dpapi', 'keystore') +# The 2026-10-18 damage audit found servermanager/ssh-key-backup-20261013 and keepassxc-nightly deleted unflagged. +EXTRA_MARKERS = ('ssh-key', 'keepass', '.kdbx', 'id_rsa', 'id_ed25519', '.pfx', '.p12', '.clixml') +PROTECTED_MARKERS = RULE_MARKERS + EXTRA_MARKERS +# CryptProtectData blobs start with a version dword and the DPAPI provider GUID df9d8cd0-1501-11d1-8c7a-00c04fc297eb. +DPAPI_HEADER = bytes.fromhex('01000000d08c9ddf0115d1118c7a00c04fc297eb') +PEM_PRIVATE = (b'-----BEGIN PRIVATE KEY', b'-----BEGIN RSA PRIVATE KEY', b'-----BEGIN EC PRIVATE KEY', + b'-----BEGIN OPENSSH PRIVATE KEY', b'-----BEGIN ENCRYPTED PRIVATE KEY', b'-----BEGIN DSA PRIVATE KEY') +SNIFF_BYTES = 4096 + + +class ProtectedPathError(ValueError): + def __init__(self, hits): + self.hits = hits + super().__init__(f'{len(hits)} protected path(s) refused, nothing deleted: ' + '; '.join(f"{h['path']} ({h['reason']})" for h in hits[:5])) + + +def name_reason(path, root=None): + """Return the marker that protects path by name, checking every component (below root when given).""" + path = Path(path) + if root is not None: + try: + path = Path(os.path.abspath(path)).relative_to(os.path.abspath(root)) + except ValueError: + pass + for part in path.parts: + lower = part.lower() + for marker in PROTECTED_MARKERS: + if marker in lower: + return f'name contains {marker!r} in {part!r}' + return None + + +def content_reason(path): + """Return a reason when a regular file starts like a DPAPI blob or carries a PEM private key.""" + try: + with open(path, 'rb') as stream: + head = stream.read(SNIFF_BYTES) + except OSError: + return None + if head.startswith(DPAPI_HEADER): + return 'DPAPI blob header' + # DPAPI output is often stored base64-encoded; AQAAANCMnd8BFdERjHoAwE/Cl+s is the encoded header. + if head.lstrip(b'\xef\xbb\xbf"\' \r\n').startswith(b'AQAAANCMnd8BFdERjHoAwE/Cl+s'): + return 'base64 DPAPI blob header' + for marker in PEM_PRIVATE: + if marker in head: + return 'PEM private key header' + return None + + +def reason(path, root=None, content=True): + path = Path(path) + found = name_reason(path, root) + if found is None and content and path.is_file() and not path.is_symlink(): + found = content_reason(path) + return found + + +def scan(paths, root=None, content=True): + """List every protected path at or below the given paths, without following links.""" + hits = [] + for top in [Path(p) for p in ([paths] if isinstance(paths, (str, os.PathLike)) else paths)]: + found = reason(top, root, content) + if found: + hits.append(dict(path=str(top), reason=found)) + continue + if top.is_dir() and not top.is_symlink(): + for parent, dirs, files in os.walk(top, followlinks=False): + keep = [] + for name in dirs: + found = name_reason(Path(parent) / name, root) + if found: + hits.append(dict(path=str(Path(parent) / name), reason=found)) + else: + keep.append(name) + dirs[:] = keep + for name in files: + found = reason(Path(parent) / name, root, content) + if found: + hits.append(dict(path=str(Path(parent) / name), reason=found)) + return hits + + +def log_skips(hits, log=None, tool=None, dry=False): + """Print each skip to stderr and append it to a JSONL log when one is given.""" + stamp = dt.datetime.now(dt.timezone.utc).isoformat(timespec='seconds').replace('+00:00', 'Z') + for hit in hits: + print(f"protected: skip {hit['path']} ({hit['reason']})", file=sys.stderr) + if log and hits: + log = Path(log) + log.parent.mkdir(parents=True, exist_ok=True) + with log.open('a', encoding='utf-8') as stream: + for hit in hits: + stream.write(json.dumps(dict(time=stamp, tool=tool, dry=dry, **hit)) + '\n') + + +def guard(paths, root=None, log=None, tool=None, dry=False, content=True): + """Raise ProtectedPathError (after logging every hit) if any planned path is or contains a protected path.""" + hits = scan(paths, root, content) + if hits: + log_skips(hits, log, tool, dry) + raise ProtectedPathError(hits) + return [] + + +def safe_rmtree(path, root=None, log=None, tool=None): + guard([path], root, log, tool) + shutil.rmtree(path) + + +def safe_unlink(path, root=None, log=None, tool=None, missing_ok=False): + path = Path(path) + if missing_ok and not path.exists(): + return + guard([path], root, log, tool) + path.unlink() + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + sub = parser.add_subparsers(dest='cmd', required=True) + p = sub.add_parser('check', help='Exit 2 if any path is or contains a protected path') + p.add_argument('paths', nargs='+'); p.add_argument('--root') + p = sub.add_parser('scan', help='List protected paths under ROOT') + p.add_argument('root'); p.add_argument('--json', action='store_true') + args = parser.parse_args(argv) + if args.cmd == 'check': + hits = scan(args.paths, args.root) + else: + hits = scan([args.root], args.root) + if getattr(args, 'json', False): + print(json.dumps(hits, indent=2)) + else: + for hit in hits: + print(f"{hit['path']}\t{hit['reason']}") + return 2 if hits else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/common/workpaths.py b/common/workpaths.py index 39a4668..de1b8dc 100644 --- a/common/workpaths.py +++ b/common/workpaths.py @@ -4,14 +4,31 @@ import hashlib +def external_path(path: Path, checkout: Path | None = None) -> Path: + """Reject physical output/environment locations inside any Git checkout.""" + path = Path(path).expanduser().resolve() + checkout = (checkout or Path(__file__).resolve().parents[1]).resolve() + if path == checkout or checkout in path.parents: + raise ValueError('Generated storage must be outside the checkout. Set SPRITEMOTION_WORK_ROOT / SPRITEMOTION_CONTENT_HOME.') + for parent in (path, *path.parents): + if (parent / '.git').exists(): + raise ValueError('Generated storage must be outside every Git checkout: ' + str(path)) + return path + + def work_root() -> Path: """Environment override, then the fleet shared work area (portable off Windows).""" default = 'D:/_agentShared/work/spritemotion' if os.name == 'nt' else '~/.cache/spritemotion' - return Path(os.environ.get('SPRITEMOTION_WORK_ROOT') or default).expanduser().resolve() + return external_path(Path(os.environ.get('SPRITEMOTION_WORK_ROOT') or default)) def content_home() -> Path: - return Path(os.environ.get('SPRITEMOTION_CONTENT_HOME') or work_root() / 'content-studio').expanduser().resolve() + return external_path(Path(os.environ.get('SPRITEMOTION_CONTENT_HOME') or work_root() / 'content-studio')) + + +def backend_home() -> Path: + """One external canonical input installation, shared by all checkouts.""" + return external_path(Path(os.environ.get('SPRITEMOTION_BACKEND') or work_root() / 'inputs/canonical-model')) def checkout_venv(checkout: Path) -> Path: diff --git a/launchers/README.md b/launchers/README.md index 628a976..8ce6374 100644 --- a/launchers/README.md +++ b/launchers/README.md @@ -36,7 +36,7 @@ Content tools (Content Studio, Fit Lab, the workbench) are described in [editor/ | Launcher | What it does | Arguments | |---|---|---| -| `pipeline/0-setup` | Create .venvs\spritemotion, install SpriteMotion (editable, with test extras) and fetch the tested Godot build into tools\godot unless SPRITEMOTION_GODOT is set. | | +| `pipeline/0-setup` | Create an external checkout environment and install the tested Godot build in shared storage unless SPRITEMOTION_GODOT is set. | | | `pipeline/1-extract-uo` | Extract a character from YOUR UO client into workspace\ and apply the bundled annotations, then print its status. | `[character] (default: body-400)` | | `pipeline/2-status` | Print annotation coverage, review state and fingerprint mismatches for a dataset (exit 1 when it does not validate). | `[dataset] (default: SPRITEMOTION_DATASET)` | | `pipeline/3-fit` | Export your model's rig from Blender, then fit one sequence to the dataset's annotations (writes workspace\fits\rig.json and workspace\fits\.json). | ` [approved\|independent\|all]` | @@ -70,6 +70,8 @@ Content tools (Content Studio, Fit Lab, the workbench) are described in [editor/ | `dev/worktree-venv` | Build a shared-area venv for this checkout, install it editable with the test extras, and verify its import path. | `[--rebuild] (default: reuse the venv and refresh the editable install)` | | `dev/whole-outfit` | Build a whole outfit from a Fit Lab catalogue and composite contact sheets in client layer order (local output, prints a per-slot table). | `build, body or composite ... (see tools\whole-outfit\run.py --help)` | +| `dev/check-storage` | Validate external storage and the matching Python environment before starting this or a historical checkout. | `[--checkout PATH] [--python PATH] [--paths-only]` | + ## Not launchers - Fit Lab's poke **Measure** runs in the browser (there is no command line for it), so it has no launcher: start `editor/fit-lab` and press Measure. diff --git a/launchers/_shared/common.bat b/launchers/_shared/common.bat index f23428f..293b595 100644 --- a/launchers/_shared/common.bat +++ b/launchers/_shared/common.bat @@ -3,13 +3,17 @@ @echo off for %%I in ("%~dp0..\..") do set "SM_ROOT=%%~fI" call "%~dp0config.bat" +python "%SM_ROOT%\tools\storage\run.py" startup --checkout "%SM_ROOT%" --paths-only >nul || exit /b 1 -@rem Prefer the shared-area venv keyed to THIS checkout, then legacy environments. +@rem Use only the external environment keyed to THIS checkout or an explicit override. set "SM_CHECKOUT_VENV=" for /f "delims=" %%V in ('python "%SM_ROOT%\tools\worktree-venv\run.py" --path 2^>nul') do set "SM_CHECKOUT_VENV=%%V" if not defined SPRITEMOTION_PYTHON if defined SM_CHECKOUT_VENV if exist "%SM_CHECKOUT_VENV%\Scripts\python.exe" set "SPRITEMOTION_PYTHON=%SM_CHECKOUT_VENV%\Scripts\python.exe" -if not defined SPRITEMOTION_PYTHON if exist "%SM_ROOT%\workspace\venv\Scripts\python.exe" set "SPRITEMOTION_PYTHON=%SM_ROOT%\workspace\venv\Scripts\python.exe" -if not defined SPRITEMOTION_PYTHON set "SPRITEMOTION_PYTHON=%SM_ROOT%\.venvs\spritemotion\Scripts\python.exe" +if not defined SPRITEMOTION_PYTHON if defined SM_CHECKOUT_VENV set "SPRITEMOTION_PYTHON=%SM_CHECKOUT_VENV%\Scripts\python.exe" +if exist "%SPRITEMOTION_PYTHON%" "%SPRITEMOTION_PYTHON%" "%SM_ROOT%\tools\storage\run.py" startup --checkout "%SM_ROOT%" --python "%SPRITEMOTION_PYTHON%" >nul || exit /b 1 +for /f "delims=" %%G in ('python "%SM_ROOT%\tools\storage\run.py" godot-path') do set "SM_GODOT_CACHE=%%G" +if not defined SPRITEMOTION_GODOT if exist "%SM_GODOT_CACHE%\Godot_v4.7-stable_win64.exe" set "SPRITEMOTION_GODOT=%SM_GODOT_CACHE%\Godot_v4.7-stable_win64.exe" +if not defined SPRITEMOTION_GODOT_CONSOLE if exist "%SM_GODOT_CACHE%\Godot_v4.7-stable_win64_console.exe" set "SPRITEMOTION_GODOT_CONSOLE=%SM_GODOT_CACHE%\Godot_v4.7-stable_win64_console.exe" if not defined SPRITEMOTION_GODOT if exist "%SM_ROOT%\tools\godot\Godot_v4.7-stable_win64.exe" set "SPRITEMOTION_GODOT=%SM_ROOT%\tools\godot\Godot_v4.7-stable_win64.exe" if not defined SPRITEMOTION_GODOT_CONSOLE if exist "%SM_ROOT%\tools\godot\Godot_v4.7-stable_win64_console.exe" set "SPRITEMOTION_GODOT_CONSOLE=%SM_ROOT%\tools\godot\Godot_v4.7-stable_win64_console.exe" if not defined SPRITEMOTION_GODOT_CONSOLE set "SPRITEMOTION_GODOT_CONSOLE=%SPRITEMOTION_GODOT%" diff --git a/launchers/_shared/common.sh b/launchers/_shared/common.sh index b4379c1..6f18a71 100755 --- a/launchers/_shared/common.sh +++ b/launchers/_shared/common.sh @@ -4,7 +4,7 @@ SM_ROOT=$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd) _sm_here="$SM_ROOT/launchers/_shared" -_sm_keys="SPRITEMOTION_UO_SOURCE SPRITEMOTION_DATASET SPRITEMOTION_GODOT SPRITEMOTION_GODOT_CONSOLE SPRITEMOTION_BLENDER SPRITEMOTION_BLEND SPRITEMOTION_ARMATURE SPRITEMOTION_MAPPING SPRITEMOTION_FRAME_START SPRITEMOTION_FRAME_STEP SPRITEMOTION_PYTHON SPRITEMOTION_WORK_ROOT SPRITEMOTION_CONTENT_HOME" +_sm_keys="SPRITEMOTION_UO_SOURCE SPRITEMOTION_DATASET SPRITEMOTION_GODOT SPRITEMOTION_GODOT_CONSOLE SPRITEMOTION_BLENDER SPRITEMOTION_BLEND SPRITEMOTION_ARMATURE SPRITEMOTION_MAPPING SPRITEMOTION_FRAME_START SPRITEMOTION_FRAME_STEP SPRITEMOTION_PYTHON SPRITEMOTION_WORK_ROOT SPRITEMOTION_CONTENT_HOME SPRITEMOTION_BACKEND SPRITEMOTION_MAX_WORK_GB SPRITEMOTION_OUTPUT_DAYS SPRITEMOTION_KEEP_OUTPUT_JOBS" # Remember what the environment already says (a blank value counts as unset, like in common.bat). _sm_env="" @@ -16,15 +16,11 @@ done [ -f "$_sm_here/config.local.sh" ] && . "$_sm_here/config.local.sh" for _k in $_sm_env; do eval "$_k=\$_sm_saved_$_k"; done -sm_find_venv_python() { - local c - for c in "$SM_ROOT/.venvs/spritemotion/bin/python" "$SM_ROOT/.venvs/spritemotion/Scripts/python.exe"; do - if [ -x "$c" ]; then printf '%s\n' "$c"; return 0; fi - done - return 1 -} -SM_VENV_PYTHON=$(sm_find_venv_python || true) -export SPRITEMOTION_WORK_ROOT SPRITEMOTION_CONTENT_HOME +export SPRITEMOTION_WORK_ROOT SPRITEMOTION_CONTENT_HOME SPRITEMOTION_BACKEND +export SPRITEMOTION_MAX_WORK_GB SPRITEMOTION_OUTPUT_DAYS SPRITEMOTION_KEEP_OUTPUT_JOBS +_sm_bootstrap=$(command -v python3 || command -v python || true) +if [ -z "$_sm_bootstrap" ]; then echo '[SpriteMotion] Python is required for external-storage validation.' >&2; return 1; fi +"$_sm_bootstrap" "$SM_ROOT/tools/storage/run.py" startup --checkout "$SM_ROOT" --paths-only >/dev/null || return 1 if [ -z "${SPRITEMOTION_PYTHON:-}" ]; then _sm_bootstrap=$(command -v python3 || command -v python || true) if [ -n "$_sm_bootstrap" ]; then @@ -35,16 +31,18 @@ if [ -z "${SPRITEMOTION_PYTHON:-}" ]; then fi fi if [ -z "${SPRITEMOTION_PYTHON:-}" ]; then - # workspace/venv (launchers/dev/worktree-venv) tests THIS checkout; .venvs/spritemotion is main's editable install. - for _c in "$SM_ROOT/workspace/venv/bin/python" "$SM_ROOT/workspace/venv/Scripts/python.exe"; do - if [ -x "$_c" ]; then SPRITEMOTION_PYTHON=$_c; break; fi - done + SPRITEMOTION_PYTHON="$SM_CHECKOUT_VENV/bin/python" fi -if [ -z "${SPRITEMOTION_PYTHON:-}" ]; then - if [ -n "$SM_VENV_PYTHON" ]; then SPRITEMOTION_PYTHON=$SM_VENV_PYTHON - else SPRITEMOTION_PYTHON=$(command -v python3 || command -v python || true); fi +if [ -x "$SPRITEMOTION_PYTHON" ]; then + "$SPRITEMOTION_PYTHON" "$SM_ROOT/tools/storage/run.py" startup --checkout "$SM_ROOT" --python "$SPRITEMOTION_PYTHON" >/dev/null || return 1 fi +if [ -z "${SPRITEMOTION_GODOT:-}" ]; then + SM_GODOT_CACHE=$("$_sm_bootstrap" "$SM_ROOT/tools/storage/run.py" godot-path) + for _c in "$SM_GODOT_CACHE/Godot_v4.7-stable_win64.exe" "$SM_GODOT_CACHE/Godot_v4.7-stable_linux.x86_64" "$SM_GODOT_CACHE/Godot.app/Contents/MacOS/Godot"; do + if [ -x "$_c" ]; then SPRITEMOTION_GODOT=$_c; break; fi + done +fi if [ -z "${SPRITEMOTION_GODOT:-}" ]; then for _c in "$SM_ROOT/tools/godot/Godot_v4.7-stable_win64.exe" "$SM_ROOT/tools/godot/Godot_v4.7-stable_linux.x86_64" \ "$SM_ROOT/tools/godot/Godot.app/Contents/MacOS/Godot"; do @@ -52,7 +50,9 @@ if [ -z "${SPRITEMOTION_GODOT:-}" ]; then done fi if [ -z "${SPRITEMOTION_GODOT_CONSOLE:-}" ]; then - if [ -x "$SM_ROOT/tools/godot/Godot_v4.7-stable_win64_console.exe" ]; then + if [ -x "${SM_GODOT_CACHE:-}/Godot_v4.7-stable_win64_console.exe" ]; then + SPRITEMOTION_GODOT_CONSOLE="$SM_GODOT_CACHE/Godot_v4.7-stable_win64_console.exe" + elif [ -x "$SM_ROOT/tools/godot/Godot_v4.7-stable_win64_console.exe" ]; then SPRITEMOTION_GODOT_CONSOLE="$SM_ROOT/tools/godot/Godot_v4.7-stable_win64_console.exe" else SPRITEMOTION_GODOT_CONSOLE=${SPRITEMOTION_GODOT:-}; fi fi diff --git a/launchers/_shared/config.bat b/launchers/_shared/config.bat index c60d39b..a0bdcf4 100644 --- a/launchers/_shared/config.bat +++ b/launchers/_shared/config.bat @@ -22,3 +22,8 @@ if not defined SPRITEMOTION_MAPPING set "SPRITEMOTION_MAPPING=games\ultima-onlin @rem Blender timeline: source frame f is keyed at FRAME_START + f * FRAME_STEP. if not defined SPRITEMOTION_FRAME_START set "SPRITEMOTION_FRAME_START=1" if not defined SPRITEMOTION_FRAME_STEP set "SPRITEMOTION_FRAME_STEP=4" + +@rem Content-store limits: positive values; inputs and editable scenes never expire. +if not defined SPRITEMOTION_MAX_WORK_GB set "SPRITEMOTION_MAX_WORK_GB=5" +if not defined SPRITEMOTION_OUTPUT_DAYS set "SPRITEMOTION_OUTPUT_DAYS=7" +if not defined SPRITEMOTION_KEEP_OUTPUT_JOBS set "SPRITEMOTION_KEEP_OUTPUT_JOBS=10" diff --git a/launchers/_shared/config.sh b/launchers/_shared/config.sh index 42f19b7..2fee02d 100755 --- a/launchers/_shared/config.sh +++ b/launchers/_shared/config.sh @@ -20,3 +20,8 @@ SPRITEMOTION_MAPPING="games/ultima-online/skeletons/rig-mappings/rigify.json" # Blender timeline: source frame f is keyed at FRAME_START + f * FRAME_STEP. SPRITEMOTION_FRAME_START=1 SPRITEMOTION_FRAME_STEP=4 + +# Content-store limits. Inputs and editable scenes never expire. +SPRITEMOTION_MAX_WORK_GB=5 +SPRITEMOTION_OUTPUT_DAYS=7 +SPRITEMOTION_KEEP_OUTPUT_JOBS=10 diff --git a/launchers/dev/check-storage.bat b/launchers/dev/check-storage.bat new file mode 100644 index 0000000..063b6a3 --- /dev/null +++ b/launchers/dev/check-storage.bat @@ -0,0 +1,6 @@ +@echo off +rem Validate external storage and the matching Python environment before starting this or a historical checkout. +rem args: [--checkout PATH] [--python PATH] [--paths-only] +call "%~dp0..\_shared\common.bat" || exit /b 1 +python "%SM_ROOT%\tools\storage\run.py" startup %* +exit /b %errorlevel% diff --git a/launchers/dev/check-storage.sh b/launchers/dev/check-storage.sh new file mode 100755 index 0000000..03cd042 --- /dev/null +++ b/launchers/dev/check-storage.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Validate external storage and the matching Python environment before starting this or a historical checkout. +# args: [--checkout PATH] [--python PATH] [--paths-only] +set -euo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/../_shared/common.sh" +exec "$_sm_bootstrap" "$SM_ROOT/tools/storage/run.py" startup "$@" diff --git a/launchers/editor/content-studio.ps1 b/launchers/editor/content-studio.ps1 index 9b01b52..833da61 100644 --- a/launchers/editor/content-studio.ps1 +++ b/launchers/editor/content-studio.ps1 @@ -1,12 +1,14 @@ $ErrorActionPreference = 'Stop' $studioRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path +$bootstrap = (Get-Command python -ErrorAction Stop).Source +& $bootstrap (Join-Path $studioRoot 'tools/storage/run.py') startup --checkout $studioRoot --paths-only +if ($LASTEXITCODE -ne 0) { throw 'External storage validation failed.' } +$checkoutVenv = & $bootstrap (Join-Path $studioRoot 'tools/worktree-venv/run.py') --path +if ($LASTEXITCODE -ne 0) { throw 'Cannot resolve external checkout environment.' } $candidates = @( $env:SPRITEMOTION_PYTHON, - (Join-Path $studioRoot '.venvs/spritemotion/Scripts/python.exe'), - (Join-Path $env:USERPROFILE '.cache/codex-runtimes/codex-primary-runtime/dependencies/python/python.exe') + (Join-Path $checkoutVenv 'Scripts/python.exe') ) -$pythonCommand = Get-Command python -ErrorAction SilentlyContinue -if ($pythonCommand) { $candidates += $pythonCommand.Source } $studioPython = $null foreach ($candidate in $candidates) { if ($candidate -and (Test-Path -LiteralPath $candidate)) { @@ -15,6 +17,8 @@ foreach ($candidate in $candidates) { } } if (-not $studioPython) { throw 'Install Python with Pillow and NumPy, or set SPRITEMOTION_PYTHON.' } +& $bootstrap (Join-Path $studioRoot 'tools/storage/run.py') startup --checkout $studioRoot --python $studioPython +if ($LASTEXITCODE -ne 0) { throw 'Python environment belongs to another checkout.' } Set-Location -LiteralPath $studioRoot Write-Host 'Open http://127.0.0.1:8772 in your browser. Close this terminal to stop the studio.' & $studioPython tools/uo-content/studio.py diff --git a/launchers/editor/female-live-pose.ps1 b/launchers/editor/female-live-pose.ps1 index ce95374..37441bb 100644 --- a/launchers/editor/female-live-pose.ps1 +++ b/launchers/editor/female-live-pose.ps1 @@ -1,8 +1,13 @@ param([int]$Port = 8768) $ErrorActionPreference = 'Stop' $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path -$pythonExe = Join-Path $repoRoot '.venvs/spritemotion/Scripts/python.exe' -if (-not (Test-Path -LiteralPath $pythonExe)) { $pythonExe = (Get-Command python -ErrorAction Stop).Source } +$bootstrap = (Get-Command python -ErrorAction Stop).Source +$checkoutVenv = & $bootstrap (Join-Path $repoRoot 'tools/worktree-venv/run.py') --path +if ($LASTEXITCODE -ne 0) { throw 'Cannot resolve external checkout environment.' } +$pythonExe = $env:SPRITEMOTION_PYTHON +if (-not $pythonExe) { $pythonExe = Join-Path $checkoutVenv 'Scripts/python.exe' } +& $bootstrap (Join-Path $repoRoot 'tools/storage/run.py') startup --checkout $repoRoot --python $pythonExe +if ($LASTEXITCODE -ne 0) { throw 'Storage/environment validation failed.' } $editorUrl = "http://127.0.0.1:$Port/editor/" try { $response = Invoke-WebRequest $editorUrl -TimeoutSec 2 diff --git a/launchers/pipeline/0-setup.bat b/launchers/pipeline/0-setup.bat index 9423d59..b5c20c8 100644 --- a/launchers/pipeline/0-setup.bat +++ b/launchers/pipeline/0-setup.bat @@ -1,11 +1,7 @@ @echo off -rem Create .venvs\spritemotion, install SpriteMotion (editable, with test extras) and fetch the tested Godot build into tools\godot unless SPRITEMOTION_GODOT is set. +rem Create an external checkout environment and install the tested Godot build in shared storage unless SPRITEMOTION_GODOT is set. call "%~dp0..\_shared\common.bat" || exit /b 1 -if not exist "%SPRITEMOTION_PYTHON%" ( - py -3 -m venv "%SM_ROOT%\.venvs\spritemotion" 2>nul || python -m venv "%SM_ROOT%\.venvs\spritemotion" || exit /b 1 -) -"%SPRITEMOTION_PYTHON%" -m pip install --upgrade pip >nul -"%SPRITEMOTION_PYTHON%" -m pip install -e ".[test]" || exit /b 1 +python "%SM_ROOT%\tools\worktree-venv\run.py" || exit /b 1 %SM_CLI% --version if defined SPRITEMOTION_GODOT ( echo Using Godot at %SPRITEMOTION_GODOT% diff --git a/launchers/pipeline/0-setup.sh b/launchers/pipeline/0-setup.sh index 5a2e318..e4d4c26 100755 --- a/launchers/pipeline/0-setup.sh +++ b/launchers/pipeline/0-setup.sh @@ -1,13 +1,9 @@ #!/usr/bin/env bash -# Create .venvs/spritemotion, install SpriteMotion (editable, with test extras) and fetch the tested Godot build into tools/godot unless SPRITEMOTION_GODOT is set. +# Create an external checkout environment and install the tested Godot build in shared storage unless SPRITEMOTION_GODOT is set. set -euo pipefail . "$(dirname "${BASH_SOURCE[0]}")/../_shared/common.sh" -if [ -z "${SM_VENV_PYTHON:-}" ]; then - python3 -m venv "$SM_ROOT/.venvs/spritemotion" - SPRITEMOTION_PYTHON=$(sm_find_venv_python) -fi -"$SPRITEMOTION_PYTHON" -m pip install --upgrade pip >/dev/null -"$SPRITEMOTION_PYTHON" -m pip install -e ".[test]" +"$_sm_bootstrap" "$SM_ROOT/tools/worktree-venv/run.py" +. "$(dirname "${BASH_SOURCE[0]}")/../_shared/common.sh" "$SPRITEMOTION_PYTHON" -m spritemotion --version if [ -n "${SPRITEMOTION_GODOT:-}" ]; then echo "Using Godot at $SPRITEMOTION_GODOT" diff --git a/tests/unit/test_client_staging.py b/tests/unit/test_client_staging.py index d9b72c1..1e6cfdb 100644 --- a/tests/unit/test_client_staging.py +++ b/tests/unit/test_client_staging.py @@ -56,7 +56,7 @@ def test_anim_stage_missing_client_file_fails_before_output(tmp_path): def test_equipment_failure_after_staging_removes_partial_output(tmp_path, monkeypatch): folder = job(tmp_path) - def fake_stage(vd, source, body, out): + def fake_stage(vd, source, body, out, **kwargs): Path(out).mkdir(); (Path(out) / 'anim.mul').write_bytes(b'partial') return {'source_hashes': {}, 'staged_hashes': {}} diff --git a/tests/unit/test_fit_rebuild.py b/tests/unit/test_fit_rebuild.py index bc6854e..77da8d7 100644 --- a/tests/unit/test_fit_rebuild.py +++ b/tests/unit/test_fit_rebuild.py @@ -47,5 +47,7 @@ def test_partial_rebuild_rejects_changed_inputs(tmp_path,monkeypatch,change,reas if change=='model': spec['backend_sha256']='old' elif change=='renderer': spec['render_fingerprint']='old' else: source.write_bytes(b'changed') - (tmp_path/'job.json').write_text(json.dumps(spec)) - with pytest.raises(ValueError,match=reason): rebuild_job(tmp_path,{'parts':{},'items':{}}) + home=tmp_path/'content'; job=home/'jobs/000000000001'; job.mkdir(parents=True) + monkeypatch.setattr(pipeline,'HOME',home) + (job/'job.json').write_text(json.dumps(spec)) + with pytest.raises(ValueError,match=reason): rebuild_job(job,{'parts':{},'items':{}}) diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py new file mode 100644 index 0000000..22e8932 --- /dev/null +++ b/tests/unit/test_storage.py @@ -0,0 +1,238 @@ +"""Storage limits preserve inputs/authoring and stop real runaway producers.""" +import json +import importlib.util +import os +from pathlib import Path +import subprocess +import sys +import time + +import pytest + +from spritemotion import storage, workpaths + + +def job(home, number, state='complete', age=20): + p = home / 'jobs' / f'{number:012x}' + (p / 'render').mkdir(parents=True) + (p / 'render/frame.png').write_bytes(b'render') + (p / 'input').mkdir() + (p / 'input/source.fbx').write_bytes(b'original input') + (p / 'item.blend').write_bytes(b'authored scene') + (p / 'job.json').write_text('{}') + status = p / 'status.json' + status.write_text(json.dumps(dict(state=state))) + stamp = time.time() - age * 86400 + os.utime(status, (stamp, stamp)) + return p + + +def test_retention_preserves_inputs_scenes_active_recent_and_pinned(tmp_path): + home = tmp_path / 'content' + old = job(home, 1) + active = job(home, 2, 'building') + queued = job(home, 3, 'queued') + recent = job(home, 4, age=1) + pinned = job(home, 5) + (pinned / '.keep').touch() + store = storage.Store(home, storage.Policy(max_bytes=100000, days=7, keep=1)) + with store.session(): + pass + assert not (old / 'render').exists() + assert (old / 'item.blend').read_bytes() == b'authored scene' + assert (old / 'input/source.fbx').read_bytes() == b'original input' + assert (old / 'job.json').exists() + assert json.loads((old / 'status.json').read_text())['state'] == 'failed' + assert all((p / 'render/frame.png').exists() for p in [active, queued, recent, pinned]) + + +def test_protected_plan_aborts_before_any_deletion(tmp_path): + home = tmp_path / 'content' + first = job(home, 1) + second = job(home, 2) + (first / 'render/private.png').write_bytes(b'protected fixture') + job(home, 3, age=0) + with pytest.raises(ValueError, match='protected'): + storage.Store(home, storage.Policy(max_bytes=100000, keep=1)).prune() + assert (first / 'render/frame.png').exists() + assert (second / 'render/frame.png').exists() + + +def test_unknown_authored_file_is_not_expired(tmp_path): + home = tmp_path / 'content' + old = job(home, 1) + (old / 'render/notes.txt').write_text('author notes') + job(home, 2, age=0) + with pytest.raises(ValueError, match='Unknown file'): + storage.Store(home, storage.Policy(max_bytes=100000, keep=1)).prune() + assert (old / 'render/notes.txt').read_text() == 'author notes' + assert (old / 'render/frame.png').exists() + + +def test_capacity_and_concurrent_session_fail_without_deletion(tmp_path): + home = tmp_path / 'content' + store = storage.Store(home, storage.Policy(max_bytes=10000)) + with store.session(): + with pytest.raises(ValueError, match='Another content build'): + with store.session(): + pass + with pytest.raises(storage.StorageLimit): + store.require(20000) + assert not (home / '.storage.lock').exists() + + +def test_rebuild_pin_protects_old_source_between_sessions(tmp_path): + home = tmp_path / 'content' + old = job(home, 1) + job(home, 2, age=0) + store = storage.Store(home, storage.Policy(max_bytes=100000, keep=1)) + with storage.pin(old): + store.prune() + assert (old / 'render/frame.png').exists() + assert not (old / '.storage.pin').exists() + store.prune() + assert not (old / 'render').exists() + + +@pytest.mark.parametrize('existing_scene', [False, True]) +def test_runaway_child_stopped_and_only_new_derivatives_removed(tmp_path, monkeypatch, existing_scene): + import pipeline + home = tmp_path / 'content' + p = job(home, 1, 'queued', age=0) + if not existing_scene: + (p / 'item.blend').unlink() + monkeypatch.setattr(pipeline, 'HOME', home) + monkeypatch.setenv('SPRITEMOTION_MAX_WORK_GB', '.00004') + finished = tmp_path / 'child-finished' + code = ("from pathlib import Path; import time\n" + f"job=Path({str(p)!r}); (job/'item.blend').write_bytes(b'generated')\n" + "with (job/'render/large.png').open('wb') as f:\n" + " for i in range(150): f.write(b'x'*8192); f.flush(); time.sleep(.02)\n" + f"Path({str(finished)!r}).touch()\n") + # A real child writes the bytes; exercise pipeline's actual limit-failure wrapper. + monkeypatch.setattr(pipeline, '_run_job', lambda path, store: store.process([sys.executable, '-c', code], path)) + if existing_scene: + with pytest.raises(ValueError, match='Existing editable scene'): + pipeline.run_job(p) + assert (p / 'item.blend').read_bytes() == b'authored scene' + assert (p / 'render/frame.png').exists() + return + with pytest.raises(pipeline.storage.StorageLimit): + pipeline.run_job(p) + assert not finished.exists() + assert not (p / 'render').exists() + assert (p / 'input/source.fbx').read_bytes() == b'original input' + assert (p / 'job.json').exists() + assert (p / 'item.blend').exists() == existing_scene + assert json.loads((p / 'status.json').read_text())['state'] == 'failed' + assert not (home / '.storage.lock').exists() + + +def test_large_input_refused_before_copy_or_job_creation(tmp_path, monkeypatch): + import pipeline + home = tmp_path / 'content' + source = tmp_path / 'large.glb' + source.write_bytes(b'x' * 100000) + monkeypatch.setattr(pipeline, 'HOME', home) + monkeypatch.setenv('SPRITEMOTION_MAX_WORK_GB', '.00004') + with pytest.raises(pipeline.storage.StorageLimit): + pipeline.create_job({}, source) + assert source.stat().st_size == 100000 + assert not (home / 'jobs').exists() + + +def test_output_override_in_any_checkout_is_refused(tmp_path, monkeypatch): + checkout = tmp_path / 'old-checkout' + (checkout / '.git').mkdir(parents=True) + monkeypatch.setenv('SPRITEMOTION_WORK_ROOT', str(checkout / 'output')) + with pytest.raises(ValueError, match='outside'): + workpaths.work_root() + assert not (checkout / 'output').exists() + + +def test_historical_checkout_fails_closed_without_generated_fallback(tmp_path): + checkout = tmp_path / 'historical' + checkout.mkdir() + with pytest.raises(ValueError, match='Historical checkout'): + storage.startup(checkout, sys.executable) + assert list(checkout.iterdir()) == [] + + +def test_corrupt_status_is_preserved(tmp_path): + home = tmp_path / 'content' + old = job(home, 1) + (old / 'status.json').write_text('[]') + job(home, 2, age=0) + storage.Store(home, storage.Policy(max_bytes=100000, keep=1)).prune() + assert (old / 'render/frame.png').exists() + + +@pytest.mark.parametrize('setting,value', [('SPRITEMOTION_MAX_WORK_GB','0'), + ('SPRITEMOTION_OUTPUT_DAYS','-1'), ('SPRITEMOTION_KEEP_OUTPUT_JOBS','0')]) +def test_invalid_limits_refused(setting, value, monkeypatch): + monkeypatch.setenv(setting, value) + with pytest.raises(ValueError): + storage.Policy() + + +def test_download_extraction_refuses_expansion_before_unpack(tmp_path, monkeypatch): + import hashlib + import zipfile + root = Path(__file__).resolve().parents[2] + spec = importlib.util.spec_from_file_location('godot_fetch_test', root / 'tools/godot/fetch.py') + fetch = importlib.util.module_from_spec(spec); spec.loader.exec_module(fetch) + archive = tmp_path / 'synthetic.zip' + with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED) as z: + z.writestr('synthetic-tool', b'x' * 100000) + data = archive.read_bytes() + monkeypatch.setattr(fetch, 'HERE', tmp_path / 'godot') + monkeypatch.setattr(fetch, 'BUILDS', {'linux': ('synthetic.zip', hashlib.sha512(data).hexdigest(), ['synthetic-tool'])}) + def fake_download(url, target): + target.write_bytes(data) + return hashlib.sha512(data).hexdigest() + monkeypatch.setattr(fetch, 'download', fake_download) + monkeypatch.setenv('SPRITEMOTION_MAX_WORK_GB', '.00001') + with pytest.raises(fetch._storage.StorageLimit): + fetch.main(['--platform', 'linux']) + assert not (fetch.HERE / 'synthetic-tool').exists() + assert not (fetch.HERE / '.storage.lock').exists() + + +def test_missing_keyed_environment_never_creates_local_fallback(tmp_path, monkeypatch): + root = Path(__file__).resolve().parents[2] + monkeypatch.setenv('SPRITEMOTION_WORK_ROOT', str(tmp_path / 'external')) + with pytest.raises(OSError): + storage.startup(root) + assert not (tmp_path / 'external').exists() + + +def test_wrong_editable_install_rejected(tmp_path, monkeypatch): + checkout = tmp_path / 'different' + (checkout / 'common').mkdir(parents=True) + (checkout / 'common/storage.py').write_text('STORAGE_GUARD_VERSION = 1') + launchers = checkout / 'launchers/_shared' + launchers.mkdir(parents=True) + for name in ['common.bat', 'common.sh']: + (launchers / name).write_text('tools/storage/run.py') + monkeypatch.setenv('SPRITEMOTION_WORK_ROOT', str(tmp_path / 'external')) + monkeypatch.delenv('SPRITEMOTION_CONTENT_HOME', raising=False) + with pytest.raises(ValueError, match='does not import this checkout'): + storage.startup(checkout, sys.executable) + + +@pytest.mark.skipif(os.name != 'nt', reason='Windows launcher verification') +def test_real_launcher_refuses_local_override_and_historical_resume(tmp_path): + root = Path(__file__).resolve().parents[2] + command = ['cmd', '/c', str(root / 'launchers/dev/check-storage.bat')] + env = dict(os.environ, SPRITEMOTION_PYTHON=sys.executable, SPRITEMOTION_WORK_ROOT=str(tmp_path / 'external')) + env.pop('SPRITEMOTION_CONTENT_HOME', None) + result = subprocess.run(command + ['--paths-only'], env=env, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + historical = tmp_path / 'historical' + historical.mkdir() + result = subprocess.run(command + ['--checkout', str(historical)], env=env, capture_output=True, text=True) + assert result.returncode != 0 and 'Historical checkout' in result.stderr + env['SPRITEMOTION_CONTENT_HOME'] = str(root / 'forbidden-generated') + result = subprocess.run(command + ['--paths-only'], env=env, capture_output=True, text=True) + assert result.returncode != 0 and 'outside' in result.stderr + assert not (root / 'forbidden-generated').exists() diff --git a/tools/fit-lab/export_blender.py b/tools/fit-lab/export_blender.py index f8480fc..2b93edd 100644 --- a/tools/fit-lab/export_blender.py +++ b/tools/fit-lab/export_blender.py @@ -7,6 +7,7 @@ skinned to the same rig, with the pack palette. manifest.json camera, directions, actions and the item list (merged with an earlier export). """ +import importlib.util import json import sys from pathlib import Path @@ -24,9 +25,12 @@ argv = sys.argv[sys.argv.index('--') + 1:] items_doc = labpaths.load(argv[0]) -OUT = Path(argv[1]); FORCE = '--force' in argv +_paths_spec = importlib.util.spec_from_file_location('export_workpaths', REPO / 'common/workpaths.py') +_paths = importlib.util.module_from_spec(_paths_spec) +_paths_spec.loader.exec_module(_paths) +OUT = _paths.external_path(Path(argv[1])); FORCE = '--force' in argv (OUT / 'items').mkdir(parents=True, exist_ok=True) -MODEL = Path(items_doc.get('model', REPO / 'workspace/ultima-online/canonical-model/model/UO_Body_0x190.blend')) +MODEL = Path(items_doc.get('model', _paths.backend_home() / 'model/UO_Body_0x190.blend')) STEP = 3 bpy.ops.wm.open_mainfile(filepath=str(MODEL), load_ui=False, use_scripts=False) diff --git a/tools/godot/fetch.py b/tools/godot/fetch.py index 9689ae6..f27df7c 100644 --- a/tools/godot/fetch.py +++ b/tools/godot/fetch.py @@ -1,4 +1,4 @@ -"""Download the Godot build the Sprite Pose Editor is tested with into tools/godot/. +"""Download the tested Godot build into the external shared work area. python tools/godot/fetch.py # this platform python tools/godot/fetch.py --force # replace an existing download @@ -12,6 +12,7 @@ import argparse import hashlib +import importlib.util import platform import sys import tempfile @@ -21,7 +22,13 @@ VERSION = "4.7-stable" RELEASE = f"https://github.com/godotengine/godot/releases/download/{VERSION}/" -HERE = Path(__file__).resolve().parent +_spec = importlib.util.spec_from_file_location('workpaths', Path(__file__).resolve().parents[2] / 'common/workpaths.py') +_paths = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_paths) +HERE = _paths.work_root() / 'godot' +_guard_spec = importlib.util.spec_from_file_location('storage', Path(__file__).resolve().parents[2] / 'common/storage.py') +_storage = importlib.util.module_from_spec(_guard_spec) +_guard_spec.loader.exec_module(_storage) # platform -> (zip name, sha512, files it must contain) BUILDS = { @@ -58,6 +65,7 @@ def download(url: str, target: Path) -> str: total = int(response.headers.get("Content-Length") or 0) done = 0 while block := response.read(1 << 20): + _storage.Store(HERE).require(len(block)) out.write(block) digest.update(block) done += len(block) @@ -72,18 +80,29 @@ def main(argv=None) -> int: parser.add_argument("--platform", choices=sorted(BUILDS), default=None) parser.add_argument("--force", action="store_true", help="download even if Godot is already here") args = parser.parse_args(argv) + with _storage.Store(HERE).session() as store: + return install(args, store) + + +def install(args, store): name, sha512, files = BUILDS[args.platform or current_platform()] if all((HERE / f).exists() for f in files) and not args.force: print(f"Godot {VERSION} is already in {HERE}") return 0 print(f"Downloading {RELEASE}{name}") - with tempfile.TemporaryDirectory() as tmp: + HERE.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=HERE) as tmp: archive = Path(tmp) / name actual = download(RELEASE + name, archive) if actual != sha512: print(f"Checksum mismatch for {name}:\n expected {sha512}\n got {actual}", file=sys.stderr) return 1 with zipfile.ZipFile(archive) as zf: + for info in zf.infolist(): + target = (HERE / info.filename).resolve() + if HERE.resolve() not in target.parents: + raise ValueError('Archive member escapes the external Godot directory.') + store.require(sum(info.file_size for info in zf.infolist())) zf.extractall(HERE) if sys.platform != "win32": # zipfile drops the executable bit for info in zf.infolist(): diff --git a/tools/storage/README.md b/tools/storage/README.md new file mode 100644 index 0000000..eb805e5 --- /dev/null +++ b/tools/storage/README.md @@ -0,0 +1,66 @@ +# Generated storage + +Content builds, block rebuilds, client staging and checkout environment setup +check their external stores before allocating large outputs. Content builds +monitor the Blender child every 250 ms and terminate it on overflow. A failed +fresh build removes its regenerable derivatives and newly generated scene; +inputs and any pre-existing editable scene are preserved. Building over an +existing editable scene is refused: create a new job instead. + +Settings resolve environment > launcher configuration > defaults: + +| Setting | Default | Meaning | +|---|---|---| +| `SPRITEMOTION_MAX_WORK_GB` | `5` | Decimal GB per content/environment/Godot or explicit staging store | +| `SPRITEMOTION_OUTPUT_DAYS` | `7` | Minimum age before completed/failed derivatives expire | +| `SPRITEMOTION_KEEP_OUTPUT_JOBS` | `10` | Retain at least this many newest finished jobs | +| `SPRITEMOTION_WORK_ROOT` | Fleet shared work area; user cache off Windows | External work root | +| `SPRITEMOTION_CONTENT_HOME` | `/content-studio` | Content job store | +| `SPRITEMOTION_BACKEND` | `/inputs/canonical-model` | One canonical input installation shared by checkouts | + +Positive limits are required. The byte cap counts preserved inputs and scenes +too; when these fill the store, new work is refused. Choose another external +store or explicitly manage inputs; retention never deletes them. Set +`SPRITEMOTION_BACKEND` to an existing external canonical installation when +migrating, or run pipeline `setup --source` once. No input migration is automatic. + +Every content producer session expires only finished jobs that are BOTH older +than the age threshold AND outside the newest-job allowance. Only the fixed +derivative allowlist in `common/storage.py` is eligible. Inputs, job settings, +editable `.blend` files, staging/client payloads, unknown files and protected +fleet filenames never expire. Queued/building jobs, `.keep` jobs and pinned +rebuild sources are excluded. Protected/unknown contents abort the deletion +plan before any candidate is removed. Links are never traversed for deletion. + +`python tools/storage/run.py check` reports current use; `prune` runs the same +retention policy immediately. Locks serialize builds and cleanup. After a +crash, verify the PID in `.storage.lock` or `.storage.pin` is no longer active +before removing that marker. There is no unattended timer or stale-lock bypass. + +This is a producer guard, not an operating-system disk quota. A child can +overshoot during one write/poll interval. ZIPs and client copies reserve a +conservative size before allocation. Environment setup stops its subprocess +on overflow but retains the partial external environment for explicit rebuild. +Independent explicit stores have independent caps; this is not a fleet-wide +quota. Direct manual writes and historical scripts are outside the producer +guard. Review story stores and remove obsolete derivatives at story closure. + +## Startup and historical checkouts + +Run `launchers/dev/check-storage.bat` (`.sh` elsewhere) before resuming work. +For an old checkout use `--checkout PATH --python EXTERNAL_PYTHON`. Missing +guard code or launcher checks fail closed; update/rebase that checkout before +launching it. No script can retrofit protection into an untouched historical +checkout: the resume procedure must invoke this guard first. + +Updated launchers reject output roots and interpreters inside any Git checkout, +verify the editable install maps to that exact checkout, and use the keyed +external environment. `--paths-only` supports first-time setup without an +existing environment. Legacy in-checkout venv fallback was removed. Godot +downloads now target `/godot`; existing configured executables remain +usable. Download chunks and archive extraction check available store capacity. +Content-studio and pipeline CLI also validate direct startup. + +Other explicit exporters must select external destinations; not every legacy +one-off Blender script has automatic age retention. Do not call an untouched +historical producer directly as a substitute for the guarded launchers. diff --git a/tools/storage/run.py b/tools/storage/run.py new file mode 100644 index 0000000..a98f001 --- /dev/null +++ b/tools/storage/run.py @@ -0,0 +1,46 @@ +"""Validate a checkout before resuming it, or expire obsolete generated derivatives.""" +import argparse +import importlib.util +import json +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('storage', ROOT / 'common/storage.py') +storage = importlib.util.module_from_spec(spec) +spec.loader.exec_module(storage) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + sub = p.add_subparsers(dest='command', required=True) + s = sub.add_parser('startup') + s.add_argument('--checkout', type=Path, default=ROOT) + s.add_argument('--python') + s.add_argument('--paths-only', action='store_true') + sub.add_parser('prune') + sub.add_parser('check') + sub.add_parser('godot-path') + args = p.parse_args() + try: + if args.command == 'godot-path': + print(storage.paths.work_root() / 'godot') + return 0 + if args.command == 'startup': + result = storage.startup(args.checkout, args.python, args.paths_only) + else: + store = storage.Store(storage.paths.content_home()) + if args.command == 'prune': + with store.session() as session: + result = dict(bytes=session.require(), limit_bytes=session.policy.max_bytes) + else: + result = dict(bytes=store.require(), limit_bytes=store.policy.max_bytes) + print(json.dumps(result)) + return 0 + except (ValueError, OSError) as error: + print('[SpriteMotion storage] ' + str(error), file=sys.stderr) + return 2 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/uo-content/client_import.py b/tools/uo-content/client_import.py index 25e0283..d7d5ac7 100644 --- a/tools/uo-content/client_import.py +++ b/tools/uo-content/client_import.py @@ -53,7 +53,23 @@ def require_client_files(client, names): missing=[n for n in names if not (Path(client)/n).is_file()] if missing: raise ValueError(f'Client folder is missing {", ".join(missing)}: choose a classic MUL client folder.') -def stage(vd, client, body, out): +def staging_store(out): + import importlib.util + spec=importlib.util.spec_from_file_location('staging_storage',Path(__file__).resolve().parents[2]/'common/storage.py') + storage=importlib.util.module_from_spec(spec);spec.loader.exec_module(storage) + out=storage.paths.external_path(Path(out)) + home=storage.paths.content_home() + return storage.Store(home if home in out.parents else out.parent) + + +def stage(vd, client, body, out, _store=None): + if _store is not None: + return _stage(vd,client,body,out,_store) + with staging_store(out).session() as store: + return _stage(vd,client,body,out,store) + + +def _stage(vd, client, body, out, store): vd,client,out=Path(vd).resolve(),Path(client).resolve(),Path(out).resolve() if not 400<=body<=2047: raise ValueError('Choose a people animation ID from 400 to 2047.') require_client_files(client,ANIM_FILES) @@ -82,6 +98,9 @@ def stage(vd, client, body, out): except (TypeError,IndexError): pass except ValueError as e: if 'routed' in str(e): raise + # Include the entire client copy, appended VD blocks, index expansion and report. + store.require((client/'anim.mul').stat().st_size + sum(map(len,blocks.values())) + + max(len(index),(first+175)*12) + 16384) out.mkdir(parents=True) try: shutil.copy2(client/'anim.mul',out/'anim.mul') diff --git a/tools/uo-content/equipment.py b/tools/uo-content/equipment.py index e7c5631..1df7a11 100644 --- a/tools/uo-content/equipment.py +++ b/tools/uo-content/equipment.py @@ -104,6 +104,12 @@ def server_source(name,graphic,part,flavor='modernuo'): ''' def stage_equipment(job,client,body,graphic,flavor='modernuo'): + from client_import import staging_store + with staging_store(Path(job)/'staged-client').session(protect=[job]) as store: + return _stage_equipment(job,client,body,graphic,flavor,store) + + +def _stage_equipment(job,client,body,graphic,flavor,store): job,client=Path(job).resolve(),Path(client).resolve() require_client_files(client,EQUIPMENT_FILES) spec=json.loads((job/'job.json').read_text(encoding='utf-8')) @@ -122,8 +128,10 @@ def stage_equipment(job,client,body,graphic,flavor='modernuo'): if not np.array_equal(decode_art(encoded)[...,3]>0,np.asarray(image)[...,3]>=128): raise ValueError('Inventory art round-trip failed.') out=job/'staged-client' - result=stage(job/'item.vd',client,body,out) # refuses an existing out, so out is ours from here + result=stage(job/'item.vd',client,body,out,_store=store) # refuses an existing out try: + store.require(len(data)+(client/'art.mul').stat().st_size+len(encoded) + +max(len(artidx),index+12)+32768) data[target:target+size]=data[source:source+size] struct.pack_into('___.. All of it is built from game-derived frames: -keep the output local (workspace/ is ignored). +keep the output external and private. """ import argparse import json @@ -51,12 +51,15 @@ def build(args): def body(args): import pipeline - backend = ROOT/'workspace/ultima-online/canonical-model' + backend = pipeline.BACKEND blend = next((backend/'model').glob('*.blend')) out = Path(args.work)/'body'; out.mkdir(parents=True, exist_ok=True) command = [pipeline.blender_path(), '--background', '--factory-startup', '--python-exit-code', '1', str(blend), '--python', str(Path(__file__).with_name('body_render.py')), '--', str(backend), str(out), ','.join(ACTIONS.values())] - sys.exit(subprocess.run(command).returncode) + with pipeline.storage.Store(Path(args.work)).session() as store: + result = store.process(command, out) + if result: + raise SystemExit(result) def composite(args): @@ -137,6 +140,11 @@ def main(): c.add_argument('--out', required=True); c.add_argument('--direction', type=int, default=2) c.add_argument('--story', default='whole-outfit', help='story id used in the evidence file names') args = parser.parse_args() + import pipeline + pipeline.storage.startup(ROOT, sys.executable) + args.work = str(pipeline.storage.paths.external_path(Path(args.work))) + if hasattr(args, 'out'): + args.out = str(pipeline.storage.paths.external_path(Path(args.out))) {'build': build, 'body': body, 'composite': composite}[args.command](args) diff --git a/tools/worktree-venv/run.py b/tools/worktree-venv/run.py index 9d35a11..5668f93 100644 --- a/tools/worktree-venv/run.py +++ b/tools/worktree-venv/run.py @@ -18,7 +18,6 @@ import subprocess import sys import tempfile -import venv from pathlib import Path ROOT = Path(__file__).resolve().parents[2] @@ -26,6 +25,9 @@ _paths = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(_paths) VENV = _paths.checkout_venv(ROOT) +_guard_spec = importlib.util.spec_from_file_location('storage', ROOT / 'common/storage.py') +_storage = importlib.util.module_from_spec(_guard_spec) +_guard_spec.loader.exec_module(_storage) CHECK = ( "import sys, spritemotion, numpy, scipy, PIL, jsonschema, pytest;" @@ -47,6 +49,13 @@ def main(argv: list[str] | None = None) -> int: print(VENV) return 0 + _storage.startup(ROOT, paths_only=True) + with _storage.Store(VENV.parent).session() as store: + return prepare(args, store) + + +def prepare(args, store): + if args.rebuild and VENV.exists(): print(f"Removing {VENV}") shutil.rmtree(VENV) @@ -54,16 +63,18 @@ def main(argv: list[str] | None = None) -> int: print(f"Reusing {VENV}") else: print(f"Creating {VENV}") - venv.EnvBuilder(with_pip=True, clear=False).create(VENV) + if store.process([sys.executable, '-m', 'venv', str(VENV)], VENV): + return 1 (VENV / 'README.md').write_text( '# SpriteMotion checkout environment\n\n' 'Regenerate from its checkout with `python tools/worktree-venv/run.py`. ' 'The checkout path determines this directory key. Do not reuse this editable install ' 'for another checkout. `--path` prints the directory without creating it.\n', encoding='utf-8') python = str(venv_python()) - env = dict(os.environ, PIP_DISABLE_PIP_VERSION_CHECK="1") + env = dict(os.environ, PIP_DISABLE_PIP_VERSION_CHECK="1", PIP_CACHE_DIR=str(VENV.parent / '.pip-cache'), + PYTHONPYCACHEPREFIX=str(VENV / 'pycache')) - if subprocess.run([python, "-m", "pip", "install", "-e", ".[test]"], cwd=ROOT, env=env).returncode: + if store.process([python, "-m", "pip", "install", "-e", ".[test]"], VENV, cwd=ROOT, env=env): print("pip install -e .[test] failed", file=sys.stderr) return 1 @@ -85,4 +96,8 @@ def main(argv: list[str] | None = None) -> int: if __name__ == "__main__": - raise SystemExit(main()) + try: + raise SystemExit(main()) + except (ValueError, OSError) as error: + print('[SpriteMotion environment] ' + str(error), file=sys.stderr) + raise SystemExit(2) From fc8ba4dfb1dff759bcda983f7a2f8bd87e94c41a Mon Sep 17 00:00:00 2001 From: Moshu <1709219+DatMoshu@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:12:21 -0500 Subject: [PATCH 2/5] Honor explicit Python environment in standalone storage checks --- common/storage.py | 2 +- tests/unit/test_storage.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/common/storage.py b/common/storage.py index 5e2e2dc..cd9e39d 100644 --- a/common/storage.py +++ b/common/storage.py @@ -218,7 +218,7 @@ def startup(checkout, python=None, paths_only=False): paths.external_path(content, checkout) if not paths_only: if not python: - python = paths.checkout_venv(checkout) / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python') + python = os.environ.get('SPRITEMOTION_PYTHON') or paths.checkout_venv(checkout) / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python') exe = Path(python).resolve() paths.external_path(exe, checkout) result = subprocess.run([str(exe), '-c', 'import spritemotion; print(spritemotion.__file__)'], diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py index 22e8932..7370776 100644 --- a/tests/unit/test_storage.py +++ b/tests/unit/test_storage.py @@ -201,6 +201,7 @@ def fake_download(url, target): def test_missing_keyed_environment_never_creates_local_fallback(tmp_path, monkeypatch): root = Path(__file__).resolve().parents[2] monkeypatch.setenv('SPRITEMOTION_WORK_ROOT', str(tmp_path / 'external')) + monkeypatch.delenv('SPRITEMOTION_PYTHON', raising=False) with pytest.raises(OSError): storage.startup(root) assert not (tmp_path / 'external').exists() @@ -228,6 +229,8 @@ def test_real_launcher_refuses_local_override_and_historical_resume(tmp_path): env.pop('SPRITEMOTION_CONTENT_HOME', None) result = subprocess.run(command + ['--paths-only'], env=env, capture_output=True, text=True) assert result.returncode == 0, result.stdout + result.stderr + result = subprocess.run(command, env=env, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr historical = tmp_path / 'historical' historical.mkdir() result = subprocess.run(command + ['--checkout', str(historical)], env=env, capture_output=True, text=True) From 4244eaad6a5d7057a6c06bf258ec5296189bedb0 Mon Sep 17 00:00:00 2001 From: Moshu <1709219+DatMoshu@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:12:48 -0500 Subject: [PATCH 3/5] Make storage regression tests independent of collection order --- tests/unit/test_storage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py index 7370776..e25414f 100644 --- a/tests/unit/test_storage.py +++ b/tests/unit/test_storage.py @@ -11,6 +11,8 @@ from spritemotion import storage, workpaths +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'tools/uo-content')) + def job(home, number, state='complete', age=20): p = home / 'jobs' / f'{number:012x}' From 1b611ca8aa861f41df0a3d2a26ce56a1d2e5c419 Mon Sep 17 00:00:00 2001 From: Moshu <1709219+DatMoshu@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:19:40 -0500 Subject: [PATCH 4/5] Serialize rebuild pins with retention planning and deletion --- common/storage.py | 43 ++++++++++++----- tests/unit/test_storage.py | 98 ++++++++++++++++++++++++++++++++++++++ tools/storage/README.md | 9 +++- 3 files changed, 137 insertions(+), 13 deletions(-) diff --git a/common/storage.py b/common/storage.py index cd9e39d..13ec658 100644 --- a/common/storage.py +++ b/common/storage.py @@ -109,17 +109,28 @@ def remove_derived(job, initial_scene=False): @contextmanager def pin(job): """Keep an immutable rebuild source alive across patch/render/merge sessions.""" - marker = Path(job) / '.storage.pin' + job = Path(job).resolve() + if job.parent.name != 'jobs': + raise ValueError('A rebuild pin must belong to an external store jobs directory.') + marker = job / '.storage.pin' + fd = None try: - fd = os.open(marker, os.O_CREAT | os.O_EXCL | os.O_WRONLY) - except FileExistsError: - raise ValueError('This source job is already pinned by another rebuild.') - try: - os.write(fd, str(os.getpid()).encode()) + # Serialize marker creation with the ENTIRE retention plan/delete pass. + # Release the store lock before yielding so rebuild create/run sessions + # can acquire it while the persistent marker protects their source. + with Store(job.parent.parent).lock(): + try: + fd = os.open(marker, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except FileExistsError: + raise ValueError('This source job is already pinned by another rebuild.') + os.write(fd, str(os.getpid()).encode()) yield finally: - os.close(fd) - marker.unlink() + if fd is not None: + os.close(fd) + # The source is no longer in use. Removing its marker needs no + # store lock and must not fail merely because another build started. + marker.unlink() class Store: @@ -137,6 +148,11 @@ def require(self, extra=0): return size def prune(self, protect=()): + with self.lock(): + return self._prune(protect) + + def _prune(self, protect=()): + """Plan and delete while the caller owns this store's lock.""" protected = {Path(p).resolve() for p in protect} jobs = [] if linked(self.home / 'jobs'): @@ -166,7 +182,7 @@ def prune(self, protect=()): return removed @contextmanager - def session(self, protect=()): + def lock(self): self.home.mkdir(parents=True, exist_ok=True) lock = self.home / '.storage.lock' try: @@ -175,13 +191,18 @@ def session(self, protect=()): raise ValueError('Another content build/cleanup holds .storage.lock. Wait; after a crash verify no build is active before removing the lock.') try: os.write(fd, str(os.getpid()).encode()) - self.prune(protect) - self.require() yield self finally: os.close(fd) lock.unlink() + @contextmanager + def session(self, protect=()): + with self.lock(): + self._prune(protect) + self.require() + yield self + def process(self, command, job, **kwargs): """Stop the child on overflow; polling may allow one write interval of overshoot.""" self.require() diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py index e25414f..f6720e8 100644 --- a/tests/unit/test_storage.py +++ b/tests/unit/test_storage.py @@ -6,6 +6,8 @@ import subprocess import sys import time +from concurrent.futures import ThreadPoolExecutor +from threading import Event import pytest @@ -96,6 +98,102 @@ def test_rebuild_pin_protects_old_source_between_sessions(tmp_path): assert not (old / 'render').exists() +@pytest.mark.parametrize('entry', ['prune', 'session']) +def test_retention_plan_blocks_late_pin_before_rebuild_body(tmp_path, monkeypatch, entry): + home = tmp_path / 'content' + old = job(home, 1) + job(home, 2, age=0) + planned, resume = Event(), Event() + guard = storage.protected_guard + + def paused_guard(candidates): + # Retention has already selected this source but has not deleted it. + assert old / 'render' in candidates + planned.set() + assert resume.wait(10), 'test never released retention barrier' + guard(candidates) + + monkeypatch.setattr(storage, 'protected_guard', paused_guard) + + def prune(): + store = storage.Store(home, storage.Policy(max_bytes=100000, keep=1)) + if entry == 'prune': + store.prune() + else: + with store.session(): + pass + + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(prune) + try: + assert planned.wait(10), 'retention did not reach planned-deletion barrier' + with pytest.raises(ValueError, match='Another content build'): + with storage.pin(old): + pytest.fail('rebuild entered while its source was selected for deletion') + assert not (old / '.storage.pin').exists() + assert (old / 'render/frame.png').exists() + finally: + resume.set() + future.result(timeout=10) + assert not (old / 'render').exists() + assert not (home / '.storage.lock').exists() + assert (old / 'item.blend').read_bytes() == b'authored scene' + + +def test_acquired_pin_survives_concurrent_prune_and_releases_without_store_lock(tmp_path): + home = tmp_path / 'content' + old = job(home, 1) + job(home, 2, age=0) + acquired, release = Event(), Event() + + def rebuild(): + with storage.pin(old): + acquired.set() + assert release.wait(10), 'test never released pinned rebuild' + + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(rebuild) + try: + assert acquired.wait(10), 'rebuild never acquired pin' + store = storage.Store(home, storage.Policy(max_bytes=100000, keep=1)) + store.prune() + with store.session(): + assert (old / 'render/frame.png').exists() + # Releasing a finished rebuild must not contend with this lock. + release.set() + future.result(timeout=10) + assert not (old / '.storage.pin').exists() + finally: + release.set() + future.result(timeout=10) + store.prune() + assert not (old / 'render').exists() + + +def test_pinned_source_allows_real_create_and_run_sessions(tmp_path, monkeypatch): + import pipeline + home = tmp_path / 'content' + old = job(home, 1) + backend = tmp_path / 'backend' + (backend / 'model').mkdir(parents=True) + (backend / 'model/UO_Body_0x190.blend').write_bytes(b'synthetic model') + (backend / 'provenance.json').write_text('{}') + monkeypatch.setattr(pipeline, 'HOME', home) + monkeypatch.setattr(pipeline, 'BACKEND', backend) + monkeypatch.setattr(pipeline, 'render_fingerprint', lambda: 'synthetic renderer') + monkeypatch.setenv('SPRITEMOTION_KEEP_OUTPUT_JOBS', '1') + def render(path, store): + assert (home / '.storage.lock').exists() + assert (old / 'render/frame.png').exists() + return 'render entered' + monkeypatch.setattr(pipeline, '_run_job', render) + with storage.pin(old): + patch = pipeline.create_job({'name': 'Synthetic patch'}) + assert pipeline.run_job(patch) == 'render entered' + assert not (home / '.storage.lock').exists() + assert not (old / '.storage.pin').exists() + + @pytest.mark.parametrize('existing_scene', [False, True]) def test_runaway_child_stopped_and_only_new_derivatives_removed(tmp_path, monkeypatch, existing_scene): import pipeline diff --git a/tools/storage/README.md b/tools/storage/README.md index eb805e5..02f43a7 100644 --- a/tools/storage/README.md +++ b/tools/storage/README.md @@ -33,8 +33,13 @@ rebuild sources are excluded. Protected/unknown contents abort the deletion plan before any candidate is removed. Links are never traversed for deletion. `python tools/storage/run.py check` reports current use; `prune` runs the same -retention policy immediately. Locks serialize builds and cleanup. After a -crash, verify the PID in `.storage.lock` or `.storage.pin` is no longer active +retention policy immediately. Locks serialize builds and cleanup. An acquired +pin protects the source across separately locked +create/render/merge sessions. Creating a pin uses the same store lock as the +entire retention selection/deletion pass. If cleanup already owns that lock, +pin acquisition fails before the rebuild enters its body; retry after cleanup. +No store lock is held across the pin body, so rebuild sessions do not deadlock. +After a crash, verify the PID in `.storage.lock` or `.storage.pin` is no longer active before removing that marker. There is no unattended timer or stale-lock bypass. This is a producer guard, not an operating-system disk quota. A child can From 0af1ade16dae541a01ee82afe2b57817420368e2 Mon Sep 17 00:00:00 2001 From: Moshu <1709219+DatMoshu@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:30:05 -0600 Subject: [PATCH 5/5] Fix storage guard review findings - startup() runs the interpreter path as given; following a venv's bin/python symlink started the base interpreter without the venv's site-packages. The physical file and the environment are both checked. - Retention ranks jobs by when they finished. Expiry records completed_at because its status.json rewrite refreshes the file's mtime, which let expired jobs displace the jobs meant to be kept. - Staging a VD protects its source job from the session's retention. - usage() tolerates files a running producer removes mid-walk, which aborted worktree-venv creation. - The Blender import check supplies the spec global import_asset now reads. Co-Authored-By: Claude Opus 5.5 --- common/storage.py | 34 ++++++-- .../integration/blender_import_asset_check.py | 1 + tests/unit/test_client_staging.py | 22 +++++ tests/unit/test_storage.py | 86 ++++++++++++++++++- tools/storage/README.md | 3 + tools/uo-content/client_import.py | 3 +- 6 files changed, 139 insertions(+), 10 deletions(-) diff --git a/common/storage.py b/common/storage.py index 13ec658..db57643 100644 --- a/common/storage.py +++ b/common/storage.py @@ -50,7 +50,10 @@ def usage(root): p = Path(parent, name) if linked(p): continue - s = p.stat() + try: + s = p.stat() + except FileNotFoundError: + continue # A running producer removed its temporary file mid-walk. key = (s.st_dev, s.st_ino) if key not in seen: total += s.st_size @@ -133,6 +136,14 @@ def pin(job): marker.unlink() +def finished_at(status, path): + """When a job finished: the stamp expiry recorded, else its status file's mtime.""" + stamp = status.get('completed_at') + if isinstance(stamp, (int, float)) and not isinstance(stamp, bool) and math.isfinite(stamp): + return stamp + return path.stat().st_mtime + + class Store: def __init__(self, home, policy=None): self.home = paths.external_path(Path(home)) @@ -165,18 +176,21 @@ def _prune(self, protect=()): except (OSError, ValueError): continue # Unknown or corrupt state is not proof of obsolescence. if isinstance(status, dict) and status.get('state') in ('complete', 'failed'): - jobs.append((p.stat().st_mtime, p, status)) + jobs.append((finished_at(status, p), p, status)) plans = [] - for index, (stamp, p, status) in enumerate(sorted(jobs, reverse=True)): + for index, (stamp, p, status) in enumerate(sorted(jobs, key=lambda job: job[0], reverse=True)): if index < self.policy.keep or time.time() - stamp < self.policy.days * 86400: continue - plans.append((p, status, derivative_paths(p.parent))) - protected_guard([q for p, status, candidates in plans for q in candidates]) + plans.append((stamp, p, status, derivative_paths(p.parent))) + protected_guard([q for stamp, p, status, candidates in plans for q in candidates]) removed = [] - for p, status, candidates in plans: + for stamp, p, status, candidates in plans: deleted = delete_paths(candidates) if deleted: - status.update(state='failed', error='Generated derivatives expired; retained inputs/settings/scene can rebuild.') + # This rewrite refreshes the file's mtime, so record when the job + # really finished or it would outrank the jobs meant to be kept. + status.update(state='failed', completed_at=stamp, + error='Generated derivatives expired; retained inputs/settings/scene can rebuild.') p.write_text(json.dumps(status, indent=2), encoding='utf-8') removed.extend(deleted) return removed @@ -240,8 +254,12 @@ def startup(checkout, python=None, paths_only=False): if not paths_only: if not python: python = os.environ.get('SPRITEMOTION_PYTHON') or paths.checkout_venv(checkout) / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python') - exe = Path(python).resolve() + # Run the path as given: a venv's bin/python is a symlink, and following + # it would start the base interpreter without the venv's site-packages. + exe = Path(os.path.abspath(python)) + # Check both the interpreter's physical file and the environment holding the link. paths.external_path(exe, checkout) + paths.external_path(exe.parent, checkout) result = subprocess.run([str(exe), '-c', 'import spritemotion; print(spritemotion.__file__)'], cwd=checkout.parent, capture_output=True, text=True) if result.returncode or Path(result.stdout.strip()).resolve().parent != checkout / 'common': diff --git a/tests/integration/blender_import_asset_check.py b/tests/integration/blender_import_asset_check.py index 63a7db4..236639c 100644 --- a/tests/integration/blender_import_asset_check.py +++ b/tests/integration/blender_import_asset_check.py @@ -11,6 +11,7 @@ root = Path(__file__).resolve().parents[2] tree = ast.parse((root / 'tools/uo-content/blender_build.py').read_text(encoding='utf-8')) function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == 'import_asset') +spec = {} # the build script's job settings, read as a module global by import_asset exec(compile(ast.Module(body=[function], type_ignores=[]), 'import_asset', 'exec')) with tempfile.TemporaryDirectory() as temp: diff --git a/tests/unit/test_client_staging.py b/tests/unit/test_client_staging.py index 1e6cfdb..7a5f486 100644 --- a/tests/unit/test_client_staging.py +++ b/tests/unit/test_client_staging.py @@ -3,8 +3,10 @@ Fixtures are synthetic byte buffers, never client data. """ import json +import os from pathlib import Path import sys +import time import pytest from PIL import Image @@ -53,6 +55,26 @@ def test_anim_stage_missing_client_file_fails_before_output(tmp_path): assert not out.exists() +def test_staging_a_vd_keeps_its_old_job_through_retention(tmp_path, monkeypatch): + home = tmp_path / 'content' + monkeypatch.setenv('SPRITEMOTION_CONTENT_HOME', str(home)) + monkeypatch.setenv('SPRITEMOTION_KEEP_OUTPUT_JOBS', '1') + jobs = {} + for name, age in (('000000000001', 30), ('000000000002', 0)): + folder = home / 'jobs' / name; folder.mkdir(parents=True) + (folder / 'item.vd').write_bytes(b'generated') + status = folder / 'status.json'; status.write_text(json.dumps({'state': 'complete'}), encoding='utf-8') + stamp = time.time() - age * 86400 + os.utime(status, (stamp, stamp)) + jobs[name] = folder + source = jobs['000000000001'] / 'item.vd' + seen = [] + monkeypatch.setattr(client_import, '_stage', lambda vd, *rest: seen.append(Path(vd).read_bytes())) + client_import.stage(source, client(tmp_path), 400, home / 'staged') + assert seen == [b'generated'] + assert source.exists() + + def test_equipment_failure_after_staging_removes_partial_output(tmp_path, monkeypatch): folder = job(tmp_path) diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py index f6720e8..b003383 100644 --- a/tests/unit/test_storage.py +++ b/tests/unit/test_storage.py @@ -50,6 +50,21 @@ def test_retention_preserves_inputs_scenes_active_recent_and_pinned(tmp_path): assert all((p / 'render/frame.png').exists() for p in [active, queued, recent, pinned]) +def test_usage_tolerates_a_producer_removing_its_temporary_file(tmp_path, monkeypatch): + (tmp_path / 'kept.bin').write_bytes(b'12345') + temporary = tmp_path / 'module.pyc.1234' + temporary.write_bytes(b'transient') + original = storage.linked + + def removed_after_listing(path): + if path == temporary: + temporary.unlink() + return original(path) + + monkeypatch.setattr(storage, 'linked', removed_after_listing) + assert storage.usage(tmp_path) == 5 + + def test_protected_plan_aborts_before_any_deletion(tmp_path): home = tmp_path / 'content' first = job(home, 1) @@ -307,7 +322,7 @@ def test_missing_keyed_environment_never_creates_local_fallback(tmp_path, monkey assert not (tmp_path / 'external').exists() -def test_wrong_editable_install_rejected(tmp_path, monkeypatch): +def guarded_checkout(tmp_path, monkeypatch): checkout = tmp_path / 'different' (checkout / 'common').mkdir(parents=True) (checkout / 'common/storage.py').write_text('STORAGE_GUARD_VERSION = 1') @@ -317,10 +332,79 @@ def test_wrong_editable_install_rejected(tmp_path, monkeypatch): (launchers / name).write_text('tools/storage/run.py') monkeypatch.setenv('SPRITEMOTION_WORK_ROOT', str(tmp_path / 'external')) monkeypatch.delenv('SPRITEMOTION_CONTENT_HOME', raising=False) + return checkout + + +def test_wrong_editable_install_rejected(tmp_path, monkeypatch): + checkout = guarded_checkout(tmp_path, monkeypatch) with pytest.raises(ValueError, match='does not import this checkout'): storage.startup(checkout, sys.executable) +def venv_link(folder, target): + folder.mkdir(parents=True) + link = folder / 'python' + try: + link.symlink_to(target) + except OSError as error: + pytest.skip(f'symlinks unavailable: {error}') + return link + + +def test_startup_runs_venv_link_not_its_base_interpreter(tmp_path, monkeypatch): + checkout = guarded_checkout(tmp_path, monkeypatch) + base = tmp_path / 'base-python' + base.write_bytes(b'interpreter') + link = venv_link(tmp_path / 'external/venv/bin', base) + commands = [] + + def imports_checkout(command, **kwargs): + commands.append(command) + return subprocess.CompletedProcess(command, 0, stdout=str(checkout / 'common/__init__.py')) + + monkeypatch.setattr(storage.subprocess, 'run', imports_checkout) + storage.startup(checkout, link) + assert commands[0][0] == str(link) + + +@pytest.mark.parametrize('inside', ['link', 'target']) +def test_startup_refuses_environment_or_interpreter_inside_checkout(tmp_path, monkeypatch, inside): + checkout = guarded_checkout(tmp_path, monkeypatch) + base = (checkout if inside == 'target' else tmp_path) / 'base-python' + base.write_bytes(b'interpreter') + link = venv_link((checkout if inside == 'link' else tmp_path / 'external') / 'venv/bin', base) + monkeypatch.setattr(storage.subprocess, 'run', lambda *a, **k: pytest.fail('checkout environment was executed')) + with pytest.raises(ValueError, match='outside'): + storage.startup(checkout, link) + + +def test_expired_jobs_do_not_displace_the_jobs_kept(tmp_path): + home = tmp_path / 'content' + kept = [job(home, 1, age=20), job(home, 2, age=21)] + expired = [job(home, 3, age=22), job(home, 4, age=23)] + finished = [(p / 'status.json').stat().st_mtime for p in expired] + store = storage.Store(home, storage.Policy(max_bytes=100000, days=7, keep=2)) + store.prune() + # A second pass is what sees the first pass's rewritten status files. + store.prune() + assert all((p / 'render/frame.png').exists() for p in kept) + assert not any((p / 'render').exists() for p in expired) + assert [json.loads((p / 'status.json').read_text())['completed_at'] for p in expired] == finished + + +@pytest.mark.parametrize('stamp', ['yesterday', True, float('nan')]) +def test_unusable_completion_stamp_falls_back_to_file_time(tmp_path, stamp): + home = tmp_path / 'content' + old = job(home, 1) + status = old / 'status.json' + mtime = status.stat().st_mtime + status.write_text(json.dumps(dict(state='complete', completed_at=stamp))) + os.utime(status, (mtime, mtime)) + job(home, 2, age=0) + storage.Store(home, storage.Policy(max_bytes=100000, days=7, keep=1)).prune() + assert not (old / 'render').exists() + + @pytest.mark.skipif(os.name != 'nt', reason='Windows launcher verification') def test_real_launcher_refuses_local_override_and_historical_resume(tmp_path): root = Path(__file__).resolve().parents[2] diff --git a/tools/storage/README.md b/tools/storage/README.md index 02f43a7..3498c51 100644 --- a/tools/storage/README.md +++ b/tools/storage/README.md @@ -31,6 +31,9 @@ editable `.blend` files, staging/client payloads, unknown files and protected fleet filenames never expire. Queued/building jobs, `.keep` jobs and pinned rebuild sources are excluded. Protected/unknown contents abort the deletion plan before any candidate is removed. Links are never traversed for deletion. +Jobs rank by when they finished: the status file's modification time, or the +`completed_at` Unix timestamp retention writes to `status.json` when it expires a +job, so that rewrite cannot make an expired job look new. `python tools/storage/run.py check` reports current use; `prune` runs the same retention policy immediately. Locks serialize builds and cleanup. An acquired diff --git a/tools/uo-content/client_import.py b/tools/uo-content/client_import.py index d7d5ac7..e2c57b5 100644 --- a/tools/uo-content/client_import.py +++ b/tools/uo-content/client_import.py @@ -65,7 +65,8 @@ def staging_store(out): def stage(vd, client, body, out, _store=None): if _store is not None: return _stage(vd,client,body,out,_store) - with staging_store(out).session() as store: + # The VD usually sits in a finished job, which this session's retention could expire. + with staging_store(out).session(protect=[Path(vd).resolve().parent]) as store: return _stage(vd,client,body,out,store)