From de0fe07792a868706ff194ba1ec9a6753cb31e57 Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:51:38 +0200 Subject: [PATCH 01/22] Extract the runtime seam from the target model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v3 Phase A: a target says which machine a box runs on; a runtime says what runs inside it. Those facts lived in one table, which made every target adapter a statement that a box is a Python box and would have made a second runtime a fork of it rather than one more adapter beside it. The runtime half moves to src/contract/runtimes.mjs — layout, execution kinds, discovery, shell-free argv, self-test probe — mirrored by rust/src/contract/runtimes.rs and python/src/scrollcase_consumer/_contract.py and proven against a new shared fixture, runtime-contract.json. The builder-side half moves under src/runtimes/python/: launcher repair, the pip requirements reader, and the starter files `new scroll` writes. The three places that had bypassed the adapter and hard-coded `venv/`, or re-derived the Windows standard-library path in a branch, now ask the runtime. The four Node call sites that each carried a copy of the unsupported-version message share one helper. No wire format, document, schema or existing fixture changes. --- CHANGELOG.md | 34 ++ docs/reference/api.md | 2 +- docs/white-paper.md | 200 +++++-- python/src/scrollcase_consumer/_contract.py | 357 +++++++++--- python/src/scrollcase_consumer/environment.py | 4 +- python/src/scrollcase_consumer/run.py | 21 +- python/src/scrollcase_consumer/verify.py | 6 +- python/tests/test_contract.py | 170 ++++++ rust/fixtures/runtime-contract.json | 269 +++++++++ rust/scripts/sync-assets.mjs | 1 + rust/src/contract/mod.rs | 1 + rust/src/contract/runtimes.rs | 512 ++++++++++++++++++ rust/src/contract/targets.rs | 105 +--- rust/src/execution.rs | 113 ++-- rust/src/prepare.rs | 3 +- rust/src/release.rs | 28 + rust/src/run.rs | 30 +- rust/tests/contract.rs | 258 +++++++++ rust/tests/release_document.rs | 7 +- src/build/archive.d.mts | 3 +- src/build/archive.mjs | 33 +- src/build/authoring.mjs | 48 +- src/build/box.mjs | 15 +- src/build/dependencies.mjs | 68 +-- src/build/execution.d.mts | 19 +- src/build/execution.mjs | 67 +-- src/build/index.d.mts | 2 +- src/build/index.mjs | 6 +- src/build/launchers.d.mts | 9 - src/build/pixi.d.mts | 23 +- src/build/pixi.mjs | 28 +- src/build/scroll.mjs | 12 +- src/build/verify.mjs | 35 +- src/cli.mjs | 3 +- src/consumer/run-extracted.mjs | 23 +- src/consumer/verify-and-extract.mjs | 5 +- src/contract/document-shape.d.mts | 14 + src/contract/document-shape.mjs | 19 + src/contract/documents.d.mts | 2 +- src/contract/documents.mjs | 5 +- src/contract/fixtures/runtime-contract.json | 269 +++++++++ src/contract/runtimes.d.mts | 284 ++++++++++ src/contract/runtimes.mjs | 341 ++++++++++++ src/contract/targets.d.mts | 29 +- src/contract/targets.mjs | 83 +-- src/runtimes/index.mjs | 37 ++ src/runtimes/python/dependencies.mjs | 72 +++ src/runtimes/python/index.mjs | 44 ++ src/runtimes/python/launchers.d.mts | 10 + src/{build => runtimes/python}/launchers.mjs | 18 +- src/runtimes/python/templates/index.mjs | 54 ++ src/sign/keys.mjs | 8 +- tests/helpers/consumer-box-fixture.mjs | 6 +- tests/helpers/consumer-conformance.mjs | 3 +- tests/unit/build-pipeline.test.mjs | 75 ++- tests/unit/contract-runtimes.test.mjs | 144 +++++ tests/unit/contract-targets.test.mjs | 18 +- tests/unit/execution-contract.test.mjs | 12 +- tests/unit/llm-demo.test.mjs | 3 +- tests/unit/scroll-editing.test.mjs | 3 +- 60 files changed, 3467 insertions(+), 606 deletions(-) create mode 100644 rust/fixtures/runtime-contract.json create mode 100644 rust/src/contract/runtimes.rs delete mode 100644 src/build/launchers.d.mts create mode 100644 src/contract/fixtures/runtime-contract.json create mode 100644 src/contract/runtimes.d.mts create mode 100644 src/contract/runtimes.mjs create mode 100644 src/runtimes/index.mjs create mode 100644 src/runtimes/python/dependencies.mjs create mode 100644 src/runtimes/python/index.mjs create mode 100644 src/runtimes/python/launchers.d.mts rename src/{build => runtimes/python}/launchers.mjs (70%) create mode 100644 src/runtimes/python/templates/index.mjs create mode 100644 tests/unit/contract-runtimes.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 66654d6..2b4290c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,40 @@ All notable changes to Scrollcase are documented here. The format follows ## [Unreleased] +### Changed + +- The box format now models the **runtime** separately from the **target**. A target says which + machine a box runs on; a runtime says what runs inside it — where the interpreter sits, which + execution kinds exist, how a declared entry point becomes a command line, and which inherited + environment variables can change what that command loads. Those facts lived inside the target + adapter, which made every target a statement that a box is a Python box and would have made a + second runtime a fork of that table. They now live in `src/contract/runtimes.mjs`, mirrored by + `rust/src/contract/runtimes.rs` and `python/src/scrollcase_consumer/_contract.py` and proven + against a new shared fixture, `src/contract/fixtures/runtime-contract.json`. **No wire format, + document, schema or existing fixture changes**, and the archive a given commit produces is + byte-for-byte what it produced before. + +- `boxTargetAdapter()` no longer returns a `python` block or a `selfTestPython` string, and its + `executionAffectingEnvironmentVariables` is now the operating system's half of the list only — + `DYLD_INSERT_LIBRARIES` on macOS, `LD_PRELOAD` on Linux, nothing on Windows. The runtime + contributes the `PYTHON*` half, and `executionAffectingVariables(runtimeId, adapter)` joins the + two in the order a diagnostic report prints them. The Rust `BoxTargetAdapter` and the Python + `TargetAdapter` lost the same fields, for the same reason. `assertPythonEntryPoint` keeps its + published name and signature in all three, and delegates to the runtime rule. + +- The builder-side half of a runtime now lives under `src/runtimes//`. `repairPosixLaunchers` + moved from `src/build/launchers.mjs` to `src/runtimes/python/launchers.mjs` — the conda shebang + trampoline it parses is a Python fact, not a build fact — and is still re-exported from + `scrollcase/build` under the same name. The pip `requirements.txt` reader moved beside it, and the + starter script, starter self-test and interpreter constraint that `new scroll` writes moved to + `src/runtimes/python/templates/`. The three places that had bypassed the adapter and hard-coded + `venv/`, or re-derived the Windows standard-library path in a branch, now ask the runtime. + +- The four Node call sites that each carried their own copy of the unsupported-`schemaVersion` + message share one `unsupportedSchemaVersionMessage()` in `src/contract/document-shape.mjs`. The + wording is unchanged; the next format version now has one sentence to edit per language rather + than four. + ## [0.12.0] — 2026-08-22 ### Added diff --git a/docs/reference/api.md b/docs/reference/api.md index 8f59b76..78a0789 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -397,7 +397,7 @@ The single source of truth for what a box is. See [The Box Format](/reference/bo | `condaSubdir` | `(target) => string` | The conda platform subdir (`osx-arm64`, `linux-64`, `win-64`) | | `pixiAccelerator` | `(scroll) => { accelerator, cudaVersion }` | The conda accelerator descriptor a scroll selects, rejecting target drift | | `assertNativeHost` | `(adapter, host = process) => void` | Throws unless the current host matches the adapter's OS and architecture | -| `assertPythonEntryPoint` | `(adapter, entryPoint) => void` | Throws unless the entry point matches the adapter's layout | +| `assertPythonEntryPoint` | `(adapter, entryPoint) => void` | Throws unless the entry point matches the runtime's layout for the target | ```js import { boxTargetId } from 'scrollcase/contract'; diff --git a/docs/white-paper.md b/docs/white-paper.md index b9ad607..e07bceb 100644 --- a/docs/white-paper.md +++ b/docs/white-paper.md @@ -1560,28 +1560,27 @@ different one. #### Target adapters An adapter states what a target implies for the built payload. It is part of the format rather than -an implementation detail, because a consumer unpacking a box relies on that layout to find the -interpreter. +an implementation detail, because a consumer unpacking a box relies on it. | Field | `macos-aarch64` | `linux-x86_64` | `windows-x86_64` | | --- | --- | --- | --- | | `host.platform` / `host.arch` | `darwin` / `arm64` | `linux` / `x64` | `win32` / `x64` | | `condaSubdir` | `osx-arm64` | `linux-64` | `win-64` | -| `python.payloadRoot` | `venv` | `venv` | `venv` | -| `python.entryPoint` | `venv/bin/python` | `venv/bin/python` | `venv/python.exe` | -| `python.scriptsDirectory` | `venv/bin` | `venv/bin` | `venv/Scripts` | -| `python.executableSuffix` | *(empty)* | *(empty)* | `.exe` | -| `python.launcherKind` | `posix-polyglot` | `posix-polyglot` | `uv-windows-pe` | | `nativeLibraryInspection` | `otool -L`, `.dylib` `.so` | `ldd`, `.so` | `dumpbin /DEPENDENTS`, `.dll` `.pyd` | | `validationEnvironments` | `cpu`, `metal` | `cpu`, `cuda` | `cpu`, `cuda` | -| `executionAffectingEnvironmentVariables` | Python controls + `DYLD_INSERT_LIBRARIES` | Python controls + `LD_PRELOAD` | Python controls | -| `selfTestPython` | `assert sys.platform == 'darwin'` | `assert sys.platform.startswith('linux')` | `assert sys.platform == 'win32'` | +| `executionAffectingEnvironmentVariables` | `DYLD_INSERT_LIBRARIES` | `LD_PRELOAD` | *(none)* | | `archive` | shared backend descriptor | shared backend descriptor | shared backend descriptor | Every adapter is deeply frozen, and `boxTargetAdapters()` hands out a fresh array, so a caller cannot mutate the format for everyone else in the process. -Three details deserve their own note. +**What an adapter deliberately does not state is the runtime inside the box.** The interpreter +layout, the execution kinds and the runtime's own environment variables belong to +[the runtime model](#_5-2a-the-runtime-model-—-runtimes-mjs). While they lived here, every target +adapter was also a statement that a box is a Python box, and a second runtime would have been a fork +of this table rather than one more adapter beside it. + +Two details deserve their own note. **`validationEnvironments` are how an accelerator is forced.** Each is a small environment map applied to validation runs — `CUDA_VISIBLE_DEVICES: ''` to force CPU, `CUDA_VISIBLE_DEVICES: '0'` to @@ -1589,37 +1588,30 @@ force CUDA, `PYTORCH_ENABLE_MPS_FALLBACK: '0'` so a Metal run fails loudly inste falling back to CPU. Without that last one, a [parity](#parity) check comparing Metal against CPU could pass by comparing CPU against itself. -**`executionAffectingEnvironmentVariables` drives diagnostics, not policy.** The shared Python set -is `PYTHONPATH`, `PYTHONHOME`, `PYTHONSTARTUP`, and `PYTHONBREAKPOINT`; the two POSIX loaders add -their platform-specific injection variable. Their presence is reported because it can change which -code runs. No adapter filters them. - -**`selfTestPython` is prepended to every self-test**, so the check begins by asserting it is running -on the platform the box claims. A box that somehow reached the wrong operating system fails at the -first line rather than at an import that happens to exist on both. - -**`launcherKind: 'uv-windows-pe'` is a frozen wire string.** It reads like a reference to a tool this -project does not use, and it is: the value is inert, and only names a launcher shape. It is recorded -here because it is the single most likely thing in the contract for a well-meaning cleanup to -"correct", and changing it would change the format for every client that already reads it. +**`executionAffectingEnvironmentVariables` drives diagnostics, not policy**, and is only half the +list. A target contributes the operating system's own dynamic-linker controls — the two POSIX +loaders have one each, and Windows has none worth naming, because `PATH` decides DLL resolution and +is far too broad. The runtime contributes the rest, and +`executionAffectingVariables(runtimeId, adapter)` is what joins the two halves, runtime first. +Their presence is reported because it can change which code runs. No adapter filters them.
-#### Host and layout assertions - -Two guards are exported beside the model, and both are refusals rather than conveniences. +#### Host assertion `assertNativeHost(adapter, host)` refuses to build or lock a target on a machine that is not the one it ships for. There is no cross-compilation: the environment being packed contains native code solved and installed for one platform, and a self-test run on the wrong host would prove nothing about the box. -`assertPythonEntryPoint(adapter, entryPoint)` refuses a scroll whose declared interpreter path -disagrees with the adapter's layout. The entry point is not free-form input — it is a fact about the -target — and accepting a disagreement would produce a signed release whose `pythonEntryPoint` -pointed at nothing. +The layout assertion beside it — `assertPythonEntryPoint(adapter, entryPoint)` — keeps its published +name while the wire format still spells the field `pythonEntryPoint`, and delegates to +`assertRuntimeEntryPoint()` in the runtime model. It refuses a scroll whose declared interpreter +path disagrees with the runtime's layout for that target. The entry point is not free-form input — +it is a fact about the runtime and the target together — and accepting a disagreement would produce +a signed release whose `pythonEntryPoint` pointed at nothing.
@@ -1641,6 +1633,95 @@ Reference: `tests/unit/contract-targets.test.mjs`.
+### 5.2a The runtime model — `runtimes.mjs` + + +A [target](#target) says which machine a box runs on. A **runtime** says what runs inside it: where +the interpreter sits in the payload, which `execution.kind` values exist, how a declared entry point +becomes a command line, and which inherited environment variables can change what that command +loads. Those are different questions, and until the seam was cut they lived in one table. + +The module is contract-level for the same reason `targets.mjs` is: a consumer unpacking a box relies +on the layout, and a consumer running one relies on the argv rule. `fixtures/runtime-contract.json` +is what "the implementations agree" means here, and both mirrors — `rust/src/contract/runtimes.rs` +and `python/src/scrollcase_consumer/_contract.py` — validate themselves against it. + +
+ +
+ +#### What an adapter states + +| Member | Answer for `python` | +| --- | --- | +| `id` | `python` | +| `executionKinds` | `python-script`, `python-module` | +| `executionEnvironmentVariables` | `PYTHONPATH`, `PYTHONHOME`, `PYTHONSTARTUP`, `PYTHONBREAKPOINT` | +| `layout(target)` | `root`, `entryPoint`, `scriptsDirectory`, `standardLibrary`, `executableSuffix`, `launcherKind` | +| `executablePayloadPaths(target)` | the interpreter by name, and the scripts directory by prefix | +| `resolveExecutionFiles({ execution, runtimeVersion, target })` | every payload path the declaration could resolve to, and the message for when none does | +| `buildArgv({ execution, target })` | the shell-free command line, in payload-relative terms | +| `selfTestArgv({ probe, target })` | the arguments that follow the interpreter for a self-test | + +The layout is `venv` on every target; what differs is where the interpreter and its generated +scripts land inside it — `venv/bin/python` and `venv/bin` on POSIX, `venv/python.exe` and +`venv/Scripts` on Windows — and where the standard library is, which is `venv/lib/python.` +on POSIX and `venv/Lib` on Windows, with no interpreter version in the path. + +**`launcherKind: 'uv-windows-pe'` is a frozen wire string.** It reads like a reference to a tool this +project does not use, and it is: the value is inert, and only names a launcher shape. It is recorded +here because it is the single most likely thing in the contract for a well-meaning cleanup to +"correct", and changing it would change the format for every client that already reads it. + +**The self-test opens with a platform assertion**, so the check begins by proving it is running on +the platform the box claims. A box that somehow reached the wrong operating system fails at the +first line rather than at an import that happens to exist on both. + +
+ +
+ +#### Two shapes, and why they are shaped that way + +**`buildArgv` returns payload-relative paths tagged as paths, not a joined command line.** Each +element is `{ kind: 'literal' | 'payload-path', value }`, and the caller resolves the payload paths +against the box root it is holding. A box root is a real filesystem path; returning a joined string +would put "what a Windows path looks like" inside the format, and would make the golden fixture +depend on the host that happened to read it. The three consumers join in their own platform's +terms — Node with `path.join`, Rust with `PathBuf`, Python with `Path.joinpath` — and the fixture +pins the part they must agree on. + +**`resolveExecutionFiles` returns candidates plus a message, rather than throwing.** The caller owns +the error path: `fail()` in the builder, a typed error in each consumer. The wording is part of the +contract, so it lives beside the rule that produces it instead of being restated at every call site. + +Nothing in the module reads a file, joins a host path or starts a process. Every function is a +statement about names, which is what makes the mirror provable at all. The builder-side half — +launcher repair, authoring templates, the pixi dependency a runtime contributes — lives under +`src/runtimes//`, where all three are allowed. + +
+ +
+ +#### One runtime, registered rather than assumed + +Only `python` is registered, and `runtimeAdapter('node')` is a `TypeError` rather than a stub. A +registry that answered for a runtime no build can produce would move the failure somewhere further +down, where the message no longer says what went wrong. + +The wire format carries no runtime declaration yet — a box records a Python entry point and Python +execution kinds and nothing that says "Python" — so a reader that must name one names +`IMPLICIT_RUNTIME_ID`, from a single place. Adding the declaration later changes an argument rather +than starting a hunt for hard-coded strings. + +Reference: `tests/unit/contract-runtimes.test.mjs`, `rust/tests/contract.rs`, +`python/tests/test_contract.py`. + +
+ +
+ ### 5.3 The envelope — `document-shape.mjs` and `documents.mjs` @@ -1735,7 +1816,10 @@ signature. 1. **Refuse `schemaVersion: 1` explicitly**, with the remedy in the message — `Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.` A v1 document is not - reinterpreted, and it is not rejected as merely malformed either; it is named. + reinterpreted, and it is not rejected as merely malformed either; it is named. The wording comes + from `unsupportedSchemaVersionMessage()` in `document-shape.mjs`, so the payload decoder, the key + loader and the release verifier say one thing rather than three copies of it — which is what the + next version bump has to change in one place instead of four. 2. **Refuse anything that fails the shape check.** 3. **Hash the decoded bytes and compare against `payloadSha256`** *before* parsing them as JSON. A truncated or edited document is caught before its contents are read at all. @@ -2353,7 +2437,7 @@ interpreter first runs should be able to see it without following a call graph. | 2 | Validate the build options | `box.mjs` | Channel in `CHANNELS`; weights mode; on-demand refuses `assetArchives` | | 3 | Refuse an unusable host, toolchain or tree | `targets.mjs`, `pixi.mjs`, `scroll.mjs` | `assertNativeHost`; pinned pixi and conda-pack located; `pixi.lock` present and hashed; git revision read, dirty tree refused | | 4 | Prepare the build tree | `box.mjs` | Removes and recreates `//payload/`; clears the target's object directory under `dist/` | -| 5 | Solve, pack and relocate | `pixi.mjs`, `launchers.mjs` | Installs into a build-local pixi workspace, packs it, extracts into `payload/venv/`, repairs it, deletes the workspace and tarball | +| 5 | Solve, pack and relocate | `pixi.mjs`, `runtimes/python/launchers.mjs` | Installs into a build-local pixi workspace, packs it, extracts into `payload/venv/`, repairs it, deletes the workspace and tarball | | 6 | Stage assets | `assets.mjs` | Downloads verified assets, copies verified local files, expands asset archives — the downloads and the archives only when weights are embedded, the local files always | | 7 | Prune | `box.mjs` | Deletes each `prunePaths` entry from the payload | | 8 | Licence inventory | `licenses.mjs` | When the scroll declares a reviewed audit: recomputes from the lock, compares against it, writes `payload/THIRD_PARTY_NOTICES/conda-distributions.json` | @@ -2481,7 +2565,8 @@ installed. path. 6. **The directory names agree with the declarations.** The parent directory must equal `boxId` and the child must equal the canonical [target ID](#target-id). -7. **The entry point agrees with the adapter**, via `assertPythonEntryPoint`. +7. **The entry point agrees with the runtime's layout for the target**, via + `assertRuntimeEntryPoint`. Check 6 deserves its reasoning. The layout is `scrolls///`, and the directory names are *checked context*, not identity: the scroll declares both facts, and the filesystem is required @@ -2550,8 +2635,9 @@ on its own. Deriving in the reader rather than at each use is the whole point. The alternative — a `??` at every call site — spreads the definition of "what this field means when absent" across the builder, where two of them eventually disagree. Here there is one place to read, and a scroll that spells a derived -field out explicitly produces exactly the same object as one that omits it; `assertPythonEntryPoint` -still runs either way, so declaring the wrong interpreter is as much an error as it ever was. +field out explicitly produces exactly the same object as one that omits it; +`assertRuntimeEntryPoint` still runs either way, so declaring the wrong interpreter is as much an +error as it ever was.
@@ -2808,7 +2894,7 @@ lexical check cannot see — and both must say yes.
-### 6.6 Repairing launchers — `launchers.mjs` +### 6.6 Repairing launchers — `runtimes/python/launchers.mjs` Console scripts generated at solve time (`tqdm`, `isympy`, `f2py`, …) carry the build machine's @@ -3018,18 +3104,20 @@ For a **module**, the check enumerates the places Python would find it and asks as a regular file: ```js -// src/build/execution.mjs +// src/contract/runtimes.mjs const relativeCandidates = [`${modulePath}.py`, `${modulePath}/__main__.py`]; -const standardLibrary = adapter.platform === 'windows' - ? 'venv/Lib' - : `venv/lib/python${pythonMajorMinor(pythonVersion)}`; +const standardLibrary = target.platform === 'windows' + ? layout.standardLibrary + : `${layout.standardLibrary}/python${pythonMajorMinor(runtimeVersion)}`; const roots = ['', standardLibrary, `${standardLibrary}/site-packages`]; ``` Both `foo/bar.py` and `foo/bar/__main__.py` are accepted, since `python -m` runs either. The roots are the payload root, the standard library and `site-packages`, and the Windows standard library lives at `venv/Lib` rather than under a version-named directory — one of the three-target -differences that no single-host test suite can catch. +differences that no single-host test suite can catch. It is a *data* difference now rather than a +branch: `standardLibrary` is a field of the runtime layout, and `src/build/execution.mjs` asks for +candidates rather than deriving them. **Rejected:** proving a module by importing it. Importing runs `__init__.py`, which is application code, and would turn validation into execution before the trust chain has finished. The whole point @@ -3592,7 +3680,9 @@ No version is looked up. An added dependency defaults to `*` and the committed ` what was actually solved. **Rejected:** asking the network for a "latest" to write into the manifest, which would put a second, weaker pin beside the real one and leave the two to drift. -`readRequirements` translates a pip `requirements.txt`. The table of PyPI names whose conda-forge +`readRequirements`, in `src/runtimes/python/dependencies.mjs`, translates a pip `requirements.txt` — +a Python fact, kept beside the runtime rather than in the substrate module that edits the manifest. +The table of PyPI names whose conda-forge package is called something else is deliberately short — every entry is one this project can state with confidence — and **every rename and every skip is reported**, because a name guessed wrongly produces a lock that resolves and a box that cannot import what it was built for. That failure @@ -3920,7 +4010,7 @@ theatre. Every code path that consumes a signed document — `build` re-verifyin // src/sign/keys.mjs export function decodeSignedDocument(document) { if (document?.schemaVersion === 1) { - fail('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); + fail(unsupportedSchemaVersionMessage(1)); } if (document?.schemaVersion !== BOX_SCHEMA_VERSION || document?.payloadEncoding !== PAYLOAD_ENCODING) { fail('Unsupported signed document.'); @@ -6087,6 +6177,7 @@ line over all of them. The Rust crate follows at the end, since it ships separat | `src/contract/index.mjs` | The contract entry point: the single source of truth for what a box is | 5.1 | | `src/contract/browser.mjs` | The same model without any Node built-in, for a browser or an edge runtime | 5.1 | | `src/contract/targets.mjs` | The [target](#target) model, the identity rule, and the adapter per target | 5.2 | +| `src/contract/runtimes.mjs` | The runtime model: layout, execution kinds, discovery and shell-free argv | 5.2 | | `src/contract/document-shape.mjs` | The platform-neutral parts of the [envelope](#envelope): shape checks and namespacing | 5.3 | | `src/contract/documents.mjs` | The envelope reference implementation, including payload decoding | 5.3 | | `src/contract/links.mjs` | The rule deciding which [symbolic links](#symbolic-link) a payload may carry | 5.4 | @@ -6106,7 +6197,6 @@ line over all of them. The Rust crate follows at the end, since it ships separat | `src/build/workspace.mjs` | [Workspace](#workspace) discovery and path resolution | 6.3 | | `src/build/schema-validation.mjs` | Dependency-free runtime validation against the shipped schemas | 6.4 | | `src/build/pixi.mjs` | Tool discovery, the exact pixi and conda-pack arguments, packing and relocation | 6.5 | -| `src/build/launchers.mjs` | Repairing the console scripts a conda environment generates | 6.6 | | `src/build/assets.mjs` | Verified download, verified copy, archive expansion, and the publish-ready move | 6.7 | | `src/build/licenses.mjs` | The [SPDX](#spdx) licence inventory derived from the [lockfile](#lockfile) | 6.8 | | `src/build/audit.mjs` | `auditScroll` — the inventory as a command, with the reviewed-copy comparison | 6.8 | @@ -6128,6 +6218,24 @@ line over all of them. The Rust crate follows at the end, since it ships separat
+#### `src/runtimes/` — what the substrate packs + +The builder-side half of the runtime seam. `src/contract/runtimes.mjs` states what a consumer must +agree with and is mirrored in every language; these modules are the builder's alone, and they are +what a box's runtime is allowed to need that another runtime would not. + +| Module | Role | Section | +| --- | --- | --- | +| `src/runtimes/index.mjs` | The registry of builder-side runtime adapters | 6.6 | +| `src/runtimes/python/index.mjs` | The Python adapter: its pixi dependency, its launcher repair, its starter files | 6.6 | +| `src/runtimes/python/launchers.mjs` | Repairing the console scripts a conda environment generates | 6.6 | +| `src/runtimes/python/dependencies.mjs` | Reading a pip `requirements.txt` into conda-forge terms | 6.16 | +| `src/runtimes/python/templates/index.mjs` | The Python source `new scroll` writes, and the interpreter constraint a generated manifest declares | 6.16 | + +
+ +
+ #### `src/sign/` and `src/consumer/` | Module | Role | Section | @@ -6171,7 +6279,7 @@ Published separately, and listed here because it implements the same section 8 a | --- | --- | --- | | `error.rs` | One opaque error type and the `fail!` macro — the single failure path, deliberately not an enum a caller could match on and come to depend on | 8.1 | | `path.rs` | The path-safety primitive every extraction and attachment goes through | 8.2 | -| `contract/` | The mirror: `targets.rs`, `documents.rs`, `links.rs`, `payload_digest.rs` | 5.2–5.5 | +| `contract/` | The mirror: `targets.rs`, `runtimes.rs`, `documents.rs`, `links.rs`, `payload_digest.rs` | 5.2–5.5 | | `trust.rs` | Trust anchors from either source, key rotation, and strict ed25519 verification | 7.4 | | `release.rs` | The typed release and box manifests, refusing an unknown field where the others run a schema — except in `compatibility`, the one object the schema leaves open, whose unfamiliar constraints are carried to the caller | 8.1 | | `archive.rs` | Defensive reading and extraction, including the duplicate-name check the ZIP backend cannot make; that check locates EOCD or EOCD64 and streams only the declared central-directory records, because identical index bytes inside stored nested archives are payload data | 8.2, 8.6 | @@ -6231,7 +6339,7 @@ consumer-only dependent avoid the entire build layer. | `condaSubdir` | function | The [conda subdir](#conda-subdir) a target maps to | | `pixiAccelerator` | function | The accelerator descriptor a scroll selects | | `assertNativeHost` | function | Refuses a build on a host that is not the target it ships for | -| `assertPythonEntryPoint` | function | Refuses an entry point that disagrees with the adapter layout | +| `assertPythonEntryPoint` | function | Refuses an entry point that disagrees with the runtime's layout for the target | | `documentKinds` | function | The three `kind` strings for a publishing project's namespace | | `parseDocumentKind` | function | Splits a `kind` back into namespace and document type | | `isSignedBoxDocument` | function | The structural envelope guard — shape only, never trust | diff --git a/python/src/scrollcase_consumer/_contract.py b/python/src/scrollcase_consumer/_contract.py index ab6cb38..6df7e93 100644 --- a/python/src/scrollcase_consumer/_contract.py +++ b/python/src/scrollcase_consumer/_contract.py @@ -42,14 +42,16 @@ @dataclass(frozen=True, slots=True) class TargetAdapter: - """Runtime-relevant target layout mirrored from the canonical target adapters.""" + """What a target implies for the extracted tree, mirrored from the canonical adapters. + + Deliberately no interpreter layout and no Python environment variables: those are facts about + what a box *runs*, not about the machine it runs on, and they live on :class:`RuntimeAdapter`. + """ platform: str arch: str host_platform: str host_arch: str - python_entry_point: str - standard_library: str execution_affecting_environment_variables: tuple[str, ...] @@ -59,44 +61,23 @@ class TargetAdapter: arch="aarch64", host_platform="darwin", host_arch="aarch64", - python_entry_point="venv/bin/python", - standard_library="venv/lib", - execution_affecting_environment_variables=( - "PYTHONPATH", - "PYTHONHOME", - "PYTHONSTARTUP", - "PYTHONBREAKPOINT", - "DYLD_INSERT_LIBRARIES", - ), + execution_affecting_environment_variables=("DYLD_INSERT_LIBRARIES",), ), ("linux", "x86_64"): TargetAdapter( platform="linux", arch="x86_64", host_platform="linux", host_arch="x86_64", - python_entry_point="venv/bin/python", - standard_library="venv/lib", - execution_affecting_environment_variables=( - "PYTHONPATH", - "PYTHONHOME", - "PYTHONSTARTUP", - "PYTHONBREAKPOINT", - "LD_PRELOAD", - ), + execution_affecting_environment_variables=("LD_PRELOAD",), ), ("windows", "x86_64"): TargetAdapter( platform="windows", arch="x86_64", host_platform="win32", host_arch="x86_64", - python_entry_point="venv/python.exe", - standard_library="venv/Lib", - execution_affecting_environment_variables=( - "PYTHONPATH", - "PYTHONHOME", - "PYTHONSTARTUP", - "PYTHONBREAKPOINT", - ), + # Windows has no inherited loader control of its own worth reporting: ``PATH`` decides DLL + # resolution and is far too broad to name here, so the whole list is the runtime's. + execution_affecting_environment_variables=(), ), } _ACCELERATORS = { @@ -106,6 +87,278 @@ class TargetAdapter: } +def _python_major_minor(version: str) -> str: + """The ``major.minor`` prefix naming the standard-library directory a packed prefix carries. + + A patch component is dropped rather than rejected: a scroll may pin ``3.14.2``, and the + directory conda-forge writes is ``python3.14`` either way. + """ + + match = re.match(r"^(\d+)\.(\d+)(?:\.|$)", version) + if match is None: + raise ScrollcaseConsumerError( + f"Invalid Python version for execution discovery: {version}." + ) + return f"{match.group(1)}.{match.group(2)}" + + +@dataclass(frozen=True, slots=True) +class RuntimeLayout: + """Where a runtime lives inside an extracted box.""" + + root: str + entry_point: str + scripts_directory: str + standard_library: str + executable_suffix: str + launcher_kind: str + + +@dataclass(frozen=True, slots=True) +class ExecutablePayloadPaths: + """Payload paths a runtime requires the executable bit on, as a rule rather than a list. + + A conda prefix carries hundreds of generated console scripts and no scroll could name them by + hand, so the scripts directory matches by prefix while the runtime's own entry point — which + lives outside it on Windows — matches by name. + """ + + files: tuple[str, ...] + directories: tuple[str, ...] + + def matches(self, relative_path: str) -> bool: + """Whether this path is one the runtime needs the executable bit on.""" + + if relative_path in self.files: + return True + return any( + relative_path.startswith(f"{directory}/") for directory in self.directories + ) + + +@dataclass(frozen=True, slots=True) +class RuntimeArgument: + """One element of a shell-free command line. + + A ``payload-path`` stays relative and tagged rather than joined, because a box root is a real + path on this host and each implementation joins one in its own terms. + """ + + kind: str + value: str + + +@dataclass(frozen=True, slots=True) +class RuntimeInvocation: + """A shell-free command line, before the caller's own arguments.""" + + command: RuntimeArgument + args: tuple[RuntimeArgument, ...] + + +@dataclass(frozen=True, slots=True) +class ResolvedExecutionFiles: + """Every payload path a declaration could resolve to, and what to say when none does.""" + + candidates: tuple[str, ...] + missing: str + + +@dataclass(frozen=True, slots=True) +class RuntimeAdapter: + """What a runtime implies for a box, independent of the machine it runs on. + + Mirrored from ``src/contract/runtimes.mjs`` and proven against + ``src/contract/fixtures/runtime-contract.json``. Keeping the interpreter layout here rather than + on :class:`TargetAdapter` is what stops every target from being a statement that a box is a + Python box. + """ + + id: str + execution_kinds: tuple[str, ...] + execution_environment_variables: tuple[str, ...] + _layouts: Mapping[str, RuntimeLayout] + _platform_assertions: Mapping[str, str] + + def layout(self, platform: str) -> RuntimeLayout: + """Where this runtime sits inside a box built for *platform*.""" + + layout = self._layouts.get(platform) + if layout is None: + raise ScrollcaseConsumerError( + f"No {self.id} runtime layout exists for platform {platform}" + ) + return layout + + def executable_payload_paths(self, platform: str) -> ExecutablePayloadPaths: + """Payload paths this runtime requires the executable bit on.""" + + layout = self.layout(platform) + return ExecutablePayloadPaths( + files=(layout.entry_point,), directories=(layout.scripts_directory,) + ) + + def resolve_execution_files( + self, + execution: BoxExecution, + platform: str, + runtime_version: str, + ) -> ResolvedExecutionFiles: + """Every payload path a declaration could resolve to, and the message when none does.""" + + if execution.kind not in self.execution_kinds: + raise ScrollcaseConsumerError( + f"Unsupported execution kind: {execution.kind}." + ) + layout = self.layout(platform) + if isinstance(execution, PythonScriptExecution): + return ResolvedExecutionFiles( + candidates=(execution.script,), + missing=( + f"Execution script is missing from the box: {execution.script}." + ), + ) + module_path = execution.module.replace(".", "/") + relative = (f"{module_path}.py", f"{module_path}/__main__.py") + # Windows names its standard library once, with no interpreter version in the path; every + # other platform carries ``python.`` under it. + standard_library = ( + layout.standard_library + if platform == "windows" + else f"{layout.standard_library}/python{_python_major_minor(runtime_version)}" + ) + roots = ("", standard_library, f"{standard_library}/site-packages") + return ResolvedExecutionFiles( + candidates=tuple( + f"{root}/{candidate}" if root else candidate + for root in roots + for candidate in relative + ), + missing=( + f"Execution module is not discoverable in the box: {execution.module}." + ), + ) + + def build_argv(self, execution: BoxExecution, platform: str) -> RuntimeInvocation: + """The shell-free command line that runs a declaration, in payload-relative terms.""" + + if execution.kind not in self.execution_kinds: + raise ScrollcaseConsumerError( + f"Unsupported execution kind: {execution.kind}." + ) + layout = self.layout(platform) + if isinstance(execution, PythonScriptExecution): + args = [RuntimeArgument("payload-path", execution.script)] + else: + args = [ + RuntimeArgument("literal", "-m"), + RuntimeArgument("literal", execution.module), + ] + args.extend( + RuntimeArgument("literal", value) for value in execution.default_args + ) + return RuntimeInvocation( + command=RuntimeArgument("payload-path", layout.entry_point), + args=tuple(args), + ) + + def self_test_argv( + self, imports: Iterable[str], platform: str, code: str | None = None + ) -> tuple[str, ...]: + """The arguments that follow this runtime's entry point when it runs a self-test probe.""" + + assertion = self._platform_assertions.get(platform) + if assertion is None: + raise ScrollcaseConsumerError( + f"No {self.id} self-test assertion exists for platform {platform}" + ) + body = f"import {', '.join(imports)}" + source = f"{assertion}\n{body}\n{code}" if code else f"{assertion}\n{body}" + return ("-c", source) + + +_POSIX_PYTHON_LAYOUT = RuntimeLayout( + root="venv", + entry_point="venv/bin/python", + scripts_directory="venv/bin", + standard_library="venv/lib", + executable_suffix="", + launcher_kind="posix-polyglot", +) + +_RUNTIMES = { + "python": RuntimeAdapter( + id="python", + execution_kinds=("python-script", "python-module"), + execution_environment_variables=( + "PYTHONPATH", + "PYTHONHOME", + "PYTHONSTARTUP", + "PYTHONBREAKPOINT", + ), + _layouts={ + "macos": _POSIX_PYTHON_LAYOUT, + "linux": _POSIX_PYTHON_LAYOUT, + "windows": RuntimeLayout( + root="venv", + entry_point="venv/python.exe", + scripts_directory="venv/Scripts", + standard_library="venv/Lib", + executable_suffix=".exe", + # Reads like a stale reference to a tool this project does not use. It is a frozen + # wire string under the published format; it must not be "cleaned". + launcher_kind="uv-windows-pe", + ), + }, + _platform_assertions={ + "macos": "import sys; assert sys.platform == 'darwin'", + "linux": "import sys; assert sys.platform.startswith('linux')", + "windows": "import sys; assert sys.platform == 'win32'", + }, + ) +} + +#: The runtime every box built by this schema version implicitly declares. +#: +#: The wire format has no runtime field: a box records a Python entry point and Python execution +#: kinds and nothing that says "Python". So a reader that must name a runtime names this one, from +#: one place. +IMPLICIT_RUNTIME_ID = "python" + + +def runtime_adapter(runtime_id: str = IMPLICIT_RUNTIME_ID) -> RuntimeAdapter: + """Return the runtime adapter for a runtime id.""" + + runtime = _RUNTIMES.get(runtime_id) + if runtime is None: + raise ScrollcaseConsumerError( + f"No box runtime adapter exists for {runtime_id}" + ) + return runtime + + +def runtime_adapters() -> tuple[RuntimeAdapter, ...]: + """Every runtime adapter, for contract tests and callers enumerating what a box may be.""" + + return tuple(_RUNTIMES.values()) + + +def execution_affecting_variables( + adapter: TargetAdapter, runtime_id: str = IMPLICIT_RUNTIME_ID +) -> tuple[str, ...]: + """The complete list of inherited variables that can change what a box executes. + + Two halves, because they have two owners: the runtime contributes the variables its own loader + reads, and the target contributes the operating system's dynamic-linker controls. The order is + what a diagnostic report is printed in, so it is part of the answer. + """ + + return ( + *runtime_adapter(runtime_id).execution_environment_variables, + *adapter.execution_affecting_environment_variables, + ) + + def safe_relative_path(value: object) -> str: """Return a forward-slash relative path that cannot leave a box root.""" @@ -244,54 +497,32 @@ def required_assets_from_json(values: list[Mapping[str, Any]] | None) -> tuple[R ) -def _python_major_minor(version: str) -> str: - match = re.match(r"^(\d+)\.(\d+)(?:\.|$)", version) - if match is None: - raise ScrollcaseConsumerError( - f"Invalid Python version for execution discovery: {version}." - ) - return f"{match.group(1)}.{match.group(2)}" - - def assert_execution_files( execution: BoxExecution | None, target: BoxTarget, - python_version: str, + runtime_version: str, resolvable_paths: Collection[str], ) -> None: """Prove a signed script or module resolves from a payload path. A payload link resolves to a regular file inside the same payload, so the caller passes links alongside regular files: a box may reach its entry point through one. + + Which paths a declaration could resolve to is the runtime's rule; what stays here is the + traversal rule every candidate goes through, applied to all of them rather than only the one a + scroll wrote by hand. """ if execution is None: return - if isinstance(execution, PythonScriptExecution): - script = safe_relative_path(execution.script) - if script not in resolvable_paths: - raise ScrollcaseConsumerError( - f"Execution script is missing from the box: {script}." - ) - return - adapter = target_adapter(target) - module_path = execution.module.replace(".", "/") - candidates = (f"{module_path}.py", f"{module_path}/__main__.py") - version = _python_major_minor(python_version) - standard_library = ( - adapter.standard_library - if target.platform == "windows" - else f"{adapter.standard_library}/python{version}" + target_adapter(target) + resolved = runtime_adapter().resolve_execution_files( + execution, target.platform, runtime_version ) - roots = ("", standard_library, f"{standard_library}/site-packages") - if not any( - (f"{root}/{candidate}" if root else candidate) in resolvable_paths - for root in roots - for candidate in candidates - ): - raise ScrollcaseConsumerError( - f"Execution module is not discoverable in the box: {execution.module}." - ) + for candidate in resolved.candidates: + if safe_relative_path(candidate) in resolvable_paths: + return + raise ScrollcaseConsumerError(resolved.missing) @lru_cache(maxsize=1) diff --git a/python/src/scrollcase_consumer/environment.py b/python/src/scrollcase_consumer/environment.py index 92b5032..e75e7e5 100644 --- a/python/src/scrollcase_consumer/environment.py +++ b/python/src/scrollcase_consumer/environment.py @@ -10,7 +10,7 @@ from collections.abc import Mapping, Sequence from typing import cast -from ._contract import target_adapter +from ._contract import execution_affecting_variables, target_adapter from .errors import ScrollcaseConsumerError from .models import ( BoxTarget, @@ -76,7 +76,7 @@ def resolve_environment( dangerous = { _normalized_name(name, target.platform) - for name in adapter.execution_affecting_environment_variables + for name in execution_affecting_variables(adapter) } variables: list[tuple[EnvironmentVariableReport, bool]] = [] for normalized, sources in records.items(): diff --git a/python/src/scrollcase_consumer/run.py b/python/src/scrollcase_consumer/run.py index 3febe98..f6651e5 100644 --- a/python/src/scrollcase_consumer/run.py +++ b/python/src/scrollcase_consumer/run.py @@ -24,6 +24,7 @@ assert_execution_files, assert_native_host, path_under, + runtime_adapter, ) from .errors import ScrollcaseConsumerError from .environment import resolve_environment @@ -32,8 +33,6 @@ BoxRunResult, EnvironmentReport, PreparedBox, - PythonModuleExecution, - PythonScriptExecution, ) from .verify import prepared_box_state, verify_and_extract_box, verify_required_assets @@ -165,15 +164,15 @@ def run_extracted_box( verify_required_assets(Path(prepared.root), prepared.required_assets) python = path_under(root, prepared.python_entry_point) - if isinstance(execution, PythonScriptExecution): - execution_args = [str(path_under(root, execution.script))] - elif isinstance(execution, PythonModuleExecution): - execution_args = ["-m", execution.module] - else: - raise ScrollcaseConsumerError( - f"Unsupported execution kind: {execution.kind}." - ) - execution_args.extend(execution.default_args) + # The runtime states the command line in payload-relative terms and this end joins it: a box + # root is a real path on this host, and the format has no business deciding what one looks like. + invocation = runtime_adapter().build_argv(execution, state.target.platform) + execution_args = [ + str(path_under(root, argument.value)) + if argument.kind == "payload-path" + else argument.value + for argument in invocation.args + ] execution_args.extend(caller_args) environment, environment_report = resolve_environment( state.target, diff --git a/python/src/scrollcase_consumer/verify.py b/python/src/scrollcase_consumer/verify.py index 5433a82..dbbe069 100644 --- a/python/src/scrollcase_consumer/verify.py +++ b/python/src/scrollcase_consumer/verify.py @@ -34,6 +34,7 @@ parse_payload_digest_stream, path_under, required_assets_from_json, + runtime_adapter, target_adapter, target_from_json, target_id, @@ -301,10 +302,11 @@ def _inspect_release_document( target = target_from_json(cast(dict[str, Any], release["target"])) adapter = target_adapter(target) - if release["pythonEntryPoint"] != adapter.python_entry_point: + expected_entry_point = runtime_adapter().layout(adapter.platform).entry_point + if release["pythonEntryPoint"] != expected_entry_point: raise ScrollcaseConsumerError( f"{adapter.platform}-{adapter.arch} boxes must use Python entry point " - f"{adapter.python_entry_point}" + f"{expected_entry_point}" ) return _InspectedRelease( release_path=release_path, diff --git a/python/tests/test_contract.py b/python/tests/test_contract.py index 07230a0..1e6f6a3 100644 --- a/python/tests/test_contract.py +++ b/python/tests/test_contract.py @@ -14,13 +14,19 @@ from typing import Any, cast from scrollcase_consumer._contract import ( + IMPLICIT_RUNTIME_ID, PAYLOAD_DIGEST_FILE, PAYLOAD_DIGEST_FORMAT, SCHEMA_FILES, PayloadDigestEntry, absolute_path, + execution_affecting_variables, + execution_from_json, parse_payload_digest_stream, payload_digest_stream, + runtime_adapter, + runtime_adapters, + target_adapter, target_from_json, target_id, ) @@ -69,6 +75,170 @@ def test_bundled_schemas_are_exact_generated_copies(self) -> None: self.assertEqual(result.returncode, 0, result.stderr) +class RuntimeContractTests(unittest.TestCase): + """The Python half of the shared runtime vectors. + + These drive ``src/contract/fixtures/runtime-contract.json``, the same file + ``tests/unit/contract-runtimes.test.mjs`` and ``rust/tests/contract.rs`` drive. Where a runtime + lives inside a box, which paths it needs the executable bit on, what a declared execution could + resolve to and the command line that runs it are all statements three implementations have to + agree on, and this is where this one is held to them. + """ + + @staticmethod + def _load() -> dict[str, Any]: + repo_root = Path(__file__).resolve().parents[2] + return cast( + dict[str, Any], + json.loads( + ( + repo_root + / "src" + / "contract" + / "fixtures" + / "runtime-contract.json" + ).read_text(encoding="utf-8") + ), + ) + + def test_exposes_exactly_the_runtimes_the_fixture_describes(self) -> None: + fixture = self._load() + self.assertEqual( + [runtime.id for runtime in runtime_adapters()], + [case["id"] for case in fixture["runtimes"]], + ) + + def test_refuses_a_runtime_the_format_does_not_define(self) -> None: + for runtime_id in ("node", "native", ""): + with self.subTest(runtime_id=runtime_id): + with self.assertRaises(ScrollcaseConsumerError): + runtime_adapter(runtime_id) + + def test_reproduces_every_golden_layout_and_executable_rule(self) -> None: + for case in self._load()["runtimes"]: + runtime = runtime_adapter(case["id"]) + self.assertEqual( + list(runtime.execution_kinds), case["executionKinds"] + ) + self.assertEqual( + list(runtime.execution_environment_variables), + case["executionEnvironmentVariables"], + ) + for platform in case["layouts"]: + with self.subTest(runtime=case["id"], platform=platform["platform"]): + layout = runtime.layout(platform["platform"]) + self.assertEqual( + { + "root": layout.root, + "entryPoint": layout.entry_point, + "scriptsDirectory": layout.scripts_directory, + "standardLibrary": layout.standard_library, + "executableSuffix": layout.executable_suffix, + "launcherKind": layout.launcher_kind, + }, + platform["layout"], + ) + rule = runtime.executable_payload_paths(platform["platform"]) + self.assertEqual( + { + "files": list(rule.files), + "directories": list(rule.directories), + }, + platform["executablePayloadPaths"], + ) + + def test_answers_the_executable_question_the_same_way(self) -> None: + for case in self._load()["executableMatches"]: + with self.subTest(case=case["name"]): + rule = runtime_adapter(case["runtime"]).executable_payload_paths( + case["platform"] + ) + self.assertEqual(rule.matches(case["path"]), case["executable"]) + + def test_derives_exactly_the_golden_candidate_list(self) -> None: + for case in self._load()["executionDiscovery"]: + with self.subTest(case=case["name"]): + execution = execution_from_json(case["execution"]) + assert execution is not None + resolved = runtime_adapter(case["runtime"]).resolve_execution_files( + execution, case["platform"], case["runtimeVersion"] + ) + self.assertEqual(list(resolved.candidates), case["candidates"]) + + def test_refuses_a_runtime_version_that_cannot_name_a_standard_library(self) -> None: + execution = execution_from_json( + {"kind": "python-module", "module": "pkg", "defaultArgs": []} + ) + assert execution is not None + for invalid in self._load()["invalidRuntimeVersions"]: + with self.subTest(runtime_version=invalid): + with self.assertRaisesRegex( + ScrollcaseConsumerError, "Invalid Python version" + ): + runtime_adapter().resolve_execution_files( + execution, "linux", invalid + ) + + def test_builds_exactly_the_golden_shell_free_command_line(self) -> None: + for case in self._load()["argv"]: + with self.subTest(case=case["name"]): + execution = execution_from_json(case["execution"]) + assert execution is not None + invocation = runtime_adapter(case["runtime"]).build_argv( + execution, case["platform"] + ) + self.assertEqual( + { + "kind": invocation.command.kind, + "value": invocation.command.value, + }, + case["command"], + ) + self.assertEqual( + [ + {"kind": argument.kind, "value": argument.value} + for argument in invocation.args + ], + case["args"], + ) + + def test_turns_every_golden_probe_into_the_same_arguments(self) -> None: + for case in self._load()["selfTest"]: + with self.subTest(case=case["name"]): + argv = runtime_adapter(case["runtime"]).self_test_argv( + case["probe"]["imports"], + case["platform"], + case["probe"].get("code"), + ) + self.assertEqual(list(argv), case["args"]) + + def test_joins_the_runtime_half_to_the_target_half_runtime_first(self) -> None: + # The order is what a diagnostic report is printed in, so it is part of the answer rather + # than an accident of how the two lists happened to be concatenated. + for platform, arch, operating_system in ( + ("macos", "aarch64", "DYLD_INSERT_LIBRARIES"), + ("linux", "x86_64", "LD_PRELOAD"), + ): + with self.subTest(platform=platform): + adapter = target_adapter( + target_from_json( + {"platform": platform, "arch": arch, "accelerator": "cpu"} + ) + ) + merged = execution_affecting_variables(adapter) + self.assertEqual( + list(merged), + [ + *runtime_adapter( + IMPLICIT_RUNTIME_ID + ).execution_environment_variables, + *adapter.execution_affecting_environment_variables, + ], + ) + self.assertIn("PYTHONPATH", merged) + self.assertIn(operating_system, merged) + + class PayloadDigestContractTests(unittest.TestCase): """The Python half of the canonical entry list, against the shared vectors. diff --git a/rust/fixtures/runtime-contract.json b/rust/fixtures/runtime-contract.json new file mode 100644 index 0000000..f567304 --- /dev/null +++ b/rust/fixtures/runtime-contract.json @@ -0,0 +1,269 @@ +{ + "description": "Golden cases for the Scrollcase runtime model: where a runtime lives inside a box, which payload paths it needs the executable bit on, which paths a declared execution could resolve to, and the shell-free command line that runs it. Every implementation of the format proves its mirror against this file. Paths stay payload-relative on purpose: a box root is a real filesystem path and each language joins one in its own terms, so a joined expectation here would only pin the host that read it.", + "runtimes": [ + { + "id": "python", + "executionKinds": ["python-script", "python-module"], + "executionEnvironmentVariables": [ + "PYTHONPATH", + "PYTHONHOME", + "PYTHONSTARTUP", + "PYTHONBREAKPOINT" + ], + "layouts": [ + { + "platform": "macos", + "layout": { + "root": "venv", + "entryPoint": "venv/bin/python", + "scriptsDirectory": "venv/bin", + "standardLibrary": "venv/lib", + "executableSuffix": "", + "launcherKind": "posix-polyglot" + }, + "executablePayloadPaths": { + "files": ["venv/bin/python"], + "directories": ["venv/bin"] + } + }, + { + "platform": "linux", + "layout": { + "root": "venv", + "entryPoint": "venv/bin/python", + "scriptsDirectory": "venv/bin", + "standardLibrary": "venv/lib", + "executableSuffix": "", + "launcherKind": "posix-polyglot" + }, + "executablePayloadPaths": { + "files": ["venv/bin/python"], + "directories": ["venv/bin"] + } + }, + { + "platform": "windows", + "layout": { + "root": "venv", + "entryPoint": "venv/python.exe", + "scriptsDirectory": "venv/Scripts", + "standardLibrary": "venv/Lib", + "executableSuffix": ".exe", + "launcherKind": "uv-windows-pe" + }, + "executablePayloadPaths": { + "files": ["venv/python.exe"], + "directories": ["venv/Scripts"] + } + } + ] + } + ], + "executableMatches": [ + { + "name": "the interpreter itself, by exact name", + "runtime": "python", + "platform": "linux", + "path": "venv/bin/python", + "executable": true + }, + { + "name": "a generated console script, by directory", + "runtime": "python", + "platform": "linux", + "path": "venv/bin/tqdm", + "executable": true + }, + { + "name": "a nested path under the scripts directory", + "runtime": "python", + "platform": "linux", + "path": "venv/bin/nested/tool", + "executable": true + }, + { + "name": "a library the interpreter loads but never executes", + "runtime": "python", + "platform": "linux", + "path": "venv/lib/python3.14/os.py", + "executable": false + }, + { + "name": "a sibling directory whose name only starts the same way", + "runtime": "python", + "platform": "linux", + "path": "venv/binary-blob", + "executable": false + }, + { + "name": "the application's own entry point, which the runtime never claims", + "runtime": "python", + "platform": "linux", + "path": "entrypoint.py", + "executable": false + }, + { + "name": "the Windows interpreter, which sits outside its scripts directory", + "runtime": "python", + "platform": "windows", + "path": "venv/python.exe", + "executable": true + }, + { + "name": "a Windows console script", + "runtime": "python", + "platform": "windows", + "path": "venv/Scripts/pip.exe", + "executable": true + }, + { + "name": "the POSIX scripts directory on a Windows box", + "runtime": "python", + "platform": "windows", + "path": "venv/bin/tqdm", + "executable": false + } + ], + "executionDiscovery": [ + { + "name": "a script resolves at exactly the path it declares", + "runtime": "python", + "platform": "linux", + "runtimeVersion": "3.11.15", + "execution": { "kind": "python-script", "script": "app/main.py", "defaultArgs": [] }, + "candidates": ["app/main.py"] + }, + { + "name": "a POSIX module is looked for at the root, in the standard library, and in site-packages", + "runtime": "python", + "platform": "linux", + "runtimeVersion": "3.11.15", + "execution": { + "kind": "python-module", + "module": "example_model.main", + "defaultArgs": ["--serve"] + }, + "candidates": [ + "example_model/main.py", + "example_model/main/__main__.py", + "venv/lib/python3.11/example_model/main.py", + "venv/lib/python3.11/example_model/main/__main__.py", + "venv/lib/python3.11/site-packages/example_model/main.py", + "venv/lib/python3.11/site-packages/example_model/main/__main__.py" + ] + }, + { + "name": "a patch component names the same standard library directory", + "runtime": "python", + "platform": "macos", + "runtimeVersion": "3.12.4", + "execution": { "kind": "python-module", "module": "pkg", "defaultArgs": [] }, + "candidates": [ + "pkg.py", + "pkg/__main__.py", + "venv/lib/python3.12/pkg.py", + "venv/lib/python3.12/pkg/__main__.py", + "venv/lib/python3.12/site-packages/pkg.py", + "venv/lib/python3.12/site-packages/pkg/__main__.py" + ] + }, + { + "name": "Windows names its standard library once, without an interpreter version", + "runtime": "python", + "platform": "windows", + "runtimeVersion": "3.11.15", + "execution": { "kind": "python-module", "module": "pkg", "defaultArgs": [] }, + "candidates": [ + "pkg.py", + "pkg/__main__.py", + "venv/Lib/pkg.py", + "venv/Lib/pkg/__main__.py", + "venv/Lib/site-packages/pkg.py", + "venv/Lib/site-packages/pkg/__main__.py" + ] + } + ], + "invalidRuntimeVersions": ["", "3", "3.x", "x.1", "3."], + "argv": [ + { + "name": "a script runs as a payload path, with its declared arguments after it", + "runtime": "python", + "platform": "linux", + "execution": { + "kind": "python-script", + "script": "app/main.py", + "defaultArgs": ["--serve", "--port", "8080"] + }, + "command": { "kind": "payload-path", "value": "venv/bin/python" }, + "args": [ + { "kind": "payload-path", "value": "app/main.py" }, + { "kind": "literal", "value": "--serve" }, + { "kind": "literal", "value": "--port" }, + { "kind": "literal", "value": "8080" } + ] + }, + { + "name": "a module runs through -m and is never a path", + "runtime": "python", + "platform": "macos", + "execution": { + "kind": "python-module", + "module": "example_model.main", + "defaultArgs": [] + }, + "command": { "kind": "payload-path", "value": "venv/bin/python" }, + "args": [ + { "kind": "literal", "value": "-m" }, + { "kind": "literal", "value": "example_model.main" } + ] + }, + { + "name": "a Windows box runs the same declaration through its own interpreter", + "runtime": "python", + "platform": "windows", + "execution": { + "kind": "python-script", + "script": "app/main.py", + "defaultArgs": [] + }, + "command": { "kind": "payload-path", "value": "venv/python.exe" }, + "args": [{ "kind": "payload-path", "value": "app/main.py" }] + } + ], + "selfTest": [ + { + "name": "macOS asserts Darwin before importing anything", + "runtime": "python", + "platform": "macos", + "probe": { "imports": ["json"] }, + "args": ["-c", "import sys; assert sys.platform == 'darwin'\nimport json"] + }, + { + "name": "Linux accepts any linux variant", + "runtime": "python", + "platform": "linux", + "probe": { "imports": ["json", "numpy"] }, + "args": [ + "-c", + "import sys; assert sys.platform.startswith('linux')\nimport json, numpy" + ] + }, + { + "name": "Windows asserts win32", + "runtime": "python", + "platform": "windows", + "probe": { "imports": ["json"] }, + "args": ["-c", "import sys; assert sys.platform == 'win32'\nimport json"] + }, + { + "name": "the builder appends the extra source a scroll declared", + "runtime": "python", + "platform": "linux", + "probe": { "imports": ["json"], "code": "print(\"self-test ok\")\n" }, + "args": [ + "-c", + "import sys; assert sys.platform.startswith('linux')\nimport json\nprint(\"self-test ok\")\n" + ] + } + ] +} diff --git a/rust/scripts/sync-assets.mjs b/rust/scripts/sync-assets.mjs index 5f02510..c4878cb 100644 --- a/rust/scripts/sync-assets.mjs +++ b/rust/scripts/sync-assets.mjs @@ -18,6 +18,7 @@ const REPO_ROOT = dirname(CRATE_ROOT); /** Canonical source → crate destination, both relative to the repository root. */ const ASSETS = [ ['src/contract/fixtures/target-id-contract.json', 'rust/fixtures/target-id-contract.json'], + ['src/contract/fixtures/runtime-contract.json', 'rust/fixtures/runtime-contract.json'], ['src/contract/fixtures/payload-digest-contract.json', 'rust/fixtures/payload-digest-contract.json'], ['src/contract/fixtures/consumer-conformance.json', 'rust/fixtures/consumer-conformance.json'], ['src/contract/schema/signed-document.schema.json', 'rust/src/contract/schema/signed-document.schema.json'], diff --git a/rust/src/contract/mod.rs b/rust/src/contract/mod.rs index 9f7a2a4..bd6d301 100644 --- a/rust/src/contract/mod.rs +++ b/rust/src/contract/mod.rs @@ -11,4 +11,5 @@ pub mod documents; pub mod links; pub mod payload_digest; +pub mod runtimes; pub mod targets; diff --git a/rust/src/contract/runtimes.rs b/rust/src/contract/runtimes.rs new file mode 100644 index 0000000..87bd84e --- /dev/null +++ b/rust/src/contract/runtimes.rs @@ -0,0 +1,512 @@ +//! Mirror of the Scrollcase box-format runtime model. +//! +//! A target says which machine a box runs on; a runtime says what runs *inside* it — where the +//! interpreter sits, which execution kinds exist, how a declared entry point becomes a command +//! line, and which inherited environment variables can change what that command loads. Keeping +//! those facts in [`super::targets`] made every target adapter a statement that a box is a Python +//! box; they live here instead, and `fixtures/runtime-contract.json` is what proves this mirror +//! agrees with the reference implementation. +//! +//! Nothing here touches a filesystem or a process. [`BoxRuntimeAdapter::build_argv`] in particular +//! returns payload-*relative* paths tagged as paths rather than a joined command line: a box root +//! is a real path on this host, and each language joins one in its own terms. + +use crate::error::{fail, Result}; + +/// Where a runtime lives inside an extracted box. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RuntimeLayout { + /// Directory the packed prefix was relocated into. + pub root: &'static str, + /// The runtime's own executable, relative to the box root. + pub entry_point: &'static str, + /// Directory holding generated console scripts. + pub scripts_directory: &'static str, + /// Directory holding the runtime's bundled library. + pub standard_library: &'static str, + /// Suffix an executable carries on this platform. + pub executable_suffix: &'static str, + /// Frozen wire string naming how launchers were repaired. + pub launcher_kind: &'static str, +} + +/// Payload paths a runtime requires the executable bit on, as a rule rather than a list: a conda +/// prefix carries hundreds of console scripts and no scroll could name them by hand. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ExecutablePayloadPaths { + /// Paths that match exactly. + pub files: &'static [&'static str], + /// Directories every path beneath which matches. + pub directories: &'static [&'static str], +} + +impl ExecutablePayloadPaths { + /// Whether a payload path is one the runtime requires the executable bit on. + #[must_use] + pub fn matches(&self, relative_path: &str) -> bool { + self.files.contains(&relative_path) + || self + .directories + .iter() + .any(|directory| relative_path.starts_with(&format!("{directory}/"))) + } +} + +/// A declared execution, in the terms the runtime rules need. +/// +/// Borrowed from whatever document carried it, so this mirror never has to own a document model — +/// the release manifest's own `Execution` converts into it and the contract stays a statement about +/// names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimeExecution<'a> { + /// One regular payload file, run by the runtime's own entry point. + Script { + /// Payload-relative path to the file. + script: &'a str, + /// Arguments always passed before a caller's own. + default_args: &'a [String], + }, + /// One importable unit, resolved by the runtime rather than named as a path. + Module { + /// Dotted module name. + module: &'a str, + /// Arguments always passed before a caller's own. + default_args: &'a [String], + }, +} + +impl RuntimeExecution<'_> { + /// The wire `kind` this declaration carries. + #[must_use] + pub fn kind(&self) -> &'static str { + match self { + RuntimeExecution::Script { .. } => "python-script", + RuntimeExecution::Module { .. } => "python-module", + } + } + + fn default_args(&self) -> &[String] { + match self { + RuntimeExecution::Script { default_args, .. } + | RuntimeExecution::Module { default_args, .. } => default_args, + } + } +} + +/// Every payload path a declaration could resolve to, and what to say when none of them does. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedExecutionFiles { + /// Payload paths, any one of which resolving satisfies the declaration. + pub candidates: Vec, + /// The message for a box where none of them do. + pub missing: String, +} + +/// One element of a shell-free command line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RuntimeArgument { + /// Passed through exactly as written. + Literal(String), + /// A payload-relative path the caller resolves against the box root. + PayloadPath(String), +} + +/// A shell-free command line, before the caller's own arguments. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeInvocation { + /// The runtime's own entry point. + pub command: RuntimeArgument, + /// Everything the box declared. + pub args: Vec, +} + +/// What a self-test asks the runtime to prove, plus the builder-only extension a scroll may add. +#[derive(Debug, Clone, Copy)] +pub struct SelfTestProbe<'a> { + /// Modules the box must be able to import. + pub imports: &'a [String], + /// Extra source the builder appends; never part of the signed subset. + pub code: Option<&'a str>, +} + +/// What a runtime implies for a box, independent of the machine it runs on. +/// +/// The per-runtime rules are function pointers rather than a `match` on [`Self::id`]: a second +/// runtime is then one more entry in [`RUNTIME_ADAPTERS`], which is the whole point of splitting +/// the model in the first place. +#[derive(Debug, Clone, Copy)] +pub struct BoxRuntimeAdapter { + /// Canonical runtime id, for example `python`. + pub id: &'static str, + /// The `execution.kind` values this runtime defines. + pub execution_kinds: &'static [&'static str], + /// Inherited variables whose presence can change which code this runtime loads — the runtime + /// half of the diagnostic list, to which the target adapter adds the operating system's own. + pub execution_environment_variables: &'static [&'static str], + layouts: &'static [(&'static str, RuntimeLayout)], + platform_assertions: &'static [(&'static str, &'static str)], + resolve: fn(&RuntimeExecution<'_>, &str, &RuntimeLayout, &str) -> Result, +} + +const PYTHON_EXECUTION_ENVIRONMENT: &[&str] = &[ + "PYTHONPATH", + "PYTHONHOME", + "PYTHONSTARTUP", + "PYTHONBREAKPOINT", +]; + +const POSIX_PYTHON_LAYOUT: RuntimeLayout = RuntimeLayout { + root: "venv", + entry_point: "venv/bin/python", + scripts_directory: "venv/bin", + standard_library: "venv/lib", + executable_suffix: "", + launcher_kind: "posix-polyglot", +}; + +const WINDOWS_PYTHON_LAYOUT: RuntimeLayout = RuntimeLayout { + root: "venv", + entry_point: "venv/python.exe", + scripts_directory: "venv/Scripts", + standard_library: "venv/Lib", + executable_suffix: ".exe", + // Reads like a stale reference to a tool this project does not use. It is a frozen wire string + // under the published format; it is not a typo and must not be "cleaned". + launcher_kind: "uv-windows-pe", +}; + +const RUNTIME_ADAPTERS: &[BoxRuntimeAdapter] = &[BoxRuntimeAdapter { + id: "python", + execution_kinds: &["python-script", "python-module"], + execution_environment_variables: PYTHON_EXECUTION_ENVIRONMENT, + layouts: &[ + ("macos", POSIX_PYTHON_LAYOUT), + ("linux", POSIX_PYTHON_LAYOUT), + ("windows", WINDOWS_PYTHON_LAYOUT), + ], + platform_assertions: &[ + ("macos", "import sys; assert sys.platform == 'darwin'"), + ("linux", "import sys; assert sys.platform.startswith('linux')"), + ("windows", "import sys; assert sys.platform == 'win32'"), + ], + resolve: resolve_python_execution_files, +}]; + +/// The `major.minor` prefix naming the standard-library directory a packed prefix carries. +/// +/// A patch component is dropped rather than rejected: a scroll may pin `3.14.2`, and the directory +/// conda-forge writes is `python3.14` either way. +fn python_major_minor(version: &str) -> Result { + let mut parts = version.split('.'); + let (Some(major), Some(minor)) = (parts.next(), parts.next()) else { + fail!("Invalid Python version for execution discovery: {version}."); + }; + if major.is_empty() + || minor.is_empty() + || !major.bytes().all(|byte| byte.is_ascii_digit()) + || !minor.bytes().all(|byte| byte.is_ascii_digit()) + { + fail!("Invalid Python version for execution discovery: {version}."); + } + Ok(format!("{major}.{minor}")) +} + +fn resolve_python_execution_files( + execution: &RuntimeExecution<'_>, + platform: &str, + layout: &RuntimeLayout, + runtime_version: &str, +) -> Result { + let module = match execution { + RuntimeExecution::Script { script, .. } => { + return Ok(ResolvedExecutionFiles { + candidates: vec![(*script).to_string()], + missing: format!("Execution script is missing from the box: {script}."), + }) + } + RuntimeExecution::Module { module, .. } => module, + }; + let module_path = module.replace('.', "/"); + let relative = [ + format!("{module_path}.py"), + format!("{module_path}/__main__.py"), + ]; + // Windows names its standard library once, with no interpreter version in the path; every + // other platform carries `python.` under it. + let standard_library = if platform == "windows" { + layout.standard_library.to_string() + } else { + format!( + "{}/python{}", + layout.standard_library, + python_major_minor(runtime_version)? + ) + }; + let roots = [ + String::new(), + standard_library.clone(), + format!("{standard_library}/site-packages"), + ]; + let candidates = roots + .iter() + .flat_map(|root| { + relative.iter().map(move |candidate| { + if root.is_empty() { + candidate.clone() + } else { + format!("{root}/{candidate}") + } + }) + }) + .collect(); + Ok(ResolvedExecutionFiles { + candidates, + missing: format!("Execution module is not discoverable in the box: {module}."), + }) +} + +impl BoxRuntimeAdapter { + /// Where this runtime sits inside a box built for `platform`. + /// + /// # Errors + /// + /// When the runtime has no layout for that platform. + pub fn layout(&self, platform: &str) -> Result<&'static RuntimeLayout> { + let Some((_, layout)) = self.layouts.iter().find(|(name, _)| *name == platform) else { + fail!("No {} runtime layout exists for platform {platform}", self.id); + }; + Ok(layout) + } + + /// Payload paths this runtime requires the executable bit on. + /// + /// # Errors + /// + /// When the runtime has no layout for that platform. + pub fn executable_payload_paths(&self, platform: &str) -> Result { + let layout = self.layout(platform)?; + // The interpreter by name, and the console-script directory wholesale. A conda prefix + // generates that directory's contents at solve time and nothing declares them, so the rule + // is the only way they can carry the bit at all. + Ok(ExecutablePayloadPaths { + files: std::slice::from_ref(&layout.entry_point), + directories: std::slice::from_ref(&layout.scripts_directory), + }) + } + + /// Every payload path a declaration could resolve to, and what to say when none of them does. + /// + /// # Errors + /// + /// When the execution kind is not this runtime's, the platform is unknown, or the runtime + /// version cannot name a standard library. + pub fn resolve_execution_files( + &self, + execution: &RuntimeExecution<'_>, + platform: &str, + runtime_version: &str, + ) -> Result { + if !self.execution_kinds.contains(&execution.kind()) { + fail!("Unsupported execution kind: {}.", execution.kind()); + } + let layout = self.layout(platform)?; + (self.resolve)(execution, platform, layout, runtime_version) + } + + /// The shell-free command line that runs a declaration, in payload-relative terms. + /// + /// # Errors + /// + /// When the execution kind is not this runtime's, or the platform is unknown. + pub fn build_argv( + &self, + execution: &RuntimeExecution<'_>, + platform: &str, + ) -> Result { + if !self.execution_kinds.contains(&execution.kind()) { + fail!("Unsupported execution kind: {}.", execution.kind()); + } + let layout = self.layout(platform)?; + let mut args = match execution { + RuntimeExecution::Script { script, .. } => { + vec![RuntimeArgument::PayloadPath((*script).to_string())] + } + RuntimeExecution::Module { module, .. } => vec![ + RuntimeArgument::Literal("-m".to_string()), + RuntimeArgument::Literal((*module).to_string()), + ], + }; + args.extend( + execution + .default_args() + .iter() + .map(|value| RuntimeArgument::Literal(value.clone())), + ); + Ok(RuntimeInvocation { + command: RuntimeArgument::PayloadPath(layout.entry_point.to_string()), + args, + }) + } + + /// The arguments that follow this runtime's entry point when it runs a self-test probe. + /// + /// # Errors + /// + /// When the runtime has no platform assertion for that platform. + pub fn self_test_argv(&self, probe: &SelfTestProbe<'_>, platform: &str) -> Result> { + let Some((_, assertion)) = self + .platform_assertions + .iter() + .find(|(name, _)| *name == platform) + else { + fail!( + "No {} self-test assertion exists for platform {platform}", + self.id + ); + }; + let imports = format!("import {}", probe.imports.join(", ")); + let code = match probe.code { + Some(extra) => format!("{assertion}\n{imports}\n{extra}"), + None => format!("{assertion}\n{imports}"), + }; + Ok(vec!["-c".to_string(), code]) + } +} + +/// Returns the runtime adapter for a runtime id. +/// +/// # Errors +/// +/// When no runtime with that id exists. +pub fn runtime_adapter(runtime_id: &str) -> Result<&'static BoxRuntimeAdapter> { + let Some(adapter) = RUNTIME_ADAPTERS + .iter() + .find(|candidate| candidate.id == runtime_id) + else { + fail!("No box runtime adapter exists for {runtime_id}"); + }; + Ok(adapter) +} + +/// Lists every runtime adapter, for contract tests and for callers enumerating what a box may be. +#[must_use] +pub fn runtime_adapters() -> &'static [BoxRuntimeAdapter] { + RUNTIME_ADAPTERS +} + +/// The runtime every box built by this schema version implicitly declares. +/// +/// The wire format has no runtime field: a box records a Python entry point and Python execution +/// kinds and nothing that says "Python". So a reader that must name a runtime names this one, from +/// one place. +pub const IMPLICIT_RUNTIME_ID: &str = "python"; + +/// The complete list of inherited variables that can change what a box executes. +/// +/// Two halves, because they have two owners: the runtime contributes the variables its own loader +/// reads, and the target contributes the operating system's dynamic-linker controls. The order is +/// what a diagnostic report is printed in, so it is part of the answer. +/// +/// # Errors +/// +/// When no runtime with that id exists. +pub fn execution_affecting_variables( + runtime_id: &str, + adapter: &super::targets::BoxTargetAdapter, +) -> Result> { + let runtime = runtime_adapter(runtime_id)?; + Ok(runtime + .execution_environment_variables + .iter() + .chain(adapter.execution_affecting_environment_variables.iter()) + .copied() + .collect()) +} + +/// Ensures a declared entry point agrees with where the runtime actually sits in the payload. +/// +/// # Errors +/// +/// When the entry point is not the one the runtime defines for this target. +pub fn assert_runtime_entry_point( + runtime_id: &str, + adapter: &super::targets::BoxTargetAdapter, + entry_point: &str, +) -> Result<()> { + let expected = runtime_adapter(runtime_id)?.layout(adapter.platform)?.entry_point; + if entry_point != expected { + // The wording still names Python because the wire format still does: a release declares + // `pythonEntryPoint`, and an error that called it something else would name a field nobody + // can find. + fail!("{} boxes must use Python entry point {expected}", adapter.id); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ + python_major_minor, runtime_adapter, RuntimeArgument, RuntimeExecution, SelfTestProbe, + IMPLICIT_RUNTIME_ID, + }; + + #[test] + fn a_runtime_the_format_does_not_define_is_refused() { + assert!(runtime_adapter("node").is_err()); + assert!(runtime_adapter("").is_err()); + assert!(runtime_adapter(IMPLICIT_RUNTIME_ID).is_ok()); + } + + #[test] + fn a_platform_with_no_layout_is_refused_rather_than_guessed() { + let python = runtime_adapter(IMPLICIT_RUNTIME_ID).unwrap(); + assert!(python.layout("plan9").is_err()); + assert!(python.layout("linux").is_ok()); + } + + #[test] + fn a_module_never_becomes_a_payload_path() { + let python = runtime_adapter(IMPLICIT_RUNTIME_ID).unwrap(); + let default_args = vec![]; + let invocation = python + .build_argv( + &RuntimeExecution::Module { + module: "example_model.main", + default_args: &default_args, + }, + "linux", + ) + .unwrap(); + assert_eq!( + invocation.command, + RuntimeArgument::PayloadPath("venv/bin/python".to_string()) + ); + assert!(invocation + .args + .iter() + .all(|argument| matches!(argument, RuntimeArgument::Literal(_)))); + } + + #[test] + fn a_python_version_that_cannot_locate_a_standard_library_is_refused() { + assert_eq!(python_major_minor("3.11.9").unwrap(), "3.11"); + assert_eq!(python_major_minor("3.12").unwrap(), "3.12"); + for invalid in ["", "3", "3.x", "x.1", "3."] { + assert!(python_major_minor(invalid).is_err(), "{invalid} was accepted"); + } + } + + #[test] + fn a_self_test_opens_with_the_platform_it_was_built_for() { + let python = runtime_adapter(IMPLICIT_RUNTIME_ID).unwrap(); + let imports = vec!["json".to_string()]; + let argv = python + .self_test_argv(&SelfTestProbe { imports: &imports, code: None }, "macos") + .unwrap(); + assert_eq!(argv[0], "-c"); + assert!(argv[1].starts_with("import sys; assert sys.platform == 'darwin'")); + assert!(python + .self_test_argv(&SelfTestProbe { imports: &imports, code: None }, "plan9") + .is_err()); + } +} diff --git a/rust/src/contract/targets.rs b/rust/src/contract/targets.rs index b06da38..9066d8e 100644 --- a/rust/src/contract/targets.rs +++ b/rust/src/contract/targets.rs @@ -7,11 +7,15 @@ //! what "agree" means, and `tests/contract.rs` proves this mirror against them. //! //! The adapter describes what a target implies for the extracted tree. Only the parts a consumer -//! relies on are carried here: the interpreter layout it must find, the inherited variables that can -//! change which code that interpreter loads, and the platform assertion a self-test opens with. The -//! builder's own adapter additionally names the archive backend, the conda subdir and the native -//! library inspector — all of them decisions taken while a box is produced, none of them observable -//! by something that only unpacks and runs one. +//! relies on are carried here: the native host it may run on, and the operating system's own +//! dynamic-linker controls. The builder's own adapter additionally names the archive backend, the +//! conda subdir and the native library inspector — all of them decisions taken while a box is +//! produced, none of them observable by something that only unpacks and runs one. +//! +//! What a target deliberately no longer describes is the *runtime* inside the box. The interpreter +//! layout, the execution kinds and the runtime's own environment variables live in +//! [`super::runtimes`], because they are facts about what a box runs rather than about the machine +//! it runs on. //! //! The native host is expressed in Rust's own `OS`/`ARCH` vocabulary rather than Node's //! `darwin`/`arm64`. Those strings never appear in a signed document; they only answer "may this @@ -36,21 +40,6 @@ pub struct BoxTarget { pub cuda_version: Option, } -/// Layout of the interpreter inside an extracted box. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PythonLayout { - /// Directory the packed prefix was relocated into. - pub payload_root: &'static str, - /// Interpreter path, relative to the box root. - pub entry_point: &'static str, - /// Directory holding console scripts. - pub scripts_directory: &'static str, - /// Suffix an executable carries on this platform. - pub executable_suffix: &'static str, - /// Frozen wire string naming how launchers were repaired. - pub launcher_kind: &'static str, -} - /// What a target implies for the extracted tree. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct BoxTargetAdapter { @@ -64,44 +53,18 @@ pub struct BoxTargetAdapter { pub host_os: &'static str, /// `std::env::consts::ARCH` value a host must report to run this box. pub host_arch: &'static str, - /// Interpreter layout inside the box. - pub python: PythonLayout, - /// Inherited variables whose presence can change which code the interpreter loads. + /// The operating system's own dynamic-linker controls; the runtime adds the variables its + /// loader reads, and [`super::runtimes::execution_affecting_variables`] joins the two halves. pub execution_affecting_environment_variables: &'static [&'static str], - /// The platform assertion prepended to every self-test. - pub self_test_python: &'static str, } -const PYTHON_EXECUTION_ENVIRONMENT: &[&str] = &[ - "PYTHONPATH", - "PYTHONHOME", - "PYTHONSTARTUP", - "PYTHONBREAKPOINT", -]; - -const MACOS_EXECUTION_ENVIRONMENT: &[&str] = &[ - "PYTHONPATH", - "PYTHONHOME", - "PYTHONSTARTUP", - "PYTHONBREAKPOINT", - "DYLD_INSERT_LIBRARIES", -]; +const MACOS_EXECUTION_ENVIRONMENT: &[&str] = &["DYLD_INSERT_LIBRARIES"]; -const LINUX_EXECUTION_ENVIRONMENT: &[&str] = &[ - "PYTHONPATH", - "PYTHONHOME", - "PYTHONSTARTUP", - "PYTHONBREAKPOINT", - "LD_PRELOAD", -]; +const LINUX_EXECUTION_ENVIRONMENT: &[&str] = &["LD_PRELOAD"]; -const POSIX_PYTHON: PythonLayout = PythonLayout { - payload_root: "venv", - entry_point: "venv/bin/python", - scripts_directory: "venv/bin", - executable_suffix: "", - launcher_kind: "posix-polyglot", -}; +// Windows has no inherited loader control of its own worth reporting: `PATH` decides DLL +// resolution and is far too broad to name here, so the whole list is the runtime's. +const WINDOWS_EXECUTION_ENVIRONMENT: &[&str] = &[]; const TARGET_ADAPTERS: &[BoxTargetAdapter] = &[ BoxTargetAdapter { @@ -110,9 +73,7 @@ const TARGET_ADAPTERS: &[BoxTargetAdapter] = &[ arch: "aarch64", host_os: "macos", host_arch: "aarch64", - python: POSIX_PYTHON, execution_affecting_environment_variables: MACOS_EXECUTION_ENVIRONMENT, - self_test_python: "import sys; assert sys.platform == 'darwin'", }, BoxTargetAdapter { id: "linux-x86_64", @@ -120,9 +81,7 @@ const TARGET_ADAPTERS: &[BoxTargetAdapter] = &[ arch: "x86_64", host_os: "linux", host_arch: "x86_64", - python: POSIX_PYTHON, execution_affecting_environment_variables: LINUX_EXECUTION_ENVIRONMENT, - self_test_python: "import sys; assert sys.platform.startswith('linux')", }, BoxTargetAdapter { id: "windows-x86_64", @@ -130,17 +89,7 @@ const TARGET_ADAPTERS: &[BoxTargetAdapter] = &[ arch: "x86_64", host_os: "windows", host_arch: "x86_64", - python: PythonLayout { - payload_root: "venv", - entry_point: "venv/python.exe", - scripts_directory: "venv/Scripts", - executable_suffix: ".exe", - // Reads like a stale reference to a tool this project does not use. It is a frozen wire - // string under the published format; it is not a typo and must not be "cleaned". - launcher_kind: "uv-windows-pe", - }, - execution_affecting_environment_variables: PYTHON_EXECUTION_ENVIRONMENT, - self_test_python: "import sys; assert sys.platform == 'win32'", + execution_affecting_environment_variables: WINDOWS_EXECUTION_ENVIRONMENT, }, ]; @@ -255,20 +204,20 @@ pub fn assert_host(adapter: &BoxTargetAdapter, os: &str, arch: &str) -> Result<( Ok(()) } -/// Ensures a release's entry point agrees with the adapter's standalone Python layout. +/// Ensures a release's entry point agrees with the standalone Python layout for this target. +/// +/// Kept under its published name while the wire format still spells the field `pythonEntryPoint`. +/// The rule itself lives in [`super::runtimes`], where it can be asked about any runtime. /// /// # Errors /// -/// When the entry point is not the one the adapter defines. +/// When the entry point is not the one the runtime defines for this target. pub fn assert_python_entry_point(adapter: &BoxTargetAdapter, entry_point: &str) -> Result<()> { - if entry_point != adapter.python.entry_point { - fail!( - "{} boxes must use Python entry point {}", - adapter.id, - adapter.python.entry_point - ); - } - Ok(()) + super::runtimes::assert_runtime_entry_point( + super::runtimes::IMPLICIT_RUNTIME_ID, + adapter, + entry_point, + ) } #[cfg(test)] diff --git a/rust/src/execution.rs b/rust/src/execution.rs index 42fff9f..912ff3c 100644 --- a/rust/src/execution.rs +++ b/rust/src/execution.rs @@ -1,68 +1,22 @@ //! Static execution prerequisites. //! //! Execution metadata is not a command string: it names either one regular payload file or one -//! dotted Python module. Checking the file set proves those names can resolve without importing a -//! package, running an `__init__.py`, or starting the application — so the check itself cannot be -//! the thing that executes box code before the trust chain has finished. +//! importable unit of the box's runtime. Checking the file set proves those names can resolve +//! without importing a package, running an `__init__.py`, or starting the application — so the +//! check itself cannot be the thing that executes box code before the trust chain has finished. +//! +//! Which paths a declaration could resolve to is a runtime question, asked of +//! [`crate::contract::runtimes`] rather than answered here. What stays is the path-safety rule +//! every candidate goes through. use std::collections::BTreeSet; +use crate::contract::runtimes::{runtime_adapter, IMPLICIT_RUNTIME_ID}; use crate::contract::targets::BoxTargetAdapter; -use crate::error::{fail, Result}; +use crate::error::{Error, Result}; use crate::path::safe_relative_path; use crate::release::Execution; -/// The `major.minor` prefix used to locate a standard library directory. -fn python_major_minor(version: &str) -> Result { - let mut parts = version.split('.'); - let (Some(major), Some(minor)) = (parts.next(), parts.next()) else { - fail!("Invalid Python version for execution discovery: {version}."); - }; - if major.is_empty() - || minor.is_empty() - || !major.bytes().all(|byte| byte.is_ascii_digit()) - || !minor.bytes().all(|byte| byte.is_ascii_digit()) - { - fail!("Invalid Python version for execution discovery: {version}."); - } - Ok(format!("{major}.{minor}")) -} - -/// Every path a dotted module could legitimately resolve to inside a box. -fn module_entry_points( - adapter: &BoxTargetAdapter, - module: &str, - python_version: &str, -) -> Result> { - let module_path = module.replace('.', "/"); - let relative = [ - format!("{module_path}.py"), - format!("{module_path}/__main__.py"), - ]; - let standard_library = if adapter.platform == "windows" { - "venv/Lib".to_string() - } else { - format!("venv/lib/python{}", python_major_minor(python_version)?) - }; - let roots = [ - String::new(), - standard_library.clone(), - format!("{standard_library}/site-packages"), - ]; - Ok(roots - .iter() - .flat_map(|root| { - relative.iter().map(move |candidate| { - if root.is_empty() { - candidate.clone() - } else { - format!("{root}/{candidate}") - } - }) - }) - .collect()) -} - /// Confirms optional execution metadata names something runnable in a payload or archive. /// /// `files` must hold only regular entries: a link resolves, but the thing that finally runs has to @@ -74,37 +28,40 @@ fn module_entry_points( pub fn assert_execution_files( execution: Option<&Execution>, adapter: &BoxTargetAdapter, - python_version: &str, + runtime_version: &str, files: &BTreeSet, ) -> Result<()> { let Some(execution) = execution else { return Ok(()); }; - match execution { - Execution::PythonScript { script, .. } => { - let safe = safe_relative_path(script)?; - if !files.contains(&safe) { - fail!("Execution script is missing from the box: {safe}."); - } - } - Execution::PythonModule { module, .. } => { - let candidates = module_entry_points(adapter, module, python_version)?; - if !candidates.iter().any(|path| files.contains(path)) { - fail!("Execution module is not discoverable in the box: {module}."); - } + let runtime = runtime_adapter(IMPLICIT_RUNTIME_ID)?; + let resolved = runtime.resolve_execution_files( + &execution.as_runtime(), + adapter.platform, + runtime_version, + )?; + // Every candidate goes through the traversal rule, not just the one a scroll wrote by hand: a + // path the format derived is still a path this process is about to look for. + for candidate in &resolved.candidates { + if files.contains(&safe_relative_path(candidate)?) { + return Ok(()); } } - Ok(()) + Err(Error::new(resolved.missing)) } #[cfg(test)] mod tests { - use super::{assert_execution_files, python_major_minor}; + use super::assert_execution_files; use crate::contract::targets::{box_target_adapter, BoxTarget}; use crate::release::Execution; use std::collections::BTreeSet; - fn adapter(platform: &str, arch: &str, accelerator: &str) -> &'static crate::contract::targets::BoxTargetAdapter { + fn adapter( + platform: &str, + arch: &str, + accelerator: &str, + ) -> &'static crate::contract::targets::BoxTargetAdapter { box_target_adapter(&BoxTarget { platform: platform.to_string(), arch: arch.to_string(), @@ -189,11 +146,17 @@ mod tests { } #[test] - fn a_python_version_that_cannot_locate_a_standard_library_is_refused() { - assert_eq!(python_major_minor("3.11.9").unwrap(), "3.11"); - assert_eq!(python_major_minor("3.12").unwrap(), "3.12"); + fn a_runtime_version_that_cannot_locate_a_standard_library_is_refused() { + let adapter = adapter("linux", "x86_64", "cpu"); + let execution = Execution::PythonModule { + module: "pkg".to_string(), + default_args: vec![], + }; for invalid in ["", "3", "3.x", "x.1", "3."] { - assert!(python_major_minor(invalid).is_err(), "{invalid} was accepted"); + assert!( + assert_execution_files(Some(&execution), adapter, invalid, &files(&[])).is_err(), + "{invalid} was accepted" + ); } } diff --git a/rust/src/prepare.rs b/rust/src/prepare.rs index da9a3e2..00d6da4 100644 --- a/rust/src/prepare.rs +++ b/rust/src/prepare.rs @@ -20,6 +20,7 @@ use crate::archive::extract_zip_archive; use crate::contract::payload_digest::{ parse_payload_digest_stream, PayloadDigestKind, MAX_PAYLOAD_DIGEST_BYTES, PAYLOAD_DIGEST_FILE, }; +use crate::contract::runtimes::{execution_affecting_variables, IMPLICIT_RUNTIME_ID}; use crate::contract::targets::{assert_native_host, box_target_id, BoxTargetAdapter}; use crate::environment::{ resolve_environment, EnvironmentLayer, EnvironmentReport, EnvironmentSource, ResolveOptions, @@ -311,7 +312,7 @@ fn release_environment_report( values: release_pairs, }, ], - execution_affecting_variables: adapter.execution_affecting_environment_variables, + execution_affecting_variables: &execution_affecting_variables(IMPLICIT_RUNTIME_ID, adapter)?, expanded: options.env_report || options.env_report_values, reveal_host_values: options.env_report_values, })? diff --git a/rust/src/release.rs b/rust/src/release.rs index 72de397..3df3a3b 100644 --- a/rust/src/release.rs +++ b/rust/src/release.rs @@ -25,6 +25,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use crate::contract::documents::{parse_document_kind, DocumentType, BOX_SCHEMA_VERSION}; +use crate::contract::runtimes::RuntimeExecution; use crate::contract::targets::BoxTarget; use crate::error::{fail, Result}; use crate::path::safe_relative_path; @@ -127,6 +128,33 @@ pub enum Execution { }, } +impl Execution { + /// Borrows this declaration in the terms the runtime rules read. + /// + /// The conversion lives here rather than in the contract mirror so that + /// [`crate::contract::runtimes`] never has to know the document model — it states rules about + /// names, and this is the document handing it the names. + #[must_use] + pub fn as_runtime(&self) -> RuntimeExecution<'_> { + match self { + Execution::PythonScript { + script, + default_args, + } => RuntimeExecution::Script { + script, + default_args, + }, + Execution::PythonModule { + module, + default_args, + } => RuntimeExecution::Module { + module, + default_args, + }, + } + } +} + /// How the box was produced. Recorded, signed, and never fabricated. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] diff --git a/rust/src/run.rs b/rust/src/run.rs index 51f9f88..81aee1d 100644 --- a/rust/src/run.rs +++ b/rust/src/run.rs @@ -20,6 +20,9 @@ use std::time::Duration; use crate::environment::{ resolve_environment, EnvironmentLayer, EnvironmentReport, EnvironmentSource, ResolveOptions, }; +use crate::contract::runtimes::{ + execution_affecting_variables, runtime_adapter, RuntimeArgument, IMPLICIT_RUNTIME_ID, +}; use crate::error::{fail, Error, Result}; use crate::execution::assert_execution_files; use crate::filesystem::collect_files; @@ -28,7 +31,6 @@ use crate::prepare::{ verify_and_extract_box, verify_required_assets, EnvironmentReportOptions, PrepareOptions, PreparedBox, }; -use crate::release::Execution; use crate::trust::TrustAnchors; /// What would be spawned, once the trust chain has finished and the environment is resolved. @@ -241,7 +243,7 @@ fn resolve_run_environment( .collect(), }, ], - execution_affecting_variables: adapter.execution_affecting_environment_variables, + execution_affecting_variables: &execution_affecting_variables(IMPLICIT_RUNTIME_ID, adapter)?, expanded: options.environment.env_report || options.environment.env_report_values, reveal_host_values: options.environment.env_report_values, }) @@ -289,17 +291,19 @@ pub fn run_extracted_box(prepared: &PreparedBox, options: &RunOptions<'_>) -> Re verify_required_assets(root, prepared.required_assets())?; let python = join_relative(root, &safe_relative_path(&release.python_entry_point)?); - let mut arguments: Vec = match execution { - Execution::PythonScript { script, .. } => vec![join_relative(root, &safe_relative_path(script)?) - .to_string_lossy() - .into_owned()], - Execution::PythonModule { module, .. } => vec!["-m".to_string(), module.clone()], - }; - match execution { - Execution::PythonScript { default_args, .. } - | Execution::PythonModule { default_args, .. } => { - arguments.extend(default_args.iter().cloned()); - } + // The runtime states the command line in payload-relative terms and this end joins it: a box + // root is a real path on this host, and the format has no business deciding what one looks + // like. + let invocation = runtime_adapter(IMPLICIT_RUNTIME_ID)? + .build_argv(&execution.as_runtime(), adapter.platform)?; + let mut arguments: Vec = Vec::with_capacity(invocation.args.len() + options.args.len()); + for argument in &invocation.args { + arguments.push(match argument { + RuntimeArgument::Literal(value) => value.clone(), + RuntimeArgument::PayloadPath(value) => join_relative(root, &safe_relative_path(value)?) + .to_string_lossy() + .into_owned(), + }); } arguments.extend(options.args.iter().cloned()); diff --git a/rust/tests/contract.rs b/rust/tests/contract.rs index a64e715..b40ac94 100644 --- a/rust/tests/contract.rs +++ b/rust/tests/contract.rs @@ -12,10 +12,14 @@ use sha2::{Digest, Sha256}; use scrollcase_consumer::contract::payload_digest::{ payload_digest_stream, PayloadDigestEntry, PayloadDigestKind, }; +use scrollcase_consumer::contract::runtimes::{ + runtime_adapter, runtime_adapters, RuntimeArgument, RuntimeExecution, SelfTestProbe, +}; use scrollcase_consumer::contract::targets::{box_target_id, BoxTarget}; const TARGET_ID_CONTRACT: &str = include_str!("../fixtures/target-id-contract.json"); const PAYLOAD_DIGEST_CONTRACT: &str = include_str!("../fixtures/payload-digest-contract.json"); +const RUNTIME_CONTRACT: &str = include_str!("../fixtures/runtime-contract.json"); fn sha256_hex(bytes: &[u8]) -> String { use std::fmt::Write as _; @@ -70,6 +74,259 @@ fn matches_the_shared_target_id_contract() { } } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct RuntimeContract { + runtimes: Vec, + executable_matches: Vec, + execution_discovery: Vec, + invalid_runtime_versions: Vec, + argv: Vec, + self_test: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct RuntimeCase { + id: String, + execution_kinds: Vec, + execution_environment_variables: Vec, + layouts: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct LayoutCase { + platform: String, + layout: LayoutFields, + executable_payload_paths: ExecutablePathsFields, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct LayoutFields { + root: String, + entry_point: String, + scripts_directory: String, + standard_library: String, + executable_suffix: String, + launcher_kind: String, +} + +#[derive(Deserialize)] +struct ExecutablePathsFields { + files: Vec, + directories: Vec, +} + +#[derive(Deserialize)] +struct ExecutableMatchCase { + name: String, + runtime: String, + platform: String, + path: String, + executable: bool, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ExecutionDiscoveryCase { + name: String, + runtime: String, + platform: String, + runtime_version: String, + execution: ExecutionFields, + candidates: Vec, +} + +#[derive(Deserialize)] +struct ArgvCase { + name: String, + runtime: String, + platform: String, + execution: ExecutionFields, + command: ArgumentFields, + args: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct SelfTestCase { + name: String, + runtime: String, + platform: String, + probe: ProbeFields, + args: Vec, +} + +#[derive(Deserialize)] +struct ProbeFields { + imports: Vec, + #[serde(default)] + code: Option, +} + +/// The execution declaration as the fixture spells it, so the vectors are read exactly as the other +/// implementations read them rather than through this crate's own release model. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ExecutionFields { + kind: String, + #[serde(default)] + script: Option, + #[serde(default)] + module: Option, + default_args: Vec, +} + +impl ExecutionFields { + fn as_runtime(&self) -> RuntimeExecution<'_> { + match self.kind.as_str() { + "python-script" => RuntimeExecution::Script { + script: self.script.as_deref().expect("a script case declares a script"), + default_args: &self.default_args, + }, + "python-module" => RuntimeExecution::Module { + module: self.module.as_deref().expect("a module case declares a module"), + default_args: &self.default_args, + }, + other => panic!("the fixture declares an execution kind this mirror has no shape for: {other}"), + } + } +} + +#[derive(Deserialize)] +struct ArgumentFields { + kind: String, + value: String, +} + +impl ArgumentFields { + fn matches(&self, argument: &RuntimeArgument) -> bool { + match (self.kind.as_str(), argument) { + ("literal", RuntimeArgument::Literal(value)) + | ("payload-path", RuntimeArgument::PayloadPath(value)) => value == &self.value, + _ => false, + } + } +} + +/// The Rust half of the shared runtime vectors. +/// +/// Everything the runtime model states about a box — where the interpreter sits, which paths need +/// the executable bit, what a declaration could resolve to, and the command line that runs it — is +/// asserted here against the same file the Node and Python implementations read. +#[test] +fn matches_the_shared_runtime_contract() { + let contract: RuntimeContract = serde_json::from_str(RUNTIME_CONTRACT).unwrap(); + + let mirrored: Vec<&str> = runtime_adapters().iter().map(|runtime| runtime.id).collect(); + let declared: Vec<&str> = contract.runtimes.iter().map(|case| case.id.as_str()).collect(); + assert_eq!(mirrored, declared); + + for case in &contract.runtimes { + let runtime = runtime_adapter(&case.id).unwrap(); + assert_eq!(runtime.execution_kinds, case.execution_kinds, "{}", case.id); + assert_eq!( + runtime.execution_environment_variables, case.execution_environment_variables, + "{}", + case.id + ); + for platform in &case.layouts { + let layout = runtime.layout(&platform.platform).unwrap(); + let expected = &platform.layout; + assert_eq!(layout.root, expected.root, "{}", platform.platform); + assert_eq!(layout.entry_point, expected.entry_point, "{}", platform.platform); + assert_eq!( + layout.scripts_directory, expected.scripts_directory, + "{}", + platform.platform + ); + assert_eq!( + layout.standard_library, expected.standard_library, + "{}", + platform.platform + ); + assert_eq!( + layout.executable_suffix, expected.executable_suffix, + "{}", + platform.platform + ); + assert_eq!(layout.launcher_kind, expected.launcher_kind, "{}", platform.platform); + + let rule = runtime.executable_payload_paths(&platform.platform).unwrap(); + assert_eq!(rule.files, platform.executable_payload_paths.files, "{}", platform.platform); + assert_eq!( + rule.directories, platform.executable_payload_paths.directories, + "{}", + platform.platform + ); + } + } + + for case in &contract.executable_matches { + let rule = runtime_adapter(&case.runtime) + .unwrap() + .executable_payload_paths(&case.platform) + .unwrap(); + assert_eq!(rule.matches(&case.path), case.executable, "{}", case.name); + } + + for case in &contract.execution_discovery { + let resolved = runtime_adapter(&case.runtime) + .unwrap() + .resolve_execution_files( + &case.execution.as_runtime(), + &case.platform, + &case.runtime_version, + ) + .unwrap_or_else(|error| panic!("{} was refused: {error}", case.name)); + assert_eq!(resolved.candidates, case.candidates, "{}", case.name); + } + + let python = runtime_adapter("python").unwrap(); + let module = ExecutionFields { + kind: "python-module".to_string(), + script: None, + module: Some("pkg".to_string()), + default_args: vec![], + }; + for invalid in &contract.invalid_runtime_versions { + assert!( + python + .resolve_execution_files(&module.as_runtime(), "linux", invalid) + .is_err(), + "{invalid:?} was accepted" + ); + } + + for case in &contract.argv { + let invocation = runtime_adapter(&case.runtime) + .unwrap() + .build_argv(&case.execution.as_runtime(), &case.platform) + .unwrap(); + assert!(case.command.matches(&invocation.command), "{}", case.name); + assert_eq!(invocation.args.len(), case.args.len(), "{}", case.name); + for (expected, produced) in case.args.iter().zip(invocation.args.iter()) { + assert!(expected.matches(produced), "{}", case.name); + } + } + + for case in &contract.self_test { + let argv = runtime_adapter(&case.runtime) + .unwrap() + .self_test_argv( + &SelfTestProbe { + imports: &case.probe.imports, + code: case.probe.code.as_deref(), + }, + &case.platform, + ) + .unwrap(); + assert_eq!(argv, case.args, "{}", case.name); + } +} + #[derive(Deserialize)] struct PayloadDigestContract { format: String, @@ -155,6 +412,7 @@ fn bundled_assets_match_the_canonical_sources() { let assets = [ ("fixtures/target-id-contract.json", "fixtures/target-id-contract.json"), + ("fixtures/runtime-contract.json", "fixtures/runtime-contract.json"), ("fixtures/payload-digest-contract.json", "fixtures/payload-digest-contract.json"), ("fixtures/consumer-conformance.json", "fixtures/consumer-conformance.json"), ("schema/signed-document.schema.json", "src/contract/schema/signed-document.schema.json"), diff --git a/rust/tests/release_document.rs b/rust/tests/release_document.rs index 6408d5b..51cb99f 100644 --- a/rust/tests/release_document.rs +++ b/rust/tests/release_document.rs @@ -7,6 +7,7 @@ use std::path::{Path, PathBuf}; +use scrollcase_consumer::contract::runtimes::{runtime_adapter, IMPLICIT_RUNTIME_ID}; use scrollcase_consumer::trust::TrustAnchors; use scrollcase_consumer::verify::inspect_release_document; @@ -50,7 +51,11 @@ fn a_genuine_signed_release_is_accepted_and_fully_interpreted() { // The adapter is resolved from the signed target, and the entry point agreed with it. assert_eq!( inspected.release.python_entry_point, - inspected.adapter.python.entry_point + runtime_adapter(IMPLICIT_RUNTIME_ID) + .unwrap() + .layout(inspected.adapter.platform) + .unwrap() + .entry_point ); assert_eq!(inspected.signed.signatures.len(), 1); } diff --git a/src/build/archive.d.mts b/src/build/archive.d.mts index e7e4988..a40d22e 100644 --- a/src/build/archive.d.mts +++ b/src/build/archive.d.mts @@ -11,9 +11,10 @@ * @param {string} archivePath * @param {import('../contract/targets.mjs').BoxTargetAdapter} adapter * @param {readonly string[]} [uncompressedPaths] payload paths stored rather than deflated + * @param {string} [runtimeId] whose layout decides which entries carry the executable bit * @returns {Promise} */ -export function createDeterministicZip(payloadDir: string, archivePath: string, adapter: import("../contract/targets.mjs").BoxTargetAdapter, uncompressedPaths?: readonly string[]): Promise; +export function createDeterministicZip(payloadDir: string, archivePath: string, adapter: import("../contract/targets.mjs").BoxTargetAdapter, uncompressedPaths?: readonly string[], runtimeId?: string): Promise; /** * Lists and validates all entries before any ZIP data is trusted or extracted. * diff --git a/src/build/archive.mjs b/src/build/archive.mjs index 3f3f10d..81e3cc1 100644 --- a/src/build/archive.mjs +++ b/src/build/archive.mjs @@ -19,6 +19,11 @@ import * as tar from 'tar'; import yauzl from 'yauzl'; import yazl from 'yazl'; import { findEntryThroughLink, findUnresolvableLink } from '../contract/links.mjs'; +import { + IMPLICIT_RUNTIME_ID, + isExecutablePayloadPath, + runtimeAdapter, +} from '../contract/runtimes.mjs'; import { FIXED_ARCHIVE_TIME, collectEntries, @@ -34,13 +39,17 @@ const ZIP_REGULAR_FILE = 0o100000; const ZIP_DIRECTORY = 0o040000; const ZIP_SYMBOLIC_LINK = 0o120000; -/** Returns the stable archive mode for a box payload file. */ -function archiveFileMode(adapter, relativePath) { +/** + * Returns the stable archive mode for a box payload file. + * + * Which paths need the bit is the runtime's rule, not the target's: a conda prefix generates + * hundreds of console scripts that no scroll could name, and the runtime is what knows where they + * land. The mode is still *synthesised* rather than read off disk, which is what keeps two builds + * of one commit byte-identical and keeps `payload-digest.v1` — which excludes mode — honest. + */ +function archiveFileMode(adapter, relativePath, executablePaths) { if (adapter.host.platform === 'win32') return 0o100644; - const scriptsDirectory = adapter.python.scriptsDirectory; - return relativePath === adapter.python.entryPoint || relativePath.startsWith(`${scriptsDirectory}/`) - ? 0o100755 - : 0o100644; + return isExecutablePayloadPath(executablePaths, relativePath) ? 0o100755 : 0o100644; } /** @@ -72,10 +81,18 @@ function isDeclaredUncompressed(path, declared) { * @param {string} archivePath * @param {import('../contract/targets.mjs').BoxTargetAdapter} adapter * @param {readonly string[]} [uncompressedPaths] payload paths stored rather than deflated + * @param {string} [runtimeId] whose layout decides which entries carry the executable bit * @returns {Promise} */ -export async function createDeterministicZip(payloadDir, archivePath, adapter, uncompressedPaths = []) { +export async function createDeterministicZip( + payloadDir, + archivePath, + adapter, + uncompressedPaths = [], + runtimeId = IMPLICIT_RUNTIME_ID, +) { const entries = await collectEntries(payloadDir); + const executablePaths = runtimeAdapter(runtimeId).executablePayloadPaths(adapter); assertPayloadLinksAreCarryable(entries); await rm(archivePath, { force: true }); await mkdir(dirname(archivePath), { recursive: true }); @@ -100,7 +117,7 @@ export async function createDeterministicZip(payloadDir, archivePath, adapter, u compress: compressionLevel !== 0, compressionLevel, mtime: FIXED_ARCHIVE_TIME, - mode: archiveFileMode(adapter, entry.path), + mode: archiveFileMode(adapter, entry.path, executablePaths), forceDosTimestamp: true, }); } diff --git a/src/build/authoring.mjs b/src/build/authoring.mjs index 1d2f650..b8cd636 100644 --- a/src/build/authoring.mjs +++ b/src/build/authoring.mjs @@ -15,6 +15,8 @@ import { lstat, mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; import { dirname, isAbsolute, join, relative, sep } from 'node:path'; import { boxTargetAdapter, boxTargetId, condaSubdir } from '../contract/targets.mjs'; +import { IMPLICIT_RUNTIME_ID } from '../contract/runtimes.mjs'; +import { runtimeBuilder } from '../runtimes/index.mjs'; import { fileExists, safeRelativePath } from './filesystem.mjs'; import { fail } from './process.mjs'; import { schemaValidationError } from './schema-validation.mjs'; @@ -66,33 +68,6 @@ export function resolvePythonVersion(requested) { return requested === 'latest' ? LATEST_PYTHON_VERSION : requested; } -const STARTER_SCRIPT = `"""Minimal application entry point generated by Scrollcase.""" - -import sys - - -def main() -> int: - print("Scrollcase box is ready.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) -`; - -const STARTER_SELF_TEST = `"""Self-test for this box, run by \`scrollcase build\` before the box is archived. - -It runs with the box's own interpreter, from the payload root, after the imports declared in -scroll.json have already succeeded — so it can read the files the box ships and import the code it -packs. Any exception fails the build, which is the point: this is the last check that the box works -before anyone downloads it. - -Replace the line below with something that would actually notice a broken box. -""" - -print("self-test ok") -`; - const TYPESCRIPT_CONSUMER_TEMPLATE = `/** * Runs a local box through the typed Node consumer. * @@ -259,10 +234,10 @@ function projectRelativePath(projectRoot, path) { return relativePath.split(sep).join('/'); } -function pixiManifest(environmentName, target, pythonVersion) { - const pythonConstraint = /^\d+\.\d+$/.test(pythonVersion) - ? `${pythonVersion}.*` - : pythonVersion; +function pixiManifest(environmentName, target, runtimeVersion, runtimeId = IMPLICIT_RUNTIME_ID) { + // The workspace table is substrate — one channel, one platform, whatever the box runs. Only the + // dependency line knows which runtime is being packed, and the runtime is what writes it. + const runtime = runtimeBuilder(runtimeId).pixiDependency(runtimeVersion); return `# Solved by \`scrollcase lock\` into pixi.lock, which is committed and reviewed. # \`platforms\` must equal the target's conda subdirectory, or the solve produces an environment # that cannot run on the machine the box is for. @@ -272,7 +247,7 @@ channels = ["conda-forge"] platforms = ["${condaSubdir(target)}"] [dependencies] -python = "${pythonConstraint}" +${runtime.name} = "${runtime.spec}" `; } @@ -372,7 +347,7 @@ export async function createScroll({ if (await fileExists(generatedScriptPath)) { fail(`Generated script already exists: ${sourcePath}.`); } - generatedSource = STARTER_SCRIPT; + generatedSource = runtimeBuilder(IMPLICIT_RUNTIME_ID).templates.script; } else { sourcePath = safeRelativePath(scriptSourcePath); const source = join(workspace.root, ...sourcePath.split('/')); @@ -444,7 +419,12 @@ export async function createScroll({ join(staging, 'pixi.toml'), pixiManifest(`${identity.boxId}-${targetId}`, target, identity.pythonVersion), ); - if (selfTestPath) await writeFile(join(staging, 'self_test.py'), STARTER_SELF_TEST); + if (selfTestPath) { + await writeFile( + join(staging, 'self_test.py'), + runtimeBuilder(IMPLICIT_RUNTIME_ID).templates.selfTest, + ); + } if (generatedScriptPath) { await mkdir(dirname(generatedScriptPath), { recursive: true }); await writeFile(generatedScriptPath, generatedSource, { flag: 'wx' }); diff --git a/src/build/box.mjs b/src/build/box.mjs index b9328c6..e545642 100644 --- a/src/build/box.mjs +++ b/src/build/box.mjs @@ -19,6 +19,7 @@ import { createHash } from 'node:crypto'; import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { assertNativeHost, boxTargetId } from '../contract/targets.mjs'; +import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../contract/runtimes.mjs'; import { CHANNELS, documentKinds } from '../contract/documents.mjs'; import { mergeEnvironmentLayers } from '../environment.mjs'; import { @@ -67,11 +68,13 @@ async function selfTestExtraCode(scroll, projectRoot) { /** Runs the scroll's self-test with the payload's own interpreter, under the target's environment. */ function runSelfTest({ interpreter, adapter, scroll, payloadDir, run, extraCode = null }) { - const imports = `import ${scroll.selfTest.imports.join(', ')}`; - const code = extraCode - ? `${adapter.selfTestPython}\n${imports}\n${extraCode}` - : `${adapter.selfTestPython}\n${imports}`; - run(interpreter, ['-c', code], { + // The builder's probe is the signed one plus whatever extra source the scroll declared, and the + // runtime is the only thing that knows how to turn either into a command line. + const argv = runtimeAdapter(IMPLICIT_RUNTIME_ID).selfTestArgv({ + probe: { imports: scroll.selfTest.imports, code: extraCode }, + target: adapter, + }); + run(interpreter, argv, { cwd: payloadDir, env: mergeEnvironmentLayers( adapter.platform, @@ -230,7 +233,7 @@ export async function buildBox(name, options = {}) { assertExecutionFiles({ execution: scroll.execution, adapter, - pythonVersion: scroll.pythonVersion, + runtimeVersion: scroll.pythonVersion, files: new Set(await collectFiles(payloadDir)), }); // Everything needed to answer "where did this box come from, and could I rebuild it?". diff --git a/src/build/dependencies.mjs b/src/build/dependencies.mjs index b210084..6de5d2f 100644 --- a/src/build/dependencies.mjs +++ b/src/build/dependencies.mjs @@ -15,6 +15,10 @@ * actually solved, so an added dependency defaults to `*` and the lock — committed and reviewed — * pins the newest version that fits everything else. Asking the network for a "latest" to write here * would put a second, weaker pin next to the real one. + * + * Everything here is substrate: a conda package name and a pixi table, whatever runtime the box + * packs. Reading a *Python* project's `requirements.txt` into these terms is a Python fact and lives + * at `src/runtimes/python/dependencies.mjs`. */ import { readFile, writeFile } from 'node:fs/promises'; @@ -22,23 +26,6 @@ import { fail } from './process.mjs'; const DEPENDENCIES_TABLE = '[dependencies]'; -/** - * PyPI names whose conda-forge package is called something else. - * - * Deliberately short. Every entry is one this project can state with confidence; anything else is - * lowercased and passed through, and every rename is reported so the author can check it before - * locking. A wrong guess here produces a lock that resolves and a box that cannot import what it - * was built for, which is worse than an unmapped name the author has to look up. - */ -const CONDA_FORGE_NAMES = Object.freeze({ - 'opencv-python': 'opencv', - 'opencv-python-headless': 'opencv', - 'psycopg2-binary': 'psycopg2', - 'msgpack': 'msgpack-python', - 'tables': 'pytables', - 'torch': 'pytorch', -}); - /** A conda package name, as conda-forge spells them. */ const PACKAGE_NAME = /^[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$/; @@ -104,50 +91,3 @@ export async function addDependency({ manifests, name, spec = '*' }) { } return { written, replaced }; } - -/** - * Reads a pip `requirements.txt` and reports what it would mean on conda-forge. - * - * A project arriving from pip already has this file, and retyping it is the sort of work that - * invites a typo. What it cannot do is decide: names are translated where this module is sure and - * lowercased otherwise, and every translation and every skip is reported so the author reviews them - * before locking rather than after a build fails. - * - * @param {string} contents - * @returns {{ dependencies: { name: string, spec: string }[], - * renamed: { from: string, to: string }[], skipped: { line: string, reason: string }[] }} - */ -export function readRequirements(contents) { - const dependencies = []; - const renamed = []; - const skipped = []; - for (const raw of contents.split('\n')) { - const line = raw.split('#')[0].trim(); - if (!line) continue; - if (line.startsWith('-')) { - skipped.push({ line, reason: 'a pip option, which has no conda-forge equivalent' }); - continue; - } - if (/^[a-z][a-z0-9+.-]*:\/\//i.test(line) || line.includes('@')) { - skipped.push({ line, reason: 'a direct URL or VCS reference, which conda-forge cannot express' }); - continue; - } - // `name[extra1,extra2] >= 1.2 ; python_version < "3.12"` — the name runs to the first of these. - const [requirement] = line.split(';'); - const match = /^([A-Za-z0-9._-]+)\s*(\[[^\]]*\])?\s*(.*)$/.exec(requirement.trim()); - if (!match) { - skipped.push({ line, reason: 'not a requirement this reader understands' }); - continue; - } - const [, rawName, extras, rawSpec] = match; - const normalized = rawName.toLowerCase().replace(/_/g, '-'); - const name = CONDA_FORGE_NAMES[normalized] ?? normalized; - if (name !== rawName) renamed.push({ from: rawName, to: name }); - if (extras) { - skipped.push({ line: `${rawName}${extras}`, reason: 'extras are a pip concept; add the packages they pull in yourself' }); - } - const spec = rawSpec.trim().replace(/\s+/g, ''); - dependencies.push({ name, spec: spec === '' ? '*' : spec }); - } - return { dependencies, renamed, skipped }; -} diff --git a/src/build/execution.d.mts b/src/build/execution.d.mts index 38f9d27..fdf7b1f 100644 --- a/src/build/execution.d.mts +++ b/src/build/execution.d.mts @@ -3,10 +3,19 @@ * * `files` must contain only regular archive entries. Both collectFiles() during build and the ZIP * entry classifier during verify provide exactly that representation. + * + * @param {object} options + * @param {object | null | undefined} options.execution + * @param {import('../contract/targets.mjs').BoxTargetAdapter} options.adapter + * @param {string} options.runtimeVersion the interpreter version a module search needs + * @param {Set} options.files + * @param {string} [options.runtimeId] + * @returns {void} */ -export function assertExecutionFiles({ execution, adapter, pythonVersion, files, }: { - execution: any; - adapter: any; - pythonVersion: any; - files: any; +export function assertExecutionFiles({ execution, adapter, runtimeVersion, files, runtimeId, }: { + execution: object | null | undefined; + adapter: import("../contract/targets.mjs").BoxTargetAdapter; + runtimeVersion: string; + files: Set; + runtimeId?: string; }): void; diff --git a/src/build/execution.mjs b/src/build/execution.mjs index 965097c..3424d97 100644 --- a/src/build/execution.mjs +++ b/src/build/execution.mjs @@ -2,59 +2,52 @@ * Static execution prerequisites shared by the builder and verifier. * * Execution metadata is not a command string: it names either one regular payload file or one - * dotted Python module. Checking the archive file set proves those names can resolve without - * importing a package, running an `__init__.py`, or starting the application. The later consumer - * may therefore launch only after the complete trust chain has passed. + * importable unit of the box's runtime. Checking the archive file set proves those names can + * resolve without importing a package, running an `__init__.py`, or starting the application. The + * later consumer may therefore launch only after the complete trust chain has passed. + * + * Which paths a declaration could resolve to is a runtime question, and it is asked of + * `contract/runtimes.mjs` rather than answered here. This module keeps the two things that are + * genuinely the builder's: the path-safety rule every candidate goes through, and the single error + * path every validation failure in the tool takes. */ +import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../contract/runtimes.mjs'; import { safeRelativePath } from './filesystem.mjs'; import { fail } from './process.mjs'; -function pythonMajorMinor(version) { - const match = /^(\d+)\.(\d+)(?:\.|$)/.exec(version); - if (!match) fail(`Invalid Python version for execution discovery: ${version}.`); - return `${match[1]}.${match[2]}`; -} - -function moduleEntryPoints({ adapter, module, pythonVersion }) { - const modulePath = module.split('.').join('/'); - const relativeCandidates = [`${modulePath}.py`, `${modulePath}/__main__.py`]; - const standardLibrary = adapter.platform === 'windows' - ? 'venv/Lib' - : `venv/lib/python${pythonMajorMinor(pythonVersion)}`; - const roots = ['', standardLibrary, `${standardLibrary}/site-packages`]; - return roots.flatMap((root) => - relativeCandidates.map((path) => (root ? `${root}/${path}` : path))); -} - /** * Confirms that optional execution metadata names runnable regular files in a payload/archive. * * `files` must contain only regular archive entries. Both collectFiles() during build and the ZIP * entry classifier during verify provide exactly that representation. + * + * @param {object} options + * @param {object | null | undefined} options.execution + * @param {import('../contract/targets.mjs').BoxTargetAdapter} options.adapter + * @param {string} options.runtimeVersion the interpreter version a module search needs + * @param {Set} options.files + * @param {string} [options.runtimeId] + * @returns {void} */ export function assertExecutionFiles({ execution, adapter, - pythonVersion, + runtimeVersion, files, + runtimeId = IMPLICIT_RUNTIME_ID, }) { if (!execution) return; - if (execution.kind === 'python-script') { - const script = safeRelativePath(execution.script); - if (!files.has(script)) fail(`Execution script is missing from the box: ${script}.`); - return; - } - if (execution.kind === 'python-module') { - const candidates = moduleEntryPoints({ - adapter, - module: execution.module, - pythonVersion, - }); - if (!candidates.some((path) => files.has(path))) { - fail(`Execution module is not discoverable in the box: ${execution.module}.`); - } - return; + const runtime = runtimeAdapter(runtimeId); + if (!runtime.executionKinds.includes(execution.kind)) { + fail(`Unsupported execution kind: ${String(execution.kind)}.`); } - fail(`Unsupported execution kind: ${String(execution.kind)}.`); + const { candidates, missing } = runtime.resolveExecutionFiles({ + execution, + runtimeVersion, + target: adapter, + }); + // Every candidate goes through the traversal rule, not just the one a scroll wrote by hand: a + // path the format derived is still a path this process is about to look for. + if (!candidates.some((path) => files.has(safeRelativePath(path)))) fail(missing); } diff --git a/src/build/index.d.mts b/src/build/index.d.mts index 1264a18..d63a4ab 100644 --- a/src/build/index.d.mts +++ b/src/build/index.d.mts @@ -1,4 +1,4 @@ -export { repairPosixLaunchers } from "./launchers.mjs"; +export { repairPosixLaunchers } from "../runtimes/python/launchers.mjs"; export { CONDA_PACK_VERSION } from "./toolchain.mjs"; export { createDeterministicZip, extractZipArchive, listZipEntries } from "./archive.mjs"; export { collectFiles, fileExists, payloadDigest, sha256File } from "./filesystem.mjs"; diff --git a/src/build/index.mjs b/src/build/index.mjs index 275bbef..b1f7f24 100644 --- a/src/build/index.mjs +++ b/src/build/index.mjs @@ -4,12 +4,16 @@ * One substrate only — pixi solves a conda-forge environment from a committed `pixi.lock`, conda-pack * relocates it, and the result is extracted into the box's `venv/`. There is deliberately no second * dependency backend: a packaging tool with two substrates has to prove every guarantee twice. + * + * What the substrate packs is a separate question, and it is answered under `src/runtimes//`. + * `repairPosixLaunchers` is re-exported from there rather than owned here: the conda shebang + * trampoline it rewrites is a Python fact, and the export name is what a caller depends on. */ export { createDeterministicZip, extractZipArchive, listZipEntries } from './archive.mjs'; export { collectFiles, fileExists, payloadDigest, sha256File } from './filesystem.mjs'; export { boxReleaseObjectPrefix, boxReleaseStem, builderVersionFields } from './identity.mjs'; -export { repairPosixLaunchers } from './launchers.mjs'; +export { repairPosixLaunchers } from '../runtimes/python/launchers.mjs'; export { createCondaDependencyLicenseAudit, lockedCondaDistributions, diff --git a/src/build/launchers.d.mts b/src/build/launchers.d.mts deleted file mode 100644 index e64ac3c..0000000 --- a/src/build/launchers.d.mts +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Makes generated POSIX console scripts resolve Python relative to their own installed path. - * - * @param {import('../contract/targets.mjs').BoxTargetAdapter} adapter - * @param {string} payloadDir - * @param {readonly string[]} forbiddenPaths - * @returns {Promise} - */ -export function repairPosixLaunchers(adapter: import("../contract/targets.mjs").BoxTargetAdapter, payloadDir: string, forbiddenPaths: readonly string[]): Promise; diff --git a/src/build/pixi.d.mts b/src/build/pixi.d.mts index 9a7eefb..084138f 100644 --- a/src/build/pixi.d.mts +++ b/src/build/pixi.d.mts @@ -84,14 +84,18 @@ export function findCondaPack({ path, runResult }?: { runResult?: typeof defaultRunResult; }): string; /** - * Builds the box's `venv/` prefix from a scroll's committed pixi.lock and packs it for relocation. + * Builds the box's runtime prefix from a scroll's committed pixi.lock and packs it for relocation. * * Flow: install the exact locked env into an isolated workspace so pixi's `.pixi/envs` never * lands in the tracked scroll dir; conda-pack the prefix into a relocatable tarball; extract it - * into `payloadDir/venv`; remove the service files that carry the build prefix (conda-unpack is - * never run — see below); then dereference every symlink so the payload is link-free for the - * archive layer. The multi-gigabyte workspace and tarball are removed before the payload is - * archived. + * into the runtime's payload root; remove the service files that carry the build prefix + * (conda-unpack is never run — see below); then dereference every symlink so the payload is + * link-free for the archive layer. The multi-gigabyte workspace and tarball are removed before the + * payload is archived. + * + * Where that prefix lands and what the interpreter inside it is called are the runtime's answers, + * not this module's. Packing is substrate work — one pixi, one conda-pack, one tarball, whatever is + * inside it. * * `run` is injected so this composes with the orchestrator's logging and error model. * @@ -104,10 +108,11 @@ export function findCondaPack({ path, runResult }?: { * payloadDir: string, * adapter: import('../contract/targets.mjs').BoxTargetAdapter, * run: typeof import('./process.mjs').run, + * runtimeId?: string, * }} options - * @returns {Promise<{ interpreter: string, prefix: string }>} + * @returns {Promise<{ interpreter: string, venvDir: string, sitePackagesRelative: string }>} */ -export function installAndPackPixiEnvironment({ pixi, condaPack, manifestPath, lockPath, buildDir, payloadDir, adapter, run, }: { +export function installAndPackPixiEnvironment({ pixi, condaPack, manifestPath, lockPath, buildDir, payloadDir, adapter, run, runtimeId, }: { pixi: string; condaPack: string; manifestPath: string; @@ -116,8 +121,10 @@ export function installAndPackPixiEnvironment({ pixi, condaPack, manifestPath, l payloadDir: string; adapter: import("../contract/targets.mjs").BoxTargetAdapter; run: typeof import("./process.mjs").run; + runtimeId?: string; }): Promise<{ interpreter: string; - prefix: string; + venvDir: string; + sitePackagesRelative: string; }>; import { runResult as defaultRunResult } from './process.mjs'; diff --git a/src/build/pixi.mjs b/src/build/pixi.mjs index f7f5b85..0de7a6e 100644 --- a/src/build/pixi.mjs +++ b/src/build/pixi.mjs @@ -18,9 +18,10 @@ import { chmod, copyFile, cp, mkdir, readFile, readdir, readlink, realpath, rm, import { dirname, join, relative, resolve, sep } from 'node:path'; import * as tar from 'tar'; import { resolvePayloadLinkTarget, targetCarriesLinks } from '../contract/links.mjs'; +import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../contract/runtimes.mjs'; import { compareStableStrings, safeRelativePath } from './filesystem.mjs'; import { fail, runResult as defaultRunResult } from './process.mjs'; -import { repairPosixLaunchers } from './launchers.mjs'; +import { repairPosixLaunchers } from '../runtimes/python/launchers.mjs'; import { CONDA_PACK_VERSION, toolchainPaths } from './toolchain.mjs'; import { getWorkspace } from './workspace.mjs'; @@ -313,14 +314,18 @@ async function keepsAsLink(root, linkPath, canonicalRoot) { } /** - * Builds the box's `venv/` prefix from a scroll's committed pixi.lock and packs it for relocation. + * Builds the box's runtime prefix from a scroll's committed pixi.lock and packs it for relocation. * * Flow: install the exact locked env into an isolated workspace so pixi's `.pixi/envs` never * lands in the tracked scroll dir; conda-pack the prefix into a relocatable tarball; extract it - * into `payloadDir/venv`; remove the service files that carry the build prefix (conda-unpack is - * never run — see below); then dereference every symlink so the payload is link-free for the - * archive layer. The multi-gigabyte workspace and tarball are removed before the payload is - * archived. + * into the runtime's payload root; remove the service files that carry the build prefix + * (conda-unpack is never run — see below); then dereference every symlink so the payload is + * link-free for the archive layer. The multi-gigabyte workspace and tarball are removed before the + * payload is archived. + * + * Where that prefix lands and what the interpreter inside it is called are the runtime's answers, + * not this module's. Packing is substrate work — one pixi, one conda-pack, one tarball, whatever is + * inside it. * * `run` is injected so this composes with the orchestrator's logging and error model. * @@ -333,8 +338,9 @@ async function keepsAsLink(root, linkPath, canonicalRoot) { * payloadDir: string, * adapter: import('../contract/targets.mjs').BoxTargetAdapter, * run: typeof import('./process.mjs').run, + * runtimeId?: string, * }} options - * @returns {Promise<{ interpreter: string, prefix: string }>} + * @returns {Promise<{ interpreter: string, venvDir: string, sitePackagesRelative: string }>} */ export async function installAndPackPixiEnvironment({ pixi, @@ -345,7 +351,9 @@ export async function installAndPackPixiEnvironment({ payloadDir, adapter, run, + runtimeId = IMPLICIT_RUNTIME_ID, }) { + const layout = runtimeAdapter(runtimeId).layout(adapter); const workspace = join(buildDir, 'pixi-workspace'); await rm(workspace, { recursive: true, force: true }); await mkdir(workspace, { recursive: true }); @@ -360,7 +368,7 @@ export async function installAndPackPixiEnvironment({ await rm(packPath, { force: true }); run(condaPack, condaPackArguments(prefix, packPath)); - const venvDir = join(payloadDir, 'venv'); + const venvDir = join(payloadDir, ...layout.root.split('/')); await rm(venvDir, { recursive: true, force: true }); await mkdir(venvDir, { recursive: true }); // conda-pack emits the prefix contents at the tar root, so extracting into `venv` yields the @@ -405,7 +413,7 @@ export async function installAndPackPixiEnvironment({ await symlink(link.target, linkPath, type); } - const interpreter = join(payloadDir, ...adapter.python.entryPoint.split('/')); + const interpreter = join(payloadDir, ...layout.entryPoint.split('/')); // Deliberately do NOT run conda-unpack. conda-pack already replaces the build prefix with a // neutral placeholder, and the box imports and runs fine that way (a cold import from a moved // prefix was proven before any fixer). Running the fixer here would stamp the *build machine's* @@ -429,7 +437,7 @@ export async function installAndPackPixiEnvironment({ // conda console scripts (tqdm, isympy, …) embed the absolute build interpreter in a shell // trampoline shebang. Rewrite them to resolve Python next to themselves, so no build path // ships inside the box. - await repairPosixLaunchers(adapter, payloadDir, [prefix, workspace, payloadDir]); + await repairPosixLaunchers(layout, payloadDir, [prefix, workspace, payloadDir]); await rm(workspace, { recursive: true, force: true }); await rm(packPath, { force: true }); diff --git a/src/build/scroll.mjs b/src/build/scroll.mjs index d543ca7..55ab5fa 100644 --- a/src/build/scroll.mjs +++ b/src/build/scroll.mjs @@ -14,7 +14,12 @@ import { readFile, readdir } from 'node:fs/promises'; import { join, resolve, sep } from 'node:path'; -import { assertPythonEntryPoint, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; +import { boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; +import { + IMPLICIT_RUNTIME_ID, + assertRuntimeEntryPoint, + runtimeAdapter, +} from '../contract/runtimes.mjs'; import { compareStableStrings, fileExists, safeRelativePath } from './filesystem.mjs'; import { fail, runResult } from './process.mjs'; import { schemaValidationError } from './schema-validation.mjs'; @@ -218,7 +223,8 @@ function effectiveScroll(scroll, adapter, targetId) { scrollId: scroll.scrollId ?? `${scroll.boxId}-${targetId}`, scrollVersion: scroll.scrollVersion ?? '1.0.0', compatibility: scroll.compatibility ?? {}, - pythonEntryPoint: scroll.pythonEntryPoint ?? adapter.python.entryPoint, + pythonEntryPoint: scroll.pythonEntryPoint + ?? runtimeAdapter(IMPLICIT_RUNTIME_ID).layout(adapter).entryPoint, modelCacheSubdir: scroll.modelCacheSubdir ?? `model-cache/${scroll.boxId}`, assets: scroll.assets ?? [], selfTest: { ...scroll.selfTest, files: scroll.selfTest.files ?? [] }, @@ -274,7 +280,7 @@ async function readExactScroll(reference) { if (targetDirectory !== targetId) { fail(`Nested scroll target directory ${targetDirectory} does not match declared target ${targetId}.`); } - assertPythonEntryPoint(adapter, scroll.pythonEntryPoint); + assertRuntimeEntryPoint(IMPLICIT_RUNTIME_ID, adapter, scroll.pythonEntryPoint); return { adapter, dir, scroll, reference: normalized, targetId }; } diff --git a/src/build/verify.mjs b/src/build/verify.mjs index c3a774a..f520310 100644 --- a/src/build/verify.mjs +++ b/src/build/verify.mjs @@ -13,8 +13,18 @@ import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; -import { assertNativeHost, assertPythonEntryPoint, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; -import { BOX_SCHEMA_VERSION, parseDocumentKind } from '../contract/documents.mjs'; +import { assertNativeHost, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; +import { + IMPLICIT_RUNTIME_ID, + assertRuntimeEntryPoint, + executionAffectingVariables, + runtimeAdapter, +} from '../contract/runtimes.mjs'; +import { + BOX_SCHEMA_VERSION, + parseDocumentKind, + unsupportedSchemaVersionMessage, +} from '../contract/documents.mjs'; import { resolveTrustedKeys, verifySignedDocument } from '../sign/index.mjs'; const AGREEMENT_FIELDS = [ @@ -83,18 +93,15 @@ export async function inspectReleaseDocument(releaseDocumentPath, { publicPath, const releasePath = resolve(releaseDocumentPath); const signed = JSON.parse(await readFile(releasePath, 'utf8')); if (signed?.schemaVersion === 1) { - fail('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); + fail(unsupportedSchemaVersionMessage(1)); } const [releaseSchema, boxSchema, targetSchema, executionSchema, signedSchema] = await loadManifestSchemas(); const signedError = schemaValidationError(signed, signedSchema); if (signedError) fail(`Invalid signed document: ${signedError}.`); const release = await verifySignedDocument(signed, await resolveTrustedKeys({ publicPath, trustedKeys })); - if (release?.schemaVersion === 1) { - fail('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); - } if (release?.schemaVersion !== BOX_SCHEMA_VERSION) { - fail(`Unsupported schemaVersion ${String(release?.schemaVersion)}; expected ${BOX_SCHEMA_VERSION}.`); + fail(unsupportedSchemaVersionMessage(release?.schemaVersion)); } const releaseError = schemaValidationError( release, @@ -104,7 +111,7 @@ export async function inspectReleaseDocument(releaseDocumentPath, { publicPath, if (releaseError) fail(`Invalid release manifest: ${releaseError}.`); if (parseDocumentKind(release.kind)?.type !== 'release') fail('Document is not a box release.'); const adapter = boxTargetAdapter(release.target); - assertPythonEntryPoint(adapter, release.pythonEntryPoint); + assertRuntimeEntryPoint(IMPLICIT_RUNTIME_ID, adapter, release.pythonEntryPoint); // Describes the extracted tree rather than the archive, so it belongs on this side of the split. // `payloadDigest` needs no companion check: its `format` is a schema `const`, so a release naming // a format this build cannot read is already refused above, by name, as an invalid manifest. @@ -164,7 +171,7 @@ export async function inspectBoxArchive(releaseDocumentPath, options = {}) { assertExecutionFiles({ execution: release.execution, adapter, - pythonVersion: release.provenance.pythonVersion, + runtimeVersion: release.provenance.pythonVersion, files: resolvablePaths, }); @@ -213,7 +220,7 @@ export async function verifyBox(releaseDocumentPath, options = {}) { const resolvedEnvironment = resolveEnvironment({ platform: adapter.platform, layers: environmentLayers, - executionAffectingVariables: adapter.executionAffectingEnvironmentVariables, + executionAffectingVariables: executionAffectingVariables(IMPLICIT_RUNTIME_ID, adapter), expanded: Boolean(options.envReport || options.envReportValues), revealHostValues: Boolean(options.envReportValues), }); @@ -239,7 +246,13 @@ export async function verifyBox(releaseDocumentPath, options = {}) { fail('Extracted payload does not match the signed release.'); } const python = join(extracted, safeRelativePath(release.pythonEntryPoint)); - run(python, ['-c', `${adapter.selfTestPython}\nimport ${release.selfTest.pythonImports.join(', ')}`], { + // The signed subset only: a consumer repeating this check has the imports and nothing else, + // and the runtime is the only thing that knows how to turn them into a command line. + const argv = runtimeAdapter(IMPLICIT_RUNTIME_ID).selfTestArgv({ + probe: { imports: release.selfTest.pythonImports }, + target: adapter, + }); + run(python, argv, { cwd: extracted, env: resolvedEnvironment.environment, }); diff --git a/src/cli.mjs b/src/cli.mjs index a0a357e..950f047 100755 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -33,7 +33,8 @@ import { installTypeScriptConsumerDependencies, SCROLLCASE_NPM_VERSION, } from './build/consumer-setup.mjs'; -import { addDependency, readRequirements } from './build/dependencies.mjs'; +import { addDependency } from './build/dependencies.mjs'; +import { readRequirements } from './runtimes/python/dependencies.mjs'; import { findPixi, pixiLockArguments } from './build/pixi.mjs'; import { fail, run } from './build/process.mjs'; import { diagnose, ensureToolchain, initProject } from './build/project.mjs'; diff --git a/src/consumer/run-extracted.mjs b/src/consumer/run-extracted.mjs index 7af033e..5f5b226 100644 --- a/src/consumer/run-extracted.mjs +++ b/src/consumer/run-extracted.mjs @@ -13,6 +13,11 @@ import { collectFiles, safeRelativePath } from '../build/filesystem.mjs'; import { assertExecutionFiles } from '../build/execution.mjs'; import { fail } from '../build/process.mjs'; import { assertNativeHost, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; +import { + IMPLICIT_RUNTIME_ID, + executionAffectingVariables, + runtimeAdapter, +} from '../contract/runtimes.mjs'; import { resolveEnvironment } from '../environment.mjs'; import { preparedBoxState, verifyRequiredAssets } from './verify-and-extract.mjs'; @@ -119,16 +124,22 @@ export async function runExtractedBox(prepared, options = {}) { assertExecutionFiles({ execution: release.execution, adapter, - pythonVersion: release.provenance.pythonVersion, + runtimeVersion: release.provenance.pythonVersion, files, }); await verifyRequiredAssets(prepared.root, prepared.requiredAssets); const python = join(prepared.root, ...safeRelativePath(release.pythonEntryPoint).split('/')); - const executionArgs = release.execution.kind === 'python-script' - ? [join(prepared.root, ...safeRelativePath(release.execution.script).split('/'))] - : ['-m', release.execution.module]; - executionArgs.push(...release.execution.defaultArgs, ...callerArgs); + // The runtime states the command line in payload-relative terms and this end joins it: a box root + // is a real path on this host, and the format has no business deciding what one looks like. + const { args } = runtimeAdapter(IMPLICIT_RUNTIME_ID).buildArgv({ + execution: release.execution, + target: adapter, + }); + const executionArgs = args.map((argument) => (argument.kind === 'payload-path' + ? join(prepared.root, ...safeRelativePath(argument.value).split('/')) + : argument.value)); + executionArgs.push(...callerArgs); const { environment, report: environmentReport } = resolveEnvironment({ platform: adapter.platform, @@ -137,7 +148,7 @@ export async function runExtractedBox(prepared, options = {}) { { source: 'caller', values: options.env }, { source: 'release', values: release.environment }, ], - executionAffectingVariables: adapter.executionAffectingEnvironmentVariables, + executionAffectingVariables: executionAffectingVariables(IMPLICIT_RUNTIME_ID, adapter), expanded: Boolean(options.envReport || options.envReportValues), revealHostValues: Boolean(options.envReportValues), }); diff --git a/src/consumer/verify-and-extract.mjs b/src/consumer/verify-and-extract.mjs index c9013e0..121f88f 100644 --- a/src/consumer/verify-and-extract.mjs +++ b/src/consumer/verify-and-extract.mjs @@ -29,6 +29,7 @@ import { parsePayloadDigestStream, } from '../contract/payload-digest.mjs'; import { assertNativeHost, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; +import { IMPLICIT_RUNTIME_ID, executionAffectingVariables } from '../contract/runtimes.mjs'; import { resolveEnvironment } from '../environment.mjs'; /** @@ -152,7 +153,7 @@ function releaseEnvironmentReport(release, options = {}) { { source: 'host', values: process.env }, { source: 'release', values: release.environment }, ], - executionAffectingVariables: adapter.executionAffectingEnvironmentVariables, + executionAffectingVariables: executionAffectingVariables(IMPLICIT_RUNTIME_ID, adapter), expanded: Boolean(options.envReport || options.envReportValues), revealHostValues: Boolean(options.envReportValues), }).report; @@ -322,7 +323,7 @@ export async function attachExtractedBox(releaseDocumentPath, { assertExecutionFiles({ execution: release.execution, adapter, - pythonVersion: release.provenance.pythonVersion, + runtimeVersion: release.provenance.pythonVersion, files, }); await verifyRequiredAssets(boxRoot, requiredAssetsOf(release)); diff --git a/src/contract/document-shape.d.mts b/src/contract/document-shape.d.mts index 569047f..8119583 100644 --- a/src/contract/document-shape.d.mts +++ b/src/contract/document-shape.d.mts @@ -24,6 +24,20 @@ export function parseDocumentKind(kind: unknown): { namespace: string; type: "release" | "channel" | "revocations"; } | null; +/** + * The message any reader gives a document written to a format version it cannot read. + * + * Four Node call sites answer this question — the payload decoder, the key loader, and the release + * verifier twice — and until now each carried its own copy of the sentence. That is not a style + * problem: the next version bump has to change what a v1 document is told, and a message duplicated + * per call site is a message that gets changed in three of four places. There is one wording per + * language now, and each language keeps its own because the string is user-facing text, not wire + * data that has to match across implementations. + * + * @param {unknown} version the `schemaVersion` the document declared + * @returns {string} + */ +export function unsupportedSchemaVersionMessage(version: unknown): string; /** * Reports whether a value is a structurally valid signed envelope. * diff --git a/src/contract/document-shape.mjs b/src/contract/document-shape.mjs index 2aa9fbc..b877f31 100644 --- a/src/contract/document-shape.mjs +++ b/src/contract/document-shape.mjs @@ -64,6 +64,25 @@ export function parseDocumentKind(kind) { return { namespace, type }; } +/** + * The message any reader gives a document written to a format version it cannot read. + * + * Four Node call sites answer this question — the payload decoder, the key loader, and the release + * verifier twice — and until now each carried its own copy of the sentence. That is not a style + * problem: the next version bump has to change what a v1 document is told, and a message duplicated + * per call site is a message that gets changed in three of four places. There is one wording per + * language now, and each language keeps its own because the string is user-facing text, not wire + * data that has to match across implementations. + * + * @param {unknown} version the `schemaVersion` the document declared + * @returns {string} + */ +export function unsupportedSchemaVersionMessage(version) { + return version === 1 + ? 'Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.' + : `Unsupported schemaVersion ${String(version)}; expected ${BOX_SCHEMA_VERSION}.`; +} + /** Channels a box may be published to, ordered from least to most stable. */ export const CHANNELS = Object.freeze(['nightly', 'beta', 'stable']); diff --git a/src/contract/documents.d.mts b/src/contract/documents.d.mts index ea21198..3a5dbc6 100644 --- a/src/contract/documents.d.mts +++ b/src/contract/documents.d.mts @@ -10,4 +10,4 @@ * @throws {Error} when the embedded payload hash does not match the bytes */ export function decodeDocumentPayload(document: import("./types/index.d.ts").SignedBoxDocument): unknown; -export { BOX_SCHEMA_VERSION, CHANNELS, DEFAULT_DOCUMENT_NAMESPACE, PAYLOAD_ENCODING, SIGNATURE_ALGORITHM, documentKinds, isSignedBoxDocument, parseDocumentKind } from "./document-shape.mjs"; +export { BOX_SCHEMA_VERSION, CHANNELS, DEFAULT_DOCUMENT_NAMESPACE, PAYLOAD_ENCODING, SIGNATURE_ALGORITHM, documentKinds, isSignedBoxDocument, parseDocumentKind, unsupportedSchemaVersionMessage } from "./document-shape.mjs"; diff --git a/src/contract/documents.mjs b/src/contract/documents.mjs index 78f12a4..9429c9e 100644 --- a/src/contract/documents.mjs +++ b/src/contract/documents.mjs @@ -8,7 +8,7 @@ */ import { createHash } from 'node:crypto'; -import { isSignedBoxDocument } from './document-shape.mjs'; +import { isSignedBoxDocument, unsupportedSchemaVersionMessage } from './document-shape.mjs'; export { BOX_SCHEMA_VERSION, @@ -19,6 +19,7 @@ export { documentKinds, isSignedBoxDocument, parseDocumentKind, + unsupportedSchemaVersionMessage, } from './document-shape.mjs'; /** @@ -34,7 +35,7 @@ export { */ export function decodeDocumentPayload(document) { if (document?.schemaVersion === 1) { - throw new TypeError('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); + throw new TypeError(unsupportedSchemaVersionMessage(1)); } if (!isSignedBoxDocument(document)) { throw new TypeError('Not a signed box document'); diff --git a/src/contract/fixtures/runtime-contract.json b/src/contract/fixtures/runtime-contract.json new file mode 100644 index 0000000..f567304 --- /dev/null +++ b/src/contract/fixtures/runtime-contract.json @@ -0,0 +1,269 @@ +{ + "description": "Golden cases for the Scrollcase runtime model: where a runtime lives inside a box, which payload paths it needs the executable bit on, which paths a declared execution could resolve to, and the shell-free command line that runs it. Every implementation of the format proves its mirror against this file. Paths stay payload-relative on purpose: a box root is a real filesystem path and each language joins one in its own terms, so a joined expectation here would only pin the host that read it.", + "runtimes": [ + { + "id": "python", + "executionKinds": ["python-script", "python-module"], + "executionEnvironmentVariables": [ + "PYTHONPATH", + "PYTHONHOME", + "PYTHONSTARTUP", + "PYTHONBREAKPOINT" + ], + "layouts": [ + { + "platform": "macos", + "layout": { + "root": "venv", + "entryPoint": "venv/bin/python", + "scriptsDirectory": "venv/bin", + "standardLibrary": "venv/lib", + "executableSuffix": "", + "launcherKind": "posix-polyglot" + }, + "executablePayloadPaths": { + "files": ["venv/bin/python"], + "directories": ["venv/bin"] + } + }, + { + "platform": "linux", + "layout": { + "root": "venv", + "entryPoint": "venv/bin/python", + "scriptsDirectory": "venv/bin", + "standardLibrary": "venv/lib", + "executableSuffix": "", + "launcherKind": "posix-polyglot" + }, + "executablePayloadPaths": { + "files": ["venv/bin/python"], + "directories": ["venv/bin"] + } + }, + { + "platform": "windows", + "layout": { + "root": "venv", + "entryPoint": "venv/python.exe", + "scriptsDirectory": "venv/Scripts", + "standardLibrary": "venv/Lib", + "executableSuffix": ".exe", + "launcherKind": "uv-windows-pe" + }, + "executablePayloadPaths": { + "files": ["venv/python.exe"], + "directories": ["venv/Scripts"] + } + } + ] + } + ], + "executableMatches": [ + { + "name": "the interpreter itself, by exact name", + "runtime": "python", + "platform": "linux", + "path": "venv/bin/python", + "executable": true + }, + { + "name": "a generated console script, by directory", + "runtime": "python", + "platform": "linux", + "path": "venv/bin/tqdm", + "executable": true + }, + { + "name": "a nested path under the scripts directory", + "runtime": "python", + "platform": "linux", + "path": "venv/bin/nested/tool", + "executable": true + }, + { + "name": "a library the interpreter loads but never executes", + "runtime": "python", + "platform": "linux", + "path": "venv/lib/python3.14/os.py", + "executable": false + }, + { + "name": "a sibling directory whose name only starts the same way", + "runtime": "python", + "platform": "linux", + "path": "venv/binary-blob", + "executable": false + }, + { + "name": "the application's own entry point, which the runtime never claims", + "runtime": "python", + "platform": "linux", + "path": "entrypoint.py", + "executable": false + }, + { + "name": "the Windows interpreter, which sits outside its scripts directory", + "runtime": "python", + "platform": "windows", + "path": "venv/python.exe", + "executable": true + }, + { + "name": "a Windows console script", + "runtime": "python", + "platform": "windows", + "path": "venv/Scripts/pip.exe", + "executable": true + }, + { + "name": "the POSIX scripts directory on a Windows box", + "runtime": "python", + "platform": "windows", + "path": "venv/bin/tqdm", + "executable": false + } + ], + "executionDiscovery": [ + { + "name": "a script resolves at exactly the path it declares", + "runtime": "python", + "platform": "linux", + "runtimeVersion": "3.11.15", + "execution": { "kind": "python-script", "script": "app/main.py", "defaultArgs": [] }, + "candidates": ["app/main.py"] + }, + { + "name": "a POSIX module is looked for at the root, in the standard library, and in site-packages", + "runtime": "python", + "platform": "linux", + "runtimeVersion": "3.11.15", + "execution": { + "kind": "python-module", + "module": "example_model.main", + "defaultArgs": ["--serve"] + }, + "candidates": [ + "example_model/main.py", + "example_model/main/__main__.py", + "venv/lib/python3.11/example_model/main.py", + "venv/lib/python3.11/example_model/main/__main__.py", + "venv/lib/python3.11/site-packages/example_model/main.py", + "venv/lib/python3.11/site-packages/example_model/main/__main__.py" + ] + }, + { + "name": "a patch component names the same standard library directory", + "runtime": "python", + "platform": "macos", + "runtimeVersion": "3.12.4", + "execution": { "kind": "python-module", "module": "pkg", "defaultArgs": [] }, + "candidates": [ + "pkg.py", + "pkg/__main__.py", + "venv/lib/python3.12/pkg.py", + "venv/lib/python3.12/pkg/__main__.py", + "venv/lib/python3.12/site-packages/pkg.py", + "venv/lib/python3.12/site-packages/pkg/__main__.py" + ] + }, + { + "name": "Windows names its standard library once, without an interpreter version", + "runtime": "python", + "platform": "windows", + "runtimeVersion": "3.11.15", + "execution": { "kind": "python-module", "module": "pkg", "defaultArgs": [] }, + "candidates": [ + "pkg.py", + "pkg/__main__.py", + "venv/Lib/pkg.py", + "venv/Lib/pkg/__main__.py", + "venv/Lib/site-packages/pkg.py", + "venv/Lib/site-packages/pkg/__main__.py" + ] + } + ], + "invalidRuntimeVersions": ["", "3", "3.x", "x.1", "3."], + "argv": [ + { + "name": "a script runs as a payload path, with its declared arguments after it", + "runtime": "python", + "platform": "linux", + "execution": { + "kind": "python-script", + "script": "app/main.py", + "defaultArgs": ["--serve", "--port", "8080"] + }, + "command": { "kind": "payload-path", "value": "venv/bin/python" }, + "args": [ + { "kind": "payload-path", "value": "app/main.py" }, + { "kind": "literal", "value": "--serve" }, + { "kind": "literal", "value": "--port" }, + { "kind": "literal", "value": "8080" } + ] + }, + { + "name": "a module runs through -m and is never a path", + "runtime": "python", + "platform": "macos", + "execution": { + "kind": "python-module", + "module": "example_model.main", + "defaultArgs": [] + }, + "command": { "kind": "payload-path", "value": "venv/bin/python" }, + "args": [ + { "kind": "literal", "value": "-m" }, + { "kind": "literal", "value": "example_model.main" } + ] + }, + { + "name": "a Windows box runs the same declaration through its own interpreter", + "runtime": "python", + "platform": "windows", + "execution": { + "kind": "python-script", + "script": "app/main.py", + "defaultArgs": [] + }, + "command": { "kind": "payload-path", "value": "venv/python.exe" }, + "args": [{ "kind": "payload-path", "value": "app/main.py" }] + } + ], + "selfTest": [ + { + "name": "macOS asserts Darwin before importing anything", + "runtime": "python", + "platform": "macos", + "probe": { "imports": ["json"] }, + "args": ["-c", "import sys; assert sys.platform == 'darwin'\nimport json"] + }, + { + "name": "Linux accepts any linux variant", + "runtime": "python", + "platform": "linux", + "probe": { "imports": ["json", "numpy"] }, + "args": [ + "-c", + "import sys; assert sys.platform.startswith('linux')\nimport json, numpy" + ] + }, + { + "name": "Windows asserts win32", + "runtime": "python", + "platform": "windows", + "probe": { "imports": ["json"] }, + "args": ["-c", "import sys; assert sys.platform == 'win32'\nimport json"] + }, + { + "name": "the builder appends the extra source a scroll declared", + "runtime": "python", + "platform": "linux", + "probe": { "imports": ["json"], "code": "print(\"self-test ok\")\n" }, + "args": [ + "-c", + "import sys; assert sys.platform.startswith('linux')\nimport json\nprint(\"self-test ok\")\n" + ] + } + ] +} diff --git a/src/contract/runtimes.d.mts b/src/contract/runtimes.d.mts new file mode 100644 index 0000000..a44f3ac --- /dev/null +++ b/src/contract/runtimes.d.mts @@ -0,0 +1,284 @@ +/** + * Returns the runtime adapter for a runtime id. + * + * @param {string} runtimeId + * @returns {BoxRuntimeAdapter} + * @throws {TypeError} when no runtime with that id exists + */ +export function runtimeAdapter(runtimeId: string): BoxRuntimeAdapter; +/** + * Lists every runtime adapter, for contract tests and for callers enumerating what a box may be. + * + * @returns {BoxRuntimeAdapter[]} every runtime adapter, as a fresh array + */ +export function runtimeAdapters(): BoxRuntimeAdapter[]; +/** + * Ensures a declared entry point agrees with where the runtime actually sits in the payload. + * + * @param {string} runtimeId + * @param {import('./targets.mjs').BoxTargetAdapter} adapter the resolved target adapter, whose id + * names the layout the entry point is being judged against + * @param {string} entryPoint + * @returns {void} + * @throws {TypeError} when the entry point is not the one the runtime defines for this target + */ +export function assertRuntimeEntryPoint(runtimeId: string, adapter: import("./targets.mjs").BoxTargetAdapter, entryPoint: string): void; +/** + * Whether a payload path is one the runtime requires the executable bit on. + * + * A directory matches by prefix so one rule covers a whole generated scripts tree; an exact file + * match covers the runtime's own entry point, which lives outside it on Windows. + * + * @param {ExecutablePayloadPaths} rule + * @param {string} relativePath forward-slash path relative to the box root + * @returns {boolean} + */ +export function isExecutablePayloadPath(rule: ExecutablePayloadPaths, relativePath: string): boolean; +/** + * The complete list of inherited variables that can change what a box executes. + * + * Two halves, because they have two owners: the runtime contributes the variables its own loader + * reads, and the target contributes the operating system's dynamic-linker controls. Callers want + * one list, and assembling it here rather than at each of them is what keeps a diagnostic report + * from depending on which call site produced it. + * + * @param {string} runtimeId + * @param {import('./targets.mjs').BoxTargetAdapter} adapter + * @returns {readonly string[]} the runtime's variables followed by the target's + */ +export function executionAffectingVariables(runtimeId: string, adapter: import("./targets.mjs").BoxTargetAdapter): readonly string[]; +/** + * Reference implementation of the Scrollcase box-format runtime model. + * + * A target says which machine a box runs on; a runtime says what runs *inside* it — where the + * interpreter sits, which execution kinds exist, how a declared entry point becomes a command line, + * and which inherited environment variables can change what that command loads. Those are different + * questions with different answers, and until now they lived in one table: `targets.mjs` carried a + * nested `python: {…}` block and a Python self-test assertion, so every target adapter was also a + * statement that a box is a Python box. + * + * Splitting them is what makes a second runtime an adapter rather than a fork. This module is the + * runtime half, and it is contract-level for the same reason `targets.mjs` is: a consumer unpacking + * a box relies on the layout, and a consumer running one relies on the argv rule. The golden cases + * in `fixtures/runtime-contract.json` are what "agree" means, and are what the Python and Rust + * mirrors validate themselves against. + * + * Only the pure half lives here. Nothing in this module reads a file, joins a host path, or starts + * a process: every function is a statement about names, so the same inputs give the same answer in + * every language and on every host. Builder-side behaviour — environment preparation, launcher + * repair, authoring templates — lives under `src/runtimes//`, which may do all three. + * + * Two shapes deserve their reasons stated: + * + * - `buildArgv` returns payload-*relative* paths tagged as paths rather than a joined command line. + * A box root is a real filesystem path and the three consumers join one in their own platform's + * terms; returning a joined string would put "what a Windows path looks like" inside the format, + * and would make the golden fixture depend on the host that reads it. + * - `resolveExecutionFiles` returns candidates plus the message for when none of them resolve, + * instead of throwing. The caller owns the error path — `fail()` in the builder, a typed error in + * each consumer — and the wording is part of the contract, so it belongs beside the rule that + * produces it rather than being restated at every call site. + */ +/** + * What a runtime implies for a box, independent of the machine it runs on. + * + * @typedef {object} BoxRuntimeAdapter + * @property {string} id canonical runtime id, e.g. `python` + * @property {readonly string[]} executionKinds the `execution.kind` values this runtime defines + * @property {readonly string[]} executionEnvironmentVariables inherited variables whose presence + * can change which code this runtime loads — the runtime half of the diagnostic list, to which + * the target adapter adds the operating system's own + * @property {(target: BoxRuntimeTarget) => BoxRuntimeLayout} layout where the runtime lives inside + * the payload + * @property {(target: BoxRuntimeTarget) => ExecutablePayloadPaths} executablePayloadPaths payload + * paths the runtime itself requires the executable bit on + * @property {(options: { execution: object, runtimeVersion: string, + * target: BoxRuntimeTarget }) => ResolvedExecutionFiles} resolveExecutionFiles + * @property {(options: { execution: object, + * target: BoxRuntimeTarget }) => BoxRuntimeInvocation} buildArgv + * @property {(options: { probe: BoxRuntimeSelfTestProbe, + * target: BoxRuntimeTarget }) => readonly string[]} selfTestArgv the arguments that follow the + * runtime's own entry point when it runs a self-test probe + */ +/** + * The part of a target a runtime rule reads. A `BoxTarget` and the `BoxTargetAdapter` resolved from + * one both satisfy it, so callers pass whichever they are already holding. + * + * @typedef {{ platform: string }} BoxRuntimeTarget + */ +/** + * Where a runtime lives inside an extracted box. + * + * @typedef {object} BoxRuntimeLayout + * @property {string} root directory the runtime was relocated into + * @property {string} entryPoint the runtime's own executable, relative to the box root + * @property {string} scriptsDirectory directory holding generated console scripts + * @property {string} standardLibrary directory holding the runtime's bundled library + * @property {string} executableSuffix suffix an executable carries on this platform + * @property {string} launcherKind frozen wire string naming how launchers were repaired + */ +/** + * Payload paths a runtime requires the executable bit on, as a rule rather than a list: a conda + * prefix carries hundreds of console scripts and no scroll could name them by hand. + * + * @typedef {{ files: readonly string[], directories: readonly string[] }} ExecutablePayloadPaths + */ +/** + * @typedef {object} ResolvedExecutionFiles + * @property {readonly string[]} candidates payload paths, any one of which resolving satisfies the + * declaration + * @property {string} missing the message for a box where none of them do + */ +/** + * One element of a shell-free command line: either a literal argument or a payload path the caller + * resolves against the box root. + * + * @typedef {{ kind: 'literal' | 'payload-path', value: string }} BoxRuntimeArgument + */ +/** + * @typedef {object} BoxRuntimeInvocation + * @property {BoxRuntimeArgument} command the runtime's own entry point + * @property {readonly BoxRuntimeArgument[]} args everything the box declared, before the caller's + * own arguments + */ +/** + * What a self-test asks the runtime to prove, plus the builder-only extension a scroll may add. + * + * @typedef {{ imports: readonly string[], code?: string | null }} BoxRuntimeSelfTestProbe + */ +/** + * The runtime every box built by this schema version implicitly declares. + * + * The wire format has no runtime field: a box records a Python entry point and Python execution + * kinds and nothing that says "Python". So a reader that must name a runtime names this one, from + * one place — the point being that adding the declaration later changes an argument rather than + * starting a hunt for hard-coded strings. + */ +export const IMPLICIT_RUNTIME_ID: "python"; +/** + * What a runtime implies for a box, independent of the machine it runs on. + */ +export type BoxRuntimeAdapter = { + /** + * canonical runtime id, e.g. `python` + */ + id: string; + /** + * the `execution.kind` values this runtime defines + */ + executionKinds: readonly string[]; + /** + * inherited variables whose presence + * can change which code this runtime loads — the runtime half of the diagnostic list, to which + * the target adapter adds the operating system's own + */ + executionEnvironmentVariables: readonly string[]; + /** + * where the runtime lives inside + * the payload + */ + layout: (target: BoxRuntimeTarget) => BoxRuntimeLayout; + /** + * payload + * paths the runtime itself requires the executable bit on + */ + executablePayloadPaths: (target: BoxRuntimeTarget) => ExecutablePayloadPaths; + resolveExecutionFiles: (options: { + execution: object; + runtimeVersion: string; + target: BoxRuntimeTarget; + }) => ResolvedExecutionFiles; + buildArgv: (options: { + execution: object; + target: BoxRuntimeTarget; + }) => BoxRuntimeInvocation; + /** + * the arguments that follow the + * runtime's own entry point when it runs a self-test probe + */ + selfTestArgv: (options: { + probe: BoxRuntimeSelfTestProbe; + target: BoxRuntimeTarget; + }) => readonly string[]; +}; +/** + * The part of a target a runtime rule reads. A `BoxTarget` and the `BoxTargetAdapter` resolved from + * one both satisfy it, so callers pass whichever they are already holding. + */ +export type BoxRuntimeTarget = { + platform: string; +}; +/** + * Where a runtime lives inside an extracted box. + */ +export type BoxRuntimeLayout = { + /** + * directory the runtime was relocated into + */ + root: string; + /** + * the runtime's own executable, relative to the box root + */ + entryPoint: string; + /** + * directory holding generated console scripts + */ + scriptsDirectory: string; + /** + * directory holding the runtime's bundled library + */ + standardLibrary: string; + /** + * suffix an executable carries on this platform + */ + executableSuffix: string; + /** + * frozen wire string naming how launchers were repaired + */ + launcherKind: string; +}; +/** + * Payload paths a runtime requires the executable bit on, as a rule rather than a list: a conda + * prefix carries hundreds of console scripts and no scroll could name them by hand. + */ +export type ExecutablePayloadPaths = { + files: readonly string[]; + directories: readonly string[]; +}; +export type ResolvedExecutionFiles = { + /** + * payload paths, any one of which resolving satisfies the + * declaration + */ + candidates: readonly string[]; + /** + * the message for a box where none of them do + */ + missing: string; +}; +/** + * One element of a shell-free command line: either a literal argument or a payload path the caller + * resolves against the box root. + */ +export type BoxRuntimeArgument = { + kind: "literal" | "payload-path"; + value: string; +}; +export type BoxRuntimeInvocation = { + /** + * the runtime's own entry point + */ + command: BoxRuntimeArgument; + /** + * everything the box declared, before the caller's + * own arguments + */ + args: readonly BoxRuntimeArgument[]; +}; +/** + * What a self-test asks the runtime to prove, plus the builder-only extension a scroll may add. + */ +export type BoxRuntimeSelfTestProbe = { + imports: readonly string[]; + code?: string | null; +}; diff --git a/src/contract/runtimes.mjs b/src/contract/runtimes.mjs new file mode 100644 index 0000000..be7bd08 --- /dev/null +++ b/src/contract/runtimes.mjs @@ -0,0 +1,341 @@ +/** + * Reference implementation of the Scrollcase box-format runtime model. + * + * A target says which machine a box runs on; a runtime says what runs *inside* it — where the + * interpreter sits, which execution kinds exist, how a declared entry point becomes a command line, + * and which inherited environment variables can change what that command loads. Those are different + * questions with different answers, and until now they lived in one table: `targets.mjs` carried a + * nested `python: {…}` block and a Python self-test assertion, so every target adapter was also a + * statement that a box is a Python box. + * + * Splitting them is what makes a second runtime an adapter rather than a fork. This module is the + * runtime half, and it is contract-level for the same reason `targets.mjs` is: a consumer unpacking + * a box relies on the layout, and a consumer running one relies on the argv rule. The golden cases + * in `fixtures/runtime-contract.json` are what "agree" means, and are what the Python and Rust + * mirrors validate themselves against. + * + * Only the pure half lives here. Nothing in this module reads a file, joins a host path, or starts + * a process: every function is a statement about names, so the same inputs give the same answer in + * every language and on every host. Builder-side behaviour — environment preparation, launcher + * repair, authoring templates — lives under `src/runtimes//`, which may do all three. + * + * Two shapes deserve their reasons stated: + * + * - `buildArgv` returns payload-*relative* paths tagged as paths rather than a joined command line. + * A box root is a real filesystem path and the three consumers join one in their own platform's + * terms; returning a joined string would put "what a Windows path looks like" inside the format, + * and would make the golden fixture depend on the host that reads it. + * - `resolveExecutionFiles` returns candidates plus the message for when none of them resolve, + * instead of throwing. The caller owns the error path — `fail()` in the builder, a typed error in + * each consumer — and the wording is part of the contract, so it belongs beside the rule that + * produces it rather than being restated at every call site. + */ + +/** + * What a runtime implies for a box, independent of the machine it runs on. + * + * @typedef {object} BoxRuntimeAdapter + * @property {string} id canonical runtime id, e.g. `python` + * @property {readonly string[]} executionKinds the `execution.kind` values this runtime defines + * @property {readonly string[]} executionEnvironmentVariables inherited variables whose presence + * can change which code this runtime loads — the runtime half of the diagnostic list, to which + * the target adapter adds the operating system's own + * @property {(target: BoxRuntimeTarget) => BoxRuntimeLayout} layout where the runtime lives inside + * the payload + * @property {(target: BoxRuntimeTarget) => ExecutablePayloadPaths} executablePayloadPaths payload + * paths the runtime itself requires the executable bit on + * @property {(options: { execution: object, runtimeVersion: string, + * target: BoxRuntimeTarget }) => ResolvedExecutionFiles} resolveExecutionFiles + * @property {(options: { execution: object, + * target: BoxRuntimeTarget }) => BoxRuntimeInvocation} buildArgv + * @property {(options: { probe: BoxRuntimeSelfTestProbe, + * target: BoxRuntimeTarget }) => readonly string[]} selfTestArgv the arguments that follow the + * runtime's own entry point when it runs a self-test probe + */ + +/** + * The part of a target a runtime rule reads. A `BoxTarget` and the `BoxTargetAdapter` resolved from + * one both satisfy it, so callers pass whichever they are already holding. + * + * @typedef {{ platform: string }} BoxRuntimeTarget + */ + +/** + * Where a runtime lives inside an extracted box. + * + * @typedef {object} BoxRuntimeLayout + * @property {string} root directory the runtime was relocated into + * @property {string} entryPoint the runtime's own executable, relative to the box root + * @property {string} scriptsDirectory directory holding generated console scripts + * @property {string} standardLibrary directory holding the runtime's bundled library + * @property {string} executableSuffix suffix an executable carries on this platform + * @property {string} launcherKind frozen wire string naming how launchers were repaired + */ + +/** + * Payload paths a runtime requires the executable bit on, as a rule rather than a list: a conda + * prefix carries hundreds of console scripts and no scroll could name them by hand. + * + * @typedef {{ files: readonly string[], directories: readonly string[] }} ExecutablePayloadPaths + */ + +/** + * @typedef {object} ResolvedExecutionFiles + * @property {readonly string[]} candidates payload paths, any one of which resolving satisfies the + * declaration + * @property {string} missing the message for a box where none of them do + */ + +/** + * One element of a shell-free command line: either a literal argument or a payload path the caller + * resolves against the box root. + * + * @typedef {{ kind: 'literal' | 'payload-path', value: string }} BoxRuntimeArgument + */ + +/** + * @typedef {object} BoxRuntimeInvocation + * @property {BoxRuntimeArgument} command the runtime's own entry point + * @property {readonly BoxRuntimeArgument[]} args everything the box declared, before the caller's + * own arguments + */ + +/** + * What a self-test asks the runtime to prove, plus the builder-only extension a scroll may add. + * + * @typedef {{ imports: readonly string[], code?: string | null }} BoxRuntimeSelfTestProbe + */ + +/** + * The runtime every box built by this schema version implicitly declares. + * + * The wire format has no runtime field: a box records a Python entry point and Python execution + * kinds and nothing that says "Python". So a reader that must name a runtime names this one, from + * one place — the point being that adding the declaration later changes an argument rather than + * starting a hunt for hard-coded strings. + */ +export const IMPLICIT_RUNTIME_ID = 'python'; + +const PYTHON_EXECUTION_ENVIRONMENT = Object.freeze([ + 'PYTHONPATH', + 'PYTHONHOME', + 'PYTHONSTARTUP', + 'PYTHONBREAKPOINT', +]); + +const POSIX_PYTHON_LAYOUT = Object.freeze({ + root: 'venv', + entryPoint: 'venv/bin/python', + scriptsDirectory: 'venv/bin', + standardLibrary: 'venv/lib', + executableSuffix: '', + launcherKind: 'posix-polyglot', +}); + +const PYTHON_LAYOUTS = Object.freeze({ + macos: POSIX_PYTHON_LAYOUT, + linux: POSIX_PYTHON_LAYOUT, + windows: Object.freeze({ + root: 'venv', + entryPoint: 'venv/python.exe', + scriptsDirectory: 'venv/Scripts', + standardLibrary: 'venv/Lib', + executableSuffix: '.exe', + // Reads like a stale reference to a tool this project does not use. It is a frozen wire string + // under the published format; it is not a typo and must not be "cleaned". + launcherKind: 'uv-windows-pe', + }), +}); + +/** + * The assertion every Python self-test opens with, so the check begins by proving it is running on + * the platform the box was built for rather than on whatever interpreter answered first. + */ +const PYTHON_PLATFORM_ASSERTIONS = Object.freeze({ + macos: "import sys; assert sys.platform == 'darwin'", + linux: "import sys; assert sys.platform.startswith('linux')", + windows: "import sys; assert sys.platform == 'win32'", +}); + +const PYTHON_MAJOR_MINOR = /^(\d+)\.(\d+)(?:\.|$)/; + +function pythonLayout(target) { + const layout = PYTHON_LAYOUTS[target?.platform]; + if (!layout) { + throw new TypeError(`No python runtime layout exists for platform ${String(target?.platform)}`); + } + return layout; +} + +/** + * The `major.minor` prefix naming the standard-library directory a packed prefix carries. + * + * A patch component is deliberately dropped rather than rejected: a scroll may pin `3.14.2`, and + * the directory conda-forge writes is `python3.14` either way. + */ +function pythonMajorMinor(version) { + const match = PYTHON_MAJOR_MINOR.exec(String(version)); + if (!match) { + throw new TypeError(`Invalid Python version for execution discovery: ${version}.`); + } + return `${match[1]}.${match[2]}`; +} + +/** + * Every path a dotted module could legitimately resolve to inside a box. + * + * The payload root comes first because a box may ship its application beside the environment rather + * than installed into it, which is what a scroll's `localFiles` produce. + */ +function pythonModuleEntryPoints({ module, runtimeVersion, target }) { + const layout = pythonLayout(target); + const modulePath = String(module).split('.').join('/'); + const relativeCandidates = [`${modulePath}.py`, `${modulePath}/__main__.py`]; + // Windows names its standard library once, with no interpreter version in the path; every other + // platform carries `python.` under it. + const standardLibrary = target.platform === 'windows' + ? layout.standardLibrary + : `${layout.standardLibrary}/python${pythonMajorMinor(runtimeVersion)}`; + const roots = ['', standardLibrary, `${standardLibrary}/site-packages`]; + return roots.flatMap((root) => + relativeCandidates.map((path) => (root ? `${root}/${path}` : path))); +} + +const PYTHON_RUNTIME = Object.freeze({ + id: 'python', + executionKinds: Object.freeze(['python-script', 'python-module']), + executionEnvironmentVariables: PYTHON_EXECUTION_ENVIRONMENT, + + layout: pythonLayout, + + executablePayloadPaths(target) { + const layout = pythonLayout(target); + // The interpreter by name, and the console-script directory wholesale. A conda prefix generates + // that directory's contents at solve time and nothing declares them, so the rule is the only + // way they can carry the bit at all. + return Object.freeze({ + files: Object.freeze([layout.entryPoint]), + directories: Object.freeze([layout.scriptsDirectory]), + }); + }, + + resolveExecutionFiles({ execution, runtimeVersion, target }) { + if (execution.kind === 'python-script') { + return Object.freeze({ + candidates: Object.freeze([execution.script]), + missing: `Execution script is missing from the box: ${execution.script}.`, + }); + } + return Object.freeze({ + candidates: Object.freeze(pythonModuleEntryPoints({ + module: execution.module, + runtimeVersion, + target, + })), + missing: `Execution module is not discoverable in the box: ${execution.module}.`, + }); + }, + + buildArgv({ execution, target }) { + const layout = pythonLayout(target); + const args = execution.kind === 'python-script' + ? [{ kind: 'payload-path', value: execution.script }] + : [{ kind: 'literal', value: '-m' }, { kind: 'literal', value: execution.module }]; + for (const value of execution.defaultArgs ?? []) args.push({ kind: 'literal', value }); + return Object.freeze({ + command: Object.freeze({ kind: 'payload-path', value: layout.entryPoint }), + args: Object.freeze(args.map((argument) => Object.freeze(argument))), + }); + }, + + selfTestArgv({ probe, target }) { + const assertion = PYTHON_PLATFORM_ASSERTIONS[target?.platform]; + if (!assertion) { + throw new TypeError(`No python self-test assertion exists for platform ${String(target?.platform)}`); + } + const imports = `import ${probe.imports.join(', ')}`; + const code = probe.code + ? `${assertion}\n${imports}\n${probe.code}` + : `${assertion}\n${imports}`; + return Object.freeze(['-c', code]); + }, +}); + +const RUNTIME_ADAPTERS = Object.freeze([PYTHON_RUNTIME]); + +/** + * Returns the runtime adapter for a runtime id. + * + * @param {string} runtimeId + * @returns {BoxRuntimeAdapter} + * @throws {TypeError} when no runtime with that id exists + */ +export function runtimeAdapter(runtimeId) { + const adapter = RUNTIME_ADAPTERS.find((candidate) => candidate.id === runtimeId); + if (!adapter) throw new TypeError(`No box runtime adapter exists for ${String(runtimeId)}`); + return adapter; +} + +/** + * Lists every runtime adapter, for contract tests and for callers enumerating what a box may be. + * + * @returns {BoxRuntimeAdapter[]} every runtime adapter, as a fresh array + */ +export function runtimeAdapters() { + return [...RUNTIME_ADAPTERS]; +} + +/** + * Ensures a declared entry point agrees with where the runtime actually sits in the payload. + * + * @param {string} runtimeId + * @param {import('./targets.mjs').BoxTargetAdapter} adapter the resolved target adapter, whose id + * names the layout the entry point is being judged against + * @param {string} entryPoint + * @returns {void} + * @throws {TypeError} when the entry point is not the one the runtime defines for this target + */ +export function assertRuntimeEntryPoint(runtimeId, adapter, entryPoint) { + const runtime = runtimeAdapter(runtimeId); + const expected = runtime.layout(adapter).entryPoint; + if (entryPoint !== expected) { + // The wording still names Python because the wire format still does: a scroll declares + // `pythonEntryPoint`, and an error that called it something else would name a field the author + // cannot find. It generalises with the field, in the same version bump. + throw new TypeError(`${adapter.id} scrolls must use Python entry point ${expected}`); + } +} + +/** + * Whether a payload path is one the runtime requires the executable bit on. + * + * A directory matches by prefix so one rule covers a whole generated scripts tree; an exact file + * match covers the runtime's own entry point, which lives outside it on Windows. + * + * @param {ExecutablePayloadPaths} rule + * @param {string} relativePath forward-slash path relative to the box root + * @returns {boolean} + */ +export function isExecutablePayloadPath(rule, relativePath) { + if (rule.files.includes(relativePath)) return true; + return rule.directories.some((directory) => relativePath.startsWith(`${directory}/`)); +} + +/** + * The complete list of inherited variables that can change what a box executes. + * + * Two halves, because they have two owners: the runtime contributes the variables its own loader + * reads, and the target contributes the operating system's dynamic-linker controls. Callers want + * one list, and assembling it here rather than at each of them is what keeps a diagnostic report + * from depending on which call site produced it. + * + * @param {string} runtimeId + * @param {import('./targets.mjs').BoxTargetAdapter} adapter + * @returns {readonly string[]} the runtime's variables followed by the target's + */ +export function executionAffectingVariables(runtimeId, adapter) { + return Object.freeze([ + ...runtimeAdapter(runtimeId).executionEnvironmentVariables, + ...adapter.executionAffectingEnvironmentVariables, + ]); +} diff --git a/src/contract/targets.d.mts b/src/contract/targets.d.mts index 9f810bf..dc5be64 100644 --- a/src/contract/targets.d.mts +++ b/src/contract/targets.d.mts @@ -28,12 +28,16 @@ export function assertNativeHost(adapter: BoxTargetAdapter, host?: { arch: string; }): void; /** - * Ensures the scroll entry point agrees with the adapter's standalone Python layout. + * Ensures the scroll entry point agrees with the standalone Python layout for this target. + * + * Kept under its published name while the wire format still spells the field `pythonEntryPoint`. + * The rule itself moved to `runtimes.mjs`, where it can be asked about any runtime; this is the one + * public spelling of it, and it goes when the field does. * * @param {BoxTargetAdapter} adapter * @param {string} entryPoint * @returns {void} - * @throws {TypeError} when the entry point does not match the adapter's layout + * @throws {TypeError} when the entry point does not match the runtime's layout for this target */ export function assertPythonEntryPoint(adapter: BoxTargetAdapter, entryPoint: string): void; /** @@ -64,7 +68,7 @@ export function pixiAccelerator(scroll: Pick>>>; /** - * inherited variables whose - * presence can change which code the box interpreter loads or executes + * the operating system's own + * dynamic-linker controls; the runtime adds the variables its loader reads, and + * `executionAffectingVariables()` in `runtimes.mjs` is what joins the two halves */ executionAffectingEnvironmentVariables: readonly string[]; - /** - * the platform assertion prepended to every self-test - */ - selfTestPython: string; }; diff --git a/src/contract/targets.mjs b/src/contract/targets.mjs index dfd6ef8..27fa00b 100644 --- a/src/contract/targets.mjs +++ b/src/contract/targets.mjs @@ -8,13 +8,20 @@ * `fixtures/target-id-contract.json` are what "agree" means, and are the fixtures other languages * validate their mirrors against. * - * The adapters below describe what a target implies for the built payload: the Python layout inside - * the box, the archive backend, how native libraries are inspected, and the environment a validation - * run gets. They are part of the format because a consumer unpacking a box relies on that layout. + * The adapters below describe what a target implies for the built payload: the conda subdir it + * solves for, the archive backend, how native libraries are inspected, and the environment a + * validation run gets. They are part of the format because a consumer unpacking a box relies on + * them. + * + * What a target deliberately no longer describes is the *runtime* inside the box. The interpreter + * layout, the execution kinds and the runtime's own environment variables live in `runtimes.mjs`, + * because they are facts about what a box runs rather than about the machine it runs on. Keeping + * them here made every target adapter a statement that a box is a Python box, and made a second + * runtime a fork of this table rather than one more adapter beside it. */ /** * What a target implies for the built payload. Part of the format rather than an implementation - * detail: a consumer unpacking a box relies on this layout. + * detail: a consumer unpacking a box relies on this. * * @typedef {object} BoxTargetAdapter * @property {string} id canonical adapter id, e.g. `macos-aarch64` @@ -22,31 +29,25 @@ * @property {'aarch64' | 'x86_64'} arch * @property {{ platform: string, arch: string }} host the Node platform/arch a build must run on * @property {'osx-arm64' | 'linux-64' | 'win-64'} condaSubdir the scroll's pixi `platforms` value - * @property {{ payloadRoot: string, entryPoint: string, scriptsDirectory: string, - * executableSuffix: string, launcherKind: string }} python layout of the interpreter in the box * @property {{ format: 'zip', writer: string, reader: string, assetTarReader: string, * zip64: boolean }} archive the pinned archive backend * @property {{ command: string, argsPrefix: readonly string[], * extensions: readonly string[] }} nativeLibraryInspection * @property {Readonly>>>} validationEnvironments * the environment that forces a run onto one accelerator, keyed by accelerator - * @property {readonly string[]} executionAffectingEnvironmentVariables inherited variables whose - * presence can change which code the box interpreter loads or executes - * @property {string} selfTestPython the platform assertion prepended to every self-test + * @property {readonly string[]} executionAffectingEnvironmentVariables the operating system's own + * dynamic-linker controls; the runtime adds the variables its loader reads, and + * `executionAffectingVariables()` in `runtimes.mjs` is what joins the two halves */ +import { IMPLICIT_RUNTIME_ID, assertRuntimeEntryPoint } from './runtimes.mjs'; + const TARGET_ACCELERATORS = { macos: { aarch64: ['metal', 'cpu'] }, linux: { x86_64: ['cpu', 'cuda'] }, windows: { x86_64: ['cpu', 'cuda'] }, }; const CUDA_VERSION = /^[1-9][0-9]*\.[0-9]+$/; -const PYTHON_EXECUTION_ENVIRONMENT = Object.freeze([ - 'PYTHONPATH', - 'PYTHONHOME', - 'PYTHONSTARTUP', - 'PYTHONBREAKPOINT', -]); // The exact libraries that wrote and read a box, so a consumer knows what produced the bytes it // holds rather than inferring it. Each version is the one this package installs: they are pinned in @@ -68,13 +69,6 @@ const TARGET_ADAPTERS = Object.freeze([ host: Object.freeze({ platform: 'darwin', arch: 'arm64' }), // conda platform subdir: the `platforms` value in the scroll's pixi.toml. condaSubdir: 'osx-arm64', - python: Object.freeze({ - payloadRoot: 'venv', - entryPoint: 'venv/bin/python', - scriptsDirectory: 'venv/bin', - executableSuffix: '', - launcherKind: 'posix-polyglot', - }), archive: ARCHIVE_BACKEND, nativeLibraryInspection: Object.freeze({ command: 'otool', @@ -85,11 +79,7 @@ const TARGET_ADAPTERS = Object.freeze([ cpu: Object.freeze({ CUDA_VISIBLE_DEVICES: '' }), metal: Object.freeze({ PYTORCH_ENABLE_MPS_FALLBACK: '0' }), }), - executionAffectingEnvironmentVariables: Object.freeze([ - ...PYTHON_EXECUTION_ENVIRONMENT, - 'DYLD_INSERT_LIBRARIES', - ]), - selfTestPython: "import sys; assert sys.platform == 'darwin'", + executionAffectingEnvironmentVariables: Object.freeze(['DYLD_INSERT_LIBRARIES']), }), Object.freeze({ id: 'linux-x86_64', @@ -97,13 +87,6 @@ const TARGET_ADAPTERS = Object.freeze([ arch: 'x86_64', host: Object.freeze({ platform: 'linux', arch: 'x64' }), condaSubdir: 'linux-64', - python: Object.freeze({ - payloadRoot: 'venv', - entryPoint: 'venv/bin/python', - scriptsDirectory: 'venv/bin', - executableSuffix: '', - launcherKind: 'posix-polyglot', - }), archive: ARCHIVE_BACKEND, nativeLibraryInspection: Object.freeze({ command: 'ldd', @@ -114,11 +97,7 @@ const TARGET_ADAPTERS = Object.freeze([ cpu: Object.freeze({ CUDA_VISIBLE_DEVICES: '' }), cuda: Object.freeze({ CUDA_VISIBLE_DEVICES: '0' }), }), - executionAffectingEnvironmentVariables: Object.freeze([ - ...PYTHON_EXECUTION_ENVIRONMENT, - 'LD_PRELOAD', - ]), - selfTestPython: "import sys; assert sys.platform.startswith('linux')", + executionAffectingEnvironmentVariables: Object.freeze(['LD_PRELOAD']), }), Object.freeze({ id: 'windows-x86_64', @@ -126,13 +105,6 @@ const TARGET_ADAPTERS = Object.freeze([ arch: 'x86_64', host: Object.freeze({ platform: 'win32', arch: 'x64' }), condaSubdir: 'win-64', - python: Object.freeze({ - payloadRoot: 'venv', - entryPoint: 'venv/python.exe', - scriptsDirectory: 'venv/Scripts', - executableSuffix: '.exe', - launcherKind: 'uv-windows-pe', - }), archive: ARCHIVE_BACKEND, nativeLibraryInspection: Object.freeze({ command: 'dumpbin', @@ -143,8 +115,9 @@ const TARGET_ADAPTERS = Object.freeze([ cpu: Object.freeze({ CUDA_VISIBLE_DEVICES: '' }), cuda: Object.freeze({ CUDA_VISIBLE_DEVICES: '0' }), }), - executionAffectingEnvironmentVariables: PYTHON_EXECUTION_ENVIRONMENT, - selfTestPython: "import sys; assert sys.platform == 'win32'", + // Windows has no inherited loader control of its own worth reporting: `PATH` decides DLL + // resolution and is far too broad to name here, so the whole list is the runtime's. + executionAffectingEnvironmentVariables: Object.freeze([]), }), ]); @@ -211,19 +184,19 @@ export function assertNativeHost(adapter, host = process) { } /** - * Ensures the scroll entry point agrees with the adapter's standalone Python layout. + * Ensures the scroll entry point agrees with the standalone Python layout for this target. + * + * Kept under its published name while the wire format still spells the field `pythonEntryPoint`. + * The rule itself moved to `runtimes.mjs`, where it can be asked about any runtime; this is the one + * public spelling of it, and it goes when the field does. * * @param {BoxTargetAdapter} adapter * @param {string} entryPoint * @returns {void} - * @throws {TypeError} when the entry point does not match the adapter's layout + * @throws {TypeError} when the entry point does not match the runtime's layout for this target */ export function assertPythonEntryPoint(adapter, entryPoint) { - if (entryPoint !== adapter.python.entryPoint) { - throw new TypeError( - `${adapter.id} scrolls must use Python entry point ${adapter.python.entryPoint}`, - ); - } + assertRuntimeEntryPoint(IMPLICIT_RUNTIME_ID, adapter, entryPoint); } /** diff --git a/src/runtimes/index.mjs b/src/runtimes/index.mjs new file mode 100644 index 0000000..33b3692 --- /dev/null +++ b/src/runtimes/index.mjs @@ -0,0 +1,37 @@ +/** + * The registry of builder-side runtime adapters. + * + * One entry per runtime Scrollcase can actually pack. It is deliberately not seeded with the + * runtimes that are planned but unimplemented: a registry that answers for a runtime no build can + * produce turns "unsupported" into a failure somewhere further down, where the message no longer + * says what went wrong. + * + * The contract half of a runtime lives in `src/contract/runtimes.mjs` and is mirrored in every + * consumer language. This half is the builder's alone. + */ + +import { pythonRuntimeBuilder } from './python/index.mjs'; + +const RUNTIME_BUILDERS = Object.freeze([pythonRuntimeBuilder]); + +/** + * Returns the builder-side adapter for a runtime id. + * + * @param {string} runtimeId + * @returns {import('./python/index.mjs').RuntimeBuilder} + * @throws {TypeError} when Scrollcase cannot build boxes for that runtime + */ +export function runtimeBuilder(runtimeId) { + const builder = RUNTIME_BUILDERS.find((candidate) => candidate.id === runtimeId); + if (!builder) throw new TypeError(`Scrollcase cannot build a ${String(runtimeId)} box`); + return builder; +} + +/** + * Lists every runtime the builder can pack, for the CLI's own listings and for contract tests. + * + * @returns {import('./python/index.mjs').RuntimeBuilder[]} every builder, as a fresh array + */ +export function runtimeBuilders() { + return [...RUNTIME_BUILDERS]; +} diff --git a/src/runtimes/python/dependencies.mjs b/src/runtimes/python/dependencies.mjs new file mode 100644 index 0000000..b893310 --- /dev/null +++ b/src/runtimes/python/dependencies.mjs @@ -0,0 +1,72 @@ +/** + * Reading a Python project's existing dependency list into conda-forge terms. + * + * A project arriving from pip already has `requirements.txt`, and retyping it is the sort of work + * that invites a typo. What this cannot do is decide: names are translated where this module is + * sure and lowercased otherwise, and every translation and every skip is reported so the author + * reviews them before locking rather than after a build fails. + * + * It lives under the Python runtime because `requirements.txt`, extras, environment markers and the + * PyPI spelling of a package are all Python facts. What stays in `src/build/dependencies.mjs` is + * the substrate half — editing the `[dependencies]` table of a pixi manifest — which is the same + * job whatever runtime the box packs. + */ + +/** + * PyPI names whose conda-forge package is called something else. + * + * Deliberately short. Every entry is one this project can state with confidence; anything else is + * lowercased and passed through, and every rename is reported so the author can check it before + * locking. A wrong guess here produces a lock that resolves and a box that cannot import what it + * was built for, which is worse than an unmapped name the author has to look up. + */ +const CONDA_FORGE_NAMES = Object.freeze({ + 'opencv-python': 'opencv', + 'opencv-python-headless': 'opencv', + 'psycopg2-binary': 'psycopg2', + 'msgpack': 'msgpack-python', + 'tables': 'pytables', + 'torch': 'pytorch', +}); + +/** + * Reads a pip `requirements.txt` and reports what it would mean on conda-forge. + * + * @param {string} contents + * @returns {{ dependencies: { name: string, spec: string }[], + * renamed: { from: string, to: string }[], skipped: { line: string, reason: string }[] }} + */ +export function readRequirements(contents) { + const dependencies = []; + const renamed = []; + const skipped = []; + for (const raw of contents.split('\n')) { + const line = raw.split('#')[0].trim(); + if (!line) continue; + if (line.startsWith('-')) { + skipped.push({ line, reason: 'a pip option, which has no conda-forge equivalent' }); + continue; + } + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(line) || line.includes('@')) { + skipped.push({ line, reason: 'a direct URL or VCS reference, which conda-forge cannot express' }); + continue; + } + // `name[extra1,extra2] >= 1.2 ; python_version < "3.12"` — the name runs to the first of these. + const [requirement] = line.split(';'); + const match = /^([A-Za-z0-9._-]+)\s*(\[[^\]]*\])?\s*(.*)$/.exec(requirement.trim()); + if (!match) { + skipped.push({ line, reason: 'not a requirement this reader understands' }); + continue; + } + const [, rawName, extras, rawSpec] = match; + const normalized = rawName.toLowerCase().replace(/_/g, '-'); + const name = CONDA_FORGE_NAMES[normalized] ?? normalized; + if (name !== rawName) renamed.push({ from: rawName, to: name }); + if (extras) { + skipped.push({ line: `${rawName}${extras}`, reason: 'extras are a pip concept; add the packages they pull in yourself' }); + } + const spec = rawSpec.trim().replace(/\s+/g, ''); + dependencies.push({ name, spec: spec === '' ? '*' : spec }); + } + return { dependencies, renamed, skipped }; +} diff --git a/src/runtimes/python/index.mjs b/src/runtimes/python/index.mjs new file mode 100644 index 0000000..a5da147 --- /dev/null +++ b/src/runtimes/python/index.mjs @@ -0,0 +1,44 @@ +/** + * The builder-side Python runtime adapter. + * + * `src/contract/runtimes.mjs` holds what a *consumer* must agree with — layout, execution kinds, + * argv, discovery — and is pure for that reason. This is the other half: what the builder has to do + * to produce a Python box in the first place, which is allowed to touch a filesystem and is not + * mirrored in any other language. + * + * The split is what makes a second runtime an adapter. Everything a Python box needs that a native + * or Node box would not — the interpreter's pixi dependency, the starter files `new scroll` writes, + * the conda shebang trampoline `pixi.mjs` repairs after packing — is reachable from here, and + * nothing above it names Python to get at them. + */ + +import { runtimeAdapter } from '../../contract/runtimes.mjs'; +import { repairPosixLaunchers } from './launchers.mjs'; +import { STARTER_SCRIPT, STARTER_SELF_TEST, pixiDependency } from './templates/index.mjs'; + +/** + * What the builder needs from a runtime, beyond what the contract already states. + * + * @typedef {object} RuntimeBuilder + * @property {string} id + * @property {import('../../contract/runtimes.mjs').BoxRuntimeAdapter} contract the pure half, so a + * caller holding a builder never has to look the same runtime up twice + * @property {(runtimeVersion: string) => { name: string, spec: string }} pixiDependency the + * `[dependencies]` entry a generated pixi manifest declares for this runtime + * @property {(layout: import('../../contract/runtimes.mjs').BoxRuntimeLayout, payloadDir: string, + * forbiddenPaths: readonly string[]) => Promise} repairLaunchers rewrites generated console + * scripts so nothing in the box points at the build machine + * @property {{ script: string, selfTest: string }} templates the source `new scroll` writes + */ + +/** @type {RuntimeBuilder} */ +export const pythonRuntimeBuilder = Object.freeze({ + id: 'python', + contract: runtimeAdapter('python'), + pixiDependency, + repairLaunchers: repairPosixLaunchers, + templates: Object.freeze({ + script: STARTER_SCRIPT, + selfTest: STARTER_SELF_TEST, + }), +}); diff --git a/src/runtimes/python/launchers.d.mts b/src/runtimes/python/launchers.d.mts new file mode 100644 index 0000000..fd7afcc --- /dev/null +++ b/src/runtimes/python/launchers.d.mts @@ -0,0 +1,10 @@ +/** + * Makes generated POSIX console scripts resolve Python relative to their own installed path. + * + * @param {import('../../contract/runtimes.mjs').BoxRuntimeLayout} layout where the runtime sits in + * the payload, for the target being packed + * @param {string} payloadDir + * @param {readonly string[]} forbiddenPaths + * @returns {Promise} + */ +export function repairPosixLaunchers(layout: import("../../contract/runtimes.mjs").BoxRuntimeLayout, payloadDir: string, forbiddenPaths: readonly string[]): Promise; diff --git a/src/build/launchers.mjs b/src/runtimes/python/launchers.mjs similarity index 70% rename from src/build/launchers.mjs rename to src/runtimes/python/launchers.mjs index a153592..a423d9c 100644 --- a/src/build/launchers.mjs +++ b/src/runtimes/python/launchers.mjs @@ -5,11 +5,18 @@ * absolute interpreter path in their shebang. That path means nothing on a user's machine, and * shipping it also leaks a developer's directory layout. Rewriting them to resolve Python next to * themselves is what makes the packed environment genuinely relocatable. + * + * It lives under the Python runtime rather than in `src/build/` because the trampoline it parses is + * a Python fact: the `'''exec'` header is what setuptools and conda generate when an absolute + * shebang would exceed the POSIX length limit, and the body left behind is Python source. Another + * runtime packed from the same conda-forge substrate may well need its shebangs rewritten too, but + * it will not need *this* parser, and pretending otherwise is how a shared helper acquires a + * per-runtime branch. */ import { chmod, readFile, writeFile } from 'node:fs/promises'; import { basename, join } from 'node:path'; -import { collectFiles, fileExists } from './filesystem.mjs'; +import { collectFiles, fileExists } from '../../build/filesystem.mjs'; /** * Removes either a direct shebang or a shell trampoline header from a launcher, leaving the Python @@ -31,15 +38,16 @@ function posixLauncherBody(text) { /** * Makes generated POSIX console scripts resolve Python relative to their own installed path. * - * @param {import('../contract/targets.mjs').BoxTargetAdapter} adapter + * @param {import('../../contract/runtimes.mjs').BoxRuntimeLayout} layout where the runtime sits in + * the payload, for the target being packed * @param {string} payloadDir * @param {readonly string[]} forbiddenPaths * @returns {Promise} */ -export async function repairPosixLaunchers(adapter, payloadDir, forbiddenPaths) { - const scriptsRoot = join(payloadDir, ...adapter.python.scriptsDirectory.split('/')); +export async function repairPosixLaunchers(layout, payloadDir, forbiddenPaths) { + const scriptsRoot = join(payloadDir, ...layout.scriptsDirectory.split('/')); if (!await fileExists(scriptsRoot)) return; - const pythonName = basename(adapter.python.entryPoint); + const pythonName = basename(layout.entryPoint); for (const file of await collectFiles(scriptsRoot)) { const path = join(scriptsRoot, ...file.split('/')); const bytes = await readFile(path); diff --git a/src/runtimes/python/templates/index.mjs b/src/runtimes/python/templates/index.mjs new file mode 100644 index 0000000..e6f0b15 --- /dev/null +++ b/src/runtimes/python/templates/index.mjs @@ -0,0 +1,54 @@ +/** + * The Python source `scrollcase new scroll` writes for a project that has none yet. + * + * These are starting points, not scaffolding to be extended: each is short enough to read in one + * screen and says what the author is expected to replace. They live under the runtime because they + * are Python source — a Node box or a native box needs different files, and the authoring path + * should reach for the runtime's own rather than acquire a branch per runtime. + */ + +/** The application a generated `python-script` scroll points at. */ +export const STARTER_SCRIPT = `"""Minimal application entry point generated by Scrollcase.""" + +import sys + + +def main() -> int: + print("Scrollcase box is ready.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +`; + +/** The self-test a generated scroll runs with the box's own interpreter. */ +export const STARTER_SELF_TEST = `"""Self-test for this box, run by \`scrollcase build\` before the box is archived. + +It runs with the box's own interpreter, from the payload root, after the imports declared in +scroll.json have already succeeded — so it can read the files the box ships and import the code it +packs. Any exception fails the build, which is the point: this is the last check that the box works +before anyone downloads it. + +Replace the line below with something that would actually notice a broken box. +""" + +print("self-test ok") +`; + +/** + * The interpreter constraint a generated pixi manifest declares. + * + * A bare `major.minor` becomes `major.minor.*` so the solve is free to take a patch release; a + * version the author spelled out in full is written through unchanged, because someone who typed + * `3.14.2` meant `3.14.2`. + * + * @param {string} runtimeVersion + * @returns {{ name: string, spec: string }} the `[dependencies]` entry this runtime contributes + */ +export function pixiDependency(runtimeVersion) { + return { + name: 'python', + spec: /^\d+\.\d+$/.test(runtimeVersion) ? `${runtimeVersion}.*` : runtimeVersion, + }; +} diff --git a/src/sign/keys.mjs b/src/sign/keys.mjs index cf5383e..158e630 100644 --- a/src/sign/keys.mjs +++ b/src/sign/keys.mjs @@ -20,7 +20,11 @@ import { chmod, mkdir, readFile, writeFile } from 'node:fs/promises'; import { dirname } from 'node:path'; import { fail } from '../build/process.mjs'; import { fileExists } from '../build/filesystem.mjs'; -import { BOX_SCHEMA_VERSION, PAYLOAD_ENCODING } from '../contract/document-shape.mjs'; +import { + BOX_SCHEMA_VERSION, + PAYLOAD_ENCODING, + unsupportedSchemaVersionMessage, +} from '../contract/document-shape.mjs'; /** * A published public key, as written by `keygen` and read back when verifying. @@ -184,7 +188,7 @@ export function signWithLocalKey(payloadBytes, { privateKey, metadata }) { */ export function decodeSignedDocument(document) { if (document?.schemaVersion === 1) { - fail('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); + fail(unsupportedSchemaVersionMessage(1)); } if (document?.schemaVersion !== BOX_SCHEMA_VERSION || document?.payloadEncoding !== PAYLOAD_ENCODING) { fail('Unsupported signed document.'); diff --git a/tests/helpers/consumer-box-fixture.mjs b/tests/helpers/consumer-box-fixture.mjs index ec5c571..67269ad 100644 --- a/tests/helpers/consumer-box-fixture.mjs +++ b/tests/helpers/consumer-box-fixture.mjs @@ -16,6 +16,7 @@ import { } from '../../src/contract/payload-digest.mjs'; import { documentKinds } from '../../src/contract/documents.mjs'; import { boxTargetAdapter } from '../../src/contract/targets.mjs'; +import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../../src/contract/runtimes.mjs'; import { generateSigningKey, signDocument } from '../../src/sign/index.mjs'; export function nativeTarget() { @@ -75,7 +76,8 @@ export async function createConsumerBoxFixture({ const payload = join(root, 'payload'); await mkdir(payload); const adapter = boxTargetAdapter(target); - const pythonPath = join(payload, ...adapter.python.entryPoint.split('/')); + const layout = runtimeAdapter(IMPLICIT_RUNTIME_ID).layout(adapter); + const pythonPath = join(payload, ...layout.entryPoint.split('/')); await mkdir(dirname(pythonPath), { recursive: true }); await writeFile(pythonPath, interpreterContents); @@ -96,7 +98,7 @@ export async function createConsumerBoxFixture({ runtimeId: 'example-consumer-runtime', version: '2.0.0', target, - pythonEntryPoint: adapter.python.entryPoint, + pythonEntryPoint: layout.entryPoint, modelCacheSubdir: 'model-cache/consumer-fixture', selfTest: { pythonImports: ['json'], diff --git a/tests/helpers/consumer-conformance.mjs b/tests/helpers/consumer-conformance.mjs index abdcb16..637274a 100644 --- a/tests/helpers/consumer-conformance.mjs +++ b/tests/helpers/consumer-conformance.mjs @@ -29,6 +29,7 @@ import { payloadDigestStream, } from '../../src/contract/payload-digest.mjs'; import { boxTargetAdapter, boxTargetId } from '../../src/contract/targets.mjs'; +import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../../src/contract/runtimes.mjs'; import { attachExtractedBox, runBox, @@ -413,7 +414,7 @@ function replaceTokens(value, root = null) { if (typeof value === 'string') { const adapter = boxTargetAdapter(nativeTarget()); return value - .replaceAll('$NATIVE_PYTHON', adapter.python.entryPoint) + .replaceAll('$NATIVE_PYTHON', runtimeAdapter(IMPLICIT_RUNTIME_ID).layout(adapter).entryPoint) .replaceAll('$NATIVE_TARGET', boxTargetId(nativeTarget())) .replaceAll('$BOX', root ?? '$BOX'); } diff --git a/tests/unit/build-pipeline.test.mjs b/tests/unit/build-pipeline.test.mjs index 5ea069b..64c7fd4 100644 --- a/tests/unit/build-pipeline.test.mjs +++ b/tests/unit/build-pipeline.test.mjs @@ -22,6 +22,7 @@ import { assertBoxManifestAgreement, verifyBox } from '../../src/build/verify.mj import { configureWorkspace, resetWorkspace } from '../../src/build/workspace.mjs'; import { generateSigningKey, signDocument } from '../../src/sign/index.mjs'; import { boxTargetAdapters, boxTargetId, decodeDocumentPayload, documentKinds } from '../../src/contract/index.mjs'; +import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../../src/contract/runtimes.mjs'; // The pipeline is the same on every platform, but the native-host gate (rightly) refuses to build // a box for any other one — so the test scroll targets whatever host the suite is running on. @@ -29,6 +30,7 @@ import { boxTargetAdapters, boxTargetId, decodeDocumentPayload, documentKinds } const HOST_ADAPTER = boxTargetAdapters().find((adapter) => adapter.host.platform === process.platform && adapter.host.arch === process.arch) ?? (() => { throw new Error(`No box target adapter for this host: ${process.platform}/${process.arch}`); })(); +const HOST_LAYOUT = runtimeAdapter(IMPLICIT_RUNTIME_ID).layout(HOST_ADAPTER); const SCROLL = { schemaVersion: 2, @@ -43,7 +45,7 @@ const SCROLL = { compatibility: { minHostAppVersion: '1.0.0' }, pythonVersion: '3.11.15', pixiVersion: '0.73.0', - pythonEntryPoint: HOST_ADAPTER.python.entryPoint, + pythonEntryPoint: HOST_LAYOUT.entryPoint, modelCacheSubdir: 'model-cache/example-model', assetBaseUrl: 'https://assets.example.org/boxes', assets: [], @@ -52,7 +54,7 @@ const SCROLL = { const SCROLL_REF = `${SCROLL.boxId}/${boxTargetId(SCROLL.target)}`; // The interpreter's path inside the payload, split for platform-correct joins. -const ENTRY_SEGMENTS = HOST_ADAPTER.python.entryPoint.split('/'); +const ENTRY_SEGMENTS = HOST_LAYOUT.entryPoint.split('/'); function writeDeep(path, contents) { mkdirSync(dirname(path), { recursive: true }); @@ -78,6 +80,20 @@ async function zipCompressionMethods(archivePath) { return methods; } +/** The Unix mode each entry carries, out of the high half of the external attributes. */ +async function zipModes(archivePath) { + const zip = await yauzl.openPromise(archivePath, { autoClose: false, lazyEntries: true }); + const modes = new Map(); + try { + for await (const entry of zip.eachEntry()) { + modes.set(entry.fileName, entry.externalFileAttributes >>> 16); + } + } finally { + await zip.close(); + } + return modes; +} + /** * One of conda's per-package records, as the installer writes it. * @@ -136,12 +152,24 @@ function plantPrefixSymlinks(prefix) { * asset staging, pruning, the self-test gate, box.json, the deterministic archive, signing — is the * real implementation, which is what this test is here to exercise. */ -function fakeToolchain(payloadDir, { module = null, onSelfTest = null } = {}) { +function fakeToolchain(payloadDir, { module = null, onSelfTest = null, consoleScript = null } = {}) { const run = function run(command, args = [], options = {}) { if (command === 'pixi' && args[0] === 'install') { const manifest = args[args.indexOf('--manifest-path') + 1]; const prefix = join(dirname(manifest), '.pixi', 'envs', 'default'); writeDeep(join(prefix, ...ENTRY_SEGMENTS.slice(1)), '#!/bin/sh\nexit 0\n'); + if (consoleScript) { + // What conda actually generates: the *build machine's* interpreter, reached through the + // shell trampoline it falls back to when an absolute shebang would be too long. + const scriptsRoot = HOST_LAYOUT.scriptsDirectory.split('/').slice(1); + writeDeep(join(prefix, ...scriptsRoot, consoleScript), [ + '#!/bin/sh', + `'''exec' "${join(prefix, ...ENTRY_SEGMENTS.slice(1))}" "$0" "$@"`, + "' '''", + 'print("console script")', + '', + ].join('\n')); + } writeDeep(join(prefix, 'conda-meta', 'history'), '==> 2026-07-27 05:29:00 <==\n'); writeDeep(join(prefix, 'conda-meta', 'bzip2-1.0.8-hd037594_9.json'), `${JSON.stringify(CONDA_RECORD, null, 2)}\n`); @@ -759,6 +787,41 @@ describe('the build pipeline', () => { expect(receipt.status).toBe('passed'); }); + it('synthesises the executable bit from the runtime layout, and repairs the launcher', async () => { + // Mode is not read off the build machine — a payload assembled under any umask has to archive + // identically, and `payload-digest.v1` deliberately excludes mode, so the archive is the only + // place the bit is stated. Which paths get it is the runtime's rule: the interpreter by name + // and its generated scripts by directory, and nothing else. + const { keys, payloadDir } = await makeProject(); + const built = await buildBox(SCROLL_REF, { + ...keys, + ...fakeToolchain(payloadDir, { consoleScript: 'tqdm' }), + log: () => {}, + }); + const modes = await zipModes(built.archivePath); + const scriptInPayload = `${HOST_LAYOUT.scriptsDirectory}/tqdm`; + expect(modes.has(scriptInPayload)).toBe(true); + + // A Windows host has no Unix mode to synthesise, and every entry is archived 0644 there. + const executable = process.platform === 'win32' ? 0o100644 : 0o100755; + expect(modes.get(HOST_LAYOUT.entryPoint)).toBe(executable); + expect(modes.get(scriptInPayload)).toBe(executable); + expect(modes.get('box.json')).toBe(0o100644); + expect(modes.get(PAYLOAD_DIGEST_FILE)).toBe(0o100644); + + // The launcher the packed prefix carried named the build machine; the shipped one resolves the + // interpreter next to itself instead. Nothing in a box may point at the machine that built it. + const extracted = await mkdtemp(join(tmpdir(), 'scrollcase-launcher-')); + created.push(extracted); + await extractZipArchive(built.archivePath, extracted); + const launcher = await readFile(join(extracted, ...scriptInPayload.split('/')), 'utf8'); + expect(launcher).not.toContain(payloadDir); + if (process.platform !== 'win32') { + expect(launcher).toContain('dirname -- "$0"'); + expect(launcher).toContain('print("console script")'); + } + }); + it('stores declared weights instead of deflating them, and still rebuilds identically', async () => { // Incompressible on purpose: deflate makes this *larger*, which is the whole reason the rule // exists. Text elsewhere in the payload still compresses, so one archive proves both halves. @@ -799,7 +862,7 @@ describe('the build pipeline', () => { expect(methods.get('corpus/data.bin')).toBe(ZIP_STORED); // The interpreter and the box's own manifest are ordinary files and must still be compressed; // otherwise this rule would have quietly turned compression off for the whole box. - expect(methods.get(HOST_ADAPTER.python.entryPoint)).toBe(ZIP_DEFLATED); + expect(methods.get(HOST_LAYOUT.entryPoint)).toBe(ZIP_DEFLATED); expect(methods.get('box.json')).toBe(ZIP_DEFLATED); // Stored is only worth anything if the bytes come back exactly, so read them back out. @@ -839,7 +902,7 @@ describe('the build pipeline', () => { const paths = listed.map((entry) => entry.path); expect(paths).not.toContain(PAYLOAD_DIGEST_FILE); expect(paths).toContain('box.json'); - expect(paths).toContain(HOST_ADAPTER.python.entryPoint); + expect(paths).toContain(HOST_LAYOUT.entryPoint); }); it('notices a payload byte that changed after the box was built', async () => { @@ -1059,7 +1122,7 @@ describe('the build pipeline', () => { log: () => {}, }); expect(receipt.selfTest).toBe('passed'); - expect(invocations[0].command).toContain(HOST_ADAPTER.python.entryPoint.split('/').at(-1)); + expect(invocations[0].command).toContain(HOST_LAYOUT.entryPoint.split('/').at(-1)); // Re-sign the same archive under a digest it does not have. Every archive check still passes, // so this isolates the one comparison: the tree the archive extracts to is not the tree the diff --git a/tests/unit/contract-runtimes.test.mjs b/tests/unit/contract-runtimes.test.mjs new file mode 100644 index 0000000..61af68f --- /dev/null +++ b/tests/unit/contract-runtimes.test.mjs @@ -0,0 +1,144 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { fixtureUrl } from '../../src/contract/index.mjs'; +import { boxTargetAdapters } from '../../src/contract/targets.mjs'; +import { + IMPLICIT_RUNTIME_ID, + executionAffectingVariables, + isExecutablePayloadPath, + runtimeAdapter, + runtimeAdapters, +} from '../../src/contract/runtimes.mjs'; + +/** + * The Node half of the shared runtime vectors. + * + * `src/contract/fixtures/runtime-contract.json` is what the Python and Rust mirrors validate + * themselves against, and this suite is what keeps the reference implementation honest about the + * same file. Every case here is one another language runs too; a change that only satisfies this + * one is a change that has broken the mirrors. + */ +const contract = JSON.parse(readFileSync(fixtureUrl('runtime-contract'), 'utf8')); + +/** A target the layout rules can be asked about, from a platform name alone. */ +function targetFor(platform) { + const adapter = boxTargetAdapters().find((candidate) => candidate.platform === platform); + if (!adapter) throw new Error(`No target adapter for platform ${platform}`); + return adapter; +} + +describe('runtime adapters', () => { + it('exposes exactly the runtimes the fixture describes', () => { + expect(runtimeAdapters().map((runtime) => runtime.id)) + .toEqual(contract.runtimes.map((fixture) => fixture.id)); + }); + + it('refuses a runtime the format does not define', () => { + for (const id of ['node', 'native', '', undefined, null, 42]) { + expect(() => runtimeAdapter(id)).toThrow(TypeError); + } + }); + + it('reproduces every golden layout and executable-path rule', () => { + for (const fixture of contract.runtimes) { + const runtime = runtimeAdapter(fixture.id); + expect([...runtime.executionKinds], fixture.id).toEqual(fixture.executionKinds); + expect([...runtime.executionEnvironmentVariables], fixture.id) + .toEqual(fixture.executionEnvironmentVariables); + for (const platform of fixture.layouts) { + const target = targetFor(platform.platform); + expect({ ...runtime.layout(target) }, platform.platform).toEqual(platform.layout); + const rule = runtime.executablePayloadPaths(target); + expect({ files: [...rule.files], directories: [...rule.directories] }, platform.platform) + .toEqual(platform.executablePayloadPaths); + } + } + }); + + it('answers the executable question the same way for every golden path', () => { + for (const testCase of contract.executableMatches) { + const runtime = runtimeAdapter(testCase.runtime); + const rule = runtime.executablePayloadPaths(targetFor(testCase.platform)); + expect(isExecutablePayloadPath(rule, testCase.path), testCase.name) + .toBe(testCase.executable); + } + }); + + it('derives exactly the golden candidate list for every declared execution', () => { + for (const testCase of contract.executionDiscovery) { + const runtime = runtimeAdapter(testCase.runtime); + const { candidates } = runtime.resolveExecutionFiles({ + execution: testCase.execution, + runtimeVersion: testCase.runtimeVersion, + target: targetFor(testCase.platform), + }); + expect([...candidates], testCase.name).toEqual(testCase.candidates); + } + }); + + it('refuses a runtime version that cannot name a standard library', () => { + const target = targetFor('linux'); + for (const runtimeVersion of contract.invalidRuntimeVersions) { + expect(() => runtimeAdapter('python').resolveExecutionFiles({ + execution: { kind: 'python-module', module: 'pkg', defaultArgs: [] }, + runtimeVersion, + target, + }), JSON.stringify(runtimeVersion)).toThrow(/Invalid Python version/); + } + }); + + it('builds exactly the golden shell-free command line', () => { + for (const testCase of contract.argv) { + const runtime = runtimeAdapter(testCase.runtime); + const invocation = runtime.buildArgv({ + execution: testCase.execution, + target: targetFor(testCase.platform), + }); + expect({ ...invocation.command }, testCase.name).toEqual(testCase.command); + expect(invocation.args.map((argument) => ({ ...argument })), testCase.name) + .toEqual(testCase.args); + } + }); + + it('turns every golden self-test probe into the same arguments', () => { + for (const testCase of contract.selfTest) { + const runtime = runtimeAdapter(testCase.runtime); + expect([...runtime.selfTestArgv({ + probe: testCase.probe, + target: targetFor(testCase.platform), + })], testCase.name).toEqual(testCase.args); + } + }); + + it('rejects a target no runtime has a layout for', () => { + const runtime = runtimeAdapter(IMPLICIT_RUNTIME_ID); + expect(() => runtime.layout({ platform: 'plan9' })).toThrow(/No python runtime layout/); + expect(() => runtime.selfTestArgv({ probe: { imports: [] }, target: { platform: 'plan9' } })) + .toThrow(/No python self-test assertion/); + }); +}); + +describe('execution-affecting variables', () => { + it('joins the runtime half to the target half, runtime first', () => { + // The order is what a diagnostic report is printed in, so it is part of the answer rather than + // an accident of how the two lists happened to be concatenated. + for (const adapter of boxTargetAdapters()) { + const merged = executionAffectingVariables(IMPLICIT_RUNTIME_ID, adapter); + expect([...merged], adapter.id).toEqual([ + ...runtimeAdapter(IMPLICIT_RUNTIME_ID).executionEnvironmentVariables, + ...adapter.executionAffectingEnvironmentVariables, + ]); + // Neither half may be dropped: this is the list that decides which inherited values a report + // calls out, and a short one is a quiet one. + expect(merged, adapter.id).toContain('PYTHONPATH'); + } + }); + + it('names the operating system control each platform actually has', () => { + const named = new Map(boxTargetAdapters() + .map((adapter) => [adapter.platform, executionAffectingVariables(IMPLICIT_RUNTIME_ID, adapter)])); + expect(named.get('macos')).toContain('DYLD_INSERT_LIBRARIES'); + expect(named.get('linux')).toContain('LD_PRELOAD'); + expect(named.get('windows')).not.toContain('LD_PRELOAD'); + }); +}); diff --git a/tests/unit/contract-targets.test.mjs b/tests/unit/contract-targets.test.mjs index 0046b4e..20148a1 100644 --- a/tests/unit/contract-targets.test.mjs +++ b/tests/unit/contract-targets.test.mjs @@ -46,13 +46,21 @@ describe('target adapters', () => { } }); - it('describes a payload layout consumers can rely on', () => { + it('describes the substrate and archive facts a build depends on', () => { for (const adapter of boxTargetAdapters()) { - expect(adapter.python.payloadRoot, adapter.id).toBe('venv'); - expect(adapter.python.entryPoint, adapter.id).toMatch(/^venv\//); expect(adapter.archive.format, adapter.id).toBe('zip'); - // The scripts directory must sit inside the payload root, or an installed box cannot find it. - expect(adapter.python.scriptsDirectory, adapter.id).toMatch(/^venv/); + expect(adapter.condaSubdir, adapter.id).toMatch(/^(osx-arm64|linux-64|win-64)$/); + expect(adapter.nativeLibraryInspection.command, adapter.id).toBeTruthy(); + } + }); + + it('carries only the operating system half of the execution-affecting variables', () => { + // The runtime contributes the rest. A target adapter that named PYTHONPATH would be saying a + // box is a Python box, which is exactly the coupling `runtimes.mjs` exists to remove. + for (const adapter of boxTargetAdapters()) { + for (const variable of adapter.executionAffectingEnvironmentVariables) { + expect(variable, adapter.id).not.toMatch(/^PYTHON/); + } } }); diff --git a/tests/unit/execution-contract.test.mjs b/tests/unit/execution-contract.test.mjs index 16af1cb..85d166a 100644 --- a/tests/unit/execution-contract.test.mjs +++ b/tests/unit/execution-contract.test.mjs @@ -9,13 +9,13 @@ describe('execution payload prerequisites', () => { expect(() => assertExecutionFiles({ execution, adapter, - pythonVersion: '3.11.15', + runtimeVersion: '3.11.15', files: new Set(['app/main.py']), })).not.toThrow(); expect(() => assertExecutionFiles({ execution, adapter, - pythonVersion: '3.11.15', + runtimeVersion: '3.11.15', files: new Set(), })).toThrow(/Execution script is missing/); }); @@ -30,7 +30,7 @@ describe('execution payload prerequisites', () => { expect(() => assertExecutionFiles({ execution: moduleExecution, adapter: linux, - pythonVersion: '3.11.15', + runtimeVersion: '3.11.15', files: new Set(['venv/lib/python3.11/site-packages/example_model/main.py']), })).not.toThrow(); @@ -38,14 +38,14 @@ describe('execution payload prerequisites', () => { expect(() => assertExecutionFiles({ execution: moduleExecution, adapter: windows, - pythonVersion: '3.11.15', + runtimeVersion: '3.11.15', files: new Set(['venv/Lib/site-packages/example_model/main.py']), })).not.toThrow(); expect(() => assertExecutionFiles({ execution: { ...moduleExecution, module: 'json.tool' }, adapter: linux, - pythonVersion: '3.11.15', + runtimeVersion: '3.11.15', files: new Set(['venv/lib/python3.11/json/tool.py']), })).not.toThrow(); }); @@ -55,7 +55,7 @@ describe('execution payload prerequisites', () => { expect(() => assertExecutionFiles({ execution: { kind: 'python-module', module: 'missing.main', defaultArgs: [] }, adapter, - pythonVersion: '3.12.4', + runtimeVersion: '3.12.4', files: new Set(['venv/bin/python']), })).toThrow(/Execution module is not discoverable/); }); diff --git a/tests/unit/llm-demo.test.mjs b/tests/unit/llm-demo.test.mjs index 0a3fbfe..ac0f466 100644 --- a/tests/unit/llm-demo.test.mjs +++ b/tests/unit/llm-demo.test.mjs @@ -8,6 +8,7 @@ import { auditScroll } from '../../src/build/audit.mjs'; import { readScroll } from '../../src/build/scroll.mjs'; import { configureWorkspace, resetWorkspace } from '../../src/build/workspace.mjs'; import { boxTargetAdapter, condaSubdir } from '../../src/contract/targets.mjs'; +import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../../src/contract/runtimes.mjs'; const root = fileURLToPath(new URL('../..', import.meta.url)); const example = join(root, 'examples', 'llm-demo'); @@ -190,7 +191,7 @@ describe('published local LLM demo box', () => { const manifest = await readFile(join(example, target, 'pixi.toml'), 'utf8'); expect(manifest, target).toContain(`platforms = ["${condaSubdir(scroll.target)}"]`); expect(scroll.pythonEntryPoint, target) - .toBe(boxTargetAdapter(scroll.target).python.entryPoint); + .toBe(runtimeAdapter(IMPLICIT_RUNTIME_ID).layout(boxTargetAdapter(scroll.target)).entryPoint); } }); diff --git a/tests/unit/scroll-editing.test.mjs b/tests/unit/scroll-editing.test.mjs index 2c95a56..e16b495 100644 --- a/tests/unit/scroll-editing.test.mjs +++ b/tests/unit/scroll-editing.test.mjs @@ -11,7 +11,8 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; -import { addDependency, readRequirements, withDependency } from '../../src/build/dependencies.mjs'; +import { addDependency, withDependency } from '../../src/build/dependencies.mjs'; +import { readRequirements } from '../../src/runtimes/python/dependencies.mjs'; import { sha256File } from '../../src/build/filesystem.mjs'; import { readScroll } from '../../src/build/scroll.mjs'; import { From 4743ecf74ca3a7fefd05ade49a95464ecec83cbf Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:45:44 +0200 Subject: [PATCH 02/22] Untrack the staging copy npm run types leaves behind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A temp workspace escaped into a commit in August, leaving a second, stale copy of src/ tracked in the tree — including a copy of the contract that no longer agreed with the real one. --- .gitignore | 4 + .../src/build/archive.mjs | 356 ------------ .../src/build/assets.mjs | 123 ----- .../src/build/audit.mjs | 69 --- .../src/build/authoring.mjs | 403 -------------- .../src/build/box.mjs | 339 ------------ .../src/build/consumer-setup.mjs | 110 ---- .../src/build/execution.mjs | 60 -- .../src/build/filesystem.mjs | 198 ------- .../src/build/identity.mjs | 39 -- .../src/build/index.mjs | 38 -- .../src/build/launchers.mjs | 60 -- .../src/build/licenses.mjs | 138 ----- .../src/build/parity.mjs | 111 ---- .../src/build/pixi.mjs | 437 --------------- .../src/build/process.mjs | 66 --- .../src/build/project.mjs | 276 ---------- .../src/build/schema-validation.mjs | 194 ------- .../src/build/scroll.mjs | 200 ------- .../src/build/toolchain.mjs | 246 --------- .../src/build/verify.mjs | 204 ------- .../src/build/workspace.mjs | 245 --------- .../src/cli-args.mjs | 31 -- .../src/cli-authoring.mjs | 187 ------- .../src/cli-init.mjs | 46 -- .../src/cli-menu.mjs | 102 ---- .../src/cli-output.mjs | 34 -- .../src/cli-run.mjs | 56 -- .../src/cli-signing.mjs | 33 -- .../src/cli-targets.mjs | 192 ------- .scrollcase-runtime-types-RzUfxr/src/cli.mjs | 501 ----------------- .../src/consumer/index.mjs | 17 - .../src/consumer/run-box.mjs | 46 -- .../src/consumer/run-extracted.mjs | 161 ------ .../src/consumer/verify-and-extract.mjs | 175 ------ .../src/contract/browser.mjs | 28 - .../src/contract/document-shape.mjs | 92 ---- .../src/contract/documents.mjs | 48 -- .../fixtures/consumer-conformance.json | 516 ------------------ .../examples/box-manifest.example.json | 32 -- .../examples/channel-manifest.example.json | 20 - .../examples/release-manifest.example.json | 45 -- .../examples/scroll-pixi.example.json | 72 --- .../fixtures/examples/scroll.example.json | 44 -- .../examples/signed-release.example.json | 13 - .../examples/signed-release.public-key.json | 5 - .../contract/fixtures/target-id-contract.json | 64 --- .../src/contract/index.mjs | 55 -- .../src/contract/links.mjs | 162 ------ .../contract/schema/box-manifest.schema.json | 124 ----- .../schema/channel-manifest.schema.json | 80 --- .../src/contract/schema/execution.schema.json | 94 ---- .../schema/release-manifest.schema.json | 272 --------- .../schema/revocations-manifest.schema.json | 64 --- .../src/contract/schema/scroll.schema.json | 339 ------------ .../schema/signed-document.schema.json | 43 -- .../src/contract/schema/target.schema.json | 57 -- .../src/contract/targets.mjs | 254 --------- .../src/contract/types/index.d.ts | 467 ---------------- .../src/sign/index.mjs | 132 ----- .../src/sign/keys.mjs | 160 ------ 61 files changed, 4 insertions(+), 8775 deletions(-) delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/archive.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/assets.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/audit.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/authoring.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/box.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/consumer-setup.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/execution.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/filesystem.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/identity.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/index.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/launchers.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/licenses.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/parity.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/pixi.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/process.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/project.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/schema-validation.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/scroll.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/toolchain.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/verify.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/workspace.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-args.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-authoring.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-init.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-menu.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-output.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-run.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-signing.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-targets.mjs delete mode 100755 .scrollcase-runtime-types-RzUfxr/src/cli.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/consumer/index.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/consumer/run-box.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/consumer/run-extracted.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/consumer/verify-and-extract.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/browser.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/document-shape.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/documents.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/consumer-conformance.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/box-manifest.example.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/channel-manifest.example.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/release-manifest.example.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll-pixi.example.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll.example.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.example.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.public-key.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/target-id-contract.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/index.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/links.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/box-manifest.schema.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/channel-manifest.schema.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/execution.schema.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/release-manifest.schema.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/revocations-manifest.schema.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/scroll.schema.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/signed-document.schema.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/target.schema.json delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/targets.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/types/index.d.ts delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/sign/index.mjs delete mode 100644 .scrollcase-runtime-types-RzUfxr/src/sign/keys.mjs diff --git a/.gitignore b/.gitignore index ec1f9d9..a4a676d 100644 --- a/.gitignore +++ b/.gitignore @@ -156,6 +156,10 @@ vite.config.ts.timestamp-* # scrollcase build state .scrollcase/ +# `npm run types` stages a copy of src/ here before running tsc over it. One escaped into a commit +# once, leaving a stale second copy of the contract in the tree. +.scrollcase-runtime-types-*/ + # Python development and packaging state __pycache__/ *.py[cod] diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/archive.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/archive.mjs deleted file mode 100644 index fa2cc33..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/archive.mjs +++ /dev/null @@ -1,356 +0,0 @@ -/** - * Deterministic archive creation and defensive extraction. - * - * Writing: every box ships as a ZIP whose bytes depend only on its contents — fixed timestamps, - * stable file ordering, and modes derived from the target adapter — so rebuilding the same commit - * reproduces the archive bit for bit. - * - * Reading: nothing from inside an archive is trusted before it is validated. Entry names are - * checked against path traversal, links and special entries are rejected outright, and both ZIP - * and TAR are handled by pinned Node implementations rather than whatever tools the host happens - * to have — an archive behaves the same on every machine that opens it. - */ -import { constants, createWriteStream } from 'node:fs'; -import { copyFile, mkdir, mkdtemp, rm, stat, symlink } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; -import { pipeline } from 'node:stream/promises'; -import * as tar from 'tar'; -import yauzl from 'yauzl'; -import yazl from 'yazl'; -import { findEntryThroughLink, findUnresolvableLink } from '../contract/links.mjs'; -import { - FIXED_ARCHIVE_TIME, - collectEntries, - collectFiles, - fileExists, - safeRelativePath, - validateExtractedTree, -} from './filesystem.mjs'; -import { fail } from './process.mjs'; - -const ZIP_FILE_TYPE_MASK = 0o170000; -const ZIP_REGULAR_FILE = 0o100000; -const ZIP_DIRECTORY = 0o040000; -const ZIP_SYMBOLIC_LINK = 0o120000; - -/** Returns the stable archive mode for a box payload file. */ -function archiveFileMode(adapter, relativePath) { - if (adapter.host.platform === 'win32') return 0o100644; - const scriptsDirectory = adapter.python.scriptsDirectory; - return relativePath === adapter.python.entryPoint || relativePath.startsWith(`${scriptsDirectory}/`) - ? 0o100755 - : 0o100644; -} - -/** - * Streams a deterministic, Zip64-capable box archive using the pinned Node backend. - * - * @param {string} payloadDir - * @param {string} archivePath - * @param {import('../contract/targets.mjs').BoxTargetAdapter} adapter - * @returns {Promise} - */ -export async function createDeterministicZip(payloadDir, archivePath, adapter) { - const entries = await collectEntries(payloadDir); - assertPayloadLinksAreCarryable(entries); - await rm(archivePath, { force: true }); - await mkdir(dirname(archivePath), { recursive: true }); - const zip = new yazl.ZipFile(); - const output = pipeline(zip.outputStream, createWriteStream(archivePath, { flags: 'wx' })); - for (const entry of entries) { - if (entry.kind === 'link') { - // A link is its target string, stored under a mode whose type bits say what it is — the same - // two facts every ZIP implementation reads it back from. - zip.addBuffer(Buffer.from(entry.linkTarget, 'utf8'), entry.path, { - compress: false, - mtime: FIXED_ARCHIVE_TIME, - mode: ZIP_SYMBOLIC_LINK | 0o777, - forceDosTimestamp: true, - }); - continue; - } - zip.addFile(join(payloadDir, ...entry.path.split('/')), entry.path, { - compress: true, - compressionLevel: 6, - mtime: FIXED_ARCHIVE_TIME, - mode: archiveFileMode(adapter, entry.path), - forceDosTimestamp: true, - }); - } - zip.end({ forceZip64Format: false }); - await output; -} - -/** - * Refuses to archive a payload whose links do not satisfy the contract rule. - * - * The builder settles links against the real filesystem; this asks the same question of the entry - * set that will actually be written, which is what a consumer will later be handed. A failure here - * is a bug in this repository rather than bad input — but shipping a box a consumer must reject is - * worse than not building one. - * - * @param {Array<{ path: string, kind: string, linkTarget?: string }>} entries - */ -function assertPayloadLinksAreCarryable(entries) { - const unresolvable = findUnresolvableLink(entries); - if (unresolvable) fail(`Box link does not resolve to a file inside the payload: ${unresolvable}`); - const throughLink = findEntryThroughLink(entries); - if (throughLink) fail(`Box entry would be written through a link: ${throughLink}`); -} - -/** - * The longest link target a payload may carry. A real one is a file name; anything approaching a - * path limit is either corrupt or an attempt to make reading the archive expensive. - */ -const MAX_LINK_TARGET_BYTES = 1024; - -/** Classifies a ZIP entry and rejects special entries and encrypted files. */ -function classifyZipEntry(entry) { - if ((entry.generalPurposeBitFlag & 0x1) !== 0) fail(`Encrypted ZIP entries are not allowed: ${entry.fileName}`); - const path = safeRelativePath(entry.fileName.endsWith('/') ? entry.fileName.slice(0, -1) : entry.fileName); - const unixType = (entry.externalFileAttributes >>> 16) & ZIP_FILE_TYPE_MASK; - if (unixType === ZIP_SYMBOLIC_LINK) { - if (entry.uncompressedSize > MAX_LINK_TARGET_BYTES) fail(`Archive link target is too long: ${path}`); - // The target itself is the entry's content, so it is not known yet; listZipEntries reads it - // before anything is validated, and nothing may be extracted until it has. - return { path, kind: 'link', size: entry.uncompressedSize, mode: 0o777, linkTarget: null }; - } - const directory = entry.fileName.endsWith('/') || unixType === ZIP_DIRECTORY; - if (!directory && unixType !== 0 && unixType !== ZIP_REGULAR_FILE) { - fail(`Archive special entries are not allowed: ${path}`); - } - return { - path, - kind: directory ? 'directory' : 'file', - size: entry.uncompressedSize, - mode: (entry.externalFileAttributes >>> 16) & 0o777, - }; -} - -/** Rejects duplicate paths and file/directory collisions before extraction begins. */ -function assertNoZipEntryCollisions(entries) { - const seen = new Map(); - const parentsWithChildren = new Set(); - for (const entry of entries) { - if (seen.has(entry.path)) fail(`Archive entry collides with another entry: ${entry.path}`); - const parts = entry.path.split('/'); - for (let index = 1; index < parts.length; index += 1) { - const parent = parts.slice(0, index).join('/'); - if (seen.get(parent) === 'file') { - fail(`Archive entry collides with another entry: ${entry.path}`); - } - parentsWithChildren.add(parent); - } - if (entry.kind === 'file' && parentsWithChildren.has(entry.path)) { - fail(`Archive entry collides with another entry: ${entry.path}`); - } - seen.set(entry.path, entry.kind); - } -} - -/** Opens a ZIP with strict names, path validation, and uncompressed-size checks enabled. */ -async function openZip(archivePath) { - return yauzl.openPromise(archivePath, { - autoClose: false, - decodeStrings: true, - lazyEntries: true, - strictFileNames: true, - validateEntrySizes: true, - }); -} - -/** - * Lists and validates all entries before any ZIP data is trusted or extracted. - * - * @param {string} archivePath - * @returns {Promise>} - */ -export async function listZipEntries(archivePath) { - const zip = await openZip(archivePath); - const entries = []; - try { - for await (const entry of zip.eachEntry()) { - const classified = classifyZipEntry(entry); - if (classified.kind === 'link') { - const chunks = []; - const stream = await zip.openReadStreamPromise(entry); - for await (const chunk of stream) chunks.push(chunk); - classified.linkTarget = Buffer.concat(chunks).toString('utf8'); - } - entries.push(classified); - } - } finally { - await zip.close(); - } - assertNoZipEntryCollisions(entries); - // Every link is judged by the same rule the builder applied, against the archive as received - // rather than as intended. A box assembled by hand gets no benefit of the doubt here. - const unresolvable = findUnresolvableLink(entries); - if (unresolvable) fail(`Archive link does not resolve to a file inside the payload: ${unresolvable}`); - const throughLink = findEntryThroughLink(entries); - if (throughLink) fail(`Archive entry would be written through a link: ${throughLink}`); - return entries; -} - -/** - * Reads one small ZIP metadata entry without extracting the surrounding archive. - * - * @param {string} archivePath - * @param {string} wantedPath - * @param {number} [maximumBytes] - * @returns {Promise} - */ -export async function readZipEntry(archivePath, wantedPath, maximumBytes = 1024 * 1024) { - const safePath = safeRelativePath(wantedPath); - const zip = await openZip(archivePath); - try { - for await (const entry of zip.eachEntry()) { - const classified = classifyZipEntry(entry); - if (classified.path !== safePath || classified.kind !== 'file') continue; - if (classified.size > maximumBytes) fail(`ZIP entry is too large to read as metadata: ${safePath}`); - const stream = await zip.openReadStreamPromise(entry); - const chunks = []; - let length = 0; - for await (const chunk of stream) { - length += chunk.length; - if (length > maximumBytes) fail(`ZIP entry is too large to read as metadata: ${safePath}`); - chunks.push(chunk); - } - return Buffer.concat(chunks).toString('utf8'); - } - } finally { - await zip.close(); - } - fail(`ZIP archive does not contain ${safePath}`); -} - -/** - * Extracts a prevalidated ZIP without shelling out to whatever unzip the host provides. - * - * @param {string} archivePath - * @param {string} destination - * @returns {Promise} - */ -export async function extractZipArchive(archivePath, destination) { - // Validated in full first, and the targets it returns are the only ones written below: reading - // the link target twice would let a concurrently rewritten archive pass the check with one value - // and extract with another. - const validated = await listZipEntries(archivePath); - const linkTargets = new Map(validated - .filter((entry) => entry.kind === 'link') - .map((entry) => [entry.path, entry.linkTarget])); - await mkdir(destination, { recursive: true }); - const zip = await openZip(archivePath); - try { - for await (const entry of zip.eachEntry()) { - const classified = classifyZipEntry(entry); - const outputPath = join(destination, ...classified.path.split('/')); - if (classified.kind === 'directory') { - await mkdir(outputPath, { recursive: true }); - continue; - } - await mkdir(dirname(outputPath), { recursive: true }); - if (classified.kind === 'link') { - // Written as the relative string it was validated as, never as a resolved absolute path: - // the link must mean the same thing wherever the box is extracted. - await symlink(linkTargets.get(classified.path), outputPath); - continue; - } - const stream = await zip.openReadStreamPromise(entry); - await pipeline(stream, createWriteStream(outputPath, { - flags: 'wx', - mode: classified.mode || 0o644, - })); - } - } finally { - await zip.close(); - } - await validateExtractedTree(destination, { allowLinks: true }); -} - -/** Lists TAR assets and rejects paths, links, and special entries before extraction. */ -async function validateTarArchive(archivePath) { - let violation; - await tar.t({ - file: archivePath, - gzip: true, - strict: true, - onentry(entry) { - if (violation) return; - try { - safeRelativePath(entry.path.endsWith('/') ? entry.path.slice(0, -1) : entry.path); - if (!['File', 'OldFile', 'Directory'].includes(entry.type)) { - violation = `Archive links and special entries are not allowed: ${entry.path}`; - } - } catch (error) { - violation = error instanceof Error ? error.message : String(error); - } - }, - }); - if (violation) fail(violation); -} - -/** - * Extracts scroll assets using only pinned Node archive implementations. - * - * @param {string} archivePath - * @param {'zip' | 'tar.gz'} format - * @param {string} destination - * @param {number} [stripComponents] - * @returns {Promise} - */ -export async function extractScrollArchive(archivePath, format, destination, stripComponents = 0) { - const tempRoot = await mkdtemp(join(tmpdir(), 'scrollcase-extract-')); - try { - if (format === 'zip') { - await extractZipArchive(archivePath, tempRoot); - } else if (format === 'tar.gz') { - await validateTarArchive(archivePath); - await tar.x({ - file: archivePath, - cwd: tempRoot, - gzip: true, - preservePaths: false, - strict: true, - }); - await validateExtractedTree(tempRoot); - } else { - fail(`Unsupported archive format: ${format}`); - } - - let source = tempRoot; - for (let index = 0; index < stripComponents; index += 1) { - const entries = await collectFiles(source); - const topLevels = [...new Set(entries - .map((entry) => entry.split('/')[0]) - .filter((entry) => entry !== '__MACOSX'))]; - if (topLevels.length !== 1) fail(`Cannot strip archive component ${index + 1}: expected one root directory`); - const nextSource = join(source, topLevels[0]); - if (!(await stat(nextSource)).isDirectory()) { - fail(`Cannot strip archive component ${index + 1}: expected one root directory`); - } - source = nextSource; - } - const files = await collectFiles(source); - // Archives may add a subtree beside verified assets, but must never replace those assets. - for (const file of files) { - if (await fileExists(join(destination, ...file.split('/')))) { - fail(`Scroll archive entry already exists in destination: ${file}`); - } - } - await mkdir(destination, { recursive: true }); - for (const file of files) { - const outputPath = join(destination, ...file.split('/')); - await mkdir(dirname(outputPath), { recursive: true }); - await copyFile(join(source, ...file.split('/')), outputPath, constants.COPYFILE_EXCL); - } - } finally { - await rm(tempRoot, { recursive: true, force: true }); - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/assets.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/assets.mjs deleted file mode 100644 index 247b8e0..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/assets.mjs +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Fetching and staging the files a box carries. - * - * Every asset is declared with a size and a SHA-256 in the scroll, and nothing enters the payload - * before both match. That is what makes a box reproducible even though its inputs live on servers - * outside anyone's control: if an upstream file is moved, replaced, or silently re-uploaded, the - * build fails instead of quietly producing a different box under the same version. - */ - -import { createWriteStream } from 'node:fs'; -import { copyFile, mkdir, rename, rm, stat } from 'node:fs/promises'; -import { dirname, join, sep } from 'node:path'; -import { pipeline } from 'node:stream/promises'; -import { extractScrollArchive as extractArchive } from './archive.mjs'; -import { fileExists, safeRelativePath, sha256File } from './filesystem.mjs'; -import { fail } from './process.mjs'; - -const MAX_DOWNLOAD_ATTEMPTS = 5; - -/** - * Downloads an asset and enforces the scroll's declared size and hash. - * - * Model files are large, so retries inside one download operation resume from a `.part` file with a - * Range request. Two safeguards matter — a complete destination is reused only after size and hash - * verification, and the `.part` file is renamed into place only *after* the hash matches, so an - * interrupted or corrupted transfer can never masquerade as a finished asset. The build scratch - * tree is recreated at process start, so this is intentionally not a cross-process cache. - */ -export async function downloadVerified(asset, destination, options = {}) { - const { - fetchImpl = fetch, - log = console.error, - wait = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)), - } = options; - const expectedPath = safeRelativePath(asset.relativePath).split('/').join(sep); - if (!destination.endsWith(expectedPath)) fail(`Unexpected asset destination: ${destination}`); - await mkdir(dirname(destination), { recursive: true }); - if (await fileExists(destination)) { - const current = await stat(destination); - if (current.size === asset.sizeBytes && await sha256File(destination) === asset.sha256) return; - } - const partPath = `${destination}.part`; - // Large assets come from hosts that occasionally drop a connection mid-stream. Retry with backoff, - // resuming from the partial via Range, so a reset near the end is not paid for in full. - for (let attempt = 1; ; attempt += 1) { - const resumeAt = await fileExists(partPath) ? (await stat(partPath)).size : 0; - try { - const response = await fetchImpl(asset.url, { - headers: resumeAt > 0 ? { Range: `bytes=${resumeAt}-` } : undefined, - redirect: 'follow', - }); - if (!response.ok) fail(`Asset download failed (${response.status}): ${asset.url}`); - // Only append when the server actually honoured the range (206). A server that ignores Range - // replies 200 with the whole body, which must overwrite rather than be appended to a partial. - const append = resumeAt > 0 && response.status === 206; - await pipeline(response.body, createWriteStream(partPath, { flags: append ? 'a' : 'w' })); - break; - } catch (error) { - // A failed status is a hard error, not a transient network drop — do not retry it. - const message = error instanceof Error ? error.message : String(error); - if (message.startsWith('Asset download failed') || attempt >= MAX_DOWNLOAD_ATTEMPTS) throw error; - log(`scrollcase: asset ${asset.relativePath} attempt ${attempt} failed (${message}); retrying.`); - await wait(2000 * attempt); - } - } - const downloaded = await stat(partPath); - if (downloaded.size !== asset.sizeBytes) fail(`Asset size mismatch for ${asset.relativePath}.`); - if (await sha256File(partPath) !== asset.sha256) { - // A full-size partial with the wrong digest cannot be resumed: asking for bytes after its end - // would either fail forever or append unrelated data. Remove it so the next build starts from - // byte zero and has a chance to recover from a corrupt mirror response. - await rm(partPath, { force: true }); - fail(`Asset SHA-256 mismatch for ${asset.relativePath}.`); - } - await rename(partPath, destination); -} - -/** Copies a file from the project into the payload after verifying its declared hash. */ -export async function copyVerifiedLocalFile(file, payloadDir, projectRoot) { - const source = join(projectRoot, safeRelativePath(file.sourcePath)); - if (!await fileExists(source) || !(await stat(source)).isFile()) { - fail(`Local box file is missing: ${file.sourcePath}`); - } - if (await sha256File(source) !== file.sha256) { - fail(`Local box file SHA-256 mismatch: ${file.sourcePath}`); - } - const destination = join(payloadDir, safeRelativePath(file.relativePath)); - await mkdir(dirname(destination), { recursive: true }); - await copyFile(source, destination); -} - -/** - * Moves a built file to the name it is published under, without ever leaving two copies behind. - * - * A box archive is measured in gigabytes, so this renames rather than copies: on one filesystem the - * bytes never move at all. The copy-and-remove fallback is for the case a project points its build - * and dist directories at different volumes, where rename cannot work. - */ -export async function moveIntoPlace(source, destination) { - await rm(destination, { force: true }); - try { - await rename(source, destination); - } catch { - await copyFile(source, destination); - await rm(source, { force: true }); - } -} - -/** - * Unpacks a downloaded archive into the payload tree. - * - * Entries are listed and validated *before* extraction (archive-slip defence). `stripComponents` - * drops the redundant top-level wrapper directory many published archives carry; it insists on - * finding exactly one directory to strip, so a surprising layout fails loudly rather than producing - * a wrong tree. - */ -export async function expandAssetArchive(payloadDir, archive) { - const archivePath = join(payloadDir, safeRelativePath(archive.relativePath)); - const destination = join(payloadDir, safeRelativePath(archive.destination)); - await extractArchive(archivePath, archive.format, destination, Number(archive.stripComponents ?? 0)); - // The compressed original is dead weight inside the payload once unpacked. - if (archive.removeAfterExtract !== false) await rm(archivePath, { force: true }); -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/audit.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/audit.mjs deleted file mode 100644 index c284042..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/audit.mjs +++ /dev/null @@ -1,69 +0,0 @@ -/** - * `audit` — the dependency licence inventory, without building anything. - * - * The inventory is a pure function of the committed lock, so it can be produced, reviewed and - * checked into a repository long before any box exists. That matters because licence review is a - * human step: it should happen when dependencies change, not in the middle of a multi-gigabyte build - * that then fails at the end. - * - * The same function the build runs is used here, so a reviewed audit and the one a build produces - * cannot disagree by construction. - */ - -import { mkdir, readFile, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import { boxTargetId } from '../contract/targets.mjs'; -import { compareStableStrings, fileExists, safeRelativePath } from './filesystem.mjs'; -import { createCondaDependencyLicenseAudit, validateCondaDependencyLicenseAudit } from './licenses.mjs'; -import { fail } from './process.mjs'; -import { readScroll } from './scroll.mjs'; -import { getWorkspace } from './workspace.mjs'; - -/** - * Produces the inventory for a scroll, and either checks it against the reviewed copy or writes it. - * - * Writing is explicit (`write: true`) because overwriting the reviewed file is how an unreviewed - * licence change would slip through: the default is to compare and fail on any difference. - */ -export async function auditScroll(name, { write = false, namespace } = {}) { - const workspace = getWorkspace(); - const { dir, scroll } = await readScroll(name); - const lockPath = join(dir, 'pixi.lock'); - if (!await fileExists(lockPath)) fail(`Missing dependency lock: ${lockPath}`); - const inventory = createCondaDependencyLicenseAudit({ - lockBytes: await readFile(lockPath), - targetId: boxTargetId(scroll.target), - ...(namespace ? { namespace } : {}), - }); - - // A package with no declared licence never reaches here: parsing the lock rejects it outright, - // which is the point — an unlicensed dependency is a legal problem, not a reporting gap. - const licences = new Map(); - for (const entry of inventory.packages) { - licences.set(entry.declaredLicense, (licences.get(entry.declaredLicense) ?? 0) + 1); - } - const summary = { - scrollId: scroll.scrollId, - targetId: inventory.targetId, - packageCount: inventory.packages.length, - licenses: [...licences] - .sort((left, right) => right[1] - left[1] || compareStableStrings(left[0], right[0])) - .map(([license, count]) => ({ license, count })), - }; - - if (!scroll.condaDependencyLicenseAudit) { - if (write) fail('The scroll declares no condaDependencyLicenseAudit path to write to.'); - return { inventory, summary, reviewed: null }; - } - const reviewedPath = join(workspace.root, safeRelativePath(scroll.condaDependencyLicenseAudit)); - if (write) { - await mkdir(dirname(reviewedPath), { recursive: true }); - await writeFile(reviewedPath, `${JSON.stringify(inventory, null, 2)}\n`); - return { inventory, summary, reviewed: reviewedPath, written: true }; - } - if (!await fileExists(reviewedPath)) { - fail(`Reviewed licence audit is missing: ${reviewedPath}. Run audit --write and review the result.`); - } - validateCondaDependencyLicenseAudit(JSON.parse(await readFile(reviewedPath, 'utf8')), inventory); - return { inventory, summary, reviewed: reviewedPath, written: false }; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/authoring.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/authoring.mjs deleted file mode 100644 index bb3adf4..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/authoring.mjs +++ /dev/null @@ -1,403 +0,0 @@ -/** - * Authoring one scroll inside an initialized workspace. - * - * `init` owns workspace structure; this module owns the atomic creation of one target-specific - * scroll. All material input is validated before the first write, existing paths are never - * overwritten, and a generated starter script is hashed from the exact bytes written to disk. - * Execution metadata is authored here and later copied unchanged into both signed manifests by the - * builder; keeping creation separate prevents this module from acquiring build or execution policy. - */ - -import { createHash } from 'node:crypto'; -import { lstat, mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import { boxTargetAdapter, boxTargetId, condaSubdir } from '../contract/targets.mjs'; -import { fileExists, safeRelativePath, sha256File } from './filesystem.mjs'; -import { fail } from './process.mjs'; -import { schemaValidationError } from './schema-validation.mjs'; - -const scrollSchemaUrl = new URL('../contract/schema/scroll.schema.json', import.meta.url); -const targetSchemaUrl = new URL('../contract/schema/target.schema.json', import.meta.url); -const executionSchemaUrl = new URL('../contract/schema/execution.schema.json', import.meta.url); -const EXECUTION_KINDS = Object.freeze(['python-script', 'python-module', 'library-only']); -const WEIGHTS_MODES = Object.freeze(['embed', 'on-demand']); -export const EXAMPLE_PIXI_VERSION = '0.73.0'; - -const STARTER_SCRIPT = `"""Minimal application entry point generated by Scrollcase.""" - -import sys - - -def main() -> int: - print("Scrollcase box is ready.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) -`; - -const TYPESCRIPT_CONSUMER_TEMPLATE = `/** - * Runs a local box through the typed Node consumer. - * - * SETUP (once): - * npm install scrollcase - * npm install --save-dev tsx typescript - * - * RUN: - * npx tsx consumer-templates/run-box.ts - * - * Replace and below with the values printed by scrollcase build. - */ -import { runBox } from 'scrollcase/consumer'; - -const releaseToRun = - '.scrollcase/dist/boxes/example-box/1.0.0//.release.json'; - -runBox(releaseToRun, { - publicPath: '.scrollcase/keys/signing-public.json', - args: [], - stdin: 'inherit', - stdout: 'inherit', - stderr: 'inherit', - onPrepared: ({ boxId, version, targetId }) => { - console.log(\`Running \${boxId} \${version} (\${targetId})\`); - }, -}).then((result) => { - if (result.signal) console.error(\`Box exited after \${result.signal}.\`); - process.exitCode = result.exitCode ?? 1; -}); -`; - -const CONSUMER_PACKAGE_JSON = `${JSON.stringify({ - private: true, - type: 'module', -}, null, 2)}\n`; - -const PYTHON_CONSUMER_TEMPLATE = `""" -Runs a local box through the typed Python consumer. - -The Python consumer is published separately on PyPI. -npm install scrollcase does not install this Python package. - -SETUP (once): - - python -m pip install scrollcase-consumer - -RUN (from the project root): - - python consumer-templates/run_box.py - -Replace and below with the values printed by scrollcase build. -""" - -from __future__ import annotations - -import sys - -from scrollcase_consumer import PreparedBox, run_box - - -RELEASE_TO_RUN = ( - ".scrollcase/dist/boxes/example-box/1.0.0//.release.json" -) - - -def _report(prepared: PreparedBox) -> None: - print( - f"Running {prepared.box_id} {prepared.version} ({prepared.target_id})" - ) - - -def main() -> int: - result = run_box( - RELEASE_TO_RUN, - public_key_path=".scrollcase/keys/signing-public.json", - args=[], - on_prepared=_report, - ) - - if result.signal is not None: - print(f"Box exited after {result.signal}.", file=sys.stderr) - return result.exit_code if result.exit_code is not None else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) -`; - -const textHash = (value) => createHash('sha256').update(value, 'utf8').digest('hex'); - -async function ensureTextFile(path, contents) { - await mkdir(dirname(path), { recursive: true }); - try { - await writeFile(path, contents, { flag: 'wx' }); - return true; - } catch (error) { - if (error?.code === 'EEXIST') return false; - throw error; - } -} - -function requiredText(value, name) { - if (typeof value !== 'string' || value.trim() === '') fail(`${name} is required.`); - return value.trim(); -} - -function pixiManifest(environmentName, target, pythonVersion) { - const pythonConstraint = /^\d+\.\d+$/.test(pythonVersion) - ? `${pythonVersion}.*` - : pythonVersion; - return `# Solved by \`scrollcase lock\` into pixi.lock, which is committed and reviewed. -# \`platforms\` must equal the target's conda subdirectory, or the solve produces an environment -# that cannot run on the machine the box is for. -[workspace] -name = "${environmentName}" -channels = ["conda-forge"] -platforms = ["${condaSubdir(target)}"] - -[dependencies] -python = "${pythonConstraint}" -`; -} - -async function validateScroll(scroll) { - const [scrollSchema, targetSchema, executionSchema] = await Promise.all( - [scrollSchemaUrl, targetSchemaUrl, executionSchemaUrl] - .map(async (url) => JSON.parse(await readFile(url, 'utf8'))), - ); - const error = schemaValidationError(scroll, scrollSchema, [targetSchema, executionSchema]); - if (error) fail(`Generated scroll is invalid: ${error}.`); -} - -/** - * Creates one nested `/` scroll without overwriting any authored file. - * - * @param {object} options - * @returns {Promise<{ written: string[], scroll: object, scrollDir: string, scrollRef: string, - * targetId: string, generatedScriptPath: string | null }>} - */ -export async function createScroll({ - workspace, - boxId, - target, - modelId, - runtimeId, - version, - scrollVersion, - sourceRevision, - pythonVersion, - pixiVersion, - compatibility, - assetBaseUrl, - weights, - executionKind, - scriptSourcePath = null, - generateScript = false, - generatedScriptSourcePath = null, - scriptRelativePath = 'entrypoint.py', - module = null, - defaultArgs = [], -}) { - if (!workspace?.configPath || !await fileExists(workspace.configPath) - || !await fileExists(workspace.scrollsDir)) { - fail('No initialized Scrollcase workspace; run scrollcase init first.'); - } - - const identity = { - boxId: requiredText(boxId, 'boxId'), - modelId: requiredText(modelId, 'modelId'), - runtimeId: requiredText(runtimeId, 'runtimeId'), - version: requiredText(version, 'version'), - scrollVersion: requiredText(scrollVersion, 'scrollVersion'), - sourceRevision: requiredText(sourceRevision, 'sourceRevision'), - pythonVersion: requiredText(pythonVersion, 'pythonVersion'), - pixiVersion: requiredText(pixiVersion, 'pixiVersion'), - assetBaseUrl: requiredText(assetBaseUrl, 'assetBaseUrl'), - }; - if (!compatibility || typeof compatibility !== 'object' || Array.isArray(compatibility)) { - fail('compatibility must be an object.'); - } - if (!WEIGHTS_MODES.includes(weights)) { - fail(`Unsupported weights mode: ${weights}. Use ${WEIGHTS_MODES.join(' or ')}.`); - } - if (!EXECUTION_KINDS.includes(executionKind)) { - fail(`Unsupported execution kind: ${executionKind}. Use ${EXECUTION_KINDS.join(', ')}.`); - } - if (!Array.isArray(defaultArgs) || defaultArgs.some((value) => typeof value !== 'string')) { - fail('defaultArgs must be an array of strings.'); - } - - const adapter = boxTargetAdapter(target); - const targetId = boxTargetId(target); - const scrollRef = `${identity.boxId}/${targetId}`; - const scrollDir = join(workspace.scrollsDir, identity.boxId, targetId); - if (await fileExists(scrollDir)) fail(`Scroll already exists: ${scrollRef}.`); - - let localFile = null; - let execution; - let generatedScriptPath = null; - let generatedSource = null; - if (executionKind === 'python-script') { - if (generateScript && scriptSourcePath) { - fail('Choose either an existing script or --generate-script, not both.'); - } - if (!generateScript && !scriptSourcePath) { - fail('python-script execution requires an existing script or --generate-script.'); - } - const relativePath = safeRelativePath(scriptRelativePath); - let sourcePath; - let sha256; - if (generateScript) { - sourcePath = safeRelativePath(generatedScriptSourcePath - ?? `box-entrypoints/${identity.boxId}/${targetId}/entrypoint.py`); - generatedScriptPath = join(workspace.root, ...sourcePath.split('/')); - if (await fileExists(generatedScriptPath)) { - fail(`Generated script already exists: ${sourcePath}.`); - } - generatedSource = STARTER_SCRIPT; - sha256 = textHash(generatedSource); - } else { - sourcePath = safeRelativePath(scriptSourcePath); - const source = join(workspace.root, ...sourcePath.split('/')); - let details; - try { - details = await lstat(source); - } catch { - fail(`Project script is missing: ${sourcePath}.`); - } - if (!details.isFile() || details.isSymbolicLink()) { - fail(`Project script must be a regular file: ${sourcePath}.`); - } - sha256 = await sha256File(source); - } - localFile = { sourcePath, relativePath, sha256 }; - execution = { kind: 'python-script', script: relativePath, defaultArgs: [...defaultArgs] }; - } else if (executionKind === 'python-module') { - execution = { - kind: 'python-module', - module: requiredText(module, 'module'), - defaultArgs: [...defaultArgs], - }; - } else if (module || scriptSourcePath || generateScript || defaultArgs.length > 0) { - fail('library-only execution cannot declare a script, module, or default arguments.'); - } - - const scroll = { - $schema: 'https://scrollcase.dev/schema/v2/scroll.schema.json', - schemaVersion: 2, - scrollVersion: identity.scrollVersion, - boxId: identity.boxId, - modelId: identity.modelId, - runtimeId: identity.runtimeId, - version: identity.version, - sourceRevision: identity.sourceRevision, - target, - compatibility: { ...compatibility }, - pythonVersion: identity.pythonVersion, - pixiVersion: identity.pixiVersion, - pythonEntryPoint: adapter.python.entryPoint, - modelCacheSubdir: `model-cache/${identity.boxId}`, - assetBaseUrl: identity.assetBaseUrl, - assets: [], - selfTest: { - imports: ['json'], - files: localFile ? [localFile.relativePath] : [], - }, - weights, - ...(localFile ? { localFiles: [localFile] } : {}), - ...(execution ? { execution } : {}), - }; - await validateScroll(scroll); - - const boxDir = dirname(scrollDir); - await mkdir(boxDir, { recursive: true }); - const staging = await mkdtemp(join(boxDir, '.scrollcase-new-')); - let generatedWritten = false; - try { - await writeFile(join(staging, 'scroll.json'), `${JSON.stringify(scroll, null, 2)}\n`); - await writeFile( - join(staging, 'pixi.toml'), - pixiManifest(`${identity.boxId}-${targetId}`, target, identity.pythonVersion), - ); - if (generatedScriptPath) { - await mkdir(dirname(generatedScriptPath), { recursive: true }); - await writeFile(generatedScriptPath, generatedSource, { flag: 'wx' }); - generatedWritten = true; - } - await rename(staging, scrollDir); - } catch (error) { - await rm(staging, { recursive: true, force: true }); - if (generatedWritten) await rm(generatedScriptPath, { force: true }); - throw error; - } - - const written = [ - join(scrollDir, 'scroll.json'), - join(scrollDir, 'pixi.toml'), - ...(generatedScriptPath ? [generatedScriptPath] : []), - ]; - return { written, scroll, scrollDir, scrollRef, targetId, generatedScriptPath }; -} - -/** - * Ensures the disposable example created by `init` exists for one native target. - * - * The example uses the same authoring path as every real scroll. An existing target directory is - * treated as authored input and left untouched, including when a user has edited the starter. - * - * @param {{ workspace: object, target: object, pixiVersion?: string }} options - * @returns {Promise} - */ -export async function ensureExampleScroll({ - workspace, - target, - pixiVersion = EXAMPLE_PIXI_VERSION, -}) { - const targetId = boxTargetId(target); - const scrollRef = `example-box/${targetId}`; - const scrollDir = join(workspace.scrollsDir, 'example-box', targetId); - let result; - if (await fileExists(scrollDir)) { - result = { - created: false, - written: [], - scrollDir, - scrollRef, - targetId, - generatedScriptPath: null, - }; - } else { - result = { - created: true, - ...await createScroll({ - workspace, - boxId: 'example-box', - target, - modelId: 'example-org-example-box', - runtimeId: 'example-box-runtime', - version: '1.0.0', - scrollVersion: '1.0.0', - sourceRevision: 'example-source-1.0.0', - pythonVersion: '3.11', - pixiVersion, - compatibility: { minHostAppVersion: '1.0.0' }, - assetBaseUrl: 'https://example.org/boxes', - weights: 'embed', - executionKind: 'python-script', - generateScript: true, - }), - }; - } - - const consumerFiles = [ - [join(workspace.root, 'package.json'), CONSUMER_PACKAGE_JSON], - [join(workspace.root, 'consumer-templates', 'run-box.ts'), TYPESCRIPT_CONSUMER_TEMPLATE], - [join(workspace.root, 'consumer-templates', 'run_box.py'), PYTHON_CONSUMER_TEMPLATE], - ]; - const consumerFilesWritten = []; - for (const [path, contents] of consumerFiles) { - if (await ensureTextFile(path, contents)) consumerFilesWritten.push(path); - } - return { ...result, written: [...result.written, ...consumerFilesWritten] }; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/box.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/box.mjs deleted file mode 100644 index 41b7ac0..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/box.mjs +++ /dev/null @@ -1,339 +0,0 @@ -/** - * `build` — assemble the environment, prove it works, archive it, sign it. - * - * In order: solve and pack the conda-forge environment from the committed lock, fetch and unpack the - * declared assets, prune what is not needed at run time, audit dependency licences, self-test with - * the payload's *own* interpreter, normalise timestamps, zip deterministically, and emit a signed - * release plus a signed channel pointer. - * - * The self-test is the step that earns the box its name. The builder runs target, import, optional - * Python-code, and file assertions; the release signs the import subset that a consumer can repeat - * after extraction. The distinction is deliberate rather than pretending the narrower consumer - * check reproduces scroll-only assertions it cannot see. - * - * The archive is content-addressed by its own hash, so the release document can commit to it and any - * consumer can verify it byte for byte. - */ - -import { createHash } from 'node:crypto'; -import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import { assertNativeHost, boxTargetId } from '../contract/targets.mjs'; -import { CHANNELS, documentKinds } from '../contract/documents.mjs'; -import { signDocument } from '../sign/index.mjs'; -import { copyVerifiedLocalFile, downloadVerified, expandAssetArchive, moveIntoPlace } from './assets.mjs'; -import { createDeterministicZip } from './archive.mjs'; -import { - collectFiles, - fileExists, - normalizeTree, - payloadSize, - safeRelativePath, - sha256File, -} from './filesystem.mjs'; -import { assertExecutionFiles } from './execution.mjs'; -import { boxReleaseObjectPrefix, boxReleaseStem, builderVersionFields } from './identity.mjs'; -import { createCondaDependencyLicenseAudit, validateCondaDependencyLicenseAudit } from './licenses.mjs'; -import { checkParity } from './parity.mjs'; -import { findCondaPack, findPixi, installAndPackPixiEnvironment } from './pixi.mjs'; -import { fail, run as runProcess } from './process.mjs'; -import { readScroll, sourceBuildState, sourceBuildTime } from './scroll.mjs'; -import { getWorkspace } from './workspace.mjs'; - -const SELF_TEST_TIMEOUT_SECONDS = 180; -const sha256Hex = (bytes) => createHash('sha256').update(bytes).digest('hex'); - -/** Runs the scroll's self-test with the payload's own interpreter, under the target's environment. */ -function runSelfTest({ interpreter, adapter, scroll, payloadDir, run }) { - const imports = `import ${scroll.selfTest.imports.join(', ')}`; - const code = scroll.selfTest.pythonCode - ? `${adapter.selfTestPython}\n${imports}\n${scroll.selfTest.pythonCode}` - : `${adapter.selfTestPython}\n${imports}`; - run(interpreter, ['-c', code], { - cwd: payloadDir, - env: adapter.validationEnvironments[scroll.target.accelerator], - }); -} - -/** Writes the licence inventory the box ships, after proving it still matches the reviewed one. */ -async function writeLicenceAudit({ scroll, lockPath, payloadDir, projectRoot }) { - if (!scroll.condaDependencyLicenseAudit) return; - const actual = createCondaDependencyLicenseAudit({ - lockBytes: await readFile(lockPath), - targetId: boxTargetId(scroll.target), - }); - const reviewedPath = join(projectRoot, safeRelativePath(scroll.condaDependencyLicenseAudit)); - const reviewed = JSON.parse(await readFile(reviewedPath, 'utf8')); - validateCondaDependencyLicenseAudit(reviewed, actual); - const auditPath = join(payloadDir, 'THIRD_PARTY_NOTICES', 'conda-distributions.json'); - await mkdir(dirname(auditPath), { recursive: true }); - await writeFile(auditPath, `${JSON.stringify(actual, null, 2)}\n`); -} - -/** - * Builds, self-tests, archives, and signs the box a scroll describes — the whole pipeline the - * module header narrates. `name` is an exact scroll reference, or an unambiguous box shorthand; - * options override signing, channel, weights mode, namespace, and toolchain paths. `run`, - * `runResult`, and `fetchImpl` are the injection seams the tests use to substitute the toolchain - * and asset transport. - */ -export async function buildBox(name, options = {}) { - const { - allowDirty = false, - channel = 'beta', - weights = null, - assetBaseUrl: assetBaseUrlOverride = null, - namespace, - signerCommand = null, - privatePath, - publicPath, - pixiPath = null, - condaPackPath = null, - run = runProcess, - runResult = null, - fetchImpl = fetch, - log = console.log, - } = options; - // Tool discovery probes with its own runner; a caller may substitute one to drive a build without - // the real toolchain on PATH. - const probe = runResult ? { runResult } : {}; - const workspace = getWorkspace(); - const { adapter, dir, scroll } = await readScroll(name); - const weightsMode = weights || scroll.weights || 'embed'; - if (!CHANNELS.includes(channel)) { - fail(`Unsupported channel: ${channel}. Use ${CHANNELS.join(' or ')}.`); - } - if (weightsMode !== 'embed' && weightsMode !== 'on-demand') { - fail(`Unsupported weights mode: ${weightsMode}. Use embed or on-demand.`); - } - if (weightsMode === 'on-demand' && (scroll.assetArchives ?? []).length > 0) { - fail('on-demand weights cannot be combined with assetArchives, which are expanded at build time.'); - } - // Wheels, native libraries, and the interpreter are proven on the exact OS/architecture they ship for. - assertNativeHost(adapter); - const pixi = findPixi({ requiredVersion: scroll.pixiVersion, path: pixiPath, ...probe }); - const condaPack = findCondaPack({ path: condaPackPath, ...probe }); - const lockPath = join(dir, 'pixi.lock'); - // A build installs from the lock and never resolves, so a missing lock is a hard error rather than - // an invitation to resolve dependencies on the fly. - if (!await fileExists(lockPath)) fail(`Missing dependency lock: ${lockPath}`); - const lockSha = await sha256File(lockPath); - - const source = sourceBuildState(workspace.root); - if (!source) fail('A box records the commit it was built from; run inside a git checkout.'); - // If the tree is dirty that record is a lie — the artefact would not be reproducible from that - // revision — so refuse unless the caller explicitly accepts it for local development. - if (source.dirty && !allowDirty) { - fail('Refusing to build from a dirty source tree. Commit first, or pass --allow-dirty for local development.'); - } - - const buildDir = join(workspace.buildDir, scroll.scrollId); - const payloadDir = join(buildDir, 'payload'); - // `dist` is laid out as the two things a publisher does with it, and nothing else. `boxes/` is - // the tree that goes under the asset base URL verbatim — the same prefix the signed documents - // write into their own URLs, so uploading it is a copy rather than a mapping. `channels/` is - // separate because a channel is not part of any one version: it is a pointer that moves to the - // next one, and filing it under 1.0.0 would leave a stale copy claiming to be current the moment - // 1.0.1 ships. Nothing is written twice: what is on disk here is what gets published. - const objectPrefix = boxReleaseObjectPrefix(scroll); - const objectDir = join(workspace.distDir, ...objectPrefix.split('/')); - const archivePath = join(buildDir, `${boxReleaseStem(scroll)}.zip`); - // Always start from an empty tree: leftovers from a previous build would end up in the archive. - await rm(buildDir, { recursive: true, force: true }); - await rm(objectDir, { recursive: true, force: true }); - await mkdir(payloadDir, { recursive: true }); - - const { interpreter } = await installAndPackPixiEnvironment({ - pixi, - condaPack, - manifestPath: join(dir, 'pixi.toml'), - lockPath, - buildDir, - payloadDir, - adapter, - run, - }); - - // `embed` packs the assets into the archive, so an installed box needs no network and works - // air-gapped. `on-demand` leaves them out for the caller's distribution layer to materialize from - // descriptors carried in the signed release. Consumers verify those bytes before execution; the - // declared hash is what keeps that safe. The choice trades archive size against an install-time - // dependency on the asset host, so it is the project's to make, per build. - const embedded = weightsMode === 'embed'; - for (const asset of embedded ? scroll.assets : []) { - log(`Downloading ${asset.relativePath}`); - await downloadVerified(asset, join(payloadDir, safeRelativePath(asset.relativePath)), { - fetchImpl, - log, - }); - } - const deferredAssets = new Set(embedded ? [] : scroll.assets.map((asset) => asset.relativePath)); - for (const file of scroll.localFiles ?? []) { - await copyVerifiedLocalFile(file, payloadDir, workspace.root); - } - for (const archive of embedded ? scroll.assetArchives ?? [] : []) { - await expandAssetArchive(payloadDir, archive); - } - // Drops what is only needed to build (tests, docs, bundled sample data). A box is a multi-gigabyte - // download for an end user, so pruning is a user-facing concern rather than tidiness. - for (const prunePath of scroll.prunePaths ?? []) { - await rm(join(payloadDir, safeRelativePath(prunePath)), { recursive: true, force: true }); - } - await writeLicenceAudit({ scroll, lockPath, payloadDir, projectRoot: workspace.root }); - // Guards against over-pruning: the files the box needs at run time must still be there. - for (const requiredFile of scroll.selfTest.files ?? []) { - // A deferred asset is legitimately absent from the payload; anything else missing means pruning - // removed something the box needs at run time. - if (deferredAssets.has(requiredFile)) continue; - if (!await fileExists(join(payloadDir, safeRelativePath(requiredFile)))) { - fail(`Missing self-test file: ${requiredFile}`); - } - } - assertExecutionFiles({ - execution: scroll.execution, - adapter, - pythonVersion: scroll.pythonVersion, - files: new Set(await collectFiles(payloadDir)), - }); - runSelfTest({ interpreter, adapter, scroll, payloadDir, run }); - // Parity runs after the self-test, on the same payload: there is no point comparing accelerators - // in a box that cannot import its dependencies in the first place. - const parity = await checkParity({ - parity: scroll.parity, - adapter, - interpreter, - payloadDir, - run, - }); - if (parity) { - log(`Parity passed on ${parity.comparisons.map((c) => c.accelerator).join(', ')} against ${parity.comparisons[0].reference}`); - } - - // Everything needed to answer "where did this box come from, and could I rebuild it?". - const provenance = { - scrollId: scroll.scrollId, - scrollVersion: scroll.scrollVersion, - builderRevision: source.revision, - sourceTreeDirty: source.dirty, - sourceRevision: scroll.sourceRevision, - pythonVersion: scroll.pythonVersion, - ...builderVersionFields(scroll), - dependencyLockSha256: lockSha, - builtAt: sourceBuildTime(workspace.root), - }; - const selfTest = { - pythonImports: scroll.selfTest.imports, - timeoutSeconds: SELF_TEST_TIMEOUT_SECONDS, - }; - // Descriptors travel with the box only when the consumer has to fetch the assets itself. - const deferred = embedded ? {} : { - weights: 'on-demand', - assets: scroll.assets.map(({ url, relativePath, sizeBytes, sha256 }) => ({ - url, relativePath, sizeBytes, sha256, - })), - }; - const identity = { - boxId: scroll.boxId, - modelId: scroll.modelId, - runtimeId: scroll.runtimeId, - version: scroll.version, - }; - const execution = scroll.execution ? { execution: scroll.execution } : {}; - // box.json travels *inside* the archive. A consumer compares it field by field against the signed - // release, which is what binds the archive's contents to its signed metadata. - await writeFile(join(payloadDir, 'box.json'), `${JSON.stringify({ - schemaVersion: 2, - ...identity, - target: scroll.target, - pythonEntryPoint: scroll.pythonEntryPoint, - modelCacheSubdir: scroll.modelCacheSubdir, - selfTest, - ...execution, - ...deferred, - provenance, - }, null, 2)}\n`); - await normalizeTree(payloadDir); - const installedSizeBytes = await payloadSize(payloadDir); - await mkdir(workspace.distDir, { recursive: true }); - await createDeterministicZip(payloadDir, archivePath, adapter); - - const archiveSha = await sha256File(archivePath); - const archiveSize = (await stat(archivePath)).size; - // Content-addressed: the object is named after its own hash, so publishing is idempotent and an - // object can never be replaced with different bytes under the same URL. - const archiveObject = `${objectPrefix}/${archiveSha}.zip`; - const assetBaseUrl = String(assetBaseUrlOverride || scroll.assetBaseUrl || '').replace(/\/$/, ''); - if (!assetBaseUrl) fail('No asset base URL: declare assetBaseUrl in the scroll or pass --asset-base-url.'); - const kinds = documentKinds(namespace); - const signing = { signerCommand, privatePath, publicPath }; - - const release = { - schemaVersion: 2, - kind: kinds.release, - ...identity, - target: scroll.target, - compatibility: scroll.compatibility, - archive: { format: 'zip', url: `${assetBaseUrl}/${archiveObject}`, sha256: archiveSha, sizeBytes: archiveSize }, - installedSizeBytes, - pythonEntryPoint: scroll.pythonEntryPoint, - modelCacheSubdir: scroll.modelCacheSubdir, - selfTest, - ...execution, - ...deferred, - provenance, - }; - // Written beside the archive, under the same scratch rule: named for its own hash once it has one. - const stagedReleasePath = join(buildDir, 'release.json'); - await writeFile(stagedReleasePath, `${JSON.stringify(await signDocument(release, signing), null, 2)}\n`); - - // The channel points at the release document by *its* hash too, so the whole chain is - // content-addressed: channel -> release document -> archive. - const releaseDocumentSha = await sha256File(stagedReleasePath); - const channelDocument = { - schemaVersion: 2, - kind: kinds.channel, - channel, - boxId: scroll.boxId, - target: scroll.target, - updatedAt: provenance.builtAt, - // Derived from box and version rather than random, so rebuilding the same release reproduces the - // same cohort assignment instead of reshuffling which users receive it. - cohortSalt: sha256Hex(Buffer.from(`${scroll.boxId}:${scroll.version}`)).slice(0, 32), - // A freshly built channel goes out at 100%; a staged rollout is arranged by editing this document - // rather than by the builder. - releases: [{ - version: scroll.version, - releaseManifestUrl: `${assetBaseUrl}/${objectPrefix}/${releaseDocumentSha}.release.json`, - rolloutPercentage: 100, - }], - }; - // One file per channel per target, filed by channel rather than by version: it is a pointer, and - // the next release moves it rather than adding a second one. - const channelDir = join(workspace.distDir, 'channels', scroll.boxId, channel); - const channelPath = join(channelDir, `${boxTargetId(scroll.target)}.json`); - await mkdir(channelDir, { recursive: true }); - await writeFile(channelPath, `${JSON.stringify(await signDocument(channelDocument, signing), null, 2)}\n`); - - // Both documents move — not copy — into the tree a publisher uploads, so the only copy that - // exists is the one that gets published and there is no second name for the same bytes. - await mkdir(objectDir, { recursive: true }); - const publishedArchive = join(objectDir, `${archiveSha}.zip`); - const publishedRelease = join(objectDir, `${releaseDocumentSha}.release.json`); - await moveIntoPlace(archivePath, publishedArchive); - await moveIntoPlace(stagedReleasePath, publishedRelease); - log(`Box: ${publishedArchive}`); - log(`Release: ${publishedRelease}`); - log(`Channel: ${channelPath}`); - log(''); - log(`Publish: upload ${join(workspace.distDir, 'boxes')} under ${assetBaseUrl}, keeping its paths,`); - log(' then publish the channel document where your clients look for it.'); - return { - archivePath: publishedArchive, - releasePath: publishedRelease, - channelPath, - archiveSha256: archiveSha, - installedSizeBytes, - weights: weightsMode, - parity, - }; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/consumer-setup.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/consumer-setup.mjs deleted file mode 100644 index 358c0c3..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/consumer-setup.mjs +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Optional dependencies for the generated consumer templates. - * - * These installations belong to the initialized project, not Scrollcase's managed build - * toolchain. Every command therefore runs from the workspace root, beside - * `scrollcase.config.json`. Node uses the root package and `node_modules`; Python uses the - * interpreter selected from the caller's environment. Consent and the Python package source are - * chosen at the CLI edge and passed in explicitly. - */ - -import { readFileSync } from 'node:fs'; -import { fail, run as defaultRun, runResult as defaultRunResult } from './process.mjs'; - -const packageJson = JSON.parse(readFileSync( - new URL('../../package.json', import.meta.url), - 'utf8', -)); - -export const SCROLLCASE_NPM_VERSION = packageJson.version; - -export function installTypeScriptConsumerDependencies({ - root, - scrollcaseVersion = SCROLLCASE_NPM_VERSION, - platform = process.platform, - comspec = process.env.ComSpec || 'cmd.exe', - run = defaultRun, -}) { - const runNpm = (args) => { - if (platform === 'win32') { - // npm is a .cmd shim on Windows, which spawnSync cannot execute directly. - run(comspec, ['/d', '/s', '/c', 'npm', ...args], { cwd: root }); - return; - } - run('npm', args, { cwd: root }); - }; - runNpm(['install', `scrollcase@${scrollcaseVersion}`]); - runNpm(['install', '--save-dev', 'tsx', 'typescript']); - return { scrollcaseVersion }; -} - -function findPython({ root, runResult }) { - for (const command of ['python', 'python3', 'py']) { - const result = runResult(command, ['--version'], { capture: true, cwd: root }); - if (!result.error && result.status === 0) return command; - } - fail('Python was not found. Install Python 3.10 or newer, then re-run scrollcase init.'); -} - -export function isCondaAvailable({ - root, - runResult = defaultRunResult, -}) { - const result = runResult('conda', ['--version'], { capture: true, cwd: root }); - return !result.error && result.status === 0; -} - -export function installPythonConsumerDependency({ - root, - source, - run = defaultRun, - runResult = defaultRunResult, -}) { - if (!['pypi', 'conda-forge'].includes(source)) { - fail(`Unsupported Python consumer source ${source}.`); - } - - if (source === 'pypi') { - const command = findPython({ root, runResult }); - const args = ['-m', 'pip', 'install', 'scrollcase-consumer']; - const result = runResult(command, args, { capture: true, cwd: root }); - if (result.error) fail(`${command} failed to start: ${result.error.message}`); - if (result.status === 0) return { source, command }; - - const detail = `${result.stderr || ''}\n${result.stdout || ''}`; - if (/externally-managed-environment/i.test(detail)) { - // PEP 668 blocks even user installs unless pip receives the override. Pairing it with - // --user keeps package files out of Homebrew's or the distribution's managed prefix. - run( - command, - [ - '-m', - 'pip', - 'install', - '--user', - '--break-system-packages', - 'scrollcase-consumer', - ], - { cwd: root }, - ); - return { source, command }; - } - fail(`${command} exited with status ${result.status}\n${detail.trim()}`); - } - - if (!isCondaAvailable({ root, runResult })) { - fail('Conda is not installed. Re-run scrollcase init and choose PyPI with pip.'); - } - run( - 'conda', - [ - 'install', - '--yes', - '--channel', - 'conda-forge', - 'scrollcase-consumer', - ], - { cwd: root }, - ); - return { source, command: 'python' }; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/execution.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/execution.mjs deleted file mode 100644 index 965097c..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/execution.mjs +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Static execution prerequisites shared by the builder and verifier. - * - * Execution metadata is not a command string: it names either one regular payload file or one - * dotted Python module. Checking the archive file set proves those names can resolve without - * importing a package, running an `__init__.py`, or starting the application. The later consumer - * may therefore launch only after the complete trust chain has passed. - */ - -import { safeRelativePath } from './filesystem.mjs'; -import { fail } from './process.mjs'; - -function pythonMajorMinor(version) { - const match = /^(\d+)\.(\d+)(?:\.|$)/.exec(version); - if (!match) fail(`Invalid Python version for execution discovery: ${version}.`); - return `${match[1]}.${match[2]}`; -} - -function moduleEntryPoints({ adapter, module, pythonVersion }) { - const modulePath = module.split('.').join('/'); - const relativeCandidates = [`${modulePath}.py`, `${modulePath}/__main__.py`]; - const standardLibrary = adapter.platform === 'windows' - ? 'venv/Lib' - : `venv/lib/python${pythonMajorMinor(pythonVersion)}`; - const roots = ['', standardLibrary, `${standardLibrary}/site-packages`]; - return roots.flatMap((root) => - relativeCandidates.map((path) => (root ? `${root}/${path}` : path))); -} - -/** - * Confirms that optional execution metadata names runnable regular files in a payload/archive. - * - * `files` must contain only regular archive entries. Both collectFiles() during build and the ZIP - * entry classifier during verify provide exactly that representation. - */ -export function assertExecutionFiles({ - execution, - adapter, - pythonVersion, - files, -}) { - if (!execution) return; - if (execution.kind === 'python-script') { - const script = safeRelativePath(execution.script); - if (!files.has(script)) fail(`Execution script is missing from the box: ${script}.`); - return; - } - if (execution.kind === 'python-module') { - const candidates = moduleEntryPoints({ - adapter, - module: execution.module, - pythonVersion, - }); - if (!candidates.some((path) => files.has(path))) { - fail(`Execution module is not discoverable in the box: ${execution.module}.`); - } - return; - } - fail(`Unsupported execution kind: ${String(execution.kind)}.`); -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/filesystem.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/filesystem.mjs deleted file mode 100644 index 0da0cd0..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/filesystem.mjs +++ /dev/null @@ -1,198 +0,0 @@ -/** - * Filesystem primitives shared by the build, archive, and verify layers. - * - * Two invariants live here. Determinism: payload files are always enumerated in one stable order - * and stamped with one fixed timestamp, so hashing and archiving the same tree twice produces the - * same bytes. Safety: every relative path that will be joined to a directory is screened against - * traversal, and trees that will enter a box are refused if they contain links or special nodes. - */ -import { createHash } from 'node:crypto'; -import { createReadStream } from 'node:fs'; -import { access, lstat, lutimes, readdir, readlink } from 'node:fs/promises'; -import { join, relative, sep } from 'node:path'; -import { fail } from './process.mjs'; - -/** - * The single mtime every archived file carries. Any fixed instant works — what matters is that it - * never varies between builds; this one is simply a recognisable round date safely past the 1980 - * floor of DOS/ZIP timestamps. - */ -export const FIXED_ARCHIVE_TIME = new Date('2000-01-01T00:00:00.000Z'); - -/** - * Returns whether a filesystem entry exists without exposing platform-specific error codes. - * - * @param {string} path - * @returns {Promise} - */ -export async function fileExists(path) { - try { - await access(path); - return true; - } catch { - return false; - } -} - -/** - * Compares machine-facing identifiers by code unit, independent of host locale and ICU data. - * - * @param {string} left - * @param {string} right - * @returns {-1 | 0 | 1} - */ -export function compareStableStrings(left, right) { - return left < right ? -1 : left > right ? 1 : 0; -} - -/** - * Rejects paths that could escape a box staging directory. - * - * @param {unknown} value - * @returns {string} the path, normalised to forward slashes - * @throws {Error} when the path is absolute, empty, contains `..`, a drive letter or a NUL - */ -export function safeRelativePath(value) { - const normalized = String(value).replaceAll('\\', '/'); - if (!normalized || normalized.startsWith('/') || normalized.includes('\0')) { - fail(`Unsafe relative path: ${value}`); - } - if (/^[A-Za-z]:\//.test(normalized) - || normalized.split('/').some((part) => part === '..' || part === '')) { - fail(`Unsafe relative path: ${value}`); - } - return normalized; -} - -/** - * Lists payload entries in the stable order used by hashing and archive creation. - * - * A payload may hold regular files and the narrow class of symbolic links `src/contract/links.mjs` - * permits; anything else — a socket, a device, a fifo — is still refused, because nothing that is - * not one of those two things can be archived, hashed or relocated meaningfully. - * - * @param {string} root - * @param {string} [current] - * @returns {Promise>} - */ -export async function collectEntries(root, current = root) { - const entries = await readdir(current, { withFileTypes: true }); - const collected = []; - for (const entry of entries.sort((a, b) => compareStableStrings(a.name, b.name))) { - if (entry.name === '__pycache__' || entry.name === '.DS_Store' || entry.name.endsWith('.pyc')) continue; - const fullPath = join(current, entry.name); - const path = relative(root, fullPath).split(sep).join('/'); - // Order matters: a symlink to a directory reports isDirectory() as false but would be walked - // into by isDirectory() checks that stat rather than lstat, so links are classified first. - if (entry.isSymbolicLink()) { - collected.push({ path, kind: 'link', linkTarget: (await readlink(fullPath)).split(sep).join('/') }); - } else if (entry.isDirectory()) { - collected.push(...await collectEntries(root, fullPath)); - } else if (entry.isFile()) { - collected.push({ path, kind: 'file' }); - } else { - fail(`box special entries are not allowed: ${path}`); - } - } - return collected; -} - -/** - * Lists every payload path — files and links alike — in the stable archive order. - * - * Callers asking "is this path in the box" want a link to count, because a linked path is a path - * that resolves. Callers that must read or rewrite bytes want `collectRegularFiles` instead. - * - * @param {string} root - * @returns {Promise} - */ -export async function collectFiles(root) { - return (await collectEntries(root)).map((entry) => entry.path); -} - -/** - * Lists only the payload paths backed by their own bytes. - * - * Anything that rewrites file contents belongs here rather than on `collectFiles`: writing through - * a link would edit the target a second time, once under its own name and once under the link's. - * - * @param {string} root - * @returns {Promise} - */ -export async function collectRegularFiles(root) { - return (await collectEntries(root)).filter((entry) => entry.kind === 'file').map((entry) => entry.path); -} - -/** - * Sums what a box actually occupies once extracted. - * - * `lstat`, not `stat`: a link costs its own few bytes, not the size of what it points at. Counting - * the target would restore on paper exactly the duplication that preserving links removes from - * disk, and this number is what a consumer checks free space against. - * - * @param {string} root - * @returns {Promise} - */ -export async function payloadSize(root) { - let total = 0; - for (const file of await collectFiles(root)) { - total += (await lstat(join(root, ...file.split('/')))).size; - if (!Number.isSafeInteger(total)) fail('box installed size exceeds the safe integer range.'); - } - return total; -} - -/** - * Rejects links and special nodes before an extracted tree is copied. - * - * This guards a *scroll-declared asset archive* — a third-party tar or zip a project points at — - * whose contents are then copied into the payload. A link here is not the narrow, checked kind a - * box may carry: it arrives from outside, and the copy that follows would write through it. The - * links a payload does carry come from the packed conda prefix, which is a different path with its - * own contract check, so this stays as strict as it has always been. - * - * A box archive is the one caller that passes `allowLinks`, because its links were each checked - * against the contract rule before extraction wrote them. Every other caller keeps the default. - * - * @param {string} root - * @param {{ allowLinks?: boolean, current?: string }} [options] - * @returns {Promise} - */ -export async function validateExtractedTree(root, { allowLinks = false, current = root } = {}) { - for (const entry of await readdir(current, { withFileTypes: true })) { - const fullPath = join(current, entry.name); - if (entry.isSymbolicLink()) { - if (allowLinks) continue; - fail(`Archive links and special entries are not allowed: ${relative(root, fullPath)}`); - } - if (entry.isDirectory()) await validateExtractedTree(root, { allowLinks, current: fullPath }); - else if (!entry.isFile()) fail(`Archive special entries are not allowed: ${relative(root, fullPath)}`); - } -} - -/** - * Applies the archive timestamp to every payload entry. - * - * `lutimes` stamps the link itself rather than following it to its target, which would otherwise be - * stamped once under its own name and again through every link that points at it. - * - * @param {string} root - * @returns {Promise} - */ -export async function normalizeTree(root) { - for (const file of await collectFiles(root)) { - await lutimes(join(root, ...file.split('/')), FIXED_ARCHIVE_TIME, FIXED_ARCHIVE_TIME); - } -} - -/** - * Streams a file into SHA-256 without buffering large boxes in memory. - * - * @param {string} path - * @returns {Promise} - */ -export async function sha256File(path) { - const hash = createHash('sha256'); - for await (const chunk of createReadStream(path)) hash.update(chunk); - return hash.digest('hex'); -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/identity.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/identity.mjs deleted file mode 100644 index 026be4b..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/identity.mjs +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Naming: where a release's artefacts live relative to everything else. - * - * The stem and object prefix are derived from the release's identity fields alone, so the archive, - * its release document, and the staged objects always agree on their names without any of them - * recording the others' paths. Whatever a consumer uses to serve boxes, laying storage out under - * this prefix means the URLs inside the signed documents already point at the right objects. - */ -import { boxTargetId } from '../contract/targets.mjs'; - -/** - * Returns the single filename stem shared by an archive and its release document. - * - * @param {Pick} release - * @returns {string} `--` - */ -export function boxReleaseStem(release) { - return `${release.boxId}-${release.version}-${boxTargetId(release.target)}`; -} - -/** - * Returns the immutable object prefix for one box release target. - * - * @param {Pick} release - * @returns {string} `boxes///` - */ -export function boxReleaseObjectPrefix(release) { - return `boxes/${release.boxId}/${release.version}/${boxTargetId(release.target)}`; -} - -/** - * Returns the builder-identity field recorded in provenance: the pixi release that solved the box. - * - * @param {{ pixiVersion?: string } | null | undefined} source - * @returns {{ pixiVersion: string | undefined }} - */ -export function builderVersionFields(source) { - return { pixiVersion: source?.pixiVersion }; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/index.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/index.mjs deleted file mode 100644 index dbd079d..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/index.mjs +++ /dev/null @@ -1,38 +0,0 @@ -/** - * The build layer: everything needed to turn a scroll into a packed, relocatable box. - * - * One substrate only — pixi solves a conda-forge environment from a committed `pixi.lock`, conda-pack - * relocates it, and the result is extracted into the box's `venv/`. There is deliberately no second - * dependency backend: a packaging tool with two substrates has to prove every guarantee twice. - */ - -export { createDeterministicZip, extractZipArchive, listZipEntries } from './archive.mjs'; -export { collectFiles, fileExists, sha256File } from './filesystem.mjs'; -export { boxReleaseObjectPrefix, boxReleaseStem, builderVersionFields } from './identity.mjs'; -export { repairPosixLaunchers } from './launchers.mjs'; -export { - createCondaDependencyLicenseAudit, - lockedCondaDistributions, - parseCondaPackageReference, - validateCondaDependencyLicenseAudit, -} from './licenses.mjs'; -export { - condaPackArguments, - findCondaPack, - findPixi, - installAndPackPixiEnvironment, - pixiInstallArguments, - pixiLockArguments, -} from './pixi.mjs'; -export { fail, run, runResult } from './process.mjs'; -export { CONDA_PACK_VERSION } from './toolchain.mjs'; -export { - DEFAULT_WORKSPACE_PATHS, - SCROLLCASE_CONFIG_FILENAME, - configureWorkspace, - findWorkspaceConfig, - getWorkspace, - resolveWorkspace, - workspaceOverridesFromArgv, - workspaceOverridesFromFlags, -} from './workspace.mjs'; diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/launchers.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/launchers.mjs deleted file mode 100644 index a153592..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/launchers.mjs +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Repairs the console scripts a conda environment generates. - * - * Console scripts (tqdm, isympy, f2py, …) are written at solve time with the *build machine's* - * absolute interpreter path in their shebang. That path means nothing on a user's machine, and - * shipping it also leaks a developer's directory layout. Rewriting them to resolve Python next to - * themselves is what makes the packed environment genuinely relocatable. - */ - -import { chmod, readFile, writeFile } from 'node:fs/promises'; -import { basename, join } from 'node:path'; -import { collectFiles, fileExists } from './filesystem.mjs'; - -/** - * Removes either a direct shebang or a shell trampoline header from a launcher, leaving the Python - * body. The trampoline appears when an absolute shebang would exceed the POSIX length limit; it - * closes its quote either on its own `' '''` line or at the end of the same line (`… "$@" #'''`), - * so both are handled by scanning forward to the line that closes the quote. - */ -function posixLauncherBody(text) { - const lines = text.split('\n'); - if (lines.length === 0 || !lines[0].startsWith('#!')) return text; - if (lines[1]?.startsWith("'''exec'")) { - for (let index = 1; index < lines.length; index += 1) { - if (lines[index].trimEnd().endsWith("'''")) return lines.slice(index + 1).join('\n'); - } - } - return lines.slice(1).join('\n'); -} - -/** - * Makes generated POSIX console scripts resolve Python relative to their own installed path. - * - * @param {import('../contract/targets.mjs').BoxTargetAdapter} adapter - * @param {string} payloadDir - * @param {readonly string[]} forbiddenPaths - * @returns {Promise} - */ -export async function repairPosixLaunchers(adapter, payloadDir, forbiddenPaths) { - const scriptsRoot = join(payloadDir, ...adapter.python.scriptsDirectory.split('/')); - if (!await fileExists(scriptsRoot)) return; - const pythonName = basename(adapter.python.entryPoint); - for (const file of await collectFiles(scriptsRoot)) { - const path = join(scriptsRoot, ...file.split('/')); - const bytes = await readFile(path); - if (!bytes.subarray(0, 2).equals(Buffer.from('#!'))) continue; - const text = bytes.toString('utf8'); - // Search the complete generated launcher, since a trampoline hides the path below line one. - if (!forbiddenPaths.some((value) => text.includes(value))) continue; - const body = posixLauncherBody(text); - const launcher = [ - '#!/bin/sh', - `'''exec' "$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)/${pythonName}" "$0" "$@"`, - "' '''", - body, - ].join('\n'); - await writeFile(path, launcher); - await chmod(path, 0o755); - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/licenses.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/licenses.mjs deleted file mode 100644 index ae7a58b..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/licenses.mjs +++ /dev/null @@ -1,138 +0,0 @@ -/** - * Builds the dependency licence inventory shipped inside every box. - * - * The inventory is derived from the committed lock file rather than from the installed tree: the - * lock already carries an SPDX licence per package, and `pixi install --frozen` guarantees the - * installed set equals it. That makes the audit a pure function of a file the user reviews, so it - * can be computed without a built prefix and cannot drift from what was approved. - */ - -import { createHash } from 'node:crypto'; -import { DEFAULT_DOCUMENT_NAMESPACE } from '../contract/documents.mjs'; -import { compareStableStrings } from './filesystem.mjs'; - -/** - * One package as the lock declares it. - * - * @typedef {object} LockedDistribution - * @property {string} name - * @property {string} version - * @property {string} declaredLicense the SPDX expression carried by the lock - * @property {'conda' | 'pypi'} source - */ - -function fail(message) { - throw new Error(`box licence audit: ${message}`); -} - -function sha256(value) { - return createHash('sha256').update(value).digest('hex'); -} - -/** conda ships packages as `.conda` or the older `.tar.bz2`; both encode name-version-build. */ -const CONDA_PACKAGE_FILE = /\.(?:conda|tar\.bz2)$/; - -/** - * Derives (name, version) from a conda package filename: `name-version-build.conda`. - * - * @param {string} url a conda package URL or filename - * @returns {{ name: string, version: string }} - * @throws {Error} when the filename is not `name-version-build.conda` - */ -export function parseCondaPackageReference(url) { - const file = String(url).split('/').pop() ?? ''; - const stem = file.replace(CONDA_PACKAGE_FILE, ''); - const parts = stem.split('-'); - // conda names may contain '-', but version and build never do, so they are the last two segments. - if (parts.length < 3 || stem === file) fail(`unparseable conda package filename: ${file}`); - parts.pop(); // build string - const version = parts.pop(); - return { name: parts.join('-'), version }; -} - -/** - * Parses the exact conda + pypi distributions and their declared licenses from a pixi.lock. - * - * The `packages:` section is a YAML list of `- conda: ` / `- pypi: ` items, each followed - * by indented `key: value` fields. This scans that regular, machine-generated structure directly - * rather than taking a transitive YAML dependency. - * - * @param {Buffer} lockBytes the committed `pixi.lock` - * @returns {LockedDistribution[]} sorted by name then version - * @throws {Error} when the lock is unparseable or a package lacks a licence - */ -export function lockedCondaDistributions(lockBytes) { - const lines = lockBytes.toString('utf8').split(/\r?\n/); - const start = lines.findIndex((line) => line === 'packages:'); - if (start === -1) fail('pixi.lock has no packages section'); - const distributions = []; - let current = null; - const flush = () => { - if (!current) return; - let { name, version } = current; - if (current.source === 'conda') ({ name, version } = parseCondaPackageReference(current.url)); - if (!name || !version) fail(`pixi.lock package lacks a name or version: ${current.url}`); - if (!current.license || current.license.toUpperCase() === 'UNKNOWN') { - fail(`${name}==${version} lacks a declared license in pixi.lock`); - } - // conda/pypi filenames already carry the canonical name, so keep raw names — normalizing - // would mangle legitimate leading-underscore conda names like `_openmp_mutex`. - distributions.push({ name, version, declaredLicense: current.license, source: current.source }); - current = null; - }; - for (let index = start + 1; index < lines.length; index += 1) { - const line = lines[index]; - const entry = /^- (conda|pypi): (.+)$/.exec(line); - if (entry) { - flush(); - current = { source: entry[1], url: entry[2], name: null, version: null, license: null }; - continue; - } - if (!current) continue; - // A non-indented, non-empty line ends the packages section (defensive; it is normally last). - if (line !== '' && !line.startsWith(' ')) { flush(); break; } - const field = /^ {2}(\w[\w-]*): (.*)$/.exec(line); - if (!field) continue; - const [, key, value] = field; - if (key === 'license' && current.license === null) current.license = value.trim(); - else if (key === 'name' && current.name === null) current.name = value.trim(); - else if (key === 'version' && current.version === null) current.version = value.trim(); - } - flush(); - return distributions.sort((left, right) => - compareStableStrings(left.name, right.name) || compareStableStrings(left.version, right.version)); -} - -/** - * Builds the deterministic conda license audit bound to one pixi.lock and target. - * - * @param {{ lockBytes: Buffer, targetId: string, namespace?: string }} options - * @returns {{ schemaVersion: 2, kind: string, targetId: string, dependencyLockSha256: string, - * packages: LockedDistribution[] }} - * @throws {Error} when a locked package declares no licence - */ -export function createCondaDependencyLicenseAudit({ lockBytes, targetId, namespace = DEFAULT_DOCUMENT_NAMESPACE }) { - return { - schemaVersion: 2, - kind: `${namespace}.dependency-license-audit`, - targetId, - dependencyLockSha256: sha256(lockBytes), - packages: lockedCondaDistributions(lockBytes), - }; -} - -/** - * Ensures a reviewed conda audit still matches the current pixi.lock exactly. - * - * @param {unknown} reviewed the audit committed to the repository - * @param {ReturnType} actual - * @returns {ReturnType} `actual`, when they agree - * @throws {Error} when the lock no longer matches what was reviewed - */ -export function validateCondaDependencyLicenseAudit(reviewed, actual) { - if (reviewed?.schemaVersion !== 2 || reviewed.kind !== actual.kind) fail('reviewed conda audit contract is invalid'); - if (JSON.stringify(reviewed) !== JSON.stringify(actual)) { - fail('locked conda dependency licenses differ from the reviewed audit'); - } - return actual; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/parity.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/parity.mjs deleted file mode 100644 index 6a67d90..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/parity.mjs +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Accelerator parity: does this box compute the same thing on the GPU as on the CPU? - * - * That question sounds scientific but is a packaging question. It catches the failures this tool is - * responsible for — the wrong wheels solved in, a CPU-only build shipped as CUDA, a broken BLAS — - * and it catches them on the build machine rather than on a user's. - * - * The division of labour matters. Scrollcase owns the mechanism: run the declared check inside the - * box once per accelerator, compare the numbers, enforce the declared tolerances, report what was - * measured. The project owns the meaning: which input to feed the model, which tensor to read, and - * what closeness is acceptable for it. The tool never decides what is scientifically correct — it - * enforces a threshold its user wrote down. - */ - -import { fail } from './process.mjs'; - -/** Reads the array of numbers a parity check prints, rejecting anything else. */ -function readValues(output, accelerator) { - let parsed; - try { - parsed = JSON.parse(output); - } catch { - fail(`Parity check on ${accelerator} did not print JSON: ${String(output).trim().slice(0, 200)}`); - } - const values = Array.isArray(parsed) ? parsed : parsed?.values; - if (!Array.isArray(values) || values.length === 0) { - fail(`Parity check on ${accelerator} printed no "values" array.`); - } - if (!values.every((value) => typeof value === 'number' && Number.isFinite(value))) { - // A NaN or an infinity is the classic symptom of a broken accelerator build, so it is reported - // as such rather than being allowed to poison the comparison arithmetic below. - fail(`Parity check on ${accelerator} produced non-finite values.`); - } - return values; -} - -/** Compares two runs: largest absolute and relative difference, and cosine similarity. */ -export function compareValues(reference, candidate) { - if (reference.length !== candidate.length) { - fail(`Parity outputs differ in length: ${reference.length} vs ${candidate.length}.`); - } - let maximumAbsolute = 0; - let maximumRelative = 0; - let dot = 0; - let referenceNorm = 0; - let candidateNorm = 0; - for (const [index, expected] of reference.entries()) { - const actual = candidate[index]; - const absolute = Math.abs(actual - expected); - maximumAbsolute = Math.max(maximumAbsolute, absolute); - // Relative error is meaningless around zero, so it is only counted where the reference has - // magnitude; the absolute bound is what guards the near-zero entries. - if (Math.abs(expected) > 0) maximumRelative = Math.max(maximumRelative, absolute / Math.abs(expected)); - dot += expected * actual; - referenceNorm += expected * expected; - candidateNorm += actual * actual; - } - const norms = Math.sqrt(referenceNorm) * Math.sqrt(candidateNorm); - return { - maximumAbsoluteError: maximumAbsolute, - maximumRelativeError: maximumRelative, - cosineSimilarity: norms > 0 ? dot / norms : 1, - }; -} - -/** Reports which declared tolerance a measurement breaches, or null when it satisfies them all. */ -export function breachedTolerance(measured, tolerances) { - const { absolute = null, relative = null, minimumCosine = null } = tolerances ?? {}; - if (absolute !== null && measured.maximumAbsoluteError > absolute) { - return `maximum absolute error ${measured.maximumAbsoluteError} exceeds ${absolute}`; - } - if (relative !== null && measured.maximumRelativeError > relative) { - return `maximum relative error ${measured.maximumRelativeError} exceeds ${relative}`; - } - if (minimumCosine !== null && measured.cosineSimilarity < minimumCosine) { - return `cosine similarity ${measured.cosineSimilarity} is below ${minimumCosine}`; - } - return null; -} - -/** - * Runs the scroll's parity check across the declared accelerators and enforces its tolerances. - * - * The first accelerator listed is the reference every other run is compared against — conventionally - * `cpu`, because it is the one available everywhere and the least likely to be wrong. Returns the - * measurements so they can be recorded as evidence even when nothing failed. - */ -export async function checkParity({ parity, adapter, interpreter, payloadDir, run }) { - if (!parity) return null; - const { script, accelerators, tolerances } = parity; - if (!Array.isArray(accelerators) || accelerators.length < 2) { - fail('A parity check needs at least two accelerators to compare.'); - } - const runs = []; - for (const accelerator of accelerators) { - const environment = adapter.validationEnvironments[accelerator]; - if (!environment) { - fail(`Target ${adapter.id} defines no validation environment for accelerator ${accelerator}.`); - } - const output = run(interpreter, [script], { cwd: payloadDir, env: environment, capture: true }); - runs.push({ accelerator, values: readValues(output, accelerator) }); - } - const [reference, ...others] = runs; - const comparisons = others.map(({ accelerator, values }) => { - const measured = compareValues(reference.values, values); - const breach = breachedTolerance(measured, tolerances); - if (breach) fail(`Parity check ${reference.accelerator} vs ${accelerator}: ${breach}.`); - return { accelerator, reference: reference.accelerator, ...measured }; - }); - return { script, tolerances, valueCount: reference.values.length, comparisons }; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/pixi.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/pixi.mjs deleted file mode 100644 index f7f5b85..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/pixi.mjs +++ /dev/null @@ -1,437 +0,0 @@ -/** - * pixi + conda-forge builder helpers. - * - * This module owns the deterministic, side-effect-free pieces (tool discovery and exact argument - * vectors) so they can be unit-tested; the orchestration that actually installs and packs a - * prefix lives below in installAndPackPixiEnvironment, composed with an injected runner. - * - * Relocation model: build the env with pixi from the committed pixi.lock, pack it with - * conda-pack, and extract it into the box as `venv/`. conda-pack already rewrites the build - * prefix to a neutral placeholder, and a conda-forge prefix imports and runs from any location - * with **no activation environment and no relocation fixer** (proven cold on macOS and Windows, - * CPU + GPU). So conda-unpack is deliberately never run: doing so would bake the build machine's - * path into the shipped box. A box needs no relocation step at install time. - */ - -import { existsSync } from 'node:fs'; -import { chmod, copyFile, cp, mkdir, readFile, readdir, readlink, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; -import { dirname, join, relative, resolve, sep } from 'node:path'; -import * as tar from 'tar'; -import { resolvePayloadLinkTarget, targetCarriesLinks } from '../contract/links.mjs'; -import { compareStableStrings, safeRelativePath } from './filesystem.mjs'; -import { fail, runResult as defaultRunResult } from './process.mjs'; -import { repairPosixLaunchers } from './launchers.mjs'; -import { CONDA_PACK_VERSION, toolchainPaths } from './toolchain.mjs'; -import { getWorkspace } from './workspace.mjs'; - -/** - * Resolves a tool, highest precedence first: an explicit path, the environment override, a - * toolchain the project installed for itself, and finally the bare name on PATH. The project-local - * toolchain is looked up rather than configured so that `init --install-toolchain` is enough on its - * own: nothing has to be added to PATH for the next command to find what was just installed. - */ -function toolCandidate({ path, environmentVariable, toolchainKey, name }) { - if (path) return String(path); - const fromEnvironment = process.env[environmentVariable]; - if (fromEnvironment) return String(fromEnvironment); - let installed = null; - try { - installed = toolchainPaths(getWorkspace().toolchainDir)[toolchainKey]; - } catch { - // No resolvable workspace (an unusual cwd, a test): fall through to PATH. - } - return installed && existsSync(installed) ? installed : name; -} - -/** - * Verifies the pinned pixi is installed. `build` and `lock` must use the same pixi the scroll was - * pinned against, never whatever happens to be on PATH: a different resolver version can select - * different packages and silently change the box. - * `runResult` is injectable so a caller can drive discovery without a real pixi on PATH. - * - * @param {{ requiredVersion: string, path?: string | null, runResult?: typeof defaultRunResult }} options - * @returns {string} the executable to invoke - * @throws {Error} when pixi is absent or is not the pinned version - */ -export function findPixi({ requiredVersion, path = null, runResult = defaultRunResult }) { - const found = probePixi({ path, runResult }); - if (!found) { - fail(`pixi ${requiredVersion} is required. Install it from https://pixi.sh/, run \`scrollcase init --install-toolchain\`, or pass --pixi .`); - } - if (found.version !== requiredVersion) fail(`Scroll requires pixi ${requiredVersion}, found ${found.version}.`); - return found.path; -} - -/** - * Reports which pixi is available and at what version, without requiring a particular one. - * - * `findPixi` answers "is the pinned pixi here?"; this answers "is there a pixi at all?", which is - * what `init` needs before it can offer to install one. Returns null when nothing runs. - * - * @param {{ path?: string | null, runResult?: typeof defaultRunResult }} [options] - * @returns {{ path: string, version: string | null } | null} null when nothing runs - */ -export function probePixi({ path = null, runResult = defaultRunResult } = {}) { - const candidate = toolCandidate({ - path, - environmentVariable: 'SCROLLCASE_PIXI', - toolchainKey: 'pixi', - name: 'pixi', - }); - const result = runResult(candidate, ['--version'], { capture: true }); - if (result.error || result.status !== 0) return null; - // `pixi --version` prints "pixi 0.x.y"; the version is the second token. - return { path: candidate, version: String(result.stdout ?? '').trim().split(/\s+/)[1] ?? null }; -} - -/** - * Reports whether conda-pack is available, and where. Returns null when nothing runs. - * - * @param {{ path?: string | null, runResult?: typeof defaultRunResult }} [options] - * @returns {{ path: string } | null} null when nothing runs - */ -export function probeCondaPack({ path = null, runResult = defaultRunResult } = {}) { - const candidate = toolCandidate({ - path, - environmentVariable: 'SCROLLCASE_CONDA_PACK', - toolchainKey: 'condaPack', - name: 'conda-pack', - }); - const result = runResult(candidate, ['--help'], { capture: true }); - return result.error || result.status !== 0 ? null : { path: candidate }; -} - -/** - * `lock` — resolves a scroll's pixi.toml into its committed pixi.lock without installing anything. - * Run by a human when dependencies change; the lock is committed and reviewed, and `build` then - * only installs from it. The manifest itself pins the channels and the single target platform, so - * resolution is host-independent without any per-invocation platform flag. - * - * @param {string} manifestPath - * @returns {string[]} - */ -export function pixiLockArguments(manifestPath) { - return ['lock', '--manifest-path', manifestPath]; -} - -/** - * `build` install — materializes the env from the committed lock, never re-resolving. `--frozen` - * installs exactly the locked packages without touching or re-checking the lock, so what ships is - * byte-for-byte what was reviewed: install-from-lock, never-resolve. - * Lock freshness against the manifest is a separate CI `check` concern, not a build-time resolve. - * - * @param {string} manifestPath - * @returns {string[]} - */ -export function pixiInstallArguments(manifestPath) { - return ['install', '--manifest-path', manifestPath, '--frozen']; -} - -/** - * conda-pack arguments to pack an installed conda prefix into a relocatable tarball. The tarball - * is extracted into the box as `venv/`; the embedded conda-unpack fixer is deliberately removed - * rather than run (see installAndPackPixiEnvironment). - * - * @param {string} prefix - * @param {string} outputPath - * @returns {string[]} - */ -export function condaPackArguments(prefix, outputPath) { - return ['-p', prefix, '-o', outputPath, '--format', 'tar.gz']; -} - -/** - * Verifies conda-pack is available. Its `--version` is unreliable (prints 0.0.0), so we only - * confirm it runs; the exact version pin is recorded elsewhere (via the pixi global manifest). - * - * @param {{ path?: string | null, runResult?: typeof defaultRunResult }} [options] - * @returns {string} the executable to invoke - * @throws {Error} when conda-pack is absent - */ -export function findCondaPack({ path = null, runResult = defaultRunResult } = {}) { - const found = probeCondaPack({ path, runResult }); - if (!found) { - fail(`conda-pack ${CONDA_PACK_VERSION} is required. Install it with \`scrollcase init --install-toolchain\` or \`pixi global install "conda-pack==${CONDA_PACK_VERSION}"\`, or pass --conda-pack .`); - } - return found.path; -} - -/** - * The only fields a box keeps from conda's per-package records: which exact binary this is, and - * what it is licensed under. Everything else is dropped. - * - * `build` earns its place because name and version do not identify a conda binary — one version is - * published in many builds, and a CPU and a CUDA build of the same library can differ in nothing - * else. All four are properties of the package as published rather than of the install that placed - * it, which is what makes them stable across rebuilds: `license` here was measured equal to the - * lock's declared licence for every package, and the lock is already the source the shipped licence - * inventory is derived from. A package declaring no licence simply has no such field, which is - * equally a property of the package and not of the run. - * - * The dependency graph is deliberately not here: nothing resolves dependencies inside a box, and - * the lock records them where they can actually be acted on. - */ -const CONDA_RECORD_FIELDS = Object.freeze([ - 'name', - 'version', - 'build', - 'license', -]); - -/** - * Rewrites `conda-meta/` into a canonical, build-independent form. - * - * These records are written by the installer, not by the package, and two installs of the identical - * lock do not produce identical ones: a per-file `sha256_in_prefix` is recorded on one run and not - * the next. They also carry absolute paths into the build machine's package cache. The first breaks - * the promise the whole trust chain rests on — rebuild a commit, get the same bytes — and the second - * ships a developer's directory layout to users, which is the very leak conda-unpack is refused - * over. Anything that is not a record (conda's `history` log) goes entirely. - * - * Nothing in a box reads any of this: conda is never shipped inside one, and package versions stay - * readable from `site-packages` where a Python tool actually looks. So the kept fields are copied - * verbatim and the rule is an allowlist rather than a list of known-volatile fields — deliberately, - * because a field pixi starts writing in a later release then cannot reintroduce the drift. It was - * never eligible to be written in the first place. - */ -async function canonicalizeCondaRecords(venvDir) { - const metaDir = join(venvDir, 'conda-meta'); - if (!existsSync(metaDir)) return; - for (const entry of (await readdir(metaDir)).sort(compareStableStrings)) { - const entryPath = join(metaDir, entry); - if (!entry.endsWith('.json')) { - await rm(entryPath, { recursive: true, force: true }); - continue; - } - let record; - try { - record = JSON.parse(await readFile(entryPath, 'utf8')); - } catch (error) { - return fail(`Unreadable conda package record ${entry}: ${error instanceof Error ? error.message : String(error)}`); - } - const canonical = {}; - for (const field of CONDA_RECORD_FIELDS) { - if (record[field] !== undefined) canonical[field] = record[field]; - } - await writeFile(entryPath, `${JSON.stringify(canonical, null, 2)}\n`); - } -} - -/** - * Settles every symbolic link under `root`: kept when the payload may carry it, materialized into - * real content when it may not, dropped when it points nowhere useful. - * - * A conda prefix is dense with links, and materializing all of them was expensive in a way nobody - * had measured: on Linux the soname convention alone (`libfoo.so` → `.so.N` → `.so.N.M`) meant - * roughly 60% of an extracted box was duplicates of its own bytes. What may be kept is decided by - * `src/contract/links.mjs`, not here — this function only supplies the filesystem facts that rule - * needs and applies its answer. - * - * Two conditions are checked against the disk rather than the path string, because only the disk - * knows them: that the link resolves inside the prefix even after passing through other links, and - * that what it ends at is a regular file. A link to a directory is materialized, which is what - * keeps anything from ever being written *through* a link. - * - * @param {string} root - * @param {boolean} keepLinks whether this target can extract links at all - * @param {string} [current] - * @returns {Promise} - */ -async function settleSymlinksInPlace(root, keepLinks, current = root) { - const canonicalRoot = await realpath(root); - // Sorted, because whether a link is kept can depend on what an earlier entry became, and - // readdir order is the filesystem's business. Two builds must settle the tree identically. - const children = (await readdir(current, { withFileTypes: true })) - .sort((left, right) => compareStableStrings(left.name, right.name)); - for (const entry of children) { - const path = join(current, entry.name); - if (entry.isSymbolicLink()) { - let target; - try { - target = await realpath(path); - } catch { - await rm(path, { force: true }); // dangling link - continue; - } - const insideTree = target === canonicalRoot - || target.startsWith(`${canonicalRoot}${sep}`); - let info; - try { - info = await stat(target); - } catch { - await rm(path, { force: true }); - continue; - } - if (!insideTree) { - // A link escaping the prefix would drag a host file into the box; drop it instead. - await rm(path, { force: true }); - continue; - } - if (keepLinks && !info.isDirectory() && await keepsAsLink(root, path, canonicalRoot)) continue; - await rm(path, { force: true }); - if (info.isDirectory()) { - await cp(target, path, { recursive: true, dereference: true }); - await settleSymlinksInPlace(root, keepLinks, path); - } else { - await copyFile(target, path); - await chmod(path, info.mode & 0o777); - } - } else if (entry.isDirectory()) { - await settleSymlinksInPlace(root, keepLinks, path); - } - } -} - -/** - * Whether one link satisfies the payload rule, judged against the tree it actually sits in. - * - * The raw target is what gets archived, so it is what must be checked: an absolute target names the - * build machine and is exactly what relocation exists to erase, and a relative one has to land - * inside the prefix both lexically and after the filesystem has followed it. - * - * @param {string} root - * @param {string} linkPath - * @param {string} canonicalRoot - * @returns {Promise} - */ -async function keepsAsLink(root, linkPath, canonicalRoot) { - const rawTarget = (await readlink(linkPath)).split(sep).join('/'); - const relativeLink = relative(root, linkPath).split(sep).join('/'); - const resolved = resolvePayloadLinkTarget(relativeLink, rawTarget); - if (resolved === null) return false; - // The lexical answer and the filesystem's answer must agree. They can differ when the target is - // reached through another link, which is precisely the case a purely lexical check cannot see. - const lexicalPath = join(root, ...resolved.split('/')); - let lexicalReal; - try { - lexicalReal = await realpath(lexicalPath); - } catch { - return false; - } - if (lexicalReal !== canonicalRoot && !lexicalReal.startsWith(`${canonicalRoot}${sep}`)) return false; - return (await stat(lexicalPath)).isFile(); -} - -/** - * Builds the box's `venv/` prefix from a scroll's committed pixi.lock and packs it for relocation. - * - * Flow: install the exact locked env into an isolated workspace so pixi's `.pixi/envs` never - * lands in the tracked scroll dir; conda-pack the prefix into a relocatable tarball; extract it - * into `payloadDir/venv`; remove the service files that carry the build prefix (conda-unpack is - * never run — see below); then dereference every symlink so the payload is link-free for the - * archive layer. The multi-gigabyte workspace and tarball are removed before the payload is - * archived. - * - * `run` is injected so this composes with the orchestrator's logging and error model. - * - * @param {{ - * pixi: string, - * condaPack: string, - * manifestPath: string, - * lockPath: string, - * buildDir: string, - * payloadDir: string, - * adapter: import('../contract/targets.mjs').BoxTargetAdapter, - * run: typeof import('./process.mjs').run, - * }} options - * @returns {Promise<{ interpreter: string, prefix: string }>} - */ -export async function installAndPackPixiEnvironment({ - pixi, - condaPack, - manifestPath, - lockPath, - buildDir, - payloadDir, - adapter, - run, -}) { - const workspace = join(buildDir, 'pixi-workspace'); - await rm(workspace, { recursive: true, force: true }); - await mkdir(workspace, { recursive: true }); - // pixi installs from the manifest+lock sitting next to each other; stage both into the workspace - // so the resulting `.pixi/envs/default` prefix is build-local, never inside the scroll. - await copyFile(manifestPath, join(workspace, 'pixi.toml')); - await copyFile(lockPath, join(workspace, 'pixi.lock')); - run(pixi, pixiInstallArguments(join(workspace, 'pixi.toml'))); - const prefix = join(workspace, '.pixi', 'envs', 'default'); - - const packPath = join(buildDir, 'pixi-env.tar.gz'); - await rm(packPath, { force: true }); - run(condaPack, condaPackArguments(prefix, packPath)); - - const venvDir = join(payloadDir, 'venv'); - await rm(venvDir, { recursive: true, force: true }); - await mkdir(venvDir, { recursive: true }); - // conda-pack emits the prefix contents at the tar root, so extracting into `venv` yields the - // conda layout (bin/, lib/, conda-meta/) directly under it. Use the pinned Node implementation - // rather than a host `tar`: builds then have exactly the dependencies `doctor` reports, and the - // archive behaves the same on macOS, Linux and Windows. Symlinks are expected in a conda prefix - // and are deliberately handled by dereferenceSymlinksInPlace immediately below. - // - // They cannot, however, be created *during* extraction. The extractor refuses a link whose target - // leaves the tree, and refuses one whose target passes through another link — both are defences - // against writing file content through a link, and neither is negotiable. A conda prefix trips - // the second routinely: icu ships `current -> ` and then `pkgdata.inc -> - // current/pkgdata.inc`, which arrives in a plain `python` environment that never asked for icu, - // and made the whole box unbuildable. - // - // So links are extracted in a second pass, once every regular entry is already on disk and there - // is nothing left that could be written through one. Creating a link is not traversing it: the - // targets are resolved and checked immediately below, where anything leaving the tree is dropped. - const deferredLinks = []; - await tar.x({ - file: packPath, - cwd: venvDir, - gzip: true, - preservePaths: false, - strict: true, - filter: (entryPath, entry) => { - if (entry.type !== 'SymbolicLink') return true; - deferredLinks.push({ path: safeRelativePath(entryPath), target: String(entry.linkpath) }); - return false; - }, - }); - // Sorted so the tree is built the same way whatever order the tar happened to list them in. - for (const link of deferredLinks.sort((left, right) => compareStableStrings(left.path, right.path))) { - const linkPath = join(venvDir, ...link.path.split('/')); - // A regular entry already holding the path wins: content beats an alias to it. - if (existsSync(linkPath)) continue; - await mkdir(dirname(linkPath), { recursive: true }); - // The type argument is inert on POSIX and decides junction-vs-file on Windows, where a conda - // prefix carries no links at all — so a target that is not yet a directory is simply a file. - const resolved = resolve(dirname(linkPath), link.target); - const type = existsSync(resolved) && (await stat(resolved)).isDirectory() ? 'dir' : 'file'; - await symlink(link.target, linkPath, type); - } - - const interpreter = join(payloadDir, ...adapter.python.entryPoint.split('/')); - // Deliberately do NOT run conda-unpack. conda-pack already replaces the build prefix with a - // neutral placeholder, and the box imports and runs fine that way (a cold import from a moved - // prefix was proven before any fixer). Running the fixer here would stamp the *build machine's* - // absolute path into dozens of files that then ship to users — measured on a probe env: 0 files - // carry the prefix before, 36 after — leaking a developer path while still being wrong at the - // user's install location. Instead drop the few service files that do carry the build prefix. - for (const servicePath of [ - ['conda-meta', 'pixi_env_prefix'], - ['conda-meta', 'pixi'], - ['bin', 'conda-unpack'], - ['Scripts', 'conda-unpack.exe'], - ['Scripts', 'conda-unpack-script.py'], - ]) { - await rm(join(venvDir, ...servicePath), { force: true }); - } - // The rest of conda-meta carries the same two problems in a less obvious form; see above. - await canonicalizeCondaRecords(venvDir); - // Order matters: settle the links first, so the launcher repair that follows walks a tree whose - // shape is final and rewrites each script's bytes exactly once, under its own name. - await settleSymlinksInPlace(venvDir, targetCarriesLinks(adapter.platform)); - // conda console scripts (tqdm, isympy, …) embed the absolute build interpreter in a shell - // trampoline shebang. Rewrite them to resolve Python next to themselves, so no build path - // ships inside the box. - await repairPosixLaunchers(adapter, payloadDir, [prefix, workspace, payloadDir]); - - await rm(workspace, { recursive: true, force: true }); - await rm(packPath, { force: true }); - return { interpreter, venvDir, sitePackagesRelative: relative(payloadDir, venvDir) }; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/process.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/process.mjs deleted file mode 100644 index 0c9e560..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/process.mjs +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Error and subprocess primitives for the whole tool. - * - * Every validation failure funnels through `fail`, and every external command through `run` / - * `runResult` — which is also the seam the tests use: injecting a fake runner is how the pipeline - * suite builds boxes without pixi or conda-pack installed. - */ -import { spawnSync } from 'node:child_process'; - -/** - * Subprocess options shared by the library surface and its injected test seams. - * - * @typedef {object} RunOptions - * @property {string} [cwd] - * @property {NodeJS.ProcessEnv} [env] - * @property {string | Uint8Array} [input] - * @property {number} [maxBuffer] - * @property {boolean} [capture] - */ - -/** - * Throws a consistent CLI error from validation helpers. - * - * @param {unknown} message - * @returns {never} - */ -export function fail(message) { - throw new Error(message); -} - -/** - * Runs a subprocess without interpreting its result. - * - * @param {string} command - * @param {readonly string[]} args - * @param {RunOptions} [options] - * @returns {import('node:child_process').SpawnSyncReturns} - */ -export function runResult(command, args, options = {}) { - return spawnSync(command, args, { - cwd: options.cwd, - env: { ...process.env, ...options.env }, - encoding: 'utf8', - input: options.input, - maxBuffer: options.maxBuffer ?? 64 * 1024 * 1024, - stdio: options.capture ? 'pipe' : ['pipe', 'inherit', 'inherit'], - }); -} - -/** - * Runs a subprocess and throws when it cannot start or exits unsuccessfully. - * - * @param {string} command - * @param {readonly string[]} args - * @param {RunOptions} [options] - * @returns {string} - */ -export function run(command, args, options = {}) { - const result = runResult(command, args, options); - if (result.error) fail(`${command} failed to start: ${result.error.message}`); - if (result.status !== 0) { - const detail = options.capture ? `\n${result.stderr || result.stdout}` : ''; - fail(`${command} exited with status ${result.status}${detail}`); - } - return (result.stdout ?? '').trim(); -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/project.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/project.mjs deleted file mode 100644 index 5770678..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/project.mjs +++ /dev/null @@ -1,276 +0,0 @@ -/** - * Setting a project up, and telling it what is wrong. - * - * `initProject` scaffolds only the workspace; the CLI may compose it with the explicitly - * disposable example used for onboarding. `doctor` inspects and never writes. Real scroll - * authoring remains a separate operation because a workspace may carry many boxes and targets. - * - * `init` may also install the build toolchain, but only after asking: scaffolding never reaches for - * the network on its own, and the download is verified against a pinned checksum. See - * `ensureToolchain` below and `toolchain.mjs` for why the consent and the pin are the design rather - * than a nicety. - */ - -import { readFile, mkdir, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; -import { fileExists } from './filesystem.mjs'; -import { findCondaPack, findPixi, probeCondaPack, probePixi } from './pixi.mjs'; -import { fail, run as defaultRun, runResult as defaultRunResult } from './process.mjs'; -import { - CONDA_PACK_VERSION, - installCondaPack, - installPixi, - latestPixiVersion, - pixiReleaseAsset, -} from './toolchain.mjs'; -import { DEFAULT_WORKSPACE_PATHS, SCROLLCASE_CONFIG_FILENAME } from './workspace.mjs'; - -// Written into a project's .gitignore and matched on re-run to stay idempotent. Changing the text -// makes an already-scaffolded project look unmarked and append the rules a second time. -const GITIGNORE_MARKER = '# scrollcase build state'; - -const PROJECT_GUIDE = `[Scrollcase documentation](https://scrollcase.dev/) - -# Scrollcase in this project - -Scrollcase turns a declarative [scroll](https://scrollcase.dev/reference/scroll) into a signed, -portable [box](https://scrollcase.dev/reference/box-format) for one [target](https://scrollcase.dev/reference/box-format#targets). - -## Usual workflow - -Run \`npm install scrollcase\` to install Scrollcase CLI. Then: - -1. \`scrollcase init\` -2. \`scrollcase lock /\` -3. \`scrollcase keygen\` -4. \`scrollcase build /\` -5. \`scrollcase verify --self-test\` or \`scrollcase run \` - -See the [CLI reference](https://scrollcase.dev/reference/cli) and -[signing guidance](https://scrollcase.dev/guides/signing-and-custody). The \`consumer-templates/\` -files demonstrate the [consumer APIs](https://scrollcase.dev/reference/api) against local releases. - -## Node consumer - -\`\`\`sh -npm install scrollcase -npm install --save-dev tsx typescript -npx tsx consumer-templates/run-box.ts -\`\`\` - -## Python consumer - -npm does not install the Python consumer. A Python-only application does not need the Node CLI: - -\`\`\`sh -python -m pip install scrollcase-consumer -python consumer-templates/run_box.py -\`\`\` - -[Scrollcase documentation](https://scrollcase.dev/) -`; - -/** - * Scaffolds a workspace config, concise project guide, scroll root, and generated-state ignores. - * - * This low-level primitive deliberately creates no scroll. Existing files are never overwritten, - * so a half-configured workspace can be completed by running the command again without changing - * authored inputs. - */ -export async function initProject({ - root, - scrollsDir = join(root, DEFAULT_WORKSPACE_PATHS.scrolls), -}) { - const written = []; - const skipped = []; - const write = async (path, contents) => { - if (await fileExists(path)) return skipped.push(path); - await mkdir(join(path, '..'), { recursive: true }); - await writeFile(path, contents); - return written.push(path); - }; - - await write(join(root, SCROLLCASE_CONFIG_FILENAME), `${JSON.stringify({ - version: 1, - paths: { ...DEFAULT_WORKSPACE_PATHS }, - }, null, 2)}\n`); - await write(join(root, 'SCROLLCASE.md'), PROJECT_GUIDE); - - if (await fileExists(scrollsDir)) skipped.push(scrollsDir); - else { - await mkdir(scrollsDir, { recursive: true }); - written.push(scrollsDir); - } - - // Build state is regenerated on every build and must never be committed; the lock and the scroll - // must be. Appending rather than rewriting leaves an existing .gitignore alone. - const gitignorePath = join(root, '.gitignore'); - const existing = await fileExists(gitignorePath) ? await readFile(gitignorePath, 'utf8') : ''; - if (!existing.includes(GITIGNORE_MARKER)) { - const rules = `${existing.endsWith('\n') || existing === '' ? '' : '\n'}${GITIGNORE_MARKER}\n.scrollcase/\n`; - await writeFile(gitignorePath, `${existing}${rules}`); - written.push(gitignorePath); - } else { - skipped.push(gitignorePath); - } - return { - written, - skipped, - root, - scrollsDir, - }; -} - -/** Reads the project config back, so a toolchain pin is added to it rather than replacing it. */ -async function readConfig(configPath) { - if (!await fileExists(configPath)) return { version: 1, paths: { ...DEFAULT_WORKSPACE_PATHS } }; - return JSON.parse(await readFile(configPath, 'utf8')); -} - -/** - * Installs the build toolchain into the project, if it is missing and only if allowed. - * - * `confirm` is the consent, injected rather than assumed: the CLI asks a human, a scripted setup - * passes a flag, and CI without a terminal answers no. Nothing is downloaded before it returns - * true, which is what keeps `init` a command that is always safe to run. - * - * The pixi version is the caller's requested pin, the installed pixi's version when one is already - * present, and otherwise the newest release. Managed installs record that choice and the verified - * archive digest in the workspace config; each scroll separately declares which resolver it uses. - * - * The archive's verified digest and managed conda-pack version are recorded under `toolchain` in - * the project config. The first pixi install trusts the checksum published beside the release; - * every later one is checked against the value the project committed, so a teammate or a CI runner - * cannot silently receive different bytes. - */ -export async function ensureToolchain({ - workspace, - pixiVersion = null, - confirm, - host = process, - fetchImpl = fetch, - run = defaultRun, - runResult = defaultRunResult, - log = console.log, -}) { - const discoveredPixi = probePixi({ runResult }); - // A present but different pixi is still missing for this project: resolver versions are part of - // the scroll's reproducibility contract, so `init --pixi-version` must install what it promises. - const pixi = discoveredPixi && (!pixiVersion || discoveredPixi.version === pixiVersion) - ? discoveredPixi - : null; - const condaPack = probeCondaPack({ runResult }); - const missing = [!pixi && 'pixi', !condaPack && 'conda-pack'].filter(Boolean); - if (missing.length === 0) { - return { - installed: [], - missing: [], - pixiVersion: pixi.version, - condaPackVersion: CONDA_PACK_VERSION, - declined: false, - }; - } - if (!pixiReleaseAsset(host)) { - return { installed: [], missing, declined: false, unsupportedHost: `${host.platform}/${host.arch}` }; - } - if (!await confirm(missing)) return { installed: [], missing, declined: true }; - - const configPath = join(workspace.root, SCROLLCASE_CONFIG_FILENAME); - const config = await readConfig(configPath); - const installed = []; - let pixiPath = pixi?.path ?? null; - let version = pixiVersion ?? pixi?.version ?? null; - - if (!pixi) { - if (!version) { - version = await latestPixiVersion({ fetchImpl }); - log(`Newest pixi release is ${version}; recording it for the workspace toolchain.`); - } - const pinned = config.toolchain?.pixi?.version === version - ? config.toolchain?.pixi?.assets?.[pixiReleaseAsset(host).asset] ?? null - : null; - const result = await installPixi({ - version, - toolchainDir: workspace.toolchainDir, - expectedSha256: pinned, - host, - fetchImpl, - log, - }); - pixiPath = result.path; - installed.push(`pixi ${version}`); - // Record the digest that was actually verified, so the next machine checks against it. - config.toolchain = { - ...config.toolchain, - pixi: { - version, - assets: { ...config.toolchain?.pixi?.assets, [result.asset]: result.sha256 }, - }, - }; - await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`); - } - - if (!condaPack) { - if (!pixiPath) fail('conda-pack is installed with pixi, but no pixi is available.'); - await installCondaPack({ pixi: pixiPath, toolchainDir: workspace.toolchainDir, run, log }); - installed.push(`conda-pack ${CONDA_PACK_VERSION}`); - config.toolchain = { - ...config.toolchain, - condaPack: { version: CONDA_PACK_VERSION }, - }; - await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`); - } - - return { - installed, - missing: [], - declined: false, - pixiVersion: version, - condaPackVersion: CONDA_PACK_VERSION, - configPath, - }; -} - -/** - * Diagnoses whether this machine can build a box, and says what to do when it cannot. - * - * Every check reports rather than throws, so one missing tool does not hide the next problem: a - * user with neither pixi nor conda-pack should learn both in one run, not one per attempt. - */ -export async function diagnose({ workspace, pixiVersion = null, pixiPath = null, condaPackPath = null, runResult = defaultRunResult }) { - const checks = []; - const record = (name, ok, detail, remedy = null) => checks.push({ name, ok, detail, remedy }); - - record('workspace', true, workspace.configPath - ? `config ${workspace.configPath}` - : `no ${SCROLLCASE_CONFIG_FILENAME} found; using defaults under ${workspace.root}`); - record('scrolls', await fileExists(workspace.scrollsDir), workspace.scrollsDir, - `Create it, or point "paths.scrolls" at where your scrolls live.`); - - const git = runResult('git', ['rev-parse', 'HEAD'], { capture: true, cwd: workspace.root }); - record('git', git.status === 0, - git.status === 0 ? `HEAD ${git.stdout.trim().slice(0, 12)}` : 'not a git checkout', - 'A box records the commit it was built from. Initialise a repository and commit your scrolls.'); - - if (pixiVersion) { - try { - const pixi = findPixi({ requiredVersion: pixiVersion, path: pixiPath, runResult }); - record('pixi', true, `${pixi} at ${pixiVersion}`); - } catch (error) { - record('pixi', false, error.message, - `Install pixi ${pixiVersion} from https://pixi.sh/, or pass --pixi .`); - } - } else { - record('pixi', true, 'not checked: pass --pixi-version, or run doctor with a scroll'); - } - - try { - const condaPack = findCondaPack({ path: condaPackPath, runResult }); - record('conda-pack', true, condaPack); - } catch (error) { - record('conda-pack', false, error.message, - `Install conda-pack ${CONDA_PACK_VERSION} with \`scrollcase init --install-toolchain\`, or pass --conda-pack .`); - } - - return { checks, ok: checks.every((check) => check.ok) }; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/schema-validation.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/schema-validation.mjs deleted file mode 100644 index 5821cd3..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/schema-validation.mjs +++ /dev/null @@ -1,194 +0,0 @@ -/** - * Runtime validation for the JSON Schemas Scrollcase ships. - * - * Ajv remains a development dependency because adding a fourth runtime package would widen the - * installed surface for one narrow job. This validator implements the 2020-12 keywords used by the - * scroll and target schemas, including local and absolute references and target conditionals. The - * schemas remain the source of truth; this module deliberately contains no scroll field list. - */ - -import { isDeepStrictEqual } from 'node:util'; - -const own = (value, key) => Object.prototype.hasOwnProperty.call(value, key); -const objectValue = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); - -function valueType(value) { - if (Array.isArray(value)) return 'array'; - if (value === null) return 'null'; - if (Number.isInteger(value)) return 'integer'; - return typeof value; -} - -function matchesType(value, expected) { - if (expected === 'object') return objectValue(value); - if (expected === 'array') return Array.isArray(value); - if (expected === 'integer') return Number.isInteger(value); - if (expected === 'number') return typeof value === 'number' && Number.isFinite(value); - return typeof value === expected; -} - -function decodePointerPart(part) { - return decodeURIComponent(part).replaceAll('~1', '/').replaceAll('~0', '~'); -} - -function resolveReference(reference, rootSchema, registry) { - const [id, fragment = ''] = reference.split('#', 2); - const document = id ? registry.get(id) : rootSchema; - if (!document) throw new Error(`Schema reference is not registered: ${reference}`); - if (!fragment) return { schema: document, rootSchema: document }; - if (!fragment.startsWith('/')) throw new Error(`Unsupported schema fragment: #${fragment}`); - const schema = fragment.slice(1).split('/').map(decodePointerPart) - .reduce((current, part) => current?.[part], document); - if (!schema) throw new Error(`Schema reference does not resolve: ${reference}`); - return { schema, rootSchema: document }; -} - -function validate(value, schema, context, path, errors) { - if (schema.$ref) { - const resolved = resolveReference(schema.$ref, context.rootSchema, context.registry); - validate(value, resolved.schema, { ...context, rootSchema: resolved.rootSchema }, path, errors); - if (errors.length > 0) return; - } - - if (schema.const !== undefined && !isDeepStrictEqual(value, schema.const)) { - errors.push(`${path} must equal ${JSON.stringify(schema.const)}`); - return; - } - if (schema.enum && !schema.enum.some((candidate) => isDeepStrictEqual(value, candidate))) { - errors.push(`${path} must be one of ${schema.enum.map((item) => JSON.stringify(item)).join(', ')}`); - return; - } - if (schema.type && !matchesType(value, schema.type)) { - errors.push(`${path} must be ${schema.type}, received ${valueType(value)}`); - return; - } - - if (typeof value === 'string') { - if (schema.minLength !== undefined && value.length < schema.minLength) { - errors.push(`${path} must contain at least ${schema.minLength} character${schema.minLength === 1 ? '' : 's'}`); - return; - } - if (schema.pattern && !new RegExp(schema.pattern, 'u').test(value)) { - errors.push(`${path} does not match the required pattern`); - return; - } - } - - if (typeof value === 'number') { - if (!Number.isFinite(value)) { - errors.push(`${path} must be finite`); - return; - } - if (schema.minimum !== undefined && value < schema.minimum) { - errors.push(`${path} must be at least ${schema.minimum}`); - return; - } - if (schema.exclusiveMinimum !== undefined && value <= schema.exclusiveMinimum) { - errors.push(`${path} must be greater than ${schema.exclusiveMinimum}`); - return; - } - if (schema.maximum !== undefined && value > schema.maximum) { - errors.push(`${path} must be at most ${schema.maximum}`); - return; - } - } - - if (Array.isArray(value)) { - if (schema.minItems !== undefined && value.length < schema.minItems) { - errors.push(`${path} must contain at least ${schema.minItems} item${schema.minItems === 1 ? '' : 's'}`); - return; - } - if (schema.items) { - for (let index = 0; index < value.length && errors.length === 0; index += 1) { - validate(value[index], schema.items, context, `${path}[${index}]`, errors); - } - } - } - - if (objectValue(value)) { - if (schema.minProperties !== undefined && Object.keys(value).length < schema.minProperties) { - errors.push(`${path} must contain at least ${schema.minProperties} property`); - return; - } - for (const required of schema.required ?? []) { - if (!own(value, required)) { - errors.push(`${path}.${required} is required`); - return; - } - } - for (const [key, propertySchema] of Object.entries(schema.properties ?? {})) { - if (own(value, key)) validate(value[key], propertySchema, context, `${path}.${key}`, errors); - if (errors.length > 0) return; - } - if (schema.additionalProperties === false) { - const allowed = new Set(Object.keys(schema.properties ?? {})); - const unexpected = Object.keys(value).find((key) => !allowed.has(key)); - if (unexpected) { - errors.push(`${path}.${unexpected} is not allowed`); - return; - } - } - for (const [key, dependencies] of Object.entries(schema.dependentRequired ?? {})) { - if (!own(value, key)) continue; - const missing = dependencies.find((dependency) => !own(value, dependency)); - if (missing) { - errors.push(`${path}.${missing} is required when ${key} is present`); - return; - } - } - } - - for (const branch of schema.allOf ?? []) { - validate(value, branch, context, path, errors); - if (errors.length > 0) return; - } - - if (schema.oneOf) { - const results = schema.oneOf.map((branch) => { - const branchErrors = []; - validate(value, branch, context, path, branchErrors); - return branchErrors; - }); - const matches = results.filter((branchErrors) => branchErrors.length === 0); - if (matches.length === 0) { - const closest = results.reduce((best, candidate) => - candidate.length < best.length ? candidate : best); - errors.push(closest[0] ?? `${path} does not match an allowed shape`); - return; - } - if (matches.length > 1) { - errors.push(`${path} must match exactly one allowed shape`); - return; - } - } - - if (schema.if) { - const conditionErrors = []; - validate(value, schema.if, context, path, conditionErrors); - const branch = conditionErrors.length === 0 ? schema.then : schema.else; - if (branch) validate(value, branch, context, path, errors); - } - - if (schema.not) { - const forbiddenErrors = []; - validate(value, schema.not, context, path, forbiddenErrors); - if (forbiddenErrors.length === 0) errors.push(`${path} matches a forbidden shape`); - } -} - -/** - * Returns the first structural disagreement with a schema, or null when the value matches. - * - * @param {unknown} value - * @param {object} schema - * @param {object[]} [relatedSchemas] - * @returns {string | null} - */ -export function schemaValidationError(value, schema, relatedSchemas = []) { - const registry = new Map([schema, ...relatedSchemas] - .filter((candidate) => candidate.$id) - .map((candidate) => [candidate.$id, candidate])); - const errors = []; - validate(value, schema, { registry, rootSchema: schema }, '$', errors); - return errors[0] ?? null; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/scroll.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/scroll.mjs deleted file mode 100644 index 4a1ece7..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/scroll.mjs +++ /dev/null @@ -1,200 +0,0 @@ -/** - * Reading a scroll, and the provenance of the build that reads it. - * - * A scroll is the only input a build accepts, so it is validated before anything is installed. In - * the nested layout, the meaningful declarations police the path: `boxId` names the parent and the - * canonical target names the child. Python layout is checked against the target before the scroll - * reaches any tool discovery or build mutation. - */ - -import { readFile, readdir } from 'node:fs/promises'; -import { join, resolve, sep } from 'node:path'; -import { assertPythonEntryPoint, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; -import { compareStableStrings, fileExists, safeRelativePath } from './filesystem.mjs'; -import { fail, runResult } from './process.mjs'; -import { schemaValidationError } from './schema-validation.mjs'; -import { getWorkspace } from './workspace.mjs'; - -const scrollSchemaUrl = new URL('../contract/schema/scroll.schema.json', import.meta.url); -const targetSchemaUrl = new URL('../contract/schema/target.schema.json', import.meta.url); -const executionSchemaUrl = new URL('../contract/schema/execution.schema.json', import.meta.url); -let scrollSchemas; - -async function loadScrollSchemas() { - scrollSchemas ??= Promise.all([scrollSchemaUrl, targetSchemaUrl, executionSchemaUrl] - .map(async (url) => JSON.parse(await readFile(url, 'utf8')))); - return scrollSchemas; -} - -/** Resolves an exact scroll reference to its directory, refusing anything outside the scrolls root. */ -export function scrollDirectory(reference) { - const root = getWorkspace().scrollsDir; - const normalized = safeRelativePath(reference); - const path = resolve(root, ...normalized.split('/')); - if (path === root || !path.startsWith(`${root}${sep}`)) fail(`Invalid scroll: ${reference}`); - return path; -} - -/** Loads one exact nested scroll reference and normalises its provenance identity. */ -async function readExactScroll(reference) { - const normalized = safeRelativePath(reference); - const parts = normalized.split('/'); - if (parts.length !== 2) fail(`Invalid scroll reference ${reference}; use /.`); - const dir = scrollDirectory(normalized); - const scroll = JSON.parse(await readFile(resolve(dir, 'scroll.json'), 'utf8')); - const [scrollSchema, targetSchema, executionSchema] = await loadScrollSchemas(); - const validationError = schemaValidationError(scroll, scrollSchema, [targetSchema, executionSchema]); - if (validationError) fail(`Invalid scroll ${normalized}: ${validationError}.`); - if (scroll.weights === 'on-demand' && (scroll.assetArchives ?? []).length > 0) { - fail('on-demand weights cannot be combined with assetArchives, which are expanded at build time.'); - } - const payloadPaths = [ - scroll.modelCacheSubdir, - ...scroll.assets.map((asset) => asset.relativePath), - ...(scroll.assetArchives ?? []).flatMap((archive) => [archive.relativePath, archive.destination]), - ...(scroll.localFiles ?? []).flatMap((file) => [file.sourcePath, file.relativePath]), - ...(scroll.prunePaths ?? []), - ...scroll.selfTest.files, - ...(scroll.execution?.kind === 'python-script' ? [scroll.execution.script] : []), - ...(scroll.parity ? [scroll.parity.script] : []), - ...(scroll.condaDependencyLicenseAudit ? [scroll.condaDependencyLicenseAudit] : []), - ]; - for (const path of payloadPaths) safeRelativePath(path); - const adapter = boxTargetAdapter(scroll.target); - const targetId = boxTargetId(scroll.target); - if (parts.length === 2) { - const [boxDirectory, targetDirectory] = parts; - if (boxDirectory !== scroll.boxId) { - fail(`Nested scroll box directory ${boxDirectory} does not match scroll boxId ${scroll.boxId}.`); - } - if (targetDirectory !== targetId) { - fail(`Nested scroll target directory ${targetDirectory} does not match declared target ${targetId}.`); - } - } - assertPythonEntryPoint(adapter, scroll.pythonEntryPoint); - return { - adapter, - dir, - scroll: { - ...scroll, - // Provenance needs a stable source identity. It is derived when the scroll does not name one, - // so the directory layout remains checked context rather than a second wire identity. - scrollId: scroll.scrollId ?? `${scroll.boxId}-${targetId}`, - }, - reference: normalized, - targetId, - }; -} - -/** - * Lists the scrolls named by a CLI/library reference. - * - * An exact `/` reference loads one scroll. A single box name expands to its - * `scrolls///` children. Omitting the name discovers every nested scroll in the - * workspace for CLI selection. Every child is validated before it is offered, so a misleading - * directory never becomes a selectable target. - */ -export async function scrollCandidates(name = null) { - if (name === null || name === undefined) { - let boxes; - try { - boxes = await readdir(getWorkspace().scrollsDir, { withFileTypes: true }); - } catch { - return fail('No scrolls found; run scrollcase init or scrollcase new scroll.'); - } - const candidates = []; - for (const box of boxes.sort((left, right) => compareStableStrings(left.name, right.name))) { - if (!box.isDirectory()) continue; - let targets; - try { - targets = await readdir(scrollDirectory(box.name), { withFileTypes: true }); - } catch { - continue; - } - for (const target of targets.sort((left, right) => - compareStableStrings(left.name, right.name))) { - if (!target.isDirectory()) continue; - const nestedReference = `${box.name}/${target.name}`; - if (await fileExists(join(scrollDirectory(nestedReference), 'scroll.json'))) { - candidates.push(await readExactScroll(nestedReference)); - } - } - } - if (candidates.length === 0) { - fail('No scrolls found; run scrollcase init or scrollcase new scroll.'); - } - return candidates; - } - - const reference = safeRelativePath(name); - if (reference.includes('/')) { - if (reference.split('/').length !== 2 - || !await fileExists(join(scrollDirectory(reference), 'scroll.json'))) { - fail(`Scroll not found: ${reference}.`); - } - return [await readExactScroll(reference)]; - } - - let entries; - try { - entries = await readdir(scrollDirectory(reference), { withFileTypes: true }); - } catch { - return fail(`Scroll or box not found: ${reference}.`); - } - const candidates = []; - for (const entry of entries.sort((left, right) => compareStableStrings(left.name, right.name))) { - if (!entry.isDirectory()) continue; - const nestedReference = `${reference}/${entry.name}`; - if (await fileExists(join(scrollDirectory(nestedReference), 'scroll.json'))) { - candidates.push(await readExactScroll(nestedReference)); - } - } - if (candidates.length === 0) fail(`Box ${reference} contains no target scrolls.`); - return candidates; -} - -/** - * Loads a scroll without prompting. - * - * Library callers may select a target explicitly. An unambiguous box shorthand is also accepted; - * ambiguity is a hard error here because only the CLI edge is allowed to ask a person. - */ -export async function readScroll(name, { targetId = null } = {}) { - let candidates = await scrollCandidates(name); - if (targetId) { - candidates = candidates.filter((candidate) => candidate.targetId === targetId); - if (candidates.length === 0) { - fail(`Target ${targetId} is not available for ${name}.`); - } - } - if (candidates.length > 1) { - fail( - `Box ${name} has multiple scroll targets (${candidates.map((candidate) => candidate.targetId).join(', ')}); ` - + 'use / or select a target explicitly.', - ); - } - return candidates[0]; -} - -/** - * Build timestamp taken from the HEAD commit rather than the clock, so rebuilding the same commit - * produces the same provenance. Falls back to the epoch outside a git checkout — deliberately a - * constant, since a wall-clock fallback would reintroduce the nondeterminism this avoids. - */ -export function sourceBuildTime(cwd) { - const result = runResult('git', ['show', '-s', '--format=%cI', 'HEAD'], { capture: true, cwd }); - return result.status === 0 ? result.stdout.trim() : new Date(0).toISOString(); -} - -/** - * The commit a box was built from, and whether the tree had uncommitted changes at the time. - * - * Outside a git checkout there is no revision to record, which callers must handle explicitly rather - * than inventing one: an unversioned build is reproducible by nobody. - */ -export function sourceBuildState(cwd) { - const revision = runResult('git', ['rev-parse', 'HEAD'], { capture: true, cwd }); - if (revision.status !== 0) return null; - const status = runResult('git', ['status', '--porcelain', '--untracked-files=all'], { capture: true, cwd }); - return { revision: revision.stdout.trim(), dirty: status.stdout.trim().length > 0 }; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/toolchain.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/toolchain.mjs deleted file mode 100644 index 72cd7a1..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/toolchain.mjs +++ /dev/null @@ -1,246 +0,0 @@ -/** - * Installing the build toolchain, only when a human says so. - * - * `init` prepares a workspace without touching the network. When pixi or conda-pack is missing it - * *offers* to install them and downloads nothing until an explicit yes. That consent is the design, - * not a courtesy: a command that quietly fetched and ran a binary would be one nobody dares re-run, - * and the whole point of `init` is that it is always safe to run again. - * - * What is downloaded is verified before it is used. The archive's SHA-256 is checked against the - * checksum pixi publishes beside it, and the verified digest is then recorded in the project's - * config, so every later install — a teammate's machine, CI — is checked against a value the - * project reviewed rather than against whatever the server serves that day. A mismatch is a hard - * failure: an unverified toolchain would undermine every guarantee built on top of it. - * - * The toolchain is installed inside the project, under the workspace's toolchain directory. Nothing - * is placed on PATH, nothing is installed system-wide, and removing the directory undoes it. - */ - -import { createWriteStream } from 'node:fs'; -import { chmod, copyFile, mkdir, mkdtemp, rm, rename } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { pipeline } from 'node:stream/promises'; -import { extractScrollArchive } from './archive.mjs'; -import { collectFiles, fileExists, sha256File } from './filesystem.mjs'; -import { fail, run as defaultRun } from './process.mjs'; - -const PIXI_RELEASES = 'https://github.com/prefix-dev/pixi/releases'; -const PIXI_LATEST_API = 'https://api.github.com/repos/prefix-dev/pixi/releases/latest'; -const SHA256_TOKEN = /\b[a-f0-9]{64}\b/; - -/** - * One host-specific archive published by pixi. - * - * @typedef {object} PixiReleaseAsset - * @property {string} asset - * @property {'zip' | 'tar.gz'} format - * @property {string} binary - */ - -// conda-pack changes the bytes staged into a box, so letting the resolver select a newer release -// would make the same Scrollcase version produce a different payload over time. Keep the pin with -// the implementation that relies on its output; changing it is a reviewed Scrollcase release. -export const CONDA_PACK_VERSION = '0.9.2'; - -/** - * The release asset for each host pixi publishes a build for, keyed by `platform/arch` as Node - * reports them. A host outside this table is not a failure of the project — it just means the - * toolchain has to be installed by hand. - */ -/** @type {Readonly>>} */ -export const PIXI_RELEASE_ASSETS = Object.freeze({ - 'darwin/arm64': Object.freeze({ asset: 'pixi-aarch64-apple-darwin.tar.gz', format: 'tar.gz', binary: 'pixi' }), - 'darwin/x64': Object.freeze({ asset: 'pixi-x86_64-apple-darwin.tar.gz', format: 'tar.gz', binary: 'pixi' }), - 'linux/x64': Object.freeze({ asset: 'pixi-x86_64-unknown-linux-musl.tar.gz', format: 'tar.gz', binary: 'pixi' }), - 'linux/arm64': Object.freeze({ asset: 'pixi-aarch64-unknown-linux-musl.tar.gz', format: 'tar.gz', binary: 'pixi' }), - 'win32/x64': Object.freeze({ asset: 'pixi-x86_64-pc-windows-msvc.zip', format: 'zip', binary: 'pixi.exe' }), - 'win32/arm64': Object.freeze({ asset: 'pixi-aarch64-pc-windows-msvc.zip', format: 'zip', binary: 'pixi.exe' }), -}); - -/** - * Where the project keeps the tools it installed for itself. - * - * @param {string} toolchainDir - * @returns {{ binDir: string, pixi: string, condaPack: string }} - */ -export function toolchainPaths(toolchainDir) { - const binDir = join(toolchainDir, 'bin'); - const suffix = process.platform === 'win32' ? '.exe' : ''; - return { - binDir, - pixi: join(binDir, `pixi${suffix}`), - condaPack: join(binDir, `conda-pack${suffix}`), - }; -} - -/** - * Returns the release asset for this host, or null when pixi publishes no build for it. - * - * @param {{ platform: string, arch: string }} [host] - * @returns {Readonly | null} - */ -export function pixiReleaseAsset(host = process) { - return PIXI_RELEASE_ASSETS[`${host.platform}/${host.arch}`] ?? null; -} - -/** - * The archive and checksum URLs for one pixi release. - * - * @param {string} version - * @param {string} asset - * @returns {{ archiveUrl: string, checksumUrl: string }} - */ -export function pixiAssetUrls(version, asset) { - const base = `${PIXI_RELEASES}/download/v${version}/${asset}`; - return { archiveUrl: base, checksumUrl: `${base}.sha256` }; -} - -/** - * Reads the digest out of a published checksum file, which may or may not name the file beside it. - * - * @param {unknown} text - * @returns {string} - */ -export function parseChecksumFile(text) { - const match = SHA256_TOKEN.exec(String(text).toLowerCase()); - if (!match) fail('Published pixi checksum file does not contain a SHA-256 digest.'); - return match[0]; -} - -/** - * Resolves the newest pixi release, for a project that has not pinned a version yet. - * - * @param {{ fetchImpl?: typeof fetch }} [options] - * @returns {Promise} - */ -export async function latestPixiVersion({ fetchImpl = fetch } = {}) { - const response = await fetchImpl(PIXI_LATEST_API, { headers: { accept: 'application/vnd.github+json' } }); - if (!response.ok) fail(`Could not look up the latest pixi release (${response.status}).`); - const tag = (await response.json())?.tag_name; - if (typeof tag !== 'string' || !tag) fail('The pixi release feed returned no version.'); - return tag.replace(/^v/, ''); -} - -async function fetchText(url, fetchImpl) { - const response = await fetchImpl(url); - if (!response.ok) fail(`Download failed (${response.status}): ${url}`); - return response.text(); -} - -async function fetchToFile(url, destination, fetchImpl) { - const response = await fetchImpl(url); - if (!response.ok) fail(`Download failed (${response.status}): ${url}`); - await pipeline(response.body, createWriteStream(destination)); -} - -/** - * Moves a staged file onto its final path, falling back to a copy across filesystems. - * - * Staging happens in the OS temp directory while the toolchain lives inside the project, and those - * are routinely on different volumes: on Windows temp sits on `C:` while a checkout commonly sits - * on another drive, which is the default on a GitHub runner and ordinary on a developer's machine. - * `rename` cannot cross a volume boundary and fails with `EXDEV`, so a plain rename left the - * toolchain uninstalled for those users. Copying is slower and only needed on that path, which is - * why it is the fallback rather than the rule. - * - * @param {string} source - * @param {string} destination - * @returns {Promise} - */ -async function moveInto(source, destination) { - try { - await rename(source, destination); - } catch (error) { - if (error?.code !== 'EXDEV') throw error; - // The staging directory is removed by the caller's `finally`, so the copy needs no cleanup. - await copyFile(source, destination); - } -} - -/** - * Downloads one pixi release and installs its binary into the project's toolchain directory. - * - * `expectedSha256` is the digest the project has already reviewed, when it has one; without it the - * checksum published beside the archive is used and returned, so the caller can pin it. Either way - * the bytes on disk are hashed and compared before anything is installed. - * - * @param {{ - * version: string, - * toolchainDir: string, - * expectedSha256?: string | null, - * host?: { platform: string, arch: string }, - * fetchImpl?: typeof fetch, - * log?: (message: string) => void, - * }} options - * @returns {Promise<{ path: string, version: string, sha256: string, asset: string }>} - */ -export async function installPixi({ - version, - toolchainDir, - expectedSha256 = null, - host = process, - fetchImpl = fetch, - log = console.log, -}) { - const release = pixiReleaseAsset(host); - if (!release) { - fail(`pixi publishes no build for ${host.platform}/${host.arch}; install it manually from https://pixi.sh/.`); - } - const { archiveUrl, checksumUrl } = pixiAssetUrls(version, release.asset); - const staging = await mkdtemp(join(tmpdir(), 'scrollcase-toolchain-')); - try { - const expected = expectedSha256 ?? parseChecksumFile(await fetchText(checksumUrl, fetchImpl)); - log(`Downloading pixi ${version} (${release.asset})`); - const archivePath = join(staging, release.asset); - await fetchToFile(archiveUrl, archivePath, fetchImpl); - - const actual = await sha256File(archivePath); - if (actual !== expected) { - fail(`pixi ${version} failed its checksum: expected ${expected}, got ${actual}. Nothing was installed.`); - } - - // Unpacked through the same guarded extractor the payload uses, so a hostile archive cannot - // write outside the staging directory even though this one came from a known publisher. - const unpacked = join(staging, 'unpacked'); - await extractScrollArchive(archivePath, release.format, unpacked); - const entry = (await collectFiles(unpacked)).find((file) => file.split('/').pop() === release.binary); - if (!entry) fail(`The pixi archive did not contain ${release.binary}.`); - - const { binDir, pixi } = toolchainPaths(toolchainDir); - await mkdir(binDir, { recursive: true }); - await rm(pixi, { force: true }); - await moveInto(join(unpacked, ...entry.split('/')), pixi); - if (process.platform !== 'win32') await chmod(pixi, 0o755); - return { path: pixi, version, sha256: expected, asset: release.asset }; - } finally { - await rm(staging, { recursive: true, force: true }); - } -} - -/** - * Installs conda-pack with the project's own pixi, into the project's own toolchain directory. - * - * `PIXI_HOME` points pixi at the toolchain directory, so the result lands beside pixi instead of in - * the user's home. Integrity here is conda-forge's to provide: the package is resolved and verified - * by pixi exactly as any other dependency is. - * - * @param {{ - * pixi: string, - * toolchainDir: string, - * run?: typeof defaultRun, - * log?: (message: string) => void, - * }} options - * @returns {Promise<{ path: string, version: typeof CONDA_PACK_VERSION }>} - */ -export async function installCondaPack({ pixi, toolchainDir, run = defaultRun, log = console.log }) { - log(`Installing conda-pack ${CONDA_PACK_VERSION} with pixi`); - run(pixi, ['global', 'install', `conda-pack==${CONDA_PACK_VERSION}`], { - env: { PIXI_HOME: toolchainDir }, - }); - const { condaPack } = toolchainPaths(toolchainDir); - if (!await fileExists(condaPack)) { - fail(`pixi reported success but ${condaPack} is missing.`); - } - return { path: condaPack, version: CONDA_PACK_VERSION }; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/verify.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/verify.mjs deleted file mode 100644 index 52e737c..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/verify.mjs +++ /dev/null @@ -1,204 +0,0 @@ -/** - * `verify` — re-run a consumer's install-time checks locally, before anything is published. - * - * This deliberately mirrors what an installing client does on a user's machine: check the signature, - * the archive's size and hash, that entry names are safe, that `box.json` agrees with the signed - * release, and that the declared interpreter is actually present. `selfTest` goes one step further - * and imports the modules from a real extraction, which is the closest thing to a dry-run install. - * - * The point is that a box which would fail on a user's machine fails here instead. - */ - -import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { dirname, join, resolve } from 'node:path'; -import { isDeepStrictEqual } from 'node:util'; -import { assertNativeHost, assertPythonEntryPoint, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; -import { BOX_SCHEMA_VERSION, parseDocumentKind } from '../contract/documents.mjs'; -import { verifySignedDocument } from '../sign/index.mjs'; - -const AGREEMENT_FIELDS = [ - 'schemaVersion', - 'boxId', - 'modelId', - 'runtimeId', - 'version', - 'target', - 'pythonEntryPoint', - 'modelCacheSubdir', - 'selfTest', - 'execution', - 'weights', - 'assets', - 'provenance', -]; - -/** - * Binds the self-description inside the archive to the signed release outside it. - * - * Only fields present in both schema-version-2 documents belong here. Release-only transport data - * has no counterpart in box.json; every shared identity, target, layout, consumer self-test, - * asset-policy, and provenance field must agree recursively. - */ -export function assertBoxManifestAgreement(box, release) { - for (const field of AGREEMENT_FIELDS) { - if (!isDeepStrictEqual(box[field], release[field])) fail(`box.json mismatch: ${field}`); - } -} -import { extractZipArchive, listZipEntries, readZipEntry } from './archive.mjs'; -import { assertExecutionFiles } from './execution.mjs'; -import { fileExists, payloadSize, safeRelativePath, sha256File } from './filesystem.mjs'; -import { fail, run as runProcess } from './process.mjs'; -import { schemaValidationError } from './schema-validation.mjs'; - -const schemaUrls = [ - new URL('../contract/schema/release-manifest.schema.json', import.meta.url), - new URL('../contract/schema/box-manifest.schema.json', import.meta.url), - new URL('../contract/schema/target.schema.json', import.meta.url), - new URL('../contract/schema/execution.schema.json', import.meta.url), - new URL('../contract/schema/signed-document.schema.json', import.meta.url), -]; -let manifestSchemas; - -async function loadManifestSchemas() { - manifestSchemas ??= Promise.all(schemaUrls.map(async (url) => JSON.parse(await readFile(url, 'utf8')))); - return manifestSchemas; -} - -/** - * Performs the complete read-only trust chain shared by `verify` and the local consumer. - * - * Keeping this as one operation matters: adding an execution API must not create a second, - * subtly different interpretation of a signed release. The caller receives the validated - * in-memory objects and exact archive path, but extraction and execution remain separate steps. - */ -export async function inspectBoxArchive(releaseDocumentPath, options = {}) { - const { publicPath, archive: archiveOverride = null } = options; - const releasePath = resolve(releaseDocumentPath); - const signed = JSON.parse(await readFile(releasePath, 'utf8')); - if (signed?.schemaVersion === 1) { - fail('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); - } - const [releaseSchema, boxSchema, targetSchema, executionSchema, signedSchema] = - await loadManifestSchemas(); - const signedError = schemaValidationError(signed, signedSchema); - if (signedError) fail(`Invalid signed document: ${signedError}.`); - const release = await verifySignedDocument(signed, publicPath); - if (release?.schemaVersion === 1) { - fail('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); - } - if (release?.schemaVersion !== BOX_SCHEMA_VERSION) { - fail(`Unsupported schemaVersion ${String(release?.schemaVersion)}; expected ${BOX_SCHEMA_VERSION}.`); - } - const releaseError = schemaValidationError( - release, - releaseSchema, - [boxSchema, targetSchema, executionSchema], - ); - if (releaseError) fail(`Invalid release manifest: ${releaseError}.`); - if (parseDocumentKind(release.kind)?.type !== 'release') fail('Document is not a box release.'); - const adapter = boxTargetAdapter(release.target); - assertPythonEntryPoint(adapter, release.pythonEntryPoint); - - // The archive sits next to its release document under the hash that document commits to — the - // same name it is published under, so this resolves identically against a local dist tree and a - // directory downloaded from a mirror. - const archivePath = archiveOverride - ? resolve(archiveOverride) - : join(dirname(releasePath), `${release.archive.sha256}.zip`); - if (!await fileExists(archivePath)) fail(`Archive not found: ${archivePath}`); - if ((await stat(archivePath)).size !== release.archive.sizeBytes) fail('Archive size mismatch.'); - if (await sha256File(archivePath) !== release.archive.sha256) fail('Archive SHA-256 mismatch.'); - if (release.installedSizeBytes !== undefined - && (!Number.isSafeInteger(release.installedSizeBytes) || release.installedSizeBytes <= 0)) { - fail('Invalid installed size.'); - } - - const entries = await listZipEntries(archivePath); - // Two questions, deliberately not the same set. `box.json` is read out of the archive, so it must - // be an entry with its own bytes. Everything else asks only whether a path resolves — and a link - // does resolve, to a file inside this same payload, because nothing else was allowed in. - const files = new Set(entries.filter((entry) => entry.kind === 'file').map((entry) => entry.path)); - const resolvablePaths = new Set(entries - .filter((entry) => entry.kind === 'file' || entry.kind === 'link') - .map((entry) => entry.path)); - if (!files.has('box.json')) fail('Archive is missing box.json.'); - const box = JSON.parse(await readZipEntry(archivePath, 'box.json')); - const boxError = schemaValidationError( - box, - boxSchema, - [releaseSchema, targetSchema, executionSchema], - ); - if (boxError) fail(`Invalid box.json: ${boxError}.`); - assertBoxManifestAgreement(box, release); - if (!resolvablePaths.has(release.pythonEntryPoint)) fail(`Archive is missing ${release.pythonEntryPoint}.`); - assertExecutionFiles({ - execution: release.execution, - adapter, - pythonVersion: release.provenance.pythonVersion, - files: resolvablePaths, - }); - - return { - releasePath, - archivePath, - signed, - release, - box, - adapter, - entries, - files, - }; -} - -/** - * Verifies a signed release document and the archive it commits to. - * - * `publicPath` names the trusted key file; `archive` overrides the convention of the archive - * sitting next to its release document; `selfTest` additionally extracts the box and runs its own - * interpreter, which only works on a matching native host. Returns a summary of what was checked. - */ -export async function verifyBox(releaseDocumentPath, options = {}) { - const { - selfTest = false, - run = runProcess, - log = console.log, - } = options; - const inspected = await inspectBoxArchive(releaseDocumentPath, options); - const { - archivePath, - signed, - release, - adapter, - } = inspected; - - if (selfTest) { - assertNativeHost(adapter); - const extracted = await mkdtemp(join(tmpdir(), 'scrollcase-verify-')); - try { - await extractZipArchive(archivePath, extracted); - if (release.installedSizeBytes !== undefined - && await payloadSize(extracted) !== release.installedSizeBytes) { - fail('Extracted payload size does not match the signed release.'); - } - const python = join(extracted, safeRelativePath(release.pythonEntryPoint)); - run(python, ['-c', `${adapter.selfTestPython}\nimport ${release.selfTest.pythonImports.join(', ')}`], { - cwd: extracted, - env: adapter.validationEnvironments[release.target.accelerator], - }); - } finally { - await rm(extracted, { recursive: true, force: true }); - } - } - - log(`Verified ${release.boxId} ${release.version} (${boxTargetId(release.target)})`); - return { - status: 'passed', - localSignatureVerified: true, - signingKeyIds: signed.signatures.map((signature) => signature.keyId), - releasePayloadSha256: signed.payloadSha256, - archiveSha256: release.archive.sha256, - archiveSizeBytes: release.archive.sizeBytes, - selfTest: selfTest ? 'passed' : 'not-requested', - }; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/workspace.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/workspace.mjs deleted file mode 100644 index 0823c5b..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/build/workspace.mjs +++ /dev/null @@ -1,245 +0,0 @@ -/** - * Scrollcase workspace resolution. - * - * Where a project keeps its scrolls, and where the tool writes what it builds, is the project's - * decision, not the tool's. A workspace is declared by a `scrollcase.config.json` at the project - * root, discovered by walking up from the working directory and overridable per invocation by CLI - * flags. A project that declares nothing gets the defaults below. - * - * Precedence, highest first: CLI flag, then `scrollcase.config.json`, then the built-in default. - * Flag values resolve against the current working directory (what a shell user expects); config - * values resolve against the project root (so a config file is portable). - */ -import { existsSync, readFileSync } from 'node:fs'; -import { dirname, isAbsolute, parse as parsePath, resolve } from 'node:path'; -import { fail } from './process.mjs'; - -/** - * The absolute layout a command works against. Every path is resolved and the object is frozen. - * - * @typedef {object} Workspace - * @property {string} root the project root the config was found in, and the git checkout provenance - * is recorded from - * @property {string | null} configPath the config that produced it, or null when defaults applied - * @property {string} scrollsDir - * @property {string} buildDir - * @property {string} distDir - * @property {string} keysDir - * @property {string} toolchainDir - */ - -/** - * Per-invocation overrides, highest precedence in workspace resolution. - * - * @typedef {object} WorkspaceOverrides - * @property {string} [projectRoot] - * @property {string} [config] - * @property {string} [scrolls] - * @property {string} [build] - * @property {string} [dist] - * @property {string} [keys] - * @property {string} [toolchain] - */ - -export const SCROLLCASE_CONFIG_FILENAME = 'scrollcase.config.json'; - -/** - * The layout a project gets when it declares nothing. A project that already keeps its scrolls - * elsewhere — or that adopted the tool after building its own convention — overrides these in its - * config rather than moving its files. - */ -export const DEFAULT_WORKSPACE_PATHS = Object.freeze({ - scrolls: 'scrolls', - build: '.scrollcase/build', - dist: '.scrollcase/dist', - keys: '.scrollcase/keys', - toolchain: '.scrollcase/toolchain', -}); - -/** Config path key -> resolved workspace field. */ -const PATH_FIELDS = Object.freeze({ - scrolls: 'scrollsDir', - build: 'buildDir', - dist: 'distDir', - keys: 'keysDir', - toolchain: 'toolchainDir', -}); - -/** CLI flag -> config path key. */ -const PATH_FLAGS = Object.freeze({ - 'scrolls-dir': 'scrolls', - 'build-dir': 'build', - 'out-dir': 'dist', - 'keys-dir': 'keys', - 'toolchain-dir': 'toolchain', -}); - -/** - * Walks up from `startDir` to the filesystem root looking for a workspace config. - * - * @param {string} startDir - * @returns {string | null} the nearest config path, or null at the filesystem root - */ -export function findWorkspaceConfig(startDir) { - let current = resolve(startDir); - const { root } = parsePath(current); - for (;;) { - const candidate = resolve(current, SCROLLCASE_CONFIG_FILENAME); - if (existsSync(candidate)) return candidate; - if (current === root) return null; - const parent = dirname(current); - if (parent === current) return null; - current = parent; - } -} - -/** Reads and shape-checks a config file; an unreadable or malformed config is a hard error. */ -function readWorkspaceConfig(configPath) { - let config; - try { - config = JSON.parse(readFileSync(configPath, 'utf8')); - } catch (error) { - return fail(`Invalid ${SCROLLCASE_CONFIG_FILENAME} at ${configPath}: ${error instanceof Error ? error.message : String(error)}`); - } - if (!config || typeof config !== 'object' || Array.isArray(config)) { - return fail(`Invalid ${SCROLLCASE_CONFIG_FILENAME} at ${configPath}: expected a JSON object`); - } - if (config.version !== undefined && config.version !== 1) { - return fail(`Unsupported ${SCROLLCASE_CONFIG_FILENAME} version ${config.version} at ${configPath}; this builder understands version 1`); - } - const paths = config.paths ?? {}; - if (typeof paths !== 'object' || paths === null || Array.isArray(paths)) { - return fail(`Invalid ${SCROLLCASE_CONFIG_FILENAME} at ${configPath}: "paths" must be an object`); - } - for (const [key, value] of Object.entries(paths)) { - if (!(key in PATH_FIELDS)) { - fail(`Unknown "paths" entry "${key}" in ${configPath}; expected one of ${Object.keys(PATH_FIELDS).join(', ')}`); - } - if (typeof value !== 'string' || value.trim() === '') { - fail(`Invalid "paths.${key}" in ${configPath}: expected a non-empty path string`); - } - } - return { ...config, paths }; -} - -/** - * Collects workspace overrides from an already-parsed CLI flag map. - * - * @param {ReadonlyMap | null | undefined} flags - * @returns {WorkspaceOverrides} - */ -export function workspaceOverridesFromFlags(flags) { - const overrides = {}; - const stringFlag = (name) => { - const value = flags?.get(name); - if (value === undefined) return undefined; - if (typeof value !== 'string' || value.trim() === '') fail(`--${name} requires a path value`); - return value; - }; - const projectRoot = stringFlag('project-root'); - if (projectRoot !== undefined) overrides.projectRoot = projectRoot; - const config = stringFlag('config'); - if (config !== undefined) overrides.config = config; - for (const [flag, key] of Object.entries(PATH_FLAGS)) { - const value = stringFlag(flag); - if (value !== undefined) overrides[key] = value; - } - return overrides; -} - -/** - * Collects workspace overrides directly from raw arguments, for entry points that parse the rest of - * their command line themselves. Only the workspace flags are read, in `--name value` or - * `--name=value` form; anything else is left untouched for the caller's own parser. - * - * @param {readonly string[]} values - * @returns {WorkspaceOverrides} - */ -export function workspaceOverridesFromArgv(values) { - const flags = new Map(); - const known = new Set(['project-root', 'config', ...Object.keys(PATH_FLAGS)]); - for (let index = 0; index < values.length; index += 1) { - const value = values[index]; - if (typeof value !== 'string' || !value.startsWith('--')) continue; - const [name, inline] = value.slice(2).split('=', 2); - if (!known.has(name)) continue; - if (inline !== undefined) flags.set(name, inline); - else if (values[index + 1] !== undefined) flags.set(name, values[index + 1]); - else fail(`--${name} requires a path value`); - } - return workspaceOverridesFromFlags(flags); -} - -/** - * Resolves the absolute workspace layout. - * - * Root selection, highest precedence first: `--project-root`, the directory of an explicit - * `--config`, the nearest `scrollcase.config.json` above the working directory, and finally the - * working directory itself. - * - * @param {{ cwd?: string, overrides?: WorkspaceOverrides }} [options] - * @returns {Workspace} frozen, with every path absolute - * @throws {Error} when a named config is missing or malformed - */ -export function resolveWorkspace({ cwd = process.cwd(), overrides = {} } = {}) { - const base = resolve(cwd); - let configPath = null; - let root; - if (overrides.config !== undefined) { - // An explicitly named config must exist: silently ignoring it would hide a typo behind defaults. - configPath = resolve(base, overrides.config); - if (!existsSync(configPath)) fail(`Workspace config not found: ${configPath}`); - root = overrides.projectRoot !== undefined ? resolve(base, overrides.projectRoot) : dirname(configPath); - } else if (overrides.projectRoot !== undefined) { - root = resolve(base, overrides.projectRoot); - const candidate = resolve(root, SCROLLCASE_CONFIG_FILENAME); - configPath = existsSync(candidate) ? candidate : null; - } else { - configPath = findWorkspaceConfig(base); - root = configPath ? dirname(configPath) : base; - } - const config = configPath ? readWorkspaceConfig(configPath) : { paths: {} }; - const workspace = { root, configPath }; - for (const [key, field] of Object.entries(PATH_FIELDS)) { - const override = overrides[key]; - if (override !== undefined) { - // A flag is typed by a user standing in some directory, so it resolves from there. - workspace[field] = isAbsolute(override) ? override : resolve(base, override); - continue; - } - const declared = config.paths[key]; - // Config and default values belong to the project, so they resolve from its root. - workspace[field] = resolve(root, declared ?? DEFAULT_WORKSPACE_PATHS[key]); - } - return Object.freeze(workspace); -} - -let current = null; - -/** - * Installs the workspace for this process. Entry points call this once, before any other work, so - * every module downstream observes the same resolved layout. - * - * @param {{ cwd?: string, overrides?: WorkspaceOverrides }} [options] - * @returns {Workspace} - */ -export function configureWorkspace(options = {}) { - current = resolveWorkspace(options); - return current; -} - -/** - * Returns the process workspace, resolving a flag-free default on first use. Modules read paths - * through this rather than at import time, so an entry point can still configure them from flags. - * - * @returns {Workspace} resolving a flag-free default on first use - */ -export function getWorkspace() { - if (!current) current = resolveWorkspace(); - return current; -} - -/** Test seam: forgets the resolved workspace so the next read re-resolves it. */ -export function resetWorkspace() { - current = null; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-args.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-args.mjs deleted file mode 100644 index 1c93473..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/cli-args.mjs +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Argument parsing at the CLI edge. - * - * The `--` separator is a hard boundary: every string after it belongs unchanged to the box - * application, even when it looks like a Scrollcase flag or contains shell syntax. - */ - -/** Parses `--name=value`, `--name value`, bare flags, and an application argument tail. */ -export function parseArgs(values) { - const positional = []; - const flags = new Map(); - let passthrough = []; - for (let index = 0; index < values.length; index += 1) { - const value = values[index]; - if (value === '--') { - passthrough = values.slice(index + 1); - break; - } - if (!value.startsWith('--')) { - positional.push(value); - continue; - } - const [name, inline] = value.slice(2).split('=', 2); - if (inline !== undefined) flags.set(name, inline); - else if (values[index + 1] && !values[index + 1].startsWith('--')) { - flags.set(name, values[index + 1]); - index += 1; - } else flags.set(name, true); - } - return { positional, flags, passthrough }; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-authoring.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-authoring.mjs deleted file mode 100644 index 2f5316d..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/cli-authoring.mjs +++ /dev/null @@ -1,187 +0,0 @@ -/** - * Interactive and scripted input collection for `scrollcase new scroll`. - * - * This is a CLI-edge module: finite decisions use the shared navigable menu, free-form values use - * explicit text prompts, and a non-terminal process must provide every material value as a flag. - * The build layer receives one complete object and never reads a terminal. - */ - -import { createInterface } from 'node:readline/promises'; -import { fail } from './build/process.mjs'; -import { chooseCliValue } from './cli-menu.mjs'; -import { chooseTarget, cliTargetFamilies, parseCliTarget } from './cli-targets.mjs'; - -const flagText = (flags, name) => { - if (!flags.has(name)) return null; - const value = flags.get(name); - if (typeof value !== 'string' || value.trim() === '') fail(`--${name} requires a value.`); - return value.trim(); -}; - -async function promptText(question, { - defaultValue = null, - optional = false, - input = process.stdin, - output = process.stdout, -} = {}) { - const readline = createInterface({ input, output }); - try { - const suffix = defaultValue === null ? '' : ` [${defaultValue}]`; - const value = (await readline.question(`${question}${suffix}: `)).trim(); - if (value) return value; - if (defaultValue !== null) return defaultValue; - if (optional) return null; - fail(`${question} is required.`); - } finally { - readline.close(); - } -} - -function parseDefaultArgs(value) { - if (value === null) return []; - let parsed; - try { - parsed = JSON.parse(value); - } catch { - fail('--default-args must be a JSON array of strings.'); - } - if (!Array.isArray(parsed) || parsed.some((item) => typeof item !== 'string')) { - fail('--default-args must be a JSON array of strings.'); - } - return parsed; -} - -async function collectTarget(flags, { terminal, ask, chooseTargetValue }) { - const requested = flagText(flags, 'target'); - if (requested) return parseCliTarget(requested); - if (!terminal) fail('new scroll requires --target without a terminal.'); - - const selected = await chooseTargetValue(cliTargetFamilies(), { terminal: true }); - if (selected.target.accelerator !== 'cuda') return parseCliTarget(selected.targetId); - const cudaVersion = await ask('CUDA version (major.minor)'); - return parseCliTarget(`${selected.targetId}${cudaVersion}`); -} - -/** - * Collects a complete `createScroll` argument object from flags or interactive prompts. - * - * @param {ReadonlyMap} flags - * @param {object} [options] - * @returns {Promise} - */ -export async function collectNewScrollOptions(flags, { - terminal = Boolean(process.stdin.isTTY && process.stdout.isTTY), - ask = promptText, - choose = chooseCliValue, - chooseTargetValue = chooseTarget, -} = {}) { - const required = async (flag, question, defaultValue = null) => { - const supplied = flagText(flags, flag); - if (supplied !== null) return supplied; - if (!terminal) fail(`new scroll requires --${flag} without a terminal.`); - return ask(question, { defaultValue }); - }; - const optional = async (flag, question) => { - const supplied = flagText(flags, flag); - if (supplied !== null) return supplied; - if (!terminal) return null; - return ask(question, { optional: true }); - }; - const finite = async (flag, question, choices) => { - const supplied = flagText(flags, flag); - if (!supplied && !terminal) { - fail(`new scroll requires --${flag} <${choices.join('|')}> without a terminal.`); - } - return choose(question, choices, { flag: supplied, terminal }); - }; - - const target = await collectTarget(flags, { terminal, ask, chooseTargetValue }); - const boxId = await required('box-id', 'Box ID'); - const modelId = await required('model-id', 'Model ID'); - const runtimeId = await required('runtime-id', 'Runtime ID'); - const version = await required('version', 'Box version', '1.0.0'); - const scrollVersion = await required('scroll-version', 'Scroll version', '1.0.0'); - const sourceRevision = await required('source-revision', 'Upstream source revision'); - const pythonVersion = await required('python-version', 'Python version', '3.11'); - const pixiVersion = await required('pixi-version', 'pixi version'); - const minHostAppVersion = await required( - 'min-host-app-version', - 'Minimum host application version', - '1.0.0', - ); - const compatibility = { minHostAppVersion }; - const maxHostAppVersionExclusive = await optional( - 'max-host-app-version-exclusive', - 'Maximum host application version (exclusive, optional)', - ); - if (maxHostAppVersionExclusive) compatibility.maxHostAppVersionExclusive = maxHostAppVersionExclusive; - if (target.platform === 'macos') { - const minMacosVersion = await optional('min-macos-version', 'Minimum macOS version (optional)'); - if (minMacosVersion) compatibility.minMacosVersion = minMacosVersion; - } - const minRam = await optional('min-ram-gb', 'Minimum RAM in GB (optional)'); - if (minRam !== null) { - const minRamGb = Number(minRam); - if (!Number.isFinite(minRamGb) || minRamGb <= 0) fail('--min-ram-gb must be a positive number.'); - compatibility.minRamGb = minRamGb; - } - if (target.accelerator === 'cuda') { - const minNvidiaDriverVersion = await optional( - 'min-nvidia-driver-version', - 'Minimum NVIDIA driver version (optional)', - ); - if (minNvidiaDriverVersion) compatibility.minNvidiaDriverVersion = minNvidiaDriverVersion; - } - const assetBaseUrl = await required('asset-base-url', 'Asset base URL'); - const weights = await finite('weights', 'weights mode', ['embed', 'on-demand']); - const executionKind = await finite( - 'execution', - 'execution kind', - ['python-script', 'python-module', 'library-only'], - ); - const defaultArgs = parseDefaultArgs(flagText(flags, 'default-args')); - - const result = { - boxId, - target, - modelId, - runtimeId, - version, - scrollVersion, - sourceRevision, - pythonVersion, - pixiVersion, - compatibility, - assetBaseUrl, - weights, - executionKind, - defaultArgs, - }; - if (executionKind === 'python-module') { - result.module = await required('module', 'Python module'); - } else if (executionKind === 'python-script') { - const existing = flagText(flags, 'script'); - const generateScript = Boolean(flags.get('generate-script')); - if (existing && generateScript) { - fail('Choose either --script or --generate-script, not both.'); - } - if (existing) result.scriptSourcePath = existing; - else if (generateScript) result.generateScript = true; - else if (!terminal) { - fail('python-script execution requires --script or --generate-script without a terminal.'); - } else { - const source = await choose( - 'script source', - ['existing project script', 'generate starter script'], - { terminal: true }, - ); - if (source === 'existing project script') { - result.scriptSourcePath = await ask('Project-relative script path'); - } else result.generateScript = true; - } - result.scriptRelativePath = flagText(flags, 'script-destination') ?? 'entrypoint.py'; - const generatedScriptSourcePath = flagText(flags, 'generated-script-path'); - if (generatedScriptSourcePath) result.generatedScriptSourcePath = generatedScriptSourcePath; - } - return result; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-init.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-init.mjs deleted file mode 100644 index 4ad5ea8..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/cli-init.mjs +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Orders the optional work performed by `scrollcase init`. - * - * Every answer is collected before the first installer runs. Besides making the interaction easier - * to review, this prevents an early download or package install from interrupting the remaining - * questions and leaving the user's choices only half collected. - */ - -export async function resolvePythonConsumerSource({ - selectedSource, - condaAvailable, - confirmPyPIFallback, -}) { - if (selectedSource !== 'conda-forge' || condaAvailable) return selectedSource; - return await confirmPyPIFallback() ? 'pypi' : null; -} - -export async function runInitDependencySetup({ - hasExample, - confirmTypeScript, - confirmPython, - choosePythonSource, - installToolchain, - installTypeScript, - installPython, -}) { - let shouldInstallTypeScript = false; - let pythonSource = null; - - if (hasExample) { - shouldInstallTypeScript = await confirmTypeScript(); - if (await confirmPython()) pythonSource = await choosePythonSource(); - } - - const toolchain = await installToolchain(); - const typescript = shouldInstallTypeScript ? installTypeScript() : null; - const python = pythonSource ? installPython(pythonSource) : null; - - return { - installTypeScript: shouldInstallTypeScript, - pythonSource, - toolchain, - typescript, - python, - }; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-menu.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-menu.mjs deleted file mode 100644 index 3abfb74..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/cli-menu.mjs +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Navigable choices at the CLI edge. - * - * Closed choices use one raw-key menu instead of several subtly different text prompts. Free-form - * values and safety consent remain explicit flags or text input: a menu must not pretend they are - * finite choices. - */ - -import { emitKeypressEvents } from 'node:readline'; -import { fail } from './build/process.mjs'; - -/** Shows a raw-key menu and resolves to the selected index. */ -export function selectCliMenu(question, choices, { - initialIndex = null, - input = process.stdin, - output = process.stdout, -} = {}) { - if (!input.isTTY || typeof input.setRawMode !== 'function') { - fail(`${question} selection requires an interactive terminal.`); - } - - return new Promise((resolve, reject) => { - let selectedIndex = initialIndex; - const previousRawMode = Boolean(input.isRaw); - const frameLines = choices.length + 1; - let firstFrame = true; - - const render = () => { - if (!firstFrame) output.write(`\x1b[${frameLines}A`); - for (let index = 0; index < choices.length; index += 1) { - const marker = index === selectedIndex ? '❯' : ' '; - output.write(`\x1b[2K\r${marker} ${choices[index]}\n`); - } - output.write('\x1b[2K\rUse ↑/↓ to move, Enter to select.\n'); - firstFrame = false; - }; - - const cleanup = () => { - input.removeListener('keypress', onKeypress); - input.setRawMode(previousRawMode); - input.pause(); - output.write('\x1b[?25h'); - }; - - const onKeypress = (_character, key = {}) => { - if (key.ctrl && key.name === 'c') { - cleanup(); - reject(new Error(`${question} selection cancelled.`)); - return; - } - if (key.name === 'up') { - selectedIndex = selectedIndex === null - ? choices.length - 1 - : (selectedIndex - 1 + choices.length) % choices.length; - render(); - } else if (key.name === 'down') { - selectedIndex = selectedIndex === null ? 0 : (selectedIndex + 1) % choices.length; - render(); - } else if ((key.name === 'return' || key.name === 'enter') && selectedIndex !== null) { - cleanup(); - resolve(selectedIndex); - } - }; - - emitKeypressEvents(input); - input.on('keypress', onKeypress); - input.setRawMode(true); - input.resume(); - output.write(`Which ${question}?\n\x1b[?25l`); - render(); - }); -} - -/** - * Resolves a CLI choice from a flag, a menu, or the reported non-terminal default. - * - * `open` applies only to explicit flags; custom values cannot be represented by a finite menu. - */ -export async function chooseCliValue(question, choices, { - flag = null, - open = false, - terminal = Boolean(process.stdin.isTTY && process.stdout.isTTY), - menu = selectCliMenu, - log = console.log, -} = {}) { - const [fallback] = choices; - if (flag) { - if (!open && !choices.includes(flag)) { - fail(`Unsupported ${question}: ${flag}. Use ${choices.join(' or ')}.`); - } - return flag; - } - if (!terminal) { - log(`scrollcase: no terminal to ask which ${question}; using ${fallback}.`); - return fallback; - } - const selectedIndex = await menu(question, choices, { initialIndex: 0 }); - if (!Number.isInteger(selectedIndex) || selectedIndex < 0 || selectedIndex >= choices.length) { - fail(`${question} menu returned an invalid selection.`); - } - return choices[selectedIndex]; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-output.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-output.mjs deleted file mode 100644 index 4aaecb2..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/cli-output.mjs +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Restrained terminal presentation for the human CLI. - * - * Symbols keep redirected logs readable on their own; ANSI colour is an optional enhancement only - * for a real terminal, and `NO_COLOR` always wins. The library modules remain presentation-free. - */ - -import { dirname, relative, sep } from 'node:path'; - -const styles = Object.freeze({ - success: { symbol: '✓', ansi: 32 }, - step: { symbol: '→', ansi: 36 }, - info: { symbol: '·', ansi: 90 }, - warning: { symbol: '⚠', ansi: 33 }, - error: { symbol: '✗', ansi: 31 }, -}); - -/** Formats one CLI status line, colouring only its symbol when the terminal supports it. */ -export function statusLine(kind, message, { - stream = process.stdout, - env = process.env, -} = {}) { - const style = styles[kind]; - const colour = Boolean(stream.isTTY && !Object.hasOwn(env, 'NO_COLOR') && env.TERM !== 'dumb'); - const symbol = colour ? `\x1b[${style.ansi}m${style.symbol}\x1b[0m` : style.symbol; - return `${symbol} ${message}`; -} - -/** Builds the concise, relative distribution instruction printed after a successful build. */ -export function buildDistributionSummary({ archivePath, channelPath }, distDir) { - const displayPath = (path) => relative(distDir, path).split(sep).join('/'); - return `Build complete — you can distribute the 2 files under ${displayPath(dirname(archivePath))}/ ` - + `and ${displayPath(channelPath)}`; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-run.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-run.mjs deleted file mode 100644 index 5c2f44e..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/cli-run.mjs +++ /dev/null @@ -1,56 +0,0 @@ -/** - * The `run` command's deliberately thin edge over the Node consumer. - * - * Verification, extraction, execution, signals, and cleanup remain owned by `runBox`. This module - * adds only terminal presentation and translates the child's terminal result into CLI process - * semantics. - */ - -import { runBox } from './consumer/index.mjs'; - -/** - * Runs one local release through the consumer and applies its terminal result to this process. - * - * @param {string} releaseDocumentPath - * @param {{ - * publicPath: string, - * archive?: string | null, - * args?: readonly string[], - * run?: typeof runBox, - * log?: (message: string) => void, - * setExitCode?: (code: number) => void, - * terminate?: (signal: NodeJS.Signals) => void, - * }} options - * @returns {Promise} - */ -export async function runCliBox(releaseDocumentPath, { - publicPath, - archive = null, - args = [], - run = runBox, - log = console.log, - setExitCode = (code) => { - process.exitCode = code; - }, - terminate = (signal) => { - process.kill(process.pid, signal); - }, -}) { - const result = await run(releaseDocumentPath, { - publicPath, - archive, - args, - stdin: 'inherit', - stdout: 'inherit', - stderr: 'inherit', - onPrepared: (prepared) => { - log( - `Running ${prepared.boxId} ${prepared.version} ` - + `(${prepared.targetId}, ${prepared.execution?.kind ?? 'library-only'})`, - ); - }, - }); - if (result.signal) terminate(result.signal); - else setExitCode(result.exitCode ?? 1); - return result; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-signing.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-signing.mjs deleted file mode 100644 index 4df0d3c..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/cli-signing.mjs +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Signing readiness at the CLI edge. - * - * Signing readiness is a read-only preflight. `build` never creates or repairs identity material: - * doing so would mutate the project before provenance checks and could silently rotate the - * identity used by already-published documents. - */ - -import { fileExists } from './build/filesystem.mjs'; -import { fail } from './build/process.mjs'; - -/** Ensures the selected signing path is ready before any expensive build work begins. */ -export async function ensureBuildSigningKeys({ - privatePath, - publicPath, - signerCommand = null, -}) { - const publicExists = await fileExists(publicPath); - if (signerCommand) { - if (!publicExists) { - fail(`Trusted public key not found: ${publicPath}. Supply the key used to verify the external signer.`); - } - return; - } - - const privateExists = await fileExists(privatePath); - if (privateExists && publicExists) return; - if (privateExists || publicExists) { - const missing = privateExists ? publicPath : privatePath; - fail(`Signing key pair is incomplete; missing ${missing}. Refusing to replace the existing key.`); - } - fail('Signing keys not found. Run scrollcase keygen before building.'); -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-targets.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-targets.mjs deleted file mode 100644 index 3c9267d..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/cli-targets.mjs +++ /dev/null @@ -1,192 +0,0 @@ -/** - * Target choices at the CLI edge. - * - * Modules beneath the CLI receive a resolved scroll or target and never read a terminal. This file - * owns the one interactive policy: choices are made through a keyboard menu, with a sole native - * target as the default and Metal preferred on macOS. Non-interactive callers get the same - * decision without ever blocking. - */ - -import { boxTargetAdapters, boxTargetId } from './contract/targets.mjs'; -import { compareStableStrings } from './build/filesystem.mjs'; -import { fail } from './build/process.mjs'; -import { selectCliMenu } from './cli-menu.mjs'; - -/** Parses a complete canonical target ID back into the target it names. */ -export function parseCliTarget(value) { - const targetId = String(value); - for (const adapter of boxTargetAdapters()) { - for (const accelerator of Object.keys(adapter.validationEnvironments)) { - const target = { platform: adapter.platform, arch: adapter.arch, accelerator }; - if (accelerator === 'cuda') { - const prefix = `${adapter.platform}-${adapter.arch}-cuda`; - if (!targetId.startsWith(prefix)) continue; - target.cudaVersion = targetId.slice(prefix.length); - } - try { - if (boxTargetId(target) === targetId) return target; - } catch { - // Keep looking. A partial CUDA ID reaches here, then receives the one canonical error below. - } - } - } - return fail( - `Invalid target ${targetId}; specify a complete target such as ` - + 'macos-aarch64-metal or linux-x86_64-cuda12.4.', - ); -} - -/** - * Lists target families for `new scroll`. CUDA is shown without an ABI version; selecting it is - * followed by the separate version question that turns it into a complete canonical target. - */ -export function cliTargetFamilies(platform) { - const families = []; - for (const adapter of boxTargetAdapters()) { - if (platform && adapter.platform !== platform) continue; - for (const accelerator of Object.keys(adapter.validationEnvironments)) { - families.push({ - adapter, - target: { platform: adapter.platform, arch: adapter.arch, accelerator }, - targetId: `${adapter.platform}-${adapter.arch}-${accelerator}`, - }); - } - } - return families.sort((left, right) => compareStableStrings(left.targetId, right.targetId)); -} - -/** - * Chooses the deterministic native target for the example created by `init`. - * - * The demo prefers Metal on Apple Silicon and CPU elsewhere, so it never guesses a CUDA ABI and - * remains usable from a non-interactive setup. - */ -export function nativeExampleTarget( - host = { platform: process.platform, arch: process.arch }, -) { - const adapter = boxTargetAdapters().find((candidate) => - candidate.host.platform === host.platform && candidate.host.arch === host.arch); - if (!adapter) { - return fail( - `No example target is available for ${host.platform}/${host.arch}; ` - + 'use scrollcase init --no-example.', - ); - } - return { - platform: adapter.platform, - arch: adapter.arch, - accelerator: adapter.platform === 'macos' ? 'metal' : 'cpu', - }; -} - -/** - * Selects one complete scroll reference when a CLI caller omitted the positional argument. - * - * Unlike target selection, this has no non-terminal default: locking or building an arbitrary - * first scroll would mutate or package the wrong input without consent. - * - * @template {{ reference: string }} T - * @param {T[]} candidates - * @returns {Promise} - */ -export async function chooseScroll(candidates, { - terminal = Boolean(process.stdin.isTTY && process.stdout.isTTY), - menu = selectCliMenu, -} = {}) { - if (candidates.length === 0) fail('No scrolls are available.'); - const choices = [...candidates] - .sort((left, right) => compareStableStrings(left.reference, right.reference)); - if (new Set(choices.map(({ reference }) => reference)).size !== choices.length) { - fail('Scroll choices must have unique references.'); - } - if (!terminal) { - fail( - 'scroll selection requires an interactive terminal; ' - + 'pass / explicitly.', - ); - } - const selectedIndex = await menu( - 'scroll', - choices.map(({ reference }) => reference), - { initialIndex: 0 }, - ); - if (!Number.isInteger(selectedIndex) || selectedIndex < 0 || selectedIndex >= choices.length) { - fail('scroll menu returned an invalid selection.'); - } - return choices[selectedIndex]; -} - -/** Shows a raw-key target menu and resolves to the selected index. */ -export function selectTargetMenu(targetIds, { - initialIndex = null, - input = process.stdin, - output = process.stdout, -} = {}) { - return selectCliMenu('target', targetIds, { initialIndex, input, output }); -} - -/** - * Chooses one target candidate under the CLI's terminal policy. - * - * @template {{ targetId: string, adapter: { host: { platform: string, arch: string } } }} T - * @param {T[]} candidates - * @param {{ requested?: string | null, terminal?: boolean, - * host?: { platform: string, arch: string }, - * menu?: (targetIds: string[], options: { initialIndex: number | null }) => Promise, - * log?: (message: string) => void }} [options] - * @returns {Promise} - */ -export async function chooseTarget(candidates, { - requested = null, - terminal = Boolean(process.stdin.isTTY && process.stdout.isTTY), - host = { platform: process.platform, arch: process.arch }, - menu = selectTargetMenu, - log = console.log, -} = {}) { - if (candidates.length === 0) fail('No supported targets are available.'); - const choices = [...candidates] - .sort((left, right) => compareStableStrings(left.targetId, right.targetId)); - if (new Set(choices.map(({ targetId }) => targetId)).size !== choices.length) { - fail('Target choices must have unique canonical IDs.'); - } - - if (requested) { - const selected = choices.find((candidate) => candidate.targetId === requested); - if (!selected) { - fail(`Target ${requested} is not available; choose one of ${choices.map(({ targetId }) => targetId).join(', ')}.`); - } - return selected; - } - if (choices.length === 1) return choices[0]; - - const native = choices.filter(({ adapter }) => - adapter.host.platform === host.platform && adapter.host.arch === host.arch); - const macMetal = host.platform === 'darwin' - ? native.find(({ targetId }) => targetId.endsWith('-metal')) - : null; - const fallback = native.length === 1 ? native[0] : macMetal; - if (!terminal) { - if (fallback) { - log(`scrollcase: no terminal to ask which target; using host target ${fallback.targetId}.`); - return fallback; - } - if (native.length > 1) { - fail( - `This host can build more than one available target (${native.map(({ targetId }) => targetId).join(', ')}); ` - + 'specify --target .', - ); - } - fail( - `No available target is an unambiguous match for this host; specify --target ` - + `from ${choices.map(({ targetId }) => targetId).join(', ')}.`, - ); - } - - const selectedIndex = await menu(choices.map(({ targetId }) => targetId), { - initialIndex: fallback ? choices.indexOf(fallback) : null, - }); - if (!Number.isInteger(selectedIndex) || selectedIndex < 0 || selectedIndex >= choices.length) { - fail('Target menu returned an invalid selection.'); - } - return choices[selectedIndex]; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli.mjs deleted file mode 100755 index 82cdd38..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/cli.mjs +++ /dev/null @@ -1,501 +0,0 @@ -#!/usr/bin/env node - -/** - * The Scrollcase command line. - * - * One job: turn a scroll into a portable, locked, self-contained box and prove it works. `init` - * prepares the workspace, `new scroll` authors one input, `doctor` checks the machine, `lock` - * resolves dependencies once so a human can review and commit the result, `audit` reports what - * licences that pulls in, `build` installs only from the lock, `verify` re-runs a consumer's - * install-time checks, `run` executes one caller-supplied local release through that consumer, and - * `keygen` produces the signing key that makes any of it trustworthy. - * - * Every command resolves its paths through the workspace, so the tool runs from anywhere against any - * project that declares a scrollcase.config.json. - */ - -import { createInterface } from 'node:readline/promises'; -import { join, resolve } from 'node:path'; -import { auditScroll } from './build/audit.mjs'; -import { - createScroll, - ensureExampleScroll, - EXAMPLE_PIXI_VERSION, -} from './build/authoring.mjs'; -import { buildBox } from './build/box.mjs'; -import { - isCondaAvailable, - installPythonConsumerDependency, - installTypeScriptConsumerDependencies, - SCROLLCASE_NPM_VERSION, -} from './build/consumer-setup.mjs'; -import { findPixi, pixiLockArguments } from './build/pixi.mjs'; -import { fail, run } from './build/process.mjs'; -import { diagnose, ensureToolchain, initProject } from './build/project.mjs'; -import { scrollCandidates, readScroll } from './build/scroll.mjs'; -import { verifyBox } from './build/verify.mjs'; -import { - configureWorkspace, - getWorkspace, - SCROLLCASE_CONFIG_FILENAME, - workspaceOverridesFromFlags, -} from './build/workspace.mjs'; -import { collectNewScrollOptions } from './cli-authoring.mjs'; -import { parseArgs } from './cli-args.mjs'; -import { - resolvePythonConsumerSource, - runInitDependencySetup, -} from './cli-init.mjs'; -import { chooseCliValue } from './cli-menu.mjs'; -import { buildDistributionSummary, statusLine } from './cli-output.mjs'; -import { runCliBox } from './cli-run.mjs'; -import { ensureBuildSigningKeys } from './cli-signing.mjs'; -import { chooseScroll, chooseTarget, nativeExampleTarget } from './cli-targets.mjs'; -import { CHANNELS } from './contract/index.mjs'; -import { generateSigningKey } from './sign/index.mjs'; - -const success = (message) => console.log(statusLine('success', message)); -const step = (message) => console.log(statusLine('step', message)); -const info = (message) => console.log(statusLine('info', message)); -const warning = (message) => console.log(statusLine('warning', message)); - -const text = (flags, name) => (flags.has(name) ? String(flags.get(name)) : null); - -/** Signing key locations, defaulting into the workspace's key directory. */ -function keyPaths(flags) { - const keysDir = getWorkspace().keysDir; - return { - privatePath: resolve(text(flags, 'private-key') || join(keysDir, 'signing-private.pem')), - publicPath: resolve(text(flags, 'public-key') || join(keysDir, 'signing-public.json')), - }; -} - -async function keygen(flags) { - const { privatePath, publicPath } = keyPaths(flags); - const created = await generateSigningKey({ - privatePath, - publicPath, - keyId: text(flags, 'key-id'), - force: Boolean(flags.get('force')), - }); - success(`Created signing key ${created.keyId}`); - info(`Private: ${created.privatePath}`); - info(`Public: ${created.publicPath}`); -} - -/** - * `lock` — resolve the scroll's pixi manifest into a fully pinned lock file. - * - * Run by a human when dependencies change; the result is committed and reviewed. Builds then only - * *install* from it, so what ships is exactly what was reviewed. The manifest pins the channels and - * the single target platform, which is what makes resolution independent of the machine doing it. - */ -async function lock(name, flags) { - const reference = await selectScrollReference(name, flags); - const { dir, scroll } = await readScroll(reference); - const pixi = findPixi({ requiredVersion: scroll.pixiVersion, path: text(flags, 'pixi') }); - run(pixi, pixiLockArguments(join(dir, 'pixi.toml'))); - success(`Updated ${join(dir, 'pixi.lock')}`); -} - -/** - * Asks a yes/no question, defaulting to no. - * - * Only ever asks when both ends are a terminal. Without one — CI, a pipe — there is nobody to - * answer, and silence must not be read as consent, so the answer is no. - */ -async function confirm(question) { - if (!process.stdin.isTTY || !process.stdout.isTTY) return false; - console.log(); - const readline = createInterface({ input: process.stdin, output: process.stdout }); - try { - return /^y(es)?$/i.test((await readline.question(`${question} [y/N] `)).trim()); - } finally { - readline.close(); - } -} - -/** Resolves a box shorthand at the CLI edge, where an ambiguous target can be asked about. */ -async function selectScrollReference(name, flags) { - const candidates = await scrollCandidates(name); - if (!name) return (await chooseScroll(candidates)).reference; - return (await chooseTarget(candidates, { requested: text(flags, 'target') })).reference; -} - -/** - * `init` — scaffold the workspace and its disposable runnable example, then offer its dependencies. - * - * Real scroll creation remains separate: the fixed `example-box` is onboarding material, never a - * guess at the project's identity. Toolchain and consumer installs each require explicit consent. - */ -async function init(flags) { - const workspace = getWorkspace(); - const authoringFlags = [ - 'target', - 'platform', - 'accelerator', - 'cuda-version', - 'box-id', - 'model-id', - 'runtime-id', - ].filter((name) => flags.has(name)); - if (authoringFlags.length > 0) { - fail(`init accepts only the fixed example; pass ${authoringFlags.map((name) => `--${name}`).join(', ')} to scrollcase new scroll.`); - } - const exampleTarget = flags.get('no-example') ? null : nativeExampleTarget(); - const result = await initProject({ root: workspace.root, scrollsDir: workspace.scrollsDir }); - for (const path of result.written) success(`Created ${path}`); - for (const path of result.skipped) info(`Kept ${path} (already present)`); - - let example = null; - const pixiVersion = text(flags, 'pixi-version') - ?? (exampleTarget ? EXAMPLE_PIXI_VERSION : null); - if (exampleTarget) { - const initializedWorkspace = workspace.configPath - ? workspace - : { ...workspace, configPath: join(workspace.root, SCROLLCASE_CONFIG_FILENAME) }; - example = await ensureExampleScroll({ - workspace: initializedWorkspace, - target: exampleTarget, - pixiVersion, - }); - if (example.created) { - success(`Created example scroll ${example.scrollRef}`); - } else { - info(`Kept example scroll ${example.scrollRef} (already present)`); - } - for (const path of example.written) info(path); - } - - const always = Boolean(flags.get('install-toolchain')); - const never = Boolean(flags.get('no-install-toolchain')); - const setup = await runInitDependencySetup({ - hasExample: Boolean(example), - confirmTypeScript: () => confirm( - `Install scrollcase, TypeScript, and tsx in ${workspace.root}?`, - ), - confirmPython: () => confirm( - 'Install scrollcase-consumer for Python?', - ), - choosePythonSource: async () => { - console.log(); - const selectedSource = await chooseCliValue( - 'Python consumer package source', - ['PyPI with pip', 'conda-forge with conda'], - ); - const source = selectedSource.startsWith('PyPI') ? 'pypi' : 'conda-forge'; - return resolvePythonConsumerSource({ - selectedSource: source, - condaAvailable: source === 'pypi' || isCondaAvailable({ root: workspace.root }), - confirmPyPIFallback: () => confirm( - 'Conda is not installed. Install scrollcase-consumer from PyPI with pip instead?', - ), - }); - }, - installToolchain: () => ensureToolchain({ - workspace, - pixiVersion, - confirm: async (missing) => { - if (never) return false; - if (always) return true; - console.log(); - return confirm(`This project needs ${missing.join(' and ')} to build a box.\nInstall ${missing.length > 1 ? 'them' : 'it'} into ${workspace.toolchainDir}?`); - }, - }), - installTypeScript: () => installTypeScriptConsumerDependencies({ root: workspace.root }), - installPython: (source) => installPythonConsumerDependency({ - root: workspace.root, - source, - }), - }); - const { toolchain } = setup; - - if (toolchain.installed.length > 0) { - success(`Installed ${toolchain.installed.join(' and ')} into ${workspace.toolchainDir}`); - info('Nothing was added to PATH; scrollcase finds them there on its own.'); - if (toolchain.configPath) success(`Recorded the toolchain pins in ${toolchain.configPath}`); - } else if (toolchain.unsupportedHost) { - warning(`pixi publishes no build for ${toolchain.unsupportedHost}; install ${toolchain.missing.join(' and ')} manually.`); - } else if (toolchain.missing.length > 0) { - warning(`Skipped installing ${toolchain.missing.join(' and ')}.`); - info('Install them yourself, or re-run with --install-toolchain. `scrollcase doctor` reports what is missing.'); - } - - if (setup.typescript) { - const installed = setup.typescript; - success( - `Installed scrollcase ${installed.scrollcaseVersion}, TypeScript, and tsx in ${workspace.root}`, - ); - } - - if (setup.python) { - const installed = setup.python; - success( - `Installed scrollcase-consumer from ${installed.source} using ${installed.command}`, - ); - } - - success('Workspace initialized'); - if (example) step(`Example: scrollcase lock ${example.scrollRef}`); - step(example ? 'Create your own: scrollcase new scroll' : 'Next: scrollcase new scroll'); -} - -/** `new scroll` — collect one complete authoring decision and create it atomically. */ -async function newScroll(flags) { - const workspace = getWorkspace(); - const options = await collectNewScrollOptions(flags); - const result = await createScroll({ workspace, ...options }); - success(`Created scroll ${result.scrollRef}`); - for (const path of result.written) info(path); - step(`Next: scrollcase lock ${result.scrollRef}`); -} - -/** `doctor` — report whether this machine can build a box. Reads only; never writes. */ -async function doctor(flags) { - let pixiVersion = text(flags, 'pixi-version'); - const scrollName = text(flags, 'scroll'); - if (!pixiVersion && scrollName) { - const reference = await selectScrollReference(scrollName, flags); - pixiVersion = (await readScroll(reference)).scroll.pixiVersion; - } - const { checks, ok } = await diagnose({ - workspace: getWorkspace(), - pixiVersion, - pixiPath: text(flags, 'pixi'), - condaPackPath: text(flags, 'conda-pack'), - }); - for (const check of checks) { - console.log(statusLine(check.ok ? 'success' : 'error', `${check.name.padEnd(11)} ${check.detail}`)); - if (!check.ok && check.remedy) console.log(` ${statusLine('step', check.remedy)}`); - } - if (!ok) fail('Some checks failed; see the remedies above.'); -} - -/** `audit` — the dependency licence inventory, derived from the lock without building. */ -async function audit(name, flags) { - const reference = await selectScrollReference(name, flags); - const write = Boolean(flags.get('write')); - const { summary, reviewed, written } = await auditScroll(reference, { - write, - namespace: text(flags, 'namespace') || undefined, - }); - info(`${summary.packageCount} packages for ${summary.scrollId} (${summary.targetId})`); - for (const entry of summary.licenses) console.log(` ${String(entry.count).padStart(4)} ${entry.license}`); - if (written) success(`Wrote reviewed audit: ${reviewed}`); - else if (reviewed) success(`Matches the reviewed audit: ${reviewed}`); -} - -async function build(name, flags) { - const reference = await selectScrollReference(name, flags); - const signing = { - ...keyPaths(flags), - signerCommand: text(flags, 'signer-command'), - }; - await ensureBuildSigningKeys(signing); - // Asked at the CLI edge and passed down: buildBox never reads a terminal itself. - const channel = await chooseCliValue( - 'channel', - ['beta', ...CHANNELS.filter((value) => value !== 'beta')], - { flag: text(flags, 'channel') }, - ); - const weights = await chooseCliValue( - 'weights mode', - ['embed', 'on-demand'], - { flag: text(flags, 'weights') }, - ); - step(`Building ${reference} (${channel}, ${weights})`); - const built = await buildBox(reference, { - ...signing, - allowDirty: Boolean(flags.get('allow-dirty')), - channel, - weights, - assetBaseUrl: text(flags, 'asset-base-url'), - namespace: text(flags, 'namespace') || undefined, - pixiPath: text(flags, 'pixi'), - condaPackPath: text(flags, 'conda-pack'), - log: (message) => { - if (!message || /^(Box:|Release:|Channel:|Publish:| {9}then )/.test(message)) return; - step(message); - }, - }); - const workspace = getWorkspace(); - success(buildDistributionSummary(built, workspace.distDir)); -} - -async function verify(path, flags) { - await verifyBox(path, { - publicPath: keyPaths(flags).publicPath, - archive: text(flags, 'archive'), - selfTest: Boolean(flags.get('self-test')), - }); -} - -async function runRelease(path, flags, args) { - return runCliBox(path, { - publicPath: keyPaths(flags).publicPath, - archive: text(flags, 'archive'), - args, - log: step, - }); -} - -function usage() { - console.log(`Usage: scrollcase [options] - scrollcase -v | --version - -Commands: - init Initialize a workspace with a runnable example - new scroll Create one guided target-specific scroll - doctor Report whether this machine can build a box - keygen Create a local ed25519 signing key - lock [] Resolve the scroll's pixi manifest into pixi.lock - audit Dependency licence inventory, derived from the lock - build [] Build, self-test, archive, and sign a box - verify Verify signature, archive hash, and layout - run Verify, temporarily extract, and run a local box - -Init options: - --pixi-version Install this pixi release when setup is approved - --no-example Initialize an empty workspace without example-box - --install-toolchain Install missing pixi/conda-pack without asking - --no-install-toolchain Never install them; just report what is missing - With neither flag, init asks before downloading anything, and - installs into after a verified checksum check. - When the example is present, init separately offers to install - its TypeScript and Python consumer dependencies in the project - root. Missing Conda offers a PyPI fallback. - -New scroll options: - --target Complete target, including the CUDA ABI when applicable - --box-id Box identity - --model-id Packaged model identity - --runtime-id Runtime identity - --version Box version - --scroll-version Scroll authoring version - --source-revision Upstream source revision recorded in provenance - --python-version Python dependency version - --pixi-version pixi resolver version - --min-host-app-version Minimum compatible host application version - --asset-base-url Base URL used in built release documents - --weights embed or on-demand - --execution python-script, python-module, or library-only - --script Existing project script for python-script - --generate-script Generate a minimal project script instead - --script-destination Payload path for the script (default entrypoint.py) - --generated-script-path Project path for a generated starter - --module Dotted module name for python-module - --default-args JSON array of default application arguments - --max-host-app-version-exclusive - --min-macos-version - --min-ram-gb - --min-nvidia-driver-version - Without a terminal, every material value must be supplied. - -Doctor options: - --scroll Take the required pixi version from this scroll - --target Select a target when is a box with several scrolls - --pixi-version Check for this pixi release - -Keygen options: - --key-id Identifier recorded in signatures (default derived from key) - --force Overwrite both named key files; unsafe for rotation - -Audit options: - --target Select a target when names a box - --write Write the inventory to the scroll's reviewed audit path - --namespace Document kind namespace (default scrollcase.box) - -Build options: - --target Select a target when names a box - --channel Channel the signed pointer names (nightly, beta, or stable; - default beta) - --weights embed (default: assets packed in, works air-gapped) or - on-demand (caller-materialized; verified before execution) - Without either flag, build shows an arrow-key menu. With no - terminal to ask, it says which default it took and carries on. - --asset-base-url Override the scroll's published base URL - --namespace Document kind namespace (default scrollcase.box) - --allow-dirty Permit a build from an uncommitted source tree - --pixi Use this pixi executable - --conda-pack Use this conda-pack executable (managed installs pin 0.9.2) - -Scroll targets: - lock, audit and build accept either / or a box ID plus - --target . With only a box ID, a terminal shows an arrow-key menu. - lock and build also let an interactive terminal choose from every workspace - scroll when the argument is omitted; non-interactive callers must name one. - A sole target for this host is the default; Metal is preferred on macOS. - Without a terminal, any other ambiguous target is an error. - -Verify options: - --archive Archive to check, if not beside the release document - --self-test Extract and import with the box's own interpreter - -Run: - scrollcase run [--archive ] -- [application args] - --archive Local archive, if not beside the release document - Uses --public-key from Signing below, attaches terminal stdio, - forwards signals, and exits with the application result. - -Signing: - --private-key Local signing key (default /signing-private.pem) - --public-key Trusted key set (default /signing-public.json) - --signer-command Sign through an external command instead of a local key. - It receives the payload on stdin and returns the signed - document as JSON on stdout; the result is verified locally. - Before build work starts, missing local keys fail with an - explicit instruction to run scrollcase keygen. - -Workspace: - Paths come from scrollcase.config.json at the project root, discovered by walking - up from the working directory, and can be overridden per invocation: - --config Use this workspace config explicitly - --project-root Treat this directory as the project root - --scrolls-dir Where scrolls live (default scrolls) - --build-dir Payload scratch space (default .scrollcase/build) - --out-dir Built artefacts (default .scrollcase/dist) - --keys-dir Local signing keys (default .scrollcase/keys) - --toolchain-dir Project-local pixi/conda-pack (default .scrollcase/toolchain) -`); -} - -async function main() { - const [command, ...rest] = process.argv.slice(2); - if (command === '-v' || command === '--version') { - console.log(SCROLLCASE_NPM_VERSION); - return; - } - const { positional, flags, passthrough } = parseArgs(rest); - if (!command || command === 'help' || command === '--help') return usage(); - // Resolve the workspace before any command touches a path, so flags win over the project config. - configureWorkspace({ overrides: workspaceOverridesFromFlags(flags) }); - if (command === 'init') return init(flags); - if (command === 'new') { - if (positional[0] !== 'scroll' || positional.length !== 1) { - fail('Usage: scrollcase new scroll [options]'); - } - return newScroll(flags); - } - if (command === 'doctor') return doctor(flags); - if (command === 'keygen') return keygen(flags); - if (command === 'audit') return audit(positional[0] || fail('audit requires a scroll name.'), flags); - if (command === 'lock') return lock(positional[0], flags); - if (command === 'build') return build(positional[0], flags); - if (command === 'verify') return verify(positional[0] || fail('verify requires a signed release document.'), flags); - if (command === 'run') { - if (positional.length !== 1) fail('Usage: scrollcase run [--archive ] -- [application args]'); - return runRelease(positional[0], flags, passthrough); - } - fail(`Unknown command: ${command}`); -} - -// Single failure path: every `fail()` anywhere lands here as a one-line message and a non-zero exit -// code, so CI and shell callers can rely on the status. -main().catch((error) => { - console.error(statusLine( - 'error', - `scrollcase: ${error instanceof Error ? error.message : String(error)}`, - { stream: process.stderr }, - )); - process.exitCode = 1; -}); diff --git a/.scrollcase-runtime-types-RzUfxr/src/consumer/index.mjs b/.scrollcase-runtime-types-RzUfxr/src/consumer/index.mjs deleted file mode 100644 index 22ef358..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/consumer/index.mjs +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Local box preparation and execution. - * - * This surface composes with a caller's distribution policy; it does not become that policy. Every - * path, trust anchor, archive and destination comes from the caller, and no box code runs until the - * complete signed release and archive trust chain has passed. - */ - -/** @typedef {import('./verify-and-extract.mjs').PreparedBox} PreparedBox */ -/** @typedef {import('./verify-and-extract.mjs').RequiredAsset} RequiredAsset */ -/** @typedef {import('./run-extracted.mjs').BoxRunResult} BoxRunResult */ -/** @typedef {import('./run-extracted.mjs').RunExtractedBoxOptions} RunExtractedBoxOptions */ -/** @typedef {import('./run-box.mjs').RunBoxOptions} RunBoxOptions */ - -export { verifyAndExtractBox } from './verify-and-extract.mjs'; -export { runExtractedBox } from './run-extracted.mjs'; -export { runBox } from './run-box.mjs'; diff --git a/.scrollcase-runtime-types-RzUfxr/src/consumer/run-box.mjs b/.scrollcase-runtime-types-RzUfxr/src/consumer/run-box.mjs deleted file mode 100644 index e4af208..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/consumer/run-box.mjs +++ /dev/null @@ -1,46 +0,0 @@ -/** - * One-shot local execution: prepare into a private temporary root, run, then remove every byte. - * - * The `finally` owns cleanup for every terminal path — normal exit, non-zero exit, spawn failure, - * or a forwarded signal. The child result is returned unchanged so callers retain application exit - * semantics instead of having them translated into a Scrollcase success/failure convention. - */ - -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join, resolve } from 'node:path'; -import { runExtractedBox } from './run-extracted.mjs'; -import { verifyAndExtractBox } from './verify-and-extract.mjs'; - -/** - * @typedef {import('./run-extracted.mjs').RunExtractedBoxOptions & { - * publicPath: string, - * archive?: string | null, - * temporaryDirectory?: string, - * onPrepared?: (prepared: Readonly) => - * void | Promise, - * }} RunBoxOptions - */ - -/** - * Verifies, temporarily extracts, and executes one caller-supplied local box. - * - * @param {string} releaseDocumentPath - * @param {RunBoxOptions} options - * @returns {Promise} - */ -export async function runBox(releaseDocumentPath, options) { - const temporaryParent = resolve(options.temporaryDirectory ?? tmpdir()); - const temporaryRoot = await mkdtemp(join(temporaryParent, 'scrollcase-run-')); - try { - const prepared = await verifyAndExtractBox(releaseDocumentPath, { - publicPath: options.publicPath, - archive: options.archive, - destination: join(temporaryRoot, 'box'), - }); - await options.onPrepared?.(prepared); - return await runExtractedBox(prepared, options); - } finally { - await rm(temporaryRoot, { recursive: true, force: true }); - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/consumer/run-extracted.mjs b/.scrollcase-runtime-types-RzUfxr/src/consumer/run-extracted.mjs deleted file mode 100644 index 4fe2f29..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/consumer/run-extracted.mjs +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Shell-free execution of a box that this process has already prepared. - * - * The verified manifest supplies the interpreter and script/module identity; the caller supplies - * only additional argument strings, streams, and environment values. Signals are forwarded while - * the child is alive and listeners are removed at the same point the result settles. - */ - -import { spawn as spawnProcess } from 'node:child_process'; -import { lstat } from 'node:fs/promises'; -import { join } from 'node:path'; -import { collectFiles, safeRelativePath, sha256File } from '../build/filesystem.mjs'; -import { assertExecutionFiles } from '../build/execution.mjs'; -import { fail } from '../build/process.mjs'; -import { assertNativeHost, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; -import { preparedBoxState } from './verify-and-extract.mjs'; - -/** - * @typedef {'pipe' | 'overlapped' | 'ignore' | 'inherit' | number | - * import('node:stream').Stream | null | undefined} BoxStdio - */ - -/** - * @typedef {object} RunExtractedBoxOptions - * @property {readonly string[]} [args] - * @property {NodeJS.ProcessEnv} [env] values merged over the current process environment - * @property {BoxStdio} [stdin] - * @property {BoxStdio} [stdout] - * @property {BoxStdio} [stderr] - * @property {typeof spawnProcess} [spawn] injectable process seam - * @property {Pick} [signalSource] injectable signal seam - */ - -/** - * @typedef {object} BoxRunResult - * @property {number | null} exitCode - * @property {NodeJS.Signals | null} signal - */ - -const FORWARDED_SIGNALS = /** @type {const} */ (['SIGINT', 'SIGTERM', 'SIGHUP']); - -function stringArguments(values) { - if (!Array.isArray(values) || !values.every((value) => typeof value === 'string')) { - fail('Box execution arguments must be an array of strings.'); - } - return values; -} - -async function verifyRequiredAssets(root, assets) { - for (const asset of assets) { - const path = join(root, ...safeRelativePath(asset.relativePath).split('/')); - let metadata; - try { - metadata = await lstat(path); - } catch (error) { - if (error?.code === 'ENOENT') { - fail(`Required on-demand asset is missing: ${asset.relativePath}.`); - } - throw error; - } - if (!metadata.isFile()) fail(`Required on-demand asset is not a regular file: ${asset.relativePath}.`); - if (metadata.size !== asset.sizeBytes) { - fail(`Required on-demand asset size mismatch: ${asset.relativePath}.`); - } - if (await sha256File(path) !== asset.sha256) { - fail(`Required on-demand asset SHA-256 mismatch: ${asset.relativePath}.`); - } - } -} - -function waitForChild(child, signalSource) { - return new Promise((resolve, reject) => { - const handlers = new Map(); - const cleanup = () => { - for (const [signal, handler] of handlers) signalSource.removeListener(signal, handler); - handlers.clear(); - }; - const onError = (error) => { - cleanup(); - reject(error); - }; - const onClose = (exitCode, signal) => { - cleanup(); - resolve({ exitCode, signal }); - }; - child.once('error', onError); - child.once('close', onClose); - for (const signal of FORWARDED_SIGNALS) { - const handler = () => child.kill(signal); - handlers.set(signal, handler); - signalSource.on(signal, handler); - } - }); -} - -/** - * Executes a prepared box with its own interpreter and returns its terminal result. - * - * @param {import('./verify-and-extract.mjs').PreparedBox} prepared - * @param {RunExtractedBoxOptions} [options] - * @returns {Promise} - */ -export async function runExtractedBox(prepared, options = {}) { - const { release, rootIdentity } = preparedBoxState(prepared); - if (!release.execution) fail('Box does not declare an execution entry point.'); - const callerArgs = stringArguments(options.args ?? []); - const adapter = boxTargetAdapter(release.target); - try { - assertNativeHost(adapter); - } catch { - fail( - `Box target ${boxTargetId(release.target)} cannot run on ${process.platform}/${process.arch}; ` - + `requires ${adapter.host.platform}/${adapter.host.arch}.`, - ); - } - - let rootMetadata; - try { - rootMetadata = await lstat(prepared.root); - } catch (error) { - if (error?.code === 'ENOENT') { - fail('Prepared box root no longer matches the prepared box.'); - } - throw error; - } - if (!rootMetadata.isDirectory() - || rootMetadata.dev !== rootIdentity.device - || rootMetadata.ino !== rootIdentity.inode) { - fail('Prepared box root no longer matches the prepared box.'); - } - const files = new Set(await collectFiles(prepared.root)); - if (!files.has(release.pythonEntryPoint)) { - fail(`Prepared box is missing ${release.pythonEntryPoint}.`); - } - assertExecutionFiles({ - execution: release.execution, - adapter, - pythonVersion: release.provenance.pythonVersion, - files, - }); - await verifyRequiredAssets(prepared.root, prepared.requiredAssets); - - const python = join(prepared.root, ...safeRelativePath(release.pythonEntryPoint).split('/')); - const executionArgs = release.execution.kind === 'python-script' - ? [join(prepared.root, ...safeRelativePath(release.execution.script).split('/'))] - : ['-m', release.execution.module]; - executionArgs.push(...release.execution.defaultArgs, ...callerArgs); - - const spawn = options.spawn ?? spawnProcess; - const child = spawn(python, executionArgs, { - cwd: prepared.root, - env: { ...process.env, ...options.env }, - stdio: [ - options.stdin ?? 'inherit', - options.stdout ?? 'inherit', - options.stderr ?? 'inherit', - ], - shell: false, - }); - return waitForChild(child, options.signalSource ?? process); -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/consumer/verify-and-extract.mjs b/.scrollcase-runtime-types-RzUfxr/src/consumer/verify-and-extract.mjs deleted file mode 100644 index 32e14f5..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/consumer/verify-and-extract.mjs +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Verification and durable preparation of a caller-supplied local box. - * - * A prepared box is deliberately opaque. Its public receipt contains useful signed identity and - * audit data, while private state binds that exact object to the release Scrollcase verified. A - * caller therefore cannot construct an object that looks prepared and use it to bypass the trust - * chain before execution. - */ - -import { - lstat, - mkdir, - mkdtemp, - rename, - rm, -} from 'node:fs/promises'; -import { basename, dirname, join, resolve } from 'node:path'; -import { extractZipArchive } from '../build/archive.mjs'; -import { payloadSize, safeRelativePath, sha256File } from '../build/filesystem.mjs'; -import { fail } from '../build/process.mjs'; -import { inspectBoxArchive } from '../build/verify.mjs'; -import { boxTargetId } from '../contract/targets.mjs'; - -/** - * An on-demand asset whose signed bytes the caller must place under `root` before execution. - * - * @typedef {object} RequiredAsset - * @property {string} url - * @property {string} relativePath - * @property {number} sizeBytes - * @property {string} sha256 - */ - -/** - * The immutable result of a successfully verified and atomically prepared local box. - * - * @typedef {object} PreparedBox - * @property {'prepared'} status - * @property {string} root absolute extracted box root - * @property {string} boxId - * @property {string} modelId - * @property {string} runtimeId - * @property {string} version - * @property {import('../contract/types/index.d.ts').BoxTarget} target - * @property {string} targetId - * @property {string} pythonEntryPoint - * @property {import('../contract/types/index.d.ts').BoxExecution | null} execution - * @property {readonly RequiredAsset[]} requiredAssets assets the caller must materialize, never - * downloaded by Scrollcase - * @property {readonly string[]} signingKeyIds - * @property {string} releasePayloadSha256 - * @property {string} archiveSha256 - * @property {number} archiveSizeBytes - * @property {number} installedSizeBytes logical size of the verified extracted payload - */ - -/** @type {WeakMap} */ -const preparedBoxes = new WeakMap(); - -function freezeValue(value) { - if (!value || typeof value !== 'object' || Object.isFrozen(value)) return value; - for (const nested of Object.values(value)) freezeValue(nested); - return Object.freeze(value); -} - -async function pathExists(path) { - try { - await lstat(path); - return true; - } catch (error) { - if (error?.code === 'ENOENT') return false; - throw error; - } -} - -/** - * Returns the private, verified release bound to a prepared receipt. - * - * This is internal to the consumer module graph; it is not re-exported from the package surface. - */ -export function preparedBoxState(/** @type {unknown} */ prepared) { - const state = preparedBoxes.get(prepared); - if (!state) fail('Expected a PreparedBox returned by verifyAndExtractBox().'); - return state; -} - -/** - * Verifies and extracts one local box without executing any code from it. - * - * The destination must not exist. Extraction happens in a fresh sibling directory so the final - * rename stays on one filesystem and exposes either the complete verified tree or nothing. - * - * @param {string} releaseDocumentPath - * @param {{ publicPath: string, archive?: string | null, destination: string }} options - * @returns {Promise>} - */ -export async function verifyAndExtractBox(releaseDocumentPath, { - publicPath, - archive = null, - destination, -}) { - if (!destination) fail('A destination is required to prepare a box.'); - const finalRoot = resolve(destination); - if (await pathExists(finalRoot)) fail(`Destination already exists: ${finalRoot}`); - - const inspected = await inspectBoxArchive(releaseDocumentPath, { publicPath, archive }); - const { - archivePath, - signed, - release, - } = inspected; - const requiredAssets = release.weights === 'on-demand' ? release.assets : []; - for (const asset of requiredAssets) safeRelativePath(asset.relativePath); - - const parent = dirname(finalRoot); - await mkdir(parent, { recursive: true }); - if (await pathExists(finalRoot)) fail(`Destination already exists: ${finalRoot}`); - - const stageRoot = await mkdtemp(join(parent, `.scrollcase-prepare-${basename(finalRoot)}-`)); - const extractedRoot = join(stageRoot, 'payload'); - try { - await extractZipArchive(archivePath, extractedRoot); - const extractedSize = await payloadSize(extractedRoot); - if (release.installedSizeBytes !== undefined - && extractedSize !== release.installedSizeBytes) { - fail('Extracted payload size does not match the signed release.'); - } - - // Re-check the source after extraction. This catches a local archive being replaced between - // the initial trust decision and the move into the caller's durable destination. - if (await sha256File(archivePath) !== release.archive.sha256) { - fail('Archive SHA-256 changed during extraction.'); - } - const stagedMetadata = await lstat(extractedRoot); - if (await pathExists(finalRoot)) fail(`Destination already exists: ${finalRoot}`); - await rename(extractedRoot, finalRoot); - const installedMetadata = await lstat(finalRoot); - if (installedMetadata.dev !== stagedMetadata.dev || installedMetadata.ino !== stagedMetadata.ino) { - fail('Prepared destination identity changed during installation.'); - } - - const frozenRelease = freezeValue(release); - const receipt = freezeValue({ - status: 'prepared', - root: finalRoot, - boxId: release.boxId, - modelId: release.modelId, - runtimeId: release.runtimeId, - version: release.version, - target: release.target, - targetId: boxTargetId(release.target), - pythonEntryPoint: release.pythonEntryPoint, - execution: release.execution ?? null, - requiredAssets, - signingKeyIds: signed.signatures.map((signature) => signature.keyId), - releasePayloadSha256: signed.payloadSha256, - archiveSha256: release.archive.sha256, - archiveSizeBytes: release.archive.sizeBytes, - installedSizeBytes: extractedSize, - }); - preparedBoxes.set(receipt, { - release: frozenRelease, - rootIdentity: { - device: installedMetadata.dev, - inode: installedMetadata.ino, - }, - }); - return receipt; - } finally { - await rm(stageRoot, { recursive: true, force: true }); - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/browser.mjs b/.scrollcase-runtime-types-RzUfxr/src/contract/browser.mjs deleted file mode 100644 index 1141613..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/browser.mjs +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Browser-safe reference helpers for the Scrollcase contract. - * - * The full `scrollcase/contract` entry point also decodes and hashes signed payloads through Node's - * crypto implementation. Consumers that only need target identity, document names, constants, or - * the structural envelope guard can use this entry point in browsers, Workers, and Node alike. - */ - -export { - assertNativeHost, - assertPythonEntryPoint, - condaSubdir, - pixiAccelerator, - boxTargetAdapter, - boxTargetAdapters, - boxTargetId, -} from './targets.mjs'; - -export { - CHANNELS, - DEFAULT_DOCUMENT_NAMESPACE, - PAYLOAD_ENCODING, - BOX_SCHEMA_VERSION, - SIGNATURE_ALGORITHM, - documentKinds, - isSignedBoxDocument, - parseDocumentKind, -} from './document-shape.mjs'; diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/document-shape.mjs b/.scrollcase-runtime-types-RzUfxr/src/contract/document-shape.mjs deleted file mode 100644 index 2aa9fbc..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/document-shape.mjs +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Platform-neutral parts of the signed-document contract. - * - * Shape checks and namespacing have no reason to depend on Node. Keeping them in this module lets - * browser and Worker consumers share the reference implementation without pulling in the - * cryptographic decoder, while the main contract entry point continues to expose the complete API. - */ - -/** Format version carried by every document this contract describes. */ -export const BOX_SCHEMA_VERSION = 2; - -/** The only payload encoding the format defines. */ -export const PAYLOAD_ENCODING = 'base64-json-utf8'; - -/** The only signature algorithm the format defines. */ -export const SIGNATURE_ALGORITHM = 'ed25519'; - -/** - * Namespace prefixing every document's `kind` discriminator. - * - * A project that already publishes boxes owns its own namespace and must keep emitting it, or its - * installed clients stop recognizing the documents they are handed. So the namespace is the - * consumer's to declare, not the tool's to impose: this is only the default used by a project that - * has no published history to preserve. - */ -export const DEFAULT_DOCUMENT_NAMESPACE = 'scrollcase.box'; - -const DOCUMENT_TYPES = Object.freeze(['release', 'channel', 'revocations']); -const NAMESPACE_PATTERN = /^[a-z0-9]+(?:[.-][a-z0-9]+)*$/; - -/** - * Returns the `kind` discriminator for each document type under a namespace. - * - * Pass the namespace a project has already published under to keep its documents byte-compatible; - * omit it for a new project. - * - * @param {string} [namespace] defaults to `scrollcase.box` - * @returns {Readonly<{ release: string, channel: string, revocations: string }>} - * @throws {TypeError} when the namespace is not a dotted lowercase identifier - */ -export function documentKinds(namespace = DEFAULT_DOCUMENT_NAMESPACE) { - if (typeof namespace !== 'string' || !NAMESPACE_PATTERN.test(namespace)) { - throw new TypeError(`Invalid document namespace: ${namespace}`); - } - return Object.freeze(Object.fromEntries( - DOCUMENT_TYPES.map((type) => [type, `${namespace}.${type}`]), - )); -} - -/** - * Splits a `kind` back into its namespace and document type, or returns null if it is not one. - * - * @param {unknown} kind - * @returns {{ namespace: string, type: 'release' | 'channel' | 'revocations' } | null} null when - * the value is not a document kind at all - */ -export function parseDocumentKind(kind) { - if (typeof kind !== 'string') return null; - const separator = kind.lastIndexOf('.'); - if (separator <= 0) return null; - const namespace = kind.slice(0, separator); - const type = kind.slice(separator + 1); - if (!DOCUMENT_TYPES.includes(type) || !NAMESPACE_PATTERN.test(namespace)) return null; - return { namespace, type }; -} - -/** Channels a box may be published to, ordered from least to most stable. */ -export const CHANNELS = Object.freeze(['nightly', 'beta', 'stable']); - -/** - * Reports whether a value is a structurally valid signed envelope. - * - * This is a shape check, not a verification: it says the document is worth attempting to verify, - * never that its signature is good. Callers must still verify the payload hash and at least one - * signature against a trusted key before acting on the contents. - * - * @param {unknown} value - * @returns {value is import('./types/index.d.ts').SignedBoxDocument} true when the envelope is - * well formed and therefore worth verifying — never that its signature is valid - */ -export function isSignedBoxDocument(value) { - if (!value || typeof value !== 'object') return false; - return value.schemaVersion === BOX_SCHEMA_VERSION - && value.payloadEncoding === PAYLOAD_ENCODING - && typeof value.payloadBase64 === 'string' - && typeof value.payloadSha256 === 'string' - && Array.isArray(value.signatures) - && value.signatures.length > 0 - && value.signatures.every((signature) => signature?.algorithm === SIGNATURE_ALGORITHM - && typeof signature.keyId === 'string' - && typeof signature.signatureBase64 === 'string'); -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/documents.mjs b/.scrollcase-runtime-types-RzUfxr/src/contract/documents.mjs deleted file mode 100644 index 78f12a4..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/documents.mjs +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Reference implementation of the Scrollcase signed-document envelope. - * - * Signed documents carry their payload as exact base64-encoded JSON rather than canonicalized JSON. - * That choice is deliberate: verifying a signature then means hashing bytes that were transmitted - * verbatim, so Node, Rust, a Worker, and any future client agree without each maintaining a - * canonical-JSON implementation — historically the richest source of cross-language signature bugs. - */ - -import { createHash } from 'node:crypto'; -import { isSignedBoxDocument } from './document-shape.mjs'; - -export { - BOX_SCHEMA_VERSION, - CHANNELS, - DEFAULT_DOCUMENT_NAMESPACE, - PAYLOAD_ENCODING, - SIGNATURE_ALGORITHM, - documentKinds, - isSignedBoxDocument, - parseDocumentKind, -} from './document-shape.mjs'; - -/** - * Decodes an envelope's payload without verifying any signature. - * - * Throws when the envelope is malformed or when the embedded payload hash does not match the bytes, - * which catches a truncated or edited document before its contents are ever read. - * - * @param {import('./types/index.d.ts').SignedBoxDocument} document - * @returns {unknown} the decoded payload, still unverified - * @throws {TypeError} when the envelope is malformed - * @throws {Error} when the embedded payload hash does not match the bytes - */ -export function decodeDocumentPayload(document) { - if (document?.schemaVersion === 1) { - throw new TypeError('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); - } - if (!isSignedBoxDocument(document)) { - throw new TypeError('Not a signed box document'); - } - const bytes = Buffer.from(document.payloadBase64, 'base64'); - const digest = createHash('sha256').update(bytes).digest('hex'); - if (digest !== document.payloadSha256) { - throw new Error('Signed box payload hash does not match its bytes'); - } - return JSON.parse(bytes.toString('utf8')); -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/consumer-conformance.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/consumer-conformance.json deleted file mode 100644 index 6b6f236..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/consumer-conformance.json +++ /dev/null @@ -1,516 +0,0 @@ -{ - "schemaVersion": 1, - "description": "Language-neutral semantic cases shared by the Node and Python Scrollcase consumers.", - "errorPatterns": { - "invalid-signature": "no valid signature", - "altered-payload": "Signed payload SHA-256 mismatch", - "archive-hash": "Archive SHA-256 mismatch", - "archive-size": "Archive size mismatch", - "manifest-disagreement": "box.json mismatch: modelId", - "execution-disagreement": "box.json mismatch: execution", - "missing-interpreter": "Archive is missing venv/", - "missing-script": "Execution script is missing", - "missing-module": "Execution module is not discoverable", - "unsafe-path": "Unsafe relative path|invalid relative path|absolute path", - "link-entry": "link does not resolve to a file inside the payload|would be written through a link|link target is too long", - "special-entry": "special entries", - "encrypted-entry": "Encrypted ZIP entries", - "entry-collision": "Archive entry collides with another entry", - "existing-destination": "Destination already exists", - "asset-missing": "asset is missing", - "asset-size": "asset size mismatch", - "asset-hash": "asset SHA-256 mismatch", - "spawn-failure": "failed to start|fixture spawn failed" - }, - "cases": [ - { - "id": "valid-local-signer", - "action": "prepare", - "fixture": { - "signer": "local" - }, - "expected": { - "outcome": "prepared", - "receipt": { - "status": "prepared", - "boxId": "consumer-fixture", - "executionKind": "python-script", - "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", - "targetId": "$NATIVE_TARGET" - } - } - }, - { - "id": "valid-external-signer", - "action": "prepare", - "fixture": { - "signer": "external" - }, - "expected": { - "outcome": "prepared", - "receipt": { - "status": "prepared", - "boxId": "consumer-fixture", - "executionKind": "python-script", - "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", - "targetId": "$NATIVE_TARGET" - } - } - }, - { - "id": "altered-signature", - "action": "prepare", - "mutation": "alter-signature", - "expected": { - "outcome": "rejected", - "error": "invalid-signature", - "destinationExists": false - } - }, - { - "id": "altered-payload", - "action": "prepare", - "mutation": "alter-payload", - "expected": { - "outcome": "rejected", - "error": "altered-payload", - "destinationExists": false - } - }, - { - "id": "altered-archive-hash", - "action": "prepare", - "mutation": "alter-archive-bytes", - "expected": { - "outcome": "rejected", - "error": "archive-hash", - "destinationExists": false - } - }, - { - "id": "altered-archive-size", - "action": "prepare", - "mutation": "alter-archive-size", - "expected": { - "outcome": "rejected", - "error": "archive-size", - "destinationExists": false - } - }, - { - "id": "release-box-disagreement", - "action": "prepare", - "mutation": "alter-release-model", - "expected": { - "outcome": "rejected", - "error": "manifest-disagreement", - "destinationExists": false - } - }, - { - "id": "altered-execution-metadata", - "action": "prepare", - "mutation": "alter-release-execution", - "expected": { - "outcome": "rejected", - "error": "execution-disagreement", - "destinationExists": false - } - }, - { - "id": "missing-interpreter", - "action": "prepare", - "mutation": "remove-interpreter", - "expected": { - "outcome": "rejected", - "error": "missing-interpreter", - "destinationExists": false - } - }, - { - "id": "missing-script", - "action": "prepare", - "mutation": "remove-script", - "expected": { - "outcome": "rejected", - "error": "missing-script", - "destinationExists": false - } - }, - { - "id": "missing-module", - "action": "prepare", - "fixture": { - "execution": "module" - }, - "mutation": "remove-module", - "expected": { - "outcome": "rejected", - "error": "missing-module", - "destinationExists": false - } - }, - { - "id": "traversal-entry", - "action": "prepare", - "mutation": "add-traversal-entry", - "expected": { - "outcome": "rejected", - "error": "unsafe-path", - "destinationExists": false - } - }, - { - "id": "absolute-entry", - "action": "prepare", - "mutation": "add-absolute-entry", - "expected": { - "outcome": "rejected", - "error": "unsafe-path", - "destinationExists": false - } - }, - { - "id": "link-entry", - "action": "prepare", - "mutation": "add-link-entry", - "expected": { - "outcome": "rejected", - "error": "link-entry", - "destinationExists": false - } - }, - { - "id": "linked-interpreter", - "action": "prepare", - "mutation": "link-interpreter", - "requiresSymlinks": true, - "expected": { - "outcome": "prepared", - "receipt": { - "status": "prepared", - "boxId": "consumer-fixture", - "executionKind": "python-script", - "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", - "targetId": "$NATIVE_TARGET" - } - } - }, - { - "id": "special-entry", - "action": "prepare", - "mutation": "add-special-entry", - "expected": { - "outcome": "rejected", - "error": "special-entry", - "destinationExists": false - } - }, - { - "id": "encrypted-entry", - "action": "prepare", - "mutation": "encrypt-entry", - "expected": { - "outcome": "rejected", - "error": "encrypted-entry", - "destinationExists": false - } - }, - { - "id": "extraction-collision", - "action": "prepare", - "mutation": "duplicate-entry", - "expected": { - "outcome": "rejected", - "error": "entry-collision", - "destinationExists": false - } - }, - { - "id": "file-directory-collision", - "action": "prepare", - "mutation": "file-directory-collision", - "expected": { - "outcome": "rejected", - "error": "entry-collision", - "destinationExists": false - } - }, - { - "id": "existing-destination", - "action": "prepare", - "mutation": "create-destination", - "expected": { - "outcome": "rejected", - "error": "existing-destination", - "destinationExists": true - } - }, - { - "id": "macos-entry-point", - "action": "prepare", - "fixture": { - "target": "macos-aarch64-cpu" - }, - "expected": { - "outcome": "prepared", - "receipt": { - "status": "prepared", - "boxId": "consumer-fixture", - "executionKind": "python-script", - "requiredAssetCount": 0, - "pythonEntryPoint": "venv/bin/python", - "targetId": "macos-aarch64-cpu" - } - } - }, - { - "id": "linux-entry-point", - "action": "prepare", - "fixture": { - "target": "linux-x86_64-cpu" - }, - "expected": { - "outcome": "prepared", - "receipt": { - "status": "prepared", - "boxId": "consumer-fixture", - "executionKind": "python-script", - "requiredAssetCount": 0, - "pythonEntryPoint": "venv/bin/python", - "targetId": "linux-x86_64-cpu" - } - } - }, - { - "id": "windows-entry-point", - "action": "prepare", - "fixture": { - "target": "windows-x86_64-cpu" - }, - "expected": { - "outcome": "prepared", - "receipt": { - "status": "prepared", - "boxId": "consumer-fixture", - "executionKind": "python-script", - "requiredAssetCount": 0, - "pythonEntryPoint": "venv/python.exe", - "targetId": "windows-x86_64-cpu" - } - } - }, - { - "id": "persistent-prepared-execution", - "action": "run-prepared", - "runtime": { - "exitCode": 0 - }, - "expected": { - "outcome": "completed", - "result": { - "exitCode": 0, - "signal": null - }, - "persistentRootExists": true, - "spawned": true - } - }, - { - "id": "default-user-argument-ordering", - "action": "run-prepared", - "runtime": { - "args": [ - "--caller", - "caller value" - ], - "exitCode": 0, - "inspectInvocation": true - }, - "expected": { - "outcome": "completed", - "result": { - "exitCode": 0, - "signal": null - }, - "argv": [ - "$BOX/$NATIVE_PYTHON", - "$BOX/app/main.py", - "--default", - "value with spaces", - "--caller", - "caller value" - ], - "cwd": "$BOX", - "shell": false - } - }, - { - "id": "shell-metacharacter-preservation", - "action": "run-prepared", - "runtime": { - "args": [ - "$(touch never)", - "semi;colon", - "quote'\"value" - ], - "exitCode": 0, - "inspectInvocation": true - }, - "expected": { - "outcome": "completed", - "result": { - "exitCode": 0, - "signal": null - }, - "argv": [ - "$BOX/$NATIVE_PYTHON", - "$BOX/app/main.py", - "--default", - "value with spaces", - "$(touch never)", - "semi;colon", - "quote'\"value" - ], - "cwd": "$BOX", - "shell": false - } - }, - { - "id": "standard-stream-forwarding", - "action": "run-prepared", - "runtime": { - "exitCode": 0, - "streams": true - }, - "expected": { - "outcome": "completed", - "result": { - "exitCode": 0, - "signal": null - }, - "streamsPreserved": true - } - }, - { - "id": "child-non-zero-exit", - "action": "run-prepared", - "runtime": { - "exitCode": 23 - }, - "expected": { - "outcome": "completed", - "result": { - "exitCode": 23, - "signal": null - }, - "spawned": true - } - }, - { - "id": "signal-forwarding", - "action": "run-prepared", - "runtime": { - "signal": "SIGTERM" - }, - "expected": { - "outcome": "completed", - "result": { - "exitCode": null, - "signal": "SIGTERM" - }, - "forwardedSignal": "SIGTERM" - } - }, - { - "id": "temporary-cleanup-success", - "action": "run-box", - "runtime": { - "exitCode": 0 - }, - "expected": { - "outcome": "completed", - "result": { - "exitCode": 0, - "signal": null - }, - "temporaryDirectoryEmpty": true - } - }, - { - "id": "temporary-cleanup-failure", - "action": "run-box", - "runtime": { - "spawnError": true - }, - "expected": { - "outcome": "rejected", - "error": "spawn-failure", - "temporaryDirectoryEmpty": true - } - }, - { - "id": "temporary-cleanup-signal", - "action": "run-box", - "runtime": { - "signal": "SIGTERM" - }, - "expected": { - "outcome": "completed", - "result": { - "exitCode": null, - "signal": "SIGTERM" - }, - "temporaryDirectoryEmpty": true - } - }, - { - "id": "on-demand-asset-missing", - "action": "run-prepared", - "fixture": { - "requiredAsset": true - }, - "runtime": { - "assetState": "missing" - }, - "expected": { - "outcome": "rejected", - "error": "asset-missing", - "spawned": false - } - }, - { - "id": "on-demand-asset-size-mismatch", - "action": "run-prepared", - "fixture": { - "requiredAsset": true - }, - "runtime": { - "assetState": "wrong-size" - }, - "expected": { - "outcome": "rejected", - "error": "asset-size", - "spawned": false - } - }, - { - "id": "on-demand-asset-hash-mismatch", - "action": "run-prepared", - "fixture": { - "requiredAsset": true - }, - "runtime": { - "assetState": "wrong-hash" - }, - "expected": { - "outcome": "rejected", - "error": "asset-hash", - "spawned": false - } - } - ] -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/box-manifest.example.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/box-manifest.example.json deleted file mode 100644 index 3613501..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/box-manifest.example.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "schemaVersion": 2, - "boxId": "example-model", - "modelId": "example-org-example-model", - "runtimeId": "example-model-runtime", - "version": "1.0.0", - "target": { - "platform": "macos", - "arch": "aarch64", - "accelerator": "metal" - }, - "pythonEntryPoint": "venv/bin/python", - "modelCacheSubdir": "model-cache/example-model", - "selfTest": { - "pythonImports": [ - "torch", - "numpy" - ], - "timeoutSeconds": 180 - }, - "provenance": { - "scrollId": "example-model-macos-arm64-metal", - "scrollVersion": "1.0.0", - "builderRevision": "4c1d9b7e2a0f5836d419e7c05b3a8f61d2704e93", - "sourceTreeDirty": false, - "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", - "pythonVersion": "3.11.15", - "pixiVersion": "0.73.0", - "dependencyLockSha256": "3b1f8c47a2d9e05b6c7418af23d5e69017b4c8ad91e2f350768bd4ca19e0f5b7", - "builtAt": "2026-07-25T12:00:00+00:00" - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/channel-manifest.example.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/channel-manifest.example.json deleted file mode 100644 index 7682e91..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/channel-manifest.example.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "schemaVersion": 2, - "kind": "scrollcase.box.channel", - "channel": "beta", - "boxId": "example-model", - "target": { - "platform": "macos", - "arch": "aarch64", - "accelerator": "metal" - }, - "updatedAt": "2026-07-25T12:05:00+00:00", - "cohortSalt": "41547ba146d88df877b97a331854567e", - "releases": [ - { - "version": "1.0.0", - "releaseManifestUrl": "https://assets.example.org/boxes/boxes/example-model/1.0.0/macos-aarch64-metal/c1a0f67d97543c9f0deccffdd00fbad45662ceed828f84e866b997ab4b019d1f.release.json", - "rolloutPercentage": 100 - } - ] -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/release-manifest.example.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/release-manifest.example.json deleted file mode 100644 index 9a04e5b..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/release-manifest.example.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "schemaVersion": 2, - "kind": "scrollcase.box.release", - "boxId": "example-model", - "modelId": "example-org-example-model", - "runtimeId": "example-model-runtime", - "version": "1.0.0", - "target": { - "platform": "macos", - "arch": "aarch64", - "accelerator": "metal" - }, - "compatibility": { - "minHostAppVersion": "1.0.0", - "minMacosVersion": "13.0", - "minRamGb": 8 - }, - "archive": { - "format": "zip", - "url": "https://assets.example.org/boxes/boxes/example-model/1.0.0/macos-aarch64-metal/7d2c9a41e8b350f6c174a9de20358bf41c6e97d05a8b3f2619e4c7081da5b3f2.zip", - "sha256": "7d2c9a41e8b350f6c174a9de20358bf41c6e97d05a8b3f2619e4c7081da5b3f2", - "sizeBytes": 655752216 - }, - "installedSizeBytes": 1892340112, - "pythonEntryPoint": "venv/bin/python", - "modelCacheSubdir": "model-cache/example-model", - "selfTest": { - "pythonImports": [ - "torch", - "numpy" - ], - "timeoutSeconds": 180 - }, - "provenance": { - "scrollId": "example-model-macos-arm64-metal", - "scrollVersion": "1.0.0", - "builderRevision": "4c1d9b7e2a0f5836d419e7c05b3a8f61d2704e93", - "sourceTreeDirty": false, - "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", - "pythonVersion": "3.11.15", - "pixiVersion": "0.73.0", - "dependencyLockSha256": "3b1f8c47a2d9e05b6c7418af23d5e69017b4c8ad91e2f350768bd4ca19e0f5b7", - "builtAt": "2026-07-25T12:00:00+00:00" - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll-pixi.example.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll-pixi.example.json deleted file mode 100644 index b758030..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll-pixi.example.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "$schema": "https://scrollcase.dev/schema/v2/scroll.schema.json", - "schemaVersion": 2, - "scrollId": "example-model-linux-x86_64-cuda12.9", - "scrollVersion": "1.0.0", - "boxId": "example-model", - "modelId": "example-org-example-model", - "runtimeId": "example-model-runtime", - "version": "1.0.0", - "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", - "target": { - "platform": "linux", - "arch": "x86_64", - "accelerator": "cuda", - "cudaVersion": "12.9" - }, - "compatibility": { - "minHostAppVersion": "1.0.0", - "minRamGb": 16, - "minNvidiaDriverVersion": "525.60.13" - }, - "pythonVersion": "3.11.15", - "pixiVersion": "0.73.0", - "condaDependencyLicenseAudit": "legal/audits/example-model-linux-x86_64-cuda12.9.json", - "pythonEntryPoint": "venv/bin/python", - "modelCacheSubdir": "model-cache/example-model", - "assetBaseUrl": "https://assets.example.org/boxes", - "assets": [ - { - "url": "https://assets.example.org/example-model/weights.safetensors", - "relativePath": "model-cache/example-model/weights.safetensors", - "sizeBytes": 205385258, - "sha256": "6cb5d451ab5c4b33eb673adbe4fddc61d2389df1b89b7651a9fe2e557572b922" - }, - { - "url": "https://codeload.example.org/example-org/example-model/zip/9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", - "relativePath": ".sources/example-model-source.zip", - "sizeBytes": 12117557, - "sha256": "1b825a687b4855dbb8c7b530f17cc246ef677d93d0e2df3fbc82330ff9d189db" - } - ], - "assetArchives": [ - { - "relativePath": ".sources/example-model-source.zip", - "format": "zip", - "destination": "source/example-model", - "stripComponents": 1, - "removeAfterExtract": true - } - ], - "localFiles": [ - { - "sourcePath": "legal/notices/example-model-THIRD-PARTY.txt", - "relativePath": "THIRD_PARTY_NOTICES/example-model.txt", - "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - } - ], - "prunePaths": [ - "source/example-model/docs", - "source/example-model/tests", - "venv/lib/python3.11/site-packages/pip", - "venv/lib/python3.11/tkinter" - ], - "selfTest": { - "imports": ["torch", "numpy"], - "files": [ - "model-cache/example-model/weights.safetensors", - "source/example-model/LICENSE" - ], - "pythonCode": "assert torch.cuda.is_available()" - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll.example.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll.example.json deleted file mode 100644 index 4790c73..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll.example.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "$schema": "https://scrollcase.dev/schema/v2/scroll.schema.json", - "schemaVersion": 2, - "scrollId": "example-model-macos-arm64-metal", - "scrollVersion": "1.0.0", - "boxId": "example-model", - "modelId": "example-org-example-model", - "runtimeId": "example-model-runtime", - "version": "1.0.0", - "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", - "target": { - "platform": "macos", - "arch": "aarch64", - "accelerator": "metal" - }, - "compatibility": { - "minHostAppVersion": "1.0.0", - "minMacosVersion": "13.0", - "minRamGb": 8 - }, - "pythonVersion": "3.11.15", - "pythonEntryPoint": "venv/bin/python", - "modelCacheSubdir": "model-cache/example-model", - "assetBaseUrl": "https://assets.example.org/boxes", - "assets": [ - { - "url": "https://assets.example.org/example-model/weights.safetensors", - "relativePath": "model-cache/example-model/weights.safetensors", - "sizeBytes": 205385258, - "sha256": "6cb5d451ab5c4b33eb673adbe4fddc61d2389df1b89b7651a9fe2e557572b922" - } - ], - "selfTest": { - "imports": [ - "torch", - "numpy" - ], - "files": [ - "model-cache/example-model/weights.safetensors" - ] - }, - "pixiVersion": "0.73.0", - "condaDependencyLicenseAudit": "legal/audits/example-model-macos-arm64-metal.json" -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.example.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.example.json deleted file mode 100644 index 64b1a63..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.example.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schemaVersion": 2, - "payloadEncoding": "base64-json-utf8", - "payloadBase64": "ewogICJzY2hlbWFWZXJzaW9uIjogMiwKICAia2luZCI6ICJzY3JvbGxjYXNlLmJveC5yZWxlYXNlIiwKICAiYm94SWQiOiAiZXhhbXBsZS1tb2RlbCIsCiAgIm1vZGVsSWQiOiAiZXhhbXBsZS1vcmctZXhhbXBsZS1tb2RlbCIsCiAgInJ1bnRpbWVJZCI6ICJleGFtcGxlLW1vZGVsLXJ1bnRpbWUiLAogICJ2ZXJzaW9uIjogIjEuMC4wIiwKICAidGFyZ2V0IjogewogICAgInBsYXRmb3JtIjogIm1hY29zIiwKICAgICJhcmNoIjogImFhcmNoNjQiLAogICAgImFjY2VsZXJhdG9yIjogIm1ldGFsIgogIH0sCiAgImNvbXBhdGliaWxpdHkiOiB7CiAgICAibWluSG9zdEFwcFZlcnNpb24iOiAiMS4wLjAiLAogICAgIm1pbk1hY29zVmVyc2lvbiI6ICIxMy4wIiwKICAgICJtaW5SYW1HYiI6IDgKICB9LAogICJhcmNoaXZlIjogewogICAgImZvcm1hdCI6ICJ6aXAiLAogICAgInVybCI6ICJodHRwczovL2Fzc2V0cy5leGFtcGxlLm9yZy9ib3hlcy9ib3hlcy9leGFtcGxlLW1vZGVsLzEuMC4wL21hY29zLWFhcmNoNjQtbWV0YWwvN2QyYzlhNDFlOGIzNTBmNmMxNzRhOWRlMjAzNThiZjQxYzZlOTdkMDVhOGIzZjI2MTllNGM3MDgxZGE1YjNmMi56aXAiLAogICAgInNoYTI1NiI6ICI3ZDJjOWE0MWU4YjM1MGY2YzE3NGE5ZGUyMDM1OGJmNDFjNmU5N2QwNWE4YjNmMjYxOWU0YzcwODFkYTViM2YyIiwKICAgICJzaXplQnl0ZXMiOiA2NTU3NTIyMTYKICB9LAogICJpbnN0YWxsZWRTaXplQnl0ZXMiOiAxODkyMzQwMTEyLAogICJweXRob25FbnRyeVBvaW50IjogInZlbnYvYmluL3B5dGhvbiIsCiAgIm1vZGVsQ2FjaGVTdWJkaXIiOiAibW9kZWwtY2FjaGUvZXhhbXBsZS1tb2RlbCIsCiAgInNlbGZUZXN0IjogewogICAgInB5dGhvbkltcG9ydHMiOiBbCiAgICAgICJ0b3JjaCIsCiAgICAgICJudW1weSIKICAgIF0sCiAgICAidGltZW91dFNlY29uZHMiOiAxODAKICB9LAogICJwcm92ZW5hbmNlIjogewogICAgInNjcm9sbElkIjogImV4YW1wbGUtbW9kZWwtbWFjb3MtYXJtNjQtbWV0YWwiLAogICAgInNjcm9sbFZlcnNpb24iOiAiMS4wLjAiLAogICAgImJ1aWxkZXJSZXZpc2lvbiI6ICI0YzFkOWI3ZTJhMGY1ODM2ZDQxOWU3YzA1YjNhOGY2MWQyNzA0ZTkzIiwKICAgICJzb3VyY2VUcmVlRGlydHkiOiBmYWxzZSwKICAgICJzb3VyY2VSZXZpc2lvbiI6ICI5ZjBkM2ExYzRiNWU2ZjcwODE5MjBhM2I0YzVkNmU3ZjgwOTEwYTJiIiwKICAgICJweXRob25WZXJzaW9uIjogIjMuMTEuMTUiLAogICAgInBpeGlWZXJzaW9uIjogIjAuNzMuMCIsCiAgICAiZGVwZW5kZW5jeUxvY2tTaGEyNTYiOiAiM2IxZjhjNDdhMmQ5ZTA1YjZjNzQxOGFmMjNkNWU2OTAxN2I0YzhhZDkxZTJmMzUwNzY4YmQ0Y2ExOWUwZjViNyIsCiAgICAiYnVpbHRBdCI6ICIyMDI2LTA3LTI1VDEyOjAwOjAwKzAwOjAwIgogIH0KfQo=", - "payloadSha256": "bbf60de7d31035b2bfcb98c6c57624220f3bf59900391189f5e55da955055bc6", - "signatures": [ - { - "algorithm": "ed25519", - "keyId": "scrollcase-example-v2", - "signatureBase64": "c4ftYhvfGwJicd3kK9DzyTj+HNvJeiCfwk049BKk8hxSgAMxxU8BE51Q51ZkSK0mPolXRJ9pz52HO3KoKD1mAA==" - } - ] -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.public-key.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.public-key.json deleted file mode 100644 index f506853..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.public-key.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "algorithm": "ed25519", - "keyId": "scrollcase-example-v2", - "publicKeyBase64": "frGNF6Fa2cw9m5HvWYBacydXujD4+ldqo6xGSCnCFyc=" -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/target-id-contract.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/target-id-contract.json deleted file mode 100644 index a18a9ed..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/target-id-contract.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "valid": [ - { - "name": "macOS arm64 Metal", - "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, - "targetId": "macos-aarch64-metal" - }, - { - "name": "macOS arm64 CPU", - "target": { "platform": "macos", "arch": "aarch64", "accelerator": "cpu" }, - "targetId": "macos-aarch64-cpu" - }, - { - "name": "Linux x86_64 CPU", - "target": { "platform": "linux", "arch": "x86_64", "accelerator": "cpu" }, - "targetId": "linux-x86_64-cpu" - }, - { - "name": "Linux x86_64 CUDA 12.4", - "target": { "platform": "linux", "arch": "x86_64", "accelerator": "cuda", "cudaVersion": "12.4" }, - "targetId": "linux-x86_64-cuda12.4" - }, - { - "name": "Windows x86_64 CPU", - "target": { "platform": "windows", "arch": "x86_64", "accelerator": "cpu" }, - "targetId": "windows-x86_64-cpu" - }, - { - "name": "Windows x86_64 CUDA 12.4", - "target": { "platform": "windows", "arch": "x86_64", "accelerator": "cuda", "cudaVersion": "12.4" }, - "targetId": "windows-x86_64-cuda12.4" - } - ], - "invalid": [ - { - "name": "CUDA without a version", - "target": { "platform": "linux", "arch": "x86_64", "accelerator": "cuda" } - }, - { - "name": "CPU with a CUDA version", - "target": { "platform": "linux", "arch": "x86_64", "accelerator": "cpu", "cudaVersion": "12.4" } - }, - { - "name": "Metal with a CUDA version", - "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal", "cudaVersion": "12.4" } - }, - { - "name": "CUDA version with a prefix", - "target": { "platform": "windows", "arch": "x86_64", "accelerator": "cuda", "cudaVersion": "cuda12.4" } - }, - { - "name": "CUDA version without a minor component", - "target": { "platform": "linux", "arch": "x86_64", "accelerator": "cuda", "cudaVersion": "12" } - }, - { - "name": "unsupported macOS Intel target", - "target": { "platform": "macos", "arch": "x86_64", "accelerator": "cpu" } - }, - { - "name": "unsupported Linux arm64 target", - "target": { "platform": "linux", "arch": "aarch64", "accelerator": "cpu" } - } - ] -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/index.mjs b/.scrollcase-runtime-types-RzUfxr/src/contract/index.mjs deleted file mode 100644 index f3ec625..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/index.mjs +++ /dev/null @@ -1,55 +0,0 @@ -/** - * The Scrollcase box-format contract. - * - * This module is the single source of truth for what a box *is*: which targets exist, how a - * target is named, what layout the payload has, and the shape of every document a build emits. It - * ships three things that must never disagree — a reference implementation (this code), a - * machine-readable spec (`schema/*.json`), and golden fixtures (`fixtures/*.json`) that any other - * implementation can validate itself against. - * - * A consumer written in another language does not import this code; it mirrors the rules and proves - * the mirror against the fixtures. That is how clients in other languages stay honest. - */ - -export { - assertNativeHost, - assertPythonEntryPoint, - condaSubdir, - pixiAccelerator, - boxTargetAdapter, - boxTargetAdapters, - boxTargetId, -} from './targets.mjs'; - -export { - CHANNELS, - DEFAULT_DOCUMENT_NAMESPACE, - PAYLOAD_ENCODING, - BOX_SCHEMA_VERSION, - SIGNATURE_ALGORITHM, - decodeDocumentPayload, - documentKinds, - isSignedBoxDocument, - parseDocumentKind, -} from './documents.mjs'; - -/** - * Absolute URL of a shipped JSON Schema, for consumers that validate documents themselves. - * - * @param {'target' | 'scroll' | 'box-manifest' | 'release-manifest' | 'channel-manifest' - * | 'revocations-manifest' | 'signed-document'} name - * @returns {URL} - */ -export function schemaUrl(name) { - return new URL(`./schema/${name}.schema.json`, import.meta.url); -} - -/** - * Absolute URL of a shipped fixture file, for consumers proving a mirror implementation. - * - * @param {string} name fixture file name without its extension - * @returns {URL} - */ -export function fixtureUrl(name) { - return new URL(`./fixtures/${name}.json`, import.meta.url); -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/links.mjs b/.scrollcase-runtime-types-RzUfxr/src/contract/links.mjs deleted file mode 100644 index 6619706..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/links.mjs +++ /dev/null @@ -1,162 +0,0 @@ -/** - * The rule deciding which symbolic links a box payload may carry. - * - * A conda prefix is dense with links: the shared-library soname convention alone stores every large - * library two or three times (`libfoo.so` → `libfoo.so.N` → `libfoo.so.N.M`), and `bin` carries - * interpreter aliases. Materialising all of them produced a Linux box where roughly 60% of the - * bytes were duplicates of other bytes in the same box. Preserving them costs nothing to store and - * everything to get wrong, because a link is the classic way an archive writes outside the - * directory it was extracted into. - * - * So the rule is deliberately narrow and purely lexical, which is what makes it provable: - * - * 1. a target is relative — never absolute, never a drive letter, never a backslash; - * 2. resolved against the link's own directory it stays inside the payload, so `..` is allowed - * exactly as far as it cannot escape; - * 3. a link resolves to a *regular file*, never to a directory; - * 4. no entry may have a link as a path prefix, so nothing is ever written *through* a link; - * 5. chains terminate, within a small bound, without a cycle. - * - * Rule 3 is what keeps the rest small. A directory link is legitimate in a conda prefix — - * `lib/python3.1` → `python3.11` is real — but it is also the only reason an entry could ever be - * written *through* a link and land somewhere its own name does not describe. Refusing directory - * links costs one duplicated standard library and removes an entire class of escape, so rule 4 - * survives only as a second lock on a door that rule 3 already welded shut. - * - * Nothing here consults the filesystem: the same inputs give the same answer on every host, which - * is what lets the builder, the Node consumer and the Python consumer apply one rule rather than - * three approximations of it. The builder additionally confirms its own links with `realpath`, - * because it can — but no consumer trusts that, and every rule here is re-checked before extraction - * writes anything. - * - * Targets that fail this rule are not an error at build time; they are simply materialised into - * real files, which is what every link used to become. - */ - -/** - * How many links a single resolution may traverse before it is treated as hostile. Real prefixes - * use one or two hops (`python` → `python3.11`, `libfoo.so` → `.so.N` → `.so.N.M`); a longer chain - * has no legitimate source and is the cheap way to make resolution expensive. - */ -export const MAX_PAYLOAD_LINK_DEPTH = 8; - -/** - * Whether a raw link target is shaped like one a payload may carry, before resolving it. - * - * @param {unknown} target - * @returns {boolean} - */ -export function isRelativeLinkTarget(target) { - if (typeof target !== 'string' || target === '') return false; - if (target.includes('\0') || target.includes('\\')) return false; - if (target.startsWith('/')) return false; - return !/^[A-Za-z]:/.test(target); -} - -/** - * Resolves a link target against the link's own location, staying inside the payload. - * - * @param {string} linkPath forward-slash path of the link itself, relative to the payload root - * @param {string} target the raw link body - * @returns {string | null} the resolved payload-relative path, or null when the link may not be - * carried — an absolute target, an escape through `..`, or a link onto itself - */ -export function resolvePayloadLinkTarget(linkPath, target) { - if (!isRelativeLinkTarget(target)) return null; - const segments = String(linkPath).split('/'); - // The link's own name is not part of the directory its target resolves against. - const stack = segments.slice(0, -1); - if (segments.length === 0 || segments.at(-1) === '') return null; - for (const part of target.split('/')) { - if (part === '' || part === '.') continue; - if (part === '..') { - // Underflow means the target climbed past the payload root: exactly the escape being - // guarded against, and the reason this is checked per segment rather than on the result. - if (stack.length === 0) return null; - stack.pop(); - continue; - } - stack.push(part); - } - if (stack.length === 0) return null; - const resolved = stack.join('/'); - return resolved === linkPath ? null : resolved; -} - -/** - * Rejects an entry set in which anything could be written through a link. - * - * A directory link is legitimate — conda ships `lib/python3.1` → `python3.11` — but it means an - * entry named under that link lands wherever the link points. Forbidding a link as any entry's path - * prefix removes the question entirely, and is why resolution never has to model what earlier - * entries did to the filesystem. - * - * @param {Array<{ path: string, kind: string }>} entries - * @returns {string | null} the offending entry path, or null when the set is safe - */ -export function findEntryThroughLink(entries) { - const links = new Set(entries.filter((entry) => entry.kind === 'link').map((entry) => entry.path)); - if (links.size === 0) return null; - for (const entry of entries) { - const parts = entry.path.split('/'); - for (let index = 1; index < parts.length; index += 1) { - if (links.has(parts.slice(0, index).join('/'))) return entry.path; - } - } - return null; -} - -/** - * Follows every link in an entry set until it reaches a regular file. - * - * A chain that ends anywhere else is refused: at a directory (rule 3), at nothing at all, at - * itself, or at more hops than a real prefix ever needs. The terminal entry must exist in the same - * archive, which is what makes a link a statement about this payload rather than about the host. - * - * @param {Array<{ path: string, kind: string, linkTarget?: string }>} entries - * @returns {string | null} the offending link path, or null when every chain ends at a file - */ -export function findUnresolvableLink(entries) { - const byPath = new Map(entries.map((entry) => [entry.path, entry])); - const directories = new Set(); - for (const entry of entries) { - if (entry.kind === 'directory') directories.add(entry.path); - const parts = entry.path.split('/'); - for (let index = 1; index < parts.length; index += 1) directories.add(parts.slice(0, index).join('/')); - } - for (const entry of entries) { - if (entry.kind !== 'link') continue; - const seen = new Set([entry.path]); - let current = entry; - for (let depth = 0; ; depth += 1) { - if (depth >= MAX_PAYLOAD_LINK_DEPTH) return entry.path; - const resolved = resolvePayloadLinkTarget(current.path, current.linkTarget ?? ''); - if (resolved === null) return entry.path; - // A directory may exist implicitly, through its children, without an entry of its own — so - // this has to be asked before looking the path up as an entry. - if (directories.has(resolved)) return entry.path; - const next = byPath.get(resolved); - if (!next) return entry.path; - if (next.kind === 'file') break; - if (next.kind !== 'link') return entry.path; - if (seen.has(next.path)) return entry.path; - seen.add(next.path); - current = next; - } - } - return null; -} - -/** - * Whether a target platform can extract a payload containing links. - * - * Creating a symbolic link on Windows needs Developer Mode or elevation, so a Windows box keeps - * materialising every link rather than producing an archive that fails to extract on an ordinary - * machine. - * - * @param {string} platform the target platform, as a scroll declares it - * @returns {boolean} - */ -export function targetCarriesLinks(platform) { - return platform !== 'windows'; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/box-manifest.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/box-manifest.schema.json deleted file mode 100644 index 61b0288..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/box-manifest.schema.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/box-manifest.schema.json", - "title": "Box manifest (box.json)", - "description": "The manifest packed inside the archive at box.json. It restates the box's identity, layout and provenance so an extracted box is self-describing: a consumer that has the directory but not the release document can still tell what it is holding and how it was built.", - "type": "object", - "additionalProperties": false, - "required": [ - "schemaVersion", - "boxId", - "modelId", - "runtimeId", - "version", - "target", - "pythonEntryPoint", - "modelCacheSubdir", - "selfTest", - "provenance" - ], - "properties": { - "schemaVersion": { - "const": 2 - }, - "boxId": { - "type": "string", - "minLength": 1 - }, - "modelId": { - "type": "string", - "minLength": 1 - }, - "runtimeId": { - "type": "string", - "minLength": 1 - }, - "version": { - "type": "string", - "minLength": 1 - }, - "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" - }, - "pythonEntryPoint": { - "type": "string", - "minLength": 1 - }, - "modelCacheSubdir": { - "type": "string", - "minLength": 1 - }, - "selfTest": { - "type": "object", - "additionalProperties": false, - "required": [ - "pythonImports", - "timeoutSeconds" - ], - "properties": { - "pythonImports": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "minLength": 1 - } - }, - "timeoutSeconds": { - "type": "integer", - "exclusiveMinimum": 0 - } - } - }, - "execution": { - "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" - }, - "provenance": { - "$ref": "https://scrollcase.dev/schema/v2/release-manifest.schema.json#/$defs/provenance" - }, - "weights": { - "const": "on-demand", - "description": "Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it." - }, - "assets": { - "type": "array", - "minItems": 1, - "description": "Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe.", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "url", - "relativePath", - "sizeBytes", - "sha256" - ], - "properties": { - "url": { - "type": "string", - "minLength": 1 - }, - "relativePath": { - "type": "string", - "minLength": 1 - }, - "sizeBytes": { - "type": "integer", - "exclusiveMinimum": 0 - }, - "sha256": { - "$ref": "https://scrollcase.dev/schema/v2/release-manifest.schema.json#/$defs/sha256" - } - } - } - } - }, - "dependentRequired": { - "assets": [ - "weights" - ], - "weights": [ - "assets" - ] - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/channel-manifest.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/channel-manifest.schema.json deleted file mode 100644 index 70102ed..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/channel-manifest.schema.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/channel-manifest.schema.json", - "title": "Box channel manifest", - "description": "A small mutable pointer from a channel to the releases it currently serves. Signed independently from releases, so promoting a build never requires re-signing it.", - "type": "object", - "additionalProperties": false, - "required": [ - "schemaVersion", - "kind", - "channel", - "boxId", - "target", - "updatedAt", - "cohortSalt", - "releases" - ], - "properties": { - "schemaVersion": { - "const": 2 - }, - "kind": { - "type": "string", - "pattern": "^[a-z0-9]+(?:[.-][a-z0-9]+)*\\.channel$", - "description": "Wire discriminator, \".channel\", carrying the same namespace as the releases it refers to." - }, - "channel": { - "enum": [ - "nightly", - "beta", - "stable" - ] - }, - "boxId": { - "type": "string", - "minLength": 1 - }, - "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" - }, - "updatedAt": { - "type": "string", - "minLength": 1 - }, - "cohortSalt": { - "type": "string", - "minLength": 1, - "description": "Salt mixed into a client's rollout hash. It makes cohort assignment stable per client and unpredictable across channels, so a staged rollout cannot be gamed by reinstalling." - }, - "releases": { - "type": "array", - "minItems": 1, - "description": "Candidate releases in evaluation order. A client takes the first entry whose rollout cohort it falls into.", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "version", - "releaseManifestUrl", - "rolloutPercentage" - ], - "properties": { - "version": { - "type": "string", - "minLength": 1 - }, - "releaseManifestUrl": { - "type": "string", - "minLength": 1 - }, - "rolloutPercentage": { - "type": "integer", - "minimum": 1, - "maximum": 100 - } - } - } - } - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/execution.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/execution.schema.json deleted file mode 100644 index a37ebfb..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/execution.schema.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/execution.schema.json", - "title": "Box execution", - "description": "The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only.", - "oneOf": [ - { - "title": "Python script", - "description": "Run one regular payload file with the box's own Python interpreter.", - "type": "object", - "additionalProperties": false, - "required": [ - "kind", - "script", - "defaultArgs" - ], - "properties": { - "kind": { - "const": "python-script", - "description": "Selects direct script execution." - }, - "script": { - "$ref": "#/$defs/payloadPath", - "description": "Safe path to a regular Python file inside the box." - }, - "defaultArgs": { - "$ref": "#/$defs/defaultArgs" - } - } - }, - { - "title": "Python module", - "description": "Run an importable dotted module with Python's -m option.", - "type": "object", - "additionalProperties": false, - "required": [ - "kind", - "module", - "defaultArgs" - ], - "properties": { - "kind": { - "const": "python-module", - "description": "Selects dotted-module execution." - }, - "module": { - "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*)*$", - "description": "Strict Python dotted-module name, without command-line syntax or shell fragments.", - "examples": [ - "example_model.main" - ] - }, - "defaultArgs": { - "$ref": "#/$defs/defaultArgs" - } - } - } - ], - "examples": [ - { - "kind": "python-script", - "script": "entrypoint.py", - "defaultArgs": [] - }, - { - "kind": "python-module", - "module": "example_model.main", - "defaultArgs": [ - "--serve" - ] - } - ], - "$defs": { - "defaultArgs": { - "type": "array", - "description": "Arguments placed before caller-supplied arguments. Every item is passed directly without a shell.", - "default": [], - "items": { - "type": "string" - } - }, - "payloadPath": { - "type": "string", - "minLength": 1, - "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*//).+$", - "description": "A forward-slash path inside the box payload: relative, non-empty, and unable to escape the payload root.", - "examples": [ - "entrypoint.py", - "app/main.py" - ] - } - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/release-manifest.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/release-manifest.schema.json deleted file mode 100644 index 2437bcf..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/release-manifest.schema.json +++ /dev/null @@ -1,272 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/release-manifest.schema.json", - "title": "Box release manifest", - "description": "The immutable description of one built box: what it is, which target it runs on, where its archive lives, and how it was produced. Published as the payload of a signed document and never edited after signing; a correction ships as a new version.", - "type": "object", - "additionalProperties": false, - "required": [ - "schemaVersion", - "kind", - "boxId", - "modelId", - "runtimeId", - "version", - "target", - "compatibility", - "archive", - "pythonEntryPoint", - "modelCacheSubdir", - "selfTest", - "provenance" - ], - "properties": { - "schemaVersion": { - "const": 2 - }, - "kind": { - "$ref": "#/$defs/kind", - "description": "Wire discriminator, \".release\". The namespace belongs to the publishing project \u2014 a project with boxes already in the field must keep emitting the one its clients recognise \u2014 and defaults to scrollcase.box for a new one." - }, - "boxId": { - "$ref": "#/$defs/identifier" - }, - "modelId": { - "$ref": "#/$defs/identifier" - }, - "runtimeId": { - "$ref": "#/$defs/identifier" - }, - "version": { - "type": "string", - "minLength": 1 - }, - "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" - }, - "compatibility": { - "type": "object", - "additionalProperties": true, - "description": "What the host must satisfy before this box may be installed. The builder copies these constraints through verbatim and never interprets them, so a project may add its own alongside the ones defined here. A consumer that cannot evaluate a constraint must refuse the box rather than assume it passes.", - "properties": { - "minHostAppVersion": { - "type": "string", - "minLength": 1, - "description": "Lowest version of the installing application this box supports." - }, - "maxHostAppVersionExclusive": { - "type": "string", - "minLength": 1 - }, - "minMacosVersion": { - "type": "string", - "minLength": 1 - }, - "minRamGb": { - "type": "number", - "exclusiveMinimum": 0, - "description": "Installed memory in decimal gigabytes (1 GB = 1,000,000,000 bytes)." - }, - "minNvidiaDriverVersion": { - "type": "string", - "minLength": 1 - }, - "hostEnvironments": { - "type": "array", - "minItems": 1, - "items": { - "enum": [ - "native", - "windows-wsl2" - ] - }, - "description": "Host environments this payload was validated on." - } - } - }, - "archive": { - "type": "object", - "additionalProperties": false, - "required": [ - "format", - "url", - "sha256", - "sizeBytes" - ], - "properties": { - "format": { - "const": "zip" - }, - "url": { - "type": "string", - "minLength": 1 - }, - "sha256": { - "$ref": "#/$defs/sha256" - }, - "sizeBytes": { - "type": "integer", - "exclusiveMinimum": 0 - } - } - }, - "installedSizeBytes": { - "type": "integer", - "exclusiveMinimum": 0, - "description": "Sum of extracted payload file sizes before activation metadata is written, so a consumer can check free space before downloading." - }, - "pythonEntryPoint": { - "type": "string", - "minLength": 1, - "description": "Interpreter path relative to the extracted box root, for example venv/bin/python. Fixed per target by the adapter." - }, - "modelCacheSubdir": { - "type": "string", - "minLength": 1, - "description": "Directory relative to the extracted box root holding model assets." - }, - "selfTest": { - "type": "object", - "additionalProperties": false, - "required": [ - "pythonImports", - "timeoutSeconds" - ], - "description": "The import check a consumer can repeat after extraction with the box's own interpreter. The builder also ran the scroll's Python-code and file assertions, which are builder-only checks.", - "properties": { - "pythonImports": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "minLength": 1 - } - }, - "timeoutSeconds": { - "type": "integer", - "exclusiveMinimum": 0 - } - } - }, - "execution": { - "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" - }, - "provenance": { - "$ref": "#/$defs/provenance" - }, - "weights": { - "const": "on-demand", - "description": "Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it." - }, - "assets": { - "type": "array", - "minItems": 1, - "description": "Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe.", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "url", - "relativePath", - "sizeBytes", - "sha256" - ], - "properties": { - "url": { - "type": "string", - "minLength": 1 - }, - "relativePath": { - "type": "string", - "minLength": 1 - }, - "sizeBytes": { - "type": "integer", - "exclusiveMinimum": 0 - }, - "sha256": { - "$ref": "#/$defs/sha256" - } - } - } - } - }, - "$defs": { - "identifier": { - "type": "string", - "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" - }, - "kind": { - "type": "string", - "pattern": "^[a-z0-9]+(?:[.-][a-z0-9]+)*\\.release$" - }, - "sha256": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "provenance": { - "type": "object", - "additionalProperties": false, - "description": "How this box was produced. Every field is recorded by the builder from observed state, never accepted from caller input, so the record cannot be dressed up after the fact.", - "required": [ - "scrollId", - "scrollVersion", - "builderRevision", - "sourceTreeDirty", - "sourceRevision", - "pythonVersion", - "dependencyLockSha256", - "builtAt", - "pixiVersion" - ], - "properties": { - "scrollId": { - "type": "string", - "minLength": 1 - }, - "scrollVersion": { - "type": "string", - "minLength": 1 - }, - "builderRevision": { - "type": "string", - "pattern": "^[a-f0-9]{40}$", - "description": "Exact commit of the builder source that produced the box." - }, - "sourceTreeDirty": { - "type": "boolean", - "description": "Whether the builder's working tree carried uncommitted changes. True means the build is not reproducible from the recorded revision alone." - }, - "sourceRevision": { - "type": "string", - "minLength": 1, - "description": "Upstream revision of the packaged model source, as declared by the scroll." - }, - "pythonVersion": { - "type": "string", - "minLength": 1 - }, - "pixiVersion": { - "type": "string", - "minLength": 1 - }, - "dependencyLockSha256": { - "$ref": "#/$defs/sha256", - "description": "Hash of the pixi.lock the environment was solved from." - }, - "builtAt": { - "type": "string", - "minLength": 1 - } - } - } - }, - "dependentRequired": { - "assets": [ - "weights" - ], - "weights": [ - "assets" - ] - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/revocations-manifest.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/revocations-manifest.schema.json deleted file mode 100644 index 67470c7..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/revocations-manifest.schema.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/revocations-manifest.schema.json", - "title": "Box revocations manifest", - "description": "The signed list of releases that must no longer be installed or activated. A published release is immutable, so withdrawing one is an explicit statement rather than a deletion: clients keep honouring this list even when the archive is still reachable.", - "type": "object", - "additionalProperties": false, - "required": [ - "schemaVersion", - "kind", - "updatedAt", - "revocations" - ], - "properties": { - "schemaVersion": { - "const": 2 - }, - "kind": { - "type": "string", - "pattern": "^[a-z0-9]+(?:[.-][a-z0-9]+)*\\.revocations$", - "description": "Wire discriminator, \".revocations\", carrying the same namespace as the releases it refers to." - }, - "updatedAt": { - "type": "string", - "minLength": 1 - }, - "revocations": { - "type": "array", - "description": "May be empty: an empty signed list is a positive statement that nothing is revoked, which a client can distinguish from a missing or withheld document.", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "boxId", - "version", - "reason", - "revokedAt" - ], - "properties": { - "boxId": { - "type": "string", - "minLength": 1 - }, - "version": { - "type": "string", - "minLength": 1 - }, - "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json", - "description": "Omitted when every target of that version is revoked." - }, - "reason": { - "type": "string", - "minLength": 1 - }, - "revokedAt": { - "type": "string", - "minLength": 1 - } - } - } - } - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/scroll.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/scroll.schema.json deleted file mode 100644 index 9cb4ee6..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/scroll.schema.json +++ /dev/null @@ -1,339 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/scroll.schema.json", - "title": "Box scroll", - "description": "The declarative input to a build: an identity, a target, a pinned dependency environment, the assets to fetch, and the self-test the result must pass. A scroll is checked into the consumer's repository next to its lock file; everything a build produces is derived from it.", - "type": "object", - "additionalProperties": false, - "required": [ - "schemaVersion", - "scrollVersion", - "boxId", - "modelId", - "runtimeId", - "version", - "sourceRevision", - "target", - "compatibility", - "pythonVersion", - "pythonEntryPoint", - "modelCacheSubdir", - "assets", - "selfTest", - "pixiVersion" - ], - "properties": { - "$schema": { - "const": "https://scrollcase.dev/schema/v2/scroll.schema.json", - "description": "Associates this file with the published Scrollcase v2 schema for editor validation, completion, and hover help." - }, - "schemaVersion": { - "const": 2, - "description": "Scrollcase wire version. Version 2 is the only active format.", - "examples": [ - 2 - ] - }, - "scrollId": { - "type": "string", - "minLength": 1, - "description": "Optional provenance identity. When omitted, Scrollcase derives it deterministically from boxId and the canonical target." - }, - "scrollVersion": { - "type": "string", - "minLength": 1, - "description": "Version of this declarative build input, recorded in provenance.", - "examples": [ - "1.0.0" - ] - }, - "boxId": { - "$ref": "#/$defs/identifier" - }, - "modelId": { - "$ref": "#/$defs/identifier" - }, - "runtimeId": { - "$ref": "#/$defs/identifier" - }, - "version": { - "type": "string", - "minLength": 1, - "description": "Version of the box this scroll produces, as it will appear in the release manifest." - }, - "sourceRevision": { - "type": "string", - "minLength": 1, - "description": "Upstream revision of the packaged source, recorded verbatim into provenance." - }, - "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" - }, - "compatibility": { - "type": "object", - "additionalProperties": true, - "properties": { - "minHostAppVersion": { - "type": "string", - "minLength": 1, - "description": "Lowest version of the installing application this box supports." - }, - "maxHostAppVersionExclusive": { - "type": "string", - "minLength": 1 - }, - "minMacosVersion": { - "type": "string", - "minLength": 1 - }, - "minRamGb": { - "type": "number", - "exclusiveMinimum": 0 - }, - "minNvidiaDriverVersion": { - "type": "string", - "minLength": 1 - } - }, - "description": "Constraints the installing host must satisfy. Copied through into the release manifest verbatim and never interpreted by the builder, so a project may declare its own alongside these." - }, - "pythonVersion": { - "type": "string", - "minLength": 1, - "description": "Python version solved into the box.", - "examples": [ - "3.11.15" - ] - }, - "pixiVersion": { - "type": "string", - "minLength": 1, - "description": "Pins the pixi release used to solve and install the conda-forge environment from the committed pixi.lock." - }, - "condaDependencyLicenseAudit": { - "type": "string", - "minLength": 1, - "description": "Path to the reviewed licence inventory derived from pixi.lock, which carries an SPDX licence per package. The build fails if the lock no longer matches what was reviewed." - }, - "pythonEntryPoint": { - "type": "string", - "minLength": 1, - "description": "Interpreter path relative to the box root. Must match the target adapter's layout." - }, - "modelCacheSubdir": { - "type": "string", - "minLength": 1 - }, - "assetBaseUrl": { - "type": "string", - "minLength": 1, - "description": "Base URL of the mirror the built archive and its objects are published under." - }, - "assets": { - "type": "array", - "description": "Files fetched during the build. Every entry is size- and hash-checked before use, so a moved or replaced upstream file fails the build instead of silently changing the box. May be empty.", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "url", - "relativePath", - "sizeBytes", - "sha256" - ], - "properties": { - "url": { - "type": "string", - "minLength": 1 - }, - "relativePath": { - "$ref": "#/$defs/payloadPath" - }, - "sizeBytes": { - "type": "integer", - "exclusiveMinimum": 0 - }, - "sha256": { - "$ref": "#/$defs/sha256" - } - } - } - }, - "assetArchives": { - "type": "array", - "description": "Downloaded archives to expand into the payload. Extraction preserves files already present in the destination and refuses to overwrite them.", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "relativePath", - "format", - "destination" - ], - "properties": { - "relativePath": { - "$ref": "#/$defs/payloadPath" - }, - "format": { - "enum": [ - "zip", - "tar.gz" - ] - }, - "destination": { - "$ref": "#/$defs/payloadPath" - }, - "stripComponents": { - "type": "integer", - "minimum": 0 - }, - "removeAfterExtract": { - "type": "boolean" - } - } - } - }, - "localFiles": { - "type": "array", - "description": "Files copied from the consumer's own repository into the payload, each verified against a declared hash so a licence notice or runtime shim cannot drift from what was reviewed.", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "sourcePath", - "relativePath", - "sha256" - ], - "properties": { - "sourcePath": { - "type": "string", - "minLength": 1 - }, - "relativePath": { - "$ref": "#/$defs/payloadPath" - }, - "sha256": { - "$ref": "#/$defs/sha256" - } - } - } - }, - "prunePaths": { - "type": "array", - "description": "Payload paths deleted before packing, to keep the box to what it actually needs at run time. Pruning a distribution the lock requires is rejected.", - "items": { - "$ref": "#/$defs/payloadPath" - } - }, - "selfTest": { - "type": "object", - "additionalProperties": false, - "required": [ - "imports", - "files" - ], - "description": "Builder checks run with the payload's own interpreter before archiving. Schema version 2 signs only the import subset for a consumer to repeat; file and optional Python-code assertions remain builder-only.", - "properties": { - "imports": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "minLength": 1 - } - }, - "files": { - "type": "array", - "description": "Files that must still exist after pruning, which is what stops an over-aggressive prune from shipping a broken box.", - "items": { - "$ref": "#/$defs/payloadPath" - } - }, - "pythonCode": { - "type": "string", - "minLength": 1, - "description": "Extra Python executed after the imports succeed, for checks a bare import cannot make." - } - } - }, - "weights": { - "enum": [ - "embed", - "on-demand" - ], - "default": "embed", - "description": "Whether assets are packed into the archive (embed, the default: the box installs with no network and works air-gapped) or left out for the caller to materialize from descriptors in the signed release (on-demand). Consumers verify materialized assets before execution and do not download them. A build may override this." - }, - "execution": { - "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" - }, - "parity": { - "type": "object", - "additionalProperties": false, - "required": [ - "script", - "accelerators", - "tolerances" - ], - "description": "An optional numerical gate: run a check inside the box on more than one accelerator and require the results to agree. This catches a mis-solved environment \u2014 CPU-only wheels shipped as CUDA, a broken BLAS \u2014 on the build machine rather than on a user's. The tool runs the check and enforces the thresholds; what the check computes, and what closeness is acceptable, belong to the project.", - "properties": { - "script": { - "type": "string", - "minLength": 1, - "description": "Path inside the box, run with the box's own interpreter. It must print a JSON array of numbers, or an object with a \"values\" array." - }, - "accelerators": { - "type": "array", - "minItems": 2, - "items": { - "enum": [ - "cpu", - "metal", - "cuda" - ] - }, - "description": "Accelerators to run under, each with its target's validation environment. The first is the reference the others are compared against \u2014 conventionally cpu, being the one available everywhere." - }, - "tolerances": { - "type": "object", - "additionalProperties": false, - "minProperties": 1, - "description": "At least one bound. Absolute guards entries near zero, where relative error is meaningless; cosine similarity catches a result that drifted in direction rather than magnitude.", - "properties": { - "absolute": { - "type": "number", - "exclusiveMinimum": 0 - }, - "relative": { - "type": "number", - "exclusiveMinimum": 0 - }, - "minimumCosine": { - "type": "number", - "maximum": 1 - } - } - } - } - } - }, - "$defs": { - "identifier": { - "type": "string", - "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" - }, - "sha256": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "payloadPath": { - "type": "string", - "minLength": 1, - "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*//).+$", - "description": "A forward-slash path inside the box payload: relative, non-empty, and unable to escape the payload root.", - "examples": [ - "model-cache/example-model/weights.safetensors" - ] - } - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/signed-document.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/signed-document.schema.json deleted file mode 100644 index 602af0c..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/signed-document.schema.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/signed-document.schema.json", - "title": "Signed box document", - "description": "The envelope wrapping every signed document. The payload travels as exact base64-encoded JSON so that verifying a signature means hashing the bytes as transmitted, with no canonical-JSON implementation to keep in sync across languages. Passing this schema means the envelope is well-formed, never that its signature is valid.", - "type": "object", - "additionalProperties": false, - "required": ["schemaVersion", "payloadEncoding", "payloadBase64", "payloadSha256", "signatures"], - "properties": { - "schemaVersion": { "const": 2 }, - "payloadEncoding": { "const": "base64-json-utf8" }, - "payloadBase64": { - "type": "string", - "minLength": 1, - "description": "The document payload: UTF-8 JSON, base64-encoded, signed and hashed exactly as it appears here." - }, - "payloadSha256": { - "$ref": "#/$defs/sha256", - "description": "SHA-256 of the decoded payload bytes." - }, - "signatures": { - "type": "array", - "minItems": 1, - "description": "Detached signatures over the decoded payload bytes. A verifier accepts the document when any one signature verifies against a trusted key, which is what allows a key to be rotated without reissuing every document.", - "items": { - "type": "object", - "additionalProperties": false, - "required": ["algorithm", "keyId", "signatureBase64"], - "properties": { - "algorithm": { "const": "ed25519" }, - "keyId": { "type": "string", "minLength": 1 }, - "signatureBase64": { "type": "string", "minLength": 1 } - } - } - } - }, - "$defs": { - "sha256": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - } - } -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/target.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/target.schema.json deleted file mode 100644 index 6894c12..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/target.schema.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/target.schema.json", - "title": "Box target", - "description": "The platform, architecture and accelerator a box is built for. The supported combinations are closed: a target outside this matrix has no defined identifier and cannot be built, signed, or routed.", - "type": "object", - "additionalProperties": false, - "required": ["platform", "arch", "accelerator"], - "properties": { - "platform": { - "enum": ["macos", "linux", "windows"], - "description": "Operating system the box runs on.", - "examples": ["linux"] - }, - "arch": { - "enum": ["aarch64", "x86_64"], - "description": "CPU architecture the box runs on; supported combinations are constrained below.", - "examples": ["x86_64"] - }, - "accelerator": { - "enum": ["cpu", "metal", "cuda"], - "description": "Acceleration backend built into the environment.", - "default": "cpu", - "examples": ["cpu"] - }, - "cudaVersion": { - "type": "string", - "pattern": "^[1-9][0-9]*\\.[0-9]+$", - "description": "CUDA ABI as major.minor, for example \"12.8\". Required for a CUDA target and forbidden for any other, so an identifier can never be ambiguous." - } - }, - "allOf": [ - { - "if": { "properties": { "accelerator": { "const": "cuda" } }, "required": ["accelerator"] }, - "then": { "required": ["cudaVersion"] }, - "else": { "not": { "required": ["cudaVersion"] } } - }, - { - "if": { "properties": { "platform": { "const": "macos" } }, "required": ["platform"] }, - "then": { - "properties": { - "arch": { "const": "aarch64" }, - "accelerator": { "enum": ["metal", "cpu"] } - } - } - }, - { - "if": { "properties": { "platform": { "enum": ["linux", "windows"] } }, "required": ["platform"] }, - "then": { - "properties": { - "arch": { "const": "x86_64" }, - "accelerator": { "enum": ["cpu", "cuda"] } - } - } - } - ] -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/targets.mjs b/.scrollcase-runtime-types-RzUfxr/src/contract/targets.mjs deleted file mode 100644 index a4fa2b3..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/targets.mjs +++ /dev/null @@ -1,254 +0,0 @@ -/** - * Reference implementation of the Scrollcase box-format target model. - * - * A target is the (platform, arch, accelerator) triple a box is built for, plus a CUDA ABI version - * when the accelerator is CUDA. `boxTargetId()` turns it into the canonical slug that appears - * in archive names, object keys, and registry routes, so every implementation of the format — this - * one, a consumer's own client, a signer — must agree character for character. The golden cases in - * `fixtures/target-id-contract.json` are what "agree" means, and are the fixtures other languages - * validate their mirrors against. - * - * The adapters below describe what a target implies for the built payload: the Python layout inside - * the box, the archive backend, how native libraries are inspected, and the environment a validation - * run gets. They are part of the format because a consumer unpacking a box relies on that layout. - */ -/** - * What a target implies for the built payload. Part of the format rather than an implementation - * detail: a consumer unpacking a box relies on this layout. - * - * @typedef {object} BoxTargetAdapter - * @property {string} id canonical adapter id, e.g. `macos-aarch64` - * @property {'macos' | 'linux' | 'windows'} platform - * @property {'aarch64' | 'x86_64'} arch - * @property {{ platform: string, arch: string }} host the Node platform/arch a build must run on - * @property {'osx-arm64' | 'linux-64' | 'win-64'} condaSubdir the scroll's pixi `platforms` value - * @property {{ payloadRoot: string, entryPoint: string, scriptsDirectory: string, - * executableSuffix: string, launcherKind: string }} python layout of the interpreter in the box - * @property {{ format: 'zip', writer: string, reader: string, assetTarReader: string, - * zip64: boolean }} archive the pinned archive backend - * @property {{ command: string, argsPrefix: readonly string[], - * extensions: readonly string[] }} nativeLibraryInspection - * @property {Readonly>>>} validationEnvironments - * the environment that forces a run onto one accelerator, keyed by accelerator - * @property {string} selfTestPython the platform assertion prepended to every self-test - */ - -const TARGET_ACCELERATORS = { - macos: { aarch64: ['metal', 'cpu'] }, - linux: { x86_64: ['cpu', 'cuda'] }, - windows: { x86_64: ['cpu', 'cuda'] }, -}; -const CUDA_VERSION = /^[1-9][0-9]*\.[0-9]+$/; - -// The exact libraries that wrote and read a box, so a consumer knows what produced the bytes it -// holds rather than inferring it. Each version is the one this package installs: they are pinned in -// `package.json` and `tests/unit/contract-targets.test.mjs` fails when the two drift, because a -// descriptor naming a release that never touched the archive is worse than no descriptor at all. -const ARCHIVE_BACKEND = Object.freeze({ - format: 'zip', - writer: 'yazl@3.3.1', - reader: 'yauzl@3.4.0', - assetTarReader: 'tar@7.5.22', - zip64: true, -}); - -const TARGET_ADAPTERS = Object.freeze([ - Object.freeze({ - id: 'macos-aarch64', - platform: 'macos', - arch: 'aarch64', - host: Object.freeze({ platform: 'darwin', arch: 'arm64' }), - // conda platform subdir: the `platforms` value in the scroll's pixi.toml. - condaSubdir: 'osx-arm64', - python: Object.freeze({ - payloadRoot: 'venv', - entryPoint: 'venv/bin/python', - scriptsDirectory: 'venv/bin', - executableSuffix: '', - launcherKind: 'posix-polyglot', - }), - archive: ARCHIVE_BACKEND, - nativeLibraryInspection: Object.freeze({ - command: 'otool', - argsPrefix: Object.freeze(['-L']), - extensions: Object.freeze(['.dylib', '.so']), - }), - validationEnvironments: Object.freeze({ - cpu: Object.freeze({ CUDA_VISIBLE_DEVICES: '' }), - metal: Object.freeze({ PYTORCH_ENABLE_MPS_FALLBACK: '0' }), - }), - selfTestPython: "import sys; assert sys.platform == 'darwin'", - }), - Object.freeze({ - id: 'linux-x86_64', - platform: 'linux', - arch: 'x86_64', - host: Object.freeze({ platform: 'linux', arch: 'x64' }), - condaSubdir: 'linux-64', - python: Object.freeze({ - payloadRoot: 'venv', - entryPoint: 'venv/bin/python', - scriptsDirectory: 'venv/bin', - executableSuffix: '', - launcherKind: 'posix-polyglot', - }), - archive: ARCHIVE_BACKEND, - nativeLibraryInspection: Object.freeze({ - command: 'ldd', - argsPrefix: Object.freeze([]), - extensions: Object.freeze(['.so']), - }), - validationEnvironments: Object.freeze({ - cpu: Object.freeze({ CUDA_VISIBLE_DEVICES: '' }), - cuda: Object.freeze({ CUDA_VISIBLE_DEVICES: '0' }), - }), - selfTestPython: "import sys; assert sys.platform.startswith('linux')", - }), - Object.freeze({ - id: 'windows-x86_64', - platform: 'windows', - arch: 'x86_64', - host: Object.freeze({ platform: 'win32', arch: 'x64' }), - condaSubdir: 'win-64', - python: Object.freeze({ - payloadRoot: 'venv', - entryPoint: 'venv/python.exe', - scriptsDirectory: 'venv/Scripts', - executableSuffix: '.exe', - launcherKind: 'uv-windows-pe', - }), - archive: ARCHIVE_BACKEND, - nativeLibraryInspection: Object.freeze({ - command: 'dumpbin', - argsPrefix: Object.freeze(['/DEPENDENTS']), - extensions: Object.freeze(['.dll', '.pyd']), - }), - validationEnvironments: Object.freeze({ - cpu: Object.freeze({ CUDA_VISIBLE_DEVICES: '' }), - cuda: Object.freeze({ CUDA_VISIBLE_DEVICES: '0' }), - }), - selfTestPython: "import sys; assert sys.platform == 'win32'", - }), -]); - -/** - * Returns the canonical target slug used in box filenames, object keys, and routes. - * - * @param {import('./types/index.d.ts').BoxTarget} target - * @returns {string} the canonical slug, e.g. `linux-x86_64-cuda12.4` - * @throws {TypeError} when the target is outside the supported matrix, or its CUDA version is - * missing on a CUDA target or present on any other - */ -export function boxTargetId(target) { - if (!target || typeof target !== 'object') { - throw new TypeError('Box target must be an object'); - } - const accelerators = TARGET_ACCELERATORS[target?.platform]?.[target?.arch]; - if (!accelerators?.includes(target?.accelerator)) { - throw new TypeError( - `Unsupported box target: ${target?.platform}/${target?.arch}/${target?.accelerator}`, - ); - } - if (target.accelerator === 'cuda') { - if (typeof target.cudaVersion !== 'string' || !CUDA_VERSION.test(target.cudaVersion)) { - throw new TypeError('A CUDA box target requires a numeric major.minor CUDA version'); - } - return `${target.platform}-${target.arch}-cuda${target.cudaVersion}`; - } - if (target.cudaVersion !== undefined) { - throw new TypeError('Only CUDA box targets may declare a CUDA version'); - } - return `${target.platform}-${target.arch}-${target.accelerator}`; -} - -/** - * Returns the native builder adapter for a validated box target. - * - * @param {import('./types/index.d.ts').BoxTarget} target - * @returns {BoxTargetAdapter} - * @throws {TypeError} when the target is unsupported - */ -export function boxTargetAdapter(target) { - boxTargetId(target); - const adapter = TARGET_ADAPTERS.find((candidate) => - candidate.platform === target.platform && candidate.arch === target.arch); - if (!adapter) throw new TypeError(`No box target adapter exists for ${target.platform}/${target.arch}`); - return adapter; -} - -/** - * Ensures a build or target lock runs on the OS and architecture it will ship for. - * - * @param {BoxTargetAdapter} adapter - * @param {{ platform: string, arch: string }} [host] defaults to the current process - * @returns {void} - * @throws {TypeError} when the host is not the OS and architecture the box ships for - */ -export function assertNativeHost(adapter, host = process) { - if (host.platform !== adapter.host.platform || host.arch !== adapter.host.arch) { - throw new TypeError( - `${adapter.id} boxes must be built natively on ${adapter.host.platform}/${adapter.host.arch}; ` - + `current host is ${host.platform}/${host.arch}`, - ); - } -} - -/** - * Ensures the scroll entry point agrees with the adapter's standalone Python layout. - * - * @param {BoxTargetAdapter} adapter - * @param {string} entryPoint - * @returns {void} - * @throws {TypeError} when the entry point does not match the adapter's layout - */ -export function assertPythonEntryPoint(adapter, entryPoint) { - if (entryPoint !== adapter.python.entryPoint) { - throw new TypeError( - `${adapter.id} scrolls must use Python entry point ${adapter.python.entryPoint}`, - ); - } -} - -/** - * Lists every adapter, for contract tests and for consumers enumerating supported targets. - * - * @returns {BoxTargetAdapter[]} every supported adapter, as a fresh array - */ -export function boxTargetAdapters() { - return [...TARGET_ADAPTERS]; -} - -/** - * Maps a validated box target to its conda platform subdir (the pixi `platforms` value). - * - * @param {import('./types/index.d.ts').BoxTarget} target - * @returns {'osx-arm64' | 'linux-64' | 'win-64'} the pixi `platforms` value for the target - */ -export function condaSubdir(target) { - const adapter = boxTargetAdapter(target); - return adapter.condaSubdir; -} - -/** - * Returns the conda accelerator descriptor a scroll selects, rejecting target drift. `metal` and - * `cpu` need no extra conda knobs (osx-arm64 ships MPS in the pytorch build; cpu is the default build); `cuda` pins a - * `cuda-version` and declares a CUDA system requirement so the solver picks the GPU pytorch build. - * - * @param {Pick} scroll - * @returns {{ accelerator: 'cpu' | 'metal' | 'cuda', cudaVersion: string | null }} - * @throws {TypeError} when the accelerator is unsupported, or a CUDA target lacks a version - */ -export function pixiAccelerator(scroll) { - const accelerator = scroll?.target?.accelerator; - if (accelerator === 'metal' || accelerator === 'cpu') { - return Object.freeze({ accelerator, cudaVersion: null }); - } - if (accelerator === 'cuda') { - const cudaVersion = scroll?.target?.cudaVersion; - if (typeof cudaVersion !== 'string' || !CUDA_VERSION.test(cudaVersion)) { - throw new TypeError('A CUDA box target requires a numeric major.minor CUDA version'); - } - return Object.freeze({ accelerator, cudaVersion }); - } - throw new TypeError(`Unsupported box accelerator: ${accelerator}`); -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/types/index.d.ts b/.scrollcase-runtime-types-RzUfxr/src/contract/types/index.d.ts deleted file mode 100644 index da0a0cd..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/contract/types/index.d.ts +++ /dev/null @@ -1,467 +0,0 @@ -/** - * Types for the scrollcase box format, generated from the JSON Schemas in - * src/contract/schema/. Do not edit by hand: run `npm run types` instead. - * - * The schemas are the source of truth. These types are a projection of them, and the test suite - * fails if the two disagree. - */ - -export type BoxTarget = { - [k: string]: unknown; -} & { - /** - * Operating system the box runs on. - */ - platform: 'macos' | 'linux' | 'windows'; - /** - * CPU architecture the box runs on; supported combinations are constrained below. - */ - arch: 'aarch64' | 'x86_64'; - /** - * Acceleration backend built into the environment. - */ - accelerator: 'cpu' | 'metal' | 'cuda'; - /** - * CUDA ABI as major.minor, for example "12.8". Required for a CUDA target and forbidden for any other, so an identifier can never be ambiguous. - */ - cudaVersion?: string; -}; - -/** - * The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only. - */ -export type BoxExecution = PythonScript | PythonModule; -/** - * Arguments placed before caller-supplied arguments. Every item is passed directly without a shell. - */ -export type DefaultArgs = string[]; - -/** - * Run one regular payload file with the box's own Python interpreter. - */ -export interface PythonScript { - /** - * Selects direct script execution. - */ - kind: 'python-script'; - /** - * Safe path to a regular Python file inside the box. - */ - script: string; - defaultArgs: DefaultArgs; -} -/** - * Run an importable dotted module with Python's -m option. - */ -export interface PythonModule { - /** - * Selects dotted-module execution. - */ - kind: 'python-module'; - /** - * Strict Python dotted-module name, without command-line syntax or shell fragments. - */ - module: string; - defaultArgs: DefaultArgs; -} - -export type Identifier = string; -/** - * The platform, architecture and accelerator a box is built for. The supported combinations are closed: a target outside this matrix has no defined identifier and cannot be built, signed, or routed. - */ -export type PayloadPath = string; -export type Sha256 = string; -/** - * The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only. - */ -export interface BoxScroll { - /** - * Associates this file with the published Scrollcase v2 schema for editor validation, completion, and hover help. - */ - $schema?: 'https://scrollcase.dev/schema/v2/scroll.schema.json'; - /** - * Scrollcase wire version. Version 2 is the only active format. - */ - schemaVersion: 2; - /** - * Optional provenance identity. When omitted, Scrollcase derives it deterministically from boxId and the canonical target. - */ - scrollId?: string; - /** - * Version of this declarative build input, recorded in provenance. - */ - scrollVersion: string; - boxId: Identifier; - modelId: Identifier; - runtimeId: Identifier; - /** - * Version of the box this scroll produces, as it will appear in the release manifest. - */ - version: string; - /** - * Upstream revision of the packaged source, recorded verbatim into provenance. - */ - sourceRevision: string; - target: BoxTarget; - /** - * Constraints the installing host must satisfy. Copied through into the release manifest verbatim and never interpreted by the builder, so a project may declare its own alongside these. - */ - compatibility: { - /** - * Lowest version of the installing application this box supports. - */ - minHostAppVersion?: string; - maxHostAppVersionExclusive?: string; - minMacosVersion?: string; - minRamGb?: number; - minNvidiaDriverVersion?: string; - [k: string]: unknown; - }; - /** - * Python version solved into the box. - */ - pythonVersion: string; - /** - * Pins the pixi release used to solve and install the conda-forge environment from the committed pixi.lock. - */ - pixiVersion: string; - /** - * Path to the reviewed licence inventory derived from pixi.lock, which carries an SPDX licence per package. The build fails if the lock no longer matches what was reviewed. - */ - condaDependencyLicenseAudit?: string; - /** - * Interpreter path relative to the box root. Must match the target adapter's layout. - */ - pythonEntryPoint: string; - modelCacheSubdir: string; - /** - * Base URL of the mirror the built archive and its objects are published under. - */ - assetBaseUrl?: string; - /** - * Files fetched during the build. Every entry is size- and hash-checked before use, so a moved or replaced upstream file fails the build instead of silently changing the box. May be empty. - */ - assets: { - url: string; - relativePath: PayloadPath; - sizeBytes: number; - sha256: Sha256; - }[]; - /** - * Downloaded archives to expand into the payload. Extraction preserves files already present in the destination and refuses to overwrite them. - */ - assetArchives?: { - relativePath: PayloadPath; - format: 'zip' | 'tar.gz'; - destination: PayloadPath; - stripComponents?: number; - removeAfterExtract?: boolean; - }[]; - /** - * Files copied from the consumer's own repository into the payload, each verified against a declared hash so a licence notice or runtime shim cannot drift from what was reviewed. - */ - localFiles?: { - sourcePath: string; - relativePath: PayloadPath; - sha256: Sha256; - }[]; - /** - * Payload paths deleted before packing, to keep the box to what it actually needs at run time. Pruning a distribution the lock requires is rejected. - */ - prunePaths?: PayloadPath[]; - /** - * Builder checks run with the payload's own interpreter before archiving. Schema version 2 signs only the import subset for a consumer to repeat; file and optional Python-code assertions remain builder-only. - */ - selfTest: { - /** - * @minItems 1 - */ - imports: [string, ...string[]]; - /** - * Files that must still exist after pruning, which is what stops an over-aggressive prune from shipping a broken box. - */ - files: PayloadPath[]; - /** - * Extra Python executed after the imports succeed, for checks a bare import cannot make. - */ - pythonCode?: string; - }; - /** - * Whether assets are packed into the archive (embed, the default: the box installs with no network and works air-gapped) or left out for the caller to materialize from descriptors in the signed release (on-demand). Consumers verify materialized assets before execution and do not download them. A build may override this. - */ - weights?: 'embed' | 'on-demand'; - execution?: BoxExecution; - /** - * An optional numerical gate: run a check inside the box on more than one accelerator and require the results to agree. This catches a mis-solved environment — CPU-only wheels shipped as CUDA, a broken BLAS — on the build machine rather than on a user's. The tool runs the check and enforces the thresholds; what the check computes, and what closeness is acceptable, belong to the project. - */ - parity?: { - /** - * Path inside the box, run with the box's own interpreter. It must print a JSON array of numbers, or an object with a "values" array. - */ - script: string; - /** - * Accelerators to run under, each with its target's validation environment. The first is the reference the others are compared against — conventionally cpu, being the one available everywhere. - * - * @minItems 2 - */ - accelerators: ['cpu' | 'metal' | 'cuda', 'cpu' | 'metal' | 'cuda', ...('cpu' | 'metal' | 'cuda')[]]; - /** - * At least one bound. Absolute guards entries near zero, where relative error is meaningless; cosine similarity catches a result that drifted in direction rather than magnitude. - */ - tolerances: { - absolute?: number; - relative?: number; - minimumCosine?: number; - }; - }; -} -/** - * Run one regular payload file with the box's own Python interpreter. - */ -export interface BoxManifest { - schemaVersion: 2; - boxId: string; - modelId: string; - runtimeId: string; - version: string; - target: BoxTarget; - pythonEntryPoint: string; - modelCacheSubdir: string; - selfTest: { - /** - * @minItems 1 - */ - pythonImports: [string, ...string[]]; - timeoutSeconds: number; - }; - execution?: BoxExecution; - provenance: Provenance; - /** - * Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it. - */ - weights?: 'on-demand'; - /** - * Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe. - * - * @minItems 1 - */ - assets?: [ - { - url: string; - relativePath: string; - sizeBytes: number; - sha256: string; - }, - ...{ - url: string; - relativePath: string; - sizeBytes: number; - sha256: string; - }[] - ]; -} -/** - * Run one regular payload file with the box's own Python interpreter. - */ -export interface Provenance { - scrollId: string; - scrollVersion: string; - /** - * Exact commit of the builder source that produced the box. - */ - builderRevision: string; - /** - * Whether the builder's working tree carried uncommitted changes. True means the build is not reproducible from the recorded revision alone. - */ - sourceTreeDirty: boolean; - /** - * Upstream revision of the packaged model source, as declared by the scroll. - */ - sourceRevision: string; - pythonVersion: string; - pixiVersion: string; - /** - * Hash of the pixi.lock the environment was solved from. - */ - dependencyLockSha256: string; - builtAt: string; -} - -export interface BoxReleaseManifest { - schemaVersion: 2; - /** - * Wire discriminator, ".release". The namespace belongs to the publishing project — a project with boxes already in the field must keep emitting the one its clients recognise — and defaults to scrollcase.box for a new one. - */ - kind: string; - boxId: Identifier; - modelId: Identifier; - runtimeId: Identifier; - version: string; - target: BoxTarget; - /** - * What the host must satisfy before this box may be installed. The builder copies these constraints through verbatim and never interprets them, so a project may add its own alongside the ones defined here. A consumer that cannot evaluate a constraint must refuse the box rather than assume it passes. - */ - compatibility: { - /** - * Lowest version of the installing application this box supports. - */ - minHostAppVersion?: string; - maxHostAppVersionExclusive?: string; - minMacosVersion?: string; - /** - * Installed memory in decimal gigabytes (1 GB = 1,000,000,000 bytes). - */ - minRamGb?: number; - minNvidiaDriverVersion?: string; - /** - * Host environments this payload was validated on. - * - * @minItems 1 - */ - hostEnvironments?: ['native' | 'windows-wsl2', ...('native' | 'windows-wsl2')[]]; - [k: string]: unknown; - }; - archive: { - format: 'zip'; - url: string; - sha256: Sha256; - sizeBytes: number; - }; - /** - * Sum of extracted payload file sizes before activation metadata is written, so a consumer can check free space before downloading. - */ - installedSizeBytes?: number; - /** - * Interpreter path relative to the extracted box root, for example venv/bin/python. Fixed per target by the adapter. - */ - pythonEntryPoint: string; - /** - * Directory relative to the extracted box root holding model assets. - */ - modelCacheSubdir: string; - /** - * The import check a consumer can repeat after extraction with the box's own interpreter. The builder also ran the scroll's Python-code and file assertions, which are builder-only checks. - */ - selfTest: { - /** - * @minItems 1 - */ - pythonImports: [string, ...string[]]; - timeoutSeconds: number; - }; - execution?: BoxExecution; - provenance: Provenance; - /** - * Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it. - */ - weights?: 'on-demand'; - /** - * Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe. - * - * @minItems 1 - */ - assets?: [ - { - url: string; - relativePath: string; - sizeBytes: number; - sha256: Sha256; - }, - ...{ - url: string; - relativePath: string; - sizeBytes: number; - sha256: Sha256; - }[] - ]; -} -/** - * Run one regular payload file with the box's own Python interpreter. - */ -export interface BoxChannelManifest { - schemaVersion: 2; - /** - * Wire discriminator, ".channel", carrying the same namespace as the releases it refers to. - */ - kind: string; - channel: 'nightly' | 'beta' | 'stable'; - boxId: string; - target: BoxTarget; - updatedAt: string; - /** - * Salt mixed into a client's rollout hash. It makes cohort assignment stable per client and unpredictable across channels, so a staged rollout cannot be gamed by reinstalling. - */ - cohortSalt: string; - /** - * Candidate releases in evaluation order. A client takes the first entry whose rollout cohort it falls into. - * - * @minItems 1 - */ - releases: [ - { - version: string; - releaseManifestUrl: string; - rolloutPercentage: number; - }, - ...{ - version: string; - releaseManifestUrl: string; - rolloutPercentage: number; - }[] - ]; -} - -/** - * Omitted when every target of that version is revoked. - */ -export interface BoxRevocationsManifest { - schemaVersion: 2; - /** - * Wire discriminator, ".revocations", carrying the same namespace as the releases it refers to. - */ - kind: string; - updatedAt: string; - /** - * May be empty: an empty signed list is a positive statement that nothing is revoked, which a client can distinguish from a missing or withheld document. - */ - revocations: { - boxId: string; - version: string; - target?: BoxTarget; - reason: string; - revokedAt: string; - }[]; -} - -/** - * The envelope wrapping every signed document. The payload travels as exact base64-encoded JSON so that verifying a signature means hashing the bytes as transmitted, with no canonical-JSON implementation to keep in sync across languages. Passing this schema means the envelope is well-formed, never that its signature is valid. - */ -export interface SignedBoxDocument { - schemaVersion: 2; - payloadEncoding: 'base64-json-utf8'; - /** - * The document payload: UTF-8 JSON, base64-encoded, signed and hashed exactly as it appears here. - */ - payloadBase64: string; - /** - * SHA-256 of the decoded payload bytes. - */ - payloadSha256: string; - /** - * Detached signatures over the decoded payload bytes. A verifier accepts the document when any one signature verifies against a trusted key, which is what allows a key to be rotated without reissuing every document. - * - * @minItems 1 - */ - signatures: [ - { - algorithm: 'ed25519'; - keyId: string; - signatureBase64: string; - }, - ...{ - algorithm: 'ed25519'; - keyId: string; - signatureBase64: string; - }[] - ]; -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/sign/index.mjs b/.scrollcase-runtime-types-RzUfxr/src/sign/index.mjs deleted file mode 100644 index 29a0b11..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/sign/index.mjs +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Signing, with key custody left to the operator. - * - * Two paths, one envelope. The built-in path signs with a local ed25519 key, which is enough for - * development and for anyone happy to hold their own key. The external path hands the payload to a - * command the operator configures — a KMS, an HSM, a signing service — so the private key never - * touches the build machine and Scrollcase never learns anything about the custody model. - * - * What the external path does *not* do is take the result on faith. The returned document must echo - * back exactly the payload that was sent, and its signature is verified locally before the build - * continues. A signer that substitutes a payload, or returns a signature that does not verify, fails - * the build rather than producing a box nobody can install. - */ - -import { fail, runResult as defaultRunResult } from '../build/process.mjs'; -import { readSigningKey, signWithLocalKey, verifySignedDocument } from './keys.mjs'; - -export { - decodeSignedDocument, - generateSigningKey, - readSigningKey, - verifySignedDocument, -} from './keys.mjs'; - -/** - * Runs an external signer command. - * - * The contract is deliberately the simplest thing that composes with anything: the command receives - * the payload bytes on stdin and writes the complete signed document as JSON on stdout. Any language, - * any credential mechanism, no plugin API to keep compatible. - */ -function commandArguments(command) { - if (Array.isArray(command)) { - if (!command.every((part) => typeof part === 'string')) { - fail('External signer command array must contain only strings.'); - } - return command; - } - const source = String(command); - const args = []; - let current = ''; - let quote = null; - let tokenStarted = false; - for (let index = 0; index < source.length; index += 1) { - const character = source[index]; - if (quote === "'") { - if (character === quote) quote = null; - else current += character; - } else if (quote === '"') { - if (character === quote) { - quote = null; - } else if (character === '\\' && ['\\', '"'].includes(source[index + 1])) { - current += source[index + 1]; - index += 1; - } else { - current += character; - } - } else if (character === '"' || character === "'") { - quote = character; - tokenStarted = true; - } else if (/\s/.test(character)) { - if (tokenStarted) { - args.push(current); - current = ''; - tokenStarted = false; - } - } else if (character === '\\' && source[index + 1] - && (/[\s'"\\]/).test(source[index + 1])) { - current += source[index + 1]; - tokenStarted = true; - index += 1; - } else { - current += character; - tokenStarted = true; - } - } - if (quote) fail('External signer command has an unmatched quote.'); - if (tokenStarted) args.push(current); - return args; -} - -function signWithCommand(payloadBytes, command, runResult) { - const [executable, ...args] = commandArguments(command); - if (!executable) fail('External signer command is empty.'); - const result = runResult(executable, args, { - input: payloadBytes, - capture: true, - maxBuffer: 16 * 1024 * 1024, - }); - if (result.error) fail(`External signer failed to start: ${result.error.message}`); - if (result.status !== 0) { - const stderr = (result.stderr?.toString() || '').trim(); - fail(`External signer exited with ${result.status}${stderr ? `: ${stderr}` : ''}`); - } - try { - return JSON.parse(result.stdout.toString('utf8')); - } catch (error) { - fail(`External signer did not return a JSON document: ${error instanceof Error ? error.message : String(error)}`); - } -} - -/** - * Wraps a manifest in the signed envelope, through whichever signer is configured. - * - * The payload is serialised once and both hashed and signed as-is, so what gets signed is - * byte-for-byte what gets published. - * - * @param {unknown} payload the manifest to wrap; serialised once and signed exactly as serialised - * @param {{ signerCommand?: string | string[] | null, privatePath?: string, publicPath: string, - * runResult?: typeof defaultRunResult }} signing - * @returns {Promise} - * @throws {Error} when an external signer fails, alters the payload, or returns an unverifiable - * signature - */ -export async function signDocument(payload, { - signerCommand = null, - privatePath, - publicPath, - runResult = defaultRunResult, -}) { - const payloadBytes = Buffer.from(`${JSON.stringify(payload, null, 2)}\n`, 'utf8'); - if (signerCommand) { - const document = signWithCommand(payloadBytes, signerCommand, runResult); - if (document?.payloadBase64 !== payloadBytes.toString('base64')) { - fail('External signer returned a different payload than the one it was given.'); - } - // Verified against the trust anchor the operator points at, not against the signer's word. - await verifySignedDocument(document, publicPath); - return document; - } - return signWithLocalKey(payloadBytes, await readSigningKey({ privatePath, publicPath })); -} diff --git a/.scrollcase-runtime-types-RzUfxr/src/sign/keys.mjs b/.scrollcase-runtime-types-RzUfxr/src/sign/keys.mjs deleted file mode 100644 index e58d23f..0000000 --- a/.scrollcase-runtime-types-RzUfxr/src/sign/keys.mjs +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Local signing keys and signature verification. - * - * A box is only worth as much as the signature over its release document, so the tool ships a - * working signer out of the box: `keygen` produces an ed25519 pair, and every document it emits can - * be verified with the matching public key. Production key custody is a separate concern — see the - * external signer in `index.mjs` — but verification always lives here, because a signature nobody - * checks is theatre. - */ - -import { - createPrivateKey, - createPublicKey, - generateKeyPairSync, - createHash, - sign as edSign, - verify as edVerify, -} from 'node:crypto'; -import { chmod, mkdir, readFile, writeFile } from 'node:fs/promises'; -import { dirname } from 'node:path'; -import { fail } from '../build/process.mjs'; -import { fileExists } from '../build/filesystem.mjs'; -import { BOX_SCHEMA_VERSION, PAYLOAD_ENCODING } from '../contract/document-shape.mjs'; - -/** - * A published public key, as written by `keygen` and read back when verifying. - * - * @typedef {object} TrustedKey - * @property {'ed25519'} algorithm - * @property {string} keyId stable identifier derived from the key itself - * @property {string} publicKeyBase64 the raw 32-byte key, for non-Node verifiers - * @property {string} publicKeyPem - */ - -const sha256Hex = (bytes) => createHash('sha256').update(bytes).digest('hex'); - -/** - * Creates an ed25519 signing pair. - * - * Overwriting an existing key is gated behind `force` because doing so silently would invalidate - * every document previously signed with it, with no way to tell which. - * - * @param {{ privatePath: string, publicPath: string, keyId?: string | null, force?: boolean }} options - * @returns {Promise<{ keyId: string, privatePath: string, publicPath: string }>} - * @throws {Error} when a key already exists and `force` was not passed - */ -export async function generateSigningKey({ privatePath, publicPath, keyId, force = false }) { - if (await fileExists(privatePath) && !force) { - fail(`Signing key already exists: ${privatePath}. Pass --force to rotate it explicitly.`); - } - const { privateKey, publicKey } = generateKeyPairSync('ed25519'); - const privatePem = privateKey.export({ type: 'pkcs8', format: 'pem' }); - const publicPem = publicKey.export({ type: 'spki', format: 'pem' }); - const publicDer = publicKey.export({ type: 'spki', format: 'der' }); - // An ed25519 SPKI DER is a fixed 12-byte header followed by the 32-byte key, so the raw key is - // simply the tail. That raw form is what non-Node verifiers expect in base64. - const rawPublicKey = publicDer.subarray(publicDer.length - 32); - // Deriving the ID from the key itself makes it stable and collision-resistant without a registry. - const resolvedKeyId = keyId || `scrollcase-${sha256Hex(rawPublicKey).slice(0, 16)}`; - await mkdir(dirname(privatePath), { recursive: true }); - await mkdir(dirname(publicPath), { recursive: true }); - // Owner-only, and chmod again afterwards in case a permissive umask widened the mode on create. - await writeFile(privatePath, privatePem, { mode: 0o600 }); - await chmod(privatePath, 0o600); - await writeFile(publicPath, `${JSON.stringify({ - algorithm: 'ed25519', - keyId: resolvedKeyId, - publicKeyBase64: rawPublicKey.toString('base64'), - publicKeyPem: publicPem, - }, null, 2)}\n`); - return { keyId: resolvedKeyId, privatePath, publicPath }; -} - -/** - * Loads the private key and cross-checks it against the published public key file, so a mismatched - * pair is caught here rather than producing documents nobody can verify. - * - * @param {{ privatePath: string, publicPath: string }} options - * @returns {Promise<{ privateKey: import('node:crypto').KeyObject, metadata: TrustedKey }>} - * @throws {Error} when the key is missing, or the pair does not match - */ -export async function readSigningKey({ privatePath, publicPath }) { - if (!await fileExists(privatePath)) fail(`Signing key not found: ${privatePath}. Run keygen first.`); - const privateKey = createPrivateKey(await readFile(privatePath, 'utf8')); - const publicKey = createPublicKey(privateKey); - const rawPublicKey = publicKey.export({ type: 'spki', format: 'der' }).subarray(-32); - const metadata = JSON.parse(await readFile(publicPath, 'utf8')); - if (metadata.publicKeyBase64 !== rawPublicKey.toString('base64')) { - fail('Private and public signing keys do not match.'); - } - return { privateKey, metadata }; -} - -/** Accepts both trust-file shapes: a bundle of keys, or a single bare key. */ -function trustedKeyEntries(value) { - return Array.isArray(value?.keys) ? value.keys : [value]; -} - -/** - * Signs payload bytes with a local key, producing the envelope the format defines. - * - * @param {Buffer} payloadBytes the exact bytes to sign, which are also the bytes published - * @param {{ privateKey: import('node:crypto').KeyObject, metadata: TrustedKey }} key - * @returns {import('../contract/types/index.d.ts').SignedBoxDocument} - */ -export function signWithLocalKey(payloadBytes, { privateKey, metadata }) { - return { - schemaVersion: BOX_SCHEMA_VERSION, - payloadEncoding: PAYLOAD_ENCODING, - payloadBase64: payloadBytes.toString('base64'), - payloadSha256: sha256Hex(payloadBytes), - signatures: [{ - algorithm: 'ed25519', - keyId: metadata.keyId, - signatureBase64: edSign(null, payloadBytes, privateKey).toString('base64'), - }], - }; -} - -/** - * Unwraps an envelope and checks its checksum. Does *not* check the signature. - * - * @param {import('../contract/types/index.d.ts').SignedBoxDocument} document - * @returns {{ bytes: Buffer, payload: unknown }} - * @throws {Error} when the envelope is unsupported or its checksum does not match - */ -export function decodeSignedDocument(document) { - if (document?.schemaVersion === 1) { - fail('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); - } - if (document?.schemaVersion !== BOX_SCHEMA_VERSION || document?.payloadEncoding !== PAYLOAD_ENCODING) { - fail('Unsupported signed document.'); - } - const bytes = Buffer.from(document.payloadBase64, 'base64'); - if (sha256Hex(bytes) !== document.payloadSha256) fail('Signed payload SHA-256 mismatch.'); - return { bytes, payload: JSON.parse(bytes.toString('utf8')) }; -} - -/** - * Verifies a signed document against a trusted key file and returns its payload. - * - * The document is accepted when *any one* signature verifies against a trusted key, which is what - * allows a document signed by both an outgoing and an incoming key to stay valid across a rotation. - * - * @param {import('../contract/types/index.d.ts').SignedBoxDocument} document - * @param {string} publicKeyPath a single trusted key, or a `{ keys: [...] }` bundle - * @returns {Promise} the payload, once a signature has verified against a trusted key - * @throws {Error} when no signature verifies - */ -export async function verifySignedDocument(document, publicKeyPath) { - const trusted = trustedKeyEntries(JSON.parse(await readFile(publicKeyPath, 'utf8'))); - const { bytes, payload } = decodeSignedDocument(document); - const valid = document.signatures?.some((signature) => { - const key = trusted.find((candidate) => candidate.keyId === signature.keyId); - return key?.publicKeyPem - && edVerify(null, bytes, createPublicKey(key.publicKeyPem), Buffer.from(signature.signatureBase64, 'base64')); - }); - if (!valid) fail('Document has no valid signature from a trusted ed25519 key.'); - return payload; -} From e76783dd139bb0b3abfa9b9b9f421fe3ebabc23f Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:09:05 +0200 Subject: [PATCH 03/22] Break the wire to version 3: a declared runtime, per-asset embed, declared executables The single format break the v3 plan allows. Every change that touches a schema, a document or a fixture lands here, so implementing the runtimes the vocabulary now names is pure code. - A box declares $(printf %s 'runtime: { id, version, entryPoint }') instead of leaving a reader to infer Python from a Python-shaped entry point. The wire vocabulary names python, node and native; only python has an adapter, and a box naming another is refused by name rather than misread. - modelId and runtimeId are gone, replaced by an optional labels map the tool never reads. modelCacheSubdir becomes cacheSubdir; pythonVersion and pythonEntryPoint move into the runtime block; provenance.pythonVersion becomes provenance.runtimeVersion. - weights is gone. Whether an asset ships inside the archive is per entry, so a box can embed a small entry point and defer a large dataset in one build. --weights goes with it: a build-time override of a per-asset declaration is the silent-repack bug the flag already warned about. - The self-test generalises. selfTest.probe carries imports and commands, and the runtime is the only thing that turns either into a command line. - The executable bit is declared rather than inferred from venv/bin: assets and local files may ask for it, and extraction chmods explicitly so a strict umask cannot take it away again. Still to do in this phase: the tri-language consumers, the conformance fixture, the examples and the docs. --- src/build/archive.d.mts | 12 +- src/build/archive.mjs | 72 ++-- src/build/authoring.mjs | 47 +- src/build/box.mjs | 189 ++++---- src/build/execution.d.mts | 10 +- src/build/execution.mjs | 8 +- src/build/pixi.d.mts | 4 +- src/build/pixi.mjs | 6 +- src/build/process.d.mts | 7 + src/build/process.mjs | 8 +- src/build/scroll-edit.mjs | 60 ++- src/build/scroll.mjs | 74 ++-- src/build/verify.d.mts | 10 +- src/build/verify.mjs | 63 ++- src/cli-authoring.mjs | 38 +- src/cli.mjs | 45 +- src/consumer/run-extracted.mjs | 24 +- src/consumer/verify-and-extract.d.mts | 13 +- src/consumer/verify-and-extract.mjs | 31 +- src/contract/browser.d.mts | 2 +- src/contract/document-shape.d.mts | 16 +- src/contract/document-shape.mjs | 23 +- .../examples/box-manifest.example.json | 28 +- .../examples/channel-manifest.example.json | 2 +- .../examples/release-manifest.example.json | 28 +- .../examples/scroll-pixi.example.json | 32 +- .../fixtures/examples/scroll.example.json | 23 +- .../examples/signed-release.example.json | 10 +- .../examples/signed-release.public-key.json | 4 +- src/contract/fixtures/runtime-contract.json | 404 ++++++++++++++++-- src/contract/index.d.mts | 3 +- src/contract/index.mjs | 20 +- src/contract/runtimes.d.mts | 97 ++++- src/contract/runtimes.mjs | 135 ++++-- src/contract/schema/box-manifest.schema.json | 96 +---- .../schema/channel-manifest.schema.json | 6 +- src/contract/schema/execution.schema.json | 52 ++- .../schema/release-manifest.schema.json | 176 +++++--- .../schema/revocations-manifest.schema.json | 6 +- src/contract/schema/scroll.schema.json | 179 +++++--- .../schema/signed-document.schema.json | 4 +- src/contract/schema/target.schema.json | 2 +- src/contract/targets.d.mts | 13 - src/contract/targets.mjs | 18 - src/contract/types/index.d.ts | 260 ++++++----- tests/helpers/consumer-box-fixture.mjs | 29 +- tests/unit/build-pipeline.test.mjs | 237 +++++++--- tests/unit/consumer.test.mjs | 2 +- tests/unit/contract-runtimes.test.mjs | 63 ++- tests/unit/contract-schema.test.mjs | 17 +- 50 files changed, 1851 insertions(+), 857 deletions(-) diff --git a/src/build/archive.d.mts b/src/build/archive.d.mts index a40d22e..1ebf337 100644 --- a/src/build/archive.d.mts +++ b/src/build/archive.d.mts @@ -10,11 +10,17 @@ * @param {string} payloadDir * @param {string} archivePath * @param {import('../contract/targets.mjs').BoxTargetAdapter} adapter - * @param {readonly string[]} [uncompressedPaths] payload paths stored rather than deflated - * @param {string} [runtimeId] whose layout decides which entries carry the executable bit + * @param {object} options + * @param {string} options.runtimeId whose rule decides which entries carry the executable bit + * @param {readonly string[]} [options.uncompressedPaths] payload paths stored rather than deflated + * @param {readonly string[]} [options.executablePaths] payload paths the scroll declared executable * @returns {Promise} */ -export function createDeterministicZip(payloadDir: string, archivePath: string, adapter: import("../contract/targets.mjs").BoxTargetAdapter, uncompressedPaths?: readonly string[], runtimeId?: string): Promise; +export function createDeterministicZip(payloadDir: string, archivePath: string, adapter: import("../contract/targets.mjs").BoxTargetAdapter, options: { + runtimeId: string; + uncompressedPaths?: readonly string[]; + executablePaths?: readonly string[]; +}): Promise; /** * Lists and validates all entries before any ZIP data is trusted or extracted. * diff --git a/src/build/archive.mjs b/src/build/archive.mjs index 81e3cc1..a11aca4 100644 --- a/src/build/archive.mjs +++ b/src/build/archive.mjs @@ -2,8 +2,8 @@ * Deterministic archive creation and defensive extraction. * * Writing: every box ships as a ZIP whose bytes depend only on its contents — fixed timestamps, - * stable file ordering, and modes derived from the target adapter — so rebuilding the same commit - * reproduces the archive bit for bit. + * stable file ordering, and modes synthesised from what the runtime and the scroll declared rather + * than read off the build machine — so rebuilding the same commit reproduces the archive bit for bit. * * Reading: nothing from inside an archive is trusted before it is validated. Entry names are * checked against path traversal, links and special entries are rejected outright, and both ZIP @@ -11,7 +11,7 @@ * to have — an archive behaves the same on every machine that opens it. */ import { constants, createWriteStream } from 'node:fs'; -import { copyFile, mkdir, mkdtemp, rm, stat, symlink } from 'node:fs/promises'; +import { chmod, copyFile, mkdir, mkdtemp, rm, stat, symlink } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { pipeline } from 'node:stream/promises'; @@ -19,11 +19,7 @@ import * as tar from 'tar'; import yauzl from 'yauzl'; import yazl from 'yazl'; import { findEntryThroughLink, findUnresolvableLink } from '../contract/links.mjs'; -import { - IMPLICIT_RUNTIME_ID, - isExecutablePayloadPath, - runtimeAdapter, -} from '../contract/runtimes.mjs'; +import { isExecutablePayloadPath, runtimeAdapter } from '../contract/runtimes.mjs'; import { FIXED_ARCHIVE_TIME, collectEntries, @@ -42,20 +38,38 @@ const ZIP_SYMBOLIC_LINK = 0o120000; /** * Returns the stable archive mode for a box payload file. * - * Which paths need the bit is the runtime's rule, not the target's: a conda prefix generates - * hundreds of console scripts that no scroll could name, and the runtime is what knows where they - * land. The mode is still *synthesised* rather than read off disk, which is what keeps two builds - * of one commit byte-identical and keeps `payload-digest.v1` — which excludes mode — honest. + * Executability is *declared*, from two sources joined into one rule. The runtime contributes what + * no scroll could name by hand — a conda prefix generates hundreds of console scripts, and the + * runtime is what knows where they land. The scroll contributes everything it brought in itself: + * an asset arrives over HTTP, which carries content and not permissions, and a local file is copied + * rather than moved, so neither has a mode to inherit and both would otherwise be unrunnable. + * + * The mode is still *synthesised* rather than read off disk, which is what keeps two builds of one + * commit byte-identical whatever umask each ran under, and keeps `payload-digest.v1` — which + * excludes mode — honest. */ function archiveFileMode(adapter, relativePath, executablePaths) { if (adapter.host.platform === 'win32') return 0o100644; return isExecutablePayloadPath(executablePaths, relativePath) ? 0o100755 : 0o100644; } +/** + * Joins the runtime's executable rule with the paths the scroll declared executable. + * + * @param {string} runtimeId + * @param {import('../contract/targets.mjs').BoxTargetAdapter} adapter + * @param {readonly string[]} declared payload paths the scroll marked executable + * @returns {import('../contract/runtimes.mjs').ExecutablePayloadPaths} + */ +function declaredExecutablePaths(runtimeId, adapter, declared) { + const rule = runtimeAdapter(runtimeId).executablePayloadPaths(adapter); + return { files: [...rule.files, ...declared], directories: rule.directories }; +} + /** * Whether a payload path was declared as one whose bytes are already compressed. * - * A match is exact or by directory prefix, so one declaration can name a single weights file or + * A match is exact or by directory prefix, so one declaration can name a single large file or * the whole tree an expanded asset archive landed in. Nothing here opens the file or reads its * extension: the answer depends only on the scroll and the path, which is what keeps two builds of * the same commit byte-identical. @@ -73,26 +87,23 @@ function isDeclaredUncompressed(path, declared) { * * Deflating an already-compressed file is pure loss: measured on incompressible bytes, level 6 * runs at 47 MB/s and the result is 0.03% *larger* than the input, and dropping to level 1 buys - * 4 MB/s because the search fails either way. Weights are the only thing in a box large enough for - * that to matter, so `uncompressedPaths` names them and they are stored instead. Everything else — + * 4 MB/s because the search fails either way. Declared assets are the only thing in a box large + * enough for that to matter, so they and `uncompressedPaths` are stored instead. Everything else — * the interpreter, the site-packages tree, the notices — compresses genuinely and still does. * * @param {string} payloadDir * @param {string} archivePath * @param {import('../contract/targets.mjs').BoxTargetAdapter} adapter - * @param {readonly string[]} [uncompressedPaths] payload paths stored rather than deflated - * @param {string} [runtimeId] whose layout decides which entries carry the executable bit + * @param {object} options + * @param {string} options.runtimeId whose rule decides which entries carry the executable bit + * @param {readonly string[]} [options.uncompressedPaths] payload paths stored rather than deflated + * @param {readonly string[]} [options.executablePaths] payload paths the scroll declared executable * @returns {Promise} */ -export async function createDeterministicZip( - payloadDir, - archivePath, - adapter, - uncompressedPaths = [], - runtimeId = IMPLICIT_RUNTIME_ID, -) { +export async function createDeterministicZip(payloadDir, archivePath, adapter, options) { + const { runtimeId, uncompressedPaths = [], executablePaths: declared = [] } = options; const entries = await collectEntries(payloadDir); - const executablePaths = runtimeAdapter(runtimeId).executablePayloadPaths(adapter); + const executablePaths = declaredExecutablePaths(runtimeId, adapter, declared); assertPayloadLinksAreCarryable(entries); await rm(archivePath, { force: true }); await mkdir(dirname(archivePath), { recursive: true }); @@ -306,10 +317,13 @@ export async function extractZipArchive(archivePath, destination) { continue; } const stream = await zip.openReadStreamPromise(entry); - await pipeline(stream, createWriteStream(outputPath, { - flags: 'wx', - mode: classified.mode || 0o644, - })); + const mode = classified.mode || 0o644; + await pipeline(stream, createWriteStream(outputPath, { flags: 'wx', mode })); + // `open(2)` masks the mode it is given by the process umask, so a box extracted under 077 + // would silently lose the executable bit the archive states — and the box would fail to run + // for reasons nothing in it explains. Say the mode again, explicitly, the way key writing + // already has to (`src/sign/keys.mjs`). Windows has no bit to restore. + if (process.platform !== 'win32') await chmod(outputPath, mode & 0o7777); } } finally { await zip.close(); diff --git a/src/build/authoring.mjs b/src/build/authoring.mjs index b8cd636..bfe8b9d 100644 --- a/src/build/authoring.mjs +++ b/src/build/authoring.mjs @@ -15,7 +15,7 @@ import { lstat, mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; import { dirname, isAbsolute, join, relative, sep } from 'node:path'; import { boxTargetAdapter, boxTargetId, condaSubdir } from '../contract/targets.mjs'; -import { IMPLICIT_RUNTIME_ID } from '../contract/runtimes.mjs'; +import { BOX_SCHEMA_VERSION } from '../contract/documents.mjs'; import { runtimeBuilder } from '../runtimes/index.mjs'; import { fileExists, safeRelativePath } from './filesystem.mjs'; import { fail } from './process.mjs'; @@ -25,9 +25,12 @@ const scrollSchemaUrl = new URL('../contract/schema/scroll.schema.json', import. const targetSchemaUrl = new URL('../contract/schema/target.schema.json', import.meta.url); const executionSchemaUrl = new URL('../contract/schema/execution.schema.json', import.meta.url); const EXECUTION_KINDS = Object.freeze(['python-script', 'python-module', 'library-only']); -export const WEIGHTS_MODES = Object.freeze(['embed', 'on-demand']); -/** The schema's own default, so a scroll that takes it says nothing about weights at all. */ -export const DEFAULT_WEIGHTS_MODE = 'embed'; +/** + * The runtime `scrollcase new` writes. Authoring is deliberately narrower than the format: the + * wire vocabulary names every runtime the format defines, and this names the one a generated + * scroll can actually be built from today. + */ +export const AUTHORED_RUNTIME_ID = 'python'; export const EXAMPLE_PIXI_VERSION = '0.73.0'; export const DEFAULT_SCROLL_VERSION = '1.0.0'; @@ -234,7 +237,7 @@ function projectRelativePath(projectRoot, path) { return relativePath.split(sep).join('/'); } -function pixiManifest(environmentName, target, runtimeVersion, runtimeId = IMPLICIT_RUNTIME_ID) { +function pixiManifest(environmentName, target, runtimeVersion, runtimeId = AUTHORED_RUNTIME_ID) { // The workspace table is substrate — one channel, one platform, whatever the box runs. Only the // dependency line knows which runtime is being packed, and the runtime is what writes it. const runtime = runtimeBuilder(runtimeId).pixiDependency(runtimeVersion); @@ -271,8 +274,7 @@ export async function createScroll({ workspace, boxId, target, - modelId, - runtimeId, + labels = {}, version, scrollVersion = DEFAULT_SCROLL_VERSION, sourceRevision, @@ -280,7 +282,6 @@ export async function createScroll({ pixiVersion, compatibility = {}, assetBaseUrl, - weights = DEFAULT_WEIGHTS_MODE, executionKind, scriptSourcePath = null, generateScript = false, @@ -296,22 +297,20 @@ export async function createScroll({ const identity = { boxId: requiredText(boxId, 'boxId'), - modelId: requiredText(modelId, 'modelId'), - runtimeId: requiredText(runtimeId, 'runtimeId'), version: requiredText(version, 'version'), scrollVersion: requiredText(scrollVersion, 'scrollVersion'), sourceRevision: requiredText(sourceRevision, 'sourceRevision'), pythonVersion: requiredText(pythonVersion, 'pythonVersion'), pixiVersion: requiredText(pixiVersion, 'pixiVersion'), - // Required whatever the weights mode: the release manifest names the archive's own published - // URL, not just the assets'. + // Required whether or not any asset is deferred: the release manifest names the archive's own + // published URL, not just the assets'. assetBaseUrl: requiredText(assetBaseUrl, 'assetBaseUrl'), }; if (!compatibility || typeof compatibility !== 'object' || Array.isArray(compatibility)) { fail('compatibility must be an object.'); } - if (!WEIGHTS_MODES.includes(weights)) { - fail(`Unsupported weights mode: ${weights}. Use ${WEIGHTS_MODES.join(' or ')}.`); + if (!labels || typeof labels !== 'object' || Array.isArray(labels)) { + fail('labels must be an object.'); } if (!EXECUTION_KINDS.includes(executionKind)) { fail(`Unsupported execution kind: ${executionKind}. Use ${EXECUTION_KINDS.join(', ')}.`); @@ -347,7 +346,7 @@ export async function createScroll({ if (await fileExists(generatedScriptPath)) { fail(`Generated script already exists: ${sourcePath}.`); } - generatedSource = runtimeBuilder(IMPLICIT_RUNTIME_ID).templates.script; + generatedSource = runtimeBuilder(AUTHORED_RUNTIME_ID).templates.script; } else { sourcePath = safeRelativePath(scriptSourcePath); const source = join(workspace.root, ...sourcePath.split('/')); @@ -381,11 +380,10 @@ export async function createScroll({ // person had to choose. const selfTestPath = projectRelativePath(workspace.root, join(scrollDir, 'self_test.py')); const scroll = { - $schema: 'https://scrollcase.dev/schema/v2/scroll.schema.json', - schemaVersion: 2, + $schema: 'https://scrollcase.dev/schema/v3/scroll.schema.json', + schemaVersion: BOX_SCHEMA_VERSION, boxId: identity.boxId, - modelId: identity.modelId, - runtimeId: identity.runtimeId, + ...(Object.keys(labels).length > 0 ? { labels: { ...labels } } : {}), version: identity.version, sourceRevision: identity.sourceRevision, target, @@ -393,17 +391,14 @@ export async function createScroll({ ? {} : { scrollVersion: identity.scrollVersion }), ...(Object.keys(compatibility).length > 0 ? { compatibility: { ...compatibility } } : {}), - pythonVersion: identity.pythonVersion, + runtime: { id: AUTHORED_RUNTIME_ID, version: identity.pythonVersion }, pixiVersion: identity.pixiVersion, assetBaseUrl: identity.assetBaseUrl, selfTest: { imports: ['json'], ...(localFile ? { files: [localFile.relativePath] } : {}), - ...(selfTestPath ? { pythonFile: selfTestPath } : {}), + ...(selfTestPath ? { script: selfTestPath } : {}), }, - // Left out when it is the schema's default: a scroll should read like the decisions its author - // made, and most boxes have no asset to leave out of the archive in the first place. - ...(weights === DEFAULT_WEIGHTS_MODE ? {} : { weights }), ...(localFile ? { localFiles: [localFile] } : {}), ...(execution ? { execution } : {}), }; @@ -422,7 +417,7 @@ export async function createScroll({ if (selfTestPath) { await writeFile( join(staging, 'self_test.py'), - runtimeBuilder(IMPLICIT_RUNTIME_ID).templates.selfTest, + runtimeBuilder(AUTHORED_RUNTIME_ID).templates.selfTest, ); } if (generatedScriptPath) { @@ -479,8 +474,6 @@ export async function ensureExampleScroll({ workspace, boxId: 'example-box', target, - modelId: 'example-org-example-box', - runtimeId: 'example-box-runtime', version: '1.0.0', sourceRevision: 'example-source-1.0.0', pixiVersion, diff --git a/src/build/box.mjs b/src/build/box.mjs index e545642..d758f64 100644 --- a/src/build/box.mjs +++ b/src/build/box.mjs @@ -6,10 +6,11 @@ * the payload's *own* interpreter, normalise timestamps, zip deterministically, and emit a signed * release plus a signed channel pointer. * - * The self-test is the step that earns the box its name. The builder runs target, import, optional - * Python-code, and file assertions; the release signs the import subset that a consumer can repeat - * after extraction. The distinction is deliberate rather than pretending the narrower consumer - * check reproduces scroll-only assertions it cannot see. + * The self-test is the step that earns the box its name. The builder runs the platform assertion, + * the probe the scroll declared, its file assertions and any extra source it named; the release + * signs the probe alone, which is what a consumer can repeat after extraction. The distinction is + * deliberate rather than pretending the narrower consumer check reproduces assertions it cannot + * see. * * The archive is content-addressed by its own hash, so the release document can commit to it and any * consumer can verify it byte for byte. @@ -19,8 +20,8 @@ import { createHash } from 'node:crypto'; import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { assertNativeHost, boxTargetId } from '../contract/targets.mjs'; -import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../contract/runtimes.mjs'; -import { CHANNELS, documentKinds } from '../contract/documents.mjs'; +import { runtimeAdapter } from '../contract/runtimes.mjs'; +import { BOX_SCHEMA_VERSION, CHANNELS, documentKinds } from '../contract/documents.mjs'; import { mergeEnvironmentLayers } from '../environment.mjs'; import { PAYLOAD_DIGEST_FILE, @@ -52,36 +53,50 @@ const SELF_TEST_TIMEOUT_SECONDS = 180; const sha256Hex = (bytes) => createHash('sha256').update(bytes).digest('hex'); /** - * The extra self-test Python a scroll declares, read from the file it names or taken inline. + * The extra self-test source a scroll declares, read from the file it names or taken inline. * - * A real self-test outgrows a JSON string quickly, so `pythonFile` points at a file in the project - * that an editor, a linter and a diff can all see. It is read here rather than at scroll-read time + * A real self-test outgrows a JSON string quickly, so `script` points at a file in the project that + * an editor, a linter and a diff can all see. It is read here rather than at scroll-read time * because it is build input, not part of the box format. */ async function selfTestExtraCode(scroll, projectRoot) { - const { pythonCode, pythonFile } = scroll.selfTest; - if (!pythonFile) return pythonCode ?? null; - const path = join(projectRoot, safeRelativePath(pythonFile)); - if (!await fileExists(path)) fail(`Self-test Python file is missing: ${pythonFile}`); + const { code, script } = scroll.selfTest; + if (!script) return code ?? null; + const path = join(projectRoot, safeRelativePath(script)); + if (!await fileExists(path)) fail(`Self-test script is missing: ${script}`); return readFile(path, 'utf8'); } -/** Runs the scroll's self-test with the payload's own interpreter, under the target's environment. */ -function runSelfTest({ interpreter, adapter, scroll, payloadDir, run, extraCode = null }) { - // The builder's probe is the signed one plus whatever extra source the scroll declared, and the - // runtime is the only thing that knows how to turn either into a command line. - const argv = runtimeAdapter(IMPLICIT_RUNTIME_ID).selfTestArgv({ - probe: { imports: scroll.selfTest.imports, code: extraCode }, +/** + * Runs the scroll's self-test against the payload, under the target's validation environment. + * + * The builder's probe is the signed one plus whatever extra source the scroll declared, and the + * runtime is the only thing that knows how to turn either into command lines. A probe may imply + * several — an import check and any number of command invocations — and each is run from the + * payload root with its own required exit status, because a check that passes by exiting non-zero + * is a check that never ran. + */ +function runSelfTest({ adapter, scroll, payloadDir, run, extraCode = null }) { + const invocations = runtimeAdapter(scroll.runtime.id).selfTestInvocations({ + probe: { ...scroll.selfTest, code: extraCode }, + execution: scroll.execution, target: adapter, }); - run(interpreter, argv, { - cwd: payloadDir, - env: mergeEnvironmentLayers( - adapter.platform, - scroll.environment ?? {}, - adapter.validationEnvironments[scroll.target.accelerator], - ), - }); + const env = mergeEnvironmentLayers( + adapter.platform, + scroll.environment ?? {}, + adapter.validationEnvironments[scroll.target.accelerator], + ); + const resolve = (argument) => (argument.kind === 'payload-path' + ? join(payloadDir, ...safeRelativePath(argument.value).split('/')) + : argument.value); + for (const invocation of invocations) { + run(resolve(invocation.command), invocation.args.map(resolve), { + cwd: payloadDir, + env, + expectExitCode: invocation.expectExitCode, + }); + } } /** Writes the licence inventory the box ships, after proving it still matches the reviewed one. */ @@ -102,7 +117,7 @@ async function writeLicenceAudit({ scroll, lockPath, payloadDir, projectRoot }) /** * Builds, self-tests, archives, and signs the box a scroll describes — the whole pipeline the * module header narrates. `name` is an exact scroll reference, or an unambiguous box shorthand; - * options override signing, channel, weights mode, namespace, and toolchain paths. `run`, + * options override signing, channel, namespace, and toolchain paths. `run`, * `runResult`, and `fetchImpl` are the injection seams the tests use to substitute the toolchain * and asset transport. */ @@ -110,7 +125,6 @@ export async function buildBox(name, options = {}) { const { allowDirty = false, channel = 'beta', - weights = null, assetBaseUrl: assetBaseUrlOverride = null, namespace, signerCommand = null, @@ -128,19 +142,15 @@ export async function buildBox(name, options = {}) { const probe = runResult ? { runResult } : {}; const workspace = getWorkspace(); const { adapter, dir, scroll } = await readScroll(name); - const weightsMode = weights || scroll.weights || 'embed'; if (!CHANNELS.includes(channel)) { fail(`Unsupported channel: ${channel}. Use ${CHANNELS.join(' or ')}.`); } - if (weightsMode !== 'embed' && weightsMode !== 'on-demand') { - fail(`Unsupported weights mode: ${weightsMode}. Use embed or on-demand.`); - } - if (weightsMode === 'on-demand' && (scroll.assetArchives ?? []).length > 0) { - fail('on-demand weights cannot be combined with assetArchives, which are expanded at build time.'); - } - // Reported rather than asked: the mode decides whether declared assets ship inside the archive, - // and the CLI no longer puts a menu in front of what the scroll already says. - log(`Weights: ${weightsMode}`); + // Whether an asset ships inside the archive is per entry and declared by the scroll, so there is + // nothing to ask and nothing to override: a build-time override would silently repack a box under + // an identity that no longer describes it. What the scroll decided is reported, not negotiated. + const deferred = scroll.assets.filter((asset) => asset.embed === false); + log(`Runtime: ${scroll.runtime.id}${scroll.runtime.version ? ` ${scroll.runtime.version}` : ''}`); + log(`Assets: ${scroll.assets.length - deferred.length} embedded, ${deferred.length} on demand`); // Wheels, native libraries, and the interpreter are proven on the exact OS/architecture they ship for. assertNativeHost(adapter); const pixi = findPixi({ requiredVersion: scroll.pixiVersion, path: pixiPath, ...probe }); @@ -192,27 +202,31 @@ export async function buildBox(name, options = {}) { payloadDir, adapter, run: runEnvironmentCommand, + runtimeId: scroll.runtime.id, }); log('Preparing payload'); - // `embed` packs the assets into the archive, so an installed box needs no network and works - // air-gapped. `on-demand` leaves them out for the caller's distribution layer to materialize from - // descriptors carried in the signed release. Consumers verify those bytes before execution; the - // declared hash is what keeps that safe. The choice trades archive size against an install-time - // dependency on the asset host, so it is the project's to make, per build. - const embedded = weightsMode === 'embed'; - for (const asset of embedded ? scroll.assets : []) { + // An embedded asset is packed into the archive, so a box made only of them installs with no + // network and works air-gapped. A deferred one is left out and its descriptor carried in the + // signed release instead, for the caller's distribution layer to materialize; consumers verify + // those bytes before execution, and the declared hash is what keeps that safe. The choice trades + // archive size against an install-time dependency on the asset host, and it is per entry so that + // a box can ship a small entry point and defer a large dataset. + for (const asset of scroll.assets) { + if (asset.embed === false) continue; log(`Downloading ${asset.relativePath}`); await downloadVerified(asset, join(payloadDir, safeRelativePath(asset.relativePath)), { fetchImpl, log, }); } - const deferredAssets = new Set(embedded ? [] : scroll.assets.map((asset) => asset.relativePath)); + const deferredAssets = new Set(deferred.map((asset) => asset.relativePath)); for (const file of scroll.localFiles ?? []) { await copyVerifiedLocalFile(file, payloadDir, workspace.root); } - for (const archive of embedded ? scroll.assetArchives ?? [] : []) { + // An asset archive is expanded at build time and has no descriptor to defer, so the schema gives + // it no `embed` field and there is nothing to skip here. + for (const archive of scroll.assetArchives ?? []) { await expandAssetArchive(payloadDir, archive); } // Drops what is only needed to build (tests, docs, bundled sample data). A box is a multi-gigabyte @@ -233,7 +247,8 @@ export async function buildBox(name, options = {}) { assertExecutionFiles({ execution: scroll.execution, adapter, - runtimeVersion: scroll.pythonVersion, + runtimeId: scroll.runtime.id, + runtimeVersion: scroll.runtime.version, files: new Set(await collectFiles(payloadDir)), }); // Everything needed to answer "where did this box come from, and could I rebuild it?". @@ -243,26 +258,42 @@ export async function buildBox(name, options = {}) { builderRevision: source.revision, sourceTreeDirty: source.dirty, sourceRevision: scroll.sourceRevision, - pythonVersion: scroll.pythonVersion, + // Omitted rather than defaulted when the runtime has no version: an invented value is a + // provenance record that lies about what was observed. + ...(scroll.runtime.version === undefined ? {} : { runtimeVersion: scroll.runtime.version }), ...builderVersionFields(scroll), dependencyLockSha256: lockSha, builtAt: sourceBuildTime(workspace.root), }; + // The signed probe is what a consumer can repeat: the imports and commands, and neither the file + // assertions nor the extra source, which no consumer can see to reproduce. const selfTest = { - pythonImports: scroll.selfTest.imports, + probe: { + ...(scroll.selfTest.imports ? { imports: scroll.selfTest.imports } : {}), + ...(scroll.selfTest.commands + ? { + commands: scroll.selfTest.commands.map(({ args, expectExitCode }) => ({ + args, + expectExitCode: expectExitCode ?? 0, + })), + } + : {}), + }, timeoutSeconds: SELF_TEST_TIMEOUT_SECONDS, }; - // Descriptors travel with the box only when the consumer has to fetch the assets itself. - const deferred = embedded ? {} : { - weights: 'on-demand', - assets: scroll.assets.map(({ url, relativePath, sizeBytes, sha256 }) => ({ - url, relativePath, sizeBytes, sha256, + // Descriptors travel with the box only for the assets the consumer has to fetch itself. + const deferredDescriptors = deferred.length === 0 ? {} : { + assets: deferred.map(({ url, relativePath, sizeBytes, sha256, executable }) => ({ + url, + relativePath, + sizeBytes, + sha256, + ...(executable ? { executable: true } : {}), })), }; const identity = { boxId: scroll.boxId, - modelId: scroll.modelId, - runtimeId: scroll.runtimeId, + ...(scroll.labels === undefined ? {} : { labels: scroll.labels }), version: scroll.version, }; const execution = scroll.execution ? { execution: scroll.execution } : {}; @@ -276,21 +307,20 @@ export async function buildBox(name, options = {}) { // self-tested at all: the check ran against a payload missing a file the box has. Nothing here // depends on the test or the parity gate, so there was never a reason for it to wait. await writeFile(join(payloadDir, 'box.json'), `${JSON.stringify({ - schemaVersion: 2, + schemaVersion: BOX_SCHEMA_VERSION, ...identity, target: scroll.target, - pythonEntryPoint: scroll.pythonEntryPoint, - modelCacheSubdir: scroll.modelCacheSubdir, + runtime: scroll.runtime, + cacheSubdir: scroll.cacheSubdir, selfTest, ...environment, ...execution, - ...deferred, + ...deferredDescriptors, provenance, }, null, 2)}\n`); log('Running self-test'); runSelfTest({ - interpreter, adapter, scroll, payloadDir, @@ -317,7 +347,7 @@ export async function buildBox(name, options = {}) { // cannot describe itself. It goes in before `normalizeTree` so it carries the same fixed mtime as // the rest, and before `payloadSize` so the size a consumer checks free space against is honest. // This costs one full sequential read of the payload — real minutes on a box carrying embedded - // weights — and it is paid here rather than folded into the archive writer, which has no business + // assets — and it is paid here rather than folded into the archive writer, which has no business // knowing a format rule that is not about archiving. const digestStream = payloadDigestStream(await payloadDigestEntries(payloadDir)); await writeFile(join(payloadDir, PAYLOAD_DIGEST_FILE), digestStream); @@ -328,14 +358,27 @@ export async function buildBox(name, options = {}) { // Declared assets are the one thing a box carries that arrives already compressed, so they are // stored rather than deflated without the project having to say so. `uncompressedPaths` covers // what only the project can know: the tree an expanded archive left behind, a bundled corpus. - // Under `on-demand` the assets are not in the payload at all and the first list simply matches - // nothing. + // A deferred asset is not in the payload at all, and simply matches nothing. const uncompressedPaths = [ ...scroll.assets.map((asset) => safeRelativePath(asset.relativePath)), ...(scroll.uncompressedPaths ?? []).map((path) => safeRelativePath(path)), ]; + // The executable bit is synthesised, never read off the build machine, so what carries it has to + // be *declared*: the runtime's own rule covers the interpreter and the console scripts a conda + // prefix generates, and the scroll covers everything it brought in itself. A downloaded file + // arrives with no mode at all — HTTP carries content, not permissions — so without this a box + // could not ship an asset that runs. + const declaredExecutablePaths = [ + ...scroll.assets, + ...(scroll.localFiles ?? []), + ].filter((entry) => entry.executable && entry.embed !== false) + .map((entry) => safeRelativePath(entry.relativePath)); log('Creating deterministic archive'); - await createDeterministicZip(payloadDir, archivePath, adapter, uncompressedPaths); + await createDeterministicZip(payloadDir, archivePath, adapter, { + uncompressedPaths, + runtimeId: scroll.runtime.id, + executablePaths: declaredExecutablePaths, + }); log('Hashing deterministic archive'); const archiveSha = await sha256File(archivePath); @@ -350,7 +393,7 @@ export async function buildBox(name, options = {}) { log('Signing release and channel'); const release = { - schemaVersion: 2, + schemaVersion: BOX_SCHEMA_VERSION, kind: kinds.release, ...identity, target: scroll.target, @@ -358,12 +401,12 @@ export async function buildBox(name, options = {}) { archive: { format: 'zip', url: `${assetBaseUrl}/${archiveObject}`, sha256: archiveSha, sizeBytes: archiveSize }, installedSizeBytes, payloadDigest: payloadDigestValue, - pythonEntryPoint: scroll.pythonEntryPoint, - modelCacheSubdir: scroll.modelCacheSubdir, + runtime: scroll.runtime, + cacheSubdir: scroll.cacheSubdir, selfTest, ...environment, ...execution, - ...deferred, + ...deferredDescriptors, provenance, }; // Written beside the archive, under the same scratch rule: named for its own hash once it has one. @@ -374,7 +417,7 @@ export async function buildBox(name, options = {}) { // content-addressed: channel -> release document -> archive. const releaseDocumentSha = await sha256File(stagedReleasePath); const channelDocument = { - schemaVersion: 2, + schemaVersion: BOX_SCHEMA_VERSION, kind: kinds.channel, channel, boxId: scroll.boxId, @@ -417,7 +460,7 @@ export async function buildBox(name, options = {}) { channelPath, archiveSha256: archiveSha, installedSizeBytes, - weights: weightsMode, + deferredAssets: deferred.length, parity, }; } diff --git a/src/build/execution.d.mts b/src/build/execution.d.mts index fdf7b1f..b2953ab 100644 --- a/src/build/execution.d.mts +++ b/src/build/execution.d.mts @@ -7,15 +7,15 @@ * @param {object} options * @param {object | null | undefined} options.execution * @param {import('../contract/targets.mjs').BoxTargetAdapter} options.adapter - * @param {string} options.runtimeVersion the interpreter version a module search needs + * @param {string} options.runtimeId the runtime the box declares + * @param {string | undefined} options.runtimeVersion its version, where a module search needs one * @param {Set} options.files - * @param {string} [options.runtimeId] * @returns {void} */ -export function assertExecutionFiles({ execution, adapter, runtimeVersion, files, runtimeId, }: { +export function assertExecutionFiles({ execution, adapter, runtimeId, runtimeVersion, files, }: { execution: object | null | undefined; adapter: import("../contract/targets.mjs").BoxTargetAdapter; - runtimeVersion: string; + runtimeId: string; + runtimeVersion: string | undefined; files: Set; - runtimeId?: string; }): void; diff --git a/src/build/execution.mjs b/src/build/execution.mjs index 3424d97..a29ee74 100644 --- a/src/build/execution.mjs +++ b/src/build/execution.mjs @@ -12,7 +12,7 @@ * path every validation failure in the tool takes. */ -import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../contract/runtimes.mjs'; +import { runtimeAdapter } from '../contract/runtimes.mjs'; import { safeRelativePath } from './filesystem.mjs'; import { fail } from './process.mjs'; @@ -25,17 +25,17 @@ import { fail } from './process.mjs'; * @param {object} options * @param {object | null | undefined} options.execution * @param {import('../contract/targets.mjs').BoxTargetAdapter} options.adapter - * @param {string} options.runtimeVersion the interpreter version a module search needs + * @param {string} options.runtimeId the runtime the box declares + * @param {string | undefined} options.runtimeVersion its version, where a module search needs one * @param {Set} options.files - * @param {string} [options.runtimeId] * @returns {void} */ export function assertExecutionFiles({ execution, adapter, + runtimeId, runtimeVersion, files, - runtimeId = IMPLICIT_RUNTIME_ID, }) { if (!execution) return; const runtime = runtimeAdapter(runtimeId); diff --git a/src/build/pixi.d.mts b/src/build/pixi.d.mts index 084138f..e1c7ecf 100644 --- a/src/build/pixi.d.mts +++ b/src/build/pixi.d.mts @@ -108,7 +108,7 @@ export function findCondaPack({ path, runResult }?: { * payloadDir: string, * adapter: import('../contract/targets.mjs').BoxTargetAdapter, * run: typeof import('./process.mjs').run, - * runtimeId?: string, + * runtimeId: string, * }} options * @returns {Promise<{ interpreter: string, venvDir: string, sitePackagesRelative: string }>} */ @@ -121,7 +121,7 @@ export function installAndPackPixiEnvironment({ pixi, condaPack, manifestPath, l payloadDir: string; adapter: import("../contract/targets.mjs").BoxTargetAdapter; run: typeof import("./process.mjs").run; - runtimeId?: string; + runtimeId: string; }): Promise<{ interpreter: string; venvDir: string; diff --git a/src/build/pixi.mjs b/src/build/pixi.mjs index 0de7a6e..4e0f58f 100644 --- a/src/build/pixi.mjs +++ b/src/build/pixi.mjs @@ -18,7 +18,7 @@ import { chmod, copyFile, cp, mkdir, readFile, readdir, readlink, realpath, rm, import { dirname, join, relative, resolve, sep } from 'node:path'; import * as tar from 'tar'; import { resolvePayloadLinkTarget, targetCarriesLinks } from '../contract/links.mjs'; -import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../contract/runtimes.mjs'; +import { runtimeAdapter } from '../contract/runtimes.mjs'; import { compareStableStrings, safeRelativePath } from './filesystem.mjs'; import { fail, runResult as defaultRunResult } from './process.mjs'; import { repairPosixLaunchers } from '../runtimes/python/launchers.mjs'; @@ -338,7 +338,7 @@ async function keepsAsLink(root, linkPath, canonicalRoot) { * payloadDir: string, * adapter: import('../contract/targets.mjs').BoxTargetAdapter, * run: typeof import('./process.mjs').run, - * runtimeId?: string, + * runtimeId: string, * }} options * @returns {Promise<{ interpreter: string, venvDir: string, sitePackagesRelative: string }>} */ @@ -351,7 +351,7 @@ export async function installAndPackPixiEnvironment({ payloadDir, adapter, run, - runtimeId = IMPLICIT_RUNTIME_ID, + runtimeId, }) { const layout = runtimeAdapter(runtimeId).layout(adapter); const workspace = join(buildDir, 'pixi-workspace'); diff --git a/src/build/process.d.mts b/src/build/process.d.mts index e48fc3d..1385db5 100644 --- a/src/build/process.d.mts +++ b/src/build/process.d.mts @@ -7,6 +7,8 @@ * @property {string | Uint8Array} [input] * @property {number} [maxBuffer] * @property {boolean} [capture] + * @property {number} [expectExitCode] the status that means success, defaulting to 0. A self-test + * command may legitimately require another one, and every other caller wants the default. */ /** * Throws a consistent CLI error from validation helpers. @@ -42,4 +44,9 @@ export type RunOptions = { input?: string | Uint8Array; maxBuffer?: number; capture?: boolean; + /** + * the status that means success, defaulting to 0. A self-test + * command may legitimately require another one, and every other caller wants the default. + */ + expectExitCode?: number; }; diff --git a/src/build/process.mjs b/src/build/process.mjs index c6e3b0e..9821acd 100644 --- a/src/build/process.mjs +++ b/src/build/process.mjs @@ -17,6 +17,8 @@ import { mergeEnvironmentLayers } from '../environment.mjs'; * @property {string | Uint8Array} [input] * @property {number} [maxBuffer] * @property {boolean} [capture] + * @property {number} [expectExitCode] the status that means success, defaulting to 0. A self-test + * command may legitimately require another one, and every other caller wants the default. */ /** @@ -57,11 +59,13 @@ export function runResult(command, args, options = {}) { * @returns {string} */ export function run(command, args, options = {}) { + const expected = options.expectExitCode ?? 0; const result = runResult(command, args, options); if (result.error) fail(`${command} failed to start: ${result.error.message}`); - if (result.status !== 0) { + if (result.status !== expected) { const detail = options.capture ? `\n${result.stderr || result.stdout}` : ''; - fail(`${command} exited with status ${result.status}${detail}`); + const wanted = expected === 0 ? '' : ` (expected ${expected})`; + fail(`${command} exited with status ${result.status}${wanted}${detail}`); } return (result.stdout ?? '').trim(); } diff --git a/src/build/scroll-edit.mjs b/src/build/scroll-edit.mjs index 484c7a9..7918284 100644 --- a/src/build/scroll-edit.mjs +++ b/src/build/scroll-edit.mjs @@ -175,11 +175,11 @@ function withoutSelfTestFile(selfTest, relativePath) { /** * The payload path an asset URL lands at when the caller does not name one. * - * The last segment of the URL path, under the box's model cache. A URL that ends in a slash, or + * The last segment of the URL path, under the box's cache directory. A URL that ends in a slash, or * whose last segment is not a filename, gets no default: guessing a name for a file whose hash is * about to be pinned would be the wrong kind of helpful. */ -function defaultAssetPath(url, modelCacheSubdir) { +function defaultAssetPath(url, cacheSubdir) { let name; try { name = decodeURIComponent(new URL(url).pathname.split('/').filter(Boolean).at(-1) ?? ''); @@ -189,7 +189,7 @@ function defaultAssetPath(url, modelCacheSubdir) { if (!name || name === '.' || name === '..') { fail(`Cannot tell what to call ${url} in the box; pass --to .`); } - return `${modelCacheSubdir}/${name}`; + return `${cacheSubdir}/${name}`; } /** @@ -222,22 +222,40 @@ async function measureAsset(url, { fetchImpl = fetch, log = () => {} } = {}) { /** * `add asset` — records a remote file in the scroll, with the size and hash it actually has. * - * @param {{ boxId: string, target: string, url: string, to?: string | null, - * fetchImpl?: typeof fetch, log?: (message: string) => void }} options + * @param {{ boxId: string, target: string, url: string, to?: string | null, embed?: boolean, + * executable?: boolean, fetchImpl?: typeof fetch, log?: (message: string) => void }} options */ -export async function addAsset({ boxId, target, url, to = null, fetchImpl = fetch, log = () => {} }) { +export async function addAsset({ + boxId, + target, + url, + to = null, + embed = true, + executable = false, + fetchImpl = fetch, + log = () => {}, +}) { let relativePath; if (to) { relativePath = safeRelativePath(to); } else { - const modelCacheSubdir = await agreedValue(boxId, 'modelCacheSubdir'); - if (!modelCacheSubdir) { - fail(`${boxId}'s targets use different model cache directories; pass --to .`); + const cacheSubdir = await agreedValue(boxId, 'cacheSubdir'); + if (!cacheSubdir) { + fail(`${boxId}'s targets use different cache directories; pass --to .`); } - relativePath = safeRelativePath(defaultAssetPath(url, modelCacheSubdir)); + relativePath = safeRelativePath(defaultAssetPath(url, cacheSubdir)); } const { sizeBytes, sha256 } = await measureAsset(url, { fetchImpl, log }); - const entry = { url, relativePath, sizeBytes, sha256 }; + // Both flags are written only when they differ from the schema's default, so an ordinary asset + // still reads as four lines rather than six. + const entry = { + url, + relativePath, + sizeBytes, + sha256, + ...(embed ? {} : { embed: false }), + ...(executable ? { executable: true } : {}), + }; const { written } = await updateScrollFiles(boxId, target, (scroll) => ({ ...scroll, assets: [...(scroll.assets ?? []), entry], @@ -253,9 +271,10 @@ export async function addAsset({ boxId, target, url, to = null, fetchImpl = fetc * added is usually one being worked on, and a pin there fails the next build over an edit the * author meant to make. * - * @param {{ boxId: string, target: string, sourcePath: string, to?: string | null }} options + * @param {{ boxId: string, target: string, sourcePath: string, to?: string | null, + * executable?: boolean }} options */ -export async function addFile({ boxId, target, sourcePath, to = null }) { +export async function addFile({ boxId, target, sourcePath, to = null, executable = false }) { const source = safeRelativePath(sourcePath); const absolute = join(getWorkspace().root, ...source.split('/')); let details; @@ -268,7 +287,9 @@ export async function addFile({ boxId, target, sourcePath, to = null }) { fail(`A box file must be a regular file: ${source}`); } const relativePath = safeRelativePath(to ?? basename(source)); - const entry = { sourcePath: source, relativePath }; + // The source file's own mode is deliberately not read: it varies with the umask of whoever + // checked the project out, and a build that copied it would not rebuild byte-identically. + const entry = { sourcePath: source, relativePath, ...(executable ? { executable: true } : {}) }; const { written } = await updateScrollFiles(boxId, target, (scroll) => ({ ...scroll, localFiles: [...(scroll.localFiles ?? []), entry], @@ -405,13 +426,14 @@ export async function removeSelfTestImport({ boxId, target, module }) { * * Three kinds. Structural values a project does not choose (`$schema`, `schemaVersion`, `extends`). * Values the layout or the target fixes, where a text prompt would only let someone contradict a - * check they cannot win — `boxId` and `target` name the directories, and `pythonEntryPoint` has one - * legal value per target. And the collections, which have their own commands or their own file: - * editing a list through a single value prompt is how a list gets destroyed. + * check they cannot win — `boxId` and `target` name the directories, and `runtime` holds an id that + * decides the whole payload layout and an entry point with one legal value per target. And the + * collections, which have their own commands or their own file: editing a list through a single + * value prompt is how a list gets destroyed. */ const UNEDITABLE_FIELDS = Object.freeze(new Set([ - '$schema', 'schemaVersion', 'extends', 'boxId', 'target', 'pythonEntryPoint', - 'compatibility', 'environment', 'assets', 'assetArchives', 'localFiles', + '$schema', 'schemaVersion', 'extends', 'boxId', 'target', 'runtime', + 'labels', 'compatibility', 'environment', 'assets', 'assetArchives', 'localFiles', 'prunePaths', 'uncompressedPaths', 'selfTest', 'execution', 'parity', ])); diff --git a/src/build/scroll.mjs b/src/build/scroll.mjs index 55ab5fa..166e864 100644 --- a/src/build/scroll.mjs +++ b/src/build/scroll.mjs @@ -3,8 +3,8 @@ * * A scroll is the only input a build accepts, so it is validated before anything is installed. In * the nested layout, the meaningful declarations police the path: `boxId` names the parent and the - * canonical target names the child. Python layout is checked against the target before the scroll - * reaches any tool discovery or build mutation. + * canonical target names the child. The declared runtime's layout is checked against the target + * before the scroll reaches any tool discovery or build mutation. * * Reading is also where a scroll becomes complete. A split scroll's two halves are joined, fields * the target or the identity already determine are derived, and the result — the *effective* scroll @@ -16,9 +16,10 @@ import { readFile, readdir } from 'node:fs/promises'; import { join, resolve, sep } from 'node:path'; import { boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; import { - IMPLICIT_RUNTIME_ID, assertRuntimeEntryPoint, + isImplementedRuntime, runtimeAdapter, + unimplementedRuntimeMessage, } from '../contract/runtimes.mjs'; import { compareStableStrings, fileExists, safeRelativePath } from './filesystem.mjs'; import { fail, runResult } from './process.mjs'; @@ -101,22 +102,31 @@ function assertDistinctPayloadDestinations(scroll) { /** * Joins the base and fragment self-tests. * - * `imports` and `files` accumulate, because a target that needs one more module still needs the - * shared ones. The extra Python is one logical slot with two spellings, so a fragment naming either - * `pythonCode` or `pythonFile` replaces both: inheriting a base's file while the fragment declares - * inline code would produce a scroll the schema refuses, and silently running both would run a check - * the author did not ask for. + * `imports`, `files` and `commands` accumulate, because a target that needs one more module — or one + * more invocation — still needs the shared ones. `imports` and `files` drop repeats, since asking + * for the same module twice is the same instruction twice; `commands` does not, because two + * invocations differing only in `expectExitCode` are two different checks and comparing whole + * objects for identity would be a rule with a surprising edge rather than a simplification. + * + * The extra source is one logical slot with two spellings, so a fragment naming either `code` or + * `script` replaces both: inheriting a base's file while the fragment declares inline source would + * produce a scroll the schema refuses, and silently running both would run a check the author did + * not ask for. */ function joinSelfTests(base = {}, fragment = {}) { const joined = { ...base, ...fragment }; - joined.imports = [...new Set([...(base.imports ?? []), ...(fragment.imports ?? [])])]; + const imports = [...new Set([...(base.imports ?? []), ...(fragment.imports ?? [])])]; + if (base.imports || fragment.imports) joined.imports = imports; const files = [...new Set([...(base.files ?? []), ...(fragment.files ?? [])])]; if (base.files || fragment.files) joined.files = files; - if (fragment.pythonCode !== undefined || fragment.pythonFile !== undefined) { - delete joined.pythonCode; - delete joined.pythonFile; - if (fragment.pythonCode !== undefined) joined.pythonCode = fragment.pythonCode; - if (fragment.pythonFile !== undefined) joined.pythonFile = fragment.pythonFile; + if (base.commands || fragment.commands) { + joined.commands = [...(base.commands ?? []), ...(fragment.commands ?? [])]; + } + if (fragment.code !== undefined || fragment.script !== undefined) { + delete joined.code; + delete joined.script; + if (fragment.code !== undefined) joined.code = fragment.code; + if (fragment.script !== undefined) joined.script = fragment.script; } return joined; } @@ -131,9 +141,10 @@ function joinSelfTests(base = {}, fragment = {}) { * * | Shape | Rule | * | --- | --- | - * | Scalars, and the cohesive objects `target`, `execution`, `parity` | The fragment replaces the base | + * | Scalars, and the cohesive objects `target`, `runtime`, `execution`, `parity` | The fragment replaces the base | * | `assets`, `assetArchives`, `localFiles` | Joined base-first; a repeated `relativePath` is an error | * | `prunePaths`, `uncompressedPaths`, `selfTest.imports`, `selfTest.files` | Joined base-first, repeats dropped | + * | `selfTest.commands` | Joined base-first, repeats kept | * | `compatibility`, `environment` | Joined key by key, the fragment winning a shared key | * | `extends` | Dropped: the joined scroll extends nothing | * @@ -211,11 +222,14 @@ export function scrollDirectory(reference) { * Fills in everything a scroll does not have to say twice. * * A hand-written scroll should carry decisions, not restatements: the interpreter path is the only - * one the target admits, the model cache directory follows the box identity, and an empty list means + * one the runtime's layout admits, the cache directory follows the box identity, and an empty list means * the same thing whether or not it was typed. Deriving here rather than at each use keeps one * effective scroll — the object the rest of the build, and the provenance record, actually see. */ function effectiveScroll(scroll, adapter, targetId) { + // Only a runtime this build implements gets this far, so its layout is the one authority on where + // the entry point sits — derived when the scroll stays quiet, checked against when it does not. + const layout = runtimeAdapter(scroll.runtime.id).layout(adapter); return { ...scroll, // Provenance needs a stable source identity. It is derived when the scroll does not name one, @@ -223,9 +237,8 @@ function effectiveScroll(scroll, adapter, targetId) { scrollId: scroll.scrollId ?? `${scroll.boxId}-${targetId}`, scrollVersion: scroll.scrollVersion ?? '1.0.0', compatibility: scroll.compatibility ?? {}, - pythonEntryPoint: scroll.pythonEntryPoint - ?? runtimeAdapter(IMPLICIT_RUNTIME_ID).layout(adapter).entryPoint, - modelCacheSubdir: scroll.modelCacheSubdir ?? `model-cache/${scroll.boxId}`, + runtime: { ...scroll.runtime, entryPoint: scroll.runtime.entryPoint ?? layout.entryPoint }, + cacheSubdir: scroll.cacheSubdir ?? `cache/${scroll.boxId}`, assets: scroll.assets ?? [], selfTest: { ...scroll.selfTest, files: scroll.selfTest.files ?? [] }, }; @@ -249,8 +262,18 @@ async function readExactScroll(reference) { if (validationError) { fail(`Invalid scroll ${normalized}${extended ? ' joined with its base' : ''}: ${validationError}.`); } - if (declared.weights === 'on-demand' && (declared.assetArchives ?? []).length > 0) { - fail('on-demand weights cannot be combined with assetArchives, which are expanded at build time.'); + // The wire vocabulary is wider than what this build can run, deliberately, so the schema admits a + // runtime with no adapter here and this is where that becomes a clear refusal. + if (!isImplementedRuntime(declared.runtime.id)) fail(unimplementedRuntimeMessage(declared.runtime.id)); + const runtime = runtimeAdapter(declared.runtime.id); + if (declared.execution && !runtime.executionKinds.includes(declared.execution.kind)) { + fail(`Execution kind ${declared.execution.kind} does not belong to the ${runtime.id} runtime; ` + + `it defines ${runtime.executionKinds.join(', ')}.`); + } + // A command probe appends arguments to the box's declared execution. With none declared there is + // nothing to append them to, so the two declarations contradict each other. + if ((declared.selfTest.commands ?? []).length > 0 && !declared.execution) { + fail('selfTest.commands invokes the box\'s execution, which this scroll does not declare.'); } // Checked here rather than in the schema: a base legitimately has no target, and requiring one // there would make every base file light up in an editor. @@ -259,15 +282,16 @@ async function readExactScroll(reference) { const targetId = boxTargetId(declared.target); const scroll = effectiveScroll(declared, adapter, targetId); const payloadPaths = [ - scroll.modelCacheSubdir, + scroll.cacheSubdir, ...scroll.assets.map((asset) => asset.relativePath), ...(scroll.assetArchives ?? []).flatMap((archive) => [archive.relativePath, archive.destination]), ...(scroll.localFiles ?? []).flatMap((file) => [file.sourcePath, file.relativePath]), ...(scroll.prunePaths ?? []), ...(scroll.uncompressedPaths ?? []), ...scroll.selfTest.files, - ...(scroll.selfTest.pythonFile ? [scroll.selfTest.pythonFile] : []), - ...(scroll.execution?.kind === 'python-script' ? [scroll.execution.script] : []), + ...(scroll.selfTest.script ? [scroll.selfTest.script] : []), + ...(scroll.execution?.script ? [scroll.execution.script] : []), + ...(scroll.execution?.binary ? [scroll.execution.binary] : []), ...(scroll.parity ? [scroll.parity.script] : []), ...(scroll.condaDependencyLicenseAudit ? [scroll.condaDependencyLicenseAudit] : []), ]; @@ -280,7 +304,7 @@ async function readExactScroll(reference) { if (targetDirectory !== targetId) { fail(`Nested scroll target directory ${targetDirectory} does not match declared target ${targetId}.`); } - assertRuntimeEntryPoint(IMPLICIT_RUNTIME_ID, adapter, scroll.pythonEntryPoint); + assertRuntimeEntryPoint(scroll.runtime.id, adapter, scroll.runtime.entryPoint); return { adapter, dir, scroll, reference: normalized, targetId }; } diff --git a/src/build/verify.d.mts b/src/build/verify.d.mts index b25db30..0feb947 100644 --- a/src/build/verify.d.mts +++ b/src/build/verify.d.mts @@ -1,9 +1,13 @@ /** * Binds the self-description inside the archive to the signed release outside it. * - * Only fields present in both schema-version-2 documents belong here. Release-only transport data - * has no counterpart in box.json; every shared identity, target, layout, consumer self-test, - * asset-policy, and provenance field must agree recursively. + * Only fields present in both schema-version-3 documents belong here. Release-only transport data + * has no counterpart in box.json; every shared identity, target, runtime, layout, consumer + * self-test, deferred-asset, and provenance field must agree recursively. + * + * `assets` carries the per-entry `embed` decision by construction: it lists exactly the deferred + * entries, and it is compared deeply, so a box that quietly changed its mind about one asset + * disagrees with its release. */ export function assertBoxManifestAgreement(box: any, release: any): void; /** diff --git a/src/build/verify.mjs b/src/build/verify.mjs index f520310..19c89f3 100644 --- a/src/build/verify.mjs +++ b/src/build/verify.mjs @@ -15,10 +15,11 @@ import { dirname, join, resolve } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { assertNativeHost, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; import { - IMPLICIT_RUNTIME_ID, assertRuntimeEntryPoint, executionAffectingVariables, + isImplementedRuntime, runtimeAdapter, + unimplementedRuntimeMessage, } from '../contract/runtimes.mjs'; import { BOX_SCHEMA_VERSION, @@ -30,16 +31,14 @@ import { resolveTrustedKeys, verifySignedDocument } from '../sign/index.mjs'; const AGREEMENT_FIELDS = [ 'schemaVersion', 'boxId', - 'modelId', - 'runtimeId', + 'labels', 'version', 'target', - 'pythonEntryPoint', - 'modelCacheSubdir', + 'runtime', + 'cacheSubdir', 'environment', 'selfTest', 'execution', - 'weights', 'assets', 'provenance', ]; @@ -47,9 +46,13 @@ const AGREEMENT_FIELDS = [ /** * Binds the self-description inside the archive to the signed release outside it. * - * Only fields present in both schema-version-2 documents belong here. Release-only transport data - * has no counterpart in box.json; every shared identity, target, layout, consumer self-test, - * asset-policy, and provenance field must agree recursively. + * Only fields present in both schema-version-3 documents belong here. Release-only transport data + * has no counterpart in box.json; every shared identity, target, runtime, layout, consumer + * self-test, deferred-asset, and provenance field must agree recursively. + * + * `assets` carries the per-entry `embed` decision by construction: it lists exactly the deferred + * entries, and it is compared deeply, so a box that quietly changed its mind about one asset + * disagrees with its release. */ export function assertBoxManifestAgreement(box, release) { for (const field of AGREEMENT_FIELDS) { @@ -111,7 +114,12 @@ export async function inspectReleaseDocument(releaseDocumentPath, { publicPath, if (releaseError) fail(`Invalid release manifest: ${releaseError}.`); if (parseDocumentKind(release.kind)?.type !== 'release') fail('Document is not a box release.'); const adapter = boxTargetAdapter(release.target); - assertRuntimeEntryPoint(IMPLICIT_RUNTIME_ID, adapter, release.pythonEntryPoint); + // The format's runtime vocabulary is wider than what this build implements, so a release may name + // one there is no adapter for. That is refused by name rather than misread as another runtime. + if (!isImplementedRuntime(release.runtime.id)) fail(unimplementedRuntimeMessage(release.runtime.id)); + if (release.runtime.entryPoint !== undefined) { + assertRuntimeEntryPoint(release.runtime.id, adapter, release.runtime.entryPoint); + } // Describes the extracted tree rather than the archive, so it belongs on this side of the split. // `payloadDigest` needs no companion check: its `format` is a schema `const`, so a release naming // a format this build cannot read is already refused above, by name, as an invalid manifest. @@ -167,11 +175,15 @@ export async function inspectBoxArchive(releaseDocumentPath, options = {}) { ); if (boxError) fail(`Invalid box.json: ${boxError}.`); assertBoxManifestAgreement(box, release); - if (!resolvablePaths.has(release.pythonEntryPoint)) fail(`Archive is missing ${release.pythonEntryPoint}.`); + if (release.runtime.entryPoint !== undefined + && !resolvablePaths.has(release.runtime.entryPoint)) { + fail(`Archive is missing ${release.runtime.entryPoint}.`); + } assertExecutionFiles({ execution: release.execution, adapter, - runtimeVersion: release.provenance.pythonVersion, + runtimeId: release.runtime.id, + runtimeVersion: release.provenance.runtimeVersion, files: resolvablePaths, }); @@ -220,7 +232,7 @@ export async function verifyBox(releaseDocumentPath, options = {}) { const resolvedEnvironment = resolveEnvironment({ platform: adapter.platform, layers: environmentLayers, - executionAffectingVariables: executionAffectingVariables(IMPLICIT_RUNTIME_ID, adapter), + executionAffectingVariables: executionAffectingVariables(release.runtime.id, adapter), expanded: Boolean(options.envReport || options.envReportValues), revealHostValues: Boolean(options.envReportValues), }); @@ -245,17 +257,24 @@ export async function verifyBox(releaseDocumentPath, options = {}) { && (await payloadDigest(extracted)).sha256 !== release.payloadDigest.sha256) { fail('Extracted payload does not match the signed release.'); } - const python = join(extracted, safeRelativePath(release.pythonEntryPoint)); - // The signed subset only: a consumer repeating this check has the imports and nothing else, - // and the runtime is the only thing that knows how to turn them into a command line. - const argv = runtimeAdapter(IMPLICIT_RUNTIME_ID).selfTestArgv({ - probe: { imports: release.selfTest.pythonImports }, + // The signed probe only: a consumer repeating this check has the imports and commands and + // nothing else, and the runtime is the only thing that knows how to turn them into command + // lines. Running the builder-only extras here would report a pass the consumer cannot get. + const invocations = runtimeAdapter(release.runtime.id).selfTestInvocations({ + probe: release.selfTest.probe, + execution: release.execution, target: adapter, }); - run(python, argv, { - cwd: extracted, - env: resolvedEnvironment.environment, - }); + const resolveArgument = (argument) => (argument.kind === 'payload-path' + ? join(extracted, ...safeRelativePath(argument.value).split('/')) + : argument.value); + for (const invocation of invocations) { + run(resolveArgument(invocation.command), invocation.args.map(resolveArgument), { + cwd: extracted, + env: resolvedEnvironment.environment, + expectExitCode: invocation.expectExitCode, + }); + } } finally { await rm(extracted, { recursive: true, force: true }); } diff --git a/src/cli-authoring.mjs b/src/cli-authoring.mjs index c27c9e4..d92fa5e 100644 --- a/src/cli-authoring.mjs +++ b/src/cli-authoring.mjs @@ -15,9 +15,7 @@ import { createInterface } from 'node:readline/promises'; import { DEFAULT_PYTHON_VERSION, - DEFAULT_WEIGHTS_MODE, EXAMPLE_PIXI_VERSION, - WEIGHTS_MODES, resolvePythonVersion, } from './build/authoring.mjs'; import { probePixi } from './build/pixi.mjs'; @@ -100,6 +98,26 @@ export async function promptText(question, { } } +/** + * `--labels` as the JSON object it is, matching `--default-args` rather than inventing a second + * spelling for structured input. Scrollcase never reads a label, so nothing here interprets one + * beyond checking that it is a string keyed by a string. + */ +function parseLabels(value) { + if (value === null) return {}; + let parsed; + try { + parsed = JSON.parse(value); + } catch { + fail('--labels must be a JSON object of string values.'); + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed) + || Object.values(parsed).some((item) => typeof item !== 'string')) { + fail('--labels must be a JSON object of string values.'); + } + return parsed; +} + function parseDefaultArgs(value) { if (value === null) return []; let parsed; @@ -163,8 +181,6 @@ export async function collectNewScrollOptions(flags, { // The upstream revision is the one identity nothing here can supply: it names the version of the // thing being packaged, and inventing it would put a false claim into the box's provenance. const sourceRevision = await required('source-revision', 'Upstream revision', HINTS.sourceRevision); - const modelId = derived('model-id', boxId); - const runtimeId = derived('runtime-id', `${boxId}-runtime`); const version = derived('version', '1.0.0'); const scrollVersion = derived('scroll-version', undefined); const pythonVersion = resolvePythonVersion(derived('python-version', DEFAULT_PYTHON_VERSION)); @@ -188,13 +204,9 @@ export async function collectNewScrollOptions(flags, { // Not derivable and not optional: the signed release names the URL the archive itself is published // under, so a build has nowhere to point without it. const assetBaseUrl = await required('asset-base-url', 'Asset base URL', HINTS.assetBaseUrl); - // Not a question. A box declares assets or it does not, and one that does not — which is most of - // them, since a scroll packages Python, not necessarily a model — has nothing to leave out of its - // archive. `--weights on-demand` states the choice for a box whose assets are published beside it. - const weights = derived('weights', DEFAULT_WEIGHTS_MODE); - if (!WEIGHTS_MODES.includes(weights)) { - fail(`Unsupported weights mode: ${weights}. Use ${WEIGHTS_MODES.join(' or ')}.`); - } + // Free-form annotations, flags only and empty by default. Scrollcase reads none of them, so + // prompting for one would be asking the author to fill in a field on the tool's behalf. + const labels = parseLabels(flagText(flags, 'labels')); const executionKind = await finite( 'execution', 'execution kind', @@ -206,8 +218,7 @@ export async function collectNewScrollOptions(flags, { const result = { boxId, target, - modelId, - runtimeId, + labels, version, scrollVersion, sourceRevision, @@ -215,7 +226,6 @@ export async function collectNewScrollOptions(flags, { pixiVersion, compatibility, assetBaseUrl, - weights, executionKind, defaultArgs, }; diff --git a/src/cli.mjs b/src/cli.mjs index 950f047..b98259b 100755 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -187,8 +187,7 @@ async function init(flags) { 'accelerator', 'cuda-version', 'box-id', - 'model-id', - 'runtime-id', + 'labels', ].filter((name) => flags.has(name)); if (authoringFlags.length > 0) { fail(`init accepts only the fixed example; pass ${authoringFlags.map((name) => `--${name}`).join(', ')} to scrollcase new scroll.`); @@ -374,12 +373,21 @@ async function add(kind, positional, flags) { const [name, value] = positional; if (kind === 'dep') return addDep(name, value, flags); if (kind === 'env' || kind === 'import') return addDeclaration(kind, name, value, flags); - if (!value) fail(`Usage: scrollcase add ${kind} <${kind === 'asset' ? 'url' : 'path'}> [--to ] [--target |all]`); + if (!value) fail(`Usage: scrollcase add ${kind} <${kind === 'asset' ? 'url' : 'path'}> [--to ] [--on-demand] [--executable] [--target |all]`); const { boxId, target } = await editScope(name, flags); const to = text(flags, 'to'); + const executable = Boolean(flags.get('executable')); const result = kind === 'asset' - ? await addAsset({ boxId, target, url: value, to, log: (message) => step(message) }) - : await addFile({ boxId, target, sourcePath: value, to }); + ? await addAsset({ + boxId, + target, + url: value, + to, + embed: !flags.get('on-demand'), + executable, + log: (message) => step(message), + }) + : await addFile({ boxId, target, sourcePath: value, to, executable }); success(`Added ${result.entry.relativePath} to ${boxId}${target === ALL_TARGETS ? '' : `/${target}`}`); if (kind === 'asset') info(`${result.entry.sizeBytes} bytes, sha256 ${result.entry.sha256}`); reportWritten(result.written); @@ -559,16 +567,15 @@ async function build(name, flags) { ['beta', ...CHANNELS.filter((value) => value !== 'beta')], { flag: text(flags, 'channel') }, ); - // The weights mode is not asked. The scroll declares it, and a menu preselected on `embed` in - // front of every build was an override waiting to happen: pressing Enter silently repacked a box - // whose scroll said `on-demand`. `--weights` still overrides deliberately. - const weights = text(flags, 'weights'); - step(`Building ${reference} (${channel}${weights ? `, ${weights}` : ''})`); + // Whether an asset ships inside the archive is a per-entry scroll declaration with no build-time + // override. There was one, and a menu preselected on `embed` in front of every build turned out to + // be an override waiting to happen: pressing Enter silently repacked a box whose scroll had said + // otherwise, under an identity that no longer described it. + step(`Building ${reference} (${channel})`); const built = await buildBox(reference, { ...signing, allowDirty: Boolean(flags.get('allow-dirty')), channel, - weights, assetBaseUrl: text(flags, 'asset-base-url'), namespace: text(flags, 'namespace') || undefined, pixiPath: text(flags, 'pixi'), @@ -685,15 +692,13 @@ New scroll options: --box-id Box identity --source-revision Upstream source revision recorded in provenance --asset-base-url Base URL used in built release documents - --model-id Identity of what the box packages (default: the box id) - --runtime-id Runtime identity (default: -runtime) + --labels JSON object of free-form annotations carried into the signed + release. Scrollcase reads none of them. --version Box version (default 1.0.0) --scroll-version Scroll authoring version (default 1.0.0) --python-version Python dependency version, or latest --pixi-version pixi resolver version (default: the installed pixi) --min-host-app-version Minimum compatible host application version - --weights embed (default) or on-demand; only matters once the box declares - assets, so it is a flag rather than a question --execution python-script, python-module, or library-only --script Existing project script for python-script --generate-script Generate a minimal project script instead @@ -713,7 +718,12 @@ Add, remove and edit options: is no base. Without it, a box with one target uses that one and a box with several asks; without a terminal it stops instead. --to Where the file lands inside the box. Defaults to the URL's last - segment under the model cache, or the file's own name at the root. + segment under the box's cache directory, or the file's own name at + the root. + --on-demand For add asset: leave this file out of the archive and carry its + descriptor in the signed release for the caller to materialize. + --executable Mark the added file as one that needs the executable bit. A + download and a copy both arrive without one. --version Version constraint for add dep (default *, letting the lock pin it) --from-requirements Read dependencies from a pip requirements.txt instead --field Field for edit scroll; without it, a menu built from the schema @@ -743,9 +753,6 @@ Build options: --target Select a target when names a box --channel Channel the signed pointer names (nightly, beta, or stable; default beta) - --weights embed (assets packed in, works air-gapped) or on-demand - (caller-materialized; verified before execution). Overrides the - scroll for this build; without it the scroll's own mode is used. --asset-base-url Override the scroll's published base URL --namespace Document kind namespace (default scrollcase.box) --allow-dirty Permit a build from an uncommitted source tree diff --git a/src/consumer/run-extracted.mjs b/src/consumer/run-extracted.mjs index 5f5b226..f998f3d 100644 --- a/src/consumer/run-extracted.mjs +++ b/src/consumer/run-extracted.mjs @@ -14,7 +14,6 @@ import { assertExecutionFiles } from '../build/execution.mjs'; import { fail } from '../build/process.mjs'; import { assertNativeHost, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; import { - IMPLICIT_RUNTIME_ID, executionAffectingVariables, runtimeAdapter, } from '../contract/runtimes.mjs'; @@ -118,27 +117,30 @@ export async function runExtractedBox(prepared, options = {}) { fail('Prepared box root no longer matches the prepared box.'); } const files = new Set(await collectFiles(prepared.root)); - if (!files.has(release.pythonEntryPoint)) { - fail(`Prepared box is missing ${release.pythonEntryPoint}.`); + if (release.runtime.entryPoint !== undefined && !files.has(release.runtime.entryPoint)) { + fail(`Prepared box is missing ${release.runtime.entryPoint}.`); } assertExecutionFiles({ execution: release.execution, adapter, - runtimeVersion: release.provenance.pythonVersion, + runtimeId: release.runtime.id, + runtimeVersion: release.provenance.runtimeVersion, files, }); await verifyRequiredAssets(prepared.root, prepared.requiredAssets); - const python = join(prepared.root, ...safeRelativePath(release.pythonEntryPoint).split('/')); // The runtime states the command line in payload-relative terms and this end joins it: a box root - // is a real path on this host, and the format has no business deciding what one looks like. - const { args } = runtimeAdapter(IMPLICIT_RUNTIME_ID).buildArgv({ + // is a real path on this host, and the format has no business deciding what one looks like. Which + // runtime states it is the box's declaration, not an assumption about what a box contains. + const { command, args } = runtimeAdapter(release.runtime.id).buildArgv({ execution: release.execution, target: adapter, }); - const executionArgs = args.map((argument) => (argument.kind === 'payload-path' + const resolveArgument = (argument) => (argument.kind === 'payload-path' ? join(prepared.root, ...safeRelativePath(argument.value).split('/')) - : argument.value)); + : argument.value); + const executionCommand = resolveArgument(command); + const executionArgs = args.map(resolveArgument); executionArgs.push(...callerArgs); const { environment, report: environmentReport } = resolveEnvironment({ @@ -148,14 +150,14 @@ export async function runExtractedBox(prepared, options = {}) { { source: 'caller', values: options.env }, { source: 'release', values: release.environment }, ], - executionAffectingVariables: executionAffectingVariables(IMPLICIT_RUNTIME_ID, adapter), + executionAffectingVariables: executionAffectingVariables(release.runtime.id, adapter), expanded: Boolean(options.envReport || options.envReportValues), revealHostValues: Boolean(options.envReportValues), }); await options.onEnvironmentReport?.(environmentReport); const spawn = options.spawn ?? spawnProcess; - const child = spawn(python, executionArgs, { + const child = spawn(executionCommand, executionArgs, { cwd: prepared.root, env: environment, stdio: [ diff --git a/src/consumer/verify-and-extract.d.mts b/src/consumer/verify-and-extract.d.mts index b20c741..f2e56b7 100644 --- a/src/consumer/verify-and-extract.d.mts +++ b/src/consumer/verify-and-extract.d.mts @@ -140,12 +140,19 @@ export type PreparedBox = { */ root: string; boxId: string; - modelId: string; - runtimeId: string; + /** + * free-form annotations the publisher signed; + * empty when the box declared none + */ + labels: Readonly>; version: string; target: import("../contract/types/index.d.ts").BoxTarget; targetId: string; - pythonEntryPoint: string; + runtime: Readonly<{ + id: string; + version?: string; + entryPoint?: string; + }>; execution: import("../contract/types/index.d.ts").BoxExecution | null; /** * assets the caller must materialize, never diff --git a/src/consumer/verify-and-extract.mjs b/src/consumer/verify-and-extract.mjs index 121f88f..ee0ab94 100644 --- a/src/consumer/verify-and-extract.mjs +++ b/src/consumer/verify-and-extract.mjs @@ -29,7 +29,7 @@ import { parsePayloadDigestStream, } from '../contract/payload-digest.mjs'; import { assertNativeHost, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; -import { IMPLICIT_RUNTIME_ID, executionAffectingVariables } from '../contract/runtimes.mjs'; +import { executionAffectingVariables } from '../contract/runtimes.mjs'; import { resolveEnvironment } from '../environment.mjs'; /** @@ -54,12 +54,12 @@ import { resolveEnvironment } from '../environment.mjs'; * @property {'prepared' | 'attached'} status * @property {string} root absolute extracted box root * @property {string} boxId - * @property {string} modelId - * @property {string} runtimeId + * @property {Readonly>} labels free-form annotations the publisher signed; + * empty when the box declared none * @property {string} version * @property {import('../contract/types/index.d.ts').BoxTarget} target * @property {string} targetId - * @property {string} pythonEntryPoint + * @property {Readonly<{ id: string, version?: string, entryPoint?: string }>} runtime * @property {import('../contract/types/index.d.ts').BoxExecution | null} execution * @property {readonly RequiredAsset[]} requiredAssets assets the caller must materialize, never * downloaded by Scrollcase @@ -137,9 +137,14 @@ export async function verifyRequiredAssets(root, assets) { } } -/** The on-demand descriptors a release requires a caller to have materialised, screened for safety. */ +/** + * The deferred descriptors a release requires a caller to have materialised, screened for safety. + * + * The list is exactly the assets the scroll declared `embed: false`; a release whose assets are all + * embedded carries none, and the box needs nothing fetched before it runs. + */ function requiredAssetsOf(release) { - const assets = release.weights === 'on-demand' ? release.assets : []; + const assets = release.assets ?? []; for (const asset of assets) safeRelativePath(asset.relativePath); return assets; } @@ -153,7 +158,7 @@ function releaseEnvironmentReport(release, options = {}) { { source: 'host', values: process.env }, { source: 'release', values: release.environment }, ], - executionAffectingVariables: executionAffectingVariables(IMPLICIT_RUNTIME_ID, adapter), + executionAffectingVariables: executionAffectingVariables(release.runtime.id, adapter), expanded: Boolean(options.envReport || options.envReportValues), revealHostValues: Boolean(options.envReportValues), }).report; @@ -173,12 +178,11 @@ function mintReceipt(status, { status, root, boxId: release.boxId, - modelId: release.modelId, - runtimeId: release.runtimeId, + labels: release.labels ?? {}, version: release.version, target: release.target, targetId: boxTargetId(release.target), - pythonEntryPoint: release.pythonEntryPoint, + runtime: release.runtime, execution: release.execution ?? null, requiredAssets: requiredAssetsOf(release), signingKeyIds: signed.signatures.map((signature) => signature.keyId), @@ -317,13 +321,14 @@ export async function attachExtractedBox(releaseDocumentPath, { } const files = new Set(await collectFiles(boxRoot)); - if (!files.has(release.pythonEntryPoint)) { - fail(`Attached box is missing ${release.pythonEntryPoint}.`); + if (release.runtime.entryPoint !== undefined && !files.has(release.runtime.entryPoint)) { + fail(`Attached box is missing ${release.runtime.entryPoint}.`); } assertExecutionFiles({ execution: release.execution, adapter, - runtimeVersion: release.provenance.pythonVersion, + runtimeId: release.runtime.id, + runtimeVersion: release.provenance.runtimeVersion, files, }); await verifyRequiredAssets(boxRoot, requiredAssetsOf(release)); diff --git a/src/contract/browser.d.mts b/src/contract/browser.d.mts index 69668c3..d47c09b 100644 --- a/src/contract/browser.d.mts +++ b/src/contract/browser.d.mts @@ -1,2 +1,2 @@ -export { assertNativeHost, assertPythonEntryPoint, condaSubdir, pixiAccelerator, boxTargetAdapter, boxTargetAdapters, boxTargetId } from "./targets.mjs"; +export { assertNativeHost, condaSubdir, pixiAccelerator, boxTargetAdapter, boxTargetAdapters, boxTargetId } from "./targets.mjs"; export { CHANNELS, DEFAULT_DOCUMENT_NAMESPACE, PAYLOAD_ENCODING, BOX_SCHEMA_VERSION, SIGNATURE_ALGORITHM, documentKinds, isSignedBoxDocument, parseDocumentKind } from "./document-shape.mjs"; diff --git a/src/contract/document-shape.d.mts b/src/contract/document-shape.d.mts index 8119583..de1a8a2 100644 --- a/src/contract/document-shape.d.mts +++ b/src/contract/document-shape.d.mts @@ -28,11 +28,15 @@ export function parseDocumentKind(kind: unknown): { * The message any reader gives a document written to a format version it cannot read. * * Four Node call sites answer this question — the payload decoder, the key loader, and the release - * verifier twice — and until now each carried its own copy of the sentence. That is not a style - * problem: the next version bump has to change what a v1 document is told, and a message duplicated - * per call site is a message that gets changed in three of four places. There is one wording per - * language now, and each language keeps its own because the string is user-facing text, not wire - * data that has to match across implementations. + * verifier twice — and each used to carry its own copy of the sentence. That is not a style problem: + * the v3 bump had to change what an older document is told, and a message duplicated per call site + * is a message that gets changed in three of four places. There is one wording per language now, and + * each language keeps its own because the string is user-facing text, not wire data that has to + * match across implementations. + * + * Both superseded versions are named rather than lumped together as "too old": a v1 and a v2 box + * are different artefacts with different rebuilds ahead of them, and the reader holding one is + * entitled to know which. * * @param {unknown} version the `schemaVersion` the document declared * @returns {string} @@ -58,7 +62,7 @@ export function isSignedBoxDocument(value: unknown): value is import("./types/in * cryptographic decoder, while the main contract entry point continues to expose the complete API. */ /** Format version carried by every document this contract describes. */ -export const BOX_SCHEMA_VERSION: 2; +export const BOX_SCHEMA_VERSION: 3; /** The only payload encoding the format defines. */ export const PAYLOAD_ENCODING: "base64-json-utf8"; /** The only signature algorithm the format defines. */ diff --git a/src/contract/document-shape.mjs b/src/contract/document-shape.mjs index b877f31..fe311cd 100644 --- a/src/contract/document-shape.mjs +++ b/src/contract/document-shape.mjs @@ -7,7 +7,7 @@ */ /** Format version carried by every document this contract describes. */ -export const BOX_SCHEMA_VERSION = 2; +export const BOX_SCHEMA_VERSION = 3; /** The only payload encoding the format defines. */ export const PAYLOAD_ENCODING = 'base64-json-utf8'; @@ -68,19 +68,24 @@ export function parseDocumentKind(kind) { * The message any reader gives a document written to a format version it cannot read. * * Four Node call sites answer this question — the payload decoder, the key loader, and the release - * verifier twice — and until now each carried its own copy of the sentence. That is not a style - * problem: the next version bump has to change what a v1 document is told, and a message duplicated - * per call site is a message that gets changed in three of four places. There is one wording per - * language now, and each language keeps its own because the string is user-facing text, not wire - * data that has to match across implementations. + * verifier twice — and each used to carry its own copy of the sentence. That is not a style problem: + * the v3 bump had to change what an older document is told, and a message duplicated per call site + * is a message that gets changed in three of four places. There is one wording per language now, and + * each language keeps its own because the string is user-facing text, not wire data that has to + * match across implementations. + * + * Both superseded versions are named rather than lumped together as "too old": a v1 and a v2 box + * are different artefacts with different rebuilds ahead of them, and the reader holding one is + * entitled to know which. * * @param {unknown} version the `schemaVersion` the document declared * @returns {string} */ export function unsupportedSchemaVersionMessage(version) { - return version === 1 - ? 'Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.' - : `Unsupported schemaVersion ${String(version)}; expected ${BOX_SCHEMA_VERSION}.`; + if (version === 1 || version === 2) { + return `Unsupported schemaVersion ${version}; rebuild this box with Scrollcase v3.`; + } + return `Unsupported schemaVersion ${String(version)}; expected ${BOX_SCHEMA_VERSION}.`; } /** Channels a box may be published to, ordered from least to most stable. */ diff --git a/src/contract/fixtures/examples/box-manifest.example.json b/src/contract/fixtures/examples/box-manifest.example.json index 3613501..d324293 100644 --- a/src/contract/fixtures/examples/box-manifest.example.json +++ b/src/contract/fixtures/examples/box-manifest.example.json @@ -1,21 +1,29 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "boxId": "example-model", - "modelId": "example-org-example-model", - "runtimeId": "example-model-runtime", + "labels": { + "model": "example-org/example-model", + "owner": "platform-team" + }, "version": "1.0.0", "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, - "pythonEntryPoint": "venv/bin/python", - "modelCacheSubdir": "model-cache/example-model", + "runtime": { + "id": "python", + "version": "3.11.15", + "entryPoint": "venv/bin/python" + }, + "cacheSubdir": "cache/example-model", "selfTest": { - "pythonImports": [ - "torch", - "numpy" - ], + "probe": { + "imports": [ + "torch", + "numpy" + ] + }, "timeoutSeconds": 180 }, "provenance": { @@ -24,7 +32,7 @@ "builderRevision": "4c1d9b7e2a0f5836d419e7c05b3a8f61d2704e93", "sourceTreeDirty": false, "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", - "pythonVersion": "3.11.15", + "runtimeVersion": "3.11.15", "pixiVersion": "0.73.0", "dependencyLockSha256": "3b1f8c47a2d9e05b6c7418af23d5e69017b4c8ad91e2f350768bd4ca19e0f5b7", "builtAt": "2026-07-25T12:00:00+00:00" diff --git a/src/contract/fixtures/examples/channel-manifest.example.json b/src/contract/fixtures/examples/channel-manifest.example.json index 7682e91..1aa0605 100644 --- a/src/contract/fixtures/examples/channel-manifest.example.json +++ b/src/contract/fixtures/examples/channel-manifest.example.json @@ -1,5 +1,5 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "kind": "scrollcase.box.channel", "channel": "beta", "boxId": "example-model", diff --git a/src/contract/fixtures/examples/release-manifest.example.json b/src/contract/fixtures/examples/release-manifest.example.json index 9a04e5b..e1d83fc 100644 --- a/src/contract/fixtures/examples/release-manifest.example.json +++ b/src/contract/fixtures/examples/release-manifest.example.json @@ -1,9 +1,11 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "kind": "scrollcase.box.release", "boxId": "example-model", - "modelId": "example-org-example-model", - "runtimeId": "example-model-runtime", + "labels": { + "model": "example-org/example-model", + "owner": "platform-team" + }, "version": "1.0.0", "target": { "platform": "macos", @@ -22,13 +24,19 @@ "sizeBytes": 655752216 }, "installedSizeBytes": 1892340112, - "pythonEntryPoint": "venv/bin/python", - "modelCacheSubdir": "model-cache/example-model", + "runtime": { + "id": "python", + "version": "3.11.15", + "entryPoint": "venv/bin/python" + }, + "cacheSubdir": "cache/example-model", "selfTest": { - "pythonImports": [ - "torch", - "numpy" - ], + "probe": { + "imports": [ + "torch", + "numpy" + ] + }, "timeoutSeconds": 180 }, "provenance": { @@ -37,7 +45,7 @@ "builderRevision": "4c1d9b7e2a0f5836d419e7c05b3a8f61d2704e93", "sourceTreeDirty": false, "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", - "pythonVersion": "3.11.15", + "runtimeVersion": "3.11.15", "pixiVersion": "0.73.0", "dependencyLockSha256": "3b1f8c47a2d9e05b6c7418af23d5e69017b4c8ad91e2f350768bd4ca19e0f5b7", "builtAt": "2026-07-25T12:00:00+00:00" diff --git a/src/contract/fixtures/examples/scroll-pixi.example.json b/src/contract/fixtures/examples/scroll-pixi.example.json index b758030..b098a4b 100644 --- a/src/contract/fixtures/examples/scroll-pixi.example.json +++ b/src/contract/fixtures/examples/scroll-pixi.example.json @@ -1,11 +1,12 @@ { - "$schema": "https://scrollcase.dev/schema/v2/scroll.schema.json", - "schemaVersion": 2, + "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "schemaVersion": 3, "scrollId": "example-model-linux-x86_64-cuda12.9", "scrollVersion": "1.0.0", "boxId": "example-model", - "modelId": "example-org-example-model", - "runtimeId": "example-model-runtime", + "labels": { + "model": "example-org/example-model" + }, "version": "1.0.0", "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", "target": { @@ -19,18 +20,22 @@ "minRamGb": 16, "minNvidiaDriverVersion": "525.60.13" }, - "pythonVersion": "3.11.15", "pixiVersion": "0.73.0", + "runtime": { + "id": "python", + "version": "3.11.15", + "entryPoint": "venv/bin/python" + }, "condaDependencyLicenseAudit": "legal/audits/example-model-linux-x86_64-cuda12.9.json", - "pythonEntryPoint": "venv/bin/python", - "modelCacheSubdir": "model-cache/example-model", + "cacheSubdir": "cache/example-model", "assetBaseUrl": "https://assets.example.org/boxes", "assets": [ { "url": "https://assets.example.org/example-model/weights.safetensors", - "relativePath": "model-cache/example-model/weights.safetensors", + "relativePath": "cache/example-model/weights.safetensors", "sizeBytes": 205385258, - "sha256": "6cb5d451ab5c4b33eb673adbe4fddc61d2389df1b89b7651a9fe2e557572b922" + "sha256": "6cb5d451ab5c4b33eb673adbe4fddc61d2389df1b89b7651a9fe2e557572b922", + "embed": false }, { "url": "https://codeload.example.org/example-org/example-model/zip/9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", @@ -62,11 +67,14 @@ "venv/lib/python3.11/tkinter" ], "selfTest": { - "imports": ["torch", "numpy"], + "imports": [ + "torch", + "numpy" + ], "files": [ - "model-cache/example-model/weights.safetensors", + "cache/example-model/weights.safetensors", "source/example-model/LICENSE" ], - "pythonCode": "assert torch.cuda.is_available()" + "code": "assert torch.cuda.is_available()" } } diff --git a/src/contract/fixtures/examples/scroll.example.json b/src/contract/fixtures/examples/scroll.example.json index 4790c73..e62d4ff 100644 --- a/src/contract/fixtures/examples/scroll.example.json +++ b/src/contract/fixtures/examples/scroll.example.json @@ -1,11 +1,13 @@ { - "$schema": "https://scrollcase.dev/schema/v2/scroll.schema.json", - "schemaVersion": 2, + "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "schemaVersion": 3, "scrollId": "example-model-macos-arm64-metal", "scrollVersion": "1.0.0", "boxId": "example-model", - "modelId": "example-org-example-model", - "runtimeId": "example-model-runtime", + "labels": { + "model": "example-org/example-model", + "owner": "platform-team" + }, "version": "1.0.0", "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", "target": { @@ -18,14 +20,17 @@ "minMacosVersion": "13.0", "minRamGb": 8 }, - "pythonVersion": "3.11.15", - "pythonEntryPoint": "venv/bin/python", - "modelCacheSubdir": "model-cache/example-model", + "runtime": { + "id": "python", + "version": "3.11.15", + "entryPoint": "venv/bin/python" + }, + "cacheSubdir": "cache/example-model", "assetBaseUrl": "https://assets.example.org/boxes", "assets": [ { "url": "https://assets.example.org/example-model/weights.safetensors", - "relativePath": "model-cache/example-model/weights.safetensors", + "relativePath": "cache/example-model/weights.safetensors", "sizeBytes": 205385258, "sha256": "6cb5d451ab5c4b33eb673adbe4fddc61d2389df1b89b7651a9fe2e557572b922" } @@ -36,7 +41,7 @@ "numpy" ], "files": [ - "model-cache/example-model/weights.safetensors" + "cache/example-model/weights.safetensors" ] }, "pixiVersion": "0.73.0", diff --git a/src/contract/fixtures/examples/signed-release.example.json b/src/contract/fixtures/examples/signed-release.example.json index 64b1a63..b19c21f 100644 --- a/src/contract/fixtures/examples/signed-release.example.json +++ b/src/contract/fixtures/examples/signed-release.example.json @@ -1,13 +1,13 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "payloadEncoding": "base64-json-utf8", - "payloadBase64": "ewogICJzY2hlbWFWZXJzaW9uIjogMiwKICAia2luZCI6ICJzY3JvbGxjYXNlLmJveC5yZWxlYXNlIiwKICAiYm94SWQiOiAiZXhhbXBsZS1tb2RlbCIsCiAgIm1vZGVsSWQiOiAiZXhhbXBsZS1vcmctZXhhbXBsZS1tb2RlbCIsCiAgInJ1bnRpbWVJZCI6ICJleGFtcGxlLW1vZGVsLXJ1bnRpbWUiLAogICJ2ZXJzaW9uIjogIjEuMC4wIiwKICAidGFyZ2V0IjogewogICAgInBsYXRmb3JtIjogIm1hY29zIiwKICAgICJhcmNoIjogImFhcmNoNjQiLAogICAgImFjY2VsZXJhdG9yIjogIm1ldGFsIgogIH0sCiAgImNvbXBhdGliaWxpdHkiOiB7CiAgICAibWluSG9zdEFwcFZlcnNpb24iOiAiMS4wLjAiLAogICAgIm1pbk1hY29zVmVyc2lvbiI6ICIxMy4wIiwKICAgICJtaW5SYW1HYiI6IDgKICB9LAogICJhcmNoaXZlIjogewogICAgImZvcm1hdCI6ICJ6aXAiLAogICAgInVybCI6ICJodHRwczovL2Fzc2V0cy5leGFtcGxlLm9yZy9ib3hlcy9ib3hlcy9leGFtcGxlLW1vZGVsLzEuMC4wL21hY29zLWFhcmNoNjQtbWV0YWwvN2QyYzlhNDFlOGIzNTBmNmMxNzRhOWRlMjAzNThiZjQxYzZlOTdkMDVhOGIzZjI2MTllNGM3MDgxZGE1YjNmMi56aXAiLAogICAgInNoYTI1NiI6ICI3ZDJjOWE0MWU4YjM1MGY2YzE3NGE5ZGUyMDM1OGJmNDFjNmU5N2QwNWE4YjNmMjYxOWU0YzcwODFkYTViM2YyIiwKICAgICJzaXplQnl0ZXMiOiA2NTU3NTIyMTYKICB9LAogICJpbnN0YWxsZWRTaXplQnl0ZXMiOiAxODkyMzQwMTEyLAogICJweXRob25FbnRyeVBvaW50IjogInZlbnYvYmluL3B5dGhvbiIsCiAgIm1vZGVsQ2FjaGVTdWJkaXIiOiAibW9kZWwtY2FjaGUvZXhhbXBsZS1tb2RlbCIsCiAgInNlbGZUZXN0IjogewogICAgInB5dGhvbkltcG9ydHMiOiBbCiAgICAgICJ0b3JjaCIsCiAgICAgICJudW1weSIKICAgIF0sCiAgICAidGltZW91dFNlY29uZHMiOiAxODAKICB9LAogICJwcm92ZW5hbmNlIjogewogICAgInNjcm9sbElkIjogImV4YW1wbGUtbW9kZWwtbWFjb3MtYXJtNjQtbWV0YWwiLAogICAgInNjcm9sbFZlcnNpb24iOiAiMS4wLjAiLAogICAgImJ1aWxkZXJSZXZpc2lvbiI6ICI0YzFkOWI3ZTJhMGY1ODM2ZDQxOWU3YzA1YjNhOGY2MWQyNzA0ZTkzIiwKICAgICJzb3VyY2VUcmVlRGlydHkiOiBmYWxzZSwKICAgICJzb3VyY2VSZXZpc2lvbiI6ICI5ZjBkM2ExYzRiNWU2ZjcwODE5MjBhM2I0YzVkNmU3ZjgwOTEwYTJiIiwKICAgICJweXRob25WZXJzaW9uIjogIjMuMTEuMTUiLAogICAgInBpeGlWZXJzaW9uIjogIjAuNzMuMCIsCiAgICAiZGVwZW5kZW5jeUxvY2tTaGEyNTYiOiAiM2IxZjhjNDdhMmQ5ZTA1YjZjNzQxOGFmMjNkNWU2OTAxN2I0YzhhZDkxZTJmMzUwNzY4YmQ0Y2ExOWUwZjViNyIsCiAgICAiYnVpbHRBdCI6ICIyMDI2LTA3LTI1VDEyOjAwOjAwKzAwOjAwIgogIH0KfQo=", - "payloadSha256": "bbf60de7d31035b2bfcb98c6c57624220f3bf59900391189f5e55da955055bc6", + "payloadBase64": "ewogICJzY2hlbWFWZXJzaW9uIjogMywKICAia2luZCI6ICJzY3JvbGxjYXNlLmJveC5yZWxlYXNlIiwKICAiYm94SWQiOiAiZXhhbXBsZS1tb2RlbCIsCiAgImxhYmVscyI6IHsKICAgICJtb2RlbCI6ICJleGFtcGxlLW9yZy9leGFtcGxlLW1vZGVsIiwKICAgICJvd25lciI6ICJwbGF0Zm9ybS10ZWFtIgogIH0sCiAgInZlcnNpb24iOiAiMS4wLjAiLAogICJ0YXJnZXQiOiB7CiAgICAicGxhdGZvcm0iOiAibWFjb3MiLAogICAgImFyY2giOiAiYWFyY2g2NCIsCiAgICAiYWNjZWxlcmF0b3IiOiAibWV0YWwiCiAgfSwKICAiY29tcGF0aWJpbGl0eSI6IHsKICAgICJtaW5Ib3N0QXBwVmVyc2lvbiI6ICIxLjAuMCIsCiAgICAibWluTWFjb3NWZXJzaW9uIjogIjEzLjAiLAogICAgIm1pblJhbUdiIjogOAogIH0sCiAgImFyY2hpdmUiOiB7CiAgICAiZm9ybWF0IjogInppcCIsCiAgICAidXJsIjogImh0dHBzOi8vYXNzZXRzLmV4YW1wbGUub3JnL2JveGVzL2JveGVzL2V4YW1wbGUtbW9kZWwvMS4wLjAvbWFjb3MtYWFyY2g2NC1tZXRhbC83ZDJjOWE0MWU4YjM1MGY2YzE3NGE5ZGUyMDM1OGJmNDFjNmU5N2QwNWE4YjNmMjYxOWU0YzcwODFkYTViM2YyLnppcCIsCiAgICAic2hhMjU2IjogIjdkMmM5YTQxZThiMzUwZjZjMTc0YTlkZTIwMzU4YmY0MWM2ZTk3ZDA1YThiM2YyNjE5ZTRjNzA4MWRhNWIzZjIiLAogICAgInNpemVCeXRlcyI6IDY1NTc1MjIxNgogIH0sCiAgImluc3RhbGxlZFNpemVCeXRlcyI6IDE4OTIzNDAxMTIsCiAgInJ1bnRpbWUiOiB7CiAgICAiaWQiOiAicHl0aG9uIiwKICAgICJ2ZXJzaW9uIjogIjMuMTEuMTUiLAogICAgImVudHJ5UG9pbnQiOiAidmVudi9iaW4vcHl0aG9uIgogIH0sCiAgImNhY2hlU3ViZGlyIjogImNhY2hlL2V4YW1wbGUtbW9kZWwiLAogICJzZWxmVGVzdCI6IHsKICAgICJwcm9iZSI6IHsKICAgICAgImltcG9ydHMiOiBbCiAgICAgICAgInRvcmNoIiwKICAgICAgICAibnVtcHkiCiAgICAgIF0KICAgIH0sCiAgICAidGltZW91dFNlY29uZHMiOiAxODAKICB9LAogICJwcm92ZW5hbmNlIjogewogICAgInNjcm9sbElkIjogImV4YW1wbGUtbW9kZWwtbWFjb3MtYXJtNjQtbWV0YWwiLAogICAgInNjcm9sbFZlcnNpb24iOiAiMS4wLjAiLAogICAgImJ1aWxkZXJSZXZpc2lvbiI6ICI0YzFkOWI3ZTJhMGY1ODM2ZDQxOWU3YzA1YjNhOGY2MWQyNzA0ZTkzIiwKICAgICJzb3VyY2VUcmVlRGlydHkiOiBmYWxzZSwKICAgICJzb3VyY2VSZXZpc2lvbiI6ICI5ZjBkM2ExYzRiNWU2ZjcwODE5MjBhM2I0YzVkNmU3ZjgwOTEwYTJiIiwKICAgICJydW50aW1lVmVyc2lvbiI6ICIzLjExLjE1IiwKICAgICJwaXhpVmVyc2lvbiI6ICIwLjczLjAiLAogICAgImRlcGVuZGVuY3lMb2NrU2hhMjU2IjogIjNiMWY4YzQ3YTJkOWUwNWI2Yzc0MThhZjIzZDVlNjkwMTdiNGM4YWQ5MWUyZjM1MDc2OGJkNGNhMTllMGY1YjciLAogICAgImJ1aWx0QXQiOiAiMjAyNi0wNy0yNVQxMjowMDowMCswMDowMCIKICB9Cn0K", + "payloadSha256": "f264818314ac170618c9472dc6cc7a72a09eb6390a237fe00dcbe44507d1aef1", "signatures": [ { "algorithm": "ed25519", - "keyId": "scrollcase-example-v2", - "signatureBase64": "c4ftYhvfGwJicd3kK9DzyTj+HNvJeiCfwk049BKk8hxSgAMxxU8BE51Q51ZkSK0mPolXRJ9pz52HO3KoKD1mAA==" + "keyId": "scrollcase-example-v3", + "signatureBase64": "oFNLMVGTsE162A9uaoo8IRBC1Vlqdy1vCeIUCgHYoupvgJVfTHiV8SDcdF5JOFxslrqN+/fgrbK5/wqaTS+jDg==" } ] } diff --git a/src/contract/fixtures/examples/signed-release.public-key.json b/src/contract/fixtures/examples/signed-release.public-key.json index f506853..829de3c 100644 --- a/src/contract/fixtures/examples/signed-release.public-key.json +++ b/src/contract/fixtures/examples/signed-release.public-key.json @@ -1,5 +1,5 @@ { "algorithm": "ed25519", - "keyId": "scrollcase-example-v2", - "publicKeyBase64": "frGNF6Fa2cw9m5HvWYBacydXujD4+ldqo6xGSCnCFyc=" + "keyId": "scrollcase-example-v3", + "publicKeyBase64": "CXKLVpasebepvlFXtejGxiqPLkYvd/avINY7f88s64s=" } diff --git a/src/contract/fixtures/runtime-contract.json b/src/contract/fixtures/runtime-contract.json index f567304..f08f8df 100644 --- a/src/contract/fixtures/runtime-contract.json +++ b/src/contract/fixtures/runtime-contract.json @@ -1,9 +1,12 @@ { - "description": "Golden cases for the Scrollcase runtime model: where a runtime lives inside a box, which payload paths it needs the executable bit on, which paths a declared execution could resolve to, and the shell-free command line that runs it. Every implementation of the format proves its mirror against this file. Paths stay payload-relative on purpose: a box root is a real filesystem path and each language joins one in its own terms, so a joined expectation here would only pin the host that read it.", + "description": "Golden cases for the box-format runtime model. Every implementation of the format - the Node reference in src/contract/runtimes.mjs, the Rust mirror, the Python mirror - must produce these answers exactly. A target says which machine a box runs on; a runtime says what runs inside it, and these are the rules that answer the second question.", "runtimes": [ { "id": "python", - "executionKinds": ["python-script", "python-module"], + "executionKinds": [ + "python-script", + "python-module" + ], "executionEnvironmentVariables": [ "PYTHONPATH", "PYTHONHOME", @@ -22,8 +25,12 @@ "launcherKind": "posix-polyglot" }, "executablePayloadPaths": { - "files": ["venv/bin/python"], - "directories": ["venv/bin"] + "files": [ + "venv/bin/python" + ], + "directories": [ + "venv/bin" + ] } }, { @@ -37,8 +44,12 @@ "launcherKind": "posix-polyglot" }, "executablePayloadPaths": { - "files": ["venv/bin/python"], - "directories": ["venv/bin"] + "files": [ + "venv/bin/python" + ], + "directories": [ + "venv/bin" + ] } }, { @@ -52,8 +63,12 @@ "launcherKind": "uv-windows-pe" }, "executablePayloadPaths": { - "files": ["venv/python.exe"], - "directories": ["venv/Scripts"] + "files": [ + "venv/python.exe" + ], + "directories": [ + "venv/Scripts" + ] } } ] @@ -130,8 +145,14 @@ "runtime": "python", "platform": "linux", "runtimeVersion": "3.11.15", - "execution": { "kind": "python-script", "script": "app/main.py", "defaultArgs": [] }, - "candidates": ["app/main.py"] + "execution": { + "kind": "python-script", + "script": "app/main.py", + "defaultArgs": [] + }, + "candidates": [ + "app/main.py" + ] }, { "name": "a POSIX module is looked for at the root, in the standard library, and in site-packages", @@ -141,7 +162,9 @@ "execution": { "kind": "python-module", "module": "example_model.main", - "defaultArgs": ["--serve"] + "defaultArgs": [ + "--serve" + ] }, "candidates": [ "example_model/main.py", @@ -157,7 +180,11 @@ "runtime": "python", "platform": "macos", "runtimeVersion": "3.12.4", - "execution": { "kind": "python-module", "module": "pkg", "defaultArgs": [] }, + "execution": { + "kind": "python-module", + "module": "pkg", + "defaultArgs": [] + }, "candidates": [ "pkg.py", "pkg/__main__.py", @@ -172,7 +199,11 @@ "runtime": "python", "platform": "windows", "runtimeVersion": "3.11.15", - "execution": { "kind": "python-module", "module": "pkg", "defaultArgs": [] }, + "execution": { + "kind": "python-module", + "module": "pkg", + "defaultArgs": [] + }, "candidates": [ "pkg.py", "pkg/__main__.py", @@ -183,7 +214,13 @@ ] } ], - "invalidRuntimeVersions": ["", "3", "3.x", "x.1", "3."], + "invalidRuntimeVersions": [ + "", + "3", + "3.x", + "x.1", + "3." + ], "argv": [ { "name": "a script runs as a payload path, with its declared arguments after it", @@ -192,14 +229,33 @@ "execution": { "kind": "python-script", "script": "app/main.py", - "defaultArgs": ["--serve", "--port", "8080"] + "defaultArgs": [ + "--serve", + "--port", + "8080" + ] + }, + "command": { + "kind": "payload-path", + "value": "venv/bin/python" }, - "command": { "kind": "payload-path", "value": "venv/bin/python" }, "args": [ - { "kind": "payload-path", "value": "app/main.py" }, - { "kind": "literal", "value": "--serve" }, - { "kind": "literal", "value": "--port" }, - { "kind": "literal", "value": "8080" } + { + "kind": "payload-path", + "value": "app/main.py" + }, + { + "kind": "literal", + "value": "--serve" + }, + { + "kind": "literal", + "value": "--port" + }, + { + "kind": "literal", + "value": "8080" + } ] }, { @@ -211,10 +267,19 @@ "module": "example_model.main", "defaultArgs": [] }, - "command": { "kind": "payload-path", "value": "venv/bin/python" }, + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, "args": [ - { "kind": "literal", "value": "-m" }, - { "kind": "literal", "value": "example_model.main" } + { + "kind": "literal", + "value": "-m" + }, + { + "kind": "literal", + "value": "example_model.main" + } ] }, { @@ -226,8 +291,16 @@ "script": "app/main.py", "defaultArgs": [] }, - "command": { "kind": "payload-path", "value": "venv/python.exe" }, - "args": [{ "kind": "payload-path", "value": "app/main.py" }] + "command": { + "kind": "payload-path", + "value": "venv/python.exe" + }, + "args": [ + { + "kind": "payload-path", + "value": "app/main.py" + } + ] } ], "selfTest": [ @@ -235,35 +308,292 @@ "name": "macOS asserts Darwin before importing anything", "runtime": "python", "platform": "macos", - "probe": { "imports": ["json"] }, - "args": ["-c", "import sys; assert sys.platform == 'darwin'\nimport json"] + "probe": { + "imports": [ + "json" + ] + }, + "execution": null, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "literal", + "value": "-c" + }, + { + "kind": "literal", + "value": "import sys; assert sys.platform == 'darwin'\nimport json" + } + ], + "expectExitCode": 0 + } + ] }, { "name": "Linux accepts any linux variant", "runtime": "python", "platform": "linux", - "probe": { "imports": ["json", "numpy"] }, - "args": [ - "-c", - "import sys; assert sys.platform.startswith('linux')\nimport json, numpy" + "probe": { + "imports": [ + "json", + "numpy" + ] + }, + "execution": null, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "literal", + "value": "-c" + }, + { + "kind": "literal", + "value": "import sys; assert sys.platform.startswith('linux')\nimport json, numpy" + } + ], + "expectExitCode": 0 + } ] }, { "name": "Windows asserts win32", "runtime": "python", "platform": "windows", - "probe": { "imports": ["json"] }, - "args": ["-c", "import sys; assert sys.platform == 'win32'\nimport json"] + "probe": { + "imports": [ + "json" + ] + }, + "execution": null, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/python.exe" + }, + "args": [ + { + "kind": "literal", + "value": "-c" + }, + { + "kind": "literal", + "value": "import sys; assert sys.platform == 'win32'\nimport json" + } + ], + "expectExitCode": 0 + } + ] }, { "name": "the builder appends the extra source a scroll declared", "runtime": "python", "platform": "linux", - "probe": { "imports": ["json"], "code": "print(\"self-test ok\")\n" }, - "args": [ - "-c", - "import sys; assert sys.platform.startswith('linux')\nimport json\nprint(\"self-test ok\")\n" + "probe": { + "imports": [ + "json" + ], + "code": "print(\"self-test ok\")\n" + }, + "execution": null, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "literal", + "value": "-c" + }, + { + "kind": "literal", + "value": "import sys; assert sys.platform.startswith('linux')\nimport json\nprint(\"self-test ok\")\n" + } + ], + "expectExitCode": 0 + } + ] + }, + { + "name": "a command probe invokes the declared execution, after its default arguments", + "runtime": "python", + "platform": "linux", + "probe": { + "commands": [ + { + "args": [ + "--version" + ], + "expectExitCode": 0 + } + ] + }, + "execution": { + "kind": "python-script", + "script": "app/main.py", + "defaultArgs": [ + "--serve" + ] + }, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "payload-path", + "value": "app/main.py" + }, + { + "kind": "literal", + "value": "--serve" + }, + { + "kind": "literal", + "value": "--version" + } + ], + "expectExitCode": 0 + } + ] + }, + { + "name": "a command may require a non-zero exit status", + "runtime": "python", + "platform": "macos", + "probe": { + "commands": [ + { + "args": [ + "--help" + ], + "expectExitCode": 2 + } + ] + }, + "execution": { + "kind": "python-module", + "module": "example.cli", + "defaultArgs": [] + }, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "literal", + "value": "-m" + }, + { + "kind": "literal", + "value": "example.cli" + }, + { + "kind": "literal", + "value": "--help" + } + ], + "expectExitCode": 2 + } + ] + }, + { + "name": "imports and commands both run, imports first", + "runtime": "python", + "platform": "linux", + "probe": { + "imports": [ + "json" + ], + "commands": [ + { + "args": [], + "expectExitCode": 0 + }, + { + "args": [ + "--check" + ], + "expectExitCode": 1 + } + ] + }, + "execution": { + "kind": "python-script", + "script": "run.py", + "defaultArgs": [] + }, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "literal", + "value": "-c" + }, + { + "kind": "literal", + "value": "import sys; assert sys.platform.startswith('linux')\nimport json" + } + ], + "expectExitCode": 0 + }, + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "payload-path", + "value": "run.py" + } + ], + "expectExitCode": 0 + }, + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "payload-path", + "value": "run.py" + }, + { + "kind": "literal", + "value": "--check" + } + ], + "expectExitCode": 1 + } ] } + ], + "runtimeIds": [ + "python", + "node", + "native" ] } diff --git a/src/contract/index.d.mts b/src/contract/index.d.mts index 3992db8..b0b1192 100644 --- a/src/contract/index.d.mts +++ b/src/contract/index.d.mts @@ -13,5 +13,6 @@ export function schemaUrl(name: "target" | "scroll" | "box-manifest" | "release- * @returns {URL} */ export function fixtureUrl(name: string): URL; -export { assertNativeHost, assertPythonEntryPoint, condaSubdir, pixiAccelerator, boxTargetAdapter, boxTargetAdapters, boxTargetId } from "./targets.mjs"; +export { assertNativeHost, condaSubdir, pixiAccelerator, boxTargetAdapter, boxTargetAdapters, boxTargetId } from "./targets.mjs"; +export { RUNTIME_IDS, assertRuntimeEntryPoint, executionAffectingVariables, isExecutablePayloadPath, isImplementedRuntime, runtimeAdapter, runtimeAdapters, unimplementedRuntimeMessage } from "./runtimes.mjs"; export { CHANNELS, DEFAULT_DOCUMENT_NAMESPACE, PAYLOAD_ENCODING, BOX_SCHEMA_VERSION, SIGNATURE_ALGORITHM, decodeDocumentPayload, documentKinds, isSignedBoxDocument, parseDocumentKind } from "./documents.mjs"; diff --git a/src/contract/index.mjs b/src/contract/index.mjs index f3ec625..228e50a 100644 --- a/src/contract/index.mjs +++ b/src/contract/index.mjs @@ -2,10 +2,10 @@ * The Scrollcase box-format contract. * * This module is the single source of truth for what a box *is*: which targets exist, how a - * target is named, what layout the payload has, and the shape of every document a build emits. It - * ships three things that must never disagree — a reference implementation (this code), a - * machine-readable spec (`schema/*.json`), and golden fixtures (`fixtures/*.json`) that any other - * implementation can validate itself against. + * target is named, which runtimes a box may declare and what each implies for the payload, and the + * shape of every document a build emits. It ships three things that must never disagree — a + * reference implementation (this code), a machine-readable spec (`schema/*.json`), and golden + * fixtures (`fixtures/*.json`) that any other implementation can validate itself against. * * A consumer written in another language does not import this code; it mirrors the rules and proves * the mirror against the fixtures. That is how clients in other languages stay honest. @@ -13,7 +13,6 @@ export { assertNativeHost, - assertPythonEntryPoint, condaSubdir, pixiAccelerator, boxTargetAdapter, @@ -21,6 +20,17 @@ export { boxTargetId, } from './targets.mjs'; +export { + RUNTIME_IDS, + assertRuntimeEntryPoint, + executionAffectingVariables, + isExecutablePayloadPath, + isImplementedRuntime, + runtimeAdapter, + runtimeAdapters, + unimplementedRuntimeMessage, +} from './runtimes.mjs'; + export { CHANNELS, DEFAULT_DOCUMENT_NAMESPACE, diff --git a/src/contract/runtimes.d.mts b/src/contract/runtimes.d.mts index a44f3ac..a1e4eb8 100644 --- a/src/contract/runtimes.d.mts +++ b/src/contract/runtimes.d.mts @@ -23,6 +23,26 @@ export function runtimeAdapters(): BoxRuntimeAdapter[]; * @throws {TypeError} when the entry point is not the one the runtime defines for this target */ export function assertRuntimeEntryPoint(runtimeId: string, adapter: import("./targets.mjs").BoxTargetAdapter, entryPoint: string): void; +/** + * The message for a box declaring a runtime this build has no adapter for. + * + * The wire vocabulary is fixed and the implemented set is not, so this case is expected rather than + * exceptional, and the wording says which of the two the box fell foul of. It lives here so the + * builder and all three consumers report an unimplemented runtime identically instead of each + * inventing a phrasing. + * + * @param {string} runtimeId + * @returns {string} + */ +export function unimplementedRuntimeMessage(runtimeId: string): string; +/** + * Whether this build carries an adapter for a runtime id — the question every caller asks before + * `runtimeAdapter`, which throws rather than returning nothing. + * + * @param {string} runtimeId + * @returns {boolean} + */ +export function isImplementedRuntime(runtimeId: string): boolean; /** * Whether a payload path is one the runtime requires the executable bit on. * @@ -63,6 +83,11 @@ export function executionAffectingVariables(runtimeId: string, adapter: import(" * in `fixtures/runtime-contract.json` are what "agree" means, and are what the Python and Rust * mirrors validate themselves against. * + * Schema version 3 made the runtime a declaration: a box says `runtime: { id, version, entryPoint }` + * instead of leaving a reader to infer Python from a Python-shaped entry point. `RUNTIME_IDS` is the + * vocabulary that declaration may use and `RUNTIME_ADAPTERS` is what this build can actually run — + * two different lists on purpose, so implementing `node` later is code and not another wire break. + * * Only the pure half lives here. Nothing in this module reads a file, joins a host path, or starts * a process: every function is a statement about names, so the same inputs give the same answer in * every language and on every host. Builder-side behaviour — environment preparation, launcher @@ -96,9 +121,9 @@ export function executionAffectingVariables(runtimeId: string, adapter: import(" * target: BoxRuntimeTarget }) => ResolvedExecutionFiles} resolveExecutionFiles * @property {(options: { execution: object, * target: BoxRuntimeTarget }) => BoxRuntimeInvocation} buildArgv - * @property {(options: { probe: BoxRuntimeSelfTestProbe, - * target: BoxRuntimeTarget }) => readonly string[]} selfTestArgv the arguments that follow the - * runtime's own entry point when it runs a self-test probe + * @property {(options: { probe: BoxRuntimeSelfTestProbe, execution: object | null | undefined, + * target: BoxRuntimeTarget }) => readonly BoxRuntimeSelfTestInvocation[]} selfTestInvocations + * every command a self-test probe implies, in declaration order */ /** * The part of a target a runtime rule reads. A `BoxTarget` and the `BoxTargetAdapter` resolved from @@ -142,19 +167,34 @@ export function executionAffectingVariables(runtimeId: string, adapter: import(" * own arguments */ /** - * What a self-test asks the runtime to prove, plus the builder-only extension a scroll may add. + * What a self-test asks the box to prove, plus the builder-only extension a scroll may add. + * + * `imports` asks the runtime's loader a question and only means something to a runtime that has + * one. `commands` asks the box's declared execution a question, which every runtime can answer and + * a native one can answer *only* that way. A probe carries whichever apply; `code` never travels on + * the wire, because signing it would claim a consumer had repeated a check it cannot see. * - * @typedef {{ imports: readonly string[], code?: string | null }} BoxRuntimeSelfTestProbe + * @typedef {object} BoxRuntimeSelfTestProbe + * @property {readonly string[]} [imports] modules the runtime must be able to load + * @property {readonly { args: readonly string[], expectExitCode?: number }[]} [commands] + * @property {string | null} [code] builder-only extra source in the runtime's own language */ /** - * The runtime every box built by this schema version implicitly declares. + * One command the self-test runs, and the status it must exit with. * - * The wire format has no runtime field: a box records a Python entry point and Python execution - * kinds and nothing that says "Python". So a reader that must name a runtime names this one, from - * one place — the point being that adding the declaration later changes an argument rather than - * starting a hunt for hard-coded strings. + * @typedef {object} BoxRuntimeSelfTestInvocation + * @property {BoxRuntimeArgument} command + * @property {readonly BoxRuntimeArgument[]} args + * @property {number} expectExitCode */ -export const IMPLICIT_RUNTIME_ID: "python"; +/** + * Every runtime id the box format admits, in the order the schema lists them. + * + * The wire enum and the implemented set are deliberately two different things: schema version 3 + * fixes the vocabulary once, so a later release can implement `node` without another wire break. + * A box naming a runtime this build has no adapter for is refused by name, not misread. + */ +export const RUNTIME_IDS: readonly string[]; /** * What a runtime implies for a box, independent of the machine it runs on. */ @@ -193,13 +233,13 @@ export type BoxRuntimeAdapter = { target: BoxRuntimeTarget; }) => BoxRuntimeInvocation; /** - * the arguments that follow the - * runtime's own entry point when it runs a self-test probe + * every command a self-test probe implies, in declaration order */ - selfTestArgv: (options: { + selfTestInvocations: (options: { probe: BoxRuntimeSelfTestProbe; + execution: object | null | undefined; target: BoxRuntimeTarget; - }) => readonly string[]; + }) => readonly BoxRuntimeSelfTestInvocation[]; }; /** * The part of a target a runtime rule reads. A `BoxTarget` and the `BoxTargetAdapter` resolved from @@ -276,9 +316,32 @@ export type BoxRuntimeInvocation = { args: readonly BoxRuntimeArgument[]; }; /** - * What a self-test asks the runtime to prove, plus the builder-only extension a scroll may add. + * What a self-test asks the box to prove, plus the builder-only extension a scroll may add. + * + * `imports` asks the runtime's loader a question and only means something to a runtime that has + * one. `commands` asks the box's declared execution a question, which every runtime can answer and + * a native one can answer *only* that way. A probe carries whichever apply; `code` never travels on + * the wire, because signing it would claim a consumer had repeated a check it cannot see. */ export type BoxRuntimeSelfTestProbe = { - imports: readonly string[]; + /** + * modules the runtime must be able to load + */ + imports?: readonly string[]; + commands?: readonly { + args: readonly string[]; + expectExitCode?: number; + }[]; + /** + * builder-only extra source in the runtime's own language + */ code?: string | null; }; +/** + * One command the self-test runs, and the status it must exit with. + */ +export type BoxRuntimeSelfTestInvocation = { + command: BoxRuntimeArgument; + args: readonly BoxRuntimeArgument[]; + expectExitCode: number; +}; diff --git a/src/contract/runtimes.mjs b/src/contract/runtimes.mjs index be7bd08..10b4b06 100644 --- a/src/contract/runtimes.mjs +++ b/src/contract/runtimes.mjs @@ -14,6 +14,11 @@ * in `fixtures/runtime-contract.json` are what "agree" means, and are what the Python and Rust * mirrors validate themselves against. * + * Schema version 3 made the runtime a declaration: a box says `runtime: { id, version, entryPoint }` + * instead of leaving a reader to infer Python from a Python-shaped entry point. `RUNTIME_IDS` is the + * vocabulary that declaration may use and `RUNTIME_ADAPTERS` is what this build can actually run — + * two different lists on purpose, so implementing `node` later is code and not another wire break. + * * Only the pure half lives here. Nothing in this module reads a file, joins a host path, or starts * a process: every function is a statement about names, so the same inputs give the same answer in * every language and on every host. Builder-side behaviour — environment preparation, launcher @@ -48,9 +53,9 @@ * target: BoxRuntimeTarget }) => ResolvedExecutionFiles} resolveExecutionFiles * @property {(options: { execution: object, * target: BoxRuntimeTarget }) => BoxRuntimeInvocation} buildArgv - * @property {(options: { probe: BoxRuntimeSelfTestProbe, - * target: BoxRuntimeTarget }) => readonly string[]} selfTestArgv the arguments that follow the - * runtime's own entry point when it runs a self-test probe + * @property {(options: { probe: BoxRuntimeSelfTestProbe, execution: object | null | undefined, + * target: BoxRuntimeTarget }) => readonly BoxRuntimeSelfTestInvocation[]} selfTestInvocations + * every command a self-test probe implies, in declaration order */ /** @@ -101,20 +106,36 @@ */ /** - * What a self-test asks the runtime to prove, plus the builder-only extension a scroll may add. + * What a self-test asks the box to prove, plus the builder-only extension a scroll may add. + * + * `imports` asks the runtime's loader a question and only means something to a runtime that has + * one. `commands` asks the box's declared execution a question, which every runtime can answer and + * a native one can answer *only* that way. A probe carries whichever apply; `code` never travels on + * the wire, because signing it would claim a consumer had repeated a check it cannot see. + * + * @typedef {object} BoxRuntimeSelfTestProbe + * @property {readonly string[]} [imports] modules the runtime must be able to load + * @property {readonly { args: readonly string[], expectExitCode?: number }[]} [commands] + * @property {string | null} [code] builder-only extra source in the runtime's own language + */ + +/** + * One command the self-test runs, and the status it must exit with. * - * @typedef {{ imports: readonly string[], code?: string | null }} BoxRuntimeSelfTestProbe + * @typedef {object} BoxRuntimeSelfTestInvocation + * @property {BoxRuntimeArgument} command + * @property {readonly BoxRuntimeArgument[]} args + * @property {number} expectExitCode */ /** - * The runtime every box built by this schema version implicitly declares. + * Every runtime id the box format admits, in the order the schema lists them. * - * The wire format has no runtime field: a box records a Python entry point and Python execution - * kinds and nothing that says "Python". So a reader that must name a runtime names this one, from - * one place — the point being that adding the declaration later changes an argument rather than - * starting a hunt for hard-coded strings. + * The wire enum and the implemented set are deliberately two different things: schema version 3 + * fixes the vocabulary once, so a later release can implement `node` without another wire break. + * A box naming a runtime this build has no adapter for is refused by name, not misread. */ -export const IMPLICIT_RUNTIME_ID = 'python'; +export const RUNTIME_IDS = Object.freeze(['python', 'node', 'native']); const PYTHON_EXECUTION_ENVIRONMENT = Object.freeze([ 'PYTHONPATH', @@ -201,6 +222,35 @@ function pythonModuleEntryPoints({ module, runtimeVersion, target }) { relativeCandidates.map((path) => (root ? `${root}/${path}` : path))); } +/** + * A command probe, turned into an invocation of the box's own declared execution. + * + * Shared by every runtime, because it is not a runtime-specific rule: the box says how it is run, + * and the probe appends arguments to that. A probe of this shape in a box that declares no + * execution has nothing to invoke, which is a contradiction in the declaration rather than a + * property of the box, so it is refused where scrolls are read and reported here as a programming + * error if it ever gets this far. + */ +function commandInvocation(runtime, { command, execution, target }) { + if (!execution) { + throw new TypeError('A self-test command needs a declared execution to invoke'); + } + const { command: entryPoint, args } = runtime.buildArgv({ execution, target }); + return { + command: entryPoint, + args: [...args, ...command.args.map((value) => ({ kind: 'literal', value }))], + expectExitCode: command.expectExitCode ?? 0, + }; +} + +function freezeInvocations(invocations) { + return Object.freeze(invocations.map((invocation) => Object.freeze({ + command: Object.freeze(invocation.command), + args: Object.freeze(invocation.args.map((argument) => Object.freeze(argument))), + expectExitCode: invocation.expectExitCode, + }))); +} + const PYTHON_RUNTIME = Object.freeze({ id: 'python', executionKinds: Object.freeze(['python-script', 'python-module']), @@ -248,16 +298,27 @@ const PYTHON_RUNTIME = Object.freeze({ }); }, - selfTestArgv({ probe, target }) { - const assertion = PYTHON_PLATFORM_ASSERTIONS[target?.platform]; - if (!assertion) { - throw new TypeError(`No python self-test assertion exists for platform ${String(target?.platform)}`); + selfTestInvocations({ probe, execution, target }) { + const invocations = []; + if (probe.imports?.length) { + const assertion = PYTHON_PLATFORM_ASSERTIONS[target?.platform]; + if (!assertion) { + throw new TypeError(`No python self-test assertion exists for platform ${String(target?.platform)}`); + } + const imports = `import ${probe.imports.join(', ')}`; + const code = probe.code + ? `${assertion}\n${imports}\n${probe.code}` + : `${assertion}\n${imports}`; + invocations.push({ + command: { kind: 'payload-path', value: pythonLayout(target).entryPoint }, + args: ['-c', code].map((value) => ({ kind: 'literal', value })), + expectExitCode: 0, + }); } - const imports = `import ${probe.imports.join(', ')}`; - const code = probe.code - ? `${assertion}\n${imports}\n${probe.code}` - : `${assertion}\n${imports}`; - return Object.freeze(['-c', code]); + for (const command of probe.commands ?? []) { + invocations.push(commandInvocation(PYTHON_RUNTIME, { command, execution, target })); + } + return freezeInvocations(invocations); }, }); @@ -299,13 +360,39 @@ export function assertRuntimeEntryPoint(runtimeId, adapter, entryPoint) { const runtime = runtimeAdapter(runtimeId); const expected = runtime.layout(adapter).entryPoint; if (entryPoint !== expected) { - // The wording still names Python because the wire format still does: a scroll declares - // `pythonEntryPoint`, and an error that called it something else would name a field the author - // cannot find. It generalises with the field, in the same version bump. - throw new TypeError(`${adapter.id} scrolls must use Python entry point ${expected}`); + throw new TypeError(`${adapter.id} boxes with the ${runtime.id} runtime must use entry point ${expected}`); } } +/** + * The message for a box declaring a runtime this build has no adapter for. + * + * The wire vocabulary is fixed and the implemented set is not, so this case is expected rather than + * exceptional, and the wording says which of the two the box fell foul of. It lives here so the + * builder and all three consumers report an unimplemented runtime identically instead of each + * inventing a phrasing. + * + * @param {string} runtimeId + * @returns {string} + */ +export function unimplementedRuntimeMessage(runtimeId) { + const implemented = RUNTIME_ADAPTERS.map((adapter) => adapter.id).join(', '); + return RUNTIME_IDS.includes(runtimeId) + ? `Runtime ${runtimeId} is not implemented by this version of Scrollcase; it implements ${implemented}.` + : `Unknown runtime: ${String(runtimeId)}. The box format defines ${RUNTIME_IDS.join(', ')}.`; +} + +/** + * Whether this build carries an adapter for a runtime id — the question every caller asks before + * `runtimeAdapter`, which throws rather than returning nothing. + * + * @param {string} runtimeId + * @returns {boolean} + */ +export function isImplementedRuntime(runtimeId) { + return RUNTIME_ADAPTERS.some((adapter) => adapter.id === runtimeId); +} + /** * Whether a payload path is one the runtime requires the executable bit on. * diff --git a/src/contract/schema/box-manifest.schema.json b/src/contract/schema/box-manifest.schema.json index 8b0ddb8..6060ec5 100644 --- a/src/contract/schema/box-manifest.schema.json +++ b/src/contract/schema/box-manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/box-manifest.schema.json", + "$id": "https://scrollcase.dev/schema/v3/box-manifest.schema.json", "title": "Box manifest (box.json)", "description": "The manifest packed inside the archive at box.json. It restates the box's identity, layout and provenance so an extracted box is self-describing: a consumer that has the directory but not the release document can still tell what it is holding and how it was built.", "type": "object", @@ -8,43 +8,35 @@ "required": [ "schemaVersion", "boxId", - "modelId", - "runtimeId", "version", "target", - "pythonEntryPoint", - "modelCacheSubdir", + "runtime", + "cacheSubdir", "selfTest", "provenance" ], "properties": { "schemaVersion": { - "const": 2 + "const": 3 }, "boxId": { "type": "string", "minLength": 1 }, - "modelId": { - "type": "string", - "minLength": 1 - }, - "runtimeId": { - "type": "string", - "minLength": 1 + "labels": { + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/labels" }, "version": { "type": "string", "minLength": 1 }, "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + "$ref": "https://scrollcase.dev/schema/v3/target.schema.json" }, - "pythonEntryPoint": { - "type": "string", - "minLength": 1 + "runtime": { + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/runtime" }, - "modelCacheSubdir": { + "cacheSubdir": { "type": "string", "minLength": 1 }, @@ -61,76 +53,16 @@ } }, "selfTest": { - "type": "object", - "additionalProperties": false, - "required": [ - "pythonImports", - "timeoutSeconds" - ], - "properties": { - "pythonImports": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "minLength": 1 - } - }, - "timeoutSeconds": { - "type": "integer", - "exclusiveMinimum": 0 - } - } + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/selfTest" }, "execution": { - "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" + "$ref": "https://scrollcase.dev/schema/v3/execution.schema.json" }, "provenance": { - "$ref": "https://scrollcase.dev/schema/v2/release-manifest.schema.json#/$defs/provenance" - }, - "weights": { - "const": "on-demand", - "description": "Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it." + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/provenance" }, "assets": { - "type": "array", - "minItems": 1, - "description": "Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe.", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "url", - "relativePath", - "sizeBytes", - "sha256" - ], - "properties": { - "url": { - "type": "string", - "minLength": 1 - }, - "relativePath": { - "type": "string", - "minLength": 1 - }, - "sizeBytes": { - "type": "integer", - "exclusiveMinimum": 0 - }, - "sha256": { - "$ref": "https://scrollcase.dev/schema/v2/release-manifest.schema.json#/$defs/sha256" - } - } - } + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/deferredAssets" } - }, - "dependentRequired": { - "assets": [ - "weights" - ], - "weights": [ - "assets" - ] } } diff --git a/src/contract/schema/channel-manifest.schema.json b/src/contract/schema/channel-manifest.schema.json index 70102ed..0e04054 100644 --- a/src/contract/schema/channel-manifest.schema.json +++ b/src/contract/schema/channel-manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/channel-manifest.schema.json", + "$id": "https://scrollcase.dev/schema/v3/channel-manifest.schema.json", "title": "Box channel manifest", "description": "A small mutable pointer from a channel to the releases it currently serves. Signed independently from releases, so promoting a build never requires re-signing it.", "type": "object", @@ -17,7 +17,7 @@ ], "properties": { "schemaVersion": { - "const": 2 + "const": 3 }, "kind": { "type": "string", @@ -36,7 +36,7 @@ "minLength": 1 }, "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + "$ref": "https://scrollcase.dev/schema/v3/target.schema.json" }, "updatedAt": { "type": "string", diff --git a/src/contract/schema/execution.schema.json b/src/contract/schema/execution.schema.json index a37ebfb..62b4da2 100644 --- a/src/contract/schema/execution.schema.json +++ b/src/contract/schema/execution.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/execution.schema.json", + "$id": "https://scrollcase.dev/schema/v3/execution.schema.json", "title": "Box execution", - "description": "The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only.", + "description": "The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only.\n\nEach kind is named -, and the runtime half must be the one the box declares: a python-script in a box whose runtime is native describes something that cannot be run, and is refused rather than guessed at.", "oneOf": [ { "title": "Python script", @@ -55,6 +55,54 @@ "$ref": "#/$defs/defaultArgs" } } + }, + { + "title": "Node script", + "description": "Run one regular payload file with the box's own Node runtime.", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "script", + "defaultArgs" + ], + "properties": { + "kind": { + "const": "node-script", + "description": "Selects direct script execution." + }, + "script": { + "$ref": "#/$defs/payloadPath", + "description": "Safe path to a regular JavaScript file inside the box." + }, + "defaultArgs": { + "$ref": "#/$defs/defaultArgs" + } + } + }, + { + "title": "Native binary", + "description": "Run a compiled executable that the box carries directly, with no interpreter in front of it. The only shape a runtime with no module system has.", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "binary", + "defaultArgs" + ], + "properties": { + "kind": { + "const": "native-binary", + "description": "Selects direct execution of a payload file." + }, + "binary": { + "$ref": "#/$defs/payloadPath", + "description": "Safe path to the executable inside the box. It carries the executable bit because the scroll declared it, not because the build machine happened to have it set." + }, + "defaultArgs": { + "$ref": "#/$defs/defaultArgs" + } + } } ], "examples": [ diff --git a/src/contract/schema/release-manifest.schema.json b/src/contract/schema/release-manifest.schema.json index 07d5458..90d1b23 100644 --- a/src/contract/schema/release-manifest.schema.json +++ b/src/contract/schema/release-manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/release-manifest.schema.json", + "$id": "https://scrollcase.dev/schema/v3/release-manifest.schema.json", "title": "Box release manifest", "description": "The immutable description of one built box: what it is, which target it runs on, where its archive lives, and how it was produced. Published as the payload of a signed document and never edited after signing; a correction ships as a new version.", "type": "object", @@ -9,20 +9,18 @@ "schemaVersion", "kind", "boxId", - "modelId", - "runtimeId", "version", "target", "compatibility", "archive", - "pythonEntryPoint", - "modelCacheSubdir", + "runtime", + "cacheSubdir", "selfTest", "provenance" ], "properties": { "schemaVersion": { - "const": 2 + "const": 3 }, "kind": { "$ref": "#/$defs/kind", @@ -31,18 +29,15 @@ "boxId": { "$ref": "#/$defs/identifier" }, - "modelId": { - "$ref": "#/$defs/identifier" - }, - "runtimeId": { - "$ref": "#/$defs/identifier" + "labels": { + "$ref": "#/$defs/labels" }, "version": { "type": "string", "minLength": 1 }, "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + "$ref": "https://scrollcase.dev/schema/v3/target.schema.json" }, "compatibility": { "type": "object", @@ -132,15 +127,13 @@ } } }, - "pythonEntryPoint": { - "type": "string", - "minLength": 1, - "description": "Interpreter path relative to the extracted box root, for example venv/bin/python. Fixed per target by the adapter." + "runtime": { + "$ref": "#/$defs/runtime" }, - "modelCacheSubdir": { + "cacheSubdir": { "type": "string", "minLength": 1, - "description": "Directory relative to the extracted box root holding model assets." + "description": "Directory relative to the extracted box root holding the box's own large files." }, "environment": { "type": "object", @@ -155,42 +148,127 @@ } }, "selfTest": { + "$ref": "#/$defs/selfTest" + }, + "execution": { + "$ref": "https://scrollcase.dev/schema/v3/execution.schema.json" + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "assets": { + "$ref": "#/$defs/deferredAssets" + } + }, + "$defs": { + "identifier": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" + }, + "runtime": { "type": "object", "additionalProperties": false, "required": [ - "pythonImports", + "id" + ], + "description": "What runs inside the box: the runtime, its version, and where its own executable sits in the payload. A consumer needs all three to run the box, and none of them are derivable from the target.", + "properties": { + "id": { + "enum": [ + "python", + "node", + "native" + ], + "description": "The runtime the box carries. A consumer that does not recognise the id must refuse the box: the id decides the payload layout and the argv rule, so guessing would mean executing something on an assumption." + }, + "version": { + "type": "string", + "minLength": 1, + "description": "The runtime's own version. Absent for a runtime that has no interpreter to version." + }, + "entryPoint": { + "type": "string", + "minLength": 1, + "description": "The runtime's own executable relative to the extracted box root, for example venv/bin/python. Fixed per (runtime, target) by the runtime's layout. Absent for a runtime that has no separate executable to name." + } + } + }, + "labels": { + "type": "object", + "description": "Free-form annotations the publishing project declared, signed and carried through untouched. Scrollcase attaches no meaning to any key; a consumer that reads one is reading its own project's convention, not the box format.", + "propertyNames": { + "$ref": "#/$defs/identifier" + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } + }, + "selfTest": { + "type": "object", + "additionalProperties": false, + "required": [ + "probe", "timeoutSeconds" ], - "description": "The import check a consumer can repeat after extraction with the box's own interpreter. The builder also ran the scroll's Python-code and file assertions, which are builder-only checks.", + "description": "The check a consumer can repeat against an extracted box. The builder also ran the scroll's file assertions and any extra source it declared, which are builder-only: signing them would claim a consumer had reproduced a check it cannot see.", "properties": { - "pythonImports": { + "probe": { + "$ref": "#/$defs/selfTestProbe" + }, + "timeoutSeconds": { + "type": "integer", + "exclusiveMinimum": 0 + } + } + }, + "selfTestProbe": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "description": "What the box proves about itself, in whichever shapes its runtime supports. The runtime turns this into command lines; nothing here is a command line, and nothing here is source in any language.", + "properties": { + "imports": { "type": "array", "minItems": 1, + "description": "Modules the runtime must be able to load.", "items": { "type": "string", "minLength": 1 } }, - "timeoutSeconds": { - "type": "integer", - "exclusiveMinimum": 0 + "commands": { + "type": "array", + "minItems": 1, + "description": "Invocations of the box's declared execution and the exit status each must produce.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "args", + "expectExitCode" + ], + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "expectExitCode": { + "type": "integer", + "minimum": 0, + "maximum": 255 + } + } + } } } }, - "execution": { - "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" - }, - "provenance": { - "$ref": "#/$defs/provenance" - }, - "weights": { - "const": "on-demand", - "description": "Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it." - }, - "assets": { + "deferredAssets": { "type": "array", "minItems": 1, - "description": "Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe.", + "description": "Assets the consumer must fetch and place under the box root before first use — the entries the scroll declared with embed false, and only those. A box whose assets are all embedded carries no such list. The declared size and hash are what make fetching them safe; a Scrollcase consumer verifies them and never downloads them itself.", "items": { "type": "object", "additionalProperties": false, @@ -215,15 +293,13 @@ }, "sha256": { "$ref": "#/$defs/sha256" + }, + "executable": { + "type": "boolean", + "description": "Present and true when the scroll declared the file executable. Whoever materializes it owns setting the bit: the file never passes through the archive, so nothing Scrollcase writes can carry a mode for it." } } } - } - }, - "$defs": { - "identifier": { - "type": "string", - "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" }, "kind": { "type": "string", @@ -243,7 +319,6 @@ "builderRevision", "sourceTreeDirty", "sourceRevision", - "pythonVersion", "dependencyLockSha256", "builtAt", "pixiVersion" @@ -269,11 +344,12 @@ "sourceRevision": { "type": "string", "minLength": 1, - "description": "Upstream revision of the packaged model source, as declared by the scroll." + "description": "Upstream revision of the packaged source, as declared by the scroll." }, - "pythonVersion": { + "runtimeVersion": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "The runtime version the environment was solved with, repeated from runtime.version. Absent exactly when the runtime has none: provenance records what was observed and never invents a value to fill a field." }, "pixiVersion": { "type": "string", @@ -289,13 +365,5 @@ } } } - }, - "dependentRequired": { - "assets": [ - "weights" - ], - "weights": [ - "assets" - ] } } diff --git a/src/contract/schema/revocations-manifest.schema.json b/src/contract/schema/revocations-manifest.schema.json index 67470c7..6de992b 100644 --- a/src/contract/schema/revocations-manifest.schema.json +++ b/src/contract/schema/revocations-manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/revocations-manifest.schema.json", + "$id": "https://scrollcase.dev/schema/v3/revocations-manifest.schema.json", "title": "Box revocations manifest", "description": "The signed list of releases that must no longer be installed or activated. A published release is immutable, so withdrawing one is an explicit statement rather than a deletion: clients keep honouring this list even when the archive is still reachable.", "type": "object", @@ -13,7 +13,7 @@ ], "properties": { "schemaVersion": { - "const": 2 + "const": 3 }, "kind": { "type": "string", @@ -46,7 +46,7 @@ "minLength": 1 }, "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json", + "$ref": "https://scrollcase.dev/schema/v3/target.schema.json", "description": "Omitted when every target of that version is revoked." }, "reason": { diff --git a/src/contract/schema/scroll.schema.json b/src/contract/schema/scroll.schema.json index b076c7d..57239f3 100644 --- a/src/contract/schema/scroll.schema.json +++ b/src/contract/schema/scroll.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/scroll.schema.json", + "$id": "https://scrollcase.dev/schema/v3/scroll.schema.json", "title": "Box scroll", "description": "The declarative input to a build: an identity, a target, a pinned dependency environment, the assets to fetch, and the self-test the result must pass. A scroll is checked into the consumer's repository next to its lock file; everything a build produces is derived from it.\n\nOnly what a build cannot work out for itself is required. Anything the target or the identity already determines is optional here and filled in when the scroll is read, so a hand-written scroll declares decisions rather than restating them.\n\nOne box's targets differ in a handful of lines and agree on the rest, so a scroll may also be split: scrolls//scroll.json holds what they share, and each scrolls///scroll.json declares `extends` plus its own differences. Both halves are files of this shape; the joined result is what a build reads.", "type": "object", @@ -8,28 +8,26 @@ "required": [ "schemaVersion", "boxId", - "modelId", - "runtimeId", "version", "sourceRevision", - "pythonVersion", + "runtime", "selfTest", "pixiVersion" ], "properties": { "$schema": { - "const": "https://scrollcase.dev/schema/v2/scroll.schema.json", - "description": "Associates this file with the published Scrollcase v2 schema for editor validation, completion, and hover help." + "const": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "description": "Associates this file with the published Scrollcase v3 schema for editor validation, completion, and hover help." }, "extends": { "const": "../scroll.json", "description": "Marks this file as one target's fragment of a box whose shared declarations live in scrolls//scroll.json. The value is fixed: a base is always the box directory's own scroll.json, so there is no path to get wrong and no chain to follow. The base and the fragment are joined into one effective scroll before anything else happens, and that effective scroll is what the build reads and what provenance records." }, "schemaVersion": { - "const": 2, - "description": "Scrollcase wire version. Version 2 is the only active format.", + "const": 3, + "description": "Scrollcase wire version. Version 3 is the only active format.", "examples": [ - 2 + 3 ] }, "scrollId": { @@ -49,11 +47,8 @@ "boxId": { "$ref": "#/$defs/identifier" }, - "modelId": { - "$ref": "#/$defs/identifier" - }, - "runtimeId": { - "$ref": "#/$defs/identifier" + "labels": { + "$ref": "#/$defs/labels" }, "version": { "type": "string", @@ -66,7 +61,7 @@ "description": "Upstream revision of the packaged source, recorded verbatim into provenance." }, "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json", + "$ref": "https://scrollcase.dev/schema/v3/target.schema.json", "description": "The (platform, arch, accelerator) triple this box is built for. Required in every scroll a build reads, and absent from a base: a base holds what its targets share, so declaring one there would name a target the box does not build. Enforced when the scroll is read rather than here, so a base file still validates in an editor." }, "compatibility": { @@ -97,13 +92,8 @@ }, "description": "Constraints the installing host must satisfy. Copied through into the release manifest verbatim and never interpreted by the builder, so a project may declare its own alongside these. Defaults to empty: declaring no constraint is a legitimate answer, and inventing one would be a claim the project never made." }, - "pythonVersion": { - "type": "string", - "minLength": 1, - "description": "Python version solved into the box.", - "examples": [ - "3.11.15" - ] + "runtime": { + "$ref": "#/$defs/runtime" }, "pixiVersion": { "type": "string", @@ -115,15 +105,10 @@ "minLength": 1, "description": "Path to the reviewed licence inventory derived from pixi.lock, which carries an SPDX licence per package. The build fails if the lock no longer matches what was reviewed." }, - "pythonEntryPoint": { + "cacheSubdir": { "type": "string", "minLength": 1, - "description": "Interpreter path relative to the box root. The target adapter's layout admits exactly one value, so this is derived from the target when omitted and still checked against it when declared." - }, - "modelCacheSubdir": { - "type": "string", - "minLength": 1, - "description": "Payload directory the box's model files live under. Defaults to model-cache/." + "description": "Payload directory the box's own large files live under — the destination a scroll's assets conventionally share. Defaults to cache/." }, "environment": { "type": "object", @@ -168,6 +153,16 @@ }, "sha256": { "$ref": "#/$defs/sha256" + }, + "embed": { + "type": "boolean", + "default": true, + "description": "Whether this file is packed into the archive. True, the default, makes the box self-contained: it installs with no network and works air-gapped. False leaves it out and carries its descriptor in the signed release instead, for the caller's distribution layer to materialize. The choice is per entry, so a box may ship a small entry point and defer a large dataset; consumers verify what was materialized before execution and never download it themselves." + }, + "executable": { + "type": "boolean", + "default": false, + "description": "Whether the file needs the executable bit. HTTP carries content and not permissions, so a downloaded file arrives with none; declaring it here is the only way a box can ship one that runs. The bit is synthesised into the archive from this declaration, never read off the build machine." } } } @@ -227,6 +222,11 @@ "sha256": { "$ref": "#/$defs/sha256", "description": "Optional pin. When present the build refuses a file whose contents no longer match." + }, + "executable": { + "type": "boolean", + "default": false, + "description": "Whether the file needs the executable bit. A copy does not carry the source file's mode, because a mode read off the build machine would vary with its umask and break the byte-identical rebuild; the bit is synthesised into the archive from this declaration instead." } } } @@ -240,7 +240,7 @@ }, "uncompressedPaths": { "type": "array", - "description": "Payload paths stored in the archive instead of deflated, because their bytes are already compressed and re-compressing them costs build time while making the archive marginally larger. A path matches itself and everything beneath it, so one entry can name a weights file or the directory an expanded asset archive landed in. Declared assets are stored automatically; this is for anything else the project knows to be already compressed.", + "description": "Payload paths stored in the archive instead of deflated, because their bytes are already compressed and re-compressing them costs build time while making the archive marginally larger. A path matches itself and everything beneath it, so one entry can name a single large file or the directory an expanded asset archive landed in. Declared assets are stored automatically; this is for anything else the project knows to be already compressed.", "items": { "$ref": "#/$defs/payloadPath" } @@ -248,25 +248,63 @@ "selfTest": { "type": "object", "additionalProperties": false, - "required": [ - "imports" + "anyOf": [ + { + "required": [ + "imports" + ] + }, + { + "required": [ + "commands" + ] + } ], "not": { "required": [ - "pythonCode", - "pythonFile" + "code", + "script" ] }, - "description": "Builder checks run with the payload's own interpreter before archiving. Schema version 2 signs only the import subset for a consumer to repeat; file and optional Python-code assertions remain builder-only.", + "description": "Builder checks run against the payload before archiving. The signed release carries the probe — the imports and commands a consumer can repeat after extraction — while file assertions and the optional extra source stay builder-only. At least one of imports and commands is required: a box that proves nothing about itself is not a box worth signing.", "properties": { "imports": { "type": "array", "minItems": 1, + "description": "Modules the runtime must be able to load. Meaningful to a runtime with a module system, which is why it is not the only shape a probe can take.", "items": { "type": "string", "minLength": 1 } }, + "commands": { + "type": "array", + "minItems": 1, + "description": "Invocations of the box's own declared execution, each with the exit code it must produce. This is the only probe shape available to a runtime with no module system, and it needs execution to be declared — there is nothing else to invoke.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "args" + ], + "properties": { + "args": { + "type": "array", + "description": "Arguments appended to the box's declared execution. Passed directly, without a shell. May be empty, which runs the entry point as the box would.", + "items": { + "type": "string" + } + }, + "expectExitCode": { + "type": "integer", + "minimum": 0, + "maximum": 255, + "default": 0, + "description": "Exit status the invocation must produce. Defaults to 0; a non-zero value suits a tool whose --version or --help deliberately exits otherwise." + } + } + } + }, "files": { "type": "array", "description": "Files that must still exist after pruning, which is what stops an over-aggressive prune from shipping a broken box. Defaults to empty.", @@ -274,28 +312,20 @@ "$ref": "#/$defs/payloadPath" } }, - "pythonCode": { + "code": { "type": "string", "minLength": 1, - "description": "Extra Python executed after the imports succeed, for checks a bare import cannot make. Anything longer than an assertion belongs in pythonFile, where an editor can see it is Python." + "description": "Extra source in the runtime's own language, executed after the imports succeed, for checks a bare import cannot make. Anything longer than an assertion belongs in script, where an editor can see what language it is." }, - "pythonFile": { + "script": { "type": "string", "minLength": 1, - "description": "Project path to a Python file executed after the imports succeed, in place of pythonCode. The file is read at build time and run from the payload root, so a real self-test keeps its syntax highlighting, its linter, and its diffs instead of living inside a JSON string." + "description": "Project path to a source file executed after the imports succeed, in place of code. The file is read at build time and run from the payload root, so a real self-test keeps its syntax highlighting, its linter, and its diffs instead of living inside a JSON string." } } }, - "weights": { - "enum": [ - "embed", - "on-demand" - ], - "default": "embed", - "description": "Whether assets are packed into the archive (embed, the default: the box installs with no network and works air-gapped) or left out for the caller to materialize from descriptors in the signed release (on-demand). Consumers verify materialized assets before execution and do not download them. A build may override this." - }, "execution": { - "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" + "$ref": "https://scrollcase.dev/schema/v3/execution.schema.json" }, "parity": { "type": "object", @@ -352,6 +382,57 @@ "type": "string", "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" }, + "runtime": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "description": "What runs inside the box. A target says which machine the box is for; this says what executes on it — which is a different question, and until version 3 the format never asked it: a box declared a Python interpreter path and Python execution kinds and nothing that said \"Python\".", + "properties": { + "id": { + "enum": [ + "python", + "node", + "native" + ], + "description": "The runtime the box carries. The list is closed rather than free-form: each id implies a payload layout, a set of execution kinds and an argv rule that a consumer has to already know, so an unrecognised one is a box that cannot be run, not a box with an unusual label." + }, + "version": { + "type": "string", + "minLength": 1, + "description": "The runtime's own version, solved into the box and recorded in provenance. Required by any runtime whose layout depends on it — Python names its standard library after major.minor — and legitimately absent for one that has no interpreter to version, which is why the format does not demand it.", + "examples": [ + "3.11.15" + ] + }, + "entryPoint": { + "type": "string", + "minLength": 1, + "description": "The runtime's own executable, relative to the box root. The runtime's layout for a given target admits exactly one value, so this is derived when omitted and still checked against the layout when declared. Absent for a runtime that has no separate executable to name.", + "examples": [ + "venv/bin/python" + ] + } + } + }, + "labels": { + "type": "object", + "description": "Free-form annotations carried through into the signed release untouched. Scrollcase never reads a label; it exists so a project can record what it needs to record — the upstream model a box packages, the team that owns it, the ticket it came from — without the format having to grow a field, and without the format claiming to know what any project's boxes are about.", + "propertyNames": { + "$ref": "#/$defs/identifier" + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + }, + "examples": [ + { + "model": "example-org/example-model", + "owner": "platform-team" + } + ] + }, "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" @@ -362,7 +443,7 @@ "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*//).+$", "description": "A forward-slash path inside the box payload: relative, non-empty, and unable to escape the payload root.", "examples": [ - "model-cache/example-model/weights.safetensors" + "cache/example-box/data.bin" ] } } diff --git a/src/contract/schema/signed-document.schema.json b/src/contract/schema/signed-document.schema.json index 602af0c..ad06de0 100644 --- a/src/contract/schema/signed-document.schema.json +++ b/src/contract/schema/signed-document.schema.json @@ -1,13 +1,13 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/signed-document.schema.json", + "$id": "https://scrollcase.dev/schema/v3/signed-document.schema.json", "title": "Signed box document", "description": "The envelope wrapping every signed document. The payload travels as exact base64-encoded JSON so that verifying a signature means hashing the bytes as transmitted, with no canonical-JSON implementation to keep in sync across languages. Passing this schema means the envelope is well-formed, never that its signature is valid.", "type": "object", "additionalProperties": false, "required": ["schemaVersion", "payloadEncoding", "payloadBase64", "payloadSha256", "signatures"], "properties": { - "schemaVersion": { "const": 2 }, + "schemaVersion": { "const": 3 }, "payloadEncoding": { "const": "base64-json-utf8" }, "payloadBase64": { "type": "string", diff --git a/src/contract/schema/target.schema.json b/src/contract/schema/target.schema.json index 6894c12..1573b29 100644 --- a/src/contract/schema/target.schema.json +++ b/src/contract/schema/target.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/target.schema.json", + "$id": "https://scrollcase.dev/schema/v3/target.schema.json", "title": "Box target", "description": "The platform, architecture and accelerator a box is built for. The supported combinations are closed: a target outside this matrix has no defined identifier and cannot be built, signed, or routed.", "type": "object", diff --git a/src/contract/targets.d.mts b/src/contract/targets.d.mts index dc5be64..6827a8b 100644 --- a/src/contract/targets.d.mts +++ b/src/contract/targets.d.mts @@ -27,19 +27,6 @@ export function assertNativeHost(adapter: BoxTargetAdapter, host?: { platform: string; arch: string; }): void; -/** - * Ensures the scroll entry point agrees with the standalone Python layout for this target. - * - * Kept under its published name while the wire format still spells the field `pythonEntryPoint`. - * The rule itself moved to `runtimes.mjs`, where it can be asked about any runtime; this is the one - * public spelling of it, and it goes when the field does. - * - * @param {BoxTargetAdapter} adapter - * @param {string} entryPoint - * @returns {void} - * @throws {TypeError} when the entry point does not match the runtime's layout for this target - */ -export function assertPythonEntryPoint(adapter: BoxTargetAdapter, entryPoint: string): void; /** * Lists every adapter, for contract tests and for consumers enumerating supported targets. * diff --git a/src/contract/targets.mjs b/src/contract/targets.mjs index 27fa00b..19411dd 100644 --- a/src/contract/targets.mjs +++ b/src/contract/targets.mjs @@ -40,8 +40,6 @@ * `executionAffectingVariables()` in `runtimes.mjs` is what joins the two halves */ -import { IMPLICIT_RUNTIME_ID, assertRuntimeEntryPoint } from './runtimes.mjs'; - const TARGET_ACCELERATORS = { macos: { aarch64: ['metal', 'cpu'] }, linux: { x86_64: ['cpu', 'cuda'] }, @@ -183,22 +181,6 @@ export function assertNativeHost(adapter, host = process) { } } -/** - * Ensures the scroll entry point agrees with the standalone Python layout for this target. - * - * Kept under its published name while the wire format still spells the field `pythonEntryPoint`. - * The rule itself moved to `runtimes.mjs`, where it can be asked about any runtime; this is the one - * public spelling of it, and it goes when the field does. - * - * @param {BoxTargetAdapter} adapter - * @param {string} entryPoint - * @returns {void} - * @throws {TypeError} when the entry point does not match the runtime's layout for this target - */ -export function assertPythonEntryPoint(adapter, entryPoint) { - assertRuntimeEntryPoint(IMPLICIT_RUNTIME_ID, adapter, entryPoint); -} - /** * Lists every adapter, for contract tests and for consumers enumerating supported targets. * diff --git a/src/contract/types/index.d.ts b/src/contract/types/index.d.ts index fe1c26c..5d57579 100644 --- a/src/contract/types/index.d.ts +++ b/src/contract/types/index.d.ts @@ -29,8 +29,10 @@ export type BoxTarget = { /** * The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only. + * + * Each kind is named -, and the runtime half must be the one the box declares: a python-script in a box whose runtime is native describes something that cannot be run, and is refused rather than guessed at. */ -export type BoxExecution = PythonScript | PythonModule; +export type BoxExecution = PythonScript | PythonModule | NodeScript | NativeBinary; /** * Arguments placed before caller-supplied arguments. Every item is passed directly without a shell. */ @@ -64,6 +66,34 @@ export interface PythonModule { module: string; defaultArgs: DefaultArgs; } +/** + * Run one regular payload file with the box's own Node runtime. + */ +export interface NodeScript { + /** + * Selects direct script execution. + */ + kind: 'node-script'; + /** + * Safe path to a regular JavaScript file inside the box. + */ + script: string; + defaultArgs: DefaultArgs; +} +/** + * Run a compiled executable that the box carries directly, with no interpreter in front of it. The only shape a runtime with no module system has. + */ +export interface NativeBinary { + /** + * Selects direct execution of a payload file. + */ + kind: 'native-binary'; + /** + * Safe path to the executable inside the box. It carries the executable bit because the scroll declared it, not because the build machine happened to have it set. + */ + binary: string; + defaultArgs: DefaultArgs; +} export type Identifier = string; /** @@ -73,20 +103,22 @@ export type PayloadPath = string; export type Sha256 = string; /** * The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only. + * + * Each kind is named -, and the runtime half must be the one the box declares: a python-script in a box whose runtime is native describes something that cannot be run, and is refused rather than guessed at. */ export interface BoxScroll { /** - * Associates this file with the published Scrollcase v2 schema for editor validation, completion, and hover help. + * Associates this file with the published Scrollcase v3 schema for editor validation, completion, and hover help. */ - $schema?: 'https://scrollcase.dev/schema/v2/scroll.schema.json'; + $schema?: 'https://scrollcase.dev/schema/v3/scroll.schema.json'; /** * Marks this file as one target's fragment of a box whose shared declarations live in scrolls//scroll.json. The value is fixed: a base is always the box directory's own scroll.json, so there is no path to get wrong and no chain to follow. The base and the fragment are joined into one effective scroll before anything else happens, and that effective scroll is what the build reads and what provenance records. */ extends?: '../scroll.json'; /** - * Scrollcase wire version. Version 2 is the only active format. + * Scrollcase wire version. Version 3 is the only active format. */ - schemaVersion: 2; + schemaVersion: 3; /** * Optional provenance identity. When omitted, Scrollcase derives it deterministically from boxId and the canonical target. */ @@ -96,8 +128,7 @@ export interface BoxScroll { */ scrollVersion?: string; boxId: Identifier; - modelId: Identifier; - runtimeId: Identifier; + labels?: Labels; /** * Version of the box this scroll produces, as it will appear in the release manifest. */ @@ -121,10 +152,7 @@ export interface BoxScroll { minNvidiaDriverVersion?: string; [k: string]: unknown; }; - /** - * Python version solved into the box. - */ - pythonVersion: string; + runtime: Runtime; /** * Pins the pixi release used to solve and install the conda-forge environment from the committed pixi.lock. */ @@ -134,13 +162,9 @@ export interface BoxScroll { */ condaDependencyLicenseAudit?: string; /** - * Interpreter path relative to the box root. The target adapter's layout admits exactly one value, so this is derived from the target when omitted and still checked against it when declared. - */ - pythonEntryPoint?: string; - /** - * Payload directory the box's model files live under. Defaults to model-cache/. + * Payload directory the box's own large files live under — the destination a scroll's assets conventionally share. Defaults to cache/. */ - modelCacheSubdir?: string; + cacheSubdir?: string; /** * Environment variables the box requires when its interpreter runs. The declaration is copied into box.json and the signed release; its values override both the inherited host environment and caller-supplied values. */ @@ -159,6 +183,14 @@ export interface BoxScroll { relativePath: PayloadPath; sizeBytes: number; sha256: Sha256; + /** + * Whether this file is packed into the archive. True, the default, makes the box self-contained: it installs with no network and works air-gapped. False leaves it out and carries its descriptor in the signed release instead, for the caller's distribution layer to materialize. The choice is per entry, so a box may ship a small entry point and defer a large dataset; consumers verify what was materialized before execution and never download it themselves. + */ + embed?: boolean; + /** + * Whether the file needs the executable bit. HTTP carries content and not permissions, so a downloaded file arrives with none; declaring it here is the only way a box can ship one that runs. The bit is synthesised into the archive from this declaration, never read off the build machine. + */ + executable?: boolean; }[]; /** * Downloaded archives to expand into the payload. Extraction preserves files already present in the destination and refuses to overwrite them. @@ -180,40 +212,25 @@ export interface BoxScroll { * Optional pin. When present the build refuses a file whose contents no longer match. */ sha256?: string; + /** + * Whether the file needs the executable bit. A copy does not carry the source file's mode, because a mode read off the build machine would vary with its umask and break the byte-identical rebuild; the bit is synthesised into the archive from this declaration instead. + */ + executable?: boolean; }[]; /** * Payload paths deleted before packing, to keep the box to what it actually needs at run time. Pruning a distribution the lock requires is rejected. */ prunePaths?: PayloadPath[]; /** - * Payload paths stored in the archive instead of deflated, because their bytes are already compressed and re-compressing them costs build time while making the archive marginally larger. A path matches itself and everything beneath it, so one entry can name a weights file or the directory an expanded asset archive landed in. Declared assets are stored automatically; this is for anything else the project knows to be already compressed. + * Payload paths stored in the archive instead of deflated, because their bytes are already compressed and re-compressing them costs build time while making the archive marginally larger. A path matches itself and everything beneath it, so one entry can name a single large file or the directory an expanded asset archive landed in. Declared assets are stored automatically; this is for anything else the project knows to be already compressed. */ uncompressedPaths?: PayloadPath[]; /** - * Builder checks run with the payload's own interpreter before archiving. Schema version 2 signs only the import subset for a consumer to repeat; file and optional Python-code assertions remain builder-only. + * Builder checks run against the payload before archiving. The signed release carries the probe — the imports and commands a consumer can repeat after extraction — while file assertions and the optional extra source stay builder-only. At least one of imports and commands is required: a box that proves nothing about itself is not a box worth signing. */ selfTest: { - /** - * @minItems 1 - */ - imports: [string, ...string[]]; - /** - * Files that must still exist after pruning, which is what stops an over-aggressive prune from shipping a broken box. Defaults to empty. - */ - files?: PayloadPath[]; - /** - * Extra Python executed after the imports succeed, for checks a bare import cannot make. Anything longer than an assertion belongs in pythonFile, where an editor can see it is Python. - */ - pythonCode?: string; - /** - * Project path to a Python file executed after the imports succeed, in place of pythonCode. The file is read at build time and run from the payload root, so a real self-test keeps its syntax highlighting, its linter, and its diffs instead of living inside a JSON string. - */ - pythonFile?: string; + [k: string]: unknown; }; - /** - * Whether assets are packed into the archive (embed, the default: the box installs with no network and works air-gapped) or left out for the caller to materialize from descriptors in the signed release (on-demand). Consumers verify materialized assets before execution and do not download them. A build may override this. - */ - weights?: 'embed' | 'on-demand'; execution?: BoxExecution; /** * An optional numerical gate: run a check inside the box on more than one accelerator and require the results to agree. This catches a mis-solved environment — CPU-only wheels shipped as CUDA, a broken BLAS — on the build machine rather than on a user's. The tool runs the check and enforces the thresholds; what the check computes, and what closeness is acceptable, belong to the project. @@ -239,54 +256,107 @@ export interface BoxScroll { }; }; } +/** + * Free-form annotations carried through into the signed release untouched. Scrollcase never reads a label; it exists so a project can record what it needs to record — the upstream model a box packages, the team that owns it, the ticket it came from — without the format having to grow a field, and without the format claiming to know what any project's boxes are about. + */ +export interface Labels { + [k: string]: string; +} +/** + * What runs inside the box. A target says which machine the box is for; this says what executes on it — which is a different question, and until version 3 the format never asked it: a box declared a Python interpreter path and Python execution kinds and nothing that said "Python". + */ +export interface Runtime { + /** + * The runtime the box carries. The list is closed rather than free-form: each id implies a payload layout, a set of execution kinds and an argv rule that a consumer has to already know, so an unrecognised one is a box that cannot be run, not a box with an unusual label. + */ + id: 'python' | 'node' | 'native'; + /** + * The runtime's own version, solved into the box and recorded in provenance. Required by any runtime whose layout depends on it — Python names its standard library after major.minor — and legitimately absent for one that has no interpreter to version, which is why the format does not demand it. + */ + version?: string; + /** + * The runtime's own executable, relative to the box root. The runtime's layout for a given target admits exactly one value, so this is derived when omitted and still checked against the layout when declared. Absent for a runtime that has no separate executable to name. + */ + entryPoint?: string; +} /** * Run one regular payload file with the box's own Python interpreter. */ +export type DeferredAssets = [ + { + url: string; + relativePath: string; + sizeBytes: number; + sha256: string; + /** + * Present and true when the scroll declared the file executable. Whoever materializes it owns setting the bit: the file never passes through the archive, so nothing Scrollcase writes can carry a mode for it. + */ + executable?: boolean; + }, + ...{ + url: string; + relativePath: string; + sizeBytes: number; + sha256: string; + /** + * Present and true when the scroll declared the file executable. Whoever materializes it owns setting the bit: the file never passes through the archive, so nothing Scrollcase writes can carry a mode for it. + */ + executable?: boolean; + }[] +]; + +/** + * The manifest packed inside the archive at box.json. It restates the box's identity, layout and provenance so an extracted box is self-describing: a consumer that has the directory but not the release document can still tell what it is holding and how it was built. + */ export interface BoxManifest { - schemaVersion: 2; + schemaVersion: 3; boxId: string; - modelId: string; - runtimeId: string; + labels?: Labels; version: string; target: BoxTarget; - pythonEntryPoint: string; - modelCacheSubdir: string; + runtime: Runtime; + cacheSubdir: string; /** * Environment variables repeated from the signed release. Scrollcase consumers compare this declaration before execution and apply it over inherited and caller-supplied values. */ environment?: { [k: string]: string; }; - selfTest: { - /** - * @minItems 1 - */ - pythonImports: [string, ...string[]]; - timeoutSeconds: number; - }; + selfTest: SelfTest; execution?: BoxExecution; provenance: Provenance; + assets?: DeferredAssets; +} +/** + * Free-form annotations the publishing project declared, signed and carried through untouched. Scrollcase attaches no meaning to any key; a consumer that reads one is reading its own project's convention, not the box format. + */ +export interface SelfTest { + probe: SelfTestProbe; + timeoutSeconds: number; +} +/** + * What the box proves about itself, in whichever shapes its runtime supports. The runtime turns this into command lines; nothing here is a command line, and nothing here is source in any language. + */ +export interface SelfTestProbe { /** - * Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it. + * Modules the runtime must be able to load. + * + * @minItems 1 */ - weights?: 'on-demand'; + imports?: [string, ...string[]]; /** - * Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe. + * Invocations of the box's declared execution and the exit status each must produce. * * @minItems 1 */ - assets?: [ + commands?: [ { - url: string; - relativePath: string; - sizeBytes: number; - sha256: string; + args: string[]; + expectExitCode: number; }, ...{ - url: string; - relativePath: string; - sizeBytes: number; - sha256: string; + args: string[]; + expectExitCode: number; }[] ]; } @@ -305,10 +375,13 @@ export interface Provenance { */ sourceTreeDirty: boolean; /** - * Upstream revision of the packaged model source, as declared by the scroll. + * Upstream revision of the packaged source, as declared by the scroll. */ sourceRevision: string; - pythonVersion: string; + /** + * The runtime version the environment was solved with, repeated from runtime.version. Absent exactly when the runtime has none: provenance records what was observed and never invents a value to fill a field. + */ + runtimeVersion?: string; pixiVersion: string; /** * Hash of the pixi.lock the environment was solved from. @@ -318,14 +391,13 @@ export interface Provenance { } export interface BoxReleaseManifest { - schemaVersion: 2; + schemaVersion: 3; /** * Wire discriminator, ".release". The namespace belongs to the publishing project — a project with boxes already in the field must keep emitting the one its clients recognise — and defaults to scrollcase.box for a new one. */ kind: string; boxId: Identifier; - modelId: Identifier; - runtimeId: Identifier; + labels?: Labels; version: string; target: BoxTarget; /** @@ -368,61 +440,27 @@ export interface BoxReleaseManifest { format: 'sha256-path-list-v1'; sha256: Sha256; }; + runtime: Runtime; /** - * Interpreter path relative to the extracted box root, for example venv/bin/python. Fixed per target by the adapter. + * Directory relative to the extracted box root holding the box's own large files. */ - pythonEntryPoint: string; - /** - * Directory relative to the extracted box root holding model assets. - */ - modelCacheSubdir: string; + cacheSubdir: string; /** * Signed environment variables applied whenever Scrollcase runs the box interpreter. These values override both the inherited host environment and caller-supplied values. */ environment?: { [k: string]: string; }; - /** - * The import check a consumer can repeat after extraction with the box's own interpreter. The builder also ran the scroll's Python-code and file assertions, which are builder-only checks. - */ - selfTest: { - /** - * @minItems 1 - */ - pythonImports: [string, ...string[]]; - timeoutSeconds: number; - }; + selfTest: SelfTest; execution?: BoxExecution; provenance: Provenance; - /** - * Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it. - */ - weights?: 'on-demand'; - /** - * Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe. - * - * @minItems 1 - */ - assets?: [ - { - url: string; - relativePath: string; - sizeBytes: number; - sha256: Sha256; - }, - ...{ - url: string; - relativePath: string; - sizeBytes: number; - sha256: Sha256; - }[] - ]; + assets?: DeferredAssets; } /** - * Run one regular payload file with the box's own Python interpreter. + * Free-form annotations the publishing project declared, signed and carried through untouched. Scrollcase attaches no meaning to any key; a consumer that reads one is reading its own project's convention, not the box format. */ export interface BoxChannelManifest { - schemaVersion: 2; + schemaVersion: 3; /** * Wire discriminator, ".channel", carrying the same namespace as the releases it refers to. */ @@ -458,7 +496,7 @@ export interface BoxChannelManifest { * Omitted when every target of that version is revoked. */ export interface BoxRevocationsManifest { - schemaVersion: 2; + schemaVersion: 3; /** * Wire discriminator, ".revocations", carrying the same namespace as the releases it refers to. */ @@ -480,7 +518,7 @@ export interface BoxRevocationsManifest { * The envelope wrapping every signed document. The payload travels as exact base64-encoded JSON so that verifying a signature means hashing the bytes as transmitted, with no canonical-JSON implementation to keep in sync across languages. Passing this schema means the envelope is well-formed, never that its signature is valid. */ export interface SignedBoxDocument { - schemaVersion: 2; + schemaVersion: 3; payloadEncoding: 'base64-json-utf8'; /** * The document payload: UTF-8 JSON, base64-encoded, signed and hashed exactly as it appears here. diff --git a/tests/helpers/consumer-box-fixture.mjs b/tests/helpers/consumer-box-fixture.mjs index 67269ad..edd0b70 100644 --- a/tests/helpers/consumer-box-fixture.mjs +++ b/tests/helpers/consumer-box-fixture.mjs @@ -14,9 +14,9 @@ import { PAYLOAD_DIGEST_FORMAT, payloadDigestStream, } from '../../src/contract/payload-digest.mjs'; -import { documentKinds } from '../../src/contract/documents.mjs'; +import { BOX_SCHEMA_VERSION, documentKinds } from '../../src/contract/documents.mjs'; import { boxTargetAdapter } from '../../src/contract/targets.mjs'; -import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../../src/contract/runtimes.mjs'; +import { runtimeAdapter } from '../../src/contract/runtimes.mjs'; import { generateSigningKey, signDocument } from '../../src/sign/index.mjs'; export function nativeTarget() { @@ -71,12 +71,15 @@ export async function createConsumerBoxFixture({ scriptContents = 'print("consumer fixture")\n', payloadDigest = true, environment = undefined, + labels = undefined, + runtimeId = 'python', + executablePaths = [], } = {}) { const root = await mkdtemp(join(tmpdir(), 'scrollcase-consumer-fixture-')); const payload = join(root, 'payload'); await mkdir(payload); const adapter = boxTargetAdapter(target); - const layout = runtimeAdapter(IMPLICIT_RUNTIME_ID).layout(adapter); + const layout = runtimeAdapter(runtimeId).layout(adapter); const pythonPath = join(payload, ...layout.entryPoint.split('/')); await mkdir(dirname(pythonPath), { recursive: true }); await writeFile(pythonPath, interpreterContents); @@ -92,16 +95,15 @@ export async function createConsumerBoxFixture({ } const shared = { - schemaVersion: 2, + schemaVersion: BOX_SCHEMA_VERSION, boxId: 'consumer-fixture', - modelId: 'example-consumer-model', - runtimeId: 'example-consumer-runtime', + ...(labels === undefined ? {} : { labels }), version: '2.0.0', target, - pythonEntryPoint: layout.entryPoint, - modelCacheSubdir: 'model-cache/consumer-fixture', + runtime: { id: runtimeId, version: '3.11.15', entryPoint: layout.entryPoint }, + cacheSubdir: 'cache/consumer-fixture', selfTest: { - pythonImports: ['json'], + probe: { imports: ['json'] }, timeoutSeconds: 30, }, ...(environment === undefined ? {} : { environment }), @@ -112,15 +114,12 @@ export async function createConsumerBoxFixture({ builderRevision: '0123456789abcdef0123456789abcdef01234567', sourceTreeDirty: false, sourceRevision: 'fedcba9876543210', - pythonVersion: '3.11.15', + runtimeVersion: '3.11.15', pixiVersion: '0.73.0', dependencyLockSha256: 'a'.repeat(64), builtAt: '2026-07-29T00:00:00.000Z', }, - ...(requiredAsset ? { - weights: 'on-demand', - assets: [requiredAsset], - } : {}), + ...(requiredAsset ? { assets: [requiredAsset] } : {}), }; await writeFile(join(payload, 'box.json'), `${JSON.stringify(shared, null, 2)}\n`); // Written last and never listed in itself, exactly as the build does it — a fixture that skipped @@ -136,7 +135,7 @@ export async function createConsumerBoxFixture({ } const installedSizeBytes = await payloadSize(payload); const archivePath = join(root, 'box.zip'); - await createDeterministicZip(payload, archivePath, adapter); + await createDeterministicZip(payload, archivePath, adapter, { runtimeId, executablePaths }); const archiveMetadata = await stat(archivePath); const release = { ...shared, diff --git a/tests/unit/build-pipeline.test.mjs b/tests/unit/build-pipeline.test.mjs index 64c7fd4..5b3d8f9 100644 --- a/tests/unit/build-pipeline.test.mjs +++ b/tests/unit/build-pipeline.test.mjs @@ -22,7 +22,7 @@ import { assertBoxManifestAgreement, verifyBox } from '../../src/build/verify.mj import { configureWorkspace, resetWorkspace } from '../../src/build/workspace.mjs'; import { generateSigningKey, signDocument } from '../../src/sign/index.mjs'; import { boxTargetAdapters, boxTargetId, decodeDocumentPayload, documentKinds } from '../../src/contract/index.mjs'; -import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../../src/contract/runtimes.mjs'; +import { runtimeAdapter } from '../../src/contract/runtimes.mjs'; // The pipeline is the same on every platform, but the native-host gate (rightly) refuses to build // a box for any other one — so the test scroll targets whatever host the suite is running on. @@ -30,23 +30,23 @@ import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../../src/contract/runtimes const HOST_ADAPTER = boxTargetAdapters().find((adapter) => adapter.host.platform === process.platform && adapter.host.arch === process.arch) ?? (() => { throw new Error(`No box target adapter for this host: ${process.platform}/${process.arch}`); })(); -const HOST_LAYOUT = runtimeAdapter(IMPLICIT_RUNTIME_ID).layout(HOST_ADAPTER); +const RUNTIME_ID = 'python'; +const HOST_LAYOUT = runtimeAdapter(RUNTIME_ID).layout(HOST_ADAPTER); +const PYTHON_VERSION = '3.11.15'; const SCROLL = { - schemaVersion: 2, + schemaVersion: 3, scrollId: 'example-model-native-cpu', scrollVersion: '1.0.0', boxId: 'example-model', - modelId: 'example-org-example-model', - runtimeId: 'example-model-runtime', + labels: { model: 'example-org/example-model' }, version: '1.0.0', sourceRevision: 'a'.repeat(40), target: { platform: HOST_ADAPTER.platform, arch: HOST_ADAPTER.arch, accelerator: 'cpu' }, compatibility: { minHostAppVersion: '1.0.0' }, - pythonVersion: '3.11.15', + runtime: { id: RUNTIME_ID, version: PYTHON_VERSION, entryPoint: HOST_LAYOUT.entryPoint }, pixiVersion: '0.73.0', - pythonEntryPoint: HOST_LAYOUT.entryPoint, - modelCacheSubdir: 'model-cache/example-model', + cacheSubdir: 'cache/example-model', assetBaseUrl: 'https://assets.example.org/boxes', assets: [], selfTest: { imports: ['json'], files: [] }, @@ -177,7 +177,7 @@ function fakeToolchain(payloadDir, { module = null, onSelfTest = null, consoleSc const modulePath = module.split('.'); const sitePackages = HOST_ADAPTER.platform === 'windows' ? ['Lib', 'site-packages'] - : ['lib', `python${SCROLL.pythonVersion.split('.').slice(0, 2).join('.')}`, 'site-packages']; + : ['lib', `python${PYTHON_VERSION.split('.').slice(0, 2).join('.')}`, 'site-packages']; writeDeep(join(prefix, ...sitePackages, ...modulePath.slice(0, -1), `${modulePath.at(-1)}.py`), 'print("module ready")\n'); } @@ -324,10 +324,37 @@ describe('the build pipeline', () => { resetWorkspace(); // An entry point belonging to any *other* target must be refused on this one. const foreignEntryPoint = HOST_ADAPTER.platform === 'windows' ? 'venv/bin/python' : 'venv/python.exe'; - await makeProject({ ...SCROLL, pythonEntryPoint: foreignEntryPoint }, { commit: false }); + await makeProject( + { ...SCROLL, runtime: { ...SCROLL.runtime, entryPoint: foreignEntryPoint } }, + { commit: false }, + ); await expect(readScroll(SCROLL_REF)).rejects.toThrow(/entry point/); }); + it('refuses a runtime the format defines but this build cannot produce', async () => { + // The wire vocabulary is deliberately wider than the implemented set, so this is an expected + // refusal with a message that says which of the two the scroll fell foul of. + await makeProject({ ...SCROLL, runtime: { id: 'native' } }, { commit: false }); + await expect(readScroll(SCROLL_REF)).rejects.toThrow(/native is not implemented/); + }); + + it('refuses an execution kind belonging to another runtime', async () => { + await makeProject({ + ...SCROLL, + execution: { kind: 'native-binary', binary: 'bin/tool', defaultArgs: [] }, + }, { commit: false }); + await expect(readScroll(SCROLL_REF)) + .rejects.toThrow(/native-binary does not belong to the python runtime/); + }); + + it('refuses a command probe with no execution to invoke', async () => { + await makeProject({ + ...SCROLL, + selfTest: { commands: [{ args: ['--version'] }], files: [] }, + }, { commit: false }); + await expect(readScroll(SCROLL_REF)).rejects.toThrow(/does not declare/); + }); + it.each([ ['an identity the release schema cannot carry', { ...SCROLL, boxId: 'Example Model' }], ['an invalid nested asset field', { @@ -558,14 +585,17 @@ describe('the build pipeline', () => { expect(calls).toEqual([]); }); - it('rejects an on-demand archive before probing tools, fetching, or mutating the build tree', async () => { + it('gives an asset archive no way to be deferred, before touching anything', async () => { + // An archive is expanded at build time, so "leave it out and let the caller fetch it" names + // nothing that could happen. Version 2 refused the combination with a cross-field check in two + // places; version 3 gives the entry no `embed` field at all, and the schema settles it. const scroll = { ...SCROLL, - weights: 'on-demand', assetArchives: [{ - relativePath: 'model-cache/weights.zip', + relativePath: 'cache/data.zip', format: 'zip', - destination: 'model-cache', + destination: 'cache', + embed: false, }], }; const { keys, payloadDir } = await makeProject(scroll); @@ -582,7 +612,7 @@ describe('the build pipeline', () => { throw new Error('unexpected fetch'); }, log: () => {}, - })).rejects.toThrow(/on-demand weights cannot be combined with assetArchives/); + })).rejects.toThrow(/Invalid scroll/); expect(calls).toEqual([]); expect(await fileExists(payloadDir)).toBe(false); }); @@ -654,17 +684,19 @@ describe('the build pipeline', () => { expect(result.environmentReport.releaseVariableCount).toBe(2); }); - it('rejects a signed v1 release payload even inside a valid v2 envelope', async () => { + it.each([1, 2])('rejects a signed v%i release payload, by version', async (schemaVersion) => { + // Both superseded versions are named rather than lumped together: they are different artefacts + // with different rebuilds ahead of them, and the reader holding one is entitled to know which. const { root, keys } = await makeProject(); - const releasePath = join(root, 'v1.release.json'); + const releasePath = join(root, `v${schemaVersion}.release.json`); const signed = await signDocument({ - schemaVersion: 1, + schemaVersion, kind: documentKinds().release, }, keys); await writeFile(releasePath, `${JSON.stringify(signed, null, 2)}\n`); await expect(verifyBox(releasePath, { publicPath: keys.publicPath, log: () => {} })) - .rejects.toThrow('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); + .rejects.toThrow(`Unsupported schemaVersion ${schemaVersion}; rebuild this box with Scrollcase v3.`); }); it('does not fall back to the pre-v2 stem-based archive name', async () => { @@ -787,6 +819,67 @@ describe('the build pipeline', () => { expect(receipt.status).toBe('passed'); }); + it('synthesises the executable bit for a declared file, and keeps it under a strict umask', async () => { + // HTTP carries content and not permissions, and a local file is copied rather than moved, so + // neither arrives with a mode to inherit. The scroll is the only place either can say it needs + // one, and the bit is put into the archive from that declaration — never read off the machine, + // which would make two builds of one commit differ by the umask each ran under. + const tool = { + url: 'https://assets.example.org/tool', + relativePath: 'bin/tool', + sizeBytes: 17, + sha256: createHash('sha256').update('#!/bin/sh\nexit 0\n').digest('hex'), + executable: true, + }; + const scroll = { + ...SCROLL, + assets: [tool], + localFiles: [ + { sourcePath: 'scripts/launch.sh', relativePath: 'bin/launch.sh', executable: true }, + { sourcePath: 'legal/NOTICE.txt', relativePath: 'NOTICE.txt' }, + ], + selfTest: { imports: ['json'], files: ['bin/tool', 'bin/launch.sh'] }, + }; + const { keys, payloadDir } = await makeProject(scroll, { + projectFiles: { + 'scripts/launch.sh': '#!/bin/sh\nexec ./bin/tool "$@"\n', + 'legal/NOTICE.txt': 'notices\n', + }, + }); + const built = await buildBox(SCROLL_REF, { + ...keys, + ...fakeToolchain(payloadDir), + fetchImpl: async () => new Response( + Readable.toWeb(Readable.from([Buffer.from('#!/bin/sh\nexit 0\n')])), + { status: 200 }, + ), + log: () => {}, + }); + + const executable = process.platform === 'win32' ? 0o100644 : 0o100755; + const modes = await zipModes(built.archivePath); + expect(modes.get('bin/tool')).toBe(executable); + expect(modes.get('bin/launch.sh')).toBe(executable); + // Undeclared neighbours stay 0644: the rule is a declaration, not a directory. + expect(modes.get('NOTICE.txt')).toBe(0o100644); + + // And extraction must not hand the bit back to the umask. `open(2)` masks the mode it is given, + // so a box unpacked under 077 would silently lose it and fail to run for reasons nothing in the + // box explains. + if (process.platform !== 'win32') { + const previous = process.umask(0o077); + try { + const extracted = await mkdtemp(join(tmpdir(), 'scrollcase-umask-')); + created.push(extracted); + await extractZipArchive(built.archivePath, extracted); + expect((await lstat(join(extracted, 'bin', 'tool'))).mode & 0o777).toBe(0o755); + expect((await lstat(join(extracted, 'NOTICE.txt'))).mode & 0o777).toBe(0o644); + } finally { + process.umask(previous); + } + } + }); + it('synthesises the executable bit from the runtime layout, and repairs the launcher', async () => { // Mode is not read off the build machine — a payload assembled under any umask has to archive // identically, and `payload-digest.v1` deliberately excludes mode, so the archive is the only @@ -1010,11 +1103,11 @@ describe('the build pipeline', () => { .rejects.toThrow(/git checkout/); }); - it('runs the self-test Python a scroll keeps in a file', async () => { + it('runs the extra self-test source a scroll keeps in a file', async () => { const source = 'assert 2 + 2 == 4, "arithmetic is broken"\n'; const scroll = { ...SCROLL, - selfTest: { imports: ['json'], files: [], pythonFile: 'checks/self_test.py' }, + selfTest: { imports: ['json'], files: [], script: 'checks/self_test.py' }, }; const { keys, payloadDir } = await makeProject(scroll, { projectFiles: { 'checks/self_test.py': source }, @@ -1032,17 +1125,17 @@ describe('the build pipeline', () => { }); it('runs the self-test against a payload that already contains box.json', async () => { - // An application finds its own files by reading the modelCacheSubdir its box declares, rather + // An application finds its own files by reading the cacheSubdir its box declares, rather // than hard-coding a path the scroll would then have to be bent to match. That only works if // box.json is there when the self-test runs: writing it afterwards meant the check ran against // a payload missing a file the shipped box has, so exactly the applications doing the right // thing were the ones that could not be tested. const source = 'import json, pathlib\n' - + 'declared = json.loads(pathlib.Path("box.json").read_text())["modelCacheSubdir"]\n' - + 'assert declared == "model-cache/example-model", declared\n'; + + 'declared = json.loads(pathlib.Path("box.json").read_text())["cacheSubdir"]\n' + + 'assert declared == "cache/example-model", declared\n'; const scroll = { ...SCROLL, - selfTest: { imports: ['json'], files: [], pythonFile: 'checks/self_test.py' }, + selfTest: { imports: ['json'], files: [], script: 'checks/self_test.py' }, }; const { keys, payloadDir } = await makeProject(scroll, { projectFiles: { 'checks/self_test.py': source }, @@ -1062,43 +1155,62 @@ describe('the build pipeline', () => { it('fails the build when the self-test file a scroll names is gone', async () => { const scroll = { ...SCROLL, - selfTest: { imports: ['json'], files: [], pythonFile: 'checks/self_test.py' }, + selfTest: { imports: ['json'], files: [], script: 'checks/self_test.py' }, }; const { keys, payloadDir } = await makeProject(scroll); await expect(buildBox(SCROLL_REF, { ...keys, ...fakeToolchain(payloadDir), log: () => {} })) - .rejects.toThrow(/Self-test Python file is missing/); + .rejects.toThrow(/Self-test script is missing/); }); it('fails the build when pruning removed a file the self-test needs', async () => { - const scroll = { ...SCROLL, selfTest: { imports: ['json'], files: ['model-cache/weights.bin'] } }; + const scroll = { ...SCROLL, selfTest: { imports: ['json'], files: ['cache/data.bin'] } }; const { keys, payloadDir } = await makeProject(scroll); await expect(buildBox(SCROLL_REF, { ...keys, ...fakeToolchain(payloadDir), log: () => {} })) .rejects.toThrow(/Missing self-test file/); }); - it('leaves assets out of the archive on demand, and carries their descriptors instead', async () => { - const asset = { - url: 'https://assets.example.org/weights.bin', - relativePath: 'model-cache/example-model/weights.bin', + it('defers only the assets declared deferred, and carries their descriptors instead', async () => { + // Per entry, which is the whole point of the change: this box ships a small file inside the + // archive and leaves a large one out, in one build, which version 2 could not express at all. + const embedded = { + url: 'https://assets.example.org/config.json', + relativePath: 'cache/example-model/config.json', + sizeBytes: 3, + sha256: createHash('sha256').update('{}\n').digest('hex'), + }; + const deferred = { + url: 'https://assets.example.org/data.bin', + relativePath: 'cache/example-model/data.bin', sizeBytes: 4, sha256: 'b'.repeat(64), + embed: false, }; const scroll = { ...SCROLL, - assets: [asset], - selfTest: { imports: ['json'], files: [asset.relativePath] }, + assets: [embedded, deferred], + selfTest: { imports: ['json'], files: [embedded.relativePath, deferred.relativePath] }, }; const { keys, payloadDir } = await makeProject(scroll); - // Nothing is downloaded: the fake toolchain would throw on an unexpected command, and the - // self-test file that lives at the asset's path is legitimately absent from the payload. + const fetched = []; const built = await buildBox(SCROLL_REF, { - ...keys, weights: 'on-demand', ...fakeToolchain(payloadDir), log: () => {}, + ...keys, + ...fakeToolchain(payloadDir), + fetchImpl: async (url) => { + fetched.push(url); + return new Response(Readable.toWeb(Readable.from([Buffer.from('{}\n')])), { status: 200 }); + }, + log: () => {}, }); - expect(built.weights).toBe('on-demand'); + // Only the embedded one is fetched; the deferred one's self-test file is legitimately absent. + expect(fetched).toEqual([embedded.url]); + expect(built.deferredAssets).toBe(1); const release = decodeDocumentPayload(JSON.parse(await readFile(built.releasePath, 'utf8'))); - expect(release.weights).toBe('on-demand'); - // The hash travels with the descriptor, which is what makes fetching it later safe. - expect(release.assets).toEqual([asset]); + // The descriptor list is exactly the deferred half, without the `embed: false` that produced + // it: on this side of the wire the list itself is the statement. + const { embed: _embed, ...descriptor } = deferred; + expect(release.assets).toEqual([descriptor]); + expect(await listZipEntries(built.archivePath).then((entries) => + entries.map((entry) => entry.path))).toContain(embedded.relativePath); await expect(verifyBox(built.releasePath, { publicPath: keys.publicPath, log: () => {} })) .resolves.toMatchObject({ status: 'passed' }); }); @@ -1156,22 +1268,21 @@ describe('the build pipeline', () => { describe('box manifest agreement', () => { const shared = { - schemaVersion: 2, + schemaVersion: 3, boxId: 'example-model', - modelId: 'example-org-example-model', - runtimeId: 'example-runtime', + labels: { model: 'example-org/example-model' }, version: '1.0.0', target: { platform: 'linux', arch: 'x86_64', accelerator: 'cpu' }, - pythonEntryPoint: 'venv/bin/python', - modelCacheSubdir: 'model-cache/example-model', - selfTest: { pythonImports: ['json'], timeoutSeconds: 180 }, + runtime: { id: 'python', version: '3.11.15', entryPoint: 'venv/bin/python' }, + cacheSubdir: 'cache/example-model', + selfTest: { probe: { imports: ['json'] }, timeoutSeconds: 180 }, provenance: { scrollId: 'example-model-linux', scrollVersion: '1.0.0', builderRevision: 'a'.repeat(40), sourceTreeDirty: false, sourceRevision: 'b'.repeat(40), - pythonVersion: '3.11.15', + runtimeVersion: '3.11.15', pixiVersion: '0.73.0', dependencyLockSha256: 'c'.repeat(64), builtAt: '2026-01-01T00:00:00Z', @@ -1179,15 +1290,14 @@ describe('box manifest agreement', () => { }; it.each([ - ['schemaVersion', 1], + ['schemaVersion', 2], ['boxId', 'other-box'], - ['modelId', 'other-model'], - ['runtimeId', 'other-runtime'], + ['labels', { model: 'other-org/other-model' }], ['version', '2.0.0'], ['target', { platform: 'linux', arch: 'x86_64', accelerator: 'cuda', cudaVersion: '12.8' }], - ['pythonEntryPoint', 'venv/python.exe'], - ['modelCacheSubdir', 'other-cache'], - ['selfTest', { pythonImports: ['math'], timeoutSeconds: 180 }], + ['runtime', { id: 'python', version: '3.11.15', entryPoint: 'venv/python.exe' }], + ['cacheSubdir', 'other-cache'], + ['selfTest', { probe: { imports: ['math'] }, timeoutSeconds: 180 }], ['provenance', { ...shared.provenance, sourceTreeDirty: true }], ['execution', { kind: 'python-module', module: 'other.main', defaultArgs: [] }], ])('rejects a %s mismatch', (field, value) => { @@ -1195,20 +1305,29 @@ describe('box manifest agreement', () => { .toThrow(new RegExp(`box\\.json mismatch: ${field}`)); }); - it('compares the complete on-demand asset policy', () => { + it('compares the deferred asset list entry by entry', () => { + // There is no box-wide asset switch left to compare: the list *is* the decision, one entry at a + // time, so a box that changed its mind about a single asset disagrees with its release here. const asset = { - url: 'https://assets.example.org/weights.bin', - relativePath: 'model-cache/example-model/weights.bin', + url: 'https://assets.example.org/data.bin', + relativePath: 'cache/example-model/data.bin', sizeBytes: 4, sha256: 'd'.repeat(64), }; - const release = { ...shared, weights: 'on-demand', assets: [asset] }; + const release = { ...shared, assets: [asset] }; expect(() => assertBoxManifestAgreement({ ...release }, release)).not.toThrow(); expect(() => assertBoxManifestAgreement({ ...release, assets: [{ ...asset, sha256: 'e'.repeat(64) }], }, release)).toThrow(/box\.json mismatch: assets/); + expect(() => assertBoxManifestAgreement({ + ...release, + assets: [{ ...asset, executable: true }], + }, release)).toThrow(/box\.json mismatch: assets/); + // A box claiming to be self-contained against a release that defers an asset, and the reverse. expect(() => assertBoxManifestAgreement({ ...shared }, release)) - .toThrow(/box\.json mismatch: weights/); + .toThrow(/box\.json mismatch: assets/); + expect(() => assertBoxManifestAgreement(release, { ...shared })) + .toThrow(/box\.json mismatch: assets/); }); }); diff --git a/tests/unit/consumer.test.mjs b/tests/unit/consumer.test.mjs index fde7936..5b8f60d 100644 --- a/tests/unit/consumer.test.mjs +++ b/tests/unit/consumer.test.mjs @@ -516,7 +516,7 @@ describe('Node consumer execution', () => { expect(result).toMatchObject({ exitCode: 17, signal: null }); expect(fake.calls).toHaveLength(1); - expect(fake.calls[0].command).toBe(join(prepared.root, ...prepared.pythonEntryPoint.split('/'))); + expect(fake.calls[0].command).toBe(join(prepared.root, ...prepared.runtime.entryPoint.split('/'))); expect(fake.calls[0].args).toEqual([ join(prepared.root, 'app/main.py'), '--default', diff --git a/tests/unit/contract-runtimes.test.mjs b/tests/unit/contract-runtimes.test.mjs index 61af68f..1963116 100644 --- a/tests/unit/contract-runtimes.test.mjs +++ b/tests/unit/contract-runtimes.test.mjs @@ -3,13 +3,17 @@ import { describe, expect, it } from 'vitest'; import { fixtureUrl } from '../../src/contract/index.mjs'; import { boxTargetAdapters } from '../../src/contract/targets.mjs'; import { - IMPLICIT_RUNTIME_ID, + RUNTIME_IDS, executionAffectingVariables, isExecutablePayloadPath, + isImplementedRuntime, runtimeAdapter, runtimeAdapters, + unimplementedRuntimeMessage, } from '../../src/contract/runtimes.mjs'; +const PYTHON = 'python'; + /** * The Node half of the shared runtime vectors. * @@ -33,9 +37,25 @@ describe('runtime adapters', () => { .toEqual(contract.runtimes.map((fixture) => fixture.id)); }); - it('refuses a runtime the format does not define', () => { - for (const id of ['node', 'native', '', undefined, null, 42]) { - expect(() => runtimeAdapter(id)).toThrow(TypeError); + it('names every runtime the format defines, whether or not it implements one', () => { + // Two different lists on purpose: the wire vocabulary was fixed once, in the version 3 break, so + // that implementing a second runtime is code rather than another format change. + expect([...RUNTIME_IDS]).toEqual(contract.runtimeIds); + for (const id of RUNTIME_IDS) { + expect(isImplementedRuntime(id), id).toBe(runtimeAdapters().some((r) => r.id === id)); + } + }); + + it('refuses a runtime it has no adapter for, and says which kind of refusal it is', () => { + for (const id of ['node', 'native']) { + expect(() => runtimeAdapter(id), id).toThrow(TypeError); + expect(isImplementedRuntime(id), id).toBe(false); + expect(unimplementedRuntimeMessage(id)).toContain('not implemented by this version'); + } + for (const id of ['', undefined, null, 42, 'ruby']) { + expect(() => runtimeAdapter(id), String(id)).toThrow(TypeError); + expect(isImplementedRuntime(id), String(id)).toBe(false); + expect(unimplementedRuntimeMessage(id)).toContain('Unknown runtime'); } }); @@ -100,21 +120,38 @@ describe('runtime adapters', () => { } }); - it('turns every golden self-test probe into the same arguments', () => { + it('turns every golden self-test probe into the same invocations', () => { for (const testCase of contract.selfTest) { const runtime = runtimeAdapter(testCase.runtime); - expect([...runtime.selfTestArgv({ + const invocations = runtime.selfTestInvocations({ probe: testCase.probe, + execution: testCase.execution, target: targetFor(testCase.platform), - })], testCase.name).toEqual(testCase.args); + }); + expect(invocations.map((invocation) => ({ + command: { ...invocation.command }, + args: invocation.args.map((argument) => ({ ...argument })), + expectExitCode: invocation.expectExitCode, + })), testCase.name).toEqual(testCase.invocations); } }); + it('refuses a command probe with no execution to invoke', () => { + expect(() => runtimeAdapter(PYTHON).selfTestInvocations({ + probe: { commands: [{ args: [], expectExitCode: 0 }] }, + execution: null, + target: targetFor('linux'), + })).toThrow(/needs a declared execution/); + }); + it('rejects a target no runtime has a layout for', () => { - const runtime = runtimeAdapter(IMPLICIT_RUNTIME_ID); + const runtime = runtimeAdapter(PYTHON); expect(() => runtime.layout({ platform: 'plan9' })).toThrow(/No python runtime layout/); - expect(() => runtime.selfTestArgv({ probe: { imports: [] }, target: { platform: 'plan9' } })) - .toThrow(/No python self-test assertion/); + expect(() => runtime.selfTestInvocations({ + probe: { imports: ['json'] }, + execution: null, + target: { platform: 'plan9' }, + })).toThrow(/No python self-test assertion/); }); }); @@ -123,9 +160,9 @@ describe('execution-affecting variables', () => { // The order is what a diagnostic report is printed in, so it is part of the answer rather than // an accident of how the two lists happened to be concatenated. for (const adapter of boxTargetAdapters()) { - const merged = executionAffectingVariables(IMPLICIT_RUNTIME_ID, adapter); + const merged = executionAffectingVariables(PYTHON, adapter); expect([...merged], adapter.id).toEqual([ - ...runtimeAdapter(IMPLICIT_RUNTIME_ID).executionEnvironmentVariables, + ...runtimeAdapter(PYTHON).executionEnvironmentVariables, ...adapter.executionAffectingEnvironmentVariables, ]); // Neither half may be dropped: this is the list that decides which inherited values a report @@ -136,7 +173,7 @@ describe('execution-affecting variables', () => { it('names the operating system control each platform actually has', () => { const named = new Map(boxTargetAdapters() - .map((adapter) => [adapter.platform, executionAffectingVariables(IMPLICIT_RUNTIME_ID, adapter)])); + .map((adapter) => [adapter.platform, executionAffectingVariables(PYTHON, adapter)])); expect(named.get('macos')).toContain('DYLD_INSERT_LIBRARIES'); expect(named.get('linux')).toContain('LD_PRELOAD'); expect(named.get('windows')).not.toContain('LD_PRELOAD'); diff --git a/tests/unit/contract-schema.test.mjs b/tests/unit/contract-schema.test.mjs index 85c1cc3..ab4e710 100644 --- a/tests/unit/contract-schema.test.mjs +++ b/tests/unit/contract-schema.test.mjs @@ -41,7 +41,7 @@ function createValidator() { } const ajv = createValidator(); -const validatorFor = (name) => ajv.getSchema(`https://scrollcase.dev/schema/v2/${name}.schema.json`); +const validatorFor = (name) => ajv.getSchema(`https://scrollcase.dev/schema/v3/${name}.schema.json`); /** Reports why a document failed, instead of a bare boolean, when a schema and reality disagree. */ function expectValid(name, document, label) { @@ -54,7 +54,7 @@ describe('published schemas', () => { it('ships one well-formed schema per document the format defines', () => { for (const name of SCHEMA_NAMES) { const schema = readJson(schemaUrl(name)); - expect(schema.$id, name).toBe(`https://scrollcase.dev/schema/v2/${name}.schema.json`); + expect(schema.$id, name).toBe(`https://scrollcase.dev/schema/v3/${name}.schema.json`); expect(schema.title, name).toBeTruthy(); expect(schema.description, name).toBeTruthy(); expect(validatorFor(name), name).toBeTypeOf('function'); @@ -149,10 +149,17 @@ describe('schemas describe what the builder actually emits', () => { const execution = readJson(schemaUrl('execution')); expect(scroll.properties.$schema.const).toBe(scroll.$id); - expect(scroll.properties.weights.default).toBe('embed'); - expect(scroll.properties.weights.description).toBeTruthy(); + // Both per-asset switches carry the default that lets an ordinary entry say nothing at all. + const asset = scroll.properties.assets.items.properties; + expect(asset.embed.default).toBe(true); + expect(asset.embed.description).toBeTruthy(); + expect(asset.executable.default).toBe(false); + expect(scroll.properties.localFiles.items.properties.executable.default).toBe(false); expect(scroll.properties.execution.$ref).toBe(execution.$id); - expect(execution.oneOf).toHaveLength(2); + // One branch per execution kind the format defines, including the two no runtime implements + // yet: the vocabulary was fixed in the version 3 break so Phase C never touches the wire. + expect(execution.oneOf.map((branch) => branch.properties.kind.const)) + .toEqual(['python-script', 'python-module', 'node-script', 'native-binary']); expect(execution.examples).toHaveLength(2); expect(execution.oneOf.every((branch) => branch.additionalProperties === false)).toBe(true); for (const field of ['platform', 'arch', 'accelerator']) { From 525e6b833bb5a4d1fe663c3b7c2edc2b6241f945 Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:14:56 +0200 Subject: [PATCH 04/22] Carry the version 3 break through the Node tests, fixtures and examples The conformance fixture keeps its 81 cases and gains three: a declared-executable box that keeps its bit under umask 077, a release naming a runtime this build has no adapter for, and a runtime block that disagrees with box.json. The signed example release is re-signed rather than edited, under a fresh key, because a fixture whose ed25519 signature no longer checks would prove nothing. The twelve example scrolls migrate to v3, and model-cache/ becomes cache/ in the examples and in the two demo entrypoints that read the field by name from box.json. --- examples/hello-box/scroll.json | 11 +- examples/llm-demo/scroll.json | 28 ++-- examples/llm-demo/shared/MODEL_NOTICE.md | 2 +- examples/llm-demo/shared/entrypoint.py | 6 +- examples/sentiment-demo/scroll.json | 36 +++-- .../sentiment-demo/shared/MODEL_NOTICE.md | 6 +- examples/sentiment-demo/shared/entrypoint.py | 2 +- src/contract/documents.mjs | 4 +- .../fixtures/consumer-conformance.json | 147 +++++++++++++----- src/sign/keys.mjs | 6 +- tests/helpers/consumer-box-fixture.mjs | 7 + tests/helpers/consumer-conformance.mjs | 75 +++++++-- tests/unit/contract-targets.test.mjs | 9 +- tests/unit/execution-contract.test.mjs | 6 + tests/unit/llm-demo.test.mjs | 15 +- tests/unit/package-surface.test.mjs | 2 +- tests/unit/scroll-authoring.test.mjs | 40 +++-- tests/unit/scroll-editing.test.mjs | 50 +++--- tests/unit/scroll-extends.test.mjs | 39 +++-- tests/unit/sentiment-demo.test.mjs | 2 +- tests/unit/signing.test.mjs | 2 +- ...gration.test.mjs => v3-migration.test.mjs} | 21 +-- 22 files changed, 325 insertions(+), 191 deletions(-) rename tests/unit/{v2-migration.test.mjs => v3-migration.test.mjs} (73%) diff --git a/examples/hello-box/scroll.json b/examples/hello-box/scroll.json index ed00e21..1db6fea 100644 --- a/examples/hello-box/scroll.json +++ b/examples/hello-box/scroll.json @@ -1,18 +1,19 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "scrollVersion": "1.0.0", "boxId": "hello-box", - "modelId": "example-org-hello", - "runtimeId": "hello-box-runtime", "version": "1.0.0", "sourceRevision": "example-hello-v1", "compatibility": { "minHostAppVersion": "1.0.0", "minRamGb": 1 }, - "pythonVersion": "3.11", + "runtime": { + "id": "python", + "version": "3.11" + }, "pixiVersion": "0.73.0", - "modelCacheSubdir": "model-cache/hello", + "cacheSubdir": "cache/hello", "environment": {}, "assetBaseUrl": "https://assets.example.org/boxes", "assets": [], diff --git a/examples/llm-demo/scroll.json b/examples/llm-demo/scroll.json index ef87adc..ecb850b 100644 --- a/examples/llm-demo/scroll.json +++ b/examples/llm-demo/scroll.json @@ -1,19 +1,24 @@ { - "$schema": "https://scrollcase.dev/schema/v2/scroll.schema.json", - "schemaVersion": 2, + "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "schemaVersion": 3, "scrollVersion": "1.0.0", "boxId": "llm-demo", - "modelId": "smollm2-1.7b-instruct-gguf-q4-k-m", - "runtimeId": "llama-cpp-cpu", + "labels": { + "model": "smollm2-1.7b-instruct-gguf-q4-k-m", + "stack": "llama-cpp-cpu" + }, "version": "1.0.0", "sourceRevision": "2d4a76a30b4af41ecd395c35725ac11688d4cfe4", "compatibility": { "minHostAppVersion": "1.0.0", "minRamGb": 4 }, - "pythonVersion": "3.11.*", + "runtime": { + "id": "python", + "version": "3.11.*" + }, "pixiVersion": "0.73.0", - "modelCacheSubdir": "model-cache/llm-demo", + "cacheSubdir": "cache/llm-demo", "environment": { "PYTHONDONTWRITEBYTECODE": "1" }, @@ -21,7 +26,7 @@ "assets": [ { "url": "https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF/resolve/2d4a76a30b4af41ecd395c35725ac11688d4cfe4/smollm2-1.7b-instruct-q4_k_m.gguf", - "relativePath": "model-cache/llm-demo/smollm2-1.7b-instruct-q4_k_m.gguf", + "relativePath": "cache/llm-demo/smollm2-1.7b-instruct-q4_k_m.gguf", "sizeBytes": 1055609536, "sha256": "decd2598bc2c8ed08c19adc3c8fdd461ee19ed5708679d1c54ef54a5a30d4f33" } @@ -30,12 +35,12 @@ { "sourcePath": "examples/llm-demo/shared/entrypoint.py", "relativePath": "entrypoint.py", - "sha256": "c8b15f057a90ddba07b07d2cd983e2f200090a6b2dd2ac33fd9e54dfda2afa06" + "sha256": "ff344a7a91e0baceb6b3230923f03e21bfbc549f1b519ecfba4634af7fbe7943" }, { "sourcePath": "examples/llm-demo/shared/MODEL_NOTICE.md", "relativePath": "THIRD_PARTY_NOTICES/smollm2/MODEL_NOTICE.md", - "sha256": "32c8619c5c6c5d94699d37fd78b90723aed8ae08f85bb037a92aa2a6d149454c" + "sha256": "6a289f5a4c501e959bb7a646cd6b40a456ff8598844c1e4dd17684ff7a92c1b2" }, { "sourcePath": "examples/llm-demo/shared/APACHE-2.0.txt", @@ -43,18 +48,17 @@ "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" } ], - "weights": "embed", "selfTest": { "imports": [ "llama_cpp" ], "files": [ "entrypoint.py", - "model-cache/llm-demo/smollm2-1.7b-instruct-q4_k_m.gguf", + "cache/llm-demo/smollm2-1.7b-instruct-q4_k_m.gguf", "THIRD_PARTY_NOTICES/smollm2/MODEL_NOTICE.md", "THIRD_PARTY_NOTICES/smollm2/APACHE-2.0.txt" ], - "pythonFile": "examples/llm-demo/shared/self_test.py" + "script": "examples/llm-demo/shared/self_test.py" }, "execution": { "kind": "python-script", diff --git a/examples/llm-demo/shared/MODEL_NOTICE.md b/examples/llm-demo/shared/MODEL_NOTICE.md index 5d5f3fe..9919c99 100644 --- a/examples/llm-demo/shared/MODEL_NOTICE.md +++ b/examples/llm-demo/shared/MODEL_NOTICE.md @@ -7,7 +7,7 @@ comes from, what it may reasonably be used for, and where its documented limitat | File | Origin | | --- | --- | -| `model-cache/llm-demo/smollm2-1.7b-instruct-q4_k_m.gguf` | GGUF Q4_K_M conversion, revision `2d4a76a30b4af41ecd395c35725ac11688d4cfe4` | +| `cache/llm-demo/smollm2-1.7b-instruct-q4_k_m.gguf` | GGUF Q4_K_M conversion, revision `2d4a76a30b4af41ecd395c35725ac11688d4cfe4` | One file, and that is the whole model: a GGUF carries the weights, the tokenizer and the chat template together, so there is no separate `tokenizer.json` or `config.json` to ship alongside it. diff --git a/examples/llm-demo/shared/entrypoint.py b/examples/llm-demo/shared/entrypoint.py index 500a32f..af97219 100644 --- a/examples/llm-demo/shared/entrypoint.py +++ b/examples/llm-demo/shared/entrypoint.py @@ -87,7 +87,7 @@ def model_dir() -> Path: """Model directory, as the box itself declares it. A box ships a `box.json` at its root, and one of the things it states is - `modelCacheSubdir` -- where the model files were placed. Reading it here means + `cacheSubdir` -- where the model files were placed. Reading it here means the application never has to guess a path, and the scroll never has to be bent to match a constant compiled into this file. Change where the model lives and this keeps working; hard-code it and the two drift apart silently. @@ -98,9 +98,9 @@ def model_dir() -> Path: raise DemoError(f"missing box manifest: {manifest}") try: with manifest.open(encoding="utf-8") as handle: - subdirectory = json.load(handle)["modelCacheSubdir"] + subdirectory = json.load(handle)["cacheSubdir"] except (KeyError, ValueError) as error: - raise DemoError(f"{BOX_MANIFEST} declares no usable modelCacheSubdir: {error}") from None + raise DemoError(f"{BOX_MANIFEST} declares no usable cacheSubdir: {error}") from None return root.joinpath(*str(subdirectory).split("/")) diff --git a/examples/sentiment-demo/scroll.json b/examples/sentiment-demo/scroll.json index 742d1fc..e20f22c 100644 --- a/examples/sentiment-demo/scroll.json +++ b/examples/sentiment-demo/scroll.json @@ -1,19 +1,24 @@ { - "$schema": "https://scrollcase.dev/schema/v2/scroll.schema.json", - "schemaVersion": 2, + "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "schemaVersion": 3, "scrollVersion": "1.0.0", "boxId": "sentiment-demo", - "modelId": "distilbert-sst2-onnx-int8", - "runtimeId": "onnxruntime-cpu", + "labels": { + "model": "distilbert-sst2-onnx-int8", + "stack": "onnxruntime-cpu" + }, "version": "1.0.0", "sourceRevision": "fd49941c1b822846cb14970cdf430a7cfbe0f5b9", "compatibility": { "minHostAppVersion": "1.0.0", "minRamGb": 2 }, - "pythonVersion": "3.11.*", + "runtime": { + "id": "python", + "version": "3.11.*" + }, "pixiVersion": "0.73.0", - "modelCacheSubdir": "model-cache/distilbert-sst2", + "cacheSubdir": "cache/distilbert-sst2", "environment": { "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", @@ -23,19 +28,19 @@ "assets": [ { "url": "https://huggingface.co/onnx-community/distilbert-base-uncased-finetuned-sst-2-english-ONNX/resolve/fd49941c1b822846cb14970cdf430a7cfbe0f5b9/onnx/model_int8.onnx", - "relativePath": "model-cache/distilbert-sst2/model_int8.onnx", + "relativePath": "cache/distilbert-sst2/model_int8.onnx", "sizeBytes": 67537148, "sha256": "1bc93de9f1da185c67028dbac37df6c14939256e0851d28e8f9c2994d338ac4c" }, { "url": "https://huggingface.co/onnx-community/distilbert-base-uncased-finetuned-sst-2-english-ONNX/resolve/fd49941c1b822846cb14970cdf430a7cfbe0f5b9/tokenizer.json", - "relativePath": "model-cache/distilbert-sst2/tokenizer.json", + "relativePath": "cache/distilbert-sst2/tokenizer.json", "sizeBytes": 711396, "sha256": "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66" }, { "url": "https://huggingface.co/onnx-community/distilbert-base-uncased-finetuned-sst-2-english-ONNX/resolve/fd49941c1b822846cb14970cdf430a7cfbe0f5b9/config.json", - "relativePath": "model-cache/distilbert-sst2/config.json", + "relativePath": "cache/distilbert-sst2/config.json", "sizeBytes": 786, "sha256": "27475a0750e539c105a51c59dbef1f0ab75615b0a06e96f2f4d585c46f160c2f" } @@ -44,12 +49,12 @@ { "sourcePath": "examples/sentiment-demo/shared/entrypoint.py", "relativePath": "entrypoint.py", - "sha256": "247033008de5ab88847115ad4e643c7aeec97e22ea0e74c202b2e0e280206dc3" + "sha256": "9e6dd813fccb8c9a0abfa419bc8911312afe48820b6ae3f1c5516765eed2eecc" }, { "sourcePath": "examples/sentiment-demo/shared/MODEL_NOTICE.md", "relativePath": "THIRD_PARTY_NOTICES/distilbert/MODEL_NOTICE.md", - "sha256": "eb945ca2676fe6faeafb708bedb0e5846ac4fad27449c43ddabadef886a6a5ba" + "sha256": "77040dbe4f875714ad617c1016fd44afa367814dba5968d4aec02ce21f5d84be" }, { "sourcePath": "examples/sentiment-demo/shared/APACHE-2.0.txt", @@ -57,7 +62,6 @@ "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" } ], - "weights": "embed", "selfTest": { "imports": [ "onnxruntime", @@ -66,13 +70,13 @@ ], "files": [ "entrypoint.py", - "model-cache/distilbert-sst2/model_int8.onnx", - "model-cache/distilbert-sst2/tokenizer.json", - "model-cache/distilbert-sst2/config.json", + "cache/distilbert-sst2/model_int8.onnx", + "cache/distilbert-sst2/tokenizer.json", + "cache/distilbert-sst2/config.json", "THIRD_PARTY_NOTICES/distilbert/MODEL_NOTICE.md", "THIRD_PARTY_NOTICES/distilbert/APACHE-2.0.txt" ], - "pythonCode": "import math\nimport os\nimport sys\n\nsys.path.insert(0, os.getcwd())\n\nfrom entrypoint import predict\n\nCASES = (\n ('This product is surprisingly easy to use.', 'POSITIVE'),\n ('This was a frustrating and disappointing experience.', 'NEGATIVE'),\n)\n\nfor sentence, expected in CASES:\n label, confidence = predict(sentence)\n assert label == expected, f'{sentence!r}: expected {expected}, got {label}'\n assert math.isfinite(confidence), f'{sentence!r}: confidence is not finite'\n assert 0.0 <= confidence <= 1.0, f'{sentence!r}: confidence {confidence} outside [0, 1]'\n\nprint('self-test ok')\n" + "code": "import math\nimport os\nimport sys\n\nsys.path.insert(0, os.getcwd())\n\nfrom entrypoint import predict\n\nCASES = (\n ('This product is surprisingly easy to use.', 'POSITIVE'),\n ('This was a frustrating and disappointing experience.', 'NEGATIVE'),\n)\n\nfor sentence, expected in CASES:\n label, confidence = predict(sentence)\n assert label == expected, f'{sentence!r}: expected {expected}, got {label}'\n assert math.isfinite(confidence), f'{sentence!r}: confidence is not finite'\n assert 0.0 <= confidence <= 1.0, f'{sentence!r}: confidence {confidence} outside [0, 1]'\n\nprint('self-test ok')\n" }, "execution": { "kind": "python-script", diff --git a/examples/sentiment-demo/shared/MODEL_NOTICE.md b/examples/sentiment-demo/shared/MODEL_NOTICE.md index 6ae8263..b4e24b0 100644 --- a/examples/sentiment-demo/shared/MODEL_NOTICE.md +++ b/examples/sentiment-demo/shared/MODEL_NOTICE.md @@ -7,9 +7,9 @@ comes from, what it may reasonably be used for, and where its documented limitat | File | Origin | | --- | --- | -| `model-cache/distilbert-sst2/model_int8.onnx` | ONNX INT8 conversion, revision `fd49941c1b822846cb14970cdf430a7cfbe0f5b9` | -| `model-cache/distilbert-sst2/tokenizer.json` | ONNX INT8 conversion, revision `fd49941c1b822846cb14970cdf430a7cfbe0f5b9` | -| `model-cache/distilbert-sst2/config.json` | ONNX INT8 conversion, revision `fd49941c1b822846cb14970cdf430a7cfbe0f5b9` | +| `cache/distilbert-sst2/model_int8.onnx` | ONNX INT8 conversion, revision `fd49941c1b822846cb14970cdf430a7cfbe0f5b9` | +| `cache/distilbert-sst2/tokenizer.json` | ONNX INT8 conversion, revision `fd49941c1b822846cb14970cdf430a7cfbe0f5b9` | +| `cache/distilbert-sst2/config.json` | ONNX INT8 conversion, revision `fd49941c1b822846cb14970cdf430a7cfbe0f5b9` | Every file is fetched from an immutable, commit-pinned URL and hashed in the scroll. The box performs no download at run time. diff --git a/examples/sentiment-demo/shared/entrypoint.py b/examples/sentiment-demo/shared/entrypoint.py index 6cb5cb0..49b3679 100644 --- a/examples/sentiment-demo/shared/entrypoint.py +++ b/examples/sentiment-demo/shared/entrypoint.py @@ -20,7 +20,7 @@ import sys from pathlib import Path -MODEL_SUBDIR = ("model-cache", "distilbert-sst2") +MODEL_SUBDIR = ("cache", "distilbert-sst2") MODEL_FILE = "model_int8.onnx" TOKENIZER_FILE = "tokenizer.json" CONFIG_FILE = "config.json" diff --git a/src/contract/documents.mjs b/src/contract/documents.mjs index 9429c9e..57e38a0 100644 --- a/src/contract/documents.mjs +++ b/src/contract/documents.mjs @@ -34,8 +34,8 @@ export { * @throws {Error} when the embedded payload hash does not match the bytes */ export function decodeDocumentPayload(document) { - if (document?.schemaVersion === 1) { - throw new TypeError(unsupportedSchemaVersionMessage(1)); + if (document?.schemaVersion === 1 || document?.schemaVersion === 2) { + throw new TypeError(unsupportedSchemaVersionMessage(document.schemaVersion)); } if (!isSignedBoxDocument(document)) { throw new TypeError('Not a signed box document'); diff --git a/src/contract/fixtures/consumer-conformance.json b/src/contract/fixtures/consumer-conformance.json index 39f6023..ed34cd4 100644 --- a/src/contract/fixtures/consumer-conformance.json +++ b/src/contract/fixtures/consumer-conformance.json @@ -1,36 +1,39 @@ { - "schemaVersion": 1, + "schemaVersion": 3, "description": "Language-neutral semantic cases shared by the Node, Python and Rust Scrollcase consumers.", "errorPatterns": { - "invalid-trust-file": "Invalid trusted ed25519 key file", - "invalid-signature": "no valid signature", "altered-payload": "Signed payload SHA-256 mismatch", "archive-hash": "Archive SHA-256 mismatch", "archive-size": "Archive size mismatch", - "manifest-disagreement": "box.json mismatch: modelId", - "execution-disagreement": "box.json mismatch: execution", - "environment-disagreement": "box.json mismatch: environment", - "missing-interpreter": "Archive is missing venv/", - "missing-script": "Execution script is missing", - "missing-module": "Execution module is not discoverable", - "unsafe-path": "Unsafe relative path|invalid relative path|absolute path", - "link-entry": "link does not resolve to a file inside the payload|would be written through a link|link target is too long", - "special-entry": "special entries", - "encrypted-entry": "Encrypted ZIP entries", - "entry-collision": "Archive entry collides with another entry", - "existing-destination": "Destination already exists", + "asset-hash": "asset SHA-256 mismatch", "asset-missing": "asset is missing", + "asset-not-executable": "is not executable", "asset-size": "asset size mismatch", - "asset-hash": "asset SHA-256 mismatch", - "spawn-failure": "failed to start|fixture spawn failed", - "attach-root": "is not an extracted box directory", "attach-missing-interpreter": "Attached box is missing venv/", + "attach-root": "is not an extracted box directory", + "encrypted-entry": "Encrypted ZIP entries", + "entry-collision": "Archive entry collides with another entry", + "environment-disagreement": "box.json mismatch: environment", + "execution-disagreement": "box.json mismatch: execution", + "existing-destination": "Destination already exists", "foreign-target": "cannot run on", - "payload-list-missing": "missing its payload digest list", - "payload-list-mismatch": "Payload digest list does not match the signed release", + "invalid-signature": "no valid signature", + "invalid-trust-file": "Invalid trusted ed25519 key file", + "link-entry": "link does not resolve to a file inside the payload|would be written through a link|link target is too long", + "manifest-disagreement": "box.json mismatch: labels", + "missing-interpreter": "Archive is missing venv/", + "missing-module": "Execution module is not discoverable", + "missing-script": "Execution script is missing", "payload-digest-absent": "does not commit to a payload digest", + "payload-list-mismatch": "Payload digest list does not match the signed release", + "payload-list-missing": "missing its payload digest list", "payload-mismatch": "^Payload does not match the signed release:", - "unsupported-schema-version": "Unsupported schemaVersion 1" + "runtime-disagreement": "box.json mismatch: runtime", + "spawn-failure": "failed to start|fixture spawn failed", + "special-entry": "special entries", + "unimplemented-runtime": "is not implemented by this version", + "unsafe-path": "Unsafe relative path|invalid relative path|absolute path", + "unsupported-schema-version": "Unsupported schemaVersion [12]" }, "cases": [ { @@ -50,7 +53,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -69,7 +73,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -88,7 +93,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -107,7 +113,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -278,7 +285,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -294,7 +302,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -319,7 +328,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET", "environmentReport": { "mode": "summary", @@ -370,7 +380,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET", "environmentReport": { "mode": "full", @@ -626,7 +637,7 @@ { "id": "release-box-disagreement", "action": "prepare", - "mutation": "alter-release-model", + "mutation": "alter-release-labels", "expected": { "outcome": "rejected", "error": "manifest-disagreement", @@ -718,7 +729,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -786,7 +798,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "venv/bin/python", + "runtimeId": "python", + "entryPoint": "venv/bin/python", "targetId": "macos-aarch64-cpu" } } @@ -804,7 +817,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "venv/bin/python", + "runtimeId": "python", + "entryPoint": "venv/bin/python", "targetId": "linux-x86_64-cpu" } } @@ -822,7 +836,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "venv/python.exe", + "runtimeId": "python", + "entryPoint": "venv/python.exe", "targetId": "windows-x86_64-cpu" } } @@ -861,7 +876,7 @@ "signal": null }, "argv": [ - "$BOX/$NATIVE_PYTHON", + "$BOX/$NATIVE_ENTRY_POINT", "$BOX/app/main.py", "--default", "value with spaces", @@ -891,7 +906,7 @@ "signal": null }, "argv": [ - "$BOX/$NATIVE_PYTHON", + "$BOX/$NATIVE_ENTRY_POINT", "$BOX/app/main.py", "--default", "value with spaces", @@ -1046,7 +1061,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -1069,7 +1085,7 @@ "signal": null }, "argv": [ - "$BOX/$NATIVE_PYTHON", + "$BOX/$NATIVE_ENTRY_POINT", "$BOX/app/main.py", "--default", "value with spaces", @@ -1092,7 +1108,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -1113,7 +1130,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 1, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -1129,7 +1147,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -1352,6 +1371,52 @@ "outcome": "rejected", "error": "payload-list-mismatch" } + }, + { + "id": "declared-executable-survives-a-restrictive-umask", + "action": "prepare", + "fixture": { + "executableAsset": true + }, + "runtime": { + "umask": "077" + }, + "expected": { + "outcome": "prepared", + "receipt": { + "status": "prepared", + "boxId": "consumer-fixture", + "executionKind": "python-script", + "requiredAssetCount": 0, + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", + "targetId": "$NATIVE_TARGET", + "executableModes": { + "bin/tool": "755", + "box.json": "644" + } + } + } + }, + { + "id": "runtime-this-build-cannot-run", + "action": "prepare", + "mutation": "alter-release-runtime-id", + "expected": { + "outcome": "rejected", + "error": "unimplemented-runtime", + "destinationExists": false + } + }, + { + "id": "runtime-block-disagreement", + "action": "prepare", + "mutation": "alter-release-runtime-version", + "expected": { + "outcome": "rejected", + "error": "runtime-disagreement", + "destinationExists": false + } } ] } diff --git a/src/sign/keys.mjs b/src/sign/keys.mjs index 158e630..ac1093f 100644 --- a/src/sign/keys.mjs +++ b/src/sign/keys.mjs @@ -187,8 +187,10 @@ export function signWithLocalKey(payloadBytes, { privateKey, metadata }) { * @throws {Error} when the envelope is unsupported or its checksum does not match */ export function decodeSignedDocument(document) { - if (document?.schemaVersion === 1) { - fail(unsupportedSchemaVersionMessage(1)); + // Named by version before the generic refusal: a superseded document is a common thing to be + // holding, and "unsupported signed document" would not tell its owner what to do about it. + if (document?.schemaVersion === 1 || document?.schemaVersion === 2) { + fail(unsupportedSchemaVersionMessage(document.schemaVersion)); } if (document?.schemaVersion !== BOX_SCHEMA_VERSION || document?.payloadEncoding !== PAYLOAD_ENCODING) { fail('Unsupported signed document.'); diff --git a/tests/helpers/consumer-box-fixture.mjs b/tests/helpers/consumer-box-fixture.mjs index edd0b70..f81ca9c 100644 --- a/tests/helpers/consumer-box-fixture.mjs +++ b/tests/helpers/consumer-box-fixture.mjs @@ -74,6 +74,7 @@ export async function createConsumerBoxFixture({ labels = undefined, runtimeId = 'python', executablePaths = [], + extraFiles = {}, } = {}) { const root = await mkdtemp(join(tmpdir(), 'scrollcase-consumer-fixture-')); const payload = join(root, 'payload'); @@ -84,6 +85,12 @@ export async function createConsumerBoxFixture({ await mkdir(dirname(pythonPath), { recursive: true }); await writeFile(pythonPath, interpreterContents); + for (const [relativePath, contents] of Object.entries(extraFiles)) { + const filePath = join(payload, ...relativePath.split('/')); + await mkdir(dirname(filePath), { recursive: true }); + await writeFile(filePath, contents); + } + if (execution?.kind === 'python-script') { const scriptPath = join(payload, ...execution.script.split('/')); await mkdir(dirname(scriptPath), { recursive: true }); diff --git a/tests/helpers/consumer-conformance.mjs b/tests/helpers/consumer-conformance.mjs index 637274a..11d6f5d 100644 --- a/tests/helpers/consumer-conformance.mjs +++ b/tests/helpers/consumer-conformance.mjs @@ -29,7 +29,7 @@ import { payloadDigestStream, } from '../../src/contract/payload-digest.mjs'; import { boxTargetAdapter, boxTargetId } from '../../src/contract/targets.mjs'; -import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../../src/contract/runtimes.mjs'; +import { runtimeAdapter } from '../../src/contract/runtimes.mjs'; import { attachExtractedBox, runBox, @@ -157,7 +157,7 @@ async function mutateFixture(fixture, mutation, destination) { } if (mutation === 'downgrade-envelope-version') { // The envelope's own version is outside the signed payload, so this is what a genuine v1 - // document looks like to a v2 consumer: refusable by name before any signature is checked. + // document looks like to a v3 consumer: refusable by name before any signature is checked. const signed = JSON.parse(await readFile(fixture.releasePath, 'utf8')); signed.schemaVersion = 1; await writeFile(fixture.releasePath, `${JSON.stringify(signed, null, 2)}\n`); @@ -174,8 +174,20 @@ async function mutateFixture(fixture, mutation, destination) { await writeSignedRelease(fixture, fixture.release); return; } - if (mutation === 'alter-release-model') { - fixture.release.modelId = 'altered-model'; + if (mutation === 'alter-release-labels') { + fixture.release.labels = { model: 'altered-model' }; + await writeSignedRelease(fixture, fixture.release); + return; + } + if (mutation === 'alter-release-runtime-version') { + fixture.release.runtime = { ...fixture.release.runtime, version: '3.99.0' }; + await writeSignedRelease(fixture, fixture.release); + return; + } + if (mutation === 'alter-release-runtime-id') { + // A runtime the format names and this build has no adapter for. The consumer must refuse the + // box rather than fall back to reading it as the runtime it happens to be shaped like. + fixture.release.runtime = { ...fixture.release.runtime, id: 'native' }; await writeSignedRelease(fixture, fixture.release); return; } @@ -208,7 +220,7 @@ async function mutateFixture(fixture, mutation, destination) { return; } const removePath = { - 'remove-interpreter': fixture.release.pythonEntryPoint, + 'remove-interpreter': fixture.release.runtime.entryPoint, 'remove-script': fixture.release.execution?.script, 'remove-module': fixture.release.execution?.module?.split('.').join('/') + '.py', }[mutation]; @@ -222,17 +234,17 @@ async function mutateFixture(fixture, mutation, destination) { // — `venv/bin/python` is a link to the versioned binary beside it — so a consumer that only // accepts regular files here rejects every box the builder produces on macOS and Linux. if (mutation === 'link-interpreter') { - const parts = fixture.release.pythonEntryPoint.split('/'); + const parts = fixture.release.runtime.entryPoint.split('/'); const linkTarget = `${parts[parts.length - 1]}-real`; const directory = join(fixture.payload, ...parts.slice(0, -1)); await rename(join(fixture.payload, ...parts), join(directory, linkTarget)); await refreshPayloadDigest(fixture, [{ - path: fixture.release.pythonEntryPoint, + path: fixture.release.runtime.entryPoint, kind: 'link', contentSha256: createHash('sha256').update(linkTarget, 'utf8').digest('hex'), }]); await writeZip(fixture.archivePath, fixture.payload, { - path: fixture.release.pythonEntryPoint, + path: fixture.release.runtime.entryPoint, contents: linkTarget, options: { mode: 0o120777 }, }); @@ -305,7 +317,7 @@ async function mutateExtractedRoot(fixture, mutation, root) { return root; } if (mutation === 'remove-interpreter') { - await rm(join(root, ...fixture.release.pythonEntryPoint.split('/'))); + await rm(join(root, ...fixture.release.runtime.entryPoint.split('/'))); return root; } if (mutation === 'remove-script') { @@ -313,7 +325,7 @@ async function mutateExtractedRoot(fixture, mutation, root) { return root; } if (mutation === 'retarget-interpreter-link') { - const interpreter = join(root, ...fixture.release.pythonEntryPoint.split('/')); + const interpreter = join(root, ...fixture.release.runtime.entryPoint.split('/')); const target = await readlink(interpreter); await rm(interpreter); await symlink(`${target}-retargeted`, interpreter); @@ -347,9 +359,10 @@ function fixtureOptions(spec = {}) { const execution = spec.execution === 'module' ? { kind: 'python-module', module: 'example.application', defaultArgs: ['--default'] } : undefined; + const executablePaths = spec.executableAsset ? ['bin/tool'] : []; const requiredAsset = spec.requiredAsset ? { - url: 'https://assets.example.org/weights.bin', - relativePath: 'model-cache/consumer-fixture/weights.bin', + url: 'https://assets.example.org/data.bin', + relativePath: 'cache/consumer-fixture/data.bin', sizeBytes: ASSET_BYTES.length, sha256: createHash('sha256').update(ASSET_BYTES).digest('hex'), } : null; @@ -360,6 +373,9 @@ function fixtureOptions(spec = {}) { requiredAsset, payloadDigest: spec.payloadDigest !== false, environment: spec.environment, + executablePaths, + ...(spec.executableAsset ? { extraFiles: { 'bin/tool': '#!/bin/sh\nexit 0\n' } } : {}), + ...(spec.labels ? { labels: spec.labels } : {}), }; } @@ -414,7 +430,7 @@ function replaceTokens(value, root = null) { if (typeof value === 'string') { const adapter = boxTargetAdapter(nativeTarget()); return value - .replaceAll('$NATIVE_PYTHON', runtimeAdapter(IMPLICIT_RUNTIME_ID).layout(adapter).entryPoint) + .replaceAll('$NATIVE_ENTRY_POINT', runtimeAdapter('python').layout(adapter).entryPoint) .replaceAll('$NATIVE_TARGET', boxTargetId(nativeTarget())) .replaceAll('$BOX', root ?? '$BOX'); } @@ -462,6 +478,11 @@ export async function runNodeConformanceCase(testCase) { previousHostEnvironment.set(name, process.env[name]); process.env[name] = value; } + // A restrictive umask is the condition under which the three consumers used to disagree: two + // applied the archive's mode through open(2) and lost it, one chmod'd and kept it. + const previousUmask = runtime.umask === undefined || process.platform === 'win32' + ? null + : process.umask(Number.parseInt(runtime.umask, 8)); try { if (testCase.fixture?.linkedInterpreter) { await mutateFixture(fixture, 'link-interpreter', destination); @@ -486,9 +507,16 @@ export async function runNodeConformanceCase(testCase) { boxId: prepared.boxId, executionKind: prepared.execution?.kind ?? null, requiredAssetCount: prepared.requiredAssets.length, - pythonEntryPoint: prepared.pythonEntryPoint, + runtimeId: prepared.runtime.id, + entryPoint: prepared.runtime.entryPoint, targetId: prepared.targetId, }; + if (testCase.expected.receipt?.executableModes) { + receipt.executableModes = await executableModes( + prepared.root, + testCase.expected.receipt.executableModes, + ); + } if (testCase.expected.receipt?.environmentReport) { receipt.environmentReport = environmentReport( prepared.environmentReport, @@ -531,7 +559,8 @@ export async function runNodeConformanceCase(testCase) { boxId: attached.boxId, executionKind: attached.execution?.kind ?? null, requiredAssetCount: attached.requiredAssets.length, - pythonEntryPoint: attached.pythonEntryPoint, + runtimeId: attached.runtime.id, + entryPoint: attached.runtime.entryPoint, targetId: attached.targetId, }; if (testCase.expected.receipt?.environmentReport) { @@ -688,6 +717,7 @@ export async function runNodeConformanceCase(testCase) { } return { actual, expected, root: fixture.root }; } finally { + if (previousUmask !== null) process.umask(previousUmask); for (const [name, value] of previousHostEnvironment) { if (value === undefined) delete process.env[name]; else process.env[name] = value; @@ -695,6 +725,21 @@ export async function runNodeConformanceCase(testCase) { } } +/** + * The permission bits an extracted box actually carries, for the paths a case names. + * + * Windows has no bit to read, so every path reports the same value there and the fixture says so + * rather than the driver quietly skipping the case. + */ +async function executableModes(root, paths) { + const modes = {}; + for (const path of Object.keys(paths)) { + const details = await stat(join(root, ...path.split('/'))); + modes[path] = process.platform === 'win32' ? null : (details.mode & 0o777).toString(8); + } + return modes; +} + async function pathExists(path) { try { await stat(path); diff --git a/tests/unit/contract-targets.test.mjs b/tests/unit/contract-targets.test.mjs index 20148a1..8b7b9cf 100644 --- a/tests/unit/contract-targets.test.mjs +++ b/tests/unit/contract-targets.test.mjs @@ -2,7 +2,7 @@ import { readFileSync } from 'node:fs'; import { describe, expect, it } from 'vitest'; import { assertNativeHost, - assertPythonEntryPoint, + assertRuntimeEntryPoint, boxTargetAdapter, boxTargetAdapters, boxTargetId, @@ -90,10 +90,11 @@ describe('target adapters', () => { expect(() => assertNativeHost(adapter, { platform: 'darwin', arch: 'arm64' })).toThrow(/must be built natively/); }); - it('refuses a scroll entry point that disagrees with the adapter layout', () => { + it('refuses a scroll entry point that disagrees with the runtime layout', () => { const adapter = boxTargetAdapter({ platform: 'windows', arch: 'x86_64', accelerator: 'cpu' }); - expect(() => assertPythonEntryPoint(adapter, 'venv/python.exe')).not.toThrow(); - expect(() => assertPythonEntryPoint(adapter, 'venv/bin/python')).toThrow(/entry point/); + expect(() => assertRuntimeEntryPoint('python', adapter, 'venv/python.exe')).not.toThrow(); + expect(() => assertRuntimeEntryPoint('python', adapter, 'venv/bin/python')) + .toThrow(/entry point/); }); }); diff --git a/tests/unit/execution-contract.test.mjs b/tests/unit/execution-contract.test.mjs index 85d166a..0564326 100644 --- a/tests/unit/execution-contract.test.mjs +++ b/tests/unit/execution-contract.test.mjs @@ -9,12 +9,14 @@ describe('execution payload prerequisites', () => { expect(() => assertExecutionFiles({ execution, adapter, + runtimeId: 'python', runtimeVersion: '3.11.15', files: new Set(['app/main.py']), })).not.toThrow(); expect(() => assertExecutionFiles({ execution, adapter, + runtimeId: 'python', runtimeVersion: '3.11.15', files: new Set(), })).toThrow(/Execution script is missing/); @@ -30,6 +32,7 @@ describe('execution payload prerequisites', () => { expect(() => assertExecutionFiles({ execution: moduleExecution, adapter: linux, + runtimeId: 'python', runtimeVersion: '3.11.15', files: new Set(['venv/lib/python3.11/site-packages/example_model/main.py']), })).not.toThrow(); @@ -38,6 +41,7 @@ describe('execution payload prerequisites', () => { expect(() => assertExecutionFiles({ execution: moduleExecution, adapter: windows, + runtimeId: 'python', runtimeVersion: '3.11.15', files: new Set(['venv/Lib/site-packages/example_model/main.py']), })).not.toThrow(); @@ -45,6 +49,7 @@ describe('execution payload prerequisites', () => { expect(() => assertExecutionFiles({ execution: { ...moduleExecution, module: 'json.tool' }, adapter: linux, + runtimeId: 'python', runtimeVersion: '3.11.15', files: new Set(['venv/lib/python3.11/json/tool.py']), })).not.toThrow(); @@ -55,6 +60,7 @@ describe('execution payload prerequisites', () => { expect(() => assertExecutionFiles({ execution: { kind: 'python-module', module: 'missing.main', defaultArgs: [] }, adapter, + runtimeId: 'python', runtimeVersion: '3.12.4', files: new Set(['venv/bin/python']), })).toThrow(/Execution module is not discoverable/); diff --git a/tests/unit/llm-demo.test.mjs b/tests/unit/llm-demo.test.mjs index ac0f466..4322680 100644 --- a/tests/unit/llm-demo.test.mjs +++ b/tests/unit/llm-demo.test.mjs @@ -8,7 +8,7 @@ import { auditScroll } from '../../src/build/audit.mjs'; import { readScroll } from '../../src/build/scroll.mjs'; import { configureWorkspace, resetWorkspace } from '../../src/build/workspace.mjs'; import { boxTargetAdapter, condaSubdir } from '../../src/contract/targets.mjs'; -import { IMPLICIT_RUNTIME_ID, runtimeAdapter } from '../../src/contract/runtimes.mjs'; +import { runtimeAdapter } from '../../src/contract/runtimes.mjs'; const root = fileURLToPath(new URL('../..', import.meta.url)); const example = join(root, 'examples', 'llm-demo'); @@ -126,7 +126,7 @@ describe('published local LLM demo box', () => { scrollId: null, target: null, condaDependencyLicenseAudit: null, - pythonEntryPoint: null, + runtime: null, environment: null, })); @@ -152,11 +152,12 @@ describe('published local LLM demo box', () => { expect(asset.url, target).toContain(`/resolve/${modelRevision}/`); expect(asset.url, target).not.toContain('/resolve/main/'); expect(asset.relativePath, target).toBe( - 'model-cache/llm-demo/smollm2-1.7b-instruct-q4_k_m.gguf', + 'cache/llm-demo/smollm2-1.7b-instruct-q4_k_m.gguf', ); expect(asset.sizeBytes, target).toBe(modelSizeBytes); expect(asset.sha256, target).toBe(modelSha256); - expect(scroll.weights, target).toBe('embed'); + // Embedded, which is now what saying nothing means: the box installs with no network. + expect(asset.embed, target).toBeUndefined(); } }); @@ -190,8 +191,10 @@ describe('published local LLM demo box', () => { for (const { target, scroll } of await scrolls()) { const manifest = await readFile(join(example, target, 'pixi.toml'), 'utf8'); expect(manifest, target).toContain(`platforms = ["${condaSubdir(scroll.target)}"]`); - expect(scroll.pythonEntryPoint, target) - .toBe(runtimeAdapter(IMPLICIT_RUNTIME_ID).layout(boxTargetAdapter(scroll.target)).entryPoint); + // Derived rather than declared: the runtime's layout admits exactly one value per target, so + // an example that wrote one down would only be restating what reading the scroll fills in. + expect(scroll.runtime.entryPoint, target) + .toBe(runtimeAdapter(scroll.runtime.id).layout(boxTargetAdapter(scroll.target)).entryPoint); } }); diff --git a/tests/unit/package-surface.test.mjs b/tests/unit/package-surface.test.mjs index 71e08b8..3731f53 100644 --- a/tests/unit/package-surface.test.mjs +++ b/tests/unit/package-surface.test.mjs @@ -153,7 +153,7 @@ describe('the package surface', () => { const scroll = await import('scrollcase/contract/schema/scroll.schema.json', { with: { type: 'json' } }); const fixture = await import('scrollcase/contract/fixtures/target-id-contract.json', { with: { type: 'json' } }); expect(schema.default.$id).toMatch(/target\.schema\.json$/); - expect(scroll.default.$id).toBe('https://scrollcase.dev/schema/v2/scroll.schema.json'); + expect(scroll.default.$id).toBe('https://scrollcase.dev/schema/v3/scroll.schema.json'); expect(fixture.default.valid.length).toBeGreaterThan(0); }); }); diff --git a/tests/unit/scroll-authoring.test.mjs b/tests/unit/scroll-authoring.test.mjs index 566d653..2df3e50 100644 --- a/tests/unit/scroll-authoring.test.mjs +++ b/tests/unit/scroll-authoring.test.mjs @@ -23,8 +23,7 @@ const TARGET = { platform: 'macos', arch: 'aarch64', accelerator: 'metal' }; const BASE = { boxId: 'example-model', target: TARGET, - modelId: 'example-org-example-model', - runtimeId: 'example-model-runtime', + labels: { model: 'example-org-example-model' }, version: '1.0.0', scrollVersion: '1.0.0', sourceRevision: 'upstream-v1', @@ -32,7 +31,6 @@ const BASE = { pixiVersion: '0.73.0', compatibility: { minHostAppVersion: '1.0.0' }, assetBaseUrl: 'https://assets.example.org', - weights: 'embed', }; describe('scroll authoring', () => { @@ -62,7 +60,7 @@ describe('scroll authoring', () => { expect(result.scrollRef).toBe('example-model/macos-aarch64-metal'); expect(result.scroll.execution).toBeUndefined(); expect(result.scroll.localFiles).toBeUndefined(); - expect(result.scroll.$schema).toBe('https://scrollcase.dev/schema/v2/scroll.schema.json'); + expect(result.scroll.$schema).toBe('https://scrollcase.dev/schema/v3/scroll.schema.json'); expect(await readScroll(result.scrollRef)).toMatchObject({ scroll: { boxId: BASE.boxId, target: TARGET, pixiVersion: BASE.pixiVersion }, }); @@ -107,13 +105,13 @@ describe('scroll authoring', () => { // Four questions, not nine: identity, provenance, where it will be published, how it runs. expect(asked).toEqual([...answers.keys()]); - // The weights mode is not among the menus. It decides where declared assets live, and a scroll - // packaging plain Python declares none — so it is a flag, and the scroll takes the default. + // Labels are not among the menus, and there is nothing to derive: Scrollcase reads none of + // them, so prompting for one would be asking the author to fill in a field on the tool's + // behalf. A generated scroll carries none. expect(chosen).toEqual(['execution kind']); - expect(options.weights).toBe('embed'); - expect(result.scroll.weights).toBeUndefined(); - expect(options.modelId).toBe(BASE.boxId); - expect(options.runtimeId).toBe(`${BASE.boxId}-runtime`); + expect(options.labels).toEqual({}); + expect(result.scroll.labels).toBeUndefined(); + expect(result.scroll.runtime).toEqual({ id: 'python', version: DEFAULT_PYTHON_VERSION }); expect(options.version).toBe('1.0.0'); expect(options.pythonVersion).toBe(DEFAULT_PYTHON_VERSION); expect(options.pixiVersion).toBe(BASE.pixiVersion); @@ -128,7 +126,7 @@ describe('scroll authoring', () => { it('pins the pixi that is installed, since a build refuses any other', async () => { const options = await collectNewScrollOptions( new Map([['box-id', 'example-model'], ['source-revision', 'upstream-v1'], - ['asset-base-url', 'https://assets.example.org'], ['weights', 'embed'], + ['asset-base-url', 'https://assets.example.org'], ['execution', 'library-only'], ['target', 'macos-aarch64-metal']]), { terminal: false, probe: () => ({ path: 'pixi', version: '9.9.9' }) }, ); @@ -219,11 +217,11 @@ describe('scroll authoring', () => { }); const written = JSON.parse(await readFile(join(result.scrollDir, 'scroll.json'), 'utf8')); - expect(written.pythonEntryPoint).toBeUndefined(); - expect(written.modelCacheSubdir).toBeUndefined(); + expect(written.runtime.entryPoint).toBeUndefined(); + expect(written.cacheSubdir).toBeUndefined(); const { scroll } = await readScroll(result.scrollRef); - expect(scroll.pythonEntryPoint).toBe('venv/python.exe'); - expect(scroll.modelCacheSubdir).toBe('model-cache/example-model'); + expect(scroll.runtime.entryPoint).toBe('venv/python.exe'); + expect(scroll.cacheSubdir).toBe('cache/example-model'); expect(await readFile(join(result.scrollDir, 'pixi.toml'), 'utf8')) .toContain('platforms = ["win-64"]'); }); @@ -243,8 +241,8 @@ describe('scroll authoring', () => { await writeFile(join(result.scrollDir, 'scroll.json'), `${JSON.stringify({ ...minimal, scrollVersion: '1.0.0', - pythonEntryPoint: 'venv/bin/python', - modelCacheSubdir: 'model-cache/example-model', + runtime: { ...minimal.runtime, entryPoint: 'venv/bin/python' }, + cacheSubdir: 'cache/example-model', assets: [], selfTest: { ...minimal.selfTest, files: [] }, }, null, 2)}\n`); @@ -263,10 +261,10 @@ describe('scroll authoring', () => { const scroll = JSON.parse(await readFile(join(result.scrollDir, 'scroll.json'), 'utf8')); await writeFile(join(result.scrollDir, 'scroll.json'), `${JSON.stringify({ ...scroll, - pythonEntryPoint: 'venv/python.exe', + runtime: { ...scroll.runtime, entryPoint: 'venv/python.exe' }, }, null, 2)}\n`); - await expect(readScroll(result.scrollRef)).rejects.toThrow(/must use Python entry point/); + await expect(readScroll(result.scrollRef)).rejects.toThrow(/must use entry point/); }); it('generates a runnable self-test as a Python file, not a JSON string', async () => { @@ -277,9 +275,9 @@ describe('scroll authoring', () => { executionKind: 'library-only', }); - expect(result.scroll.selfTest.pythonCode).toBeUndefined(); + expect(result.scroll.selfTest.code).toBeUndefined(); const selfTestPath = join(result.scrollDir, 'self_test.py'); - expect(result.scroll.selfTest.pythonFile) + expect(result.scroll.selfTest.script) .toBe(`scrolls/example-model/macos-aarch64-metal/self_test.py`); expect(await readFile(selfTestPath, 'utf8')).toContain('self-test ok'); expect(result.written).toContain(selfTestPath); diff --git a/tests/unit/scroll-editing.test.mjs b/tests/unit/scroll-editing.test.mjs index e16b495..727fdb5 100644 --- a/tests/unit/scroll-editing.test.mjs +++ b/tests/unit/scroll-editing.test.mjs @@ -38,14 +38,13 @@ const OTHER_TARGET = { platform: 'macos', arch: 'aarch64', accelerator: 'cpu' }; const REFERENCE = `example-model/${TARGET_ID}`; const SHARED = { - $schema: 'https://scrollcase.dev/schema/v2/scroll.schema.json', - schemaVersion: 2, + $schema: 'https://scrollcase.dev/schema/v3/scroll.schema.json', + schemaVersion: 3, boxId: 'example-model', - modelId: 'example-org-example-model', - runtimeId: 'example-model-runtime', + labels: { model: 'example-org-example-model' }, version: '1.0.0', sourceRevision: 'upstream-v1', - pythonVersion: '3.14', + runtime: { id: 'python', version: '3.14' }, pixiVersion: '0.73.0', assetBaseUrl: 'https://assets.example.org/boxes', selfTest: { imports: ['json'] }, @@ -92,21 +91,21 @@ describe('editing an existing scroll', () => { const { entry } = await addAsset({ boxId: 'example-model', target: ALL_TARGETS, - url: 'https://assets.example.org/weights.bin', - fetchImpl: servingBytes('weights'), + url: 'https://assets.example.org/data.bin', + fetchImpl: servingBytes('data'), }); // The two values nobody can write by hand, taken from the bytes rather than from the author. expect(entry).toEqual({ - url: 'https://assets.example.org/weights.bin', - relativePath: 'model-cache/example-model/weights.bin', - sizeBytes: 7, - sha256: createHash('sha256').update('weights').digest('hex'), + url: 'https://assets.example.org/data.bin', + relativePath: 'cache/example-model/data.bin', + sizeBytes: 4, + sha256: createHash('sha256').update('data').digest('hex'), }); const { scroll } = await readScroll(REFERENCE); expect(scroll.assets).toHaveLength(1); // Both targets share it, and the self-test now guards it against an over-eager prune. - expect(scroll.selfTest.files).toContain('model-cache/example-model/weights.bin'); + expect(scroll.selfTest.files).toContain('cache/example-model/data.bin'); const other = await readScroll(`example-model/${OTHER_TARGET_ID}`); expect(other.scroll.assets).toHaveLength(1); }); @@ -171,8 +170,8 @@ describe('editing an existing scroll', () => { boxId: 'example-model', target: ALL_TARGETS, field: 'assets', - relativePath: 'model-cache/absent.bin', - })).rejects.toThrow(/No asset at model-cache\/absent\.bin/); + relativePath: 'cache/absent.bin', + })).rejects.toThrow(/No asset at cache\/absent\.bin/); }); it('restores every file when an edit would leave the box unreadable', async () => { @@ -186,7 +185,7 @@ describe('editing an existing scroll', () => { target: TARGET, assets: [{ url: 'https://assets.example.org/other.bin', - relativePath: 'model-cache/example-model/weights.bin', + relativePath: 'cache/example-model/data.bin', sizeBytes: 4, sha256: 'a'.repeat(64), }], @@ -195,8 +194,8 @@ describe('editing an existing scroll', () => { await expect(addAsset({ boxId: 'example-model', target: ALL_TARGETS, - url: 'https://assets.example.org/weights.bin', - fetchImpl: servingBytes('weights'), + url: 'https://assets.example.org/data.bin', + fetchImpl: servingBytes('data'), })).rejects.toThrow(/both claim that path/); expect(await readFile(basePath, 'utf8')).toBe(before); @@ -273,11 +272,8 @@ describe('editing an existing scroll', () => { expect((await readScroll(REFERENCE)).scroll.version).toBe('2.0.0'); await expect(setScrollField({ - boxId: 'example-model', target: ALL_TARGETS, field: 'pythonEntryPoint', value: 'venv/python.exe', + boxId: 'example-model', target: ALL_TARGETS, field: 'runtime', value: 'node', })).rejects.toThrow(/not an editable scroll field/); - await expect(setScrollField({ - boxId: 'example-model', target: ALL_TARGETS, field: 'weights', value: 'maybe', - })).rejects.toThrow(/Unsupported weights/); }); it('offers editable fields from the schema, never the derived or structural ones', async () => { @@ -285,7 +281,7 @@ describe('editing an existing scroll', () => { expect(names).toContain('version'); expect(names).toContain('assetBaseUrl'); - for (const excluded of ['boxId', 'target', 'pythonEntryPoint', 'schemaVersion', 'extends', 'assets']) { + for (const excluded of ['boxId', 'target', 'runtime', 'schemaVersion', 'extends', 'assets']) { expect(names, excluded).not.toContain(excluded); } }); @@ -315,8 +311,8 @@ describe('editing an existing scroll', () => { await addFile({ boxId: 'example-model', target: ALL_TARGETS, sourcePath: 'NOTICE.md' }); const base = JSON.parse(await readFile(join(boxDir, 'scroll.json'), 'utf8')); base.assets = [{ - url: 'https://assets.example.org/weights.bin', - relativePath: 'model-cache/example-model/weights.bin', + url: 'https://assets.example.org/data.bin', + relativePath: 'cache/example-model/data.bin', sizeBytes: 7, sha256: 'a'.repeat(64), }]; @@ -331,8 +327,8 @@ describe('editing an existing scroll', () => { const { boxDir } = await splitBox(); const base = JSON.parse(await readFile(join(boxDir, 'scroll.json'), 'utf8')); base.assets = [{ - url: 'https://assets.example.org/weights.bin', - relativePath: 'model-cache/example-model/weights.bin', + url: 'https://assets.example.org/data.bin', + relativePath: 'cache/example-model/data.bin', sizeBytes: 7, sha256: 'a'.repeat(64), }]; @@ -346,7 +342,7 @@ describe('editing an existing scroll', () => { expect((await readScroll(REFERENCE)).scroll.assets[0].sha256).toBe('a'.repeat(64)); const accepted = await refreshScroll({ boxId: 'example-model', repin: true, fetchImpl }); - expect(accepted.repinned).toEqual(['model-cache/example-model/weights.bin']); + expect(accepted.repinned).toEqual(['cache/example-model/data.bin']); expect((await readScroll(REFERENCE)).scroll.assets[0].sha256).not.toBe('a'.repeat(64)); }); diff --git a/tests/unit/scroll-extends.test.mjs b/tests/unit/scroll-extends.test.mjs index 1fed34a..1797f5a 100644 --- a/tests/unit/scroll-extends.test.mjs +++ b/tests/unit/scroll-extends.test.mjs @@ -19,14 +19,13 @@ const REFERENCE = `example-model/${TARGET_ID}`; /** Everything the targets of one box share. A base declares no target of its own. */ const BASE = { - $schema: 'https://scrollcase.dev/schema/v2/scroll.schema.json', - schemaVersion: 2, + $schema: 'https://scrollcase.dev/schema/v3/scroll.schema.json', + schemaVersion: 3, boxId: 'example-model', - modelId: 'example-org-example-model', - runtimeId: 'example-model-runtime', + labels: { model: 'example-org-example-model' }, version: '1.0.0', sourceRevision: 'upstream-v1', - pythonVersion: '3.14', + runtime: { id: 'python', version: '3.14' }, pixiVersion: '0.73.0', assetBaseUrl: 'https://assets.example.org/boxes', selfTest: { imports: ['json'] }, @@ -34,7 +33,7 @@ const BASE = { const asset = (name, hash) => ({ url: `https://assets.example.org/${name}`, - relativePath: `model-cache/${name}`, + relativePath: `cache/${name}`, sizeBytes: 4, sha256: hash.repeat(64), }); @@ -68,7 +67,7 @@ describe('joining a base scroll with a target fragment', () => { expect(scroll.boxId).toBe('example-model'); expect(scroll.target).toEqual(TARGET); - expect(scroll.pythonVersion).toBe('3.14'); + expect(scroll.runtime.version).toBe('3.14'); // The joined scroll extends nothing: it is the whole scroll, not half of one. expect(scroll.extends).toBeUndefined(); expect(scroll.scrollId).toBe(`example-model-${TARGET_ID}`); @@ -114,23 +113,23 @@ describe('joining a base scroll with a target fragment', () => { // A target that adds one asset must not lose the shared ones. expect(scroll.assets.map(({ relativePath }) => relativePath)) - .toEqual(['model-cache/shared.bin', 'model-cache/metal.bin']); + .toEqual(['cache/shared.bin', 'cache/metal.bin']); expect(scroll.localFiles.map(({ relativePath }) => relativePath)) .toEqual(['NOTICE.md', 'metal.py']); }); it('refuses two entries claiming one payload path', async () => { await family( - { ...BASE, assets: [asset('weights.bin', 'a')] }, + { ...BASE, assets: [asset('data.bin', 'a')] }, { extends: '../scroll.json', target: TARGET, - assets: [{ ...asset('weights.bin', 'b'), url: 'https://assets.example.org/other.bin' }], + assets: [{ ...asset('data.bin', 'b'), url: 'https://assets.example.org/other.bin' }], }, ); await expect(readScroll(REFERENCE)) - .rejects.toThrow(/asset and the asset at model-cache\/weights\.bin both claim that path/); + .rejects.toThrow(/asset and the asset at cache\/data\.bin both claim that path/); }); it('refuses an asset and a local file claiming one path, whichever half declared them', async () => { @@ -139,13 +138,13 @@ describe('joining a base scroll with a target fragment', () => { { extends: '../scroll.json', target: TARGET, - localFiles: [{ sourcePath: 'shim.py', relativePath: 'model-cache/shim.py' }], + localFiles: [{ sourcePath: 'shim.py', relativePath: 'cache/shim.py' }], }, ); // The conflict is about the destination, not about which list an author wrote it in. await expect(readScroll(REFERENCE)) - .rejects.toThrow(/asset and the local file at model-cache\/shim\.py both claim that path/); + .rejects.toThrow(/asset and the local file at cache\/shim\.py both claim that path/); }); it('joins string lists and drops repeats', async () => { @@ -197,20 +196,20 @@ describe('joining a base scroll with a target fragment', () => { expect(scroll.environment).toEqual({ HF_HUB_OFFLINE: '1', LOG_LEVEL: 'debug' }); }); - it('lets a fragment replace the extra self-test Python in either spelling', async () => { + it('lets a fragment replace the extra self-test source in either spelling', async () => { await family( - { ...BASE, selfTest: { imports: ['json'], pythonFile: 'checks/shared.py' } }, + { ...BASE, selfTest: { imports: ['json'], script: 'checks/shared.py' } }, { extends: '../scroll.json', target: TARGET, - selfTest: { imports: ['json'], pythonCode: 'assert True' }, + selfTest: { imports: ['json'], code: 'assert True' }, }, ); const { scroll } = await readScroll(REFERENCE); // One logical slot, two spellings: keeping both would produce a scroll the schema refuses. - expect(scroll.selfTest.pythonCode).toBe('assert True'); - expect(scroll.selfTest.pythonFile).toBeUndefined(); + expect(scroll.selfTest.code).toBe('assert True'); + expect(scroll.selfTest.script).toBeUndefined(); }); it('refuses a base that declares a target', async () => { @@ -257,9 +256,9 @@ describe('joining a base scroll with a target fragment', () => { ...BASE, compatibility: { minHostAppVersion: '1.0.0' }, environment: { HF_HUB_OFFLINE: '1' }, - assets: [asset('weights.bin', 'a')], + assets: [asset('data.bin', 'a')], prunePaths: ['venv/share/doc'], - selfTest: { imports: ['json'], files: ['model-cache/weights.bin'] }, + selfTest: { imports: ['json'], files: ['cache/data.bin'] }, }; await family(shared, { extends: '../scroll.json', target: TARGET }); const { scroll: split } = await readScroll(REFERENCE); diff --git a/tests/unit/sentiment-demo.test.mjs b/tests/unit/sentiment-demo.test.mjs index 4cf3c31..ceb64ae 100644 --- a/tests/unit/sentiment-demo.test.mjs +++ b/tests/unit/sentiment-demo.test.mjs @@ -80,7 +80,7 @@ describe('published sentiment demo box', () => { scrollId: null, target: null, condaDependencyLicenseAudit: null, - pythonEntryPoint: null, + runtime: null, })); expect(new Set(normalised).size).toBe(1); diff --git a/tests/unit/signing.test.mjs b/tests/unit/signing.test.mjs index 1669cd4..1e06611 100644 --- a/tests/unit/signing.test.mjs +++ b/tests/unit/signing.test.mjs @@ -36,7 +36,7 @@ if (mode === 'substitute') payload = Buffer.from('{"substituted":true}\\n'); const metadata = JSON.parse(readFileSync(publicPath, 'utf8')); const signature = sign(null, payload, createPrivateKey(readFileSync(privatePath))); const document = { - schemaVersion: 2, + schemaVersion: 3, payloadEncoding: 'base64-json-utf8', payloadBase64: payload.toString('base64'), payloadSha256: createHash('sha256').update(payload).digest('hex'), diff --git a/tests/unit/v2-migration.test.mjs b/tests/unit/v3-migration.test.mjs similarity index 73% rename from tests/unit/v2-migration.test.mjs rename to tests/unit/v3-migration.test.mjs index 725807a..b652aad 100644 --- a/tests/unit/v2-migration.test.mjs +++ b/tests/unit/v3-migration.test.mjs @@ -15,25 +15,28 @@ import { resolveWorkspace } from '../../src/build/workspace.mjs'; const root = fileURLToPath(new URL('../..', import.meta.url)); -describe('the v2-only contract boundary', () => { - it('publishes only the canonical v2 scroll schema', async () => { - expect(BOX_SCHEMA_VERSION).toBe(2); +describe('the v3-only contract boundary', () => { + it('publishes only the canonical v3 scroll schema', async () => { + expect(BOX_SCHEMA_VERSION).toBe(3); const schema = JSON.parse(await readFile(schemaUrl('scroll'), 'utf8')); - expect(schema.$id).toBe('https://scrollcase.dev/schema/v2/scroll.schema.json'); - expect(schema.properties.schemaVersion.const).toBe(2); + expect(schema.$id).toBe('https://scrollcase.dev/schema/v3/scroll.schema.json'); + expect(schema.properties.schemaVersion.const).toBe(3); }); - it('rejects a v1 signed document with the migration remedy', () => { + // Both superseded versions, each named. Published v1 and v2 boxes stay historical artefacts and + // there is no dual-read path anywhere: a reader holding one is told which rebuild it needs, not + // handed a guess at what the document meant. + it.each([1, 2])('rejects a v%i signed document with the migration remedy', (schemaVersion) => { const bytes = Buffer.from('{}'); const document = { - schemaVersion: 1, + schemaVersion, payloadEncoding: 'base64-json-utf8', payloadBase64: bytes.toString('base64'), payloadSha256: createHash('sha256').update(bytes).digest('hex'), signatures: [{ algorithm: 'ed25519', keyId: 'test', signatureBase64: 'test' }], }; expect(() => decodeSignedDocument(document)) - .toThrow('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); + .toThrow(`Unsupported schemaVersion ${schemaVersion}; rebuild this box with Scrollcase v3.`); }); it('closes the channel vocabulary to nightly, beta, and stable', () => { @@ -43,7 +46,7 @@ describe('the v2-only contract boundary', () => { describe('canonical scroll workspace names', () => { it('uses scrolls and exposes no legacy compatibility field', () => { - const cwd = join(tmpdir(), 'scrollcase-v2-workspace'); + const cwd = join(tmpdir(), 'scrollcase-v3-workspace'); const workspace = resolveWorkspace({ cwd }); expect(workspace.scrollsDir).toBe(join(cwd, 'scrolls')); const legacyField = ['re', 'cipesDir'].join(''); From 46d5c91e44ebff2a8582abffdc3937a211a6b6b5 Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:23:28 +0200 Subject: [PATCH 05/22] Rewrite the documentation for version 3 The schema route moves to /schema/v3/ everywhere it is derived or asserted, and box-format.md gains a table of exactly what changed between the two versions and why. managing-weights.md becomes managing-assets.md: the guide was built around a box-wide switch that no longer exists, and its subject was never weights in the first place. It now covers per-asset embed, the executable declaration, and what a deferred asset puts in the release. The white paper drops the weights-mode glossary entry for a deferred-asset one, names the new contract exports, and records that --weights was removed rather than replaced. --- docs/.vitepress/api-catalog.mjs | 8 +- docs/.vitepress/config.mts | 2 +- docs/.vitepress/llms.mjs | 2 +- docs/concepts/architecture.md | 4 +- docs/concepts/design-decisions.md | 8 +- docs/concepts/security-and-trust.md | 4 +- docs/concepts/why-pixi.md | 4 +- docs/demos/box-dev-demo.md | 2 +- docs/demos/llm-box-demo.md | 4 +- docs/demos/sentiment-demo.md | 6 +- docs/getting-started/quickstart.md | 12 +- docs/guides/accelerator-parity.md | 2 +- docs/guides/distributing-boxes.md | 6 +- ...managing-weights.md => managing-assets.md} | 122 ++++++----- docs/guides/offline-airgap.md | 6 +- docs/guides/packaging-cuda.md | 25 ++- docs/guides/signing-and-custody.md | 2 +- .../public/schema/v2/box-manifest.schema.json | 136 ------------- .../public/schema/v3/box-manifest.schema.json | 68 +++++++ .../{v2 => v3}/channel-manifest.schema.json | 6 +- .../schema/{v2 => v3}/execution.schema.json | 52 ++++- .../{v2 => v3}/release-manifest.schema.json | 176 +++++++++++----- .../revocations-manifest.schema.json | 6 +- .../schema/{v2 => v3}/scroll.schema.json | 179 +++++++++++----- .../{v2 => v3}/signed-document.schema.json | 4 +- .../schema/{v2 => v3}/target.schema.json | 2 +- docs/reference/api.md | 9 +- docs/reference/box-format.md | 98 +++++---- docs/reference/cli.md | 56 ++--- docs/reference/schemas.md | 14 +- docs/reference/scroll.md | 176 +++++++++------- docs/white-paper.md | 192 ++++++++++-------- scripts/sync-docs-schemas.mjs | 2 +- scripts/verify-built-docs.mjs | 2 +- src/build/archive.d.mts | 4 +- tests/unit/cli-target-choice.test.mjs | 3 +- tests/unit/docs-contract.test.mjs | 6 +- tests/unit/docs-markdown-negotiation.test.mjs | 2 +- 38 files changed, 829 insertions(+), 583 deletions(-) rename docs/guides/{managing-weights.md => managing-assets.md} (55%) delete mode 100644 docs/public/schema/v2/box-manifest.schema.json create mode 100644 docs/public/schema/v3/box-manifest.schema.json rename docs/public/schema/{v2 => v3}/channel-manifest.schema.json (93%) rename docs/public/schema/{v2 => v3}/execution.schema.json (58%) rename docs/public/schema/{v2 => v3}/release-manifest.schema.json (61%) rename docs/public/schema/{v2 => v3}/revocations-manifest.schema.json (92%) rename docs/public/schema/{v2 => v3}/scroll.schema.json (62%) rename docs/public/schema/{v2 => v3}/signed-document.schema.json (94%) rename docs/public/schema/{v2 => v3}/target.schema.json (96%) diff --git a/docs/.vitepress/api-catalog.mjs b/docs/.vitepress/api-catalog.mjs index 2fdf7da..a00b77e 100644 --- a/docs/.vitepress/api-catalog.mjs +++ b/docs/.vitepress/api-catalog.mjs @@ -28,13 +28,13 @@ export const CATALOG_PATH = '/.well-known/api-catalog'; * honest href — constructing one from the filename would let the two disagree. */ async function schemaTargets(outDir, hostname) { - const directory = join(outDir, 'schema', 'v2'); + const directory = join(outDir, 'schema', 'v3'); const names = (await readdir(directory)).filter((name) => name.endsWith('.schema.json')).sort(); const targets = []; for (const name of names) { const schema = JSON.parse(await readFile(join(directory, name), 'utf8')); targets.push({ - href: schema.$id ?? `${hostname}/schema/v2/${name}`, + href: schema.$id ?? `${hostname}/schema/v3/${name}`, type: 'application/schema+json', title: schema.title ?? name, }); @@ -48,12 +48,12 @@ export async function writeApiCatalog({ outDir, hostname }) { { anchor: `${hostname}${CATALOG_PATH}`, item: [ - { href: `${hostname}/schema/v2/` }, + { href: `${hostname}/schema/v3/` }, { href: `${hostname}/reference/cli` }, ], }, { - anchor: `${hostname}/schema/v2/`, + anchor: `${hostname}/schema/v3/`, 'service-desc': await schemaTargets(outDir, hostname), 'service-doc': [ { href: `${hostname}/reference/schemas`, type: 'text/html', title: 'JSON Schemas' }, diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 121a981..f6df0fc 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -95,7 +95,7 @@ const sidebar = [ link: '/guides', collapsed: false, items: [ - { text: 'Managing Model Weights', link: '/guides/managing-weights' }, + { text: 'Managing Assets', link: '/guides/managing-assets' }, { text: 'Packaging CUDA Boxes', link: '/guides/packaging-cuda' }, { text: 'Accelerator Parity', link: '/guides/accelerator-parity' }, { text: 'Signing & Key Custody', link: '/guides/signing-and-custody' }, diff --git a/docs/.vitepress/llms.mjs b/docs/.vitepress/llms.mjs index 2624251..a469d35 100644 --- a/docs/.vitepress/llms.mjs +++ b/docs/.vitepress/llms.mjs @@ -112,7 +112,7 @@ async function schemaLinks(outDir, hostname) { const names = (await readdir(join(outDir, 'schema', 'v2'))) .filter((name) => name.endsWith('.schema.json')) .sort(); - return names.map((name) => `- [${name}](${hostname}/schema/v2/${name})`); + return names.map((name) => `- [${name}](${hostname}/schema/v3/${name})`); } catch { return []; } diff --git a/docs/concepts/architecture.md b/docs/concepts/architecture.md index a53c0e1..1b7c272 100644 --- a/docs/concepts/architecture.md +++ b/docs/concepts/architecture.md @@ -87,7 +87,7 @@ Each step earns its position: **Validate first.** The complete nested scroll is checked against the shipped schemas before a tool is probed, a fetch is made, or build state is mutated. Identity, target/entry-point, -weights/archive policy, native host, lock presence, and Git state follow in that order. +the declared runtime, native host, lock presence, and Git state follow in that order. **Install, never resolve.** `pixi install --frozen` materialises exactly the locked packages without touching or re-checking the lock, so what ships is byte-for-byte what was reviewed. @@ -107,7 +107,7 @@ reviewed copy. A licence problem is a legal problem, and it is cheaper to hit it expensive checks. **Self-test with the box's own interpreter.** The builder runs post-prune file assertions, the -target assertion, imports, and optional scroll `pythonCode`. Schema version 2 signs the target +target assertion, imports, and optional scroll `code`. Schema version 2 signs the target assertion and import subset for a consumer to repeat; the richer scroll-only checks are not misrepresented as consumer checks. diff --git a/docs/concepts/design-decisions.md b/docs/concepts/design-decisions.md index 80086fe..a70d899 100644 --- a/docs/concepts/design-decisions.md +++ b/docs/concepts/design-decisions.md @@ -295,7 +295,7 @@ A scroll is a file a person writes and maintains by hand, and several of its fie restating something the file already said. `pythonEntryPoint` is the clearest case: a target admits exactly one interpreter path and the reader rejected every other value, so requiring the field obliged the author to type the one string that was already implied — and to type it again for every -target of the same box. `scrollVersion`, `compatibility`, `modelCacheSubdir`, `assets` and +target of the same box. `scrollVersion`, `compatibility`, `cacheSubdir`, `assets` and `selfTest.files` were the same kind of obligation in weaker form. Those fields are now optional and derived when the scroll is read. Derivation happens in one place, @@ -326,9 +326,9 @@ writing. A pinned file that drifts still fails the build. ### A self-test belongs in a file -`selfTest.pythonCode` puts Python inside a JSON string, with escaped newlines and no syntax +`selfTest.code` puts Python inside a JSON string, with escaped newlines and no syntax highlighting, no linter and no readable diff. It suits a single assertion and nothing more. -`selfTest.pythonFile` names a file in the project instead; it is read at build time and executed +`selfTest.script` names a file in the project instead; it is read at build time and executed from the payload root, so it can read what the box ships and import what it packs. The two are mutually exclusive, and `new scroll` generates the file rather than leaving the field empty. @@ -522,7 +522,7 @@ The public-contract audit resolved six implementation choices: - Public schema URLs are deterministic copies of `src/contract/schema/`, guarded byte for byte. - Verification compares all security-, identity-, target-, asset-policy-, self-test-, and provenance fields duplicated by schema version 2. -- Consumer self-test is documented as the signed import subset; scroll `pythonCode` and file +- Consumer self-test is documented as the signed import subset; scroll `code` and file assertions stay builder-only until a future wire version can carry them. - Scroll structure is validated at runtime from the shipped schemas by a dependency-free internal validator before tool discovery or build-directory mutation. diff --git a/docs/concepts/security-and-trust.md b/docs/concepts/security-and-trust.md index a6e263a..2388876 100644 --- a/docs/concepts/security-and-trust.md +++ b/docs/concepts/security-and-trust.md @@ -58,7 +58,7 @@ together. Never commit the private key under `.scrollcase/keys/`. 4. List ZIP entries defensively, rejecting traversal, links, and special entries before extraction. 5. Require `box.json` and recursively compare every shared schema-v2 field: identity and version, complete target, entry point, cache subdirectory, declared environment, consumer self-test, - weights/assets policy, and provenance. + the deferred-asset list, and provenance. 6. Require the declared interpreter entry inside the archive. 7. With `--self-test`, require a matching native host, extract to a temporary directory, compare the logical extracted payload size, and run the signed import check with the box's interpreter @@ -71,7 +71,7 @@ caller-supplied values are not masked. None of these diagnostics is a sandbox or The declaration is authenticated format data. The report is local consumer output that changes with the process inspecting or running the box. -The builder's richer scroll checks—optional `pythonCode` and post-prune file assertions—are not +The builder's richer scroll checks—optional `code` and post-prune file assertions—are not part of the signed release and therefore cannot be repeated by a consumer. See [The Scroll](/reference/scroll#self-test). diff --git a/docs/concepts/why-pixi.md b/docs/concepts/why-pixi.md index 20de991..9bf7fe0 100644 --- a/docs/concepts/why-pixi.md +++ b/docs/concepts/why-pixi.md @@ -95,8 +95,8 @@ Being honest about the trade-offs: the licence parser reads both, but the further you go from conda-forge the weaker the native dependency metadata gets. - **Archives are large.** A full conda prefix with a CUDA stack is measured in gigabytes. Pruning - and `--weights on-demand` exist for this — see - [Managing Model Weights](/guides/managing-weights). + and per-asset `"embed": false` exist for this — see + [Managing Assets](/guides/managing-assets). - **Builds are native.** No cross-building: a Windows box is built on Windows. The self-test runs the box's own interpreter, and that only proves anything on matching hardware. - **Two tools must be present.** `pixi` at the scroll's pinned version, and `conda-pack` 0.9.2. diff --git a/docs/demos/box-dev-demo.md b/docs/demos/box-dev-demo.md index 9998bc8..1ef06cb 100644 --- a/docs/demos/box-dev-demo.md +++ b/docs/demos/box-dev-demo.md @@ -75,7 +75,7 @@ git add . git commit -m "Initialize Scrollcase example" scrollcase keygen -scrollcase build example-box/linux-x86_64-cpu --weights embed +scrollcase build example-box/linux-x86_64-cpu scrollcase verify .scrollcase/dist/boxes/example-box/1.0.0/linux-x86_64-cpu/*.release.json --self-test ``` diff --git a/docs/demos/llm-box-demo.md b/docs/demos/llm-box-demo.md index 080584d..7172c3c 100644 --- a/docs/demos/llm-box-demo.md +++ b/docs/demos/llm-box-demo.md @@ -164,7 +164,7 @@ configuration file. *and* the chat template, so the scroll declares exactly **one** asset where the sentiment demo needs three — and there is no tokenizer that can drift out of step with the weights it belongs to. It is pinned to an immutable upstream commit, with the size and SHA-256 that `add asset` recorded when it -fetched the file. With `weights: embed` it is packed into the archive, so the box installs and runs +fetched the file. Embedded by default, it is packed into the archive, so the box installs and runs air-gapped. **Offline because there is no downloader, not because a variable says so.** The sentiment demo @@ -213,7 +213,7 @@ No hash is typed by hand anywhere. `scrollcase add asset` fetches the GGUF once and SHA-256 it found; the notices and the entrypoint are pinned the same way, and [`scrollcase refresh`](/reference/cli#refresh) moves those digests after a reviewed change. -The scroll declares `weights: embed`, a **4 GB RAM floor** and `execution` as a `python-script`. That +The scroll embeds the asset, declares a **4 GB RAM floor**, and names `execution` as a `python-script`. That floor is arithmetic rather than a guess: the quantised weights occupy about 1.0 GB and the attention cache adds 384 MiB at the 2048-token context the entrypoint asks for, which lands around 1.5–1.8 GB resident. It is a fact a consumer can check *before* unpacking a gigabyte. diff --git a/docs/demos/sentiment-demo.md b/docs/demos/sentiment-demo.md index fea26c3..6975340 100644 --- a/docs/demos/sentiment-demo.md +++ b/docs/demos/sentiment-demo.md @@ -118,7 +118,7 @@ against a public key you obtain independently of the download. **The model travels inside the box.** The ONNX weights, the tokenizer and the config are declared as assets pinned to an immutable upstream commit, each with its size and SHA-256. The build fetches -them once and fails if a byte moved. With `weights: embed` they are packed into the archive, so +them once and fails if a byte moved. Embedded by default, they are packed into the archive, so the box installs and runs air-gapped. **Defence in depth against a stray download.** The scroll declares `HF_HUB_OFFLINE=1`, @@ -135,7 +135,7 @@ multi-gigabyte build. **A self-test at two levels.** `selfTest.imports` is the part schema v2 signs, which is why `verify --self-test` can repeat it later with the box's own interpreter. `files` and the optional -`pythonCode` block stay builder-only: add `pythonCode` and the build runs real predictions and +`code` block stay builder-only: add `code` and the build runs real predictions and refuses to sign a box that answers wrong. Proof of real inference for a box you downloaded is what `run` gives you. @@ -171,7 +171,7 @@ No hash is typed by hand anywhere. `scrollcase add asset` fetches each model fil the size and SHA-256 it found; the notices and the entrypoint are pinned, and [`scrollcase refresh`](/reference/cli#refresh) moves those digests after a reviewed change. -The scroll declares `weights: embed`, a 2 GB RAM floor, and `execution` as a `python-script`. +The scroll embeds its assets, declares a 2 GB RAM floor, and names `execution` as a `python-script`. ## Measured diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 30bff67..72665e3 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -134,7 +134,7 @@ scrollcase new scroll The generated example is already ready for the remaining walkthrough steps, so you can skip this command for a first build. Use the wizard for real project metadata. It asks four questions — the complete target, the box id, the upstream revision of what you are packaging, and the base URL -boxes will be published under — plus menus for weights mode and execution kind. Each one is printed +boxes will be published under — plus a menu for the execution kind. Each one is printed as its own block: a blank line, the field's name, one line saying what the field is, then the answer typed after ` ↳ `. Everything else has a default and is available as a flag. A blank answer to a required question repeats it rather than ending the session. @@ -244,14 +244,14 @@ scrollcase verify .scrollcase/dist/boxes/example-box/1.0.0/macos-aarch64-metal/* `verify` mirrors the format checks available to an installing client: trusted signature, archive size and SHA-256, safe entry names, recursive agreement between `box.json` and the signed release, -and the declared interpreter. With `--self-test` it extracts to a temporary directory and imports -the signed modules **with the box's own Python**. Scroll-only `pythonCode` and file assertions ran -on the builder but are not carried by the signed release. +and the declared interpreter. With `--self-test` it extracts to a temporary directory and runs the +signed probe **with the box's own runtime**. Scroll-only `code` and file assertions ran on the +builder but are not carried by the signed release. ## Where to go next -- Package something real: declare model weights and data files — - [Managing Model Weights](/guides/managing-weights). +- Package something real: declare model weights and data files, and choose what ships inside the archive — + [Managing Assets](/guides/managing-assets). - Understand every field you just used: [The Scroll](/reference/scroll) and [CLI Commands](/reference/cli). - Review dependency licences before building: run `scrollcase audit ` — see diff --git a/docs/guides/accelerator-parity.md b/docs/guides/accelerator-parity.md index 6a6d64f..4ba74b6 100644 --- a/docs/guides/accelerator-parity.md +++ b/docs/guides/accelerator-parity.md @@ -149,5 +149,5 @@ record it in its own pipeline. Parity needs at least two accelerators available on the build machine — a CPU/CUDA gate needs a GPU present. If a box has only one meaningful accelerator, or your CI cannot provide the second -device, leave `parity` out and lean on the self-test's `pythonCode` instead. The gate is optional +device, leave `parity` out and lean on the self-test's `code` instead. The gate is optional by design; a scroll without it is complete. diff --git a/docs/guides/distributing-boxes.md b/docs/guides/distributing-boxes.md index 020c816..5a549e1 100644 --- a/docs/guides/distributing-boxes.md +++ b/docs/guides/distributing-boxes.md @@ -88,7 +88,7 @@ separation is the point: **promoting a build never requires re-signing it**. ```jsonc { - "schemaVersion": 2, + "schemaVersion": 3, "kind": "scrollcase.box.channel", "channel": "beta", "boxId": "my-model", @@ -161,9 +161,9 @@ Whatever installs your boxes should do exactly what `scrollcase verify` does, in 5. Download the archive; check size and SHA-256 against the release. 6. Validate every entry name before final extraction. 7. Compare all shared `box.json` fields recursively against the release. -8. Run the self-test: `selfTest.pythonImports` with `pythonEntryPoint`, bounded by +8. Run the self-test: `selfTest.probe` with `runtime.entryPoint`, bounded by `selfTest.timeoutSeconds`. -9. With on-demand weights, fetch each asset and check its size and SHA-256 before first use. +9. For each entry in `assets`, fetch it and check its size and SHA-256 before first use. Running `scrollcase verify --self-test` on the build machine covers the archive and temporary extraction checks, not final installation, compatibility policy, rollout, or activation. diff --git a/docs/guides/managing-weights.md b/docs/guides/managing-assets.md similarity index 55% rename from docs/guides/managing-weights.md rename to docs/guides/managing-assets.md index edae279..0749384 100644 --- a/docs/guides/managing-weights.md +++ b/docs/guides/managing-assets.md @@ -1,13 +1,14 @@ --- -title: Managing Model Weights -description: Declare, verify, embed or defer the assets a box carries. +title: Managing Assets +description: Declare, verify, embed or defer the files a box carries. --- -# Managing Model Weights +# Managing Assets -Model weights are usually the largest thing a box carries, and the only part fetched from a -server nobody controls. Scrollcase treats them the same way it treats everything else: declared -up front, verified before use, and committed to by hash in the signed release. +An asset is usually the largest thing a box carries, and the only part fetched from a server +nobody controls — model weights, a dataset, a compiled tool. Scrollcase treats them the same way it +treats everything else: declared up front, verified before use, and committed to by hash in the +signed release. ## Declare an asset @@ -17,7 +18,7 @@ Every asset carries a URL, a destination inside the payload, a size, and a SHA-2 "assets": [ { "url": "https://huggingface.co/example-org/model/resolve/main/model.safetensors", - "relativePath": "model-cache/hello/model.safetensors", + "relativePath": "cache/hello/model.safetensors", "sizeBytes": 438012416, "sha256": "9f2b7c1e04a83d5641b0e7c28a3d95f7c9d1a4e60b8f37c25e9a4d7081da5b3f" } @@ -32,8 +33,9 @@ scrollcase add asset my-model https://…/model.safetensors ``` It also adds the payload path to `selfTest.files`, so an over-eager `prunePaths` cannot quietly drop -it. Use `--to ` to land it somewhere other than the box's model cache, and `--target` -to give it to one target only. If you would rather write the entry yourself, the two values come +it. Use `--to ` to land it somewhere other than the box's cache directory, `--target` +to give it to one target only, `--on-demand` to leave it out of the archive, and `--executable` for +a file that has to run. If you would rather write the entry yourself, the two values come from `shasum -a 256 model.safetensors` and `wc -c < model.safetensors`. Nothing enters the payload before **both** match. That is what makes a box reproducible even @@ -56,16 +58,16 @@ When the upstream artefact is a tarball or zip, declare it as an asset and then "assets": [ { "url": "https://example.org/model/weights-v1.tar.gz", - "relativePath": "model-cache/hello/weights.tar.gz", + "relativePath": "cache/hello/weights.tar.gz", "sizeBytes": 1073741824, "sha256": "4c7e…9a" } ], "assetArchives": [ { - "relativePath": "model-cache/hello/weights.tar.gz", + "relativePath": "cache/hello/weights.tar.gz", "format": "tar.gz", - "destination": "model-cache/hello", + "destination": "cache/hello", "stripComponents": 1, "removeAfterExtract": true } @@ -83,7 +85,7 @@ When the upstream artefact is a tarball or zip, declare it as an asset and then ## Compression -Weights arrive already compressed, and deflating them again is pure loss. Measured on +An asset usually arrives already compressed, and deflating it again is pure loss. Measured on incompressible bytes: level 6 runs at 47 MB/s and the archive comes out **0.03% larger** than the input, and dropping to level 1 recovers 4 MB/s because the search fails either way. Lowering the level is not the fix — not compressing is. @@ -95,7 +97,7 @@ For anything else your box carries that is already compressed — the tree an `a expanded into, a bundled corpus of JPEGs — say so: ```jsonc -"uncompressedPaths": ["model-cache/hello", "corpora/images"] +"uncompressedPaths": ["cache/hello", "corpora/images"] ``` An entry matches that path and everything beneath it. Nothing is decided by looking at the file or @@ -115,6 +117,7 @@ scrollcase add file my-model runtime/entrypoint.py ```jsonc "localFiles": [ { "sourcePath": "runtime/entrypoint.py", "relativePath": "entrypoint.py" }, + { "sourcePath": "bin/launch.sh", "relativePath": "bin/launch.sh", "executable": true }, { "sourcePath": "legal/MODEL_LICENSE.txt", "relativePath": "MODEL_LICENSE.txt", "sha256": "…" } ] ``` @@ -131,48 +134,54 @@ writing, which would otherwise fail your next build over an edit you meant to ma ## Embed or defer -This is the one real decision, and it is per build: +This is the one real decision, and it is made **per asset**, in the scroll: -| | `embed` (default) | `on-demand` | +| | `embed: true` (default) | `embed: false` | | --- | --- | --- | -| Assets live | inside the archive | on your asset host | +| The file lives | inside the archive | on your asset host | | Install needs network | no — **works air-gapped** | yes | | Archive size | large | small | | Integrity guaranteed by | the archive's own signed hash | the per-asset size + SHA-256 in the signed release | -```sh -scrollcase build my-model/linux-x86_64-cpu --weights embed # the default -scrollcase build my-model/linux-x86_64-cpu --weights on-demand +```jsonc +"assets": [ + { "url": "https://…/entrypoint-config.json", "relativePath": "cache/hello/config.json", + "sizeBytes": 786, "sha256": "27 47…c2" }, + { "url": "https://…/model.safetensors", "relativePath": "cache/hello/model.safetensors", + "sizeBytes": 438012416, "sha256": "9f2b…3f", "embed": false } +] ``` -A scroll may set `"weights": "embed" | "on-demand"` as its own default; the flag overrides it for one -build. `build` never asks: what the scroll declares is what it uses, and the mode in effect is -printed as the build starts. A scroll that says nothing takes `embed`, which is what a box with no -declared assets wants anyway. +That box ships its small config inside the archive and defers the large weights — which version 2 +could not express at all, because the choice was a single box-wide `weights` switch. There is no +build-time override, deliberately: overriding a per-asset declaration would repack the box under an +identity that no longer describes it, and `build` prints what the scroll decided rather than +offering a menu in front of it. -### What `on-demand` puts in the release +### What deferring puts in the release -The assets are left out of the archive, and their descriptors travel in the signed release and in +A deferred asset is left out of the archive, and its descriptor travels in the signed release and in `box.json`: ```jsonc -"weights": "on-demand", "assets": [ - { "url": "https://…/model.safetensors", "relativePath": "model-cache/hello/model.safetensors", + { "url": "https://…/model.safetensors", "relativePath": "cache/hello/model.safetensors", "sizeBytes": 438012416, "sha256": "9f2b…3f" } ] ``` -The declared hash is what makes deferring safe: the release **commits to exactly which bytes the -box expects**, whatever host serves them. Retrieval belongs to the caller's distribution layer, -which places each asset at `relativePath` under the box root. The official consumers do not fetch -assets; they check every materialized file's size and hash before execution. - -::: warning Two constraints -`on-demand` cannot be combined with `assetArchives` — archives are expanded at build time, so -deferring them would declare a layout that never materialises; the build fails rather than lie. -And a file listed in `selfTest.files` that is a deferred asset is legitimately absent from the -payload, so it is skipped by the post-prune check. +The list is exactly the deferred entries and nothing else — on this side of the wire, the list +*is* the statement, so there is no separate mode field to disagree with it. The declared hash is +what makes deferring safe: the release **commits to exactly which bytes the box expects**, whatever +host serves them. Retrieval belongs to the caller's distribution layer, which places each asset at +`relativePath` under the box root. The official consumers do not fetch assets; they check every +materialized file's size and hash before execution. + +::: warning Two consequences +An `assetArchives` entry has no `embed` field at all: an archive is expanded at build time, so +deferring one would declare a layout that never materialises. And a file listed in `selfTest.files` +that is a deferred asset is legitimately absent from the payload, so it is skipped by the post-prune +check. ::: ### Choosing @@ -181,9 +190,30 @@ Embedding is the default because air-gapped installation is a property worth kee project explicitly trades it away, and because it is the behaviour that surprises nobody: what you verified is what you install. -Defer when the archive would otherwise be unreasonable to move around, when the same weights are -shared by several boxes, or when your asset host is already the thing your users download from. -Then read [Offline / Air-Gapped Installs](/guides/offline-airgap) to understand what you gave up. +Defer the entries that would make the archive unreasonable to move around, that are shared by +several boxes, or that your asset host is already the thing your users download from. Then read +[Offline / Air-Gapped Installs](/guides/offline-airgap) to understand what you gave up. + +## Files that have to run + +A downloaded file arrives with no permission bits — HTTP carries content, not permissions — and a +local file is copied rather than moved, so neither has a mode to inherit. Declare it: + +```jsonc +"assets": [ + { "url": "https://…/tool", "relativePath": "bin/tool", + "sizeBytes": 4212992, "sha256": "1b82…db", "executable": true } +] +``` + +The bit is **synthesised** into the archive from that declaration and never read off the build +machine, which is what keeps two builds of one commit byte-identical whatever umask each ran under. +Extraction sets it explicitly rather than letting `open(2)` mask it away, so a box unpacked under a +restrictive umask still runs. + +The runtime's own files need no declaration: the interpreter and the console scripts a conda prefix +generates are covered by a rule the runtime carries, because a prefix generates hundreds of them and +no scroll could name them by hand. ## Keeping the box small @@ -208,16 +238,16 @@ Guard against over-pruning by listing what must survive: ```jsonc "selfTest": { "imports": ["torch", "numpy"], - "files": ["model-cache/hello/model.safetensors", "entrypoint.py"] + "files": ["cache/hello/model.safetensors", "entrypoint.py"] } ``` A box is a multi-gigabyte download for an end user, so pruning is a user-facing concern rather than tidiness — but a box that unpacks and cannot run is worse than a large one. The self-test -runs after pruning, with the box's own interpreter, precisely to catch that. +runs after pruning, with the box's own runtime, precisely to catch that. ## Where assets live inside the box -`modelCacheSubdir` names the directory holding model assets, relative to the box root -(`model-cache/hello` above). Keep asset `relativePath` values under it so an installed box has -one obvious place where its weights are, and a consumer can find them without parsing anything. +`cacheSubdir` names the directory holding the box's own large files, relative to the box root +(`cache/hello` above). Keep asset `relativePath` values under it so an installed box has one obvious +place where its data is, and a consumer can find it without parsing anything. diff --git a/docs/guides/offline-airgap.md b/docs/guides/offline-airgap.md index c391e3c..89af9e8 100644 --- a/docs/guides/offline-airgap.md +++ b/docs/guides/offline-airgap.md @@ -15,12 +15,12 @@ This guide covers what makes that true, and what to check before relying on it. | Requirement | How to satisfy it | | --- | --- | -| Assets are inside the archive | Build with `--weights embed` (the default) | +| Assets are inside the archive | Leave every `assets[].embed` at its default of `true` | | No install-time relocation step | Guaranteed by the format — see [relocation](#why-no-install-step) | | The trust anchor is on the isolated machine | Copy `signing-public.json` across, out of band | | The verifier runs offline | `scrollcase verify` never touches the network | -The one thing that breaks air-gapped installation is `--weights on-demand`, which deliberately +The one thing that breaks air-gapped installation is `"embed": false` on an asset, which deliberately leaves the assets out for the caller's distribution layer to materialize. That is why `embed` is the default: air-gapped installation is a property worth keeping unless a project explicitly trades it away. @@ -28,7 +28,7 @@ trades it away. ## Build on the connected side ```sh -scrollcase build my-model/linux-x86_64-cpu --weights embed +scrollcase build my-model/linux-x86_64-cpu scrollcase verify .scrollcase/dist/boxes/my-model/1.0.0/linux-x86_64-cpu/*.release.json --self-test ``` diff --git a/docs/guides/packaging-cuda.md b/docs/guides/packaging-cuda.md index edbe8ed..7d9575a 100644 --- a/docs/guides/packaging-cuda.md +++ b/docs/guides/packaging-cuda.md @@ -24,11 +24,10 @@ only CUDA ABI the contract accepts. ```json { - "schemaVersion": 2, + "schemaVersion": 3, "scrollVersion": "1.0.0", "boxId": "my-model", - "modelId": "example-org-my-model", - "runtimeId": "my-model-runtime", + "labels": { "model": "example-org/my-model" }, "version": "1.0.0", "sourceRevision": "my-model-v1.2.0", "target": { @@ -42,12 +41,12 @@ only CUDA ABI the contract accepts. "minNvidiaDriverVersion": "550.54.14", "minRamGb": 16 }, - "pythonVersion": "3.14", + "runtime": { "id": "python", "version": "3.14" }, "pixiVersion": "0.73.0", "assetBaseUrl": "https://assets.example.org/boxes", "selfTest": { "imports": ["torch"], - "pythonFile": "scrolls/my-model/linux-x86_64-cuda12.4/self_test.py" + "script": "scrolls/my-model/linux-x86_64-cuda12.4/self_test.py" } } ``` @@ -60,7 +59,7 @@ Three things are CUDA-specific: the installing host to check. Scrollcase never interprets it. 3. **A self-test that actually exercises the GPU** — see below. -`pythonEntryPoint`, `modelCacheSubdir` and an empty `assets` list are left out: the target and the +`pythonEntryPoint`, `cacheSubdir` and an empty `assets` list are left out: the target and the box identity already determine them, and they are filled in when the scroll is read. A box that ships both a CUDA and a CPU target should keep what they share in one [base scroll](/reference/scroll#one-box-several-targets), with `cudaVersion`, @@ -139,8 +138,8 @@ assert torch.cuda.is_available(), "CUDA runtime not usable inside the box" assert torch.version.cuda.startswith("12.4"), f"built against CUDA {torch.version.cuda}" ``` -A single assertion can also go inline as `selfTest.pythonCode`, but anything longer belongs in a -file the editor and the linter can see — which is what `selfTest.pythonFile` names. +A single assertion can also go inline as `selfTest.code`, but anything longer belongs in a +file the editor and the linter can see — which is what `selfTest.script` names. **2. The parity gate.** Run a real computation on CPU and on CUDA and require the results to agree within a declared tolerance: @@ -163,7 +162,7 @@ box's own interpreter, on a matching native host: scrollcase verify .scrollcase/dist/boxes/my-model/1.0.0/linux-x86_64-cuda12.4/*.release.json --self-test ``` -This consumer check does **not** repeat scroll `pythonCode`, so it does not by itself prove +This consumer check does **not** repeat scroll `code`, so it does not by itself prove `torch.cuda.is_available()`. That stronger assertion and parity are builder gates. Building a target proves packaging and declared gates; it never proves scientific parity unless the scroll declares and passes a suitable parity check. @@ -206,9 +205,9 @@ the box does not need at run time, and let the self-test guard the prune: "venv/lib/python3.14/site-packages/torch/test", "venv/lib/python3.14/site-packages/torch/include" ], -"selfTest": { "imports": ["torch"], "files": [], "pythonCode": "import torch; assert torch.cuda.is_available()" } +"selfTest": { "imports": ["torch"], "files": [], "code": "import torch; assert torch.cuda.is_available()" } ``` -See [Managing Model Weights](/guides/managing-weights#keeping-the-box-small) for the general -approach, and consider `--weights on-demand` when the weights, rather than the runtime, are what -makes the archive unwieldy. +See [Managing Assets](/guides/managing-assets#keeping-the-box-small) for the general +approach, and consider `"embed": false` on the weights themselves when they, rather than the CUDA +runtime, are what makes the archive unwieldy. diff --git a/docs/guides/signing-and-custody.md b/docs/guides/signing-and-custody.md index 76ee379..64fd144 100644 --- a/docs/guides/signing-and-custody.md +++ b/docs/guides/signing-and-custody.md @@ -149,7 +149,7 @@ signature_b64=$(your-kms sign --key-id "$KEY_ID" --algorithm ed25519 --input "$p cat <-, and the runtime half must be the one the box declares: a python-script in a box whose runtime is native describes something that cannot be run, and is refused rather than guessed at.", "oneOf": [ { "title": "Python script", @@ -55,6 +55,54 @@ "$ref": "#/$defs/defaultArgs" } } + }, + { + "title": "Node script", + "description": "Run one regular payload file with the box's own Node runtime.", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "script", + "defaultArgs" + ], + "properties": { + "kind": { + "const": "node-script", + "description": "Selects direct script execution." + }, + "script": { + "$ref": "#/$defs/payloadPath", + "description": "Safe path to a regular JavaScript file inside the box." + }, + "defaultArgs": { + "$ref": "#/$defs/defaultArgs" + } + } + }, + { + "title": "Native binary", + "description": "Run a compiled executable that the box carries directly, with no interpreter in front of it. The only shape a runtime with no module system has.", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "binary", + "defaultArgs" + ], + "properties": { + "kind": { + "const": "native-binary", + "description": "Selects direct execution of a payload file." + }, + "binary": { + "$ref": "#/$defs/payloadPath", + "description": "Safe path to the executable inside the box. It carries the executable bit because the scroll declared it, not because the build machine happened to have it set." + }, + "defaultArgs": { + "$ref": "#/$defs/defaultArgs" + } + } } ], "examples": [ diff --git a/docs/public/schema/v2/release-manifest.schema.json b/docs/public/schema/v3/release-manifest.schema.json similarity index 61% rename from docs/public/schema/v2/release-manifest.schema.json rename to docs/public/schema/v3/release-manifest.schema.json index 07d5458..90d1b23 100644 --- a/docs/public/schema/v2/release-manifest.schema.json +++ b/docs/public/schema/v3/release-manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/release-manifest.schema.json", + "$id": "https://scrollcase.dev/schema/v3/release-manifest.schema.json", "title": "Box release manifest", "description": "The immutable description of one built box: what it is, which target it runs on, where its archive lives, and how it was produced. Published as the payload of a signed document and never edited after signing; a correction ships as a new version.", "type": "object", @@ -9,20 +9,18 @@ "schemaVersion", "kind", "boxId", - "modelId", - "runtimeId", "version", "target", "compatibility", "archive", - "pythonEntryPoint", - "modelCacheSubdir", + "runtime", + "cacheSubdir", "selfTest", "provenance" ], "properties": { "schemaVersion": { - "const": 2 + "const": 3 }, "kind": { "$ref": "#/$defs/kind", @@ -31,18 +29,15 @@ "boxId": { "$ref": "#/$defs/identifier" }, - "modelId": { - "$ref": "#/$defs/identifier" - }, - "runtimeId": { - "$ref": "#/$defs/identifier" + "labels": { + "$ref": "#/$defs/labels" }, "version": { "type": "string", "minLength": 1 }, "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + "$ref": "https://scrollcase.dev/schema/v3/target.schema.json" }, "compatibility": { "type": "object", @@ -132,15 +127,13 @@ } } }, - "pythonEntryPoint": { - "type": "string", - "minLength": 1, - "description": "Interpreter path relative to the extracted box root, for example venv/bin/python. Fixed per target by the adapter." + "runtime": { + "$ref": "#/$defs/runtime" }, - "modelCacheSubdir": { + "cacheSubdir": { "type": "string", "minLength": 1, - "description": "Directory relative to the extracted box root holding model assets." + "description": "Directory relative to the extracted box root holding the box's own large files." }, "environment": { "type": "object", @@ -155,42 +148,127 @@ } }, "selfTest": { + "$ref": "#/$defs/selfTest" + }, + "execution": { + "$ref": "https://scrollcase.dev/schema/v3/execution.schema.json" + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "assets": { + "$ref": "#/$defs/deferredAssets" + } + }, + "$defs": { + "identifier": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" + }, + "runtime": { "type": "object", "additionalProperties": false, "required": [ - "pythonImports", + "id" + ], + "description": "What runs inside the box: the runtime, its version, and where its own executable sits in the payload. A consumer needs all three to run the box, and none of them are derivable from the target.", + "properties": { + "id": { + "enum": [ + "python", + "node", + "native" + ], + "description": "The runtime the box carries. A consumer that does not recognise the id must refuse the box: the id decides the payload layout and the argv rule, so guessing would mean executing something on an assumption." + }, + "version": { + "type": "string", + "minLength": 1, + "description": "The runtime's own version. Absent for a runtime that has no interpreter to version." + }, + "entryPoint": { + "type": "string", + "minLength": 1, + "description": "The runtime's own executable relative to the extracted box root, for example venv/bin/python. Fixed per (runtime, target) by the runtime's layout. Absent for a runtime that has no separate executable to name." + } + } + }, + "labels": { + "type": "object", + "description": "Free-form annotations the publishing project declared, signed and carried through untouched. Scrollcase attaches no meaning to any key; a consumer that reads one is reading its own project's convention, not the box format.", + "propertyNames": { + "$ref": "#/$defs/identifier" + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } + }, + "selfTest": { + "type": "object", + "additionalProperties": false, + "required": [ + "probe", "timeoutSeconds" ], - "description": "The import check a consumer can repeat after extraction with the box's own interpreter. The builder also ran the scroll's Python-code and file assertions, which are builder-only checks.", + "description": "The check a consumer can repeat against an extracted box. The builder also ran the scroll's file assertions and any extra source it declared, which are builder-only: signing them would claim a consumer had reproduced a check it cannot see.", "properties": { - "pythonImports": { + "probe": { + "$ref": "#/$defs/selfTestProbe" + }, + "timeoutSeconds": { + "type": "integer", + "exclusiveMinimum": 0 + } + } + }, + "selfTestProbe": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "description": "What the box proves about itself, in whichever shapes its runtime supports. The runtime turns this into command lines; nothing here is a command line, and nothing here is source in any language.", + "properties": { + "imports": { "type": "array", "minItems": 1, + "description": "Modules the runtime must be able to load.", "items": { "type": "string", "minLength": 1 } }, - "timeoutSeconds": { - "type": "integer", - "exclusiveMinimum": 0 + "commands": { + "type": "array", + "minItems": 1, + "description": "Invocations of the box's declared execution and the exit status each must produce.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "args", + "expectExitCode" + ], + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "expectExitCode": { + "type": "integer", + "minimum": 0, + "maximum": 255 + } + } + } } } }, - "execution": { - "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" - }, - "provenance": { - "$ref": "#/$defs/provenance" - }, - "weights": { - "const": "on-demand", - "description": "Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it." - }, - "assets": { + "deferredAssets": { "type": "array", "minItems": 1, - "description": "Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe.", + "description": "Assets the consumer must fetch and place under the box root before first use — the entries the scroll declared with embed false, and only those. A box whose assets are all embedded carries no such list. The declared size and hash are what make fetching them safe; a Scrollcase consumer verifies them and never downloads them itself.", "items": { "type": "object", "additionalProperties": false, @@ -215,15 +293,13 @@ }, "sha256": { "$ref": "#/$defs/sha256" + }, + "executable": { + "type": "boolean", + "description": "Present and true when the scroll declared the file executable. Whoever materializes it owns setting the bit: the file never passes through the archive, so nothing Scrollcase writes can carry a mode for it." } } } - } - }, - "$defs": { - "identifier": { - "type": "string", - "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" }, "kind": { "type": "string", @@ -243,7 +319,6 @@ "builderRevision", "sourceTreeDirty", "sourceRevision", - "pythonVersion", "dependencyLockSha256", "builtAt", "pixiVersion" @@ -269,11 +344,12 @@ "sourceRevision": { "type": "string", "minLength": 1, - "description": "Upstream revision of the packaged model source, as declared by the scroll." + "description": "Upstream revision of the packaged source, as declared by the scroll." }, - "pythonVersion": { + "runtimeVersion": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "The runtime version the environment was solved with, repeated from runtime.version. Absent exactly when the runtime has none: provenance records what was observed and never invents a value to fill a field." }, "pixiVersion": { "type": "string", @@ -289,13 +365,5 @@ } } } - }, - "dependentRequired": { - "assets": [ - "weights" - ], - "weights": [ - "assets" - ] } } diff --git a/docs/public/schema/v2/revocations-manifest.schema.json b/docs/public/schema/v3/revocations-manifest.schema.json similarity index 92% rename from docs/public/schema/v2/revocations-manifest.schema.json rename to docs/public/schema/v3/revocations-manifest.schema.json index 67470c7..6de992b 100644 --- a/docs/public/schema/v2/revocations-manifest.schema.json +++ b/docs/public/schema/v3/revocations-manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/revocations-manifest.schema.json", + "$id": "https://scrollcase.dev/schema/v3/revocations-manifest.schema.json", "title": "Box revocations manifest", "description": "The signed list of releases that must no longer be installed or activated. A published release is immutable, so withdrawing one is an explicit statement rather than a deletion: clients keep honouring this list even when the archive is still reachable.", "type": "object", @@ -13,7 +13,7 @@ ], "properties": { "schemaVersion": { - "const": 2 + "const": 3 }, "kind": { "type": "string", @@ -46,7 +46,7 @@ "minLength": 1 }, "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json", + "$ref": "https://scrollcase.dev/schema/v3/target.schema.json", "description": "Omitted when every target of that version is revoked." }, "reason": { diff --git a/docs/public/schema/v2/scroll.schema.json b/docs/public/schema/v3/scroll.schema.json similarity index 62% rename from docs/public/schema/v2/scroll.schema.json rename to docs/public/schema/v3/scroll.schema.json index b076c7d..57239f3 100644 --- a/docs/public/schema/v2/scroll.schema.json +++ b/docs/public/schema/v3/scroll.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/scroll.schema.json", + "$id": "https://scrollcase.dev/schema/v3/scroll.schema.json", "title": "Box scroll", "description": "The declarative input to a build: an identity, a target, a pinned dependency environment, the assets to fetch, and the self-test the result must pass. A scroll is checked into the consumer's repository next to its lock file; everything a build produces is derived from it.\n\nOnly what a build cannot work out for itself is required. Anything the target or the identity already determines is optional here and filled in when the scroll is read, so a hand-written scroll declares decisions rather than restating them.\n\nOne box's targets differ in a handful of lines and agree on the rest, so a scroll may also be split: scrolls//scroll.json holds what they share, and each scrolls///scroll.json declares `extends` plus its own differences. Both halves are files of this shape; the joined result is what a build reads.", "type": "object", @@ -8,28 +8,26 @@ "required": [ "schemaVersion", "boxId", - "modelId", - "runtimeId", "version", "sourceRevision", - "pythonVersion", + "runtime", "selfTest", "pixiVersion" ], "properties": { "$schema": { - "const": "https://scrollcase.dev/schema/v2/scroll.schema.json", - "description": "Associates this file with the published Scrollcase v2 schema for editor validation, completion, and hover help." + "const": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "description": "Associates this file with the published Scrollcase v3 schema for editor validation, completion, and hover help." }, "extends": { "const": "../scroll.json", "description": "Marks this file as one target's fragment of a box whose shared declarations live in scrolls//scroll.json. The value is fixed: a base is always the box directory's own scroll.json, so there is no path to get wrong and no chain to follow. The base and the fragment are joined into one effective scroll before anything else happens, and that effective scroll is what the build reads and what provenance records." }, "schemaVersion": { - "const": 2, - "description": "Scrollcase wire version. Version 2 is the only active format.", + "const": 3, + "description": "Scrollcase wire version. Version 3 is the only active format.", "examples": [ - 2 + 3 ] }, "scrollId": { @@ -49,11 +47,8 @@ "boxId": { "$ref": "#/$defs/identifier" }, - "modelId": { - "$ref": "#/$defs/identifier" - }, - "runtimeId": { - "$ref": "#/$defs/identifier" + "labels": { + "$ref": "#/$defs/labels" }, "version": { "type": "string", @@ -66,7 +61,7 @@ "description": "Upstream revision of the packaged source, recorded verbatim into provenance." }, "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json", + "$ref": "https://scrollcase.dev/schema/v3/target.schema.json", "description": "The (platform, arch, accelerator) triple this box is built for. Required in every scroll a build reads, and absent from a base: a base holds what its targets share, so declaring one there would name a target the box does not build. Enforced when the scroll is read rather than here, so a base file still validates in an editor." }, "compatibility": { @@ -97,13 +92,8 @@ }, "description": "Constraints the installing host must satisfy. Copied through into the release manifest verbatim and never interpreted by the builder, so a project may declare its own alongside these. Defaults to empty: declaring no constraint is a legitimate answer, and inventing one would be a claim the project never made." }, - "pythonVersion": { - "type": "string", - "minLength": 1, - "description": "Python version solved into the box.", - "examples": [ - "3.11.15" - ] + "runtime": { + "$ref": "#/$defs/runtime" }, "pixiVersion": { "type": "string", @@ -115,15 +105,10 @@ "minLength": 1, "description": "Path to the reviewed licence inventory derived from pixi.lock, which carries an SPDX licence per package. The build fails if the lock no longer matches what was reviewed." }, - "pythonEntryPoint": { + "cacheSubdir": { "type": "string", "minLength": 1, - "description": "Interpreter path relative to the box root. The target adapter's layout admits exactly one value, so this is derived from the target when omitted and still checked against it when declared." - }, - "modelCacheSubdir": { - "type": "string", - "minLength": 1, - "description": "Payload directory the box's model files live under. Defaults to model-cache/." + "description": "Payload directory the box's own large files live under — the destination a scroll's assets conventionally share. Defaults to cache/." }, "environment": { "type": "object", @@ -168,6 +153,16 @@ }, "sha256": { "$ref": "#/$defs/sha256" + }, + "embed": { + "type": "boolean", + "default": true, + "description": "Whether this file is packed into the archive. True, the default, makes the box self-contained: it installs with no network and works air-gapped. False leaves it out and carries its descriptor in the signed release instead, for the caller's distribution layer to materialize. The choice is per entry, so a box may ship a small entry point and defer a large dataset; consumers verify what was materialized before execution and never download it themselves." + }, + "executable": { + "type": "boolean", + "default": false, + "description": "Whether the file needs the executable bit. HTTP carries content and not permissions, so a downloaded file arrives with none; declaring it here is the only way a box can ship one that runs. The bit is synthesised into the archive from this declaration, never read off the build machine." } } } @@ -227,6 +222,11 @@ "sha256": { "$ref": "#/$defs/sha256", "description": "Optional pin. When present the build refuses a file whose contents no longer match." + }, + "executable": { + "type": "boolean", + "default": false, + "description": "Whether the file needs the executable bit. A copy does not carry the source file's mode, because a mode read off the build machine would vary with its umask and break the byte-identical rebuild; the bit is synthesised into the archive from this declaration instead." } } } @@ -240,7 +240,7 @@ }, "uncompressedPaths": { "type": "array", - "description": "Payload paths stored in the archive instead of deflated, because their bytes are already compressed and re-compressing them costs build time while making the archive marginally larger. A path matches itself and everything beneath it, so one entry can name a weights file or the directory an expanded asset archive landed in. Declared assets are stored automatically; this is for anything else the project knows to be already compressed.", + "description": "Payload paths stored in the archive instead of deflated, because their bytes are already compressed and re-compressing them costs build time while making the archive marginally larger. A path matches itself and everything beneath it, so one entry can name a single large file or the directory an expanded asset archive landed in. Declared assets are stored automatically; this is for anything else the project knows to be already compressed.", "items": { "$ref": "#/$defs/payloadPath" } @@ -248,25 +248,63 @@ "selfTest": { "type": "object", "additionalProperties": false, - "required": [ - "imports" + "anyOf": [ + { + "required": [ + "imports" + ] + }, + { + "required": [ + "commands" + ] + } ], "not": { "required": [ - "pythonCode", - "pythonFile" + "code", + "script" ] }, - "description": "Builder checks run with the payload's own interpreter before archiving. Schema version 2 signs only the import subset for a consumer to repeat; file and optional Python-code assertions remain builder-only.", + "description": "Builder checks run against the payload before archiving. The signed release carries the probe — the imports and commands a consumer can repeat after extraction — while file assertions and the optional extra source stay builder-only. At least one of imports and commands is required: a box that proves nothing about itself is not a box worth signing.", "properties": { "imports": { "type": "array", "minItems": 1, + "description": "Modules the runtime must be able to load. Meaningful to a runtime with a module system, which is why it is not the only shape a probe can take.", "items": { "type": "string", "minLength": 1 } }, + "commands": { + "type": "array", + "minItems": 1, + "description": "Invocations of the box's own declared execution, each with the exit code it must produce. This is the only probe shape available to a runtime with no module system, and it needs execution to be declared — there is nothing else to invoke.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "args" + ], + "properties": { + "args": { + "type": "array", + "description": "Arguments appended to the box's declared execution. Passed directly, without a shell. May be empty, which runs the entry point as the box would.", + "items": { + "type": "string" + } + }, + "expectExitCode": { + "type": "integer", + "minimum": 0, + "maximum": 255, + "default": 0, + "description": "Exit status the invocation must produce. Defaults to 0; a non-zero value suits a tool whose --version or --help deliberately exits otherwise." + } + } + } + }, "files": { "type": "array", "description": "Files that must still exist after pruning, which is what stops an over-aggressive prune from shipping a broken box. Defaults to empty.", @@ -274,28 +312,20 @@ "$ref": "#/$defs/payloadPath" } }, - "pythonCode": { + "code": { "type": "string", "minLength": 1, - "description": "Extra Python executed after the imports succeed, for checks a bare import cannot make. Anything longer than an assertion belongs in pythonFile, where an editor can see it is Python." + "description": "Extra source in the runtime's own language, executed after the imports succeed, for checks a bare import cannot make. Anything longer than an assertion belongs in script, where an editor can see what language it is." }, - "pythonFile": { + "script": { "type": "string", "minLength": 1, - "description": "Project path to a Python file executed after the imports succeed, in place of pythonCode. The file is read at build time and run from the payload root, so a real self-test keeps its syntax highlighting, its linter, and its diffs instead of living inside a JSON string." + "description": "Project path to a source file executed after the imports succeed, in place of code. The file is read at build time and run from the payload root, so a real self-test keeps its syntax highlighting, its linter, and its diffs instead of living inside a JSON string." } } }, - "weights": { - "enum": [ - "embed", - "on-demand" - ], - "default": "embed", - "description": "Whether assets are packed into the archive (embed, the default: the box installs with no network and works air-gapped) or left out for the caller to materialize from descriptors in the signed release (on-demand). Consumers verify materialized assets before execution and do not download them. A build may override this." - }, "execution": { - "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" + "$ref": "https://scrollcase.dev/schema/v3/execution.schema.json" }, "parity": { "type": "object", @@ -352,6 +382,57 @@ "type": "string", "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" }, + "runtime": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "description": "What runs inside the box. A target says which machine the box is for; this says what executes on it — which is a different question, and until version 3 the format never asked it: a box declared a Python interpreter path and Python execution kinds and nothing that said \"Python\".", + "properties": { + "id": { + "enum": [ + "python", + "node", + "native" + ], + "description": "The runtime the box carries. The list is closed rather than free-form: each id implies a payload layout, a set of execution kinds and an argv rule that a consumer has to already know, so an unrecognised one is a box that cannot be run, not a box with an unusual label." + }, + "version": { + "type": "string", + "minLength": 1, + "description": "The runtime's own version, solved into the box and recorded in provenance. Required by any runtime whose layout depends on it — Python names its standard library after major.minor — and legitimately absent for one that has no interpreter to version, which is why the format does not demand it.", + "examples": [ + "3.11.15" + ] + }, + "entryPoint": { + "type": "string", + "minLength": 1, + "description": "The runtime's own executable, relative to the box root. The runtime's layout for a given target admits exactly one value, so this is derived when omitted and still checked against the layout when declared. Absent for a runtime that has no separate executable to name.", + "examples": [ + "venv/bin/python" + ] + } + } + }, + "labels": { + "type": "object", + "description": "Free-form annotations carried through into the signed release untouched. Scrollcase never reads a label; it exists so a project can record what it needs to record — the upstream model a box packages, the team that owns it, the ticket it came from — without the format having to grow a field, and without the format claiming to know what any project's boxes are about.", + "propertyNames": { + "$ref": "#/$defs/identifier" + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + }, + "examples": [ + { + "model": "example-org/example-model", + "owner": "platform-team" + } + ] + }, "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" @@ -362,7 +443,7 @@ "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*//).+$", "description": "A forward-slash path inside the box payload: relative, non-empty, and unable to escape the payload root.", "examples": [ - "model-cache/example-model/weights.safetensors" + "cache/example-box/data.bin" ] } } diff --git a/docs/public/schema/v2/signed-document.schema.json b/docs/public/schema/v3/signed-document.schema.json similarity index 94% rename from docs/public/schema/v2/signed-document.schema.json rename to docs/public/schema/v3/signed-document.schema.json index 602af0c..ad06de0 100644 --- a/docs/public/schema/v2/signed-document.schema.json +++ b/docs/public/schema/v3/signed-document.schema.json @@ -1,13 +1,13 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/signed-document.schema.json", + "$id": "https://scrollcase.dev/schema/v3/signed-document.schema.json", "title": "Signed box document", "description": "The envelope wrapping every signed document. The payload travels as exact base64-encoded JSON so that verifying a signature means hashing the bytes as transmitted, with no canonical-JSON implementation to keep in sync across languages. Passing this schema means the envelope is well-formed, never that its signature is valid.", "type": "object", "additionalProperties": false, "required": ["schemaVersion", "payloadEncoding", "payloadBase64", "payloadSha256", "signatures"], "properties": { - "schemaVersion": { "const": 2 }, + "schemaVersion": { "const": 3 }, "payloadEncoding": { "const": "base64-json-utf8" }, "payloadBase64": { "type": "string", diff --git a/docs/public/schema/v2/target.schema.json b/docs/public/schema/v3/target.schema.json similarity index 96% rename from docs/public/schema/v2/target.schema.json rename to docs/public/schema/v3/target.schema.json index 6894c12..1573b29 100644 --- a/docs/public/schema/v2/target.schema.json +++ b/docs/public/schema/v3/target.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/target.schema.json", + "$id": "https://scrollcase.dev/schema/v3/target.schema.json", "title": "Box target", "description": "The platform, architecture and accelerator a box is built for. The supported combinations are closed: a target outside this matrix has no defined identifier and cannot be built, signed, or routed.", "type": "object", diff --git a/docs/reference/api.md b/docs/reference/api.md index 78a0789..be6f2ba 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -397,7 +397,14 @@ The single source of truth for what a box is. See [The Box Format](/reference/bo | `condaSubdir` | `(target) => string` | The conda platform subdir (`osx-arm64`, `linux-64`, `win-64`) | | `pixiAccelerator` | `(scroll) => { accelerator, cudaVersion }` | The conda accelerator descriptor a scroll selects, rejecting target drift | | `assertNativeHost` | `(adapter, host = process) => void` | Throws unless the current host matches the adapter's OS and architecture | -| `assertPythonEntryPoint` | `(adapter, entryPoint) => void` | Throws unless the entry point matches the runtime's layout for the target | +| `assertRuntimeEntryPoint` | `(runtimeId, adapter, entryPoint) => void` | Throws unless the entry point matches that runtime's layout for the target | +| `RUNTIME_IDS` | `readonly string[]` | Every runtime id the format defines: `python`, `node`, `native`. Wider than what this build implements, on purpose | +| `runtimeAdapter` | `(runtimeId) => BoxRuntimeAdapter` | The runtime's layout, execution kinds, argv rule and self-test rule. Throws for a runtime with no adapter | +| `runtimeAdapters` | `() => BoxRuntimeAdapter[]` | Every runtime this build implements | +| `isImplementedRuntime` | `(runtimeId) => boolean` | Whether an adapter exists — the question to ask before `runtimeAdapter` | +| `unimplementedRuntimeMessage` | `(runtimeId) => string` | One wording for a box naming a runtime this build cannot run, so the builder and all three consumers report it identically | +| `executionAffectingVariables` | `(runtimeId, adapter) => readonly string[]` | Inherited variables that can change what a box executes: the runtime's loader controls, then the OS's | +| `isExecutablePayloadPath` | `(rule, relativePath) => boolean` | Whether a payload path is one the runtime requires the executable bit on | ```js import { boxTargetId } from 'scrollcase/contract'; diff --git a/docs/reference/box-format.md b/docs/reference/box-format.md index bae07ea..e1197e5 100644 --- a/docs/reference/box-format.md +++ b/docs/reference/box-format.md @@ -14,7 +14,7 @@ The normative artefacts ship inside the npm package: | Artefact | Where | What it is | | --- | --- | --- | | Reference implementation | `scrollcase/contract` | The rules as executable code | -| JSON Schemas | `scrollcase/contract/schema/*.json` and `/schema/v2/*.json` | The machine-readable spec, package-local or public | +| JSON Schemas | `scrollcase/contract/schema/*.json` and `/schema/v3/*.json` | The machine-readable spec, package-local or public | | Golden fixtures | `scrollcase/contract/fixtures/*.json` | What "agreeing" means, concretely | A client written in another language **does not import the code** — it mirrors the rules and @@ -73,7 +73,7 @@ example-model-1.0.0-macos-aarch64-metal.zip │ ├── bin/python # (venv/python.exe on Windows) │ ├── lib/… │ └── conda-meta/… -├── model-cache/… # assets, when weights are embedded +├── cache/… # the box's own large files, when embedded └── THIRD_PARTY_NOTICES/ └── conda-distributions.json # the dependency licence inventory ``` @@ -108,7 +108,7 @@ files. An entry point sitting at the payload root reaches its model with: ```python root = Path(__file__).resolve().parent -model = root / json.loads((root / "box.json").read_text())["modelCacheSubdir"] +model = root / json.loads((root / "box.json").read_text())["cacheSubdir"] ``` Rather than a hard-coded path, which the scroll then has to be bent to match and which drifts @@ -117,24 +117,23 @@ written this way exercises the same layout the shipped box has. ```jsonc { - "schemaVersion": 2, + "schemaVersion": 3, "boxId": "example-model", - "modelId": "example-org-example-model", - "runtimeId": "example-model-runtime", + "labels": { "model": "example-org/example-model" }, "version": "1.0.0", "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, - "pythonEntryPoint": "venv/bin/python", - "modelCacheSubdir": "model-cache/example", - "environment": { "MODEL_ROOT": "model-cache/example" }, - "selfTest": { "pythonImports": ["json", "sqlite3"], "timeoutSeconds": 180 }, + "runtime": { "id": "python", "version": "3.11.15", "entryPoint": "venv/bin/python" }, + "cacheSubdir": "cache/example", + "environment": { "MODEL_ROOT": "cache/example" }, + "selfTest": { "probe": { "imports": ["json", "sqlite3"] }, "timeoutSeconds": 180 }, "provenance": { "…": "see below" } } ``` -`verify` recursively checks every shared field against the signed release: schema and identity, -complete target, entry point, cache subdirectory, declared environment, consumer self-test, -weights/assets policy, and provenance. That agreement binds the archive's contents to its signed -metadata. +`verify` recursively checks every shared field against the signed release: schema, identity and +labels, complete target, the runtime block, cache subdirectory, declared environment, consumer +self-test, the deferred-asset list, and provenance. That agreement binds the archive's contents to +its signed metadata. ## Provenance @@ -146,8 +145,9 @@ cannot be dressed up after the fact: | `scrollId`, `scrollVersion` | Which scroll produced the box. New scroll inputs derive `scrollId` as `-` | | `builderRevision` | The 40-hex commit of the source tree that built it | | `sourceTreeDirty` | Whether that tree had uncommitted changes. `true` means the build is **not** reproducible from the recorded revision alone | -| `sourceRevision` | Upstream revision of the packaged model source, as declared by the scroll | -| `pythonVersion`, `pixiVersion` | The interpreter version, and the resolver that solved the environment | +| `sourceRevision` | Upstream revision of the packaged source, as declared by the scroll | +| `runtimeVersion` | The runtime version the environment was solved with. Absent exactly when the runtime has none — provenance records what was observed and never invents a value | +| `pixiVersion` | The resolver that solved the environment | | `dependencyLockSha256` | Hash of the `pixi.lock` the environment was solved from | | `builtAt` | Taken from the HEAD commit, not the clock — the same commit rebuilds to the same timestamp | @@ -157,7 +157,7 @@ Every document a build emits travels in one envelope: ```jsonc { - "schemaVersion": 2, + "schemaVersion": 3, "payloadEncoding": "base64-json-utf8", "payloadBase64": "eyJzY2hlbWFWZXJzaW9uIjoyfQ==", "payloadSha256": "7d2c9a41e8b350f6c174a9de20358bf41c6e97d05a8b3f2619e4c7081da5b3f2", @@ -198,11 +198,10 @@ lives and what it hashes to, the consumer import check to repeat, and provenance ```jsonc { - "schemaVersion": 2, + "schemaVersion": 3, "kind": "scrollcase.box.release", "boxId": "example-model", - "modelId": "example-org-example-model", - "runtimeId": "example-model-runtime", + "labels": { "model": "example-org/example-model" }, "version": "1.0.0", "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, "compatibility": { "minHostAppVersion": "1.0.0", "minMacosVersion": "13.0", "minRamGb": 8 }, @@ -217,16 +216,21 @@ lives and what it hashes to, the consumer import check to repeat, and provenance "format": "sha256-path-list-v1", "sha256": "6b8f…4c" }, - "pythonEntryPoint": "venv/bin/python", - "modelCacheSubdir": "model-cache/example", - "environment": { "MODEL_ROOT": "model-cache/example" }, - "selfTest": { "pythonImports": ["json", "sqlite3"], "timeoutSeconds": 180 }, + "runtime": { "id": "python", "version": "3.11.15", "entryPoint": "venv/bin/python" }, + "cacheSubdir": "cache/example", + "environment": { "MODEL_ROOT": "cache/example" }, + "selfTest": { "probe": { "imports": ["json", "sqlite3"] }, "timeoutSeconds": 180 }, "provenance": { "…": "…" } } ``` -`environment` is optional for compatibility with earlier schema-v2 releases. When present it is a -signed string map repeated value-for-value in `box.json`. A conforming verifier checks the +`runtime.id` is `python`, `node` or `native`. A consumer that does not recognise it must **refuse +the box**: the id decides the payload layout and the argv rule, so guessing would mean executing +something on an assumption. Only `python` can be built today; the other two are named by the format +so that implementing them is code rather than another format change. + +`environment` is optional. When present it is a signed string map repeated value-for-value in +`box.json`. A conforming verifier checks the declaration; a Scrollcase consumer additionally resolves it against its current process and may emit an environment report. That report is not part of the format and is not a guarantee of the box. @@ -237,15 +241,18 @@ need headroom for the archive, extracted files, temporary copies, allocation uni metadata. A prepared receipt reports the matching extracted measurement; an attached receipt reports the directory's current measurement without comparing it with this signed build-time value. -`weights: "on-demand"` and an `assets` array appear together only when assets were deliberately -left out; their absence means the box is self-contained. +An `assets` array appears only when at least one asset was deliberately left out of the archive, and +lists exactly those entries. Its absence means the box is self-contained. There is no box-wide mode +field: the list itself is the statement, so there is nothing that can disagree with it. An entry may +carry `executable: true`, which tells whoever materializes the file that it has to run — Scrollcase +never writes that file, so nothing it produces can carry a mode for it. #### Extracted-payload commitment `payloadDigest` signs the SHA-256 of `payload-digest.v1`, which travels inside the payload and names -every original file and symbolic link except itself. It is optional so schema version 2 releases -built before this capability remain valid; an operation specifically asked to verify an extracted -payload refuses a release without the commitment. +every original file and symbolic link except itself. It is optional so that releases built before +this capability remain valid; an operation specifically asked to verify an extracted payload refuses +a release without the commitment. The canonical byte stream starts with `sha256-path-list-v1` and LF. Each following record is: @@ -275,7 +282,7 @@ so promoting a build never requires re-signing it. ```jsonc { - "schemaVersion": 2, + "schemaVersion": 3, "kind": "scrollcase.box.channel", "channel": "beta", "boxId": "example-model", @@ -307,7 +314,7 @@ honouring the list even when the archive is still reachable. ```jsonc { - "schemaVersion": 2, + "schemaVersion": 3, "kind": "scrollcase.box.revocations", "updatedAt": "2026-07-25T10:14:03Z", "revocations": [ @@ -348,7 +355,24 @@ URL. See [Distributing Boxes](/guides/distributing-boxes). ## Versioning {#versioning} -Published v1 is immutable and remains paired with the old Scrollcase versions that emitted it. -Active v2 code accepts and emits only `schemaVersion: 2`. A future breaking change gets a **new** -`schemaVersion` — never a silent edit to a `kind` string, payload encoding, signature algorithm, -or golden fixture. +Published v1 and v2 are immutable and remain paired with the Scrollcase versions that emitted them. +Active v3 code accepts and emits only `schemaVersion: 3`, and refuses either older version **by +name** rather than reinterpreting it — a v1 and a v2 box are different artefacts with different +rebuilds ahead of them, and whoever is holding one is entitled to know which. There is no dual-read +path anywhere. + +A future breaking change gets a **new** `schemaVersion` — never a silent edit to a `kind` string, +payload encoding, signature algorithm, or golden fixture. + +### What version 3 changed + +| Version 2 | Version 3 | Why | +| --- | --- | --- | +| `modelId`, `runtimeId` (both required) | `labels`, optional and free-form | Neither was ever read by any code path. They were a consumer's vocabulary in the format: a box packaging a library still had to name a model | +| `pythonVersion`, `pythonEntryPoint` | `runtime: { id, version, entryPoint }` | A box said *where its Python was* and never *that it was Python*. A reader had to infer the runtime from the shape of a path | +| `provenance.pythonVersion` | `provenance.runtimeVersion` | Same reason, and it may now be absent for a runtime that has no version to record | +| `modelCacheSubdir` | `cacheSubdir` | The directory holds whatever the box's large files are | +| `weights: embed \| on-demand` | `assets[].embed`, per entry | A box-wide switch could not ship a small entry point and defer a large dataset. `--weights` went with it: a build-time override of a per-asset declaration repacks a box under an identity that no longer describes it | +| `selfTest.pythonImports` | `selfTest.probe` with `imports` and `commands` | Python syntax in the wire format. A runtime with no module system could not state a check at all | +| Executable bit from a `venv/bin` heuristic | `assets[].executable`, `localFiles[].executable` | A downloaded file arrives with no permissions, so a box could not ship one that runs | +| `python-script`, `python-module` | plus `node-script`, `native-binary` | Named now, so implementing them later is code rather than another wire break | diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 23a2918..16bec23 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -147,11 +147,9 @@ navigable menus for execution kind and script source. Everything else has a defe is a flag rather than a prompt. A required answer left blank repeats the question instead of ending the session. -The weights mode is one of those defaults rather than a menu. It decides whether declared assets are -packed into the archive, and a box that declares none — which is most of them, since a scroll -packages a Python environment and not necessarily a model — has nothing for it to decide. New -scrolls take `embed` and say nothing about it; `--weights on-demand` states the other choice, and -`scrollcase edit scroll` changes it later. +Labels are among those defaults rather than a menu. Scrollcase reads none of them, so prompting for +one would be asking you to fill in a field on the tool's behalf; a generated scroll carries none, +and `--labels '{"model":"…"}'` states them when there is something to record. Every question in the CLI has the same shape: a blank line, the field's name, one line saying what the field is, then the answer typed after ` ↳ `. The name is coloured and the explanation is not, so @@ -174,7 +172,6 @@ scrollcase new scroll \ --box-id example-model \ --source-revision upstream-v1 \ --asset-base-url https://assets.example.org/boxes \ - --weights embed \ --execution library-only ``` @@ -184,8 +181,7 @@ scrollcase new scroll \ | `--box-id` | Box identity and parent directory | | `--source-revision` | Upstream revision recorded in provenance | | `--asset-base-url` | Base URL copied into built release metadata | -| `--model-id` | Identity of what the box packages. Defaults to the box id | -| `--runtime-id` | Runtime identity. Defaults to `-runtime` | +| `--labels` | JSON object of free-form annotations carried into the signed release. Scrollcase reads none of them | | `--version` | Box version. Defaults to `1.0.0` | | `--scroll-version` | Version of the authoring input. Defaults to `1.0.0` | | `--python-version` | Python dependency version written into `pixi.toml`, or `latest`. Defaults to one minor behind the newest Python conda-forge publishes | @@ -195,7 +191,6 @@ scrollcase new scroll \ | `--min-macos-version` | Optional macOS floor | | `--min-ram-gb` | Optional positive RAM requirement | | `--min-nvidia-driver-version` | Optional NVIDIA driver floor | -| `--weights` | `embed` (default, and left out of the scroll) or `on-demand` | | `--execution` | `python-script`, `python-module`, or `library-only` | | `--script` | Existing project-relative Python script | | `--generate-script` | Generate a minimal starter instead of using an existing script | @@ -211,7 +206,7 @@ refuses traversal and non-regular sources, and never overwrites an existing sour Generated defaults are grouped by both box and target; `library-only` omits execution metadata. Alongside `scroll.json` and `pixi.toml`, `new scroll` writes a `self_test.py` next to them and -points `selfTest.pythonFile` at it, so the box's own check starts life as real Python rather than +points `selfTest.script` at it, so the box's own check starts life as real Python rather than an escaped JSON string. `--python-version latest` resolves once, at authoring time, and writes the resulting number into the @@ -230,8 +225,10 @@ Record something in a scroll that already exists, so the fields nobody can write written by hand. ```sh -scrollcase add asset [--to ] [--target |all] -scrollcase add file [--to ] [--target |all] +scrollcase add asset [--to ] [--on-demand] [--executable] + [--target |all] +scrollcase add file [--to ] [--executable] + [--target |all] scrollcase add dep [--version ] [--target |all] scrollcase add dep --from-requirements requirements.txt scrollcase add env NAME=VALUE [--target |all] @@ -242,12 +239,18 @@ scrollcase add import [--target |all] which are the two values a scroll cannot be written without and no author can know without fetching the file. Recording them here changes nothing about the guarantee: they are pinned once and checked on every build, exactly as before. `--to` is optional and defaults to the URL's last path segment -under the box's `modelCacheSubdir`. +under the box's `cacheSubdir`. `--on-demand` writes `"embed": false`, leaving this one file out of +the archive for your distribution layer to materialize. `add file` records a file from the project. `--to` defaults to the file's own name at the payload root. No `sha256` is written — see [`localFiles`](/reference/scroll#localfiles) — so the first edit to a file you just added does not fail your next build. +`--executable` marks either kind as a file that needs the executable bit. A download arrives with no +permissions at all and a copy does not carry the source file's mode, so this declaration is the only +way a box can ship one that runs — see [Managing +Assets](/guides/managing-assets#files-that-have-to-run). + Both also add the payload path to `selfTest.files`, so an over-eager `prunePaths` cannot quietly drop what you just declared. @@ -263,9 +266,9 @@ rest of the map alone. A map is the one shape a single-value prompt cannot edit, its own command rather than being left to a hand edit. The value may contain `=`; only the first one separates the name. -`add import` adds a module to `selfTest.imports`. Those names are signed into the release and -repeated by `verify --self-test`, so they are the part of the self-test a consumer can check for -itself. +`add import` adds a module to `selfTest.imports`. Those names are signed into the release as part of +`selfTest.probe` and repeated by `verify --self-test`, so they are the part of the self-test a +consumer can check for itself. `--from-requirements` reads a pip `requirements.txt` instead. Names are translated to conda-forge where Scrollcase is sure and lowercased otherwise, and **every translation and every skip is @@ -327,9 +330,9 @@ that is not a field cannot be typed in the first place — and an enum field off Without a terminal, `--field` and `--value` are required. Three kinds of field are not offered: structural values a project does not choose (`schemaVersion`, -`extends`), values the layout or the target fixes (`boxId` and `target` name the directories, -`pythonEntryPoint` has one legal value per target), and the collections, which have `add`/`remove` -or a file of their own. +`extends`), values the layout or the target fixes (`boxId` and `target` name the directories, and +`runtime` holds an id that decides the whole payload layout and an entry point with one legal value +per target), and the collections, which have `add`/`remove` or a file of their own. ## `refresh` @@ -453,7 +456,7 @@ plus a channel pointer. The full pipeline is narrated in ```sh scrollcase build [] [--target ] - [--channel ] [--weights embed|on-demand] + [--channel ] [--asset-base-url ] [--namespace ] [--allow-dirty] [--pixi ] [--conda-pack ] [--private-key ] [--public-key ] [--signer-command ] @@ -465,8 +468,7 @@ menu. CI and other non-interactive callers must always provide it explicitly. | Flag | Default | Meaning | | --- | --- | --- | | `--target` | ask when a box has several scrolls | Canonical target scroll to build | -| `--channel` | `beta` | Channel the signed pointer names. The v2 vocabulary is closed to `nightly`, `beta`, and `stable` | -| `--weights` | scroll's `weights`, else `embed` | Overrides the scroll for this build: `embed` packs assets into the archive (works air-gapped), `on-demand` leaves them out for the caller to materialize; consumers verify them before execution. `build` does not ask — the scroll's declaration is what it uses | +| `--channel` | `beta` | Channel the signed pointer names. The vocabulary is closed to `nightly`, `beta`, and `stable` | | `--asset-base-url` | scroll's `assetBaseUrl` | Base URL the signed documents point at; one of the two must be set | | `--namespace` | `scrollcase.box` | Document `kind` namespace — a project with boxes already in the field keeps emitting its own | | `--allow-dirty` | off | Permit a build from an uncommitted tree; recorded as `sourceTreeDirty: true` in the box | @@ -526,11 +528,11 @@ scrollcase verify --extracted [--env-report] [--env-report- Checks, in order: envelope payload hash and at least one trusted signature; release kind; coherent target and entry point; archive size and SHA-256; safe entry names; recursively equal shared -`box.json` fields (identity/version, full target, entry point, cache subdirectory, declared -environment, consumer self-test, weights/assets, and provenance); and the declared interpreter. `--self-test` -additionally requires a matching native host, extracts to a temporary directory, checks logical -payload size, and runs the signed import check. It does not repeat scroll-only `pythonCode` or file -assertions, which are builder-only checks. +`box.json` fields (identity, labels, version, full target, the runtime block, cache subdirectory, +declared environment, consumer self-test, the deferred-asset list, and provenance); and the declared +interpreter. `--self-test` additionally requires a matching native host, extracts to a temporary +directory, checks logical payload size, and runs the signed probe. It does not repeat scroll-only +`code` or file assertions, which are builder-only checks. After validating the command arguments, `verify` prints a blank line and `Verifying box` (or `Verifying extracted payload`) before it starts reading and hashing the supplied bytes, so a long diff --git a/docs/reference/schemas.md b/docs/reference/schemas.md index 3ab20e8..69e5a4f 100644 --- a/docs/reference/schemas.md +++ b/docs/reference/schemas.md @@ -8,13 +8,13 @@ description: Public schema URLs, package imports, offline registration, generate The shipped JSON Schemas are available both from the package and at stable public URLs: ```text -https://scrollcase.dev/schema/v2/target.schema.json -https://scrollcase.dev/schema/v2/scroll.schema.json -https://scrollcase.dev/schema/v2/box-manifest.schema.json -https://scrollcase.dev/schema/v2/release-manifest.schema.json -https://scrollcase.dev/schema/v2/channel-manifest.schema.json -https://scrollcase.dev/schema/v2/revocations-manifest.schema.json -https://scrollcase.dev/schema/v2/signed-document.schema.json +https://scrollcase.dev/schema/v3/target.schema.json +https://scrollcase.dev/schema/v3/scroll.schema.json +https://scrollcase.dev/schema/v3/box-manifest.schema.json +https://scrollcase.dev/schema/v3/release-manifest.schema.json +https://scrollcase.dev/schema/v3/channel-manifest.schema.json +https://scrollcase.dev/schema/v3/revocations-manifest.schema.json +https://scrollcase.dev/schema/v3/signed-document.schema.json ``` The documentation build fails unless these public files are byte-identical to diff --git a/docs/reference/scroll.md b/docs/reference/scroll.md index 6294c0c..6f1b003 100644 --- a/docs/reference/scroll.md +++ b/docs/reference/scroll.md @@ -21,7 +21,7 @@ The parent directory is the scroll's declared `boxId`; the child is the canonica its declared `target`. Scrollcase checks both, so the path cannot mislabel the scroll, but neither value is written twice inside `scroll.json`. Flat source directories are not accepted in v2. -The machine-readable definition is [`scroll.schema.json`](/schema/v2/scroll.schema.json), also shipped +The machine-readable definition is [`scroll.schema.json`](/schema/v3/scroll.schema.json), also shipped through the package export. See [JSON Schemas](/reference/schemas). Create a new target-specific input with `scrollcase new scroll`. Interactively it asks four @@ -36,15 +36,13 @@ scroll is read. Write the decisions, not the restatements: ```json { - "$schema": "https://scrollcase.dev/schema/v2/scroll.schema.json", - "schemaVersion": 2, + "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "schemaVersion": 3, "boxId": "hello-box", - "modelId": "example-org-hello", - "runtimeId": "hello-box-runtime", "version": "1.0.0", "sourceRevision": "example-hello-v1", "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, - "pythonVersion": "3.14", + "runtime": { "id": "python", "version": "3.14" }, "pixiVersion": "0.73.0", "assetBaseUrl": "https://assets.example.org/boxes", "selfTest": { "imports": ["json", "sqlite3"] } @@ -54,25 +52,22 @@ scroll is read. Write the decisions, not the restatements: ## The same scroll, fully spelled out Identical in every respect — this is what the file above becomes when it is read. Declaring a -derived field is never wrong; `pythonEntryPoint` is still checked against the target either way. +derived field is never wrong; `runtime.entryPoint` is still checked against the target either way. ```json { - "$schema": "https://scrollcase.dev/schema/v2/scroll.schema.json", - "schemaVersion": 2, + "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "schemaVersion": 3, "scrollVersion": "1.0.0", "boxId": "hello-box", - "modelId": "example-org-hello", - "runtimeId": "hello-box-runtime", "version": "1.0.0", "sourceRevision": "example-hello-v1", "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, "compatibility": { "minHostAppVersion": "1.0.0", "minMacosVersion": "13.0", "minRamGb": 1 }, - "pythonVersion": "3.14", + "runtime": { "id": "python", "version": "3.14", "entryPoint": "venv/bin/python" }, "pixiVersion": "0.73.0", - "pythonEntryPoint": "venv/bin/python", - "modelCacheSubdir": "model-cache/hello-box", - "environment": { "MODEL_ROOT": "model-cache/hello-box", "HF_HUB_OFFLINE": "1" }, + "cacheSubdir": "cache/hello-box", + "environment": { "MODEL_ROOT": "cache/hello-box", "HF_HUB_OFFLINE": "1" }, "assetBaseUrl": "https://assets.example.org/boxes", "assets": [], "selfTest": { "imports": ["json", "sqlite3"], "files": [] } @@ -83,12 +78,11 @@ derived field is never wrong; `pythonEntryPoint` is still checked against the ta | Field | Required | Meaning | | --- | --- | --- | -| `schemaVersion` | yes | Always `2`. See [versioning](/reference/box-format#versioning) | +| `schemaVersion` | yes | Always `3`. See [versioning](/reference/box-format#versioning) | | `scrollId` | no | Provenance identity. When omitted, Scrollcase derives `-` | | `scrollVersion` | no | Version of the scroll itself — bump it when you change how the box is built. Defaults to `1.0.0` | | `boxId` | yes | Identity of the box across versions. Appears in archive names, object keys, and the channel pointer | -| `modelId` | yes | Identity of what the box packages — a model, a library, an application | -| `runtimeId` | yes | Identity of the runtime environment the box provides | +| `labels` | no | Free-form annotations, carried into the signed release and never read by Scrollcase | | `version` | yes | Version of the box this scroll produces, as it appears in the release manifest | | `sourceRevision` | yes | Upstream revision of the packaged source, recorded verbatim into provenance | | `extends` | no | `"../scroll.json"`, marking this file as one target's half of a [split scroll](#one-box-several-targets) | @@ -96,22 +90,32 @@ derived field is never wrong; `pythonEntryPoint` is still checked against the ta "Required" here means required of the scroll a build reads. In a split scroll that is the two halves joined, so either half may carry any given field. -`boxId`, `modelId` and `runtimeId` are lowercase identifiers (`^[a-z0-9]+(?:[-.][a-z0-9]+)*$`) in -the published manifests — keep them to that shape. +`boxId` is a lowercase identifier (`^[a-z0-9]+(?:[-.][a-z0-9]+)*$`) in the published manifests, and +so is every `labels` key — keep them to that shape. An explicit `scrollId` lets a project choose its source identity. It may be omitted because `boxId` and `target` already contain the meaningful identity and the derived value is deterministic. -::: tip Three identifiers, three questions -`boxId` answers *which artefact is this a version of?*, `modelId` answers *what is inside?*, and -`runtimeId` answers *what environment does it provide?* Several boxes may package the same payload -with different runtimes, or the same runtime for different payloads; keeping the three separate is -what lets a consumer reason about that. +### Labels -The name is historical: the first boxes carried models. What it identifies is whatever the box -packages, and a box that packages a library or an application names that. A project with nothing -to distinguish there sets it to the `boxId` — which is what `scrollcase new scroll` does when -`--model-id` is not passed, so it is a field most scrolls never think about. +```jsonc +"labels": { + "model": "example-org/example-model", + "owner": "platform-team" +} +``` + +Whatever the project needs recorded and the format has no business defining: the upstream model a +box packages, the team that owns it, the ticket it came from. Labels are signed and carried through +untouched, and **Scrollcase never reads one** — a consumer that reads a label is reading its own +project's convention. + +::: tip Why this replaced two required fields +Version 2 required `modelId` and `runtimeId`, and no code path ever read either. They were a +consumer's vocabulary written into the format: a box that packaged a library rather than a model +still had to name a model, so most scrolls set `modelId` to the `boxId` and moved on. + +A label says the same thing when there is something to say, and says nothing when there is not. ::: ## Target @@ -183,7 +187,7 @@ leave `execution` half from each half, producing a `python-script` that inherite | `assets`, `assetArchives`, `localFiles` | Joined, base entries first. Two entries claiming one `relativePath` is an **error** | | `prunePaths`, `uncompressedPaths`, `selfTest.imports`, `selfTest.files` | Joined, base first, repeats dropped | | `compatibility`, `environment` | Joined key by key; on a shared key the fragment wins | -| `selfTest.pythonCode` / `selfTest.pythonFile` | One slot: a fragment naming either replaces both | +| `selfTest.code` / `selfTest.script` | One slot: a fragment naming either replaces both | | `extends` | Dropped — the joined scroll extends nothing | The distinction between the two list rules is deliberate. A repeated prune path or import is the @@ -205,10 +209,11 @@ build reads, and provenance records. Nothing downstream can tell which half a va | Field | Required | Meaning | | --- | --- | --- | -| `pythonVersion` | yes | Python version the box carries, recorded into provenance | +| `runtime.id` | yes | `python`, `node` or `native`. Only `python` can be built today; the other two are named by the format so implementing them is not another format change | +| `runtime.version` | for `python` | The runtime version the box carries, recorded into provenance | | `pixiVersion` | yes | The exact pixi release used to solve and install. `lock` and `build` refuse any other version | -| `pythonEntryPoint` | no | Interpreter path relative to the box root. Fixed per target: `venv/bin/python` on macOS and Linux, `venv/python.exe` on Windows. Derived from the target when omitted, and a mismatch is still rejected when declared | -| `modelCacheSubdir` | no | Directory relative to the box root holding model assets. Defaults to `model-cache/` | +| `runtime.entryPoint` | no | The runtime's own executable relative to the box root. Fixed per (runtime, target): `venv/bin/python` on macOS and Linux, `venv/python.exe` on Windows. Derived when omitted, and a mismatch is still rejected when declared | +| `cacheSubdir` | no | Directory relative to the box root holding model assets. Defaults to `cache/` | | `environment` | no | String environment variables required whenever Scrollcase runs the box interpreter | | `condaDependencyLicenseAudit` | no | Path (from the project root) to the reviewed licence inventory, written and declared by [`audit --write`](/reference/cli#audit). When declared, the build fails if the lock no longer matches what was reviewed | @@ -237,7 +242,7 @@ so they cannot drift apart, and `--from-requirements` imports an existing pip fi ```jsonc "environment": { - "MODEL_ROOT": "model-cache/hello", + "MODEL_ROOT": "cache/hello", "HF_HUB_OFFLINE": "1" } ``` @@ -329,17 +334,35 @@ cannot be known without downloading the file. "assets": [ { "url": "https://huggingface.co/example-org/model/resolve/main/model.safetensors", - "relativePath": "model-cache/hello/model.safetensors", + "relativePath": "cache/hello/model.safetensors", "sizeBytes": 438012416, - "sha256": "9f2b…c1" + "sha256": "9f2b…c1", + "embed": false, + "executable": false } ] ``` +| Field | Required | Meaning | +| --- | --- | --- | +| `url`, `relativePath`, `sizeBytes`, `sha256` | yes | Where the file comes from, where it lands, and the exact bytes expected | +| `embed` | no | Whether the file is packed into the archive. `true` by default | +| `executable` | no | Whether the file needs the executable bit. `false` by default | + +`embed: false` leaves the file out of the archive and carries its descriptor in the signed release +instead, for your distribution layer to materialize. It is **per entry**, so one box can ship a +small entry point inside the archive and defer a 30 GB dataset beside it. Scrollcase consumers +verify a materialized file before execution and never download one. See [Managing +Assets](/guides/managing-assets). + +`executable: true` is the only way a downloaded file can end up runnable: HTTP carries content, not +permissions, so an asset arrives with no mode at all. The bit is synthesised into the archive from +this declaration, never read off the build machine — which is what keeps two builds of one commit +byte-identical whatever umask each ran under. + Retries inside one download operation resume from a partial file, and a partial transfer is renamed into place only after its size and hash match. The build scratch tree is recreated at -process start, so there is no cross-process cache. See [Managing Model -Weights](/guides/managing-weights). +process start, so there is no cross-process cache. ### `assetArchives` @@ -349,18 +372,18 @@ refuses to overwrite them. ```jsonc "assetArchives": [ { - "relativePath": "model-cache/hello/weights.tar.gz", + "relativePath": "cache/hello/weights.tar.gz", "format": "tar.gz", - "destination": "model-cache/hello", + "destination": "cache/hello", "stripComponents": 1, "removeAfterExtract": true } ] ``` -`format` is `zip` or `tar.gz`. Archives are expanded at build time, so they **cannot be combined -with `on-demand` weights** — the build fails rather than declaring a layout that never -materialises. +`format` is `zip` or `tar.gz`. An archive is expanded at build time, so **it has no `embed` field**: +"leave it out and let the caller fetch it" names nothing that could happen. Version 2 refused that +combination with a cross-field check; version 3 makes it unspeakable. ### `localFiles` @@ -370,10 +393,15 @@ Files copied from your own repository into the payload. Added and removed with ```jsonc "localFiles": [ { "sourcePath": "runtime/entrypoint.py", "relativePath": "entrypoint.py" }, + { "sourcePath": "bin/launch.sh", "relativePath": "bin/launch.sh", "executable": true }, { "sourcePath": "legal/MODEL_NOTICE.md", "relativePath": "THIRD_PARTY_NOTICES/MODEL_NOTICE.md", "sha256": "4c7e…9a" } ] ``` +`executable` works exactly as it does for an asset, and for the same reason: a local file is +**copied** rather than moved, so it has no mode to inherit, and reading the source file's mode would +make the archive depend on the umask of whoever checked the project out. + `sha256` is an optional **pin**: when it is present, the build refuses a file whose contents no longer match. Leave it off the files you are still writing — a script you edit every day would otherwise fail its own build until you recomputed a digest by hand. Add it to the files that must @@ -411,7 +439,7 @@ Payload paths stored in the archive rather than deflated, because their bytes ar compressed — re-compressing them costs build time and makes the archive marginally larger. ```jsonc -"uncompressedPaths": ["model-cache/hello", "corpora/images"] +"uncompressedPaths": ["cache/hello", "corpora/images"] ``` An entry matches that path **and everything beneath it**, so one line can name a weights file or @@ -436,47 +464,51 @@ channel documents point at. Required unless passed per build with `--asset-base- Builder checks run with the payload's **own interpreter** before the box is archived. Schema version 2 signs the import subset for a consumer to repeat; it does not carry the richer file or -`pythonCode` assertions. +`code` assertions. ```jsonc "selfTest": { "imports": ["torch", "transformers"], - "files": ["model-cache/hello/model.safetensors"], - "pythonFile": "scrolls/hello-box/macos-aarch64-metal/self_test.py" + "files": ["cache/hello/model.safetensors"], + "script": "scrolls/hello-box/macos-aarch64-metal/self_test.py" } ``` | Field | Required | Meaning | | --- | --- | --- | -| `imports` | yes | One or more modules imported with the box's interpreter, added with [`add import`](/reference/cli#add). These names are signed and repeated by `verify --self-test` | +| `imports` | one of these two | Modules loaded with the box's runtime, added with [`add import`](/reference/cli#add). Signed, and repeated by `verify --self-test` | +| `commands` | one of these two | Invocations of the box's own `execution`, each with the exit status it must produce. Also signed and repeated | | `files` | no | Files that must still exist after pruning — this is what stops an over-aggressive prune from shipping a broken box. Defaults to empty | -| `pythonFile` | no | Project path to a Python file run after the imports succeed | -| `pythonCode` | no | The same thing inline, for a single assertion. Mutually exclusive with `pythonFile` | +| `script` | no | Project path to a source file run after the imports succeed | +| `code` | no | The same thing inline, for a single assertion. Mutually exclusive with `script` | -Prefer `pythonFile` for anything longer than one line. A self-test is real code and deserves an +At least one of `imports` and `commands` is required: a box that proves nothing about itself is not +a box worth signing. `imports` asks the runtime's loader a question and means something only to a +runtime that has one; `commands` asks the box's declared execution a question, which every runtime +can answer: + +```jsonc +"selfTest": { + "imports": ["json"], + "commands": [{ "args": ["--version"], "expectExitCode": 0 }] +} +``` + +A `commands` entry needs `execution` to be declared — there is nothing else to invoke — and the +scroll is refused when it is not. `expectExitCode` defaults to `0`; a non-zero value suits a tool +whose `--help` deliberately exits otherwise. + +Prefer `script` for anything longer than one line. A self-test is real code and deserves an editor that knows it: in a file it keeps its syntax highlighting, its linter, and a readable diff, where inline it is a JSON string with escaped newlines. `scrollcase new scroll` generates one next to the scroll and points the field at it. The target's own platform assertion is prepended automatically, and the run happens under the accelerator's validation environment. The file is read at build time and executed from the payload -root, so it can read what the box ships and import what it packs. A file listed in `files` that is -a deliberately deferred on-demand asset is not required to be present. After pruning, the builder -checks required files, then runs the target assertion, imports, the extra Python, and finally -optional parity. A consumer runs the target assertion and signed imports only. - -## Weights mode - -```jsonc -"weights": "embed" -``` - -`embed` (the default) packs assets into the archive: the box installs with no network and works -air-gapped, at the cost of a large artefact. `on-demand` leaves them out and carries their URL, -path, size and SHA-256 in the signed release. A caller must materialize those files; the local -consumers verify them before execution and do not download them. A build may override this with -`--weights`. See -[Managing Model Weights](/guides/managing-weights). +root, so it can read what the box ships and import what it packs. A file listed in `files` that is a +deliberately deferred asset is not required to be present. After pruning, the builder checks +required files, then runs the target assertion, the probe, the extra source, and finally optional +parity. A consumer runs the target assertion and the signed probe only. ## Parity (optional) @@ -510,10 +542,10 @@ mutation: 1. Parse `scroll.json` and, when it declares `extends`, read and join its base first — neither half of a split scroll is a complete document, so validating one alone would report the other half's fields as missing. Validate the joined result against the shipped scroll and target schemas. -2. For a nested scroll, require a target, and require the parent and child directories to match - `boxId` and the canonical target; reject invalid target/entry-point combinations and default - on-demand weights with `assetArchives`. -3. Resolve a build-time `--weights` override and repeat the archive/policy check. +2. Reject a runtime this build has no adapter for, an `execution` kind belonging to a different + runtime, and a `selfTest.commands` with no `execution` to invoke. +3. For a nested scroll, require a target, and require the parent and child directories to match + `boxId` and the canonical target; reject an entry point the runtime's layout does not admit. 4. Require a matching native host, discover the exact tools, and require `pixi.lock`. 5. Record Git provenance and reject a dirty tree unless `--allow-dirty` was explicit. 6. Only then recreate build state, install from the lock, download verified assets, and enforce diff --git a/docs/white-paper.md b/docs/white-paper.md index e07bceb..3798593 100644 --- a/docs/white-paper.md +++ b/docs/white-paper.md @@ -144,7 +144,7 @@ have separate glossary entries, and this document always makes clear which is me | Scrollcase version | 0.6.0 | | --- | --- | -| Box format | `schemaVersion: 2` | +| Box format | `schemaVersion: 3` | | Substrate | pixi + conda-pack + conda-forge | | Runtime dependencies | `tar`, `yauzl`, `yazl` | | Node engine | >= 20 | @@ -152,10 +152,10 @@ have separate glossary entries, and this document always makes clear which is me -Version 2 is a clean break from version 1. A v2 verifier rejects a `schemaVersion: 1` document with -an explicit unsupported-version error rather than reinterpreting it, and this paper describes v2 -only. Published v1 artefacts remain usable with the Scrollcase versions that produced them, and are -otherwise out of scope here. +Version 3 is a clean break from versions 1 and 2. A v3 verifier rejects an older document with an +explicit unsupported-version error naming *which* version it holds, rather than reinterpreting it, +and this paper describes v3 only. Published v1 and v2 artefacts remain usable with the Scrollcase +versions that produced them, and are otherwise out of scope here. ## 2. Glossary @@ -194,7 +194,7 @@ Reference: `src/build/box.mjs`, `docs/reference/box-format.md`. The stable identifier of the thing being packaged, declared by the [scroll](#scroll) as `boxId` and carried into `box.json` and the [release](#release). One box ID spans every version and every target -of that box. It is distinct from `modelId` and `runtimeId`, which are the publishing project's own +of that box. It is distinct from `labels`, which are the publishing project's own identifiers for what the box contains and what runs it; Scrollcase stores and transports all three without interpreting them. @@ -848,7 +848,7 @@ temporary directory. The record of where a box came from: which scroll and scroll version, the 40-hex commit of the source tree that built it, whether that tree was dirty, the upstream revision of the packaged model -source as the scroll declared it, the Python and pixi versions, the SHA-256 of the lock the +source as the scroll declared it, the runtime and pixi versions, the SHA-256 of the lock the environment was solved from, and the build timestamp taken from the commit. It is recorded from observed state and never accepted from caller input. @@ -856,12 +856,16 @@ observed state and never accepted from caller input.
-#### Weights mode +#### Deferred asset -Whether a box carries its assets or refers to them: `embed` packs them into the archive, so the box -installs air-gapped at the cost of a large artefact; `on-demand` leaves them out and carries their -URL, path, size and SHA-256 in the signed release and in `box.json`. The declared hash is what makes -deferring safe — the release commits to exactly which bytes are expected, whatever host serves them. +An [asset](#asset) the scroll declared `"embed": false`. It is left out of the archive, and its URL, +path, size and SHA-256 travel in the signed release and in `box.json` instead, for the caller's +distribution layer to materialize. The declared hash is what makes deferring safe — the release +commits to exactly which bytes are expected, whatever host serves them. + +The choice is per entry, so one box can ship a small entry point inside the archive and defer a +large dataset beside it. Everything else is embedded, which is the default and the behaviour that +installs air-gapped.
@@ -1425,7 +1429,7 @@ of consumer: | Artefact | Location | What it is | Who uses it | | --- | --- | --- | --- | | Reference implementation | `src/contract/*.mjs` | The rules as executable code | JavaScript callers, and the builder itself | -| JSON Schemas | `src/contract/schema/*.json`, published at `/schema/v2/*.json` | The machine-readable specification | Validators, editors, any language with a schema library | +| JSON Schemas | `src/contract/schema/*.json`, published at `/schema/v3/*.json` | The machine-readable specification | Validators, editors, any language with a schema library | | Golden fixtures | `src/contract/fixtures/*.json` | What "agreeing" means, concretely | Implementations in other languages, proving themselves | @@ -1606,12 +1610,12 @@ it ships for. There is no cross-compilation: the environment being packed contai solved and installed for one platform, and a self-test run on the wrong host would prove nothing about the box. -The layout assertion beside it — `assertPythonEntryPoint(adapter, entryPoint)` — keeps its published -name while the wire format still spells the field `pythonEntryPoint`, and delegates to +The layout assertion beside it — `assertRuntimeEntryPoint(runtimeId, adapter, entryPoint)` — asks +the same question of any runtime, and delegates to `assertRuntimeEntryPoint()` in the runtime model. It refuses a scroll whose declared interpreter path disagrees with the runtime's layout for that target. The entry point is not free-form input — it is a fact about the runtime and the target together — and accepting a disagreement would produce -a signed release whose `pythonEntryPoint` pointed at nothing. +a signed release whose `runtime.entryPoint` pointed at nothing. @@ -1661,7 +1665,15 @@ and `python/src/scrollcase_consumer/_contract.py` — validate themselves agains | `executablePayloadPaths(target)` | the interpreter by name, and the scripts directory by prefix | | `resolveExecutionFiles({ execution, runtimeVersion, target })` | every payload path the declaration could resolve to, and the message for when none does | | `buildArgv({ execution, target })` | the shell-free command line, in payload-relative terms | -| `selfTestArgv({ probe, target })` | the arguments that follow the interpreter for a self-test | +| `selfTestInvocations({ probe, execution, target })` | every command a self-test probe implies, each with the exit status it must produce | + +Beside the adapters, the module exports the vocabulary and the two questions every caller asks +before reaching for one: `RUNTIME_IDS` names every runtime the *format* defines, +`isImplementedRuntime()` says whether this build carries an adapter for one, and +`unimplementedRuntimeMessage()` is the single wording the builder and all three consumers use when +it does not. `isExecutablePayloadPath()` answers whether a payload path is one the runtime requires +the executable bit on, and `executionAffectingVariables()` joins the runtime's loader controls to +the operating system's. The layout is `venv` on every target; what differs is where the interpreter and its generated scripts land inside it — `venv/bin/python` and `venv/bin` on POSIX, `venv/python.exe` and @@ -1704,16 +1716,18 @@ launcher repair, authoring templates, the pixi dependency a runtime contributes
-#### One runtime, registered rather than assumed +#### Two lists, on purpose -Only `python` is registered, and `runtimeAdapter('node')` is a `TypeError` rather than a stub. A -registry that answered for a runtime no build can produce would move the failure somewhere further -down, where the message no longer says what went wrong. +`RUNTIME_IDS` is the vocabulary a box may declare — `python`, `node`, `native` — and it is fixed by +the wire format. `RUNTIME_ADAPTERS` is what this build can actually run, and today that is `python` +alone: `runtimeAdapter('node')` is a `TypeError` rather than a stub, because a registry that +answered for a runtime no build can produce would move the failure somewhere further down, where the +message no longer says what went wrong. -The wire format carries no runtime declaration yet — a box records a Python entry point and Python -execution kinds and nothing that says "Python" — so a reader that must name one names -`IMPLICIT_RUNTIME_ID`, from a single place. Adding the declaration later changes an argument rather -than starting a hunt for hard-coded strings. +Keeping the two apart is what makes implementing `node` code rather than another format break. A box +declaring a runtime with no adapter is refused by name — `isImplementedRuntime()` asks the question +and `unimplementedRuntimeMessage()` gives the one wording the builder and all three consumers use — +and never misread as the runtime it happens to be shaped like. Reference: `tests/unit/contract-runtimes.test.mjs`, `rust/tests/contract.rs`, `python/tests/test_contract.py`. @@ -1732,7 +1746,7 @@ type: the type is inside it, discriminated by the payload's `kind`. ```jsonc { - "schemaVersion": 2, + "schemaVersion": 3, "payloadEncoding": "base64-json-utf8", "payloadBase64": "eyJraW5kIjoic2Nyb2xsY2FzZS5ib3gucmVsZWFzZSIsIn0=", "payloadSha256": "7d2c9a41…", @@ -1760,7 +1774,7 @@ implementation. ```js // src/contract/document-shape.mjs -export const BOX_SCHEMA_VERSION = 2; +export const BOX_SCHEMA_VERSION = 3; export const PAYLOAD_ENCODING = 'base64-json-utf8'; export const SIGNATURE_ALGORITHM = 'ed25519'; export const DEFAULT_DOCUMENT_NAMESPACE = 'scrollcase.box'; @@ -1814,12 +1828,14 @@ signature. `decodeDocumentPayload(document)` does three things in a fixed order, and the order is the point: -1. **Refuse `schemaVersion: 1` explicitly**, with the remedy in the message — - `Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.` A v1 document is not - reinterpreted, and it is not rejected as merely malformed either; it is named. The wording comes - from `unsupportedSchemaVersionMessage()` in `document-shape.mjs`, so the payload decoder, the key +1. **Refuse a superseded `schemaVersion` explicitly**, with the remedy in the message — + `Unsupported schemaVersion 2; rebuild this box with Scrollcase v3.` Both older versions are named + rather than lumped together as "too old": a v1 and a v2 box are different artefacts with + different rebuilds ahead of them, and whoever is holding one is entitled to know which. Neither + is reinterpreted, and neither is rejected as merely malformed. The wording comes from + `unsupportedSchemaVersionMessage()` in `document-shape.mjs`, so the payload decoder, the key loader and the release verifier say one thing rather than three copies of it — which is what the - next version bump has to change in one place instead of four. + v3 bump changed in one place instead of four. 2. **Refuse anything that fails the shape check.** 3. **Hash the decoded bytes and compare against `payloadSha256`** *before* parsing them as JSON. A truncated or edited document is caught before its contents are read at all. @@ -2068,7 +2084,7 @@ evaluate a constraint must refuse the box rather than assume it passes.* | `$def` | Pattern | Used for | | --- | --- | --- | -| `identifier` | `^[a-z0-9]+(?:[-.][a-z0-9]+)*$` | `boxId`, `modelId`, `runtimeId` | +| `identifier` | `^[a-z0-9]+(?:[-.][a-z0-9]+)*$` | `boxId`, every `labels` key | | `sha256` | `^[a-f0-9]{64}$` | Every digest, lowercase hex only | | `payloadPath` | a negative-lookahead chain | Any path inside the payload | @@ -2082,7 +2098,7 @@ schema level rather than leaving it to code: Not absolute, not a drive letter, no backslash anywhere, no `..` segment, no empty segment, non-empty. A path that fails this never reaches the code that would have to reject it. -**`schemaVersion` is `const: 2` in every document schema.** Not a minimum, not a range: a v1 +**`schemaVersion` is `const: 3` in every document schema.** Not a minimum, not a range: an older document fails schema validation with the same finality as the code rejects it. **`weights` and `assets` are paired by `dependentRequired`** in both the release and the box @@ -2097,13 +2113,13 @@ both directions. #### The scroll The largest schema, and the only one describing *input* rather than output. Nine fields are -required: `schemaVersion`, `boxId`, `modelId`, `runtimeId`, `version`, `sourceRevision`, -`pythonVersion`, `pixiVersion` and `selfTest`. A tenth, `target`, is required of every scroll a +required: `schemaVersion`, `boxId`, `version`, `sourceRevision`, `runtime`, `pixiVersion` and +`selfTest`. An eighth, `target`, is required of every scroll a build reads but not by the schema, because the base of a split scroll legitimately has none; the reader enforces it, so a base file still validates in an editor. That list is shorter than the format needs, because a scroll is a file someone writes by hand and -several of its fields were only ever restatements of others. `pythonEntryPoint` is the clearest +several of its fields were only ever restatements of others. `runtime.entryPoint` is the clearest case: the target adapter admits exactly one value and the reader rejected any other, so requiring it obliged the author to type the single string that was already implied. Those fields are now derived when the scroll is read, in one place, so every consumer of a scroll still sees a complete object: @@ -2112,8 +2128,8 @@ when the scroll is read, in one place, so every consumer of a scroll still sees | --- | --- | | `scrollVersion` | `1.0.0` | | `compatibility` | `{}` — declaring no constraint is an answer, and inventing one would be a claim the project never made | -| `pythonEntryPoint` | The target adapter's interpreter path; still checked against the target when declared | -| `modelCacheSubdir` | `model-cache/` | +| `runtime.entryPoint` | The runtime's own executable for this target; still checked against the layout when declared | +| `cacheSubdir` | `cache/` | | `assets` | `[]` | | `selfTest.files` | `[]` | @@ -2145,7 +2161,7 @@ leaves the pin off what it is still writing. And `selfTest.files` lists what mus *after* pruning, which is what stops an over-aggressive `prunePaths` from shipping a broken box. `selfTest` carries one more choice: the extra Python it runs after the imports may be given inline -as `pythonCode` or, mutually exclusively, as `pythonFile` — a path to a file in the project, read at +as `code` or, mutually exclusively, as `script` — a path to a file in the project, read at build time and executed from the payload root. A self-test that is worth writing outgrows a JSON string almost immediately, and in a file it keeps its syntax highlighting, its linter and a readable diff. @@ -2198,7 +2214,7 @@ after extraction. The schema states plainly that the builder also ran the scroll file assertions and that those are builder-only, rather than implying the signed check covers them. **`provenance`** requires all nine of its fields: `scrollId`, `scrollVersion`, `builderRevision` -(exactly 40 hex characters), `sourceTreeDirty`, `sourceRevision`, `pythonVersion`, `pixiVersion`, +(exactly 40 hex characters), `sourceTreeDirty`, `sourceRevision`, `pixiVersion`, `dependencyLockSha256` and `builtAt`. `sourceTreeDirty` is a required boolean rather than an optional flag, so "clean" is always an assertion somebody made and never the absence of one. @@ -2253,8 +2269,8 @@ distributes boxes — this is the boundary of section 3, expressed as a format. #### Publication is checked, not assumed -Each schema's `$id` is an absolute `https://scrollcase.dev/schema/v2/` URL, and byte-identical -copies are published under `docs/public/schema/v2/`. Two tests enforce it: one compares every +Each schema's `$id` is an absolute `https://scrollcase.dev/schema/v3/` URL, and byte-identical +copies are published under `docs/public/schema/v3/`. Two tests enforce it: one compares every published file against its source byte for byte, and one asserts that every `$id` and every absolute `$ref` resolves to a schema that is actually published. A schema referencing a sibling that never shipped would validate locally and fail for everyone else. @@ -2434,7 +2450,7 @@ interpreter first runs should be able to see it without following a call graph. | # | Stage | Module | State and files touched | | --- | --- | --- | --- | | 1 | Read and validate the scroll | `scroll.mjs` | Reads `scrolls///scroll.json`; resolves the adapter | -| 2 | Validate the build options | `box.mjs` | Channel in `CHANNELS`; weights mode; on-demand refuses `assetArchives` | +| 2 | Validate the build options | `box.mjs` | Channel in `CHANNELS`; the deferred-asset list is read off the scroll | | 3 | Refuse an unusable host, toolchain or tree | `targets.mjs`, `pixi.mjs`, `scroll.mjs` | `assertNativeHost`; pinned pixi and conda-pack located; `pixi.lock` present and hashed; git revision read, dirty tree refused | | 4 | Prepare the build tree | `box.mjs` | Removes and recreates `//payload/`; clears the target's object directory under `dist/` | | 5 | Solve, pack and relocate | `pixi.mjs`, `runtimes/python/launchers.mjs` | Installs into a build-local pixi workspace, packs it, extracts into `payload/venv/`, repairs it, deletes the workspace and tarball | @@ -2559,7 +2575,7 @@ installed. base of a split scroll still validates on its own. 4. **Weights and archives are compatible**: `on-demand` with `assetArchives` is refused, because those archives are expanded at build time and cannot be deferred. -5. **Every declared path is safe.** One sweep screens `modelCacheSubdir`, every asset path, both +5. **Every declared path is safe.** One sweep screens `cacheSubdir`, every asset path, both ends of every asset archive, both ends of every local file, every prune path, every self-test file, the self-test Python file, the execution script, the parity script and the licence audit path. @@ -2591,7 +2607,7 @@ declares both facts. Reading is also where a scroll becomes complete. `effectiveScroll()` runs between validation and the path sweep, filling in every field the target or the identity already determines — the interpreter -path, `model-cache/`, a `scrollVersion` of `1.0.0`, and the empty collections. Everything +path, `cache/`, a `scrollVersion` of `1.0.0`, and the empty collections. Everything downstream, including the provenance record, sees that one object and never has to ask whether a field was written down. @@ -2609,7 +2625,7 @@ substance of the feature: | `assets`, `assetArchives`, `localFiles` | Joined base-first; a repeated `relativePath` is an error | | `prunePaths`, `uncompressedPaths`, `selfTest.imports`, `selfTest.files` | Joined base-first, repeats dropped | | `compatibility`, `environment` | Joined key by key, the fragment winning a shared key | -| `selfTest.pythonCode` / `selfTest.pythonFile` | One slot; a fragment naming either replaces both | +| `selfTest.code` / `selfTest.script` | One slot; a fragment naming either replaces both | | `extends` | Dropped — the joined scroll extends nothing | Each row is a rejection of the two obvious alternatives. Replace-everything would make a fragment @@ -3055,7 +3071,7 @@ The result is sorted by name then version, which is what makes the inventory its ```jsonc { - "schemaVersion": 2, + "schemaVersion": 3, "kind": "scrollcase.box.dependency-license-audit", "targetId": "macos-aarch64-metal", "dependencyLockSha256": "…", @@ -3415,7 +3431,7 @@ The two functions perform the complete read-only chain in this fixed order: 1. Refuse `schemaVersion: 1` explicitly. 2. Validate the **signed envelope** against its schema. 3. **Verify the signature** against the trusted key. -4. Refuse a payload that is not `schemaVersion: 2`. +4. Refuse a payload that is not `schemaVersion: 3`. 5. Validate the **release manifest** against its schema. 6. Confirm the document's `kind` parses as a *release*. 7. Resolve the target adapter and check the entry point against it. @@ -3441,8 +3457,8 @@ ones come after the cheap ones that could have ended the check. ```js // src/build/verify.mjs const AGREEMENT_FIELDS = [ - 'schemaVersion', 'boxId', 'modelId', 'runtimeId', 'version', 'target', - 'pythonEntryPoint', 'modelCacheSubdir', 'environment', 'selfTest', 'execution', + 'schemaVersion', 'boxId', 'labels', 'version', 'target', + 'runtime', 'cacheSubdir', 'environment', 'selfTest', 'execution', 'weights', 'assets', 'provenance', ]; ``` @@ -3576,7 +3592,7 @@ What it generates is deliberately short. Every field the reader can derive is le reads as the decisions its author made rather than a form they filled in; the generated starter script is recorded in `localFiles` **without a hash pin**, because the first thing an author does with a starter is edit it; and the self-test is written as a real `self_test.py` beside the scroll, -with `selfTest.pythonFile` pointing at it. +with `selfTest.script` pointing at it. Two constants live here rather than in a lookup: `DEFAULT_PYTHON_VERSION`, one minor behind the newest Python conda-forge publishes, and `LATEST_PYTHON_VERSION`, what `--python-version latest` @@ -3592,12 +3608,10 @@ Execution intent is a closed set at this level too — `python-script`, `python- `library-only` — and a `library-only` scroll declaring a script, a module or default arguments is refused rather than silently simplified. -The weights mode is not one of the decisions `new scroll` asks about. It says where declared assets -live — inside the archive, or beside it for the caller to materialize — and a box that declares no -assets, which is most of them, has nothing for it to decide. `createScroll` defaults it to `embed` -and then leaves it out of the generated file, because that is the schema's own default and a scroll -should read like the decisions its author actually made. `--weights on-demand` states the other -choice for a box whose assets are published separately. +Labels are not one of the decisions `new scroll` asks about. Scrollcase reads none of them, so +prompting for one would be asking the author to fill in a field on the tool's behalf; `createScroll` +leaves the map out of the generated file entirely, because a scroll should read like the decisions +its author actually made. `--labels '{"model":"…"}'` states them when there is something to record. `ensureExampleScroll()` creates the disposable `example-box` that `init` offers, through the same validated authoring path as any real scroll. An existing target directory is treated as authored @@ -4338,7 +4352,7 @@ implementation detail; it is part of the contract, and every consumer follows it ```text 1 signed document schema, signature, payload digest - 2 release manifest schema, schemaVersion 2, kind + 2 release manifest schema, schemaVersion 3, kind 3 target adapter resolved, declared interpreter path 4 archive located, size, SHA-256 5 every archive entry safe path, kind, collisions, links @@ -4525,9 +4539,9 @@ The root identity check is the one that is easy to leave out. Without it, prepar running it later would trust that nothing swapped the directory in between — which on a shared machine is precisely the assumption an attacker wants. -::: warning On-demand assets are verified, never fetched -When [weights mode](#weights-mode) is `on-demand`, the release carries signed descriptors and the -receipt exposes them as `requiredAssets`. The caller places those bytes under the box root — often +::: warning Deferred assets are verified, never fetched +For every [deferred asset](#deferred-asset) the release carries a signed descriptor, and the receipt +exposes them as `requiredAssets`. The caller places those bytes under the box root — often in an `onPrepared` callback. The consumer then checks each one's size and SHA-256 against the signed descriptor before spawning anything, and refuses to run if any is missing, is not a regular file, or does not match. Downloading them is the caller's job, always. @@ -4709,8 +4723,8 @@ to the running host's interpreter path and target ID, and `$BOX` to the prepared therefore assert a complete absolute argument vector without hard-coding a platform or a temporary path. -Note that the fixture's own `schemaVersion: 1` is the *fixture format's* version. It is unrelated to -the box format's `schemaVersion: 2`. +Note that the fixture's own `schemaVersion` is the *fixture format's* version. It is unrelated to +the box format's `schemaVersion: 3`.
@@ -4902,9 +4916,9 @@ reads. Its guarantee is worth stating exactly, because it is narrower than it lo A release built before the digest existed is refused by name rather than reported as verified. A box that carries no commitment must not be mistaken for one that satisfies it. -The weights mode changes the cost honestly. Embedded weights are payload entries and verification -reads all of them, which can mean tens of gigabytes. On-demand assets were absent when the list was -built and appear later as ignored extras; their integrity is covered separately by the signed +Deferring changes the cost honestly. An embedded asset is a payload entry and verification reads +all of them, which can mean tens of gigabytes. A deferred asset was absent when the list was built +and appears later as an ignored extra; their integrity is covered separately by the signed per-file `requiredAssets` descriptors that attachment and execution enforce. Mode and modification time are also outside the digest, because archive writing synthesises modes and extraction restores neither the build mode consistently across platforms nor the fixed build timestamp. @@ -5203,7 +5217,6 @@ everything to `buildBox`: await ensureBuildSigningKeys(signing); // Asked at the CLI edge and passed down: buildBox never reads a terminal itself. const channel = await chooseCliValue('channel', ['beta', …], { flag: text(flags, 'channel') }); -const weights = text(flags, 'weights'); ``` The order is the point. The preflight is a read-only check that the keys exist (section 7.6), and it @@ -5213,11 +5226,13 @@ second, not the twenty minutes it would cost if it were discovered at the signin `beta` is listed first so it is the highlighted default in the menu and the value taken when there is no terminal — the channel a build should land on unless someone deliberately says otherwise. -The weights mode used to be a second menu, and that was a defect rather than a convenience. It was -preselected on `embed`, so a build of a scroll declaring `on-demand` silently repacked the assets +Where assets live used to be a second menu, and that was a defect rather than a convenience. It was +preselected on `embed`, so a build of a scroll that had said otherwise silently repacked the assets into the archive for anyone who answered by pressing Enter — the scroll's own declaration overridden -by the menu's default. It is not asked any more: `buildBox` takes the scroll's mode, `--weights` -overrides it deliberately, and the mode in effect is logged rather than negotiated. +by the menu's default. Version 3 removed the question *and* the `--weights` flag that replaced it: +whether an asset ships inside the archive is a per-entry scroll declaration, and a build-time +override of one would repack the box under an identity that no longer describes it. What the scroll +decided is logged rather than negotiated. @@ -5669,7 +5684,7 @@ document different, which breaks the chain just as thoroughly. #### The exact scope of the promise Determinism is claimed **per commit, per target, per pinned toolchain**. Same commit, same host -platform, same pinned pixi and conda-pack, same committed lock, same weights mode — same bytes. +platform, same pinned pixi and conda-pack, same committed lock — same bytes. It is not a claim that two different operating systems produce the same archive: they cannot, because they package different native code, and section 5.2 is why a box is built on the host it ships for. @@ -5788,7 +5803,7 @@ guard, because a rule with no guard survives exactly as long as everyone remembe | No consuming project's name anywhere in the tool | It must stay usable by projects with nothing to do with the one that first needed it | `tests/unit/v2-migration.test.mjs` greps the whole tracked tree, content and paths | | The document namespace belongs to the publishing project | A project with boxes in the field keeps emitting the kinds its clients recognise | `documentKinds(namespace)`; the schemas accept any well-formed namespace and nothing else | | One substrate, and only one | Two dependency backends means proving every guarantee twice | The absence of any second backend, and section 4's single-substrate description | -| Published v1 is immutable; v2 is a clean break | A reinterpreted old document is a silent wrong answer | `decodeSignedDocument` rejects `schemaVersion: 1` with an explicit remedy | +| Published v1 and v2 are immutable; v3 is a clean break | A reinterpreted old document is a silent wrong answer | `decodeSignedDocument` rejects both by name, each with its own remedy | The first guard is worth a note for anyone who reads it. It runs `git grep` over every tracked file *and* every tracked path, and the retired term it searches for is assembled from two string fragments @@ -5830,21 +5845,23 @@ that assumes one is absent on POSIX, is wrong on one of the two.
-#### 2. `embed` versus `on-demand` weights +#### 2. Embedded versus [deferred](#deferred-asset) assets -The two [weights modes](#weights-mode) produce different archives, different release documents and -different consumer behaviour from the same scroll: +The two produce different archives, different release documents and different consumer behaviour +from the same scroll — and a box may now do both at once, one entry each way: -| | `embed` | `on-demand` | +| | `"embed": true` (default) | `"embed": false` | | --- | --- | --- | -| Assets in the archive | Yes | No | -| Descriptors in the signed release | Not needed | Required | -| `assetArchives` | Allowed | **Refused** | -| Air-gapped install | Works | Needs the assets materialised first | -| Consumer before execution | Nothing extra | Verifies every materialised asset against its signed hash | +| In the archive | Yes | No | +| Descriptor in the signed release | Not needed | Required | +| Air-gapped install | Works | Needs the asset materialised first | +| Consumer before execution | Nothing extra | Verifies it against its signed hash | + +`assetArchives` has no `embed` field at all, so the combination version 2 refused with a cross-field +check is now unspeakable. Any change to asset staging, to the manifest, or to what the consumer checks before spawning affects -both, and a test that only covers the default mode covers half the behaviour. +both, and a test whose assets are all embedded covers half the behaviour.
@@ -6339,7 +6356,12 @@ consumer-only dependent avoid the entire build layer. | `condaSubdir` | function | The [conda subdir](#conda-subdir) a target maps to | | `pixiAccelerator` | function | The accelerator descriptor a scroll selects | | `assertNativeHost` | function | Refuses a build on a host that is not the target it ships for | -| `assertPythonEntryPoint` | function | Refuses an entry point that disagrees with the runtime's layout for the target | +| `assertRuntimeEntryPoint` | function | Refuses an entry point that disagrees with the named runtime's layout for the target | +| `RUNTIME_IDS` | array | Every runtime id the format defines: `python`, `node`, `native` | +| `runtimeAdapter`, `runtimeAdapters` | function | The runtime model: layout, execution kinds, argv, self-test | +| `isImplementedRuntime`, `unimplementedRuntimeMessage` | function | Whether this build carries an adapter, and the one wording for when it does not | +| `executionAffectingVariables` | function | The runtime's loader controls followed by the operating system's | +| `isExecutablePayloadPath` | function | Whether a payload path is one the runtime requires the executable bit on | | `documentKinds` | function | The three `kind` strings for a publishing project's namespace | | `parseDocumentKind` | function | Splits a `kind` back into namespace and document type | | `isSignedBoxDocument` | function | The structural envelope guard — shape only, never trust | diff --git a/scripts/sync-docs-schemas.mjs b/scripts/sync-docs-schemas.mjs index 6335f54..0d342c6 100644 --- a/scripts/sync-docs-schemas.mjs +++ b/scripts/sync-docs-schemas.mjs @@ -13,7 +13,7 @@ import { join } from 'node:path'; const check = process.argv.includes('--check'); const root = fileURLToPath(new URL('..', import.meta.url)); const sourceDir = join(root, 'src', 'contract', 'schema'); -const publicDir = join(root, 'docs', 'public', 'schema', 'v2'); +const publicDir = join(root, 'docs', 'public', 'schema', 'v3'); const schemaNames = (await readdir(sourceDir)) .filter((name) => name.endsWith('.schema.json')) .sort(); diff --git a/scripts/verify-built-docs.mjs b/scripts/verify-built-docs.mjs index 358580e..b1d3ea6 100644 --- a/scripts/verify-built-docs.mjs +++ b/scripts/verify-built-docs.mjs @@ -13,7 +13,7 @@ import { join, resolve } from 'node:path'; const root = fileURLToPath(new URL('..', import.meta.url)); const distDir = resolve(process.argv[2] ?? join(root, 'docs', '.vitepress', 'dist')); const schemaSource = join(root, 'src', 'contract', 'schema'); -const schemaDist = join(distDir, 'schema', 'v2'); +const schemaDist = join(distDir, 'schema', 'v3'); async function requireFile(path, label) { try { diff --git a/src/build/archive.d.mts b/src/build/archive.d.mts index 1ebf337..1d26da9 100644 --- a/src/build/archive.d.mts +++ b/src/build/archive.d.mts @@ -3,8 +3,8 @@ * * Deflating an already-compressed file is pure loss: measured on incompressible bytes, level 6 * runs at 47 MB/s and the result is 0.03% *larger* than the input, and dropping to level 1 buys - * 4 MB/s because the search fails either way. Weights are the only thing in a box large enough for - * that to matter, so `uncompressedPaths` names them and they are stored instead. Everything else — + * 4 MB/s because the search fails either way. Declared assets are the only thing in a box large + * enough for that to matter, so they and `uncompressedPaths` are stored instead. Everything else — * the interpreter, the site-packages tree, the notices — compresses genuinely and still does. * * @param {string} payloadDir diff --git a/tests/unit/cli-target-choice.test.mjs b/tests/unit/cli-target-choice.test.mjs index 1a96e5e..1ed5735 100644 --- a/tests/unit/cli-target-choice.test.mjs +++ b/tests/unit/cli-target-choice.test.mjs @@ -170,8 +170,9 @@ describe('CLI target selection', () => { 'utf8', )); expect(scroll).toMatchObject({ - schemaVersion: 2, + schemaVersion: 3, boxId: 'example-box', + runtime: { id: 'python' }, target: { platform: adapter.platform, arch: adapter.arch, diff --git a/tests/unit/docs-contract.test.mjs b/tests/unit/docs-contract.test.mjs index 6f2f263..71569c5 100644 --- a/tests/unit/docs-contract.test.mjs +++ b/tests/unit/docs-contract.test.mjs @@ -11,7 +11,7 @@ import * as sign from '../../src/sign/index.mjs'; const root = fileURLToPath(new URL('../..', import.meta.url)); const schemaSource = join(root, 'src', 'contract', 'schema'); -const schemaPublic = join(root, 'docs', 'public', 'schema', 'v2'); +const schemaPublic = join(root, 'docs', 'public', 'schema', 'v3'); const whitePaperPath = join(root, 'docs', 'white-paper.md'); @@ -105,7 +105,7 @@ describe('public documentation routes', () => { for (const name of names) { const source = await readFile(join(schemaSource, name), 'utf8'); const schema = JSON.parse(source); - expect(new URL(schema.$id).pathname).toBe(`/schema/v2/${name}`); + expect(new URL(schema.$id).pathname).toBe(`/schema/v3/${name}`); for (const match of source.matchAll(/"\$ref":\s*"(https:\/\/scrollcase\.dev\/schema\/v2\/([^"#]+)(?:#[^"]*)?)"/g)) { expect(published.has(match[2]), match[1]).toBe(true); } @@ -182,7 +182,7 @@ describe('public documentation routes', () => { const markdown = await readFile(path, 'utf8'); for (const match of markdown.matchAll(/^```json\n([\s\S]*?)^```$/gm)) { const example = JSON.parse(match[1]); - if (example?.schemaVersion === 2 && typeof example?.scrollVersion === 'string' && example?.target) { + if (example?.schemaVersion === 3 && typeof example?.scrollVersion === 'string' && example?.target) { expect(validateScroll(example), `${path}: ${ajv.errorsText(validateScroll.errors)}`).toBe(true); } } diff --git a/tests/unit/docs-markdown-negotiation.test.mjs b/tests/unit/docs-markdown-negotiation.test.mjs index 633d4a0..c0cfbac 100644 --- a/tests/unit/docs-markdown-negotiation.test.mjs +++ b/tests/unit/docs-markdown-negotiation.test.mjs @@ -60,7 +60,7 @@ describe('markdownPathFor', () => { it('leaves anything that is not a page alone', () => { expect(markdownPathFor('/llms.txt')).toBeNull(); - expect(markdownPathFor('/schema/v2/target.schema.json')).toBeNull(); + expect(markdownPathFor('/schema/v3/target.schema.json')).toBeNull(); expect(markdownPathFor('/static/svg/logo-dark.svg')).toBeNull(); // Asking for the Markdown of a Markdown file is a loop. expect(markdownPathFor('/reference/cli.md')).toBeNull(); From 411008744aad9b7da894c987818726dbd40cd60c Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:35:14 +0200 Subject: [PATCH 06/22] Bring the Rust consumer to version 3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manifest types carry a runtime block, labels and a probe instead of modelId, runtimeId, pythonEntryPoint and pythonImports, and the deferred-asset list replaces the weights/assets co-requirement — the list is the whole statement now, so there is nothing left for it to disagree with. RuntimeExecution gains a Binary shape and the kind becomes - asked of the runtime that owns it: a script belongs to python or to node depending on the box, and the shape alone cannot say which. rust/tests/fixtures/signed-release.json is re-signed under a fresh key, its private half never having been committed, and the trusted key beside it is rewritten to match. --- rust/fixtures/consumer-conformance.json | 146 +++++-- .../examples/box-manifest.example.json | 28 +- .../examples/release-manifest.example.json | 28 +- .../examples/signed-release.example.json | 10 +- rust/fixtures/runtime-contract.json | 404 ++++++++++++++++-- rust/src/contract/documents.rs | 23 +- rust/src/contract/runtimes.rs | 280 +++++++++--- .../contract/schema/box-manifest.schema.json | 96 +---- .../src/contract/schema/execution.schema.json | 52 ++- .../schema/release-manifest.schema.json | 176 +++++--- .../schema/signed-document.schema.json | 4 +- rust/src/contract/schema/target.schema.json | 2 +- rust/src/contract/targets.rs | 25 +- rust/src/execution.rs | 19 +- rust/src/prepare.rs | 45 +- rust/src/release.rs | 255 ++++++++--- rust/src/run.rs | 27 +- rust/src/trust.rs | 2 +- rust/src/verify.rs | 60 ++- rust/tests/archive.rs | 8 +- rust/tests/conformance.rs | 88 +++- rust/tests/contract.rs | 82 +++- rust/tests/fixtures/signed-release.json | 10 +- rust/tests/fixtures/trusted-key.json | 6 +- rust/tests/prepare.rs | 22 +- rust/tests/release_document.rs | 8 +- rust/tests/run.rs | 2 +- rust/tests/schema.rs | 30 +- rust/tests/support/mod.rs | 23 +- .../fixtures/consumer-conformance.json | 3 +- 30 files changed, 1418 insertions(+), 546 deletions(-) diff --git a/rust/fixtures/consumer-conformance.json b/rust/fixtures/consumer-conformance.json index 39f6023..8d81d99 100644 --- a/rust/fixtures/consumer-conformance.json +++ b/rust/fixtures/consumer-conformance.json @@ -1,36 +1,38 @@ { - "schemaVersion": 1, + "schemaVersion": 3, "description": "Language-neutral semantic cases shared by the Node, Python and Rust Scrollcase consumers.", "errorPatterns": { - "invalid-trust-file": "Invalid trusted ed25519 key file", - "invalid-signature": "no valid signature", "altered-payload": "Signed payload SHA-256 mismatch", "archive-hash": "Archive SHA-256 mismatch", "archive-size": "Archive size mismatch", - "manifest-disagreement": "box.json mismatch: modelId", - "execution-disagreement": "box.json mismatch: execution", - "environment-disagreement": "box.json mismatch: environment", - "missing-interpreter": "Archive is missing venv/", - "missing-script": "Execution script is missing", - "missing-module": "Execution module is not discoverable", - "unsafe-path": "Unsafe relative path|invalid relative path|absolute path", - "link-entry": "link does not resolve to a file inside the payload|would be written through a link|link target is too long", - "special-entry": "special entries", - "encrypted-entry": "Encrypted ZIP entries", - "entry-collision": "Archive entry collides with another entry", - "existing-destination": "Destination already exists", + "asset-hash": "asset SHA-256 mismatch", "asset-missing": "asset is missing", "asset-size": "asset size mismatch", - "asset-hash": "asset SHA-256 mismatch", - "spawn-failure": "failed to start|fixture spawn failed", - "attach-root": "is not an extracted box directory", "attach-missing-interpreter": "Attached box is missing venv/", + "attach-root": "is not an extracted box directory", + "encrypted-entry": "Encrypted ZIP entries", + "entry-collision": "Archive entry collides with another entry", + "environment-disagreement": "box.json mismatch: environment", + "execution-disagreement": "box.json mismatch: execution", + "existing-destination": "Destination already exists", "foreign-target": "cannot run on", - "payload-list-missing": "missing its payload digest list", - "payload-list-mismatch": "Payload digest list does not match the signed release", + "invalid-signature": "no valid signature", + "invalid-trust-file": "Invalid trusted ed25519 key file", + "link-entry": "link does not resolve to a file inside the payload|would be written through a link|link target is too long", + "manifest-disagreement": "box.json mismatch: labels", + "missing-interpreter": "Archive is missing venv/", + "missing-module": "Execution module is not discoverable", + "missing-script": "Execution script is missing", "payload-digest-absent": "does not commit to a payload digest", + "payload-list-mismatch": "Payload digest list does not match the signed release", + "payload-list-missing": "missing its payload digest list", "payload-mismatch": "^Payload does not match the signed release:", - "unsupported-schema-version": "Unsupported schemaVersion 1" + "runtime-disagreement": "box.json mismatch: runtime", + "spawn-failure": "failed to start|fixture spawn failed", + "special-entry": "special entries", + "unimplemented-runtime": "is not implemented by this version", + "unsafe-path": "Unsafe relative path|invalid relative path|absolute path", + "unsupported-schema-version": "Unsupported schemaVersion 1|Unsupported schemaVersion 2" }, "cases": [ { @@ -50,7 +52,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -69,7 +72,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -88,7 +92,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -107,7 +112,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -278,7 +284,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -294,7 +301,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -319,7 +327,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET", "environmentReport": { "mode": "summary", @@ -370,7 +379,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET", "environmentReport": { "mode": "full", @@ -626,7 +636,7 @@ { "id": "release-box-disagreement", "action": "prepare", - "mutation": "alter-release-model", + "mutation": "alter-release-labels", "expected": { "outcome": "rejected", "error": "manifest-disagreement", @@ -718,7 +728,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -786,7 +797,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "venv/bin/python", + "runtimeId": "python", + "entryPoint": "venv/bin/python", "targetId": "macos-aarch64-cpu" } } @@ -804,7 +816,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "venv/bin/python", + "runtimeId": "python", + "entryPoint": "venv/bin/python", "targetId": "linux-x86_64-cpu" } } @@ -822,7 +835,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "venv/python.exe", + "runtimeId": "python", + "entryPoint": "venv/python.exe", "targetId": "windows-x86_64-cpu" } } @@ -861,7 +875,7 @@ "signal": null }, "argv": [ - "$BOX/$NATIVE_PYTHON", + "$BOX/$NATIVE_ENTRY_POINT", "$BOX/app/main.py", "--default", "value with spaces", @@ -891,7 +905,7 @@ "signal": null }, "argv": [ - "$BOX/$NATIVE_PYTHON", + "$BOX/$NATIVE_ENTRY_POINT", "$BOX/app/main.py", "--default", "value with spaces", @@ -1046,7 +1060,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -1069,7 +1084,7 @@ "signal": null }, "argv": [ - "$BOX/$NATIVE_PYTHON", + "$BOX/$NATIVE_ENTRY_POINT", "$BOX/app/main.py", "--default", "value with spaces", @@ -1092,7 +1107,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -1113,7 +1129,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 1, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -1129,7 +1146,8 @@ "boxId": "consumer-fixture", "executionKind": "python-script", "requiredAssetCount": 0, - "pythonEntryPoint": "$NATIVE_PYTHON", + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", "targetId": "$NATIVE_TARGET" } } @@ -1352,6 +1370,52 @@ "outcome": "rejected", "error": "payload-list-mismatch" } + }, + { + "id": "declared-executable-survives-a-restrictive-umask", + "action": "prepare", + "fixture": { + "executableAsset": true + }, + "runtime": { + "umask": "077" + }, + "expected": { + "outcome": "prepared", + "receipt": { + "status": "prepared", + "boxId": "consumer-fixture", + "executionKind": "python-script", + "requiredAssetCount": 0, + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", + "targetId": "$NATIVE_TARGET", + "executableModes": { + "bin/tool": "755", + "box.json": "644" + } + } + } + }, + { + "id": "runtime-this-build-cannot-run", + "action": "prepare", + "mutation": "alter-release-runtime-id", + "expected": { + "outcome": "rejected", + "error": "unimplemented-runtime", + "destinationExists": false + } + }, + { + "id": "runtime-block-disagreement", + "action": "prepare", + "mutation": "alter-release-runtime-version", + "expected": { + "outcome": "rejected", + "error": "runtime-disagreement", + "destinationExists": false + } } ] } diff --git a/rust/fixtures/examples/box-manifest.example.json b/rust/fixtures/examples/box-manifest.example.json index 3613501..d324293 100644 --- a/rust/fixtures/examples/box-manifest.example.json +++ b/rust/fixtures/examples/box-manifest.example.json @@ -1,21 +1,29 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "boxId": "example-model", - "modelId": "example-org-example-model", - "runtimeId": "example-model-runtime", + "labels": { + "model": "example-org/example-model", + "owner": "platform-team" + }, "version": "1.0.0", "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, - "pythonEntryPoint": "venv/bin/python", - "modelCacheSubdir": "model-cache/example-model", + "runtime": { + "id": "python", + "version": "3.11.15", + "entryPoint": "venv/bin/python" + }, + "cacheSubdir": "cache/example-model", "selfTest": { - "pythonImports": [ - "torch", - "numpy" - ], + "probe": { + "imports": [ + "torch", + "numpy" + ] + }, "timeoutSeconds": 180 }, "provenance": { @@ -24,7 +32,7 @@ "builderRevision": "4c1d9b7e2a0f5836d419e7c05b3a8f61d2704e93", "sourceTreeDirty": false, "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", - "pythonVersion": "3.11.15", + "runtimeVersion": "3.11.15", "pixiVersion": "0.73.0", "dependencyLockSha256": "3b1f8c47a2d9e05b6c7418af23d5e69017b4c8ad91e2f350768bd4ca19e0f5b7", "builtAt": "2026-07-25T12:00:00+00:00" diff --git a/rust/fixtures/examples/release-manifest.example.json b/rust/fixtures/examples/release-manifest.example.json index 9a04e5b..e1d83fc 100644 --- a/rust/fixtures/examples/release-manifest.example.json +++ b/rust/fixtures/examples/release-manifest.example.json @@ -1,9 +1,11 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "kind": "scrollcase.box.release", "boxId": "example-model", - "modelId": "example-org-example-model", - "runtimeId": "example-model-runtime", + "labels": { + "model": "example-org/example-model", + "owner": "platform-team" + }, "version": "1.0.0", "target": { "platform": "macos", @@ -22,13 +24,19 @@ "sizeBytes": 655752216 }, "installedSizeBytes": 1892340112, - "pythonEntryPoint": "venv/bin/python", - "modelCacheSubdir": "model-cache/example-model", + "runtime": { + "id": "python", + "version": "3.11.15", + "entryPoint": "venv/bin/python" + }, + "cacheSubdir": "cache/example-model", "selfTest": { - "pythonImports": [ - "torch", - "numpy" - ], + "probe": { + "imports": [ + "torch", + "numpy" + ] + }, "timeoutSeconds": 180 }, "provenance": { @@ -37,7 +45,7 @@ "builderRevision": "4c1d9b7e2a0f5836d419e7c05b3a8f61d2704e93", "sourceTreeDirty": false, "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", - "pythonVersion": "3.11.15", + "runtimeVersion": "3.11.15", "pixiVersion": "0.73.0", "dependencyLockSha256": "3b1f8c47a2d9e05b6c7418af23d5e69017b4c8ad91e2f350768bd4ca19e0f5b7", "builtAt": "2026-07-25T12:00:00+00:00" diff --git a/rust/fixtures/examples/signed-release.example.json b/rust/fixtures/examples/signed-release.example.json index 64b1a63..b19c21f 100644 --- a/rust/fixtures/examples/signed-release.example.json +++ b/rust/fixtures/examples/signed-release.example.json @@ -1,13 +1,13 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "payloadEncoding": "base64-json-utf8", - "payloadBase64": "ewogICJzY2hlbWFWZXJzaW9uIjogMiwKICAia2luZCI6ICJzY3JvbGxjYXNlLmJveC5yZWxlYXNlIiwKICAiYm94SWQiOiAiZXhhbXBsZS1tb2RlbCIsCiAgIm1vZGVsSWQiOiAiZXhhbXBsZS1vcmctZXhhbXBsZS1tb2RlbCIsCiAgInJ1bnRpbWVJZCI6ICJleGFtcGxlLW1vZGVsLXJ1bnRpbWUiLAogICJ2ZXJzaW9uIjogIjEuMC4wIiwKICAidGFyZ2V0IjogewogICAgInBsYXRmb3JtIjogIm1hY29zIiwKICAgICJhcmNoIjogImFhcmNoNjQiLAogICAgImFjY2VsZXJhdG9yIjogIm1ldGFsIgogIH0sCiAgImNvbXBhdGliaWxpdHkiOiB7CiAgICAibWluSG9zdEFwcFZlcnNpb24iOiAiMS4wLjAiLAogICAgIm1pbk1hY29zVmVyc2lvbiI6ICIxMy4wIiwKICAgICJtaW5SYW1HYiI6IDgKICB9LAogICJhcmNoaXZlIjogewogICAgImZvcm1hdCI6ICJ6aXAiLAogICAgInVybCI6ICJodHRwczovL2Fzc2V0cy5leGFtcGxlLm9yZy9ib3hlcy9ib3hlcy9leGFtcGxlLW1vZGVsLzEuMC4wL21hY29zLWFhcmNoNjQtbWV0YWwvN2QyYzlhNDFlOGIzNTBmNmMxNzRhOWRlMjAzNThiZjQxYzZlOTdkMDVhOGIzZjI2MTllNGM3MDgxZGE1YjNmMi56aXAiLAogICAgInNoYTI1NiI6ICI3ZDJjOWE0MWU4YjM1MGY2YzE3NGE5ZGUyMDM1OGJmNDFjNmU5N2QwNWE4YjNmMjYxOWU0YzcwODFkYTViM2YyIiwKICAgICJzaXplQnl0ZXMiOiA2NTU3NTIyMTYKICB9LAogICJpbnN0YWxsZWRTaXplQnl0ZXMiOiAxODkyMzQwMTEyLAogICJweXRob25FbnRyeVBvaW50IjogInZlbnYvYmluL3B5dGhvbiIsCiAgIm1vZGVsQ2FjaGVTdWJkaXIiOiAibW9kZWwtY2FjaGUvZXhhbXBsZS1tb2RlbCIsCiAgInNlbGZUZXN0IjogewogICAgInB5dGhvbkltcG9ydHMiOiBbCiAgICAgICJ0b3JjaCIsCiAgICAgICJudW1weSIKICAgIF0sCiAgICAidGltZW91dFNlY29uZHMiOiAxODAKICB9LAogICJwcm92ZW5hbmNlIjogewogICAgInNjcm9sbElkIjogImV4YW1wbGUtbW9kZWwtbWFjb3MtYXJtNjQtbWV0YWwiLAogICAgInNjcm9sbFZlcnNpb24iOiAiMS4wLjAiLAogICAgImJ1aWxkZXJSZXZpc2lvbiI6ICI0YzFkOWI3ZTJhMGY1ODM2ZDQxOWU3YzA1YjNhOGY2MWQyNzA0ZTkzIiwKICAgICJzb3VyY2VUcmVlRGlydHkiOiBmYWxzZSwKICAgICJzb3VyY2VSZXZpc2lvbiI6ICI5ZjBkM2ExYzRiNWU2ZjcwODE5MjBhM2I0YzVkNmU3ZjgwOTEwYTJiIiwKICAgICJweXRob25WZXJzaW9uIjogIjMuMTEuMTUiLAogICAgInBpeGlWZXJzaW9uIjogIjAuNzMuMCIsCiAgICAiZGVwZW5kZW5jeUxvY2tTaGEyNTYiOiAiM2IxZjhjNDdhMmQ5ZTA1YjZjNzQxOGFmMjNkNWU2OTAxN2I0YzhhZDkxZTJmMzUwNzY4YmQ0Y2ExOWUwZjViNyIsCiAgICAiYnVpbHRBdCI6ICIyMDI2LTA3LTI1VDEyOjAwOjAwKzAwOjAwIgogIH0KfQo=", - "payloadSha256": "bbf60de7d31035b2bfcb98c6c57624220f3bf59900391189f5e55da955055bc6", + "payloadBase64": "ewogICJzY2hlbWFWZXJzaW9uIjogMywKICAia2luZCI6ICJzY3JvbGxjYXNlLmJveC5yZWxlYXNlIiwKICAiYm94SWQiOiAiZXhhbXBsZS1tb2RlbCIsCiAgImxhYmVscyI6IHsKICAgICJtb2RlbCI6ICJleGFtcGxlLW9yZy9leGFtcGxlLW1vZGVsIiwKICAgICJvd25lciI6ICJwbGF0Zm9ybS10ZWFtIgogIH0sCiAgInZlcnNpb24iOiAiMS4wLjAiLAogICJ0YXJnZXQiOiB7CiAgICAicGxhdGZvcm0iOiAibWFjb3MiLAogICAgImFyY2giOiAiYWFyY2g2NCIsCiAgICAiYWNjZWxlcmF0b3IiOiAibWV0YWwiCiAgfSwKICAiY29tcGF0aWJpbGl0eSI6IHsKICAgICJtaW5Ib3N0QXBwVmVyc2lvbiI6ICIxLjAuMCIsCiAgICAibWluTWFjb3NWZXJzaW9uIjogIjEzLjAiLAogICAgIm1pblJhbUdiIjogOAogIH0sCiAgImFyY2hpdmUiOiB7CiAgICAiZm9ybWF0IjogInppcCIsCiAgICAidXJsIjogImh0dHBzOi8vYXNzZXRzLmV4YW1wbGUub3JnL2JveGVzL2JveGVzL2V4YW1wbGUtbW9kZWwvMS4wLjAvbWFjb3MtYWFyY2g2NC1tZXRhbC83ZDJjOWE0MWU4YjM1MGY2YzE3NGE5ZGUyMDM1OGJmNDFjNmU5N2QwNWE4YjNmMjYxOWU0YzcwODFkYTViM2YyLnppcCIsCiAgICAic2hhMjU2IjogIjdkMmM5YTQxZThiMzUwZjZjMTc0YTlkZTIwMzU4YmY0MWM2ZTk3ZDA1YThiM2YyNjE5ZTRjNzA4MWRhNWIzZjIiLAogICAgInNpemVCeXRlcyI6IDY1NTc1MjIxNgogIH0sCiAgImluc3RhbGxlZFNpemVCeXRlcyI6IDE4OTIzNDAxMTIsCiAgInJ1bnRpbWUiOiB7CiAgICAiaWQiOiAicHl0aG9uIiwKICAgICJ2ZXJzaW9uIjogIjMuMTEuMTUiLAogICAgImVudHJ5UG9pbnQiOiAidmVudi9iaW4vcHl0aG9uIgogIH0sCiAgImNhY2hlU3ViZGlyIjogImNhY2hlL2V4YW1wbGUtbW9kZWwiLAogICJzZWxmVGVzdCI6IHsKICAgICJwcm9iZSI6IHsKICAgICAgImltcG9ydHMiOiBbCiAgICAgICAgInRvcmNoIiwKICAgICAgICAibnVtcHkiCiAgICAgIF0KICAgIH0sCiAgICAidGltZW91dFNlY29uZHMiOiAxODAKICB9LAogICJwcm92ZW5hbmNlIjogewogICAgInNjcm9sbElkIjogImV4YW1wbGUtbW9kZWwtbWFjb3MtYXJtNjQtbWV0YWwiLAogICAgInNjcm9sbFZlcnNpb24iOiAiMS4wLjAiLAogICAgImJ1aWxkZXJSZXZpc2lvbiI6ICI0YzFkOWI3ZTJhMGY1ODM2ZDQxOWU3YzA1YjNhOGY2MWQyNzA0ZTkzIiwKICAgICJzb3VyY2VUcmVlRGlydHkiOiBmYWxzZSwKICAgICJzb3VyY2VSZXZpc2lvbiI6ICI5ZjBkM2ExYzRiNWU2ZjcwODE5MjBhM2I0YzVkNmU3ZjgwOTEwYTJiIiwKICAgICJydW50aW1lVmVyc2lvbiI6ICIzLjExLjE1IiwKICAgICJwaXhpVmVyc2lvbiI6ICIwLjczLjAiLAogICAgImRlcGVuZGVuY3lMb2NrU2hhMjU2IjogIjNiMWY4YzQ3YTJkOWUwNWI2Yzc0MThhZjIzZDVlNjkwMTdiNGM4YWQ5MWUyZjM1MDc2OGJkNGNhMTllMGY1YjciLAogICAgImJ1aWx0QXQiOiAiMjAyNi0wNy0yNVQxMjowMDowMCswMDowMCIKICB9Cn0K", + "payloadSha256": "f264818314ac170618c9472dc6cc7a72a09eb6390a237fe00dcbe44507d1aef1", "signatures": [ { "algorithm": "ed25519", - "keyId": "scrollcase-example-v2", - "signatureBase64": "c4ftYhvfGwJicd3kK9DzyTj+HNvJeiCfwk049BKk8hxSgAMxxU8BE51Q51ZkSK0mPolXRJ9pz52HO3KoKD1mAA==" + "keyId": "scrollcase-example-v3", + "signatureBase64": "oFNLMVGTsE162A9uaoo8IRBC1Vlqdy1vCeIUCgHYoupvgJVfTHiV8SDcdF5JOFxslrqN+/fgrbK5/wqaTS+jDg==" } ] } diff --git a/rust/fixtures/runtime-contract.json b/rust/fixtures/runtime-contract.json index f567304..f08f8df 100644 --- a/rust/fixtures/runtime-contract.json +++ b/rust/fixtures/runtime-contract.json @@ -1,9 +1,12 @@ { - "description": "Golden cases for the Scrollcase runtime model: where a runtime lives inside a box, which payload paths it needs the executable bit on, which paths a declared execution could resolve to, and the shell-free command line that runs it. Every implementation of the format proves its mirror against this file. Paths stay payload-relative on purpose: a box root is a real filesystem path and each language joins one in its own terms, so a joined expectation here would only pin the host that read it.", + "description": "Golden cases for the box-format runtime model. Every implementation of the format - the Node reference in src/contract/runtimes.mjs, the Rust mirror, the Python mirror - must produce these answers exactly. A target says which machine a box runs on; a runtime says what runs inside it, and these are the rules that answer the second question.", "runtimes": [ { "id": "python", - "executionKinds": ["python-script", "python-module"], + "executionKinds": [ + "python-script", + "python-module" + ], "executionEnvironmentVariables": [ "PYTHONPATH", "PYTHONHOME", @@ -22,8 +25,12 @@ "launcherKind": "posix-polyglot" }, "executablePayloadPaths": { - "files": ["venv/bin/python"], - "directories": ["venv/bin"] + "files": [ + "venv/bin/python" + ], + "directories": [ + "venv/bin" + ] } }, { @@ -37,8 +44,12 @@ "launcherKind": "posix-polyglot" }, "executablePayloadPaths": { - "files": ["venv/bin/python"], - "directories": ["venv/bin"] + "files": [ + "venv/bin/python" + ], + "directories": [ + "venv/bin" + ] } }, { @@ -52,8 +63,12 @@ "launcherKind": "uv-windows-pe" }, "executablePayloadPaths": { - "files": ["venv/python.exe"], - "directories": ["venv/Scripts"] + "files": [ + "venv/python.exe" + ], + "directories": [ + "venv/Scripts" + ] } } ] @@ -130,8 +145,14 @@ "runtime": "python", "platform": "linux", "runtimeVersion": "3.11.15", - "execution": { "kind": "python-script", "script": "app/main.py", "defaultArgs": [] }, - "candidates": ["app/main.py"] + "execution": { + "kind": "python-script", + "script": "app/main.py", + "defaultArgs": [] + }, + "candidates": [ + "app/main.py" + ] }, { "name": "a POSIX module is looked for at the root, in the standard library, and in site-packages", @@ -141,7 +162,9 @@ "execution": { "kind": "python-module", "module": "example_model.main", - "defaultArgs": ["--serve"] + "defaultArgs": [ + "--serve" + ] }, "candidates": [ "example_model/main.py", @@ -157,7 +180,11 @@ "runtime": "python", "platform": "macos", "runtimeVersion": "3.12.4", - "execution": { "kind": "python-module", "module": "pkg", "defaultArgs": [] }, + "execution": { + "kind": "python-module", + "module": "pkg", + "defaultArgs": [] + }, "candidates": [ "pkg.py", "pkg/__main__.py", @@ -172,7 +199,11 @@ "runtime": "python", "platform": "windows", "runtimeVersion": "3.11.15", - "execution": { "kind": "python-module", "module": "pkg", "defaultArgs": [] }, + "execution": { + "kind": "python-module", + "module": "pkg", + "defaultArgs": [] + }, "candidates": [ "pkg.py", "pkg/__main__.py", @@ -183,7 +214,13 @@ ] } ], - "invalidRuntimeVersions": ["", "3", "3.x", "x.1", "3."], + "invalidRuntimeVersions": [ + "", + "3", + "3.x", + "x.1", + "3." + ], "argv": [ { "name": "a script runs as a payload path, with its declared arguments after it", @@ -192,14 +229,33 @@ "execution": { "kind": "python-script", "script": "app/main.py", - "defaultArgs": ["--serve", "--port", "8080"] + "defaultArgs": [ + "--serve", + "--port", + "8080" + ] + }, + "command": { + "kind": "payload-path", + "value": "venv/bin/python" }, - "command": { "kind": "payload-path", "value": "venv/bin/python" }, "args": [ - { "kind": "payload-path", "value": "app/main.py" }, - { "kind": "literal", "value": "--serve" }, - { "kind": "literal", "value": "--port" }, - { "kind": "literal", "value": "8080" } + { + "kind": "payload-path", + "value": "app/main.py" + }, + { + "kind": "literal", + "value": "--serve" + }, + { + "kind": "literal", + "value": "--port" + }, + { + "kind": "literal", + "value": "8080" + } ] }, { @@ -211,10 +267,19 @@ "module": "example_model.main", "defaultArgs": [] }, - "command": { "kind": "payload-path", "value": "venv/bin/python" }, + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, "args": [ - { "kind": "literal", "value": "-m" }, - { "kind": "literal", "value": "example_model.main" } + { + "kind": "literal", + "value": "-m" + }, + { + "kind": "literal", + "value": "example_model.main" + } ] }, { @@ -226,8 +291,16 @@ "script": "app/main.py", "defaultArgs": [] }, - "command": { "kind": "payload-path", "value": "venv/python.exe" }, - "args": [{ "kind": "payload-path", "value": "app/main.py" }] + "command": { + "kind": "payload-path", + "value": "venv/python.exe" + }, + "args": [ + { + "kind": "payload-path", + "value": "app/main.py" + } + ] } ], "selfTest": [ @@ -235,35 +308,292 @@ "name": "macOS asserts Darwin before importing anything", "runtime": "python", "platform": "macos", - "probe": { "imports": ["json"] }, - "args": ["-c", "import sys; assert sys.platform == 'darwin'\nimport json"] + "probe": { + "imports": [ + "json" + ] + }, + "execution": null, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "literal", + "value": "-c" + }, + { + "kind": "literal", + "value": "import sys; assert sys.platform == 'darwin'\nimport json" + } + ], + "expectExitCode": 0 + } + ] }, { "name": "Linux accepts any linux variant", "runtime": "python", "platform": "linux", - "probe": { "imports": ["json", "numpy"] }, - "args": [ - "-c", - "import sys; assert sys.platform.startswith('linux')\nimport json, numpy" + "probe": { + "imports": [ + "json", + "numpy" + ] + }, + "execution": null, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "literal", + "value": "-c" + }, + { + "kind": "literal", + "value": "import sys; assert sys.platform.startswith('linux')\nimport json, numpy" + } + ], + "expectExitCode": 0 + } ] }, { "name": "Windows asserts win32", "runtime": "python", "platform": "windows", - "probe": { "imports": ["json"] }, - "args": ["-c", "import sys; assert sys.platform == 'win32'\nimport json"] + "probe": { + "imports": [ + "json" + ] + }, + "execution": null, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/python.exe" + }, + "args": [ + { + "kind": "literal", + "value": "-c" + }, + { + "kind": "literal", + "value": "import sys; assert sys.platform == 'win32'\nimport json" + } + ], + "expectExitCode": 0 + } + ] }, { "name": "the builder appends the extra source a scroll declared", "runtime": "python", "platform": "linux", - "probe": { "imports": ["json"], "code": "print(\"self-test ok\")\n" }, - "args": [ - "-c", - "import sys; assert sys.platform.startswith('linux')\nimport json\nprint(\"self-test ok\")\n" + "probe": { + "imports": [ + "json" + ], + "code": "print(\"self-test ok\")\n" + }, + "execution": null, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "literal", + "value": "-c" + }, + { + "kind": "literal", + "value": "import sys; assert sys.platform.startswith('linux')\nimport json\nprint(\"self-test ok\")\n" + } + ], + "expectExitCode": 0 + } + ] + }, + { + "name": "a command probe invokes the declared execution, after its default arguments", + "runtime": "python", + "platform": "linux", + "probe": { + "commands": [ + { + "args": [ + "--version" + ], + "expectExitCode": 0 + } + ] + }, + "execution": { + "kind": "python-script", + "script": "app/main.py", + "defaultArgs": [ + "--serve" + ] + }, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "payload-path", + "value": "app/main.py" + }, + { + "kind": "literal", + "value": "--serve" + }, + { + "kind": "literal", + "value": "--version" + } + ], + "expectExitCode": 0 + } + ] + }, + { + "name": "a command may require a non-zero exit status", + "runtime": "python", + "platform": "macos", + "probe": { + "commands": [ + { + "args": [ + "--help" + ], + "expectExitCode": 2 + } + ] + }, + "execution": { + "kind": "python-module", + "module": "example.cli", + "defaultArgs": [] + }, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "literal", + "value": "-m" + }, + { + "kind": "literal", + "value": "example.cli" + }, + { + "kind": "literal", + "value": "--help" + } + ], + "expectExitCode": 2 + } + ] + }, + { + "name": "imports and commands both run, imports first", + "runtime": "python", + "platform": "linux", + "probe": { + "imports": [ + "json" + ], + "commands": [ + { + "args": [], + "expectExitCode": 0 + }, + { + "args": [ + "--check" + ], + "expectExitCode": 1 + } + ] + }, + "execution": { + "kind": "python-script", + "script": "run.py", + "defaultArgs": [] + }, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "literal", + "value": "-c" + }, + { + "kind": "literal", + "value": "import sys; assert sys.platform.startswith('linux')\nimport json" + } + ], + "expectExitCode": 0 + }, + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "payload-path", + "value": "run.py" + } + ], + "expectExitCode": 0 + }, + { + "command": { + "kind": "payload-path", + "value": "venv/bin/python" + }, + "args": [ + { + "kind": "payload-path", + "value": "run.py" + }, + { + "kind": "literal", + "value": "--check" + } + ], + "expectExitCode": 1 + } ] } + ], + "runtimeIds": [ + "python", + "node", + "native" ] } diff --git a/rust/src/contract/documents.rs b/rust/src/contract/documents.rs index eaaae72..3d8dd0e 100644 --- a/rust/src/contract/documents.rs +++ b/rust/src/contract/documents.rs @@ -19,7 +19,7 @@ use sha2::{Digest, Sha256}; use crate::error::{fail, Result}; /// Format version carried by every document this contract describes. -pub const BOX_SCHEMA_VERSION: u32 = 2; +pub const BOX_SCHEMA_VERSION: u32 = 3; /// The only payload encoding the format defines. pub const PAYLOAD_ENCODING: &str = "base64-json-utf8"; @@ -173,11 +173,15 @@ impl SignedDocument { /// /// When the bytes are not a structurally valid envelope. pub fn parse(bytes: &[u8]) -> Result { - // Read the version before the typed parse, so a v1 document is refused by name instead of - // producing a shape complaint that hides why it was rejected. + // Read the version before the typed parse, so a superseded document is refused by name + // instead of producing a shape complaint that hides why it was rejected. Both older + // versions are named: they are different artefacts with different rebuilds ahead of them. if let Ok(value) = serde_json::from_slice::(bytes) { - if value.get("schemaVersion").and_then(serde_json::Value::as_u64) == Some(1) { - fail!("Unsupported schemaVersion 1; rebuild this box with Scrollcase v2."); + if let Some(version @ (1 | 2)) = value + .get("schemaVersion") + .and_then(serde_json::Value::as_u64) + { + fail!("Unsupported schemaVersion {version}; rebuild this box with Scrollcase v3."); } } serde_json::from_slice(bytes) @@ -190,8 +194,11 @@ impl SignedDocument { /// /// When the envelope is unsupported, or the payload bytes do not hash to the value it names. pub fn decode_payload(&self) -> Result> { - if self.schema_version == 1 { - fail!("Unsupported schemaVersion 1; rebuild this box with Scrollcase v2."); + if self.schema_version == 1 || self.schema_version == 2 { + fail!( + "Unsupported schemaVersion {}; rebuild this box with Scrollcase v3.", + self.schema_version + ); } if self.schema_version != BOX_SCHEMA_VERSION || self.payload_encoding != PAYLOAD_ENCODING { fail!("Unsupported signed document."); @@ -232,7 +239,7 @@ mod tests { fn envelope(payload: &[u8], sha256: &str) -> SignedDocument { serde_json::from_value(serde_json::json!({ - "schemaVersion": 2, + "schemaVersion": 3, "payloadEncoding": "base64-json-utf8", "payloadBase64": BASE64.encode(payload), "payloadSha256": sha256, diff --git a/rust/src/contract/runtimes.rs b/rust/src/contract/runtimes.rs index 87bd84e..2f59d96 100644 --- a/rust/src/contract/runtimes.rs +++ b/rust/src/contract/runtimes.rs @@ -73,22 +73,38 @@ pub enum RuntimeExecution<'a> { /// Arguments always passed before a caller's own. default_args: &'a [String], }, + /// A compiled executable the box carries, run with no interpreter in front of it. + /// + /// Named by the format so implementing the runtime is code rather than another wire break. No + /// adapter answers for it yet, so a box declaring it is refused by name. + Binary { + /// Payload-relative path to the executable. + binary: &'a str, + /// Arguments always passed before a caller's own. + default_args: &'a [String], + }, } impl RuntimeExecution<'_> { - /// The wire `kind` this declaration carries. + /// The wire `kind` this declaration carries, for the runtime that owns it. + /// + /// The kind is `-`, so the shape alone does not name it: a script belongs to + /// `python` or to `node` depending on the box, and the caller says which. #[must_use] - pub fn kind(&self) -> &'static str { - match self { - RuntimeExecution::Script { .. } => "python-script", - RuntimeExecution::Module { .. } => "python-module", - } + pub fn kind(&self, runtime_id: &str) -> String { + let shape = match self { + RuntimeExecution::Script { .. } => "script", + RuntimeExecution::Module { .. } => "module", + RuntimeExecution::Binary { .. } => "binary", + }; + format!("{runtime_id}-{shape}") } fn default_args(&self) -> &[String] { match self { RuntimeExecution::Script { default_args, .. } - | RuntimeExecution::Module { default_args, .. } => default_args, + | RuntimeExecution::Module { default_args, .. } + | RuntimeExecution::Binary { default_args, .. } => default_args, } } } @@ -120,15 +136,41 @@ pub struct RuntimeInvocation { pub args: Vec, } -/// What a self-test asks the runtime to prove, plus the builder-only extension a scroll may add. +/// One invocation a self-test probe asks for, borrowed from whatever document carried it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SelfTestCommand<'a> { + /// Arguments appended to the box's declared execution. + pub args: &'a [String], + /// The status the invocation must exit with. + pub expect_exit_code: u8, +} + +/// What a self-test asks the box to prove, plus the builder-only extension a scroll may add. +/// +/// `imports` asks the runtime's loader a question and only means something to a runtime that has +/// one. `commands` asks the box's declared execution a question, which every runtime can answer and +/// a native one can answer *only* that way. `code` never travels on the wire. #[derive(Debug, Clone, Copy)] pub struct SelfTestProbe<'a> { /// Modules the box must be able to import. pub imports: &'a [String], - /// Extra source the builder appends; never part of the signed subset. + /// Invocations of the box's declared execution. + pub commands: &'a [SelfTestCommand<'a>], + /// Extra source the builder appends; never part of the signed probe. pub code: Option<&'a str>, } +/// One command a self-test runs, and the status it must exit with. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SelfTestInvocation { + /// The command to run. + pub command: RuntimeArgument, + /// Everything before the caller's own arguments. + pub args: Vec, + /// The status it must produce. + pub expect_exit_code: u8, +} + /// What a runtime implies for a box, independent of the machine it runs on. /// /// The per-runtime rules are function pointers rather than a `match` on [`Self::id`]: a second @@ -225,6 +267,9 @@ fn resolve_python_execution_files( }) } RuntimeExecution::Module { module, .. } => module, + // Unreachable: `resolve_execution_files` refuses a kind that is not this runtime's before + // it gets here, and `python` defines only the two above. + RuntimeExecution::Binary { .. } => fail!("Unsupported execution kind: {}.", execution.kind("python")), }; let module_path = module.replace('.', "/"); let relative = [ @@ -306,9 +351,7 @@ impl BoxRuntimeAdapter { platform: &str, runtime_version: &str, ) -> Result { - if !self.execution_kinds.contains(&execution.kind()) { - fail!("Unsupported execution kind: {}.", execution.kind()); - } + self.assert_own_kind(execution)?; let layout = self.layout(platform)?; (self.resolve)(execution, platform, layout, runtime_version) } @@ -323,9 +366,7 @@ impl BoxRuntimeAdapter { execution: &RuntimeExecution<'_>, platform: &str, ) -> Result { - if !self.execution_kinds.contains(&execution.kind()) { - fail!("Unsupported execution kind: {}.", execution.kind()); - } + self.assert_own_kind(execution)?; let layout = self.layout(platform)?; let mut args = match execution { RuntimeExecution::Script { script, .. } => { @@ -335,6 +376,9 @@ impl BoxRuntimeAdapter { RuntimeArgument::Literal("-m".to_string()), RuntimeArgument::Literal((*module).to_string()), ], + RuntimeExecution::Binary { binary, .. } => { + vec![RuntimeArgument::PayloadPath((*binary).to_string())] + } }; args.extend( execution @@ -348,28 +392,77 @@ impl BoxRuntimeAdapter { }) } - /// The arguments that follow this runtime's entry point when it runs a self-test probe. + /// Refuses an execution kind belonging to another runtime. + fn assert_own_kind(&self, execution: &RuntimeExecution<'_>) -> Result<()> { + let kind = execution.kind(self.id); + if !self.execution_kinds.contains(&kind.as_str()) { + fail!("Unsupported execution kind: {kind}."); + } + Ok(()) + } + + /// Every command a self-test probe implies, in declaration order. /// /// # Errors /// - /// When the runtime has no platform assertion for that platform. - pub fn self_test_argv(&self, probe: &SelfTestProbe<'_>, platform: &str) -> Result> { - let Some((_, assertion)) = self - .platform_assertions - .iter() - .find(|(name, _)| *name == platform) - else { - fail!( - "No {} self-test assertion exists for platform {platform}", - self.id + /// When the runtime has no platform assertion for that platform, or a command probe arrives + /// with no declared execution to invoke. + pub fn self_test_invocations( + &self, + probe: &SelfTestProbe<'_>, + execution: Option<&RuntimeExecution<'_>>, + platform: &str, + ) -> Result> { + let mut invocations = Vec::new(); + if !probe.imports.is_empty() { + let Some((_, assertion)) = self + .platform_assertions + .iter() + .find(|(name, _)| *name == platform) + else { + fail!( + "No {} self-test assertion exists for platform {platform}", + self.id + ); + }; + let imports = format!("import {}", probe.imports.join(", ")); + let code = match probe.code { + Some(extra) => format!("{assertion}\n{imports}\n{extra}"), + None => format!("{assertion}\n{imports}"), + }; + invocations.push(SelfTestInvocation { + command: RuntimeArgument::PayloadPath( + self.layout(platform)?.entry_point.to_string(), + ), + args: vec![ + RuntimeArgument::Literal("-c".to_string()), + RuntimeArgument::Literal(code), + ], + expect_exit_code: 0, + }); + } + for command in probe.commands { + // A command probe appends arguments to the box's own declared execution. With none + // declared there is nothing to append them to, which is a contradiction in the + // declaration rather than a property of the box. + let Some(execution) = execution else { + fail!("A self-test command needs a declared execution to invoke"); + }; + let invocation = self.build_argv(execution, platform)?; + let mut args = invocation.args; + args.extend( + command + .args + .iter() + .map(|value| RuntimeArgument::Literal(value.clone())), ); - }; - let imports = format!("import {}", probe.imports.join(", ")); - let code = match probe.code { - Some(extra) => format!("{assertion}\n{imports}\n{extra}"), - None => format!("{assertion}\n{imports}"), - }; - Ok(vec!["-c".to_string(), code]) + invocations.push(SelfTestInvocation { + command: invocation.command, + args, + expect_exit_code: command.expect_exit_code, + }); + } + Ok(invocations) } } @@ -394,12 +487,41 @@ pub fn runtime_adapters() -> &'static [BoxRuntimeAdapter] { RUNTIME_ADAPTERS } -/// The runtime every box built by this schema version implicitly declares. +/// Every runtime id the box format admits, in the order the schema lists them. /// -/// The wire format has no runtime field: a box records a Python entry point and Python execution -/// kinds and nothing that says "Python". So a reader that must name a runtime names this one, from -/// one place. -pub const IMPLICIT_RUNTIME_ID: &str = "python"; +/// The wire enum and the implemented set are deliberately two different things: schema version 3 +/// fixes the vocabulary once, so a later release can implement `node` without another wire break. +/// A box naming a runtime this crate has no adapter for is refused by name, not misread. +pub const RUNTIME_IDS: &[&str] = &["python", "node", "native"]; + +/// Whether this build carries an adapter for a runtime id — the question every caller asks before +/// [`runtime_adapter`], which fails rather than returning nothing. +#[must_use] +pub fn is_implemented_runtime(runtime_id: &str) -> bool { + RUNTIME_ADAPTERS + .iter() + .any(|adapter| adapter.id == runtime_id) +} + +/// The message for a box declaring a runtime this build has no adapter for. +/// +/// The wire vocabulary is fixed and the implemented set is not, so this case is expected rather +/// than exceptional, and the wording says which of the two the box fell foul of. +#[must_use] +pub fn unimplemented_runtime_message(runtime_id: &str) -> String { + let implemented: Vec<&str> = RUNTIME_ADAPTERS.iter().map(|adapter| adapter.id).collect(); + if RUNTIME_IDS.contains(&runtime_id) { + format!( + "Runtime {runtime_id} is not implemented by this version of Scrollcase; it implements {}.", + implemented.join(", ") + ) + } else { + format!( + "Unknown runtime: {runtime_id}. The box format defines {}.", + RUNTIME_IDS.join(", ") + ) + } +} /// The complete list of inherited variables that can change what a box executes. /// @@ -433,12 +555,14 @@ pub fn assert_runtime_entry_point( adapter: &super::targets::BoxTargetAdapter, entry_point: &str, ) -> Result<()> { - let expected = runtime_adapter(runtime_id)?.layout(adapter.platform)?.entry_point; + let runtime = runtime_adapter(runtime_id)?; + let expected = runtime.layout(adapter.platform)?.entry_point; if entry_point != expected { - // The wording still names Python because the wire format still does: a release declares - // `pythonEntryPoint`, and an error that called it something else would name a field nobody - // can find. - fail!("{} boxes must use Python entry point {expected}", adapter.id); + fail!( + "{} boxes with the {} runtime must use entry point {expected}", + adapter.id, + runtime.id + ); } Ok(()) } @@ -446,27 +570,36 @@ pub fn assert_runtime_entry_point( #[cfg(test)] mod tests { use super::{ - python_major_minor, runtime_adapter, RuntimeArgument, RuntimeExecution, SelfTestProbe, - IMPLICIT_RUNTIME_ID, + is_implemented_runtime, python_major_minor, runtime_adapter, unimplemented_runtime_message, + RuntimeArgument, RuntimeExecution, SelfTestCommand, SelfTestProbe, RUNTIME_IDS, }; + const PYTHON: &str = "python"; + #[test] - fn a_runtime_the_format_does_not_define_is_refused() { - assert!(runtime_adapter("node").is_err()); + fn a_runtime_this_build_has_no_adapter_for_is_refused_by_name() { + // The wire vocabulary is wider than the implemented set on purpose, and the two refusals + // say which of the two the box fell foul of. + assert!(RUNTIME_IDS.contains(&"native")); + assert!(runtime_adapter("native").is_err()); + assert!(!is_implemented_runtime("native")); + assert!(unimplemented_runtime_message("native").contains("not implemented by this version")); + assert!(unimplemented_runtime_message("ruby").contains("Unknown runtime")); assert!(runtime_adapter("").is_err()); - assert!(runtime_adapter(IMPLICIT_RUNTIME_ID).is_ok()); + assert!(runtime_adapter(PYTHON).is_ok()); + assert!(is_implemented_runtime(PYTHON)); } #[test] fn a_platform_with_no_layout_is_refused_rather_than_guessed() { - let python = runtime_adapter(IMPLICIT_RUNTIME_ID).unwrap(); + let python = runtime_adapter(PYTHON).unwrap(); assert!(python.layout("plan9").is_err()); assert!(python.layout("linux").is_ok()); } #[test] fn a_module_never_becomes_a_payload_path() { - let python = runtime_adapter(IMPLICIT_RUNTIME_ID).unwrap(); + let python = runtime_adapter(PYTHON).unwrap(); let default_args = vec![]; let invocation = python .build_argv( @@ -498,15 +631,46 @@ mod tests { #[test] fn a_self_test_opens_with_the_platform_it_was_built_for() { - let python = runtime_adapter(IMPLICIT_RUNTIME_ID).unwrap(); + let python = runtime_adapter(PYTHON).unwrap(); let imports = vec!["json".to_string()]; - let argv = python - .self_test_argv(&SelfTestProbe { imports: &imports, code: None }, "macos") + let probe = SelfTestProbe { imports: &imports, commands: &[], code: None }; + let invocations = python.self_test_invocations(&probe, None, "macos").unwrap(); + assert_eq!(invocations.len(), 1); + assert_eq!( + invocations[0].args[0], + RuntimeArgument::Literal("-c".to_string()) + ); + let RuntimeArgument::Literal(code) = &invocations[0].args[1] else { + panic!("self-test source is not a literal"); + }; + assert!(code.starts_with("import sys; assert sys.platform == 'darwin'")); + assert!(python.self_test_invocations(&probe, None, "plan9").is_err()); + } + + #[test] + fn a_command_probe_needs_an_execution_to_invoke() { + let python = runtime_adapter(PYTHON).unwrap(); + let args = vec!["--version".to_string()]; + let commands = [SelfTestCommand { args: &args, expect_exit_code: 2 }]; + let probe = SelfTestProbe { imports: &[], commands: &commands, code: None }; + let error = python + .self_test_invocations(&probe, None, "linux") + .unwrap_err(); + assert!(error.message().contains("needs a declared execution"), "{error}"); + + let default_args = vec![]; + let execution = RuntimeExecution::Script { script: "app/main.py", default_args: &default_args }; + let invocations = python + .self_test_invocations(&probe, Some(&execution), "linux") .unwrap(); - assert_eq!(argv[0], "-c"); - assert!(argv[1].starts_with("import sys; assert sys.platform == 'darwin'")); - assert!(python - .self_test_argv(&SelfTestProbe { imports: &imports, code: None }, "plan9") - .is_err()); + assert_eq!(invocations.len(), 1); + assert_eq!(invocations[0].expect_exit_code, 2); + assert_eq!( + invocations[0].args, + vec![ + RuntimeArgument::PayloadPath("app/main.py".to_string()), + RuntimeArgument::Literal("--version".to_string()), + ] + ); } } diff --git a/rust/src/contract/schema/box-manifest.schema.json b/rust/src/contract/schema/box-manifest.schema.json index 8b0ddb8..6060ec5 100644 --- a/rust/src/contract/schema/box-manifest.schema.json +++ b/rust/src/contract/schema/box-manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/box-manifest.schema.json", + "$id": "https://scrollcase.dev/schema/v3/box-manifest.schema.json", "title": "Box manifest (box.json)", "description": "The manifest packed inside the archive at box.json. It restates the box's identity, layout and provenance so an extracted box is self-describing: a consumer that has the directory but not the release document can still tell what it is holding and how it was built.", "type": "object", @@ -8,43 +8,35 @@ "required": [ "schemaVersion", "boxId", - "modelId", - "runtimeId", "version", "target", - "pythonEntryPoint", - "modelCacheSubdir", + "runtime", + "cacheSubdir", "selfTest", "provenance" ], "properties": { "schemaVersion": { - "const": 2 + "const": 3 }, "boxId": { "type": "string", "minLength": 1 }, - "modelId": { - "type": "string", - "minLength": 1 - }, - "runtimeId": { - "type": "string", - "minLength": 1 + "labels": { + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/labels" }, "version": { "type": "string", "minLength": 1 }, "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + "$ref": "https://scrollcase.dev/schema/v3/target.schema.json" }, - "pythonEntryPoint": { - "type": "string", - "minLength": 1 + "runtime": { + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/runtime" }, - "modelCacheSubdir": { + "cacheSubdir": { "type": "string", "minLength": 1 }, @@ -61,76 +53,16 @@ } }, "selfTest": { - "type": "object", - "additionalProperties": false, - "required": [ - "pythonImports", - "timeoutSeconds" - ], - "properties": { - "pythonImports": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "minLength": 1 - } - }, - "timeoutSeconds": { - "type": "integer", - "exclusiveMinimum": 0 - } - } + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/selfTest" }, "execution": { - "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" + "$ref": "https://scrollcase.dev/schema/v3/execution.schema.json" }, "provenance": { - "$ref": "https://scrollcase.dev/schema/v2/release-manifest.schema.json#/$defs/provenance" - }, - "weights": { - "const": "on-demand", - "description": "Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it." + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/provenance" }, "assets": { - "type": "array", - "minItems": 1, - "description": "Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe.", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "url", - "relativePath", - "sizeBytes", - "sha256" - ], - "properties": { - "url": { - "type": "string", - "minLength": 1 - }, - "relativePath": { - "type": "string", - "minLength": 1 - }, - "sizeBytes": { - "type": "integer", - "exclusiveMinimum": 0 - }, - "sha256": { - "$ref": "https://scrollcase.dev/schema/v2/release-manifest.schema.json#/$defs/sha256" - } - } - } + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/deferredAssets" } - }, - "dependentRequired": { - "assets": [ - "weights" - ], - "weights": [ - "assets" - ] } } diff --git a/rust/src/contract/schema/execution.schema.json b/rust/src/contract/schema/execution.schema.json index a37ebfb..62b4da2 100644 --- a/rust/src/contract/schema/execution.schema.json +++ b/rust/src/contract/schema/execution.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/execution.schema.json", + "$id": "https://scrollcase.dev/schema/v3/execution.schema.json", "title": "Box execution", - "description": "The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only.", + "description": "The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only.\n\nEach kind is named -, and the runtime half must be the one the box declares: a python-script in a box whose runtime is native describes something that cannot be run, and is refused rather than guessed at.", "oneOf": [ { "title": "Python script", @@ -55,6 +55,54 @@ "$ref": "#/$defs/defaultArgs" } } + }, + { + "title": "Node script", + "description": "Run one regular payload file with the box's own Node runtime.", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "script", + "defaultArgs" + ], + "properties": { + "kind": { + "const": "node-script", + "description": "Selects direct script execution." + }, + "script": { + "$ref": "#/$defs/payloadPath", + "description": "Safe path to a regular JavaScript file inside the box." + }, + "defaultArgs": { + "$ref": "#/$defs/defaultArgs" + } + } + }, + { + "title": "Native binary", + "description": "Run a compiled executable that the box carries directly, with no interpreter in front of it. The only shape a runtime with no module system has.", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "binary", + "defaultArgs" + ], + "properties": { + "kind": { + "const": "native-binary", + "description": "Selects direct execution of a payload file." + }, + "binary": { + "$ref": "#/$defs/payloadPath", + "description": "Safe path to the executable inside the box. It carries the executable bit because the scroll declared it, not because the build machine happened to have it set." + }, + "defaultArgs": { + "$ref": "#/$defs/defaultArgs" + } + } } ], "examples": [ diff --git a/rust/src/contract/schema/release-manifest.schema.json b/rust/src/contract/schema/release-manifest.schema.json index 07d5458..90d1b23 100644 --- a/rust/src/contract/schema/release-manifest.schema.json +++ b/rust/src/contract/schema/release-manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/release-manifest.schema.json", + "$id": "https://scrollcase.dev/schema/v3/release-manifest.schema.json", "title": "Box release manifest", "description": "The immutable description of one built box: what it is, which target it runs on, where its archive lives, and how it was produced. Published as the payload of a signed document and never edited after signing; a correction ships as a new version.", "type": "object", @@ -9,20 +9,18 @@ "schemaVersion", "kind", "boxId", - "modelId", - "runtimeId", "version", "target", "compatibility", "archive", - "pythonEntryPoint", - "modelCacheSubdir", + "runtime", + "cacheSubdir", "selfTest", "provenance" ], "properties": { "schemaVersion": { - "const": 2 + "const": 3 }, "kind": { "$ref": "#/$defs/kind", @@ -31,18 +29,15 @@ "boxId": { "$ref": "#/$defs/identifier" }, - "modelId": { - "$ref": "#/$defs/identifier" - }, - "runtimeId": { - "$ref": "#/$defs/identifier" + "labels": { + "$ref": "#/$defs/labels" }, "version": { "type": "string", "minLength": 1 }, "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + "$ref": "https://scrollcase.dev/schema/v3/target.schema.json" }, "compatibility": { "type": "object", @@ -132,15 +127,13 @@ } } }, - "pythonEntryPoint": { - "type": "string", - "minLength": 1, - "description": "Interpreter path relative to the extracted box root, for example venv/bin/python. Fixed per target by the adapter." + "runtime": { + "$ref": "#/$defs/runtime" }, - "modelCacheSubdir": { + "cacheSubdir": { "type": "string", "minLength": 1, - "description": "Directory relative to the extracted box root holding model assets." + "description": "Directory relative to the extracted box root holding the box's own large files." }, "environment": { "type": "object", @@ -155,42 +148,127 @@ } }, "selfTest": { + "$ref": "#/$defs/selfTest" + }, + "execution": { + "$ref": "https://scrollcase.dev/schema/v3/execution.schema.json" + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "assets": { + "$ref": "#/$defs/deferredAssets" + } + }, + "$defs": { + "identifier": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" + }, + "runtime": { "type": "object", "additionalProperties": false, "required": [ - "pythonImports", + "id" + ], + "description": "What runs inside the box: the runtime, its version, and where its own executable sits in the payload. A consumer needs all three to run the box, and none of them are derivable from the target.", + "properties": { + "id": { + "enum": [ + "python", + "node", + "native" + ], + "description": "The runtime the box carries. A consumer that does not recognise the id must refuse the box: the id decides the payload layout and the argv rule, so guessing would mean executing something on an assumption." + }, + "version": { + "type": "string", + "minLength": 1, + "description": "The runtime's own version. Absent for a runtime that has no interpreter to version." + }, + "entryPoint": { + "type": "string", + "minLength": 1, + "description": "The runtime's own executable relative to the extracted box root, for example venv/bin/python. Fixed per (runtime, target) by the runtime's layout. Absent for a runtime that has no separate executable to name." + } + } + }, + "labels": { + "type": "object", + "description": "Free-form annotations the publishing project declared, signed and carried through untouched. Scrollcase attaches no meaning to any key; a consumer that reads one is reading its own project's convention, not the box format.", + "propertyNames": { + "$ref": "#/$defs/identifier" + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } + }, + "selfTest": { + "type": "object", + "additionalProperties": false, + "required": [ + "probe", "timeoutSeconds" ], - "description": "The import check a consumer can repeat after extraction with the box's own interpreter. The builder also ran the scroll's Python-code and file assertions, which are builder-only checks.", + "description": "The check a consumer can repeat against an extracted box. The builder also ran the scroll's file assertions and any extra source it declared, which are builder-only: signing them would claim a consumer had reproduced a check it cannot see.", "properties": { - "pythonImports": { + "probe": { + "$ref": "#/$defs/selfTestProbe" + }, + "timeoutSeconds": { + "type": "integer", + "exclusiveMinimum": 0 + } + } + }, + "selfTestProbe": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "description": "What the box proves about itself, in whichever shapes its runtime supports. The runtime turns this into command lines; nothing here is a command line, and nothing here is source in any language.", + "properties": { + "imports": { "type": "array", "minItems": 1, + "description": "Modules the runtime must be able to load.", "items": { "type": "string", "minLength": 1 } }, - "timeoutSeconds": { - "type": "integer", - "exclusiveMinimum": 0 + "commands": { + "type": "array", + "minItems": 1, + "description": "Invocations of the box's declared execution and the exit status each must produce.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "args", + "expectExitCode" + ], + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "expectExitCode": { + "type": "integer", + "minimum": 0, + "maximum": 255 + } + } + } } } }, - "execution": { - "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" - }, - "provenance": { - "$ref": "#/$defs/provenance" - }, - "weights": { - "const": "on-demand", - "description": "Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it." - }, - "assets": { + "deferredAssets": { "type": "array", "minItems": 1, - "description": "Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe.", + "description": "Assets the consumer must fetch and place under the box root before first use — the entries the scroll declared with embed false, and only those. A box whose assets are all embedded carries no such list. The declared size and hash are what make fetching them safe; a Scrollcase consumer verifies them and never downloads them itself.", "items": { "type": "object", "additionalProperties": false, @@ -215,15 +293,13 @@ }, "sha256": { "$ref": "#/$defs/sha256" + }, + "executable": { + "type": "boolean", + "description": "Present and true when the scroll declared the file executable. Whoever materializes it owns setting the bit: the file never passes through the archive, so nothing Scrollcase writes can carry a mode for it." } } } - } - }, - "$defs": { - "identifier": { - "type": "string", - "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" }, "kind": { "type": "string", @@ -243,7 +319,6 @@ "builderRevision", "sourceTreeDirty", "sourceRevision", - "pythonVersion", "dependencyLockSha256", "builtAt", "pixiVersion" @@ -269,11 +344,12 @@ "sourceRevision": { "type": "string", "minLength": 1, - "description": "Upstream revision of the packaged model source, as declared by the scroll." + "description": "Upstream revision of the packaged source, as declared by the scroll." }, - "pythonVersion": { + "runtimeVersion": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "The runtime version the environment was solved with, repeated from runtime.version. Absent exactly when the runtime has none: provenance records what was observed and never invents a value to fill a field." }, "pixiVersion": { "type": "string", @@ -289,13 +365,5 @@ } } } - }, - "dependentRequired": { - "assets": [ - "weights" - ], - "weights": [ - "assets" - ] } } diff --git a/rust/src/contract/schema/signed-document.schema.json b/rust/src/contract/schema/signed-document.schema.json index 602af0c..ad06de0 100644 --- a/rust/src/contract/schema/signed-document.schema.json +++ b/rust/src/contract/schema/signed-document.schema.json @@ -1,13 +1,13 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/signed-document.schema.json", + "$id": "https://scrollcase.dev/schema/v3/signed-document.schema.json", "title": "Signed box document", "description": "The envelope wrapping every signed document. The payload travels as exact base64-encoded JSON so that verifying a signature means hashing the bytes as transmitted, with no canonical-JSON implementation to keep in sync across languages. Passing this schema means the envelope is well-formed, never that its signature is valid.", "type": "object", "additionalProperties": false, "required": ["schemaVersion", "payloadEncoding", "payloadBase64", "payloadSha256", "signatures"], "properties": { - "schemaVersion": { "const": 2 }, + "schemaVersion": { "const": 3 }, "payloadEncoding": { "const": "base64-json-utf8" }, "payloadBase64": { "type": "string", diff --git a/rust/src/contract/schema/target.schema.json b/rust/src/contract/schema/target.schema.json index 6894c12..1573b29 100644 --- a/rust/src/contract/schema/target.schema.json +++ b/rust/src/contract/schema/target.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/target.schema.json", + "$id": "https://scrollcase.dev/schema/v3/target.schema.json", "title": "Box target", "description": "The platform, architecture and accelerator a box is built for. The supported combinations are closed: a target outside this matrix has no defined identifier and cannot be built, signed, or routed.", "type": "object", diff --git a/rust/src/contract/targets.rs b/rust/src/contract/targets.rs index 9066d8e..2d5cc18 100644 --- a/rust/src/contract/targets.rs +++ b/rust/src/contract/targets.rs @@ -204,28 +204,13 @@ pub fn assert_host(adapter: &BoxTargetAdapter, os: &str, arch: &str) -> Result<( Ok(()) } -/// Ensures a release's entry point agrees with the standalone Python layout for this target. -/// -/// Kept under its published name while the wire format still spells the field `pythonEntryPoint`. -/// The rule itself lives in [`super::runtimes`], where it can be asked about any runtime. -/// -/// # Errors -/// -/// When the entry point is not the one the runtime defines for this target. -pub fn assert_python_entry_point(adapter: &BoxTargetAdapter, entry_point: &str) -> Result<()> { - super::runtimes::assert_runtime_entry_point( - super::runtimes::IMPLICIT_RUNTIME_ID, - adapter, - entry_point, - ) -} - #[cfg(test)] mod tests { use super::{ - assert_host, assert_python_entry_point, box_target_adapter, box_target_adapters, - box_target_id, is_cuda_version, BoxTarget, + assert_host, box_target_adapter, box_target_adapters, box_target_id, is_cuda_version, + BoxTarget, }; + use crate::contract::runtimes::assert_runtime_entry_point; fn target(platform: &str, arch: &str, accelerator: &str, cuda: Option<&str>) -> BoxTarget { BoxTarget { @@ -271,8 +256,8 @@ mod tests { #[test] fn an_entry_point_from_another_platform_is_refused() { let windows = box_target_adapter(&target("windows", "x86_64", "cpu", None)).unwrap(); - assert!(assert_python_entry_point(windows, "venv/python.exe").is_ok()); - assert!(assert_python_entry_point(windows, "venv/bin/python").is_err()); + assert!(assert_runtime_entry_point("python", windows, "venv/python.exe").is_ok()); + assert!(assert_runtime_entry_point("python", windows, "venv/bin/python").is_err()); } #[test] diff --git a/rust/src/execution.rs b/rust/src/execution.rs index 912ff3c..d5122e1 100644 --- a/rust/src/execution.rs +++ b/rust/src/execution.rs @@ -11,7 +11,7 @@ use std::collections::BTreeSet; -use crate::contract::runtimes::{runtime_adapter, IMPLICIT_RUNTIME_ID}; +use crate::contract::runtimes::runtime_adapter; use crate::contract::targets::BoxTargetAdapter; use crate::error::{Error, Result}; use crate::path::safe_relative_path; @@ -28,13 +28,14 @@ use crate::release::Execution; pub fn assert_execution_files( execution: Option<&Execution>, adapter: &BoxTargetAdapter, + runtime_id: &str, runtime_version: &str, files: &BTreeSet, ) -> Result<()> { let Some(execution) = execution else { return Ok(()); }; - let runtime = runtime_adapter(IMPLICIT_RUNTIME_ID)?; + let runtime = runtime_adapter(runtime_id)?; let resolved = runtime.resolve_execution_files( &execution.as_runtime(), adapter.platform, @@ -83,11 +84,11 @@ mod tests { default_args: vec![], }; assert!( - assert_execution_files(Some(&execution), adapter, "3.11.9", &files(&["app/main.py"])) + assert_execution_files(Some(&execution), adapter, "python", "3.11.9", &files(&["app/main.py"])) .is_ok() ); let error = - assert_execution_files(Some(&execution), adapter, "3.11.9", &files(&["app/other.py"])) + assert_execution_files(Some(&execution), adapter, "python", "3.11.9", &files(&["app/other.py"])) .unwrap_err(); assert!(error.message().contains("Execution script is missing"), "{error}"); } @@ -107,13 +108,13 @@ mod tests { "venv/lib/python3.11/site-packages/example_model/main/__main__.py", ] { assert!( - assert_execution_files(Some(&execution), adapter, "3.11.9", &files(&[location])) + assert_execution_files(Some(&execution), adapter, "python", "3.11.9", &files(&[location])) .is_ok(), "{location} did not resolve" ); } let error = - assert_execution_files(Some(&execution), adapter, "3.11.9", &files(&["elsewhere.py"])) + assert_execution_files(Some(&execution), adapter, "python", "3.11.9", &files(&["elsewhere.py"])) .unwrap_err(); assert!( error.message().contains("Execution module is not discoverable"), @@ -131,6 +132,7 @@ mod tests { assert!(assert_execution_files( Some(&execution), windows, + "python", "3.11.9", &files(&["venv/Lib/site-packages/pkg/__main__.py"]) ) @@ -139,6 +141,7 @@ mod tests { assert!(assert_execution_files( Some(&execution), windows, + "python", "3.11.9", &files(&["venv/lib/python3.11/site-packages/pkg/__main__.py"]) ) @@ -154,7 +157,7 @@ mod tests { }; for invalid in ["", "3", "3.x", "x.1", "3."] { assert!( - assert_execution_files(Some(&execution), adapter, invalid, &files(&[])).is_err(), + assert_execution_files(Some(&execution), adapter, "python", invalid, &files(&[])).is_err(), "{invalid} was accepted" ); } @@ -163,6 +166,6 @@ mod tests { #[test] fn a_library_only_box_declares_no_execution() { let adapter = adapter("macos", "aarch64", "metal"); - assert!(assert_execution_files(None, adapter, "3.11.9", &files(&[])).is_ok()); + assert!(assert_execution_files(None, adapter, "python", "3.11.9", &files(&[])).is_ok()); } } diff --git a/rust/src/prepare.rs b/rust/src/prepare.rs index 00d6da4..accf35e 100644 --- a/rust/src/prepare.rs +++ b/rust/src/prepare.rs @@ -20,7 +20,7 @@ use crate::archive::extract_zip_archive; use crate::contract::payload_digest::{ parse_payload_digest_stream, PayloadDigestKind, MAX_PAYLOAD_DIGEST_BYTES, PAYLOAD_DIGEST_FILE, }; -use crate::contract::runtimes::{execution_affecting_variables, IMPLICIT_RUNTIME_ID}; +use crate::contract::runtimes::execution_affecting_variables; use crate::contract::targets::{assert_native_host, box_target_id, BoxTargetAdapter}; use crate::environment::{ resolve_environment, EnvironmentLayer, EnvironmentReport, EnvironmentSource, ResolveOptions, @@ -29,7 +29,7 @@ use crate::error::{fail, Error, Result}; use crate::execution::assert_execution_files; use crate::filesystem::{collect_files, payload_size, sha256_file}; use crate::path::{join_relative, safe_relative_path}; -use crate::release::{AssetDescriptor, Execution, ReleaseManifest}; +use crate::release::{AssetDescriptor, BoxRuntime, Execution, ReleaseManifest}; use crate::trust::TrustAnchors; use crate::verify::{inspect_archive_for, inspect_release_document, InspectedRelease}; @@ -131,16 +131,16 @@ impl PreparedBox { &self.release.box_id } - /// Model identity, from the signed release. + /// Free-form annotations the publisher signed. This crate attaches no meaning to any key. #[must_use] - pub fn model_id(&self) -> &str { - &self.release.model_id + pub fn labels(&self) -> Option<&BTreeMap> { + self.release.labels.as_ref() } - /// Installed-directory identity, from the signed release. + /// What runs inside the box, from the signed release. #[must_use] - pub fn runtime_id(&self) -> &str { - &self.release.runtime_id + pub fn runtime(&self) -> &BoxRuntime { + &self.release.runtime } /// Box version, from the signed release. @@ -155,11 +155,6 @@ impl PreparedBox { &self.target_id } - /// Interpreter path, relative to the box root. - #[must_use] - pub fn python_entry_point(&self) -> &str { - &self.release.python_entry_point - } /// The declared application entry point, if the box has one. #[must_use] @@ -234,13 +229,12 @@ impl PreparedBox { } } -/// The on-demand descriptors a release requires a caller to have materialised. +/// The deferred descriptors a release requires a caller to have materialised. +/// +/// The list is exactly the assets the scroll declared `embed: false`; a release whose assets are +/// all embedded carries none, and the box needs nothing fetched before it runs. fn required_assets_of(release: &ReleaseManifest) -> &[AssetDescriptor] { - if release.weights.as_deref() == Some("on-demand") { - release.assets.as_deref().unwrap_or(&[]) - } else { - &[] - } + release.assets.as_deref().unwrap_or(&[]) } /// Checks the assets a caller was told to place, against their signed descriptors. @@ -312,7 +306,7 @@ fn release_environment_report( values: release_pairs, }, ], - execution_affecting_variables: &execution_affecting_variables(IMPLICIT_RUNTIME_ID, adapter)?, + execution_affecting_variables: &execution_affecting_variables(&release.runtime.id, adapter)?, expanded: options.env_report || options.env_report_values, reveal_host_values: options.env_report_values, })? @@ -515,18 +509,21 @@ pub fn attach_extracted_box( } let files = collect_files(&root)?; - if !files.contains(&release.python_entry_point) { - fail!("Attached box is missing {}.", release.python_entry_point); + if let Some(entry_point) = &release.runtime.entry_point { + if !files.contains(entry_point) { + fail!("Attached box is missing {entry_point}."); + } } assert_execution_files( release.execution.as_ref(), inspected.adapter, - &release.provenance.python_version, + &release.runtime.id, + release.provenance.runtime_version.as_deref().unwrap_or_default(), &files, )?; verify_required_assets(&root, required_assets_of(release))?; - // Measured, never compared: an installed tree legitimately grows after extraction — on-demand + // Measured, never compared: an installed tree legitimately grows after extraction — deferred // assets, caches, whatever the application writes — so holding it to the signed figure would // fail honest boxes. let installed_size_bytes = payload_size(&root)?; diff --git a/rust/src/release.rs b/rust/src/release.rs index 3df3a3b..931cb49 100644 --- a/rust/src/release.rs +++ b/rust/src/release.rs @@ -96,16 +96,60 @@ pub struct PayloadDigestCommitment { pub sha256: String, } -/// The import check a box must pass with its own interpreter. +/// One invocation of the box's declared execution, and the status it must exit with. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SelfTestCommand { + /// Arguments appended to the box's declared execution. + pub args: Vec, + /// Status the invocation must produce. + pub expect_exit_code: u8, +} + +/// What a box proves about itself, in whichever shapes its runtime supports. +/// +/// `imports` asks the runtime's loader a question and only means something to a runtime that has +/// one; `commands` asks the box's own execution a question, which is the only shape a runtime with +/// no module system can answer. At least one must be present. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SelfTestProbe { + /// Modules the runtime must be able to load. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub imports: Option>, + /// Invocations of the box's declared execution. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub commands: Option>, +} + +/// The check a box must pass, and how long it may take. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SelfTest { - /// Modules to import. - pub python_imports: Vec, - /// How long the import check may take. + /// What the box proves about itself. + pub probe: SelfTestProbe, + /// How long the check may take. pub timeout_seconds: u64, } +/// What runs inside the box. +/// +/// A target says which machine a box is for; this says what executes on it. Version 2 had no such +/// field: a box recorded a Python entry point and Python execution kinds and nothing that said +/// "Python", so a reader had to infer the runtime from the shape of a path. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct BoxRuntime { + /// The runtime this box carries: `python`, `node` or `native`. + pub id: String, + /// Its own version. Absent for a runtime that has none to record. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, + /// Its own executable, relative to the box root. Absent for a runtime with no separate one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub entry_point: Option, +} + /// The optional, shell-free application entry point. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] @@ -126,6 +170,25 @@ pub enum Execution { /// Arguments always passed before a caller's own. default_args: Vec, }, + /// Run one regular payload file with the box's own Node runtime. + /// + /// Named by the format so that implementing the runtime is code rather than another wire + /// break. No adapter in this crate answers for it yet, so a box declaring it is refused. + #[serde(rename_all = "camelCase")] + NodeScript { + /// Safe path to a regular JavaScript file inside the box. + script: String, + /// Arguments always passed before a caller's own. + default_args: Vec, + }, + /// Run a compiled executable the box carries, with no interpreter in front of it. + #[serde(rename_all = "camelCase")] + NativeBinary { + /// Safe path to the executable inside the box. + binary: String, + /// Arguments always passed before a caller's own. + default_args: Vec, + }, } impl Execution { @@ -137,9 +200,15 @@ impl Execution { #[must_use] pub fn as_runtime(&self) -> RuntimeExecution<'_> { match self { + // Both script kinds carry the same shape: which runtime runs it is the box's + // declaration, not something the shape can say. Execution::PythonScript { script, default_args, + } + | Execution::NodeScript { + script, + default_args, } => RuntimeExecution::Script { script, default_args, @@ -151,6 +220,13 @@ impl Execution { module, default_args, }, + Execution::NativeBinary { + binary, + default_args, + } => RuntimeExecution::Binary { + binary, + default_args, + }, } } } @@ -169,8 +245,10 @@ pub struct Provenance { pub source_tree_dirty: bool, /// Commit the box was built from. pub source_revision: String, - /// Python version inside the box. - pub python_version: String, + /// The runtime version the environment was solved with. Absent exactly when the runtime has + /// none: provenance records what was observed and never invents a value to fill a field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub runtime_version: Option, /// SHA-256 of the dependency lock. pub dependency_lock_sha256: String, /// When the box was built. @@ -179,7 +257,7 @@ pub struct Provenance { pub pixi_version: String, } -/// One on-demand asset a caller must materialise before execution. +/// One deferred asset a caller must materialise before execution. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct AssetDescriptor { @@ -191,22 +269,25 @@ pub struct AssetDescriptor { pub size_bytes: u64, /// Lowercase hex SHA-256 the placed file must have. pub sha256: String, + /// Present and true when the scroll declared the file executable. Whoever materialises it owns + /// setting the bit: the file never passes through the archive, so nothing here carries a mode. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub executable: Option, } /// The immutable description of one built box. #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ReleaseManifest { - /// Format version; always 2. + /// Format version; always 3. pub schema_version: u32, /// `.release`, in the publishing project's own namespace. pub kind: String, /// Box identity. pub box_id: String, - /// Model identity. - pub model_id: String, - /// Installed-directory identity. - pub runtime_id: String, + /// Free-form annotations the publisher signed. This crate attaches no meaning to any key. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub labels: Option>, /// Box version. pub version: String, /// The target this box was built for. @@ -221,24 +302,22 @@ pub struct ReleaseManifest { /// Commitment to the extracted tree, absent on boxes built before it existed. #[serde(default, skip_serializing_if = "Option::is_none")] pub payload_digest: Option, - /// Interpreter path, relative to the box root. - pub python_entry_point: String, - /// Where a caller's model cache belongs inside the box. - pub model_cache_subdir: String, - /// Signed environment applied whenever Scrollcase runs the interpreter. + /// What runs inside the box. + pub runtime: BoxRuntime, + /// Where the box's own large files belong inside it. + pub cache_subdir: String, + /// Signed environment applied whenever Scrollcase runs the box. #[serde(default, skip_serializing_if = "Option::is_none")] pub environment: Option>, - /// The import check. + /// The check a consumer can repeat. pub self_test: SelfTest, /// The optional application entry point. #[serde(default, skip_serializing_if = "Option::is_none")] pub execution: Option, /// How the box was produced. pub provenance: Provenance, - /// Present only when assets are carried outside the archive. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub weights: Option, - /// Descriptors of the assets a caller must materialise. + /// Descriptors of the assets a caller must materialise, and only those. Absent when the box is + /// self-contained; there is no separate mode field, because the list itself is the statement. #[serde(default, skip_serializing_if = "Option::is_none")] pub assets: Option>, } @@ -250,40 +329,83 @@ pub struct ReleaseManifest { #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct BoxManifest { - /// Format version; always 2. + /// Format version; always 3. pub schema_version: u32, /// Box identity. pub box_id: String, - /// Model identity. - pub model_id: String, - /// Installed-directory identity. - pub runtime_id: String, + /// Free-form annotations the publisher signed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub labels: Option>, /// Box version. pub version: String, /// The target this box was built for. pub target: BoxTarget, - /// Interpreter path, relative to the box root. - pub python_entry_point: String, - /// Where a caller's model cache belongs inside the box. - pub model_cache_subdir: String, - /// Signed environment applied whenever Scrollcase runs the interpreter. + /// What runs inside the box. + pub runtime: BoxRuntime, + /// Where the box's own large files belong inside it. + pub cache_subdir: String, + /// Signed environment applied whenever Scrollcase runs the box. #[serde(default, skip_serializing_if = "Option::is_none")] pub environment: Option>, - /// The import check. + /// The check a consumer can repeat. pub self_test: SelfTest, /// The optional application entry point. #[serde(default, skip_serializing_if = "Option::is_none")] pub execution: Option, /// How the box was produced. pub provenance: Provenance, - /// Present only when assets are carried outside the archive. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub weights: Option, - /// Descriptors of the assets a caller must materialise. + /// Descriptors of the assets a caller must materialise, and only those. #[serde(default, skip_serializing_if = "Option::is_none")] pub assets: Option>, } +impl BoxRuntime { + /// The runtime block's own constraints, which the types cannot state. + /// + /// The id is checked against the format's closed vocabulary rather than against what this crate + /// implements: a box naming `native` is a well-formed v3 box that this build cannot run, and + /// saying so is `run`'s job, not the manifest reader's. + fn validate(&self) -> Result<()> { + if !crate::contract::runtimes::RUNTIME_IDS.contains(&self.id.as_str()) { + fail!( + "Invalid release manifest: unknown runtime {}. The box format defines {}.", + self.id, + crate::contract::runtimes::RUNTIME_IDS.join(", ") + ); + } + for (label, value) in [("version", &self.version), ("entryPoint", &self.entry_point)] { + if value.as_ref().is_some_and(std::string::String::is_empty) { + fail!("Invalid release manifest: runtime {label} must not be empty."); + } + } + Ok(()) + } +} + +impl SelfTest { + /// The probe's own constraints. At least one shape must be present: a box that proves nothing + /// about itself is not a box worth signing. + fn validate(&self) -> Result<()> { + let imports = self.probe.imports.as_deref().unwrap_or_default(); + let commands = self.probe.commands.as_deref().unwrap_or_default(); + if imports.is_empty() && commands.is_empty() { + fail!("Invalid release manifest: selfTest probe must declare imports or commands."); + } + if self.probe.imports.is_some() + && (imports.is_empty() || imports.iter().any(std::string::String::is_empty)) + { + fail!("Invalid release manifest: selfTest probe imports must be non-empty."); + } + if self.probe.commands.is_some() && commands.is_empty() { + fail!("Invalid release manifest: selfTest probe commands must be non-empty."); + } + if self.timeout_seconds == 0 { + fail!("Invalid release manifest: selfTest timeoutSeconds must be positive."); + } + Ok(()) + } +} + /// Whether a value is lowercase hex of the given length. fn is_lowercase_hex(value: &str, length: usize) -> bool { value.len() == length @@ -328,9 +450,13 @@ impl Execution { /// When the script path is unsafe or the module name is not a strict dotted name. pub fn validate(&self) -> Result<()> { match self { - Self::PythonScript { script, .. } => { + Self::PythonScript { script, .. } + | Self::NodeScript { script, .. } => { safe_relative_path(script)?; } + Self::NativeBinary { binary, .. } => { + safe_relative_path(binary)?; + } Self::PythonModule { module, .. } => { if !is_python_module(module) { fail!("Invalid release manifest: execution module {module} is not a dotted Python module name."); @@ -363,19 +489,18 @@ impl ReleaseManifest { // outside the supported matrix, a CUDA target without a version, and a version on a target // that may not carry one, all in the same call the slug is derived from. crate::contract::targets::box_target_id(&self.target)?; - for (label, value) in [ - ("boxId", &self.box_id), - ("modelId", &self.model_id), - ("runtimeId", &self.runtime_id), - ] { - if !is_identifier(value) { - fail!("Invalid release manifest: {label} is not a valid identifier."); + if !is_identifier(&self.box_id) { + fail!("Invalid release manifest: boxId is not a valid identifier."); + } + for key in self.labels.iter().flat_map(std::collections::BTreeMap::keys) { + if !is_identifier(key) { + fail!("Invalid release manifest: label {key} is not a valid identifier."); } } + self.runtime.validate()?; for (label, value) in [ ("version", &self.version), - ("pythonEntryPoint", &self.python_entry_point), - ("modelCacheSubdir", &self.model_cache_subdir), + ("cacheSubdir", &self.cache_subdir), ("archive.url", &self.archive.url), ] { if value.is_empty() { @@ -401,18 +526,7 @@ impl ReleaseManifest { fail!("Invalid release manifest: payloadDigest is not a supported commitment."); } } - if self.self_test.python_imports.is_empty() - || self - .self_test - .python_imports - .iter() - .any(std::string::String::is_empty) - { - fail!("Invalid release manifest: selfTest pythonImports must be non-empty."); - } - if self.self_test.timeout_seconds == 0 { - fail!("Invalid release manifest: selfTest timeoutSeconds must be positive."); - } + self.self_test.validate()?; validate_environment(self.environment.as_ref())?; if let Some(execution) = &self.execution { execution.validate()?; @@ -423,18 +537,11 @@ impl ReleaseManifest { Ok(()) } - /// The `weights`/`assets` co-requirement, and the descriptors themselves. + /// The deferred-asset descriptors, which are the whole statement in version 3: there is no + /// second field they have to agree with. fn validate_assets(&self) -> Result<()> { - let assets = match (self.weights.as_deref(), self.assets.as_deref()) { - (None, None) => return Ok(()), - (Some("on-demand"), Some(assets)) => assets, - (Some(other), Some(_)) => { - fail!("Invalid release manifest: unsupported weights value {other}.") - } - // `dependentRequired` in both directions: neither field means anything alone. - (Some(_), None) | (None, Some(_)) => { - fail!("Invalid release manifest: weights and assets must be declared together.") - } + let Some(assets) = self.assets.as_deref() else { + return Ok(()); }; if assets.is_empty() { fail!("Invalid release manifest: assets must not be empty."); @@ -470,6 +577,13 @@ fn validate_environment(environment: Option<&BTreeMap>) -> Resul } fn validate_provenance(provenance: &Provenance) -> Result<()> { + if provenance + .runtime_version + .as_ref() + .is_some_and(std::string::String::is_empty) + { + fail!("Invalid release manifest: provenance runtimeVersion must not be empty."); + } if !is_lowercase_hex(&provenance.builder_revision, 40) { fail!("Invalid release manifest: provenance builderRevision is not a commit."); } @@ -480,7 +594,6 @@ fn validate_provenance(provenance: &Provenance) -> Result<()> { ("scrollId", &provenance.scroll_id), ("scrollVersion", &provenance.scroll_version), ("sourceRevision", &provenance.source_revision), - ("pythonVersion", &provenance.python_version), ("builtAt", &provenance.built_at), ("pixiVersion", &provenance.pixi_version), ] { diff --git a/rust/src/run.rs b/rust/src/run.rs index 81aee1d..d09bdd3 100644 --- a/rust/src/run.rs +++ b/rust/src/run.rs @@ -21,7 +21,7 @@ use crate::environment::{ resolve_environment, EnvironmentLayer, EnvironmentReport, EnvironmentSource, ResolveOptions, }; use crate::contract::runtimes::{ - execution_affecting_variables, runtime_adapter, RuntimeArgument, IMPLICIT_RUNTIME_ID, + execution_affecting_variables, runtime_adapter, RuntimeArgument, }; use crate::error::{fail, Error, Result}; use crate::execution::assert_execution_files; @@ -243,7 +243,7 @@ fn resolve_run_environment( .collect(), }, ], - execution_affecting_variables: &execution_affecting_variables(IMPLICIT_RUNTIME_ID, adapter)?, + execution_affecting_variables: &execution_affecting_variables(&release.runtime.id, adapter)?, expanded: options.environment.env_report || options.environment.env_report_values, reveal_host_values: options.environment.env_report_values, }) @@ -279,23 +279,30 @@ pub fn run_extracted_box(prepared: &PreparedBox, options: &RunOptions<'_>) -> Re let root = prepared.root(); let files = collect_files(root)?; - if !files.contains(&release.python_entry_point) { - fail!("Prepared box is missing {}.", release.python_entry_point); + if let Some(entry_point) = &release.runtime.entry_point { + if !files.contains(entry_point) { + fail!("Prepared box is missing {entry_point}."); + } } assert_execution_files( Some(execution), adapter, - &release.provenance.python_version, + &release.runtime.id, + release.provenance.runtime_version.as_deref().unwrap_or_default(), &files, )?; verify_required_assets(root, prepared.required_assets())?; - let python = join_relative(root, &safe_relative_path(&release.python_entry_point)?); // The runtime states the command line in payload-relative terms and this end joins it: a box // root is a real path on this host, and the format has no business deciding what one looks - // like. - let invocation = runtime_adapter(IMPLICIT_RUNTIME_ID)? + // like. Which runtime states it is the box's declaration, not an assumption about what a box + // contains. + let invocation = runtime_adapter(&release.runtime.id)? .build_argv(&execution.as_runtime(), adapter.platform)?; + let command = match &invocation.command { + RuntimeArgument::Literal(value) => PathBuf::from(value), + RuntimeArgument::PayloadPath(value) => join_relative(root, &safe_relative_path(value)?), + }; let mut arguments: Vec = Vec::with_capacity(invocation.args.len() + options.args.len()); for argument in &invocation.args { arguments.push(match argument { @@ -313,7 +320,7 @@ pub fn run_extracted_box(prepared: &PreparedBox, options: &RunOptions<'_>) -> Re } let invocation = BoxInvocation { - program: &python, + program: &command, args: &arguments, cwd: root, environment: &resolved.environment, @@ -325,7 +332,7 @@ pub fn run_extracted_box(prepared: &PreparedBox, options: &RunOptions<'_>) -> Re let child = spawner.spawn(&invocation).map_err(|error| { Error::new(format!( "Box interpreter failed to start: {}: {error}", - python.display() + command.display() )) })?; diff --git a/rust/src/trust.rs b/rust/src/trust.rs index 36e60a3..fd6ae5e 100644 --- a/rust/src/trust.rs +++ b/rust/src/trust.rs @@ -231,7 +231,7 @@ mod tests { use base64::engine::general_purpose::STANDARD as BASE64; use base64::Engine as _; serde_json::from_value(serde_json::json!({ - "schemaVersion": 2, + "schemaVersion": 3, "payloadEncoding": "base64-json-utf8", "payloadBase64": BASE64.encode(PAYLOAD), "payloadSha256": crate::contract::documents::sha256_hex(PAYLOAD), diff --git a/rust/src/verify.rs b/rust/src/verify.rs index 77e0734..aa0d9fc 100644 --- a/rust/src/verify.rs +++ b/rust/src/verify.rs @@ -12,7 +12,10 @@ use std::path::{Path, PathBuf}; use crate::archive::{list_zip_entries, read_zip_entry_text, ArchiveEntry}; use crate::contract::documents::SignedDocument; use crate::contract::links::EntryKind; -use crate::contract::targets::{assert_python_entry_point, box_target_adapter, BoxTargetAdapter}; +use crate::contract::runtimes::{ + assert_runtime_entry_point, is_implemented_runtime, unimplemented_runtime_message, +}; +use crate::contract::targets::{box_target_adapter, BoxTargetAdapter}; use crate::error::{fail, Error, Result}; use crate::execution::assert_execution_files; use crate::filesystem::sha256_file; @@ -61,16 +64,28 @@ pub fn inspect_release_document( let signed = SignedDocument::parse(&raw)?; let payload = verify_signed_document(&signed, &trusted)?; - // A v1 payload inside a v2 envelope is refused by name rather than reinterpreted. - if payload.value.get("schemaVersion").and_then(serde_json::Value::as_u64) == Some(1) { - fail!("Unsupported schemaVersion 1; rebuild this box with Scrollcase v2."); + // A superseded payload inside a v3 envelope is refused by name rather than reinterpreted. Both + // older versions are named: they are different artefacts with different rebuilds ahead of them. + if let Some(version @ (1 | 2)) = payload + .value + .get("schemaVersion") + .and_then(serde_json::Value::as_u64) + { + fail!("Unsupported schemaVersion {version}; rebuild this box with Scrollcase v3."); } let release: ReleaseManifest = serde_json::from_value(payload.value) .map_err(|error| Error::new(format!("Invalid release manifest: {error}.")))?; release.validate()?; let adapter = box_target_adapter(&release.target)?; - assert_python_entry_point(adapter, &release.python_entry_point)?; + // The format's runtime vocabulary is wider than what this crate implements, so a release may + // name one there is no adapter for. That is refused by name rather than misread as another. + if !is_implemented_runtime(&release.runtime.id) { + fail!("{}", unimplemented_runtime_message(&release.runtime.id)); + } + if let Some(entry_point) = &release.runtime.entry_point { + assert_runtime_entry_point(&release.runtime.id, adapter, entry_point)?; + } Ok(InspectedRelease { release_path, @@ -82,10 +97,10 @@ pub fn inspect_release_document( /// Binds the self-description inside the archive to the signed release outside it. /// -/// Only fields present in both schema-version-2 documents belong here. Release-only transport data -/// has no counterpart in `box.json`; every shared identity, target, layout, self-test, environment, -/// asset-policy and provenance field must agree. Without this, a correctly hashed archive could be -/// paired with a signed manifest describing something else entirely. +/// Only fields present in both schema-version-3 documents belong here. Release-only transport data +/// has no counterpart in `box.json`; every shared identity, target, runtime, layout, self-test, +/// environment, deferred-asset and provenance field must agree. Without this, a correctly hashed +/// archive could be paired with a signed manifest describing something else entirely. /// /// # Errors /// @@ -96,31 +111,27 @@ pub fn assert_box_manifest_agreement( ) -> Result<()> { // Written as explicit pairs rather than a derived comparison so the field name in the message is // the field that actually differed — the Node and Python consumers report the same way, and the - // conformance fixture pins `box.json mismatch: modelId` among others. + // conformance fixture pins `box.json mismatch: labels` among others. let mismatch = if box_manifest.schema_version != release.schema_version { Some("schemaVersion") } else if box_manifest.box_id != release.box_id { Some("boxId") - } else if box_manifest.model_id != release.model_id { - Some("modelId") - } else if box_manifest.runtime_id != release.runtime_id { - Some("runtimeId") + } else if box_manifest.labels != release.labels { + Some("labels") } else if box_manifest.version != release.version { Some("version") } else if box_manifest.target != release.target { Some("target") - } else if box_manifest.python_entry_point != release.python_entry_point { - Some("pythonEntryPoint") - } else if box_manifest.model_cache_subdir != release.model_cache_subdir { - Some("modelCacheSubdir") + } else if box_manifest.runtime != release.runtime { + Some("runtime") + } else if box_manifest.cache_subdir != release.cache_subdir { + Some("cacheSubdir") } else if box_manifest.environment != release.environment { Some("environment") } else if box_manifest.self_test != release.self_test { Some("selfTest") } else if box_manifest.execution != release.execution { Some("execution") - } else if box_manifest.weights != release.weights { - Some("weights") } else if box_manifest.assets != release.assets { Some("assets") } else if box_manifest.provenance != release.provenance { @@ -219,13 +230,16 @@ pub fn inspect_archive_for( .map_err(|error| Error::new(format!("Invalid box.json: {error}.")))?; assert_box_manifest_agreement(&box_manifest, release)?; - if !resolvable.contains(&release.python_entry_point) { - fail!("Archive is missing {}.", release.python_entry_point); + if let Some(entry_point) = &release.runtime.entry_point { + if !resolvable.contains(entry_point) { + fail!("Archive is missing {entry_point}."); + } } assert_execution_files( release.execution.as_ref(), inspected.adapter, - &release.provenance.python_version, + &release.runtime.id, + release.provenance.runtime_version.as_deref().unwrap_or_default(), &resolvable, )?; diff --git a/rust/tests/archive.rs b/rust/tests/archive.rs index 70df61b..f394f29 100644 --- a/rust/tests/archive.rs +++ b/rust/tests/archive.rs @@ -82,9 +82,9 @@ fn box_json_must_agree_with_the_signed_release_field_by_field() { // a different box. Only comparing the two manifests catches it. for (field, mutate) in [ ( - "modelId", + "labels", Box::new(|manifest: &mut serde_json::Value| { - manifest["modelId"] = json!("another-model"); + manifest["labels"] = json!({ "model": "another-model" }); }) as Box, ), ( @@ -142,7 +142,7 @@ fn an_archive_without_its_declared_interpreter_is_refused() { |_| {}, |entries| { // The entry point differs per target, so it is asked for rather than spelled out. - let interpreter = support::native_python_entry_point(); + let interpreter = support::native_entry_point(); entries.retain(|entry| !matches!(entry, Entry::File(path, _, _) if *path == interpreter)); }, |_| {}, @@ -292,7 +292,7 @@ fn extraction_reproduces_the_payload_and_its_modes() { assert_eq!( scrollcase_consumer::filesystem::payload_size(&destination).unwrap(), std::fs::metadata(destination.join("box.json")).unwrap().len() - + std::fs::metadata(destination.join(support::native_python_entry_point())) + + std::fs::metadata(destination.join(support::native_entry_point())) .unwrap() .len() + std::fs::metadata(destination.join("app/main.py")).unwrap().len() diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index 2a5c1a8..cee1fce 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -72,7 +72,7 @@ fn named_targets() -> BTreeMap { .collect() } -fn python_entry_point(target: &Value) -> &'static str { +fn entry_point_for(target: &Value) -> &'static str { if target["platform"] == "windows" { "venv/python.exe" } else { @@ -154,7 +154,7 @@ impl Fixture { Some(name) => named_targets().get(name).cloned().unwrap(), None => native_target(), }; - let entry_point = python_entry_point(&target); + let entry_point = entry_point_for(&target); let execution = if spec.get("execution").and_then(Value::as_str) == Some("module") { json!({ @@ -167,15 +167,13 @@ impl Fixture { }; let mut manifest = json!({ - "schemaVersion": 2, + "schemaVersion": 3, "boxId": "consumer-fixture", - "modelId": "consumer-fixture-model", - "runtimeId": "consumer-fixture-runtime", "version": "1.0.0", "target": target, - "pythonEntryPoint": entry_point, - "modelCacheSubdir": "model-cache/consumer-fixture", - "selfTest": { "pythonImports": ["json"], "timeoutSeconds": 30 }, + "runtime": { "id": "python", "version": "3.11.9", "entryPoint": entry_point }, + "cacheSubdir": "cache/consumer-fixture", + "selfTest": { "probe": { "imports": ["json"] }, "timeoutSeconds": 30 }, "execution": execution.clone(), "provenance": { "scrollId": "consumer-fixture", @@ -183,7 +181,7 @@ impl Fixture { "builderRevision": "b".repeat(40), "sourceTreeDirty": false, "sourceRevision": "c".repeat(40), - "pythonVersion": "3.11.9", + "runtimeVersion": "3.11.9", "dependencyLockSha256": "d".repeat(64), "builtAt": "2026-01-01T00:00:00.000Z", "pixiVersion": "0.50.0" @@ -192,11 +190,14 @@ impl Fixture { if let Some(environment) = spec.get("environment") { manifest["environment"] = environment.clone(); } + if let Some(labels) = spec.get("labels") { + manifest["labels"] = labels.clone(); + } + // The list is exactly the deferred entries; there is no second field to keep in step. if spec.get("requiredAsset").is_some() { - manifest["weights"] = json!("on-demand"); manifest["assets"] = json!([{ - "url": "https://assets.example.org/weights.bin", - "relativePath": "model-cache/consumer-fixture/weights.bin", + "url": "https://assets.example.org/data.bin", + "relativePath": "cache/consumer-fixture/data.bin", "sizeBytes": ASSET_BYTES.len(), "sha256": support::sha256_hex(ASSET_BYTES), }]); @@ -207,6 +208,11 @@ impl Fixture { Entry::file("box.json", &serde_json::to_vec_pretty(&manifest).unwrap(), 0o644), Entry::file(entry_point, b"#!/bin/sh\nexit 0\n", 0o755), ]; + // A declared-executable asset: the mode is synthesised from the scroll's declaration, and + // extraction has to hand it back whatever umask the process is running under. + if spec.get("executableAsset").is_some() { + entries.push(Entry::file("bin/tool", b"#!/bin/sh\nexit 0\n", 0o755)); + } if execution["kind"] == "python-module" { entries.push(Entry::file("example/application.py", b"print('module')\n", 0o644)); } else { @@ -403,7 +409,7 @@ fn replace_tokens(value: &Value, root: Option<&Path>) -> Value { Value::String(text) => { let target = native_target(); let replaced = text - .replace("$NATIVE_PYTHON", python_entry_point(&target)) + .replace("$NATIVE_ENTRY_POINT", entry_point_for(&target)) .replace("$NATIVE_TARGET", &target_id_of(&target)); Value::String(match root { Some(root) => replaced.replace("$BOX", &root.to_string_lossy()), @@ -463,6 +469,8 @@ fn execution_kind(execution: Option<&Execution>) -> Value { match execution { Some(Execution::PythonScript { .. }) => json!("python-script"), Some(Execution::PythonModule { .. }) => json!("python-module"), + Some(Execution::NodeScript { .. }) => json!("node-script"), + Some(Execution::NativeBinary { .. }) => json!("native-binary"), None => Value::Null, } } @@ -497,15 +505,43 @@ fn receipt_value(prepared: &PreparedBox, expected: &Value, names: &[String]) -> "boxId": prepared.box_id(), "executionKind": execution_kind(prepared.execution()), "requiredAssetCount": prepared.required_assets().len(), - "pythonEntryPoint": prepared.python_entry_point(), + "runtimeId": prepared.runtime().id, + "entryPoint": prepared.runtime().entry_point.as_deref().unwrap_or_default(), "targetId": prepared.target_id(), }); if expected.get("environmentReport").is_some() { receipt["environmentReport"] = report_value(prepared.environment_report(), names); } + if let Some(paths) = expected.get("executableModes").and_then(Value::as_object) { + receipt["executableModes"] = executable_modes(prepared.root(), paths.keys()); + } receipt } +/// The permission bits an extracted box actually carries, for the paths a case names. +/// +/// Windows has no bit to read, so every path reports null there and the fixture says so rather +/// than the driver quietly skipping the case. +fn executable_modes<'a>(root: &Path, paths: impl Iterator) -> Value { + let mut modes = serde_json::Map::new(); + for path in paths { + let metadata = std::fs::metadata(root.join(path)).unwrap(); + modes.insert(path.clone(), mode_value(&metadata)); + } + Value::Object(modes) +} + +#[cfg(unix)] +fn mode_value(metadata: &std::fs::Metadata) -> Value { + use std::os::unix::fs::PermissionsExt as _; + json!(format!("{:o}", metadata.permissions().mode() & 0o777)) +} + +#[cfg(not(unix))] +fn mode_value(_metadata: &std::fs::Metadata) -> Value { + Value::Null +} + fn materialize_asset(prepared: &PreparedBox, state: Option<&str>) { let Some(state) = state else { return }; if state == "missing" { @@ -625,8 +661,18 @@ fn mutate_fixture(fixture: &mut Fixture, mutation: &str, destination: &Path) { fixture.release["archive"]["sizeBytes"] = json!(size + 1); fixture.sign(); } - "alter-release-model" => { - fixture.release["modelId"] = json!("altered-model"); + "alter-release-labels" => { + fixture.release["labels"] = json!({ "model": "altered-model" }); + fixture.sign(); + } + "alter-release-runtime-version" => { + fixture.release["runtime"]["version"] = json!("3.99.0"); + fixture.sign(); + } + // A runtime the format names and this crate has no adapter for. The consumer must refuse + // the box rather than read it as the runtime it happens to be shaped like. + "alter-release-runtime-id" => { + fixture.release["runtime"]["id"] = json!("native"); fixture.sign(); } "alter-release-execution" => { @@ -647,7 +693,7 @@ fn mutate_fixture(fixture: &mut Fixture, mutation: &str, destination: &Path) { "create-destination" => std::fs::create_dir_all(destination).unwrap(), "remove-interpreter" | "remove-script" | "remove-module" => { let removed = match mutation { - "remove-interpreter" => fixture.release["pythonEntryPoint"] + "remove-interpreter" => fixture.release["runtime"]["entryPoint"] .as_str() .unwrap() .to_string(), @@ -670,7 +716,7 @@ fn mutate_fixture(fixture: &mut Fixture, mutation: &str, destination: &Path) { // shape — `venv/bin/python` is a link to the versioned binary beside it — so a consumer that // only accepts regular files here rejects every box the builder produces on macOS and Linux. "link-interpreter" => { - let entry_point = fixture.release["pythonEntryPoint"] + let entry_point = fixture.release["runtime"]["entryPoint"] .as_str() .unwrap() .to_string(); @@ -773,7 +819,7 @@ fn mutate_extracted_root(fixture: &Fixture, mutation: &str, root: &Path) -> Path std::fs::write(&script, bytes).unwrap(); } "remove-interpreter" => { - std::fs::remove_file(root.join(fixture.release["pythonEntryPoint"].as_str().unwrap())) + std::fs::remove_file(root.join(fixture.release["runtime"]["entryPoint"].as_str().unwrap())) .unwrap(); } "remove-script" => std::fs::remove_file(&script).unwrap(), @@ -782,7 +828,7 @@ fn mutate_extracted_root(fixture: &Fixture, mutation: &str, root: &Path) -> Path #[cfg(unix)] { let interpreter = - root.join(fixture.release["pythonEntryPoint"].as_str().unwrap()); + root.join(fixture.release["runtime"]["entryPoint"].as_str().unwrap()); let target = std::fs::read_link(&interpreter).unwrap(); std::fs::remove_file(&interpreter).unwrap(); std::os::unix::fs::symlink( @@ -1191,7 +1237,7 @@ fn the_shared_consumer_conformance_suite_passes() { let suite: Value = serde_json::from_str(SUITE).unwrap(); let patterns = suite["errorPatterns"].as_object().unwrap(); let cases = suite["cases"].as_array().unwrap(); - assert_eq!(cases.len(), 81, "the suite changed size"); + assert_eq!(cases.len(), 84, "the suite changed size"); let mut failures: Vec = Vec::new(); let mut ran = 0usize; diff --git a/rust/tests/contract.rs b/rust/tests/contract.rs index b40ac94..d2a818a 100644 --- a/rust/tests/contract.rs +++ b/rust/tests/contract.rs @@ -13,7 +13,8 @@ use scrollcase_consumer::contract::payload_digest::{ payload_digest_stream, PayloadDigestEntry, PayloadDigestKind, }; use scrollcase_consumer::contract::runtimes::{ - runtime_adapter, runtime_adapters, RuntimeArgument, RuntimeExecution, SelfTestProbe, + is_implemented_runtime, runtime_adapter, runtime_adapters, RuntimeArgument, RuntimeExecution, + SelfTestCommand, SelfTestProbe, RUNTIME_IDS, }; use scrollcase_consumer::contract::targets::{box_target_id, BoxTarget}; @@ -77,6 +78,7 @@ fn matches_the_shared_target_id_contract() { #[derive(Deserialize)] #[serde(rename_all = "camelCase")] struct RuntimeContract { + runtime_ids: Vec, runtimes: Vec, executable_matches: Vec, execution_discovery: Vec, @@ -156,16 +158,37 @@ struct SelfTestCase { runtime: String, platform: String, probe: ProbeFields, - args: Vec, + #[serde(default)] + execution: Option, + invocations: Vec, } #[derive(Deserialize)] +#[serde(rename_all = "camelCase")] struct ProbeFields { + #[serde(default)] imports: Vec, #[serde(default)] + commands: Vec, + #[serde(default)] code: Option, } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct CommandFields { + args: Vec, + expect_exit_code: u8, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct InvocationFields { + command: ArgumentFields, + args: Vec, + expect_exit_code: u8, +} + /// The execution declaration as the fixture spells it, so the vectors are read exactly as the other /// implementations read them rather than through this crate's own release model. #[derive(Deserialize)] @@ -176,13 +199,15 @@ struct ExecutionFields { script: Option, #[serde(default)] module: Option, + #[serde(default)] + binary: Option, default_args: Vec, } impl ExecutionFields { fn as_runtime(&self) -> RuntimeExecution<'_> { match self.kind.as_str() { - "python-script" => RuntimeExecution::Script { + "python-script" | "node-script" => RuntimeExecution::Script { script: self.script.as_deref().expect("a script case declares a script"), default_args: &self.default_args, }, @@ -190,6 +215,10 @@ impl ExecutionFields { module: self.module.as_deref().expect("a module case declares a module"), default_args: &self.default_args, }, + "native-binary" => RuntimeExecution::Binary { + binary: self.binary.as_deref().expect("a binary case declares a binary"), + default_args: &self.default_args, + }, other => panic!("the fixture declares an execution kind this mirror has no shape for: {other}"), } } @@ -220,6 +249,16 @@ impl ArgumentFields { fn matches_the_shared_runtime_contract() { let contract: RuntimeContract = serde_json::from_str(RUNTIME_CONTRACT).unwrap(); + // Two lists on purpose: the vocabulary the format defines, and what this crate can run. + let ids: Vec<&str> = contract.runtime_ids.iter().map(String::as_str).collect(); + assert_eq!(RUNTIME_IDS, ids.as_slice()); + for id in RUNTIME_IDS { + assert_eq!( + is_implemented_runtime(id), + runtime_adapters().iter().any(|runtime| runtime.id == *id), + "{id}" + ); + } let mirrored: Vec<&str> = runtime_adapters().iter().map(|runtime| runtime.id).collect(); let declared: Vec<&str> = contract.runtimes.iter().map(|case| case.id.as_str()).collect(); assert_eq!(mirrored, declared); @@ -289,6 +328,7 @@ fn matches_the_shared_runtime_contract() { kind: "python-module".to_string(), script: None, module: Some("pkg".to_string()), + binary: None, default_args: vec![], }; for invalid in &contract.invalid_runtime_versions { @@ -312,18 +352,46 @@ fn matches_the_shared_runtime_contract() { } } + assert_self_test_invocations(&contract); +} + + +/// The self-test half, lifted out of the main vector test: a probe may now imply several commands, +/// each with its own required exit status, and asserting that inline made one function long enough +/// for clippy to object. +fn assert_self_test_invocations(contract: &RuntimeContract) { for case in &contract.self_test { - let argv = runtime_adapter(&case.runtime) + let commands: Vec> = case + .probe + .commands + .iter() + .map(|command| SelfTestCommand { + args: &command.args, + expect_exit_code: command.expect_exit_code, + }) + .collect(); + let execution = case.execution.as_ref().map(ExecutionFields::as_runtime); + let invocations = runtime_adapter(&case.runtime) .unwrap() - .self_test_argv( + .self_test_invocations( &SelfTestProbe { imports: &case.probe.imports, + commands: &commands, code: case.probe.code.as_deref(), }, + execution.as_ref(), &case.platform, ) - .unwrap(); - assert_eq!(argv, case.args, "{}", case.name); + .unwrap_or_else(|error| panic!("{} was refused: {error}", case.name)); + assert_eq!(invocations.len(), case.invocations.len(), "{}", case.name); + for (expected, produced) in case.invocations.iter().zip(invocations.iter()) { + assert!(expected.command.matches(&produced.command), "{}", case.name); + assert_eq!(produced.expect_exit_code, expected.expect_exit_code, "{}", case.name); + assert_eq!(produced.args.len(), expected.args.len(), "{}", case.name); + for (want, got) in expected.args.iter().zip(produced.args.iter()) { + assert!(want.matches(got), "{}", case.name); + } + } } } diff --git a/rust/tests/fixtures/signed-release.json b/rust/tests/fixtures/signed-release.json index cf92733..e569f9f 100644 --- a/rust/tests/fixtures/signed-release.json +++ b/rust/tests/fixtures/signed-release.json @@ -1,13 +1,13 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "payloadEncoding": "base64-json-utf8", - "payloadBase64": "ewogICJzY2hlbWFWZXJzaW9uIjogMiwKICAia2luZCI6ICJzY3JvbGxjYXNlLmJveC5yZWxlYXNlIiwKICAiYm94SWQiOiAiZXhhbXBsZS1tb2RlbCIsCiAgIm1vZGVsSWQiOiAiZXhhbXBsZS1vcmctZXhhbXBsZS1tb2RlbCIsCiAgInJ1bnRpbWVJZCI6ICJleGFtcGxlLW1vZGVsLXJ1bnRpbWUiLAogICJ2ZXJzaW9uIjogIjEuMC4wIiwKICAidGFyZ2V0IjogewogICAgInBsYXRmb3JtIjogIm1hY29zIiwKICAgICJhcmNoIjogImFhcmNoNjQiLAogICAgImFjY2VsZXJhdG9yIjogIm1ldGFsIgogIH0sCiAgImNvbXBhdGliaWxpdHkiOiB7CiAgICAibWluSG9zdEFwcFZlcnNpb24iOiAiMS4wLjAiLAogICAgIm1pbk1hY29zVmVyc2lvbiI6ICIxMy4wIiwKICAgICJtaW5SYW1HYiI6IDgKICB9LAogICJhcmNoaXZlIjogewogICAgImZvcm1hdCI6ICJ6aXAiLAogICAgInVybCI6ICJodHRwczovL2Fzc2V0cy5leGFtcGxlLm9yZy9ib3hlcy9ib3hlcy9leGFtcGxlLW1vZGVsLzEuMC4wL21hY29zLWFhcmNoNjQtbWV0YWwvN2QyYzlhNDFlOGIzNTBmNmMxNzRhOWRlMjAzNThiZjQxYzZlOTdkMDVhOGIzZjI2MTllNGM3MDgxZGE1YjNmMi56aXAiLAogICAgInNoYTI1NiI6ICI3ZDJjOWE0MWU4YjM1MGY2YzE3NGE5ZGUyMDM1OGJmNDFjNmU5N2QwNWE4YjNmMjYxOWU0YzcwODFkYTViM2YyIiwKICAgICJzaXplQnl0ZXMiOiA2NTU3NTIyMTYKICB9LAogICJpbnN0YWxsZWRTaXplQnl0ZXMiOiAxODkyMzQwMTEyLAogICJweXRob25FbnRyeVBvaW50IjogInZlbnYvYmluL3B5dGhvbiIsCiAgIm1vZGVsQ2FjaGVTdWJkaXIiOiAibW9kZWwtY2FjaGUvZXhhbXBsZS1tb2RlbCIsCiAgInNlbGZUZXN0IjogewogICAgInB5dGhvbkltcG9ydHMiOiBbCiAgICAgICJ0b3JjaCIsCiAgICAgICJudW1weSIKICAgIF0sCiAgICAidGltZW91dFNlY29uZHMiOiAxODAKICB9LAogICJwcm92ZW5hbmNlIjogewogICAgInNjcm9sbElkIjogImV4YW1wbGUtbW9kZWwtbWFjb3MtYXJtNjQtbWV0YWwiLAogICAgInNjcm9sbFZlcnNpb24iOiAiMS4wLjAiLAogICAgImJ1aWxkZXJSZXZpc2lvbiI6ICI0YzFkOWI3ZTJhMGY1ODM2ZDQxOWU3YzA1YjNhOGY2MWQyNzA0ZTkzIiwKICAgICJzb3VyY2VUcmVlRGlydHkiOiBmYWxzZSwKICAgICJzb3VyY2VSZXZpc2lvbiI6ICI5ZjBkM2ExYzRiNWU2ZjcwODE5MjBhM2I0YzVkNmU3ZjgwOTEwYTJiIiwKICAgICJweXRob25WZXJzaW9uIjogIjMuMTEuMTUiLAogICAgInBpeGlWZXJzaW9uIjogIjAuNzMuMCIsCiAgICAiZGVwZW5kZW5jeUxvY2tTaGEyNTYiOiAiM2IxZjhjNDdhMmQ5ZTA1YjZjNzQxOGFmMjNkNWU2OTAxN2I0YzhhZDkxZTJmMzUwNzY4YmQ0Y2ExOWUwZjViNyIsCiAgICAiYnVpbHRBdCI6ICIyMDI2LTA3LTI1VDEyOjAwOjAwKzAwOjAwIgogIH0KfQo=", - "payloadSha256": "bbf60de7d31035b2bfcb98c6c57624220f3bf59900391189f5e55da955055bc6", + "payloadBase64": "ewogICJzY2hlbWFWZXJzaW9uIjogMywKICAia2luZCI6ICJzY3JvbGxjYXNlLmJveC5yZWxlYXNlIiwKICAiYm94SWQiOiAiZXhhbXBsZS1tb2RlbCIsCiAgImxhYmVscyI6IHsKICAgICJtb2RlbCI6ICJleGFtcGxlLW9yZy9leGFtcGxlLW1vZGVsIiwKICAgICJvd25lciI6ICJwbGF0Zm9ybS10ZWFtIgogIH0sCiAgInZlcnNpb24iOiAiMS4wLjAiLAogICJ0YXJnZXQiOiB7CiAgICAicGxhdGZvcm0iOiAibWFjb3MiLAogICAgImFyY2giOiAiYWFyY2g2NCIsCiAgICAiYWNjZWxlcmF0b3IiOiAibWV0YWwiCiAgfSwKICAiY29tcGF0aWJpbGl0eSI6IHsKICAgICJtaW5Ib3N0QXBwVmVyc2lvbiI6ICIxLjAuMCIsCiAgICAibWluTWFjb3NWZXJzaW9uIjogIjEzLjAiLAogICAgIm1pblJhbUdiIjogOAogIH0sCiAgImFyY2hpdmUiOiB7CiAgICAiZm9ybWF0IjogInppcCIsCiAgICAidXJsIjogImh0dHBzOi8vYXNzZXRzLmV4YW1wbGUub3JnL2JveGVzL2JveGVzL2V4YW1wbGUtbW9kZWwvMS4wLjAvbWFjb3MtYWFyY2g2NC1tZXRhbC83ZDJjOWE0MWU4YjM1MGY2YzE3NGE5ZGUyMDM1OGJmNDFjNmU5N2QwNWE4YjNmMjYxOWU0YzcwODFkYTViM2YyLnppcCIsCiAgICAic2hhMjU2IjogIjdkMmM5YTQxZThiMzUwZjZjMTc0YTlkZTIwMzU4YmY0MWM2ZTk3ZDA1YThiM2YyNjE5ZTRjNzA4MWRhNWIzZjIiLAogICAgInNpemVCeXRlcyI6IDY1NTc1MjIxNgogIH0sCiAgImluc3RhbGxlZFNpemVCeXRlcyI6IDE4OTIzNDAxMTIsCiAgInJ1bnRpbWUiOiB7CiAgICAiaWQiOiAicHl0aG9uIiwKICAgICJ2ZXJzaW9uIjogIjMuMTEuMTUiLAogICAgImVudHJ5UG9pbnQiOiAidmVudi9iaW4vcHl0aG9uIgogIH0sCiAgImNhY2hlU3ViZGlyIjogImNhY2hlL2V4YW1wbGUtbW9kZWwiLAogICJzZWxmVGVzdCI6IHsKICAgICJwcm9iZSI6IHsKICAgICAgImltcG9ydHMiOiBbCiAgICAgICAgInRvcmNoIiwKICAgICAgICAibnVtcHkiCiAgICAgIF0KICAgIH0sCiAgICAidGltZW91dFNlY29uZHMiOiAxODAKICB9LAogICJwcm92ZW5hbmNlIjogewogICAgInNjcm9sbElkIjogImV4YW1wbGUtbW9kZWwtbWFjb3MtYXJtNjQtbWV0YWwiLAogICAgInNjcm9sbFZlcnNpb24iOiAiMS4wLjAiLAogICAgImJ1aWxkZXJSZXZpc2lvbiI6ICI0YzFkOWI3ZTJhMGY1ODM2ZDQxOWU3YzA1YjNhOGY2MWQyNzA0ZTkzIiwKICAgICJzb3VyY2VUcmVlRGlydHkiOiBmYWxzZSwKICAgICJzb3VyY2VSZXZpc2lvbiI6ICI5ZjBkM2ExYzRiNWU2ZjcwODE5MjBhM2I0YzVkNmU3ZjgwOTEwYTJiIiwKICAgICJydW50aW1lVmVyc2lvbiI6ICIzLjExLjE1IiwKICAgICJwaXhpVmVyc2lvbiI6ICIwLjczLjAiLAogICAgImRlcGVuZGVuY3lMb2NrU2hhMjU2IjogIjNiMWY4YzQ3YTJkOWUwNWI2Yzc0MThhZjIzZDVlNjkwMTdiNGM4YWQ5MWUyZjM1MDc2OGJkNGNhMTllMGY1YjciLAogICAgImJ1aWx0QXQiOiAiMjAyNi0wNy0yNVQxMjowMDowMCswMDowMCIKICB9Cn0K", + "payloadSha256": "f264818314ac170618c9472dc6cc7a72a09eb6390a237fe00dcbe44507d1aef1", "signatures": [ { "algorithm": "ed25519", - "keyId": "scrollcase-71146297b2a0d761", - "signatureBase64": "xI4YUxTUce49/jjKnzLFisLqsHlEtFFYJM1h8lZ24lwEE/vuDGVY/788e62O/2rHsQp6vAr9WHflR49G621BBQ==" + "keyId": "scrollcase-c6ec10141a73801c", + "signatureBase64": "g8hInToVHp1Cit7UiWOncHL9n1ThPBib06Z3GtucFHMZqTlQTeJpNwMn4i2pDL5JIjRVo4SVt2RRAUfnerquCQ==" } ] } diff --git a/rust/tests/fixtures/trusted-key.json b/rust/tests/fixtures/trusted-key.json index 397e6ab..3c3b3aa 100644 --- a/rust/tests/fixtures/trusted-key.json +++ b/rust/tests/fixtures/trusted-key.json @@ -1,6 +1,6 @@ { "algorithm": "ed25519", - "keyId": "scrollcase-71146297b2a0d761", - "publicKeyBase64": "Fom+OnGSoGV6kdg6WVD2OrrhLUNuCUZ6A9GhKbcmSXs=", - "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMCowBQYDK2VwAyEAFom+OnGSoGV6kdg6WVD2OrrhLUNuCUZ6A9GhKbcmSXs=\n-----END PUBLIC KEY-----\n" + "keyId": "scrollcase-c6ec10141a73801c", + "publicKeyBase64": "EjNpkXCi9bVe9VQgJrZrgHvQzwNiMPZZbyYwyAtEVzQ=", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMCowBQYDK2VwAyEAEjNpkXCi9bVe9VQgJrZrgHvQzwNiMPZZbyYwyAtEVzQ=\n-----END PUBLIC KEY-----\n" } diff --git a/rust/tests/prepare.rs b/rust/tests/prepare.rs index ef25a12..cc0b56f 100644 --- a/rust/tests/prepare.rs +++ b/rust/tests/prepare.rs @@ -49,7 +49,7 @@ fn preparing_a_box_yields_a_receipt_describing_what_was_verified() { assert_eq!(prepared.status(), PreparedStatus::Prepared); assert_eq!(prepared.root(), destination); assert_eq!(prepared.box_id(), "fixture-box"); - assert_eq!(prepared.model_id(), "fixture-model"); + assert_eq!(prepared.labels().unwrap()["model"], "fixture-model"); assert_eq!(prepared.signing_key_ids(), [support::KEY_ID]); assert!(prepared.required_assets().is_empty()); assert!(prepared.execution().is_some()); @@ -249,15 +249,15 @@ fn attaching_refuses_a_link_standing_in_for_the_box_root() { } #[test] -fn an_on_demand_asset_must_match_its_signed_descriptor_before_a_receipt_exists() { - const ASSET: &[u8] = b"trusted on-demand bytes"; +fn a_deferred_asset_must_match_its_signed_descriptor_before_a_receipt_exists() { + const ASSET: &[u8] = b"trusted deferred bytes"; - // The asset policy has to appear in both documents, or `box.json` would disagree with the + // The descriptor list has to appear in both documents, or `box.json` would disagree with the // release before the assets themselves are ever looked at. let policy = || { json!([{ - "url": "https://example.invalid/weights.bin", - "relativePath": "weights/model.bin", + "url": "https://example.invalid/data.bin", + "relativePath": "cache/model.bin", "sizeBytes": ASSET.len(), "sha256": support::sha256_hex(ASSET), }]) @@ -265,12 +265,10 @@ fn an_on_demand_asset_must_match_its_signed_descriptor_before_a_receipt_exists() let fixture = support::build( "attach-assets", |manifest| { - manifest["weights"] = json!("on-demand"); manifest["assets"] = policy(); }, |_| {}, |release| { - release["weights"] = json!("on-demand"); release["assets"] = policy(); }, ); @@ -288,22 +286,22 @@ fn an_on_demand_asset_must_match_its_signed_descriptor_before_a_receipt_exists() assert!(error.message().contains("asset is missing"), "{error}"); // Placed, but truncated. - std::fs::create_dir_all(destination.join("weights")).unwrap(); - std::fs::write(destination.join("weights/model.bin"), b"short").unwrap(); + std::fs::create_dir_all(destination.join("cache")).unwrap(); + std::fs::write(destination.join("cache/model.bin"), b"short").unwrap(); let error = attach_extracted_box(&fixture.release_path, &attach_options(&fixture, &destination)) .unwrap_err(); assert!(error.message().contains("asset size mismatch"), "{error}"); // Right size, wrong bytes — the case a size check alone waves through. - std::fs::write(destination.join("weights/model.bin"), vec![b'x'; ASSET.len()]).unwrap(); + std::fs::write(destination.join("cache/model.bin"), vec![b'x'; ASSET.len()]).unwrap(); let error = attach_extracted_box(&fixture.release_path, &attach_options(&fixture, &destination)) .unwrap_err(); assert!(error.message().contains("asset SHA-256 mismatch"), "{error}"); // The real bytes. - std::fs::write(destination.join("weights/model.bin"), ASSET).unwrap(); + std::fs::write(destination.join("cache/model.bin"), ASSET).unwrap(); let attached = attach_extracted_box(&fixture.release_path, &attach_options(&fixture, &destination)) .expect("the signed asset must be accepted"); diff --git a/rust/tests/release_document.rs b/rust/tests/release_document.rs index 51cb99f..2e3ba70 100644 --- a/rust/tests/release_document.rs +++ b/rust/tests/release_document.rs @@ -7,7 +7,7 @@ use std::path::{Path, PathBuf}; -use scrollcase_consumer::contract::runtimes::{runtime_adapter, IMPLICIT_RUNTIME_ID}; +use scrollcase_consumer::contract::runtimes::runtime_adapter; use scrollcase_consumer::trust::TrustAnchors; use scrollcase_consumer::verify::inspect_release_document; @@ -46,12 +46,12 @@ fn a_genuine_signed_release_is_accepted_and_fully_interpreted() { inspect_release_document(&fixture("signed-release.json"), TrustAnchors::KeyFile(&fixture("trusted-key.json"))) .expect("the fixture release must verify"); - assert_eq!(inspected.release.schema_version, 2); + assert_eq!(inspected.release.schema_version, 3); assert!(inspected.release.kind.ends_with(".release")); // The adapter is resolved from the signed target, and the entry point agreed with it. assert_eq!( - inspected.release.python_entry_point, - runtime_adapter(IMPLICIT_RUNTIME_ID) + inspected.release.runtime.entry_point.as_deref().unwrap(), + runtime_adapter(&inspected.release.runtime.id) .unwrap() .layout(inspected.adapter.platform) .unwrap() diff --git a/rust/tests/run.rs b/rust/tests/run.rs index 5b89218..0d54f77 100644 --- a/rust/tests/run.rs +++ b/rust/tests/run.rs @@ -219,7 +219,7 @@ fn a_missing_interpreter_is_refused_rather_than_spawned() { let fixture = support::valid("run-no-interpreter"); let destination = fixture.directory.join("installed"); let prepared = prepare(&fixture, &destination); - std::fs::remove_file(destination.join(support::native_python_entry_point())).unwrap(); + std::fs::remove_file(destination.join(support::native_entry_point())).unwrap(); let error = run_extracted_box(&prepared, &quiet()).unwrap_err(); assert!(error.message().contains("Prepared box is missing venv/"), "{error}"); diff --git a/rust/tests/schema.rs b/rust/tests/schema.rs index b4a3ea0..19cb038 100644 --- a/rust/tests/schema.rs +++ b/rust/tests/schema.rs @@ -5,7 +5,7 @@ //! actually agree. So this suite runs both over the same documents and asserts they reach the same //! verdict, on the examples and on a battery of mutations chosen to poke at the places where a typed //! parse and a schema most plausibly drift apart: an unknown field, a missing required field, a -//! pattern violation, a broken bound, and the `weights`/`assets` co-requirement. +//! pattern violation, a broken bound, and a probe that states nothing. //! //! Agreement is checked in both directions. Drifting *stricter* than the schema is as much a //! divergence as drifting looser, and it is the direction a typed parse drifts by default: the last @@ -108,7 +108,7 @@ fn the_types_and_the_schema_agree_on_every_mutation() { ( "a schema version from another format revision", Box::new(|value: &mut Value| { - value["schemaVersion"] = json!(3); + value["schemaVersion"] = json!(4); }), ), ( @@ -156,7 +156,7 @@ fn the_types_and_the_schema_agree_on_every_mutation() { ( "an empty self-test import list", Box::new(|value: &mut Value| { - value["selfTest"]["pythonImports"] = json!([]); + value["selfTest"]["probe"]["imports"] = json!([]); }), ), ( @@ -166,24 +166,34 @@ fn the_types_and_the_schema_agree_on_every_mutation() { }), ), ( - "weights without assets", + "an empty deferred-asset list", Box::new(|value: &mut Value| { - value["weights"] = json!("on-demand"); - value.as_object_mut().unwrap().remove("assets"); + value["assets"] = json!([]); }), ), ( - "assets without weights", + "a deferred asset with no digest", Box::new(|value: &mut Value| { - value.as_object_mut().unwrap().remove("weights"); value["assets"] = json!([{ "url": "https://example.invalid/w.bin", - "relativePath": "weights/w.bin", + "relativePath": "cache/w.bin", "sizeBytes": 1, - "sha256": "a".repeat(64), + "sha256": "not-a-digest", }]); }), ), + ( + "a probe that proves nothing", + Box::new(|value: &mut Value| { + value["selfTest"]["probe"] = json!({}); + }), + ), + ( + "a runtime the format does not define", + Box::new(|value: &mut Value| { + value["runtime"]["id"] = json!("ruby"); + }), + ), ( "an execution kind the format does not define", Box::new(|value: &mut Value| { diff --git a/rust/tests/support/mod.rs b/rust/tests/support/mod.rs index 49a52ba..cf66e2c 100644 --- a/rust/tests/support/mod.rs +++ b/rust/tests/support/mod.rs @@ -51,7 +51,7 @@ pub fn native_target() -> Value { } /// The interpreter path the native target's adapter fixes. -pub fn native_python_entry_point() -> &'static str { +pub fn native_entry_point() -> &'static str { if std::env::consts::OS == "windows" { "venv/python.exe" } else { @@ -103,15 +103,18 @@ fn scratch(name: &str) -> PathBuf { /// The `box.json` of a valid linux-x86_64-cpu box. pub fn box_manifest() -> Value { json!({ - "schemaVersion": 2, + "schemaVersion": 3, "boxId": "fixture-box", - "modelId": "fixture-model", - "runtimeId": "fixture-runtime", + "labels": { "model": "fixture-model" }, "version": "1.0.0", "target": native_target(), - "pythonEntryPoint": native_python_entry_point(), - "modelCacheSubdir": "model-cache/fixture", - "selfTest": { "pythonImports": ["json"], "timeoutSeconds": 30 }, + "runtime": { + "id": "python", + "version": "3.11.9", + "entryPoint": native_entry_point() + }, + "cacheSubdir": "cache/fixture", + "selfTest": { "probe": { "imports": ["json"] }, "timeoutSeconds": 30 }, "execution": { "kind": "python-script", "script": "app/main.py", "defaultArgs": [] }, "provenance": { "scrollId": "fixture-box", @@ -119,7 +122,7 @@ pub fn box_manifest() -> Value { "builderRevision": "b".repeat(40), "sourceTreeDirty": false, "sourceRevision": "c".repeat(40), - "pythonVersion": "3.11.9", + "runtimeVersion": "3.11.9", "dependencyLockSha256": "d".repeat(64), "builtAt": "2026-01-01T00:00:00.000Z", "pixiVersion": "0.50.0" @@ -146,7 +149,7 @@ pub fn default_entries(manifest: &Value) -> Vec { serde_json::to_vec_pretty(manifest).unwrap(), 0o644, ), - Entry::File(native_python_entry_point(), FIXTURE_INTERPRETER.to_vec(), 0o755), + Entry::File(native_entry_point(), FIXTURE_INTERPRETER.to_vec(), 0o755), Entry::File("app/main.py", b"print('fixture')\n".to_vec(), 0o644), ] } @@ -202,7 +205,7 @@ pub fn sign(payload: &Value) -> Value { let bytes = serde_json::to_vec(payload).unwrap(); let key = SigningKey::from_bytes(&SIGNING_SEED); json!({ - "schemaVersion": 2, + "schemaVersion": 3, "payloadEncoding": "base64-json-utf8", "payloadBase64": BASE64.encode(&bytes), "payloadSha256": sha256_hex(&bytes), diff --git a/src/contract/fixtures/consumer-conformance.json b/src/contract/fixtures/consumer-conformance.json index ed34cd4..8d81d99 100644 --- a/src/contract/fixtures/consumer-conformance.json +++ b/src/contract/fixtures/consumer-conformance.json @@ -7,7 +7,6 @@ "archive-size": "Archive size mismatch", "asset-hash": "asset SHA-256 mismatch", "asset-missing": "asset is missing", - "asset-not-executable": "is not executable", "asset-size": "asset size mismatch", "attach-missing-interpreter": "Attached box is missing venv/", "attach-root": "is not an extracted box directory", @@ -33,7 +32,7 @@ "special-entry": "special entries", "unimplemented-runtime": "is not implemented by this version", "unsafe-path": "Unsafe relative path|invalid relative path|absolute path", - "unsupported-schema-version": "Unsupported schemaVersion [12]" + "unsupported-schema-version": "Unsupported schemaVersion 1|Unsupported schemaVersion 2" }, "cases": [ { From 16a1da6f3fe2e6f526dd2bf2774334bddb406abb Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:42:04 +0200 Subject: [PATCH 07/22] Bring the Python consumer to version 3 The models carry a BoxRuntime block and a labels map instead of modelId, runtimeId and pythonEntryPoint, RequiredAsset gains the executable declaration a materializer needs, and BoxExecution grows the two shapes the format now names but no adapter answers for yet. The probe becomes SelfTestProbe with imports and commands, and self_test_argv becomes self_test_invocations: a probe may imply several commands, each with its own required exit status. resolve_environment now takes the runtime id rather than assuming one, which is the last place the package inferred Python from the shape of a box. --- python/src/scrollcase_consumer/__init__.py | 6 + python/src/scrollcase_consumer/_contract.py | 200 +++++++++++++++--- python/src/scrollcase_consumer/environment.py | 7 +- python/src/scrollcase_consumer/models.py | 53 ++++- python/src/scrollcase_consumer/run.py | 36 ++-- .../schemas/box-manifest.schema.json | 96 ++------- .../schemas/execution.schema.json | 52 ++++- .../schemas/release-manifest.schema.json | 176 ++++++++++----- .../schemas/signed-document.schema.json | 4 +- .../schemas/target.schema.json | 2 +- python/src/scrollcase_consumer/verify.py | 96 +++++---- python/tests/conformance_support.py | 72 ++++++- python/tests/support.py | 30 ++- python/tests/test_contract.py | 85 ++++++-- python/tests/test_run.py | 6 +- python/tests/test_verify.py | 8 +- 16 files changed, 661 insertions(+), 268 deletions(-) diff --git a/python/src/scrollcase_consumer/__init__.py b/python/src/scrollcase_consumer/__init__.py index a11d882..a82e1f7 100644 --- a/python/src/scrollcase_consumer/__init__.py +++ b/python/src/scrollcase_consumer/__init__.py @@ -4,11 +4,14 @@ from .models import ( BoxExecution, BoxRunResult, + BoxRuntime, BoxTarget, EnvironmentReport, EnvironmentSource, EnvironmentSourceValue, EnvironmentVariableReport, + NativeBinaryExecution, + NodeScriptExecution, PayloadVerification, PreparedBox, PythonModuleExecution, @@ -26,11 +29,14 @@ __all__ = [ "BoxExecution", "BoxRunResult", + "BoxRuntime", "BoxTarget", "EnvironmentReport", "EnvironmentSource", "EnvironmentSourceValue", "EnvironmentVariableReport", + "NativeBinaryExecution", + "NodeScriptExecution", "PayloadVerification", "PreparedBox", "PythonModuleExecution", diff --git a/python/src/scrollcase_consumer/_contract.py b/python/src/scrollcase_consumer/_contract.py index 6df7e93..68cce1a 100644 --- a/python/src/scrollcase_consumer/_contract.py +++ b/python/src/scrollcase_consumer/_contract.py @@ -24,7 +24,10 @@ from .errors import ScrollcaseConsumerError from .models import ( BoxExecution, + BoxRuntime, BoxTarget, + NativeBinaryExecution, + NodeScriptExecution, PythonModuleExecution, PythonScriptExecution, RequiredAsset, @@ -164,6 +167,37 @@ class ResolvedExecutionFiles: missing: str +@dataclass(frozen=True, slots=True) +class SelfTestCommand: + """One invocation of the box's declared execution, and the status it must exit with.""" + + args: tuple[str, ...] + expect_exit_code: int = 0 + + +@dataclass(frozen=True, slots=True) +class SelfTestProbe: + """What a self-test asks the box to prove, plus the builder-only extension a scroll may add. + + ``imports`` asks the runtime's loader a question and only means something to a runtime that has + one. ``commands`` asks the box's declared execution a question, which every runtime can answer + and a native one can answer *only* that way. ``code`` never travels on the wire. + """ + + imports: tuple[str, ...] = () + commands: tuple[SelfTestCommand, ...] = () + code: str | None = None + + +@dataclass(frozen=True, slots=True) +class SelfTestInvocation: + """One command a self-test runs, and the status it must exit with.""" + + command: RuntimeArgument + args: tuple[RuntimeArgument, ...] + expect_exit_code: int + + @dataclass(frozen=True, slots=True) class RuntimeAdapter: """What a runtime implies for a box, independent of the machine it runs on. @@ -211,13 +245,20 @@ def resolve_execution_files( f"Unsupported execution kind: {execution.kind}." ) layout = self.layout(platform) - if isinstance(execution, PythonScriptExecution): + if isinstance(execution, (PythonScriptExecution, NodeScriptExecution)): return ResolvedExecutionFiles( candidates=(execution.script,), missing=( f"Execution script is missing from the box: {execution.script}." ), ) + if isinstance(execution, NativeBinaryExecution): + return ResolvedExecutionFiles( + candidates=(execution.binary,), + missing=( + f"Execution binary is missing from the box: {execution.binary}." + ), + ) module_path = execution.module.replace(".", "/") relative = (f"{module_path}.py", f"{module_path}/__main__.py") # Windows names its standard library once, with no interpreter version in the path; every @@ -247,8 +288,10 @@ def build_argv(self, execution: BoxExecution, platform: str) -> RuntimeInvocatio f"Unsupported execution kind: {execution.kind}." ) layout = self.layout(platform) - if isinstance(execution, PythonScriptExecution): + if isinstance(execution, (PythonScriptExecution, NodeScriptExecution)): args = [RuntimeArgument("payload-path", execution.script)] + elif isinstance(execution, NativeBinaryExecution): + args = [RuntimeArgument("payload-path", execution.binary)] else: args = [ RuntimeArgument("literal", "-m"), @@ -262,19 +305,62 @@ def build_argv(self, execution: BoxExecution, platform: str) -> RuntimeInvocatio args=tuple(args), ) - def self_test_argv( - self, imports: Iterable[str], platform: str, code: str | None = None - ) -> tuple[str, ...]: - """The arguments that follow this runtime's entry point when it runs a self-test probe.""" - - assertion = self._platform_assertions.get(platform) - if assertion is None: - raise ScrollcaseConsumerError( - f"No {self.id} self-test assertion exists for platform {platform}" + def self_test_invocations( + self, + probe: SelfTestProbe, + execution: BoxExecution | None, + platform: str, + ) -> tuple[SelfTestInvocation, ...]: + """Every command a self-test probe implies, in declaration order.""" + + invocations: list[SelfTestInvocation] = [] + if probe.imports: + assertion = self._platform_assertions.get(platform) + if assertion is None: + raise ScrollcaseConsumerError( + f"No {self.id} self-test assertion exists for platform {platform}" + ) + body = f"import {', '.join(probe.imports)}" + source = ( + f"{assertion}\n{body}\n{probe.code}" + if probe.code + else f"{assertion}\n{body}" + ) + invocations.append( + SelfTestInvocation( + command=RuntimeArgument( + "payload-path", self.layout(platform).entry_point + ), + args=( + RuntimeArgument("literal", "-c"), + RuntimeArgument("literal", source), + ), + expect_exit_code=0, + ) + ) + for command in probe.commands: + # A command probe appends arguments to the box's own declared execution. With none + # declared there is nothing to append them to, which is a contradiction in the + # declaration rather than a property of the box. + if execution is None: + raise ScrollcaseConsumerError( + "A self-test command needs a declared execution to invoke" + ) + invocation = self.build_argv(execution, platform) + invocations.append( + SelfTestInvocation( + command=invocation.command, + args=( + *invocation.args, + *( + RuntimeArgument("literal", value) + for value in command.args + ), + ), + expect_exit_code=command.expect_exit_code, + ) ) - body = f"import {', '.join(imports)}" - source = f"{assertion}\n{body}\n{code}" if code else f"{assertion}\n{body}" - return ("-c", source) + return tuple(invocations) _POSIX_PYTHON_LAYOUT = RuntimeLayout( @@ -318,15 +404,37 @@ def self_test_argv( ) } -#: The runtime every box built by this schema version implicitly declares. +#: Every runtime id the box format admits, in the order the schema lists them. #: -#: The wire format has no runtime field: a box records a Python entry point and Python execution -#: kinds and nothing that says "Python". So a reader that must name a runtime names this one, from -#: one place. -IMPLICIT_RUNTIME_ID = "python" +#: The wire enum and the implemented set are deliberately two different things: schema version 3 +#: fixes the vocabulary once, so a later release can implement ``node`` without another wire break. +#: A box naming a runtime this package has no adapter for is refused by name, not misread. +RUNTIME_IDS: tuple[str, ...] = ("python", "node", "native") -def runtime_adapter(runtime_id: str = IMPLICIT_RUNTIME_ID) -> RuntimeAdapter: +def is_implemented_runtime(runtime_id: str) -> bool: + """Whether this build carries an adapter — the question to ask before ``runtime_adapter``.""" + + return runtime_id in _RUNTIMES + + +def unimplemented_runtime_message(runtime_id: str) -> str: + """The message for a box declaring a runtime this build has no adapter for. + + The wire vocabulary is fixed and the implemented set is not, so this case is expected rather + than exceptional, and the wording says which of the two the box fell foul of. + """ + + implemented = ", ".join(_RUNTIMES) + if runtime_id in RUNTIME_IDS: + return ( + f"Runtime {runtime_id} is not implemented by this version of Scrollcase; " + f"it implements {implemented}." + ) + return f"Unknown runtime: {runtime_id}. The box format defines {', '.join(RUNTIME_IDS)}." + + +def runtime_adapter(runtime_id: str) -> RuntimeAdapter: """Return the runtime adapter for a runtime id.""" runtime = _RUNTIMES.get(runtime_id) @@ -344,7 +452,7 @@ def runtime_adapters() -> tuple[RuntimeAdapter, ...]: def execution_affecting_variables( - adapter: TargetAdapter, runtime_id: str = IMPLICIT_RUNTIME_ID + adapter: TargetAdapter, runtime_id: str ) -> tuple[str, ...]: """The complete list of inherited variables that can change what a box executes. @@ -468,12 +576,25 @@ def execution_from_json(value: Mapping[str, Any] | None) -> BoxExecution | None: if value is None: return None default_args = tuple(cast(list[str], value["defaultArgs"])) - if value["kind"] == "python-script": + kind = value["kind"] + if kind == "python-script": return PythonScriptExecution( kind="python-script", script=cast(str, value["script"]), default_args=default_args, ) + if kind == "node-script": + return NodeScriptExecution( + kind="node-script", + script=cast(str, value["script"]), + default_args=default_args, + ) + if kind == "native-binary": + return NativeBinaryExecution( + kind="native-binary", + binary=cast(str, value["binary"]), + default_args=default_args, + ) return PythonModuleExecution( kind="python-module", module=cast(str, value["module"]), @@ -481,8 +602,37 @@ def execution_from_json(value: Mapping[str, Any] | None) -> BoxExecution | None: ) +def runtime_from_json(value: Mapping[str, Any]) -> BoxRuntime: + """Convert a validated runtime block into an immutable value.""" + + return BoxRuntime( + id=cast(str, value["id"]), + version=cast("str | None", value.get("version")), + entry_point=cast("str | None", value.get("entryPoint")), + ) + + +def self_test_probe_from_json(value: Mapping[str, Any]) -> SelfTestProbe: + """Convert a validated signed probe into an immutable value.""" + + return SelfTestProbe( + imports=tuple(cast(list[str], value.get("imports", ()))), + commands=tuple( + SelfTestCommand( + args=tuple(cast(list[str], command["args"])), + expect_exit_code=cast(int, command["expectExitCode"]), + ) + for command in cast(list[Mapping[str, Any]], value.get("commands", ())) + ), + ) + + def required_assets_from_json(values: list[Mapping[str, Any]] | None) -> tuple[RequiredAsset, ...]: - """Convert signed on-demand descriptors into immutable values.""" + """Convert the signed deferred descriptors into immutable values. + + The list is exactly the assets the scroll declared ``embed: false``; a release whose assets are + all embedded carries none, and the box needs nothing fetched before it runs. + """ if values is None: return () @@ -492,6 +642,7 @@ def required_assets_from_json(values: list[Mapping[str, Any]] | None) -> tuple[R relative_path=safe_relative_path(value["relativePath"]), size_bytes=cast(int, value["sizeBytes"]), sha256=cast(str, value["sha256"]), + executable=bool(value.get("executable", False)), ) for value in values ) @@ -500,6 +651,7 @@ def required_assets_from_json(values: list[Mapping[str, Any]] | None) -> tuple[R def assert_execution_files( execution: BoxExecution | None, target: BoxTarget, + runtime_id: str, runtime_version: str, resolvable_paths: Collection[str], ) -> None: @@ -516,7 +668,7 @@ def assert_execution_files( if execution is None: return target_adapter(target) - resolved = runtime_adapter().resolve_execution_files( + resolved = runtime_adapter(runtime_id).resolve_execution_files( execution, target.platform, runtime_version ) for candidate in resolved.candidates: diff --git a/python/src/scrollcase_consumer/environment.py b/python/src/scrollcase_consumer/environment.py index e75e7e5..ae5ac99 100644 --- a/python/src/scrollcase_consumer/environment.py +++ b/python/src/scrollcase_consumer/environment.py @@ -52,6 +52,7 @@ def resolve_environment( target: BoxTarget, layers: Sequence[tuple[EnvironmentSource, Mapping[str, str] | None]], *, + runtime_id: str, expanded: bool = False, reveal_host_values: bool = False, ) -> tuple[dict[str, str], EnvironmentReport]: @@ -76,7 +77,7 @@ def resolve_environment( dangerous = { _normalized_name(name, target.platform) - for name in execution_affecting_variables(adapter) + for name in execution_affecting_variables(adapter, runtime_id) } variables: list[tuple[EnvironmentVariableReport, bool]] = [] for normalized, sources in records.items(): @@ -151,10 +152,12 @@ def release_environment_report( ) -> EnvironmentReport: """Return a verification-time host plus release snapshot without executing anything.""" - declared = cast(Mapping[str, str] | None, release.get("environment")) + declared = cast("Mapping[str, str] | None", release.get("environment")) + runtime = cast(Mapping[str, object], release["runtime"]) return resolve_environment( target, (("host", os.environ), ("release", declared)), + runtime_id=cast(str, runtime["id"]), expanded=expanded, reveal_host_values=reveal_host_values, )[1] diff --git a/python/src/scrollcase_consumer/models.py b/python/src/scrollcase_consumer/models.py index a8e403d..6b9b7a4 100644 --- a/python/src/scrollcase_consumer/models.py +++ b/python/src/scrollcase_consumer/models.py @@ -6,6 +6,7 @@ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass from typing import Literal, TypeAlias @@ -74,7 +75,48 @@ class PythonModuleExecution: default_args: tuple[str, ...] -BoxExecution: TypeAlias = PythonScriptExecution | PythonModuleExecution +@dataclass(frozen=True, slots=True) +class NodeScriptExecution: + """A signed direct-script entry point for the Node runtime. + + Named by the format so implementing the runtime is code rather than another wire break. No + adapter answers for it yet, so a box declaring it is refused by name. + """ + + kind: Literal["node-script"] + script: str + default_args: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class NativeBinaryExecution: + """A signed compiled executable, run with no interpreter in front of it.""" + + kind: Literal["native-binary"] + binary: str + default_args: tuple[str, ...] + + +BoxExecution: TypeAlias = ( + PythonScriptExecution + | PythonModuleExecution + | NodeScriptExecution + | NativeBinaryExecution +) + + +@dataclass(frozen=True, slots=True) +class BoxRuntime: + """What runs inside the box. + + A target says which machine a box is for; this says what executes on it. Version 2 had no such + field: a box recorded a Python entry point and Python execution kinds and nothing that said + "Python", so a reader had to infer the runtime from the shape of a path. + """ + + id: str + version: str | None = None + entry_point: str | None = None @dataclass(frozen=True, slots=True) @@ -85,6 +127,9 @@ class RequiredAsset: relative_path: str size_bytes: int sha256: str + #: True when the scroll declared the file executable. Whoever materializes it owns setting the + #: bit: the file never passes through the archive, so nothing Scrollcase writes carries a mode. + executable: bool = False # Deliberately without ``slots=True``, unlike every other model here: the instance must keep @@ -106,12 +151,12 @@ class PreparedBox: status: Literal["prepared", "attached"] root: str box_id: str - model_id: str - runtime_id: str + #: Free-form annotations the publisher signed. Scrollcase attaches no meaning to any key. + labels: Mapping[str, str] version: str target: BoxTarget target_id: str - python_entry_point: str + runtime: BoxRuntime execution: BoxExecution | None required_assets: tuple[RequiredAsset, ...] signing_key_ids: tuple[str, ...] diff --git a/python/src/scrollcase_consumer/run.py b/python/src/scrollcase_consumer/run.py index f6651e5..da754c0 100644 --- a/python/src/scrollcase_consumer/run.py +++ b/python/src/scrollcase_consumer/run.py @@ -20,6 +20,7 @@ from typing import IO, Any, Protocol, TypeAlias, cast from ._contract import ( + RuntimeArgument, absolute_path, assert_execution_files, assert_native_host, @@ -151,28 +152,32 @@ def run_extracted_box( "Prepared box root no longer matches the prepared box." ) resolvable_paths = frozenset(collect_files(root)) - if prepared.python_entry_point not in resolvable_paths: - raise ScrollcaseConsumerError( - f"Prepared box is missing {prepared.python_entry_point}." - ) + entry_point = prepared.runtime.entry_point + if entry_point is not None and entry_point not in resolvable_paths: + raise ScrollcaseConsumerError(f"Prepared box is missing {entry_point}.") + provenance = cast(Mapping[str, object], state.release["provenance"]) assert_execution_files( execution, state.target, - cast(str, state.release["provenance"]["pythonVersion"]), + prepared.runtime.id, + cast(str, provenance.get("runtimeVersion", "")), resolvable_paths, ) verify_required_assets(Path(prepared.root), prepared.required_assets) - python = path_under(root, prepared.python_entry_point) # The runtime states the command line in payload-relative terms and this end joins it: a box - # root is a real path on this host, and the format has no business deciding what one looks like. - invocation = runtime_adapter().build_argv(execution, state.target.platform) - execution_args = [ - str(path_under(root, argument.value)) - if argument.kind == "payload-path" - else argument.value - for argument in invocation.args - ] + # root is a real path on this host, and the format has no business deciding what one looks + # like. Which runtime states it is the box's declaration, not an assumption about the payload. + invocation = runtime_adapter(prepared.runtime.id).build_argv( + execution, state.target.platform + ) + def resolve(argument: RuntimeArgument) -> str: + if argument.kind == "payload-path": + return str(path_under(root, argument.value)) + return str(argument.value) + + command = resolve(invocation.command) + execution_args = [resolve(argument) for argument in invocation.args] execution_args.extend(caller_args) environment, environment_report = resolve_environment( state.target, @@ -184,6 +189,7 @@ def run_extracted_box( cast(Mapping[str, str] | None, state.release.get("environment")), ), ), + runtime_id=prepared.runtime.id, expanded=env_report or env_report_values, reveal_host_values=env_report_values, ) @@ -191,7 +197,7 @@ def run_extracted_box( on_environment_report(environment_report) try: child = popen_factory( - [str(python), *execution_args], + [command, *execution_args], cwd=str(root), env=environment, stdin=stdin, diff --git a/python/src/scrollcase_consumer/schemas/box-manifest.schema.json b/python/src/scrollcase_consumer/schemas/box-manifest.schema.json index 8b0ddb8..6060ec5 100644 --- a/python/src/scrollcase_consumer/schemas/box-manifest.schema.json +++ b/python/src/scrollcase_consumer/schemas/box-manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/box-manifest.schema.json", + "$id": "https://scrollcase.dev/schema/v3/box-manifest.schema.json", "title": "Box manifest (box.json)", "description": "The manifest packed inside the archive at box.json. It restates the box's identity, layout and provenance so an extracted box is self-describing: a consumer that has the directory but not the release document can still tell what it is holding and how it was built.", "type": "object", @@ -8,43 +8,35 @@ "required": [ "schemaVersion", "boxId", - "modelId", - "runtimeId", "version", "target", - "pythonEntryPoint", - "modelCacheSubdir", + "runtime", + "cacheSubdir", "selfTest", "provenance" ], "properties": { "schemaVersion": { - "const": 2 + "const": 3 }, "boxId": { "type": "string", "minLength": 1 }, - "modelId": { - "type": "string", - "minLength": 1 - }, - "runtimeId": { - "type": "string", - "minLength": 1 + "labels": { + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/labels" }, "version": { "type": "string", "minLength": 1 }, "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + "$ref": "https://scrollcase.dev/schema/v3/target.schema.json" }, - "pythonEntryPoint": { - "type": "string", - "minLength": 1 + "runtime": { + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/runtime" }, - "modelCacheSubdir": { + "cacheSubdir": { "type": "string", "minLength": 1 }, @@ -61,76 +53,16 @@ } }, "selfTest": { - "type": "object", - "additionalProperties": false, - "required": [ - "pythonImports", - "timeoutSeconds" - ], - "properties": { - "pythonImports": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "minLength": 1 - } - }, - "timeoutSeconds": { - "type": "integer", - "exclusiveMinimum": 0 - } - } + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/selfTest" }, "execution": { - "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" + "$ref": "https://scrollcase.dev/schema/v3/execution.schema.json" }, "provenance": { - "$ref": "https://scrollcase.dev/schema/v2/release-manifest.schema.json#/$defs/provenance" - }, - "weights": { - "const": "on-demand", - "description": "Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it." + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/provenance" }, "assets": { - "type": "array", - "minItems": 1, - "description": "Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe.", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "url", - "relativePath", - "sizeBytes", - "sha256" - ], - "properties": { - "url": { - "type": "string", - "minLength": 1 - }, - "relativePath": { - "type": "string", - "minLength": 1 - }, - "sizeBytes": { - "type": "integer", - "exclusiveMinimum": 0 - }, - "sha256": { - "$ref": "https://scrollcase.dev/schema/v2/release-manifest.schema.json#/$defs/sha256" - } - } - } + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/deferredAssets" } - }, - "dependentRequired": { - "assets": [ - "weights" - ], - "weights": [ - "assets" - ] } } diff --git a/python/src/scrollcase_consumer/schemas/execution.schema.json b/python/src/scrollcase_consumer/schemas/execution.schema.json index a37ebfb..62b4da2 100644 --- a/python/src/scrollcase_consumer/schemas/execution.schema.json +++ b/python/src/scrollcase_consumer/schemas/execution.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/execution.schema.json", + "$id": "https://scrollcase.dev/schema/v3/execution.schema.json", "title": "Box execution", - "description": "The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only.", + "description": "The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only.\n\nEach kind is named -, and the runtime half must be the one the box declares: a python-script in a box whose runtime is native describes something that cannot be run, and is refused rather than guessed at.", "oneOf": [ { "title": "Python script", @@ -55,6 +55,54 @@ "$ref": "#/$defs/defaultArgs" } } + }, + { + "title": "Node script", + "description": "Run one regular payload file with the box's own Node runtime.", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "script", + "defaultArgs" + ], + "properties": { + "kind": { + "const": "node-script", + "description": "Selects direct script execution." + }, + "script": { + "$ref": "#/$defs/payloadPath", + "description": "Safe path to a regular JavaScript file inside the box." + }, + "defaultArgs": { + "$ref": "#/$defs/defaultArgs" + } + } + }, + { + "title": "Native binary", + "description": "Run a compiled executable that the box carries directly, with no interpreter in front of it. The only shape a runtime with no module system has.", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "binary", + "defaultArgs" + ], + "properties": { + "kind": { + "const": "native-binary", + "description": "Selects direct execution of a payload file." + }, + "binary": { + "$ref": "#/$defs/payloadPath", + "description": "Safe path to the executable inside the box. It carries the executable bit because the scroll declared it, not because the build machine happened to have it set." + }, + "defaultArgs": { + "$ref": "#/$defs/defaultArgs" + } + } } ], "examples": [ diff --git a/python/src/scrollcase_consumer/schemas/release-manifest.schema.json b/python/src/scrollcase_consumer/schemas/release-manifest.schema.json index 07d5458..90d1b23 100644 --- a/python/src/scrollcase_consumer/schemas/release-manifest.schema.json +++ b/python/src/scrollcase_consumer/schemas/release-manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/release-manifest.schema.json", + "$id": "https://scrollcase.dev/schema/v3/release-manifest.schema.json", "title": "Box release manifest", "description": "The immutable description of one built box: what it is, which target it runs on, where its archive lives, and how it was produced. Published as the payload of a signed document and never edited after signing; a correction ships as a new version.", "type": "object", @@ -9,20 +9,18 @@ "schemaVersion", "kind", "boxId", - "modelId", - "runtimeId", "version", "target", "compatibility", "archive", - "pythonEntryPoint", - "modelCacheSubdir", + "runtime", + "cacheSubdir", "selfTest", "provenance" ], "properties": { "schemaVersion": { - "const": 2 + "const": 3 }, "kind": { "$ref": "#/$defs/kind", @@ -31,18 +29,15 @@ "boxId": { "$ref": "#/$defs/identifier" }, - "modelId": { - "$ref": "#/$defs/identifier" - }, - "runtimeId": { - "$ref": "#/$defs/identifier" + "labels": { + "$ref": "#/$defs/labels" }, "version": { "type": "string", "minLength": 1 }, "target": { - "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + "$ref": "https://scrollcase.dev/schema/v3/target.schema.json" }, "compatibility": { "type": "object", @@ -132,15 +127,13 @@ } } }, - "pythonEntryPoint": { - "type": "string", - "minLength": 1, - "description": "Interpreter path relative to the extracted box root, for example venv/bin/python. Fixed per target by the adapter." + "runtime": { + "$ref": "#/$defs/runtime" }, - "modelCacheSubdir": { + "cacheSubdir": { "type": "string", "minLength": 1, - "description": "Directory relative to the extracted box root holding model assets." + "description": "Directory relative to the extracted box root holding the box's own large files." }, "environment": { "type": "object", @@ -155,42 +148,127 @@ } }, "selfTest": { + "$ref": "#/$defs/selfTest" + }, + "execution": { + "$ref": "https://scrollcase.dev/schema/v3/execution.schema.json" + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "assets": { + "$ref": "#/$defs/deferredAssets" + } + }, + "$defs": { + "identifier": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" + }, + "runtime": { "type": "object", "additionalProperties": false, "required": [ - "pythonImports", + "id" + ], + "description": "What runs inside the box: the runtime, its version, and where its own executable sits in the payload. A consumer needs all three to run the box, and none of them are derivable from the target.", + "properties": { + "id": { + "enum": [ + "python", + "node", + "native" + ], + "description": "The runtime the box carries. A consumer that does not recognise the id must refuse the box: the id decides the payload layout and the argv rule, so guessing would mean executing something on an assumption." + }, + "version": { + "type": "string", + "minLength": 1, + "description": "The runtime's own version. Absent for a runtime that has no interpreter to version." + }, + "entryPoint": { + "type": "string", + "minLength": 1, + "description": "The runtime's own executable relative to the extracted box root, for example venv/bin/python. Fixed per (runtime, target) by the runtime's layout. Absent for a runtime that has no separate executable to name." + } + } + }, + "labels": { + "type": "object", + "description": "Free-form annotations the publishing project declared, signed and carried through untouched. Scrollcase attaches no meaning to any key; a consumer that reads one is reading its own project's convention, not the box format.", + "propertyNames": { + "$ref": "#/$defs/identifier" + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } + }, + "selfTest": { + "type": "object", + "additionalProperties": false, + "required": [ + "probe", "timeoutSeconds" ], - "description": "The import check a consumer can repeat after extraction with the box's own interpreter. The builder also ran the scroll's Python-code and file assertions, which are builder-only checks.", + "description": "The check a consumer can repeat against an extracted box. The builder also ran the scroll's file assertions and any extra source it declared, which are builder-only: signing them would claim a consumer had reproduced a check it cannot see.", "properties": { - "pythonImports": { + "probe": { + "$ref": "#/$defs/selfTestProbe" + }, + "timeoutSeconds": { + "type": "integer", + "exclusiveMinimum": 0 + } + } + }, + "selfTestProbe": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "description": "What the box proves about itself, in whichever shapes its runtime supports. The runtime turns this into command lines; nothing here is a command line, and nothing here is source in any language.", + "properties": { + "imports": { "type": "array", "minItems": 1, + "description": "Modules the runtime must be able to load.", "items": { "type": "string", "minLength": 1 } }, - "timeoutSeconds": { - "type": "integer", - "exclusiveMinimum": 0 + "commands": { + "type": "array", + "minItems": 1, + "description": "Invocations of the box's declared execution and the exit status each must produce.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "args", + "expectExitCode" + ], + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "expectExitCode": { + "type": "integer", + "minimum": 0, + "maximum": 255 + } + } + } } } }, - "execution": { - "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" - }, - "provenance": { - "$ref": "#/$defs/provenance" - }, - "weights": { - "const": "on-demand", - "description": "Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it." - }, - "assets": { + "deferredAssets": { "type": "array", "minItems": 1, - "description": "Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe.", + "description": "Assets the consumer must fetch and place under the box root before first use — the entries the scroll declared with embed false, and only those. A box whose assets are all embedded carries no such list. The declared size and hash are what make fetching them safe; a Scrollcase consumer verifies them and never downloads them itself.", "items": { "type": "object", "additionalProperties": false, @@ -215,15 +293,13 @@ }, "sha256": { "$ref": "#/$defs/sha256" + }, + "executable": { + "type": "boolean", + "description": "Present and true when the scroll declared the file executable. Whoever materializes it owns setting the bit: the file never passes through the archive, so nothing Scrollcase writes can carry a mode for it." } } } - } - }, - "$defs": { - "identifier": { - "type": "string", - "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" }, "kind": { "type": "string", @@ -243,7 +319,6 @@ "builderRevision", "sourceTreeDirty", "sourceRevision", - "pythonVersion", "dependencyLockSha256", "builtAt", "pixiVersion" @@ -269,11 +344,12 @@ "sourceRevision": { "type": "string", "minLength": 1, - "description": "Upstream revision of the packaged model source, as declared by the scroll." + "description": "Upstream revision of the packaged source, as declared by the scroll." }, - "pythonVersion": { + "runtimeVersion": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "The runtime version the environment was solved with, repeated from runtime.version. Absent exactly when the runtime has none: provenance records what was observed and never invents a value to fill a field." }, "pixiVersion": { "type": "string", @@ -289,13 +365,5 @@ } } } - }, - "dependentRequired": { - "assets": [ - "weights" - ], - "weights": [ - "assets" - ] } } diff --git a/python/src/scrollcase_consumer/schemas/signed-document.schema.json b/python/src/scrollcase_consumer/schemas/signed-document.schema.json index 602af0c..ad06de0 100644 --- a/python/src/scrollcase_consumer/schemas/signed-document.schema.json +++ b/python/src/scrollcase_consumer/schemas/signed-document.schema.json @@ -1,13 +1,13 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/signed-document.schema.json", + "$id": "https://scrollcase.dev/schema/v3/signed-document.schema.json", "title": "Signed box document", "description": "The envelope wrapping every signed document. The payload travels as exact base64-encoded JSON so that verifying a signature means hashing the bytes as transmitted, with no canonical-JSON implementation to keep in sync across languages. Passing this schema means the envelope is well-formed, never that its signature is valid.", "type": "object", "additionalProperties": false, "required": ["schemaVersion", "payloadEncoding", "payloadBase64", "payloadSha256", "signatures"], "properties": { - "schemaVersion": { "const": 2 }, + "schemaVersion": { "const": 3 }, "payloadEncoding": { "const": "base64-json-utf8" }, "payloadBase64": { "type": "string", diff --git a/python/src/scrollcase_consumer/schemas/target.schema.json b/python/src/scrollcase_consumer/schemas/target.schema.json index 6894c12..1573b29 100644 --- a/python/src/scrollcase_consumer/schemas/target.schema.json +++ b/python/src/scrollcase_consumer/schemas/target.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://scrollcase.dev/schema/v2/target.schema.json", + "$id": "https://scrollcase.dev/schema/v3/target.schema.json", "title": "Box target", "description": "The platform, architecture and accelerator a box is built for. The supported combinations are closed: a target outside this matrix has no defined identifier and cannot be built, signed, or routed.", "type": "object", diff --git a/python/src/scrollcase_consumer/verify.py b/python/src/scrollcase_consumer/verify.py index dbbe069..836c8a8 100644 --- a/python/src/scrollcase_consumer/verify.py +++ b/python/src/scrollcase_consumer/verify.py @@ -31,10 +31,13 @@ assert_execution_files, assert_native_host, execution_from_json, + is_implemented_runtime, parse_payload_digest_stream, path_under, required_assets_from_json, runtime_adapter, + runtime_from_json, + unimplemented_runtime_message, target_adapter, target_from_json, target_id, @@ -55,21 +58,37 @@ _AGREEMENT_FIELDS = ( "schemaVersion", "boxId", - "modelId", - "runtimeId", + "labels", "version", "target", - "pythonEntryPoint", - "modelCacheSubdir", + "runtime", + "cacheSubdir", "environment", "selfTest", "execution", - "weights", "assets", "provenance", ) +#: The format version this package reads and nothing else. +BOX_SCHEMA_VERSION = 3 + + +def _assert_supported_schema_version(version: object) -> None: + """Refuse a superseded document by name rather than reinterpreting it. + + Both older versions are named rather than lumped together as "too old": a v1 and a v2 box are + different artefacts with different rebuilds ahead of them, and whoever is holding one is + entitled to know which. + """ + + if version in (1, 2): + raise ScrollcaseConsumerError( + f"Unsupported schemaVersion {version}; rebuild this box with Scrollcase v3." + ) + + @dataclass(frozen=True, slots=True) class _PreparedState: release: dict[str, Any] @@ -280,20 +299,16 @@ def _inspect_release_document( """ release_path = absolute_path(release_document_path) signed_value = _read_json(release_path, "signed document") - if isinstance(signed_value, Mapping) and signed_value.get("schemaVersion") == 1: - raise ScrollcaseConsumerError( - "Unsupported schemaVersion 1; rebuild this box with Scrollcase v2." - ) + if isinstance(signed_value, Mapping): + _assert_supported_schema_version(signed_value.get("schemaVersion")) validate_schema(signed_value, "signed-document.schema.json", "signed document") signed = cast(dict[str, Any], signed_value) _, release = _verify_signed_document(signed, trusted) - if release.get("schemaVersion") == 1: - raise ScrollcaseConsumerError( - "Unsupported schemaVersion 1; rebuild this box with Scrollcase v2." - ) - if release.get("schemaVersion") != 2: + _assert_supported_schema_version(release.get("schemaVersion")) + if release.get("schemaVersion") != BOX_SCHEMA_VERSION: raise ScrollcaseConsumerError( - f"Unsupported schemaVersion {release.get('schemaVersion')}; expected 2." + f"Unsupported schemaVersion {release.get('schemaVersion')}; " + f"expected {BOX_SCHEMA_VERSION}." ) validate_schema(release, "release-manifest.schema.json", "release manifest") kind = cast(str, release["kind"]) @@ -302,11 +317,16 @@ def _inspect_release_document( target = target_from_json(cast(dict[str, Any], release["target"])) adapter = target_adapter(target) - expected_entry_point = runtime_adapter().layout(adapter.platform).entry_point - if release["pythonEntryPoint"] != expected_entry_point: + runtime = runtime_from_json(cast(dict[str, Any], release["runtime"])) + # The format's runtime vocabulary is wider than what this package implements, so a release may + # name one there is no adapter for. That is refused by name rather than misread as another. + if not is_implemented_runtime(runtime.id): + raise ScrollcaseConsumerError(unimplemented_runtime_message(runtime.id)) + expected_entry_point = runtime_adapter(runtime.id).layout(adapter.platform).entry_point + if runtime.entry_point is not None and runtime.entry_point != expected_entry_point: raise ScrollcaseConsumerError( - f"{adapter.platform}-{adapter.arch} boxes must use Python entry point " - f"{expected_entry_point}" + f"{adapter.platform}-{adapter.arch} boxes with the {runtime.id} runtime must use " + f"entry point {expected_entry_point}" ) return _InspectedRelease( release_path=release_path, @@ -360,17 +380,17 @@ def _inspect_box_archive( validate_schema(box_value, "box-manifest.schema.json", "box.json") box = cast(dict[str, Any], box_value) _assert_manifest_agreement(box, release) - if release["pythonEntryPoint"] not in resolvable_paths: - raise ScrollcaseConsumerError( - f"Archive is missing {release['pythonEntryPoint']}." - ) + entry_point = cast("str | None", release["runtime"].get("entryPoint")) + if entry_point is not None and entry_point not in resolvable_paths: + raise ScrollcaseConsumerError(f"Archive is missing {entry_point}.") execution = execution_from_json( cast(dict[str, Any] | None, release.get("execution")) ) assert_execution_files( execution, target, - cast(str, release["provenance"]["pythonVersion"]), + cast(str, release["runtime"]["id"]), + cast(str, release["provenance"].get("runtimeVersion", "")), resolvable_paths, ) return _InspectedBox( @@ -405,9 +425,7 @@ def verify_and_extract_box( ) release = inspected.release required_assets = required_assets_from_json( - cast(list[Mapping[str, Any]] | None, release.get("assets")) - if release.get("weights") == "on-demand" - else None + cast("list[Mapping[str, Any]] | None", release.get("assets")) ) final_root.parent.mkdir(parents=True, exist_ok=True) if final_root.exists() or final_root.is_symlink(): @@ -450,12 +468,11 @@ def verify_and_extract_box( status="prepared", root=str(final_root), box_id=cast(str, release["boxId"]), - model_id=cast(str, release["modelId"]), - runtime_id=cast(str, release["runtimeId"]), + labels=cast("Mapping[str, str]", release.get("labels", {})), version=cast(str, release["version"]), target=target, target_id=target_id(target), - python_entry_point=cast(str, release["pythonEntryPoint"]), + runtime=runtime_from_json(cast(dict[str, Any], release["runtime"])), execution=inspected.execution, required_assets=required_assets, signing_key_ids=tuple( @@ -560,23 +577,21 @@ def attach_extracted_box( assert_native_host(target) resolvable_paths = frozenset(collect_files(box_root)) - if release["pythonEntryPoint"] not in resolvable_paths: - raise ScrollcaseConsumerError( - f"Attached box is missing {release['pythonEntryPoint']}." - ) + entry_point = cast("str | None", release["runtime"].get("entryPoint")) + if entry_point is not None and entry_point not in resolvable_paths: + raise ScrollcaseConsumerError(f"Attached box is missing {entry_point}.") execution = execution_from_json( cast(dict[str, Any] | None, release.get("execution")) ) assert_execution_files( execution, target, - cast(str, release["provenance"]["pythonVersion"]), + cast(str, release["runtime"]["id"]), + cast(str, release["provenance"].get("runtimeVersion", "")), resolvable_paths, ) required_assets = required_assets_from_json( - cast(list[Mapping[str, Any]] | None, release.get("assets")) - if release.get("weights") == "on-demand" - else None + cast("list[Mapping[str, Any]] | None", release.get("assets")) ) verify_required_assets(box_root, required_assets) @@ -592,12 +607,11 @@ def attach_extracted_box( status="attached", root=str(box_root), box_id=cast(str, release["boxId"]), - model_id=cast(str, release["modelId"]), - runtime_id=cast(str, release["runtimeId"]), + labels=cast("Mapping[str, str]", release.get("labels", {})), version=cast(str, release["version"]), target=target, target_id=target_id(target), - python_entry_point=cast(str, release["pythonEntryPoint"]), + runtime=runtime_from_json(cast(dict[str, Any], release["runtime"])), execution=execution, required_assets=required_assets, signing_key_ids=tuple( diff --git a/python/tests/conformance_support.py b/python/tests/conformance_support.py index ee820ad..4985479 100644 --- a/python/tests/conformance_support.py +++ b/python/tests/conformance_support.py @@ -8,6 +8,7 @@ import signal import stat from pathlib import Path +from collections.abc import Mapping from typing import IO, Any, cast from scrollcase_consumer import ( @@ -139,13 +140,19 @@ def _fixture_options(spec: dict[str, Any]) -> dict[str, Any]: } if spec.get("requiredAsset"): options["required_asset"] = { - "url": "https://assets.example.org/weights.bin", - "relativePath": "model-cache/consumer-fixture/weights.bin", + "url": "https://assets.example.org/data.bin", + "relativePath": "cache/consumer-fixture/data.bin", "sizeBytes": len(ASSET_BYTES), "sha256": hashlib.sha256(ASSET_BYTES).hexdigest(), } + if spec.get("executableAsset"): + # The mode is synthesised from the scroll's declaration, and extraction has to hand it back + # whatever umask the process is running under. + options["extra_files"] = {"bin/tool": (b"#!/bin/sh\nexit 0\n", 0o755)} if "environment" in spec: options["environment"] = spec["environment"] + if "labels" in spec: + options["labels"] = spec["labels"] # A consumer cannot observe key custody. The external-signer case therefore uses the same # signed-envelope wire contract with an independently generated caller trust anchor. return options @@ -212,8 +219,18 @@ def _mutate_fixture( fixture.release["archive"]["sizeBytes"] += 1 fixture.sign() return - if mutation == "alter-release-model": - fixture.release["modelId"] = "altered-model" + if mutation == "alter-release-labels": + fixture.release["labels"] = {"model": "altered-model"} + fixture.sign() + return + if mutation == "alter-release-runtime-version": + fixture.release["runtime"] = {**fixture.release["runtime"], "version": "3.99.0"} + fixture.sign() + return + if mutation == "alter-release-runtime-id": + # A runtime the format names and this package has no adapter for. The consumer must refuse + # the box rather than read it as the runtime it happens to be shaped like. + fixture.release["runtime"] = {**fixture.release["runtime"], "id": "native"} fixture.sign() return if mutation == "alter-release-execution": @@ -241,7 +258,7 @@ def _mutate_fixture( destination.mkdir() return remove_path = { - "remove-interpreter": fixture.release["pythonEntryPoint"], + "remove-interpreter": fixture.release["runtime"]["entryPoint"], "remove-script": fixture.release.get("execution", {}).get("script"), "remove-module": ( fixture.release.get("execution", {}).get("module", "").replace(".", "/") @@ -257,7 +274,7 @@ def _mutate_fixture( # shape — `venv/bin/python` is a link to the versioned binary beside it — so a consumer that # only accepts regular files here rejects every box the builder produces on macOS and Linux. if mutation == "link-interpreter": - entry_point = cast(str, fixture.release["pythonEntryPoint"]) + entry_point = cast(str, fixture.release["runtime"]["entryPoint"]) directory, _, name = entry_point.rpartition("/") link_target = f"{name}-real" renamed = f"{directory}/{link_target}" if directory else link_target @@ -340,13 +357,13 @@ def _mutate_extracted_root( script.write_bytes(script.read_bytes() + b" ") return root if mutation == "remove-interpreter": - (root / fixture.release["pythonEntryPoint"]).unlink() + (root / fixture.release["runtime"]["entryPoint"]).unlink() return root if mutation == "remove-script": (root / fixture.release["execution"]["script"]).unlink() return root if mutation == "retarget-interpreter-link": - interpreter = root / fixture.release["pythonEntryPoint"] + interpreter = root / fixture.release["runtime"]["entryPoint"] target = os.readlink(interpreter) interpreter.unlink() interpreter.symlink_to(f"{target}-retargeted") @@ -375,7 +392,7 @@ def _replace_tokens(value: Any, root: str | None = None) -> Any: (target["platform"], target["arch"], target["accelerator"]) ) return ( - value.replace("$NATIVE_PYTHON", native_python) + value.replace("$NATIVE_ENTRY_POINT", native_python) .replace("$NATIVE_TARGET", native_id) .replace("$BOX", root or "$BOX") ) @@ -501,6 +518,11 @@ def run_python_conformance_case( for name in runtime.get("hostEnvironment", {}) } os.environ.update(runtime.get("hostEnvironment", {})) + # A restrictive umask is the condition under which the three consumers used to disagree: two + # applied the archive's mode through open(2) and lost it, one chmod'd and kept it. + previous_umask = ( + os.umask(int(runtime["umask"], 8)) if "umask" in runtime else None + ) try: action = test_case["action"] mutation = test_case.get("mutation") @@ -528,7 +550,8 @@ def run_python_conformance_case( "boxId": prepared.box_id, "executionKind": execution.kind if execution is not None else None, "requiredAssetCount": len(prepared.required_assets), - "pythonEntryPoint": prepared.python_entry_point, + "runtimeId": prepared.runtime.id, + "entryPoint": prepared.runtime.entry_point or "", "targetId": prepared.target_id, } if test_case["expected"].get("receipt", {}).get("environmentReport"): @@ -536,6 +559,13 @@ def run_python_conformance_case( prepared.environment_report, runtime.get("reportVariables", []), ) + declared_modes = test_case["expected"].get("receipt", {}).get( + "executableModes" + ) + if declared_modes: + receipt["executableModes"] = _executable_modes( + Path(prepared.root), declared_modes + ) actual = { "outcome": "prepared", "receipt": receipt, @@ -573,7 +603,8 @@ def run_python_conformance_case( execution.kind if execution is not None else None ), "requiredAssetCount": len(attached.required_assets), - "pythonEntryPoint": attached.python_entry_point, + "runtimeId": attached.runtime.id, + "entryPoint": attached.runtime.entry_point or "", "targetId": attached.target_id, } if test_case["expected"].get("receipt", {}).get("environmentReport"): @@ -723,6 +754,8 @@ def run_python_conformance_case( ) return actual, expected, fixture.root finally: + if previous_umask is not None: + os.umask(previous_umask) for name, value in previous_host_environment.items(): if value is None: os.environ.pop(name, None) @@ -730,5 +763,22 @@ def run_python_conformance_case( os.environ[name] = value +def _executable_modes(root: Path, paths: Mapping[str, object]) -> dict[str, str | None]: + """The permission bits an extracted box actually carries, for the paths a case names. + + Windows has no bit to read, so every path reports ``None`` there and the fixture says so rather + than the driver quietly skipping the case. + """ + + return { + path: ( + None + if os.name == "nt" + else oct((root / path).stat().st_mode & 0o777)[2:] + ) + for path in paths + } + + def remove_conformance_root(root: Path) -> None: shutil.rmtree(root, ignore_errors=True) diff --git a/python/tests/support.py b/python/tests/support.py index 66ea8a5..e628ebd 100644 --- a/python/tests/support.py +++ b/python/tests/support.py @@ -37,7 +37,7 @@ def native_target() -> dict[str, str]: raise RuntimeError(f"Unsupported test host: {sys.platform}/{machine}") -def python_entry_point(target: dict[str, str]) -> str: +def entry_point_for(target: dict[str, str]) -> str: if target["platform"] == "windows": return "venv/python.exe" return "venv/bin/python" @@ -65,7 +65,7 @@ class ConsumerFixture: def sign(self) -> None: payload = (json.dumps(self.release, indent=2) + "\n").encode("utf-8") signed = { - "schemaVersion": 2, + "schemaVersion": 3, "payloadEncoding": "base64-json-utf8", "payloadBase64": base64.b64encode(payload).decode("ascii"), "payloadSha256": hashlib.sha256(payload).hexdigest(), @@ -141,10 +141,12 @@ def create_fixture( script: bytes = b'print("consumer fixture")\n', payload_digest: bool = True, environment: dict[str, str] | None = None, + labels: dict[str, str] | None = None, + extra_files: dict[str, tuple[bytes, int]] | None = None, ) -> ConsumerFixture: root = Path(tempfile.mkdtemp(prefix="scrollcase-python-consumer-fixture-")) resolved_target = native_target() if target is None else target - entry_point = python_entry_point(resolved_target) + entry_point = entry_point_for(resolved_target) resolved_execution = ( { "kind": "python-script", @@ -155,22 +157,24 @@ def create_fixture( else execution ) shared: dict[str, Any] = { - "schemaVersion": 2, + "schemaVersion": 3, "boxId": "consumer-fixture", - "modelId": "example-consumer-model", - "runtimeId": "example-consumer-runtime", "version": "2.0.0", "target": resolved_target, - "pythonEntryPoint": entry_point, - "modelCacheSubdir": "model-cache/consumer-fixture", - "selfTest": {"pythonImports": ["json"], "timeoutSeconds": 30}, + "runtime": { + "id": "python", + "version": "3.11.15", + "entryPoint": entry_point, + }, + "cacheSubdir": "cache/consumer-fixture", + "selfTest": {"probe": {"imports": ["json"]}, "timeoutSeconds": 30}, "provenance": { "scrollId": "consumer-fixture-scroll", "scrollVersion": "2.0.0", "builderRevision": "0123456789abcdef0123456789abcdef01234567", "sourceTreeDirty": False, "sourceRevision": "fedcba9876543210", - "pythonVersion": "3.11.15", + "runtimeVersion": "3.11.15", "pixiVersion": "0.73.0", "dependencyLockSha256": "a" * 64, "builtAt": "2026-07-29T00:00:00.000Z", @@ -180,12 +184,16 @@ def create_fixture( shared["execution"] = resolved_execution if environment is not None: shared["environment"] = environment + if labels is not None: + shared["labels"] = labels if required_asset is not None: - shared["weights"] = "on-demand" + # The list is exactly the deferred entries; there is no second field to keep in step. shared["assets"] = [required_asset] entries = [ ArchiveEntry(entry_point, interpreter, 0o755), ] + for path, (data, mode) in (extra_files or {}).items(): + entries.append(ArchiveEntry(path, data, mode)) if isinstance(resolved_execution, dict): if resolved_execution["kind"] == "python-script": entries.append(ArchiveEntry(resolved_execution["script"], script)) diff --git a/python/tests/test_contract.py b/python/tests/test_contract.py index 1e6f6a3..85f7df9 100644 --- a/python/tests/test_contract.py +++ b/python/tests/test_contract.py @@ -14,7 +14,6 @@ from typing import Any, cast from scrollcase_consumer._contract import ( - IMPLICIT_RUNTIME_ID, PAYLOAD_DIGEST_FILE, PAYLOAD_DIGEST_FORMAT, SCHEMA_FILES, @@ -23,9 +22,14 @@ execution_affecting_variables, execution_from_json, parse_payload_digest_stream, + RUNTIME_IDS, + SelfTestCommand, + SelfTestProbe, + is_implemented_runtime, payload_digest_stream, runtime_adapter, runtime_adapters, + unimplemented_runtime_message, target_adapter, target_from_json, target_id, @@ -108,11 +112,34 @@ def test_exposes_exactly_the_runtimes_the_fixture_describes(self) -> None: [case["id"] for case in fixture["runtimes"]], ) - def test_refuses_a_runtime_the_format_does_not_define(self) -> None: - for runtime_id in ("node", "native", ""): + def test_names_every_runtime_the_format_defines(self) -> None: + # Two different lists on purpose: the wire vocabulary was fixed once, in the version 3 + # break, so that implementing a second runtime is code rather than another format change. + fixture = self._load() + self.assertEqual(list(RUNTIME_IDS), fixture["runtimeIds"]) + implemented = {runtime.id for runtime in runtime_adapters()} + for runtime_id in RUNTIME_IDS: + with self.subTest(runtime_id=runtime_id): + self.assertEqual( + is_implemented_runtime(runtime_id), runtime_id in implemented + ) + + def test_refuses_a_runtime_it_has_no_adapter_for(self) -> None: + for runtime_id in ("node", "native"): with self.subTest(runtime_id=runtime_id): with self.assertRaises(ScrollcaseConsumerError): runtime_adapter(runtime_id) + self.assertIn( + "not implemented by this version", + unimplemented_runtime_message(runtime_id), + ) + for runtime_id in ("", "ruby"): + with self.subTest(runtime_id=runtime_id): + with self.assertRaises(ScrollcaseConsumerError): + runtime_adapter(runtime_id) + self.assertIn( + "Unknown runtime", unimplemented_runtime_message(runtime_id) + ) def test_reproduces_every_golden_layout_and_executable_rule(self) -> None: for case in self._load()["runtimes"]: @@ -175,7 +202,7 @@ def test_refuses_a_runtime_version_that_cannot_name_a_standard_library(self) -> with self.assertRaisesRegex( ScrollcaseConsumerError, "Invalid Python version" ): - runtime_adapter().resolve_execution_files( + runtime_adapter("python").resolve_execution_files( execution, "linux", invalid ) @@ -202,15 +229,49 @@ def test_builds_exactly_the_golden_shell_free_command_line(self) -> None: case["args"], ) - def test_turns_every_golden_probe_into_the_same_arguments(self) -> None: + def test_turns_every_golden_probe_into_the_same_invocations(self) -> None: for case in self._load()["selfTest"]: with self.subTest(case=case["name"]): - argv = runtime_adapter(case["runtime"]).self_test_argv( - case["probe"]["imports"], + probe = SelfTestProbe( + imports=tuple(case["probe"].get("imports", ())), + commands=tuple( + SelfTestCommand( + args=tuple(command["args"]), + expect_exit_code=command["expectExitCode"], + ) + for command in case["probe"].get("commands", ()) + ), + code=case["probe"].get("code"), + ) + invocations = runtime_adapter(case["runtime"]).self_test_invocations( + probe, + execution_from_json(case.get("execution")), case["platform"], - case["probe"].get("code"), ) - self.assertEqual(list(argv), case["args"]) + self.assertEqual( + [ + { + "command": { + "kind": invocation.command.kind, + "value": invocation.command.value, + }, + "args": [ + {"kind": argument.kind, "value": argument.value} + for argument in invocation.args + ], + "expectExitCode": invocation.expect_exit_code, + } + for invocation in invocations + ], + case["invocations"], + ) + + def test_refuses_a_command_probe_with_no_execution_to_invoke(self) -> None: + probe = SelfTestProbe(commands=(SelfTestCommand(args=(), expect_exit_code=0),)) + with self.assertRaisesRegex( + ScrollcaseConsumerError, "needs a declared execution" + ): + runtime_adapter("python").self_test_invocations(probe, None, "linux") def test_joins_the_runtime_half_to_the_target_half_runtime_first(self) -> None: # The order is what a diagnostic report is printed in, so it is part of the answer rather @@ -225,12 +286,12 @@ def test_joins_the_runtime_half_to_the_target_half_runtime_first(self) -> None: {"platform": platform, "arch": arch, "accelerator": "cpu"} ) ) - merged = execution_affecting_variables(adapter) + merged = execution_affecting_variables(adapter, "python") self.assertEqual( list(merged), [ *runtime_adapter( - IMPLICIT_RUNTIME_ID + "python" ).execution_environment_variables, *adapter.execution_affecting_environment_variables, ], @@ -397,7 +458,7 @@ def setUp(self) -> None: self.root = Path(tempfile.mkdtemp(prefix="scrollcase-digest-")) self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) (self.root / "venv" / "bin").mkdir(parents=True) - (self.root / "box.json").write_bytes(b'{"schemaVersion":2}\n') + (self.root / "box.json").write_bytes(b'{"schemaVersion":3}\n') (self.root / "venv" / "bin" / "python3.11").write_bytes(b"interpreter") def test_skips_the_list_file_and_the_names_python_generates(self) -> None: diff --git a/python/tests/test_run.py b/python/tests/test_run.py index d50fc95..4d9bf61 100644 --- a/python/tests/test_run.py +++ b/python/tests/test_run.py @@ -116,7 +116,7 @@ def test_preserves_signed_and_caller_arguments_without_a_shell(self) -> None: ) self.assertEqual((result.exit_code, result.signal), (23, None)) argv, options = fake.calls[0] - self.assertEqual(argv[0], str(Path(prepared.root) / prepared.python_entry_point)) + self.assertEqual(argv[0], str(Path(prepared.root) / prepared.runtime.entry_point)) self.assertEqual( argv[1:], [ @@ -248,8 +248,8 @@ def test_executes_the_real_child_and_preserves_metacharacters(self) -> None: def test_verifies_materialized_on_demand_assets_before_spawn(self) -> None: data = b"trusted on-demand bytes" asset: dict[str, Any] = { - "url": "https://assets.example.org/weights.bin", - "relativePath": "model-cache/consumer-fixture/weights.bin", + "url": "https://assets.example.org/data.bin", + "relativePath": "cache/consumer-fixture/data.bin", "sizeBytes": len(data), "sha256": hashlib.sha256(data).hexdigest(), } diff --git a/python/tests/test_verify.py b/python/tests/test_verify.py index 18806f0..5b98447 100644 --- a/python/tests/test_verify.py +++ b/python/tests/test_verify.py @@ -127,7 +127,7 @@ def test_rejects_missing_interpreters_and_execution_files(self) -> None: without_interpreter = [ entry for entry in self.fixture.entries - if entry.path != self.fixture.release["pythonEntryPoint"] + if entry.path != self.fixture.release["runtime"]["entryPoint"] ] self.fixture.write_archive(without_interpreter) with self.assertRaisesRegex( @@ -170,7 +170,7 @@ def test_prepares_all_three_interpreter_layouts_without_executing_them(self) -> if target["platform"] == "windows" else "venv/bin/python" ) - self.assertEqual(prepared.python_entry_point, expected) + self.assertEqual(prepared.runtime.entry_point, expected) finally: for fixture in extra_fixtures: shutil.rmtree(fixture.root, ignore_errors=True) @@ -373,8 +373,8 @@ def test_walks_the_list_and_not_the_directory(self) -> None: # Everything an installed box legitimately grows: the application's own output in its # working directory, Python's caches, and a model cache filled after extraction. (self.root / "output.log").write_bytes(b"the application wrote this") - (self.root / "model-cache").mkdir(parents=True, exist_ok=True) - (self.root / "model-cache" / "weights.bin").write_bytes(b"downloaded later") + (self.root / "cache").mkdir(parents=True, exist_ok=True) + (self.root / "cache" / "weights.bin").write_bytes(b"downloaded later") (self.root / "__pycache__").mkdir() (self.root / "__pycache__" / "x.pyc").write_bytes(b"compiled") self.assertEqual(self.verify().status, "verified") From 7230fb19d7ee6ce947faad920a7a7c74a06ded30 Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:43:28 +0200 Subject: [PATCH 08/22] Record the version 3 format break in the changelog --- CHANGELOG.md | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b4290c..a88b509 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,69 @@ All notable changes to Scrollcase are documented here. The format follows ## [Unreleased] +### Changed — the version 3 box format + +This is a **breaking wire change**, and the only one planned. Published v1 and v2 boxes stay +historical artefacts, usable with the Scrollcase versions that produced them; a v3 verifier refuses +either **by name**, saying which version it holds, rather than reinterpreting it. There is no +dual-read path anywhere, and no migration tool: a box is rebuilt from its scroll. + +- **A box declares its runtime.** `runtime: { id, version, entryPoint }` replaces `pythonVersion` + and `pythonEntryPoint` in the scroll, `box.json` and the signed release, and + `provenance.pythonVersion` becomes `provenance.runtimeVersion`. A version 2 box recorded a Python + interpreter path and Python execution kinds and nothing that said "Python", so a reader had to + infer the runtime from the shape of a path. `id` is one of `python`, `node` or `native`; only + `python` can be built or run today, and a box naming another is refused by name rather than + misread as the runtime it happens to be shaped like. Fixing the vocabulary now is what makes + implementing the other two code rather than a second wire break. + +- **`modelId` and `runtimeId` are gone**, replaced by an optional `labels` map that Scrollcase never + reads. Both were required and neither was ever read by any code path: they were a consumer's + vocabulary written into the format, so a box packaging a library still had to name a model, and + most scrolls set `modelId` to the `boxId` and moved on. A label says the same thing when there is + something to say and nothing when there is not. `modelCacheSubdir` becomes `cacheSubdir` for the + same reason. + +- **`weights` is gone; `assets[].embed` replaces it, per entry.** A box-wide switch could not ship a + small entry point inside the archive and defer a 30 GB dataset beside it, which is the case it + existed for. The `--weights` flag went with it rather than being kept: a build-time override of a + per-asset declaration repacks a box under an identity that no longer describes it, which is the + silent-repack bug the flag's own documentation already warned about. `assetArchives` gains no + `embed` field — an archive is expanded at build time, so deferring one names nothing that could + happen — which turns version 2's cross-field refusal into a schema-level impossibility. + +- **The self-test generalises.** `selfTest.pythonImports` put Python syntax in the wire format and + gave a runtime with no module system no way to state a check at all. The signed subset becomes + `selfTest.probe`, carrying `imports`, `commands`, or both; a command invokes the box's own + declared execution and names the exit status it must produce. In the scroll, `pythonFile` and + `pythonCode` become `script` and `code`. The runtime adapter is the only thing that turns a probe + into command lines. + +- **The executable bit is declared, not inferred.** `assets[].executable` and + `localFiles[].executable` replace the `venv/bin` heuristic that used to be the only way a payload + file could carry the bit. A downloaded file arrives with no permissions — HTTP carries content, + not modes — and a local file is copied rather than moved, so neither had one to inherit, and a box + could not ship an asset that runs. The mode is still *synthesised* rather than read off the build + machine, so two builds of one commit stay byte-identical whatever umask each ran under, and + `payload-digest.v1` is untouched. + +- **Extraction sets the mode explicitly.** All three consumers used to hand the archive's mode to + `open(2)`, which masks it by the process umask, except the Python one, which chmod'd. Under a + restrictive umask that made two of them silently drop the executable bit — a box that fails to run + for reasons nothing in it explains — and made the three disagree observably. Each now chmods after + writing, on non-Windows, and a conformance case extracts a declared-executable box under + `umask 077` and asserts the bit survives. + +- The execution union gains `node-script` and `native-binary` beside `python-script` and + `python-module`, and every schema `$id` and `$ref` moves from `/schema/v2/` to `/schema/v3/`. + +- Public API: `assertPythonEntryPoint` is replaced by `assertRuntimeEntryPoint(runtimeId, adapter, + entryPoint)`, and `scrollcase/contract` now exports the runtime model — `RUNTIME_IDS`, + `runtimeAdapter`, `runtimeAdapters`, `isImplementedRuntime`, `unimplementedRuntimeMessage`, + `executionAffectingVariables` and `isExecutablePayloadPath`. The Rust crate and the Python package + gained the equivalents. A `PreparedBox` receipt now carries `runtime` and `labels` in place of + `pythonEntryPoint`, `modelId` and `runtimeId`. + ### Changed - The box format now models the **runtime** separately from the **target**. A target says which From 97c0088f0834a504e0b2f2388047e25cd240c4eb Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:45:11 +0200 Subject: [PATCH 09/22] Update the repository's own instructions for version 3 AGENTS.md and CONTRIBUTING.md still described a v2-only line with a weights mode. The canonical terms gain runtime and deferred asset, the layout section names src/runtimes/ and runtimes.mjs, and the silently-breaking-paths list asks about embedded versus deferred rather than a mode that no longer exists. --- AGENTS.md | 40 ++++++++++++++++++++-------------- CONTRIBUTING.md | 9 ++++---- package.json | 2 +- rust/tests/release_document.rs | 2 +- 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4c23d09..623e221 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,18 +18,18 @@ dependency licence inventory. The substrate is **pixi + conda-pack + conda-forge**, and only that. `pixi` solves a committed `pixi.lock`, `conda-pack` relocates the resulting prefix, and the tree is extracted into the box's -`venv/`. The v2 CLI has thirteen verbs: `init`, `new`, `add`, `remove`, `edit`, `refresh`, `doctor`, +`venv/`. The CLI has thirteen verbs: `init`, `new`, `add`, `remove`, `edit`, `refresh`, `doctor`, `keygen`, `lock`, `audit`, `build`, `verify`, and `run`. Scrollcase is **a library as well as a CLI**. Its public Node surfaces include the contract, build -and signing APIs; v2 adds the local execution API at `scrollcase/consumer`. The Python consumer +and signing APIs, plus the local execution API at `scrollcase/consumer`. The Python consumer exposes the same semantics as `scrollcase_consumer`, and the Rust crate `scrollcase-consumer` under `rust/` exposes them again. A change to any public export is a change to a public API, in any of the three. -The accepted v2 design is authoritative even while the working tree is migrated in phases. v2 is a -clean break: do not add v1/v2 unions, legacy aliases, or dual code paths. Existing v1 releases remain -historical artefacts for the old Scrollcase versions that produced them. +The accepted v3 design is authoritative even while the working tree is migrated in phases. v3 is a +clean break: do not add v2/v3 unions, legacy aliases, or dual code paths. Published v1 and v2 +releases remain historical artefacts for the Scrollcase versions that produced them. It is open source and vendor-neutral, and must stay usable by projects that have nothing to do with the one that first needed it. @@ -60,10 +60,11 @@ This boundary is the whole point of the project, and it is the thing most likely namespace its clients recognise. Never hard-code one. 3. **One substrate.** No second dependency backend. Two backends means proving every guarantee twice, and the guarantees are the product. -4. **Published v1 is immutable; active development is v2-only.** The v2 verifier rejects - `schemaVersion: 1` clearly instead of reinterpreting it. Never silently edit a `kind` string, the - payload encoding, the signature algorithm, or a golden fixture. Any future breaking wire change - needs another new `schemaVersion`. +4. **Published v1 and v2 are immutable; active development is v3-only.** The v3 verifier rejects + either older `schemaVersion` clearly, and by name, instead of reinterpreting it — they are + different artefacts with different rebuilds ahead of them. Never silently edit a `kind` string, + the payload encoding, the signature algorithm, or a golden fixture. Any future breaking wire + change needs another new `schemaVersion`. 5. **Determinism is a promise.** Rebuilding the same commit must produce a byte-identical archive. Introduce nothing that varies per run: no clock read, no random value, no unsorted directory listing. @@ -132,10 +133,13 @@ afterwards.** - **box** — the built artefact. Never "image", never "container", never a consumer's product term. - **scroll** — the declarative input (`scroll.json`), the only input a build accepts. - **target** — the `(platform, arch, accelerator)` triple, plus `cudaVersion` for CUDA. +- **runtime** — what runs *inside* the box: `python`, `node` or `native`. The format names all + three; only `python` is implemented, and a box naming another is refused by name. - **payload** — the tree assembled before archiving. - **release / channel / revocations** — the three signed document types. - **self-test** — the import check run with the box's *own* interpreter. - **parity** — the optional cross-accelerator numerical gate. +- **deferred asset** — one the scroll declared `embed: false`. Never "weights": that field is gone. **Casing is functional, not cosmetic.** Write **Scrollcase** in prose, and `scrollcase` lowercase wherever it is an identifier: the command, the npm package, the exports, `scrollcase.config.json`, @@ -172,10 +176,13 @@ without reading each hit. ### Layout -- `src/contract/` — the format: target model and identity rule (`targets.mjs`), signed-document - envelope and namespacing (`documents.mjs`), `schema/`, `fixtures/`, generated `types/`. -- `src/build/` — solving and packing (`pixi.mjs`), toolchain bootstrap (`toolchain.mjs`), relocation - repair (`launchers.mjs`), archive and filesystem primitives, the lock-derived licence audit, +- `src/contract/` — the format: target model and identity rule (`targets.mjs`), runtime model — + layout, execution kinds, argv, self-test (`runtimes.mjs`), signed-document envelope and + namespacing (`documents.mjs`), `schema/`, `fixtures/`, generated `types/`. +- `src/runtimes//` — the builder-side half of a runtime: launcher repair, dependency reading, + authoring templates, the pixi dependency it contributes. Only `python/` exists. +- `src/build/` — solving and packing (`pixi.mjs`), toolchain bootstrap (`toolchain.mjs`), + archive and filesystem primitives, the lock-derived licence audit, workspace resolution, scroll authoring (`authoring.mjs`), reading and provenance (`scroll.mjs`), asset staging, the build core (`box.mjs`), `verify.mjs`, `audit.mjs`, `project.mjs` (init/doctor), and the parity gate. @@ -271,9 +278,10 @@ affect them, read it against each — even the ones you cannot execute here. 1. **The three targets.** macOS, Linux and Windows differ in interpreter layout (`venv/bin/python` vs `venv/python.exe`), scripts directory, launcher repair and native-library inspection. Anything touching packing, relocation or path handling must be checked against all three. -2. **`embed` vs `on-demand` weights.** The second leaves assets out of the archive, carries their - descriptors in the signed release, and refuses `assetArchives`. Asset staging and manifest - changes affect both. +2. **Embedded vs deferred assets.** `embed: false` leaves one asset out of the archive and carries + its descriptor in the signed release instead. It is per entry, so one box does both at once; + asset staging and manifest changes affect both halves, and a test whose assets are all embedded + covers half the behaviour. 3. **Local key vs external signer.** The external path must still echo back the exact payload it was given and verify locally before the build continues. 4. **Toolchain from `PATH` vs the project's own.** Discovery is flag > env > project toolchain > diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 996480e..3c824c7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,10 +7,11 @@ knowing them first will save you a rejected pull request. - **One substrate.** pixi + conda-pack + conda-forge, and only that. A second dependency backend means proving every guarantee twice, and the guarantees are the product. -- **Published v1 is immutable; the next major line is v2-only.** Existing v1 boxes stay with their old - Scrollcase versions. New code must not add a v1/v2 union, compatibility aliases, or dual paths; - the v2 verifier rejects v1 clearly. Never silently edit a `kind` string, payload encoding, - signature algorithm, or golden fixture under `src/contract/fixtures/`. +- **Published v1 and v2 are immutable; the current line is v3-only.** Existing boxes stay with the + Scrollcase versions that built them. New code must not add a v2/v3 union, compatibility aliases, + or dual paths; the v3 verifier rejects both older versions clearly, and by name. Never silently + edit a `kind` string, payload encoding, signature algorithm, or golden fixture under + `src/contract/fixtures/`. - **Determinism is a promise.** Rebuilding the same commit must produce a byte-identical archive. Do not introduce anything that varies per run: a clock read, a random value, an unsorted directory listing. diff --git a/package.json b/package.json index a19ef65..8dfb957 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "scrollcase", "version": "0.12.0", - "schemaVersion": "2", + "schemaVersion": "3", "description": "Pack an entire Python environment and the code it runs into a single, self-contained, portable and signed box.", "license": "Apache-2.0", "author": "Lorenzo S. (https://github.com/suffro)", diff --git a/rust/tests/release_document.rs b/rust/tests/release_document.rs index 2e3ba70..797296e 100644 --- a/rust/tests/release_document.rs +++ b/rust/tests/release_document.rs @@ -79,7 +79,7 @@ fn nothing_survives_an_edit_to_the_signed_bytes() { let restated = mutated(&directory, |document| { use base64::engine::general_purpose::STANDARD as BASE64; use base64::Engine as _; - let payload = br#"{"schemaVersion":2,"kind":"scrollcase.box.release"}"#; + let payload = br#"{"schemaVersion":3,"kind":"scrollcase.box.release"}"#; document["payloadBase64"] = serde_json::json!(BASE64.encode(payload)); document["payloadSha256"] = serde_json::json!(sha256_hex(payload)); }); From 4371ea3b2f2952a774c9b7c764e7818b502a00f1 Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:48:27 +0200 Subject: [PATCH 10/22] Make the Rust extractor set the mode explicitly, like the other two The last of the three consumers to hand the archive's mode to open(2), which masks it by the process umask. Under 077 a declared-executable box lost the bit in Rust and Node and kept it in Python: three implementations of one contract, disagreeing observably. The conformance case now runs under umask 077 in all three drivers. Removing the chmod from any one of them turns bin/tool from 755 into 700, which is what makes the case a guard rather than a description. --- rust/Cargo.toml | 8 ++++++++ rust/src/archive.rs | 15 +++++++++++---- rust/tests/conformance.rs | 24 ++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 68ef973..f372513 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -35,6 +35,14 @@ sha2 = "0.11" [target.'cfg(unix)'.dependencies] rustix = { version = "1.1.4", default-features = false, features = ["process", "std"] } +# The conformance suite extracts a declared-executable box under a restrictive umask, which is the +# condition the three consumers used to disagree under. Setting one needs a syscall `std` does not +# expose, and rustix puts it behind `fs`. Asked for here rather than above so a consuming +# application, which never compiles dev-dependencies, does not compile it either. +[target.'cfg(unix)'.dev-dependencies] +rustix = { version = "1.1.4", default-features = false, features = ["fs", "process", "std"] } + + [lints.rust] missing_docs = "warn" unsafe_code = "forbid" diff --git a/rust/src/archive.rs b/rust/src/archive.rs index 6aa6ab8..5990062 100644 --- a/rust/src/archive.rs +++ b/rust/src/archive.rs @@ -552,13 +552,20 @@ pub fn extract_zip_archive(archive_path: &Path, destination: &Path) -> Result<() #[cfg(unix)] fn new_file(path: &Path, mode: u32) -> Result { - use std::os::unix::fs::OpenOptionsExt as _; - std::fs::OpenOptions::new() + use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _}; + let mode = if mode == 0 { 0o644 } else { mode & 0o7777 }; + let file = std::fs::OpenOptions::new() .write(true) .create_new(true) - .mode(if mode == 0 { 0o644 } else { mode }) + .mode(mode) .open(path) - .map_err(|error| Error::new(format!("cannot write {}: {error}", path.display()))) + .map_err(|error| Error::new(format!("cannot write {}: {error}", path.display())))?; + // `open(2)` masks the mode it is given by the process umask, so a box extracted under 077 would + // silently lose the executable bit the archive states — and the box would fail to run for + // reasons nothing in it explains. Say the mode again, explicitly. + file.set_permissions(std::fs::Permissions::from_mode(mode)) + .map_err(|error| Error::new(format!("cannot set mode on {}: {error}", path.display())))?; + Ok(file) } #[cfg(not(unix))] diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index cee1fce..6e2efd7 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -499,6 +499,26 @@ fn report_value(report: &EnvironmentReport, names: &[String]) -> Value { }) } +/// Sets the process umask for as long as the returned guard lives, then puts back what was there. +#[cfg(unix)] +fn set_umask(octal: &str) -> UmaskGuard { + let mode = rustix::fs::Mode::from_bits_truncate(rustix::fs::RawMode::from_str_radix(octal, 8).unwrap()); + UmaskGuard(rustix::process::umask(mode)) +} + +#[cfg(unix)] +struct UmaskGuard(rustix::fs::Mode); + +#[cfg(unix)] +impl Drop for UmaskGuard { + fn drop(&mut self) { + rustix::process::umask(self.0); + } +} + +#[cfg(not(unix))] +fn set_umask(_octal: &str) -> () {} + fn receipt_value(prepared: &PreparedBox, expected: &Value, names: &[String]) -> Value { let mut receipt = json!({ "status": if prepared.status() == PreparedStatus::Prepared { "prepared" } else { "attached" }, @@ -947,6 +967,10 @@ fn run_case(case: &Value, patterns: &Map) -> Outcome { } } + // A restrictive umask is the condition under which the three consumers used to disagree: two + // applied the archive's mode through open(2) and lost it, one chmod'd and kept it. + let _umask = runtime.get("umask").and_then(Value::as_str).map(set_umask); + let state = Arc::new(Mutex::new(FakeSpawnState::default())); let spawner = FakeSpawner { exit_code: runtime From 10cbba3b230f3ffd71a8a665a93dab4e95f2d91a Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:17:45 +0200 Subject: [PATCH 11/22] Drop the removed --weights flag from the four box-building workflows Version 3 made where an asset lives a per-entry scroll declaration with no build-time override, and the flag went with it. Every demo and example workflow still passed it, so each would have failed at the first argument the CLI no longer knows. --- .github/workflows/demo-box.yml | 1 - .github/workflows/example-build.yml | 4 ++-- .github/workflows/llm-demo-box.yml | 1 - .github/workflows/sentiment-demo-box.yml | 1 - 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/demo-box.yml b/.github/workflows/demo-box.yml index bee5504..c2af746 100644 --- a/.github/workflows/demo-box.yml +++ b/.github/workflows/demo-box.yml @@ -92,7 +92,6 @@ jobs: run: | node src/cli.mjs build "$BOX_ID/${{ matrix.target }}" \ --scrolls-dir examples \ - --weights embed \ --private-key "$RUNNER_TEMP/demo-signing-private.pem" \ --public-key examples/keys/example-signing-public.json diff --git a/.github/workflows/example-build.yml b/.github/workflows/example-build.yml index a208860..67ca0da 100644 --- a/.github/workflows/example-build.yml +++ b/.github/workflows/example-build.yml @@ -102,7 +102,7 @@ jobs: run: node src/cli.mjs keygen - name: Build - run: node src/cli.mjs build hello-box/${{ matrix.target }} --scrolls-dir examples --weights embed + run: node src/cli.mjs build hello-box/${{ matrix.target }} --scrolls-dir examples # The check that matters: extract the box and import with the interpreter inside it. - name: Verify with self-test @@ -144,7 +144,7 @@ jobs: echo "Expected exactly one archive after the first build, found $before" >&2 exit 1 fi - node src/cli.mjs build hello-box/${{ matrix.target }} --scrolls-dir examples --weights embed + node src/cli.mjs build hello-box/${{ matrix.target }} --scrolls-dir examples after=$(ls "$dist"/*.zip | wc -l) if [ "$after" -ne 1 ]; then echo "Rebuild was not byte-identical: $after distinct archives" >&2 diff --git a/.github/workflows/llm-demo-box.yml b/.github/workflows/llm-demo-box.yml index 7a874ee..0c50820 100644 --- a/.github/workflows/llm-demo-box.yml +++ b/.github/workflows/llm-demo-box.yml @@ -116,7 +116,6 @@ jobs: df -h . || true node src/cli.mjs build "$BOX_ID/${{ matrix.target }}" \ --scrolls-dir examples \ - --weights embed \ --private-key "$RUNNER_TEMP/demo-signing-private.pem" \ --public-key examples/keys/example-signing-public.json diff --git a/.github/workflows/sentiment-demo-box.yml b/.github/workflows/sentiment-demo-box.yml index 2617090..ca1bbd6 100644 --- a/.github/workflows/sentiment-demo-box.yml +++ b/.github/workflows/sentiment-demo-box.yml @@ -107,7 +107,6 @@ jobs: run: | node src/cli.mjs build "$BOX_ID/${{ matrix.target }}" \ --scrolls-dir examples \ - --weights embed \ --private-key "$RUNNER_TEMP/demo-signing-private.pem" \ --public-key examples/keys/example-signing-public.json From be04c58463c4319cb967fdcd73fec9ed9048ae05 Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:10:39 +0200 Subject: [PATCH 12/22] Implement the native and node runtimes, and declare bundled licences MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase C of the version 3 work, on the Node side. Both runtimes arrive as adapters against the seam extracted in phase A: RUNTIME_ADAPTERS now holds the same three ids RUNTIME_IDS always named, and the wire did not move for either. A native box has no interpreter, so its layout names none and none is derived; its binary is the command itself; and a command probe is its only self-test, because there is no module system to ask for an import. Each of those is refused rather than ignored where a scroll declares otherwise. A node box is a python box with a different interpreter and no trampoline to repair, so the launcher step became the runtime's answer rather than pixi.mjs importing Python's directly. One thing the wire did need. pixi.lock declares a licence per conda package but cannot see what was linked into a binary a scroll supplies: that happened before Scrollcase saw the file, and reading the binary would be guessing. So a scroll may point at a declaration the project reviewed, the build checks every path it names is a file the box really carries, and the list is signed into the release — where a licence decision can be made before an archive is downloaded — as well as written beside the derived audit under THIRD_PARTY_NOTICES/. Authoring is runtime-dispatched throughout: --runtime chooses which execution kinds are offered, which starter is written, and which pixi dependency the generated manifest declares. --python-version becomes --runtime-version. --- .../public/schema/v3/box-manifest.schema.json | 3 + .../schema/v3/release-manifest.schema.json | 56 +++ docs/public/schema/v3/scroll.schema.json | 5 + docs/reference/cli.md | 46 ++- docs/white-paper.md | 60 ++- python/tests/conformance_support.py | 5 +- rust/tests/conformance.rs | 5 +- src/build/archive.d.mts | 15 + src/build/archive.mjs | 22 ++ src/build/authoring.mjs | 181 +++++++-- src/build/box.mjs | 95 ++++- src/build/licenses.d.mts | 32 ++ src/build/licenses.mjs | 74 +++- src/build/pixi.d.mts | 4 +- src/build/pixi.mjs | 20 +- src/build/scroll-edit.mjs | 36 +- src/build/scroll.mjs | 22 +- src/build/verify.d.mts | 4 +- src/build/verify.mjs | 5 +- src/cli-authoring.mjs | 50 ++- src/cli.mjs | 20 +- .../fixtures/consumer-conformance.json | 6 +- src/contract/fixtures/runtime-contract.json | 371 ++++++++++++++++++ src/contract/runtimes.d.mts | 63 ++- src/contract/runtimes.mjs | 321 +++++++++++++-- src/contract/schema/box-manifest.schema.json | 3 + .../schema/release-manifest.schema.json | 56 +++ src/contract/schema/scroll.schema.json | 5 + src/contract/types/index.d.ts | 55 +++ src/runtimes/index.d.mts | 68 ++++ src/runtimes/index.mjs | 52 ++- src/runtimes/launchers.d.mts | 10 + src/runtimes/launchers.mjs | 44 +++ src/runtimes/native/index.d.mts | 2 + src/runtimes/native/index.mjs | 37 ++ src/runtimes/node/index.d.mts | 2 + src/runtimes/node/index.mjs | 32 ++ src/runtimes/node/templates/index.d.mts | 26 ++ src/runtimes/node/templates/index.mjs | 49 +++ src/runtimes/python/index.d.mts | 2 + src/runtimes/python/index.mjs | 20 +- src/runtimes/python/templates/index.d.mts | 26 ++ tests/helpers/consumer-conformance.mjs | 5 +- tests/unit/build-pipeline.test.mjs | 184 ++++++++- tests/unit/contract-runtimes.test.mjs | 39 +- tests/unit/docs-contract.test.mjs | 2 +- tests/unit/project-surface.test.mjs | 5 +- tests/unit/scroll-authoring.test.mjs | 110 +++++- tests/unit/scroll-editing.test.mjs | 2 +- 49 files changed, 2135 insertions(+), 222 deletions(-) create mode 100644 src/runtimes/index.d.mts create mode 100644 src/runtimes/launchers.d.mts create mode 100644 src/runtimes/launchers.mjs create mode 100644 src/runtimes/native/index.d.mts create mode 100644 src/runtimes/native/index.mjs create mode 100644 src/runtimes/node/index.d.mts create mode 100644 src/runtimes/node/index.mjs create mode 100644 src/runtimes/node/templates/index.d.mts create mode 100644 src/runtimes/node/templates/index.mjs create mode 100644 src/runtimes/python/index.d.mts create mode 100644 src/runtimes/python/templates/index.d.mts diff --git a/docs/public/schema/v3/box-manifest.schema.json b/docs/public/schema/v3/box-manifest.schema.json index 6060ec5..de6342c 100644 --- a/docs/public/schema/v3/box-manifest.schema.json +++ b/docs/public/schema/v3/box-manifest.schema.json @@ -40,6 +40,9 @@ "type": "string", "minLength": 1 }, + "bundledLicenses": { + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/bundledLicenses" + }, "environment": { "type": "object", "description": "Environment variables repeated from the signed release. Scrollcase consumers compare this declaration before execution and apply it over inherited and caller-supplied values.", diff --git a/docs/public/schema/v3/release-manifest.schema.json b/docs/public/schema/v3/release-manifest.schema.json index 90d1b23..24df66c 100644 --- a/docs/public/schema/v3/release-manifest.schema.json +++ b/docs/public/schema/v3/release-manifest.schema.json @@ -135,6 +135,9 @@ "minLength": 1, "description": "Directory relative to the extracted box root holding the box's own large files." }, + "bundledLicenses": { + "$ref": "#/$defs/bundledLicenses" + }, "environment": { "type": "object", "description": "Signed environment variables applied whenever Scrollcase runs the box interpreter. These values override both the inherited host environment and caller-supplied values.", @@ -265,6 +268,59 @@ } } }, + "bundledLicenses": { + "type": "array", + "minItems": 1, + "description": "Dependencies compiled inside the binaries this box ships, as the publishing project declared them. The conda environment's own licences are derived from pixi.lock and travel inside the payload; this list is the half no lock can see — code linked into a supplied executable before the build began — so it is declared, reviewed by the project, and signed here unchanged. It is carried in the release rather than only in the payload so that a licence decision can be made from the document alone, before an archive is downloaded. A box that declares none carries no such list, which means the project declared none and never that the box has no dependencies.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "version", + "declaredLicense", + "linkedInto" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "The dependency as its own project names it." + }, + "version": { + "type": "string", + "minLength": 1 + }, + "declaredLicense": { + "type": "string", + "minLength": 1, + "description": "The licence the project reviewed, conventionally an SPDX expression. Scrollcase carries the string through and never parses it: what a licence permits is not a question a packaging tool answers.", + "examples": [ + "Apache-2.0 OR MIT" + ] + }, + "linkedInto": { + "type": "array", + "minItems": 1, + "description": "Payload files this dependency is compiled into. Every path must be a file the built box actually carries, which is what makes the declaration checkable rather than a free-text notice.", + "items": { + "$ref": "#/$defs/payloadPath" + } + }, + "sourceUrl": { + "type": "string", + "minLength": 1, + "description": "Where the dependency's source can be obtained, for a licence that requires the offer." + } + } + } + }, + "payloadPath": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*//).+$", + "description": "A forward-slash path inside the box payload: relative, non-empty, and unable to escape the payload root." + }, "deferredAssets": { "type": "array", "minItems": 1, diff --git a/docs/public/schema/v3/scroll.schema.json b/docs/public/schema/v3/scroll.schema.json index 57239f3..74a6464 100644 --- a/docs/public/schema/v3/scroll.schema.json +++ b/docs/public/schema/v3/scroll.schema.json @@ -105,6 +105,11 @@ "minLength": 1, "description": "Path to the reviewed licence inventory derived from pixi.lock, which carries an SPDX licence per package. The build fails if the lock no longer matches what was reviewed." }, + "bundledLicenseDeclaration": { + "type": "string", + "minLength": 1, + "description": "Path to the project's inventory of dependencies compiled *inside* the binaries this box ships. pixi.lock declares a licence per conda package, but it cannot see what was linked into a supplied executable before the build ever started, and nothing Scrollcase can read will tell it. So this half is declared rather than derived: the file is a JSON array of { name, version, declaredLicense, linkedInto } entries, and the build checks that every path it names is really in the box before carrying the list into the signed release. What belongs in it is the project's judgement; Scrollcase transports and signs what the project reviewed and never decides what a complete inventory is." + }, "cacheSubdir": { "type": "string", "minLength": 1, diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 16bec23..f5d767b 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -178,40 +178,54 @@ scrollcase new scroll \ | Flag | Meaning | | --- | --- | | `--target` | Complete canonical target; CUDA IDs include the ABI, such as `linux-x86_64-cuda12.4` | +| `--runtime` | `python`, `node` or `native`. Defaults to `python` | | `--box-id` | Box identity and parent directory | | `--source-revision` | Upstream revision recorded in provenance | | `--asset-base-url` | Base URL copied into built release metadata | | `--labels` | JSON object of free-form annotations carried into the signed release. Scrollcase reads none of them | | `--version` | Box version. Defaults to `1.0.0` | | `--scroll-version` | Version of the authoring input. Defaults to `1.0.0` | -| `--python-version` | Python dependency version written into `pixi.toml`, or `latest`. Defaults to one minor behind the newest Python conda-forge publishes | +| `--runtime-version` | Interpreter version written into `pixi.toml`, or `latest`. Refused for `native`, which installs no interpreter | | `--pixi-version` | Exact resolver version required by `lock` and `build`. Defaults to the installed pixi's version | | `--min-host-app-version` | Optional compatibility floor | | `--max-host-app-version-exclusive` | Optional compatibility ceiling | | `--min-macos-version` | Optional macOS floor | | `--min-ram-gb` | Optional positive RAM requirement | | `--min-nvidia-driver-version` | Optional NVIDIA driver floor | -| `--execution` | `python-script`, `python-module`, or `library-only` | -| `--script` | Existing project-relative Python script | -| `--generate-script` | Generate a minimal starter instead of using an existing script | -| `--script-destination` | Safe payload path, default `entrypoint.py` | -| `--generated-script-path` | Project path for the generated source; defaults to `box-entrypoints///entrypoint.py` | +| `--execution` | The runtime's own kinds, plus `library-only` where it applies — see the table below | +| `--script` | Existing project-relative file the box runs | +| `--generate-script` | Generate a minimal starter instead of using an existing file | +| `--script-destination` | Safe payload path; defaults to the runtime's own starter name | +| `--generated-script-path` | Project path for the generated source; defaults to `box-entrypoints///` | | `--module` | Strict dotted Python module name | | `--default-args` | JSON array of default application arguments | -For `python-script`, choose exactly one of `--script` and `--generate-script`. Scrollcase records the -source in `localFiles` **without a hash pin**, so the first edit to a freshly generated script does -not fail its own build; add `sha256` yourself for a file that must not change without review. It -refuses traversal and non-regular sources, and never overwrites an existing source or scroll. -Generated defaults are grouped by both box and target; `library-only` omits execution metadata. +The runtime decides what the rest of the session offers: -Alongside `scroll.json` and `pixi.toml`, `new scroll` writes a `self_test.py` next to them and -points `selfTest.script` at it, so the box's own check starts life as real Python rather than -an escaped JSON string. +| Runtime | `--execution` | Starter | Default `--runtime-version` | +| --- | --- | --- | --- | +| `python` | `python-script`, `python-module`, `library-only` | `entrypoint.py`, `self_test.py` | one minor behind the newest Python conda-forge publishes | +| `node` | `node-script`, `library-only` | `entrypoint.js`, `self_test.js` | the current Node LTS line | +| `native` | `native-binary` | none — point `--script` at the binary you built | none; a native box installs no interpreter | -`--python-version latest` resolves once, at authoring time, and writes the resulting number into the +`native` offers no `library-only` because it could not then prove anything about itself: a native box +has no module system, so an invocation of its own binary is its only self-test, and that needs an +execution to invoke. + +For every kind that names a file, choose exactly one of `--script` and `--generate-script`. +Scrollcase records the source in `localFiles` **without a hash pin**, so the first edit to a freshly +generated script does not fail its own build; add `sha256` yourself for a file that must not change +without review. It refuses traversal and non-regular sources, and never overwrites an existing +source or scroll. A `native-binary` entry is additionally recorded `executable: true`, because a +copied file carries no mode of its own and the box could not otherwise start. + +Alongside `scroll.json` and `pixi.toml`, a runtime with a starter gets a self-test file written next +to them with `selfTest.script` pointing at it, so the box's own check starts life as real source in +the runtime's own language rather than an escaped JSON string. + +`--runtime-version latest` resolves once, at authoring time, and writes the resulting number into the scroll — never the word `latest`. Both it and the default are constants moved deliberately at each -Scrollcase release by `npm run python:bump`, which asks conda-forge what it publishes: a version +Scrollcase release; for Python, `npm run python:bump` asks conda-forge what it publishes. A version looked up on every invocation would make the same command produce different scrolls in different months. diff --git a/docs/white-paper.md b/docs/white-paper.md index 3798593..308d59e 100644 --- a/docs/white-paper.md +++ b/docs/white-paper.md @@ -1719,21 +1719,61 @@ launcher repair, authoring templates, the pixi dependency a runtime contributes #### Two lists, on purpose `RUNTIME_IDS` is the vocabulary a box may declare — `python`, `node`, `native` — and it is fixed by -the wire format. `RUNTIME_ADAPTERS` is what this build can actually run, and today that is `python` -alone: `runtimeAdapter('node')` is a `TypeError` rather than a stub, because a registry that -answered for a runtime no build can produce would move the failure somewhere further down, where the -message no longer says what went wrong. +the wire format. `RUNTIME_ADAPTERS` is what this build can actually run. They now hold the same +three, which is exactly what the split was for: `node` and `native` arrived as adapters and the wire +did not move. -Keeping the two apart is what makes implementing `node` code rather than another format break. A box -declaring a runtime with no adapter is refused by name — `isImplementedRuntime()` asks the question -and `unimplementedRuntimeMessage()` gives the one wording the builder and all three consumers use — -and never misread as the runtime it happens to be shaped like. +The two lists stay separate because they answer to different release cycles. The Python and Rust +consumers version independently of the builder, so one published before a runtime landed still has +to refuse a box naming it — `isImplementedRuntime()` asks the question and +`unimplementedRuntimeMessage()` gives the one wording the builder and all three consumers use — and +never misread it as the runtime it happens to be shaped like. Reference: `tests/unit/contract-runtimes.test.mjs`, `rust/tests/contract.rs`, `python/tests/test_contract.py`. +
+ +#### What the three runtimes actually differ in + +| | `python` | `node` | `native` | +| --- | --- | --- | --- | +| Entry point | `venv/bin/python`, `venv/python.exe` | `venv/bin/node`, `venv/node.exe` | **none** | +| Execution kinds | `python-script`, `python-module` | `node-script` | `native-binary` | +| argv | interpreter, then the declaration | interpreter, then the declaration | the binary *is* the command | +| Self-test probes | `imports`, `commands` | `imports`, `commands` | `commands` only | +| Import probe source | `python -c "import a, b"` | `node -e "require('a')"` | — | +| Environment variables | `PYTHON*` | `NODE_OPTIONS`, `NODE_PATH`, `NODE_EXTRA_CA_CERTS` | none of its own | +| pixi dependency | `python` | `nodejs` | **none** | +| Launcher repair | rewrites the conda trampoline | scans and refuses | scans and refuses | +| Generated starter | `entrypoint.py`, `self_test.py` | `entrypoint.js`, `self_test.js` | **none** | + +Two of those rows are the whole of what `native` means, and both propagate. Its layout's +`entryPoint` and `standardLibrary` are `null` rather than a plausible-looking path nothing would +find, so `assertRuntimeEntryPoint` gains a third answer: a runtime with an interpreter admits +exactly one value, a runtime without one admits none and **refuses** a declaration rather than +ignoring it, and a box that declares nothing at all is checked against nothing, because +`runtime.entryPoint` is optional on the wire for exactly this reason. And its only probe shape is +`commands`, so an `imports` probe in a native box is refused where the scroll is read — +`unsupportedSelfTestProbeMessage()` — rather than silently dropped, which would report a pass for a +check that never ran. + +A native box is not "no environment", only "no interpreter". It is built from a `pixi.lock` like +every other box, its binary links against the shared libraries that lock installed, and those +libraries get the same derived licence audit. What it contributes to `[dependencies]` is nothing at +all: only the person who compiled the binary knows what it needs. + +**Link repair is deliberately out of scope.** A binary that resolves its libraries through an +absolute path recorded at compile time will not find them inside a box, and fixing that means +per-format work — rpath on Linux, `install_name` on macOS, the DLL search order on Windows — that +deserves its own pass rather than a guess. A native box must ship a binary that already resolves: +statically linked, or built with a relative rpath. This is a stated limitation, not an assumption +left for someone to discover. + +
+
### 5.3 The envelope — `document-shape.mjs` and `documents.mjs` @@ -6244,10 +6284,14 @@ what a box's runtime is allowed to need that another runtime would not. | Module | Role | Section | | --- | --- | --- | | `src/runtimes/index.mjs` | The registry of builder-side runtime adapters | 6.6 | +| `src/runtimes/launchers.mjs` | The launcher check shared by the runtimes that cannot rewrite one | 6.6 | | `src/runtimes/python/index.mjs` | The Python adapter: its pixi dependency, its launcher repair, its starter files | 6.6 | | `src/runtimes/python/launchers.mjs` | Repairing the console scripts a conda environment generates | 6.6 | | `src/runtimes/python/dependencies.mjs` | Reading a pip `requirements.txt` into conda-forge terms | 6.16 | | `src/runtimes/python/templates/index.mjs` | The Python source `new scroll` writes, and the interpreter constraint a generated manifest declares | 6.16 | +| `src/runtimes/node/index.mjs` | The Node adapter: `nodejs` from conda-forge, and nothing to repair | 6.6 | +| `src/runtimes/node/templates/index.mjs` | The JavaScript source `new scroll` writes, and the Node constraint a generated manifest declares | 6.16 | +| `src/runtimes/native/index.mjs` | The native adapter: no interpreter, no dependency of its own, nothing to generate | 6.6 |
diff --git a/python/tests/conformance_support.py b/python/tests/conformance_support.py index 4985479..25c6e8f 100644 --- a/python/tests/conformance_support.py +++ b/python/tests/conformance_support.py @@ -228,8 +228,9 @@ def _mutate_fixture( fixture.sign() return if mutation == "alter-release-runtime-id": - # A runtime the format names and this package has no adapter for. The consumer must refuse - # the box rather than read it as the runtime it happens to be shaped like. + # A Python box relabelled as native after it was built. Everything about the payload + # still says Python, so the consumer must refuse it rather than read the declaration as + # the truth about a box that disagrees with it. fixture.release["runtime"] = {**fixture.release["runtime"], "id": "native"} fixture.sign() return diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index 6e2efd7..2a1ce95 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -689,8 +689,9 @@ fn mutate_fixture(fixture: &mut Fixture, mutation: &str, destination: &Path) { fixture.release["runtime"]["version"] = json!("3.99.0"); fixture.sign(); } - // A runtime the format names and this crate has no adapter for. The consumer must refuse - // the box rather than read it as the runtime it happens to be shaped like. + // A Python box relabelled as native after it was built. Everything about the payload + // still says Python, so the consumer must refuse it rather than read the declaration as + // the truth about a box that disagrees with it. "alter-release-runtime-id" => { fixture.release["runtime"]["id"] = json!("native"); fixture.sign(); diff --git a/src/build/archive.d.mts b/src/build/archive.d.mts index 1d26da9..73a1aa2 100644 --- a/src/build/archive.d.mts +++ b/src/build/archive.d.mts @@ -1,3 +1,18 @@ +/** + * Whether the archive will give one payload path the executable bit. + * + * Asked before the archive is written, by the one check that matters for a box nobody can start: + * the file a box runs has to come out of the archive runnable. A Windows target carries no modes at + * all — `archiveFileMode` writes 0644 for every entry there and Windows decides executability by + * extension — so the question does not arise and the answer is yes. + * + * @param {import('../contract/targets.mjs').BoxTargetAdapter} adapter + * @param {string} runtimeId + * @param {readonly string[]} declared payload paths the scroll marked executable + * @param {string} relativePath + * @returns {boolean} + */ +export function archiveMarksExecutable(adapter: import("../contract/targets.mjs").BoxTargetAdapter, runtimeId: string, declared: readonly string[], relativePath: string): boolean; /** * Streams a deterministic, Zip64-capable box archive using the pinned Node backend. * diff --git a/src/build/archive.mjs b/src/build/archive.mjs index a11aca4..e34d111 100644 --- a/src/build/archive.mjs +++ b/src/build/archive.mjs @@ -66,6 +66,28 @@ function declaredExecutablePaths(runtimeId, adapter, declared) { return { files: [...rule.files, ...declared], directories: rule.directories }; } +/** + * Whether the archive will give one payload path the executable bit. + * + * Asked before the archive is written, by the one check that matters for a box nobody can start: + * the file a box runs has to come out of the archive runnable. A Windows target carries no modes at + * all — `archiveFileMode` writes 0644 for every entry there and Windows decides executability by + * extension — so the question does not arise and the answer is yes. + * + * @param {import('../contract/targets.mjs').BoxTargetAdapter} adapter + * @param {string} runtimeId + * @param {readonly string[]} declared payload paths the scroll marked executable + * @param {string} relativePath + * @returns {boolean} + */ +export function archiveMarksExecutable(adapter, runtimeId, declared, relativePath) { + if (adapter.host.platform === 'win32') return true; + return isExecutablePayloadPath( + declaredExecutablePaths(runtimeId, adapter, declared), + relativePath, + ); +} + /** * Whether a payload path was declared as one whose bytes are already compressed. * diff --git a/src/build/authoring.mjs b/src/build/authoring.mjs index bfe8b9d..c597e64 100644 --- a/src/build/authoring.mjs +++ b/src/build/authoring.mjs @@ -9,13 +9,17 @@ * * What is generated is deliberately short. A scroll that restates what the target already implies is * a scroll nobody wants to write by hand, so anything `readScroll` can derive is left out and the - * self-test is written as a real Python file rather than escaped into a JSON string. + * self-test is written as a real source file in the runtime's own language rather than escaped into + * a JSON string. Which language that is, what the starter says, and whether there is one at all are + * the runtime's answers: a native box has no source to generate, so it is pointed at a binary that + * already exists. */ import { lstat, mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; import { dirname, isAbsolute, join, relative, sep } from 'node:path'; import { boxTargetAdapter, boxTargetId, condaSubdir } from '../contract/targets.mjs'; import { BOX_SCHEMA_VERSION } from '../contract/documents.mjs'; +import { isImplementedRuntime, runtimeAdapter, unimplementedRuntimeMessage } from '../contract/runtimes.mjs'; import { runtimeBuilder } from '../runtimes/index.mjs'; import { fileExists, safeRelativePath } from './filesystem.mjs'; import { fail } from './process.mjs'; @@ -24,16 +28,41 @@ import { schemaValidationError } from './schema-validation.mjs'; const scrollSchemaUrl = new URL('../contract/schema/scroll.schema.json', import.meta.url); const targetSchemaUrl = new URL('../contract/schema/target.schema.json', import.meta.url); const executionSchemaUrl = new URL('../contract/schema/execution.schema.json', import.meta.url); -const EXECUTION_KINDS = Object.freeze(['python-script', 'python-module', 'library-only']); /** - * The runtime `scrollcase new` writes. Authoring is deliberately narrower than the format: the - * wire vocabulary names every runtime the format defines, and this names the one a generated - * scroll can actually be built from today. + * A box with no entry point at all: the one execution choice that belongs to no runtime, because it + * is the choice not to declare one. Offered wherever it makes sense, which is wherever the box can + * still prove something about itself without being started. */ -export const AUTHORED_RUNTIME_ID = 'python'; +const LIBRARY_ONLY = 'library-only'; + +/** + * The runtime `scrollcase new` writes when nobody says otherwise. + * + * Python, because it is what the overwhelming majority of boxes are and because a default that + * silently changed under existing users would be a worse kind of surprise than typing a flag. + */ +export const DEFAULT_RUNTIME_ID = 'python'; export const EXAMPLE_PIXI_VERSION = '0.73.0'; export const DEFAULT_SCROLL_VERSION = '1.0.0'; +/** + * What `scrollcase new scroll --execution` may be given, per runtime. + * + * Derived from the runtime's own execution kinds rather than listed here, so a runtime cannot be + * offered a shape it does not define. `library-only` is added for the runtimes that can still + * self-test without it — a native box cannot, since a command probe is its only probe and a command + * probe needs an execution to invoke, so offering the choice would be offering an invalid scroll. + * + * @param {string} runtimeId + * @returns {string[]} + */ +export function authoredExecutionKinds(runtimeId) { + const kinds = [...runtimeAdapter(runtimeId).executionKinds]; + return runtimeAdapter(runtimeId).selfTestProbeKinds.includes('imports') + ? [...kinds, LIBRARY_ONLY] + : kinds; +} + /** * The Python a new scroll asks for when nobody says otherwise. * @@ -59,16 +88,43 @@ export const DEFAULT_PYTHON_VERSION = '3.14'; export const LATEST_PYTHON_VERSION = '3.15'; /** - * Turns a requested Python version into the one a scroll records. + * The Node a new scroll asks for, and what `latest` means for it. + * + * Same reasoning as the Python pair above, applied to a project that releases differently: Node's + * even-numbered lines are the ones with long-term support, so the default is the current LTS rather + * than one minor behind the newest. conda-forge builds both. + */ +export const DEFAULT_NODE_VERSION = '22'; +export const LATEST_NODE_VERSION = '24'; + +/** + * The version a generated scroll asks for, per runtime, and what `latest` resolves to. * + * `native` has neither, and that is not an omission: there is no runtime to install, so there is no + * version to pin, and `runtime.version` is legitimately absent from the scroll it writes. + */ +const RUNTIME_VERSIONS = Object.freeze({ + python: Object.freeze({ default: DEFAULT_PYTHON_VERSION, latest: LATEST_PYTHON_VERSION }), + node: Object.freeze({ default: DEFAULT_NODE_VERSION, latest: LATEST_NODE_VERSION }), + native: null, +}); + +/** + * Turns a requested runtime version into the one a scroll records. + * + * @param {string} runtimeId * @param {string | null | undefined} requested a version, `latest`, or nothing - * @returns {string} + * @returns {string | null} null for a runtime that has no version to record */ -export function resolvePythonVersion(requested) { - if (requested === null || requested === undefined || requested === '') { - return DEFAULT_PYTHON_VERSION; +export function resolveRuntimeVersion(runtimeId, requested) { + const versions = RUNTIME_VERSIONS[runtimeId]; + const asked = requested === '' ? null : requested ?? null; + if (!versions) { + if (asked !== null) fail(`The ${runtimeId} runtime installs no interpreter, so it has no version to pin.`); + return null; } - return requested === 'latest' ? LATEST_PYTHON_VERSION : requested; + if (asked === null) return versions.default; + return asked === 'latest' ? versions.latest : asked; } const TYPESCRIPT_CONSUMER_TEMPLATE = `/** @@ -237,10 +293,16 @@ function projectRelativePath(projectRoot, path) { return relativePath.split(sep).join('/'); } -function pixiManifest(environmentName, target, runtimeVersion, runtimeId = AUTHORED_RUNTIME_ID) { +function pixiManifest(environmentName, target, runtimeVersion, runtimeId) { // The workspace table is substrate — one channel, one platform, whatever the box runs. Only the - // dependency line knows which runtime is being packed, and the runtime is what writes it. + // dependency line knows which runtime is being packed, and the runtime is what writes it. A + // runtime that installs nothing of its own writes none, and the table is left for the author: + // a native box's environment holds the libraries its binary links against and nothing else, and + // only the person who compiled it knows what those are. const runtime = runtimeBuilder(runtimeId).pixiDependency(runtimeVersion); + const dependencies = runtime + ? `${runtime.name} = "${runtime.spec}"\n` + : '# Add the libraries this box\'s binary links against.\n'; return `# Solved by \`scrollcase lock\` into pixi.lock, which is committed and reviewed. # \`platforms\` must equal the target's conda subdirectory, or the solve produces an environment # that cannot run on the machine the box is for. @@ -250,8 +312,7 @@ channels = ["conda-forge"] platforms = ["${condaSubdir(target)}"] [dependencies] -${runtime.name} = "${runtime.spec}" -`; +${dependencies}`; } async function validateScroll(scroll) { @@ -278,7 +339,8 @@ export async function createScroll({ version, scrollVersion = DEFAULT_SCROLL_VERSION, sourceRevision, - pythonVersion = DEFAULT_PYTHON_VERSION, + runtimeId = DEFAULT_RUNTIME_ID, + runtimeVersion, pixiVersion, compatibility = {}, assetBaseUrl, @@ -286,7 +348,7 @@ export async function createScroll({ scriptSourcePath = null, generateScript = false, generatedScriptSourcePath = null, - scriptRelativePath = 'entrypoint.py', + scriptRelativePath = null, module = null, defaultArgs = [], }) { @@ -295,12 +357,19 @@ export async function createScroll({ fail('No initialized Scrollcase workspace; run scrollcase init first.'); } + if (!isImplementedRuntime(runtimeId)) fail(unimplementedRuntimeMessage(runtimeId)); + const builder = runtimeBuilder(runtimeId); + const resolvedRuntimeVersion = runtimeVersion === undefined + ? resolveRuntimeVersion(runtimeId, null) + : resolveRuntimeVersion(runtimeId, runtimeVersion); const identity = { boxId: requiredText(boxId, 'boxId'), version: requiredText(version, 'version'), scrollVersion: requiredText(scrollVersion, 'scrollVersion'), sourceRevision: requiredText(sourceRevision, 'sourceRevision'), - pythonVersion: requiredText(pythonVersion, 'pythonVersion'), + runtimeVersion: resolvedRuntimeVersion === null + ? null + : requiredText(resolvedRuntimeVersion, 'runtimeVersion'), pixiVersion: requiredText(pixiVersion, 'pixiVersion'), // Required whether or not any asset is deferred: the release manifest names the archive's own // published URL, not just the assets'. @@ -312,8 +381,9 @@ export async function createScroll({ if (!labels || typeof labels !== 'object' || Array.isArray(labels)) { fail('labels must be an object.'); } - if (!EXECUTION_KINDS.includes(executionKind)) { - fail(`Unsupported execution kind: ${executionKind}. Use ${EXECUTION_KINDS.join(', ')}.`); + const executionKinds = authoredExecutionKinds(runtimeId); + if (!executionKinds.includes(executionKind)) { + fail(`Unsupported execution kind for a ${runtimeId} box: ${executionKind}. Use ${executionKinds.join(', ')}.`); } if (!Array.isArray(defaultArgs) || defaultArgs.some((value) => typeof value !== 'string')) { fail('defaultArgs must be an array of strings.'); @@ -326,27 +396,45 @@ export async function createScroll({ const scrollDir = join(workspace.scrollsDir, identity.boxId, targetId); if (await fileExists(scrollDir)) fail(`Scroll already exists: ${scrollRef}.`); + // Every kind that names a payload file — a Python script, a Node script, a compiled binary — + // resolves the same way: an existing project file, or one Scrollcase writes a starter for. Only + // the field it lands in and whether generation is possible differ, so the shape is stated once + // and the differences are read off the runtime. + const FILE_KINDS = Object.freeze({ + 'python-script': { field: 'script', executable: false }, + 'node-script': { field: 'script', executable: false }, + // The one entry that needs the bit. A native box runs this file directly, and a downloaded or + // copied file carries no mode of its own — so if the scroll does not say it is executable, the + // archive will not mark it and the box will not start. + 'native-binary': { field: 'binary', executable: true }, + }); + let localFile = null; let execution; let generatedScriptPath = null; let generatedSource = null; - if (executionKind === 'python-script') { + const fileKind = FILE_KINDS[executionKind]; + if (fileKind) { if (generateScript && scriptSourcePath) { - fail('Choose either an existing script or --generate-script, not both.'); + fail('Choose either an existing file or --generate-script, not both.'); + } + if (generateScript && !builder.templates) { + fail(`Scrollcase cannot generate an entry point for a ${runtimeId} box; point --script at the binary you built.`); } if (!generateScript && !scriptSourcePath) { - fail('python-script execution requires an existing script or --generate-script.'); + fail(`${executionKind} execution requires an existing file${builder.templates ? ' or --generate-script' : ''}.`); } - const relativePath = safeRelativePath(scriptRelativePath); + const defaultFileName = builder.templates?.scriptFileName ?? 'entrypoint'; + const relativePath = safeRelativePath(scriptRelativePath ?? defaultFileName); let sourcePath; if (generateScript) { sourcePath = safeRelativePath(generatedScriptSourcePath - ?? `box-entrypoints/${identity.boxId}/${targetId}/entrypoint.py`); + ?? `box-entrypoints/${identity.boxId}/${targetId}/${defaultFileName}`); generatedScriptPath = join(workspace.root, ...sourcePath.split('/')); if (await fileExists(generatedScriptPath)) { fail(`Generated script already exists: ${sourcePath}.`); } - generatedSource = runtimeBuilder(AUTHORED_RUNTIME_ID).templates.script; + generatedSource = builder.templates.script; } else { sourcePath = safeRelativePath(scriptSourcePath); const source = join(workspace.root, ...sourcePath.split('/')); @@ -354,17 +442,21 @@ export async function createScroll({ try { details = await lstat(source); } catch { - fail(`Project script is missing: ${sourcePath}.`); + fail(`Project file is missing: ${sourcePath}.`); } if (!details.isFile() || details.isSymbolicLink()) { - fail(`Project script must be a regular file: ${sourcePath}.`); + fail(`Project file must be a regular file: ${sourcePath}.`); } } // No sha256: this file is the one the author is about to start editing, and pinning it here // would make the first edit fail the build. A project pins a file it wants frozen by adding the // hash itself. - localFile = { sourcePath, relativePath }; - execution = { kind: 'python-script', script: relativePath, defaultArgs: [...defaultArgs] }; + localFile = { sourcePath, relativePath, ...(fileKind.executable ? { executable: true } : {}) }; + execution = { + kind: executionKind, + [fileKind.field]: relativePath, + defaultArgs: [...defaultArgs], + }; } else if (executionKind === 'python-module') { execution = { kind: 'python-module', @@ -378,7 +470,16 @@ export async function createScroll({ // Everything the target or the identity already determines is left out: a generated scroll should // read like the decisions its author made, and `readScroll` derives the rest. What stays is what a // person had to choose. - const selfTestPath = projectRelativePath(workspace.root, join(scrollDir, 'self_test.py')); + // The generated probe is the weakest true statement the runtime can make about a fresh box: for + // one with a module system, that its own standard library loads; for one without, that the binary + // the scroll names starts and exits cleanly. Both are placeholders the author is meant to replace, + // and neither pretends to have checked anything the box actually does. + const selfTestPath = builder.templates + ? projectRelativePath(workspace.root, join(scrollDir, builder.templates.selfTestFileName)) + : null; + const probe = builder.templates + ? { imports: [builder.templates.starterImport] } + : { commands: [{ args: [] }] }; const scroll = { $schema: 'https://scrollcase.dev/schema/v3/scroll.schema.json', schemaVersion: BOX_SCHEMA_VERSION, @@ -391,11 +492,14 @@ export async function createScroll({ ? {} : { scrollVersion: identity.scrollVersion }), ...(Object.keys(compatibility).length > 0 ? { compatibility: { ...compatibility } } : {}), - runtime: { id: AUTHORED_RUNTIME_ID, version: identity.pythonVersion }, + runtime: { + id: runtimeId, + ...(identity.runtimeVersion === null ? {} : { version: identity.runtimeVersion }), + }, pixiVersion: identity.pixiVersion, assetBaseUrl: identity.assetBaseUrl, selfTest: { - imports: ['json'], + ...probe, ...(localFile ? { files: [localFile.relativePath] } : {}), ...(selfTestPath ? { script: selfTestPath } : {}), }, @@ -412,13 +516,10 @@ export async function createScroll({ await writeFile(join(staging, 'scroll.json'), `${JSON.stringify(scroll, null, 2)}\n`); await writeFile( join(staging, 'pixi.toml'), - pixiManifest(`${identity.boxId}-${targetId}`, target, identity.pythonVersion), + pixiManifest(`${identity.boxId}-${targetId}`, target, identity.runtimeVersion, runtimeId), ); if (selfTestPath) { - await writeFile( - join(staging, 'self_test.py'), - runtimeBuilder(AUTHORED_RUNTIME_ID).templates.selfTest, - ); + await writeFile(join(staging, builder.templates.selfTestFileName), builder.templates.selfTest); } if (generatedScriptPath) { await mkdir(dirname(generatedScriptPath), { recursive: true }); @@ -435,7 +536,7 @@ export async function createScroll({ const written = [ join(scrollDir, 'scroll.json'), join(scrollDir, 'pixi.toml'), - ...(selfTestPath ? [join(scrollDir, 'self_test.py')] : []), + ...(selfTestPath ? [join(scrollDir, builder.templates.selfTestFileName)] : []), ...(generatedScriptPath ? [generatedScriptPath] : []), ]; return { written, scroll, scrollDir, scrollRef, targetId, generatedScriptPath }; diff --git a/src/build/box.mjs b/src/build/box.mjs index d758f64..0841380 100644 --- a/src/build/box.mjs +++ b/src/build/box.mjs @@ -30,7 +30,7 @@ import { } from '../contract/payload-digest.mjs'; import { signDocument } from '../sign/index.mjs'; import { copyVerifiedLocalFile, downloadVerified, expandAssetArchive, moveIntoPlace } from './assets.mjs'; -import { createDeterministicZip } from './archive.mjs'; +import { archiveMarksExecutable, createDeterministicZip } from './archive.mjs'; import { collectFiles, fileExists, @@ -42,7 +42,11 @@ import { } from './filesystem.mjs'; import { assertExecutionFiles } from './execution.mjs'; import { boxReleaseObjectPrefix, boxReleaseStem, builderVersionFields } from './identity.mjs'; -import { createCondaDependencyLicenseAudit, validateCondaDependencyLicenseAudit } from './licenses.mjs'; +import { + createCondaDependencyLicenseAudit, + validateBundledLicenses, + validateCondaDependencyLicenseAudit, +} from './licenses.mjs'; import { checkParity } from './parity.mjs'; import { findCondaPack, findPixi, installAndPackPixiEnvironment } from './pixi.mjs'; import { fail, run as runProcess } from './process.mjs'; @@ -99,19 +103,46 @@ function runSelfTest({ adapter, scroll, payloadDir, run, extraCode = null }) { } } -/** Writes the licence inventory the box ships, after proving it still matches the reviewed one. */ -async function writeLicenceAudit({ scroll, lockPath, payloadDir, projectRoot }) { - if (!scroll.condaDependencyLicenseAudit) return; - const actual = createCondaDependencyLicenseAudit({ - lockBytes: await readFile(lockPath), - targetId: boxTargetId(scroll.target), - }); - const reviewedPath = join(projectRoot, safeRelativePath(scroll.condaDependencyLicenseAudit)); - const reviewed = JSON.parse(await readFile(reviewedPath, 'utf8')); - validateCondaDependencyLicenseAudit(reviewed, actual); - const auditPath = join(payloadDir, 'THIRD_PARTY_NOTICES', 'conda-distributions.json'); - await mkdir(dirname(auditPath), { recursive: true }); - await writeFile(auditPath, `${JSON.stringify(actual, null, 2)}\n`); +/** Writes one notices file, creating the directory the first of them needs. */ +async function writeNotice(payloadDir, name, value) { + const path = join(payloadDir, 'THIRD_PARTY_NOTICES', name); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); +} + +/** + * Writes the licence inventories the box ships, after proving each still describes this box. + * + * Two halves, known two different ways — see `licenses.mjs`. Both land under `THIRD_PARTY_NOTICES/`, + * because someone opening a box to answer a licence question looks in one place and half a notices + * directory is worse than none. The declared half is also returned, because unlike the derived one + * it travels in the signed release: a licence decision is made before an archive is downloaded, and + * a list only a downloaded archive reveals is a list that arrives too late to act on. + * + * @returns {Promise} + */ +async function writeLicenceInventories({ scroll, lockPath, payloadDir, projectRoot, carriedPaths }) { + if (scroll.condaDependencyLicenseAudit) { + const actual = createCondaDependencyLicenseAudit({ + lockBytes: await readFile(lockPath), + targetId: boxTargetId(scroll.target), + }); + const reviewedPath = join(projectRoot, safeRelativePath(scroll.condaDependencyLicenseAudit)); + const reviewed = JSON.parse(await readFile(reviewedPath, 'utf8')); + validateCondaDependencyLicenseAudit(reviewed, actual); + await writeNotice(payloadDir, 'conda-distributions.json', actual); + } + if (!scroll.bundledLicenseDeclaration) return null; + const declarationPath = join(projectRoot, safeRelativePath(scroll.bundledLicenseDeclaration)); + if (!await fileExists(declarationPath)) { + fail(`Bundled licence declaration is missing: ${scroll.bundledLicenseDeclaration}`); + } + const bundled = await validateBundledLicenses( + JSON.parse(await readFile(declarationPath, 'utf8')), + carriedPaths, + ); + await writeNotice(payloadDir, 'bundled-dependencies.json', bundled); + return bundled; } /** @@ -234,7 +265,17 @@ export async function buildBox(name, options = {}) { for (const prunePath of scroll.prunePaths ?? []) { await rm(join(payloadDir, safeRelativePath(prunePath)), { recursive: true, force: true }); } - await writeLicenceAudit({ scroll, lockPath, payloadDir, projectRoot: workspace.root }); + // Read once, after every staging step and every prune: this is the payload as it will be + // archived, and both the licence declaration and the execution check are questions about that + // tree rather than about what the scroll asked for. + const payloadFiles = new Set(await collectFiles(payloadDir)); + const bundledLicenses = await writeLicenceInventories({ + scroll, + lockPath, + payloadDir, + projectRoot: workspace.root, + carriedPaths: new Set([...payloadFiles, ...deferredAssets]), + }); // Guards against over-pruning: the files the box needs at run time must still be there. for (const requiredFile of scroll.selfTest.files ?? []) { // A deferred asset is legitimately absent from the payload; anything else missing means pruning @@ -249,7 +290,7 @@ export async function buildBox(name, options = {}) { adapter, runtimeId: scroll.runtime.id, runtimeVersion: scroll.runtime.version, - files: new Set(await collectFiles(payloadDir)), + files: payloadFiles, }); // Everything needed to answer "where did this box come from, and could I rebuild it?". const provenance = { @@ -298,6 +339,9 @@ export async function buildBox(name, options = {}) { }; const execution = scroll.execution ? { execution: scroll.execution } : {}; const environment = scroll.environment === undefined ? {} : { environment: scroll.environment }; + // Absent rather than empty when the project declared nothing: an empty list would read as "this + // box bundles no third-party code", which is a claim Scrollcase is in no position to make. + const notices = bundledLicenses === null ? {} : { bundledLicenses }; // box.json travels *inside* the archive. A consumer compares it field by field against the signed // release, which is what binds the archive's contents to its signed metadata. // @@ -312,6 +356,7 @@ export async function buildBox(name, options = {}) { target: scroll.target, runtime: scroll.runtime, cacheSubdir: scroll.cacheSubdir, + ...notices, selfTest, ...environment, ...execution, @@ -373,6 +418,21 @@ export async function buildBox(name, options = {}) { ...(scroll.localFiles ?? []), ].filter((entry) => entry.executable && entry.embed !== false) .map((entry) => safeRelativePath(entry.relativePath)); + // Whatever the box starts has to come out of the archive runnable. For an interpreted runtime + // that is the interpreter, and the runtime's own rule covers it; for a native one it is a file + // the scroll brought in, and only the scroll can say the bit belongs on it. Asked through the + // argv rule rather than by naming an execution kind here, so the guard holds for every runtime + // and keeps holding for the next one. + if (scroll.execution) { + const { command } = runtimeAdapter(scroll.runtime.id).buildArgv({ + execution: scroll.execution, + target: adapter, + }); + const commandPath = safeRelativePath(command.value); + if (!archiveMarksExecutable(adapter, scroll.runtime.id, declaredExecutablePaths, commandPath)) { + fail(`This box runs ${commandPath}, which the archive would not mark executable. Declare it with "executable": true on the asset or local file that brings it in.`); + } + } log('Creating deterministic archive'); await createDeterministicZip(payloadDir, archivePath, adapter, { uncompressedPaths, @@ -403,6 +463,7 @@ export async function buildBox(name, options = {}) { payloadDigest: payloadDigestValue, runtime: scroll.runtime, cacheSubdir: scroll.cacheSubdir, + ...notices, selfTest, ...environment, ...execution, diff --git a/src/build/licenses.d.mts b/src/build/licenses.d.mts index d6948eb..b42eb26 100644 --- a/src/build/licenses.d.mts +++ b/src/build/licenses.d.mts @@ -49,6 +49,22 @@ export function createCondaDependencyLicenseAudit({ lockBytes, targetId, namespa * @throws {Error} when the lock no longer matches what was reviewed */ export function validateCondaDependencyLicenseAudit(reviewed: unknown, actual: ReturnType): ReturnType; +/** + * Checks a declared bundled inventory against its schema and against the box it describes. + * + * The second half is the part worth having. A licence file nobody can check is a licence file + * nobody maintains: a path that stopped being in the box means the entry is stale, and the build + * says so instead of signing a claim about a file that is not there. Deferred assets count as + * carried — the box declares them and a consumer materializes them — because leaving one out of the + * inventory on the grounds that it is fetched later would exempt exactly the large binaries this + * exists for. + * + * @param {unknown} declared the parsed contents of the project's declaration file + * @param {Set} carriedPaths every payload path this box carries, deferred assets included + * @returns {Promise} the declaration, unchanged, when it holds + * @throws {Error} when the shape is wrong or an entry names a file the box does not carry + */ +export function validateBundledLicenses(declared: unknown, carriedPaths: Set): Promise; /** * One package as the lock declares it. */ @@ -61,3 +77,19 @@ export type LockedDistribution = { declaredLicense: string; source: "conda" | "pypi"; }; +/** + * One dependency compiled inside a binary the box ships, as the project declared it. + */ +export type BundledDependency = { + name: string; + version: string; + /** + * the licence the project reviewed + */ + declaredLicense: string; + /** + * payload files it is compiled into + */ + linkedInto: string[]; + sourceUrl?: string; +}; diff --git a/src/build/licenses.mjs b/src/build/licenses.mjs index ae7a58b..57aba07 100644 --- a/src/build/licenses.mjs +++ b/src/build/licenses.mjs @@ -1,15 +1,25 @@ /** - * Builds the dependency licence inventory shipped inside every box. + * The dependency licence inventories a box ships, and there are two of them because they are known + * in two different ways. * - * The inventory is derived from the committed lock file rather than from the installed tree: the - * lock already carries an SPDX licence per package, and `pixi install --frozen` guarantees the + * The conda half is **derived** from the committed lock file rather than from the installed tree: + * the lock already carries an SPDX licence per package, and `pixi install --frozen` guarantees the * installed set equals it. That makes the audit a pure function of a file the user reviews, so it * can be computed without a built prefix and cannot drift from what was approved. + * + * The bundled half cannot be derived at all. A binary a scroll brings into the box was linked + * before Scrollcase saw it, and no file in the build says what went into it; reading the binary + * would be guessing, and guessing about a licence is worse than not answering. So that half is + * **declared** by the project and checked for the one thing a tool can actually check — that every + * file it claims to be linked into is a file this box really carries. What belongs in the list is + * the project's judgement, exactly as the reviewed conda audit is. */ import { createHash } from 'node:crypto'; +import { readFile } from 'node:fs/promises'; import { DEFAULT_DOCUMENT_NAMESPACE } from '../contract/documents.mjs'; -import { compareStableStrings } from './filesystem.mjs'; +import { compareStableStrings, safeRelativePath } from './filesystem.mjs'; +import { schemaValidationError } from './schema-validation.mjs'; /** * One package as the lock declares it. @@ -136,3 +146,59 @@ export function validateCondaDependencyLicenseAudit(reviewed, actual) { } return actual; } + +/** + * One dependency compiled inside a binary the box ships, as the project declared it. + * + * @typedef {object} BundledDependency + * @property {string} name + * @property {string} version + * @property {string} declaredLicense the licence the project reviewed + * @property {string[]} linkedInto payload files it is compiled into + * @property {string} [sourceUrl] + */ + +const releaseSchemaUrl = new URL('../contract/schema/release-manifest.schema.json', import.meta.url); +let bundledLicenseSchema; + +/** + * The `bundledLicenses` definition, lifted out of the release manifest schema so the declaration a + * project writes is judged against the very shape that will be signed. Re-stating the fields here + * would create a second definition of the format, which is the one thing `src/contract/` exists to + * prevent. + */ +async function loadBundledLicenseSchema() { + bundledLicenseSchema ??= readFile(releaseSchemaUrl, 'utf8').then((text) => { + const release = JSON.parse(text); + return { $id: release.$id, $defs: release.$defs, $ref: '#/$defs/bundledLicenses' }; + }); + return bundledLicenseSchema; +} + +/** + * Checks a declared bundled inventory against its schema and against the box it describes. + * + * The second half is the part worth having. A licence file nobody can check is a licence file + * nobody maintains: a path that stopped being in the box means the entry is stale, and the build + * says so instead of signing a claim about a file that is not there. Deferred assets count as + * carried — the box declares them and a consumer materializes them — because leaving one out of the + * inventory on the grounds that it is fetched later would exempt exactly the large binaries this + * exists for. + * + * @param {unknown} declared the parsed contents of the project's declaration file + * @param {Set} carriedPaths every payload path this box carries, deferred assets included + * @returns {Promise} the declaration, unchanged, when it holds + * @throws {Error} when the shape is wrong or an entry names a file the box does not carry + */ +export async function validateBundledLicenses(declared, carriedPaths) { + const error = schemaValidationError(declared, await loadBundledLicenseSchema()); + if (error) fail(`declared bundled licence inventory is invalid: ${error}`); + for (const entry of /** @type {BundledDependency[]} */ (declared)) { + for (const path of entry.linkedInto) { + if (!carriedPaths.has(safeRelativePath(path))) { + fail(`${entry.name}==${entry.version} is declared linked into ${path}, which this box does not carry`); + } + } + } + return /** @type {BundledDependency[]} */ (declared); +} diff --git a/src/build/pixi.d.mts b/src/build/pixi.d.mts index e1c7ecf..95de44c 100644 --- a/src/build/pixi.d.mts +++ b/src/build/pixi.d.mts @@ -110,7 +110,7 @@ export function findCondaPack({ path, runResult }?: { * run: typeof import('./process.mjs').run, * runtimeId: string, * }} options - * @returns {Promise<{ interpreter: string, venvDir: string, sitePackagesRelative: string }>} + * @returns {Promise<{ interpreter: string | null, venvDir: string, sitePackagesRelative: string }>} */ export function installAndPackPixiEnvironment({ pixi, condaPack, manifestPath, lockPath, buildDir, payloadDir, adapter, run, runtimeId, }: { pixi: string; @@ -123,7 +123,7 @@ export function installAndPackPixiEnvironment({ pixi, condaPack, manifestPath, l run: typeof import("./process.mjs").run; runtimeId: string; }): Promise<{ - interpreter: string; + interpreter: string | null; venvDir: string; sitePackagesRelative: string; }>; diff --git a/src/build/pixi.mjs b/src/build/pixi.mjs index 4e0f58f..60bb540 100644 --- a/src/build/pixi.mjs +++ b/src/build/pixi.mjs @@ -21,7 +21,7 @@ import { resolvePayloadLinkTarget, targetCarriesLinks } from '../contract/links. import { runtimeAdapter } from '../contract/runtimes.mjs'; import { compareStableStrings, safeRelativePath } from './filesystem.mjs'; import { fail, runResult as defaultRunResult } from './process.mjs'; -import { repairPosixLaunchers } from '../runtimes/python/launchers.mjs'; +import { runtimeBuilder } from '../runtimes/index.mjs'; import { CONDA_PACK_VERSION, toolchainPaths } from './toolchain.mjs'; import { getWorkspace } from './workspace.mjs'; @@ -340,7 +340,7 @@ async function keepsAsLink(root, linkPath, canonicalRoot) { * run: typeof import('./process.mjs').run, * runtimeId: string, * }} options - * @returns {Promise<{ interpreter: string, venvDir: string, sitePackagesRelative: string }>} + * @returns {Promise<{ interpreter: string | null, venvDir: string, sitePackagesRelative: string }>} */ export async function installAndPackPixiEnvironment({ pixi, @@ -413,7 +413,11 @@ export async function installAndPackPixiEnvironment({ await symlink(link.target, linkPath, type); } - const interpreter = join(payloadDir, ...layout.entryPoint.split('/')); + // Null for a runtime that has none. Only the parity gate wants it, and a scroll declaring parity + // for such a runtime is refused where scrolls are read — there is nothing to run a check with. + const interpreter = layout.entryPoint === null + ? null + : join(payloadDir, ...layout.entryPoint.split('/')); // Deliberately do NOT run conda-unpack. conda-pack already replaces the build prefix with a // neutral placeholder, and the box imports and runs fine that way (a cold import from a moved // prefix was proven before any fixer). Running the fixer here would stamp the *build machine's* @@ -434,10 +438,12 @@ export async function installAndPackPixiEnvironment({ // Order matters: settle the links first, so the launcher repair that follows walks a tree whose // shape is final and rewrites each script's bytes exactly once, under its own name. await settleSymlinksInPlace(venvDir, targetCarriesLinks(adapter.platform)); - // conda console scripts (tqdm, isympy, …) embed the absolute build interpreter in a shell - // trampoline shebang. Rewrite them to resolve Python next to themselves, so no build path - // ships inside the box. - await repairPosixLaunchers(layout, payloadDir, [prefix, workspace, payloadDir]); + // conda console scripts (tqdm, isympy, …) can embed the absolute build interpreter in a shell + // trampoline shebang, and no build path may ship inside a box. How that is dealt with is the + // runtime's answer, not this module's: Python rewrites them to resolve its interpreter next to + // themselves, and a runtime with no trampoline parser refuses a prefix that carries one rather + // than pretending to have fixed it. + await runtimeBuilder(runtimeId).repairLaunchers(layout, payloadDir, [prefix, workspace, payloadDir]); await rm(workspace, { recursive: true, force: true }); await rm(packPath, { force: true }); diff --git a/src/build/scroll-edit.mjs b/src/build/scroll-edit.mjs index 7918284..e56f62d 100644 --- a/src/build/scroll-edit.mjs +++ b/src/build/scroll-edit.mjs @@ -17,6 +17,10 @@ import { createHash } from 'node:crypto'; import { lstat, mkdtemp, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'; import { basename, dirname, join } from 'node:path'; +import { + runtimeAdapter, + unsupportedSelfTestProbeMessage, +} from '../contract/runtimes.mjs'; import { compareStableStrings, fileExists, safeRelativePath, sha256File } from './filesystem.mjs'; import { fail } from './process.mjs'; import { readScroll, scrollDirectory } from './scroll.mjs'; @@ -371,6 +375,34 @@ export async function removeEnvironmentVariable({ boxId, target, name }) { return { written, name }; } +/** + * What an importable name looks like, per runtime. + * + * A dotted identifier is a Python fact, not a format one: `node:path` and `@scope/pkg` are perfectly + * good Node specifiers and the Python pattern refuses both. The list here is narrow on purpose — + * enough to catch a shell fragment or a quoted string that would end up inside generated source, and + * no more, because deciding what resolves is the runtime's job at self-test time and not this + * command's. + */ +const IMPORT_SPECIFIERS = Object.freeze({ + python: /^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*$/, + node: /^(?:@[A-Za-z0-9._-]+\/)?[A-Za-z_][A-Za-z0-9._-]*(?::[A-Za-z0-9._/-]+|\/[A-Za-z0-9._/-]+)?$/, +}); + +/** Refuses a specifier this box's runtime could not be asked about, or could never resolve. */ +async function assertImportSpecifier(boxId, module) { + const scrolls = await readEffectiveScrolls(boxId); + for (const scroll of scrolls) { + const runtime = runtimeAdapter(scroll.runtime.id); + if (!runtime.selfTestProbeKinds.includes('imports')) { + fail(unsupportedSelfTestProbeMessage(runtime.id, 'imports')); + } + if (!IMPORT_SPECIFIERS[runtime.id].test(String(module))) { + fail(`Not an importable ${runtime.id} module name: ${module}`); + } + } +} + /** * `add import` — adds a module to the list the box must be able to import. * @@ -381,9 +413,7 @@ export async function removeEnvironmentVariable({ boxId, target, name }) { * @param {{ boxId: string, target: string, module: string }} options */ export async function addSelfTestImport({ boxId, target, module }) { - if (!/^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*$/.test(String(module))) { - fail(`Not an importable module name: ${module}`); - } + await assertImportSpecifier(boxId, module); let added = 0; const { written } = await updateScrollFiles(boxId, target, (scroll) => { const imports = scroll.selfTest?.imports ?? []; diff --git a/src/build/scroll.mjs b/src/build/scroll.mjs index 166e864..707070f 100644 --- a/src/build/scroll.mjs +++ b/src/build/scroll.mjs @@ -20,6 +20,7 @@ import { isImplementedRuntime, runtimeAdapter, unimplementedRuntimeMessage, + unsupportedSelfTestProbeMessage, } from '../contract/runtimes.mjs'; import { compareStableStrings, fileExists, safeRelativePath } from './filesystem.mjs'; import { fail, runResult } from './process.mjs'; @@ -229,7 +230,12 @@ export function scrollDirectory(reference) { function effectiveScroll(scroll, adapter, targetId) { // Only a runtime this build implements gets this far, so its layout is the one authority on where // the entry point sits — derived when the scroll stays quiet, checked against when it does not. + // A runtime with no interpreter has none to derive, and leaving the field out is the honest + // answer: `runtime.entryPoint` is optional on the wire precisely so a native box can omit it. const layout = runtimeAdapter(scroll.runtime.id).layout(adapter); + const runtime = layout.entryPoint === null + ? { ...scroll.runtime } + : { ...scroll.runtime, entryPoint: scroll.runtime.entryPoint ?? layout.entryPoint }; return { ...scroll, // Provenance needs a stable source identity. It is derived when the scroll does not name one, @@ -237,7 +243,7 @@ function effectiveScroll(scroll, adapter, targetId) { scrollId: scroll.scrollId ?? `${scroll.boxId}-${targetId}`, scrollVersion: scroll.scrollVersion ?? '1.0.0', compatibility: scroll.compatibility ?? {}, - runtime: { ...scroll.runtime, entryPoint: scroll.runtime.entryPoint ?? layout.entryPoint }, + runtime, cacheSubdir: scroll.cacheSubdir ?? `cache/${scroll.boxId}`, assets: scroll.assets ?? [], selfTest: { ...scroll.selfTest, files: scroll.selfTest.files ?? [] }, @@ -275,11 +281,24 @@ async function readExactScroll(reference) { if ((declared.selfTest.commands ?? []).length > 0 && !declared.execution) { fail('selfTest.commands invokes the box\'s execution, which this scroll does not declare.'); } + // An import probe asks a module system a question. A runtime without one cannot answer it, and + // running the build only to discover that at self-test time would be a worse place to find out. + for (const probeKind of ['imports', 'commands']) { + if (!(declared.selfTest[probeKind] ?? []).length) continue; + if (!runtime.selfTestProbeKinds.includes(probeKind)) { + fail(unsupportedSelfTestProbeMessage(runtime.id, probeKind)); + } + } // Checked here rather than in the schema: a base legitimately has no target, and requiring one // there would make every base file light up in an editor. if (declared.target === undefined) fail(`Scroll ${normalized} declares no target.`); const adapter = boxTargetAdapter(declared.target); const targetId = boxTargetId(declared.target); + // The parity gate runs a source file with the box's own runtime, once per accelerator. A runtime + // with no interpreter has nothing to run it with, and a compiled binary is not a check script. + if (declared.parity && runtime.layout(adapter).entryPoint === null) { + fail(`A ${runtime.id} box has no interpreter to run a parity check with; parity compares source run inside the box.`); + } const scroll = effectiveScroll(declared, adapter, targetId); const payloadPaths = [ scroll.cacheSubdir, @@ -294,6 +313,7 @@ async function readExactScroll(reference) { ...(scroll.execution?.binary ? [scroll.execution.binary] : []), ...(scroll.parity ? [scroll.parity.script] : []), ...(scroll.condaDependencyLicenseAudit ? [scroll.condaDependencyLicenseAudit] : []), + ...(scroll.bundledLicenseDeclaration ? [scroll.bundledLicenseDeclaration] : []), ]; for (const path of payloadPaths) safeRelativePath(path); assertDistinctPayloadDestinations(scroll); diff --git a/src/build/verify.d.mts b/src/build/verify.d.mts index 0feb947..5fc0fa1 100644 --- a/src/build/verify.d.mts +++ b/src/build/verify.d.mts @@ -7,7 +7,9 @@ * * `assets` carries the per-entry `embed` decision by construction: it lists exactly the deferred * entries, and it is compared deeply, so a box that quietly changed its mind about one asset - * disagrees with its release. + * disagrees with its release. `bundledLicenses` is here for the same reason it is signed at all: a + * licence inventory that could differ between the document a reviewer read and the box a user + * installed would be worth nothing. */ export function assertBoxManifestAgreement(box: any, release: any): void; /** diff --git a/src/build/verify.mjs b/src/build/verify.mjs index 19c89f3..3f3ec33 100644 --- a/src/build/verify.mjs +++ b/src/build/verify.mjs @@ -36,6 +36,7 @@ const AGREEMENT_FIELDS = [ 'target', 'runtime', 'cacheSubdir', + 'bundledLicenses', 'environment', 'selfTest', 'execution', @@ -52,7 +53,9 @@ const AGREEMENT_FIELDS = [ * * `assets` carries the per-entry `embed` decision by construction: it lists exactly the deferred * entries, and it is compared deeply, so a box that quietly changed its mind about one asset - * disagrees with its release. + * disagrees with its release. `bundledLicenses` is here for the same reason it is signed at all: a + * licence inventory that could differ between the document a reviewer read and the box a user + * installed would be worth nothing. */ export function assertBoxManifestAgreement(box, release) { for (const field of AGREEMENT_FIELDS) { diff --git a/src/cli-authoring.mjs b/src/cli-authoring.mjs index d92fa5e..c48e725 100644 --- a/src/cli-authoring.mjs +++ b/src/cli-authoring.mjs @@ -13,10 +13,12 @@ */ import { createInterface } from 'node:readline/promises'; +import { runtimeAdapters } from './contract/runtimes.mjs'; import { - DEFAULT_PYTHON_VERSION, + DEFAULT_RUNTIME_ID, EXAMPLE_PIXI_VERSION, - resolvePythonVersion, + authoredExecutionKinds, + resolveRuntimeVersion, } from './build/authoring.mjs'; import { probePixi } from './build/pixi.mjs'; import { fail } from './build/process.mjs'; @@ -40,9 +42,11 @@ const HINTS = Object.freeze({ boxId: 'Name of the box across all its versions. Used in its directory, its archives and its channel pointer.', sourceRevision: 'Which version of the thing you are packaging this is — a model commit, a release tag. Recorded verbatim in the box provenance.', assetBaseUrl: 'Where you will publish built boxes. The signed release points at it; it does not have to exist yet.', - execution: 'What `scrollcase run` starts inside the box: a script file, an importable module, or nothing at all.', - scriptSource: 'Point at a Python file you already have, or start from a generated stub.', - scriptPath: 'Path from the project root to the Python file the box should run.', + runtime: 'What runs inside the box. python and node bring an interpreter; native runs a binary you compiled yourself.', + execution: 'What `scrollcase run` starts inside the box: a file, an importable module, or nothing at all.', + scriptSource: 'Point at a file you already have, or start from a generated stub.', + scriptPath: 'Path from the project root to the file the box should run.', + binaryPath: 'Path from the project root to the compiled executable the box should run.', module: 'Dotted name of a module importable inside the box, run with python -m.', }); @@ -177,13 +181,26 @@ export async function collectNewScrollOptions(flags, { }; const target = await collectTarget(flags, { terminal, ask, chooseTargetValue }); + // Asked first among the box's own decisions, because it decides which of the later questions + // exist: a native box is never asked for a module, and is never offered a generated stub. + // Not `finite`: this is the one closed choice with a defensible default, so a scripted session + // that never mentioned a runtime keeps working and gets the runtime it has always got. + const runtimeIds = runtimeAdapters().map((runtime) => runtime.id); + if (runtimeIds[0] !== DEFAULT_RUNTIME_ID) { + fail('The runtime menu must offer the default first; it is what a non-terminal session gets.'); + } + const runtimeId = await choose('runtime', runtimeIds, { + flag: flagText(flags, 'runtime'), + hint: HINTS.runtime, + terminal, + }); const boxId = await required('box-id', 'Box ID', HINTS.boxId); // The upstream revision is the one identity nothing here can supply: it names the version of the // thing being packaged, and inventing it would put a false claim into the box's provenance. const sourceRevision = await required('source-revision', 'Upstream revision', HINTS.sourceRevision); const version = derived('version', '1.0.0'); const scrollVersion = derived('scroll-version', undefined); - const pythonVersion = resolvePythonVersion(derived('python-version', DEFAULT_PYTHON_VERSION)); + const runtimeVersion = resolveRuntimeVersion(runtimeId, flagText(flags, 'runtime-version')); // Pinning the pixi that is actually installed: `findPixi` refuses to build with any other, so a // pinned version the machine does not have is a scroll that cannot be built where it was written. const pixiVersion = derived('pixi-version', probe()?.version ?? EXAMPLE_PIXI_VERSION); @@ -210,7 +227,7 @@ export async function collectNewScrollOptions(flags, { const executionKind = await finite( 'execution', 'execution kind', - ['python-script', 'python-module', 'library-only'], + authoredExecutionKinds(runtimeId), HINTS.execution, ); const defaultArgs = parseDefaultArgs(flagText(flags, 'default-args')); @@ -222,7 +239,8 @@ export async function collectNewScrollOptions(flags, { version, scrollVersion, sourceRevision, - pythonVersion, + runtimeId, + runtimeVersion, pixiVersion, compatibility, assetBaseUrl, @@ -231,16 +249,25 @@ export async function collectNewScrollOptions(flags, { }; if (executionKind === 'python-module') { result.module = await required('module', 'Python module', HINTS.module); - } else if (executionKind === 'python-script') { + } else if (executionKind !== 'library-only') { + // Every remaining kind names a payload file. Whether Scrollcase can write a starter for it is + // the runtime's answer: it generates source, and it does not generate compiled binaries. + const generable = executionKind !== 'native-binary'; + const noun = generable ? 'script' : 'binary'; const existing = flagText(flags, 'script'); const generateScript = Boolean(flags.get('generate-script')); if (existing && generateScript) { fail('Choose either --script or --generate-script, not both.'); } + if (generateScript && !generable) { + fail(`Scrollcase cannot generate a ${noun}; point --script at the one you built.`); + } if (existing) result.scriptSourcePath = existing; else if (generateScript) result.generateScript = true; else if (!terminal) { - fail('python-script execution requires --script or --generate-script without a terminal.'); + fail(`${executionKind} execution requires --script ${generable ? ' or --generate-script' : ''} without a terminal.`); + } else if (!generable) { + result.scriptSourcePath = await ask('Binary path', { hint: HINTS.binaryPath }); } else { const source = await choose( 'script source', @@ -251,7 +278,8 @@ export async function collectNewScrollOptions(flags, { result.scriptSourcePath = await ask('Script path', { hint: HINTS.scriptPath }); } else result.generateScript = true; } - result.scriptRelativePath = flagText(flags, 'script-destination') ?? 'entrypoint.py'; + const destination = flagText(flags, 'script-destination'); + if (destination) result.scriptRelativePath = destination; const generatedScriptSourcePath = flagText(flags, 'generated-script-path'); if (generatedScriptSourcePath) result.generatedScriptSourcePath = generatedScriptSourcePath; } diff --git a/src/cli.mjs b/src/cli.mjs index b98259b..6b397cd 100755 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -685,10 +685,12 @@ Init options: offers a PyPI fallback. New scroll options: - Interactively it asks four things — target, box id, upstream - revision, and where boxes will be published — plus the execution - kind, and derives the rest. Every derived value below is a flag. + Interactively it asks five things — target, runtime, box id, + upstream revision, and where boxes will be published — plus the + execution kind, and derives the rest. Every derived value below is + a flag. --target Complete target, including the CUDA ABI when applicable + --runtime python, node or native (default python) --box-id Box identity --source-revision Upstream source revision recorded in provenance --asset-base-url Base URL used in built release documents @@ -696,13 +698,15 @@ New scroll options: release. Scrollcase reads none of them. --version Box version (default 1.0.0) --scroll-version Scroll authoring version (default 1.0.0) - --python-version Python dependency version, or latest + --runtime-version Interpreter version solved into the box, or latest. Refused for + native, which installs no interpreter --pixi-version pixi resolver version (default: the installed pixi) --min-host-app-version Minimum compatible host application version - --execution python-script, python-module, or library-only - --script Existing project script for python-script - --generate-script Generate a minimal project script instead - --script-destination Payload path for the script (default entrypoint.py) + --execution The runtime's own kinds, plus library-only where the box can still + prove something without an entry point + --script Existing project file the box runs + --generate-script Generate a minimal starter instead, where the runtime has one + --script-destination Payload path for that file (default: the runtime's own name) --generated-script-path Project path for a generated starter --module Dotted module name for python-module --default-args JSON array of default application arguments diff --git a/src/contract/fixtures/consumer-conformance.json b/src/contract/fixtures/consumer-conformance.json index 8d81d99..4e2b686 100644 --- a/src/contract/fixtures/consumer-conformance.json +++ b/src/contract/fixtures/consumer-conformance.json @@ -28,9 +28,9 @@ "payload-list-missing": "missing its payload digest list", "payload-mismatch": "^Payload does not match the signed release:", "runtime-disagreement": "box.json mismatch: runtime", + "runtime-without-entry-point": "no runtime entry point to declare", "spawn-failure": "failed to start|fixture spawn failed", "special-entry": "special entries", - "unimplemented-runtime": "is not implemented by this version", "unsafe-path": "Unsafe relative path|invalid relative path|absolute path", "unsupported-schema-version": "Unsupported schemaVersion 1|Unsupported schemaVersion 2" }, @@ -1398,12 +1398,12 @@ } }, { - "id": "runtime-this-build-cannot-run", + "id": "runtime-relabelled-after-signing", "action": "prepare", "mutation": "alter-release-runtime-id", "expected": { "outcome": "rejected", - "error": "unimplemented-runtime", + "error": "runtime-without-entry-point", "destinationExists": false } }, diff --git a/src/contract/fixtures/runtime-contract.json b/src/contract/fixtures/runtime-contract.json index f08f8df..117a645 100644 --- a/src/contract/fixtures/runtime-contract.json +++ b/src/contract/fixtures/runtime-contract.json @@ -71,6 +71,147 @@ ] } } + ], + "selfTestProbeKinds": [ + "imports", + "commands" + ] + }, + { + "id": "node", + "executionKinds": [ + "node-script" + ], + "executionEnvironmentVariables": [ + "NODE_OPTIONS", + "NODE_PATH", + "NODE_EXTRA_CA_CERTS" + ], + "selfTestProbeKinds": [ + "imports", + "commands" + ], + "layouts": [ + { + "platform": "macos", + "layout": { + "root": "venv", + "entryPoint": "venv/bin/node", + "scriptsDirectory": "venv/bin", + "standardLibrary": "venv/lib", + "executableSuffix": "", + "launcherKind": "posix-polyglot" + }, + "executablePayloadPaths": { + "files": [ + "venv/bin/node" + ], + "directories": [ + "venv/bin" + ] + } + }, + { + "platform": "linux", + "layout": { + "root": "venv", + "entryPoint": "venv/bin/node", + "scriptsDirectory": "venv/bin", + "standardLibrary": "venv/lib", + "executableSuffix": "", + "launcherKind": "posix-polyglot" + }, + "executablePayloadPaths": { + "files": [ + "venv/bin/node" + ], + "directories": [ + "venv/bin" + ] + } + }, + { + "platform": "windows", + "layout": { + "root": "venv", + "entryPoint": "venv/node.exe", + "scriptsDirectory": "venv/Scripts", + "standardLibrary": "venv/Lib", + "executableSuffix": ".exe", + "launcherKind": "uv-windows-pe" + }, + "executablePayloadPaths": { + "files": [ + "venv/node.exe" + ], + "directories": [ + "venv/Scripts" + ] + } + } + ] + }, + { + "id": "native", + "executionKinds": [ + "native-binary" + ], + "executionEnvironmentVariables": [], + "selfTestProbeKinds": [ + "commands" + ], + "layouts": [ + { + "platform": "macos", + "layout": { + "root": "venv", + "entryPoint": null, + "scriptsDirectory": "venv/bin", + "standardLibrary": null, + "executableSuffix": "", + "launcherKind": "posix-polyglot" + }, + "executablePayloadPaths": { + "files": [], + "directories": [ + "venv/bin" + ] + } + }, + { + "platform": "linux", + "layout": { + "root": "venv", + "entryPoint": null, + "scriptsDirectory": "venv/bin", + "standardLibrary": null, + "executableSuffix": "", + "launcherKind": "posix-polyglot" + }, + "executablePayloadPaths": { + "files": [], + "directories": [ + "venv/bin" + ] + } + }, + { + "platform": "windows", + "layout": { + "root": "venv", + "entryPoint": null, + "scriptsDirectory": "venv/Scripts", + "standardLibrary": null, + "executableSuffix": ".exe", + "launcherKind": "uv-windows-pe" + }, + "executablePayloadPaths": { + "files": [], + "directories": [ + "venv/Scripts" + ] + } + } ] } ], @@ -137,6 +278,55 @@ "platform": "windows", "path": "venv/bin/tqdm", "executable": false + }, + { + "name": "the Node interpreter, by exact name", + "runtime": "node", + "platform": "linux", + "path": "venv/bin/node", + "executable": true + }, + { + "name": "a Node console script the prefix generated", + "runtime": "node", + "platform": "linux", + "path": "venv/bin/npx", + "executable": true + }, + { + "name": "the application's own script, which the runtime never claims", + "runtime": "node", + "platform": "linux", + "path": "app/main.js", + "executable": false + }, + { + "name": "the Windows Node interpreter, which sits outside its scripts directory", + "runtime": "node", + "platform": "windows", + "path": "venv/node.exe", + "executable": true + }, + { + "name": "a prefix tool a native box still carries", + "runtime": "native", + "platform": "linux", + "path": "venv/bin/sqlite3", + "executable": true + }, + { + "name": "the binary a native box runs, which the runtime never claims because the scroll does", + "runtime": "native", + "platform": "linux", + "path": "bin/tool", + "executable": false + }, + { + "name": "a shared library a native binary links against", + "runtime": "native", + "platform": "linux", + "path": "venv/lib/libz.so.1", + "executable": false } ], "executionDiscovery": [ @@ -212,6 +402,34 @@ "venv/Lib/site-packages/pkg.py", "venv/Lib/site-packages/pkg/__main__.py" ] + }, + { + "name": "a Node script resolves to itself and nowhere else", + "runtime": "node", + "platform": "linux", + "runtimeVersion": "22.11.0", + "execution": { + "kind": "node-script", + "script": "app/main.js", + "defaultArgs": [] + }, + "candidates": [ + "app/main.js" + ] + }, + { + "name": "a native binary resolves to itself and nowhere else", + "runtime": "native", + "platform": "macos", + "runtimeVersion": "", + "execution": { + "kind": "native-binary", + "binary": "bin/tool", + "defaultArgs": [] + }, + "candidates": [ + "bin/tool" + ] } ], "invalidRuntimeVersions": [ @@ -301,6 +519,79 @@ "value": "app/main.py" } ] + }, + { + "name": "a Node script runs through the box's own node, with its declared arguments after it", + "runtime": "node", + "platform": "linux", + "execution": { + "kind": "node-script", + "script": "app/main.js", + "defaultArgs": [ + "--serve" + ] + }, + "command": { + "kind": "payload-path", + "value": "venv/bin/node" + }, + "args": [ + { + "kind": "payload-path", + "value": "app/main.js" + }, + { + "kind": "literal", + "value": "--serve" + } + ] + }, + { + "name": "a Windows Node box runs the same declaration through its own node", + "runtime": "node", + "platform": "windows", + "execution": { + "kind": "node-script", + "script": "app/main.js", + "defaultArgs": [] + }, + "command": { + "kind": "payload-path", + "value": "venv/node.exe" + }, + "args": [ + { + "kind": "payload-path", + "value": "app/main.js" + } + ] + }, + { + "name": "a native binary is the command itself, with nothing in front of it", + "runtime": "native", + "platform": "linux", + "execution": { + "kind": "native-binary", + "binary": "bin/tool", + "defaultArgs": [ + "--serve", + "8080" + ] + }, + "command": { + "kind": "payload-path", + "value": "bin/tool" + }, + "args": [ + { + "kind": "literal", + "value": "--serve" + }, + { + "kind": "literal", + "value": "8080" + } + ] } ], "selfTest": [ @@ -589,11 +880,91 @@ "expectExitCode": 1 } ] + }, + { + "name": "a Node import probe requires each module through the box's own node", + "runtime": "node", + "platform": "linux", + "probe": { + "imports": [ + "fs", + "node:path" + ] + }, + "execution": null, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/node" + }, + "args": [ + { + "kind": "literal", + "value": "-e" + }, + { + "kind": "literal", + "value": "if (process.platform !== 'linux') throw new Error('platform mismatch: ' + process.platform)\nrequire(\"fs\");\nrequire(\"node:path\");" + } + ], + "expectExitCode": 0 + } + ] + }, + { + "name": "a native command probe invokes the binary itself", + "runtime": "native", + "platform": "macos", + "probe": { + "commands": [ + { + "args": [ + "--version" + ], + "expectExitCode": 0 + } + ] + }, + "execution": { + "kind": "native-binary", + "binary": "bin/tool", + "defaultArgs": [ + "--quiet" + ] + }, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "bin/tool" + }, + "args": [ + { + "kind": "literal", + "value": "--quiet" + }, + { + "kind": "literal", + "value": "--version" + } + ], + "expectExitCode": 0 + } + ] } ], "runtimeIds": [ "python", "node", "native" + ], + "unsupportedProbes": [ + { + "name": "a native box has no module system to ask for an import", + "runtime": "native", + "probeKind": "imports", + "message": "The native runtime cannot answer a selfTest.imports probe; it answers selfTest.commands." + } ] } diff --git a/src/contract/runtimes.d.mts b/src/contract/runtimes.d.mts index a1e4eb8..4b3a722 100644 --- a/src/contract/runtimes.d.mts +++ b/src/contract/runtimes.d.mts @@ -15,14 +15,33 @@ export function runtimeAdapters(): BoxRuntimeAdapter[]; /** * Ensures a declared entry point agrees with where the runtime actually sits in the payload. * + * Three answers, because there are three cases. A runtime with an interpreter admits exactly one + * value for a given target, so a declaration is checked against it. A runtime without one — a + * native box — admits none, and a declaration there is refused rather than ignored: it would name a + * file the box never starts, and a reader would believe it. And a box that declares nothing at all + * is checked against nothing, because `runtime.entryPoint` is optional on the wire and its absence + * is a legitimate answer for both. + * * @param {string} runtimeId * @param {import('./targets.mjs').BoxTargetAdapter} adapter the resolved target adapter, whose id * names the layout the entry point is being judged against - * @param {string} entryPoint + * @param {string | null | undefined} entryPoint * @returns {void} * @throws {TypeError} when the entry point is not the one the runtime defines for this target */ -export function assertRuntimeEntryPoint(runtimeId: string, adapter: import("./targets.mjs").BoxTargetAdapter, entryPoint: string): void; +export function assertRuntimeEntryPoint(runtimeId: string, adapter: import("./targets.mjs").BoxTargetAdapter, entryPoint: string | null | undefined): void; +/** + * The message for a self-test probe shape the runtime cannot answer. + * + * Stated here, beside the rule, for the same reason `resolveExecutionFiles` returns its own + * `missing`: the wording is part of the contract, and the builder and all three consumers should + * refuse an impossible probe identically instead of each inventing a phrasing. + * + * @param {string} runtimeId + * @param {string} probeKind + * @returns {string} + */ +export function unsupportedSelfTestProbeMessage(runtimeId: string, probeKind: string): string; /** * The message for a box declaring a runtime this build has no adapter for. * @@ -85,8 +104,11 @@ export function executionAffectingVariables(runtimeId: string, adapter: import(" * * Schema version 3 made the runtime a declaration: a box says `runtime: { id, version, entryPoint }` * instead of leaving a reader to infer Python from a Python-shaped entry point. `RUNTIME_IDS` is the - * vocabulary that declaration may use and `RUNTIME_ADAPTERS` is what this build can actually run — - * two different lists on purpose, so implementing `node` later is code and not another wire break. + * vocabulary that declaration may use and `RUNTIME_ADAPTERS` is what this build can actually run. + * They now hold the same three, which is what the split was for: `node` and `native` arrived as + * adapters and the wire did not move. The two lists stay separate because they answer to different + * release cycles — a consumer crate published before a runtime landed still has to refuse a box + * naming it, by name, rather than misread it as another runtime. * * Only the pure half lives here. Nothing in this module reads a file, joins a host path, or starts * a process: every function is a statement about names, so the same inputs give the same answer in @@ -113,6 +135,7 @@ export function executionAffectingVariables(runtimeId: string, adapter: import(" * @property {readonly string[]} executionEnvironmentVariables inherited variables whose presence * can change which code this runtime loads — the runtime half of the diagnostic list, to which * the target adapter adds the operating system's own + * @property {readonly string[]} selfTestProbeKinds the probe shapes this runtime can answer * @property {(target: BoxRuntimeTarget) => BoxRuntimeLayout} layout where the runtime lives inside * the payload * @property {(target: BoxRuntimeTarget) => ExecutablePayloadPaths} executablePayloadPaths payload @@ -134,11 +157,19 @@ export function executionAffectingVariables(runtimeId: string, adapter: import(" /** * Where a runtime lives inside an extracted box. * + * Two fields are nullable, and both mean the same thing: the runtime does not have that. A native + * box carries no interpreter to name and no bundled library to search, so `entryPoint` and + * `standardLibrary` are null rather than a plausible-looking path nothing would find. Every caller + * that reads one already has to decide what to do when the box declares none, because + * `runtime.entryPoint` is optional on the wire for exactly this reason. + * * @typedef {object} BoxRuntimeLayout * @property {string} root directory the runtime was relocated into - * @property {string} entryPoint the runtime's own executable, relative to the box root + * @property {string | null} entryPoint the runtime's own executable, relative to the box root, or + * null for a runtime that has no separate executable to name * @property {string} scriptsDirectory directory holding generated console scripts - * @property {string} standardLibrary directory holding the runtime's bundled library + * @property {string | null} standardLibrary directory holding the runtime's bundled library, or + * null for a runtime that has none * @property {string} executableSuffix suffix an executable carries on this platform * @property {string} launcherKind frozen wire string naming how launchers were repaired */ @@ -213,6 +244,10 @@ export type BoxRuntimeAdapter = { * the target adapter adds the operating system's own */ executionEnvironmentVariables: readonly string[]; + /** + * the probe shapes this runtime can answer + */ + selfTestProbeKinds: readonly string[]; /** * where the runtime lives inside * the payload @@ -250,6 +285,12 @@ export type BoxRuntimeTarget = { }; /** * Where a runtime lives inside an extracted box. + * + * Two fields are nullable, and both mean the same thing: the runtime does not have that. A native + * box carries no interpreter to name and no bundled library to search, so `entryPoint` and + * `standardLibrary` are null rather than a plausible-looking path nothing would find. Every caller + * that reads one already has to decide what to do when the box declares none, because + * `runtime.entryPoint` is optional on the wire for exactly this reason. */ export type BoxRuntimeLayout = { /** @@ -257,17 +298,19 @@ export type BoxRuntimeLayout = { */ root: string; /** - * the runtime's own executable, relative to the box root + * the runtime's own executable, relative to the box root, or + * null for a runtime that has no separate executable to name */ - entryPoint: string; + entryPoint: string | null; /** * directory holding generated console scripts */ scriptsDirectory: string; /** - * directory holding the runtime's bundled library + * directory holding the runtime's bundled library, or + * null for a runtime that has none */ - standardLibrary: string; + standardLibrary: string | null; /** * suffix an executable carries on this platform */ diff --git a/src/contract/runtimes.mjs b/src/contract/runtimes.mjs index 10b4b06..d88653c 100644 --- a/src/contract/runtimes.mjs +++ b/src/contract/runtimes.mjs @@ -16,8 +16,11 @@ * * Schema version 3 made the runtime a declaration: a box says `runtime: { id, version, entryPoint }` * instead of leaving a reader to infer Python from a Python-shaped entry point. `RUNTIME_IDS` is the - * vocabulary that declaration may use and `RUNTIME_ADAPTERS` is what this build can actually run — - * two different lists on purpose, so implementing `node` later is code and not another wire break. + * vocabulary that declaration may use and `RUNTIME_ADAPTERS` is what this build can actually run. + * They now hold the same three, which is what the split was for: `node` and `native` arrived as + * adapters and the wire did not move. The two lists stay separate because they answer to different + * release cycles — a consumer crate published before a runtime landed still has to refuse a box + * naming it, by name, rather than misread it as another runtime. * * Only the pure half lives here. Nothing in this module reads a file, joins a host path, or starts * a process: every function is a statement about names, so the same inputs give the same answer in @@ -45,6 +48,7 @@ * @property {readonly string[]} executionEnvironmentVariables inherited variables whose presence * can change which code this runtime loads — the runtime half of the diagnostic list, to which * the target adapter adds the operating system's own + * @property {readonly string[]} selfTestProbeKinds the probe shapes this runtime can answer * @property {(target: BoxRuntimeTarget) => BoxRuntimeLayout} layout where the runtime lives inside * the payload * @property {(target: BoxRuntimeTarget) => ExecutablePayloadPaths} executablePayloadPaths payload @@ -68,11 +72,19 @@ /** * Where a runtime lives inside an extracted box. * + * Two fields are nullable, and both mean the same thing: the runtime does not have that. A native + * box carries no interpreter to name and no bundled library to search, so `entryPoint` and + * `standardLibrary` are null rather than a plausible-looking path nothing would find. Every caller + * that reads one already has to decide what to do when the box declares none, because + * `runtime.entryPoint` is optional on the wire for exactly this reason. + * * @typedef {object} BoxRuntimeLayout * @property {string} root directory the runtime was relocated into - * @property {string} entryPoint the runtime's own executable, relative to the box root + * @property {string | null} entryPoint the runtime's own executable, relative to the box root, or + * null for a runtime that has no separate executable to name * @property {string} scriptsDirectory directory holding generated console scripts - * @property {string} standardLibrary directory holding the runtime's bundled library + * @property {string | null} standardLibrary directory holding the runtime's bundled library, or + * null for a runtime that has none * @property {string} executableSuffix suffix an executable carries on this platform * @property {string} launcherKind frozen wire string naming how launchers were repaired */ @@ -231,15 +243,42 @@ function pythonModuleEntryPoints({ module, runtimeVersion, target }) { * property of the box, so it is refused where scrolls are read and reported here as a programming * error if it ever gets this far. */ -function commandInvocation(runtime, { command, execution, target }) { - if (!execution) { - throw new TypeError('A self-test command needs a declared execution to invoke'); +function commandInvocations(runtime, { probe, execution, target }) { + return (probe.commands ?? []).map((command) => { + if (!execution) { + throw new TypeError('A self-test command needs a declared execution to invoke'); + } + const { command: entryPoint, args } = runtime.buildArgv({ execution, target }); + return { + command: entryPoint, + args: [...args, ...command.args.map((value) => ({ kind: 'literal', value }))], + expectExitCode: command.expectExitCode ?? 0, + }; + }); +} + +/** + * Refuses a probe shape the runtime cannot answer, before anything tries to run it. + * + * An import probe asks a module system a question, and a runtime without one has nothing to ask. + * Silently dropping it would report a pass for a check that never ran, which is worse than the + * declaration being refused. + */ +function assertProbeKinds(runtime, probe) { + for (const kind of ['imports', 'commands']) { + if (!probe[kind]?.length) continue; + if (!runtime.selfTestProbeKinds.includes(kind)) { + throw new TypeError(unsupportedSelfTestProbeMessage(runtime.id, kind)); + } } - const { command: entryPoint, args } = runtime.buildArgv({ execution, target }); +} + +/** The one place an interpreted runtime's `-c`/`-e` probe is assembled, given its own source. */ +function sourceProbeInvocation({ runtime, target, flag, source }) { return { - command: entryPoint, - args: [...args, ...command.args.map((value) => ({ kind: 'literal', value }))], - expectExitCode: command.expectExitCode ?? 0, + command: { kind: 'payload-path', value: runtime.layout(target).entryPoint }, + args: [flag, source].map((value) => ({ kind: 'literal', value })), + expectExitCode: 0, }; } @@ -251,22 +290,31 @@ function freezeInvocations(invocations) { }))); } +/** + * The interpreter by name, and the console-script directory wholesale. + * + * A conda prefix generates that directory's contents at solve time and nothing declares them, so + * the rule is the only way they can carry the bit at all. A runtime with no interpreter of its own + * contributes only the directory: the file it runs is one the scroll declared, and the scroll is + * what says the bit belongs on it. + */ +function prefixExecutablePayloadPaths(layout) { + return Object.freeze({ + files: Object.freeze(layout.entryPoint === null ? [] : [layout.entryPoint]), + directories: Object.freeze([layout.scriptsDirectory]), + }); +} + const PYTHON_RUNTIME = Object.freeze({ id: 'python', executionKinds: Object.freeze(['python-script', 'python-module']), executionEnvironmentVariables: PYTHON_EXECUTION_ENVIRONMENT, + selfTestProbeKinds: Object.freeze(['imports', 'commands']), layout: pythonLayout, executablePayloadPaths(target) { - const layout = pythonLayout(target); - // The interpreter by name, and the console-script directory wholesale. A conda prefix generates - // that directory's contents at solve time and nothing declares them, so the rule is the only - // way they can carry the bit at all. - return Object.freeze({ - files: Object.freeze([layout.entryPoint]), - directories: Object.freeze([layout.scriptsDirectory]), - }); + return prefixExecutablePayloadPaths(pythonLayout(target)); }, resolveExecutionFiles({ execution, runtimeVersion, target }) { @@ -299,6 +347,7 @@ const PYTHON_RUNTIME = Object.freeze({ }, selfTestInvocations({ probe, execution, target }) { + assertProbeKinds(PYTHON_RUNTIME, probe); const invocations = []; if (probe.imports?.length) { const assertion = PYTHON_PLATFORM_ASSERTIONS[target?.platform]; @@ -309,20 +358,205 @@ const PYTHON_RUNTIME = Object.freeze({ const code = probe.code ? `${assertion}\n${imports}\n${probe.code}` : `${assertion}\n${imports}`; - invocations.push({ - command: { kind: 'payload-path', value: pythonLayout(target).entryPoint }, - args: ['-c', code].map((value) => ({ kind: 'literal', value })), - expectExitCode: 0, - }); + invocations.push(sourceProbeInvocation({ + runtime: PYTHON_RUNTIME, + target, + flag: '-c', + source: code, + })); } - for (const command of probe.commands ?? []) { - invocations.push(commandInvocation(PYTHON_RUNTIME, { command, execution, target })); + invocations.push(...commandInvocations(PYTHON_RUNTIME, { probe, execution, target })); + return freezeInvocations(invocations); + }, +}); + +const NODE_EXECUTION_ENVIRONMENT = Object.freeze([ + 'NODE_OPTIONS', + 'NODE_PATH', + 'NODE_EXTRA_CA_CERTS', +]); + +const POSIX_NODE_LAYOUT = Object.freeze({ + root: 'venv', + entryPoint: 'venv/bin/node', + scriptsDirectory: 'venv/bin', + standardLibrary: 'venv/lib', + executableSuffix: '', + launcherKind: 'posix-polyglot', +}); + +const NODE_LAYOUTS = Object.freeze({ + macos: POSIX_NODE_LAYOUT, + linux: POSIX_NODE_LAYOUT, + windows: Object.freeze({ + root: 'venv', + // conda-forge installs a Windows package's own executables at the prefix root and its generated + // launchers under `Scripts`, which is why node.exe sits beside python.exe rather than under it. + entryPoint: 'venv/node.exe', + scriptsDirectory: 'venv/Scripts', + standardLibrary: 'venv/Lib', + launcherKind: 'uv-windows-pe', + executableSuffix: '.exe', + }), +}); + +/** + * The assertion every Node self-test opens with. Same purpose as the Python one: prove the check is + * running on the platform the box was built for before it proves anything else. + */ +const NODE_PLATFORM_ASSERTIONS = Object.freeze({ + macos: "if (process.platform !== 'darwin') throw new Error('platform mismatch: ' + process.platform)", + linux: "if (process.platform !== 'linux') throw new Error('platform mismatch: ' + process.platform)", + windows: "if (process.platform !== 'win32') throw new Error('platform mismatch: ' + process.platform)", +}); + +function nodeLayout(target) { + const layout = NODE_LAYOUTS[target?.platform]; + if (!layout) { + throw new TypeError(`No node runtime layout exists for platform ${String(target?.platform)}`); + } + return layout; +} + +const NODE_RUNTIME = Object.freeze({ + id: 'node', + // One kind, deliberately. Node has no `-m` analogue worth inventing: a package entry point + // resolves to a file, and naming that file is what every other declaration in the format does. + executionKinds: Object.freeze(['node-script']), + executionEnvironmentVariables: NODE_EXECUTION_ENVIRONMENT, + selfTestProbeKinds: Object.freeze(['imports', 'commands']), + + layout: nodeLayout, + + executablePayloadPaths(target) { + return prefixExecutablePayloadPaths(nodeLayout(target)); + }, + + resolveExecutionFiles({ execution }) { + return Object.freeze({ + candidates: Object.freeze([execution.script]), + missing: `Execution script is missing from the box: ${execution.script}.`, + }); + }, + + buildArgv({ execution, target }) { + const args = [{ kind: 'payload-path', value: execution.script }]; + for (const value of execution.defaultArgs ?? []) args.push({ kind: 'literal', value }); + return Object.freeze({ + command: Object.freeze({ kind: 'payload-path', value: nodeLayout(target).entryPoint }), + args: Object.freeze(args.map((argument) => Object.freeze(argument))), + }); + }, + + selfTestInvocations({ probe, execution, target }) { + assertProbeKinds(NODE_RUNTIME, probe); + const invocations = []; + if (probe.imports?.length) { + const assertion = NODE_PLATFORM_ASSERTIONS[target?.platform]; + if (!assertion) { + throw new TypeError(`No node self-test assertion exists for platform ${String(target?.platform)}`); + } + // `require` rather than `import()`, because `-e` source is evaluated as CommonJS and Node 22 + // resolves an ES module through `require` as well. A box whose dependency cannot be loaded + // either way is a box whose probe should fail. + const imports = probe.imports + .map((specifier) => `require(${JSON.stringify(specifier)});`) + .join('\n'); + const code = probe.code + ? `${assertion}\n${imports}\n${probe.code}` + : `${assertion}\n${imports}`; + invocations.push(sourceProbeInvocation({ + runtime: NODE_RUNTIME, + target, + flag: '-e', + source: code, + })); } + invocations.push(...commandInvocations(NODE_RUNTIME, { probe, execution, target })); return freezeInvocations(invocations); }, }); -const RUNTIME_ADAPTERS = Object.freeze([PYTHON_RUNTIME]); +/** + * A native box has no interpreter, so its layout names none — and names no standard library either, + * because there is no loader that would search one. + * + * The packed prefix is still there. A native box is built from a `pixi.lock` like every other, and + * the binary it runs links against the shared libraries that lock installed; what changes is only + * that nothing in `venv/` is started to run the box. `scriptsDirectory` therefore stays: a conda + * prefix generates console scripts whoever depends on it, and they still need the bit. + */ +const POSIX_NATIVE_LAYOUT = Object.freeze({ + root: 'venv', + entryPoint: null, + scriptsDirectory: 'venv/bin', + standardLibrary: null, + executableSuffix: '', + launcherKind: 'posix-polyglot', +}); + +const NATIVE_LAYOUTS = Object.freeze({ + macos: POSIX_NATIVE_LAYOUT, + linux: POSIX_NATIVE_LAYOUT, + windows: Object.freeze({ + root: 'venv', + entryPoint: null, + scriptsDirectory: 'venv/Scripts', + standardLibrary: null, + executableSuffix: '.exe', + launcherKind: 'uv-windows-pe', + }), +}); + +function nativeLayout(target) { + const layout = NATIVE_LAYOUTS[target?.platform]; + if (!layout) { + throw new TypeError(`No native runtime layout exists for platform ${String(target?.platform)}`); + } + return layout; +} + +const NATIVE_RUNTIME = Object.freeze({ + id: 'native', + executionKinds: Object.freeze(['native-binary']), + // Nothing of its own. A compiled binary is loaded by the operating system's dynamic linker, and + // the variables that steer it are the target's — `LD_LIBRARY_PATH` and its siblings — which the + // target adapter already contributes. Repeating them here would double every diagnostic line. + executionEnvironmentVariables: Object.freeze([]), + // The one shape a runtime with no module system can answer. + selfTestProbeKinds: Object.freeze(['commands']), + + layout: nativeLayout, + + executablePayloadPaths(target) { + return prefixExecutablePayloadPaths(nativeLayout(target)); + }, + + resolveExecutionFiles({ execution }) { + return Object.freeze({ + candidates: Object.freeze([execution.binary]), + missing: `Execution binary is missing from the box: ${execution.binary}.`, + }); + }, + + buildArgv({ execution }) { + const args = (execution.defaultArgs ?? []).map((value) => Object.freeze({ kind: 'literal', value })); + // The binary *is* the command. Every other runtime puts its own entry point first and the + // declaration second; here there is nothing to put first, which is the whole of what `native` + // means. + return Object.freeze({ + command: Object.freeze({ kind: 'payload-path', value: execution.binary }), + args: Object.freeze(args), + }); + }, + + selfTestInvocations({ probe, execution, target }) { + assertProbeKinds(NATIVE_RUNTIME, probe); + return freezeInvocations(commandInvocations(NATIVE_RUNTIME, { probe, execution, target })); + }, +}); + +const RUNTIME_ADAPTERS = Object.freeze([PYTHON_RUNTIME, NODE_RUNTIME, NATIVE_RUNTIME]); /** * Returns the runtime adapter for a runtime id. @@ -349,21 +583,50 @@ export function runtimeAdapters() { /** * Ensures a declared entry point agrees with where the runtime actually sits in the payload. * + * Three answers, because there are three cases. A runtime with an interpreter admits exactly one + * value for a given target, so a declaration is checked against it. A runtime without one — a + * native box — admits none, and a declaration there is refused rather than ignored: it would name a + * file the box never starts, and a reader would believe it. And a box that declares nothing at all + * is checked against nothing, because `runtime.entryPoint` is optional on the wire and its absence + * is a legitimate answer for both. + * * @param {string} runtimeId * @param {import('./targets.mjs').BoxTargetAdapter} adapter the resolved target adapter, whose id * names the layout the entry point is being judged against - * @param {string} entryPoint + * @param {string | null | undefined} entryPoint * @returns {void} * @throws {TypeError} when the entry point is not the one the runtime defines for this target */ export function assertRuntimeEntryPoint(runtimeId, adapter, entryPoint) { const runtime = runtimeAdapter(runtimeId); const expected = runtime.layout(adapter).entryPoint; - if (entryPoint !== expected) { + if (expected === null) { + if (entryPoint !== undefined && entryPoint !== null) { + throw new TypeError(`${runtime.id} boxes have no runtime entry point to declare; the executable a ${runtime.id} box runs is named by its execution`); + } + return; + } + if (entryPoint !== undefined && entryPoint !== expected) { throw new TypeError(`${adapter.id} boxes with the ${runtime.id} runtime must use entry point ${expected}`); } } +/** + * The message for a self-test probe shape the runtime cannot answer. + * + * Stated here, beside the rule, for the same reason `resolveExecutionFiles` returns its own + * `missing`: the wording is part of the contract, and the builder and all three consumers should + * refuse an impossible probe identically instead of each inventing a phrasing. + * + * @param {string} runtimeId + * @param {string} probeKind + * @returns {string} + */ +export function unsupportedSelfTestProbeMessage(runtimeId, probeKind) { + const runtime = runtimeAdapter(runtimeId); + return `The ${runtime.id} runtime cannot answer a selfTest.${probeKind} probe; it answers ${runtime.selfTestProbeKinds.map((kind) => `selfTest.${kind}`).join(' and ')}.`; +} + /** * The message for a box declaring a runtime this build has no adapter for. * diff --git a/src/contract/schema/box-manifest.schema.json b/src/contract/schema/box-manifest.schema.json index 6060ec5..de6342c 100644 --- a/src/contract/schema/box-manifest.schema.json +++ b/src/contract/schema/box-manifest.schema.json @@ -40,6 +40,9 @@ "type": "string", "minLength": 1 }, + "bundledLicenses": { + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/bundledLicenses" + }, "environment": { "type": "object", "description": "Environment variables repeated from the signed release. Scrollcase consumers compare this declaration before execution and apply it over inherited and caller-supplied values.", diff --git a/src/contract/schema/release-manifest.schema.json b/src/contract/schema/release-manifest.schema.json index 90d1b23..24df66c 100644 --- a/src/contract/schema/release-manifest.schema.json +++ b/src/contract/schema/release-manifest.schema.json @@ -135,6 +135,9 @@ "minLength": 1, "description": "Directory relative to the extracted box root holding the box's own large files." }, + "bundledLicenses": { + "$ref": "#/$defs/bundledLicenses" + }, "environment": { "type": "object", "description": "Signed environment variables applied whenever Scrollcase runs the box interpreter. These values override both the inherited host environment and caller-supplied values.", @@ -265,6 +268,59 @@ } } }, + "bundledLicenses": { + "type": "array", + "minItems": 1, + "description": "Dependencies compiled inside the binaries this box ships, as the publishing project declared them. The conda environment's own licences are derived from pixi.lock and travel inside the payload; this list is the half no lock can see — code linked into a supplied executable before the build began — so it is declared, reviewed by the project, and signed here unchanged. It is carried in the release rather than only in the payload so that a licence decision can be made from the document alone, before an archive is downloaded. A box that declares none carries no such list, which means the project declared none and never that the box has no dependencies.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "version", + "declaredLicense", + "linkedInto" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "The dependency as its own project names it." + }, + "version": { + "type": "string", + "minLength": 1 + }, + "declaredLicense": { + "type": "string", + "minLength": 1, + "description": "The licence the project reviewed, conventionally an SPDX expression. Scrollcase carries the string through and never parses it: what a licence permits is not a question a packaging tool answers.", + "examples": [ + "Apache-2.0 OR MIT" + ] + }, + "linkedInto": { + "type": "array", + "minItems": 1, + "description": "Payload files this dependency is compiled into. Every path must be a file the built box actually carries, which is what makes the declaration checkable rather than a free-text notice.", + "items": { + "$ref": "#/$defs/payloadPath" + } + }, + "sourceUrl": { + "type": "string", + "minLength": 1, + "description": "Where the dependency's source can be obtained, for a licence that requires the offer." + } + } + } + }, + "payloadPath": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*//).+$", + "description": "A forward-slash path inside the box payload: relative, non-empty, and unable to escape the payload root." + }, "deferredAssets": { "type": "array", "minItems": 1, diff --git a/src/contract/schema/scroll.schema.json b/src/contract/schema/scroll.schema.json index 57239f3..74a6464 100644 --- a/src/contract/schema/scroll.schema.json +++ b/src/contract/schema/scroll.schema.json @@ -105,6 +105,11 @@ "minLength": 1, "description": "Path to the reviewed licence inventory derived from pixi.lock, which carries an SPDX licence per package. The build fails if the lock no longer matches what was reviewed." }, + "bundledLicenseDeclaration": { + "type": "string", + "minLength": 1, + "description": "Path to the project's inventory of dependencies compiled *inside* the binaries this box ships. pixi.lock declares a licence per conda package, but it cannot see what was linked into a supplied executable before the build ever started, and nothing Scrollcase can read will tell it. So this half is declared rather than derived: the file is a JSON array of { name, version, declaredLicense, linkedInto } entries, and the build checks that every path it names is really in the box before carrying the list into the signed release. What belongs in it is the project's judgement; Scrollcase transports and signs what the project reviewed and never decides what a complete inventory is." + }, "cacheSubdir": { "type": "string", "minLength": 1, diff --git a/src/contract/types/index.d.ts b/src/contract/types/index.d.ts index 5d57579..03b8e79 100644 --- a/src/contract/types/index.d.ts +++ b/src/contract/types/index.d.ts @@ -161,6 +161,10 @@ export interface BoxScroll { * Path to the reviewed licence inventory derived from pixi.lock, which carries an SPDX licence per package. The build fails if the lock no longer matches what was reviewed. */ condaDependencyLicenseAudit?: string; + /** + * Path to the project's inventory of dependencies compiled *inside* the binaries this box ships. pixi.lock declares a licence per conda package, but it cannot see what was linked into a supplied executable before the build ever started, and nothing Scrollcase can read will tell it. So this half is declared rather than derived: the file is a JSON array of { name, version, declaredLicense, linkedInto } entries, and the build checks that every path it names is really in the box before carrying the list into the signed release. What belongs in it is the project's judgement; Scrollcase transports and signs what the project reviewed and never decides what a complete inventory is. + */ + bundledLicenseDeclaration?: string; /** * Payload directory the box's own large files live under — the destination a scroll's assets conventionally share. Defaults to cache/. */ @@ -282,6 +286,55 @@ export interface Runtime { /** * Run one regular payload file with the box's own Python interpreter. */ +export type BundledLicenses = [ + { + /** + * The dependency as its own project names it. + */ + name: string; + version: string; + /** + * The licence the project reviewed, conventionally an SPDX expression. Scrollcase carries the string through and never parses it: what a licence permits is not a question a packaging tool answers. + */ + declaredLicense: string; + /** + * Payload files this dependency is compiled into. Every path must be a file the built box actually carries, which is what makes the declaration checkable rather than a free-text notice. + * + * @minItems 1 + */ + linkedInto: [string, ...string[]]; + /** + * Where the dependency's source can be obtained, for a licence that requires the offer. + */ + sourceUrl?: string; + }, + ...{ + /** + * The dependency as its own project names it. + */ + name: string; + version: string; + /** + * The licence the project reviewed, conventionally an SPDX expression. Scrollcase carries the string through and never parses it: what a licence permits is not a question a packaging tool answers. + */ + declaredLicense: string; + /** + * Payload files this dependency is compiled into. Every path must be a file the built box actually carries, which is what makes the declaration checkable rather than a free-text notice. + * + * @minItems 1 + */ + linkedInto: [string, ...string[]]; + /** + * Where the dependency's source can be obtained, for a licence that requires the offer. + */ + sourceUrl?: string; + }[] +]; +/** + * The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only. + * + * Each kind is named -, and the runtime half must be the one the box declares: a python-script in a box whose runtime is native describes something that cannot be run, and is refused rather than guessed at. + */ export type DeferredAssets = [ { url: string; @@ -316,6 +369,7 @@ export interface BoxManifest { target: BoxTarget; runtime: Runtime; cacheSubdir: string; + bundledLicenses?: BundledLicenses; /** * Environment variables repeated from the signed release. Scrollcase consumers compare this declaration before execution and apply it over inherited and caller-supplied values. */ @@ -445,6 +499,7 @@ export interface BoxReleaseManifest { * Directory relative to the extracted box root holding the box's own large files. */ cacheSubdir: string; + bundledLicenses?: BundledLicenses; /** * Signed environment variables applied whenever Scrollcase runs the box interpreter. These values override both the inherited host environment and caller-supplied values. */ diff --git a/src/runtimes/index.d.mts b/src/runtimes/index.d.mts new file mode 100644 index 0000000..b89e310 --- /dev/null +++ b/src/runtimes/index.d.mts @@ -0,0 +1,68 @@ +/** + * Returns the builder-side adapter for a runtime id. + * + * @param {string} runtimeId + * @returns {RuntimeBuilder} + * @throws {TypeError} when Scrollcase cannot build boxes for that runtime + */ +export function runtimeBuilder(runtimeId: string): RuntimeBuilder; +/** + * Lists every runtime the builder can pack, for the CLI's own listings and for contract tests. + * + * @returns {RuntimeBuilder[]} every builder, as a fresh array + */ +export function runtimeBuilders(): RuntimeBuilder[]; +/** + * What the builder needs from a runtime, beyond what the contract already states. + */ +export type RuntimeBuilder = { + id: string; + /** + * the pure half, so a + * caller holding a builder never has to look the same runtime up twice + */ + contract: import("../contract/runtimes.mjs").BoxRuntimeAdapter; + /** + * the + * `[dependencies]` entry a generated pixi manifest declares for this runtime, or null for a + * runtime that installs nothing of its own + */ + pixiDependency: (runtimeVersion: string) => { + name: string; + spec: string; + } | null; + /** + * makes generated console + * scripts stop pointing at the build machine — by rewriting them where the runtime's trampoline + * is understood, and by refusing the box where it is not + */ + repairLaunchers: (layout: import("../contract/runtimes.mjs").BoxRuntimeLayout, payloadDir: string, forbiddenPaths: readonly string[]) => Promise; + /** + * the source `new scroll` writes, or null for a + * runtime whose entry point Scrollcase cannot generate + */ + templates: RuntimeTemplates | null; +}; +export type RuntimeTemplates = { + /** + * the application entry point a generated scroll points at + */ + script: string; + /** + * the self-test a generated scroll runs + */ + selfTest: string; + /** + * what the generated entry point is called + */ + scriptFileName: string; + /** + * what the generated self-test is called + */ + selfTestFileName: string; + /** + * a module every box of this runtime can load, for the probe a + * generated scroll starts with + */ + starterImport: string; +}; diff --git a/src/runtimes/index.mjs b/src/runtimes/index.mjs index 33b3692..68c190a 100644 --- a/src/runtimes/index.mjs +++ b/src/runtimes/index.mjs @@ -1,24 +1,60 @@ /** * The registry of builder-side runtime adapters. * - * One entry per runtime Scrollcase can actually pack. It is deliberately not seeded with the - * runtimes that are planned but unimplemented: a registry that answers for a runtime no build can - * produce turns "unsupported" into a failure somewhere further down, where the message no longer - * says what went wrong. + * One entry per runtime Scrollcase can actually pack. It is deliberately not seeded with runtimes + * that are named but unimplemented: a registry that answers for a runtime no build can produce + * turns "unsupported" into a failure somewhere further down, where the message no longer says what + * went wrong. * * The contract half of a runtime lives in `src/contract/runtimes.mjs` and is mirrored in every - * consumer language. This half is the builder's alone. + * consumer language. This half is the builder's alone, and the three entries differ in exactly the + * ways the runtimes do: `python` brings an interpreter and a launcher parser, `node` brings an + * interpreter and needs no parser, and `native` brings neither and generates nothing. */ +import { nativeRuntimeBuilder } from './native/index.mjs'; +import { nodeRuntimeBuilder } from './node/index.mjs'; import { pythonRuntimeBuilder } from './python/index.mjs'; -const RUNTIME_BUILDERS = Object.freeze([pythonRuntimeBuilder]); +/** + * What the builder needs from a runtime, beyond what the contract already states. + * + * @typedef {object} RuntimeBuilder + * @property {string} id + * @property {import('../contract/runtimes.mjs').BoxRuntimeAdapter} contract the pure half, so a + * caller holding a builder never has to look the same runtime up twice + * @property {(runtimeVersion: string) => { name: string, spec: string } | null} pixiDependency the + * `[dependencies]` entry a generated pixi manifest declares for this runtime, or null for a + * runtime that installs nothing of its own + * @property {(layout: import('../contract/runtimes.mjs').BoxRuntimeLayout, payloadDir: string, + * forbiddenPaths: readonly string[]) => Promise} repairLaunchers makes generated console + * scripts stop pointing at the build machine — by rewriting them where the runtime's trampoline + * is understood, and by refusing the box where it is not + * @property {RuntimeTemplates | null} templates the source `new scroll` writes, or null for a + * runtime whose entry point Scrollcase cannot generate + */ + +/** + * @typedef {object} RuntimeTemplates + * @property {string} script the application entry point a generated scroll points at + * @property {string} selfTest the self-test a generated scroll runs + * @property {string} scriptFileName what the generated entry point is called + * @property {string} selfTestFileName what the generated self-test is called + * @property {string} starterImport a module every box of this runtime can load, for the probe a + * generated scroll starts with + */ + +const RUNTIME_BUILDERS = Object.freeze([ + pythonRuntimeBuilder, + nodeRuntimeBuilder, + nativeRuntimeBuilder, +]); /** * Returns the builder-side adapter for a runtime id. * * @param {string} runtimeId - * @returns {import('./python/index.mjs').RuntimeBuilder} + * @returns {RuntimeBuilder} * @throws {TypeError} when Scrollcase cannot build boxes for that runtime */ export function runtimeBuilder(runtimeId) { @@ -30,7 +66,7 @@ export function runtimeBuilder(runtimeId) { /** * Lists every runtime the builder can pack, for the CLI's own listings and for contract tests. * - * @returns {import('./python/index.mjs').RuntimeBuilder[]} every builder, as a fresh array + * @returns {RuntimeBuilder[]} every builder, as a fresh array */ export function runtimeBuilders() { return [...RUNTIME_BUILDERS]; diff --git a/src/runtimes/launchers.d.mts b/src/runtimes/launchers.d.mts new file mode 100644 index 0000000..82873fe --- /dev/null +++ b/src/runtimes/launchers.d.mts @@ -0,0 +1,10 @@ +/** + * Refuses a packed prefix whose generated launchers name the build machine. + * + * @param {import('../contract/runtimes.mjs').BoxRuntimeLayout} layout where the runtime sits in the + * payload, for the target being packed + * @param {string} payloadDir + * @param {readonly string[]} forbiddenPaths + * @returns {Promise} + */ +export function assertRelocatableLaunchers(layout: import("../contract/runtimes.mjs").BoxRuntimeLayout, payloadDir: string, forbiddenPaths: readonly string[]): Promise; diff --git a/src/runtimes/launchers.mjs b/src/runtimes/launchers.mjs new file mode 100644 index 0000000..581b4b9 --- /dev/null +++ b/src/runtimes/launchers.mjs @@ -0,0 +1,44 @@ +/** + * The launcher check every runtime shares, for the runtimes that cannot rewrite one. + * + * A packed conda prefix must carry no path from the machine that built it — that is the whole + * reason `conda-unpack` is refused and the Python console scripts are rewritten. Rewriting needs a + * parser for the trampoline a particular ecosystem generates, and Scrollcase has exactly one of + * those (`python/launchers.mjs`, whose `'''exec'` header is Python source pretending to be shell). + * + * A runtime with no such parser is not thereby excused from the guarantee. So it scans instead: if + * a generated launcher in the packed prefix carries a build path, the build stops and says so, + * rather than shipping a box that leaks a developer's directory layout and points at an interpreter + * that is not there. A `node` or `native` prefix normally has nothing to find — conda-forge's own + * launchers for both resolve relative to themselves — so this is a guard, not a routine step, and + * the day it fires it is telling the truth about a box that would not have worked. + */ + +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { collectFiles, fileExists } from '../build/filesystem.mjs'; +import { fail } from '../build/process.mjs'; + +/** + * Refuses a packed prefix whose generated launchers name the build machine. + * + * @param {import('../contract/runtimes.mjs').BoxRuntimeLayout} layout where the runtime sits in the + * payload, for the target being packed + * @param {string} payloadDir + * @param {readonly string[]} forbiddenPaths + * @returns {Promise} + */ +export async function assertRelocatableLaunchers(layout, payloadDir, forbiddenPaths) { + const scriptsRoot = join(payloadDir, ...layout.scriptsDirectory.split('/')); + if (!await fileExists(scriptsRoot)) return; + for (const file of await collectFiles(scriptsRoot)) { + const path = join(scriptsRoot, ...file.split('/')); + const bytes = await readFile(path); + if (!bytes.subarray(0, 2).equals(Buffer.from('#!'))) continue; + const text = bytes.toString('utf8'); + const leaked = forbiddenPaths.find((value) => text.includes(value)); + if (leaked) { + fail(`${layout.scriptsDirectory}/${file} carries a build path (${leaked}) that Scrollcase cannot rewrite for this runtime. Prune it, or package this box with the runtime whose launchers it generates.`); + } + } +} diff --git a/src/runtimes/native/index.d.mts b/src/runtimes/native/index.d.mts new file mode 100644 index 0000000..d21cd86 --- /dev/null +++ b/src/runtimes/native/index.d.mts @@ -0,0 +1,2 @@ +/** @type {import('../index.mjs').RuntimeBuilder} */ +export const nativeRuntimeBuilder: import("../index.mjs").RuntimeBuilder; diff --git a/src/runtimes/native/index.mjs b/src/runtimes/native/index.mjs new file mode 100644 index 0000000..6e94a40 --- /dev/null +++ b/src/runtimes/native/index.mjs @@ -0,0 +1,37 @@ +/** + * The builder-side native runtime adapter. + * + * A native box carries a compiled executable and starts it directly. Everything the other two + * adapters exist to arrange — an interpreter to solve for, generated launchers to repair, starter + * source to write — has no counterpart here, and saying so explicitly is the adapter's whole job. + * + * It still packs a conda prefix. `native` is not "no environment": it is "no interpreter". A + * compiled binary links against shared libraries, those libraries come from conda-forge through the + * same `pixi.lock` as everything else, and they get the same licence audit. What the scroll + * declares in `[dependencies]` is therefore the box's business rather than the runtime's, which is + * why this adapter contributes no dependency of its own. + * + * **Link repair is out of scope**, deliberately and for now. A binary that resolves its libraries + * through an absolute path recorded at compile time will not find them inside a box, and fixing + * that means per-format work — rpath on Linux, `install_name` on macOS, the DLL search order on + * Windows — that deserves its own pass rather than a guess in this one. A native box must ship a + * binary that already resolves: statically linked, or built with a relative rpath. This is stated + * in the documentation as a limitation, not left for someone to discover. + */ + +import { runtimeAdapter } from '../../contract/runtimes.mjs'; +import { assertRelocatableLaunchers } from '../launchers.mjs'; + +/** @type {import('../index.mjs').RuntimeBuilder} */ +export const nativeRuntimeBuilder = Object.freeze({ + id: 'native', + contract: runtimeAdapter('native'), + // No interpreter to install. A generated manifest for a native box declares no dependency at all, + // and the author adds the libraries their binary actually needs. + pixiDependency: () => null, + repairLaunchers: assertRelocatableLaunchers, + // Nothing to generate. Scrollcase does not compile anything, so a native scroll has to be pointed + // at a binary that already exists; writing a starter would mean writing source for a language + // this tool never sees. + templates: null, +}); diff --git a/src/runtimes/node/index.d.mts b/src/runtimes/node/index.d.mts new file mode 100644 index 0000000..8d49f18 --- /dev/null +++ b/src/runtimes/node/index.d.mts @@ -0,0 +1,2 @@ +/** @type {import('../index.mjs').RuntimeBuilder} */ +export const nodeRuntimeBuilder: import("../index.mjs").RuntimeBuilder; diff --git a/src/runtimes/node/index.mjs b/src/runtimes/node/index.mjs new file mode 100644 index 0000000..c741725 --- /dev/null +++ b/src/runtimes/node/index.mjs @@ -0,0 +1,32 @@ +/** + * The builder-side Node runtime adapter. + * + * The pure half — layout, execution kinds, argv, the import probe — lives in + * `src/contract/runtimes.mjs` and is mirrored in every consumer language. This is what the builder + * has to do to produce a Node box, which is allowed to touch a filesystem and is mirrored nowhere. + * + * There is less of it than for Python, and that is the interesting part. conda-forge's `nodejs` + * package puts `node` in the prefix's own `bin/` and writes `npm` and `npx` as links into + * `lib/node_modules`, whose shebangs resolve `node` through the environment rather than through an + * absolute path burnt in at solve time. So there is no trampoline to parse and nothing to rewrite — + * only the shared guard that says so out loud if a prefix ever turns out to carry one. + */ + +import { runtimeAdapter } from '../../contract/runtimes.mjs'; +import { assertRelocatableLaunchers } from '../launchers.mjs'; +import { STARTER_SCRIPT, STARTER_SELF_TEST, pixiDependency } from './templates/index.mjs'; + +/** @type {import('../index.mjs').RuntimeBuilder} */ +export const nodeRuntimeBuilder = Object.freeze({ + id: 'node', + contract: runtimeAdapter('node'), + pixiDependency, + repairLaunchers: assertRelocatableLaunchers, + templates: Object.freeze({ + script: STARTER_SCRIPT, + selfTest: STARTER_SELF_TEST, + scriptFileName: 'entrypoint.js', + selfTestFileName: 'self_test.js', + starterImport: 'fs', + }), +}); diff --git a/src/runtimes/node/templates/index.d.mts b/src/runtimes/node/templates/index.d.mts new file mode 100644 index 0000000..9c16c2e --- /dev/null +++ b/src/runtimes/node/templates/index.d.mts @@ -0,0 +1,26 @@ +/** + * The Node constraint a generated pixi manifest declares. + * + * A bare `major` or `major.minor` becomes a `.*` range so the solve is free to take a patch release; + * a version the author spelled out in full is written through unchanged, because someone who typed + * `22.11.0` meant `22.11.0`. + * + * @param {string} runtimeVersion + * @returns {{ name: string, spec: string }} the `[dependencies]` entry this runtime contributes + */ +export function pixiDependency(runtimeVersion: string): { + name: string; + spec: string; +}; +/** + * The JavaScript source `scrollcase new scroll --runtime node` writes for a project that has none. + * + * Same shape and same purpose as the Python starters next door: short enough to read in one screen, + * and explicit about what the author is expected to replace. They are CommonJS because a box's + * entry point is a path the runtime is handed rather than a package with a declared type, and + * `node app.js` reads a file with no `package.json` beside it as CommonJS. + */ +/** The application a generated `node-script` scroll points at. */ +export const STARTER_SCRIPT: "// Minimal application entry point generated by Scrollcase.\n\nfunction main() {\n console.log('Scrollcase box is ready.');\n return 0;\n}\n\nprocess.exitCode = main();\n"; +/** The self-test a generated scroll runs with the box's own Node. */ +export const STARTER_SELF_TEST: "// Self-test for this box, run by `scrollcase build` before the box is archived.\n//\n// It runs with the box's own Node, from the payload root, after the modules declared in scroll.json\n// have already loaded \u2014 so it can read the files the box ships and require the code it packs. A\n// thrown error fails the build, which is the point: this is the last check that the box works\n// before anyone downloads it.\n//\n// Replace the line below with something that would actually notice a broken box.\n\nconsole.log('self-test ok');\n"; diff --git a/src/runtimes/node/templates/index.mjs b/src/runtimes/node/templates/index.mjs new file mode 100644 index 0000000..efdcf79 --- /dev/null +++ b/src/runtimes/node/templates/index.mjs @@ -0,0 +1,49 @@ +/** + * The JavaScript source `scrollcase new scroll --runtime node` writes for a project that has none. + * + * Same shape and same purpose as the Python starters next door: short enough to read in one screen, + * and explicit about what the author is expected to replace. They are CommonJS because a box's + * entry point is a path the runtime is handed rather than a package with a declared type, and + * `node app.js` reads a file with no `package.json` beside it as CommonJS. + */ + +/** The application a generated `node-script` scroll points at. */ +export const STARTER_SCRIPT = `// Minimal application entry point generated by Scrollcase. + +function main() { + console.log('Scrollcase box is ready.'); + return 0; +} + +process.exitCode = main(); +`; + +/** The self-test a generated scroll runs with the box's own Node. */ +export const STARTER_SELF_TEST = `// Self-test for this box, run by \`scrollcase build\` before the box is archived. +// +// It runs with the box's own Node, from the payload root, after the modules declared in scroll.json +// have already loaded — so it can read the files the box ships and require the code it packs. A +// thrown error fails the build, which is the point: this is the last check that the box works +// before anyone downloads it. +// +// Replace the line below with something that would actually notice a broken box. + +console.log('self-test ok'); +`; + +/** + * The Node constraint a generated pixi manifest declares. + * + * A bare `major` or `major.minor` becomes a `.*` range so the solve is free to take a patch release; + * a version the author spelled out in full is written through unchanged, because someone who typed + * `22.11.0` meant `22.11.0`. + * + * @param {string} runtimeVersion + * @returns {{ name: string, spec: string }} the `[dependencies]` entry this runtime contributes + */ +export function pixiDependency(runtimeVersion) { + return { + name: 'nodejs', + spec: /^\d+(?:\.\d+)?$/.test(runtimeVersion) ? `${runtimeVersion}.*` : runtimeVersion, + }; +} diff --git a/src/runtimes/python/index.d.mts b/src/runtimes/python/index.d.mts new file mode 100644 index 0000000..8273243 --- /dev/null +++ b/src/runtimes/python/index.d.mts @@ -0,0 +1,2 @@ +/** @type {import('../index.mjs').RuntimeBuilder} */ +export const pythonRuntimeBuilder: import("../index.mjs").RuntimeBuilder; diff --git a/src/runtimes/python/index.mjs b/src/runtimes/python/index.mjs index a5da147..d645dc4 100644 --- a/src/runtimes/python/index.mjs +++ b/src/runtimes/python/index.mjs @@ -16,22 +16,7 @@ import { runtimeAdapter } from '../../contract/runtimes.mjs'; import { repairPosixLaunchers } from './launchers.mjs'; import { STARTER_SCRIPT, STARTER_SELF_TEST, pixiDependency } from './templates/index.mjs'; -/** - * What the builder needs from a runtime, beyond what the contract already states. - * - * @typedef {object} RuntimeBuilder - * @property {string} id - * @property {import('../../contract/runtimes.mjs').BoxRuntimeAdapter} contract the pure half, so a - * caller holding a builder never has to look the same runtime up twice - * @property {(runtimeVersion: string) => { name: string, spec: string }} pixiDependency the - * `[dependencies]` entry a generated pixi manifest declares for this runtime - * @property {(layout: import('../../contract/runtimes.mjs').BoxRuntimeLayout, payloadDir: string, - * forbiddenPaths: readonly string[]) => Promise} repairLaunchers rewrites generated console - * scripts so nothing in the box points at the build machine - * @property {{ script: string, selfTest: string }} templates the source `new scroll` writes - */ - -/** @type {RuntimeBuilder} */ +/** @type {import('../index.mjs').RuntimeBuilder} */ export const pythonRuntimeBuilder = Object.freeze({ id: 'python', contract: runtimeAdapter('python'), @@ -40,5 +25,8 @@ export const pythonRuntimeBuilder = Object.freeze({ templates: Object.freeze({ script: STARTER_SCRIPT, selfTest: STARTER_SELF_TEST, + scriptFileName: 'entrypoint.py', + selfTestFileName: 'self_test.py', + starterImport: 'json', }), }); diff --git a/src/runtimes/python/templates/index.d.mts b/src/runtimes/python/templates/index.d.mts new file mode 100644 index 0000000..ef9c7ec --- /dev/null +++ b/src/runtimes/python/templates/index.d.mts @@ -0,0 +1,26 @@ +/** + * The interpreter constraint a generated pixi manifest declares. + * + * A bare `major.minor` becomes `major.minor.*` so the solve is free to take a patch release; a + * version the author spelled out in full is written through unchanged, because someone who typed + * `3.14.2` meant `3.14.2`. + * + * @param {string} runtimeVersion + * @returns {{ name: string, spec: string }} the `[dependencies]` entry this runtime contributes + */ +export function pixiDependency(runtimeVersion: string): { + name: string; + spec: string; +}; +/** + * The Python source `scrollcase new scroll` writes for a project that has none yet. + * + * These are starting points, not scaffolding to be extended: each is short enough to read in one + * screen and says what the author is expected to replace. They live under the runtime because they + * are Python source — a Node box or a native box needs different files, and the authoring path + * should reach for the runtime's own rather than acquire a branch per runtime. + */ +/** The application a generated `python-script` scroll points at. */ +export const STARTER_SCRIPT: "\"\"\"Minimal application entry point generated by Scrollcase.\"\"\"\n\nimport sys\n\n\ndef main() -> int:\n print(\"Scrollcase box is ready.\")\n return 0\n\n\nif __name__ == \"__main__\":\n sys.exit(main())\n"; +/** The self-test a generated scroll runs with the box's own interpreter. */ +export const STARTER_SELF_TEST: "\"\"\"Self-test for this box, run by `scrollcase build` before the box is archived.\n\nIt runs with the box's own interpreter, from the payload root, after the imports declared in\nscroll.json have already succeeded \u2014 so it can read the files the box ships and import the code it\npacks. Any exception fails the build, which is the point: this is the last check that the box works\nbefore anyone downloads it.\n\nReplace the line below with something that would actually notice a broken box.\n\"\"\"\n\nprint(\"self-test ok\")\n"; diff --git a/tests/helpers/consumer-conformance.mjs b/tests/helpers/consumer-conformance.mjs index 11d6f5d..4053a1d 100644 --- a/tests/helpers/consumer-conformance.mjs +++ b/tests/helpers/consumer-conformance.mjs @@ -185,8 +185,9 @@ async function mutateFixture(fixture, mutation, destination) { return; } if (mutation === 'alter-release-runtime-id') { - // A runtime the format names and this build has no adapter for. The consumer must refuse the - // box rather than fall back to reading it as the runtime it happens to be shaped like. + // A Python box relabelled as native after it was built. Everything about the payload still + // says Python, so the consumer must refuse it rather than read the declaration as the truth + // about a box that disagrees with it. fixture.release.runtime = { ...fixture.release.runtime, id: 'native' }; await writeSignedRelease(fixture, fixture.release); return; diff --git a/tests/unit/build-pipeline.test.mjs b/tests/unit/build-pipeline.test.mjs index 5b3d8f9..773f3fc 100644 --- a/tests/unit/build-pipeline.test.mjs +++ b/tests/unit/build-pipeline.test.mjs @@ -152,12 +152,20 @@ function plantPrefixSymlinks(prefix) { * asset staging, pruning, the self-test gate, box.json, the deterministic archive, signing — is the * real implementation, which is what this test is here to exercise. */ -function fakeToolchain(payloadDir, { module = null, onSelfTest = null, consoleScript = null } = {}) { +function fakeToolchain(payloadDir, { + module = null, + onSelfTest = null, + consoleScript = null, + // What the box starts. A native box starts a file the scroll brought in, and its packed prefix + // carries no interpreter at all — so both are parameters rather than the Python answer baked in. + interpreter = true, + selfTestCommand = join(payloadDir, ...ENTRY_SEGMENTS), +} = {}) { const run = function run(command, args = [], options = {}) { if (command === 'pixi' && args[0] === 'install') { const manifest = args[args.indexOf('--manifest-path') + 1]; const prefix = join(dirname(manifest), '.pixi', 'envs', 'default'); - writeDeep(join(prefix, ...ENTRY_SEGMENTS.slice(1)), '#!/bin/sh\nexit 0\n'); + if (interpreter) writeDeep(join(prefix, ...ENTRY_SEGMENTS.slice(1)), '#!/bin/sh\nexit 0\n'); if (consoleScript) { // What conda actually generates: the *build machine's* interpreter, reached through the // shell trampoline it falls back to when an absolute shebang would be too long. @@ -196,8 +204,8 @@ function fakeToolchain(payloadDir, { module = null, onSelfTest = null, consoleSc tar.c({ file: output, cwd: prefix, gzip: true, sync: true }, ['.']); return ''; } - // Anything else is the box's own interpreter, running the self-test. - expect(command).toBe(join(payloadDir, ...ENTRY_SEGMENTS)); + // Anything else is the box running its own self-test, with whatever the runtime starts. + expect(command).toBe(selfTestCommand); onSelfTest?.({ command, args, options }); return ''; }; @@ -331,11 +339,169 @@ describe('the build pipeline', () => { await expect(readScroll(SCROLL_REF)).rejects.toThrow(/entry point/); }); - it('refuses a runtime the format defines but this build cannot produce', async () => { - // The wire vocabulary is deliberately wider than the implemented set, so this is an expected - // refusal with a message that says which of the two the scroll fell foul of. - await makeProject({ ...SCROLL, runtime: { id: 'native' } }, { commit: false }); - await expect(readScroll(SCROLL_REF)).rejects.toThrow(/native is not implemented/); + it('builds, signs and verifies a native box that starts a binary of its own', async () => { + const NATIVE_SCROLL = { + ...SCROLL, + runtime: { id: 'native' }, + condaDependencyLicenseAudit: undefined, + bundledLicenseDeclaration: 'legal/bundled.json', + localFiles: [{ sourcePath: 'tool', relativePath: 'bin/tool', executable: true }], + execution: { kind: 'native-binary', binary: 'bin/tool', defaultArgs: ['--quiet'] }, + selfTest: { commands: [{ args: ['--version'] }], files: ['bin/tool'] }, + }; + delete NATIVE_SCROLL.condaDependencyLicenseAudit; + const declared = [{ + name: 'zlib', + version: '1.3.1', + declaredLicense: 'Zlib', + linkedInto: ['bin/tool'], + sourceUrl: 'https://zlib.net/', + }]; + const { keys, payloadDir } = await makeProject(NATIVE_SCROLL, { + projectFiles: { + tool: '#!/bin/sh\nexit 0\n', + 'legal/bundled.json': `${JSON.stringify(declared, null, 2)}\n`, + }, + }); + const invoked = []; + const built = await buildBox(SCROLL_REF, { + ...keys, + ...fakeToolchain(payloadDir, { + interpreter: false, + selfTestCommand: join(payloadDir, 'bin', 'tool'), + onSelfTest: ({ args }) => invoked.push(args), + }), + log: () => {}, + }); + + // The binary is the command; the declaration's own arguments come first, the probe's after. + expect(invoked).toEqual([['--quiet', '--version']]); + const release = decodeDocumentPayload(JSON.parse(await readFile(built.releasePath, 'utf8'))); + expect(release.runtime).toEqual({ id: 'native' }); + // Transported and signed exactly as declared: Scrollcase derives nothing here and reorders + // nothing, because it has no way to know what is inside a binary somebody else compiled. + expect(release.bundledLicenses).toEqual(declared); + // A payload file the scroll declared executable, so the archive marks it — which is the only + // reason the box can start at all. + const modes = await zipModes(join(dirname(built.releasePath), `${built.archiveSha256}.zip`)); + if (process.platform !== 'win32') expect(modes.get('bin/tool')).toBe(0o100755); + const extracted = join(payloadDir, '..', 'extracted'); + await extractZipArchive(join(dirname(built.releasePath), `${built.archiveSha256}.zip`), extracted); + // The same list a reader of the release saw, shipped where someone opening the box will look. + expect(JSON.parse(await readFile(join(extracted, 'THIRD_PARTY_NOTICES', 'bundled-dependencies.json'), 'utf8'))) + .toEqual(declared); + const box = JSON.parse(await readFile(join(extracted, 'box.json'), 'utf8')); + expect(box.bundledLicenses).toEqual(declared); + expect(() => assertBoxManifestAgreement(box, release)).not.toThrow(); + await expect(verifyBox(built.releasePath, { publicPath: keys.publicPath, log: () => {} })) + .resolves.toMatchObject({ status: 'passed' }); + }); + + it('refuses a bundled licence entry naming a file the box does not carry', async () => { + const { keys, payloadDir } = await makeProject({ + ...SCROLL, + bundledLicenseDeclaration: 'legal/bundled.json', + }, { + projectFiles: { + 'legal/bundled.json': `${JSON.stringify([{ + name: 'zlib', + version: '1.3.1', + declaredLicense: 'Zlib', + linkedInto: ['bin/gone'], + }])}\n`, + }, + }); + await expect(buildBox(SCROLL_REF, { + ...keys, + ...fakeToolchain(payloadDir), + log: () => {}, + })).rejects.toThrow(/zlib==1\.3\.1 is declared linked into bin\/gone, which this box does not carry/); + }); + + it('refuses a bundled licence declaration the format cannot carry', async () => { + const { keys, payloadDir } = await makeProject({ + ...SCROLL, + bundledLicenseDeclaration: 'legal/bundled.json', + }, { + projectFiles: { + // No `linkedInto`: an entry that names no file is a notice, not an inventory, and nothing + // about it could ever be checked against the box. + 'legal/bundled.json': `${JSON.stringify([{ name: 'zlib', version: '1.3.1', declaredLicense: 'Zlib' }])}\n`, + }, + }); + await expect(buildBox(SCROLL_REF, { + ...keys, + ...fakeToolchain(payloadDir), + log: () => {}, + })).rejects.toThrow(/declared bundled licence inventory is invalid/); + }); + + it('refuses a box that runs a file the archive would not mark executable', async () => { + const { keys, payloadDir } = await makeProject({ + ...SCROLL, + runtime: { id: 'native' }, + // Declared without `executable`, so the archive would ship it 0644 and nothing could start it. + localFiles: [{ sourcePath: 'tool', relativePath: 'bin/tool' }], + execution: { kind: 'native-binary', binary: 'bin/tool', defaultArgs: [] }, + selfTest: { commands: [{ args: [] }], files: [] }, + }, { projectFiles: { tool: '#!/bin/sh\nexit 0\n' } }); + await expect(buildBox(SCROLL_REF, { + ...keys, + ...fakeToolchain(payloadDir, { + interpreter: false, + selfTestCommand: join(payloadDir, 'bin', 'tool'), + }), + log: () => {}, + })).rejects.toThrow(/runs bin\/tool, which the archive would not mark executable/); + }); + + it('refuses a native scroll that declares a runtime entry point', async () => { + // A native box starts a binary the scroll named; there is no interpreter to point at, and a + // declaration here would name a file the box never runs. + await makeProject({ + ...SCROLL, + runtime: { id: 'native', entryPoint: HOST_LAYOUT.entryPoint }, + execution: { kind: 'native-binary', binary: 'bin/tool', defaultArgs: [] }, + selfTest: { commands: [{ args: ['--version'] }], files: [] }, + }, { commit: false }); + await expect(readScroll(SCROLL_REF)).rejects.toThrow(/no runtime entry point to declare/); + }); + + it('refuses an import probe in a box whose runtime has no module system', async () => { + await makeProject({ + ...SCROLL, + runtime: { id: 'native' }, + execution: { kind: 'native-binary', binary: 'bin/tool', defaultArgs: [] }, + selfTest: { imports: ['json'], files: [] }, + }, { commit: false }); + await expect(readScroll(SCROLL_REF)) + .rejects.toThrow(/native runtime cannot answer a selfTest.imports probe/); + }); + + it('derives no entry point for a runtime that has none', async () => { + await makeProject({ + ...SCROLL, + runtime: { id: 'native' }, + execution: { kind: 'native-binary', binary: 'bin/tool', defaultArgs: [] }, + selfTest: { commands: [{ args: ['--version'] }], files: [] }, + }, { commit: false }); + const { scroll } = await readScroll(SCROLL_REF); + expect(scroll.runtime).toEqual({ id: 'native' }); + }); + + it('refuses a parity gate in a box with no interpreter to run it', async () => { + await makeProject({ + ...SCROLL, + runtime: { id: 'native' }, + execution: { kind: 'native-binary', binary: 'bin/tool', defaultArgs: [] }, + selfTest: { commands: [{ args: ['--version'] }], files: [] }, + parity: { + script: 'parity.py', + accelerators: ['cpu', 'cuda'], + tolerances: { absolute: 1e-6 }, + }, + }, { commit: false }); + await expect(readScroll(SCROLL_REF)).rejects.toThrow(/no interpreter to run a parity check/); }); it('refuses an execution kind belonging to another runtime', async () => { diff --git a/tests/unit/contract-runtimes.test.mjs b/tests/unit/contract-runtimes.test.mjs index 1963116..3988815 100644 --- a/tests/unit/contract-runtimes.test.mjs +++ b/tests/unit/contract-runtimes.test.mjs @@ -4,12 +4,14 @@ import { fixtureUrl } from '../../src/contract/index.mjs'; import { boxTargetAdapters } from '../../src/contract/targets.mjs'; import { RUNTIME_IDS, + assertRuntimeEntryPoint, executionAffectingVariables, isExecutablePayloadPath, isImplementedRuntime, runtimeAdapter, runtimeAdapters, unimplementedRuntimeMessage, + unsupportedSelfTestProbeMessage, } from '../../src/contract/runtimes.mjs'; const PYTHON = 'python'; @@ -46,12 +48,11 @@ describe('runtime adapters', () => { } }); - it('refuses a runtime it has no adapter for, and says which kind of refusal it is', () => { - for (const id of ['node', 'native']) { - expect(() => runtimeAdapter(id), id).toThrow(TypeError); - expect(isImplementedRuntime(id), id).toBe(false); - expect(unimplementedRuntimeMessage(id)).toContain('not implemented by this version'); - } + it('refuses a runtime the format does not define, and says so as such', () => { + // This build implements every id the format names, so the other branch of the message — a + // runtime the format defines that this build cannot run — is unreachable here. It is not dead: + // the Python and Rust consumers version independently, and one published before a runtime + // landed still has to refuse a box naming it rather than misread it. for (const id of ['', undefined, null, 42, 'ruby']) { expect(() => runtimeAdapter(id), String(id)).toThrow(TypeError); expect(isImplementedRuntime(id), String(id)).toBe(false); @@ -59,12 +60,38 @@ describe('runtime adapters', () => { } }); + it('refuses a probe shape the runtime cannot answer, in the shared wording', () => { + for (const testCase of contract.unsupportedProbes) { + expect(unsupportedSelfTestProbeMessage(testCase.runtime, testCase.probeKind), testCase.name) + .toBe(testCase.message); + expect(() => runtimeAdapter(testCase.runtime).selfTestInvocations({ + probe: { [testCase.probeKind]: ['anything'] }, + execution: null, + target: targetFor('linux'), + }), testCase.name).toThrow(testCase.message); + } + }); + + it('admits an entry point only where the runtime has one', () => { + const linux = targetFor('linux'); + // Optional on the wire, so declaring nothing is fine for either kind of runtime. + expect(() => assertRuntimeEntryPoint(PYTHON, linux, undefined)).not.toThrow(); + expect(() => assertRuntimeEntryPoint('native', linux, undefined)).not.toThrow(); + expect(() => assertRuntimeEntryPoint(PYTHON, linux, 'venv/bin/python')).not.toThrow(); + expect(() => assertRuntimeEntryPoint(PYTHON, linux, 'venv/bin/python3')) + .toThrow(/must use entry point venv\/bin\/python/); + // A native box that names one is naming a file it never starts, which a reader would believe. + expect(() => assertRuntimeEntryPoint('native', linux, 'venv/bin/python')) + .toThrow(/no runtime entry point to declare/); + }); + it('reproduces every golden layout and executable-path rule', () => { for (const fixture of contract.runtimes) { const runtime = runtimeAdapter(fixture.id); expect([...runtime.executionKinds], fixture.id).toEqual(fixture.executionKinds); expect([...runtime.executionEnvironmentVariables], fixture.id) .toEqual(fixture.executionEnvironmentVariables); + expect([...runtime.selfTestProbeKinds], fixture.id).toEqual(fixture.selfTestProbeKinds); for (const platform of fixture.layouts) { const target = targetFor(platform.platform); expect({ ...runtime.layout(target) }, platform.platform).toEqual(platform.layout); diff --git a/tests/unit/docs-contract.test.mjs b/tests/unit/docs-contract.test.mjs index 71569c5..fb725b0 100644 --- a/tests/unit/docs-contract.test.mjs +++ b/tests/unit/docs-contract.test.mjs @@ -106,7 +106,7 @@ describe('public documentation routes', () => { const source = await readFile(join(schemaSource, name), 'utf8'); const schema = JSON.parse(source); expect(new URL(schema.$id).pathname).toBe(`/schema/v3/${name}`); - for (const match of source.matchAll(/"\$ref":\s*"(https:\/\/scrollcase\.dev\/schema\/v2\/([^"#]+)(?:#[^"]*)?)"/g)) { + for (const match of source.matchAll(/"\$ref":\s*"(https:\/\/scrollcase\.dev\/schema\/v3\/([^"#]+)(?:#[^"]*)?)"/g)) { expect(published.has(match[2]), match[1]).toBe(true); } } diff --git a/tests/unit/project-surface.test.mjs b/tests/unit/project-surface.test.mjs index ecded45..3f2f64a 100644 --- a/tests/unit/project-surface.test.mjs +++ b/tests/unit/project-surface.test.mjs @@ -172,16 +172,13 @@ describe('auditing dependency licences', () => { workspace: getWorkspace(), boxId: 'example-model', target: TARGET, - modelId: 'example-org-example-model', - runtimeId: 'example-runtime', version: '1.0.0', scrollVersion: '1.0.0', sourceRevision: 'upstream-v1', - pythonVersion: '3.11.15', + runtimeVersion: '3.11.15', pixiVersion: '0.73.0', compatibility: { minHostAppVersion: '1.0.0' }, assetBaseUrl: 'https://assets.example.org', - weights: 'embed', executionKind: 'library-only', }); const scroll = JSON.parse(await readFile(join(result.scrollDir, 'scroll.json'), 'utf8')); diff --git a/tests/unit/scroll-authoring.test.mjs b/tests/unit/scroll-authoring.test.mjs index 2df3e50..7a165a6 100644 --- a/tests/unit/scroll-authoring.test.mjs +++ b/tests/unit/scroll-authoring.test.mjs @@ -6,12 +6,14 @@ import { PassThrough } from 'node:stream'; import { afterEach, describe, expect, it } from 'vitest'; import { copyVerifiedLocalFile } from '../../src/build/assets.mjs'; import { + DEFAULT_NODE_VERSION, DEFAULT_PYTHON_VERSION, + LATEST_NODE_VERSION, LATEST_PYTHON_VERSION, createScroll, ensureConsumerTemplates, ensureExampleScroll, - resolvePythonVersion, + resolveRuntimeVersion, } from '../../src/build/authoring.mjs'; import { fileExists, sha256File } from '../../src/build/filesystem.mjs'; import { initProject } from '../../src/build/project.mjs'; @@ -27,7 +29,7 @@ const BASE = { version: '1.0.0', scrollVersion: '1.0.0', sourceRevision: 'upstream-v1', - pythonVersion: '3.11.15', + runtimeVersion: '3.11.15', pixiVersion: '0.73.0', compatibility: { minHostAppVersion: '1.0.0' }, assetBaseUrl: 'https://assets.example.org', @@ -68,6 +70,87 @@ describe('scroll authoring', () => { .toContain('platforms = ["osx-arm64"]'); }); + it('writes a node scroll in the node runtime\'s own terms', async () => { + const current = await workspace(); + const result = await createScroll({ + workspace: current, + ...BASE, + runtimeId: 'node', + runtimeVersion: undefined, + executionKind: 'node-script', + generateScript: true, + }); + + expect(result.scroll.runtime).toEqual({ id: 'node', version: DEFAULT_NODE_VERSION }); + expect(result.scroll.execution) + .toEqual({ kind: 'node-script', script: 'entrypoint.js', defaultArgs: [] }); + // The probe, the generated files and the pixi dependency all come from the runtime; nothing + // above it names a language. + expect(result.scroll.selfTest.imports).toEqual(['fs']); + expect(result.scroll.selfTest.script).toMatch(/self_test\.js$/); + expect(result.generatedScriptPath).toMatch(/entrypoint\.js$/); + expect(await readFile(join(result.scrollDir, 'pixi.toml'), 'utf8')) + .toContain(`nodejs = "${DEFAULT_NODE_VERSION}.*"`); + await expect(readScroll(result.scrollRef)).resolves.toBeTruthy(); + }); + + it('writes a native scroll that points at a binary and declares it executable', async () => { + const current = await workspace(); + await writeFile(join(current.root, 'tool'), '#!/bin/sh\nexit 0\n'); + const result = await createScroll({ + workspace: current, + ...BASE, + runtimeId: 'native', + runtimeVersion: undefined, + executionKind: 'native-binary', + scriptSourcePath: 'tool', + scriptRelativePath: 'bin/tool', + }); + + // No version, because there is no interpreter to version — and `readScroll` derives no entry + // point for the same reason. + expect(result.scroll.runtime).toEqual({ id: 'native' }); + expect(result.scroll.execution) + .toEqual({ kind: 'native-binary', binary: 'bin/tool', defaultArgs: [] }); + // Without this the archive would not mark the file executable and the box could not start. + expect(result.scroll.localFiles) + .toEqual([{ sourcePath: 'tool', relativePath: 'bin/tool', executable: true }]); + // A native box has no module system, so its only probe is an invocation of its own binary. + expect(result.scroll.selfTest.imports).toBeUndefined(); + expect(result.scroll.selfTest.commands).toEqual([{ args: [] }]); + expect(result.scroll.selfTest.script).toBeUndefined(); + expect(await readFile(join(result.scrollDir, 'pixi.toml'), 'utf8')) + .toContain('# Add the libraries this box'); + const { scroll } = await readScroll(result.scrollRef); + expect(scroll.runtime.entryPoint).toBeUndefined(); + }); + + it('refuses what a runtime cannot be asked for', async () => { + const current = await workspace(); + await expect(createScroll({ + workspace: current, + ...BASE, + runtimeId: 'native', + runtimeVersion: undefined, + executionKind: 'library-only', + })).rejects.toThrow(/Unsupported execution kind for a native box/); + await expect(createScroll({ + workspace: current, + ...BASE, + runtimeId: 'native', + runtimeVersion: undefined, + executionKind: 'native-binary', + generateScript: true, + })).rejects.toThrow(/cannot generate an entry point for a native box/); + await expect(createScroll({ + workspace: current, + ...BASE, + runtimeId: 'node', + executionKind: 'python-module', + module: 'example', + })).rejects.toThrow(/Unsupported execution kind for a node box/); + }); + it('asks only what it cannot work out, and derives the rest', async () => { const current = await workspace(); const answers = new Map([ @@ -93,7 +176,7 @@ describe('scroll authoring', () => { choose: async (question, _choices, chooseOptions = {}) => { expect(chooseOptions.hint).toEqual(expect.any(String)); chosen.push(question); - return 'python-module'; + return question === 'runtime' ? 'python' : 'python-module'; }, chooseTargetValue: async (_candidates, targetOptions = {}) => { expect(targetOptions.hint).toEqual(expect.any(String)); @@ -108,12 +191,13 @@ describe('scroll authoring', () => { // Labels are not among the menus, and there is nothing to derive: Scrollcase reads none of // them, so prompting for one would be asking the author to fill in a field on the tool's // behalf. A generated scroll carries none. - expect(chosen).toEqual(['execution kind']); + expect(chosen).toEqual(['runtime', 'execution kind']); expect(options.labels).toEqual({}); expect(result.scroll.labels).toBeUndefined(); expect(result.scroll.runtime).toEqual({ id: 'python', version: DEFAULT_PYTHON_VERSION }); expect(options.version).toBe('1.0.0'); - expect(options.pythonVersion).toBe(DEFAULT_PYTHON_VERSION); + expect(options.runtimeId).toBe('python'); + expect(options.runtimeVersion).toBe(DEFAULT_PYTHON_VERSION); expect(options.pixiVersion).toBe(BASE.pixiVersion); expect(result.scroll.execution).toEqual({ kind: 'python-module', @@ -176,11 +260,17 @@ describe('scroll authoring', () => { ); }); - it('resolves --python-version latest to a number, never the word', async () => { - expect(resolvePythonVersion('latest')).toBe(LATEST_PYTHON_VERSION); - expect(resolvePythonVersion('latest')).toMatch(/^\d+\.\d+$/); - expect(resolvePythonVersion(null)).toBe(DEFAULT_PYTHON_VERSION); - expect(resolvePythonVersion('3.10.4')).toBe('3.10.4'); + it('resolves --runtime-version latest to a number, never the word', async () => { + expect(resolveRuntimeVersion('python', 'latest')).toBe(LATEST_PYTHON_VERSION); + expect(resolveRuntimeVersion('python', 'latest')).toMatch(/^\d+\.\d+$/); + expect(resolveRuntimeVersion('python', null)).toBe(DEFAULT_PYTHON_VERSION); + expect(resolveRuntimeVersion('python', '3.10.4')).toBe('3.10.4'); + expect(resolveRuntimeVersion('node', 'latest')).toBe(LATEST_NODE_VERSION); + expect(resolveRuntimeVersion('node', null)).toBe(DEFAULT_NODE_VERSION); + // A runtime that installs no interpreter has no version to pin, so asking for one is refused + // rather than answered with a number that would mean nothing. + expect(resolveRuntimeVersion('native', null)).toBeNull(); + expect(() => resolveRuntimeVersion('native', '1.0')).toThrow(/no version to pin/); }); it('records a Python module and its default arguments', async () => { diff --git a/tests/unit/scroll-editing.test.mjs b/tests/unit/scroll-editing.test.mjs index 727fdb5..8153ad1 100644 --- a/tests/unit/scroll-editing.test.mjs +++ b/tests/unit/scroll-editing.test.mjs @@ -258,7 +258,7 @@ describe('editing an existing scroll', () => { })).rejects.toThrow(/a box must prove it can import something/); await expect(addSelfTestImport({ boxId: 'example-model', target: ALL_TARGETS, module: 'not a module', - })).rejects.toThrow(/Not an importable module name/); + })).rejects.toThrow(/Not an importable python module name/); }); it('sets a field, and refuses one the format does not let a person change', async () => { From e0fb472552243b4f37b7d240845b90294a09970d Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:19:15 +0200 Subject: [PATCH 13/22] Bring the Rust consumer to the native and node runtimes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mirror gains both adapters and the declared bundled-licence inventory. A layout's entry point and standard library become optional, which is what a native box actually has; assert_runtime_entry_point gains its third answer; and the probe kinds a runtime can answer are derived from whether it has an import probe at all, rather than declared a second time beside it. The conformance suite gains a case for an inventory that disagrees between box.json and the release, and the case that used to prove an unimplemented runtime now proves a python box relabelled native after signing — every id the format names is implemented, so that is the refusal that still matters. --- python/tests/conformance_support.py | 7 + rust/fixtures/consumer-conformance.json | 17 +- rust/fixtures/runtime-contract.json | 371 +++++++++++++++ rust/src/contract/runtimes.rs | 440 +++++++++++++++--- .../contract/schema/box-manifest.schema.json | 3 + .../schema/release-manifest.schema.json | 56 +++ rust/src/release.rs | 33 +- rust/src/verify.rs | 4 + rust/tests/conformance.rs | 9 +- rust/tests/contract.rs | 113 +++-- rust/tests/release_document.rs | 2 +- .../fixtures/consumer-conformance.json | 11 + tests/helpers/consumer-conformance.mjs | 8 + 13 files changed, 957 insertions(+), 117 deletions(-) diff --git a/python/tests/conformance_support.py b/python/tests/conformance_support.py index 25c6e8f..bc37e60 100644 --- a/python/tests/conformance_support.py +++ b/python/tests/conformance_support.py @@ -245,6 +245,13 @@ def _mutate_fixture( fixture.release["environment"] = {"SCROLLCASE_CHANGED_AFTER_BUILD": "1"} fixture.sign() return + if mutation == "alter-release-bundled-licenses": + # A licence inventory added to the signed release after the box was built. It is signed, so + # the signature still verifies; what refuses it is that box.json says something else, which + # is the whole reason the inventory is compared field by field rather than merely carried. + fixture.release["bundledLicenses"] = [{"name": "zlib", "version": "1.3.1", "declaredLicense": "Zlib", "linkedInto": ["box.json"]}] + fixture.sign() + return if mutation == "add-unknown-compatibility-constraint": # Not a tamper: a signed constraint in a publishing project's own vocabulary, which the # schema allows and the builder copies through. The consumer must carry it, not refuse the diff --git a/rust/fixtures/consumer-conformance.json b/rust/fixtures/consumer-conformance.json index 8d81d99..87fc105 100644 --- a/rust/fixtures/consumer-conformance.json +++ b/rust/fixtures/consumer-conformance.json @@ -10,6 +10,7 @@ "asset-size": "asset size mismatch", "attach-missing-interpreter": "Attached box is missing venv/", "attach-root": "is not an extracted box directory", + "bundled-licenses-disagreement": "box.json mismatch: bundledLicenses", "encrypted-entry": "Encrypted ZIP entries", "entry-collision": "Archive entry collides with another entry", "environment-disagreement": "box.json mismatch: environment", @@ -28,9 +29,9 @@ "payload-list-missing": "missing its payload digest list", "payload-mismatch": "^Payload does not match the signed release:", "runtime-disagreement": "box.json mismatch: runtime", + "runtime-without-entry-point": "no runtime entry point to declare", "spawn-failure": "failed to start|fixture spawn failed", "special-entry": "special entries", - "unimplemented-runtime": "is not implemented by this version", "unsafe-path": "Unsafe relative path|invalid relative path|absolute path", "unsupported-schema-version": "Unsupported schemaVersion 1|Unsupported schemaVersion 2" }, @@ -583,6 +584,16 @@ "destinationExists": false } }, + { + "id": "altered-bundled-licence-inventory", + "action": "prepare", + "mutation": "alter-release-bundled-licenses", + "expected": { + "outcome": "rejected", + "error": "bundled-licenses-disagreement", + "destinationExists": false + } + }, { "id": "unsupported-schema-version", "action": "prepare", @@ -1398,12 +1409,12 @@ } }, { - "id": "runtime-this-build-cannot-run", + "id": "runtime-relabelled-after-signing", "action": "prepare", "mutation": "alter-release-runtime-id", "expected": { "outcome": "rejected", - "error": "unimplemented-runtime", + "error": "runtime-without-entry-point", "destinationExists": false } }, diff --git a/rust/fixtures/runtime-contract.json b/rust/fixtures/runtime-contract.json index f08f8df..117a645 100644 --- a/rust/fixtures/runtime-contract.json +++ b/rust/fixtures/runtime-contract.json @@ -71,6 +71,147 @@ ] } } + ], + "selfTestProbeKinds": [ + "imports", + "commands" + ] + }, + { + "id": "node", + "executionKinds": [ + "node-script" + ], + "executionEnvironmentVariables": [ + "NODE_OPTIONS", + "NODE_PATH", + "NODE_EXTRA_CA_CERTS" + ], + "selfTestProbeKinds": [ + "imports", + "commands" + ], + "layouts": [ + { + "platform": "macos", + "layout": { + "root": "venv", + "entryPoint": "venv/bin/node", + "scriptsDirectory": "venv/bin", + "standardLibrary": "venv/lib", + "executableSuffix": "", + "launcherKind": "posix-polyglot" + }, + "executablePayloadPaths": { + "files": [ + "venv/bin/node" + ], + "directories": [ + "venv/bin" + ] + } + }, + { + "platform": "linux", + "layout": { + "root": "venv", + "entryPoint": "venv/bin/node", + "scriptsDirectory": "venv/bin", + "standardLibrary": "venv/lib", + "executableSuffix": "", + "launcherKind": "posix-polyglot" + }, + "executablePayloadPaths": { + "files": [ + "venv/bin/node" + ], + "directories": [ + "venv/bin" + ] + } + }, + { + "platform": "windows", + "layout": { + "root": "venv", + "entryPoint": "venv/node.exe", + "scriptsDirectory": "venv/Scripts", + "standardLibrary": "venv/Lib", + "executableSuffix": ".exe", + "launcherKind": "uv-windows-pe" + }, + "executablePayloadPaths": { + "files": [ + "venv/node.exe" + ], + "directories": [ + "venv/Scripts" + ] + } + } + ] + }, + { + "id": "native", + "executionKinds": [ + "native-binary" + ], + "executionEnvironmentVariables": [], + "selfTestProbeKinds": [ + "commands" + ], + "layouts": [ + { + "platform": "macos", + "layout": { + "root": "venv", + "entryPoint": null, + "scriptsDirectory": "venv/bin", + "standardLibrary": null, + "executableSuffix": "", + "launcherKind": "posix-polyglot" + }, + "executablePayloadPaths": { + "files": [], + "directories": [ + "venv/bin" + ] + } + }, + { + "platform": "linux", + "layout": { + "root": "venv", + "entryPoint": null, + "scriptsDirectory": "venv/bin", + "standardLibrary": null, + "executableSuffix": "", + "launcherKind": "posix-polyglot" + }, + "executablePayloadPaths": { + "files": [], + "directories": [ + "venv/bin" + ] + } + }, + { + "platform": "windows", + "layout": { + "root": "venv", + "entryPoint": null, + "scriptsDirectory": "venv/Scripts", + "standardLibrary": null, + "executableSuffix": ".exe", + "launcherKind": "uv-windows-pe" + }, + "executablePayloadPaths": { + "files": [], + "directories": [ + "venv/Scripts" + ] + } + } ] } ], @@ -137,6 +278,55 @@ "platform": "windows", "path": "venv/bin/tqdm", "executable": false + }, + { + "name": "the Node interpreter, by exact name", + "runtime": "node", + "platform": "linux", + "path": "venv/bin/node", + "executable": true + }, + { + "name": "a Node console script the prefix generated", + "runtime": "node", + "platform": "linux", + "path": "venv/bin/npx", + "executable": true + }, + { + "name": "the application's own script, which the runtime never claims", + "runtime": "node", + "platform": "linux", + "path": "app/main.js", + "executable": false + }, + { + "name": "the Windows Node interpreter, which sits outside its scripts directory", + "runtime": "node", + "platform": "windows", + "path": "venv/node.exe", + "executable": true + }, + { + "name": "a prefix tool a native box still carries", + "runtime": "native", + "platform": "linux", + "path": "venv/bin/sqlite3", + "executable": true + }, + { + "name": "the binary a native box runs, which the runtime never claims because the scroll does", + "runtime": "native", + "platform": "linux", + "path": "bin/tool", + "executable": false + }, + { + "name": "a shared library a native binary links against", + "runtime": "native", + "platform": "linux", + "path": "venv/lib/libz.so.1", + "executable": false } ], "executionDiscovery": [ @@ -212,6 +402,34 @@ "venv/Lib/site-packages/pkg.py", "venv/Lib/site-packages/pkg/__main__.py" ] + }, + { + "name": "a Node script resolves to itself and nowhere else", + "runtime": "node", + "platform": "linux", + "runtimeVersion": "22.11.0", + "execution": { + "kind": "node-script", + "script": "app/main.js", + "defaultArgs": [] + }, + "candidates": [ + "app/main.js" + ] + }, + { + "name": "a native binary resolves to itself and nowhere else", + "runtime": "native", + "platform": "macos", + "runtimeVersion": "", + "execution": { + "kind": "native-binary", + "binary": "bin/tool", + "defaultArgs": [] + }, + "candidates": [ + "bin/tool" + ] } ], "invalidRuntimeVersions": [ @@ -301,6 +519,79 @@ "value": "app/main.py" } ] + }, + { + "name": "a Node script runs through the box's own node, with its declared arguments after it", + "runtime": "node", + "platform": "linux", + "execution": { + "kind": "node-script", + "script": "app/main.js", + "defaultArgs": [ + "--serve" + ] + }, + "command": { + "kind": "payload-path", + "value": "venv/bin/node" + }, + "args": [ + { + "kind": "payload-path", + "value": "app/main.js" + }, + { + "kind": "literal", + "value": "--serve" + } + ] + }, + { + "name": "a Windows Node box runs the same declaration through its own node", + "runtime": "node", + "platform": "windows", + "execution": { + "kind": "node-script", + "script": "app/main.js", + "defaultArgs": [] + }, + "command": { + "kind": "payload-path", + "value": "venv/node.exe" + }, + "args": [ + { + "kind": "payload-path", + "value": "app/main.js" + } + ] + }, + { + "name": "a native binary is the command itself, with nothing in front of it", + "runtime": "native", + "platform": "linux", + "execution": { + "kind": "native-binary", + "binary": "bin/tool", + "defaultArgs": [ + "--serve", + "8080" + ] + }, + "command": { + "kind": "payload-path", + "value": "bin/tool" + }, + "args": [ + { + "kind": "literal", + "value": "--serve" + }, + { + "kind": "literal", + "value": "8080" + } + ] } ], "selfTest": [ @@ -589,11 +880,91 @@ "expectExitCode": 1 } ] + }, + { + "name": "a Node import probe requires each module through the box's own node", + "runtime": "node", + "platform": "linux", + "probe": { + "imports": [ + "fs", + "node:path" + ] + }, + "execution": null, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "venv/bin/node" + }, + "args": [ + { + "kind": "literal", + "value": "-e" + }, + { + "kind": "literal", + "value": "if (process.platform !== 'linux') throw new Error('platform mismatch: ' + process.platform)\nrequire(\"fs\");\nrequire(\"node:path\");" + } + ], + "expectExitCode": 0 + } + ] + }, + { + "name": "a native command probe invokes the binary itself", + "runtime": "native", + "platform": "macos", + "probe": { + "commands": [ + { + "args": [ + "--version" + ], + "expectExitCode": 0 + } + ] + }, + "execution": { + "kind": "native-binary", + "binary": "bin/tool", + "defaultArgs": [ + "--quiet" + ] + }, + "invocations": [ + { + "command": { + "kind": "payload-path", + "value": "bin/tool" + }, + "args": [ + { + "kind": "literal", + "value": "--quiet" + }, + { + "kind": "literal", + "value": "--version" + } + ], + "expectExitCode": 0 + } + ] } ], "runtimeIds": [ "python", "node", "native" + ], + "unsupportedProbes": [ + { + "name": "a native box has no module system to ask for an import", + "runtime": "native", + "probeKind": "imports", + "message": "The native runtime cannot answer a selfTest.imports probe; it answers selfTest.commands." + } ] } diff --git a/rust/src/contract/runtimes.rs b/rust/src/contract/runtimes.rs index 2f59d96..5383f1d 100644 --- a/rust/src/contract/runtimes.rs +++ b/rust/src/contract/runtimes.rs @@ -14,16 +14,21 @@ use crate::error::{fail, Result}; /// Where a runtime lives inside an extracted box. +/// +/// Two fields are optional, and both mean the same thing: the runtime does not have that. A native +/// box carries no interpreter to name and no bundled library to search, so both are `None` rather +/// than a plausible-looking path nothing would find. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct RuntimeLayout { /// Directory the packed prefix was relocated into. pub root: &'static str, - /// The runtime's own executable, relative to the box root. - pub entry_point: &'static str, + /// The runtime's own executable relative to the box root, or `None` for a runtime with no + /// separate executable to name. + pub entry_point: Option<&'static str>, /// Directory holding generated console scripts. pub scripts_directory: &'static str, - /// Directory holding the runtime's bundled library. - pub standard_library: &'static str, + /// Directory holding the runtime's bundled library, or `None` for a runtime with none. + pub standard_library: Option<&'static str>, /// Suffix an executable carries on this platform. pub executable_suffix: &'static str, /// Frozen wire string naming how launchers were repaired. @@ -32,12 +37,16 @@ pub struct RuntimeLayout { /// Payload paths a runtime requires the executable bit on, as a rule rather than a list: a conda /// prefix carries hundreds of console scripts and no scroll could name them by hand. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// +/// Owned rather than borrowed from a static table, because a runtime with no interpreter of its own +/// contributes no files at all — the one it runs is named by the scroll, and the scroll is what says +/// the bit belongs on it. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ExecutablePayloadPaths { /// Paths that match exactly. - pub files: &'static [&'static str], + pub files: Vec<&'static str>, /// Directories every path beneath which matches. - pub directories: &'static [&'static str], + pub directories: Vec<&'static str>, } impl ExecutablePayloadPaths { @@ -74,9 +83,6 @@ pub enum RuntimeExecution<'a> { default_args: &'a [String], }, /// A compiled executable the box carries, run with no interpreter in front of it. - /// - /// Named by the format so implementing the runtime is code rather than another wire break. No - /// adapter answers for it yet, so a box declaring it is refused by name. Binary { /// Payload-relative path to the executable. binary: &'a str, @@ -187,9 +193,20 @@ pub struct BoxRuntimeAdapter { pub execution_environment_variables: &'static [&'static str], layouts: &'static [(&'static str, RuntimeLayout)], platform_assertions: &'static [(&'static str, &'static str)], + /// How an import probe's modules become one line of source in the runtime's own language. + import_probe: Option, resolve: fn(&RuntimeExecution<'_>, &str, &RuntimeLayout, &str) -> Result, } +/// How a runtime turns a list of module names into the source its interpreter evaluates. +#[derive(Debug, Clone, Copy)] +struct ImportProbe { + /// The flag that makes the interpreter read source from the next argument. + flag: &'static str, + /// Renders every declared module into one statement per line. + render: fn(&[String]) -> String, +} + const PYTHON_EXECUTION_ENVIRONMENT: &[&str] = &[ "PYTHONPATH", "PYTHONHOME", @@ -197,42 +214,206 @@ const PYTHON_EXECUTION_ENVIRONMENT: &[&str] = &[ "PYTHONBREAKPOINT", ]; +const NODE_EXECUTION_ENVIRONMENT: &[&str] = &["NODE_OPTIONS", "NODE_PATH", "NODE_EXTRA_CA_CERTS"]; + +/// Every runtime can answer a command probe: the box says how it is run, and the probe appends +/// arguments to that. Only a runtime with a module system can answer an import probe, which is why +/// the two lists differ by exactly that one entry. +const IMPORTS_AND_COMMANDS: &[&str] = &["imports", "commands"]; +const COMMANDS_ONLY: &[&str] = &["commands"]; + const POSIX_PYTHON_LAYOUT: RuntimeLayout = RuntimeLayout { root: "venv", - entry_point: "venv/bin/python", + entry_point: Some("venv/bin/python"), scripts_directory: "venv/bin", - standard_library: "venv/lib", + standard_library: Some("venv/lib"), executable_suffix: "", launcher_kind: "posix-polyglot", }; const WINDOWS_PYTHON_LAYOUT: RuntimeLayout = RuntimeLayout { root: "venv", - entry_point: "venv/python.exe", + entry_point: Some("venv/python.exe"), scripts_directory: "venv/Scripts", - standard_library: "venv/Lib", + standard_library: Some("venv/Lib"), executable_suffix: ".exe", // Reads like a stale reference to a tool this project does not use. It is a frozen wire string // under the published format; it is not a typo and must not be "cleaned". launcher_kind: "uv-windows-pe", }; -const RUNTIME_ADAPTERS: &[BoxRuntimeAdapter] = &[BoxRuntimeAdapter { - id: "python", - execution_kinds: &["python-script", "python-module"], - execution_environment_variables: PYTHON_EXECUTION_ENVIRONMENT, - layouts: &[ - ("macos", POSIX_PYTHON_LAYOUT), - ("linux", POSIX_PYTHON_LAYOUT), - ("windows", WINDOWS_PYTHON_LAYOUT), - ], - platform_assertions: &[ - ("macos", "import sys; assert sys.platform == 'darwin'"), - ("linux", "import sys; assert sys.platform.startswith('linux')"), - ("windows", "import sys; assert sys.platform == 'win32'"), - ], - resolve: resolve_python_execution_files, -}]; +const POSIX_NODE_LAYOUT: RuntimeLayout = RuntimeLayout { + root: "venv", + entry_point: Some("venv/bin/node"), + scripts_directory: "venv/bin", + standard_library: Some("venv/lib"), + executable_suffix: "", + launcher_kind: "posix-polyglot", +}; + +const WINDOWS_NODE_LAYOUT: RuntimeLayout = RuntimeLayout { + root: "venv", + // conda-forge installs a Windows package's own executables at the prefix root and its generated + // launchers under `Scripts`, which is why node.exe sits beside python.exe rather than under it. + entry_point: Some("venv/node.exe"), + scripts_directory: "venv/Scripts", + standard_library: Some("venv/Lib"), + executable_suffix: ".exe", + launcher_kind: "uv-windows-pe", +}; + +/// A native box has no interpreter, so its layout names none — and no standard library either, +/// because there is no loader that would search one. The packed prefix is still there: `native` is +/// not "no environment", it is "no interpreter". +const POSIX_NATIVE_LAYOUT: RuntimeLayout = RuntimeLayout { + root: "venv", + entry_point: None, + scripts_directory: "venv/bin", + standard_library: None, + executable_suffix: "", + launcher_kind: "posix-polyglot", +}; + +const WINDOWS_NATIVE_LAYOUT: RuntimeLayout = RuntimeLayout { + root: "venv", + entry_point: None, + scripts_directory: "venv/Scripts", + standard_library: None, + executable_suffix: ".exe", + launcher_kind: "uv-windows-pe", +}; + +fn python_imports(imports: &[String]) -> String { + format!("import {}", imports.join(", ")) +} + +/// `require` rather than a dynamic `import()`, because `-e` source is evaluated as `CommonJS` and +/// Node 22 resolves an ES module through `require` as well. +fn node_imports(imports: &[String]) -> String { + imports + .iter() + .map(|specifier| format!("require({});", json_string(specifier))) + .collect::>() + .join("\n") +} + +/// A JSON string literal, which is also a JavaScript one. Only the escapes JSON defines are +/// produced, so the result is safe to embed in the source a probe evaluates. +fn json_string(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for character in value.chars() { + match character { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + control if (control as u32) < 0x20 => { + use std::fmt::Write as _; + let _ = write!(out, "\\u{:04x}", control as u32); + } + other => out.push(other), + } + } + out.push('"'); + out +} + +const RUNTIME_ADAPTERS: &[BoxRuntimeAdapter] = &[ + BoxRuntimeAdapter { + id: "python", + execution_kinds: &["python-script", "python-module"], + execution_environment_variables: PYTHON_EXECUTION_ENVIRONMENT, + layouts: &[ + ("macos", POSIX_PYTHON_LAYOUT), + ("linux", POSIX_PYTHON_LAYOUT), + ("windows", WINDOWS_PYTHON_LAYOUT), + ], + platform_assertions: &[ + ("macos", "import sys; assert sys.platform == 'darwin'"), + ("linux", "import sys; assert sys.platform.startswith('linux')"), + ("windows", "import sys; assert sys.platform == 'win32'"), + ], + import_probe: Some(ImportProbe { + flag: "-c", + render: python_imports, + }), + resolve: resolve_python_execution_files, + }, + BoxRuntimeAdapter { + id: "node", + // One kind, deliberately. Node has no `-m` analogue worth inventing: a package entry point + // resolves to a file, and naming that file is what every other declaration in the format + // does. + execution_kinds: &["node-script"], + execution_environment_variables: NODE_EXECUTION_ENVIRONMENT, + layouts: &[ + ("macos", POSIX_NODE_LAYOUT), + ("linux", POSIX_NODE_LAYOUT), + ("windows", WINDOWS_NODE_LAYOUT), + ], + platform_assertions: &[ + ( + "macos", + "if (process.platform !== 'darwin') throw new Error('platform mismatch: ' + process.platform)", + ), + ( + "linux", + "if (process.platform !== 'linux') throw new Error('platform mismatch: ' + process.platform)", + ), + ( + "windows", + "if (process.platform !== 'win32') throw new Error('platform mismatch: ' + process.platform)", + ), + ], + import_probe: Some(ImportProbe { + flag: "-e", + render: node_imports, + }), + resolve: resolve_named_payload_file, + }, + BoxRuntimeAdapter { + id: "native", + execution_kinds: &["native-binary"], + // Nothing of its own. A compiled binary is loaded by the operating system's dynamic linker, + // and the variables that steer it are the target's, which the target adapter contributes. + execution_environment_variables: &[], + // The one shape a runtime with no module system can answer. + layouts: &[ + ("macos", POSIX_NATIVE_LAYOUT), + ("linux", POSIX_NATIVE_LAYOUT), + ("windows", WINDOWS_NATIVE_LAYOUT), + ], + platform_assertions: &[], + import_probe: None, + resolve: resolve_named_payload_file, + }, +]; + +/// The discovery rule for every declaration that names a payload file outright: it resolves to +/// itself, or the box does not carry it. Shared by `node` and `native`, whose declarations differ +/// only in what the file is. +fn resolve_named_payload_file( + execution: &RuntimeExecution<'_>, + _platform: &str, + _layout: &RuntimeLayout, + _runtime_version: &str, +) -> Result { + match execution { + RuntimeExecution::Script { script, .. } => Ok(ResolvedExecutionFiles { + candidates: vec![(*script).to_string()], + missing: format!("Execution script is missing from the box: {script}."), + }), + RuntimeExecution::Binary { binary, .. } => Ok(ResolvedExecutionFiles { + candidates: vec![(*binary).to_string()], + missing: format!("Execution binary is missing from the box: {binary}."), + }), + // Unreachable: `resolve_execution_files` refuses a kind that is not this runtime's first, + // and neither runtime defines a module shape. + RuntimeExecution::Module { .. } => fail!("Unsupported execution kind: {}.", execution.kind("")), + } +} /// The `major.minor` prefix naming the standard-library directory a packed prefix carries. /// @@ -278,12 +459,14 @@ fn resolve_python_execution_files( ]; // Windows names its standard library once, with no interpreter version in the path; every // other platform carries `python.` under it. + let Some(bundled_library) = layout.standard_library else { + fail!("The python runtime layout for {platform} names no standard library"); + }; let standard_library = if platform == "windows" { - layout.standard_library.to_string() + bundled_library.to_string() } else { format!( - "{}/python{}", - layout.standard_library, + "{bundled_library}/python{}", python_major_minor(runtime_version)? ) }; @@ -332,10 +515,11 @@ impl BoxRuntimeAdapter { let layout = self.layout(platform)?; // The interpreter by name, and the console-script directory wholesale. A conda prefix // generates that directory's contents at solve time and nothing declares them, so the rule - // is the only way they can carry the bit at all. + // is the only way they can carry the bit at all. A runtime with no interpreter of its own + // contributes only the directory: the file it runs is one the scroll declared. Ok(ExecutablePayloadPaths { - files: std::slice::from_ref(&layout.entry_point), - directories: std::slice::from_ref(&layout.scripts_directory), + files: layout.entry_point.into_iter().collect(), + directories: vec![layout.scripts_directory], }) } @@ -367,18 +551,25 @@ impl BoxRuntimeAdapter { platform: &str, ) -> Result { self.assert_own_kind(execution)?; - let layout = self.layout(platform)?; - let mut args = match execution { - RuntimeExecution::Script { script, .. } => { - vec![RuntimeArgument::PayloadPath((*script).to_string())] - } - RuntimeExecution::Module { module, .. } => vec![ - RuntimeArgument::Literal("-m".to_string()), - RuntimeArgument::Literal((*module).to_string()), - ], - RuntimeExecution::Binary { binary, .. } => { - vec![RuntimeArgument::PayloadPath((*binary).to_string())] - } + // A binary *is* the command. Every other runtime puts its own entry point first and the + // declaration second; here there is nothing to put first, which is the whole of what + // `native` means. + let (command, mut args) = match execution { + RuntimeExecution::Binary { binary, .. } => ( + RuntimeArgument::PayloadPath((*binary).to_string()), + Vec::new(), + ), + RuntimeExecution::Script { script, .. } => ( + RuntimeArgument::PayloadPath(self.entry_point(platform)?.to_string()), + vec![RuntimeArgument::PayloadPath((*script).to_string())], + ), + RuntimeExecution::Module { module, .. } => ( + RuntimeArgument::PayloadPath(self.entry_point(platform)?.to_string()), + vec![ + RuntimeArgument::Literal("-m".to_string()), + RuntimeArgument::Literal((*module).to_string()), + ], + ), }; args.extend( execution @@ -386,10 +577,32 @@ impl BoxRuntimeAdapter { .iter() .map(|value| RuntimeArgument::Literal(value.clone())), ); - Ok(RuntimeInvocation { - command: RuntimeArgument::PayloadPath(layout.entry_point.to_string()), - args, - }) + Ok(RuntimeInvocation { command, args }) + } + + /// The runtime's own executable for a platform, for the rules that cannot proceed without one. + /// + /// # Errors + /// + /// When the platform is unknown, or the runtime has no interpreter to name. + fn entry_point(&self, platform: &str) -> Result<&'static str> { + let Some(entry_point) = self.layout(platform)?.entry_point else { + fail!("The {} runtime has no entry point of its own", self.id); + }; + Ok(entry_point) + } + + /// The self-test probe shapes this runtime can answer. + /// + /// Derived from whether it has an import probe at all rather than declared beside it: two + /// statements of one fact are two things that can disagree, and the fixture asserts this one. + #[must_use] + pub fn self_test_probe_kinds(&self) -> &'static [&'static str] { + if self.import_probe.is_some() { + IMPORTS_AND_COMMANDS + } else { + COMMANDS_ONLY + } } /// Refuses an execution kind belonging to another runtime. @@ -415,6 +628,12 @@ impl BoxRuntimeAdapter { ) -> Result> { let mut invocations = Vec::new(); if !probe.imports.is_empty() { + // An import probe asks a module system a question, and a runtime without one has + // nothing to ask. Refused rather than silently dropped, which would report a pass for + // a check that never ran. + let Some(import_probe) = self.import_probe else { + fail!("{}", unsupported_self_test_probe_message(self.id, "imports")); + }; let Some((_, assertion)) = self .platform_assertions .iter() @@ -425,17 +644,15 @@ impl BoxRuntimeAdapter { self.id ); }; - let imports = format!("import {}", probe.imports.join(", ")); + let imports = (import_probe.render)(probe.imports); let code = match probe.code { Some(extra) => format!("{assertion}\n{imports}\n{extra}"), None => format!("{assertion}\n{imports}"), }; invocations.push(SelfTestInvocation { - command: RuntimeArgument::PayloadPath( - self.layout(platform)?.entry_point.to_string(), - ), + command: RuntimeArgument::PayloadPath(self.entry_point(platform)?.to_string()), args: vec![ - RuntimeArgument::Literal("-c".to_string()), + RuntimeArgument::Literal(import_probe.flag.to_string()), RuntimeArgument::Literal(code), ], expect_exit_code: 0, @@ -490,8 +707,10 @@ pub fn runtime_adapters() -> &'static [BoxRuntimeAdapter] { /// Every runtime id the box format admits, in the order the schema lists them. /// /// The wire enum and the implemented set are deliberately two different things: schema version 3 -/// fixes the vocabulary once, so a later release can implement `node` without another wire break. -/// A box naming a runtime this crate has no adapter for is refused by name, not misread. +/// fixed the vocabulary once, and `node` and `native` then arrived as adapters without another wire +/// break. They hold the same three today; the lists stay separate because this crate versions +/// independently of the builder, so a release published before a runtime landed still has to refuse +/// a box naming it by name rather than misread it. pub const RUNTIME_IDS: &[&str] = &["python", "node", "native"]; /// Whether this build carries an adapter for a runtime id — the question every caller asks before @@ -556,7 +775,17 @@ pub fn assert_runtime_entry_point( entry_point: &str, ) -> Result<()> { let runtime = runtime_adapter(runtime_id)?; - let expected = runtime.layout(adapter.platform)?.entry_point; + // A runtime without an interpreter admits no value at all, and a declaration there is refused + // rather than ignored: it would name a file the box never starts, and a reader would believe + // it. A box that declares nothing is checked against nothing — the caller has already skipped + // this, because `runtime.entryPoint` is optional on the wire for exactly that reason. + let Some(expected) = runtime.layout(adapter.platform)?.entry_point else { + fail!( + "{} boxes have no runtime entry point to declare; the executable a {} box runs is named by its execution", + runtime.id, + runtime.id + ); + }; if entry_point != expected { fail!( "{} boxes with the {} runtime must use entry point {expected}", @@ -567,27 +796,96 @@ pub fn assert_runtime_entry_point( Ok(()) } +/// The message for a self-test probe shape the runtime cannot answer. +/// +/// Stated here, beside the rule, for the same reason [`ResolvedExecutionFiles::missing`] is: the +/// wording is part of the contract, and the builder and all three consumers should refuse an +/// impossible probe identically instead of each inventing a phrasing. +/// +/// # Panics +/// +/// Never for a runtime this crate implements; an unknown id has no probe kinds to name. +#[must_use] +pub fn unsupported_self_test_probe_message(runtime_id: &str, probe_kind: &str) -> String { + let kinds = runtime_adapter(runtime_id).map_or_else( + |_| String::new(), + |runtime| { + runtime + .self_test_probe_kinds() + .iter() + .map(|kind| format!("selfTest.{kind}")) + .collect::>() + .join(" and ") + }, + ); + format!("The {runtime_id} runtime cannot answer a selfTest.{probe_kind} probe; it answers {kinds}.") +} + #[cfg(test)] mod tests { use super::{ - is_implemented_runtime, python_major_minor, runtime_adapter, unimplemented_runtime_message, - RuntimeArgument, RuntimeExecution, SelfTestCommand, SelfTestProbe, RUNTIME_IDS, + assert_runtime_entry_point, is_implemented_runtime, python_major_minor, runtime_adapter, + unimplemented_runtime_message, RuntimeArgument, RuntimeExecution, SelfTestCommand, + SelfTestProbe, RUNTIME_IDS, }; + use crate::contract::targets::{box_target_adapter, BoxTarget}; const PYTHON: &str = "python"; #[test] - fn a_runtime_this_build_has_no_adapter_for_is_refused_by_name() { - // The wire vocabulary is wider than the implemented set on purpose, and the two refusals - // say which of the two the box fell foul of. - assert!(RUNTIME_IDS.contains(&"native")); - assert!(runtime_adapter("native").is_err()); - assert!(!is_implemented_runtime("native")); - assert!(unimplemented_runtime_message("native").contains("not implemented by this version")); + fn a_runtime_the_format_does_not_define_is_refused_by_name() { + // This crate implements every id the format names, so the other branch of the message — a + // runtime the format defines that this crate cannot run — is unreachable here. It is not + // dead: this crate versions independently of the builder, and a release published before a + // runtime landed still has to refuse a box naming it rather than misread it. + for id in RUNTIME_IDS { + assert!(is_implemented_runtime(id), "{id}"); + assert!(runtime_adapter(id).is_ok(), "{id}"); + } assert!(unimplemented_runtime_message("ruby").contains("Unknown runtime")); + assert!(runtime_adapter("ruby").is_err()); assert!(runtime_adapter("").is_err()); - assert!(runtime_adapter(PYTHON).is_ok()); - assert!(is_implemented_runtime(PYTHON)); + assert!(!is_implemented_runtime("")); + } + + #[test] + fn a_native_box_declares_no_runtime_entry_point() { + let linux = box_target_adapter(&BoxTarget { + platform: "linux".to_string(), + arch: "x86_64".to_string(), + accelerator: "cpu".to_string(), + cuda_version: None, + }) + .unwrap(); + assert!(assert_runtime_entry_point(PYTHON, linux, "venv/bin/python").is_ok()); + assert!(assert_runtime_entry_point(PYTHON, linux, "venv/bin/python3").is_err()); + // Naming one would name a file the box never starts, and a reader would believe it. + let refused = assert_runtime_entry_point("native", linux, "venv/bin/python").unwrap_err(); + assert!( + refused.to_string().contains("no runtime entry point to declare"), + "{refused}" + ); + } + + #[test] + fn a_probe_shape_the_runtime_cannot_answer_is_refused() { + let imports = vec!["json".to_string()]; + let refused = runtime_adapter("native") + .unwrap() + .self_test_invocations( + &SelfTestProbe { + imports: &imports, + commands: &[], + code: None, + }, + None, + "linux", + ) + .unwrap_err(); + assert_eq!( + refused.to_string(), + "The native runtime cannot answer a selfTest.imports probe; it answers selfTest.commands." + ); } #[test] diff --git a/rust/src/contract/schema/box-manifest.schema.json b/rust/src/contract/schema/box-manifest.schema.json index 6060ec5..de6342c 100644 --- a/rust/src/contract/schema/box-manifest.schema.json +++ b/rust/src/contract/schema/box-manifest.schema.json @@ -40,6 +40,9 @@ "type": "string", "minLength": 1 }, + "bundledLicenses": { + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/bundledLicenses" + }, "environment": { "type": "object", "description": "Environment variables repeated from the signed release. Scrollcase consumers compare this declaration before execution and apply it over inherited and caller-supplied values.", diff --git a/rust/src/contract/schema/release-manifest.schema.json b/rust/src/contract/schema/release-manifest.schema.json index 90d1b23..24df66c 100644 --- a/rust/src/contract/schema/release-manifest.schema.json +++ b/rust/src/contract/schema/release-manifest.schema.json @@ -135,6 +135,9 @@ "minLength": 1, "description": "Directory relative to the extracted box root holding the box's own large files." }, + "bundledLicenses": { + "$ref": "#/$defs/bundledLicenses" + }, "environment": { "type": "object", "description": "Signed environment variables applied whenever Scrollcase runs the box interpreter. These values override both the inherited host environment and caller-supplied values.", @@ -265,6 +268,59 @@ } } }, + "bundledLicenses": { + "type": "array", + "minItems": 1, + "description": "Dependencies compiled inside the binaries this box ships, as the publishing project declared them. The conda environment's own licences are derived from pixi.lock and travel inside the payload; this list is the half no lock can see — code linked into a supplied executable before the build began — so it is declared, reviewed by the project, and signed here unchanged. It is carried in the release rather than only in the payload so that a licence decision can be made from the document alone, before an archive is downloaded. A box that declares none carries no such list, which means the project declared none and never that the box has no dependencies.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "version", + "declaredLicense", + "linkedInto" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "The dependency as its own project names it." + }, + "version": { + "type": "string", + "minLength": 1 + }, + "declaredLicense": { + "type": "string", + "minLength": 1, + "description": "The licence the project reviewed, conventionally an SPDX expression. Scrollcase carries the string through and never parses it: what a licence permits is not a question a packaging tool answers.", + "examples": [ + "Apache-2.0 OR MIT" + ] + }, + "linkedInto": { + "type": "array", + "minItems": 1, + "description": "Payload files this dependency is compiled into. Every path must be a file the built box actually carries, which is what makes the declaration checkable rather than a free-text notice.", + "items": { + "$ref": "#/$defs/payloadPath" + } + }, + "sourceUrl": { + "type": "string", + "minLength": 1, + "description": "Where the dependency's source can be obtained, for a licence that requires the offer." + } + } + } + }, + "payloadPath": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*//).+$", + "description": "A forward-slash path inside the box payload: relative, non-empty, and unable to escape the payload root." + }, "deferredAssets": { "type": "array", "minItems": 1, diff --git a/rust/src/release.rs b/rust/src/release.rs index 931cb49..2517381 100644 --- a/rust/src/release.rs +++ b/rust/src/release.rs @@ -171,9 +171,6 @@ pub enum Execution { default_args: Vec, }, /// Run one regular payload file with the box's own Node runtime. - /// - /// Named by the format so that implementing the runtime is code rather than another wire - /// break. No adapter in this crate answers for it yet, so a box declaring it is refused. #[serde(rename_all = "camelCase")] NodeScript { /// Safe path to a regular JavaScript file inside the box. @@ -275,6 +272,29 @@ pub struct AssetDescriptor { pub executable: Option, } +/// One dependency compiled inside a binary the box ships, as the publishing project declared it. +/// +/// `pixi.lock` declares a licence per conda package, but it cannot see what was linked into a +/// supplied executable before the build began. That half is declared rather than derived, and +/// carried here so a licence decision can be made from the signed document alone, before an archive +/// is downloaded. This crate transports it and attaches no meaning to any field: what a licence +/// permits is not a question a packaging tool answers. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct BundledLicense { + /// The dependency as its own project names it. + pub name: String, + /// Its version. + pub version: String, + /// The licence the project reviewed, conventionally an SPDX expression. + pub declared_license: String, + /// Payload files this dependency is compiled into. + pub linked_into: Vec, + /// Where the source can be obtained, for a licence that requires the offer. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_url: Option, +} + /// The immutable description of one built box. #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -306,6 +326,10 @@ pub struct ReleaseManifest { pub runtime: BoxRuntime, /// Where the box's own large files belong inside it. pub cache_subdir: String, + /// Dependencies compiled inside the binaries this box ships, as the project declared them. + /// Absent when the project declared none, which never means the box has no dependencies. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bundled_licenses: Option>, /// Signed environment applied whenever Scrollcase runs the box. #[serde(default, skip_serializing_if = "Option::is_none")] pub environment: Option>, @@ -344,6 +368,9 @@ pub struct BoxManifest { pub runtime: BoxRuntime, /// Where the box's own large files belong inside it. pub cache_subdir: String, + /// Dependencies compiled inside the binaries this box ships, repeated from the release. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bundled_licenses: Option>, /// Signed environment applied whenever Scrollcase runs the box. #[serde(default, skip_serializing_if = "Option::is_none")] pub environment: Option>, diff --git a/rust/src/verify.rs b/rust/src/verify.rs index aa0d9fc..ea140dc 100644 --- a/rust/src/verify.rs +++ b/rust/src/verify.rs @@ -126,6 +126,10 @@ pub fn assert_box_manifest_agreement( Some("runtime") } else if box_manifest.cache_subdir != release.cache_subdir { Some("cacheSubdir") + } else if box_manifest.bundled_licenses != release.bundled_licenses { + // Here for the same reason it is signed at all: a licence inventory that could differ + // between the document a reviewer read and the box a user installed would be worth nothing. + Some("bundledLicenses") } else if box_manifest.environment != release.environment { Some("environment") } else if box_manifest.self_test != release.self_test { diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index 2a1ce95..d4bc7a6 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -704,6 +704,13 @@ fn mutate_fixture(fixture: &mut Fixture, mutation: &str, destination: &Path) { fixture.release["environment"] = json!({ "SCROLLCASE_CHANGED_AFTER_BUILD": "1" }); fixture.sign(); } + // A licence inventory added to the signed release after the box was built. It is signed, + // so the signature still verifies; what refuses it is that box.json says something else, + // which is the whole reason the inventory is compared field by field rather than carried. + "alter-release-bundled-licenses" => { + fixture.release["bundledLicenses"] = json!([{"name": "zlib", "version": "1.3.1", "declaredLicense": "Zlib", "linkedInto": ["box.json"]}]); + fixture.sign(); + } // Not a tamper: a signed constraint in a publishing project's own vocabulary, which the // schema allows and the builder copies through. The consumer must carry it, not refuse the // document — refusing it takes the decision away from the application that has to make it. @@ -1262,7 +1269,7 @@ fn the_shared_consumer_conformance_suite_passes() { let suite: Value = serde_json::from_str(SUITE).unwrap(); let patterns = suite["errorPatterns"].as_object().unwrap(); let cases = suite["cases"].as_array().unwrap(); - assert_eq!(cases.len(), 84, "the suite changed size"); + assert_eq!(cases.len(), 85, "the suite changed size"); let mut failures: Vec = Vec::new(); let mut ran = 0usize; diff --git a/rust/tests/contract.rs b/rust/tests/contract.rs index d2a818a..410c05e 100644 --- a/rust/tests/contract.rs +++ b/rust/tests/contract.rs @@ -13,8 +13,8 @@ use scrollcase_consumer::contract::payload_digest::{ payload_digest_stream, PayloadDigestEntry, PayloadDigestKind, }; use scrollcase_consumer::contract::runtimes::{ - is_implemented_runtime, runtime_adapter, runtime_adapters, RuntimeArgument, RuntimeExecution, - SelfTestCommand, SelfTestProbe, RUNTIME_IDS, + is_implemented_runtime, runtime_adapter, runtime_adapters, unsupported_self_test_probe_message, + RuntimeArgument, RuntimeExecution, SelfTestCommand, SelfTestProbe, RUNTIME_IDS, }; use scrollcase_consumer::contract::targets::{box_target_id, BoxTarget}; @@ -85,6 +85,16 @@ struct RuntimeContract { invalid_runtime_versions: Vec, argv: Vec, self_test: Vec, + unsupported_probes: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct UnsupportedProbeCase { + name: String, + runtime: String, + probe_kind: String, + message: String, } #[derive(Deserialize)] @@ -93,6 +103,7 @@ struct RuntimeCase { id: String, execution_kinds: Vec, execution_environment_variables: Vec, + self_test_probe_kinds: Vec, layouts: Vec, } @@ -108,9 +119,9 @@ struct LayoutCase { #[serde(rename_all = "camelCase")] struct LayoutFields { root: String, - entry_point: String, + entry_point: Option, scripts_directory: String, - standard_library: String, + standard_library: Option, executable_suffix: String, launcher_kind: String, } @@ -242,6 +253,59 @@ impl ArgumentFields { /// The Rust half of the shared runtime vectors. /// +/// One runtime's own answers: its kinds, its variables, its probe shapes and its per-platform +/// layout. Extracted so the case that drives it stays readable at a glance. +fn assert_runtime_case(case: &RuntimeCase) { + let runtime = runtime_adapter(&case.id).unwrap(); + assert_eq!(runtime.execution_kinds, case.execution_kinds, "{}", case.id); + assert_eq!( + runtime.execution_environment_variables, case.execution_environment_variables, + "{}", + case.id + ); + assert_eq!( + runtime.self_test_probe_kinds(), case.self_test_probe_kinds, + "{}", + case.id + ); + for platform in &case.layouts { + let layout = runtime.layout(&platform.platform).unwrap(); + let expected = &platform.layout; + assert_eq!(layout.root, expected.root, "{}", platform.platform); + assert_eq!( + layout.entry_point, + expected.entry_point.as_deref(), + "{}", + platform.platform + ); + assert_eq!( + layout.scripts_directory, expected.scripts_directory, + "{}", + platform.platform + ); + assert_eq!( + layout.standard_library, + expected.standard_library.as_deref(), + "{}", + platform.platform + ); + assert_eq!( + layout.executable_suffix, expected.executable_suffix, + "{}", + platform.platform + ); + assert_eq!(layout.launcher_kind, expected.launcher_kind, "{}", platform.platform); + + let rule = runtime.executable_payload_paths(&platform.platform).unwrap(); + assert_eq!(rule.files, platform.executable_payload_paths.files, "{}", platform.platform); + assert_eq!( + rule.directories, platform.executable_payload_paths.directories, + "{}", + platform.platform + ); + } +} + /// Everything the runtime model states about a box — where the interpreter sits, which paths need /// the executable bit, what a declaration could resolve to, and the command line that runs it — is /// asserted here against the same file the Node and Python implementations read. @@ -264,43 +328,16 @@ fn matches_the_shared_runtime_contract() { assert_eq!(mirrored, declared); for case in &contract.runtimes { - let runtime = runtime_adapter(&case.id).unwrap(); - assert_eq!(runtime.execution_kinds, case.execution_kinds, "{}", case.id); + assert_runtime_case(case); + } + + for case in &contract.unsupported_probes { assert_eq!( - runtime.execution_environment_variables, case.execution_environment_variables, + unsupported_self_test_probe_message(&case.runtime, &case.probe_kind), + case.message, "{}", - case.id + case.name ); - for platform in &case.layouts { - let layout = runtime.layout(&platform.platform).unwrap(); - let expected = &platform.layout; - assert_eq!(layout.root, expected.root, "{}", platform.platform); - assert_eq!(layout.entry_point, expected.entry_point, "{}", platform.platform); - assert_eq!( - layout.scripts_directory, expected.scripts_directory, - "{}", - platform.platform - ); - assert_eq!( - layout.standard_library, expected.standard_library, - "{}", - platform.platform - ); - assert_eq!( - layout.executable_suffix, expected.executable_suffix, - "{}", - platform.platform - ); - assert_eq!(layout.launcher_kind, expected.launcher_kind, "{}", platform.platform); - - let rule = runtime.executable_payload_paths(&platform.platform).unwrap(); - assert_eq!(rule.files, platform.executable_payload_paths.files, "{}", platform.platform); - assert_eq!( - rule.directories, platform.executable_payload_paths.directories, - "{}", - platform.platform - ); - } } for case in &contract.executable_matches { diff --git a/rust/tests/release_document.rs b/rust/tests/release_document.rs index 797296e..a9ed6d5 100644 --- a/rust/tests/release_document.rs +++ b/rust/tests/release_document.rs @@ -50,7 +50,7 @@ fn a_genuine_signed_release_is_accepted_and_fully_interpreted() { assert!(inspected.release.kind.ends_with(".release")); // The adapter is resolved from the signed target, and the entry point agreed with it. assert_eq!( - inspected.release.runtime.entry_point.as_deref().unwrap(), + inspected.release.runtime.entry_point.as_deref(), runtime_adapter(&inspected.release.runtime.id) .unwrap() .layout(inspected.adapter.platform) diff --git a/src/contract/fixtures/consumer-conformance.json b/src/contract/fixtures/consumer-conformance.json index 4e2b686..87fc105 100644 --- a/src/contract/fixtures/consumer-conformance.json +++ b/src/contract/fixtures/consumer-conformance.json @@ -10,6 +10,7 @@ "asset-size": "asset size mismatch", "attach-missing-interpreter": "Attached box is missing venv/", "attach-root": "is not an extracted box directory", + "bundled-licenses-disagreement": "box.json mismatch: bundledLicenses", "encrypted-entry": "Encrypted ZIP entries", "entry-collision": "Archive entry collides with another entry", "environment-disagreement": "box.json mismatch: environment", @@ -583,6 +584,16 @@ "destinationExists": false } }, + { + "id": "altered-bundled-licence-inventory", + "action": "prepare", + "mutation": "alter-release-bundled-licenses", + "expected": { + "outcome": "rejected", + "error": "bundled-licenses-disagreement", + "destinationExists": false + } + }, { "id": "unsupported-schema-version", "action": "prepare", diff --git a/tests/helpers/consumer-conformance.mjs b/tests/helpers/consumer-conformance.mjs index 4053a1d..8c853e3 100644 --- a/tests/helpers/consumer-conformance.mjs +++ b/tests/helpers/consumer-conformance.mjs @@ -205,6 +205,14 @@ async function mutateFixture(fixture, mutation, destination) { await writeSignedRelease(fixture, fixture.release); return; } + if (mutation === 'alter-release-bundled-licenses') { + // A licence inventory added to the signed release after the box was built. It is signed, so the + // signature still verifies; what refuses it is that box.json says something else, which is the + // whole reason the inventory is compared field by field rather than merely carried. + fixture.release.bundledLicenses = [{"name": "zlib", "version": "1.3.1", "declaredLicense": "Zlib", "linkedInto": ["box.json"]}]; + await writeSignedRelease(fixture, fixture.release); + return; + } if (mutation === 'add-unknown-compatibility-constraint') { // Not a tamper: a signed constraint in a publishing project's own vocabulary, which the schema // allows and the builder copies through. The consumer must carry it, not refuse the document — From e54da851537a51f79627f3568bc7b21e24279d88 Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:22:42 +0200 Subject: [PATCH 14/22] Bring the Python consumer to the native and node runtimes The same three changes as the Rust mirror: both adapters, an optional layout entry point and standard library, and assert_runtime_entry_point as a shared function so all three consumers refuse a relabelled box in the same words. The probe kinds are derived from whether the runtime has an import probe at all, and bundledLicenses joins the fields box.json must agree with. --- python/src/scrollcase_consumer/_contract.py | 283 +++++++++++++++--- .../schemas/box-manifest.schema.json | 3 + .../schemas/release-manifest.schema.json | 56 ++++ python/src/scrollcase_consumer/verify.py | 12 +- python/tests/test_contract.py | 54 +++- 5 files changed, 357 insertions(+), 51 deletions(-) diff --git a/python/src/scrollcase_consumer/_contract.py b/python/src/scrollcase_consumer/_contract.py index 68cce1a..2c7e100 100644 --- a/python/src/scrollcase_consumer/_contract.py +++ b/python/src/scrollcase_consumer/_contract.py @@ -16,7 +16,7 @@ from functools import lru_cache from importlib.resources import files from pathlib import Path -from typing import Any, Collection, Iterable, Mapping, cast +from typing import Any, Callable, Collection, Iterable, Mapping, Sequence, cast from jsonschema import Draft202012Validator from referencing import Registry, Resource @@ -107,12 +107,17 @@ def _python_major_minor(version: str) -> str: @dataclass(frozen=True, slots=True) class RuntimeLayout: - """Where a runtime lives inside an extracted box.""" + """Where a runtime lives inside an extracted box. + + Two fields are optional, and both mean the same thing: the runtime does not have that. A native + box carries no interpreter to name and no bundled library to search, so both are ``None`` rather + than a plausible-looking path nothing would find. + """ root: str - entry_point: str + entry_point: str | None scripts_directory: str - standard_library: str + standard_library: str | None executable_suffix: str launcher_kind: str @@ -198,6 +203,31 @@ class SelfTestInvocation: expect_exit_code: int +def _python_imports(imports: Sequence[str]) -> str: + return f"import {', '.join(imports)}" + + +def _node_imports(imports: Sequence[str]) -> str: + """``require`` rather than a dynamic ``import()``. + + ``-e`` source is evaluated as CommonJS, and Node 22 resolves an ES module through ``require`` as + well. ``json.dumps`` produces a JSON string literal, which is also a JavaScript one, so a module + name is safe to embed in the source the probe evaluates. + """ + + return "\n".join(f"require({json.dumps(specifier)});" for specifier in imports) + + +@dataclass(frozen=True, slots=True) +class ImportProbe: + """How a runtime turns a list of module names into the source its interpreter evaluates.""" + + #: The flag that makes the interpreter read source from the next argument. + flag: str + #: Renders every declared module into one statement per line. + render: Callable[[Sequence[str]], str] + + @dataclass(frozen=True, slots=True) class RuntimeAdapter: """What a runtime implies for a box, independent of the machine it runs on. @@ -213,6 +243,21 @@ class RuntimeAdapter: execution_environment_variables: tuple[str, ...] _layouts: Mapping[str, RuntimeLayout] _platform_assertions: Mapping[str, str] + #: How an import probe's modules become one line of source, and the flag that evaluates it. + #: ``None`` for a runtime with no module system to ask. + _import_probe: ImportProbe | None + + @property + def self_test_probe_kinds(self) -> tuple[str, ...]: + """The probe shapes this runtime can answer. + + Derived from whether it has an import probe at all rather than declared beside it: two + statements of one fact are two things that can disagree, and the fixture asserts this one. + Every runtime can answer a command probe — the box says how it is run, and the probe appends + arguments to that — so the two lists differ by exactly the one entry. + """ + + return ("imports", "commands") if self._import_probe else ("commands",) def layout(self, platform: str) -> RuntimeLayout: """Where this runtime sits inside a box built for *platform*.""" @@ -228,8 +273,11 @@ def executable_payload_paths(self, platform: str) -> ExecutablePayloadPaths: """Payload paths this runtime requires the executable bit on.""" layout = self.layout(platform) + # A runtime with no interpreter of its own contributes only the directory: the file it runs + # is one the scroll declared, and the scroll is what says the bit belongs on it. + files = () if layout.entry_point is None else (layout.entry_point,) return ExecutablePayloadPaths( - files=(layout.entry_point,), directories=(layout.scripts_directory,) + files=files, directories=(layout.scripts_directory,) ) def resolve_execution_files( @@ -261,12 +309,17 @@ def resolve_execution_files( ) module_path = execution.module.replace(".", "/") relative = (f"{module_path}.py", f"{module_path}/__main__.py") + bundled_library = layout.standard_library + if bundled_library is None: + raise ScrollcaseConsumerError( + f"The {self.id} runtime layout for {platform} names no standard library" + ) # Windows names its standard library once, with no interpreter version in the path; every # other platform carries ``python.`` under it. standard_library = ( - layout.standard_library + bundled_library if platform == "windows" - else f"{layout.standard_library}/python{_python_major_minor(runtime_version)}" + else f"{bundled_library}/python{_python_major_minor(runtime_version)}" ) roots = ("", standard_library, f"{standard_library}/site-packages") return ResolvedExecutionFiles( @@ -287,23 +340,35 @@ def build_argv(self, execution: BoxExecution, platform: str) -> RuntimeInvocatio raise ScrollcaseConsumerError( f"Unsupported execution kind: {execution.kind}." ) - layout = self.layout(platform) - if isinstance(execution, (PythonScriptExecution, NodeScriptExecution)): - args = [RuntimeArgument("payload-path", execution.script)] - elif isinstance(execution, NativeBinaryExecution): - args = [RuntimeArgument("payload-path", execution.binary)] + # A binary *is* the command. Every other runtime puts its own entry point first and the + # declaration second; here there is nothing to put first, which is the whole of what + # ``native`` means. + if isinstance(execution, NativeBinaryExecution): + command = RuntimeArgument("payload-path", execution.binary) + args: list[RuntimeArgument] = [] else: - args = [ - RuntimeArgument("literal", "-m"), - RuntimeArgument("literal", execution.module), - ] + command = RuntimeArgument("payload-path", self._entry_point(platform)) + if isinstance(execution, (PythonScriptExecution, NodeScriptExecution)): + args = [RuntimeArgument("payload-path", execution.script)] + else: + args = [ + RuntimeArgument("literal", "-m"), + RuntimeArgument("literal", execution.module), + ] args.extend( RuntimeArgument("literal", value) for value in execution.default_args ) - return RuntimeInvocation( - command=RuntimeArgument("payload-path", layout.entry_point), - args=tuple(args), - ) + return RuntimeInvocation(command=command, args=tuple(args)) + + def _entry_point(self, platform: str) -> str: + """The runtime's own executable, for the rules that cannot proceed without one.""" + + entry_point = self.layout(platform).entry_point + if entry_point is None: + raise ScrollcaseConsumerError( + f"The {self.id} runtime has no entry point of its own" + ) + return entry_point def self_test_invocations( self, @@ -315,12 +380,20 @@ def self_test_invocations( invocations: list[SelfTestInvocation] = [] if probe.imports: + # An import probe asks a module system a question, and a runtime without one has + # nothing to ask. Refused rather than silently dropped, which would report a pass for a + # check that never ran. + import_probe = self._import_probe + if import_probe is None: + raise ScrollcaseConsumerError( + unsupported_self_test_probe_message(self.id, "imports") + ) assertion = self._platform_assertions.get(platform) if assertion is None: raise ScrollcaseConsumerError( f"No {self.id} self-test assertion exists for platform {platform}" ) - body = f"import {', '.join(probe.imports)}" + body = import_probe.render(probe.imports) source = ( f"{assertion}\n{body}\n{probe.code}" if probe.code @@ -329,10 +402,10 @@ def self_test_invocations( invocations.append( SelfTestInvocation( command=RuntimeArgument( - "payload-path", self.layout(platform).entry_point + "payload-path", self._entry_point(platform) ), args=( - RuntimeArgument("literal", "-c"), + RuntimeArgument("literal", import_probe.flag), RuntimeArgument("literal", source), ), expect_exit_code=0, @@ -372,6 +445,58 @@ def self_test_invocations( launcher_kind="posix-polyglot", ) +_WINDOWS_PYTHON_LAYOUT = RuntimeLayout( + root="venv", + entry_point="venv/python.exe", + scripts_directory="venv/Scripts", + standard_library="venv/Lib", + executable_suffix=".exe", + # Reads like a stale reference to a tool this project does not use. It is a frozen wire string + # under the published format; it must not be "cleaned". + launcher_kind="uv-windows-pe", +) + +_POSIX_NODE_LAYOUT = RuntimeLayout( + root="venv", + entry_point="venv/bin/node", + scripts_directory="venv/bin", + standard_library="venv/lib", + executable_suffix="", + launcher_kind="posix-polyglot", +) + +_WINDOWS_NODE_LAYOUT = RuntimeLayout( + root="venv", + # conda-forge installs a Windows package's own executables at the prefix root and its generated + # launchers under ``Scripts``, which is why node.exe sits beside python.exe rather than under it. + entry_point="venv/node.exe", + scripts_directory="venv/Scripts", + standard_library="venv/Lib", + executable_suffix=".exe", + launcher_kind="uv-windows-pe", +) + +#: A native box has no interpreter, so its layout names none — and no standard library either, +#: because there is no loader that would search one. The packed prefix is still there: ``native`` is +#: not "no environment", it is "no interpreter". +_POSIX_NATIVE_LAYOUT = RuntimeLayout( + root="venv", + entry_point=None, + scripts_directory="venv/bin", + standard_library=None, + executable_suffix="", + launcher_kind="posix-polyglot", +) + +_WINDOWS_NATIVE_LAYOUT = RuntimeLayout( + root="venv", + entry_point=None, + scripts_directory="venv/Scripts", + standard_library=None, + executable_suffix=".exe", + launcher_kind="uv-windows-pe", +) + _RUNTIMES = { "python": RuntimeAdapter( id="python", @@ -385,30 +510,116 @@ def self_test_invocations( _layouts={ "macos": _POSIX_PYTHON_LAYOUT, "linux": _POSIX_PYTHON_LAYOUT, - "windows": RuntimeLayout( - root="venv", - entry_point="venv/python.exe", - scripts_directory="venv/Scripts", - standard_library="venv/Lib", - executable_suffix=".exe", - # Reads like a stale reference to a tool this project does not use. It is a frozen - # wire string under the published format; it must not be "cleaned". - launcher_kind="uv-windows-pe", - ), + "windows": _WINDOWS_PYTHON_LAYOUT, }, _platform_assertions={ "macos": "import sys; assert sys.platform == 'darwin'", "linux": "import sys; assert sys.platform.startswith('linux')", "windows": "import sys; assert sys.platform == 'win32'", }, - ) + _import_probe=ImportProbe(flag="-c", render=_python_imports), + ), + "node": RuntimeAdapter( + id="node", + # One kind, deliberately. Node has no ``-m`` analogue worth inventing: a package entry point + # resolves to a file, and naming that file is what every other declaration in the format + # does. + execution_kinds=("node-script",), + execution_environment_variables=( + "NODE_OPTIONS", + "NODE_PATH", + "NODE_EXTRA_CA_CERTS", + ), + _layouts={ + "macos": _POSIX_NODE_LAYOUT, + "linux": _POSIX_NODE_LAYOUT, + "windows": _WINDOWS_NODE_LAYOUT, + }, + _platform_assertions={ + "macos": ( + "if (process.platform !== 'darwin') " + "throw new Error('platform mismatch: ' + process.platform)" + ), + "linux": ( + "if (process.platform !== 'linux') " + "throw new Error('platform mismatch: ' + process.platform)" + ), + "windows": ( + "if (process.platform !== 'win32') " + "throw new Error('platform mismatch: ' + process.platform)" + ), + }, + _import_probe=ImportProbe(flag="-e", render=_node_imports), + ), + "native": RuntimeAdapter( + id="native", + execution_kinds=("native-binary",), + # Nothing of its own. A compiled binary is loaded by the operating system's dynamic linker, + # and the variables that steer it are the target's, which the target adapter contributes. + execution_environment_variables=(), + _layouts={ + "macos": _POSIX_NATIVE_LAYOUT, + "linux": _POSIX_NATIVE_LAYOUT, + "windows": _WINDOWS_NATIVE_LAYOUT, + }, + _platform_assertions={}, + _import_probe=None, + ), } + +def assert_runtime_entry_point( + runtime_id: str, adapter: TargetAdapter, entry_point: str | None +) -> None: + """Ensure a declared entry point agrees with where the runtime sits in the payload. + + Three answers, because there are three cases. A runtime with an interpreter admits exactly one + value for a given target. A runtime without one — a native box — admits none, and a declaration + there is refused rather than ignored: it would name a file the box never starts, and a reader + would believe it. And a box that declares nothing at all is checked against nothing, because + ``runtime.entryPoint`` is optional on the wire for exactly this reason. + """ + + runtime = runtime_adapter(runtime_id) + expected = runtime.layout(adapter.platform).entry_point + if expected is None: + if entry_point is not None: + raise ScrollcaseConsumerError( + f"{runtime.id} boxes have no runtime entry point to declare; the executable a " + f"{runtime.id} box runs is named by its execution" + ) + return + if entry_point is not None and entry_point != expected: + raise ScrollcaseConsumerError( + f"{adapter.platform}-{adapter.arch} boxes with the {runtime.id} runtime must use " + f"entry point {expected}" + ) + + +def unsupported_self_test_probe_message(runtime_id: str, probe_kind: str) -> str: + """The message for a self-test probe shape the runtime cannot answer. + + Stated here, beside the rule, for the same reason :attr:`ResolvedExecutionFiles.missing` is: the + wording is part of the contract, and the builder and all three consumers should refuse an + impossible probe identically instead of each inventing a phrasing. + """ + + kinds = " and ".join( + f"selfTest.{kind}" for kind in runtime_adapter(runtime_id).self_test_probe_kinds + ) + return ( + f"The {runtime_id} runtime cannot answer a selfTest.{probe_kind} probe; " + f"it answers {kinds}." + ) + + #: Every runtime id the box format admits, in the order the schema lists them. #: #: The wire enum and the implemented set are deliberately two different things: schema version 3 -#: fixes the vocabulary once, so a later release can implement ``node`` without another wire break. -#: A box naming a runtime this package has no adapter for is refused by name, not misread. +#: fixed the vocabulary once, and ``node`` and ``native`` then arrived as adapters without another +#: wire break. They hold the same three today; the lists stay separate because this package versions +#: independently of the builder, so a release published before a runtime landed still has to refuse a +#: box naming it by name rather than misread it. RUNTIME_IDS: tuple[str, ...] = ("python", "node", "native") diff --git a/python/src/scrollcase_consumer/schemas/box-manifest.schema.json b/python/src/scrollcase_consumer/schemas/box-manifest.schema.json index 6060ec5..de6342c 100644 --- a/python/src/scrollcase_consumer/schemas/box-manifest.schema.json +++ b/python/src/scrollcase_consumer/schemas/box-manifest.schema.json @@ -40,6 +40,9 @@ "type": "string", "minLength": 1 }, + "bundledLicenses": { + "$ref": "https://scrollcase.dev/schema/v3/release-manifest.schema.json#/$defs/bundledLicenses" + }, "environment": { "type": "object", "description": "Environment variables repeated from the signed release. Scrollcase consumers compare this declaration before execution and apply it over inherited and caller-supplied values.", diff --git a/python/src/scrollcase_consumer/schemas/release-manifest.schema.json b/python/src/scrollcase_consumer/schemas/release-manifest.schema.json index 90d1b23..24df66c 100644 --- a/python/src/scrollcase_consumer/schemas/release-manifest.schema.json +++ b/python/src/scrollcase_consumer/schemas/release-manifest.schema.json @@ -135,6 +135,9 @@ "minLength": 1, "description": "Directory relative to the extracted box root holding the box's own large files." }, + "bundledLicenses": { + "$ref": "#/$defs/bundledLicenses" + }, "environment": { "type": "object", "description": "Signed environment variables applied whenever Scrollcase runs the box interpreter. These values override both the inherited host environment and caller-supplied values.", @@ -265,6 +268,59 @@ } } }, + "bundledLicenses": { + "type": "array", + "minItems": 1, + "description": "Dependencies compiled inside the binaries this box ships, as the publishing project declared them. The conda environment's own licences are derived from pixi.lock and travel inside the payload; this list is the half no lock can see — code linked into a supplied executable before the build began — so it is declared, reviewed by the project, and signed here unchanged. It is carried in the release rather than only in the payload so that a licence decision can be made from the document alone, before an archive is downloaded. A box that declares none carries no such list, which means the project declared none and never that the box has no dependencies.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "version", + "declaredLicense", + "linkedInto" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "The dependency as its own project names it." + }, + "version": { + "type": "string", + "minLength": 1 + }, + "declaredLicense": { + "type": "string", + "minLength": 1, + "description": "The licence the project reviewed, conventionally an SPDX expression. Scrollcase carries the string through and never parses it: what a licence permits is not a question a packaging tool answers.", + "examples": [ + "Apache-2.0 OR MIT" + ] + }, + "linkedInto": { + "type": "array", + "minItems": 1, + "description": "Payload files this dependency is compiled into. Every path must be a file the built box actually carries, which is what makes the declaration checkable rather than a free-text notice.", + "items": { + "$ref": "#/$defs/payloadPath" + } + }, + "sourceUrl": { + "type": "string", + "minLength": 1, + "description": "Where the dependency's source can be obtained, for a licence that requires the offer." + } + } + } + }, + "payloadPath": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*//).+$", + "description": "A forward-slash path inside the box payload: relative, non-empty, and unable to escape the payload root." + }, "deferredAssets": { "type": "array", "minItems": 1, diff --git a/python/src/scrollcase_consumer/verify.py b/python/src/scrollcase_consumer/verify.py index 836c8a8..27d7585 100644 --- a/python/src/scrollcase_consumer/verify.py +++ b/python/src/scrollcase_consumer/verify.py @@ -35,9 +35,9 @@ parse_payload_digest_stream, path_under, required_assets_from_json, - runtime_adapter, runtime_from_json, unimplemented_runtime_message, + assert_runtime_entry_point, target_adapter, target_from_json, target_id, @@ -63,6 +63,9 @@ "target", "runtime", "cacheSubdir", + # Here for the same reason it is signed at all: a licence inventory that could differ between + # the document a reviewer read and the box a user installed would be worth nothing. + "bundledLicenses", "environment", "selfTest", "execution", @@ -322,12 +325,7 @@ def _inspect_release_document( # name one there is no adapter for. That is refused by name rather than misread as another. if not is_implemented_runtime(runtime.id): raise ScrollcaseConsumerError(unimplemented_runtime_message(runtime.id)) - expected_entry_point = runtime_adapter(runtime.id).layout(adapter.platform).entry_point - if runtime.entry_point is not None and runtime.entry_point != expected_entry_point: - raise ScrollcaseConsumerError( - f"{adapter.platform}-{adapter.arch} boxes with the {runtime.id} runtime must use " - f"entry point {expected_entry_point}" - ) + assert_runtime_entry_point(runtime.id, adapter, runtime.entry_point) return _InspectedRelease( release_path=release_path, signed=signed, diff --git a/python/tests/test_contract.py b/python/tests/test_contract.py index 85f7df9..ee1d017 100644 --- a/python/tests/test_contract.py +++ b/python/tests/test_contract.py @@ -19,6 +19,7 @@ SCHEMA_FILES, PayloadDigestEntry, absolute_path, + assert_runtime_entry_point, execution_affecting_variables, execution_from_json, parse_payload_digest_stream, @@ -30,11 +31,13 @@ runtime_adapter, runtime_adapters, unimplemented_runtime_message, + unsupported_self_test_probe_message, target_adapter, target_from_json, target_id, ) from scrollcase_consumer.errors import ScrollcaseConsumerError +from scrollcase_consumer.models import BoxTarget from scrollcase_consumer.extract import payload_digest @@ -124,15 +127,15 @@ def test_names_every_runtime_the_format_defines(self) -> None: is_implemented_runtime(runtime_id), runtime_id in implemented ) - def test_refuses_a_runtime_it_has_no_adapter_for(self) -> None: - for runtime_id in ("node", "native"): + def test_refuses_a_runtime_the_format_does_not_define(self) -> None: + # This package implements every id the format names, so the other branch of the message — a + # runtime the format defines that this package cannot run — is unreachable here. It is not + # dead: this package versions independently of the builder, and a release published before + # a runtime landed still has to refuse a box naming it rather than misread it. + for runtime_id in RUNTIME_IDS: with self.subTest(runtime_id=runtime_id): - with self.assertRaises(ScrollcaseConsumerError): - runtime_adapter(runtime_id) - self.assertIn( - "not implemented by this version", - unimplemented_runtime_message(runtime_id), - ) + self.assertTrue(is_implemented_runtime(runtime_id)) + self.assertIsNotNone(runtime_adapter(runtime_id)) for runtime_id in ("", "ruby"): with self.subTest(runtime_id=runtime_id): with self.assertRaises(ScrollcaseConsumerError): @@ -141,6 +144,38 @@ def test_refuses_a_runtime_it_has_no_adapter_for(self) -> None: "Unknown runtime", unimplemented_runtime_message(runtime_id) ) + def test_admits_an_entry_point_only_where_the_runtime_has_one(self) -> None: + linux = target_adapter( + BoxTarget(platform="linux", arch="x86_64", accelerator="cpu") + ) + # Optional on the wire, so declaring nothing is fine for either kind of runtime. + assert_runtime_entry_point("python", linux, None) + assert_runtime_entry_point("native", linux, None) + assert_runtime_entry_point("python", linux, "venv/bin/python") + with self.assertRaises(ScrollcaseConsumerError): + assert_runtime_entry_point("python", linux, "venv/bin/python3") + # Naming one would name a file the box never starts, and a reader would believe it. + with self.assertRaises(ScrollcaseConsumerError) as refused: + assert_runtime_entry_point("native", linux, "venv/bin/python") + self.assertIn("no runtime entry point to declare", str(refused.exception)) + + def test_refuses_a_probe_shape_the_runtime_cannot_answer(self) -> None: + for case in self._load()["unsupportedProbes"]: + with self.subTest(case=case["name"]): + self.assertEqual( + unsupported_self_test_probe_message( + case["runtime"], case["probeKind"] + ), + case["message"], + ) + with self.assertRaises(ScrollcaseConsumerError) as refused: + runtime_adapter(case["runtime"]).self_test_invocations( + SelfTestProbe(imports=("anything",), commands=()), + None, + "linux", + ) + self.assertEqual(str(refused.exception), case["message"]) + def test_reproduces_every_golden_layout_and_executable_rule(self) -> None: for case in self._load()["runtimes"]: runtime = runtime_adapter(case["id"]) @@ -151,6 +186,9 @@ def test_reproduces_every_golden_layout_and_executable_rule(self) -> None: list(runtime.execution_environment_variables), case["executionEnvironmentVariables"], ) + self.assertEqual( + list(runtime.self_test_probe_kinds), case["selfTestProbeKinds"] + ) for platform in case["layouts"]: with self.subTest(runtime=case["id"], platform=platform["platform"]): layout = runtime.layout(platform["platform"]) From e8327510d6002f0ef8f0f98b81dcb43f4a98b37b Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Fri, 28 Aug 2026 19:01:20 +0200 Subject: [PATCH 15/22] Add worked node and native example boxes, and document both runtimes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both were built, verified with --self-test and run for real on this machine, and each turned up something the unit suite could not have. The native example started as sqlite3, whose own linkage is entirely @rpath. It failed anyway: conda-forge's ncurses, three dependencies down, ships a libncurses that re-exports libtinfo through an unrewritten build-machine path. The box was correct and a package inside it was not — which is exactly the limitation the native runtime states rather than assumes, and the self-test caught it before anything was signed. The example runs zstd instead. The node example failed its self-test against this repository's own package.json: Node reads the nearest one *above* the file it is running, so a box without one asks whichever directory it was extracted into. The node runtime now writes the box its own, unless the payload already carries one. --- AGENTS.md | 7 +- CHANGELOG.md | 54 +++++++- docs/concepts/design-decisions.md | 94 ++++++++++++- docs/reference/api.md | 3 +- docs/reference/box-format.md | 36 ++++- docs/reference/scroll.md | 124 +++++++++++++++++- docs/white-paper.md | 31 +++++ examples/README.md | 46 +++++++ .../macos-aarch64-metal/conda-licenses.json | 20 +++ .../native-box/macos-aarch64-metal/pixi.lock | 43 ++++++ .../native-box/macos-aarch64-metal/pixi.toml | 13 ++ .../macos-aarch64-metal/scroll.json | 45 +++++++ .../macos-aarch64-metal/conda-licenses.json | 50 +++++++ .../macos-aarch64-metal/entrypoint.js | 14 ++ .../node-box/macos-aarch64-metal/pixi.lock | 108 +++++++++++++++ .../node-box/macos-aarch64-metal/pixi.toml | 8 ++ .../node-box/macos-aarch64-metal/scroll.json | 49 +++++++ src/build/box.mjs | 7 + src/runtimes/index.d.mts | 6 + src/runtimes/index.mjs | 3 + src/runtimes/node/index.mjs | 2 + src/runtimes/node/payload.d.mts | 7 + src/runtimes/node/payload.mjs | 40 ++++++ tests/unit/build-pipeline.test.mjs | 69 +++++++++- 24 files changed, 851 insertions(+), 28 deletions(-) create mode 100644 examples/native-box/macos-aarch64-metal/conda-licenses.json create mode 100644 examples/native-box/macos-aarch64-metal/pixi.lock create mode 100644 examples/native-box/macos-aarch64-metal/pixi.toml create mode 100644 examples/native-box/macos-aarch64-metal/scroll.json create mode 100644 examples/node-box/macos-aarch64-metal/conda-licenses.json create mode 100644 examples/node-box/macos-aarch64-metal/entrypoint.js create mode 100644 examples/node-box/macos-aarch64-metal/pixi.lock create mode 100644 examples/node-box/macos-aarch64-metal/pixi.toml create mode 100644 examples/node-box/macos-aarch64-metal/scroll.json create mode 100644 src/runtimes/node/payload.d.mts create mode 100644 src/runtimes/node/payload.mjs diff --git a/AGENTS.md b/AGENTS.md index 623e221..4497c93 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -133,8 +133,8 @@ afterwards.** - **box** — the built artefact. Never "image", never "container", never a consumer's product term. - **scroll** — the declarative input (`scroll.json`), the only input a build accepts. - **target** — the `(platform, arch, accelerator)` triple, plus `cudaVersion` for CUDA. -- **runtime** — what runs *inside* the box: `python`, `node` or `native`. The format names all - three; only `python` is implemented, and a box naming another is refused by name. +- **runtime** — what runs *inside* the box: `python`, `node` or `native`. All three are implemented; + a box naming an id the format does not define is refused by name. - **payload** — the tree assembled before archiving. - **release / channel / revocations** — the three signed document types. - **self-test** — the import check run with the box's *own* interpreter. @@ -180,7 +180,8 @@ without reading each hit. layout, execution kinds, argv, self-test (`runtimes.mjs`), signed-document envelope and namespacing (`documents.mjs`), `schema/`, `fixtures/`, generated `types/`. - `src/runtimes//` — the builder-side half of a runtime: launcher repair, dependency reading, - authoring templates, the pixi dependency it contributes. Only `python/` exists. + authoring templates, the pixi dependency it contributes, and any payload file the runtime needs + that nothing declares. `python/`, `node/` and `native/`, plus the shared `launchers.mjs`. - `src/build/` — solving and packing (`pixi.mjs`), toolchain bootstrap (`toolchain.mjs`), archive and filesystem primitives, the lock-derived licence audit, workspace resolution, scroll authoring (`authoring.mjs`), reading and provenance (`scroll.mjs`), diff --git a/CHANGELOG.md b/CHANGELOG.md index a88b509..db3ae3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,42 @@ All notable changes to Scrollcase are documented here. The format follows ## [Unreleased] +### Added — the `node` and `native` runtimes + +- **A box can run Node, or run nothing at all.** `runtime.id: "node"` packs `nodejs` from + conda-forge and starts `venv/bin/node` (`venv/node.exe` on Windows) on a declared `node-script`. + `runtime.id: "native"` packs no interpreter and starts a compiled binary directly: the binary + *is* the command line, `runtime.version` and `runtime.entryPoint` are absent, and declaring + either is refused rather than ignored, because it would name a file the box never starts. + +- **A native box proves itself with `selfTest.commands`.** It has no module system, so + `selfTest.imports` means nothing to it and is refused where the scroll is read rather than + silently dropped, which would report a pass for a check that never ran. `parity` is refused for + the same reason: it runs a source file with the box's own interpreter, and there is not one. + +- **`scrollcase new scroll --runtime `** drives which execution kinds are + offered, which starter files are written, and which dependency the generated `pixi.toml` + declares — none, for `native`, where only the author knows what their binary needs. + **`--python-version` becomes `--runtime-version`**, and is refused for `native`. + +- **A Node box carries its own `package.json`** unless the payload already has one. Node decides + whether a `.js` file is CommonJS or an ES module from the nearest `package.json` *above* it, so a + box without one asks whichever directory it was extracted into — the same box then behaves + differently in two places. Found by building one, against this repository's own `package.json`. + +- **Whatever a box starts must come out of the archive executable.** Checked before the archive is + written, through the runtime's own argv rule rather than by naming an execution kind, so it holds + for every runtime. A `native-binary` a scroll brought in therefore needs `"executable": true`. + +- **Link repair is not attempted for `native`.** A binary that finds its libraries through an + absolute path recorded at compile time will not find them inside a box, and fixing that is + per-format work — rpath, `install_name`, the DLL search order — worth its own pass. A native box + must ship a binary that already resolves. The self-test catches the rest at build time: the first + native example built here failed on conda-forge's own `ncurses`, which carries an unrewritten + build-machine path to `libtinfo`. + +- **`examples/node-box` and `examples/native-box`**, both built, verified and run for real. + ### Changed — the version 3 box format This is a **breaking wire change**, and the only one planned. Published v1 and v2 boxes stay @@ -17,10 +53,10 @@ dual-read path anywhere, and no migration tool: a box is rebuilt from its scroll and `pythonEntryPoint` in the scroll, `box.json` and the signed release, and `provenance.pythonVersion` becomes `provenance.runtimeVersion`. A version 2 box recorded a Python interpreter path and Python execution kinds and nothing that said "Python", so a reader had to - infer the runtime from the shape of a path. `id` is one of `python`, `node` or `native`; only - `python` can be built or run today, and a box naming another is refused by name rather than - misread as the runtime it happens to be shaped like. Fixing the vocabulary now is what makes - implementing the other two code rather than a second wire break. + infer the runtime from the shape of a path. `id` is one of `python`, `node` or `native` — all three + are implemented, and a box naming an id the format does not define is refused by name rather than + misread as the runtime it happens to be shaped like. Fixing the vocabulary once is what let the + other two arrive as code rather than as a second wire break. - **`modelId` and `runtimeId` are gone**, replaced by an optional `labels` map that Scrollcase never reads. Both were required and neither was ever read by any code path: they were a consumer's @@ -37,6 +73,16 @@ dual-read path anywhere, and no migration tool: a box is rebuilt from its scroll `embed` field — an archive is expanded at build time, so deferring one names nothing that could happen — which turns version 2's cross-field refusal into a schema-level impossibility. +- **A scroll may declare the licences bundled inside a binary it ships.** + `bundledLicenseDeclaration` points at a reviewed JSON array of + `{ name, version, declaredLicense, linkedInto }` entries, and the build checks that every path it + names is a file the box actually carries before signing the list into the release and `box.json` + and writing it beside the derived audit at `THIRD_PARTY_NOTICES/bundled-dependencies.json`. + `pixi.lock` declares a licence per conda package, but it cannot see what was linked into a binary + before Scrollcase ever saw the file, and reading the binary would be guessing. It travels in the + release rather than only in the payload because a licence decision is made before an archive is + downloaded. Its absence means the project declared none, never that the box has none. + - **The self-test generalises.** `selfTest.pythonImports` put Python syntax in the wire format and gave a runtime with no module system no way to state a check at all. The signed subset becomes `selfTest.probe`, carrying `imports`, `commands`, or both; a command invokes the box's own diff --git a/docs/concepts/design-decisions.md b/docs/concepts/design-decisions.md index a70d899..b90276e 100644 --- a/docs/concepts/design-decisions.md +++ b/docs/concepts/design-decisions.md @@ -8,19 +8,80 @@ description: Why Scrollcase is shaped the way it is, and which alternatives were Each entry records the alternative that was rejected, because a decision without its discarded alternative is just an assertion. -## Version 2 is a clean break +## Version 3 is a clean break -Scrollcase v2 accepts and emits only `schemaVersion: 2`. Published v1 boxes and immutable package -releases remain usable with the old Scrollcase versions that produced them; the v2 verifier rejects -them with a clear unsupported-version error. It does not reinterpret them. +Scrollcase v3 accepts and emits only `schemaVersion: 3`. Published v1 and v2 boxes remain usable +with the old Scrollcase versions that produced them; the v3 verifier rejects either **by name**, +saying which version it holds, rather than reinterpreting it. The declarative source is a **scroll**, stored as `scroll.json` under `scrolls/`. The built artefact remains a **box**. This vocabulary applies across schemas, identifiers, paths, CLI arguments, fixtures, types, documentation, and errors. -**Rejected:** a v1/v2 union, compatibility aliases, and dual execution paths. They would make every +**Rejected:** a version union, compatibility aliases, and dual execution paths. They would make every security check and every consumer carry two meanings indefinitely, while still being unable to -change the already-published v1 wire format. +change the already-published wire formats. There is no migration tool either: a box is rebuilt from +its scroll, which is the input that was kept for exactly that reason. + +## A runtime is an adapter, not a fork + +Until version 3 the format never said what ran inside a box. It recorded a Python interpreter path +and Python execution kinds, and a reader inferred "Python" from the shape of a path. Every target +adapter was therefore also a statement that a box is a Python box, and adding a second runtime meant +either a second builder or a branch at every layer. + +So version 3 asks the question outright — `runtime: { id, version, entryPoint }` — and fixes the +vocabulary once: `python`, `node`, `native`. What a runtime *implies* moved into one contract module +mirrored in all three consumer languages: where the interpreter sits, which execution kinds exist, +how a declaration becomes a shell-free command line, which inherited variables can change what gets +loaded, and which self-test shapes it can answer. `node` and `native` then arrived as two entries in +that table plus their builder-side halves, and the wire did not move. + +The vocabulary and the implemented set stay two separate lists even though they now hold the same +three ids. They answer to different release cycles: the Python and Rust consumers version +independently of the builder, so one published before a runtime landed must still refuse a box +naming it by name rather than misread it as the runtime it happens to be shaped like. + +**Rejected:** a runtime flag that only selected a template, leaving the layout and argv rules +branching on `platform` further down. That is the arrangement version 2 already had, and it is what +made the Python assumptions impossible to find. + +### A native box has no interpreter, and says so by omission + +`runtime.entryPoint` and `runtime.version` are optional on the wire, and absent for `native`. The +alternative — a placeholder path, or the binary's own path repeated in a field meaning "the +runtime's own executable" — would put a value in the record that no reader could act on, and +provenance must never invent one. A native box that declares either is refused rather than ignored: +it names a file the box never starts, and a reader would believe it. + +The same omission propagates upward. A native box has no module system, so `selfTest.imports` is +refused where the scroll is read rather than silently dropped — dropping it would report a pass for +a check that never ran. And `parity` runs a source file with the box's own interpreter, so it is +refused too. + +**Not attempted, and stated rather than assumed:** repairing a binary's library paths. A binary that +resolves its libraries through an absolute path recorded at compile time will not find them inside a +box, and fixing that is per-binary-format work — rpath on Linux, `install_name` on macOS, the DLL +search order on Windows — worth its own pass. A native box must ship a binary that already resolves. +The self-test is what catches the rest, at build time on the author's machine rather than at run +time on a user's: the first native example built for this repository failed on conda-forge's own +`ncurses`, which ships a `libncurses` re-exporting `libtinfo` through an unrewritten build-machine +path. The box was correct; a package inside it was not. + +### A Node box carries its own `package.json` + +Node decides whether a `.js` file is CommonJS or an ES module by walking *up* from the file to the +nearest `package.json`. Inside a box there usually is none, so the walk leaves the box and asks +whichever directory the box was extracted into: the same box behaves one way under a project whose +manifest says `"type": "module"` and another way one directory higher. That is a box whose behaviour +depends on where it was put, which is the one thing a box exists not to be. + +So the builder writes one at the payload root, with fixed contents so a rebuild is still +byte-identical, and only when the payload does not already carry one — a project that ships a +`package.json` has said what it wants, and overwriting it would replace an answer with a default. + +**Rejected:** requiring every Node box to name a `.mjs` or `.cjs` entry point. It would work, and it +would push the cost onto every author to fix a problem none of them created. ## Consumers prepare and run local boxes; they do not distribute them @@ -529,7 +590,7 @@ The public-contract audit resolved six implementation choices: - Asset resume is limited to retries within one download operation. There is no persistent cache and the documentation makes that process boundary explicit. -## The licence audit is derived from the lock +## The licence audit is derived from the lock — except the half no lock can see The inventory is a pure function of the committed `pixi.lock`, which carries an SPDX licence per package, and `pixi install --frozen` guarantees the installed set equals it. So `audit` runs without @@ -537,6 +598,25 @@ building anything, and licence review can happen when dependencies change rather multi-gigabyte build. A package with no declared licence fails the parse outright: an unlicensed dependency is a legal problem, not a reporting gap. +A binary a scroll supplies is the case this cannot reach. Whatever was linked into it was linked +before Scrollcase saw the file; nothing in the build records it, and reading the binary would be +guessing — and guessing about a licence is worse than not answering. So that half is **declared**: +`bundledLicenseDeclaration` names a reviewed file, and the build checks the one thing a tool can +actually check — that every payload path the declaration claims to be linked into is a file the box +really carries. A licence file nobody can check is a licence file nobody maintains. + +What belongs in the list is the project's judgement, exactly as the reviewed conda audit is. +Scrollcase transports and signs it, and never parses `declaredLicense`: what a licence permits is +not a question a packaging tool answers. Its absence means the project declared none, never that the +box has no bundled dependencies. + +It travels in the **release manifest**, not only in the payload. A licence decision is made before +an archive is downloaded, and a list only a downloaded archive reveals arrives too late to act on. + +**Rejected:** inferring the inventory from the binary, and a free-text notices file. The first is a +guess presented as a fact; the second names nothing that could ever be checked against the box, so +it would go stale the first time a file was renamed and no one would find out. + ## Deliberately out of scope Publishing to object storage, downloading boxes, selecting or promoting a channel, updating an diff --git a/docs/reference/api.md b/docs/reference/api.md index be6f2ba..370b5c6 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -398,11 +398,12 @@ The single source of truth for what a box is. See [The Box Format](/reference/bo | `pixiAccelerator` | `(scroll) => { accelerator, cudaVersion }` | The conda accelerator descriptor a scroll selects, rejecting target drift | | `assertNativeHost` | `(adapter, host = process) => void` | Throws unless the current host matches the adapter's OS and architecture | | `assertRuntimeEntryPoint` | `(runtimeId, adapter, entryPoint) => void` | Throws unless the entry point matches that runtime's layout for the target | -| `RUNTIME_IDS` | `readonly string[]` | Every runtime id the format defines: `python`, `node`, `native`. Wider than what this build implements, on purpose | +| `RUNTIME_IDS` | `readonly string[]` | Every runtime id the format defines: `python`, `node`, `native`. A separate list from what a given build implements, on purpose — the consumers version independently | | `runtimeAdapter` | `(runtimeId) => BoxRuntimeAdapter` | The runtime's layout, execution kinds, argv rule and self-test rule. Throws for a runtime with no adapter | | `runtimeAdapters` | `() => BoxRuntimeAdapter[]` | Every runtime this build implements | | `isImplementedRuntime` | `(runtimeId) => boolean` | Whether an adapter exists — the question to ask before `runtimeAdapter` | | `unimplementedRuntimeMessage` | `(runtimeId) => string` | One wording for a box naming a runtime this build cannot run, so the builder and all three consumers report it identically | +| `unsupportedSelfTestProbeMessage` | `(runtimeId, probeKind) => string` | One wording for a probe shape the runtime cannot answer — `selfTest.imports` in a `native` box, which has no module system | | `executionAffectingVariables` | `(runtimeId, adapter) => readonly string[]` | Inherited variables that can change what a box executes: the runtime's loader controls, then the OS's | | `isExecutablePayloadPath` | `(rule, relativePath) => boolean` | Whether a payload path is one the runtime requires the executable bit on | diff --git a/docs/reference/box-format.md b/docs/reference/box-format.md index e1197e5..642a79a 100644 --- a/docs/reference/box-format.md +++ b/docs/reference/box-format.md @@ -131,9 +131,10 @@ written this way exercises the same layout the shipped box has. ``` `verify` recursively checks every shared field against the signed release: schema, identity and -labels, complete target, the runtime block, cache subdirectory, declared environment, consumer -self-test, the deferred-asset list, and provenance. That agreement binds the archive's contents to -its signed metadata. +labels, complete target, the runtime block, cache subdirectory, the bundled licence inventory, +declared environment, consumer self-test, the deferred-asset list, and provenance. That agreement +binds the archive's contents to its signed metadata — a licence inventory that could differ between +the document a reviewer read and the box a user installed would be worth nothing. ## Provenance @@ -226,8 +227,30 @@ lives and what it hashes to, the consumer import check to repeat, and provenance `runtime.id` is `python`, `node` or `native`. A consumer that does not recognise it must **refuse the box**: the id decides the payload layout and the argv rule, so guessing would mean executing -something on an assumption. Only `python` can be built today; the other two are named by the format -so that implementing them is code rather than another format change. +something on an assumption. All three are implemented — the vocabulary was fixed once, in the +version 3 break, and `node` and `native` then arrived without another one. The list of ids and the +list a given consumer implements stay separate for a reason: the Python and Rust consumers version +independently, so one published before a runtime landed still refuses a box naming it, by name. + +| | `python` | `node` | `native` | +| --- | --- | --- | --- | +| `runtime.entryPoint` | `venv/bin/python`, `venv/python.exe` | `venv/bin/node`, `venv/node.exe` | **absent** | +| `runtime.version` | required | required | **absent** | +| `execution.kind` | `python-script`, `python-module` | `node-script` | `native-binary` | +| Command line | the entry point, then the declaration | the entry point, then the declaration | the binary itself | +| `selfTest.probe` | `imports`, `commands` | `imports`, `commands` | `commands` only | + +A `native` box carries no interpreter, so it names no entry point and no version. A box that +declares one anyway is refused rather than ignored: it would name a file the box never starts, and a +reader would believe it. + +`bundledLicenses` is optional and lists dependencies compiled *inside* a binary the box ships — the +half of the licence picture `pixi.lock` cannot see, declared by the publishing project and signed +here unchanged. Each entry carries `name`, `version`, `declaredLicense` and `linkedInto` (payload +files it is compiled into), plus an optional `sourceUrl`. Scrollcase never parses `declaredLicense`: +what a licence permits is not a question a packaging tool answers. It is carried in the release +rather than only in the payload so a licence decision can be made before an archive is downloaded. +Its absence means the project declared none, never that the box has no bundled dependencies. `environment` is optional. When present it is a signed string map repeated value-for-value in `box.json`. A conforming verifier checks the @@ -375,4 +398,5 @@ payload encoding, signature algorithm, or golden fixture. | `weights: embed \| on-demand` | `assets[].embed`, per entry | A box-wide switch could not ship a small entry point and defer a large dataset. `--weights` went with it: a build-time override of a per-asset declaration repacks a box under an identity that no longer describes it | | `selfTest.pythonImports` | `selfTest.probe` with `imports` and `commands` | Python syntax in the wire format. A runtime with no module system could not state a check at all | | Executable bit from a `venv/bin` heuristic | `assets[].executable`, `localFiles[].executable` | A downloaded file arrives with no permissions, so a box could not ship one that runs | -| `python-script`, `python-module` | plus `node-script`, `native-binary` | Named now, so implementing them later is code rather than another wire break | +| `python-script`, `python-module` | plus `node-script`, `native-binary` | Named once, so implementing the runtimes was code rather than another wire break — which is exactly how `node` and `native` arrived | +| — | `bundledLicenses`, optional | The licences of what was linked *inside* a binary the box ships. `pixi.lock` cannot see them, so they are declared rather than derived, and signed so a licence decision can be made before an archive is downloaded | diff --git a/docs/reference/scroll.md b/docs/reference/scroll.md index 6f1b003..69bcf93 100644 --- a/docs/reference/scroll.md +++ b/docs/reference/scroll.md @@ -209,13 +209,14 @@ build reads, and provenance records. Nothing downstream can tell which half a va | Field | Required | Meaning | | --- | --- | --- | -| `runtime.id` | yes | `python`, `node` or `native`. Only `python` can be built today; the other two are named by the format so implementing them is not another format change | -| `runtime.version` | for `python` | The runtime version the box carries, recorded into provenance | +| `runtime.id` | yes | `python`, `node` or `native`. All three are implemented; see [Choosing a runtime](#choosing-a-runtime) | +| `runtime.version` | for `python` and `node` | The runtime version the box carries, recorded into provenance. A `native` box has no interpreter, so it declares none | | `pixiVersion` | yes | The exact pixi release used to solve and install. `lock` and `build` refuse any other version | -| `runtime.entryPoint` | no | The runtime's own executable relative to the box root. Fixed per (runtime, target): `venv/bin/python` on macOS and Linux, `venv/python.exe` on Windows. Derived when omitted, and a mismatch is still rejected when declared | +| `runtime.entryPoint` | no | The runtime's own executable relative to the box root. Fixed per (runtime, target): `venv/bin/python` or `venv/bin/node` on macOS and Linux, `venv/python.exe` or `venv/node.exe` on Windows. Derived when omitted, and a mismatch is still rejected when declared. A `native` box has none, and declaring one there is refused | | `cacheSubdir` | no | Directory relative to the box root holding model assets. Defaults to `cache/` | | `environment` | no | String environment variables required whenever Scrollcase runs the box interpreter | | `condaDependencyLicenseAudit` | no | Path (from the project root) to the reviewed licence inventory, written and declared by [`audit --write`](/reference/cli#audit). When declared, the build fails if the lock no longer matches what was reviewed | +| `bundledLicenseDeclaration` | no | Path (from the project root) to the licences of dependencies compiled *inside* a binary this box ships. See [Bundled licences](#bundled-licences) | The dependencies themselves live in `pixi.toml`, not here: @@ -235,6 +236,88 @@ or the solve produces an environment that cannot run on the machine the box is f [`scrollcase add dep `](/reference/cli#add) writes into every target's manifest at once, so they cannot drift apart, and `--from-requirements` imports an existing pip file. +### Bundled licences + +`condaDependencyLicenseAudit` is **derived**: `pixi.lock` already records an SPDX licence per conda +package, so Scrollcase computes the inventory and checks it against what you reviewed. + +It cannot do that for a binary you supply. Whatever was linked into that binary was linked before +Scrollcase saw the file, nothing in the build records it, and reading the binary would be guessing — +which is worse than not answering. So that half is **declared**: + +```jsonc +"bundledLicenseDeclaration": "legal/bundled-dependencies.json" +``` + +pointing at a JSON array your project reviews and keeps up to date: + +```jsonc +[ + { + "name": "zlib", + "version": "1.3.1", + "declaredLicense": "Zlib", + "linkedInto": ["bin/my-tool"], + "sourceUrl": "https://zlib.net/" + } +] +``` + +`name`, `version`, `declaredLicense` and `linkedInto` are required; `sourceUrl` is optional, for a +licence that requires an offer of source. Scrollcase carries `declaredLicense` through as a string +and never parses it: what a licence permits is not a question a packaging tool answers. + +What it *does* check is that every path in `linkedInto` is a file the built box actually carries — +deferred assets included, since leaving a large fetched binary out of the inventory would exempt +exactly the case this exists for. A licence file nobody can check is a licence file nobody +maintains: a path that stopped being in the box means the entry is stale, and the build says so +instead of signing a claim about a file that is not there. + +The list is signed into the release manifest and `box.json`, and written into the payload beside the +derived audit at `THIRD_PARTY_NOTICES/bundled-dependencies.json`. It is in the release rather than +only in the payload because a licence decision is made **before** an archive is downloaded, and a +list only a downloaded archive reveals arrives too late to act on. + +A box that declares none carries no such list. That means the project declared none — never that the +box has no bundled dependencies, which is not something Scrollcase is in a position to say. + +### Choosing a runtime + +`runtime.id` says what executes inside the box. It decides the payload layout, the execution kinds +the scroll may declare, how the box is started, and what its self-test is allowed to ask. + +| | `python` | `node` | `native` | +| --- | --- | --- | --- | +| `pixi.toml` dependency | `python` | `nodejs` | none — you declare what your binary needs | +| `runtime.version` | required | required | not applicable | +| `execution.kind` | `python-script`, `python-module` | `node-script` | `native-binary` | +| Started by | the box's own `python` | the box's own `node` | the binary itself | +| `selfTest.imports` | yes | yes | **no** — there is no module system to ask | +| `selfTest.commands` | yes | yes | yes | +| `parity` | yes | yes | **no** — there is no interpreter to run a check with | +| Library-only | yes | yes | **no** | +| `scrollcase new scroll` starter | `entrypoint.py` | `entrypoint.js` | none — point at the binary you built | + +Two things are worth knowing before you pick `native`: + +**Scrollcase does not repair a binary's library paths.** A binary that finds its libraries through +an absolute path recorded when it was compiled will not find them inside a box, and fixing that is +per-format work — rpath on Linux, `install_name` on macOS, the DLL search order on Windows — that +this release deliberately does not attempt. A native box must ship a binary that already resolves: +statically linked, or built with a relative rpath. The self-test catches the rest at build time, on +your machine, rather than on a user's. + +**A native box still has an environment.** `native` means "no interpreter", not "no dependencies": +it is built from a `pixi.lock` like every other box, its binary links against the shared libraries +that lock installed, and those libraries get the same derived licence audit. What it does *not* +declare for you is the dependency list — only you know what your binary needs. + +For `node`, one thing happens on your behalf: the box is given its own `package.json` unless the +payload already carries one. Node decides whether a `.js` file is CommonJS or an ES module by +looking at the nearest `package.json` **above** it, and without one inside the box that walk leaves +the box entirely and asks whichever directory the box was extracted into. Ship your own +`package.json` as a `localFile` if you want ESM or anything else in it. + ### Declared runtime environment `environment` is a map of names to string values, one per @@ -286,9 +369,38 @@ or: } ``` -Omit `execution` for a library-only box. `scrollcase new scroll` presents the three authoring -choices as `python-script`, `python-module`, and `library-only`; the last one deliberately emits no -execution object. +or, for the other two runtimes: + +```jsonc +"execution": { + "kind": "node-script", + "script": "app/main.js", + "defaultArgs": [] +} +``` + +```jsonc +"execution": { + "kind": "native-binary", + "binary": "bin/my-tool", + "defaultArgs": [] +} +``` + +Each kind is named `-`, and the runtime half must be the one the box declares: a +`python-script` in a box whose runtime is `native` describes something that cannot be run, and is +refused rather than guessed at. `scrollcase new scroll` offers only the kinds the chosen runtime +defines — see [the CLI reference](/reference/cli#new). + +Omit `execution` for a library-only box. A `native` box cannot be library-only: its only self-test +shape is an invocation of its own binary, so a native box with nothing to invoke could prove nothing +about itself. + +A `native-binary` must additionally be declared `executable: true` on the asset or local file that +brings it in, unless it comes from the packed environment's own scripts directory. The executable +bit is synthesised into the archive from what the scroll declared, never read off the build machine, +so without the declaration the box ships a binary it cannot start — and the build refuses it rather +than signing one. Script authoring either hashes an existing regular project file or generates a minimal starter. The exact SHA-256 is recorded in `localFiles`, the payload path is traversal-checked, and neither an diff --git a/docs/white-paper.md b/docs/white-paper.md index 308d59e..8dc122a 100644 --- a/docs/white-paper.md +++ b/docs/white-paper.md @@ -1772,6 +1772,36 @@ deserves its own pass rather than a guess. A native box must ship a binary that statically linked, or built with a relative rpath. This is a stated limitation, not an assumption left for someone to discover. +It is not hypothetical, and it is not usually the box author's doing. The first native example built +for this repository ran `sqlite3`, whose own linkage is entirely `@rpath` and perfectly relocatable — +but conda-forge's `ncurses`, three dependencies down, ships a `libncurses.6.dylib` that re-exports +`libtinfo.6.dylib` through an unrewritten *build machine* path. The box was correct; a package inside +it was not, and no relocation step Scrollcase performs would have fixed it. **The self-test caught it +before anything was signed**, which is the arrangement working: a native box that cannot start fails +the build rather than the user. + + + +
+ +#### The one file a Node box has to carry + +Node decides whether a `.js` file is CommonJS or an ES module by walking *up* from the file to the +nearest `package.json`. Inside a box there usually is none, so the walk **leaves the box** and asks +whatever directory the box happened to be extracted into. A box extracted under a project whose +`package.json` says `"type": "module"` runs its own entry point as ESM; the same box one directory +higher runs it as CommonJS. That is a box whose behaviour depends on where it was put, which is the +one thing a box exists not to be. + +So `src/runtimes/node/payload.mjs` writes the box its own, and the walk stops inside it. The contents +are fixed, so two builds of one commit still produce the same bytes; it is written only when the +payload does not already carry one, because a project that ships a `package.json` has said what it +wants and overwriting that would replace an answer with a default; and it is written after the prunes +and before the payload is read, so it is archived and digested like every other file. + +This too was found by building one: the example Node box failed its self-test against *this +repository's* `package.json`. +
@@ -6290,6 +6320,7 @@ what a box's runtime is allowed to need that another runtime would not. | `src/runtimes/python/dependencies.mjs` | Reading a pip `requirements.txt` into conda-forge terms | 6.16 | | `src/runtimes/python/templates/index.mjs` | The Python source `new scroll` writes, and the interpreter constraint a generated manifest declares | 6.16 | | `src/runtimes/node/index.mjs` | The Node adapter: `nodejs` from conda-forge, and nothing to repair | 6.6 | +| `src/runtimes/node/payload.mjs` | The `package.json` a Node box carries so nothing above it decides what its code is | 6.6 | | `src/runtimes/node/templates/index.mjs` | The JavaScript source `new scroll` writes, and the Node constraint a generated manifest declares | 6.16 | | `src/runtimes/native/index.mjs` | The native adapter: no interpreter, no dependency of its own, nothing to generate | 6.6 | diff --git a/examples/README.md b/examples/README.md index dbd90e5..5dc8227 100644 --- a/examples/README.md +++ b/examples/README.md @@ -97,6 +97,52 @@ declares — the build stops with a mismatch on a checkout that looks perfectly marks the affected paths in [`.gitattributes`](../.gitattributes); a project declaring its own `localFiles` needs the same for the files it names. +## `node-box` + +The same thing as `hello-box`, one runtime over: a bare Node 22 environment from conda-forge, a +`node-script` entry point, and nothing to download beyond the runtime itself. One target +(`macos-aarch64-metal`), because it exists to show the shape rather than to be published. + +Two details are the whole point of it. The scroll declares `runtime.id: "node"` and nothing else +changes shape — the target, the licence audit, the self-test, the signed release are all the same +fields. And the built archive carries a `package.json` at its root that no scroll declares: + +```jsonc +{ "name": "scrollcase-box", "private": true, "type": "commonjs" } +``` + +Node decides whether a `.js` file is CommonJS or an ES module from the nearest `package.json` +**above** it. A box without one asks whichever directory it was extracted into — this example failed +its own self-test against *this repository's* `package.json`, which says `"type": "module"`. The +builder writes one so the walk stops inside the box, and leaves it alone if the payload already has +one. Ship your own as a `localFile` if you want ESM. + +```sh +node src/cli.mjs build node-box/macos-aarch64-metal --scrolls-dir examples +``` + +## `native-box` + +A box with **no interpreter at all**. It packs conda-forge's `zstd` and runs `venv/bin/zstd` +directly: the binary is the command line, `runtime.version` and `runtime.entryPoint` are absent, and +the self-test is two invocations of the box's own execution — `--version`, and a `--test` that must +exit 1 on a file that is not a zstd archive. + +`zstd` rather than something with a console UI, and that choice is itself the lesson. The first +version of this example ran `sqlite3`, whose own linkage is entirely `@rpath` and perfectly +relocatable — but conda-forge's `ncurses`, three dependencies down, ships a `libncurses` that +re-exports `libtinfo` through an unrewritten path on the machine that *built the package*. The box +was correct; a package inside it was not, and Scrollcase does not repair a binary's library paths. +The self-test caught it before anything was signed, which is the arrangement working: a native box +that cannot start fails the build rather than the user. + +The environment is small (`zstd` and `libzlib`) and the licence audit is derived from the lock as +usual — `native` means "no interpreter", not "no dependencies". + +```sh +node src/cli.mjs build native-box/macos-aarch64-metal --scrolls-dir examples +``` + ## `sentiment-demo` The same pipeline carrying a real model: DistilBERT SST-2 quantised to INT8 in ONNX form, with the diff --git a/examples/native-box/macos-aarch64-metal/conda-licenses.json b/examples/native-box/macos-aarch64-metal/conda-licenses.json new file mode 100644 index 0000000..53980e6 --- /dev/null +++ b/examples/native-box/macos-aarch64-metal/conda-licenses.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": 2, + "kind": "scrollcase.box.dependency-license-audit", + "targetId": "macos-aarch64-metal", + "dependencyLockSha256": "a8da7e3537873bd508bf5c926dc8c5519d597cfcc147b32b2506be4d3066990c", + "packages": [ + { + "name": "libzlib", + "version": "1.3.2", + "declaredLicense": "Zlib", + "source": "conda" + }, + { + "name": "zstd", + "version": "1.5.7", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + } + ] +} diff --git a/examples/native-box/macos-aarch64-metal/pixi.lock b/examples/native-box/macos-aarch64-metal/pixi.lock new file mode 100644 index 0000000..0edb4d3 --- /dev/null +++ b/examples/native-box/macos-aarch64-metal/pixi.lock @@ -0,0 +1,43 @@ +version: 7 +platforms: +- name: osx-arm64 + virtual-packages: + - __unix=0=0 + - __osx=13.0 + - __archspec=0=m1 +environments: + default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + packages: + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda +packages: +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + sha256: a18fa5d5bac452401459f966cf0d872224e8080c4ff93c77e168d43ab42ef9d7 + md5: f39288f0ea63ae962e1a2e4f355a0d75 + depends: + - __osx >=11.0 + constrains: + - zlib 1.3.2 *_3 + license: Zlib + license_family: Other + run_exports: + weak: + - libzlib >=1.3.2,<2.0a0 + size: 47822 + timestamp: 1785277049190 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda + sha256: da867f5092eb0cb746d353694f0098031fd9817a4ce7d5743121209ae0f406ca + md5: 4ec2684c73812cc2c3d78379384a39cc + depends: + - __osx >=11.0 + - libzlib >=1.3.2,<2.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - zstd >=1.5.7,<1.6.0a0 + size: 433687 + timestamp: 1786599629846 diff --git a/examples/native-box/macos-aarch64-metal/pixi.toml b/examples/native-box/macos-aarch64-metal/pixi.toml new file mode 100644 index 0000000..c361a5f --- /dev/null +++ b/examples/native-box/macos-aarch64-metal/pixi.toml @@ -0,0 +1,13 @@ +# Minimal Scrollcase example: no interpreter at all. The box runs a compiled program that +# conda-forge built, and the environment holds exactly what that program needs. +# +# `zstd` rather than something with a console UI: a native box ships whatever the packaged binary +# was linked against, and conda-forge's `ncurses` carries an unrewritten build-machine path to +# `libtinfo` that no relocation step here fixes. See the native runtime's stated limitation. +[workspace] +name = "native-box" +channels = ["conda-forge"] +platforms = ["osx-arm64"] + +[dependencies] +zstd = "1.5.*" diff --git a/examples/native-box/macos-aarch64-metal/scroll.json b/examples/native-box/macos-aarch64-metal/scroll.json new file mode 100644 index 0000000..9bcd4d4 --- /dev/null +++ b/examples/native-box/macos-aarch64-metal/scroll.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "schemaVersion": 3, + "boxId": "native-box", + "version": "1.0.0", + "sourceRevision": "example-native-v1", + "target": { + "platform": "macos", + "arch": "aarch64", + "accelerator": "metal" + }, + "compatibility": { + "minHostAppVersion": "1.0.0", + "minMacosVersion": "13.0" + }, + "runtime": { + "id": "native" + }, + "pixiVersion": "0.73.0", + "condaDependencyLicenseAudit": "examples/native-box/macos-aarch64-metal/conda-licenses.json", + "cacheSubdir": "cache/native-box", + "assetBaseUrl": "https://assets.example.org/boxes", + "selfTest": { + "commands": [ + { + "args": [ + "--version" + ] + }, + { + "args": [ + "--test", + "--quiet", + "box.json" + ], + "expectExitCode": 1 + } + ] + }, + "execution": { + "kind": "native-binary", + "binary": "venv/bin/zstd", + "defaultArgs": [] + } +} diff --git a/examples/node-box/macos-aarch64-metal/conda-licenses.json b/examples/node-box/macos-aarch64-metal/conda-licenses.json new file mode 100644 index 0000000..ce933da --- /dev/null +++ b/examples/node-box/macos-aarch64-metal/conda-licenses.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": 2, + "kind": "scrollcase.box.dependency-license-audit", + "targetId": "macos-aarch64-metal", + "dependencyLockSha256": "402ade9ea6d0c03e4b73f5296d59c8a80872a09876a235752ec2a439fd798964", + "packages": [ + { + "name": "ca-certificates", + "version": "2026.7.22", + "declaredLicense": "ISC", + "source": "conda" + }, + { + "name": "icu", + "version": "78.3", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libcxx", + "version": "23.1.0", + "declaredLicense": "Apache-2.0 WITH LLVM-exception", + "source": "conda" + }, + { + "name": "libuv", + "version": "1.52.1", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libzlib", + "version": "1.3.2", + "declaredLicense": "Zlib", + "source": "conda" + }, + { + "name": "nodejs", + "version": "22.23.2", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "openssl", + "version": "3.6.4", + "declaredLicense": "Apache-2.0", + "source": "conda" + } + ] +} diff --git a/examples/node-box/macos-aarch64-metal/entrypoint.js b/examples/node-box/macos-aarch64-metal/entrypoint.js new file mode 100644 index 0000000..8a6d97c --- /dev/null +++ b/examples/node-box/macos-aarch64-metal/entrypoint.js @@ -0,0 +1,14 @@ +// The application a `node` box runs. Deliberately the same shape as hello-box's entrypoint.py: +// enough to prove the box starts, reads its own files, and exits cleanly, and nothing more. + +const { readFileSync } = require('node:fs'); +const { join } = require('node:path'); + +function main() { + const manifest = JSON.parse(readFileSync(join(__dirname, 'box.json'), 'utf8')); + console.log(`Hello from ${manifest.boxId} ${manifest.version} on ${process.platform}.`); + console.log(`Running Node ${process.versions.node} from inside the box.`); + return 0; +} + +process.exitCode = main(); diff --git a/examples/node-box/macos-aarch64-metal/pixi.lock b/examples/node-box/macos-aarch64-metal/pixi.lock new file mode 100644 index 0000000..16dd074 --- /dev/null +++ b/examples/node-box/macos-aarch64-metal/pixi.lock @@ -0,0 +1,108 @@ +version: 7 +platforms: +- name: osx-arm64 + virtual-packages: + - __unix=0=0 + - __osx=13.0 + - __archspec=0=m1 +environments: + default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + packages: + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-py310h579977c_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-23.1.0-h55c6f16_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libuv-1.52.1-h1a92334_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/nodejs-22.23.2-h35957e4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.4-h55eecbc_0.conda +packages: +- conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + sha256: 0a0544cf95f64394fe4959286f5c71f5444ad58feb0602e53becb27448d24da6 + md5: 0f51e2391ade309db462a55611263e9c + depends: + - __unix + license: ISC + run_exports: {} + size: 131780 + timestamp: 1784754889428 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-py310h579977c_2.conda + sha256: 6cdb5dee54c72e56ab189fb3ad33cb28533553d42590e7e831160248f4416a43 + md5: a5efc0b42bb8b42e97d0a29ae3e3c187 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - icu >=78.3,<79.0a0 + size: 14070242 + timestamp: 1786545847761 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-23.1.0-h55c6f16_0.conda + sha256: d3e5f0b767964af25ef81edffc002f8e822b1a5c55330da1ae3a857f70c4e4ee + md5: 5303ba06fab927399ed8dfb3227b0af8 + depends: + - __osx >=11.0 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + run_exports: {} + size: 575940 + timestamp: 1787698697875 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libuv-1.52.1-h1a92334_1.conda + sha256: 4f47de9de1990efd998edbbd6793f89c8f02ffde987ae8120b1c006acefd2a04 + md5: de09bd0f175611e94f21b28f8c708e80 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - libuv >=1.52.1,<2.0a0 + size: 122729 + timestamp: 1785914645797 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + sha256: a18fa5d5bac452401459f966cf0d872224e8080c4ff93c77e168d43ab42ef9d7 + md5: f39288f0ea63ae962e1a2e4f355a0d75 + depends: + - __osx >=11.0 + constrains: + - zlib 1.3.2 *_3 + license: Zlib + license_family: Other + run_exports: + weak: + - libzlib >=1.3.2,<2.0a0 + size: 47822 + timestamp: 1785277049190 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/nodejs-22.23.2-h35957e4_0.conda + sha256: f147b779805145e67bb0588652b297ed3de90490da693fac05cb3593c03b49c9 + md5: 0ae64df299d12a790bbce918f913c90c + depends: + - __osx >=12.0 + - libcxx >=19 + - openssl >=3.5.7,<4.0a0 + - libuv >=1.52.1,<2.0a0 + - icu >=78.3,<79.0a0 + - libzlib >=1.3.2,<2.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - nodejs >=22.23.2,<23.0a0 + size: 16415636 + timestamp: 1785913942109 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.4-h55eecbc_0.conda + sha256: f23239eacd75c4c50705e68fae1aa3292da473e6a3a4abe2330f1e6afa680704 + md5: ae71ab40048c19a389a7dcccb86c2481 + depends: + - __osx >=11.0 + - ca-certificates + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - openssl >=3.6.4,<4.0a0 + size: 3110142 + timestamp: 1787698648639 diff --git a/examples/node-box/macos-aarch64-metal/pixi.toml b/examples/node-box/macos-aarch64-metal/pixi.toml new file mode 100644 index 0000000..c44d839 --- /dev/null +++ b/examples/node-box/macos-aarch64-metal/pixi.toml @@ -0,0 +1,8 @@ +# Minimal Scrollcase example: a bare Node environment from conda-forge, packed as a box. +[workspace] +name = "node-box" +channels = ["conda-forge"] +platforms = ["osx-arm64"] + +[dependencies] +nodejs = "22.*" diff --git a/examples/node-box/macos-aarch64-metal/scroll.json b/examples/node-box/macos-aarch64-metal/scroll.json new file mode 100644 index 0000000..95aecc7 --- /dev/null +++ b/examples/node-box/macos-aarch64-metal/scroll.json @@ -0,0 +1,49 @@ +{ + "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "schemaVersion": 3, + "boxId": "node-box", + "version": "1.0.0", + "sourceRevision": "example-node-v1", + "target": { + "platform": "macos", + "arch": "aarch64", + "accelerator": "metal" + }, + "compatibility": { + "minHostAppVersion": "1.0.0", + "minMacosVersion": "13.0" + }, + "runtime": { + "id": "node", + "version": "22" + }, + "pixiVersion": "0.73.0", + "condaDependencyLicenseAudit": "examples/node-box/macos-aarch64-metal/conda-licenses.json", + "cacheSubdir": "cache/node-box", + "assetBaseUrl": "https://assets.example.org/boxes", + "selfTest": { + "imports": [ + "node:fs", + "node:path" + ], + "files": [ + "entrypoint.js" + ], + "commands": [ + { + "args": [] + } + ] + }, + "localFiles": [ + { + "sourcePath": "examples/node-box/macos-aarch64-metal/entrypoint.js", + "relativePath": "entrypoint.js" + } + ], + "execution": { + "kind": "node-script", + "script": "entrypoint.js", + "defaultArgs": [] + } +} diff --git a/src/build/box.mjs b/src/build/box.mjs index 0841380..88fdb7e 100644 --- a/src/build/box.mjs +++ b/src/build/box.mjs @@ -49,6 +49,7 @@ import { } from './licenses.mjs'; import { checkParity } from './parity.mjs'; import { findCondaPack, findPixi, installAndPackPixiEnvironment } from './pixi.mjs'; +import { runtimeBuilder } from '../runtimes/index.mjs'; import { fail, run as runProcess } from './process.mjs'; import { readScroll, sourceBuildState, sourceBuildTime } from './scroll.mjs'; import { getWorkspace } from './workspace.mjs'; @@ -265,6 +266,12 @@ export async function buildBox(name, options = {}) { for (const prunePath of scroll.prunePaths ?? []) { await rm(join(payloadDir, safeRelativePath(prunePath)), { recursive: true, force: true }); } + // Whatever the runtime needs in the payload that nothing declares. It runs after the prunes, so + // a project cannot prune a file the runtime is about to write, and before the payload is read, + // so what it writes is archived and digested like everything else. + for (const written of await runtimeBuilder(scroll.runtime.id).preparePayload?.(payloadDir) ?? []) { + log(`Writing ${written}`); + } // Read once, after every staging step and every prune: this is the payload as it will be // archived, and both the licence declaration and the execution check are questions about that // tree rather than about what the scroll asked for. diff --git a/src/runtimes/index.d.mts b/src/runtimes/index.d.mts index b89e310..f4f2d84 100644 --- a/src/runtimes/index.d.mts +++ b/src/runtimes/index.d.mts @@ -42,6 +42,12 @@ export type RuntimeBuilder = { * runtime whose entry point Scrollcase cannot generate */ templates: RuntimeTemplates | null; + /** + * writes the files + * this runtime needs in the payload that nothing declares, returning what it wrote. Optional: + * most runtimes need none. + */ + preparePayload?: (payloadDir: string) => Promise; }; export type RuntimeTemplates = { /** diff --git a/src/runtimes/index.mjs b/src/runtimes/index.mjs index 68c190a..4cd09af 100644 --- a/src/runtimes/index.mjs +++ b/src/runtimes/index.mjs @@ -32,6 +32,9 @@ import { pythonRuntimeBuilder } from './python/index.mjs'; * is understood, and by refusing the box where it is not * @property {RuntimeTemplates | null} templates the source `new scroll` writes, or null for a * runtime whose entry point Scrollcase cannot generate + * @property {(payloadDir: string) => Promise} [preparePayload] writes the files + * this runtime needs in the payload that nothing declares, returning what it wrote. Optional: + * most runtimes need none. */ /** diff --git a/src/runtimes/node/index.mjs b/src/runtimes/node/index.mjs index c741725..2c1422a 100644 --- a/src/runtimes/node/index.mjs +++ b/src/runtimes/node/index.mjs @@ -14,6 +14,7 @@ import { runtimeAdapter } from '../../contract/runtimes.mjs'; import { assertRelocatableLaunchers } from '../launchers.mjs'; +import { writeNodePackageManifest } from './payload.mjs'; import { STARTER_SCRIPT, STARTER_SELF_TEST, pixiDependency } from './templates/index.mjs'; /** @type {import('../index.mjs').RuntimeBuilder} */ @@ -22,6 +23,7 @@ export const nodeRuntimeBuilder = Object.freeze({ contract: runtimeAdapter('node'), pixiDependency, repairLaunchers: assertRelocatableLaunchers, + preparePayload: writeNodePackageManifest, templates: Object.freeze({ script: STARTER_SCRIPT, selfTest: STARTER_SELF_TEST, diff --git a/src/runtimes/node/payload.d.mts b/src/runtimes/node/payload.d.mts new file mode 100644 index 0000000..b424413 --- /dev/null +++ b/src/runtimes/node/payload.d.mts @@ -0,0 +1,7 @@ +/** + * Writes the box's own `package.json`, unless the payload already carries one. + * + * @param {string} payloadDir + * @returns {Promise} the payload paths written, for the build log + */ +export function writeNodePackageManifest(payloadDir: string): Promise; diff --git a/src/runtimes/node/payload.mjs b/src/runtimes/node/payload.mjs new file mode 100644 index 0000000..8af9ca8 --- /dev/null +++ b/src/runtimes/node/payload.mjs @@ -0,0 +1,40 @@ +/** + * The one file a Node box has to carry that nothing declares. + * + * Node decides whether a `.js` file is CommonJS or an ES module by walking *up* from the file to + * the nearest `package.json`. Inside a box there usually is none — so the walk leaves the box and + * asks whatever directory the box happened to be extracted into. A box extracted under a project + * whose `package.json` says `"type": "module"` runs its own entry point as ESM; the same box + * extracted one directory higher runs it as CommonJS. That is a box whose behaviour depends on + * where it was put, which is the one thing a box exists not to be. It was found by building one: + * the example box failed its self-test against this repository's own `package.json`. + * + * So the box carries its own, and the walk stops inside it. The contents are fixed, so two builds + * of one commit still produce the same bytes, and it is written only when the payload does not + * already have one — a project that ships a `package.json` of its own has said what it wants, and + * overwriting that would replace an answer with a default. + */ + +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { fileExists } from '../../build/filesystem.mjs'; + +/** The manifest a Node box gets when it declares none: Node's own default, said out loud. */ +const BOX_PACKAGE_MANIFEST = `${JSON.stringify({ + name: 'scrollcase-box', + private: true, + type: 'commonjs', +}, null, 2)}\n`; + +/** + * Writes the box's own `package.json`, unless the payload already carries one. + * + * @param {string} payloadDir + * @returns {Promise} the payload paths written, for the build log + */ +export async function writeNodePackageManifest(payloadDir) { + const path = join(payloadDir, 'package.json'); + if (await fileExists(path)) return []; + await writeFile(path, BOX_PACKAGE_MANIFEST); + return ['package.json']; +} diff --git a/tests/unit/build-pipeline.test.mjs b/tests/unit/build-pipeline.test.mjs index 773f3fc..e3f4dad 100644 --- a/tests/unit/build-pipeline.test.mjs +++ b/tests/unit/build-pipeline.test.mjs @@ -159,13 +159,14 @@ function fakeToolchain(payloadDir, { // What the box starts. A native box starts a file the scroll brought in, and its packed prefix // carries no interpreter at all — so both are parameters rather than the Python answer baked in. interpreter = true, + entrySegments = ENTRY_SEGMENTS, selfTestCommand = join(payloadDir, ...ENTRY_SEGMENTS), } = {}) { const run = function run(command, args = [], options = {}) { if (command === 'pixi' && args[0] === 'install') { const manifest = args[args.indexOf('--manifest-path') + 1]; const prefix = join(dirname(manifest), '.pixi', 'envs', 'default'); - if (interpreter) writeDeep(join(prefix, ...ENTRY_SEGMENTS.slice(1)), '#!/bin/sh\nexit 0\n'); + if (interpreter) writeDeep(join(prefix, ...entrySegments.slice(1)), '#!/bin/sh\nexit 0\n'); if (consoleScript) { // What conda actually generates: the *build machine's* interpreter, reached through the // shell trampoline it falls back to when an absolute shebang would be too long. @@ -339,6 +340,72 @@ describe('the build pipeline', () => { await expect(readScroll(SCROLL_REF)).rejects.toThrow(/entry point/); }); + // A Node box, whose layout puts `node` where a Python box puts `python`. The scroll is otherwise + // hello-box: what differs is the runtime, and that is the point. + const NODE_LAYOUT = runtimeAdapter('node').layout(HOST_ADAPTER); + const NODE_ENTRY_SEGMENTS = NODE_LAYOUT.entryPoint.split('/'); + const NODE_SCROLL = { + ...SCROLL, + runtime: { id: 'node', version: '22' }, + localFiles: [{ sourcePath: 'app.js', relativePath: 'app.js' }], + execution: { kind: 'node-script', script: 'app.js', defaultArgs: [] }, + selfTest: { imports: ['fs'], files: ['app.js'] }, + }; + + function nodeToolchain(payloadDir) { + return fakeToolchain(payloadDir, { + entrySegments: NODE_ENTRY_SEGMENTS, + selfTestCommand: join(payloadDir, ...NODE_ENTRY_SEGMENTS), + }); + } + + it('gives a node box its own package.json, so nothing above it decides what its code is', async () => { + // Node picks CommonJS or ESM from the nearest package.json *above* the file it is running. With + // none in the box, that walk leaves the box and asks whichever directory it was extracted into, + // and the same box then behaves differently in two places. Found by building one: the example + // box failed its self-test against this repository's own package.json. + const { keys, payloadDir } = await makeProject(NODE_SCROLL, { + projectFiles: { 'app.js': 'console.log("ready");\n' }, + }); + const built = await buildBox(SCROLL_REF, { + ...keys, + ...nodeToolchain(payloadDir), + log: () => {}, + }); + const extracted = join(payloadDir, '..', 'extracted-node'); + await extractZipArchive(join(dirname(built.releasePath), `${built.archiveSha256}.zip`), extracted); + expect(JSON.parse(await readFile(join(extracted, 'package.json'), 'utf8'))) + .toMatchObject({ type: 'commonjs' }); + await expect(verifyBox(built.releasePath, { publicPath: keys.publicPath, log: () => {} })) + .resolves.toMatchObject({ status: 'passed' }); + }); + + it('leaves a node box that ships its own package.json alone', async () => { + // A project that declared one has said what it wants; replacing it with a default would answer + // a question the project already answered. + const declared = `${JSON.stringify({ name: 'example-app', type: 'module' }, null, 2)}\n`; + const { keys, payloadDir } = await makeProject({ + ...NODE_SCROLL, + localFiles: [ + ...NODE_SCROLL.localFiles, + { sourcePath: 'app-package.json', relativePath: 'package.json' }, + ], + }, { + projectFiles: { + 'app.js': 'console.log("ready");\n', + 'app-package.json': declared, + }, + }); + const built = await buildBox(SCROLL_REF, { + ...keys, + ...nodeToolchain(payloadDir), + log: () => {}, + }); + const extracted = join(payloadDir, '..', 'extracted-node-own'); + await extractZipArchive(join(dirname(built.releasePath), `${built.archiveSha256}.zip`), extracted); + expect(await readFile(join(extracted, 'package.json'), 'utf8')).toBe(declared); + }); + it('builds, signs and verifies a native box that starts a binary of its own', async () => { const NATIVE_SCROLL = { ...SCROLL, From c6c46c60300b46910104713234489110c210b9cb Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Fri, 28 Aug 2026 19:06:44 +0200 Subject: [PATCH 16/22] Refuse a superseded envelope by version, not as a shape error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by feeding this repository's own published v2 demo box to the v3 verifier: it was refused, but with "Invalid signed document: $.schemaVersion must equal 3" — true, and no use to whoever is holding a box and needs to know which version it is. The envelope schema pins the version to a const, so the generic shape error got there first; version 2 also never got the by-name guard version 1 had. The Python and Rust consumers already named it. All three now say the same thing about the same published document. --- src/build/verify.d.mts | 2 +- src/build/verify.mjs | 10 +++++++--- tests/unit/build-pipeline.test.mjs | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/build/verify.d.mts b/src/build/verify.d.mts index 5fc0fa1..ba0cbca 100644 --- a/src/build/verify.d.mts +++ b/src/build/verify.d.mts @@ -16,7 +16,7 @@ export function assertBoxManifestAgreement(box: any, release: any): void; * Performs the half of the trust chain that needs no archive. * * Everything here answers questions about the signed document alone — is the signature good, is the - * payload a schema-version-2 release, does it describe a target this build understands. It is split + * payload a schema-version-3 release, does it describe a target this build understands. It is split * out because a box that is already extracted has no archive to check, and re-deriving these steps * beside the ones that do would create the second interpretation of a signed release that * `inspectBoxArchive` exists to prevent. diff --git a/src/build/verify.mjs b/src/build/verify.mjs index 3f3ec33..8babd75 100644 --- a/src/build/verify.mjs +++ b/src/build/verify.mjs @@ -87,7 +87,7 @@ async function loadManifestSchemas() { * Performs the half of the trust chain that needs no archive. * * Everything here answers questions about the signed document alone — is the signature good, is the - * payload a schema-version-2 release, does it describe a target this build understands. It is split + * payload a schema-version-3 release, does it describe a target this build understands. It is split * out because a box that is already extracted has no archive to check, and re-deriving these steps * beside the ones that do would create the second interpretation of a signed release that * `inspectBoxArchive` exists to prevent. @@ -98,8 +98,12 @@ async function loadManifestSchemas() { export async function inspectReleaseDocument(releaseDocumentPath, { publicPath, trustedKeys }) { const releasePath = resolve(releaseDocumentPath); const signed = JSON.parse(await readFile(releasePath, 'utf8')); - if (signed?.schemaVersion === 1) { - fail(unsupportedSchemaVersionMessage(1)); + // Before the envelope schema gets a look at it. The schema pins `schemaVersion` to a const, so a + // superseded document would otherwise be refused as a shape error — "must equal 3" — which does + // not tell the reader what they are holding. A published box is refused by *name*, saying which + // version it is and what to do about it. + if (signed?.schemaVersion !== undefined && signed.schemaVersion !== BOX_SCHEMA_VERSION) { + fail(unsupportedSchemaVersionMessage(signed.schemaVersion)); } const [releaseSchema, boxSchema, targetSchema, executionSchema, signedSchema] = await loadManifestSchemas(); diff --git a/tests/unit/build-pipeline.test.mjs b/tests/unit/build-pipeline.test.mjs index e3f4dad..0ee1c96 100644 --- a/tests/unit/build-pipeline.test.mjs +++ b/tests/unit/build-pipeline.test.mjs @@ -932,6 +932,21 @@ describe('the build pipeline', () => { .rejects.toThrow(`Unsupported schemaVersion ${schemaVersion}; rebuild this box with Scrollcase v3.`); }); + it.each([1, 2])('rejects a whole v%i envelope by version, not as a shape error', async (schemaVersion) => { + // A *published* older box is an older envelope, not an older payload inside a current one — + // which is the case the check above covers. The envelope schema pins schemaVersion to a const, + // so without this the refusal reads "must equal 3": true, and no use to whoever is holding a + // box and needs to know which version it is. Found by feeding this repository's own published + // v2 demo box to the v3 verifier. + const { root, keys } = await makeProject(); + const releasePath = join(root, `v${schemaVersion}-envelope.release.json`); + const signed = await signDocument({ schemaVersion, kind: documentKinds().release }, keys); + await writeFile(releasePath, `${JSON.stringify({ ...signed, schemaVersion }, null, 2)}\n`); + + await expect(verifyBox(releasePath, { publicPath: keys.publicPath, log: () => {} })) + .rejects.toThrow(`Unsupported schemaVersion ${schemaVersion}; rebuild this box with Scrollcase v3.`); + }); + it('does not fall back to the pre-v2 stem-based archive name', async () => { const { keys, payloadDir } = await makeProject(); const built = await buildBox(SCROLL_REF, { ...keys, ...fakeToolchain(payloadDir), log: () => {} }); From 12a48ceb613b12943ccecb7e5743f468648ca587 Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Fri, 28 Aug 2026 19:09:45 +0200 Subject: [PATCH 17/22] Build the node and native examples in CI, and record phase C MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two new examples are the only thing in CI that builds a box with no interpreter, and one whose interpreter is not Python — the paths where a Python assumption would hide, and where the unit suite's fake toolchain cannot follow. --- .github/workflows/example-build.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/example-build.yml b/.github/workflows/example-build.yml index 67ca0da..8b36c8c 100644 --- a/.github/workflows/example-build.yml +++ b/.github/workflows/example-build.yml @@ -132,6 +132,27 @@ jobs: console.log(`extracted ${mb(payload.installedSizeBytes)}`); ' "$release" + # The other two runtimes, on the one target their example scrolls declare. They are the only + # thing in CI that builds a box with no interpreter, and a box whose interpreter is not + # Python — the paths where a Python assumption would hide, and where the unit suite's fake + # toolchain cannot follow. Each is built, verified with its own self-test, and run. + - name: Build, verify and run the node and native examples + if: matrix.target == 'macos-aarch64-metal' + shell: bash + run: | + set -euo pipefail + release_for() { + ls .scrollcase/dist/boxes/"$1"/1.0.0/${{ matrix.target }}/*.release.json + } + + node src/cli.mjs build node-box/${{ matrix.target }} --scrolls-dir examples + node src/cli.mjs verify "$(release_for node-box)" --self-test + node src/cli.mjs run "$(release_for node-box)" + + node src/cli.mjs build native-box/${{ matrix.target }} --scrolls-dir examples + node src/cli.mjs verify "$(release_for native-box)" --self-test + node src/cli.mjs run "$(release_for native-box)" -- --version + # Determinism is a promise the repository makes that nothing else verifies end to end. The # archive is named by its own SHA-256, so a rebuild that differs lands beside the first rather # than replacing it — the file count is the assertion. From 9fbb66a0672f551d94a9b8eda98157b62ca1d049 Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Fri, 28 Aug 2026 19:11:01 +0200 Subject: [PATCH 18/22] Make the CI determinism check able to fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It counted archives after a rebuild, on the reasoning that a differing archive would land beside the first under its own hash. It would not: a build clears its own object directory before writing into it, so the count is always one. Measured — changing the scroll's sourceRevision produced a completely different archive and left the count at one. It now compares the archive's name, which is its SHA-256, before and after. --- .github/workflows/example-build.yml | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/example-build.yml b/.github/workflows/example-build.yml index 8b36c8c..824fddb 100644 --- a/.github/workflows/example-build.yml +++ b/.github/workflows/example-build.yml @@ -153,23 +153,25 @@ jobs: node src/cli.mjs verify "$(release_for native-box)" --self-test node src/cli.mjs run "$(release_for native-box)" -- --version - # Determinism is a promise the repository makes that nothing else verifies end to end. The - # archive is named by its own SHA-256, so a rebuild that differs lands beside the first rather - # than replacing it — the file count is the assertion. + # Determinism is a promise the repository makes that nothing else verifies end to end. + # + # The assertion is the archive's *name*, not the file count. The archive is named by its own + # SHA-256, but a build clears its own object directory before writing into it, so a rebuild + # replaces the first archive rather than landing beside it — counting the files afterwards + # always yields one and could never fail. Measured: changing the scroll's sourceRevision + # produced a completely different archive and left the count at one. - name: Rebuild and compare shell: bash run: | + set -euo pipefail dist=".scrollcase/dist/boxes/hello-box/1.0.0/${{ matrix.target }}" - before=$(ls "$dist"/*.zip | wc -l) - if [ "$before" -ne 1 ]; then - echo "Expected exactly one archive after the first build, found $before" >&2 - exit 1 - fi + before=$(basename "$(ls "$dist"/*.zip)") node src/cli.mjs build hello-box/${{ matrix.target }} --scrolls-dir examples - after=$(ls "$dist"/*.zip | wc -l) - if [ "$after" -ne 1 ]; then - echo "Rebuild was not byte-identical: $after distinct archives" >&2 - ls -l "$dist" >&2 + after=$(basename "$(ls "$dist"/*.zip)") + if [ "$before" != "$after" ]; then + echo "Rebuild was not byte-identical:" >&2 + echo " first $before" >&2 + echo " second $after" >&2 exit 1 fi - echo "Rebuild produced a byte-identical archive." + echo "Rebuild produced a byte-identical archive: $after" From 946e62bb3d1040d8b44f98a662f79859e3f79106 Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:11:04 +0200 Subject: [PATCH 19/22] Name the publish base URL for what it is, and make publishing optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assetBaseUrl said "asset" and meant nothing of the kind: a scroll's assets carry a URL each, and this value only ever built two links — the release naming the archive, and the channel naming the release. It is publishBaseUrl, and the flag is --publish-base-url. It is also optional now, everywhere. archive.url and a channel entry's releaseManifestUrl may be absent, and a build given no publish location omits both rather than refusing. The old refusal forced anyone packaging a program to run on their own machine to invent an address, while the tool declined to invent one itself on the grounds that a placeholder in a signed release is a false statement. Both could not be right. Nothing is lost but the address: an archive is verified by sha256 and size, and all three consumers resolve one beside its release document rather than by following a link. A strip-release-archive-url conformance case pins that in every language, and examples/hello-box-native now declares no publish location at all — built, verified and run that way. This commit also carries the in-progress documentation work already in the tree: the versioned docs site with its frozen v2 snapshot, the codon, dataset and transcode examples, and src/cli-docs.mjs. --- .gitattributes | 8 + .github/workflows/example-build.yml | 36 +- AGENTS.md | 5 +- CHANGELOG.md | 353 +- README.md | 40 +- docs/.vitepress/config.mts | 195 +- docs/.vitepress/llms.mjs | 111 +- docs/.vitepress/theme/DeprecationNotice.vue | 64 + docs/.vitepress/theme/HomePage.vue | 45 +- docs/.vitepress/theme/VersionSwitch.vue | 99 + docs/.vitepress/theme/custom.css | 13 +- docs/.vitepress/theme/index.ts | 14 +- docs/.vitepress/theme/versions.data.ts | 14 + docs/.vitepress/theme/versions.ts | 29 + docs/.vitepress/versions.mjs | 43 + docs/concepts/architecture.md | 20 +- docs/concepts/design-decisions.md | 35 +- docs/concepts/security-and-trust.md | 9 +- docs/concepts/tool-comparison.md | 2 +- docs/demos/box-dev-demo.md | 2 +- docs/demos/box-run-demo.md | 2 +- docs/functions/_middleware.js | 59 +- docs/getting-started/installation.md | 2 +- docs/getting-started/overview.md | 41 +- docs/getting-started/quickstart.md | 23 +- docs/getting-started/tl-dr.md | 10 +- docs/getting-started/why-scrollcase.md | 7 +- docs/guides/distributing-boxes.md | 5 +- docs/guides/packaging-cuda.md | 2 +- docs/index.md | 2 +- .../public/schema/v2/box-manifest.schema.json | 136 + .../schema/v2/channel-manifest.schema.json | 80 + docs/public/schema/v2/execution.schema.json | 94 + .../schema/v2/release-manifest.schema.json | 301 + .../v2/revocations-manifest.schema.json | 64 + docs/public/schema/v2/scroll.schema.json | 369 + .../schema/v2/signed-document.schema.json | 43 + docs/public/schema/v2/target.schema.json | 57 + .../schema/v3/channel-manifest.schema.json | 4 +- .../schema/v3/release-manifest.schema.json | 4 +- docs/public/schema/v3/scroll.schema.json | 7 +- docs/reference/api/index.md | 59 + docs/reference/api/node.md | 424 ++ docs/reference/api/python.md | 59 + docs/reference/api/rust.md | 91 + docs/reference/box-format.md | 19 +- docs/reference/cli.md | 46 +- docs/reference/schemas.md | 22 +- docs/reference/scroll.md | 121 +- docs/v2/concepts/architecture.md | 269 + docs/v2/concepts/design-decisions.md | 549 ++ docs/v2/concepts/index.md | 15 + docs/v2/concepts/security-and-trust.md | 101 + docs/v2/concepts/tool-comparison.md | 321 + docs/v2/concepts/why-pixi.md | 116 + docs/v2/demos/box-dev-demo.md | 122 + docs/v2/demos/box-run-demo.md | 239 + docs/v2/demos/index.md | 13 + docs/v2/demos/llm-box-demo.md | 274 + docs/v2/demos/sentiment-demo.md | 195 + docs/v2/getting-started/index.md | 15 + docs/v2/getting-started/installation.md | 271 + docs/v2/getting-started/overview.md | 269 + docs/v2/getting-started/quickstart.md | 265 + docs/v2/getting-started/tl-dr.md | 121 + docs/v2/getting-started/why-scrollcase.md | 126 + docs/v2/guides/accelerator-parity.md | 153 + docs/v2/guides/distributing-boxes.md | 239 + docs/v2/guides/index.md | 15 + docs/v2/guides/managing-weights.md | 223 + docs/v2/guides/offline-airgap.md | 135 + docs/v2/guides/packaging-cuda.md | 214 + docs/v2/guides/platform-examples.md | 109 + docs/v2/guides/signing-and-custody.md | 216 + docs/v2/guides/troubleshooting.md | 124 + docs/v2/index.md | 28 + docs/{ => v2}/reference/api.md | 16 +- docs/v2/reference/box-format.md | 354 + docs/v2/reference/cli.md | 614 ++ docs/v2/reference/configuration.md | 137 + docs/v2/reference/index.md | 15 + docs/v2/reference/schemas.md | 82 + docs/v2/reference/scroll.md | 520 ++ docs/v2/white-paper.md | 6348 +++++++++++++++++ docs/white-paper.md | 174 +- examples/README.md | 145 +- .../codon-demo/macos-aarch64-metal/codons.csv | 65 + .../macos-aarch64-metal/conda-licenses.json | 104 + .../macos-aarch64-metal/entrypoint.js | 88 + .../codon-demo/macos-aarch64-metal/pixi.lock | 249 + .../codon-demo/macos-aarch64-metal/pixi.toml | 16 + .../macos-aarch64-metal/scroll.json | 56 + .../macos-aarch64-metal/conda-licenses.json | 200 + .../macos-aarch64-metal/pixi.lock | 491 ++ .../macos-aarch64-metal/pixi.toml | 16 + .../macos-aarch64-metal/readings.conf | 8 + .../macos-aarch64-metal/readings.h5 | Bin 0 -> 2336 bytes .../macos-aarch64-metal/readings.txt | 12 + .../macos-aarch64-metal/scroll.json | 60 + .../macos-aarch64-metal/conda-licenses.json | 0 .../macos-aarch64-metal/pixi.lock | 0 .../macos-aarch64-metal/pixi.toml | 2 +- .../macos-aarch64-metal/scroll.json | 7 +- .../macos-aarch64-metal/conda-licenses.json | 0 .../macos-aarch64-metal/entrypoint.js | 0 .../macos-aarch64-metal/pixi.lock | 0 .../macos-aarch64-metal/pixi.toml | 2 +- .../macos-aarch64-metal/scroll.json | 10 +- examples/hello-box/scroll.json | 2 +- examples/llm-demo/README.md | 6 +- examples/llm-demo/scroll.json | 2 +- examples/sentiment-demo/README.md | 4 +- examples/sentiment-demo/scroll.json | 2 +- .../macos-aarch64-metal/conda-licenses.json | 548 ++ .../macos-aarch64-metal/pixi.lock | 1374 ++++ .../macos-aarch64-metal/pixi.toml | 16 + .../macos-aarch64-metal/scroll.json | 46 + python/pyproject.toml | 1 + .../schemas/release-manifest.schema.json | 4 +- python/tests/conformance_support.py | 10 + python/tests/test_dependencies.py | 113 + rust/fixtures/consumer-conformance.json | 17 + .../schema/release-manifest.schema.json | 4 +- rust/src/release.rs | 19 +- rust/tests/conformance.rs | 13 +- scripts/verify-built-docs.mjs | 134 +- src/build/authoring.mjs | 75 +- src/build/box.mjs | 39 +- src/build/project.mjs | 27 +- src/build/scroll-edit.mjs | 71 +- src/cli-authoring.mjs | 189 +- src/cli-docs.mjs | 76 + src/cli-init.mjs | 49 + src/cli-menu.mjs | 12 +- src/cli-output.mjs | 25 +- src/cli-targets.mjs | 5 +- src/cli.mjs | 110 +- .../fixtures/consumer-conformance.json | 17 + .../examples/scroll-pixi.example.json | 2 +- .../fixtures/examples/scroll.example.json | 2 +- .../schema/channel-manifest.schema.json | 4 +- .../schema/release-manifest.schema.json | 4 +- src/contract/schema/scroll.schema.json | 7 +- src/contract/types/index.d.ts | 23 +- tests/helpers/consumer-conformance.mjs | 10 + tests/unit/build-pipeline.test.mjs | 55 +- tests/unit/cli-docs.test.mjs | 90 + tests/unit/cli-init.test.mjs | 48 + tests/unit/cli-output.test.mjs | 14 + tests/unit/cli-target-choice.test.mjs | 6 +- tests/unit/docs-contract.test.mjs | 11 +- tests/unit/docs-markdown-negotiation.test.mjs | 70 + tests/unit/project-surface.test.mjs | 6 +- tests/unit/scroll-authoring.test.mjs | 260 +- tests/unit/scroll-editing.test.mjs | 67 +- tests/unit/scroll-extends.test.mjs | 2 +- tests/unit/v3-migration.test.mjs | 27 +- 157 files changed, 21113 insertions(+), 423 deletions(-) create mode 100644 docs/.vitepress/theme/DeprecationNotice.vue create mode 100644 docs/.vitepress/theme/VersionSwitch.vue create mode 100644 docs/.vitepress/theme/versions.data.ts create mode 100644 docs/.vitepress/theme/versions.ts create mode 100644 docs/.vitepress/versions.mjs create mode 100644 docs/public/schema/v2/box-manifest.schema.json create mode 100644 docs/public/schema/v2/channel-manifest.schema.json create mode 100644 docs/public/schema/v2/execution.schema.json create mode 100644 docs/public/schema/v2/release-manifest.schema.json create mode 100644 docs/public/schema/v2/revocations-manifest.schema.json create mode 100644 docs/public/schema/v2/scroll.schema.json create mode 100644 docs/public/schema/v2/signed-document.schema.json create mode 100644 docs/public/schema/v2/target.schema.json create mode 100644 docs/reference/api/index.md create mode 100644 docs/reference/api/node.md create mode 100644 docs/reference/api/python.md create mode 100644 docs/reference/api/rust.md create mode 100644 docs/v2/concepts/architecture.md create mode 100644 docs/v2/concepts/design-decisions.md create mode 100644 docs/v2/concepts/index.md create mode 100644 docs/v2/concepts/security-and-trust.md create mode 100644 docs/v2/concepts/tool-comparison.md create mode 100644 docs/v2/concepts/why-pixi.md create mode 100644 docs/v2/demos/box-dev-demo.md create mode 100644 docs/v2/demos/box-run-demo.md create mode 100644 docs/v2/demos/index.md create mode 100644 docs/v2/demos/llm-box-demo.md create mode 100644 docs/v2/demos/sentiment-demo.md create mode 100644 docs/v2/getting-started/index.md create mode 100644 docs/v2/getting-started/installation.md create mode 100644 docs/v2/getting-started/overview.md create mode 100644 docs/v2/getting-started/quickstart.md create mode 100644 docs/v2/getting-started/tl-dr.md create mode 100644 docs/v2/getting-started/why-scrollcase.md create mode 100644 docs/v2/guides/accelerator-parity.md create mode 100644 docs/v2/guides/distributing-boxes.md create mode 100644 docs/v2/guides/index.md create mode 100644 docs/v2/guides/managing-weights.md create mode 100644 docs/v2/guides/offline-airgap.md create mode 100644 docs/v2/guides/packaging-cuda.md create mode 100644 docs/v2/guides/platform-examples.md create mode 100644 docs/v2/guides/signing-and-custody.md create mode 100644 docs/v2/guides/troubleshooting.md create mode 100644 docs/v2/index.md rename docs/{ => v2}/reference/api.md (94%) create mode 100644 docs/v2/reference/box-format.md create mode 100644 docs/v2/reference/cli.md create mode 100644 docs/v2/reference/configuration.md create mode 100644 docs/v2/reference/index.md create mode 100644 docs/v2/reference/schemas.md create mode 100644 docs/v2/reference/scroll.md create mode 100644 docs/v2/white-paper.md create mode 100644 examples/codon-demo/macos-aarch64-metal/codons.csv create mode 100644 examples/codon-demo/macos-aarch64-metal/conda-licenses.json create mode 100644 examples/codon-demo/macos-aarch64-metal/entrypoint.js create mode 100644 examples/codon-demo/macos-aarch64-metal/pixi.lock create mode 100644 examples/codon-demo/macos-aarch64-metal/pixi.toml create mode 100644 examples/codon-demo/macos-aarch64-metal/scroll.json create mode 100644 examples/dataset-demo/macos-aarch64-metal/conda-licenses.json create mode 100644 examples/dataset-demo/macos-aarch64-metal/pixi.lock create mode 100644 examples/dataset-demo/macos-aarch64-metal/pixi.toml create mode 100644 examples/dataset-demo/macos-aarch64-metal/readings.conf create mode 100644 examples/dataset-demo/macos-aarch64-metal/readings.h5 create mode 100644 examples/dataset-demo/macos-aarch64-metal/readings.txt create mode 100644 examples/dataset-demo/macos-aarch64-metal/scroll.json rename examples/{native-box => hello-box-native}/macos-aarch64-metal/conda-licenses.json (100%) rename examples/{native-box => hello-box-native}/macos-aarch64-metal/pixi.lock (100%) rename examples/{native-box => hello-box-native}/macos-aarch64-metal/pixi.toml (95%) rename examples/{native-box => hello-box-native}/macos-aarch64-metal/scroll.json (78%) rename examples/{node-box => hello-box-node}/macos-aarch64-metal/conda-licenses.json (100%) rename examples/{node-box => hello-box-node}/macos-aarch64-metal/entrypoint.js (100%) rename examples/{node-box => hello-box-node}/macos-aarch64-metal/pixi.lock (100%) rename examples/{node-box => hello-box-node}/macos-aarch64-metal/pixi.toml (88%) rename examples/{node-box => hello-box-node}/macos-aarch64-metal/scroll.json (71%) create mode 100644 examples/transcode-demo/macos-aarch64-metal/conda-licenses.json create mode 100644 examples/transcode-demo/macos-aarch64-metal/pixi.lock create mode 100644 examples/transcode-demo/macos-aarch64-metal/pixi.toml create mode 100644 examples/transcode-demo/macos-aarch64-metal/scroll.json create mode 100644 python/tests/test_dependencies.py create mode 100644 src/cli-docs.mjs create mode 100644 tests/unit/cli-docs.test.mjs diff --git a/.gitattributes b/.gitattributes index 2d911a2..bf67458 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,6 +5,14 @@ # So: no conversion for anything a scroll can name. The example entry points are the files in this # repository that a scroll actually names; keep this list in step with `localFiles` if that changes. examples/**/entrypoint.py -text +examples/**/entrypoint.js -text +# Reference data a box ships and answers questions about. `codon-demo` pins its SHA-256, so a +# checkout that rewrote its newlines would fail the build rather than ship different data. +examples/**/codons.csv -text +# `dataset-demo` ships a binary HDF5 file and pins its hash; `readings.txt` is the text it was +# generated from, and a rewritten newline there would silently change what regeneration produces. +examples/**/readings.h5 -text +examples/**/readings.txt -text # Committed locks and licence inventories are compared byte for byte across platforms in CI. examples/**/pixi.lock -text diff --git a/.github/workflows/example-build.yml b/.github/workflows/example-build.yml index 824fddb..101b6ac 100644 --- a/.github/workflows/example-build.yml +++ b/.github/workflows/example-build.yml @@ -145,13 +145,35 @@ jobs: ls .scrollcase/dist/boxes/"$1"/1.0.0/${{ matrix.target }}/*.release.json } - node src/cli.mjs build node-box/${{ matrix.target }} --scrolls-dir examples - node src/cli.mjs verify "$(release_for node-box)" --self-test - node src/cli.mjs run "$(release_for node-box)" - - node src/cli.mjs build native-box/${{ matrix.target }} --scrolls-dir examples - node src/cli.mjs verify "$(release_for native-box)" --self-test - node src/cli.mjs run "$(release_for native-box)" -- --version + node src/cli.mjs build hello-box-node/${{ matrix.target }} --scrolls-dir examples + node src/cli.mjs verify "$(release_for hello-box-node)" --self-test + node src/cli.mjs run "$(release_for hello-box-node)" + + node src/cli.mjs build hello-box-native/${{ matrix.target }} --scrolls-dir examples + node src/cli.mjs verify "$(release_for hello-box-native)" --self-test + node src/cli.mjs run "$(release_for hello-box-native)" -- --version + + # A node box doing real work rather than proving it starts: reference data pinned by hash, + # queried through `node:sqlite`. It is the one example whose payload carries data the box + # is expected to answer questions about, so a wrong answer here is a build failure. + node src/cli.mjs build codon-demo/${{ matrix.target }} --scrolls-dir examples + node src/cli.mjs verify "$(release_for codon-demo)" --self-test + node src/cli.mjs run "$(release_for codon-demo)" -- Leucine + + # The native counterpart: a large compiled program with a long tail of codec libraries, + # whose self-test runs a real encode rather than a version check. Slower than the rest of + # this job put together, and the only thing here that would catch a break in the one case + # `native` exists for. + node src/cli.mjs build transcode-demo/${{ matrix.target }} --scrolls-dir examples + node src/cli.mjs verify "$(release_for transcode-demo)" --self-test + node src/cli.mjs run "$(release_for transcode-demo)" -- -version + + # The other native shape: small compiled tools reading a data file the box ships, rather + # than one large program driven by flags. Its self-test reads the shipped dataset both + # ways — structure and values — so a box whose data or reader stopped agreeing fails here. + node src/cli.mjs build dataset-demo/${{ matrix.target }} --scrolls-dir examples + node src/cli.mjs verify "$(release_for dataset-demo)" --self-test + node src/cli.mjs run "$(release_for dataset-demo)" -- -H readings.h5 # Determinism is a promise the repository makes that nothing else verifies end to end. # diff --git a/AGENTS.md b/AGENTS.md index 4497c93..53a5a21 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,9 +12,10 @@ Before planning a multi-step or expensive task, and before delegating to subagen ## Project context **Scrollcase** turns a declarative **scroll** into a **box**: a portable, locked, self-contained -Python environment for one operating system and accelerator, packed so it runs somewhere other than +environment for one operating system and accelerator, packed so it runs somewhere other than where it was built, signed so a consumer can prove what they received, and accompanied by a -dependency licence inventory. +dependency licence inventory. What runs inside is the box's **runtime** — `python`, `node`, or +`native`, which starts a compiled binary and carries no interpreter at all. The substrate is **pixi + conda-pack + conda-forge**, and only that. `pixi` solves a committed `pixi.lock`, `conda-pack` relocates the resulting prefix, and the tree is extracted into the box's diff --git a/CHANGELOG.md b/CHANGELOG.md index db3ae3f..f9e55d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ All notable changes to Scrollcase are documented here. The format follows ## [Unreleased] +### Fixed — the Python package declares `referencing` + +- `scrollcase_consumer` imports `referencing` directly, to build the schema `Registry` that resolves + the `$ref`s between the bundled canonical schemas, but declared only `cryptography` and + `jsonschema`. It worked because `jsonschema` depends on `referencing` itself — that is, the package + was relying on another project's dependency list staying what it is today. It is now a declared + dependency at the floor `jsonschema` already requires, `>=0.28.4,<1`, so nothing new is installed; + what was already installed and already imported is simply named. Reported in review of the + conda-forge submission, where the declared run requirements are what the solver builds an + environment from. `tests/test_dependencies.py` now walks the shipped source and fails on any + third-party import the package does not declare. + ### Added — the `node` and `native` runtimes - **A box can run Node, or run nothing at all.** `runtime.id: "node"` packs `nodejs` from @@ -40,7 +52,25 @@ All notable changes to Scrollcase are documented here. The format follows native example built here failed on conda-forge's own `ncurses`, which carries an unrewritten build-machine path to `libtinfo`. -- **`examples/node-box` and `examples/native-box`**, both built, verified and run for real. +- **`examples/hello-box-node` and `examples/hello-box-native`**, both built, verified and run for real. + +### Changed — publishing is optional, and the field says so + +- **`assetBaseUrl` becomes `publishBaseUrl`; `--asset-base-url` becomes `--publish-base-url`.** The + old name said "asset" and meant nothing of the kind: a scroll's assets carry a URL each, and this + value was only ever used to build two links — the release naming the archive, and the channel + naming the release. It is about publishing, so it says publishing. + +- **A box no longer needs a URL at all.** `archive.url` and a channel entry's `releaseManifestUrl` + are now optional, and a build given no publish location simply omits both instead of refusing. + That refusal forced every author who only wanted to run a box on their own machine to invent an + address — while Scrollcase declined to invent one itself, on the grounds that a placeholder in a + signed release is a false statement. Both cannot be right. + + Nothing is lost but the address. No guarantee ever rested on that URL: an archive is verified by + `sha256` and size, and all three consumers find it beside its release document rather than by + following a link. A `strip-release-archive-url` conformance case now pins that in every language. + What an unpublished box gives up is the chain a downloader follows — which it has no use for. ### Changed — the version 3 box format @@ -149,6 +179,327 @@ dual-read path anywhere, and no migration tool: a box is rebuilt from its scroll wording is unchanged; the next format version now has one sentence to edit per language rather than four. +### Added — `codon-demo`, and the two smoke tests renamed + +- **`examples/codon-demo` is a `node` box doing real work**: it ships the standard genetic code and + the tool that queries it, so the recipient needs neither Node, nor npm, nor a database. `run` with + no arguments prints what the box carries, `run -- ATG` answers forward, `run -- Leucine` answers + backwards, and an unknown term exits 1. Built, verified, run and rebuilt byte-identically on + macOS; wired into `example-build.yml` beside the other two. + + It exists because the existing `node` and `native` examples prove a box *starts* and nothing more. + This one proves a box can carry data and be trusted to answer from it: `codons.csv` is pinned by + SHA-256 in `localFiles`, so appending one fabricated row is refused — `Local box file SHA-256 + mismatch` — before anything is packed or signed. + + Two constraints it documents by example. A `node` box cannot declare an npm dependency, because + Scrollcase solves from conda-forge and nothing else — the tool uses `node:sqlite`, which is part + of Node, and the JavaScript enters through `localFiles`. And it pins Node 26, because + `node:sqlite` needs a recent Node to work without a flag and `execution.defaultArgs` land *after* + the script path, so a box could not pass `--experimental-sqlite` even if it wanted to. + +- **`examples/transcode-demo` is a `native` box doing real work**: ffmpeg, pinned, with the 90 + packages it links against, signed. 121 MB archived, 391 MB extracted — the honest cost of "just + install ffmpeg", made visible. Its self-test runs a real encode rather than a version check: a + test pattern synthesised through `lavfi`, encoded with `libx264` and discarded, so a box whose + codecs did not load fails the build. No sample media ships to make that possible. + + Its third probe declares `expectExitCode: 254`, which is the point rather than a curiosity: ffmpeg + reports the negative C error number for a missing input, `ENOENT` is 2, and an exit status is one + byte. The value was measured against the built payload after the first build failed expecting 1 — + a self-test asserts the binary's real contract, not a convention. + + It is also the example where the licence inventory earns its keep: 21 of the 90 packages are + GPL-family, including ffmpeg, `x264` and `x265` at GPL-2.0-or-later. Anyone redistributing the box + needs that before shipping, and `audit` derives it from the lock. + +- **`examples/dataset-demo` is the second `native` box**, and a different shape from the first: the + HDF5 command-line tools reading a dataset the box ships, rather than one large program driven by + flags. 36 MB. The case it answers is not "I cannot install this" but "we must all read this file + the same way" — a signed box fixes the reader, so a published inspection is a repeatable one. Its + `readings.h5` is pinned by hash and regenerable: the text it came from and the `h5import` config + ship beside it. + + It was meant to be a bioinformatics tool, and conda-forge is why it is not. `samtools`, `bwa`, + `seqkit`, `minimap2`, `hmmer`, `diamond`, `blast`, `muscle` and `fasttree` are all on **bioconda**, + a second channel that an example has no business introducing. + + `mafft`, the one that is on conda-forge, **fails as a `native` box** — and the failure is now + documented in `examples/README.md`, because it is the second instance of the limitation + `hello-box-native` exists to show. Its `venv/bin/mafft` is a shell wrapper carrying the path of + the machine that built the conda package, and its `MAFFT_BINARIES` override must be absolute, + which a box extracted to a fresh temporary directory cannot supply through a fixed signed + `environment`. The self-test caught it before anything was signed. The lesson added: check what a + program *is* before packing it — a wrapper script does not relocate, a compiled binary does. + +- **`node-box` and `native-box` are now `hello-box-node` and `hello-box-native`.** They are + `hello-box` in another runtime — smoke tests, not demos — and the old names claimed more. + +### Added — every question says where it is explained + +- **Each interactive prompt prints the documentation section that covers it**, on its own muted line + under the explanation: `new scroll`'s target, runtime, box id, upstream revision, asset base URL, + execution kind, script source and paths; `init`'s example, template, dependency, Python-source and + toolchain questions; and `build`'s channel. A prompt has room for one lead-in line, which is enough + to say what a field is and never enough to say why it exists — so it names the page that does. + `scrollcase help` ends with the site itself. + +- The links live in one module, `src/cli-docs.mjs`, and **every one of them is asserted against the + pages in this repository**: the route must be a real file and the fragment a real heading on it. + A dead link in a browser shows a 404; a dead link in a terminal shows nothing, because whoever + followed it is somewhere else by the time it fails. + +### Added — `add command` and `add file --pin` + +- **`scrollcase add command -- `** records one invocation of the box's own execution + as a self-test probe, with `--expect-exit-code` for a probe that must fail. It is the counterpart + of `add import` for a runtime with no module system: a `native` box can only prove itself by + running what it declares, and until now that probe could not be authored at all — the scroll had + to be edited by hand. `remove command -- ` is its inverse. + + The arguments come after `--` rather than as a quoted list, because they *are* a command line and + the parser already preserves everything past that boundary byte for byte. It is also the only + shape that survives arguments of their own: `-version` would otherwise be read as Scrollcase's. + + The first real probe replaces the empty placeholder `new scroll` writes — "run it with no + arguments" stops being a claim anyone made once a real one exists. + +- **`scrollcase add file --pin`** records the file's SHA-256, so a changed byte fails + the build instead of shipping different data under the same signature. Opt-in, because most added + files are about to be edited and a hash recorded then would fail the very next build; reference + data the box answers from is the case that wants it. + + Together these remove the last hand edits from the end-to-end demo walkthroughs, which are + published as their own `scrollcase-e2e-demo-*` repositories: none of them now asks a reader to + open `scroll.json`. + +### Changed — `--default-args` takes one argument as itself + +- **`--default-args -hide_banner` now works**, alongside the JSON array for several + (`--default-args '["-a", "-b"]'`). Quoting a one-element JSON array to pass one flag was a tax on + the common case, and it read as noise in a walkthrough. A value opening with `[` is still held to + being a valid JSON array of strings rather than falling back to a literal, because a malformed + array silently becoming one argument that looks almost right is the worse failure. + + The quotes around the array are the shell's, not Scrollcase's: `[...]` unquoted is a glob pattern + and never reaches the process. + +### Added — the version 2 documentation stays readable at `/v2/` + +- **`docs/v2/` carries the version 2 documentation as it was published.** Thirty-four pages copied + from the last version 2 commit, plus an index that says what they are. The site's own pages were + rewritten for version 3, so without this the only account of what a version 2 box's documents mean + disappeared with the rewrite — while the boxes themselves stay in the field, and the schemas + describing them stay served at `/schema/v2/`. + +- **Every internal link in the copies was moved under the prefix**, including the three written as + full `https://scrollcase.dev/…` URLs, which would otherwise have walked a reader out of the version + they were reading without the dead-link check ever seeing it. Links to `/schema/v2/` were left + alone: those still resolve, and they are the point. + +- **Every page under `/v2/` carries a standing deprecation notice**, above the content rather than + floating in a corner. The reader it exists for did not come through the landing page: they arrived + from a search result or an old link, onto a mid-level page they have no reason to doubt. The + navbar switch is too quiet to catch them, and a floating badge lives in the corner people learned + to ignore when it held cookie banners — so the notice sits where reaching the first heading means + passing it. It links to the same page in the current version, falling back to the landing page + and saying so where version 3 has no counterpart. Not dismissible: the whole point is preventing + one mistake, and a dismiss button removes the warning on exactly the page where it was working. + + It is one component registered in the theme, not a block written into thirty-five files: it cannot + be forgotten on a page added later, and the copied pages stay byte-identical to what version 2 + published. + +- **Their Markdown twins say it too**, since that banner is a Vue component and never reaches the + generated `.md` files — leaving the audience most likely to read a superseded manual as current as + the only one told nothing. Each deprecated twin now opens with a `> **DEPRECATED.**` paragraph and + carries `deprecated: true` in its frontmatter: said twice because a consumer that parses + frontmatter can act on the field, and everything else reads the prose. Both name the URL that + supersedes the page — `current:` is emitted **only** where version 3 really has that page, since a + field naming the home page reads to a machine as "your replacement is here", a claim it cannot + check and would be wrong to act on. Where there is none, the prose says where the current + documentation starts instead. + + Both also carry `schema-version: 2` and `current-schema-version: 3`, which is the fact a consumer + can actually act on: it holds a box whose documents carry `schemaVersion`, and comparing that + number is how it works out which of the two manuals describes what it has. Integers, not `v2` + strings, because that is how the format spells them — `"schemaVersion": 2` in every document + version 2 ever signed — and a reader comparing this against a box in hand should not have to strip + a prefix off one side first. The current number is read from `package.json`, and + `verify-built-docs.mjs` fails if a twin disagrees with it: the day schema version 4 ships, a + `current-schema-version: 3` left behind is a lie told to every machine that reads it, and nothing + else in the build would notice. + +- **`/v2/` is served `X-Robots-Tag: noindex, follow`.** Being absent from `sitemap.xml` was never + enough: the version switch links each deprecated page from its current counterpart, and internal + links are how most pages get crawled in the first place. So the whole archive would have been + indexed and would have competed for the same queries, with the obsolete page often winning on age. + `follow`, because the links inside are worth following — not least the one back out. + + Set in `functions/_middleware.js` rather than as a `` tag, because the Markdown twins are + indexable files too and no meta tag reaches them. It is the one place that covers both + representations. Not `Disallow: /v2/` in robots.txt: that blocks the crawl rather than the index, + so the URLs can still surface bare while the crawler is prevented from ever reading the very + notice telling it not to index them. And the pages keep their self-canonical — pointing it at the + version 3 page would claim these are the same document, when the whole point is that they are not. + +- **The same responses carry `Link: <…/v2/>; rel="deprecation"`** (RFC 9745), pointing at the page + that explains the deprecation. Deliberately not `successor-version`: which page supersedes this + one differs per URL, and a header that guessed would send readers to pages that do not exist. That + answer is already per page, in the twin's `current:` field and the page's own banner. No `Sunset` + either — that promises when a resource stops being served, and these stay readable for as long as + there are version 2 boxes in the field. + + The `Deprecation` field itself is **not** emitted yet. It is a Date and nothing else, and the + release that makes version 3 current has not shipped, so there is no date that is a fact rather + than a guess. `DEPRECATED_SINCE` is the one line to set when it does; a test asserts the field + appears once it is, and stays absent while it is not. + +- **`llms.txt` stops hard-coding the schema version too.** Its header line and its schema links both + had `3` typed into `llms.mjs`. Same defect as the site footer, which said `2` for the whole of the + version 3 work; both now come from `package.json`. + +- **The route mapping is now one module** (`versions.mjs`), used by the sitemap filter, the llms + files, the switch and the notice. It resolves a candidate to the spelling the build serves rather + than answering yes or no, which is what found the bug below. + +### Fixed — the version switch stranded readers of the API reference + +- **`/v2/reference/api` offered the home page instead of `/reference/api/`.** Version 3 turned that + single page into a section, so its route grew a trailing slash, and the switch was asking whether + the exact string `/reference/api` existed. It did not, so the fallback fired and a reader looking + for the current API reference was dropped on the landing page. The generated twin, computing the + same answer from a route table that normalises the slash away, advertised `/reference/api` — so + the two halves disagreed about the same page. Found by the check on the twins, not by reading. + +- **The deprecated set has its own sidebar**, so navigation inside it stays inside it, and a + **`v3` / `v2` switch in the navbar** moves between the two. The switch is a theme component rather + than a nav entry because both versions ship in one build: which one you are reading is a property + of the route, so a fixed label would be wrong on half the pages. It keeps your place where it can — + `/v2/reference/cli` switches to `/reference/cli` — and falls back to the other version's landing + page where the page does not exist, which it sometimes does not: version 3 split the API reference + into a section and renamed the weights guide. It is in the mobile navbar too, since a control that + vanishes below 768px is how a reader gets stranded in the deprecated documentation. + +- **`/v2/` is declared as a VitePress locale**, which is what gives it its own navbar menu, its own + sidebar, and — the part worth the mechanism — **its own search index**. Nothing is translated; + both locales are `lang: 'en'`. A locale is simply the one thing VitePress has that scopes all + three to a path prefix. + + Without it the navbar was the leak: the sidebar was already prefixed, but `Reference` in the top + menu still went to the current version, changing the version under a reader without saying so. + Search was the same leak in a worse place — `VPLocalSearchBox` loads `searchIndexData[localeIndex]`, + so with one locale a search made from a deprecated page answered with current-version pages. + It now searches version 2 and finds version 2. The theme's own locale dropdown is hidden: the + version switch is the control for this, and unlike the dropdown it checks that the other version + has the page before offering to go there. + +- **`verify-built-docs.mjs` checks the switch and the menu**, on every built page rather than on the + sitemap's. Both are generated per page — the switch from the route, the menu from the locale — so + neither is written down where VitePress's dead-link pass could read it, and both fail silently: + a switch link to a route the build never emitted looks entirely normal and 404s only for whoever + used it, and a menu from the wrong locale renders perfectly while walking the reader into the + other version. The menu half was itself written against the wrong attribute order first, passed + against markup it had never matched, and now fails when it finds no menu to read. + +- **It is deliberately absent from `sitemap.xml`, `llms.txt` and `llms-full.txt`.** A sitemap is a + submission rather than an inventory, and two documentation sets describing incompatible formats + would compete for the same queries with the obsolete one often winning on age; the two llms files + exist to say what Scrollcase is *now*, and a second contradictory manual makes them worse than not + existing. The pages stay served, linkable and indexable — they are simply not put forward. They + keep their Markdown twins, because `functions/_middleware.js` advertises one for every page and an + advertised 404 is worse than no offer. + +### Fixed — the version 2 schema URLs resolve again + +- **`docs/public/schema/v2/` is served again.** Rewriting the documentation for version 3 removed it, + but every scroll, release and box already in the field carries + `"$schema": "https://scrollcase.dev/schema/v2/…"` — the `$id` those documents were published with. + Dropping the directory turned each of those URLs into a 404, breaking editor validation for anyone + holding a version 2 scroll and any tooling that dereferences `$schema`. + + Published v1 and v2 are immutable, which is a promise about the artefacts as much as the format: + a v2 box is refused by a v3 verifier *by name*, and its schema stays readable. The eight files are + restored verbatim from `v0.12.0` and are frozen — nothing generates or checks them, because there + is nothing left to keep them in step with. + + The `/.well-known/api-catalog` still lists version 3 only. A catalogue is read by software + choosing what to use, and version 2 is not a choice anyone should make now. + +### Fixed — `init` no longer goes quiet about the toolchain + +- **When pixi and conda-pack are already installed, `init` says so.** It looks for them on every run + and asks only when one is missing, so on a machine that had both, the question a reader had been + told to expect never appeared and nothing explained why — silence indistinguishable from never + having looked. Every other outcome reported; this one now does too. + +- **It also names a newer pixi when there is one**, with a terminal and unless + `--no-install-toolchain` was passed. Not general news: `new scroll` records the pixi it finds and + `build` refuses any other version for that scroll, so being behind decides what every scroll + written next pins. The lookup is advisory and best-effort — an offline machine or the public API's + rate limit simply produces no line. + +- The four outcomes moved out of `cli.mjs` into `toolchainReportLines` so they can be asserted + without a host that happens to have the tools installed, which is why the silent one went + unnoticed. + +### Added — a `native` box can name a binary the environment provides + +- **`new scroll` asks a `native` box where its binary comes from**, and `--from-environment ` answers it without a terminal. A program the dependency solve installs — conda-forge's + `venv/bin/ffmpeg`, a generated console script — is named where it lands, and nothing of the project + is copied in, so no `localFiles` entry appears. + + This shape was always in the format: every `native` example in this repository uses it. It was not + in the authoring surface, which assumed a file-naming execution always pointed at a project file to + stage — so writing one meant editing `scroll.json` by hand, and `edit scroll` does not accept + `execution.binary` either. Both `native` end-to-end demo walkthroughs carried that hand-edit as a + step until this closed it. + + The menu offers the environment first: it is the common case for `native`, and the only one that + works before anything has been compiled. + +### Changed — `new scroll` asks better questions + +- **The execution-kind menu explains the kinds it is actually offering.** The line above it was one + fixed sentence for every runtime, so a `node` author was told they could pick "an importable + module" — a Python idea that has never been on their menu. It is now assembled from the kinds the + chosen runtime defines. And a runtime with a single authored kind is no longer asked at all: + `native` defines only `native-binary`, and a menu of one reads as though an option were missing. + +- **The generated starter is the preselected script source**, ahead of pointing at an existing file. + It is the answer that works with nothing else in place — a first scroll builds and runs + immediately, and the stub is a file to edit rather than a file to go and find. + +- **A malformed box id is refused at the prompt that produced it**, naming the value and the shape it + needed. It used to be accepted, and then refused by schema validation after the revision, the URL + and the execution kind had all been answered, as `$.boxId does not match the required pattern` — + which named neither what was wrong nor what to type. The rule is read from the schema, so the early + check and the late one cannot disagree. + +- **`assetBaseUrl` is optional when authoring.** It is the one field a project often does not know on + its first day, the scroll schema never required it, and forcing an answer invited a placeholder URL + into a document whose whole value is that it is true. Press Enter to skip; supply it later with + `edit scroll`, or per build with `--asset-base-url`. + +### Fixed + +- **A build with no asset base URL is refused before it solves anything.** The URL is needed only + when the release document is written, which is after the environment solve, the self-test and the + archive — so a scroll that never named one paid for the entire build before being told. It is now + checked with the other early refusals. + +- **`llms.txt` lists the published JSON Schemas again.** The generator read them from + `schema/v2/`, a directory the site stopped emitting, and its `catch` turned the resulting + `ENOENT` into an empty list — so the schema section had been silently absent rather than wrong. + It reads `schema/v3/` now, and the eight schemas are back. The header also said "box format + schema version 2". + +- **Hard rule 1 — no consuming project's name anywhere in the tool — has the mechanical guard the + white paper already claimed it had.** `v3-migration.test.mjs` greps every tracked file and every + tracked path for it, alongside the retired product term it was already checking. The tree was + clean; nothing was keeping it that way. + ## [0.12.0] — 2026-08-22 ### Added diff --git a/README.md b/README.md index d5b8423..9e71f87 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ -### Pack entire Python environments as self-contained boxes +### Pack entire environments as self-contained boxes @@ -15,9 +15,11 @@ ## The main concept -- Scrollcase packs an entire Python environment and the code it runs — like an **LLM** or a **scientific model** — into a single, **self-contained**, **portable** and **signed** archive: a **box**. +- Scrollcase packs an entire environment and the code it runs — like an **LLM** or a **scientific model** — into a single, **self-contained**, **portable** and **signed** archive: a **box**. -- You give that box to someone else. They unpack it and run it — **that's it!** **Nothing to install**: no Python, no pip install, no compiler, no Docker, **no dependencies to maintain**. +- A box declares one **runtime**, and there are three: **`python`**, **`node`**, and **`native`**, which starts a compiled binary and carries no interpreter at all. All three are built, signed, verified and run the same way — the runtime changes what is inside the box, not how you work with it. + +- You give that box to someone else. They unpack it and run it — **that's it!** **Nothing to install**: no interpreter, no pip install, no compiler, no Docker, **no dependencies to maintain**. - Every box is **signed**, so whoever receives it can check that it is exactly the one you built and not something that changed on the way over. @@ -31,35 +33,40 @@ ## The problem it removes -Getting a Python runtime onto someone else's machine normally means asking them to rebuild your -environment: the right Python, the right libraries, the right native builds for their CPU or GPU, -the right weights downloaded from the right place. It works until it doesn't — and it breaks on +Getting a working environment onto someone else's machine normally means asking them to rebuild +yours: the right interpreter, the right libraries, the right native builds for their CPU or GPU, the +right model files downloaded from the right place. It works until it doesn't — and it breaks on their machine, not yours. Scrollcase moves that work to build time, once, on a machine you control, and turns the result into a file.
-## Four words +## Six words | Word | Meaning | | --- | --- | | **scroll** | The file you write: dependencies, model files, what to run, how to test it. The only input a build accepts. → [reference](https://scrollcase.dev/reference/scroll) | | **box** | What comes out: one archive with the whole environment inside. → [format](https://scrollcase.dev/reference/box-format) | | **target** | Which machine it is for: operating system, CPU architecture, accelerator (and CUDA version). One box, one target. | +| **runtime** | What runs *inside* the box: `python`, `node`, or `native`. One box, one runtime. → [choosing one](https://scrollcase.dev/reference/scroll#choosing-a-runtime) | +| **execution** | The one thing the box starts, signed so nobody can change it later. Optional: a box with none is a library your own application drives. → [what this is for](https://scrollcase.dev/reference/scroll#why-declare-an-execution) | | **release** | The signed document that describes the box, so a consumer can verify it. → [security model](https://scrollcase.dev/concepts/security-and-trust) | +**target** and **runtime** answer two different questions, and a box declares both. The target says +which machine it runs on; the runtime says what starts when it runs. +
## What is inside a box | Entity | What's inside | | --- | --- | -| **Python interpreter** | The exact version you chose. The host does not need Python at all. | -| **Every dependency** | Conda and PyPI packages, native libraries included, at the versions your lock file pinned. | -| **Your code** | Application files, an entry script or module to start. | +| **The runtime** | A `python` box carries the exact Python you chose, a `node` box the Node it declared, and a `native` box carries no interpreter at all — it starts a compiled binary directly. Either way the host needs none of them installed. | +| **Every dependency** | conda-forge packages, native libraries included, at the versions your lock file pinned. Even a `native` box is built from a lock: the binary it runs links against what that lock installed. | +| **Your code** | Application files, and an entry point to start — a script, a module, or the binary itself. | | **Model files** | Embedded in the archive, or kept outside it with their size and hash recorded. | | **Signed metadata** | What this box is, what it contains, and its digest — so a consumer can reject anything else. | -| **Licence inventory** | Every dependency's licence, derived from the lock, not guessed. | +| **Licence inventory** | Every dependency's licence, derived from the lock, not guessed — plus, when you declare it, the licences of what was linked *inside* a binary you supply, which no lock can see. | A box is built for **one target**: one operating system, one CPU architecture, one accelerator. `macos-aarch64-metal` and `linux-x86_64-cuda12` are two boxes, not one box with options. That is @@ -126,10 +133,13 @@ starting points for your own consumer under `consumer-templates/`. Pass `--no-ex scrollcase new scroll ``` -Asks four questions — target, box id, the upstream revision of what you are packaging, and where -boxes will be published — and writes one target-specific `scroll.json`, its `pixi.toml`, and a -starter `self_test.py`. Nothing existing is overwritten. To just look around first, use the example -`init` created. → [Scroll reference](https://scrollcase.dev/reference/scroll) +Asks for the target, the **runtime**, a box id, the upstream revision of what you are packaging, and +— optionally, press Enter to decide later — where boxes will be published. Then the execution kind, +unless the runtime has only one, and writes one target-specific `scroll.json` and its `pixi.toml`. +The runtime decides the rest: a `python` box gets a starter `self_test.py`, a `node` box a +`self_test.js`, and a `native` box neither, because only you know what your binary is. Nothing +existing is overwritten. To just look around first, use the example `init` created. +→ [Scroll reference](https://scrollcase.dev/reference/scroll) ### 3. Declare what goes in diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index f6df0fc..c92a561 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -2,8 +2,13 @@ import { defineConfig, type HeadConfig } from 'vitepress' import pkg from '../../package.json' import { writeApiCatalog } from './api-catalog.mjs' import { markdownFileFor, recordPage, writeLlmsFiles } from './llms.mjs' +import { isDeprecated } from './versions.mjs' +import { text } from 'node:stream/consumers'; const packageVersion = pkg.version +// Read from package.json rather than typed into the footer, where it sat at 2 for the whole of the +// version 3 work and was wrong on every page of the site. +const schemaVersion = pkg.schemaVersion // The production origin. Every absolute URL the build emits — sitemap entries, canonical links, // the llms.txt index — is prefixed with it, so the site has one name even though Cloudflare Pages @@ -12,7 +17,7 @@ const hostname = 'https://scrollcase.dev' // Shared with the generated Markdown, where it stands in for the home page's own description: // the landing page's subject is the site, so this is that page's description too. -const description = 'Signed, self-contained Python environment boxes for scientific and AI models' +const description = 'Signed, self-contained environment boxes for scientific and AI models' // The preview image every share renders. The labelled mark rather than a composed card: it is // square, so `twitter:card: summary` shows it whole instead of cropping a wide banner, and it is @@ -115,7 +120,16 @@ const sidebar = [ { text: 'The Scroll (scroll.json)', link: '/reference/scroll' }, { text: 'The Box Format', link: '/reference/box-format' }, { text: 'JSON Schemas', link: '/reference/schemas' }, - { text: 'Library APIs', link: '/reference/api' } + { + text: 'Scrollcase APIs', + link: '/reference/api', + collapsed: true, + items: [ + { text: 'Node', link: '/reference/api/node'}, + { text: 'Python', link: '/reference/api/python'}, + { text: 'Rust', link: '/reference/api/rust'} + ] + } ] }, { @@ -139,6 +153,122 @@ const sidebar = [ }, ] +// The deprecated documentation's own navigation, so a reader inside `/v2/` is not offered a sidebar +// that walks out of the version they are reading. It is the sidebar as it stood when v2 was +// current, with every link moved under the prefix — frozen for the same reason the pages under +// `docs/v2/` are. +const v2Sidebar = [ + { + text: 'v2 Getting Started', + link: '/v2/getting-started', + collapsed: false, + items: [ + { text: 'What\'s Scrollcase ', link: '/v2/getting-started/overview' }, + { text: 'Purpose', link: '/v2/getting-started/why-scrollcase' }, + { text: 'Quickstart', link: '/v2/getting-started/quickstart' }, + { text: 'Installation', link: '/v2/getting-started/installation' }, + { text: 'TL;DR', link: '/v2/getting-started/tl-dr' }, + { + text: 'Demos', + link: '/v2/demos', + collapsed: true, + items: [ + { text: 'Basic demos', items: [ + { text: 'Box run', link: '/v2/demos/box-run-demo' }, + { text: 'Box development', link: '/v2/demos/box-dev-demo' }, + ] }, + { text: 'AI models', items: [ + { text: 'Local LLM', link: '/v2/demos/llm-box-demo' }, + { text: 'Sentiment Analysis', link: '/v2/demos/sentiment-demo' } + ]}, + ] + } + ] + }, + { + text: 'v2 Guides', + link: '/v2/guides', + collapsed: false, + items: [ + { text: 'Managing Model Weights', link: '/v2/guides/managing-weights' }, + { text: 'Packaging CUDA Boxes', link: '/v2/guides/packaging-cuda' }, + { text: 'Accelerator Parity', link: '/v2/guides/accelerator-parity' }, + { text: 'Signing & Key Custody', link: '/v2/guides/signing-and-custody' }, + { text: 'Offline / Air-Gapped Installs', link: '/v2/guides/offline-airgap' }, + { text: 'Distributing Boxes', link: '/v2/guides/distributing-boxes' }, + { text: 'Platform Examples', link: '/v2/guides/platform-examples' }, + { text: 'Troubleshooting', link: '/v2/guides/troubleshooting' } + ] + }, + { + text: 'v2 Reference', + link: '/v2/reference', + collapsed: false, + items: [ + { text: 'CLI Commands', link: '/v2/reference/cli' }, + { text: 'Workspace Configuration', link: '/v2/reference/configuration' }, + { text: 'The Scroll (scroll.json)', link: '/v2/reference/scroll' }, + { text: 'The Box Format', link: '/v2/reference/box-format' }, + { text: 'JSON Schemas', link: '/v2/reference/schemas' }, + { text: 'Library APIs', link: '/v2/reference/api' } + ] + }, + { + text: 'v2 Concepts', + link: '/v2/concepts', + collapsed: false, + items: [ + { text: 'Architecture', link: '/v2/concepts/architecture' }, + { text: 'Security & Trust', link: '/v2/concepts/security-and-trust' }, + { text: 'Why Pixi & Conda-Forge', link: '/v2/concepts/why-pixi' }, + { text: 'Design Decisions', link: '/v2/concepts/design-decisions' }, + { text: 'Tool Comparison', link: '/v2/concepts/tool-comparison' } + ] + }, + { + text: 'v2 White Paper', + collapsed: false, + link: '/v2/white-paper' + }, +] + +const nav = [ + { text: 'Home', link: '/' }, + { text: 'Quickstart', link: '/getting-started/quickstart' }, + { text: 'Resources', activeMatch: " ", items: [ + { text: 'Overview', link: '/getting-started/overview' }, + { text: 'Architecture', link: '/concepts/architecture' }, + { text: 'Concepts', link: '/concepts/' }, + { text: 'Guides', link: '/guides/' }, + { text: 'Reference', link: '/reference/' }, + { text: 'Security & trust', link: '/concepts/security-and-trust' }, + { text: 'Other tools', link: '/concepts/tool-comparison' }, + { text: 'Quick Demo', link: '/demos/' }, + { text: 'White paper', link: '/white-paper' }, + ] + } +] + +// The same menu, prefixed. A reader inside `/v2/` who reaches for `Reference` in the navbar means +// the version 2 reference; sending them to the current one changes the version under them without +// saying so, which is the one navigation mistake this whole prefix exists to prevent. +const v2Nav = [ + { text: 'Home', link: '/v2/' }, + { text: 'Quickstart', link: '/v2/getting-started/quickstart' }, + { text: 'Resources', activeMatch: " ", items: [ + { text: 'Overview', link: '/v2/getting-started/overview' }, + { text: 'Architecture', link: '/v2/concepts/architecture' }, + { text: 'Concepts', link: '/v2/concepts/' }, + { text: 'Guides', link: '/v2/guides/' }, + { text: 'Reference', link: '/v2/reference/' }, + { text: 'Security & trust', link: '/v2/concepts/security-and-trust' }, + { text: 'Other tools', link: '/v2/concepts/tool-comparison' }, + { text: 'Quick Demo', link: '/v2/demos/' }, + { text: 'White paper', link: '/v2/white-paper' }, + ] + } +] + // https://vitepress.dev/reference/site-config export default defineConfig({ title: "Scrollcase", @@ -155,8 +285,15 @@ export default defineConfig({ // Generate sitemap.xml at build time so search engines can crawl every page. // `hostname` must be the production domain — it prefixes every URL entry. + // + // The deprecated v2 documentation is left out. A sitemap is a submission, not an inventory: it + // says "these are the pages worth ranking", and two documentation sets describing incompatible + // formats would compete for the same queries with the obsolete one often winning on age. The + // pages stay served and stay linkable — nothing marks them noindex — they are simply not put + // forward. sitemap: { hostname, + transformItems: (items) => items.filter((item) => !isDeprecated(`/${item.url}`)), }, // Two build-time jobs, both about being read correctly rather than being read at all. @@ -221,10 +358,11 @@ export default defineConfig({ version: `v${packageVersion}`, sidebar, siteDescription: description, + schemaVersion, }) const entries = await writeApiCatalog({ outDir: siteConfig.outDir, hostname }) console.log( - `generated ${entries}-entry api-catalog, llms.txt (${written.pages - 1} pages, ${Math.round(written.indexBytes / 1024)} kB), ` + `generated ${entries}-entry api-catalog, llms.txt (${written.indexed} pages, ${Math.round(written.indexBytes / 1024)} kB), ` + `llms-full.txt (${Math.round(written.fullBytes / 1024)} kB) ` + `and ${written.twins} Markdown page twins`, ) @@ -256,25 +394,6 @@ export default defineConfig({ siteTitle: 'Scrollcase', search: { provider: 'local' }, - nav: [ - { text: 'Home', link: '/' }, - { text: 'Quickstart', link: '/getting-started/quickstart' }, - { text: 'Resources', activeMatch: " ", items: [ - { text: 'Overview', link: '/getting-started/overview' }, - { text: 'Architecture', link: '/concepts/architecture' }, - { text: 'Concepts', link: '/concepts/' }, - { text: 'Guides', link: '/guides/' }, - { text: 'Reference', link: '/reference/' }, - { text: 'Security & trust', link: '/concepts/security-and-trust' }, - { text: 'Other tools', link: '/concepts/tool-comparison' }, - { text: 'Quick Demo', link: '/demos/box-run-demo' }, - { text: 'White paper', link: '/white-paper' }, - ] - }, - ], - - sidebar, - socialLinks: [ { icon: 'github', link: 'https://github.com/suffro/scrollcase' } ], @@ -284,8 +403,36 @@ export default defineConfig({ }, footer: { - message: `Scrollcase v${packageVersion} · schema version 2 · Privacy · Changelog`, + message: `Scrollcase v${packageVersion} · schema version ${schemaVersion} · Privacy · Changelog`, copyright: 'Licensed under Apache-2.0' } - } + }, + + /** + * `/v2/` is declared as a locale, which in VitePress is not really about language: it is the one + * mechanism that gives a path prefix its own `nav`, its own `sidebar` **and its own search index**. + * + * The last of those is why this is a locale rather than a prefix-keyed sidebar plus a hand-rolled + * menu. `VPLocalSearchBox` loads `searchIndexData[localeIndex]`, so a reader inside the deprecated + * documentation searches the deprecated documentation, instead of being answered with + * current-version pages in the one place they are least equipped to notice the version changed + * under them. The alternative was disabling search there, which is a worse answer to the same + * problem. + * + * Both locales are `lang: 'en'`; the split is by version, and nothing here is translated. The + * theme's own locale dropdown is hidden in `custom.css` — `VersionSwitch.vue` is the control for + * this, and unlike the dropdown it checks that the other version has the page before offering it. + */ + locales: { + root: { + label: 'v3', + lang: 'en', + themeConfig: { nav, sidebar }, + }, + v2: { + label: 'v2', + lang: 'en', + themeConfig: { nav: v2Nav, sidebar: v2Sidebar }, + }, + }, }) diff --git a/docs/.vitepress/llms.mjs b/docs/.vitepress/llms.mjs index a469d35..8c3f69c 100644 --- a/docs/.vitepress/llms.mjs +++ b/docs/.vitepress/llms.mjs @@ -19,6 +19,7 @@ import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; +import { counterpart, DEPRECATED_SCHEMA_VERSION, isDeprecated } from './versions.mjs'; /** Pages seen by `transformPageData`, keyed by normalised route. Sorted before use — the build * visits pages in whatever order Vite hands them over, and the output has to be stable. */ @@ -27,6 +28,18 @@ const pages = new Map(); /** The home page renders a Vue component and carries no prose, so it is nobody's reading. */ const HOME_ROUTE = '/'; +/** + * Documentation for a deprecated box format, kept online but kept out of both llms files. + * + * These pages still get a Markdown twin — they are pages, and a twin the middleware advertises has + * to exist — but they are deliberately absent from the index and the concatenated text. The whole + * purpose of those two files is to tell a model what Scrollcase is *now*; handing it a second, + * contradictory manual for a format the current release refuses to read would make them worse than + * not existing. The same reasoning keeps these routes out of sitemap.xml, in `config.mts`. + * + * Their twins say so on their own face; see `deprecationNotice` below. + */ + /** Turn `guides/index.md` into `/guides/` and `white-paper.md` into `/white-paper`, matching the * clean URLs the site serves and the entries VitePress writes into sitemap.xml. */ export function routeOf(relativePath) { @@ -98,7 +111,8 @@ function group(sidebar) { if (items.length) sections.push({ title: section.title, items }); } const rest = [...pages.entries()] - .filter(([route, page]) => !used.has(route) && page.route !== HOME_ROUTE) + .filter(([route, page]) => + !used.has(route) && page.route !== HOME_ROUTE && !isDeprecated(page.route)) .map(([, page]) => page) .sort((a, b) => a.route.localeCompare(b.route)); if (rest.length) sections.push({ title: 'Optional', items: rest }); @@ -107,27 +121,31 @@ function group(sidebar) { /** The schemas are served as files, not pages, and they are the machine-readable half of the * answer to most questions about the box format. Listed from what the build actually emitted. */ -async function schemaLinks(outDir, hostname) { +async function schemaLinks(outDir, hostname, schemaVersion) { try { - const names = (await readdir(join(outDir, 'schema', 'v2'))) + const names = (await readdir(join(outDir, 'schema', `v${schemaVersion}`))) .filter((name) => name.endsWith('.schema.json')) .sort(); - return names.map((name) => `- [${name}](${hostname}/schema/v3/${name})`); + return names.map((name) => `- [${name}](${hostname}/schema/v${schemaVersion}/${name})`); } catch { return []; } } -function header(version) { +// The schema version is read from package.json and threaded down here rather than typed into these +// strings. Both used to say 3 in the source; the same number was typed into the site footer, where +// it said 2 for the whole of the version 3 work and was wrong on every page until someone noticed. +function header(version, schemaVersion) { return [ '# Scrollcase', '', '> Scrollcase turns a declarative **scroll** into a **box**: a portable, locked, self-contained', - '> Python environment for one operating system and accelerator, packed so it runs somewhere other', + '> environment for one operating system and accelerator, packed so it runs somewhere other', '> than where it was built, signed so a consumer can prove what they received, and accompanied by', '> a dependency licence inventory.', '', - `- Version ${version}, box format schema version 2. Apache-2.0, vendor-neutral, open source.`, + `- Version ${version}, box format schema version ${schemaVersion}. Apache-2.0, vendor-neutral, open source.`, + '- Box runtimes: `python`, `node`, and `native`, which starts a compiled binary and carries no interpreter.', '- Substrate: pixi + conda-pack + conda-forge, and only that. There is no second dependency backend.', '- CLI verbs: `init`, `new`, `add`, `remove`, `edit`, `refresh`, `doctor`, `keygen`, `lock`, `audit`, `build`, `verify`, `run`.', '- Also a library, in three languages that implement the same consumer semantics: `scrollcase` on npm, `scrollcase-consumer` on PyPI, `scrollcase-consumer` on crates.io.', @@ -138,9 +156,9 @@ function header(version) { } /** `/llms.txt` — the index. */ -async function renderIndex({ hostname, version, sidebar, outDir }) { +async function renderIndex({ hostname, version, sidebar, outDir, schemaVersion }) { const lines = [ - ...header(version), + ...header(version, schemaVersion), `Every page below, concatenated as one document: ${hostname}/llms-full.txt`, '', ]; @@ -152,7 +170,7 @@ async function renderIndex({ hostname, version, sidebar, outDir }) { } lines.push(''); } - const schemas = await schemaLinks(outDir, hostname); + const schemas = await schemaLinks(outDir, hostname, schemaVersion); if (schemas.length) { lines.push('## JSON Schemas', '', 'The box format itself, machine-readable. Normative — the prose above describes these.', '', ...schemas, ''); } @@ -207,13 +225,13 @@ function sectionOf(page, body, hostname) { /** `/llms-full.txt` — every page's Markdown, in the index's order, each under its own URL. * The white paper goes last however the sidebar orders it: it is half the site by weight, and a * reader who truncates should lose the appendix rather than the manual. */ -async function renderFull({ hostname, version, sidebar, srcDir }) { +async function renderFull({ hostname, version, sidebar, srcDir, schemaVersion }) { const ordered = group(sidebar).flatMap((section) => section.items); const body = [...ordered].sort((a, b) => Number(a.route === '/white-paper') - Number(b.route === '/white-paper')); const parts = [ - ...header(version), + ...header(version, schemaVersion), `The complete text of ${hostname}, generated at build time. Index: ${hostname}/llms.txt`, '', ]; @@ -239,6 +257,50 @@ export function markdownFileFor(route) { * both. Double-quoted with the two characters that matter escaped. */ const yamlString = (value) => `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`; +/** + * What a deprecated page's twin says before it says anything else. + * + * The banner a person sees is a Vue component, so it lives in the rendered HTML and never reaches + * these files — which are generated from the Markdown source. Without this, the audience most + * likely to read a superseded manual as current is the one told nothing: an agent asking for + * `/v2/reference/cli.md` got version 2's text with no hint that a version 3 exists. + * + * Said twice on purpose. The frontmatter is for anything that parses it and can act on a field; the + * paragraph is for everything that does not, which is most of it. Both name the URL that supersedes + * the page — "this is old" without "here is the new one" leaves the reader exactly as stuck — and + * both name the two schema versions, which is the fact a consumer can actually act on: it is + * holding a box whose documents carry `schemaVersion`, and comparing that number is how it works + * out which of these two manuals is the one describing what it has. + * + * The versions are integers, not `v2` strings, because that is how the format itself spells them: + * `"schemaVersion": 2` in every document version 2 ever signed. A reader comparing this field + * against a box in hand should not have to strip a prefix off one side of the comparison first. + * + * `current` is emitted only when the current documentation really has this page. It dropped some of + * these and split others, and a `current` field naming the home page would read to a machine as + * "your replacement is here" — a claim it cannot check and would be wrong to act on. Where there is + * no counterpart the prose says where the current documentation starts instead, which is true. + */ +function deprecationNotice(url, mirrored, schemaVersion) { + const was = DEPRECATED_SCHEMA_VERSION; + const destination = mirrored + ? `The page that supersedes this one is ${url}` + : `Schema version ${schemaVersion} has no direct replacement for this page;` + + ` its documentation starts at ${url}`; + return { + fields: [ + 'deprecated: true', + `schema-version: ${was}`, + `current-schema-version: ${schemaVersion}`, + ...(mirrored ? [`current: ${url}`] : []), + ], + body: `> **DEPRECATED.** This documents Scrollcase box format schema version ${was}, superseded` + + ` by schema version ${schemaVersion} and no longer maintained. A version ${schemaVersion}` + + ` verifier refuses a version ${was} box by name rather than reading it, so nothing here` + + ` describes the current release. ${destination}`, + }; +} + /** * One `.md` per page, so `Accept: text/markdown` has something true to return. * @@ -246,8 +308,11 @@ const yamlString = (value) => `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\ * than prose, and the most useful Markdown a landing page can hand an agent is the map of * everything behind it. */ -async function writePageFiles({ outDir, srcDir, hostname, index, siteDescription }) { +async function writePageFiles({ outDir, srcDir, hostname, index, siteDescription, schemaVersion }) { const written = []; + // `key` already collapses the trailing slash, so the recorded page is found either way; its own + // `route` is then the spelling the site serves, which is what the twin has to advertise. + const resolve = (candidate) => pages.get(key(candidate))?.route ?? null; for (const page of pages.values()) { const body = page.route === HOME_ROUTE ? index @@ -255,13 +320,19 @@ async function writePageFiles({ outDir, srcDir, hostname, index, siteDescription // The home page declares no description of its own — it is the one page whose subject is the // whole site, so the site's own description is the accurate answer rather than a stand-in. const description = page.description || siteDescription; + const current = isDeprecated(page.route) ? counterpart(page.route, resolve, false) : null; + const notice = current + ? deprecationNotice(`${hostname}${current}`, current !== '/', schemaVersion) + : null; const document = [ '---', `title: ${yamlString(page.title)}`, ...(description ? [`description: ${yamlString(description)}`] : []), `source: ${hostname}${page.route}`, + ...(notice ? notice.fields : []), '---', '', + ...(notice ? [notice.body, ''] : []), body, '', ].join('\n'); @@ -275,18 +346,24 @@ async function writePageFiles({ outDir, srcDir, hostname, index, siteDescription /** Write the generated Markdown surface into the built site. Called from `buildEnd`, so `outDir` * already holds the rendered pages and the public assets the index points at. */ -export async function writeLlmsFiles({ outDir, srcDir, hostname, version, sidebar, siteDescription }) { +export async function writeLlmsFiles({ + outDir, srcDir, hostname, version, sidebar, siteDescription, schemaVersion, +}) { const [index, full] = await Promise.all([ - renderIndex({ hostname, version, sidebar, outDir }), - renderFull({ hostname, version, sidebar, srcDir }), + renderIndex({ hostname, version, sidebar, outDir, schemaVersion }), + renderFull({ hostname, version, sidebar, srcDir, schemaVersion }), ]); await Promise.all([ writeFile(join(outDir, 'llms.txt'), index), writeFile(join(outDir, 'llms-full.txt'), full), ]); - const twins = await writePageFiles({ outDir, srcDir, hostname, index, siteDescription }); + const twins = await writePageFiles({ outDir, srcDir, hostname, index, siteDescription, schemaVersion }); return { pages: pages.size, + // What the index actually lists, which is fewer than the pages built: the home page carries no + // prose and the deprecated docs are deliberately left out. Reported separately so the build line + // stays a true statement about llms.txt rather than a count of everything that exists. + indexed: group(sidebar).reduce((total, section) => total + section.items.length, 0), twins, indexBytes: Buffer.byteLength(index), fullBytes: Buffer.byteLength(full), diff --git a/docs/.vitepress/theme/DeprecationNotice.vue b/docs/.vitepress/theme/DeprecationNotice.vue new file mode 100644 index 0000000..1c83e93 --- /dev/null +++ b/docs/.vitepress/theme/DeprecationNotice.vue @@ -0,0 +1,64 @@ + + + + + + diff --git a/docs/.vitepress/theme/HomePage.vue b/docs/.vitepress/theme/HomePage.vue index da50e44..3092d60 100644 --- a/docs/.vitepress/theme/HomePage.vue +++ b/docs/.vitepress/theme/HomePage.vue @@ -13,7 +13,7 @@ const pillars = [ { icon: 'box', title: 'No container runtime', - text: 'A box is a single archive that unpacks into a self-contained Python environment. No Docker, no daemon, no dependency resolution at install time.', + text: 'A box is a single archive that unpacks into a self-contained environment — Python, Node, or a compiled binary with no interpreter. No Docker, no daemon, no dependency resolution at install time.', }, { icon: 'network', @@ -128,20 +128,27 @@ const capabilities = [
$ scrollcase init
-
Which target? ❯ macos-aarch64-metal
-
✓ Created scrollcase.config.json
-
✓ Installed pixi and conda-pack
- -
$ scrollcase lock example-box
-
+ C python 3.11.15 h0c9c016_1_cpython
-
+ C pytorch 2.2.0 h0c9c016_1_cuda
+
✓ Workspace initialized
+ +
$ scrollcase new scroll
+
Which target?
+
❯ your-target-arch
+
Which runtime?
+
❯ python
+
❯ node
+
❯ native
+
Box ID?
+
↳ amazing-box
+
✓ Created scroll amazing-box/your-target-arch
+ +
$ scrollcase lock amazing-box
✓ Updated pixi.lock
$ scrollcase keygen
✓ Created signing key scrollcase-fa120ac69c
-
$ scrollcase build example-box
-
→ Building example-box/macos-aarch64-metal (beta, embed)
+
$ scrollcase build amazing-box
+
→ Building amazing-box/your-target-arch
[########################################] 100%
✓ Build complete — your box is ready!
@@ -524,13 +531,23 @@ html.dark .tech-glow { .c-cmd { color: var(--vp-c-text-1); font-weight: 600; } .c-muted { color: var(--vp-c-text-3); } .c-success { color: #27c93f; font-weight: 600; } -.c-info { color: var(--vp-c-brand-1); } +.c-info { color: #4a81f8; } +.c-highlight { color: var(--vp-c-brand-1); } +.c-collapsed { display: none; } +.c-hidden { + opacity: 0; + pointer-events: none; +} /* ── Light Mode Terminal Accents ───────────────────── */ /* Forziamo colori leggibili per il terminale in light mode invece del giallo di default */ html:not(.dark) .c-prompt, -html:not(.dark) .c-info { - color: #2563eb; /* Azzurro tech molto leggibile */ +html:not(.dark) .c-highlight { + color: var(--vp-c-brand-1); +} + +html:not(.dark) .c-highlight { + color: #2563eb; } html:not(.dark) .c-success { @@ -538,7 +555,7 @@ html:not(.dark) .c-success { } html:not(.dark) .c-muted { - color: #6b7280; /* Grigio scuro */ + color: #828792; /* Grigio scuro */ } /* ── Section Heads ──────────────────────────────────────── */ diff --git a/docs/.vitepress/theme/VersionSwitch.vue b/docs/.vitepress/theme/VersionSwitch.vue new file mode 100644 index 0000000..577a628 --- /dev/null +++ b/docs/.vitepress/theme/VersionSwitch.vue @@ -0,0 +1,99 @@ + + + + + + diff --git a/docs/.vitepress/theme/custom.css b/docs/.vitepress/theme/custom.css index 9741654..8d74971 100644 --- a/docs/.vitepress/theme/custom.css +++ b/docs/.vitepress/theme/custom.css @@ -283,4 +283,15 @@ body:has(.cookie-backdrop) { .floating-share { display: none; } -} \ No newline at end of file +} +/* The theme's own locale dropdown, hidden. + * + * `/v2/` is a VitePress locale for what it gives a path prefix — its own nav, sidebar and search + * index — not because anything is translated, so the globe that comes with it would offer "v3 / v2" + * a second time. `VersionSwitch.vue` is the control for this, and unlike the dropdown it checks + * that the other version has the page before offering to go there. + */ +.VPNavBarTranslations, +.VPNavScreenTranslations { + display: none !important; +} diff --git a/docs/.vitepress/theme/index.ts b/docs/.vitepress/theme/index.ts index 05fdc57..11def93 100644 --- a/docs/.vitepress/theme/index.ts +++ b/docs/.vitepress/theme/index.ts @@ -9,6 +9,8 @@ import Tab from './tabs-component/Tab.vue' import Button from './Button.vue' import Spacer from './Spacer.vue' import SubPagesList from './SubPagesList.vue' +import VersionSwitch from './VersionSwitch.vue' +import DeprecationNotice from './DeprecationNotice.vue' import './custom.css' export default { @@ -24,7 +26,17 @@ export default { nextTick(() => initMermaid()) watch(() => isDark.value, () => initMermaid()) - return h(DefaultTheme.Layout, null, {}) + // The version switch goes in both navbars, not one: the wide layout's menu is replaced by the + // hamburger screen below 768px, and a control that exists on a desktop and vanishes on a phone + // is how a reader gets stranded in the deprecated documentation. + return h(DefaultTheme.Layout, null, { + 'nav-bar-content-after': () => h(VersionSwitch), + 'nav-screen-content-after': () => h(VersionSwitch), + // Declared for every page; the component shows itself only under `/v2/`. Registering it here + // rather than writing a block into each deprecated page is what makes it impossible to forget + // on one, and what keeps the copied pages byte-identical to what version 2 published. + 'doc-before': () => h(DeprecationNotice), + }) }, enhanceApp({ app }) { app.component('HomePage', HomePage), diff --git a/docs/.vitepress/theme/versions.data.ts b/docs/.vitepress/theme/versions.data.ts new file mode 100644 index 0000000..d7c5845 --- /dev/null +++ b/docs/.vitepress/theme/versions.data.ts @@ -0,0 +1,14 @@ +import { createContentLoader } from 'vitepress' + +// Every route the site builds, section indexes included. +// +// `subpages.data.ts` drops `index.md` on purpose — a section index listing itself is noise — but the +// version switch asks a different question: does the *other* version have this page? Answering it +// from an incomplete list sends a reader from `/reference/` to `/v2/` when `/v2/reference/` was +// right there, which is the exact papercut the switch exists to avoid. +export default createContentLoader('**/*.md', { + includeSrc: false, + render: false, + excerpt: false, + transform: (raw) => raw.map(({ url }) => url).sort(), +}) diff --git a/docs/.vitepress/theme/versions.ts b/docs/.vitepress/theme/versions.ts new file mode 100644 index 0000000..0439baa --- /dev/null +++ b/docs/.vitepress/theme/versions.ts @@ -0,0 +1,29 @@ +/** + * The client half of the version mapping: the rule comes from `../versions.mjs`, this supplies the + * one thing that rule cannot know — which routes the build actually produced — from the data + * loader beside it. The build-time callers in `llms.mjs` answer the same question from the pages + * VitePress recorded, so both sides decide identically without either owning the definition. + */ + +import { counterpart as pick, isDeprecated, PREFIX } from '../versions.mjs' +import { data as allRoutes } from './versions.data.js' + +export { isDeprecated, PREFIX } + +const known = new Set(allRoutes.map((url: string) => url.replace(/\.html$/, ''))) + +/** The route a page's source path is served at: `v2/reference/cli.md` → `/v2/reference/cli`. */ +export function routeOf(relativePath: string): string { + return `/${relativePath}`.replace(/index\.md$/, '').replace(/\.md$/, '') +} + +/** A route in the spelling the build serves it at, trying both sides of the trailing slash that + * separates a page from a section index, or null when nothing was built there. */ +function resolve(candidate: string): string | null { + if (known.has(candidate)) return candidate + const alternate = candidate.endsWith('/') ? candidate.slice(0, -1) : `${candidate}/` + return alternate && known.has(alternate) ? alternate : null +} + +export const counterpart = (route: string, toDeprecated: boolean): string => + pick(route, resolve, toDeprecated) diff --git a/docs/.vitepress/versions.mjs b/docs/.vitepress/versions.mjs new file mode 100644 index 0000000..2c20ac9 --- /dev/null +++ b/docs/.vitepress/versions.mjs @@ -0,0 +1,43 @@ +/** + * Which documentation set a route belongs to, and where its opposite number is. + * + * Four callers need this rule and none of them may disagree: the sitemap and the llms files leave + * the deprecated set out, the navbar switch offers the other version, the deprecation notice offers + * the current one, and the generated Markdown twins tell a bot which URL supersedes them. The day + * two of those computed it separately is the day one starts pointing somewhere the others would not. + * + * Plain ESM with no imports so both sides can use it: `llms.mjs` and `config.mts` run in Node at + * build time, while `theme/versions.ts` is bundled into the browser. Which routes exist is the one + * thing this module cannot know — the build knows it one way, the client another — so callers pass + * that in. + */ + +export const PREFIX = '/v2'; + +/** The box format schema version the prefix names. The two move together — a third documentation + * set would be `/v3` describing schema version 3 — so they are declared side by side rather than + * one being parsed out of the other. */ +export const DEPRECATED_SCHEMA_VERSION = 2; + +/** True for the deprecated set's landing page and everything under it. */ +export const isDeprecated = (route) => route === PREFIX || route.startsWith(`${PREFIX}/`); + +/** + * The other version's copy of `route`, or that version's landing page. + * + * `resolve` takes a candidate route and returns it **in the spelling the build serves it at**, or + * null when nothing was built there. Returning the route rather than a boolean is the whole point: + * a page and a section index differ only by a trailing slash, and version 3 turned the single + * `reference/api` page into a `reference/api/` section. With a yes/no answer the two callers + * disagreed about which spelling counted, so the generated twin advertised `/reference/api` while + * the navbar switch, unable to find that exact string, silently fell back to the home page. + * + * The fallback is what keeps the rest honest: version 3 dropped some of these pages and renamed + * others, and offering a mirrored route that was never built hands the reader a 404 in place of an + * answer. + */ +export function counterpart(route, resolve, toDeprecated) { + const rest = isDeprecated(route) ? route.slice(PREFIX.length) || '/' : route; + const target = toDeprecated ? `${PREFIX}${rest}` : rest; + return resolve(target) ?? (toDeprecated ? `${PREFIX}/` : '/'); +} diff --git a/docs/concepts/architecture.md b/docs/concepts/architecture.md index 1b7c272..80f6ecc 100644 --- a/docs/concepts/architecture.md +++ b/docs/concepts/architecture.md @@ -6,28 +6,31 @@ description: How a scroll becomes a signed box, and how local consumers prepare # Architecture Scrollcase turns a declarative **scroll** into a **box**: a portable, locked, self-contained -Python environment for one operating system and accelerator, packed so it runs somewhere other +environment for one operating system and accelerator, packed so it runs somewhere other than where it was built, signed so a consumer can prove what they received, and accompanied by a -dependency licence inventory. +dependency licence inventory. What runs inside it is the box's **runtime** — `python`, `node`, or +`native`, which starts a compiled binary and carries no interpreter at all. This page explains how, and — more usefully — *why each step is where it is*. -## The v2 consumer boundary +## The consumer boundary -Scrollcase has one canonical contract and two local consumer implementations: the Node/TypeScript +Scrollcase has one canonical contract and three local consumer implementations: the Node/TypeScript API at `scrollcase/consumer`, the Python package imported as `scrollcase_consumer`, and the Rust crate `scrollcase-consumer`. -`scrollcase run` delegates to the Node API instead of implementing a third path. +`scrollcase run` delegates to the Node API instead of implementing a fourth path. ```mermaid flowchart LR - C["canonical v2 contract
schemas + fixtures"] --> N["Node consumer
scrollcase/consumer"] + C["canonical v3 contract
schemas + fixtures"] --> N["Node consumer
scrollcase/consumer"] C --> P["Python consumer
scrollcase_consumer"] C --> R["Rust consumer
scrollcase-consumer"] F["caller-supplied release, archive or root,
trust keys, destination"] --> N F --> P + F --> R N --> L["verified local box
or child process"] P --> L + R --> L ``` Every consumer must agree on verification, safe extraction, attachment across restarts, @@ -191,8 +194,9 @@ is verified locally before the build continues. See ### Verified `verify` checks signature, archive size and hash, safe entry names, recursive agreement of all -shared schema-v2 manifest fields, and the declared interpreter. With `--self-test`, it temporarily -extracts and runs the signed import subset. It does not repeat scroll-only Python or file checks. +shared schema-v3 manifest fields, and the declared runtime entry point where the runtime has one. +With `--self-test`, it temporarily extracts and answers the signed probe with the box's own runtime. +It does not repeat scroll-only file checks. ### Honest about provenance diff --git a/docs/concepts/design-decisions.md b/docs/concepts/design-decisions.md index b90276e..d608436 100644 --- a/docs/concepts/design-decisions.md +++ b/docs/concepts/design-decisions.md @@ -247,11 +247,12 @@ badly and excludes everyone using something else. ## Verification is not optional `verify` checks signature, archive size and hash, safe entry names, recursive agreement of every -shared schema-v2 field, the declared interpreter, and optional execution prerequisites. Execution is -a closed script/module union rather than a shell command. The builder and verifier inspect regular -payload/archive files to prove a script or runnable module exists; module discovery never imports -the application. With `--self-test` verification extracts temporarily and runs the signed import -subset. Scroll-only Python and file assertions remain builder checks because they are not part of +shared schema-v3 field, the declared runtime entry point where the runtime has one, and optional +execution prerequisites. Execution is a closed union of declared kinds rather than a shell command. +The builder and verifier inspect regular payload/archive files to prove a script, runnable module or +binary exists; module discovery never imports the application. With `--self-test` verification +extracts temporarily and answers the signed probe with the box's own runtime. Scroll-only file +assertions remain builder checks because they are not part of the signed release. **Rejected:** accepting a shell command or proving a module by importing it. A shell changes @@ -590,6 +591,30 @@ The public-contract audit resolved six implementation choices: - Asset resume is limited to retries within one download operation. There is no persistent cache and the documentation makes that process boundary explicit. +## A URL is routing, not trust — so a box that is not published carries none + +The signed release names where the archive is published, and the channel names where the release is +published. Both are **addresses**, and nothing verifies either: an archive is identified by its +SHA-256, and all three consumers resolve one beside its release document rather than by following a +link. A wrong URL there would break a download and no check at all. + +For a long time a build refused to proceed without one. That put the cost on exactly the wrong +person: someone packaging a program to run on their own machine, who has no publication to name, was +made to invent an address — while the tool declined to invent one itself, on the grounds that a +placeholder inside a signed document is a false statement that stays false forever. Both positions +cannot be right, and the tool's was the correct one. + +So the URL is optional everywhere, and a build without one omits both links rather than filling them +in. The box is complete: hashed, signed, self-tested, verifiable, runnable. The one thing it cannot +do is tell a stranger where to find itself, which is the one thing an unpublished box never needed. + +The field is named for what it does, too. `assetBaseUrl` said "asset" and meant nothing of the kind +— a scroll's assets carry a URL each, and this value never touched them. It is `publishBaseUrl`. + +**Rejected:** keeping the refusal and documenting the placeholder; and defaulting to something like +`https://example.invalid`. Both put an untrue statement inside a document whose whole value is that +it is true. + ## The licence audit is derived from the lock — except the half no lock can see The inventory is a pure function of the committed `pixi.lock`, which carries an SPDX licence per diff --git a/docs/concepts/security-and-trust.md b/docs/concepts/security-and-trust.md index 2388876..c576c3c 100644 --- a/docs/concepts/security-and-trust.md +++ b/docs/concepts/security-and-trust.md @@ -56,10 +56,11 @@ together. Never commit the private key under `.scrollcase/keys/`. 2. Require a release document, resolve its exact target adapter, and validate its interpreter path. 3. Locate the archive and compare its byte size and SHA-256 with the signed release. 4. List ZIP entries defensively, rejecting traversal, links, and special entries before extraction. -5. Require `box.json` and recursively compare every shared schema-v2 field: identity and version, - complete target, entry point, cache subdirectory, declared environment, consumer self-test, - the deferred-asset list, and provenance. -6. Require the declared interpreter entry inside the archive. +5. Require `box.json` and recursively compare every shared schema-v3 field: identity and version, + complete target, runtime, cache subdirectory, bundled licence inventory, declared environment, + consumer self-test, execution, the deferred-asset list, and provenance. +6. Require the declared runtime entry point inside the archive, where the runtime has one — a + `native` box declares none, and what it actually runs is checked as an execution file instead. 7. With `--self-test`, require a matching native host, extract to a temporary directory, compare the logical extracted payload size, and run the signed import check with the box's interpreter under the signed environment declaration and target validation controls. diff --git a/docs/concepts/tool-comparison.md b/docs/concepts/tool-comparison.md index 66cf9dc..510afa5 100644 --- a/docs/concepts/tool-comparison.md +++ b/docs/concepts/tool-comparison.md @@ -214,7 +214,7 @@ Scrollcase is not an application freezer like PyInstaller. PyInstaller analyzes That is often the most direct way to ship a Python desktop application. -Scrollcase does not attempt to turn the model runtime into a native-looking executable. It preserves a real Python environment and exposes declared Python scripts or modules through a verified consumer. +Scrollcase does not attempt to turn the model runtime into a native-looking executable. It preserves a real environment and exposes what the box declared — a Python script or module, a Node script, or a compiled binary — through a verified consumer. This is useful when the Python runtime is one component inside a larger product rather than the product's top-level executable. diff --git a/docs/demos/box-dev-demo.md b/docs/demos/box-dev-demo.md index 1ef06cb..cb2f207 100644 --- a/docs/demos/box-dev-demo.md +++ b/docs/demos/box-dev-demo.md @@ -119,4 +119,4 @@ them side by side so `verify`, `run`, or a consumer API can resolve the archive - `doctor` and `audit` are intentionally outside this short demo; see the complete [Quickstart](/getting-started/quickstart) and [CLI reference](/reference/cli). - To run the result from an application, start with the generated templates and the - [Library APIs reference](/reference/api). + [Library APIs reference](/reference/api/). diff --git a/docs/demos/box-run-demo.md b/docs/demos/box-run-demo.md index d973de2..0e4f25f 100644 --- a/docs/demos/box-run-demo.md +++ b/docs/demos/box-run-demo.md @@ -204,7 +204,7 @@ consumer resolve the archive beside it, under the hash that document commits to. The Python package and the Rust crate are published separately: `npm install scrollcase` installs neither, and `pip install scrollcase-consumer` or `cargo add scrollcase-consumer` needs no Node at all. There is no Rust file in the folder, but the same two calls verify and run this same box from a -native application. Full surface in the [Library APIs reference](/reference/api). +native application. Full surface in the [Library APIs reference](/reference/api/). diff --git a/docs/functions/_middleware.js b/docs/functions/_middleware.js index b9a0c6f..6e19646 100644 --- a/docs/functions/_middleware.js +++ b/docs/functions/_middleware.js @@ -28,6 +28,49 @@ const MARKDOWN_TYPE = 'text/markdown; charset=utf-8'; * `.vitepress/api-catalog.mjs`, which is Node code the Worker bundle has no business carrying. */ const CATALOG_PATH = '/.well-known/api-catalog'; +/** The deprecated documentation's prefix, spelled out for the same reason as the path above rather + * than imported from `.vitepress/versions.mjs`. `docs-markdown-negotiation.test.mjs` asserts the + * two agree, so the copy cannot drift without a test saying so. */ +const DEPRECATED_PREFIX = '/v2'; + +/** + * The date version 2 stopped being current, as a Unix timestamp, or null while there is not one. + * + * RFC 9745's `Deprecation` field is a Date and nothing else will do, so this stays null until the + * release that makes version 3 current actually ships and the date is a fact rather than a guess. + * A header stating a deprecation date that never happened is worse than no header: it is a claim + * about this project made to software that cannot check it. + * + * The signal that needs no date ships regardless — see `rel="deprecation"` below. + */ +const DEPRECATED_SINCE = null; + +/** + * What every response under the deprecated prefix carries, page and Markdown twin alike. + * + * `noindex` because the version switch links these pages from every current one, so a crawler + * finds the whole archive whether or not the sitemap offers it, and two documentation sets + * describing incompatible formats then compete for the same queries — with the obsolete one often + * winning on age. `follow` because the links inside are worth following, not least the one out. + * + * `rel="deprecation"` is RFC 9745's own relation and points at the page explaining the deprecation. + * It is deliberately not `successor-version`: that names the resource replacing *this* one, and + * which page that is differs per URL — version 3 dropped some of these and renamed others. The + * answer is already per page, in the twin's `current:` field and in the page's own banner, and a + * header that guessed would be pointing readers at pages that do not exist. + * + * No `Sunset` (RFC 8594). That field promises when a resource will stop being served, and these + * are meant to stay readable for as long as there are version 2 boxes in the field. + */ +function deprecationHeaders(pathname, origin) { + if (pathname !== DEPRECATED_PREFIX && !pathname.startsWith(`${DEPRECATED_PREFIX}/`)) return []; + return [ + ['X-Robots-Tag', 'noindex, follow'], + ['Link', `<${origin}${DEPRECATED_PREFIX}/>; rel="deprecation"`], + ...(DEPRECATED_SINCE ? [['Deprecation', `@${DEPRECATED_SINCE}`]] : []), + ]; +} + /** * True when the client explicitly asked for Markdown. * @@ -66,6 +109,14 @@ export async function onRequest(context) { const { request, next, env } = context; const url = new URL(request.url); + // Applied to whichever response is returned below. Both representations of a deprecated page get + // it — the twin is an indexable file in its own right, and it is the one a bot reads. + const deprecation = deprecationHeaders(url.pathname, url.origin); + const marked = (response) => { + for (const [name, value] of deprecation) response.headers.append(name, value); + return response; + }; + // A directly requested .md file is served as an asset; all that is missing is the promise that // it is Markdown, which depends on a mime table this code should not have to trust. if (url.pathname.endsWith('.md')) { @@ -73,7 +124,7 @@ export async function onRequest(context) { if (!asset.ok) return asset; const response = new Response(asset.body, asset); response.headers.set('Content-Type', MARKDOWN_TYPE); - return response; + return marked(response); } const markdownPath = markdownPathFor(url.pathname); @@ -89,14 +140,14 @@ export async function onRequest(context) { response.headers.append('Vary', 'Accept'); response.headers.append('Link', `<${markdownUrl}>; rel="alternate"; type="text/markdown"`); response.headers.append('Link', `<${url.origin}${CATALOG_PATH}>; rel="api-catalog"`); - return response; + return marked(response); } const markdown = await env.ASSETS.fetch(markdownUrl); if (!markdown.ok) return next(); const body = await markdown.text(); - return new Response(body, { + return marked(new Response(body, { status: 200, headers: { 'Content-Type': MARKDOWN_TYPE, @@ -108,5 +159,5 @@ export async function onRequest(context) { 'x-markdown-tokens': String(estimateTokens(body)), 'Cache-Control': 'public, max-age=3600', }, - }); + })); } diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 51b21fc..c9166eb 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -117,7 +117,7 @@ keeps the template, skips the question, and prints the command to run later inst -> Checkout the [Library APIs](/reference/api.md) section for more details. +> Checkout the [Library APIs](/reference/api/) section for more details. ## Let Scrollcase install the toolchain diff --git a/docs/getting-started/overview.md b/docs/getting-started/overview.md index 0a06f66..1220909 100644 --- a/docs/getting-started/overview.md +++ b/docs/getting-started/overview.md @@ -7,9 +7,11 @@ description: What Scrollcase is for, what a box contains, and the shape of the w ## The main concept -- Scrollcase packs an entire Python environment and the code it runs — like an **LLM** or a **scientific model** — into a single, **self-contained**, **portable** and **signed** archive: a **box**. +- Scrollcase packs an entire environment and the code it runs — like an **LLM** or a **scientific model** — into a single, **self-contained**, **portable** and **signed** archive: a **box**. -- You give that box to someone else. They unpack it and run it — **that's it!** **Nothing to install**: no Python, no pip install, no compiler, no Docker, **no dependencies to maintain**. +- A box declares one **runtime**, and there are three: **`python`**, **`node`**, and **`native`**, which starts a compiled binary and carries no interpreter at all. All three are built, signed, verified and run the same way — the runtime changes what is inside the box, not how you work with it. + +- You give that box to someone else. They unpack it and run it — **that's it!** **Nothing to install**: no interpreter, no pip install, no compiler, no Docker, **no dependencies to maintain**. - Every box is **signed**, so whoever receives it can check that it is exactly the one you built and not something that changed on the way over. @@ -19,33 +21,39 @@ This is the whole idea. ## The problem it removes -Getting a Python runtime onto someone else's machine normally means asking them to rebuild your -environment: the right Python, the right libraries, the right native builds for their CPU or GPU, -the right weights downloaded from the right place. It works until it doesn't — and it breaks on +Getting a working environment onto someone else's machine normally means asking them to rebuild +yours: the right interpreter, the right libraries, the right native builds for their CPU or GPU, the +right model files downloaded from the right place. It works until it doesn't — and it breaks on their machine, not yours. Scrollcase moves that work to build time, once, on a machine you control, and turns the result into a file. > The rest of this page is a quick overview of how it works -## Four words +## Six words | Word | Meaning | | --- | --- | | **scroll** | The file you write: dependencies, model files, what to run, how to test it. The only input a build accepts. → [reference](https://scrollcase.dev/reference/scroll) | | **box** | What comes out: one archive with the whole environment inside. → [format](https://scrollcase.dev/reference/box-format) | | **target** | Which machine it is for: operating system, CPU architecture, accelerator (and CUDA version). One box, one target. | +| **runtime** | What runs *inside* the box: `python`, `node`, or `native`. One box, one runtime. → [choosing one](https://scrollcase.dev/reference/scroll#choosing-a-runtime) | +| **execution** | The one thing the box starts, signed so nobody can change it later. Optional: a box with none is a library your own application drives. → [what this is for](https://scrollcase.dev/reference/scroll#why-declare-an-execution) | | **release** | The signed document that describes the box, so a consumer can verify it. → [security model](https://scrollcase.dev/concepts/security-and-trust) | +**target** and **runtime** answer two different questions, and a box declares both. The target says +which machine it runs on; the runtime says what starts when it runs. `macos-aarch64-metal` and +`node` are one box; the same code for Linux is another. + ## What is inside a box | Entity | What's inside | | --- | --- | -| **Python interpreter** | The exact version you chose. The host does not need Python at all. | -| **Every dependency** | Conda and PyPI packages, native libraries included, at the versions your lock file pinned. | -| **Your code** | Application files, an entry script or module to start. | +| **The runtime** | A `python` box carries the exact Python you chose, a `node` box the Node it declared, and a `native` box carries no interpreter at all — it starts a compiled binary directly. Either way the host needs none of them installed. | +| **Every dependency** | conda-forge packages, native libraries included, at the versions your lock file pinned. Even a `native` box is built from a lock: the binary it runs links against what that lock installed. | +| **Your code** | Application files, and an entry point to start — a script, a module, or the binary itself. | | **Model files** | Embedded in the archive, or kept outside it with their size and hash recorded. | | **Signed metadata** | What this box is, what it contains, and its digest — so a consumer can reject anything else. | -| **Licence inventory** | Every dependency's licence, derived from the lock, not guessed. | +| **Licence inventory** | Every dependency's licence, derived from the lock, not guessed — plus, when you declare it, the licences of what was linked *inside* a binary you supply, which no lock can see. | A box is built for **one target**: one operating system, one CPU architecture, one accelerator. `macos-aarch64-metal` and `linux-x86_64-cuda12` are two boxes, not one box with options. That is @@ -113,10 +121,13 @@ examples under `consumer-templates/`. Pass `--no-example` for an empty workspace scrollcase new scroll ``` -Asks four questions — target, box id, the upstream revision of what you are packaging, and where -boxes will be published — and writes one target-specific `scroll.json`, its `pixi.toml`, and a -starter `self_test.py`. Nothing existing is overwritten. To just look around first, use the example -`init` created. → [Scroll reference](/reference/scroll) +Asks for the target, the **runtime**, a box id, the upstream revision of what you are packaging, and +— optionally, press Enter to decide later — where boxes will be published. Then the execution kind, +unless the runtime has only one, and writes one target-specific `scroll.json` and its `pixi.toml`. +The runtime decides the rest: a `python` box gets a starter `self_test.py`, a `node` box a +`self_test.js`, and a `native` box neither, because only you know what your binary is. Nothing +existing is overwritten. To just look around first, use the example `init` created. +→ [Scroll reference](/reference/scroll) ### 3. Declare what goes in @@ -184,7 +195,7 @@ same verification, safe extraction, execution, receipt, signal, cleanup, and on- semantics, and none of them downloads anything. An application that keeps a box extracted across restarts re-attaches to it rather than unpacking -again. → [Library APIs](/reference/api) · +again. → [Library APIs](/reference/api/) · [Keeping an extracted box](/guides/distributing-boxes#keeping-an-extracted-box-across-restarts) ## Publishing diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 72665e3..1e39a98 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -132,15 +132,20 @@ scrollcase new scroll ``` The generated example is already ready for the remaining walkthrough steps, so you can skip this -command for a first build. Use the wizard for real project metadata. It asks four questions — the -complete target, the box id, the upstream revision of what you are packaging, and the base URL -boxes will be published under — plus a menu for the execution kind. Each one is printed -as its own block: a blank line, the field's name, one line saying what the field is, then the answer -typed after ` ↳ `. Everything else has a default and is available as a flag. A blank answer to a -required question repeats it rather than ending the session. - -It creates `scrolls///` with `scroll.json`, the matching `pixi.toml` and a starter -`self_test.py`, then prints the exact reference to use next. +command for a first build. Use the wizard for real project metadata. It asks for the complete target, +the **runtime**, the box id, the upstream revision of what you are packaging, and the base URL boxes +will be published under — that last one optional, since a project often does not know it yet. Then a +menu for the execution kind, which offers only the kinds the runtime you chose defines, and is +skipped entirely when that runtime defines one. Each question is printed as its own block: a blank +line, the field's name, one line saying what the field is, then the answer typed after ` ↳ `. +Everything else has a default and is available as a flag. A blank answer to a required question +repeats it rather than ending the session, and a malformed box id is refused on the spot with the +shape it needed, rather than by the schema once every other answer is in. + +It creates `scrolls///` with `scroll.json` and the matching `pixi.toml`, plus a +starter self-test where the runtime has one — `self_test.py` for `python`, `self_test.js` for `node`, +and nothing for `native`, which has no language of its own to write. It then prints the exact +reference to use next. For CI or another non-terminal caller, provide the equivalent flags shown by `scrollcase help`. Missing input that has no default fails before any file is written. diff --git a/docs/getting-started/tl-dr.md b/docs/getting-started/tl-dr.md index 2b76d11..ec9f8f1 100644 --- a/docs/getting-started/tl-dr.md +++ b/docs/getting-started/tl-dr.md @@ -15,6 +15,10 @@ define and lock dependencies build the box ``` +A scroll names one **target** — the machine the box is for — and one **runtime**, which is what +starts inside it: `python`, `node`, or `native`, which runs a compiled binary and carries no +interpreter. The three steps above are the same whichever you pick. + ```mermaid flowchart TB subgraph Dev ["1. Developer Workspace"] @@ -50,7 +54,7 @@ flowchart TB ## Initial setup 1. run `npm install -g scrollcase` to install the [CLI](/reference/cli) -2. run `scrollcase init` to create the workspace and runnable native example +2. run `scrollcase init` to create the workspace and a runnable example for your own machine 3. optionally run `scrollcase new scroll` for real project metadata 4. review the selected [scroll](/reference/scroll) 5. define the **dependencies** with `scrollcase add dep `, and declare the model files @@ -61,7 +65,7 @@ flowchart TB ## Normal update -1. update code, version, weights, or dependencies +1. update code, version, model files, or dependencies 2. re-run `scrollcase lock /` only when required 3. run `scrollcase build /` @@ -102,7 +106,7 @@ Conceptually, Scrollcase is therefore fairly linear: the developer declares the ## What Scrollcase simplifies -Scrollcase removes much of the repetitive work required to turn a Python environment into a distributable product. +Scrollcase removes much of the repetitive work required to turn an environment — Python, Node, or a compiled binary with no interpreter at all — into a distributable product. Without a tool like this, the developer would have to manage: diff --git a/docs/getting-started/why-scrollcase.md b/docs/getting-started/why-scrollcase.md index 34b0f8b..7622a05 100644 --- a/docs/getting-started/why-scrollcase.md +++ b/docs/getting-started/why-scrollcase.md @@ -8,13 +8,14 @@ description: Understand when Scrollcase is useful, compare it and see how it dif Scrollcase is not a replacement for every Python packaging or deployment tool. ::: info Scrollcase is designed for a specific problem: -A project needs to deliver a complete, target-specific Python environment as a verifiable product artifact that can run on another machine without asking the end user to assemble that environment. +A project needs to deliver a complete, target-specific environment as a verifiable product artifact that can run on another machine without asking the end user to assemble that environment. ::: That means carrying more than application code. A box may include: -- a specific Python interpreter; -- Conda and PyPI dependencies; +- a specific interpreter — Python or Node — or no interpreter at all, for a box that starts a + compiled binary; +- conda-forge dependencies; - native libraries; - model code and supporting files; - embedded or separately delivered model weights; diff --git a/docs/guides/distributing-boxes.md b/docs/guides/distributing-boxes.md index 5a549e1..5f13749 100644 --- a/docs/guides/distributing-boxes.md +++ b/docs/guides/distributing-boxes.md @@ -56,8 +56,9 @@ The chain is content-addressed end to end: **channel → release document (by it - **An object can never be replaced with different bytes under the same URL.** New bytes means a new hash means a new key. Serve `boxes/` as immutable and cache it aggressively. -The URLs inside the signed documents are `/`, so pointing -`assetBaseUrl` at wherever you serve `dist/boxes/` from is all the coordination needed. +The URLs inside the signed documents are `/`, so pointing +`publishBaseUrl` at wherever you serve `dist/boxes/` from is all the coordination needed. A box +built without one carries no such URLs at all — see [the scroll reference](/reference/scroll#publishbaseurl). ## Publishing diff --git a/docs/guides/packaging-cuda.md b/docs/guides/packaging-cuda.md index 7d9575a..661ade8 100644 --- a/docs/guides/packaging-cuda.md +++ b/docs/guides/packaging-cuda.md @@ -43,7 +43,7 @@ only CUDA ABI the contract accepts. }, "runtime": { "id": "python", "version": "3.14" }, "pixiVersion": "0.73.0", - "assetBaseUrl": "https://assets.example.org/boxes", + "publishBaseUrl": "https://boxes.example.org", "selfTest": { "imports": ["torch"], "script": "scrolls/my-model/linux-x86_64-cuda12.4/self_test.py" diff --git a/docs/index.md b/docs/index.md index 5cd0338..b0184b1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,6 @@ --- layout: page -title: Scrollcase — signed, self-contained Python environment boxes +title: Scrollcase — signed, self-contained environment boxes titleTemplate: false sidebar: false --- diff --git a/docs/public/schema/v2/box-manifest.schema.json b/docs/public/schema/v2/box-manifest.schema.json new file mode 100644 index 0000000..8b0ddb8 --- /dev/null +++ b/docs/public/schema/v2/box-manifest.schema.json @@ -0,0 +1,136 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/box-manifest.schema.json", + "title": "Box manifest (box.json)", + "description": "The manifest packed inside the archive at box.json. It restates the box's identity, layout and provenance so an extracted box is self-describing: a consumer that has the directory but not the release document can still tell what it is holding and how it was built.", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "boxId", + "modelId", + "runtimeId", + "version", + "target", + "pythonEntryPoint", + "modelCacheSubdir", + "selfTest", + "provenance" + ], + "properties": { + "schemaVersion": { + "const": 2 + }, + "boxId": { + "type": "string", + "minLength": 1 + }, + "modelId": { + "type": "string", + "minLength": 1 + }, + "runtimeId": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "target": { + "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + }, + "pythonEntryPoint": { + "type": "string", + "minLength": 1 + }, + "modelCacheSubdir": { + "type": "string", + "minLength": 1 + }, + "environment": { + "type": "object", + "description": "Environment variables repeated from the signed release. Scrollcase consumers compare this declaration before execution and apply it over inherited and caller-supplied values.", + "propertyNames": { + "minLength": 1, + "pattern": "^[^=\\u0000]+$" + }, + "additionalProperties": { + "type": "string", + "pattern": "^[^\\u0000]*$" + } + }, + "selfTest": { + "type": "object", + "additionalProperties": false, + "required": [ + "pythonImports", + "timeoutSeconds" + ], + "properties": { + "pythonImports": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "timeoutSeconds": { + "type": "integer", + "exclusiveMinimum": 0 + } + } + }, + "execution": { + "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" + }, + "provenance": { + "$ref": "https://scrollcase.dev/schema/v2/release-manifest.schema.json#/$defs/provenance" + }, + "weights": { + "const": "on-demand", + "description": "Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it." + }, + "assets": { + "type": "array", + "minItems": 1, + "description": "Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "url", + "relativePath", + "sizeBytes", + "sha256" + ], + "properties": { + "url": { + "type": "string", + "minLength": 1 + }, + "relativePath": { + "type": "string", + "minLength": 1 + }, + "sizeBytes": { + "type": "integer", + "exclusiveMinimum": 0 + }, + "sha256": { + "$ref": "https://scrollcase.dev/schema/v2/release-manifest.schema.json#/$defs/sha256" + } + } + } + } + }, + "dependentRequired": { + "assets": [ + "weights" + ], + "weights": [ + "assets" + ] + } +} diff --git a/docs/public/schema/v2/channel-manifest.schema.json b/docs/public/schema/v2/channel-manifest.schema.json new file mode 100644 index 0000000..70102ed --- /dev/null +++ b/docs/public/schema/v2/channel-manifest.schema.json @@ -0,0 +1,80 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/channel-manifest.schema.json", + "title": "Box channel manifest", + "description": "A small mutable pointer from a channel to the releases it currently serves. Signed independently from releases, so promoting a build never requires re-signing it.", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "kind", + "channel", + "boxId", + "target", + "updatedAt", + "cohortSalt", + "releases" + ], + "properties": { + "schemaVersion": { + "const": 2 + }, + "kind": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[.-][a-z0-9]+)*\\.channel$", + "description": "Wire discriminator, \".channel\", carrying the same namespace as the releases it refers to." + }, + "channel": { + "enum": [ + "nightly", + "beta", + "stable" + ] + }, + "boxId": { + "type": "string", + "minLength": 1 + }, + "target": { + "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + }, + "updatedAt": { + "type": "string", + "minLength": 1 + }, + "cohortSalt": { + "type": "string", + "minLength": 1, + "description": "Salt mixed into a client's rollout hash. It makes cohort assignment stable per client and unpredictable across channels, so a staged rollout cannot be gamed by reinstalling." + }, + "releases": { + "type": "array", + "minItems": 1, + "description": "Candidate releases in evaluation order. A client takes the first entry whose rollout cohort it falls into.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "releaseManifestUrl", + "rolloutPercentage" + ], + "properties": { + "version": { + "type": "string", + "minLength": 1 + }, + "releaseManifestUrl": { + "type": "string", + "minLength": 1 + }, + "rolloutPercentage": { + "type": "integer", + "minimum": 1, + "maximum": 100 + } + } + } + } + } +} diff --git a/docs/public/schema/v2/execution.schema.json b/docs/public/schema/v2/execution.schema.json new file mode 100644 index 0000000..a37ebfb --- /dev/null +++ b/docs/public/schema/v2/execution.schema.json @@ -0,0 +1,94 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/execution.schema.json", + "title": "Box execution", + "description": "The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only.", + "oneOf": [ + { + "title": "Python script", + "description": "Run one regular payload file with the box's own Python interpreter.", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "script", + "defaultArgs" + ], + "properties": { + "kind": { + "const": "python-script", + "description": "Selects direct script execution." + }, + "script": { + "$ref": "#/$defs/payloadPath", + "description": "Safe path to a regular Python file inside the box." + }, + "defaultArgs": { + "$ref": "#/$defs/defaultArgs" + } + } + }, + { + "title": "Python module", + "description": "Run an importable dotted module with Python's -m option.", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "module", + "defaultArgs" + ], + "properties": { + "kind": { + "const": "python-module", + "description": "Selects dotted-module execution." + }, + "module": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*)*$", + "description": "Strict Python dotted-module name, without command-line syntax or shell fragments.", + "examples": [ + "example_model.main" + ] + }, + "defaultArgs": { + "$ref": "#/$defs/defaultArgs" + } + } + } + ], + "examples": [ + { + "kind": "python-script", + "script": "entrypoint.py", + "defaultArgs": [] + }, + { + "kind": "python-module", + "module": "example_model.main", + "defaultArgs": [ + "--serve" + ] + } + ], + "$defs": { + "defaultArgs": { + "type": "array", + "description": "Arguments placed before caller-supplied arguments. Every item is passed directly without a shell.", + "default": [], + "items": { + "type": "string" + } + }, + "payloadPath": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*//).+$", + "description": "A forward-slash path inside the box payload: relative, non-empty, and unable to escape the payload root.", + "examples": [ + "entrypoint.py", + "app/main.py" + ] + } + } +} diff --git a/docs/public/schema/v2/release-manifest.schema.json b/docs/public/schema/v2/release-manifest.schema.json new file mode 100644 index 0000000..07d5458 --- /dev/null +++ b/docs/public/schema/v2/release-manifest.schema.json @@ -0,0 +1,301 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/release-manifest.schema.json", + "title": "Box release manifest", + "description": "The immutable description of one built box: what it is, which target it runs on, where its archive lives, and how it was produced. Published as the payload of a signed document and never edited after signing; a correction ships as a new version.", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "kind", + "boxId", + "modelId", + "runtimeId", + "version", + "target", + "compatibility", + "archive", + "pythonEntryPoint", + "modelCacheSubdir", + "selfTest", + "provenance" + ], + "properties": { + "schemaVersion": { + "const": 2 + }, + "kind": { + "$ref": "#/$defs/kind", + "description": "Wire discriminator, \".release\". The namespace belongs to the publishing project \u2014 a project with boxes already in the field must keep emitting the one its clients recognise \u2014 and defaults to scrollcase.box for a new one." + }, + "boxId": { + "$ref": "#/$defs/identifier" + }, + "modelId": { + "$ref": "#/$defs/identifier" + }, + "runtimeId": { + "$ref": "#/$defs/identifier" + }, + "version": { + "type": "string", + "minLength": 1 + }, + "target": { + "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + }, + "compatibility": { + "type": "object", + "additionalProperties": true, + "description": "What the host must satisfy before this box may be installed. The builder copies these constraints through verbatim and never interprets them, so a project may add its own alongside the ones defined here. A consumer that cannot evaluate a constraint must refuse the box rather than assume it passes.", + "properties": { + "minHostAppVersion": { + "type": "string", + "minLength": 1, + "description": "Lowest version of the installing application this box supports." + }, + "maxHostAppVersionExclusive": { + "type": "string", + "minLength": 1 + }, + "minMacosVersion": { + "type": "string", + "minLength": 1 + }, + "minRamGb": { + "type": "number", + "exclusiveMinimum": 0, + "description": "Installed memory in decimal gigabytes (1 GB = 1,000,000,000 bytes)." + }, + "minNvidiaDriverVersion": { + "type": "string", + "minLength": 1 + }, + "hostEnvironments": { + "type": "array", + "minItems": 1, + "items": { + "enum": [ + "native", + "windows-wsl2" + ] + }, + "description": "Host environments this payload was validated on." + } + } + }, + "archive": { + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "url", + "sha256", + "sizeBytes" + ], + "properties": { + "format": { + "const": "zip" + }, + "url": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "$ref": "#/$defs/sha256" + }, + "sizeBytes": { + "type": "integer", + "exclusiveMinimum": 0 + } + } + }, + "installedSizeBytes": { + "type": "integer", + "exclusiveMinimum": 0, + "description": "Sum of extracted payload file sizes before activation metadata is written, so a consumer can check free space before downloading." + }, + "payloadDigest": { + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "sha256" + ], + "description": "SHA-256 of the canonical entry list carried at payload-digest.v1 inside the payload, letting a consumer re-identify an extracted installation once the archive is gone. Optional: boxes built before it exists carry no such commitment.", + "properties": { + "format": { + "const": "sha256-path-list-v1" + }, + "sha256": { + "$ref": "#/$defs/sha256" + } + } + }, + "pythonEntryPoint": { + "type": "string", + "minLength": 1, + "description": "Interpreter path relative to the extracted box root, for example venv/bin/python. Fixed per target by the adapter." + }, + "modelCacheSubdir": { + "type": "string", + "minLength": 1, + "description": "Directory relative to the extracted box root holding model assets." + }, + "environment": { + "type": "object", + "description": "Signed environment variables applied whenever Scrollcase runs the box interpreter. These values override both the inherited host environment and caller-supplied values.", + "propertyNames": { + "minLength": 1, + "pattern": "^[^=\\u0000]+$" + }, + "additionalProperties": { + "type": "string", + "pattern": "^[^\\u0000]*$" + } + }, + "selfTest": { + "type": "object", + "additionalProperties": false, + "required": [ + "pythonImports", + "timeoutSeconds" + ], + "description": "The import check a consumer can repeat after extraction with the box's own interpreter. The builder also ran the scroll's Python-code and file assertions, which are builder-only checks.", + "properties": { + "pythonImports": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "timeoutSeconds": { + "type": "integer", + "exclusiveMinimum": 0 + } + } + }, + "execution": { + "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "weights": { + "const": "on-demand", + "description": "Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it." + }, + "assets": { + "type": "array", + "minItems": 1, + "description": "Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "url", + "relativePath", + "sizeBytes", + "sha256" + ], + "properties": { + "url": { + "type": "string", + "minLength": 1 + }, + "relativePath": { + "type": "string", + "minLength": 1 + }, + "sizeBytes": { + "type": "integer", + "exclusiveMinimum": 0 + }, + "sha256": { + "$ref": "#/$defs/sha256" + } + } + } + } + }, + "$defs": { + "identifier": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" + }, + "kind": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[.-][a-z0-9]+)*\\.release$" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "provenance": { + "type": "object", + "additionalProperties": false, + "description": "How this box was produced. Every field is recorded by the builder from observed state, never accepted from caller input, so the record cannot be dressed up after the fact.", + "required": [ + "scrollId", + "scrollVersion", + "builderRevision", + "sourceTreeDirty", + "sourceRevision", + "pythonVersion", + "dependencyLockSha256", + "builtAt", + "pixiVersion" + ], + "properties": { + "scrollId": { + "type": "string", + "minLength": 1 + }, + "scrollVersion": { + "type": "string", + "minLength": 1 + }, + "builderRevision": { + "type": "string", + "pattern": "^[a-f0-9]{40}$", + "description": "Exact commit of the builder source that produced the box." + }, + "sourceTreeDirty": { + "type": "boolean", + "description": "Whether the builder's working tree carried uncommitted changes. True means the build is not reproducible from the recorded revision alone." + }, + "sourceRevision": { + "type": "string", + "minLength": 1, + "description": "Upstream revision of the packaged model source, as declared by the scroll." + }, + "pythonVersion": { + "type": "string", + "minLength": 1 + }, + "pixiVersion": { + "type": "string", + "minLength": 1 + }, + "dependencyLockSha256": { + "$ref": "#/$defs/sha256", + "description": "Hash of the pixi.lock the environment was solved from." + }, + "builtAt": { + "type": "string", + "minLength": 1 + } + } + } + }, + "dependentRequired": { + "assets": [ + "weights" + ], + "weights": [ + "assets" + ] + } +} diff --git a/docs/public/schema/v2/revocations-manifest.schema.json b/docs/public/schema/v2/revocations-manifest.schema.json new file mode 100644 index 0000000..67470c7 --- /dev/null +++ b/docs/public/schema/v2/revocations-manifest.schema.json @@ -0,0 +1,64 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/revocations-manifest.schema.json", + "title": "Box revocations manifest", + "description": "The signed list of releases that must no longer be installed or activated. A published release is immutable, so withdrawing one is an explicit statement rather than a deletion: clients keep honouring this list even when the archive is still reachable.", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "kind", + "updatedAt", + "revocations" + ], + "properties": { + "schemaVersion": { + "const": 2 + }, + "kind": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[.-][a-z0-9]+)*\\.revocations$", + "description": "Wire discriminator, \".revocations\", carrying the same namespace as the releases it refers to." + }, + "updatedAt": { + "type": "string", + "minLength": 1 + }, + "revocations": { + "type": "array", + "description": "May be empty: an empty signed list is a positive statement that nothing is revoked, which a client can distinguish from a missing or withheld document.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "boxId", + "version", + "reason", + "revokedAt" + ], + "properties": { + "boxId": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "target": { + "$ref": "https://scrollcase.dev/schema/v2/target.schema.json", + "description": "Omitted when every target of that version is revoked." + }, + "reason": { + "type": "string", + "minLength": 1 + }, + "revokedAt": { + "type": "string", + "minLength": 1 + } + } + } + } + } +} diff --git a/docs/public/schema/v2/scroll.schema.json b/docs/public/schema/v2/scroll.schema.json new file mode 100644 index 0000000..b076c7d --- /dev/null +++ b/docs/public/schema/v2/scroll.schema.json @@ -0,0 +1,369 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/scroll.schema.json", + "title": "Box scroll", + "description": "The declarative input to a build: an identity, a target, a pinned dependency environment, the assets to fetch, and the self-test the result must pass. A scroll is checked into the consumer's repository next to its lock file; everything a build produces is derived from it.\n\nOnly what a build cannot work out for itself is required. Anything the target or the identity already determines is optional here and filled in when the scroll is read, so a hand-written scroll declares decisions rather than restating them.\n\nOne box's targets differ in a handful of lines and agree on the rest, so a scroll may also be split: scrolls//scroll.json holds what they share, and each scrolls///scroll.json declares `extends` plus its own differences. Both halves are files of this shape; the joined result is what a build reads.", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "boxId", + "modelId", + "runtimeId", + "version", + "sourceRevision", + "pythonVersion", + "selfTest", + "pixiVersion" + ], + "properties": { + "$schema": { + "const": "https://scrollcase.dev/schema/v2/scroll.schema.json", + "description": "Associates this file with the published Scrollcase v2 schema for editor validation, completion, and hover help." + }, + "extends": { + "const": "../scroll.json", + "description": "Marks this file as one target's fragment of a box whose shared declarations live in scrolls//scroll.json. The value is fixed: a base is always the box directory's own scroll.json, so there is no path to get wrong and no chain to follow. The base and the fragment are joined into one effective scroll before anything else happens, and that effective scroll is what the build reads and what provenance records." + }, + "schemaVersion": { + "const": 2, + "description": "Scrollcase wire version. Version 2 is the only active format.", + "examples": [ + 2 + ] + }, + "scrollId": { + "type": "string", + "minLength": 1, + "description": "Optional provenance identity. When omitted, Scrollcase derives it deterministically from boxId and the canonical target." + }, + "scrollVersion": { + "type": "string", + "minLength": 1, + "description": "Version of this declarative build input, recorded in provenance. Defaults to 1.0.0, which is what an authoring version means before anyone has had reason to change it.", + "default": "1.0.0", + "examples": [ + "1.0.0" + ] + }, + "boxId": { + "$ref": "#/$defs/identifier" + }, + "modelId": { + "$ref": "#/$defs/identifier" + }, + "runtimeId": { + "$ref": "#/$defs/identifier" + }, + "version": { + "type": "string", + "minLength": 1, + "description": "Version of the box this scroll produces, as it will appear in the release manifest." + }, + "sourceRevision": { + "type": "string", + "minLength": 1, + "description": "Upstream revision of the packaged source, recorded verbatim into provenance." + }, + "target": { + "$ref": "https://scrollcase.dev/schema/v2/target.schema.json", + "description": "The (platform, arch, accelerator) triple this box is built for. Required in every scroll a build reads, and absent from a base: a base holds what its targets share, so declaring one there would name a target the box does not build. Enforced when the scroll is read rather than here, so a base file still validates in an editor." + }, + "compatibility": { + "type": "object", + "additionalProperties": true, + "properties": { + "minHostAppVersion": { + "type": "string", + "minLength": 1, + "description": "Lowest version of the installing application this box supports." + }, + "maxHostAppVersionExclusive": { + "type": "string", + "minLength": 1 + }, + "minMacosVersion": { + "type": "string", + "minLength": 1 + }, + "minRamGb": { + "type": "number", + "exclusiveMinimum": 0 + }, + "minNvidiaDriverVersion": { + "type": "string", + "minLength": 1 + } + }, + "description": "Constraints the installing host must satisfy. Copied through into the release manifest verbatim and never interpreted by the builder, so a project may declare its own alongside these. Defaults to empty: declaring no constraint is a legitimate answer, and inventing one would be a claim the project never made." + }, + "pythonVersion": { + "type": "string", + "minLength": 1, + "description": "Python version solved into the box.", + "examples": [ + "3.11.15" + ] + }, + "pixiVersion": { + "type": "string", + "minLength": 1, + "description": "Pins the pixi release used to solve and install the conda-forge environment from the committed pixi.lock." + }, + "condaDependencyLicenseAudit": { + "type": "string", + "minLength": 1, + "description": "Path to the reviewed licence inventory derived from pixi.lock, which carries an SPDX licence per package. The build fails if the lock no longer matches what was reviewed." + }, + "pythonEntryPoint": { + "type": "string", + "minLength": 1, + "description": "Interpreter path relative to the box root. The target adapter's layout admits exactly one value, so this is derived from the target when omitted and still checked against it when declared." + }, + "modelCacheSubdir": { + "type": "string", + "minLength": 1, + "description": "Payload directory the box's model files live under. Defaults to model-cache/." + }, + "environment": { + "type": "object", + "description": "Environment variables the box requires when its interpreter runs. The declaration is copied into box.json and the signed release; its values override both the inherited host environment and caller-supplied values.", + "propertyNames": { + "minLength": 1, + "pattern": "^[^=\\u0000]+$" + }, + "additionalProperties": { + "type": "string", + "pattern": "^[^\\u0000]*$" + } + }, + "assetBaseUrl": { + "type": "string", + "minLength": 1, + "description": "Base URL of the mirror the built archive and its objects are published under." + }, + "assets": { + "type": "array", + "description": "Files fetched during the build. Every entry is size- and hash-checked before use, so a moved or replaced upstream file fails the build instead of silently changing the box. May be empty, and defaults to empty.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "url", + "relativePath", + "sizeBytes", + "sha256" + ], + "properties": { + "url": { + "type": "string", + "minLength": 1 + }, + "relativePath": { + "$ref": "#/$defs/payloadPath" + }, + "sizeBytes": { + "type": "integer", + "exclusiveMinimum": 0 + }, + "sha256": { + "$ref": "#/$defs/sha256" + } + } + } + }, + "assetArchives": { + "type": "array", + "description": "Downloaded archives to expand into the payload. Extraction preserves files already present in the destination and refuses to overwrite them.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "relativePath", + "format", + "destination" + ], + "properties": { + "relativePath": { + "$ref": "#/$defs/payloadPath" + }, + "format": { + "enum": [ + "zip", + "tar.gz" + ] + }, + "destination": { + "$ref": "#/$defs/payloadPath" + }, + "stripComponents": { + "type": "integer", + "minimum": 0 + }, + "removeAfterExtract": { + "type": "boolean" + } + } + } + }, + "localFiles": { + "type": "array", + "description": "Files copied from the consumer's own repository into the payload. A file already under the project's own version control needs no second copy of its identity here, and what ships is hashed into the signed release either way; declaring sha256 pins one that must not change without review, which suits a licence notice and not a script still being written.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "sourcePath", + "relativePath" + ], + "properties": { + "sourcePath": { + "type": "string", + "minLength": 1 + }, + "relativePath": { + "$ref": "#/$defs/payloadPath" + }, + "sha256": { + "$ref": "#/$defs/sha256", + "description": "Optional pin. When present the build refuses a file whose contents no longer match." + } + } + } + }, + "prunePaths": { + "type": "array", + "description": "Payload paths deleted before packing, to keep the box to what it actually needs at run time. Pruning a distribution the lock requires is rejected.", + "items": { + "$ref": "#/$defs/payloadPath" + } + }, + "uncompressedPaths": { + "type": "array", + "description": "Payload paths stored in the archive instead of deflated, because their bytes are already compressed and re-compressing them costs build time while making the archive marginally larger. A path matches itself and everything beneath it, so one entry can name a weights file or the directory an expanded asset archive landed in. Declared assets are stored automatically; this is for anything else the project knows to be already compressed.", + "items": { + "$ref": "#/$defs/payloadPath" + } + }, + "selfTest": { + "type": "object", + "additionalProperties": false, + "required": [ + "imports" + ], + "not": { + "required": [ + "pythonCode", + "pythonFile" + ] + }, + "description": "Builder checks run with the payload's own interpreter before archiving. Schema version 2 signs only the import subset for a consumer to repeat; file and optional Python-code assertions remain builder-only.", + "properties": { + "imports": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "files": { + "type": "array", + "description": "Files that must still exist after pruning, which is what stops an over-aggressive prune from shipping a broken box. Defaults to empty.", + "items": { + "$ref": "#/$defs/payloadPath" + } + }, + "pythonCode": { + "type": "string", + "minLength": 1, + "description": "Extra Python executed after the imports succeed, for checks a bare import cannot make. Anything longer than an assertion belongs in pythonFile, where an editor can see it is Python." + }, + "pythonFile": { + "type": "string", + "minLength": 1, + "description": "Project path to a Python file executed after the imports succeed, in place of pythonCode. The file is read at build time and run from the payload root, so a real self-test keeps its syntax highlighting, its linter, and its diffs instead of living inside a JSON string." + } + } + }, + "weights": { + "enum": [ + "embed", + "on-demand" + ], + "default": "embed", + "description": "Whether assets are packed into the archive (embed, the default: the box installs with no network and works air-gapped) or left out for the caller to materialize from descriptors in the signed release (on-demand). Consumers verify materialized assets before execution and do not download them. A build may override this." + }, + "execution": { + "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" + }, + "parity": { + "type": "object", + "additionalProperties": false, + "required": [ + "script", + "accelerators", + "tolerances" + ], + "description": "An optional numerical gate: run a check inside the box on more than one accelerator and require the results to agree. This catches a mis-solved environment \u2014 CPU-only wheels shipped as CUDA, a broken BLAS \u2014 on the build machine rather than on a user's. The tool runs the check and enforces the thresholds; what the check computes, and what closeness is acceptable, belong to the project.", + "properties": { + "script": { + "type": "string", + "minLength": 1, + "description": "Path inside the box, run with the box's own interpreter. It must print a JSON array of numbers, or an object with a \"values\" array." + }, + "accelerators": { + "type": "array", + "minItems": 2, + "items": { + "enum": [ + "cpu", + "metal", + "cuda" + ] + }, + "description": "Accelerators to run under, each with its target's validation environment. The first is the reference the others are compared against \u2014 conventionally cpu, being the one available everywhere." + }, + "tolerances": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "description": "At least one bound. Absolute guards entries near zero, where relative error is meaningless; cosine similarity catches a result that drifted in direction rather than magnitude.", + "properties": { + "absolute": { + "type": "number", + "exclusiveMinimum": 0 + }, + "relative": { + "type": "number", + "exclusiveMinimum": 0 + }, + "minimumCosine": { + "type": "number", + "maximum": 1 + } + } + } + } + } + }, + "$defs": { + "identifier": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "payloadPath": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*//).+$", + "description": "A forward-slash path inside the box payload: relative, non-empty, and unable to escape the payload root.", + "examples": [ + "model-cache/example-model/weights.safetensors" + ] + } + } +} diff --git a/docs/public/schema/v2/signed-document.schema.json b/docs/public/schema/v2/signed-document.schema.json new file mode 100644 index 0000000..602af0c --- /dev/null +++ b/docs/public/schema/v2/signed-document.schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/signed-document.schema.json", + "title": "Signed box document", + "description": "The envelope wrapping every signed document. The payload travels as exact base64-encoded JSON so that verifying a signature means hashing the bytes as transmitted, with no canonical-JSON implementation to keep in sync across languages. Passing this schema means the envelope is well-formed, never that its signature is valid.", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "payloadEncoding", "payloadBase64", "payloadSha256", "signatures"], + "properties": { + "schemaVersion": { "const": 2 }, + "payloadEncoding": { "const": "base64-json-utf8" }, + "payloadBase64": { + "type": "string", + "minLength": 1, + "description": "The document payload: UTF-8 JSON, base64-encoded, signed and hashed exactly as it appears here." + }, + "payloadSha256": { + "$ref": "#/$defs/sha256", + "description": "SHA-256 of the decoded payload bytes." + }, + "signatures": { + "type": "array", + "minItems": 1, + "description": "Detached signatures over the decoded payload bytes. A verifier accepts the document when any one signature verifies against a trusted key, which is what allows a key to be rotated without reissuing every document.", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["algorithm", "keyId", "signatureBase64"], + "properties": { + "algorithm": { "const": "ed25519" }, + "keyId": { "type": "string", "minLength": 1 }, + "signatureBase64": { "type": "string", "minLength": 1 } + } + } + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + } +} diff --git a/docs/public/schema/v2/target.schema.json b/docs/public/schema/v2/target.schema.json new file mode 100644 index 0000000..6894c12 --- /dev/null +++ b/docs/public/schema/v2/target.schema.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/target.schema.json", + "title": "Box target", + "description": "The platform, architecture and accelerator a box is built for. The supported combinations are closed: a target outside this matrix has no defined identifier and cannot be built, signed, or routed.", + "type": "object", + "additionalProperties": false, + "required": ["platform", "arch", "accelerator"], + "properties": { + "platform": { + "enum": ["macos", "linux", "windows"], + "description": "Operating system the box runs on.", + "examples": ["linux"] + }, + "arch": { + "enum": ["aarch64", "x86_64"], + "description": "CPU architecture the box runs on; supported combinations are constrained below.", + "examples": ["x86_64"] + }, + "accelerator": { + "enum": ["cpu", "metal", "cuda"], + "description": "Acceleration backend built into the environment.", + "default": "cpu", + "examples": ["cpu"] + }, + "cudaVersion": { + "type": "string", + "pattern": "^[1-9][0-9]*\\.[0-9]+$", + "description": "CUDA ABI as major.minor, for example \"12.8\". Required for a CUDA target and forbidden for any other, so an identifier can never be ambiguous." + } + }, + "allOf": [ + { + "if": { "properties": { "accelerator": { "const": "cuda" } }, "required": ["accelerator"] }, + "then": { "required": ["cudaVersion"] }, + "else": { "not": { "required": ["cudaVersion"] } } + }, + { + "if": { "properties": { "platform": { "const": "macos" } }, "required": ["platform"] }, + "then": { + "properties": { + "arch": { "const": "aarch64" }, + "accelerator": { "enum": ["metal", "cpu"] } + } + } + }, + { + "if": { "properties": { "platform": { "enum": ["linux", "windows"] } }, "required": ["platform"] }, + "then": { + "properties": { + "arch": { "const": "x86_64" }, + "accelerator": { "enum": ["cpu", "cuda"] } + } + } + } + ] +} diff --git a/docs/public/schema/v3/channel-manifest.schema.json b/docs/public/schema/v3/channel-manifest.schema.json index 0e04054..7050fac 100644 --- a/docs/public/schema/v3/channel-manifest.schema.json +++ b/docs/public/schema/v3/channel-manifest.schema.json @@ -56,7 +56,6 @@ "additionalProperties": false, "required": [ "version", - "releaseManifestUrl", "rolloutPercentage" ], "properties": { @@ -66,7 +65,8 @@ }, "releaseManifestUrl": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "Where the signed release document is published. Absent when the box was built without a publish base URL: there is then nothing for this pointer to point at, and a channel that still says which version is current is more use than one carrying an address that does not resolve." }, "rolloutPercentage": { "type": "integer", diff --git a/docs/public/schema/v3/release-manifest.schema.json b/docs/public/schema/v3/release-manifest.schema.json index 24df66c..7c4abae 100644 --- a/docs/public/schema/v3/release-manifest.schema.json +++ b/docs/public/schema/v3/release-manifest.schema.json @@ -84,7 +84,6 @@ "additionalProperties": false, "required": [ "format", - "url", "sha256", "sizeBytes" ], @@ -94,7 +93,8 @@ }, "url": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "Where the archive is published, for the distribution layer that has to fetch it. Absent when the box was built without a publish base URL, which is what a box built to run locally is.\n\nNothing verifies this value and no Scrollcase consumer reads one: an archive is resolved beside its release document and identified by sha256, so a wrong URL here would break a download and no check at all. That is why an absent one is preferred to an invented one \u2014 a false address inside a signed, immutable document stays false forever." }, "sha256": { "$ref": "#/$defs/sha256" diff --git a/docs/public/schema/v3/scroll.schema.json b/docs/public/schema/v3/scroll.schema.json index 74a6464..819ef0e 100644 --- a/docs/public/schema/v3/scroll.schema.json +++ b/docs/public/schema/v3/scroll.schema.json @@ -127,10 +127,13 @@ "pattern": "^[^\\u0000]*$" } }, - "assetBaseUrl": { + "publishBaseUrl": { "type": "string", "minLength": 1, - "description": "Base URL of the mirror the built archive and its objects are published under." + "description": "Base URL the built archive and its signed documents will be published under, so each can point at the next: the channel names the release document, and the release names the archive. It says nothing about the box's own assets \u2014 those carry a URL each \u2014 and nothing about what the box does at run time.\n\nOptional, and genuinely so. A box you build to run locally is never published, so there is nowhere for these documents to point and no value here would be true; the build omits both links rather than inventing an address. Nothing verifies this URL and no Scrollcase consumer reads one: an archive is found beside its release document and identified by its SHA-256.", + "examples": [ + "https://boxes.example.org" + ] }, "assets": { "type": "array", diff --git a/docs/reference/api/index.md b/docs/reference/api/index.md new file mode 100644 index 0000000..3b97c22 --- /dev/null +++ b/docs/reference/api/index.md @@ -0,0 +1,59 @@ +--- +title: Scrollcase APIs +description: The Node, Python, and Rust surfaces for contracts, build primitives, signing and running boxes.. +--- + +# Scrollcase APIs + +The CLI is the supported way to run the build pipeline, but Scrollcase also provides Node, Python, and Rust surfaces for contracts, local consumers, build primitives, and signing. + +::: info The pipeline verbs are CLI-only +`build`, `verify`, `audit`, `lock`, `init`, `new scroll`, and `doctor` are not part of the exported surface. +They orchestrate a process — spawning pixi, writing a workspace, exiting non-zero — and are +driven through `scrollcase `. What is exported is what a *consumer* of boxes needs. +::: + +## The local consumers + +Three implementations of one contract for Node, Python, and Rust. They have the same verification, extraction, execution, +receipt and error semantics, and are held to it by shared conformance fixtures — so the choice is +which language your application is written in, not which behaviour you get. + +## Stability + +The exported surface follows the package version, and each consumer distribution — the npm package, +the PyPI package, the crate — carries its own. The active v3 **format** — target IDs, document +kinds, payload encoding, and signature algorithm — changes only through an explicit new schema +version. The v3 API rejects v1 and v2 by name, each with its own remedy, rather than widening its +types or runtime paths into a compatibility union. + +## APIs + +
+ + + + + + + + +
diff --git a/docs/reference/api/node.md b/docs/reference/api/node.md new file mode 100644 index 0000000..bfeaf57 --- /dev/null +++ b/docs/reference/api/node.md @@ -0,0 +1,424 @@ +--- +title: Node API +description: The Node surface surface for contracts, build primitives, signing and running boxes. +--- + +# Node API + +The CLI is the supported way to run the build pipeline. The Node package additionally exports five +modules for clients that need to understand, prepare, or execute local boxes: validate a document, +derive a target ID, check a signature, resolve a workspace, or run a verified application. + +```js +import { boxTargetId, documentKinds } from 'scrollcase/contract'; +import { isSignedBoxDocument } from 'scrollcase/contract/browser'; +import { sha256File, resolveWorkspace } from 'scrollcase/build'; +import { + verifyAndExtractBox, attachExtractedBox, verifyExtractedPayload, runExtractedBox, runBox, +} from 'scrollcase/consumer'; +import { verifySignedDocument } from 'scrollcase/sign'; +``` + +The JSON Schemas and golden fixtures are exported as files too: + +```js +import scrollSchema from 'scrollcase/contract/schema/scroll.schema.json' with { type: 'json' }; +import targetCases from 'scrollcase/contract/fixtures/target-id-contract.json' with { type: 'json' }; +``` + +## TypeScript types + +The box format's types are **generated from the JSON Schemas** and shipped with the package: + +```ts +import type { + BoxTarget, + BoxScroll, + BoxManifest, + BoxReleaseManifest, + BoxChannelManifest, + BoxRevocationsManifest, + SignedBoxDocument, +} from 'scrollcase/contract/types'; + +const target: BoxTarget = { + platform: 'linux', arch: 'x86_64', accelerator: 'cuda', cudaVersion: '12.4', +}; +``` + +The schemas are the source of truth; these types are a projection of them, never a second +definition. A schema change that is not accompanied by `npm run types` fails the test suite, so +the two cannot drift — the same discipline that makes the licence audit a function of the lock. + +This subpath is **types only**: there is nothing to import at runtime, so use `import type`. +`scrollcase/contract`, `scrollcase/contract/browser`, `scrollcase/build`, +`scrollcase/consumer`, and `scrollcase/sign` also ship declarations generated from the typed JSDoc +beside their JavaScript implementations. Strict TypeScript consumers therefore get checked +parameters, return values, narrowing guards, hover documentation, and completion without a build +step or a separate types package. `npm run types:check` fails if either the schema-derived format +types or the runtime declarations drift from their source. + +## scrollcase/consumer + +`scrollcase/consumer` prepares and executes release documents and archives already present on the local +machine. Every path and trust anchor comes from the caller. It never selects a channel, downloads an +archive or asset, installs globally, updates an existing destination, or applies application +lifecycle policy. + +```js +import { + attachExtractedBox, + verifyAndExtractBox, + verifyExtractedPayload, + runExtractedBox, + runBox, +} from 'scrollcase/consumer'; + +const prepared = await verifyAndExtractBox('release.json', { + publicPath: 'trusted-keys.json', + archive: 'box.zip', + destination: '/srv/boxes/example-1.0.0', +}); + +const result = await runExtractedBox(prepared, { + args: ['--port', '8080'], + env: { APPLICATION_MODE: 'local' }, + stdin: 'ignore', + stdout: 'inherit', + stderr: 'inherit', +}); +``` + +### Preparation + +`verifyAndExtractBox(releaseDocumentPath, { publicPath, archive, destination })` verifies the signed +document against a single trusted-key file or key bundle, validates the v2 release, checks archive +size and SHA-256, rejects unsafe ZIP entries, extracts through the shared safe extractor, compares +`box.json` recursively with the signed release, checks logical installed size and execution +prerequisites, then atomically renames a fresh staging tree into `destination`. The destination must +not exist. + +It returns an immutable `PreparedBox` receipt with signed identity, target, execution, archive and +signing information, plus `environmentReport`, a masked diagnostic snapshot of this process's host +environment resolved against the signed declaration. The receipt is process-bound: +`runExtractedBox` rejects copied or constructed +lookalikes, and also rejects a prepared root replaced after verification. + +For `on-demand` weights, `prepared.requiredAssets` contains the signed URL, relative path, size and +SHA-256 descriptors. Scrollcase does not fetch them. The caller may materialize those files under +`prepared.root`; execution refuses a missing, non-regular, wrong-size, or wrong-hash asset. + +### Re-attaching across restarts + +A `PreparedBox` is bound to the process that produced it, so a long-lived application cannot keep +one across a restart — and re-extracting gigabytes at every launch is not an answer. +`attachExtractedBox(releaseDocumentPath, { publicPath, root })` mints a fresh receipt from a +directory that is already extracted, without the archive. + +It verifies the signed document, requires a target this host can run, checks that the interpreter +and execution files are present, and re-checks on-demand assets against their signed hashes. It does +not read original payload file contents, though it enumerates paths and measures their metadata, so +its original-payload work scales with entry count rather than byte size. Required on-demand assets +are hashed in full. The receipt it returns carries +`status: 'attached'` rather than `'prepared'`, because the bytes on disk were not proved — only the +release, and the shape of the directory. Its `environmentReport` is produced by the same resolver as +preparation. `runExtractedBox` accepts either. + +`root` must be a real directory; a symbolic link is refused, since execution requires a real one. + +### Verifying an installation + +`verifyExtractedPayload(releaseDocumentPath, { publicPath, root })` proves the tree on disk is the +one the release describes. New boxes carry `payload-digest.v1`, an entry list naming each original +payload path with the SHA-256 of its content, and the signed release commits to that list's own hash. +Verification hashes the list, compares it with the release, parses it only then, and checks each +listed path — walking the list, never the directory, so files that appear after installation +(`__pycache__`, the model cache, anything the application writes in its working directory) are +simply never visited. + +It is standalone and opt-in: no other operation calls it, because it reads every listed byte. Call +it after installing, on a user's request, or in a maintenance job. Embedded weights are listed and +can make the check read tens of gigabytes; on-demand assets are later extras and keep their separate +signed per-file verification. File mode and modification time are deliberately not committed. A +release built before this field existed is refused rather than silently treated as verified. +The returned `PayloadVerification` also carries `environmentReport`; it describes the inspecting +process, not the payload bytes that were just checked. + +::: warning What it does and does not prove +It binds a directory to a signed release and detects corruption. It is not a defence against a local +attacker: the tree can change between this call and any later import, and no library can close that +window — filesystem permissions can, and they belong to the operating system and to your +application. Scrollcase does not guard the directory afterwards. + +The build collector excludes `__pycache__` directories and `*.pyc` files, so the digest cannot make +any assertion about them. That is a permanent blind spot, not only a timing window. +::: + +### Execution + +`runExtractedBox(prepared, options)` runs only a receipt returned by +`verifyAndExtractBox` or `attachExtractedBox` in the current process. It rechecks the prepared tree and required assets, +enforces the native target, starts the declared script or `-m` module with the box's own Python, +uses the box root as `cwd`, and appends caller `args` after signed `defaultArgs`. It never invokes a +shell. + +`stdin`, `stdout`, and `stderr` accept Node child-process stdio values or streams. Environment +precedence is inherited host, then caller `env`, then signed release `environment`; later layers win +without filtering any inherited name. `SIGINT`, `SIGTERM`, and `SIGHUP` are forwarded while the +child is alive. The returned `{ exitCode, signal, environmentReport }` preserves the child's +terminal result and the exact diagnostic used for that spawn. + +`runBox(releaseDocumentPath, options)` composes preparation and execution in a private temporary +directory and guarantees cleanup after a normal exit, non-zero exit, spawn failure, or forwarded +signal. `temporaryDirectory` selects the parent for that private root; `onPrepared` is an optional +callback invoked after verification and extraction but before execution, which lets a CLI display +the signed identity without reimplementing or repeating the trust chain: + +```js +const result = await runBox('release.json', { + publicPath: 'trusted-keys.json', + archive: 'box.zip', + args: ['--once'], + onPrepared: ({ boxId, version, targetId }) => { + console.log(`Running ${boxId} ${version} (${targetId})`); + }, +}); +process.exitCode = result.exitCode ?? 1; +``` + +### Environment reports + +Every preparation, attachment, payload verification, and run result includes a structured report. +The compact default contains every release-declared variable, every inherited variable the target +adapter identifies as capable of changing executed code, and every conflict, plus +`remainingVariableCount`. A variable records its winning `source`, visible winning `value`, whether +it is `executionAffecting`, and all `sources` in precedence order. Release values are visible because +they are already public in the signed document; caller values are visible too. Only inherited host +values are `""` by default, so a caller must not log a report containing secrets it supplied +through `env`. + +The Node report fields are: + +| Field | Meaning | +| --- | --- | +| `mode` | `"summary"` for the compact selection, or `"full"` after expansion | +| `hostValuesRevealed` | Whether inherited host values are visible | +| `releaseVariableCount` | Number of names supplied by the signed declaration | +| `conflictCount` | Number of names whose sources supplied different values | +| `dangerousHostVariables` | Present inherited names the target adapter identifies as capable of changing executed code | +| `remainingVariableCount` | Resolved names omitted from `variables` in compact mode | +| `variables` | Selected variable reports, sorted by winning name | + +Each variable has `name`, winning `source`, visible winning `value`, `executionAffecting`, +`conflict`, and `sources`. Each source entry records its `source`, exact `name` spelling, and visible +`value`. Sources are `host`, `caller`, `release`, and — during a verification self-test — +`validation`; later entries have higher precedence. Python exposes the same fields in snake case. + +Pass `envReport: true` to any consumer operation to include every resolved variable name. Pass +`envReportValues: true` to imply the full report and reveal host values deliberately. Python uses +`env_report` and `env_report_values`. Run operations also accept `onEnvironmentReport` / +`on_environment_report`, called after resolution and before spawning. + +```js +const result = await runExtractedBox(prepared, { + envReport: true, + onEnvironmentReport(report) { + logger.info({ environment: report }); + }, +}); +``` + +::: warning Diagnostic, not guarantee +The `environment` declaration is signed format data and every verifier checks agreement. An +`environmentReport` is local consumer output: it changes with the host, caller values, flags, and +time of execution. A caller that starts `venv/bin/python` directly gets neither resolution nor a +report. Do not describe the report as a property guaranteed by the box. +::: + + +## `scrollcase/contract` + +The single source of truth for what a box is. See [The Box Format](/reference/box-format). + +### Targets + +| Export | Signature | Purpose | +| --- | --- | --- | +| `boxTargetId` | `(target) => string` | The canonical slug (`linux-x86_64-cuda12.4`). Throws `TypeError` on an unsupported or ambiguous target | +| `boxTargetAdapter` | `(target) => Adapter` | The adapter for a validated target: Python layout, archive backend, native-library inspection, validation environments | +| `boxTargetAdapters` | `() => Adapter[]` | Every adapter, for enumerating supported targets | +| `condaSubdir` | `(target) => string` | The conda platform subdir (`osx-arm64`, `linux-64`, `win-64`) | +| `pixiAccelerator` | `(scroll) => { accelerator, cudaVersion }` | The conda accelerator descriptor a scroll selects, rejecting target drift | +| `assertNativeHost` | `(adapter, host = process) => void` | Throws unless the current host matches the adapter's OS and architecture | +| `assertRuntimeEntryPoint` | `(runtimeId, adapter, entryPoint) => void` | Throws unless the entry point matches that runtime's layout for the target | +| `RUNTIME_IDS` | `readonly string[]` | Every runtime id the format defines: `python`, `node`, `native`. A separate list from what a given build implements, on purpose — the consumers version independently | +| `runtimeAdapter` | `(runtimeId) => BoxRuntimeAdapter` | The runtime's layout, execution kinds, argv rule and self-test rule. Throws for a runtime with no adapter | +| `runtimeAdapters` | `() => BoxRuntimeAdapter[]` | Every runtime this build implements | +| `isImplementedRuntime` | `(runtimeId) => boolean` | Whether an adapter exists — the question to ask before `runtimeAdapter` | +| `unimplementedRuntimeMessage` | `(runtimeId) => string` | One wording for a box naming a runtime this build cannot run, so the builder and all three consumers report it identically | +| `unsupportedSelfTestProbeMessage` | `(runtimeId, probeKind) => string` | One wording for a probe shape the runtime cannot answer — `selfTest.imports` in a `native` box, which has no module system | +| `executionAffectingVariables` | `(runtimeId, adapter) => readonly string[]` | Inherited variables that can change what a box executes: the runtime's loader controls, then the OS's | +| `isExecutablePayloadPath` | `(rule, relativePath) => boolean` | Whether a payload path is one the runtime requires the executable bit on | + +```js +import { boxTargetId } from 'scrollcase/contract'; + +boxTargetId({ platform: 'linux', arch: 'x86_64', accelerator: 'cuda', cudaVersion: '12.4' }); +// → 'linux-x86_64-cuda12.4' +``` + +### Documents + +| Export | Signature | Purpose | +| --- | --- | --- | +| `documentKinds` | `(namespace = 'scrollcase.box') => { release, channel, revocations }` | The `kind` discriminators under a namespace. Throws on an invalid namespace | +| `parseDocumentKind` | `(kind) => { namespace, type } \| null` | Splits a `kind` back apart | +| `isSignedBoxDocument` | `(value) => boolean` | Envelope shape check. **Says the document is worth verifying, never that it is valid** | +| `decodeDocumentPayload` | `(document) => object` | Decodes the payload and checks its embedded hash. **Does not verify signatures** | +| `schemaUrl` | `(name) => URL` | Absolute URL of a shipped JSON Schema | +| `fixtureUrl` | `(name) => URL` | Absolute URL of a shipped fixture | + +Constants: `BOX_SCHEMA_VERSION` (`2`), `PAYLOAD_ENCODING` (`'base64-json-utf8'`), +`SIGNATURE_ALGORITHM` (`'ed25519'`), `DEFAULT_DOCUMENT_NAMESPACE` (`'scrollcase.box'`), +`CHANNELS` (`['nightly', 'beta', 'stable']`). + +## `scrollcase/contract/browser` + +The platform-neutral subset of the contract for browsers, Workers, and Node. It exports the target +helpers plus document constants, namespacing helpers, and `isSignedBoxDocument`. Its complete module +graph contains no Node built-ins. + +```js +import { + boxTargetId, + isSignedBoxDocument, +} from 'scrollcase/contract/browser'; +``` + +The full `scrollcase/contract` entry point remains the Node surface and additionally exports +`decodeDocumentPayload`, `schemaUrl`, and `fixtureUrl`. Cryptographic verification remains under +`scrollcase/sign`; the browser guard checks envelope shape only and never establishes trust. + +::: warning Decoding is not verifying +`decodeDocumentPayload` catches a truncated or edited document, because the payload hash must +match the bytes. It says nothing about *who* produced them. Anything acted upon must first pass +`verifySignedDocument` against a trusted key. +::: + +### Proving a mirror implementation + +A client in another language mirrors these rules and validates the mirror against the fixtures: + +```js +import { boxTargetId } from 'scrollcase/contract'; +import cases from 'scrollcase/contract/fixtures/target-id-contract.json' with { type: 'json' }; + +for (const { target, targetId } of cases.valid) { + if (boxTargetId(target) !== targetId) throw new Error(`mismatch for ${targetId}`); +} +for (const { target } of cases.invalid) { + let rejected = false; + try { + boxTargetId(target); + } catch { + rejected = true; + } + if (!rejected) throw new Error(`invalid target was accepted: ${JSON.stringify(target)}`); +} +``` + +## `scrollcase/sign` + +| Export | Signature | Purpose | +| --- | --- | --- | +| `generateSigningKey` | `({ privatePath, publicPath, keyId, force }) => Promise<{ keyId, privatePath, publicPath }>` | What `keygen` runs. Refuses to overwrite without `force` | +| `readSigningKey` | `({ privatePath, publicPath }) => Promise<{ privateKey, metadata }>` | Loads the private key and cross-checks it against the published public key | +| `signDocument` | `(payload, { signerCommand, privatePath, publicPath, runResult }) => Promise` | Wraps a payload in the signed envelope, locally or through an external signer; `runResult` is an optional process seam | +| `verifySignedDocument` | `(document, trust) => Promise` | Verifies against a trusted key file path or an array of keys, and returns the payload. Throws otherwise | +| `parseTrustedKeys` | `(source) => TrustedKey[]` | Reads both trust-file shapes from text or bytes a caller already holds, rather than from a path | +| `resolveTrustedKeys` | `({ publicPath, trustedKeys }) => Promise` | Resolves exactly one named trust source into the keys verification runs against | +| `decodeSignedDocument` | `(document) => { bytes, payload }` | Unwraps and checks the payload hash. Does **not** check the signature | + +The trusted key file is either a single key object or a `{ "keys": [...] }` bundle; a document is +accepted when any one of its signatures verifies. Every consumer operation that verifies a signed +release takes `publicPath` **or** `trustedKeys`, exactly one: an application holding its keys in a +keyring, an environment variable or a secrets manager should not have to write them to disk to +verify a signature. See [Signing & Key Custody](/guides/signing-and-custody). + +## `scrollcase/build` + +Build primitives. Useful for tooling around Scrollcase — a CI check, a custom staging step, a +client that computes the same hashes. + +### Workspace + +| Export | Purpose | +| --- | --- | +| `resolveWorkspace({ cwd, overrides })` | Resolve the absolute layout without installing it | +| `configureWorkspace({ cwd, overrides })` | Resolve and install it for this process | +| `getWorkspace()` | The installed workspace, resolving defaults on first use | +| `findWorkspaceConfig(startDir)` | Walk up looking for `scrollcase.config.json` | +| `workspaceOverridesFromFlags(flags)` / `workspaceOverridesFromArgv(argv)` | Collect workspace overrides from a parsed flag map, or raw arguments | +| `DEFAULT_WORKSPACE_PATHS`, `SCROLLCASE_CONFIG_FILENAME` | The defaults and the filename | + +```js +import { resolveWorkspace } from 'scrollcase/build'; + +const workspace = resolveWorkspace({ cwd: '/work/my-project/scrolls/my-model/macos-aarch64-metal' }); +// → { root, configPath, scrollsDir, buildDir, distDir, keysDir, toolchainDir } +``` + +Details in [Workspace Configuration](/reference/configuration). + +### Archives and filesystem + +| Export | Purpose | +| --- | --- | +| `createDeterministicZip(payloadDir, archivePath, adapter)` | Write a box archive: fixed timestamps, stable ordering, adapter-derived modes | +| `extractZipArchive(archivePath, destination)` | Extract with entry-name validation; rejects traversal, links and special entries | +| `listZipEntries(archivePath)` | Enumerate entries without extracting | +| `collectFiles(root)` | Enumerate files in the one stable order hashing and archiving rely on | +| `sha256File(path)`, `fileExists(path)` | Hashing and existence checks | +| `payloadDigest(root)` | Reduce an extracted tree to the `{ format, sha256 }` a release commits to | + +### Identity and toolchain + +| Export | Purpose | +| --- | --- | +| `boxReleaseStem(release)` | `--` | +| `boxReleaseObjectPrefix(release)` | `boxes///` | +| `builderVersionFields(source)` | The builder-identity fields recorded in provenance | +| `findPixi({ requiredVersion, path, runResult })` | Locate pixi and enforce the scroll's pin | +| `findCondaPack({ path, runResult })` | Locate conda-pack | +| `CONDA_PACK_VERSION` | The exact conda-pack release installed by Scrollcase (`0.9.2`) | +| `pixiLockArguments`, `pixiInstallArguments`, `condaPackArguments` | The exact argument vectors the build uses | +| `installAndPackPixiEnvironment({ … })` | Install from the lock, pack, relocate into `venv/` | +| `repairPosixLaunchers(adapter, payloadDir, forbiddenPaths)` | Rewrite console scripts to resolve Python next to themselves | + +### Licence audit + +| Export | Purpose | +| --- | --- | +| `createCondaDependencyLicenseAudit({ lockBytes, targetId, namespace })` | The inventory, derived from a `pixi.lock` | +| `validateCondaDependencyLicenseAudit(reviewed, actual)` | Throw unless a reviewed audit still matches the lock exactly | +| `lockedCondaDistributions(lockBytes)` | The parsed distributions with their declared licences | +| `parseCondaPackageReference(url)` | `{ name, version }` from a conda package filename | + +```js +import { readFile } from 'node:fs/promises'; +import { createCondaDependencyLicenseAudit } from 'scrollcase/build'; + +const audit = createCondaDependencyLicenseAudit({ + lockBytes: await readFile('scrolls/my-model/macos-aarch64-metal/pixi.lock'), + targetId: 'macos-aarch64-metal', +}); +// → { schemaVersion, kind, targetId, dependencyLockSha256, packages: [...] } +``` + +A package without a declared licence throws rather than being reported as unknown. + +### Process + +`fail(message)` throws the single error shape the CLI turns into a one-line non-zero exit; `run` +and `runResult` are the process runners the build injects, which is how the test suite drives the +pipeline without a real toolchain. diff --git a/docs/reference/api/python.md b/docs/reference/api/python.md new file mode 100644 index 0000000..0b1ab8b --- /dev/null +++ b/docs/reference/api/python.md @@ -0,0 +1,59 @@ +--- +title: Python consumer +description: The Python surface for contracts, build primitives, signing and running boxes. +--- + +# Python consumer + +`scrollcase_consumer` mirrors the local Node consumer without depending on Node or its CLI: + +```sh +python -m pip install scrollcase-consumer +``` + +```python +from scrollcase_consumer import ( + attach_extracted_box, + run_box, + run_extracted_box, + verify_and_extract_box, + verify_extracted_payload, +) + +prepared = verify_and_extract_box( + "release.json", + public_key_path="trusted-keys.json", + archive="box.zip", + destination="/srv/boxes/example-1.0.0", +) + +result = run_extracted_box( + prepared, + args=("--port", "8080"), + env={"APPLICATION_MODE": "local"}, +) +``` + +The receipt fields use idiomatic snake case (`box_id`, `target_id`, `required_assets`, +`archive_sha256`, `environment_report`). `attach_extracted_box(release, public_key_path=…, root=…)` and +`verify_extracted_payload(release, public_key_path=…, root=…)` mirror their Node counterparts +exactly, including the `attached` status and the refusal of a release that commits to no payload +digest. `run_box` performs the same one-shot prepare/run/cleanup composition. Stream +arguments accept Python file objects or `subprocess` constants; the default inherits the parent's +streams. On the main Python thread, `SIGINT`, `SIGTERM`, and `SIGHUP` are forwarded and then the +previous handlers are restored. + +`EnvironmentReport`, `EnvironmentVariableReport`, and `EnvironmentSourceValue` are immutable public +models. Their fields mirror the Node structure in snake case; `BoxRunResult` and every verification +receipt include one. + +Every operation that verifies a signed release takes `public_key_path` **or** `trusted_keys`, exactly +one, and `parse_trusted_keys(source)` reads both trust-file shapes from text or bytes — so an application +holding its keys in a keyring, an environment variable or a secrets manager verifies against them +directly instead of writing key material to a file first. Naming both sources or neither raises a +`ScrollcaseConsumerError`; the Rust `TrustAnchors` enum makes those two invalid states +unrepresentable instead. + +The distribution is not a downloader: callers still supply local release, archive, trust-key, +destination, and on-demand asset paths. It verifies Ed25519 signatures with `cryptography` and +validates bundled, generated copies of the canonical schemas. diff --git a/docs/reference/api/rust.md b/docs/reference/api/rust.md new file mode 100644 index 0000000..4bd4c40 --- /dev/null +++ b/docs/reference/api/rust.md @@ -0,0 +1,91 @@ +--- +title: Rust consumer +description: The Rust surface for contracts, build primitives, signing and running boxes. +--- + +# Rust consumer + +The `scrollcase-consumer` crate mirrors the same local consumer for applications — a Tauri desktop client, a native +service — that would otherwise have to embed a second runtime just to start a box: + +```sh +cargo add scrollcase-consumer +``` + +```rust +use std::path::Path; + +use scrollcase_consumer::prepare::{verify_and_extract_box, PrepareOptions}; +use scrollcase_consumer::run::{run_extracted_box, RunOptions}; +use scrollcase_consumer::trust::TrustAnchors; + +let prepared = verify_and_extract_box( + Path::new("release.json"), + &PrepareOptions { + trust: TrustAnchors::KeyFile(Path::new("trusted-keys.json")), + archive: Some(Path::new("box.zip")), + destination: Path::new("/srv/boxes/example-1.0.0"), + environment: Default::default(), + }, +)?; + +let result = run_extracted_box( + &prepared, + &RunOptions { + args: vec!["--port".into(), "8080".into()], + env: vec![("APPLICATION_MODE".into(), "local".into())], + ..Default::default() + }, +)?; +``` + +### Where the trusted keys come from + +Every entry point that verifies a signed release takes a `TrustAnchors`, not a path, because the two +sources are not equivalent security decisions. `TrustAnchors::KeyFile` reads a trust file at the +moment of verification, which suits a command line whose operator is also its administrator. +`TrustAnchors::Keys` verifies against +keys the caller already holds — and an application shipped to someone else's machine usually wants +exactly that, because a trust file sitting beside the application can be edited, and whoever edits it +decides which boxes the application will accept: + +```rust +use scrollcase_consumer::trust::{parse_trusted_keys, TrustAnchors}; + +// Compiled in, so substituting a key means rebuilding the application rather than editing a file. +static ANCHORS: &str = include_str!("../anchors/production.json"); + +let keys = parse_trusted_keys(ANCHORS.as_bytes())?; +let trust = TrustAnchors::Keys(&keys); +``` + +`parse_trusted_keys` accepts the same two shapes a trust file holds — a single key object, or a +`{ "keys": [...] }` bundle — so an embedded bundle is read by the crate rather than by a second +parser at the call site. Prefer the bundle shape: keys compiled into an application can only be +rotated by releasing the application, and a bundle lets the outgoing and incoming keys both be +trusted while that release makes its way out. Verification is unchanged either way, a document being +accepted when any one of its signatures verifies against any trusted key. + +The trust-file grammar is the same in all three implementations. Every entry needs a string +`keyId`; `publicKeyPem` may be absent or `null`, and otherwise must be a string. An empty bundle is +structurally valid but cannot verify a signature. Malformed JSON, bundle shapes or entries fail as +`Invalid trusted ed25519 key file.`; a syntactically valid but unusable PEM is skipped and therefore +reaches the common `Document has no valid signature from a trusted ed25519 key.` refusal. Node and +Python also reject malformed directly supplied key lists as `Invalid trusted ed25519 keys.`; Rust's +`Vec` makes the corresponding field-type errors unrepresentable. A trust file that +cannot be read uses the same `Invalid trusted ed25519 key file` prefix and includes the path and I/O +detail. + +`attach_extracted_box` and `verify_extracted_payload` behave exactly as their Node and Python +counterparts, including the `attached` status and the refusal of a release that commits to no +payload digest; `run_box` performs the same one-shot prepare/run/cleanup composition. The receipt +fields are accessor methods (`prepared.box_id()`, `prepared.target_id()`, +`prepared.environment_report()`) rather than public fields, because `PreparedBox` has no public +constructor: the rule that verification precedes execution is carried by the type system, so a +caller cannot assemble one without having verified a box. + +Everything is synchronous and needs no async runtime. Signals are forwarded from a channel the +caller owns rather than through process-wide handlers, which a library embedded in someone else's +application has no business installing. The crate forbids `unsafe`, and the modules are the same +concerns as the other two consumers: `contract`, `trust`, `release`, `archive`, `filesystem`, +`execution`, `environment`, `verify`, `prepare`, `run`. diff --git a/docs/reference/box-format.md b/docs/reference/box-format.md index 642a79a..a383757 100644 --- a/docs/reference/box-format.md +++ b/docs/reference/box-format.md @@ -69,15 +69,26 @@ A box ships as a ZIP (ZIP64-capable) whose bytes depend only on its contents: example-model-1.0.0-macos-aarch64-metal.zip ├── box.json # the self-describing manifest ├── payload-digest.v1 # canonical hashes of every original payload entry +├── package.json # node boxes only, unless the payload ships its own ├── venv/ # the packed, relocated conda-forge environment -│ ├── bin/python # (venv/python.exe on Windows) +│ ├── bin/python # the runtime's entry point: venv/bin/node for a node +│ │ # box, none at all for a native one +│ │ # (venv/python.exe, venv/node.exe on Windows) │ ├── lib/… │ └── conda-meta/… ├── cache/… # the box's own large files, when embedded └── THIRD_PARTY_NOTICES/ - └── conda-distributions.json # the dependency licence inventory + ├── conda-distributions.json # the lock-derived dependency licence inventory + └── bundled-dependencies.json # what was linked inside a binary the box ships, + # when the scroll declared it ``` +The runtime decides two of those entries. A `node` box is given its own `package.json` unless the +payload already carries one, because Node picks CommonJS or ESM from the nearest `package.json` +*above* the file it runs and a box without one asks whichever directory it was extracted into. A +`native` box has no interpreter under `venv/` to name at all: the binary it runs is one the scroll +brought in, and `venv/` holds only the shared libraries that binary links against. + Guarantees the archive layer enforces: - **Deterministic.** Fixed timestamps (`2000-01-01T00:00:00Z`), stable file ordering, and modes @@ -323,11 +334,11 @@ so promoting a build never requires re-signing it. ``` Channels are `nightly`, `beta`, and `stable`, and a fresh build emits one release at 100%. -Schema version 2 carries `cohortSalt` and rollout percentages but intentionally lacks a normative +Schema version 3 carries `cohortSalt` and rollout percentages but intentionally lacks a normative cohort algorithm and golden fixtures. It does not specify identity normalisation, byte framing, hashing, integer extraction, percentage mapping, ordering, or boundary behavior. A project can define those rules for its own clients, but cross-implementation rollout interoperability is not a -schema-v2 guarantee. +schema-v3 guarantee. ### Revocations manifest diff --git a/docs/reference/cli.md b/docs/reference/cli.md index f5d767b..c27dfbe 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -103,9 +103,16 @@ product flags passed to `init` are rejected with the same remedy instead of bein ### The toolchain step -With neither flag and a terminal attached, `init` prompts, defaulting to **yes**. Without a -terminal — CI, a pipe — it never installs and simply reports what is missing: silence is not -consent. +`init` looks for pixi and conda-pack on every run and **prompts only when one of them is missing**, +defaulting to **yes** with a terminal attached. Without a terminal — CI, a pipe — it never installs +and simply reports what is missing: silence is not consent. + +When both are already present there is nothing to ask, and `init` says so rather than passing over +it: which pixi it found, and — with a terminal, unless `--no-install-toolchain` was passed — whether +a newer one has been released. That last line matters because of what happens next rather than as +general news: `new scroll` records the pixi it finds, and `build` refuses any other version for that +scroll. The lookup is advisory and best-effort, so an offline machine or a rate-limited API simply +produces no line. When you agree, `init`: @@ -171,7 +178,7 @@ scrollcase new scroll \ --target linux-x86_64-cpu \ --box-id example-model \ --source-revision upstream-v1 \ - --asset-base-url https://assets.example.org/boxes \ + --publish-base-url https://boxes.example.org \ --execution library-only ``` @@ -181,7 +188,7 @@ scrollcase new scroll \ | `--runtime` | `python`, `node` or `native`. Defaults to `python` | | `--box-id` | Box identity and parent directory | | `--source-revision` | Upstream revision recorded in provenance | -| `--asset-base-url` | Base URL copied into built release metadata | +| `--publish-base-url` | Where built boxes will be published, so the signed documents can point at each other. Optional — the wizard lets you press Enter past it, `build` takes it as a flag, and a box you only run locally never needs one | | `--labels` | JSON object of free-form annotations carried into the signed release. Scrollcase reads none of them | | `--version` | Box version. Defaults to `1.0.0` | | `--scroll-version` | Version of the authoring input. Defaults to `1.0.0` | @@ -193,12 +200,13 @@ scrollcase new scroll \ | `--min-ram-gb` | Optional positive RAM requirement | | `--min-nvidia-driver-version` | Optional NVIDIA driver floor | | `--execution` | The runtime's own kinds, plus `library-only` where it applies — see the table below | +| `--from-environment` | Payload path to an entry point a package already installs into the box, such as `venv/bin/ffmpeg`. Nothing of the project is copied in | | `--script` | Existing project-relative file the box runs | | `--generate-script` | Generate a minimal starter instead of using an existing file | | `--script-destination` | Safe payload path; defaults to the runtime's own starter name | | `--generated-script-path` | Project path for the generated source; defaults to `box-entrypoints///` | | `--module` | Strict dotted Python module name | -| `--default-args` | JSON array of default application arguments | +| `--default-args` | Arguments the box always passes to its entry point, before any the caller adds. One argument as itself (`--default-args -hide_banner`), several as a JSON array (`--default-args '["-a", "-b"]'`). The quotes are the shell's requirement: `[...]` unquoted is a glob pattern | The runtime decides what the rest of the session offers: @@ -241,12 +249,13 @@ written by hand. ```sh scrollcase add asset [--to ] [--on-demand] [--executable] [--target |all] -scrollcase add file [--to ] [--executable] +scrollcase add file [--to ] [--executable] [--pin] [--target |all] scrollcase add dep [--version ] [--target |all] scrollcase add dep --from-requirements requirements.txt scrollcase add env NAME=VALUE [--target |all] scrollcase add import [--target |all] +scrollcase add command [--expect-exit-code ] [--target |all] -- ``` `add asset` **downloads the URL once** and records the `sizeBytes` and `sha256` it actually found, @@ -257,8 +266,12 @@ under the box's `cacheSubdir`. `--on-demand` writes `"embed": false`, leaving th the archive for your distribution layer to materialize. `add file` records a file from the project. `--to` defaults to the file's own name at the payload -root. No `sha256` is written — see [`localFiles`](/reference/scroll#localfiles) — so the first edit -to a file you just added does not fail your next build. +root. No `sha256` is written by default — see [`localFiles`](/reference/scroll#localfiles) — so the +first edit to a file you just added does not fail your next build. + +`--pin` records it anyway, which is what reference data wants: a file the box answers questions +from should stop the build when a byte changes rather than ship a different answer under the same +signature. It stays opt-in because most added files are about to be edited. `--executable` marks either kind as a file that needs the executable bit. A download arrives with no permissions at all and a copy does not carry the source file's mode, so this declaration is the only @@ -268,6 +281,17 @@ Assets](/guides/managing-assets#files-that-have-to-run). Both also add the payload path to `selfTest.files`, so an over-eager `prunePaths` cannot quietly drop what you just declared. +`add command` records one invocation of the box's own execution as a self-test probe, and +`remove command` takes it away again. It is the counterpart of `add import` for a runtime with no +module system: a [`native`](/reference/scroll#choosing-a-runtime) box proves itself by running what +it declares, and a command probe is its only shape. `--expect-exit-code` declares the status the +probe must exit with, for a check whose point is that the box *fails* correctly. + +The arguments come after `--` rather than as a quoted list, because they are a command line and the +parser preserves everything past that boundary byte for byte — and because `-version` would +otherwise be read as one of Scrollcase's own flags. The first real probe replaces the empty +placeholder `new scroll` writes. + `add dep` writes into the `[dependencies]` table of the box's `pixi.toml` files, editing the text rather than re-emitting the manifest, so comments and spacing survive. The default constraint is `*`: `pixi.lock` is the pin that matters and it records the exact version solved, so a second, @@ -471,7 +495,7 @@ plus a channel pointer. The full pipeline is narrated in ```sh scrollcase build [] [--target ] [--channel ] - [--asset-base-url ] [--namespace ] [--allow-dirty] + [--publish-base-url ] [--namespace ] [--allow-dirty] [--pixi ] [--conda-pack ] [--private-key ] [--public-key ] [--signer-command ] ``` @@ -483,7 +507,7 @@ menu. CI and other non-interactive callers must always provide it explicitly. | --- | --- | --- | | `--target` | ask when a box has several scrolls | Canonical target scroll to build | | `--channel` | `beta` | Channel the signed pointer names. The vocabulary is closed to `nightly`, `beta`, and `stable` | -| `--asset-base-url` | scroll's `assetBaseUrl` | Base URL the signed documents point at; one of the two must be set | +| `--publish-base-url` | scroll's `publishBaseUrl` | Where the signed documents will point. With neither, the release and channel carry no links and the box is local-only — see [the scroll reference](/reference/scroll#publishbaseurl) | | `--namespace` | `scrollcase.box` | Document `kind` namespace — a project with boxes already in the field keeps emitting its own | | `--allow-dirty` | off | Permit a build from an uncommitted tree; recorded as `sourceTreeDirty: true` in the box | | `--signer-command` | none | Sign through an external command instead of the local key — see [Signing & Key Custody](/guides/signing-and-custody#external-signers) | diff --git a/docs/reference/schemas.md b/docs/reference/schemas.md index 69e5a4f..5d5ab94 100644 --- a/docs/reference/schemas.md +++ b/docs/reference/schemas.md @@ -75,8 +75,20 @@ Never hand-edit `src/contract/types/index.d.ts`; the drift test checks the gener ## Compatibility -All active schemas describe `schemaVersion: 2`. A v2 verifier rejects v1 rather than interpreting -it through the new contract; historical v1 boxes remain usable with the immutable Scrollcase -versions that produced them. Target IDs, document-kind strings, payload encoding, signature -algorithm, and golden fixtures do not change silently at an existing `$id`. A future breaking -change requires another schema version. +All active schemas describe `schemaVersion: 3`, and are published under `/schema/v3/`. A v3 verifier +refuses a v1 or a v2 document **by name** rather than reinterpreting it: they are different +artefacts with different rebuilds ahead of them. Target IDs, document-kind strings, payload +encoding, signature algorithm and golden fixtures never change silently at an existing `$id`; a +breaking change gets a new schema version instead. + +### Version 2 is still readable + +The version 2 schemas remain served, verbatim, at +[`/schema/v2/`](https://scrollcase.dev/schema/v2/scroll.schema.json). Every scroll, release and box +built under version 2 carries one of those URLs in its own `$schema`, and an `$id` that stopped +resolving would break editor validation and any tool that dereferences it. "Immutable" is a promise +about the artefacts as much as about the format. + +They are frozen: nothing generates or checks them, because there is nothing left to keep them in +step with. They are not an alternative to build against — a version 2 box is rebuilt from its scroll +under version 3 — and for that reason `/.well-known/api-catalog` lists version 3 only. diff --git a/docs/reference/scroll.md b/docs/reference/scroll.md index 69bcf93..e709b16 100644 --- a/docs/reference/scroll.md +++ b/docs/reference/scroll.md @@ -19,15 +19,15 @@ scrolls/ The parent directory is the scroll's declared `boxId`; the child is the canonical ID computed from its declared `target`. Scrollcase checks both, so the path cannot mislabel the scroll, but neither -value is written twice inside `scroll.json`. Flat source directories are not accepted in v2. +value is written twice inside `scroll.json`. Flat source directories are not accepted. The machine-readable definition is [`scroll.schema.json`](/schema/v3/scroll.schema.json), also shipped through the package export. See [JSON Schemas](/reference/schemas). -Create a new target-specific input with `scrollcase new scroll`. Interactively it asks four -things — the target, the box id, the upstream revision, and where boxes will be published — and -derives everything else, generating the matching `pixi.toml` and a starter `self_test.py`. It -refuses to overwrite an existing scroll. +Create a new target-specific input with `scrollcase new scroll`. Interactively it asks for the +target, the [runtime](#choosing-a-runtime), the box id, the upstream revision, and — optionally — +where boxes will be published, then derives everything else and generates the matching `pixi.toml` +plus a starter self-test where the runtime has one. It refuses to overwrite an existing scroll. ## The shortest scroll that builds @@ -44,7 +44,7 @@ scroll is read. Write the decisions, not the restatements: "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, "runtime": { "id": "python", "version": "3.14" }, "pixiVersion": "0.73.0", - "assetBaseUrl": "https://assets.example.org/boxes", + "publishBaseUrl": "https://boxes.example.org", "selfTest": { "imports": ["json", "sqlite3"] } } ``` @@ -68,7 +68,7 @@ derived field is never wrong; `runtime.entryPoint` is still checked against the "pixiVersion": "0.73.0", "cacheSubdir": "cache/hello-box", "environment": { "MODEL_ROOT": "cache/hello-box", "HF_HUB_OFFLINE": "1" }, - "assetBaseUrl": "https://assets.example.org/boxes", + "publishBaseUrl": "https://boxes.example.org", "assets": [], "selfTest": { "imports": ["json", "sqlite3"], "files": [] } } @@ -344,11 +344,59 @@ library ships anyway — `"GGML_METAL_DEVICES": "0"` for llama.cpp on macOS. See This does **not** replace or filter the host environment. A box inherits it exactly as before. Scrollcase reports the resulting provenance through its CLI and Node/Python consumer APIs; see -[Environment reports](/reference/api#environment-reports). Names must be non-empty and contain +[Environment reports](/reference/api/node#environment-reports). Names must be non-empty and contain neither `=` nor NUL; values are strings and may be empty but cannot contain NUL. ## Execution intent +### Why declare an execution + +A box is an environment plus files. Nothing in it says which of those files is *the* thing to start +— so by default, whoever receives the box has to already know. `execution` is the box answering that +question about itself, and the answer is **signed into the release** along with everything else. + +That is the whole difference. With it, `scrollcase run ` starts the box, and so does +any of the three consumers, without the caller knowing what is inside or deciding what to launch. +Without it, the caller extracts the box and drives it themselves. + +It is a trust property before it is a convenience one. Because the command line is fixed at build +time and signed, nobody holding the box afterwards can change what it runs. A consumer that took a +path from its own configuration would be running whatever that configuration said; a consumer +reading a signed `execution` is running what the publisher built and proved. + +### Which kind to declare + +Each kind is a different way of naming the one thing to start, and each produces a shell-free command +line. These are the real ones: + +| Kind | You declare | The command line becomes | +| --- | --- | --- | +| `python-script` | `script`, a file path inside the box | `venv/bin/python app/main.py --serve` | +| `python-module` | `module`, a dotted importable name | `venv/bin/python -m example_model.main --serve` | +| `node-script` | `script`, a file path inside the box | `venv/bin/node app.js` | +| `native-binary` | `binary`, a file path inside the box | `bin/tool --quiet` | +| *(omitted)* | nothing | there is none — the box is library-only | + +**Script or module** is a question about how your code got into the box, not about style. A file you +shipped with [`localFiles`](#localfiles) sits at a path you chose, so name that path. A package that +was *installed* by the dependency solve lands in `site-packages` under a directory whose name +includes the interpreter version — a path you would not want to write down, and one that changes +when the interpreter does. Name it as a module and the box's own import system finds it. The builder +proves the module resolves by inspecting files, never by importing your application. + +**Library-only** — omitting `execution` — is not "the box does nothing". It is a box whose purpose is +to be imported: your application prepares it with a consumer, then imports from the environment +inside it and calls whatever it likes. Pick it when there is no single entry point that would mean +anything, which is the normal case for a box that exists to provide a model and its dependencies to +an application that already knows how to use them. `scrollcase run` refuses such a box by name, +because there is nothing it could honestly start. + +A `native` box cannot be library-only, for a reason worth stating: its only self-test shape is an +invocation of its own binary, so a native box with nothing to invoke could prove nothing about +itself at build time. + +### The declarations + `execution` records how a consumer may start the box: ```jsonc @@ -392,9 +440,21 @@ Each kind is named `-`, and the runtime half must be the one the refused rather than guessed at. `scrollcase new scroll` offers only the kinds the chosen runtime defines — see [the CLI reference](/reference/cli#new). -Omit `execution` for a library-only box. A `native` box cannot be library-only: its only self-test -shape is an invocation of its own binary, so a native box with nothing to invoke could prove nothing -about itself. +### Where the entry point comes from + +A file-naming execution has two possible origins, and `scrollcase new scroll` asks which: + +- **The environment provides it.** A package the dependency solve installs already puts the program + in the payload — conda-forge's `venv/bin/ffmpeg`, a console script the solve generated. The scroll + names that path and nothing of the project is copied in, so there is no `localFiles` entry. Pass + `--from-environment ` to say so without a terminal. +- **The project provides it.** A script you wrote or a binary you compiled, living in your + repository. `--script ` records it in `localFiles`, and the build copies it into the box at + the payload path you chose. For a `python` or `node` box, `--generate-script` writes a starter + instead. + +The distinction matters most for `native`, where both are common: packaging an existing program and +shipping one you built are different jobs, and only the second involves a file of yours. A `native-binary` must additionally be declared `executable: true` on the asset or local file that brings it in, unless it comes from the packed environment's own scripts directory. The executable @@ -567,15 +627,44 @@ Every path inside the payload (`relativePath`, `destination`, `prunePaths`, `unc and drive letters are rejected. ::: -### `assetBaseUrl` +### `publishBaseUrl` + +Base URL the built archive and its signed documents will be published under, so each can point at +the next: the channel names the release document, and the release names the archive. + +It says nothing about the box's own assets — those carry a URL each — and nothing about what the box +does when it runs. A box that transcodes video on your laptop touches no network and needs no URL +anywhere. + +**Optional, and genuinely so.** A box you build to run where you built it is never published, so +there is nowhere for its documents to point and no value here would be true. Omit it and the build +simply leaves both links out: + +```jsonc +// with a publish base URL +"archive": { "format": "zip", "url": "https://boxes.example.org/…", "sha256": "…", "sizeBytes": 1234 } + +// without one +"archive": { "format": "zip", "sha256": "…", "sizeBytes": 1234 } +``` + +**Nothing is lost but the address.** No guarantee depends on this URL: the archive is verified by +`sha256` and size, the documents are signed, `box.json` still has to agree with the release, and +`verify` passes either way. No Scrollcase consumer even reads it — all three find the archive beside +its release document and identify it by hash. It exists for the distribution layer that has to fetch +the bytes, and for nothing else. + +That is also why an absent URL beats an invented one. Since nothing checks it, a wrong address is a +false statement inside a signed, immutable document, and it stays false forever. `new scroll` lets +you press Enter past it for the same reason. -Base URL the built archive and its objects are published under. It is what the signed release and -channel documents point at. Required unless passed per build with `--asset-base-url`. +Supply it later with [`edit scroll`](/reference/cli#edit-scroll), or per build with +`--publish-base-url`. ## Self-test -Builder checks run with the payload's **own interpreter** before the box is archived. Schema -version 2 signs the import subset for a consumer to repeat; it does not carry the richer file or +Builder checks run with the payload's **own runtime** before the box is archived. Schema +version 3 signs the probe for a consumer to repeat; it does not carry the richer file or `code` assertions. ```jsonc diff --git a/docs/v2/concepts/architecture.md b/docs/v2/concepts/architecture.md new file mode 100644 index 0000000..065febd --- /dev/null +++ b/docs/v2/concepts/architecture.md @@ -0,0 +1,269 @@ +--- +title: Architecture +description: How a scroll becomes a signed box, and how local consumers prepare and run it. +--- + +# Architecture + +Scrollcase turns a declarative **scroll** into a **box**: a portable, locked, self-contained +Python environment for one operating system and accelerator, packed so it runs somewhere other +than where it was built, signed so a consumer can prove what they received, and accompanied by a +dependency licence inventory. + +This page explains how, and — more usefully — *why each step is where it is*. + +## The v2 consumer boundary + +Scrollcase has one canonical contract and two local consumer implementations: the Node/TypeScript +API at `scrollcase/consumer`, the Python package imported as `scrollcase_consumer`, and the Rust +crate `scrollcase-consumer`. +`scrollcase run` delegates to the Node API instead of implementing a third path. + +```mermaid +flowchart LR + C["canonical v2 contract
schemas + fixtures"] --> N["Node consumer
scrollcase/consumer"] + C --> P["Python consumer
scrollcase_consumer"] + C --> R["Rust consumer
scrollcase-consumer"] + F["caller-supplied release, archive or root,
trust keys, destination"] --> N + F --> P + N --> L["verified local box
or child process"] + P --> L +``` + +Every consumer must agree on verification, safe extraction, attachment across restarts, +installed-payload checking, execution, receipts, errors, signals, cleanup, and on-demand assets by +passing the same language-neutral conformance cases. Generated or checked schema copies are +projections of the canonical contract, never independent definitions. + +The security order is fixed: validate the signed document and release shape, verify the archive +size and hash, validate every archive entry, extract safely, compare `box.json` with the signed +release, and validate execution prerequisites before starting box code. A consumer never launches +the interpreter, a script, a module, or an import earlier. + +All inputs are local and caller-selected. Consumer code does not choose a channel, download a box, +update an installation, promote, revoke, publish, serve, allocate a runner, or own application +lifecycle policy. + +A persistent installation earns a new process-bound receipt through attachment rather than loading +one from disk. Byte verification stays separate and opt-in: the signed release commits to the +`payload-digest.v1` list inside the box, and the verifier checks that list instead of treating later +extra files as corruption. The result is point-in-time integrity; operating-system permissions and +the embedding application remain responsible for guarding the directory afterwards. + +## The substrate + +One substrate, and only one: **pixi + conda-pack + conda-forge**. + +```mermaid +flowchart LR + A["pixi.toml"] -->|scrollcase lock| B["pixi.lock"] + B -->|pixi install --frozen| C["conda prefix"] + C -->|conda-pack| D["relocatable tarball"] + D -->|extract| E["box payload: venv/"] +``` + +`pixi` solves a committed `pixi.lock` against conda-forge, `conda-pack` relocates the resulting +prefix, and the tree is extracted into the box as `venv/`. There is deliberately no second +dependency backend — the reasoning is in [Why Pixi & Conda-Forge](/v2/concepts/why-pixi). + +## The build pipeline + +```mermaid +flowchart TD + R["scroll.json + pixi.lock"] --> V["1. validate
identity, target, host, lock, git state"] + V --> P["2. install from lock, pack, relocate"] + P --> A["3. stage assets
download + verify, local files, archives"] + A --> PR["4. prune what is not needed at run time"] + PR --> L["5. licence audit vs the reviewed copy"] + L --> S["6. self-test with the box's OWN interpreter"] + S --> PA["7. parity gate (optional)"] + PA --> N["8. normalise timestamps and ordering"] + N --> Z["9. deterministic zip"] + Z --> SG["10. sign release, then channel pointer"] + SG --> O["11. content-addressed staging tree"] +``` + +Each step earns its position: + +**Validate first.** The complete nested scroll is checked against the shipped schemas before a +tool is probed, a fetch is made, or build state is mutated. Identity, target/entry-point, +weights/archive policy, native host, lock presence, and Git state follow in that order. + +**Install, never resolve.** `pixi install --frozen` materialises exactly the locked packages +without touching or re-checking the lock, so what ships is byte-for-byte what was reviewed. +Resolution is a separate, human-initiated step (`lock`). + +**Relocate.** See [below](#relocation). + +**Stage assets.** Every declared asset is size- and hash-checked before it enters the payload. +Network retries within one download resume a partial file, which is renamed into place only after +its size and hash match. Build scratch is reset between processes; there is no persistent cache. + +**Prune, then check.** Pruning keeps the box to what it needs at run time; `selfTest.files` is +what stops an over-aggressive prune from shipping a broken box. + +**Audit before self-testing.** The licence inventory is derived from the lock and compared to the +reviewed copy. A licence problem is a legal problem, and it is cheaper to hit it before the +expensive checks. + +**Self-test with the box's own interpreter.** The builder runs post-prune file assertions, the +target assertion, imports, and optional scroll `pythonCode`. Schema version 2 signs the target +assertion and import subset for a consumer to repeat; the richer scroll-only checks are not +misrepresented as consumer checks. + +**Parity after the self-test, on the same payload.** There is no point comparing accelerators in +a box that cannot import its dependencies in the first place. + +**Commit, normalise, then archive.** After `box.json`, the builder writes `payload-digest.v1` and +places the list's hash in the signed release. Timestamps are then stamped to a fixed instant and +files enumerated in one stable order, which is what makes the ZIP deterministic. + +**Sign last, stage after.** The release commits to the archive by hash; the channel commits to +the release document by *its* hash. The staging tree is then laid out exactly as a bucket would +be, so whatever publishes it uses the keys the manifests already point to. + +## The guarantees + +Everything above exists to hold six promises. They are the product; features that would weaken +one are refused. + +### Locked + +The environment is a pure function of a committed `pixi.lock`. `build` installs; it never +resolves. A missing lock is a hard error rather than an invitation to resolve on the fly. + +### Deterministic {#determinism} + +Rebuilding the same commit produces a **byte-identical archive**. Three things make that true: + +- timestamps are normalised to a fixed instant (`2000-01-01T00:00:00Z`) and files are enumerated + in one stable order; +- the build time comes from the HEAD commit, not the clock — outside a git checkout it falls back + to a constant, because a wall-clock fallback would reintroduce exactly the nondeterminism this + avoids; +- the channel cohort salt is derived from `boxId` and `version` rather than randomly. + +A test asserts this directly. Anything that varies per run — a clock read, a random value, an +unsorted directory listing — breaks it. + +### Relocatable {#relocation} + +conda-pack already replaces the build prefix with a neutral placeholder, and a conda-forge prefix +imports and runs from any location with no activation environment. So the box needs **no +relocation step at install time**, and the embedded `conda-unpack` fixer is deliberately never +run: doing so would stamp the build machine's absolute paths into dozens of files that then ship +to users — measured on a probe environment, zero files carried the build prefix before running it +and thirty-six after — leaking a developer's directory layout while still being wrong at the +user's install location. + +Instead, four repairs happen at build time: + +1. The few service files that carry the build prefix are removed + (`conda-meta/pixi_env_prefix`, `bin/conda-unpack`, and friends). +2. Every symlink is settled: kept when it provably resolves, inside the payload, to a regular file; + materialised into real content when it does not; dropped when it dangles or escapes the prefix, + rather than pulling host files into the box. A link to a *directory* is always materialised — + that is the only way an entry could be written through a link and land somewhere its own name + does not describe, so a prefix whose links chain through directory links (icu's `current`, for + instance) unpacks like any other. Keeping the rest matters: the soname convention alone stores + every large shared library two or three times, and materialising all of it doubled an extracted + Linux box. The rule lives in `src/contract/links.mjs` and is re-applied by every consumer against + the archive as received, never trusted from the builder. +3. Generated console scripts, whose shebangs embed the build interpreter's absolute path, are + rewritten to resolve Python next to themselves. +4. conda's per-package records in `conda-meta/` are reduced to name, version, build and licence, + and its `history` log is dropped. As the installer writes them, those records name the build + machine's package cache and vary between two installs of the same lock, which would leak a + developer's paths and break the byte-identical rebuild. The kept fields + are copied verbatim and chosen by allowlist, so a field a later pixi starts writing cannot + reintroduce either problem. Nothing in a box reads them — conda is never shipped inside one, and + package versions stay readable from `site-packages`. + +### Signed + +Every release and channel document travels in one envelope, with the payload as exact +base64-encoded JSON so that verifying a signature means hashing the bytes as transmitted. A local +ed25519 key works out of the box; an external signer plugs in through `--signer-command` and is +**not trusted on its word** — it must echo back the exact payload it was given, and its signature +is verified locally before the build continues. See +[Signing & Key Custody](/v2/guides/signing-and-custody). + +### Verified + +`verify` checks signature, archive size and hash, safe entry names, recursive agreement of all +shared schema-v2 manifest fields, and the declared interpreter. With `--self-test`, it temporarily +extracts and runs the signed import subset. It does not repeat scroll-only Python or file checks. + +### Honest about provenance + +A box records the commit it was built from and whether that tree was dirty, including untracked +files while respecting Git ignore rules. Building outside a git checkout **fails** rather than +inventing a revision; a dirty tree requires `--allow-dirty` and +is recorded as `sourceTreeDirty: true` in the box itself. A build that cannot be reproduced from +its recorded revision says so. + +## The code + +```text +src/ +├── contract/ the box format itself — the source of truth +│ ├── targets.mjs target model, identity rule, per-target adapters +│ ├── documents.mjs signed-document envelope, namespacing +│ ├── payload-digest.mjs canonical extracted-entry list bytes +│ ├── schema/ eight JSON Schemas +│ └── fixtures/ golden fixtures other implementations prove themselves against +├── build/ solving, packing, staging, auditing, verifying +│ ├── pixi.mjs tool discovery, argument vectors, install + pack + relocate +│ ├── launchers.mjs the console-script repair +│ ├── archive.mjs deterministic zip, defensive extraction +│ ├── filesystem.mjs stable ordering, fixed timestamps, path safety +│ ├── assets.mjs verified downloads, local files, archive expansion +│ ├── licenses.mjs the lock-derived licence inventory +│ ├── workspace.mjs project path resolution +│ ├── scroll.mjs scroll reading and provenance +│ ├── box.mjs the build core +│ ├── verify.mjs the consumer's checks, run locally +│ ├── audit.mjs the licence audit verb +│ ├── project.mjs init and doctor +│ └── parity.mjs the accelerator parity gate +├── consumer/ verified local preparation, attachment, checking and execution +│ ├── verify-and-extract.mjs staged extraction, attachment, payload checking, opaque receipts +│ ├── run-extracted.mjs interpreter invocation, assets, stdio, signals +│ └── run-box.mjs one-shot temporary execution and cleanup +├── sign/ key generation, local signing, external dispatch, verification +└── cli.mjs argument parsing and dispatch — thin, logic lives in the modules +rust/ +├── src/ the crate: contract mirror, verification, extraction, execution +├── fixtures/ bundled copies of the shared fixtures, drift-checked +└── tests/ contract, schema agreement, hostile archives, conformance +python/ +├── src/scrollcase_consumer/ typed Python verification, extraction, and execution +├── scripts/ schema sync and distribution inspection +└── tests/ local signed-box and hostile-archive regressions +``` + +`src/contract/` is the source of truth for the format. Other languages **mirror** it and prove +the mirror against `fixtures/target-id-contract.json`; they do not import it. That is how the Rust +crate, a Worker, and this builder stay in agreement without sharing a runtime. + +## Boundaries + +Two boundaries hold everything else up. + +**Paths come from the project, not from Scrollcase.** A `scrollcase.config.json` declares where +scrolls live and where artefacts go, discovered by walking up from the working directory, +overridable per invocation. A tool that derives its paths from its own location on disk only works +while it lives inside the project it serves; making the layout the project's declaration is what +lets Scrollcase run from anywhere against any project. + +**The document namespace belongs to the publishing project.** Document kinds are +`.release` / `.channel` / `.revocations`, defaulting to `scrollcase.box`. A project +with boxes already installed in the field keeps emitting the namespace its clients recognise, and +Scrollcase carries nobody's brand. + +## What is deliberately outside + +Publishing to object storage, serving or promoting a channel, revoking a release, allocating CI +runners, and model-specific scientific validation. Scrollcase stops at a signed, verified box on +disk — see [Distributing Boxes](/v2/guides/distributing-boxes) for what to build on top, and +[Design Decisions](/v2/concepts/design-decisions) for why each was left out. diff --git a/docs/v2/concepts/design-decisions.md b/docs/v2/concepts/design-decisions.md new file mode 100644 index 0000000..80086fe --- /dev/null +++ b/docs/v2/concepts/design-decisions.md @@ -0,0 +1,549 @@ +--- +title: Design decisions +description: Why Scrollcase is shaped the way it is, and which alternatives were rejected. +--- + +# Design decisions + +Each entry records the alternative that was rejected, because a decision without its discarded +alternative is just an assertion. + +## Version 2 is a clean break + +Scrollcase v2 accepts and emits only `schemaVersion: 2`. Published v1 boxes and immutable package +releases remain usable with the old Scrollcase versions that produced them; the v2 verifier rejects +them with a clear unsupported-version error. It does not reinterpret them. + +The declarative source is a **scroll**, stored as `scroll.json` under `scrolls/`. The built artefact +remains a **box**. This vocabulary applies across schemas, identifiers, paths, CLI arguments, +fixtures, types, documentation, and errors. + +**Rejected:** a v1/v2 union, compatibility aliases, and dual execution paths. They would make every +security check and every consumer carry two meanings indefinitely, while still being unable to +change the already-published v1 wire format. + +## Consumers prepare and run local boxes; they do not distribute them + +The v2 consumers operate on release documents, archives, trust keys, and destinations supplied by +the caller. They may verify, safely extract, inspect, and execute a box. Verification is ordered so +no box interpreter, script, module, or import runs before the signature, payload shape, archive +size/hash, safe entries, and shared manifest agreement have succeeded. + +The official Node API is `scrollcase/consumer`; the Python package is imported as +`scrollcase_consumer`. `scrollcase run` is a thin CLI wrapper over the Node API. The utilities live +in those consumer SDKs, not as a JavaScript helper copied into every box. + +This local execution surface does not choose channels, fetch archives, update installations, +promote, revoke, publish, serve, allocate runners, or decide application lifecycle policy. + +**Rejected:** folding registry, download, update, and lifecycle policy into the consumer. Those +responsibilities require project-specific trust and rollout choices and would turn a local, +composable verifier into a distribution system. + +## One contract, multiple consumer implementations + +`src/contract/` and its schemas remain the single source of truth. Node, Python, and Rust expose the +same verification, extraction, execution, receipt, error, signal, cleanup, and on-demand-asset +semantics. Language-neutral fixtures and expected results prove their parity. The Python package +carries checked generated copies of the canonical schemas, and the crate checked copies of the same +schemas and fixtures; neither hand-maintains a second format. + +**Rejected:** independent per-language contracts that merely look similar. Security behavior +drifts at edge cases — links, traversal, collisions, signals, or argument handling — unless every +implementation is held to the same observable cases. + +## Persistent installations earn a new receipt; payload verification stays separate + +A prepared receipt is process-bound execution authority. Serialising it would let anyone who can +write the receipt file manufacture an object that appears to have passed the trust chain. A process +that starts later therefore calls `attachExtractedBox` / `attach_extracted_box`: it re-verifies the +signed release, requires a target the current host can execute, checks the interpreter and execution +shape, verifies on-demand assets, and binds a fresh receipt to the real directory's device and inode. +The receipt says `attached`, not `prepared`, because no archive established the payload bytes in that +process. + +Byte verification is an independent, opt-in operation. New builds write `payload-digest.v1` inside +the payload and add optional `payloadDigest: { format, sha256 }` to the signed release. The list has +one byte-sorted record per original file or link and is excluded from itself; the release signs its +hash. `verifyExtractedPayload` / `verify_extracted_payload` authenticates the bounded list before +parsing it, then visits only the paths it names. The field is additive, so `schemaVersion` stays 2 +and older v2 releases remain valid, while the specific payload-verification operation refuses one +that carries no commitment. + +**Rejected:** storing the whole per-file table in the release. A conda environment routinely holds +10,000–30,000 files, which would add megabytes to every signed document. One signed digest plus the +list inside the payload keeps the document small without weakening which bytes it commits to. + +**Rejected:** a single root hash recomputed by walking the installed directory. Honest installations +grow: Python creates caches, applications write in their working directory, and on-demand assets are +materialised after extraction. If the directory is the input, every legitimate extra file changes +the answer. Walking the signed list makes extras invisible by construction. + +**Rejected:** folding byte verification into attachment or execution, or adding a verification flag +to attachment. Embedded weights can make the scan read tens of gigabytes, and a result at attach +time does not guarantee the tree at a later spawn or lazy Python import. Separate operations keep +both cost and meaning explicit: attachment answers whether a directory can mint a receipt now; +payload verification answers whether its listed bytes match now. + +**Rejected:** committing file mode or modification time. Archive writing synthesises modes from the +target and path, Windows extraction does not apply `chmod`, and no extractor restores the fixed +build timestamp. Including either would make an honest extraction disagree with its build. + +The limit is stated rather than hidden. Payload verification has a check-to-use window and is not a +defence against a live local attacker; operating-system permissions and application ownership guard +the directory. `__pycache__` directories and `*.pyc` files are excluded by the collector and are +therefore a permanent blind spot, not merely part of that timing window. Embedded assets are listed +and expensive to re-read; on-demand assets are ignored extras whose separate signed descriptors are +checked during attachment and execution. + +## One substrate: pixi + conda-pack + conda-forge + +Scrollcase supports exactly one dependency backend. + +A packaging tool's product is its guarantees — this environment installs, relocates, self-tests, and +is reproducible from a lock. Two backends means proving every guarantee twice, on every platform, for +every release. The conda-forge path also solves the problem a wheel-based one cannot: native +libraries. Scientific stacks are mostly compiled code, and conda-forge distributes it as a coherent, +licence-annotated package set rather than as wheels of varying provenance. + +**Rejected:** a second backend for projects already on `uv`. Those projects convert their scrolls +once; Scrollcase avoids a permanent double burden. + +## conda-pack, and deliberately *not* running conda-unpack + +`conda-pack` produces a ready-to-run tree, so a consumer pays no install-time work beyond extraction. +The embedded `conda-unpack` fixer is deliberately **not** run: it would stamp the build machine's +absolute paths into dozens of files that then ship to users — measured on a probe environment, zero +files carried the build prefix before running it and thirty-six after — leaking a developer's +directory layout while still being wrong at the user's install location. Instead the few service +files that do carry the prefix are removed, symlinks are settled against a rule that keeps only the +ones provably resolving inside the payload, and generated console scripts are rewritten to resolve +Python next to themselves. + +**Rejected:** `pixi-pack`, which ships packages rather than a tree and needs a per-user install plus a +bundled unpacker at the other end. The slow step (compression) is better paid once by whoever builds +than on every install. + +## A payload carries the symlinks it can prove safe + +A conda prefix is dense with symbolic links. The shared-library soname convention alone stores every +large library under two or three names — `libfoo.so` → `libfoo.so.N` → `libfoo.so.N.M` — and `bin` +carries interpreter aliases. Scrollcase used to materialise all of them, which was simple and +correct and, once measured, expensive: roughly 60% of an extracted Linux box was duplicates of its +own bytes. The example box weighed 191 MB archived and 483 MB extracted, against 48 MB and 126 MB +for the identical scroll on macOS, where dylibs use far fewer such chains. + +A link is now kept when it **provably** resolves, inside the payload, to a regular file: the target +must be relative, must stay inside after `..` is applied segment by segment, and must end at a file +rather than a directory. Everything else is materialised exactly as before. That took the example +box to 90 MB archived and 228 MB extracted. + +The narrowness is the point. A symbolic link is the classic way an archive writes outside the +directory it was extracted into, so the rule is purely lexical — the same inputs give the same +answer on every host — and it is applied three times: by the builder against the real filesystem, by +the archive writer against the entry set it is about to write, and again by each consumer against +the archive **as received**. No consumer trusts the builder, and a box assembled by hand gets no +benefit of the doubt. + +**Rejected:** carrying directory links too. They are legitimate in a prefix — `lib/python3.1` → +`python3.11` is real — and worth about one duplicated standard library. But a directory link is the +only way an entry can be written *through* a link and land somewhere its own name does not describe, +which turns a size optimisation into a question about what every other entry does to the filesystem. +Refusing them keeps the rule small enough to state in five lines and prove in two languages, and +that was worth more than the last 35 MB. + +**Rejected:** a `schemaVersion` bump. The signed document is unchanged; only what the archive may +contain grew. A consumer predating the rule rejects a link entry with a clear error rather than +misreading it, which is the only thing a version bump would have bought. + +## The document namespace belongs to the publishing project + +Every signed document carries a `kind` like `scrollcase.box.release`. The namespace is configurable +and defaults to `scrollcase.box`. + +This exists because a project that already has boxes installed in the field cannot have a tool rename +its documents underneath it — its clients would stop recognising them. Making the namespace the +project's own declaration means byte-compatibility for existing publishers and a tool that carries +nobody's brand. + +**Rejected:** hard-coding a single namespace. Byte-compatibility for existing publishers turned out to +cost nothing, and independence from any one consumer is not negotiable. + +## Signing is built in; key custody is not + +Scrollcase signs with a local ed25519 key out of the box, so anyone gets verifiable boxes without +infrastructure. An operator with real key custody — a KMS, an HSM, a signing service — configures an +external signer command instead: it receives the payload on stdin and returns the signed document on +stdout. Any language, any credential mechanism, no plugin API to keep compatible. + +An external signer is not trusted on its word. The returned document must echo back the exact payload +it was given, and its signature is verified locally before the build continues. A signer that +substitutes a payload fails the build instead of producing a box nobody can install. + +**Rejected:** a provider-specific integration. Cloud-specific authentication in a packaging tool ages +badly and excludes everyone using something else. + +## Verification is not optional + +`verify` checks signature, archive size and hash, safe entry names, recursive agreement of every +shared schema-v2 field, the declared interpreter, and optional execution prerequisites. Execution is +a closed script/module union rather than a shell command. The builder and verifier inspect regular +payload/archive files to prove a script or runnable module exists; module discovery never imports +the application. With `--self-test` verification extracts temporarily and runs the signed import +subset. Scroll-only Python and file assertions remain builder checks because they are not part of +the signed release. + +**Rejected:** accepting a shell command or proving a module by importing it. A shell changes +argument meaning and creates an injection surface; importing application code turns validation into +execution before the trust chain has finished. + +## Weights: embedded by default, on demand when asked + +`embed` packs assets into the archive: the box installs with no network and works air-gapped, at the +cost of a large artefact. `on-demand` leaves them out and carries their url, path, size and SHA-256 in +the signed release and in `box.json`. Retrieval belongs to the caller's distribution layer; the +local consumers verify caller-materialized files before execution and never download them. + +The declared hash is what makes deferring safe: the release commits to exactly which bytes the box +expects, whatever host serves them. + +**Rejected:** making on-demand the default. Air-gapped installation is a property worth keeping +unless a project explicitly trades it away, and it is the behaviour that surprises nobody. + +## Accelerator parity is a packaging concern + +A scroll may declare a `parity` block: a check script inside the box, the accelerators to run it +under, and tolerances (`absolute`, `relative`, `minimumCosine`). Scrollcase runs the check once per +accelerator using each target's validation environment, compares every run against the first, and +fails the build on a breach. + +The question — *does this box compute the same thing on the GPU as on the CPU?* — sounds scientific +but is not. It catches the failures a packaging tool is responsible for: the wrong wheels solved in, a +CPU-only build shipped as CUDA, a broken BLAS. + +The division of labour is deliberate. Scrollcase owns the mechanism and enforces the declared +threshold; the project owns the check script, the fixture, and what closeness means for its model. +Non-finite output is rejected explicitly, being the classic symptom of a broken accelerator build, and +relative error is only counted where the reference has magnitude — the absolute bound guards entries +near zero, where relative error is meaningless. + +**Rejected:** hard-coding tolerances inside Scrollcase. What counts as close enough is a property of +the model, not of the packaging step, so it is declared per scroll rather than assumed. + +## The toolchain is installed on request, and pinned once installed + +`init` can install `pixi` and `conda-pack`, but only after asking, and only into the project's own +toolchain directory. Nothing is added to `PATH`, nothing is installed system-wide, and deleting the +directory undoes it. Without a terminal to answer the question — CI, a pipe — nothing is installed +at all: silence is not consent. + +The download is verified before use. The release archive's SHA-256 is checked against the checksum +the publisher ships beside it, and the verified digest is then recorded in the project's config, so +every later install is checked against a value the project committed rather than against whatever +the server offers that day. A mismatch aborts before anything is installed. The conda-pack +dependency is installed as the exact `conda-pack==0.9.2` match specification and that version is +recorded beside the pixi pin; floating it would let the same Scrollcase release produce different +payload bytes over time. + +**Rejected:** installing silently, and the `curl | sh` convention it would imitate. A packaging tool +whose whole product is verified artefacts cannot begin by running unverified bytes it fetched +without being asked. + +## Paths come from the project, not from Scrollcase + +A workspace is declared by a `scrollcase.config.json` at the project root, discovered by walking up +from the working directory, with per-invocation flag overrides. Defaults are `scrolls/` and +`.scrollcase/{build,dist,keys}`. + +A tool that derives its paths from its own location on disk only works while it lives inside the +project it serves. Making the layout the project's declaration is what lets Scrollcase run from +anywhere against any project that declares one. + +## Workspace setup keeps real authoring separate from the disposable example + +`scrollcase init` creates project structure and, by default, one clearly named `example-box` for the +native host. The example is a complete runnable v2 scroll produced through the same validated +authoring path as any other scroll. It prefers Metal on Apple Silicon and CPU elsewhere, never +guesses a CUDA ABI, and never overwrites an existing example. Whether to create it at all is the +one of the two questions asked first, and it defaults to yes; `--no-example` answers it in advance, +and a run without a terminal keeps the example rather than dropping it, since scaffolding a +disposable directory installs nothing. +Its application starter lives at +`box-entrypoints///entrypoint.py`: executable input is grouped by the same box and +target it belongs to, without adding a redundant tool-named directory. +Three adjacent, non-overwriting consumer templates show the other side of the boundary: +`scrollcase/consumer` from TypeScript, `scrollcase_consumer` from Python, and +`scrollcase-consumer` from Rust. They accept local release and trust inputs; they do not add +download or distribution behavior. They live under `consumer-templates/`, with Rust in its own +small Cargo crate, and they are the second question — separate from the example, because a project +that wants no demo still has a consumer application to write, and these are where it starts. The +dependency offers that follow belong to them. A short non-overwriting `SCROLLCASE.md` keeps the +basic workflow and links to the canonical documentation visible in the project. + +`scrollcase new scroll` remains the only command that authors real project identity, target, +versions, compatibility, weights, and execution intent. A non-terminal authoring call must provide +every value that has no default and fails before writing when one is missing; an interactive +terminal uses the same finite-choice menus as the rest of the CLI. + +**Rejected:** either treating setup metadata as the project's real scroll or leaving a newcomer with +only an empty directory. The fixed example is explicitly disposable onboarding material; real +inputs are created independently rather than edited from guessed product metadata. + +## A scroll declares decisions, not restatements + +A scroll is a file a person writes and maintains by hand, and several of its fields were only ever +restating something the file already said. `pythonEntryPoint` is the clearest case: a target admits +exactly one interpreter path and the reader rejected every other value, so requiring the field +obliged the author to type the one string that was already implied — and to type it again for every +target of the same box. `scrollVersion`, `compatibility`, `modelCacheSubdir`, `assets` and +`selfTest.files` were the same kind of obligation in weaker form. + +Those fields are now optional and derived when the scroll is read. Derivation happens in one place, +so everything downstream — including the provenance record — still sees a complete object, and a +scroll that spells a derived field out produces an identical result. A declared `pythonEntryPoint` +that disagrees with its target is still refused. + +**Rejected:** a `??` fallback at each point of use. That spreads the meaning of an absent field +across the builder, where two of them eventually disagree and the disagreement is invisible. + +**Rejected also:** deriving `condaDependencyLicenseAudit` from a file sitting next to the scroll. +That field carries enforcement — declaring it means the build fails when the lock no longer matches +what was reviewed — and a guarantee that switches itself on because of a file's presence is a +guarantee nobody decided to make. It stays explicit; it costs one line. + +### The hash on a local file is a pin, not a checksum + +`assets` arrive over a network nobody controls, so their size and hash are mandatory: that check is +the only thing standing between a replaced upstream file and a silently different box. +`localFiles` come out of the project's own checkout, where git already records what changed, and +what ships is hashed into the signed release regardless. Requiring a hash there bought little and +cost a great deal: every edit to a generated entry point failed the next build until its digest was +recomputed by hand, which taught authors to distrust the check rather than rely on it. + +`sha256` on a local file is therefore optional, and means *pin this*. A project pins what must not +change without review — a licence notice, a reviewed shim — and leaves the pin off what it is still +writing. A pinned file that drifts still fails the build. + +### A self-test belongs in a file + +`selfTest.pythonCode` puts Python inside a JSON string, with escaped newlines and no syntax +highlighting, no linter and no readable diff. It suits a single assertion and nothing more. +`selfTest.pythonFile` names a file in the project instead; it is read at build time and executed +from the payload root, so it can read what the box ships and import what it packs. The two are +mutually exclusive, and `new scroll` generates the file rather than leaving the field empty. + +### One box's targets share a scroll + +Three targets of one box agreed about ninety-odd lines and differed in four. Every change had to be +made three times, correctly, and a divergence nobody intended stayed invisible until a user hit it. + +A scroll may now be split: `scrolls//scroll.json` holds what the targets share, and each +`scrolls///scroll.json` declares `extends` plus its own differences. The two halves +are joined before anything else happens, and that joined result — the effective scroll — is what the +schema validates, what the build reads, and what provenance records. + +**Rejected:** a free path for `extends`. A path parameter invites traversal screening, chains of +bases, and a scroll that reaches outside its workspace. The value is fixed at `"../scroll.json"`, so +the base is always the box directory's own file: nothing to get wrong, and one level rather than a +hierarchy. + +**Rejected also:** generating the target files from one command and leaving them independent +afterwards. That solves writing them once and nothing else; the duplication returns at the first +edit, which is where it actually hurts. + +#### The join rule is per field + +A single blanket rule is wrong in both directions. Replacing everything makes a fragment that adds +one asset lose the shared ones. Merging everything leaves `execution` half from each half — a +`python-script` carrying a `module` inherited from the base, an object no author wrote. + +So: scalars and the cohesive objects (`target`, `execution`, `parity`) are replaced. Payload entry +lists and string lists are joined base-first. `compatibility` and `environment` are joined key by +key, because both hold independent entries that a base and a target legitimately contribute to — a +shared floor plus a macOS-only one, shared variables plus a CUDA-only one. The extra self-test Python +is one slot with two spellings, so a fragment naming either replaces both. + +The two list rules differ deliberately. A prune path or an import repeated by both halves is the same +instruction twice, so the repeat is dropped. A `relativePath` claimed by both is two different +sources for one file in the box — the second would silently overwrite the first — so it is an error. +**Rejected:** resolving that conflict by precedence. A rule saying which source wins is a rule +nobody remembers at the moment it matters, and the loser vanishes without a word. + +Order is declaration order, base first, and nothing is sorted. Determinism asks that one pair of +files always produce one result, which declaration order already gives. The visible consequence is +stated rather than hidden: a split scroll and a hand-written whole one hold the same entries, while a +joined map may serialise its keys in a different order. **Rejected:** sorting the joined keys, which +would change the bytes of every box whose map was not already alphabetical, to fix nothing. + +### The values nobody can type are not typed + +An asset's `sizeBytes` and `sha256` cannot be known without fetching the file, so writing a scroll by +hand meant downloading it, hashing it and pasting two values per asset — for every target. `add +asset` fetches once and records what it found. This does not weaken the check it feeds: the +guarantee has always been that those values are pinned once and verified on every build, and that is +unchanged. What changes is who does the transcription. + +`add file`, `add dep`, `remove` and `edit scroll` follow from the same idea. **Rejected:** commands +that only add. A tool where arriving is a command and leaving is a hand edit has not removed the +hand edit, it has moved it. + +Every edit is atomic and then verified against the whole box, not just the file it touched: a base +and its fragments only mean something together, so an entry added to the base can collide with one a +fragment already declares. If the result would not load, the originals go back. **Rejected:** writing +first and reporting afterwards, which turns one bad command into a box nobody can build until +someone works out what changed. + +### `refresh` maintains pins; it does not launder them + +A `localFiles` pin says "this must not change without review". After a reviewed change the digest has +to move, and doing that by hand is the toil the pin never meant to impose — so `refresh` recomputes +it. + +A remote asset's hash is a different thing: it is what stands between a replaced upstream file and a +silently different box. **Rejected:** refreshing those the same way. If `refresh` re-fetched and +rewrote them, every substitution upstream would be adopted without a word and the next build would go +green — the protection removed by the command meant to maintain it. So the network is untouched +unless asked, a difference is reported and refused, and accepting it takes a separate, explicit +`--repin`. + +### A dependency is added to the manifest; the lock still pins it + +`add dep` writes `name = "*"` and lets `pixi.lock` — committed and reviewed — record the version +actually solved. **Rejected:** looking up the newest version and writing it into the manifest. That +puts a second, weaker pin beside the real one and leaves the two to drift, and it makes the same +command produce different manifests on different days. + +Importing a `requirements.txt` translates PyPI names to conda-forge where the tool is sure and +lowercases otherwise, and reports every rename and every skip. **Rejected:** a large mapping table +applied silently. A name guessed wrongly gives a lock that resolves and a box that cannot import what +it was built for — a failure that arrives long after the command that caused it. + +### Two committed Python versions instead of a lookup + +`new scroll` defaults to one minor behind the newest Python conda-forge publishes, and +`--python-version latest` resolves to the newest itself. Both are constants in the repository, moved +deliberately at release time by `npm run python:bump`. + +**Rejected:** resolving the newest Python on each invocation. That would make the same command +produce different scrolls in different months, which is precisely the variability a scroll exists to +eliminate. `latest` resolves once, at authoring time, and the resolved number — never the word — is +what the file records. + +**Rejected also:** defaulting to the newest release. conda-forge builds the heavy compiled packages +for a new minor months after the interpreter lands, so that default hands a first-time user a solve +that cannot succeed, with an error that says nothing about why. + +## Scrolls are grouped by box, then target + +The default layout is `scrolls///`. Both directory names are checked against the +meaningful fields in `scroll.json`: `boxId` and the canonical ID computed from `target`. This makes +all target variants of one box visible together without making a directory name the source of the +box's identity. + +`scrollId` is optional input. Release schema version 2 requires a provenance `scrollId`, so a scroll +that omits it derives the value deterministically as `-`. Source directories are +always nested; the flat v1 layout is deliberately not a compatibility path. + +At the CLI edge, a box name expands to its target scrolls and a terminal presents them as a +navigable menu. One target matching the current host may be the default; on macOS, Metal is the +explicit preference when CPU and Metal both match. A non-interactive process uses that same policy +and fails on any remaining ambiguity instead of silently choosing CPU or CUDA. + +For the mutating `lock` and expensive `build` commands, omitting the scroll entirely opens a +workspace-wide menu of complete references. This is interactive convenience, not a default: +non-interactive callers must name the scroll, and even a single discovered candidate still requires +terminal confirmation rather than being selected silently. + +**Rejected:** requiring `scrollId` to repeat the directory name. That check made the filesystem a +second identity layer and encouraged product-plus-machine directory names even though the scroll +already declares both facts. + +## The environment is declared; inheritance is reported, not policed + +A scroll may declare the string map its interpreter requires. The builder copies it into +`box.json` and the signed release, applies it to its own self-test and parity gate, and consumers +apply it over inherited host and caller values. Target validation controls remain last for the +accelerator checks. A wrong declared path therefore fails during the build instead of first failing +on a user's machine. + +The inherited environment is intentionally not filtered. Scrollcase is responsible for integrity, +verifiable declarations, and truthful diagnostics; it is not a sandbox for the developer or the +application launching a box. Consumers instead return a masked provenance report, and the CLI can +expand it with `--env-report`. Revealing inherited values requires the separate, deliberate +`--env-report-values` flag because a generic verbosity switch is routinely enabled in public CI +logs. + +The boundary is permanent: the declaration is part of the format and can be verified by any +implementation. The report is output from a particular consumer process and must never be +documented as a guarantee of the box. Starting the packed interpreter directly bypasses the report, +just as it bypasses every other consumer check. + +**Rejected:** a default-deny environment with a hand-maintained minimal base per platform. It would +silently make the tool a sandbox policy, and a mistaken Windows base could prevent the packed +interpreter from starting at all. + +### A declared variable is how a target keeps its accelerator promise + +A packed library can carry a backend the box never declared. conda-forge's `llama.cpp` for +`osx-arm64` ships the Metal backend even in its CPU build, and llama.cpp registers a Metal device +however the application configures offloading — so creating a context initialises a GPU backend +inside a box whose target says `cpu`, and a host where that initialisation fails takes the box down +with it. The target switches the backend off by declaring it (`GGML_METAL_DEVICES: "0"`), in the +target fragment rather than the base, and the box then does what its name says. + +The trade is real and worth stating: with no GPU backend registered, llama.cpp can no longer offload +large-batch matrix multiplication — prompt evaluation — to a device it was never going to hold +weights on. Token generation is unaffected. A box named for an accelerator it quietly uses anyway is +the worse side of that trade. + +**Rejected:** renaming the target to the accelerator, which promises hardware the box does not use +and moves the same failure to the first host without it. Also rejected: having the entrypoint pass +an empty device list through `ctypes`, which buries a library-version-specific patch in an example +whose purpose is to be read. + +## Provenance refuses to lie + +A box records the commit it was built from and whether that working tree was dirty, including +untracked files while respecting Git ignore rules. Building outside a git checkout fails rather +than inventing a revision, and building from a dirty tree requires +`--allow-dirty` and is recorded as `sourceTreeDirty: true` in the box itself. A build that cannot be +reproduced from its recorded revision says so. + +Rebuilding the same commit produces a byte-identical archive: timestamps are normalised, the build +time comes from the commit rather than the clock, and the channel cohort salt is derived from box +and version rather than randomly. + +## Documentation audit decisions (2026-07-26) + +The public-contract audit resolved six implementation choices: + +- The maintainer chose to preserve the existing privacy banner and analytics behavior. The linked + `/privacy` route documents that behavior; consent controls were not added. +- Public schema URLs are deterministic copies of `src/contract/schema/`, guarded byte for byte. +- Verification compares all security-, identity-, target-, asset-policy-, self-test-, and + provenance fields duplicated by schema version 2. +- Consumer self-test is documented as the signed import subset; scroll `pythonCode` and file + assertions stay builder-only until a future wire version can carry them. +- Scroll structure is validated at runtime from the shipped schemas by a dependency-free internal + validator before tool discovery or build-directory mutation. +- Asset resume is limited to retries within one download operation. There is no persistent cache + and the documentation makes that process boundary explicit. + +## The licence audit is derived from the lock + +The inventory is a pure function of the committed `pixi.lock`, which carries an SPDX licence per +package, and `pixi install --frozen` guarantees the installed set equals it. So `audit` runs without +building anything, and licence review can happen when dependencies change rather than at the end of a +multi-gigabyte build. A package with no declared licence fails the parse outright: an unlicensed +dependency is a legal problem, not a reporting gap. + +## Deliberately out of scope + +Publishing to object storage, downloading boxes, selecting or promoting a channel, updating an +installation, revoking a release, serving a registry, allocating CI runners, application lifecycle +policy, and model-specific scientific validation all belong to the consuming project. Scrollcase +stops at building a signed box or preparing and running caller-supplied local box inputs. + +The boundary is what keeps the guarantees provable. A packaging tool that also serves a registry has +to keep proving both sets of guarantees at once; one that stays local composes with any distribution +mechanism a project already has. diff --git a/docs/v2/concepts/index.md b/docs/v2/concepts/index.md new file mode 100644 index 0000000..2bea991 --- /dev/null +++ b/docs/v2/concepts/index.md @@ -0,0 +1,15 @@ +--- +title: Concepts +description: Understand the core architecture and mental models behind Scrollcase. +aside: false +next: false +prev: false +--- + +# Concepts + +Understand the core mental model of Scrollcase: from box determinism and signing custody to multi-platform parity and runtime security guarantees. + +--- + + diff --git a/docs/v2/concepts/security-and-trust.md b/docs/v2/concepts/security-and-trust.md new file mode 100644 index 0000000..a257dee --- /dev/null +++ b/docs/v2/concepts/security-and-trust.md @@ -0,0 +1,101 @@ +--- +title: Security and Trust +description: Threat model, trust anchors, verification order, key custody, rotation, and consumer responsibilities. +--- + +# Security and Trust + +Scrollcase protects the identity and bytes of a box after a project has decided what to build. It +does not decide whether a model is scientifically correct, whether a release should be promoted, +or whether a host is authorised to install it. + +## Threat model + +The format is designed to detect a modified or substituted archive, a modified signed document, +unsafe archive paths, a signer that returns a different payload, dependency or local-file drift, +and incomplete or corrupted downloaded assets. It records the source commit and dirty state so a +builder cannot silently present an uncommitted build as reproducible. + +The format does not protect a consumer that trusts an attacker's public key, a compromised signing +key, a malicious scroll approved by the project, or a client that skips compatibility, +revocation, rollout, and path-safety policy. Scrollcase is not a registry, transport, promotion +service, revocation authority, or host installer. + +## The five objects + +| Object | Role | Trust property | +| --- | --- | --- | +| archive | Deterministic ZIP containing the payload | Size and SHA-256 are committed by the release | +| `box.json` | Self-description inside the archive | Shared fields must agree recursively with the release | +| release | Immutable identity, target, archive digest, consumer import check, and provenance | Signed | +| channel | Mutable pointer to one or more releases | Signed independently | +| revocations | Signed statements identifying withdrawn releases | Defined and verifiable; distribution and enforcement are consumer policy | + +A signature is carried by the adjacent release document, not embedded in the archive. + +## Bootstrap a trust anchor + +Obtain the project's public key through a channel independent of the box download: a reviewed +source repository, managed device configuration, a pinned application release, or another +authenticated administrative path. Copy only the public key into a tracked project-owned trust +directory and pass it explicitly: + +```sh +scrollcase verify release.json --public-key trust/scrollcase-signing-public.json +``` + +Never treat a public key downloaded beside the archive as trusted merely because it arrived +together. Never commit the private key under `.scrollcase/keys/`. + +## Verification order + +`scrollcase verify` performs these checks in order: + +1. Decode the envelope, confirm its payload SHA-256, and verify at least one ed25519 signature + against the trusted key file. +2. Require a release document, resolve its exact target adapter, and validate its interpreter path. +3. Locate the archive and compare its byte size and SHA-256 with the signed release. +4. List ZIP entries defensively, rejecting traversal, links, and special entries before extraction. +5. Require `box.json` and recursively compare every shared schema-v2 field: identity and version, + complete target, entry point, cache subdirectory, declared environment, consumer self-test, + weights/assets policy, and provenance. +6. Require the declared interpreter entry inside the archive. +7. With `--self-test`, require a matching native host, extract to a temporary directory, compare + the logical extracted payload size, and run the signed import check with the box's interpreter + under the signed environment declaration and target validation controls. + +The process still inherits the launching host's environment. `run` and `verify --self-test` print a +compact report automatically when there is something actionable; `--env-report` expands it to every +name, while `--env-report-values` deliberately reveals inherited host values. Release-declared and +caller-supplied values are not masked. None of these diagnostics is a sandbox or a signed guarantee. +The declaration is authenticated format data. The report is local consumer output that changes with +the process inspecting or running the box. + +The builder's richer scroll checks—optional `pythonCode` and post-prune file assertions—are not +part of the signed release and therefore cannot be repeated by a consumer. See +[The Scroll](/v2/reference/scroll#self-test). + +## Key custody + +A local key is suitable for controlled development, but the private PEM remains on the build +machine. An external signer is the supported custody boundary for CI: Scrollcase sends the exact +payload on stdin, requires a complete signed envelope on stdout, checks that the returned payload +bytes are identical, and verifies the signature locally before continuing. + +For rotation, preserve and identify the outgoing public key first. Generate the incoming key at a +different explicit path, distribute a trust bundle containing both public keys, switch signing +only after consumers trust the incoming key, then retire the outgoing key after the compatibility +window. `keygen --force` is not rotation: it can destroy the only signing identity at a path and +invalidate every document that depended on it. + +If a key may be compromised, stop signing with it, distribute a new trust bundle through the +bootstrap channel, and let the consuming project publish and enforce revocation policy. Scrollcase +defines and verifies the document format; it does not distribute or globally enforce it. + +## Consumer responsibilities + +A conforming consumer still owns transport, archive storage, enough disk headroom, host +compatibility evaluation, safe final extraction, on-demand asset retrieval and hash verification, +channel freshness/anti-replay state, rollout selection, revocation enforcement, activation, and +rollback. If it cannot evaluate a compatibility constraint, it must refuse the box rather than +assume the constraint passes. diff --git a/docs/v2/concepts/tool-comparison.md b/docs/v2/concepts/tool-comparison.md new file mode 100644 index 0000000..911d036 --- /dev/null +++ b/docs/v2/concepts/tool-comparison.md @@ -0,0 +1,321 @@ +--- +title: Scrollcase vs Other Packaging Tools +description: Choosing between Scrollcase and other solutions and packaging tools. +aside: false +--- + +# Scrollcase *vs* Other Packaging Tools + +Complete comparison between Scrollcase and other packaging tools, understanding when Scrollcase is a better fit, and how it differs from Docker, Pixi, conda-pack, PEX, PyInstaller and other tools, and when a simpler solution is the better choice. + +**Scrollcase** primary goal is: + +> “Build a target-specific scientific models or AI runtime once, publish it as an immutable signed artifact, and let another application verify, install, and run it without resolving dependencies or requiring a container runtime.” + +## Tools and decision guide + + + + +### Scrollcase vs Pixi alone + +Pixi is the environment manager underneath Scrollcase. + +It is responsible for work such as: + +- declaring Conda and PyPI dependencies; +- resolving compatible packages; +- recording exact package builds in `pixi.lock`; +- installing and running project environments; +- supporting platform-specific environments and tasks. + +For many projects, that is the complete solution. + +A developer can commit `pixi.toml` and `pixi.lock`, ask another developer or CI runner to install Pixi, and recreate the environment locally. + +Scrollcase starts where that workflow stops. + +A Scrollcase consumer is not asked to install Pixi, resolve packages, or reconstruct an environment. It receives an already built artifact and verifies what it received before using it. + +Scrollcase adds: + +- installation strictly from the committed lock during the build; +- a target-specific, relocatable environment; +- deterministic archive construction; +- declared local files and verified assets; +- self-tests using the interpreter inside the box; +- source and lock provenance; +- dependency licence inventory; +- signed release metadata; +- content-addressed archives; +- safe extraction and execution through Node, Python, and Rust consumers. + +### Choose Pixi alone when + +- every machine may install Pixi; +- reconstructing the environment at installation time is acceptable; +- the environment is mainly for developers, notebooks, CI, or internal jobs; +- you do not need a signed distributable artifact; +- you do not need a stable contract between a publisher and an external consuming application. + +### Choose Scrollcase with Pixi when + +- the environment must be built once and delivered as an artifact; +- the end user should not resolve or install dependencies; +- the consuming application must verify the exact bytes it receives; +- builds need recorded provenance and reproducible output; +- the project distributes several operating-system or accelerator variants. + + + + +### Scrollcase vs conda-pack alone + +Scrollcase uses conda-pack, but is not only conda-pack. Conda environments are not generally relocatable by copying their directory, so conda-pack packages an existing environment and applies relocation logic. + +Scrollcase deliberately uses that implementation rather than inventing another environment packer. + +Using conda-pack directly can be the right design: + +```text +create environment +↓ +run conda-pack +↓ +upload archive +↓ +extract it elsewhere +``` + +Scrollcase turns that operation into a stricter release pipeline: + +```text +describe target and runtime +↓ +resolve and commit the exact lock +↓ +install and pack from that lock +↓ +stage code and verified assets +↓ +prune and repair the payload +↓ +self-test with the payload interpreter +↓ +build a deterministic archive +↓ +sign the release document +↓ +verify and consume it through a defined contract +``` + +### Choose conda-pack alone when + +- you already have a working Conda environment; +- a normal archive is sufficient; +- your deployment system owns all metadata, signing, validation, and extraction policy; +- byte-identical rebuilds and a public artifact contract are not requirements. + +### Choose Scrollcase when + +- the archive must be tied to a declarative source and committed lock; +- the release needs signed identity, hashes, provenance, and runtime metadata; +- hostile or malformed archives must be rejected before extraction; +- Node, Python, and Rust applications need the same documented consumption semantics; +- model files, licence inventory, and target metadata belong in the build contract. + + + + +### Scrollcase vs container systems (eg. Docker) + +Container systems like Docker packages an application and its runtime into a container image. A container runs as an isolated process through a container runtime and is a natural fit for services, servers, CI, orchestration, and container-native infrastructure. + +Scrollcase produces a host-native environment archive. + +A box is extracted onto the machine and its own Python interpreter is run directly. It does not provide: + +- process isolation; +- kernel namespaces; +- container networking; +- image layers; +- volumes; +- service orchestration; +- a registry; +- a daemon or container runtime. + +This difference is intentional. + +A desktop application may need to install a local scientific model and invoke it like an ordinary child process. Requiring Docker Desktop, a daemon, container permissions, image management, and host integration may be inappropriate for that product. + +Conversely, a backend service already deployed on Kubernetes usually benefits more from a container image than from a Scrollcase box. + +### Choose a container system when + +- the deployment environment already supports containers; +- process isolation is part of the requirement; +- the application is a service or infrastructure component; +- standard container registries and orchestration solve your distribution problem; +- including a Linux userland is acceptable. + +### Choose Scrollcase when + +- a desktop or local application must run Python directly on the host; +- installing a container runtime is undesirable; +- macOS Metal, Windows, or host-specific accelerator integration matters; +- the application wants to own download, installation, activation, rollback, and removal; +- the delivered environment must be verified independently of its download path. + +::: warning A box is not a sandbox +Signature and archive verification establish what was received. They do not make the Python code safe to execute. A consuming application must trust the publisher whose public key it accepts. +::: + + + + +### Scrollcase vs PEX + +Scrollcase is environment-oriented; PEX is Python-application-oriented. PEX builds executable Python environments from Python distributions. It is especially useful for Python applications and command-line tools that should be distributed as a single executable environment. + +Scrollcase has a different unit of delivery. + +A box contains a complete target-specific Python prefix built from Conda and PyPI dependencies. That makes it suitable for scientific stacks whose runtime may depend on: + +- a particular Python interpreter; +- Conda-provided native libraries; +- compiled extension modules; +- BLAS or other numerical libraries; +- accelerator-specific packages; +- files and model assets that are not Python distributions. + +Scrollcase also separates the runtime artifact from the application that consumes it. A desktop application can download and prepare a box, retain a verified receipt, and invoke a declared script or module when needed. + +### Choose PEX when + +- the deliverable is fundamentally one Python application or CLI; +- dependencies are naturally represented as Python distributions; +- PEX's execution and interpreter model fits the target systems; +- you do not need Scrollcase release, channel, asset, or consumer semantics. + +### Choose Scrollcase when + +- the deliverable is a reusable scientific or model runtime; +- Conda packages and native dependencies are first-class inputs; +- the box is installed and managed by another application; +- release identity, signatures, hashes, self-tests, and target metadata must travel together. + + + + + +### Scrollcase vs PyInstaller + +Scrollcase is not an application freezer like PyInstaller. PyInstaller analyzes a Python application and bundles it with the interpreter and dependencies needed to run it. It can produce a one-directory bundle or a single executable. + +That is often the most direct way to ship a Python desktop application. + +Scrollcase does not attempt to turn the model runtime into a native-looking executable. It preserves a real Python environment and exposes declared Python scripts or modules through a verified consumer. + +This is useful when the Python runtime is one component inside a larger product rather than the product's top-level executable. + +For example: + +```text +native or web-based desktop application +↓ +downloads the correct model box +↓ +verifies and prepares it +↓ +runs the declared Python entry point +↓ +handles UI, updates, storage, and lifecycle itself +``` + +### Choose PyInstaller when + +- you are shipping one Python application directly to the end user; +- a frozen executable or application bundle is the desired product; +- automatic import analysis and application-centric packaging fit the project; +- you do not need an independently installable model environment. + +### Choose Scrollcase when + +- the main application is not necessarily written in Python; +- Python is a managed runtime component of a larger application; +- several boxes or versions may be installed independently; +- the application needs explicit release documents and verification; +- the environment must remain inspectable as a normal Python prefix. + + + + + +### Scrollcase vs AppImage + +Scrollcase is not a Linux application format like AppImage. AppImage packages a Linux application and the dependencies that cannot be assumed to exist on the target system into one executable file. Users can download it, mark it executable, and run it without a traditional installation or root privileges. + +Scrollcase packages a different unit: a target-specific Python runtime intended to be verified, +prepared, and invoked by another application. It supports Linux, macOS, and Windows targets and +treats operating system, architecture, accelerator, dependency lock, release identity, and +verification metadata as part of the artifact contract. + +### Choose AppImage when + +- the deliverable is a complete Linux desktop application; +- one downloadable executable file is the desired user experience; +- support for macOS and Windows is handled through separate packaging formats; +- the application itself owns its top-level UI and lifecycle; +- a separately managed Python-runtime contract is unnecessary. + +### Choose Scrollcase when + +- Python is one runtime component inside a larger product; +- the consuming application must install or switch between several runtime boxes; +- CPU, CUDA, Metal, macOS, Windows, and Linux variants need explicit identities; +- signed release documents and independent archive verification are requirements; +- the runtime should remain separate from the application's own distribution format. + + + + +| Tool | Primary job | Best fit | What Scrollcase adds | +| --- | --- | --- | --- | +| [Pixi](https://pixi.sh/) | Resolve, lock, install, and run project environments | Development, CI, and reproducible environment management | Relocation, packaging, signed release metadata, deterministic archives, verification, and consumer APIs | +| [conda-pack](https://conda.github.io/conda-pack/) | Archive an existing Conda environment so it can be moved | Direct environment deployment with a small custom delivery layer | A declarative source, locked build pipeline, pruning, assets, provenance, signing, manifests, verification, and safe consumers | +| [Docker](https://docs.docker.com/get-started/docker-overview/) | Package and run applications as isolated containers | Services, infrastructure, reproducible server deployment, and container-native systems | Host-native execution without a container runtime, target-specific accelerator boxes, signed local artifacts, and application-owned installation | +| [PEX](https://pex.readthedocs.io/) | Build executable Python environments from Python distributions | Python applications and command-line tools distributed as executable environments | A complete Conda-based prefix, non-Python native dependencies, model assets, signed release documents, and a separate consumer contract | +| [PyInstaller](https://pyinstaller.org/en/stable/) | Freeze a Python application and its dependencies into an executable bundle | Shipping a standalone end-user application | A reusable environment box rather than one frozen application, plus locks, provenance, content addressing, release channels, verification, and consumer APIs | +| [AppImage](https://appimage.org/) | Distribute a Linux desktop application as one portable executable file | Shipping self-contained applications across Linux distributions without installation | Cross-platform scientific runtime boxes, dependency locks, target and accelerator metadata, signed releases, deterministic archives, and consumer APIs | + + + + + +## Next steps + +- Follow the [Quickstart](/v2/getting-started/quickstart) to build and run the example box. +- Read [Why Scrollcase](/v2/getting-started/why-scrollcase) to understand if Scrollcase is the right choice for your needs. +- Read the [Overview](/v2/getting-started/overview) for the complete developer and consumer workflow. +- Read [Architecture](/v2/concepts/architecture) to see how the builder, artifacts, publisher, and consumer fit together. +- Read [Security & Trust](/v2/concepts/security-and-trust) for the exact guarantees and non-guarantees. +- Read [Why Pixi & Conda-Forge](/v2/concepts/why-pixi) for the dependency substrate decision. + + + \ No newline at end of file diff --git a/docs/v2/concepts/why-pixi.md b/docs/v2/concepts/why-pixi.md new file mode 100644 index 0000000..2608d38 --- /dev/null +++ b/docs/v2/concepts/why-pixi.md @@ -0,0 +1,116 @@ +--- +title: Why Pixi & Conda-Forge +description: One substrate, chosen for native libraries, licence metadata, and a lock worth trusting. +--- + +# Why Pixi & Conda-Forge + +Scrollcase is built on exactly one substrate: **pixi solves a committed `pixi.lock` against +conda-forge, conda-pack relocates the resulting prefix, and the tree ships inside the box as +`venv/`.** There is no second dependency backend, and adding one is out of scope. + +This page explains what that choice buys, and what it costs. + +## One substrate, on purpose + +Supporting a second backend — `uv` with a standalone Python distribution, say — was considered and +rejected. + +A packaging tool's product is its **guarantees** — this environment installs, relocates, +self-tests, and is reproducible from a lock. Two backends means proving every guarantee twice, on +every platform, for every release: two relocation strategies, two lock formats, two licence +parsers, two sets of platform quirks. The guarantees are the product, so Scrollcase keeps exactly +one way of producing them. + +Projects already on `uv` convert their scrolls once. Scrollcase avoids a permanent double burden. + +## Why conda-forge + +**Native libraries are the actual problem.** Scientific stacks are mostly compiled code — BLAS, +CUDA runtimes, HDF5, Arrow, image and compression codecs — and the interesting failures are +linkage failures, not Python ones. conda-forge distributes that compiled code as a coherent, +centrally built package set with real dependency metadata for the native pieces, rather than as +wheels of varying provenance that each vendor their own copy of a shared library. + +**Licences come with the packages.** conda-forge records an SPDX licence per package, and it ends +up in the lock. That is what makes Scrollcase's [licence audit](/v2/reference/cli#audit) a pure +function of a committed file rather than a scraping exercise: the inventory can be produced, +reviewed and checked into a repository long before any box exists, and a package with no declared +licence fails the parse outright. + +**The interpreter is just another package.** Python itself comes from the same channel, solved +together with everything else, so there is no separate "which standalone Python build?" question +and no mismatch between the interpreter's ABI and the wheels around it. + +## Why pixi + +**A lock worth trusting.** `pixi.lock` pins every package, for the exact target platform declared +in the manifest, so resolution is independent of the machine doing it. Scrollcase splits that +into two verbs on purpose: + +```sh +scrollcase lock my-model/linux-x86_64-cpu # resolve — a human step, reviewed and committed +scrollcase build my-model/linux-x86_64-cpu # install --frozen — never resolves +``` + +`pixi install --frozen` installs exactly the locked packages without touching or re-checking the +lock, so **what ships is byte-for-byte what was reviewed**. + +**The resolver version is part of the scroll.** A scroll pins `pixiVersion`, and both `lock` and +`build` refuse any other version — a different resolver can select different packages and +silently change the box. Determinism is not just about the lock; it is about everything that +touched it. + +**One target per environment.** The manifest declares a single `platforms` entry matching the +box's target (`osx-arm64`, `linux-64`, `win-64`). One environment, one target, one box — nothing +multiplexed, nothing to disambiguate at install time. + +## Why conda-pack, and why `conda-unpack` is never run + +conda-pack produces a **ready-to-run tree**, so a consumer pays no install-time work beyond +extraction. That is what makes an install an unzip. + +Its embedded `conda-unpack` fixer is deliberately **not** run. Running it would stamp the build +machine's absolute paths into dozens of files that then ship to users — measured on a probe +environment, zero files carried the build prefix before running it and thirty-six after — leaking +a developer's directory layout while still being wrong at the user's install location. + +Instead the few service files that do carry the prefix are removed, symlinks are settled, +and generated console scripts are rewritten to resolve Python next to themselves. The result runs +from any location with no activation, no environment variables, and no fixer. Details in +[Architecture](/v2/concepts/architecture#relocation). + +### Rejected: pixi-pack + +`pixi-pack` ships *packages* rather than a tree, and needs a per-user install step plus a bundled +unpacker at the other end. Scrollcase's position is that the slow step — compression — is better +paid once by whoever builds than on every install, and that the fewer moving parts a consumer needs, +the better. An archive plus a hash is the smallest possible install contract. + +## What this costs + +Being honest about the trade-offs: + +- **PyPI-only packages need care.** pixi can install PyPI dependencies alongside conda ones, and + the licence parser reads both, but the further you go from conda-forge the weaker the native + dependency metadata gets. +- **Archives are large.** A full conda prefix with a CUDA stack is measured in gigabytes. Pruning + and `--weights on-demand` exist for this — see + [Managing Model Weights](/v2/guides/managing-weights). +- **Builds are native.** No cross-building: a Windows box is built on Windows. The self-test runs + the box's own interpreter, and that only proves anything on matching hardware. +- **Two tools must be present.** `pixi` at the scroll's pinned version, and `conda-pack` 0.9.2. + Scrollcase installs that exact conda-pack release itself; for externally managed executables, + `doctor` can confirm only that conda-pack runs because its version output is unreliable. + +## Not a container + +A box is not an image and there is no runtime. Nothing is layered, no daemon is involved, and +installing does not require root, a virtualisation stack, or a registry login. Containers solve +process isolation; Scrollcase solves *this environment, verifiably, on this machine* — including +machines where a container runtime is not available or not permitted. + +## Further reading + +- [Architecture](/v2/concepts/architecture) — how the pipeline holds its guarantees. +- [Design Decisions](/v2/concepts/design-decisions) — each decision with the alternative it rejected. diff --git a/docs/v2/demos/box-dev-demo.md b/docs/v2/demos/box-dev-demo.md new file mode 100644 index 0000000..a6e80ee --- /dev/null +++ b/docs/v2/demos/box-dev-demo.md @@ -0,0 +1,122 @@ +--- +title: Box development +description: Quick demo to learn how to develop a box with Scrollcase. +outline: [2,3] +--- + +# Box development demo + + **Build and verify a real box from an empty project** + +## Try it now + +See how to initialize, lock, sign, build and verify a Scrollcase box with our guided scenario, all in a disposable cloud Linux environment. Every Scrollcase command and its result are shown in the terminal. + + + +> All from your browser, no setup needed + +--- + +**Prefer a real development environment?** + +Open the demo in **GitHub Codespaces** to get an instant VM with a clean repository and an easy walktrough: + + + +> *Both paths perform a real Linux x86_64 CPU build. They download the project toolchain and +> locked Python environment, so allow a few minutes. Codespaces runs on your GitHub account.* + +## What the demo does + +The demo uses the disposable `example-box` created by `scrollcase init`. It contains only Python +and a small entry point, keeping the result easy to understand while still exercising the real +pipeline: + +```text +init → lock → commit → keygen → build → verify +``` + +The guided Killercoda scenario groups that path into four short steps: + +1. install the CLI and initialize the project-local toolchain; +2. resolve `pixi.lock` and commit the generated project; +3. create a local signing key and build the box; +4. verify the signed release and run its self-test with the box's own Python. + +Nothing is prebuilt. The background setup only prepares the disposable Linux machine, Node.js and +Git; the Scrollcase commands and their output remain visible. + +## Follow it in Codespaces + +The Codespace starts as an empty Scrollcase project inside a Git repository. Open its terminal and +follow the rendered README, or run the essential sequence directly: + +```sh +npm install --global scrollcase +scrollcase init --install-toolchain < /dev/null +scrollcase lock example-box/linux-x86_64-cpu + +git add . +git commit -m "Initialize Scrollcase example" + +scrollcase keygen +scrollcase build example-box/linux-x86_64-cpu --weights embed +scrollcase verify .scrollcase/dist/boxes/example-box/1.0.0/linux-x86_64-cpu/*.release.json --self-test +``` + +Redirecting `init` from `/dev/null` keeps this walkthrough non-interactive: the required toolchain +is installed because `--install-toolchain` explicitly authorizes it, while the optional Node, +Python, and Rust consumer packages are skipped. Their ready-to-customize templates are still written under +`consumer-templates/`. + +The commit is not ceremony. Every box records the exact Git revision it came from, and `build` +refuses a dirty tree unless that loss of reproducibility is explicitly accepted. + +::: warning Demo signing key +`scrollcase keygen` creates a local key for this disposable walkthrough. Its private half stays +under the ignored `.scrollcase/` directory. Production signing and key rotation need deliberate +custody — see [Signing & Key Custody](/v2/guides/signing-and-custody). +::: + +## What verification proves + +The final command checks the trusted signature, archive size and SHA-256, safe entry names, and +agreement between the signed release and the box manifest. `--self-test` then extracts the box to a +temporary directory and exercises its declared imports with the Python interpreter contained in +the box. + +At that point you have produced the two files a consumer needs: + +```text +.scrollcase/dist/boxes/example-box/1.0.0/linux-x86_64-cpu/ +├── .zip +└── .release.json +``` + +The archive is the box. Its signed release document identifies it and commits to its bytes; keep +them side by side so `verify`, `run`, or a consumer API can resolve the archive from the release. + +## Go further + +- Want only to verify and execute an already-built box? Try the [Box-run demo](/v2/demos/box-run-demo). +- To create real project metadata, targets, assets, and execution settings, use + [`scrollcase new scroll`](/v2/reference/cli#new). +- `doctor` and `audit` are intentionally outside this short demo; see the complete + [Quickstart](/v2/getting-started/quickstart) and [CLI reference](/v2/reference/cli). +- To run the result from an application, start with the generated templates and the + [Library APIs reference](/v2/reference/api). diff --git a/docs/v2/demos/box-run-demo.md b/docs/v2/demos/box-run-demo.md new file mode 100644 index 0000000..b04aa67 --- /dev/null +++ b/docs/v2/demos/box-run-demo.md @@ -0,0 +1,239 @@ +--- +title: Box-run +description: Try a box run, without installing a toolchain. +outline: [2,3] +--- + +# Box-run demo + + **Run a demo box easily, without installing any toolchain** + +## Try it now + +See how to run a Scrollcase box with our guided scenario, all in a disposable cloud Linux environment. Every Scrollcase command and its result are shown in the terminal. + + + +> All from your browser, no setup needed + +--- + +#### Prefer a real development environment? + +Open the demo in **GitHub Codespaces** to get an instant VM with the repository ready to run and an easy walktrough: + + + +> *Runs the Linux x86_64 CPU demo using your GitHub Codespaces account.* + + +## Local setup + +Building a box needs pixi and conda-pack. But **consuming does not**:
+If you only want to see what a box is, and how to run it, try this public demo. +> You can find the demo box **GitHub release** [here](https://github.com/suffro/scrollcase/releases/tag/demo-box-v1). + +### Downloads + +Download the demo for your system: + +|macOS (Metal)|Linux (CPU)|Windows (CPU)| +|--|--|--| +|[`macos-aarch64-metal`](https://github.com/suffro/scrollcase/releases/download/demo-box-v1/hello-box-1.0.0-macos-aarch64-metal.zip)|[`linux-x86_64-cpu`](https://github.com/suffro/scrollcase/releases/download/demo-box-v1/hello-box-1.0.0-linux-x86_64-cpu.zip)|[`windows-x86_64-cpu`](https://github.com/suffro/scrollcase/releases/download/demo-box-v1/hello-box-1.0.0-windows-x86_64-cpu.zip)| + + +::: tip NOTE + +The file you download (eg. hello-box-1.0.0-macos-aarch64-metal.zip) is **NOT** the demo box — it is a container, named so you can tell which machine it is for, holding the box together with two ready-to-run examples. + +The demo box is the .zip inside it under box/, next to its .release.json. Do not unzip that one: **it's ready to run**. Leave both named as they are and side by side, because that is how `verify` finds the box. + +::: + +### Run the box + +Once you have downloaded the demo, follow these steps: + +1. **Unpack the demo into a folder of its own:** + +```sh +unzip hello-box-1.0.0-.zip -d scrollcase-demo +cd scrollcase-demo +``` + +> **box/** holds 2 files: the **demo box** `.zip` to run, and its matching `.release.json`. Beside it +> you already have `run-box.ts` and `run_box.py` — nothing to retype.
+ +--- + +2. **Download the demo public key, next to the box rather than inside it:** + +```sh +mkdir keys +curl -o keys/example-signing-public.json \ + https://raw.githubusercontent.com/suffro/scrollcase/main/examples/keys/example-signing-public.json +``` + +or alternatively here's its GitHub link: [`example-signing-public.json`](https://github.com/suffro/scrollcase/blob/main/examples/keys/example-signing-public.json) + +::: tip Why the key lives outside the box +A signature only proves where something came from if the key does not travel with it. Keeping the +key in its own folder — and downloading it from the repository rather than the release — is the +habit to carry into a real project, where the key will not be a demo key. +::: + +--- + +3. **Verify and run the box:** + + + + +This path needs the CLI, and nothing else — no pixi, no conda-pack, no build: + +```sh +npm install -g scrollcase +``` + + + + +```sh +scrollcase verify box/*.release.json --public-key keys/example-signing-public.json +scrollcase run box/*.release.json --public-key keys/example-signing-public.json +``` + + + + +```powershell +scrollcase verify (Get-ChildItem box\*.release.json).FullName --public-key keys\example-signing-public.json +scrollcase run (Get-ChildItem box\*.release.json).FullName --public-key keys\example-signing-public.json +``` + + + + +> The `box/*.release.json` above is a real shell glob, not a placeholder — your shell replaces +> it with the one release document in the folder. PowerShell does not expand globs for a command like +> this, which is why it needs `Get-ChildItem`. Either way you can always type the file name you see +> after unzipping. You never name the box archive: `verify` finds it beside the release document, +> under the hash that document commits to. + +#### What just happened + +`verify` checks the signature, the archive's size and hash, the entry names and manifest agreement, +and works on any machine. `run` extracts the box to a temporary directory and executes its entry +point with the interpreter *inside* it — so it needs a machine matching the box's target. The box +output makes that outcome explicit instead of presenting its temporary extraction paths as the +demo: + +```text +Hello from inside a Scrollcase box! + + signed -> verified -> relocated -> running + +Success: the box's own Python runtime executed this program. +No dependencies were resolved or installed to make this run. + + Runtime Python 3.11.15 + Host Linux / x86_64 +``` + +The final two lines reflect the box you downloaded and the matching machine running it. + + + + + +#### Run it from your own app + +The CLI is the quickest way to see a box work, but an application does not shell out to it: every +consumer exposes the same verify-then-run semantics as a library. `run-box.ts` and `run_box.py` are +already in the folder you unpacked, so this is two commands, not a copy-paste. + + + + +```sh +npm install +npx tsx run-box.ts +``` + +::: details run-box.ts — the file you just ran +<<< @/../examples/demo-consumers/run-box.ts +::: + + + + +```sh +python -m pip install scrollcase-consumer +python run_box.py +``` + +::: details run_box.py — the file you just ran +<<< @/../examples/demo-consumers/run_box.py +::: + + + + +`runBox` verifies the signature, extracts to a private temporary directory, executes, and cleans up +after itself — the same chain `scrollcase run` performs, minus the terminal. `onPrepared` fires +after verification and before execution, which is how an application shows what it is about to run +without repeating the trust chain itself. + +Neither file names the box archive. Both find the release document by its suffix and let the +consumer resolve the archive beside it, under the hash that document commits to. + +The Python package and the Rust crate are published separately: `npm install scrollcase` installs +neither, and `pip install scrollcase-consumer` or `cargo add scrollcase-consumer` needs no Node at +all. There is no Rust file in the folder, but the same two calls verify and run this same box from a +native application. Full surface in the [Library APIs reference](/v2/reference/api). + + + + +::: warning The demo key is a demo key +Those boxes are signed with a key that exists only for the example. It signs nothing else and no +trust chain depends on it. A signature from it means the example is intact — nothing more. +::: + + +--- + +4. **Check out the results, that's it.** + +At this point the folder looks like this — the box untouched in its own directory, the key in +another, the runnable examples above both: + +```text +scrollcase-demo/ +├── box/ # from the download, left exactly as it arrived +│ ├── .zip +│ └── .release.json +├── keys/ # step 2, from the repository — never from the release +│ └── example-signing-public.json +├── run-box.ts # from the download +├── run_box.py # from the download +├── package.json # from the download +└── README.md # from the download +``` + +Everything except `keys/` came out of the one file you downloaded. The key is the deliberate +exception, and the reason is above. diff --git a/docs/v2/demos/index.md b/docs/v2/demos/index.md new file mode 100644 index 0000000..69bb933 --- /dev/null +++ b/docs/v2/demos/index.md @@ -0,0 +1,13 @@ +--- +title: Demos +description: Interactive, hands-on demos that showcase Scrollcase features and common use cases to help you learn by example. +aside: false +next: false +prev: false +--- + +# Try a demo + +Explore our collection of easy interactive demos to see Scrollcase in action. Each demo showcases different features and use cases to help you understand how to leverage Scrollcase in your projects. + + diff --git a/docs/v2/demos/llm-box-demo.md b/docs/v2/demos/llm-box-demo.md new file mode 100644 index 0000000..d2f4f60 --- /dev/null +++ b/docs/v2/demos/llm-box-demo.md @@ -0,0 +1,274 @@ +--- +title: Local LLM +description: Build a real use-case box with SmolLM2-1.7B-Instruct, a language model that runs with no network, no API key and no account. +outline: [2,3] +--- + +# Local LLM demo box + +Model: *[SmolLM2-1.7B-Instruct (GGUF Q4_K_M)](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct)* + +--- + +A large language model is the thing behind a chat assistant: you give it text, it continues it. Every +one you have used through a website answers somewhere else, on someone else's hardware, behind an +account and an API key. + +This demo takes SmolLM2-1.7B-Instruct, quantised to 4 bits in GGUF form, and ships it as a signed, +self-contained box that answers on your own machine. + +```text +$ scrollcase run .scrollcase/.../*.release.json \ + -- "What is the capital of Italy?" + +Rome. +``` + +::: info You define what the box should contain, Scrollcase handles it: +No Python environment to prepare, no model to download at run time, no container. The model and +everything it needs are inside the box, which runs it in its own environment. +::: + +There is no network call in that command, no key, and no account — and unlike a hosted assistant, +nothing about the question leaves the machine. + +## Try the demo + +Download a signed box and run it in a minute, or package one yourself in a Codespace. Both end with +the same box; only one of them asks you to build it. + + + + +### Build it yourself + +The demo repository is almost empty on purpose: you package the model yourself, and its README is +the walkthrough. Install the CLI, initialise the workspace, create the scroll, declare the model's +pinned file, lock, commit, sign and build. Longer than the [sentiment demo](/v2/demos/sentiment-demo): +most of the wait is the 1.06 GB model, fetched once to pin its hash and once to build. + + + + + + +### Download the prebuilt box + +Signed boxes for Linux, macOS and Windows are published on the Scrollcase repository. Fetch the public key from the repository, verify, run. This path needs neither pixi nor a build, and nothing is downloaded while the box runs. + +Download the one matching your machine: + +|macOS (Apple silicon)|Linux (CPU)|Windows (CPU)| +|--|--|--| +|[`macos-aarch64-cpu`](https://github.com/suffro/scrollcase/releases/download/llm-demo-v1/llm-demo-1.0.0-macos-aarch64-cpu.zip)|[`linux-x86_64-cpu`](https://github.com/suffro/scrollcase/releases/download/llm-demo-v1/llm-demo-1.0.0-linux-x86_64-cpu.zip)|[`windows-x86_64-cpu`](https://github.com/suffro/scrollcase/releases/download/llm-demo-v1/llm-demo-1.0.0-windows-x86_64-cpu.zip)| + + + +::: tip NOTE + +The file you download is **NOT** the box — it is a container, named so you can tell which machine it +is for, holding the box together with two ready-to-run examples. The box is the .zip +inside it under box/, next to its .release.json. Do not unzip that one: +it's ready to run. Leave both named as they are and side by side, because that is how `verify` finds +the box. + +::: + +Unpack it, fetch the key beside it rather than inside it, then verify and run: + +```sh +unzip llm-demo-1.0.0-.zip -d llm-box-demo +cd llm-box-demo +mkdir keys +curl -o keys/example-signing-public.json \ + https://raw.githubusercontent.com/suffro/scrollcase/main/examples/keys/example-signing-public.json + +npm install -g scrollcase +scrollcase verify box/*.release.json --public-key keys/example-signing-public.json +``` + +```sh +scrollcase run box/*.release.json --public-key keys/example-signing-public.json \ + -- "What is the capital of Italy?" +``` + +A sentence is answered once. The same command with no sentence opens an interactive +[chat](#two-modes-one-box) instead: + +```sh +scrollcase run box/*.release.json --public-key keys/example-signing-public.json +``` + +`run-box.ts` and `run_box.py` ship in the same folder and reach both modes from Node and from +Python: `npx tsx run-box.ts "Who wrote the Divine Comedy?"`, or nothing after it for the chat. + +> `box/*.release.json` is a real shell glob, not a placeholder. PowerShell does not expand it +> for a command like this, so use `(Get-ChildItem box\*.release.json).FullName` or type the file name +> you see under `box/`. You never name the box archive: `verify` finds it beside the release +> document, under the hash that document commits to. + + + + +## Two modes, one box + +Give it words and it answers once. Give it nothing and it opens a chat: + +```text +$ scrollcase run .scrollcase/.../*.release.json + +loading smollm2-1.7b-instruct-q4_k_m.gguf … +ready in 1.4s · 2 threads · 2048-token context +/exit or Ctrl-D to quit, Ctrl-C to cancel an answer +> what is a hash function? +generating … +A hash function maps data of any size to a fixed-size value. … +> give me an example of one +``` + +Shape of a session, not a recording. + +The second question has no subject in it, and it still works: the box keeps the conversation and +sends it back each turn, on a single load of the weights. That is the difference between a chat and +a loop that calls a one-shot twice — and on CPU, where loading a gigabyte is a visible cost, it is +also the difference between the box feeling like a program and feeling like a conversation. + +Nothing is rebuilt to get it. Same release file, same signature, same entrypoint: the mode is +decided by whether there are arguments, because `execution.defaultArgs` is `[]` and a bare `run` +therefore reaches the box with an empty argument list. A `--chat` flag would have had to be declared +in the scroll and signed into the release. + +## What the demo shows + +**A signed release, verified before execution.** The release document commits to the archive by size +and SHA-256, and the archive's filename *is* its content hash. `verify` checks the signature against +a public key you obtain independently of the download. The mechanics are the same as any other box; +what changes here is what the guarantee covers, which is a whole language model rather than a +configuration file. + +**The whole model is one file.** A GGUF is a single container holding the weights, the tokenizer +*and* the chat template, so the scroll declares exactly **one** asset where the sentiment demo needs +three — and there is no tokenizer that can drift out of step with the weights it belongs to. It is +pinned to an immutable upstream commit, with the size and SHA-256 that `add asset` recorded when it +fetched the file. With `weights: embed` it is packed into the archive, so the box installs and runs +air-gapped. + +**Offline because there is no downloader, not because a variable says so.** The sentiment demo +declares `HF_HUB_OFFLINE=1` and two siblings as defence in depth, and it needs them: a Hugging Face +client really is present in its environment, pulled in transitively. This stack has none. +`entrypoint.py` imports `llama_cpp` and nothing else, and what keeps the box offline is that there is +no code in it that could phone home. Copying those variables across would have looked reassuring and +guaranteed nothing, so they are deliberately absent. + +**The one environment variable that does earn its place.** `PYTHONDONTWRITEBYTECODE=1`, because a +`.pyc` carries a timestamp. Without it the self-test's own `import entrypoint` writes one into the +payload before the payload is hashed, and a box that was extracted, run, and verified again fails the +second verification — twice defeated by a cache file nobody asked for. + +**A self-test that has to actually generate.** It loads the gigabyte with the box's own interpreter +and asserts that the answer to *What is the capital of Italy?* contains `rome`. Greedy decoding +(`temperature=0.0`) is what makes that reproducible enough to assert on content rather than merely on +the model having emitted something — and it asserts a substring, not a sentence, so a llama.cpp point +release that rewords the answer does not fail a build for a reason nobody cares about. + +**One declared dependency is not one dependency shipped.** The scroll declares +`llama-cpp-python` and nothing else. What arrives with it is the compiled `llama.cpp` — and also +`fastapi`, `uvicorn`, `pydantic-settings`, `numpy` and `diskcache`, because the upstream package +ships an OpenAI-compatible server this box never starts. Every one of them appears by name in the +licence inventory that `audit` derives from the lock file, which is the point of having one. + +## What you write + +The walkthrough packages a single target, so unlike the sentiment demo there is no +[split scroll](/v2/reference/scroll#one-box-several-targets) — just one directory: + +```text +scrolls/llm-demo/ + linux-x86_64-cpu/ + scroll.json # identity, the asset, the notices, the environment, the self-test + pixi.toml # python 3.11 + llama-cpp-python + self_test.py # the one file you open in an editor +``` + +`scrollcase new scroll` asks eight questions and writes the rest. The model and runtime identity, the +box version, the pixi version and the interpreter path are all defaults worth taking; what it cannot +guess is the target, what the box is called, which revision of the model is inside, where you will +publish it, and what runs when someone starts it. + +No hash is typed by hand anywhere. `scrollcase add asset` fetches the GGUF once and records the size +and SHA-256 it found; the notices and the entrypoint are pinned the same way, and +[`scrollcase refresh`](/v2/reference/cli#refresh) moves those digests after a reviewed change. + +The scroll declares `weights: embed`, a **4 GB RAM floor** and `execution` as a `python-script`. That +floor is arithmetic rather than a guess: the quantised weights occupy about 1.0 GB and the attention +cache adds 384 MiB at the 2048-token context the entrypoint asks for, which lands around 1.5–1.8 GB +resident. It is a fact a consumer can check *before* unpacking a gigabyte. + +The packaged version of the same box, the one CI builds for all three operating systems, is +`examples/llm-demo/` in the Scrollcase repository. There the three targets *do* share a +[split scroll](/v2/reference/scroll#one-box-several-targets): one base carrying the identity, the asset, +the environment and the self-test, and three target files of nine lines each — twelve on macOS, which +switches the packaged Metal backend off with `GGML_METAL_DEVICES=0` so that a box named `cpu` is +one ([why that is not automatic](/v2/guides/troubleshooting#running-a-box)). Its `entrypoint.py` is byte +for byte the one the walkthrough ships, and a test asserts the declared hashes still match, so the +two copies cannot drift apart quietly. + +## Measured + +The box has been built, self-tested, verified and run on all three CPU targets — Linux, macOS and +Windows — by the workflow that publishes it. Each one loads its own gigabyte with its own interpreter +and has to answer *What is the capital of Italy?* with Rome before it is allowed to be signed. + +On an M1 MacBook Air the published archive is 1.16 GB, unpacks to 1.3 GB, loads in two to four +seconds and generates about 13 tokens per second on eight threads. + +## What to expect + +**Running one:** + +- **Generation is CPU-bound.** On the 2 vCPU a default Codespace gives you, expect single-digit + tokens per second, so a long answer takes tens of seconds. Output is capped at 160 tokens, and a + `generating …` line on stderr keeps the wait from looking like a hang. +- **The context is 2048 tokens**, shared between the conversation and the answer. In chat mode the + oldest exchanges are dropped when it fills, and the box says so on stderr rather than failing. + +**Building one:** + +- **About 2.1 GB is downloaded** — the 1.06 GB GGUF once when `add asset` records its hash, and + again when the build fetches it. Fast inside a Codespace, but worth knowing before you start. +- **5–6 GB of disk** goes to environment, payload, archive and downloads, against a Codespace's 32 + GB. It fits; a second build in the same session does not leave much room. + +## Scope and limitations + +This is a **demonstration of packaging**, not an assistant product, and the model is small: 1.7 +billion parameters. It states false things fluently and gives no signal that it is doing so, it knows +nothing of events after its training data, and it cannot reliably do arithmetic or cite sources. + +The 4-bit quantisation is a **lossy** transformation of the original bfloat16 checkpoint — it is what +turns 3.4 GB of parameters into a 1.06 GB file that loads on a laptop, and it shifts outputs +unevenly. A prompt the original answers correctly is not guaranteed to be answered correctly here, so +every limitation the model card documents applies at least as strongly to this box. + +It was trained primarily on **English** and its outputs reflect the biases of that data. Do not use +it for factual lookup, for decisions about people, or for anything you would not check yourself. + +- [Original model card and limitations](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct#limitations) +- [GGUF conversion, pinned revision](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF/commit/2d4a76a30b4af41ecd395c35725ac11688d4cfe4) + +Both the original checkpoint and the GGUF conversion are published by the same upstream party under +Apache-2.0, and the box ships the full licence text next to the model notice. diff --git a/docs/v2/demos/sentiment-demo.md b/docs/v2/demos/sentiment-demo.md new file mode 100644 index 0000000..3f85aa8 --- /dev/null +++ b/docs/v2/demos/sentiment-demo.md @@ -0,0 +1,195 @@ +--- +title: Sentiment analysis +description: Build a real use-case box with DistilBERT sentiment analysis model. +outline: [2,3] +--- + +# Sentiment Analysis demo box + +Model: *[DistilBERT SST-2 (ONNX INT8)](https://huggingface.co/distilbert/distilbert-base-uncased-finetuned-sst-2-english)* + +--- + +Sentiment analysis reads a piece of text and judges the opinion in it. This model is binary: it says +whether a sentence reads as positive or negative, and how confident it is about that. + +This demo takes DistilBERT, fine-tuned on SST-2 and quantised to INT8 in ONNX form, and ships it as a +signed, self-contained box. + +```text +$ scrollcase run .scrollcase/.../*.release.json \ + -- "This product is surprisingly easy to use." + +Sentiment: POSITIVE +Confidence: 99.9% +``` + +::: info You define what the box should contain, Scrollcase handles it: +No Python environment to prepare, no model to download at run time, no container. The model and everything it needs are inside the box, which runs it in its own environment. +::: + +## Try the demo + + + + +### Build it yourself + +The demo repository is almost empty on purpose: you package the model yourself, and its README is +the walkthrough. Install the CLI, initialise the workspace, create the scroll, declare the model's +pinned files, lock, commit, sign and build. About fifteen minutes, most of it spent waiting on the +build. + + + + + + +### Download the prebuilt box + +Signed boxes for Linux, macOS and Windows are published on the Scrollcase repository. Fetch the public key from the repository, verify, run. This path +needs neither pixi nor a build, and nothing is downloaded while the box runs. + +Download the one matching your machine: + +|macOS (Apple silicon)|Linux (CPU)|Windows (CPU)| +|--|--|--| +|[`macos-aarch64-cpu`](https://github.com/suffro/scrollcase/releases/download/sentiment-demo-v1/sentiment-demo-1.0.0-macos-aarch64-cpu.zip)|[`linux-x86_64-cpu`](https://github.com/suffro/scrollcase/releases/download/sentiment-demo-v1/sentiment-demo-1.0.0-linux-x86_64-cpu.zip)|[`windows-x86_64-cpu`](https://github.com/suffro/scrollcase/releases/download/sentiment-demo-v1/sentiment-demo-1.0.0-windows-x86_64-cpu.zip)| + + + +::: tip NOTE + +The file you download is **NOT** the box — it is a container, named so you can tell which machine it +is for, holding the box together with two ready-to-run examples. The box is the .zip +inside it under box/, next to its .release.json. Do not unzip that one: +it's ready to run. Leave both named as they are and side by side, because that is how `verify` finds +the box. + +::: + +Unpack it, fetch the key beside it rather than inside it, then verify and run: + +```sh +unzip sentiment-demo-1.0.0-.zip -d sentiment-demo +cd sentiment-demo +mkdir keys +curl -o keys/example-signing-public.json \ + https://raw.githubusercontent.com/suffro/scrollcase/main/examples/keys/example-signing-public.json + +npm install -g scrollcase +scrollcase verify box/*.release.json --public-key keys/example-signing-public.json +scrollcase run box/*.release.json --public-key keys/example-signing-public.json \ + -- "This product is surprisingly easy to use." +``` + +The sentence is an argument because the box declares no default one: without it the box answers with +a usage line on stderr rather than classifying something you did not ask about. `run-box.ts` and +`run_box.py` ship in the same folder and pass it for you — `npx tsx run-box.ts`, or with a sentence +of your own as their first argument. + +> `box/*.release.json` is a real shell glob, not a placeholder. PowerShell does not expand it +> for a command like this, so use `(Get-ChildItem box\*.release.json).FullName` or type the file name +> you see under `box/`. You never name the box archive: `verify` finds it beside the release +> document, under the hash that document commits to. + + + + +## What the demo shows + +**A signed release, verified before execution.** The release document commits to the archive by +size and SHA-256, and the archive's filename *is* its content hash. `verify` checks the signature +against a public key you obtain independently of the download. + +**The model travels inside the box.** The ONNX weights, the tokenizer and the config are declared +as assets pinned to an immutable upstream commit, each with its size and SHA-256. The build fetches +them once and fails if a byte moved. With `weights: embed` they are packed into the archive, so +the box installs and runs air-gapped. + +**Defence in depth against a stray download.** The scroll declares `HF_HUB_OFFLINE=1`, +`TRANSFORMERS_OFFLINE=1` and `TOKENIZERS_PARALLELISM=false`, and those values are signed into the +release and override the host environment. This matters in practice: `tokenizers` pulls +`huggingface_hub` in transitively, so a downloader *is* present in the environment — the guarantee +comes from the entrypoint importing no client and from the signed environment, not from absence. + +**A locked, audited environment.** `lock` resolves the four conda dependencies into a `pixi.lock`, +and the build installs only from it — nothing is resolved while building. `audit` derives the +licence inventory from that same lock, and the build recomputes it and fails on any difference, so +a dependency whose licence changed is caught when it changes rather than at the end of a +multi-gigabyte build. + +**A self-test at two levels.** `selfTest.imports` is the part schema v2 signs, which is why +`verify --self-test` can repeat it later with the box's own interpreter. `files` and the optional +`pythonCode` block stay builder-only: add `pythonCode` and the build runs real predictions and +refuses to sign a box that answers wrong. Proof of real inference for a box you downloaded is what +`run` gives you. + +**More than one way to call it.** The CLI; the `run-box.ts` and `run_box.py` that ship beside the +downloaded box; and, in a workspace of your own, the Node, Python and Rust consumer templates `init` +writes under `consumer-templates/`. Whichever starts it, the two lines above are the whole of stdout +— progress, diagnostics and failures go to stderr — so `… > verdict.txt` is a file with the verdict +in it and nothing else. + +## What you write + +Three CPU targets that package the same model agree about everything except the target itself, so +the demo is a [split scroll](/v2/reference/scroll#one-box-several-targets): + +```text +examples/sentiment-demo/ + scroll.json # everything the three targets share + shared/ # entrypoint, model notice, Apache-2.0 text + linux-x86_64-cpu/scroll.json # extends + target + its licence audit path + macos-aarch64-cpu/scroll.json + windows-x86_64-cpu/scroll.json +``` + +The base carries the identity, the three model files as commit-pinned assets with their sizes and +SHA-256, the offline environment, the self-test and the licence files to carry into the box. Each +target file is nine lines. A change to the model is one edit, not three. + +Beside every target sits its own **`pixi.toml`** — four conda dependencies: `python`, +`onnxruntime`, `tokenizers`, `numpy` — because the solved environment is what genuinely differs. +`lock` and `audit` then write `pixi.lock` and `conda-licenses.json` next to it. + +No hash is typed by hand anywhere. `scrollcase add asset` fetches each model file once and records +the size and SHA-256 it found; the notices and the entrypoint are pinned, and +[`scrollcase refresh`](/v2/reference/cli#refresh) moves those digests after a reviewed change. + +The scroll declares `weights: embed`, a 2 GB RAM floor, and `execution` as a `python-script`. + +## Measured + +The box has been built, self-tested, verified and run on all three CPU targets — Linux, macOS and +Windows — and a rebuild produces a byte-identical archive. On Apple Silicon the archive is about +192 MiB, almost all of it model. It answers the sentence above `POSITIVE` at 99.9% confidence, and +*This was a frustrating and disappointing experience.* `NEGATIVE` at 100.0%. + +## Scope and limitations + +This is a **demonstration of packaging**, not a sentiment product. The model reads short **English** +sentences and answers `POSITIVE` or `NEGATIVE`; input beyond 128 tokens is truncated. Its model +card documents biases inherited from the training data, including predictions that differ +systematically for sentences mentioning underrepresented populations, and INT8 quantisation does +not remove them. Do not use it for decisions about people. + +- [Original model card and limitations](https://huggingface.co/distilbert/distilbert-base-uncased-finetuned-sst-2-english#risks-limitations-and-biases) +- [ONNX conversion, pinned revision](https://huggingface.co/onnx-community/distilbert-base-uncased-finetuned-sst-2-english-ONNX/commit/fd49941c1b822846cb14970cdf430a7cfbe0f5b9) + +The original checkpoint is Apache-2.0; the community conversion is attributed separately, and every +box ships the full licence text next to the model notice. diff --git a/docs/v2/getting-started/index.md b/docs/v2/getting-started/index.md new file mode 100644 index 0000000..dd6f4d3 --- /dev/null +++ b/docs/v2/getting-started/index.md @@ -0,0 +1,15 @@ +--- +title: Getting Started +description: Everything you need to go from an empty repository to a signed, reproducible box on disk. +aside: false +next: false +prev: false +--- + +# Getting Started + +Welcome to Scrollcase. Learn how to go from an empty repo to a fully configured, locked, and cryptographically verified self-contained box ready for distribution. + +--- + + \ No newline at end of file diff --git a/docs/v2/getting-started/installation.md b/docs/v2/getting-started/installation.md new file mode 100644 index 0000000..e84b953 --- /dev/null +++ b/docs/v2/getting-started/installation.md @@ -0,0 +1,271 @@ +--- +title: Installation +description: Install the Scrollcase CLI, and the pixi + conda-pack toolchain real builds need. +--- + +# Installation + +Scrollcase is a Node.js command line tool. The CLI itself has no native dependencies; building a +box for real additionally needs `pixi` and `conda-pack` on the machine that builds — and +`scrollcase init` can install those for you, after asking. + +The Python and Rust consumers are separate distributions. Installing `scrollcase` through npm +provides neither the `scrollcase_consumer` Python module nor the `scrollcase-consumer` crate. + +## Requirements at a glance + +| You want to… | You need | +| --- | --- | +| Scaffold, audit, keygen, or verify without a self-test | Node.js ≥ 20 | +| Resolve a lock (`lock`) | Node.js ≥ 20 and `pixi` at the scroll's pinned version | +| Build a box (`build`) | Node.js ≥ 20, pinned `pixi`, conda-pack, and a local key or external signer | +| Verify with `--self-test` | The same OS and architecture the box targets | +| Consume an existing local box from Python | Python ≥ 3.10 and `scrollcase-consumer` | +| Consume an existing local box from Rust | Rust ≥ 1.88 and the `scrollcase-consumer` crate | + +Auditing, key generation, signing primitives, and verification need no dependency toolchain. +`lock` invokes pixi; `build` invokes both pixi and conda-pack. + +## Install the CLI + +```sh +npm install -g scrollcase +``` + +Check the install: + +```sh +scrollcase --version +``` + +## Install consumers + + + + +### Install the Node consumer + +The Node consumer needs the scrollcase package to be installed, therefor if you already installed the scrollcase CLI, you do not have to install anything else, otherwise [install the CLI](#install-the-cli) first. + +The import name: + +```js +import { runBox } from 'scrollcase/consumer'; +``` + +The Node consumer prepares and executes release documents and archives already present on the local machine. Every path and trust anchor comes from the caller. It never selects a channel, downloads an archive or asset, installs globally, updates an existing destination, or applies application lifecycle policy. + + + + +### Install the Python consumer + +A Python application that only verifies and runs caller-supplied local boxes does not need the +Scrollcase CLI or a Node.js runtime: + +```sh +python -m pip install scrollcase-consumer +``` + +The import name uses an underscore: + +```python +from scrollcase_consumer import run_box +``` + +This package does not build or download boxes. The publishing project builds the box; the consuming +application supplies the local release document, archive, and trusted public key. + +When `scrollcase init` creates the consumer templates, it can perform this installation for you. +It asks separately from the build toolchain and can use either PyPI with pip or conda-forge: + +```sh +conda install --yes --channel conda-forge scrollcase-consumer +``` + +The TypeScript and Rust templates have their own optional prompts. If approved, `init` runs npm from +the same project root to install `scrollcase`, `typescript`, and `tsx`, and Cargo against the +generated Rust manifest to add `scrollcase-consumer`. `init` collects every answer before starting +any installation. If pip reports a PEP 668 externally managed interpreter, `init` +automatically retries as a user install, keeping package files outside the managed Python prefix. +If you select conda-forge but the `conda` command is unavailable, it asks whether to continue with +PyPI instead. + + + + +### Install the Rust consumer + +A Rust application — a Tauri desktop client, a native service — consumes local boxes without a Node +or Python runtime: + +```sh +cargo add scrollcase-consumer +``` + +The crate name uses a hyphen and the import name an underscore: + +```rust +use scrollcase_consumer::run::run_box; +``` + +Like the Python package it builds and downloads nothing. When `scrollcase init` creates its +non-publishable template crate, it can run the same `cargo add` command against that crate's own +manifest after asking `Install scrollcase-consumer for Rust?`. If Cargo is not installed, `init` +keeps the template, skips the question, and prints the command to run later instead of failing. + + + + +> Checkout the [Library APIs](/v2/reference/api.md) section for more details. + +## Let Scrollcase install the toolchain + +`scrollcase init` initializes a workspace and then **offers** to install what is missing: + +```text +Install pixi and conda-pack into /work/my-project/.scrollcase/toolchain? +This project needs them to build a box: + ↳ [Y/n] +``` + +Nothing is downloaded before you answer, and the interactive default is yes. Say yes (or press +Enter) and Scrollcase installs +both **inside the project**, under `.scrollcase/toolchain/` — nothing is added to `PATH`, nothing +is installed system-wide, and deleting the directory undoes it. Later commands find the tools +there on their own. + +What you get is verified, not just fetched: + +- the release archive's SHA-256 is checked against the checksum pixi publishes beside it, and a + mismatch aborts before anything is installed; +- the verified digest is recorded under `toolchain` in `scrollcase.config.json`, so the next + machine — a teammate's, a CI runner's — is checked against the value **your project committed** + rather than whatever the server offers that day; +- the generated example and managed toolchain share the same pixi pin; every project scroll created + by `scrollcase new scroll` still declares its own exact `pixiVersion`. + +For unattended setups, answer up front: + +```sh +scrollcase init --install-toolchain # install without asking +scrollcase init --no-install-toolchain # never install; just report what is missing +scrollcase init --no-example # initialize without example-box, without being asked +``` + +With no terminal to prompt — CI, a pipe — Scrollcase never installs anything and simply reports +what is missing. Silence is not consent. + +::: tip Pin the version you want +`--pixi-version 0.73.0` uses exactly that release for both the generated example and an approved +managed install. With `--no-example`, omitting the flag uses the installed release or newest +available release for the workspace toolchain. `new scroll` pins the pixi it finds installed, since +`build` refuses any other; pass `--pixi-version` to pin a different one. +::: + +## Install the toolchain yourself + +If you would rather manage the toolchain — a shared machine, a company mirror, an existing pixi +install — Scrollcase is happy to use it. Install both tools and skip the step above. + +### pixi + +[pixi](https://pixi.sh) solves and installs the conda-forge environment. Every scroll **pins the +exact pixi version** it was locked with (`pixiVersion` in `scroll.json`), and Scrollcase refuses +to run `lock` or `build` with any other version — a different resolver can select different +packages and silently change the box. + +Install it following the [pixi installation docs](https://pixi.sh/latest/#installation), for +example: + +```sh +curl -fsSL https://pixi.sh/install.sh | sh +``` + +If you need a specific release to match a scroll's pin, download the matching release from the +pixi GitHub releases page, or use the version-pinned form of the install script documented by +pixi. + +### conda-pack + +[conda-pack](https://conda.github.io/conda-pack/) turns the installed environment into a +relocatable tree. The recommended install is through pixi itself: + +```sh +pixi global install "conda-pack==0.9.2" +``` + +Scrollcase's managed installer uses this exact release. `conda-pack --version` currently reports +`0.0.0` regardless of the installed package release, so Scrollcase can pin what it installs but +cannot reliably validate the version of an executable supplied through a flag, environment +variable, or `PATH`. + +## Point Scrollcase at the toolchain + +If `pixi` and `conda-pack` are on `PATH`, nothing more is needed. If they live elsewhere — a +dedicated toolchain directory, a CI cache — point Scrollcase at them per invocation: + +```sh +scrollcase build my-box/linux-x86_64-cpu \ + --pixi /opt/toolchain/bin/pixi \ + --conda-pack /opt/toolchain/bin/conda-pack +``` + +or once, through the environment: + +```sh +export SCROLLCASE_PIXI=/opt/toolchain/bin/pixi +export SCROLLCASE_CONDA_PACK=/opt/toolchain/bin/conda-pack +``` + +A `--pixi` / `--conda-pack` flag wins over the environment variable, which wins over the +project-local toolchain, which wins over `PATH`. + +## Upgrade Pixi intentionally + +Changing resolver versions is a dependency change, not a tool repair: + +1. edit the scroll's `pixiVersion` to the intended release; +2. initialise or install that exact version with explicit consent, or point `--pixi` at it; +3. run `scrollcase lock ` and review the new `pixi.lock`; +4. run `scrollcase audit ` and review/write any intentional licence change; +5. commit the scroll, lock, audit, and toolchain digest; +6. rebuild the box. + +Do not delete the pin and accept whichever resolver happens to be newest. + +## Check the machine + +`doctor` reports whether this machine can build, and says exactly what to do about anything +missing. It only reads; it never writes and never touches the network. + +```sh +scrollcase doctor --pixi-version 0.73.0 +# or take the required pixi version from a scroll: +scrollcase doctor --scroll my-box/linux-x86_64-cpu +``` + +Sample output: + +```text +ok workspace config /work/my-project/scrollcase.config.json +ok scrolls /work/my-project/scrolls +ok git HEAD 3f9c2ab17d42 +ok pixi pixi at 0.73.0 +ok conda-pack conda-pack +``` + +Every check reports rather than aborting at the first failure, so a machine missing both tools +learns both in one run. + +::: tip Builds are native +A box is always built on the OS and architecture it ships for: macOS arm64 boxes on an Apple +Silicon Mac, Linux x86_64 boxes on Linux, Windows boxes on Windows. There is no cross-building — +the self-test runs the box's own interpreter, which only proves anything on matching hardware. +::: + +## Next + +Continue with the [Quickstart](/v2/getting-started/quickstart) to initialize a workspace, author a +scroll, and build your +first box. diff --git a/docs/v2/getting-started/overview.md b/docs/v2/getting-started/overview.md new file mode 100644 index 0000000..9d2a359 --- /dev/null +++ b/docs/v2/getting-started/overview.md @@ -0,0 +1,269 @@ +--- +title: Overview +description: What Scrollcase is for, what a box contains, and the shape of the workflow — in one page. +--- + +# Overview + +## The main concept + +- Scrollcase packs an entire Python environment and the code it runs — like an **LLM** or a **scientific model** — into a single, **self-contained**, **portable** and **signed** archive: a **box**. + +- You give that box to someone else. They unpack it and run it — **that's it!** **Nothing to install**: no Python, no pip install, no compiler, no Docker, **no dependencies to maintain**. + +- Every box is **signed**, so whoever receives it can check that it is exactly the one you built and not something that changed on the way over. + +- Builds are **deterministic**: rebuilding the same commit gives the same bytes back — anyone can reproduce what you shipped. + +This is the whole idea. + +## The problem it removes + +Getting a Python runtime onto someone else's machine normally means asking them to rebuild your +environment: the right Python, the right libraries, the right native builds for their CPU or GPU, +the right weights downloaded from the right place. It works until it doesn't — and it breaks on +their machine, not yours. Scrollcase moves that work to build time, once, on a machine you control, +and turns the result into a file. + +> The rest of this page is a quick overview of how it works + +## Four words + +| Word | Meaning | +| --- | --- | +| **scroll** | The file you write: dependencies, model files, what to run, how to test it. The only input a build accepts. → [reference](https://scrollcase.dev/v2/reference/scroll) | +| **box** | What comes out: one archive with the whole environment inside. → [format](https://scrollcase.dev/v2/reference/box-format) | +| **target** | Which machine it is for: operating system, CPU architecture, accelerator (and CUDA version). One box, one target. | +| **release** | The signed document that describes the box, so a consumer can verify it. → [security model](https://scrollcase.dev/v2/concepts/security-and-trust) | + +## What is inside a box + +| Entity | What's inside | +| --- | --- | +| **Python interpreter** | The exact version you chose. The host does not need Python at all. | +| **Every dependency** | Conda and PyPI packages, native libraries included, at the versions your lock file pinned. | +| **Your code** | Application files, an entry script or module to start. | +| **Model files** | Embedded in the archive, or kept outside it with their size and hash recorded. | +| **Signed metadata** | What this box is, what it contains, and its digest — so a consumer can reject anything else. | +| **Licence inventory** | Every dependency's licence, derived from the lock, not guessed. | + +A box is built for **one target**: one operating system, one CPU architecture, one accelerator. +`macos-aarch64-metal` and `linux-x86_64-cuda12` are two boxes, not one box with options. That is +deliberate — a box that promised to work everywhere would have to decide things at install time, +which is the problem being removed. + +## Who does what + + + + +The person or team packaging the thing. + +- describes the environment in a **scroll**; +- declares dependencies, files, and model assets; +- chooses the target; +- runs `lock` and `build`; +- publishes the resulting files wherever they like. + + + + +The build. + +Creates the environment from the lock, downloads and hash-checks declared assets, makes the tree +relocatable, copies your files in, runs the tests the scroll declares, produces the archive, +computes the hashes, writes the release documents, and signs them. + +The developer does not run those steps one by one, and does not repair environment paths, write +manifests, or sign files by hand. + + + + +The application that installs and uses the box. + +- picks the right box for the user's machine; +- downloads it; +- hands the local release, archive, and trust keys to a conforming consumer; +- owns updates, activation, rollback, and removal. + +The official Node, Python, and Rust consumers verify, safely extract, and run a local box the caller +already holds. They do not choose channels and they do not download. + + + + +## The workflow + +Six steps, once. After that, shipping a new version is usually a single `build`. + +### 1. Initialise the workspace + +```bash +scrollcase init +``` + +Creates the project structure and — after asking, defaulting to yes — a disposable runnable +`example-box` for your own machine, a short `SCROLLCASE.md`, and TypeScript, Python, and Rust +examples under `consumer-templates/`. Pass `--no-example` for an empty workspace. + +### 2. Create a scroll + +```bash +scrollcase new scroll +``` + +Asks four questions — target, box id, the upstream revision of what you are packaging, and where +boxes will be published — and writes one target-specific `scroll.json`, its `pixi.toml`, and a +starter `self_test.py`. Nothing existing is overwritten. To just look around first, use the example +`init` created. → [Scroll reference](/v2/reference/scroll) + +### 3. Declare what goes in + +Everything the scroll declares is added by command, not by hand-editing files: + +```bash +scrollcase add dep my-model onnxruntime # a dependency +scrollcase add asset my-model https://…/model.safetensors # downloads once, records size and hash +scrollcase add file my-model runtime/entrypoint.py # a file from this project +``` + +`remove`, `edit scroll`, and `refresh` are the counterparts. +→ [CLI reference](/v2/reference/cli#add) + +### 4. Lock the versions + +```bash +scrollcase lock my-model/macos-aarch64-metal +``` + +Resolves the dependencies once into a `pixi.lock` you commit to Git. From then on the build +*installs* — it never resolves — which is what makes two builds of the same commit produce the same +bytes. + +Re-run `lock` when a dependency changes, and only then. → [Why pixi](/v2/concepts/why-pixi) + +### 5. Get a signing key + +```bash +scrollcase keygen +``` + +Creates the key pair used to sign releases. The private key never goes into the repository; the +public key goes to the consuming application, which is how it can tell your box from anyone else's. +Real key custody — a KMS, an HSM, a signing service — plugs in instead of the local key. +→ [Signing and key custody](/v2/guides/signing-and-custody) + +### 6. Build + +```bash +scrollcase doctor --scroll my-model/macos-aarch64-metal # optional: can this machine build it? +scrollcase build my-model/macos-aarch64-metal +``` + +The build must run on a machine compatible with the target. It produces the box archive, the signed +release and channel documents, and a publication-ready directory tree. + +If a declared import fails, an asset hash does not match, or a parity check breaches its tolerance, +there is no box. A failed gate never produces a signed artefact. + +## Running a box + +For a one-shot run from the terminal: + +```bash +scrollcase run ./release.json --archive ./box.zip -- --help +``` + +It verifies first, runs the signed script or module without a shell, preserves the child's exit +status, and removes its temporary extraction. + +An application does the same thing through a library: the Node API at `scrollcase/consumer`, the +Python package `scrollcase_consumer`, or the Rust crate `scrollcase-consumer`. All three share the +same verification, safe extraction, execution, receipt, signal, cleanup, and on-demand asset +semantics, and none of them downloads anything. + +An application that keeps a box extracted across restarts re-attaches to it rather than unpacking +again. → [Library APIs](/v2/reference/api) · +[Keeping an extracted box](/v2/guides/distributing-boxes#keeping-an-extracted-box-across-restarts) + +## Publishing + +Scrollcase writes files and stops. Uploading them is yours to do — by hand, with a script, from +CI/CD, or through an object-storage pipeline. + +```text +Scrollcase builds the files +↓ +your deployment system uploads them +↓ +your application downloads, verifies, and runs them +``` + +This boundary is the reason the format works with object storage, GitHub Releases, a private +server, or a desktop updater you already have. +→ [Distributing boxes](/v2/guides/distributing-boxes) + +## Shipping a new version + +| What changed | What to run | +| --- | --- | +| Code or included files | bump the version, `build` | +| Dependencies | `lock`, review the result, `build` | +| Model weights | update the asset in the scroll, bump the version, `build` | + +Several targets mean several builds, each on a machine compatible with its target — and that, rather +than Scrollcase itself, is usually where a wide platform matrix gets expensive. +→ [Platform examples](/v2/guides/platform-examples) + +## What the consuming application still owns + +Scrollcase builds the box; it does not implement the application around it. That application +detects the machine, chooses a release, downloads the manifests and archive, verifies the +signatures, checks runtime requirements, extracts the box, fetches any on-demand assets, starts the +runtime, and handles updates and uninstallation. + +The official consumers cover verification, extraction, and execution. Selection, download, and +update policy stay with the product — see [Why Scrollcase?](/v2/getting-started/why-scrollcase) for +why that line is drawn where it is. + +For the end user, the point is that none of it is visible: they pick a feature, press install, and +the application does the rest. + +## Is it complicated? + +Conceptually, no: + +```text +describe the environment +↓ +lock the versions +↓ +build the box +↓ +publish the files +``` + +In practice the difficulty comes from the environment you are packaging, not from the tool: +awkward native libraries, old scientific packages, several CUDA versions, very large weights, a +dependency that is not on a supported channel. Scrollcase does not make those disappear — it makes +them a build-time problem you solve once, instead of a support ticket from every user. + +## Try a demo + +Each demo showcases different features and use cases to help you understand how to leverage Scrollcase in your projects. + + + +## Next + +- [TL;DR](/v2/getting-started/tl-dr) — the same thing in a page you can read in a minute. +- [Quickstart](/v2/getting-started/quickstart) — build the example box now. +- [Why Scrollcase?](/v2/getting-started/why-scrollcase) — and when a simpler tool is the better choice. +- [Try a demo](/v2/demos/) — worked examples that show different features and use cases, and how to put Scrollcase to work in your own project. diff --git a/docs/v2/getting-started/quickstart.md b/docs/v2/getting-started/quickstart.md new file mode 100644 index 0000000..96163a5 --- /dev/null +++ b/docs/v2/getting-started/quickstart.md @@ -0,0 +1,265 @@ +--- +title: Quickstart +description: From an empty directory to a signed, verified box through workspace setup and guided authoring. +--- + +# Quickstart + +This walkthrough goes from an empty directory to a signed, verified box on disk. The guided +authoring step creates a library-only Python environment, so it remains small enough to inspect by +hand while exercising the complete packaging and signing pipeline. + +Prerequisites: the CLI and toolchain from [Installation](/v2/getting-started/installation). + +
+ +## Try a demo + +
+ +::: info Before that + +
+ +
+ +**You can try a Scrollcase demo** + +
+ +Each demo showcases different features and use cases to help you understand how to leverage Scrollcase in your projects. + + + + + + + +::: + +## 1. Create a project + +A box records the commit it was built from, so a Scrollcase project **must be a git checkout** — +building outside one fails rather than inventing a revision. + +```sh +mkdir my-boxes && cd my-boxes +git init +``` + +## 2. Install the CLI + +```sh +npm install -g scrollcase +``` + +Check the install: + +```sh +scrollcase --version +``` + +For more details check the [installation page](/v2/getting-started/installation). + + +## 3. `init` — initialize the workspace + +```sh +scrollcase init +``` + +`init` writes the workspace plus a disposable runnable example, and never overwrites anything that +already exists: + +- `scrollcase.config.json` — the [workspace declaration](/v2/reference/configuration): where scrolls + live and where builds, artefacts and keys go. +- `scrolls/example-box//` — a complete v2 scroll and pixi manifest for the native + host: Metal on Apple Silicon, CPU on Linux and Windows. +- `box-entrypoints/example-box//entrypoint.py` — the application executed inside + that box and target. +- `consumer-templates/run-box.ts` — a typed Node consumer using `scrollcase/consumer`. +- `consumer-templates/run_box.py` — the equivalent Python consumer using `scrollcase_consumer`. +- `consumer-templates/rust/` — the equivalent Rust consumer as a small Cargo crate. +- `package.json` — when absent, a private Node package with `"type": "module"` for the TypeScript + consumer; an existing package file is never overwritten. +- `SCROLLCASE.md` — a short project-local workflow guide linked to the full documentation. +- `.gitignore` rules for `.scrollcase/`, the regenerated build state that must never be + committed. + +Then, if `pixi` or `conda-pack` is missing, `init` **asks** whether to install it: + +```text +Install pixi and conda-pack into /work/my-boxes/.scrollcase/toolchain? +This project needs them to build a box: + ↳ [Y/n] +``` + +Answer yes and both land inside the project, with the pixi download checksum-verified and +conda-pack pinned to 0.9.2. Answer no and nothing is downloaded — install them yourself as +described in [Installation](/v2/getting-started/installation). Either way `init` never downloads +anything you did not agree to, which is what makes it safe to re-run. + +Because the example includes consumer templates, `init` asks separately whether to install +`scrollcase`, `typescript`, and `tsx`, whether to install the Python `scrollcase-consumer` package, +and whether to add the Rust crate to the generated Cargo manifest. For Python you choose PyPI with +pip or conda-forge with conda. It collects all answers before starting any installation, with a +blank line separating each question. Interactive yes/no questions default to yes (`[Y/n]`); +without a terminal these optional installs remain no. If Conda is unavailable after selecting +conda-forge, a separate default-yes question offers to continue with PyPI. + +Use `--install-toolchain` or `--no-install-toolchain` to answer up front in a script. The example +itself is the first thing `init` asks about, defaulting to yes; answer no, or pass `--no-example`, +when an explicitly empty workspace is preferable. + +It finishes with: + +```text +✓ Workspace initialized +→ Example: scrollcase lock example-box/macos-aarch64-metal +→ Create your own: scrollcase new scroll +``` + +## 4. `new scroll` — optionally author your own target + +```sh +scrollcase new scroll +``` + +The generated example is already ready for the remaining walkthrough steps, so you can skip this +command for a first build. Use the wizard for real project metadata. It asks four questions — the +complete target, the box id, the upstream revision of what you are packaging, and the base URL +boxes will be published under — plus menus for weights mode and execution kind. Each one is printed +as its own block: a blank line, the field's name, one line saying what the field is, then the answer +typed after ` ↳ `. Everything else has a default and is available as a flag. A blank answer to a +required question repeats it rather than ending the session. + +It creates `scrolls///` with `scroll.json`, the matching `pixi.toml` and a starter +`self_test.py`, then prints the exact reference to use next. + +For CI or another non-terminal caller, provide the equivalent flags shown by +`scrollcase help`. Missing input that has no default fails before any file is written. + +## 5. `doctor` — check the machine + +```sh +scrollcase doctor --scroll example-box/macos-aarch64-metal +``` + +Every failing check comes with a remedy. Fix what it names and re-run; `doctor` never modifies +anything, so it is always safe. + +::: info Scroll references +The exact reference is `/` under `scrolls/` — here +`example-box/macos-aarch64-metal`, assuming an Apple Silicon Mac. Substitute the example reference +printed by `init`, or the reference printed by `new scroll`, throughout. You may also pass `example-box --target +macos-aarch64-metal`. +::: + +## 6. `lock` — resolve dependencies, once + +```sh +scrollcase lock example-box/macos-aarch64-metal +``` + +`lock` runs the pinned pixi against the scroll's `pixi.toml` and writes `pixi.lock` next to it. +This is the only step that resolves anything: `build` later *installs* exactly what the lock +pins, and never resolves. Commit the lock — it is what makes a build reproducible and what the +licence audit reads. + +```sh +git add . && git commit -m "Example box scroll and lock" +``` + +Committing now also matters for the next steps: `build` refuses a dirty tree without +`--allow-dirty`, because an artefact built from uncommitted changes is reproducible by nobody. + +After the build, the three templates under `consumer-templates/` show how an application can run the +local signed release through the Node, Python, or Rust public consumer API. Replace the `` +and `` placeholders in the chosen template, then follow its setup and run instructions. The +corresponding consumer package must be installed in the application that runs the template. + +For Python, npm does not install `scrollcase_consumer`. The generated template includes the complete +setup; the equivalent commands are: + +```sh +python -m pip install scrollcase-consumer +python consumer-templates/run_box.py +``` + +A Python consumer-only application does not need the Scrollcase CLI or Node.js. + +For Rust, the generated crate includes the equivalent commands: + +```sh +cargo add --manifest-path consumer-templates/rust/Cargo.toml scrollcase-consumer +cargo run --manifest-path consumer-templates/rust/Cargo.toml +``` + +## 7. `keygen` — create a signing key + +```sh +scrollcase keygen +``` + +This writes a private ed25519 key (`.scrollcase/keys/signing-private.pem`, owner-only +permissions) and the matching public key file (`signing-public.json`). Every document the build +emits is signed; `verify` checks signatures against the public key file. For production custody — +a KMS, an HSM — see [Signing & Key Custody](/v2/guides/signing-and-custody). + +## 8. `build` — install, self-test, archive, sign + +```sh +scrollcase build example-box/macos-aarch64-metal +``` + +The pipeline, in order: install the locked environment, pack and relocate it with conda-pack, +stage declared assets, prune, self-test **with the interpreter inside the box**, normalise +timestamps, zip deterministically, and sign. The result lands in `.scrollcase/dist/`: + +```text +.scrollcase/dist/ +├── boxes/example-box/1.0.0/macos-aarch64-metal/ # upload this tree as it stands +│ ├── .zip # the box archive +│ └── .release.json # signed release document +└── channels/example-box/beta/macos-aarch64-metal.json # signed channel pointer +``` + +The build prints both paths and what to do with each. Files are named for their own hash because +that is the name they are published under — see [Distributing Boxes](/v2/guides/distributing-boxes). + +Rebuilding the same commit produces a byte-identical archive — see +[Architecture](/v2/concepts/architecture#determinism) for what makes that true. + +## 9. `verify` — prove what you built + +```sh +scrollcase verify .scrollcase/dist/boxes/example-box/1.0.0/macos-aarch64-metal/*.release.json --self-test +``` + +`verify` mirrors the format checks available to an installing client: trusted signature, archive +size and SHA-256, safe entry names, recursive agreement between `box.json` and the signed release, +and the declared interpreter. With `--self-test` it extracts to a temporary directory and imports +the signed modules **with the box's own Python**. Scroll-only `pythonCode` and file assertions ran +on the builder but are not carried by the signed release. + +## Where to go next + +- Package something real: declare model weights and data files — + [Managing Model Weights](/v2/guides/managing-weights). +- Understand every field you just used: [The Scroll](/v2/reference/scroll) and + [CLI Commands](/v2/reference/cli). +- Review dependency licences before building: run `scrollcase audit ` — see + [CLI Commands](/v2/reference/cli#audit). +- See how the whole pipeline fits together: [Architecture](/v2/concepts/architecture). + +The repository also ships a proven example, +[`examples/hello-box`](https://github.com/suffro/scrollcase/tree/main/examples/hello-box), with a +committed lock per target — the same walkthrough with nothing left to fill in, and a worked +[split scroll](/v2/reference/scroll#one-box-several-targets): one base file plus a short fragment for +each of its three targets. diff --git a/docs/v2/getting-started/tl-dr.md b/docs/v2/getting-started/tl-dr.md new file mode 100644 index 0000000..e4385a4 --- /dev/null +++ b/docs/v2/getting-started/tl-dr.md @@ -0,0 +1,121 @@ +--- +title: TL;DR +description: Scrollcase in a nutshell. +--- + +# TL;DR + +The Scrollcase mental model is simple: + +```text +write a scroll +↓ +define and lock dependencies +↓ +build the box +``` + +```mermaid +flowchart TB + subgraph Dev ["1. Developer Workspace"] + direction TB + A["Scroll & Code"] + B["Dependencies"] + C["Scrollcase CLI"] + end + + subgraph CLI ["2. Scrollcase Engine"] + direction TB + D["scrollcase lock"] --> E["scrollcase build"] + end + + subgraph Package ["3. Build Result"] + direction TB + F["Box (.zip)"] + G["Signed Manifest"] + end + + subgraph Consumer ["4. Consuming Application"] + direction TB + H["Download, Verify & Run"] + end + + Dev --> CLI + CLI --> Package + Package --> I["Distribution"] + I["Distribution"] --> Consumer + Consumer --> L["End User"] +``` + +## Initial setup + +1. run `npm install -g scrollcase` to install the [CLI](/v2/reference/cli) +2. run `scrollcase init` to create the workspace and runnable native example +3. optionally run `scrollcase new scroll` for real project metadata +4. review the selected [scroll](/v2/reference/scroll) +5. define the **dependencies** with `scrollcase add dep `, and declare the model files + with `scrollcase add asset ` +6. run `scrollcase lock /` +7. generate or configure the [signing key](/v2/guides/signing-and-custody) +8. run `scrollcase build /` + +## Normal update + +1. update code, version, weights, or dependencies +2. re-run `scrollcase lock /` only when required +3. run `scrollcase build /` + +## Responsibility split + + + + +Decides what the box contains, writes the scroll, defines dependencies, runs the commands, publishes releases, and implements integration in the application. + + + + +Builds the environment, downloads and verifies assets, prepares the box, runs tests, creates archives, generates manifests, calculates hashes, and signs the release. + + + + +Selects and downloads the box, supplies local inputs to a conforming consumer, and manages updates, +activation, rollback, and removal. + + + + +Uses the feature exposed by the application; does not interact with Scrollcase directly. + + + + + +The most demanding parts are usually: + +- defining a correct scroll; +- dealing with difficult scientific dependencies; +- integrating distribution and lifecycle policy with the Node or Python consumer. + +Conceptually, Scrollcase is therefore fairly linear: the developer declares the desired environment, and the tool turns that declaration into a distributable and verifiable release. + +## What Scrollcase simplifies + +Scrollcase removes much of the repetitive work required to turn a Python environment into a distributable product. + +Without a tool like this, the developer would have to manage: + +- environment creation; +- exact dependency versions; +- relocatability; +- native dependencies; +- verified downloads; +- manifests; +- signatures; +- hashes; +- release structure; +- tests; +- distribution conventions. + +With Scrollcase, these concerns are collected into a scroll and a small number of commands. diff --git a/docs/v2/getting-started/why-scrollcase.md b/docs/v2/getting-started/why-scrollcase.md new file mode 100644 index 0000000..a5e4a7a --- /dev/null +++ b/docs/v2/getting-started/why-scrollcase.md @@ -0,0 +1,126 @@ +--- +title: Why Scrollcase? +description: Understand when Scrollcase is useful, compare it and see how it differs from other tools and when a simpler solution is the better choice. +--- + +# Why Scrollcase? + +Scrollcase is not a replacement for every Python packaging or deployment tool. + +::: info Scrollcase is designed for a specific problem: +A project needs to deliver a complete, target-specific Python environment as a verifiable product artifact that can run on another machine without asking the end user to assemble that environment. +::: + +That means carrying more than application code. A box may include: + +- a specific Python interpreter; +- Conda and PyPI dependencies; +- native libraries; +- model code and supporting files; +- embedded or separately delivered model weights; +- target-specific CPU, CUDA, or Metal builds; +- signed metadata describing exactly what was built; +- enough information for a consumer to reject a corrupted, substituted, or incompatible artifact. + +Other tools solve important parts of this problem. Scrollcase exists to join those parts into one explicit build and consumption contract. + +## What it does not do + +Scrollcase stops at verified local artifacts and local consumption. + +It does not: + +- host archives; +- provide a package registry; +- detect which release an application should install; +- download boxes in the official consumers; +- manage application updates; +- activate or roll back installed versions; +- allocate build runners; +- provide container isolation; +- prove that a scientific model is correct. + +Those responsibilities remain with the product using Scrollcase. + +This boundary keeps the box format and verification logic usable with object storage, GitHub Releases, private servers, desktop updaters, or an organization's existing deployment system. + +## Decision and comparison guide + +Sometimes there might be a simplest tool that fits. If you only need a small reproducible development environment, use Pixi. If you only need to archive an existing Conda environment, conda-pack may be enough. If your users already run containers, Docker may be the better delivery format. Scrollcase is useful when the artifact itself must be portable, signed, inspected, verified, and consumed without a container runtime. + +Use **Scrollcase** when the goal is: + +> “Build a target-specific scientific models or AI runtime once, publish it as an immutable signed artifact, and let another application verify, install, and run it without resolving dependencies or requiring a container runtime.” + +### Other tools comparison and best fits + +| Tool | Primary job | Best fit | What Scrollcase adds | +| --- | --- | --- | --- | +| [Pixi](https://pixi.sh/) | Resolve, lock, install, and run project environments | Development, CI, and reproducible environment management | Relocation, packaging, signed release metadata, deterministic archives, verification, and consumer APIs | +| [conda-pack](https://conda.github.io/conda-pack/) | Archive an existing Conda environment so it can be moved | Direct environment deployment with a small custom delivery layer | A declarative source, locked build pipeline, pruning, assets, provenance, signing, manifests, verification, and safe consumers | +| [Docker](https://docs.docker.com/get-started/docker-overview/) | Package and run applications as isolated containers | Services, infrastructure, reproducible server deployment, and container-native systems | Host-native execution without a container runtime, target-specific accelerator boxes, signed local artifacts, and application-owned installation | +| [PEX](https://pex.readthedocs.io/) | Build executable Python environments from Python distributions | Python applications and command-line tools distributed as executable environments | A complete Conda-based prefix, non-Python native dependencies, model assets, signed release documents, and a separate consumer contract | +| [PyInstaller](https://pyinstaller.org/en/stable/) | Freeze a Python application and its dependencies into an executable bundle | Shipping a standalone end-user application | A reusable environment box rather than one frozen application, plus locks, provenance, content addressing, release channels, verification, and consumer APIs | +| [AppImage](https://appimage.org/) | Distribute a Linux desktop application as one portable executable file | Shipping self-contained applications across Linux distributions without installation | Cross-platform scientific runtime boxes, dependency locks, target and accelerator metadata, signed releases, deterministic archives, and consumer APIs | + +> for a complete comparison checkout this [page](/v2/concepts/tool-comparison). + +## When is a good fit + +- a desktop or local application embeds Python-powered features; +- scientific or AI dependencies include native Conda packages; +- releases must support CPU, CUDA, Metal, or multiple operating systems; +- environments or model assets are large enough that integrity and lifecycle matter; +- the publisher needs deterministic, content-addressed artifacts; +- consumers must verify signatures, archive bytes, paths, and manifests before execution; +- offline or air-gapped installation is required; +- the project wants Node, Python, and Rust consumers to follow the same contract. + +## When to prefer simpler tools + +- the environment is only for the project's own developers; +- users can install dependencies directly from a lock file; +- the deliverable is one small pure-Python command-line program; +- the deployment platform already requires Docker; +- the application does not need signed release metadata; +- a custom script around conda-pack already provides every required guarantee; +- reproducibility, provenance, and independent verification are not product requirements. + +Scrollcase intentionally accepts more structure than a plain archive. That structure is worthwhile only when the additional guarantees are requirements. + + +## Scrollcase terminology + +The terminology reflects separate responsibilities rather than extra steps the developer must perform manually. + +| Term | Meaning | Why it exists | +| --- | --- | --- | +| **Scroll** | The declarative source describing one box target | Separates project intent from the generated artifact | +| **Lock** | The exact resolved dependency graph in `pixi.lock` | Makes package selection reviewable and repeatable | +| **Target** | One operating system, architecture, and accelerator combination | A native environment cannot honestly be universal across incompatible platforms | +| **Box** | The built, self-contained runtime payload | This is the environment the consuming application installs and runs | +| **Release** | The signed document binding box identity to archive hashes and metadata | Lets consumers verify an artifact obtained through an untrusted transport | +| **Channel** | A signed pointer such as `beta` or `stable` to a release | Lets a publisher move an audience to a newer immutable release | +| **Key** | The signing identity trusted by the consumer | Establishes which publisher is allowed to issue releases | +| **Consumer** | The Node or Python code that verifies, extracts, and runs a local box | Keeps security checks consistent outside the builder | +| **Asset mode** | Whether model assets are embedded or materialized separately | Makes the archive-size versus offline-install trade-off explicit | + +The normal developer workflow is still small: + +```bash +scrollcase init +scrollcase lock my-box/macos-aarch64-metal +scrollcase build my-box/macos-aarch64-metal +scrollcase verify path/to/release.json --self-test +``` + +Most of the named objects are generated outputs or concepts used by the publishing and consuming systems. They are not nine unrelated configuration files the developer must maintain. + +## Next steps + +- Follow the [Quickstart](/v2/getting-started/quickstart) to build and run the example box. +- Read the [Overview](/v2/getting-started/overview) for the complete developer and consumer workflow. +- Read [Architecture](/v2/concepts/architecture) to see how the builder, artifacts, publisher, and consumer fit together. +- Read [Security & Trust](/v2/concepts/security-and-trust) for the exact guarantees and non-guarantees. +- Read [Why Pixi & Conda-Forge](/v2/concepts/why-pixi) for the dependency substrate decision. +- Read [Tool Comparison](/v2/concepts/tool-comparison) for a complete comparison between Scrollcase and other packaging tools. diff --git a/docs/v2/guides/accelerator-parity.md b/docs/v2/guides/accelerator-parity.md new file mode 100644 index 0000000..135d1bf --- /dev/null +++ b/docs/v2/guides/accelerator-parity.md @@ -0,0 +1,153 @@ +--- +title: Accelerator Parity +description: Require a box to compute the same thing on the GPU as on the CPU, within tolerances you declare. +--- + +# Accelerator Parity + +*Does this box compute the same thing on the GPU as on the CPU?* The question sounds scientific, +but it is a packaging question. It catches the failures a packaging tool is responsible for — the +wrong wheels solved in, a CPU-only build shipped as CUDA, a broken BLAS — and it catches them on +the build machine rather than on a user's. + +The division of labour is deliberate: + +| Scrollcase owns | Your project owns | +| --- | --- | +| Running the check once per accelerator, under each target's validation environment | What the check computes — which input, which tensor, which model | +| Comparing every run against the first | What closeness means for your model | +| Enforcing the tolerances you declared, and failing the build on a breach | The fixture, and reviewing the numbers | + +Scrollcase never decides what is scientifically correct. It enforces a threshold you wrote down. + +## Declaring the gate + +```jsonc +"parity": { + "script": "checks/parity.py", + "accelerators": ["cpu", "cuda"], + "tolerances": { "absolute": 1e-4, "relative": 1e-3, "minimumCosine": 0.9999 } +} +``` + +| Field | Meaning | +| --- | --- | +| `script` | A path **inside the box**, run with the box's own interpreter, from the payload root | +| `accelerators` | At least two. The **first is the reference**; every other run is compared against it. Conventionally `cpu`, being the one available everywhere and the least likely to be wrong | +| `tolerances` | At least one of `absolute`, `relative`, `minimumCosine` | + +Valid accelerator combinations follow the target: `["cpu", "metal"]` on macOS, `["cpu", "cuda"]` +on Linux and Windows. An accelerator the target defines no validation environment for is +rejected. + +## The check script + +The script ships inside the box — either produced by the environment, or copied in through +[`localFiles`](/v2/reference/scroll#localfiles): + +```sh +scrollcase add file my-model checks/parity.py --to checks/parity.py +``` + +```jsonc +"localFiles": [ + { "sourcePath": "checks/parity.py", "relativePath": "checks/parity.py", "sha256": "…" } +] +``` + +The `sha256` is optional, and a parity check is a good candidate for one: it decides whether a box +is numerically sound, so freezing it against an unreviewed edit is worth the pin. +[`scrollcase refresh`](/v2/reference/cli#refresh) moves the digest after a reviewed change. + +It must print **a JSON array of numbers**, or an object with a `values` array: + +```python +# checks/parity.py +import json, torch + +torch.manual_seed(0) # a fixed input: the comparison is between accelerators, +x = torch.randn(1, 64) # not between random draws + +model = load_model() # your model, loaded from the box's own weights +model.eval() +with torch.no_grad(): + out = model(x.to(model.device)) + +print(json.dumps({"values": out.flatten().tolist()})) +``` + +Two rules make the comparison meaningful: + +- **Deterministic input.** Seed it, or read a committed fixture. Comparing two different inputs + tells you nothing. +- **Let the environment choose the device.** Scrollcase sets `CUDA_VISIBLE_DEVICES` per run + (`""` for `cpu`, `"0"` for `cuda`; `PYTORCH_ENABLE_MPS_FALLBACK=0` for `metal`). Write the + script so it picks up what the environment offers rather than hard-coding a device. + +## How the comparison works + +The first accelerator is the reference. Every later accelerator is a candidate compared against +that same reference, in declaration order. + +For each run after the reference, three quantities are computed element-wise: + +| Measurement | Meaning | Bounded by | +| --- | --- | --- | +| Maximum absolute error | The largest `\|candidate − reference\|` | `absolute` | +| Maximum relative error | The largest `\|candidate − reference\| / \|reference\|`, **counted only where the reference has magnitude** | `relative` | +| Cosine similarity | Agreement in direction across the whole vector | `minimumCosine` | + +Relative error is meaningless around zero, so it is skipped there — the absolute bound is what +guards near-zero entries. Cosine similarity catches a result that drifted in direction rather +than magnitude, which element-wise bounds can miss. + +Outputs of differing length fail immediately, and **non-finite values are rejected explicitly**: +a `NaN` or an infinity is the classic symptom of a broken accelerator build, so it is reported as +such rather than allowed to poison the arithmetic. + +All declared tolerances are conjunctive. If `absolute`, `relative`, and `minimumCosine` are +present, the candidate must pass all three. Absolute and relative bounds are finite numbers greater +than zero; minimum cosine is finite and at most 1. Values below -1 are accepted by schema v1 but +are vacuous because cosine similarity cannot be lower than -1. Output must be a non-empty JSON array of +finite numbers, directly or under a `values` property. + +## Choosing tolerances + +There is no universally right answer, which is exactly why Scrollcase does not pick one. A useful +procedure: + +1. Build with a deliberately loose tolerance and read what the build reports. +2. Set the bound a little above the observed spread — tight enough to catch a real regression, + loose enough to survive legitimate floating-point differences between devices. +3. Record why you chose it, next to the scroll. + +The project must derive bounds from its workload, numeric precision, fixture, and scientific +requirements. Scrollcase cannot infer a safe threshold from the accelerator name. + +## When it runs, and what it prints + +Parity runs during `build`, **after** the self-test and on the same payload — there is no point +comparing accelerators in a box that cannot import its dependencies in the first place. On +success the build logs the comparison: + +```text +Parity passed on cuda against cpu +``` + +On a breach the build fails with the measurement and the bound it exceeded: + +```text +scrollcase: Parity check cpu vs cuda: maximum relative error 0.041 exceeds 0.001. +``` + +Measurements are used internally for the gate. The public CLI logs only a short success summary or +the first breached bound; it does not persist or sign the measurements, and `scrollcase/build` +does not export the build orchestrator. A project that needs retained scientific evidence must +record it in its own pipeline. + +## When not to use it + +Parity needs at least two accelerators available on the build machine — a CPU/CUDA gate needs a +GPU present. If a box has only one meaningful accelerator, or your CI cannot provide the second +device, leave `parity` out and lean on the self-test's `pythonCode` instead. The gate is optional +by design; a scroll without it is complete. diff --git a/docs/v2/guides/distributing-boxes.md b/docs/v2/guides/distributing-boxes.md new file mode 100644 index 0000000..edb2c56 --- /dev/null +++ b/docs/v2/guides/distributing-boxes.md @@ -0,0 +1,239 @@ +--- +title: Distributing Boxes +description: What a build hands you, how it is laid out, and where Scrollcase deliberately stops. +--- + +# Distributing Boxes + +Scrollcase stops at a signed, verified box on disk. Uploading it, serving a registry, promoting a +channel, revoking a release — all of that belongs to whoever consumes Scrollcase. This page +explains what the build hands you, why it is shaped that way, and how to build distribution on +top of it without fighting the format. + +::: info Why the boundary +A packaging tool that also serves a registry has to keep proving both sets of guarantees; one +that stops at a file on disk composes with any distribution mechanism you already have. Every one +of these features was left out deliberately, not overlooked. The reasoning is recorded in [Design Decisions](/v2/concepts/design-decisions). +::: + +## What a build produces + +```text +.scrollcase/dist/ +├── boxes/my-model/1.0.0/macos-aarch64-metal/ +│ ├── 7d2c9a41e8b350f6c174a9de20358bf41c6e97d05a8b3f2619e4c7081da5b3f2.zip +│ └── 4e81f0c93ab27d5e6081cf24b9a7d3e05f18c6b24a90d7e3518fc0a29b46d7e1.release.json +└── channels/my-model/beta/macos-aarch64-metal.json +``` + +Two directories, because there are exactly two things to do with a build. **`boxes/` is uploaded +verbatim**: it is laid out as the bucket already is, under the same keys the signed documents point +to, so publishing is a copy rather than a mapping. **`channels/` is separate** because a channel +belongs to a box rather than to any one version — the next release moves the pointer instead of +adding a second one beside it, which is why filing it under `1.0.0/` would leave a stale copy +claiming to be current. + +Nothing is written twice. What is on disk is what gets published, under the name it is published +under. + +## Content addressing + +Names derive from identity alone, so the archive, its release document and the staged objects +agree without any of them recording the others' paths: + +| Thing | Name | +| --- | --- | +| Object prefix | `boxes///` | +| Archive object | `/.zip` | +| Release object | `/.release.json` | +| Channel pointer | `channels///.json` | + +The chain is content-addressed end to end: **channel → release document (by its hash) → archive +(by its hash)**. Two consequences worth designing around: + +- **Publishing is idempotent.** Re-uploading the same build writes the same keys with the same + bytes. Combined with deterministic archives, a rebuild of the same commit is a no-op. +- **An object can never be replaced with different bytes under the same URL.** New bytes means a + new hash means a new key. Serve `boxes/` as immutable and cache it aggressively. + +The URLs inside the signed documents are `/`, so pointing +`assetBaseUrl` at wherever you serve `dist/boxes/` from is all the coordination needed. + +## Publishing + +Any object store or static host will do. The shape of the operation is: + +```sh +# Immutable objects — safe to cache forever. +aws s3 sync .scrollcase/dist/boxes/ s3://my-bucket/boxes/ \ + --cache-control "public, max-age=31536000, immutable" + +# The mutable pointer — short cache, uploaded last. +aws s3 sync .scrollcase/dist/channels/ s3://my-bucket/channels/ \ + --cache-control "public, max-age=60" +``` + +Two rules: + +1. **Objects first, pointer last.** The channel names a release document by hash; publishing the + pointer before the object it names gives clients a dangling reference. +2. **The channel key is yours to choose.** The build does not dictate where the channel document + lives — only the release document's URL, which it embeds. Pick a stable route your clients + already know how to reach. + +## Channels and rollout + +The channel document is a small mutable pointer, signed independently from the release. That +separation is the point: **promoting a build never requires re-signing it**. + +```jsonc +{ + "schemaVersion": 2, + "kind": "scrollcase.box.channel", + "channel": "beta", + "boxId": "my-model", + "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, + "updatedAt": "2026-07-25T10:14:03+02:00", + "cohortSalt": "9f2b7c1e04a83d5641b0e7c28a3d95f7", + "releases": [ + { "version": "1.0.0", "releaseManifestUrl": "https://…/4e81….release.json", "rolloutPercentage": 100 } + ] +} +``` + +A freshly built channel goes out at 100%. The schema can represent multiple release percentages, +but schema version 2 does **not** specify an interoperable cohort algorithm: it defines no identity +normalisation, byte framing, hash algorithm, integer extraction, percentage mapping, ordering, or +boundary fixtures. A consuming project may define and test those rules for its own clients, but +must not claim that unrelated implementations derive the same cohort from this format alone. + +`cohortSalt` is deterministic builder output derived from `boxId` and `version`; the format does +not define how a client combines it with an identity. Until a future version supplies normative +fixtures, the only cross-client behavior documented here is a 100% release. + +Promotion between channels (`nightly` → `beta` → `stable`) is publishing a channel document +naming the release you already built. Build once per channel name with `--channel`, or write the +promoted document yourself and sign it with the same key. + +## Revocation + +A published release is immutable, so withdrawing one is an explicit statement rather than a +deletion: clients keep honouring a revocations list even when the archive is still reachable. + +The format defines the [revocations manifest](/v2/reference/box-format#revocations-manifest) and the +`.revocations` kind; Scrollcase does not emit it. Publish and sign it yourself, with +the same envelope and the same key, and have clients fetch it before installing or activating. + +An empty `revocations` array is meaningful — it is a positive, signed statement that nothing is +revoked, which a client can distinguish from a missing or withheld document. + +## The client's side + +```mermaid +flowchart TD + C["fetch channel document"] --> CV{"signature valid?"} + CV -->|no| X["refuse"] + CV -->|yes| P["apply the consuming project's release policy"] + P --> R["fetch release document by URL"] + R --> RV{"signature valid?"} + RV -->|no| X + RV -->|yes| CO{"compatibility satisfied?
space available?"} + CO -->|no| X + CO -->|yes| D["download archive"] + D --> H{"size + sha256 match?"} + H -->|no| X + H -->|yes| E["validate entry names, extract"] + E --> M{"box.json agrees with release?"} + M -->|no| X + M -->|yes| T["run the self-test with the box's own Python"] + T --> OK["installed"] +``` + +Whatever installs your boxes should do exactly what `scrollcase verify` does, in the same order: + +1. Fetch the channel document, verify its signature, and apply the consuming project's release + policy. Schema version 2 guarantees interoperability only for the 100% case. +2. Fetch the release document by the URL the channel names; verify its signature. +3. Check `compatibility` against the host — and **refuse a constraint it cannot evaluate** rather + than assuming it passes. +4. Treat `installedSizeBytes` as a logical extracted-size lower bound. Require headroom for the + archive, extracted files, temporary copies, and filesystem overhead. +5. Download the archive; check size and SHA-256 against the release. +6. Validate every entry name before final extraction. +7. Compare all shared `box.json` fields recursively against the release. +8. Run the self-test: `selfTest.pythonImports` with `pythonEntryPoint`, bounded by + `selfTest.timeoutSeconds`. +9. With on-demand weights, fetch each asset and check its size and SHA-256 before first use. + +Running `scrollcase verify --self-test` on the build machine covers the archive and temporary +extraction checks, not final installation, compatibility policy, rollout, or activation. + +## Keeping an extracted box across restarts + +The archive proves the payload while it exists. A persistent installation usually discards that +archive, and a `PreparedBox` receipt cannot be serialised across process restarts: doing so would +turn a writable file into a forgeable execution capability. A new process earns a fresh receipt by +re-checking the signed release and the directory's execution prerequisites: + +```js +import { + attachExtractedBox, + runExtractedBox, + verifyExtractedPayload, +} from 'scrollcase/consumer'; + +// Optional and potentially expensive: proves the installed bytes at this moment. +await verifyExtractedPayload('release.json', { + publicPath: 'trusted-key.json', + root: '/srv/boxes/my-model/1.0.0/macos-aarch64-metal', +}); + +// Does not re-read original payload bytes; on-demand assets are hashed separately. +const attached = await attachExtractedBox('release.json', { + publicPath: 'trusted-key.json', + root: '/srv/boxes/my-model/1.0.0/macos-aarch64-metal', +}); +await runExtractedBox(attached); +``` + +Python exposes the same sequence as `verify_extracted_payload`, `attach_extracted_box`, and +`run_extracted_box`. Attachment deliberately does not verify every payload byte: it enumerates the +tree and measures metadata, but the full content read is a separate decision so launch does not +silently become a multi-gigabyte integrity scan. + +For a manual or maintenance check, the CLI reaches the same Node consumer operation: + +```sh +scrollcase verify release.json \ + --extracted /srv/boxes/my-model/1.0.0/macos-aarch64-metal \ + --public-key trusted-key.json +``` + +`--extracted` needs no archive and cannot be combined with `--archive` or `--self-test`. It checks +the signed `payload-digest.v1` entry list rather than walking the directory, so unrelated files that +appeared after installation do not fail an honest box. Embedded assets are original entries and are +read in full; on-demand assets are later extras, ignored by this digest and checked separately by +their signed descriptors during attachment and execution. + +::: warning Integrity is a point-in-time result +Payload verification detects ordinary corruption and identifies a directory against a signed +release. It does not stop the directory changing after the check. Protect persistent installations +with operating-system permissions and the embedding application's ownership policy. + +`__pycache__` directories and `*.pyc` files are excluded when the build collects payload entries, +so the digest can never see them. Do not treat the check as proof about compiled Python caches. +::: + +## Namespaces for existing publishers + +If you already have boxes installed in the field under your own document kinds, keep emitting +them: + +```sh +scrollcase build my-model/macos-aarch64-metal --namespace acme.model-pack +# → kinds: acme.model-pack.release, acme.model-pack.channel, acme.model-pack.revocations +``` + +The namespace belongs to the publishing project precisely so that adopting Scrollcase does not +rename documents underneath clients that already recognise them. New projects can leave the +default, `scrollcase.box`. diff --git a/docs/v2/guides/index.md b/docs/v2/guides/index.md new file mode 100644 index 0000000..6afad44 --- /dev/null +++ b/docs/v2/guides/index.md @@ -0,0 +1,15 @@ +--- +title: Guides +description: Comprehensive guides for mastering the Scrollcase workflows. +aside: false +next: false +prev: false +--- + +# Guides + +A comprehensive collection of guides and resources for developers. Learn the basics of Scrollcase integration through practical guides to help you build custom scrolls, configure multi-platform GPU-accelerated builds, automate CI/CD pipelines, and deploy verified AI runtimes boxes in production. + +--- + + diff --git a/docs/v2/guides/managing-weights.md b/docs/v2/guides/managing-weights.md new file mode 100644 index 0000000..962a36d --- /dev/null +++ b/docs/v2/guides/managing-weights.md @@ -0,0 +1,223 @@ +--- +title: Managing Model Weights +description: Declare, verify, embed or defer the assets a box carries. +--- + +# Managing Model Weights + +Model weights are usually the largest thing a box carries, and the only part fetched from a +server nobody controls. Scrollcase treats them the same way it treats everything else: declared +up front, verified before use, and committed to by hash in the signed release. + +## Declare an asset + +Every asset carries a URL, a destination inside the payload, a size, and a SHA-256: + +```jsonc +"assets": [ + { + "url": "https://huggingface.co/example-org/model/resolve/main/model.safetensors", + "relativePath": "model-cache/hello/model.safetensors", + "sizeBytes": 438012416, + "sha256": "9f2b7c1e04a83d5641b0e7c28a3d95f7c9d1a4e60b8f37c25e9a4d7081da5b3f" + } +] +``` + +Do not fill those two in by hand. `add asset` fetches the URL once and records the size and hash it +actually found: + +```sh +scrollcase add asset my-model https://…/model.safetensors +``` + +It also adds the payload path to `selfTest.files`, so an over-eager `prunePaths` cannot quietly drop +it. Use `--to ` to land it somewhere other than the box's model cache, and `--target` +to give it to one target only. If you would rather write the entry yourself, the two values come +from `shasum -a 256 model.safetensors` and `wc -c < model.safetensors`. + +Nothing enters the payload before **both** match. That is what makes a box reproducible even +though its inputs live on servers outside anyone's control: if an upstream file is moved, +replaced, or silently re-uploaded, the build fails instead of quietly producing a different box +under the same version. + +::: tip Resume boundary +A dropped connection is retried inside one download operation, resuming its `.part` file with a +Range request. The partial is renamed only after size and hash match. Build scratch is recreated +at process start, so a new build process does not reuse an earlier partial or provide a persistent +asset cache. +::: + +## Archives that need unpacking + +When the upstream artefact is a tarball or zip, declare it as an asset and then expand it: + +```jsonc +"assets": [ + { + "url": "https://example.org/model/weights-v1.tar.gz", + "relativePath": "model-cache/hello/weights.tar.gz", + "sizeBytes": 1073741824, + "sha256": "4c7e…9a" + } +], +"assetArchives": [ + { + "relativePath": "model-cache/hello/weights.tar.gz", + "format": "tar.gz", + "destination": "model-cache/hello", + "stripComponents": 1, + "removeAfterExtract": true + } +] +``` + +- Entries are listed and validated **before** extraction, so a malicious archive cannot write + outside its destination. +- `stripComponents` drops the redundant top-level wrapper directory many published archives + carry. It insists on finding exactly one directory to strip, so a surprising layout fails + loudly rather than producing a wrong tree. +- Extraction never overwrites a file already present in the destination. +- `removeAfterExtract` defaults to `true`: the compressed original is dead weight inside the + payload once unpacked. + +## Compression + +Weights arrive already compressed, and deflating them again is pure loss. Measured on +incompressible bytes: level 6 runs at 47 MB/s and the archive comes out **0.03% larger** than the +input, and dropping to level 1 recovers 4 MB/s because the search fails either way. Lowering the +level is not the fix — not compressing is. + +So every path you declare in `assets` is **stored** in the archive rather than deflated. You do not +have to ask for this and there is nothing to configure. + +For anything else your box carries that is already compressed — the tree an `assetArchives` entry +expanded into, a bundled corpus of JPEGs — say so: + +```jsonc +"uncompressedPaths": ["model-cache/hello", "corpora/images"] +``` + +An entry matches that path and everything beneath it. Nothing is decided by looking at the file or +its extension: the choice comes from the scroll alone, which is what keeps two builds of the same +commit byte-identical. The interpreter, `site-packages` and the notices compress genuinely and +still do. + +## Files from your own repository + +Runtime shims, licence notices, a parity check script — anything you maintain yourself — go in +`localFiles`: + +```sh +scrollcase add file my-model runtime/entrypoint.py +``` + +```jsonc +"localFiles": [ + { "sourcePath": "runtime/entrypoint.py", "relativePath": "entrypoint.py" }, + { "sourcePath": "legal/MODEL_LICENSE.txt", "relativePath": "MODEL_LICENSE.txt", "sha256": "…" } +] +``` + +`sourcePath` is relative to the project root; `relativePath` is inside the payload. + +`sha256` here is an optional **pin**, and the difference from an asset's is the point. An asset +arrives over a network nobody controls, so its hash is what stands between a substituted file and a +silently different box. A local file comes out of your own checkout, where git already records what +changed, and what ships is hashed into the signed release either way. So pin what must not change +without review — a licence notice, a reviewed shim — and leave the pin off the shim you are still +writing, which would otherwise fail your next build over an edit you meant to make. +[`scrollcase refresh`](/v2/reference/cli#refresh) recomputes a pin after a reviewed change. + +## Embed or defer + +This is the one real decision, and it is per build: + +| | `embed` (default) | `on-demand` | +| --- | --- | --- | +| Assets live | inside the archive | on your asset host | +| Install needs network | no — **works air-gapped** | yes | +| Archive size | large | small | +| Integrity guaranteed by | the archive's own signed hash | the per-asset size + SHA-256 in the signed release | + +```sh +scrollcase build my-model/linux-x86_64-cpu --weights embed # the default +scrollcase build my-model/linux-x86_64-cpu --weights on-demand +``` + +A scroll may set `"weights": "embed" | "on-demand"` as its own default; the flag overrides it for one +build. `build` never asks: what the scroll declares is what it uses, and the mode in effect is +printed as the build starts. A scroll that says nothing takes `embed`, which is what a box with no +declared assets wants anyway. + +### What `on-demand` puts in the release + +The assets are left out of the archive, and their descriptors travel in the signed release and in +`box.json`: + +```jsonc +"weights": "on-demand", +"assets": [ + { "url": "https://…/model.safetensors", "relativePath": "model-cache/hello/model.safetensors", + "sizeBytes": 438012416, "sha256": "9f2b…3f" } +] +``` + +The declared hash is what makes deferring safe: the release **commits to exactly which bytes the +box expects**, whatever host serves them. Retrieval belongs to the caller's distribution layer, +which places each asset at `relativePath` under the box root. The official consumers do not fetch +assets; they check every materialized file's size and hash before execution. + +::: warning Two constraints +`on-demand` cannot be combined with `assetArchives` — archives are expanded at build time, so +deferring them would declare a layout that never materialises; the build fails rather than lie. +And a file listed in `selfTest.files` that is a deferred asset is legitimately absent from the +payload, so it is skipped by the post-prune check. +::: + +### Choosing + +Embedding is the default because air-gapped installation is a property worth keeping unless a +project explicitly trades it away, and because it is the behaviour that surprises nobody: what +you verified is what you install. + +Defer when the archive would otherwise be unreasonable to move around, when the same weights are +shared by several boxes, or when your asset host is already the thing your users download from. +Then read [Offline / Air-Gapped Installs](/v2/guides/offline-airgap) to understand what you gave up. + +## Keeping the box small + +Everything the environment does not need at run time can be pruned before packing: + +```jsonc +"prunePaths": [ + "venv/share/doc", + "venv/lib/python3.11/site-packages/numpy/tests", + "venv/lib/python3.11/site-packages/scipy/io/tests" +] +``` + +::: warning Literal paths, not globs +Each entry is a single path removed recursively — there is no pattern matching. A glob such as +`.../**/tests` matches nothing and is removed silently, so list the directories explicitly and +check the resulting archive size to confirm the prune did what you expected. +::: + +Guard against over-pruning by listing what must survive: + +```jsonc +"selfTest": { + "imports": ["torch", "numpy"], + "files": ["model-cache/hello/model.safetensors", "entrypoint.py"] +} +``` + +A box is a multi-gigabyte download for an end user, so pruning is a user-facing concern rather +than tidiness — but a box that unpacks and cannot run is worse than a large one. The self-test +runs after pruning, with the box's own interpreter, precisely to catch that. + +## Where assets live inside the box + +`modelCacheSubdir` names the directory holding model assets, relative to the box root +(`model-cache/hello` above). Keep asset `relativePath` values under it so an installed box has +one obvious place where its weights are, and a consumer can find them without parsing anything. diff --git a/docs/v2/guides/offline-airgap.md b/docs/v2/guides/offline-airgap.md new file mode 100644 index 0000000..7fca73d --- /dev/null +++ b/docs/v2/guides/offline-airgap.md @@ -0,0 +1,135 @@ +--- +title: Offline / Air-Gapped Installs +description: Build once, carry the archive across, verify and install with no network at all. +--- + +# Offline / Air-Gapped Installs + +An embedded box is self-contained: everything it needs is inside the archive. A consumer verifies +the signed release and archive, then performs its own safe extraction and activation. Nothing is +fetched or resolved, and no daemon or container runtime is involved. + +This guide covers what makes that true, and what to check before relying on it. + +## What has to be true + +| Requirement | How to satisfy it | +| --- | --- | +| Assets are inside the archive | Build with `--weights embed` (the default) | +| No install-time relocation step | Guaranteed by the format — see [relocation](#why-no-install-step) | +| The trust anchor is on the isolated machine | Copy `signing-public.json` across, out of band | +| The verifier runs offline | `scrollcase verify` never touches the network | + +The one thing that breaks air-gapped installation is `--weights on-demand`, which deliberately +leaves the assets out for the caller's distribution layer to materialize. That is why `embed` is +the default: air-gapped installation is a property worth keeping unless a project explicitly +trades it away. + +## Build on the connected side + +```sh +scrollcase build my-model/linux-x86_64-cpu --weights embed +scrollcase verify .scrollcase/dist/boxes/my-model/1.0.0/linux-x86_64-cpu/*.release.json --self-test +``` + +Verify **before** transferring, on a machine matching the target. `--self-test` extracts the +archive and imports the declared modules with the box's own interpreter — the check that proves +the environment runs somewhere other than where it was built. Doing it now means a broken box +never makes the trip. + +## Transfer + +Three files travel: + +```text +.zip # the box +.release.json # the signed release document +signing-public.json # the trust anchor +``` + +The trust anchor should travel by a **different route** than the box, or be already present on +the isolated machine. A signature checked against a key that arrived alongside the artefact +proves only that they were produced together. + +Keep the archive beside the release document under its content-addressed filename. `verify` reads +the archive SHA-256 from the signed release and resolves `.zip`; if the files are +stored separately, pass `--archive`. + +## Verify on the isolated side + +```sh +scrollcase verify RELEASE_DOCUMENT_SHA256.release.json \ + --archive ARCHIVE_SHA256.zip \ + --public-key ./signing-public.json \ + --self-test +``` + +This validates through a temporary extraction and runs with no network: + +- at least one signature verifies against the trusted key; +- the archive's size and SHA-256 match what the signed release commits to; +- entry names are safe — no traversal, no links, no special entries; +- every shared `box.json` field agrees recursively with the signed release; +- the declared interpreter is present; +- with `--self-test`, the extracted payload size matches and the declared modules import with the + box's own Python. + +Scrollcase does not install or extract into an arbitrary final destination. After verification, +the consuming project may extract into a fresh destination using a path-safe extractor: + +```sh +unzip ARCHIVE_SHA256.zip -d /opt/boxes/my-model-1.0.0 +/opt/boxes/my-model-1.0.0/venv/bin/python -c "import torch; print(torch.__version__)" +``` + +::: warning Verification is not final installation +`verify --self-test` extracts only into a temporary directory and removes it afterwards. A generic +`unzip` command is illustrative, not a Scrollcase-managed installation. The consumer must preserve +the same path-safety checks and extract only after signature, size, and hash verification. +::: + +## Why there is no install step {#why-no-install-step} + +Three properties of the format make extraction sufficient: + +1. **The tree is packed ready to run.** conda-pack produces a complete prefix, so a consumer pays + no install-time work beyond extraction and decompression. +2. **`conda-unpack` is deliberately never run.** Running it would stamp the *build machine's* + absolute paths into dozens of files that then ship to users — measured on a probe environment, + zero files carried the build prefix before running it and thirty-six after — leaking a + developer's directory layout while still being wrong at the user's install location. Instead + the few service files that carry the prefix are removed at build time. +3. **Launchers resolve Python next to themselves.** Generated console scripts are rewritten so + they find the interpreter relative to their own location, symlinks point only inside the box, + so the extracted tree does not depend on where it landed. + +The result: the same archive works at `/opt/boxes/…`, in a user's home directory, or on a +read-only mount, with no fixer, no activation, and no environment variables. + +## Verifying without Scrollcase + +An isolated machine may not have Node. The checks are simple enough to reimplement, and the +format is specified precisely so that reimplementation stays honest — see +[The Box Format](/v2/reference/box-format). The minimum a client must do: + +1. Decode `payloadBase64`, check its SHA-256 against `payloadSha256`. +2. Verify at least one ed25519 signature over **those decoded bytes** against a trusted key. +3. Parse the payload; check the archive's size and SHA-256 against `archive.sizeBytes` and + `archive.sha256`. +4. Validate every ZIP entry name before extracting; reject links and special entries. +5. Compare all shared `box.json` fields recursively against the release manifest. + +The JSON Schemas and golden fixtures ship in the package for exactly this purpose. + +## Offline builds + +Building itself is not offline: `lock` resolves against conda-forge, and `build` needs the +packages the lock names. On a machine that cannot reach conda-forge, populate pixi's cache from a +connected machine or an internal mirror before building — `build` installs strictly from the +committed `pixi.lock` and never resolves, so nothing new is chosen, but the package files still +have to come from somewhere. + +Assets declared in the scroll are downloaded at build time too. The payload directory is wiped at +the start of every build, so pre-staging files there does not help — mirror the assets behind a +URL the build machine can reach and point the scroll's `url` at it. The declared size and SHA-256 +are what keep that safe: a mirror serving different bytes fails the build. diff --git a/docs/v2/guides/packaging-cuda.md b/docs/v2/guides/packaging-cuda.md new file mode 100644 index 0000000..3862e46 --- /dev/null +++ b/docs/v2/guides/packaging-cuda.md @@ -0,0 +1,214 @@ +--- +title: Packaging CUDA Boxes +description: Build a GPU box whose CUDA ABI is part of its identity, and prove it is really a GPU build. +--- + +# Packaging CUDA Boxes + +A CUDA box is an ordinary box with one extra property: the CUDA ABI it was built against is part +of its identity, so a CUDA 12.4 build can never be mistaken for a 12.8 one. This page covers what +a CUDA scroll declares, and how to prove the result is genuinely a GPU build rather than CPU +wheels wearing a CUDA name. + +## Supported CUDA targets + +| `platform` | `arch` | `accelerator` | Target ID | +| --- | --- | --- | --- | +| `linux` | `x86_64` | `cuda` | `linux-x86_64-cuda` | +| `windows` | `x86_64` | `cuda` | `windows-x86_64-cuda` | + +macOS has no CUDA target — use `metal`. The `12.4` values below are one concrete example, not the +only CUDA ABI the contract accepts. + +## The scroll + +```json +{ + "schemaVersion": 2, + "scrollVersion": "1.0.0", + "boxId": "my-model", + "modelId": "example-org-my-model", + "runtimeId": "my-model-runtime", + "version": "1.0.0", + "sourceRevision": "my-model-v1.2.0", + "target": { + "platform": "linux", + "arch": "x86_64", + "accelerator": "cuda", + "cudaVersion": "12.4" + }, + "compatibility": { + "minHostAppVersion": "1.0.0", + "minNvidiaDriverVersion": "550.54.14", + "minRamGb": 16 + }, + "pythonVersion": "3.14", + "pixiVersion": "0.73.0", + "assetBaseUrl": "https://assets.example.org/boxes", + "selfTest": { + "imports": ["torch"], + "pythonFile": "scrolls/my-model/linux-x86_64-cuda12.4/self_test.py" + } +} +``` + +Three things are CUDA-specific: + +1. **`cudaVersion`** — `major.minor`, required for a CUDA target and forbidden on any other. It + becomes part of the target ID, the archive name, and the object key. +2. **`minNvidiaDriverVersion`** in `compatibility` — copied verbatim into the release manifest for + the installing host to check. Scrollcase never interprets it. +3. **A self-test that actually exercises the GPU** — see below. + +`pythonEntryPoint`, `modelCacheSubdir` and an empty `assets` list are left out: the target and the +box identity already determine them, and they are filled in when the scroll is read. A box that +ships both a CUDA and a CPU target should keep what they share in one +[base scroll](/v2/reference/scroll#one-box-several-targets), with `cudaVersion`, +`minNvidiaDriverVersion` and the GPU self-test in the CUDA fragment. + +## The pixi manifest + +The solve is where a CUDA box is really made or broken. `platforms` must be the target's conda +subdirectory, and the CUDA version is declared both as a package pin and as a system requirement, +so the solver picks the GPU build rather than the CPU one: + +```toml +[workspace] +name = "my-model-linux-x86_64-cuda12.4" +channels = ["conda-forge"] +platforms = ["linux-64"] + +[system-requirements] +cuda = "12.4" + +[dependencies] +python = "3.14.*" +pytorch = { version = "2.*", build = "cuda*" } +cuda-version = "12.4.*" +``` + +Then resolve and commit the lock: + +```sh +scrollcase lock my-model/linux-x86_64-cuda12.4 +git add scrolls/my-model/linux-x86_64-cuda12.4/pixi.lock +``` + +Run those commands with Pixi `0.73.0`, because that is the exact version this example scroll pins; +do not substitute another resolver version. + +::: warning `cuda-version` pins the ABI, not the driver +The `cuda-version` package constrains which CUDA runtime the conda-forge packages are built +against. The host still needs a driver new enough for that ABI — that is what +`minNvidiaDriverVersion` communicates to the installer. +::: + +## Building + +CUDA boxes, like all boxes, are **built natively**: a `linux-x86_64-cuda12.4` box is built on +Linux x86_64. There is no cross-building, because the self-test runs the box's own interpreter, +and that only proves anything on matching hardware. + +```sh +scrollcase doctor --scroll my-model/linux-x86_64-cuda12.4 +scrollcase build my-model/linux-x86_64-cuda12.4 +``` + +The self-test runs under the target's CUDA validation environment +(`CUDA_VISIBLE_DEVICES=0`), so a `torch.cuda.is_available()` assertion is meaningful. + +::: tip A GPU is needed to build, not just to run +Solving a CUDA environment does not require a GPU, but a self-test that asserts +`torch.cuda.is_available()` does. Build CUDA boxes on a GPU machine — otherwise weaken the +self-test to something that passes without a device, and you lose the check that matters most. +::: + +## Proving it is really a GPU build + +The failure this guide exists to prevent is a box that solves, packs, installs, and then runs on +the CPU — CPU-only wheels shipped under a CUDA target ID. Three layers catch it: + +**1. The self-test.** The cheapest and most direct. It runs with the box's own interpreter, so a +CPU-only wheel fails it: + +```python +# scrolls/my-model/linux-x86_64-cuda12.4/self_test.py +import torch + +assert torch.cuda.is_available(), "CUDA runtime not usable inside the box" +assert torch.version.cuda.startswith("12.4"), f"built against CUDA {torch.version.cuda}" +``` + +A single assertion can also go inline as `selfTest.pythonCode`, but anything longer belongs in a +file the editor and the linter can see — which is what `selfTest.pythonFile` names. + +**2. The parity gate.** Run a real computation on CPU and on CUDA and require the results to +agree within a declared tolerance: + +```jsonc +"parity": { + "script": "checks/parity.py", + "accelerators": ["cpu", "cuda"], + "tolerances": { "absolute": 1e-4, "relative": 1e-3, "minimumCosine": 0.9999 } +} +``` + +This catches a broken BLAS, a mis-solved kernel, and a GPU path that silently falls back — see +[Accelerator Parity](/v2/guides/accelerator-parity). + +**3. `verify --self-test`.** Extracts the built archive and re-runs the signed imports with the +box's own interpreter, on a matching native host: + +```sh +scrollcase verify .scrollcase/dist/boxes/my-model/1.0.0/linux-x86_64-cuda12.4/*.release.json --self-test +``` + +This consumer check does **not** repeat scroll `pythonCode`, so it does not by itself prove +`torch.cuda.is_available()`. That stronger assertion and parity are builder gates. Building a +target proves packaging and declared gates; it never proves scientific parity unless the scroll +declares and passes a suitable parity check. + +## One box per CUDA ABI + +Because `cudaVersion` is part of the identity, supporting two ABIs means two scrolls, two locks, +two boxes: + +```text +scrolls/ +└── my-model/ + ├── scroll.json # what both ABIs share + ├── linux-x86_64-cuda12.4/ + └── linux-x86_64-cuda12.8/ +``` + +Two ABIs of one model agree about everything except the ABI, so this is the case a +[split scroll](/v2/reference/scroll#one-box-several-targets) is for: the base holds the identity, the +dependencies and the self-test, and each fragment declares its `cudaVersion` and its +`minNvidiaDriverVersion`. The `pixi.toml` and `pixi.lock` stay per target, since the solve is what +differs. + +They share a `boxId` and `version` and differ in target, so they publish under distinct object +keys and a client picks the one matching its driver: + +```text +boxes/my-model/1.0.0/linux-x86_64-cuda12.4/… +boxes/my-model/1.0.0/linux-x86_64-cuda12.8/… +``` + +## Size + +CUDA environments are large — the runtime libraries alone can dominate the archive. Prune what +the box does not need at run time, and let the self-test guard the prune: + +```jsonc +"prunePaths": [ + "venv/share/doc", + "venv/lib/python3.14/site-packages/torch/test", + "venv/lib/python3.14/site-packages/torch/include" +], +"selfTest": { "imports": ["torch"], "files": [], "pythonCode": "import torch; assert torch.cuda.is_available()" } +``` + +See [Managing Model Weights](/v2/guides/managing-weights#keeping-the-box-small) for the general +approach, and consider `--weights on-demand` when the weights, rather than the runtime, are what +makes the archive unwieldy. diff --git a/docs/v2/guides/platform-examples.md b/docs/v2/guides/platform-examples.md new file mode 100644 index 0000000..f12f355 --- /dev/null +++ b/docs/v2/guides/platform-examples.md @@ -0,0 +1,109 @@ +--- +title: Platform Examples +description: Minimal target and shell examples for macOS CPU, Linux CPU/CUDA, and Windows CPU/CUDA. +--- + +# Platform Examples + +Every box is built natively for one exact target. These snippets show target declarations and shell +syntax; they do not claim that a particular scroll or scientific workload has been validated. +Replace the example `/` references and use the exact `pixiVersion` pinned by each +scroll. + + + + +```jsonc +"target": { "platform": "macos", "arch": "aarch64", "accelerator": "cpu" } +``` + +```sh +scrollcase doctor --scroll my-box/macos-aarch64-cpu +scrollcase lock my-box/macos-aarch64-cpu +scrollcase audit my-box/macos-aarch64-cpu +scrollcase build my-box/macos-aarch64-cpu +``` + +The interpreter is `venv/bin/python`; the target ID is `macos-aarch64-cpu`. + + + + +```jsonc +"target": { "platform": "linux", "arch": "x86_64", "accelerator": "cpu" } +``` + +```sh +scrollcase doctor --scroll my-box/linux-x86_64-cpu +scrollcase lock my-box/linux-x86_64-cpu +scrollcase audit my-box/linux-x86_64-cpu +scrollcase build my-box/linux-x86_64-cpu +``` + +The interpreter is `venv/bin/python`; the target ID is `linux-x86_64-cpu`. + + + + +```jsonc +"target": { + "platform": "linux", + "arch": "x86_64", + "accelerator": "cuda", + "cudaVersion": "12.4" +} +``` + +```sh +scrollcase doctor --scroll my-box/linux-x86_64-cuda12.4 +scrollcase lock my-box/linux-x86_64-cuda12.4 +scrollcase audit my-box/linux-x86_64-cuda12.4 +scrollcase build my-box/linux-x86_64-cuda12.4 +``` + +`12.4` is an example ABI. The generic target suffix is `cuda`. A successful native +build proves packaging and declared gates, not scientific parity with another accelerator. + + + + +```jsonc +"target": { "platform": "windows", "arch": "x86_64", "accelerator": "cpu" } +``` + +```powershell +scrollcase doctor --scroll my-box/windows-x86_64-cpu +scrollcase lock my-box/windows-x86_64-cpu +scrollcase audit my-box/windows-x86_64-cpu +scrollcase build my-box/windows-x86_64-cpu +``` + +The interpreter is `venv/python.exe`; the target ID is `windows-x86_64-cpu`. + + + + +```jsonc +"target": { + "platform": "windows", + "arch": "x86_64", + "accelerator": "cuda", + "cudaVersion": "12.4" +} +``` + +```powershell +scrollcase doctor --scroll my-box/windows-x86_64-cuda12.4 +scrollcase lock my-box/windows-x86_64-cuda12.4 +scrollcase audit my-box/windows-x86_64-cuda12.4 +scrollcase build my-box/windows-x86_64-cuda12.4 +``` + +Windows CUDA is defined by the target contract and is buildable on a matching native host. This +example is not a claim that a specific box, GPU, driver, or scientific workload is supported. + + + + +`verify --self-test` also requires the matching native OS and architecture. CPU and CUDA boxes +share neither identity nor implied compatibility; build and verify each target independently. diff --git a/docs/v2/guides/signing-and-custody.md b/docs/v2/guides/signing-and-custody.md new file mode 100644 index 0000000..2d4a230 --- /dev/null +++ b/docs/v2/guides/signing-and-custody.md @@ -0,0 +1,216 @@ +--- +title: Signing & Key Custody +description: Local ed25519 keys, external signers (KMS/HSM), verification, and key rotation. +--- + +# Signing & Key Custody + +A box is only worth as much as the signature over its release document. Scrollcase ships a +working signer so anyone gets verifiable boxes with no infrastructure, and a plug for operators +who hold their keys somewhere serious — without ever learning anything about the custody model. + +## The local key + +```sh +scrollcase keygen +``` + +Writes two files into the workspace's keys directory: + +| File | What it is | +| --- | --- | +| `signing-private.pem` | The ed25519 private key, PKCS#8 PEM, written **owner-only** (`0600`) | +| `signing-public.json` | The trust anchor: algorithm, key ID, raw public key in base64, and the PEM | + +The key ID is derived from the key itself (`scrollcase-`), so it is +stable and collision-resistant without a registry. Override it with `--key-id`. + +```jsonc +{ + "algorithm": "ed25519", + "keyId": "scrollcase-9f2b7c1e04a83d56", + "publicKeyBase64": "kV3x…=", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\n…\n-----END PUBLIC KEY-----\n" +} +``` + +::: danger Never commit the private key +`init` adds `.scrollcase/` to `.gitignore` for exactly this reason. Keep the PEM out of history, +backups, and CI logs. Copy the public file to a tracked, project-owned trust directory and name it +explicitly: + +```sh +mkdir -p trust +cp .scrollcase/keys/signing-public.json trust/scrollcase-signing-public.json +git add trust/scrollcase-signing-public.json +scrollcase verify release.json --public-key trust/scrollcase-signing-public.json +``` +::: + +`keygen` refuses to overwrite an existing key without `--force`, because rotating silently would +invalidate every document previously signed with it, with no way to tell which. Never use +`keygen --force` as a mismatch repair or rotation procedure. + +## What gets signed + +Two documents per build, each independently signed: + +- the **release manifest** — immutable, committing to the archive by size and SHA-256; +- the **channel pointer** — mutable, naming which release the channel currently serves. Signing + it separately is what lets you promote a build without re-signing it. + +The payload is serialised once and both hashed and signed as-is, so **what is signed is +byte-for-byte what is published**. Details of the envelope in +[The Box Format](/v2/reference/box-format#signed-documents). + +## Verifying + +```sh +scrollcase verify .scrollcase/dist/boxes/my-model/1.0.0/macos-aarch64-metal/*.release.json --self-test +``` + +Verification loads the trusted key file (`--public-key`, default `/signing-public.json`) +and accepts the document when **any one** of its signatures verifies against a trusted key. The +file may hold a single key, or a bundle: + +```jsonc +{ + "keys": [ + { "algorithm": "ed25519", "keyId": "scrollcase-9f2b…", "publicKeyPem": "…" }, + { "algorithm": "ed25519", "keyId": "scrollcase-4c7e…", "publicKeyPem": "…" } + ] +} +``` + +That is also the mechanism for rotation, below. + +## External signers {#external-signers} + +An operator with real key custody — a KMS, an HSM, a signing service — configures a command +instead of a local key. The private key never touches the build machine. + +```sh +scrollcase build my-model/macos-aarch64-metal \ + --signer-command "./tools/kms-sign.sh" \ + --public-key ./trust/production-keys.json +``` + +When the command itself needs arguments, pass the whole value as one shell argument; quoted groups +inside it are preserved, including executable and argument paths containing spaces: + +```sh +scrollcase build my-model/macos-aarch64-metal \ + --signer-command '"/opt/signing tools/kms-sign" --key "production release"' +``` + +The contract is the simplest thing that composes with anything: + +1. The command receives the **payload bytes on stdin**. +2. It writes the **complete signed document as JSON on stdout**. +3. A non-zero exit, or non-JSON output, fails the build. + +Any language, any credential mechanism, no plugin API to keep compatible. + +```mermaid +sequenceDiagram + participant B as Scrollcase build + participant S as signer command + participant K as KMS / HSM + B->>S: payload bytes on stdin + S->>K: sign these bytes (ed25519) + K-->>S: signature + S-->>B: signed document on stdout + B->>B: payload echoed back unchanged? + B->>B: signature verifies against --public-key? + Note over B: either check fails → build fails +``` + +### The signer is not trusted on its word + +Two checks run on what comes back, before the build continues: + +- **The returned document must echo back the exact payload it was given.** A signer that + substitutes a payload fails the build instead of producing a box nobody can install. +- **Its signature is verified locally**, against the trust anchor `--public-key` names — not + against the signer's claim. + +### A minimal signer + +```sh +#!/bin/sh +# Reads payload bytes on stdin, prints a signed document on stdout. +set -eu +payload=$(mktemp); trap 'rm -f "$payload"' EXIT +cat > "$payload" + +payload_b64=$(base64 < "$payload" | tr -d '\n') +payload_sha=$(shasum -a 256 "$payload" | cut -d' ' -f1) +signature_b64=$(your-kms sign --key-id "$KEY_ID" --algorithm ed25519 --input "$payload") + +cat </`. Do not delete a guard, +replace a lock opportunistically, or overwrite a key to make an error disappear. + +## Toolchain + +### Missing or wrong Pixi version + +- **Symptom:** `pixi not found` or `Scroll requires pixi X, found Y`. +- **Cause:** discovery found no executable, or not the scroll's `pixiVersion`. +- **Diagnose:** run `scrollcase doctor --scroll /` and inspect flag, environment, + project-toolchain, then `PATH` precedence. +- **Correct:** install or select the exact pinned release, then relock only if intentionally + changing that pin. +- **Never:** use a different resolver version for a supposedly unchanged lock. + +### Missing conda-pack + +- **Symptom:** doctor or build reports that conda-pack is unavailable. +- **Cause:** no executable was found through flag, environment, project toolchain, or `PATH`. +- **Correct:** let `init` install it with explicit consent, or point `--conda-pack` at a managed + executable. +- **Never:** replace the pixi/conda-pack substrate with a second backend. + +## Workspace and lock + +### Dirty or non-git workspace + +- **Symptom:** build refuses a dirty tree or says it cannot record a commit. +- **Cause:** tracked changes, untracked source/build inputs, or no Git checkout. +- **Diagnose:** run `git status --short --untracked-files=all` at the resolved project root. +- **Correct:** review and commit inputs, or use `--allow-dirty` only for deliberate local work; the + release will truthfully record `sourceTreeDirty: true`. +- **Never:** hide an input or fabricate a revision. + +### Missing or outdated `pixi.lock` + +- **Symptom:** `Missing dependency lock`, or a reviewed licence audit no longer matches. +- **Cause:** the scroll has not been locked, or dependency declarations changed. +- **Correct:** run `scrollcase lock /`, review and commit the lock, then run + `scrollcase audit / --write` only after reviewing the new inventory. +- **Never:** let a production build resolve dependencies on the fly. + +### Licence audit drift + +- **Symptom:** build or audit reports that the reviewed inventory differs from the lock. +- **Cause:** the lock changed after review. +- **Correct:** inspect package and licence changes, approve them, then rewrite the reviewed audit. +- **Never:** delete the check or label unknown licence data as reviewed. + +## Assets + +### Size/hash mismatch or interrupted download + +- **Symptom:** asset size/SHA-256 mismatch, or retry messages after a dropped connection. +- **Cause:** partial transport, a server ignoring ranges, corruption, or upstream bytes changed. +- **Correct:** verify the authoritative bytes and update the scroll only if the project intends to + accept new content. Retries within one download resume from `.part`. +- **Never:** promote a partial file or change the expected digest merely to match a mirror. + +The build scratch directory is recreated on process start. There is no cross-process asset cache; +an interrupted process may need to download again. + +## Signing and verification + +### Public/private key mismatch + +- **Symptom:** signing reports that the private key does not match the public metadata. +- **Cause:** paths identify different key pairs. +- **Correct:** resolve the intended pair and pass both paths explicitly. +- **Never:** run `keygen --force`; it can destroy the only copy of the established signing identity. + +### Native-host mismatch + +- **Symptom:** build or `verify --self-test` refuses the current OS/architecture. +- **Cause:** the box target differs from the host. +- **Correct:** run on a matching native host, or omit `--self-test` when only signature/hash/layout + verification is intended. +- **Never:** bypass the native guard and report the result as target validation. + +### External signer payload mismatch + +- **Symptom:** Scrollcase rejects an external signer's result even though it contains a signature. +- **Cause:** the signer re-serialised, wrapped, or otherwise changed `payloadBase64`. +- **Correct:** echo the exact payload fields supplied on stdin and sign the decoded payload bytes. +- **Never:** accept a signature over a different representation. + +## Running a box + +### A GPU backend fails inside a `cpu` box + +- **Symptom:** the box verifies and extracts, then the application fails on its own first call. For + the [LLM demo](/v2/demos/llm-box-demo) that is `could not load …: Failed to create llama_context`, + with nothing after it. The same box runs on another host, or in a Codespace. +- **Cause:** the packaged library carries an accelerator backend that its runtime registers whatever + the application asks for. conda-forge's `llama-cpp-python` for `osx-arm64` is built with Metal, + and llama.cpp registers a Metal device however `n_gpu_layers` is set; creating a context + initialises *every* registered backend, so a host where Metal will not initialise takes down a box + that was never going to offload a layer to it. +- **Diagnose:** re-run with the application's own log unmuted — `LLM_DEMO_VERBOSE=1` for the LLM + demo — and read what the backend reports rather than what the wrapper raises. + `ggml_metal_init: picking default device: (null)` is a statement about the host: on macOS, + `MTLCreateSystemDefaultDevice()` returning nil while `MTLCopyAllDevices()` reports a working GPU + is a machine condition, not a packaging one, and it fails every library that asks for the default + device. +- **Correct:** switch the accelerator off in that target's `environment` so the box matches the name + it carries — `"GGML_METAL_DEVICES": "0"` for llama.cpp on macOS — and rebuild. `extends` merges + `environment` key by key, so a variable only one operating system needs belongs in the target + fragment and not in the base. +- **Never:** rename the target to the accelerator to make the error go away. A box declaring `metal` + promises an accelerator it does not use, and moves the same failure to the first host without one. + +## Windows specifics + +Use PowerShell syntax, preserve forward slashes in scroll payload paths, and declare +`venv/python.exe` as the interpreter. Windows launchers and native-library inspection differ from +POSIX targets; do not copy a macOS/Linux entry point or assume `venv/bin/`. Build and self-test on +native Windows x86_64. diff --git a/docs/v2/index.md b/docs/v2/index.md new file mode 100644 index 0000000..433ac53 --- /dev/null +++ b/docs/v2/index.md @@ -0,0 +1,28 @@ +--- +title: v2 Documentation [deprecated] +description: The documentation for box format schema version 2, kept as it was published. +--- + +# Scrollcase v2 + +This is the documentation for **box format schema version 2**, kept exactly as it was published. +Nothing here is maintained. It describes a format the current release refuses to read: Scrollcase v3 +rejects a v1 or v2 signed document by name rather than reinterpreting it, and tells the reader which +rebuild it needs. + +Read it for one reason only — you are holding a box that was built with Scrollcase v2 and you need +to know what its documents meant. For anything you are building now, go to +[the current documentation](/). + +Two things behave differently here from the rest of the site. These pages are outside the sitemap, +`llms.txt` and `llms-full.txt`, so a search engine or a model is not handed a superseded format as if +it were current. And the JSON Schemas they reference stay served at `/schema/v2/`, unchanged, because +a v2 box's documents name those URLs in their own `$schema` fields. + +## Sections + +- [Getting Started](/v2/getting-started/) — what Scrollcase v2 was, and how a box was built +- [Guides](/v2/guides/) — model weights, CUDA, parity, signing, air-gapped installs, distribution +- [Reference](/v2/reference/) — the CLI, the scroll, the box format, the schemas, the library APIs +- [Concepts](/v2/concepts/) — architecture, the security model, and the decisions behind both +- [White Paper](/v2/white-paper) — the v2 codebase and format, module by module diff --git a/docs/reference/api.md b/docs/v2/reference/api.md similarity index 94% rename from docs/reference/api.md rename to docs/v2/reference/api.md index 370b5c6..315556a 100644 --- a/docs/reference/api.md +++ b/docs/v2/reference/api.md @@ -385,7 +385,7 @@ concerns as the other two consumers: `contract`, `trust`, `release`, `archive`, ## `scrollcase/contract` -The single source of truth for what a box is. See [The Box Format](/reference/box-format). +The single source of truth for what a box is. See [The Box Format](/v2/reference/box-format). ### Targets @@ -397,15 +397,7 @@ The single source of truth for what a box is. See [The Box Format](/reference/bo | `condaSubdir` | `(target) => string` | The conda platform subdir (`osx-arm64`, `linux-64`, `win-64`) | | `pixiAccelerator` | `(scroll) => { accelerator, cudaVersion }` | The conda accelerator descriptor a scroll selects, rejecting target drift | | `assertNativeHost` | `(adapter, host = process) => void` | Throws unless the current host matches the adapter's OS and architecture | -| `assertRuntimeEntryPoint` | `(runtimeId, adapter, entryPoint) => void` | Throws unless the entry point matches that runtime's layout for the target | -| `RUNTIME_IDS` | `readonly string[]` | Every runtime id the format defines: `python`, `node`, `native`. A separate list from what a given build implements, on purpose — the consumers version independently | -| `runtimeAdapter` | `(runtimeId) => BoxRuntimeAdapter` | The runtime's layout, execution kinds, argv rule and self-test rule. Throws for a runtime with no adapter | -| `runtimeAdapters` | `() => BoxRuntimeAdapter[]` | Every runtime this build implements | -| `isImplementedRuntime` | `(runtimeId) => boolean` | Whether an adapter exists — the question to ask before `runtimeAdapter` | -| `unimplementedRuntimeMessage` | `(runtimeId) => string` | One wording for a box naming a runtime this build cannot run, so the builder and all three consumers report it identically | -| `unsupportedSelfTestProbeMessage` | `(runtimeId, probeKind) => string` | One wording for a probe shape the runtime cannot answer — `selfTest.imports` in a `native` box, which has no module system | -| `executionAffectingVariables` | `(runtimeId, adapter) => readonly string[]` | Inherited variables that can change what a box executes: the runtime's loader controls, then the OS's | -| `isExecutablePayloadPath` | `(rule, relativePath) => boolean` | Whether a payload path is one the runtime requires the executable bit on | +| `assertPythonEntryPoint` | `(adapter, entryPoint) => void` | Throws unless the entry point matches the adapter's layout | ```js import { boxTargetId } from 'scrollcase/contract'; @@ -490,7 +482,7 @@ The trusted key file is either a single key object or a `{ "keys": [...] }` bund accepted when any one of its signatures verifies. Every consumer operation that verifies a signed release takes `publicPath` **or** `trustedKeys`, exactly one: an application holding its keys in a keyring, an environment variable or a secrets manager should not have to write them to disk to -verify a signature. See [Signing & Key Custody](/guides/signing-and-custody). +verify a signature. See [Signing & Key Custody](/v2/guides/signing-and-custody). ## `scrollcase/build` @@ -515,7 +507,7 @@ const workspace = resolveWorkspace({ cwd: '/work/my-project/scrolls/my-model/mac // → { root, configPath, scrollsDir, buildDir, distDir, keysDir, toolchainDir } ``` -Details in [Workspace Configuration](/reference/configuration). +Details in [Workspace Configuration](/v2/reference/configuration). ### Archives and filesystem diff --git a/docs/v2/reference/box-format.md b/docs/v2/reference/box-format.md new file mode 100644 index 0000000..7fc3e31 --- /dev/null +++ b/docs/v2/reference/box-format.md @@ -0,0 +1,354 @@ +--- +title: The Box Format +description: What a box is on disk and on the wire — targets, archive layout, box.json, signed documents. +--- + +# The Box Format + +The format is the product. This page is the v2 contract a builder, a signer, and any client — in any +language — must agree on. Active documents carry `schemaVersion: 2`; v1 is rejected rather than +silently reinterpreted. + +The normative artefacts ship inside the npm package: + +| Artefact | Where | What it is | +| --- | --- | --- | +| Reference implementation | `scrollcase/contract` | The rules as executable code | +| JSON Schemas | `scrollcase/contract/schema/*.json` and `/schema/v2/*.json` | The machine-readable spec, package-local or public | +| Golden fixtures | `scrollcase/contract/fixtures/*.json` | What "agreeing" means, concretely | + +A client written in another language **does not import the code** — it mirrors the rules and +proves the mirror against `fixtures/target-id-contract.json`, +`fixtures/payload-digest-contract.json`, and the shared consumer conformance matrix. That is how +implementations stay honest without sharing a runtime. + +## Targets + +A target is the `(platform, arch, accelerator)` triple a box is built for, plus a CUDA ABI +version when the accelerator is CUDA. The supported matrix is closed: + +| `platform` | `arch` | `accelerator` | conda subdir | Interpreter | +| --- | --- | --- | --- | --- | +| `macos` | `aarch64` | `metal`, `cpu` | `osx-arm64` | `venv/bin/python` | +| `linux` | `x86_64` | `cpu`, `cuda` | `linux-64` | `venv/bin/python` | +| `windows` | `x86_64` | `cpu`, `cuda` | `win-64` | `venv/python.exe` | + +### Target identity + +`boxTargetId()` turns a target into the canonical slug that appears in archive names, object +keys, and routes. Every implementation must produce it character for character: + +```text +macos-aarch64-metal +macos-aarch64-cpu +linux-x86_64-cpu +linux-x86_64-cuda12.4 +windows-x86_64-cpu +windows-x86_64-cuda12.4 +``` + +The rule: `--`, except CUDA, which appends the version with no +separator — `cuda12.4`. `cudaVersion` is **required for CUDA and forbidden for everything else**, +so an identifier is never ambiguous. + +These, and the invalid cases that must be rejected, are the golden fixtures in +`fixtures/target-id-contract.json`. + +### Target adapters + +Each target also carries what it implies for the built payload: the Python layout, the archive +backend, how native libraries are inspected, the environment a validation run gets, and the +platform assertion the self-test prepends. Consumers unpacking a box rely on that layout, so it +is part of the format rather than an implementation detail. + +## The archive + +A box ships as a ZIP (ZIP64-capable) whose bytes depend only on its contents: + +```text +example-model-1.0.0-macos-aarch64-metal.zip +├── box.json # the self-describing manifest +├── payload-digest.v1 # canonical hashes of every original payload entry +├── venv/ # the packed, relocated conda-forge environment +│ ├── bin/python # (venv/python.exe on Windows) +│ ├── lib/… +│ └── conda-meta/… +├── model-cache/… # assets, when weights are embedded +└── THIRD_PARTY_NOTICES/ + └── conda-distributions.json # the dependency licence inventory +``` + +Guarantees the archive layer enforces: + +- **Deterministic.** Fixed timestamps (`2000-01-01T00:00:00Z`), stable file ordering, and modes + derived from the target adapter. The same commit rebuilds to identical bytes. +- **Links only where they are provably safe.** A symbolic link is carried when its target is + relative, resolves inside the payload, and ends at a regular file. Everything else — an absolute + target, one that climbs out through `..`, a link to a directory, a cycle — is materialised into + real content instead, and no entry may have a link as a path prefix, so nothing is ever written + *through* one. Windows boxes carry no links at all, because creating one there needs elevation. + Special entries are rejected outright. + + This is not a convenience: a conda prefix stores every large shared library two or three times + through the soname convention, and materialising all of it made most of an extracted Linux box + duplicates of its own bytes. See [Design decisions](/v2/concepts/design-decisions). +- **Safe to extract.** Entry names are validated against path traversal on the way out, by both + `verify` and any conforming client. +- **Relocatable.** Nothing inside depends on the build machine's paths — see + [Architecture](/v2/concepts/architecture#relocation). + +## `box.json` + +The manifest packed **inside** the archive, so an extracted box is self-describing: a consumer +holding the directory but not the release document can still tell what it is and how it was +built. + +The application inside the box can read it too, and that is the supported way to find the box's own +files. An entry point sitting at the payload root reaches its model with: + +```python +root = Path(__file__).resolve().parent +model = root / json.loads((root / "box.json").read_text())["modelCacheSubdir"] +``` + +Rather than a hard-coded path, which the scroll then has to be bent to match and which drifts +silently the day either side changes. `box.json` is written before the self-test runs, so a check +written this way exercises the same layout the shipped box has. + +```jsonc +{ + "schemaVersion": 2, + "boxId": "example-model", + "modelId": "example-org-example-model", + "runtimeId": "example-model-runtime", + "version": "1.0.0", + "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, + "pythonEntryPoint": "venv/bin/python", + "modelCacheSubdir": "model-cache/example", + "environment": { "MODEL_ROOT": "model-cache/example" }, + "selfTest": { "pythonImports": ["json", "sqlite3"], "timeoutSeconds": 180 }, + "provenance": { "…": "see below" } +} +``` + +`verify` recursively checks every shared field against the signed release: schema and identity, +complete target, entry point, cache subdirectory, declared environment, consumer self-test, +weights/assets policy, and provenance. That agreement binds the archive's contents to its signed +metadata. + +## Provenance + +Recorded by Scrollcase from observed state, never accepted from caller input, so the record +cannot be dressed up after the fact: + +| Field | Meaning | +| --- | --- | +| `scrollId`, `scrollVersion` | Which scroll produced the box. New scroll inputs derive `scrollId` as `-` | +| `builderRevision` | The 40-hex commit of the source tree that built it | +| `sourceTreeDirty` | Whether that tree had uncommitted changes. `true` means the build is **not** reproducible from the recorded revision alone | +| `sourceRevision` | Upstream revision of the packaged model source, as declared by the scroll | +| `pythonVersion`, `pixiVersion` | The interpreter version, and the resolver that solved the environment | +| `dependencyLockSha256` | Hash of the `pixi.lock` the environment was solved from | +| `builtAt` | Taken from the HEAD commit, not the clock — the same commit rebuilds to the same timestamp | + +## Signed documents + +Every document a build emits travels in one envelope: + +```jsonc +{ + "schemaVersion": 2, + "payloadEncoding": "base64-json-utf8", + "payloadBase64": "eyJzY2hlbWFWZXJzaW9uIjoyfQ==", + "payloadSha256": "7d2c9a41e8b350f6c174a9de20358bf41c6e97d05a8b3f2619e4c7081da5b3f2", + "signatures": [ + { "algorithm": "ed25519", "keyId": "scrollcase-9f2b7c1e04a83d56", "signatureBase64": "…" } + ] +} +``` + +The payload is **exact base64-encoded JSON, not canonicalised JSON**. Verifying a signature +therefore means hashing the bytes as transmitted, so Node, Rust, a Worker and any future client +agree without each maintaining a canonical-JSON implementation — historically the richest source +of cross-language signature bugs. + +A verifier accepts the document when **any one** signature verifies against a trusted key, which +is what lets a key rotate without reissuing every document. Passing the envelope schema means the +document is well-formed and worth verifying — never that its signature is valid. + +### Document kinds and namespaces + +Three document types, each discriminated by a `kind` of `.`: + +| Type | `kind` | Emitted by | +| --- | --- | --- | +| Release | `.release` | `scrollcase build` | +| Channel | `.channel` | `scrollcase build` | +| Revocations | `.revocations` | Defined by the format; published by whoever distributes boxes | + +The namespace **belongs to the publishing project**, and defaults to `scrollcase.box`. A project +that already has boxes installed in the field keeps emitting the namespace its clients recognise, +by passing `--namespace`. Scrollcase never hard-codes one, and carries nobody's brand. + +### Release manifest + +The immutable description of one built box: identity, target, compatibility, where the archive +lives and what it hashes to, the consumer import check to repeat, and provenance. Never edited after signing +— a correction ships as a new version. + +```jsonc +{ + "schemaVersion": 2, + "kind": "scrollcase.box.release", + "boxId": "example-model", + "modelId": "example-org-example-model", + "runtimeId": "example-model-runtime", + "version": "1.0.0", + "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, + "compatibility": { "minHostAppVersion": "1.0.0", "minMacosVersion": "13.0", "minRamGb": 8 }, + "archive": { + "format": "zip", + "url": "https://assets.example.org/boxes/boxes/example-model/1.0.0/macos-aarch64-metal/7d2c….zip", + "sha256": "7d2c…f2", + "sizeBytes": 49812054 + }, + "installedSizeBytes": 132145920, + "payloadDigest": { + "format": "sha256-path-list-v1", + "sha256": "6b8f…4c" + }, + "pythonEntryPoint": "venv/bin/python", + "modelCacheSubdir": "model-cache/example", + "environment": { "MODEL_ROOT": "model-cache/example" }, + "selfTest": { "pythonImports": ["json", "sqlite3"], "timeoutSeconds": 180 }, + "provenance": { "…": "…" } +} +``` + +`environment` is optional for compatibility with earlier schema-v2 releases. When present it is a +signed string map repeated value-for-value in `box.json`. A conforming verifier checks the +declaration; a Scrollcase consumer additionally resolves it against its current process and may +emit an environment report. That report is not part of the format and is not a guarantee of the +box. + +`installedSizeBytes` is the sum of logical extracted payload file and link sizes, including the +digest list. It is an estimate and lower bound, not an identity or free-space guarantee: consumers +need headroom for the archive, extracted files, temporary copies, allocation units, and filesystem +metadata. A prepared receipt reports the matching extracted measurement; an attached receipt +reports the directory's current measurement without comparing it with this signed build-time value. + +`weights: "on-demand"` and an `assets` array appear together only when assets were deliberately +left out; their absence means the box is self-contained. + +#### Extracted-payload commitment + +`payloadDigest` signs the SHA-256 of `payload-digest.v1`, which travels inside the payload and names +every original file and symbolic link except itself. It is optional so schema version 2 releases +built before this capability remain valid; an operation specifically asked to verify an extracted +payload refuses a release without the commitment. + +The canonical byte stream starts with `sha256-path-list-v1` and LF. Each following record is: + +```text +utf8(path) NUL ('f' | 'l') NUL lowercase-sha256 LF +``` + +Whole records are sorted bytewise. A file digest covers its bytes; a link digest covers the UTF-8 +bytes of its target string without following it. The list deliberately omits modes, modification +times, and directories: archive modes are synthesised, extraction does not restore build mtimes, +and empty directories do not survive the archive model. + +A verifier hashes the bounded list before parsing it, then checks only the paths it names. Files +added later are therefore ignored, including on-demand assets and application output. Embedded +assets are named and can make verification read tens of gigabytes; on-demand assets are absent from +the list and retain their separate signed per-file hashes. + +This commitment detects ordinary corruption and binds a directory to a signed release at the +moment it is checked. It is not protection against later modification or a live local attacker. +The collector also excludes `__pycache__` directories and `*.pyc` files, so those paths are a +permanent blind spot rather than merely part of the check-to-use timing window. + +### Channel manifest + +A small mutable pointer from a channel to the releases it currently serves. Signed independently, +so promoting a build never requires re-signing it. + +```jsonc +{ + "schemaVersion": 2, + "kind": "scrollcase.box.channel", + "channel": "beta", + "boxId": "example-model", + "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, + "updatedAt": "2026-07-25T10:14:03+02:00", + "cohortSalt": "9f2b7c1e04a83d5641b0e7c28a3d95f7", + "releases": [ + { + "version": "1.0.0", + "releaseManifestUrl": "https://assets.example.org/boxes/boxes/example-model/1.0.0/macos-aarch64-metal/4e81….release.json", + "rolloutPercentage": 100 + } + ] +} +``` + +Channels are `nightly`, `beta`, and `stable`, and a fresh build emits one release at 100%. +Schema version 2 carries `cohortSalt` and rollout percentages but intentionally lacks a normative +cohort algorithm and golden fixtures. It does not specify identity normalisation, byte framing, +hashing, integer extraction, percentage mapping, ordering, or boundary behavior. A project can +define those rules for its own clients, but cross-implementation rollout interoperability is not a +schema-v2 guarantee. + +### Revocations manifest + +The signed list of releases that must no longer be installed or activated. A published release is +immutable, so withdrawing one is an explicit statement rather than a deletion: clients keep +honouring the list even when the archive is still reachable. + +```jsonc +{ + "schemaVersion": 2, + "kind": "scrollcase.box.revocations", + "updatedAt": "2026-07-25T10:14:03Z", + "revocations": [ + { "boxId": "example-model", "version": "1.0.0", "reason": "mis-solved CUDA build", "revokedAt": "2026-07-26T09:00:00Z" } + ] +} +``` + +An empty `revocations` array is a positive statement that nothing is revoked, which a client can +distinguish from a missing or withheld document. Scrollcase defines this document but does not +emit it — revocation is a distribution concern, and +[distribution is deliberately out of scope](/v2/concepts/design-decisions). + +## Content addressing + +Names are derived from identity alone, so the archive, its release document and the staged +objects agree without any of them recording the others' paths: + +```text +stem -- +object prefix boxes/// +archive /.zip +release /.release.json +``` + +The whole chain is content-addressed, and every link is a hash: + +```mermaid +flowchart LR + C["channel document
signed, mutable"] -->|releaseManifestUrl
= sha256 of the document| R["release document
signed, immutable"] + R -->|archive.sha256| A["archive .zip"] + A -->|packed inside| B["box.json"] + R -.->|verify: shared fields agree recursively| B +``` + +Publishing is idempotent, and an object can never be replaced with different bytes under the same +URL. See [Distributing Boxes](/v2/guides/distributing-boxes). + +## Versioning {#versioning} + +Published v1 is immutable and remains paired with the old Scrollcase versions that emitted it. +Active v2 code accepts and emits only `schemaVersion: 2`. A future breaking change gets a **new** +`schemaVersion` — never a silent edit to a `kind` string, payload encoding, signature algorithm, +or golden fixture. diff --git a/docs/v2/reference/cli.md b/docs/v2/reference/cli.md new file mode 100644 index 0000000..eef8358 --- /dev/null +++ b/docs/v2/reference/cli.md @@ -0,0 +1,614 @@ +--- +title: CLI Commands +description: Every Scrollcase command, flag, environment variable, and exit convention. +--- + +# CLI Commands + +```text +scrollcase [options] +scrollcase -v | --version +``` + +Thirteen verbs: `init`, `new`, `add`, `remove`, `edit`, `refresh`, `doctor`, `keygen`, `lock`, `audit`, `build`, `verify`, `run`. +`scrollcase help` (or no command) prints the full usage text. +`scrollcase -v` and `scrollcase --version` print only the installed package version and do not +require a workspace. + +**Flag syntax.** Flags accept `--name value` or `--name=value`; a bare `--name` means `true`. + +**Exit convention.** Every failure, anywhere in the pipeline, exits non-zero with a single +`scrollcase: ` line on stderr — safe to rely on from shell scripts and CI. + +**Workspace flags** (`--config`, `--project-root`, `--scrolls-dir`, `--build-dir`, `--out-dir`, +`--keys-dir`, `--toolchain-dir`) apply to every command and are resolved before anything else runs. They are +documented in [Workspace Configuration](/v2/reference/configuration). + +## Scroll arguments and target selection + +`lock`, `audit` and `build` accept an exact nested reference: + +```sh +scrollcase build hello-box/macos-aarch64-metal +``` + +They also accept a box ID, with an optional target flag: + +```sh +scrollcase build hello-box --target macos-aarch64-metal +``` + +With only `hello-box`, a terminal shows a navigable target menu for the scrolls under +`scrolls/hello-box/`: use ↑/↓ and Enter. Exactly one target matching the host OS and architecture is +offered as the default; on macOS, Metal is preferred when both CPU and Metal are available. With no +terminal, the same default is selected and reported; any other ambiguous selection fails and tells +the caller to pass `--target`. v2 accepts only the nested +`scrolls///scroll.json` layout. + +The editing verbs — `add`, `remove`, `edit` and `refresh` — take a **box**, not a scroll reference, +because a change may belong to every target of that box or to one of them. Their `--target` answers +that question and is described under [`add`](#where-an-edit-goes). + +## `init` + +Initialize a workspace, and offer two independent extras: a fixed, disposable `example-box` for the +native host, and the consumer templates. A concise, linked `SCROLLCASE.md` is always created unless +one already exists. + +The **example** is a complete runnable v2 scroll: Metal on Apple Silicon and CPU on Linux or +Windows. It is created through the normal validated authoring path and never overwritten. It exists +to be built once and deleted. + +The **consumer templates** are `consumer-templates/run-box.ts`, `consumer-templates/run_box.py`, and +a small Rust crate at `consumer-templates/rust/`. They demonstrate the public Node, Python, and Rust +consumer APIs against a caller-supplied local release and include their setup commands, and they +name no particular box: the release path in each is a placeholder for the project's own. If no +`package.json` exists, `init` creates a private one with `"type": "module"`; an existing package +file is never changed. The Rust crate has its own non-overwriting `Cargo.toml` and ignores only its +generated `target/` directory. + +Both questions come before anything is written and default to yes (`[Y/n]`), and they are separate +because they answer different needs: a project that does not want a throwaway demo still has an +application to write against its boxes. `--no-example` and `--no-templates` answer without asking; +passing both leaves the workspace and its guide alone. Without a terminal both are included, as they +always were: unlike the installs below, writing a scaffold is not an act silence has to withhold +consent for. + +When it generated the templates, `init` asks in **one multi-select menu** which of their +dependencies to install — TypeScript, Python, Rust — using ↑/↓ to move, Space to select and Enter to +confirm. Nothing is preselected, and confirming an empty selection installs nothing. Selecting +Python then asks whether to take `scrollcase-consumer` from PyPI with pip or conda-forge with conda. +`init` also offers to install `pixi` and `conda-pack` if they are missing. Every answer is collected +before the first installer runs. If conda-forge is selected but `conda` cannot start, a default-yes +question offers PyPI instead. Without a terminal nothing is installed: a pipe or CI job does not +grant installation consent by being silent. + +```sh +scrollcase init [--pixi-version ] + [--no-example] [--no-templates] + [--install-toolchain | --no-install-toolchain] +``` + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--pixi-version` | example pin | Use this exact pixi release for the example and managed toolchain | +| `--no-example` | ask | Initialize without the `example-box` scroll, without asking | +| `--no-templates` | ask | Initialize without `consumer-templates/` and its package file, without asking | +| `--install-toolchain` | ask | Install missing tools without prompting | +| `--no-install-toolchain` | ask | Never install; just report what is missing | + +The final guidance names the example's exact lock command and points to `scrollcase new scroll` for +real project metadata. Without the example, it is simply `Next: scrollcase new scroll`. Target and +product flags passed to `init` are rejected with the same remedy instead of being silently ignored. + +### The toolchain step + +With neither flag and a terminal attached, `init` prompts, defaulting to **yes**. Without a +terminal — CI, a pipe — it never installs and simply reports what is missing: silence is not +consent. + +When you agree, `init`: + +1. resolves the pixi version — `--pixi-version`, else the example's repository pin; with + `--no-example`, the installed pixi's or newest release; +2. downloads the release for this host and checks its SHA-256 against the checksum pixi publishes + beside it. **A mismatch aborts and installs nothing**; +3. installs pixi into the workspace's toolchain directory, then uses it to run + `pixi global install "conda-pack==0.9.2"` with `PIXI_HOME` pointing there, so both land in the + project; +4. records the verified pixi digest and the conda-pack version under `toolchain` in + `scrollcase.config.json`, so later pixi installs are checked against the committed digest — see + [Workspace Configuration](/v2/reference/configuration#toolchain); +5. keeps each scroll's own `pixiVersion`; the generated example and managed toolchain use the same + resolved pin. + +Nothing is added to `PATH` and nothing is installed system-wide; later commands find the tools +because [tool discovery](#tool-discovery) looks in the toolchain directory. Deleting +`.scrollcase/toolchain/` undoes the whole thing. + +The consumer selections are independent of this managed build toolchain. Selecting TypeScript runs +npm in the project root to install `scrollcase`, `typescript`, and `tsx`. Selecting Python installs +`scrollcase-consumer` with either pip or conda-forge. For a PEP 668 externally managed interpreter, +`init` retries as a user-scoped installation and keeps package files outside the managed prefix. The +conda-forge path checks Conda before installation and offers the PyPI fallback when it is missing. +Selecting Rust runs `cargo add --manifest-path consumer-templates/rust/Cargo.toml +scrollcase-consumer`, modifying only the generated template crate. If Cargo is unavailable, `init` +leaves Rust out of the menu entirely without failing, keeps the Rust template, and prints the same +command so it can be run after Rust is installed. + +The example follows Scrollcase's supported box target matrix. On another host, decline it or +initialize with `--no-example`. Toolchain-only setup can still use any host for which pixi publishes a build. + +## `new` + +Create one `scrolls///` input. With a terminal it asks **four questions** — the +target, the box id, the upstream revision, and the base URL boxes will be published under — plus +navigable menus for execution kind and script source. Everything else has a defensible default and +is a flag rather than a prompt. A required answer left blank repeats the question instead of ending +the session. + +The weights mode is one of those defaults rather than a menu. It decides whether declared assets are +packed into the archive, and a box that declares none — which is most of them, since a scroll +packages a Python environment and not necessarily a model — has nothing for it to decide. New +scrolls take `embed` and say nothing about it; `--weights on-demand` states the other choice, and +`scrollcase edit scroll` changes it later. + +Every question in the CLI has the same shape: a blank line, the field's name, one line saying what +the field is, then the answer typed after ` ↳ `. The name is coloured and the explanation is not, so +a session of several questions stays readable rather than running together: + +```text +Upstream revision +Which version of the thing you are packaging this is — a model commit, a release tag. Recorded +verbatim in the box provenance: + ↳ upstream-v1 +``` + +Without a terminal, every value that has no default must be supplied explicitly, and missing input +fails before anything is written. + +```sh +scrollcase new scroll +scrollcase new scroll \ + --target linux-x86_64-cpu \ + --box-id example-model \ + --source-revision upstream-v1 \ + --asset-base-url https://assets.example.org/boxes \ + --weights embed \ + --execution library-only +``` + +| Flag | Meaning | +| --- | --- | +| `--target` | Complete canonical target; CUDA IDs include the ABI, such as `linux-x86_64-cuda12.4` | +| `--box-id` | Box identity and parent directory | +| `--source-revision` | Upstream revision recorded in provenance | +| `--asset-base-url` | Base URL copied into built release metadata | +| `--model-id` | Identity of what the box packages. Defaults to the box id | +| `--runtime-id` | Runtime identity. Defaults to `-runtime` | +| `--version` | Box version. Defaults to `1.0.0` | +| `--scroll-version` | Version of the authoring input. Defaults to `1.0.0` | +| `--python-version` | Python dependency version written into `pixi.toml`, or `latest`. Defaults to one minor behind the newest Python conda-forge publishes | +| `--pixi-version` | Exact resolver version required by `lock` and `build`. Defaults to the installed pixi's version | +| `--min-host-app-version` | Optional compatibility floor | +| `--max-host-app-version-exclusive` | Optional compatibility ceiling | +| `--min-macos-version` | Optional macOS floor | +| `--min-ram-gb` | Optional positive RAM requirement | +| `--min-nvidia-driver-version` | Optional NVIDIA driver floor | +| `--weights` | `embed` (default, and left out of the scroll) or `on-demand` | +| `--execution` | `python-script`, `python-module`, or `library-only` | +| `--script` | Existing project-relative Python script | +| `--generate-script` | Generate a minimal starter instead of using an existing script | +| `--script-destination` | Safe payload path, default `entrypoint.py` | +| `--generated-script-path` | Project path for the generated source; defaults to `box-entrypoints///entrypoint.py` | +| `--module` | Strict dotted Python module name | +| `--default-args` | JSON array of default application arguments | + +For `python-script`, choose exactly one of `--script` and `--generate-script`. Scrollcase records the +source in `localFiles` **without a hash pin**, so the first edit to a freshly generated script does +not fail its own build; add `sha256` yourself for a file that must not change without review. It +refuses traversal and non-regular sources, and never overwrites an existing source or scroll. +Generated defaults are grouped by both box and target; `library-only` omits execution metadata. + +Alongside `scroll.json` and `pixi.toml`, `new scroll` writes a `self_test.py` next to them and +points `selfTest.pythonFile` at it, so the box's own check starts life as real Python rather than +an escaped JSON string. + +`--python-version latest` resolves once, at authoring time, and writes the resulting number into the +scroll — never the word `latest`. Both it and the default are constants moved deliberately at each +Scrollcase release by `npm run python:bump`, which asks conda-forge what it publishes: a version +looked up on every invocation would make the same command produce different scrolls in different +months. + +Execution metadata is copied into the signed release and `box.json`. Before archiving, the builder +requires a script to remain a regular payload file or a dotted module to be discoverable in the +built environment without importing it. Library-only scrolls omit the field. + +## `add` + +Record something in a scroll that already exists, so the fields nobody can write by hand are not +written by hand. + +```sh +scrollcase add asset [--to ] [--target |all] +scrollcase add file [--to ] [--target |all] +scrollcase add dep [--version ] [--target |all] +scrollcase add dep --from-requirements requirements.txt +scrollcase add env NAME=VALUE [--target |all] +scrollcase add import [--target |all] +``` + +`add asset` **downloads the URL once** and records the `sizeBytes` and `sha256` it actually found, +which are the two values a scroll cannot be written without and no author can know without fetching +the file. Recording them here changes nothing about the guarantee: they are pinned once and checked +on every build, exactly as before. `--to` is optional and defaults to the URL's last path segment +under the box's `modelCacheSubdir`. + +`add file` records a file from the project. `--to` defaults to the file's own name at the payload +root. No `sha256` is written — see [`localFiles`](/v2/reference/scroll#localfiles) — so the first edit +to a file you just added does not fail your next build. + +Both also add the payload path to `selfTest.files`, so an over-eager `prunePaths` cannot quietly +drop what you just declared. + +`add dep` writes into the `[dependencies]` table of the box's `pixi.toml` files, editing the text +rather than re-emitting the manifest, so comments and spacing survive. The default constraint is +`*`: `pixi.lock` is the pin that matters and it records the exact version solved, so a second, +weaker pin in the manifest would only be something else to keep in step. Pass `--version ">=2,<3"` +when a project wants a bound. It closes by reminding you that a dependency is not proven until the +box imports it — the module name is yours to give, so the reminder names the command and no module. + +`add env` declares one environment variable the box needs whenever its interpreter runs, leaving the +rest of the map alone. A map is the one shape a single-value prompt cannot edit, which is why it has +its own command rather than being left to a hand edit. The value may contain `=`; only the first one +separates the name. + +`add import` adds a module to `selfTest.imports`. Those names are signed into the release and +repeated by `verify --self-test`, so they are the part of the self-test a consumer can check for +itself. + +`--from-requirements` reads a pip `requirements.txt` instead. Names are translated to conda-forge +where Scrollcase is sure and lowercased otherwise, and **every translation and every skip is +reported** rather than applied quietly: a name translated wrongly gives a lock that resolves and a +box that cannot import what it was built for. Extras, pip options and direct URLs are skipped with a +reason. + +### Where an edit goes + +A box may keep its shared declarations in a base and its differences in per-target fragments (see +[one box, several targets](/v2/reference/scroll#one-box-several-targets)), so every one of these +commands has to know which file to write: + +| `--target` | Writes to | +| --- | --- | +| `all` | What the targets share: the base of a split scroll, or every target file when there is no base | +| A target ID | Only that target's scroll | +| Omitted, box has one target | That target | +| Omitted, box has several, terminal | A menu, with "every target" first | +| Omitted, box has several, no terminal | Nothing — the command stops and asks for `--target` | + +It is never guessed. Both answers are reasonable, only the author knows which was meant, and a +declaration that lands on one target instead of all of them is silent until a build somewhere is +missing a file. + +Every edit is atomic and verified: the new bytes go to a staging file and are moved into place with +one rename, then the whole box is read back through the same path a build uses. If the result would +not load — a payload path claimed twice, a value the schema refuses — the originals are put back and +the command fails. + +## `remove` + +The exact inverse of `add`, because a tool where arriving is a command and leaving is a hand edit +has not removed the hand edit. + +```sh +scrollcase remove asset [--target |all] +scrollcase remove file [--target |all] +scrollcase remove env NAME [--target |all] +scrollcase remove import [--target |all] +``` + +For `asset` and `file` the entry is dropped and so is its `selfTest.files` line. Removing the last +environment variable takes the empty map with it rather than leaving `"environment": {}` behind. +Removing the last self-test import is refused: a box has to prove it can import something. + +A path, name or module that matched nothing is an error, not a quiet success. + +## `edit` + +Change one field of a scroll that exists. + +```sh +scrollcase edit scroll [] [--field --value ] [--target |all] +``` + +With a terminal and no flags, the field comes from a **menu built out of the schema** — so a name +that is not a field cannot be typed in the first place — and an enum field offers its values. +Without a terminal, `--field` and `--value` are required. + +Three kinds of field are not offered: structural values a project does not choose (`schemaVersion`, +`extends`), values the layout or the target fixes (`boxId` and `target` name the directories, +`pythonEntryPoint` has one legal value per target), and the collections, which have `add`/`remove` +or a file of their own. + +## `refresh` + +Bring a scroll back into agreement with the project it describes. + +```sh +scrollcase refresh [] [--check-assets] [--repin] +``` + +By default it recomputes only the pins a project asked for: a `localFiles` entry that declares +`sha256` means "this must not change without review", and after a reviewed change the digest has to +move with it. That is the edit worth automating; nothing else is touched and the network is not +used. + +Remote assets are deliberately different. Their hashes are what stands between a replaced upstream +file and a silently different box. If `refresh` re-fetched and rewrote them, then every time someone +swapped a file on that server the next `refresh` would adopt it without a word and the build would +go green — the protection would be gone. So `--check-assets` is opt-in (it downloads every asset), a +difference is **reported and refused**, and accepting it takes a separate `--repin`. Find out why +upstream changed before you use it. + +## `doctor` + +Report whether this machine can build a box. Reads only; never writes. Each failing check prints +a remedy, and all checks run even when an early one fails. + +```sh +scrollcase doctor [--scroll ] [--target ] [--pixi-version ] + [--pixi ] [--conda-pack ] +``` + +Checks: the workspace resolution, the scrolls directory, being inside a git checkout, pixi at the +required version (from `--pixi-version` or `--scroll`; skipped when neither is given), and +conda-pack. The managed installer pins conda-pack 0.9.2; because its `--version` output is not +reliable, `doctor` can only prove that an externally supplied conda-pack executable runs. Exits +non-zero if any check fails. + +## `keygen` + +Create a local ed25519 signing key pair: a private PEM written with owner-only permissions, and a +public key JSON file used as the trust anchor by `verify`. + +```sh +scrollcase keygen [--key-id ] [--force] + [--private-key ] [--public-key ] +``` + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--key-id` | `scrollcase-` | Identifier recorded in every signature | +| `--force` | off | Overwrite an existing key. Guarded because rotating silently would invalidate every previously signed document | +| `--private-key` | `/signing-private.pem` | Where the private key is written | +| `--public-key` | `/signing-public.json` | Where the public key file is written | + +See [Signing & Key Custody](/v2/guides/signing-and-custody) for rotation and external signers. +`--force` is not a rotation workflow or a safe way to repair mismatched paths: it can overwrite +the only copy of an established signing identity. + +## `lock` + +Resolve the scroll's `pixi.toml` into a fully pinned `pixi.lock`, written next to the manifest. +Run by a human when dependencies change; the lock is committed and reviewed, and `build` then +only installs from it. Requires pixi at the scroll's pinned version. + +```sh +scrollcase lock [] [--target ] [--pixi ] +``` + +The manifest itself pins the channels and the single target platform, so resolution does not +depend on the machine doing it. + +When `` is omitted in an interactive terminal, Scrollcase discovers every valid nested +scroll in the workspace and presents their complete `/` references in a navigable +menu. A non-interactive caller must provide the reference explicitly. + +## `audit` + +The dependency licence inventory, derived from the committed `pixi.lock` without building +anything. The lock carries an SPDX licence per package; a package **without a declared licence +fails the parse outright** — an unlicensed dependency is a legal problem, not a reporting gap. + +```sh +scrollcase audit [--target ] [--write] [--namespace ] +``` + +Two modes: + +- **Check (default).** If the scroll declares a `condaDependencyLicenseAudit` path, the computed + inventory is compared byte-for-byte against that reviewed file and any difference fails. This + is what `build` enforces too, so licence review happens when dependencies change — not at the + end of a multi-gigabyte build. +- **Write (`--write`).** Write the inventory to the scroll's declared path, for a human to review + and commit. Writing is explicit because silently overwriting the reviewed file is exactly how + an unreviewed licence change would slip through. + +A scroll that declares no path gets one: `--write` places `conda-licenses.json` beside the scroll +and records the declaration, reporting both. The path is a convention rather than a decision, so +there is nothing gained by making you type it. The **declaration** stays deliberate: a build +enforces the audit only for a scroll that names a path, so the check is switched on by running this +command and never by a file appearing on disk. + +`--namespace` sets the namespace of the inventory's `kind` +(`.dependency-license-audit`, default `scrollcase.box`). + +Output is a per-licence package count, for example: + +```text +23 packages for hello-box-macos-aarch64-metal (macos-aarch64-metal) + 9 MIT + 4 Apache-2.0 + ... +``` + +## `build` + +Turn a scroll into a signed box: install the locked environment, pack and relocate it, stage +assets, prune, audit licences, self-test with the box's own interpreter, run the optional +[parity gate](/v2/guides/accelerator-parity), archive deterministically, and sign a release document +plus a channel pointer. The full pipeline is narrated in +[Architecture](/v2/concepts/architecture). + +```sh +scrollcase build [] [--target ] + [--channel ] [--weights embed|on-demand] + [--asset-base-url ] [--namespace ] [--allow-dirty] + [--pixi ] [--conda-pack ] + [--private-key ] [--public-key ] [--signer-command ] +``` + +As with `lock`, omitting `` in an interactive terminal opens the workspace-wide scroll +menu. CI and other non-interactive callers must always provide it explicitly. + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--target` | ask when a box has several scrolls | Canonical target scroll to build | +| `--channel` | `beta` | Channel the signed pointer names. The v2 vocabulary is closed to `nightly`, `beta`, and `stable` | +| `--weights` | scroll's `weights`, else `embed` | Overrides the scroll for this build: `embed` packs assets into the archive (works air-gapped), `on-demand` leaves them out for the caller to materialize; consumers verify them before execution. `build` does not ask — the scroll's declaration is what it uses | +| `--asset-base-url` | scroll's `assetBaseUrl` | Base URL the signed documents point at; one of the two must be set | +| `--namespace` | `scrollcase.box` | Document `kind` namespace — a project with boxes already in the field keeps emitting its own | +| `--allow-dirty` | off | Permit a build from an uncommitted tree; recorded as `sourceTreeDirty: true` in the box | +| `--signer-command` | none | Sign through an external command instead of the local key — see [Signing & Key Custody](/v2/guides/signing-and-custody#external-signers) | + +Before starting the environment build, Scrollcase checks that signing is ready. If both default +local key files are absent, it fails immediately with `Signing keys not found. Run scrollcase +keygen before building.` The build command never generates identity material itself. An incomplete +pair is never overwritten; an external signer instead requires its trusted public key to be +present. + +The progress bar ending in `100% Completed` belongs to `conda-pack`, not to the whole build. After +that handoff Scrollcase reports the remaining long phases while it extracts and relocates the +packed environment, prepares and self-tests the payload, creates and hashes the deterministic +archive, and signs the release and channel documents. + +A successful build ends with a compact relative-path summary: you can distribute the two immutable files +under `boxes////` and the signed pointer at +`channels///.json`. The individual content-addressed filenames remain +unchanged. + +`build` refuses to run when: the workspace is not a git checkout; the tree is dirty and +`--allow-dirty` is absent; `pixi.lock` is missing; the pixi on hand is not the scroll's pinned +version; or the host OS/architecture does not match the target — boxes are proven on the hardware +they ship for. Dirty detection includes untracked files and excludes files ignored by Git. + +Outputs, under the workspace's `dist` directory: + +| File | What it is | +| --- | --- | +| `boxes////.zip` | The box archive | +| `boxes////.release.json` | The signed release document committing to the archive by size and SHA-256 | +| `channels///.json` | The signed channel pointer | + +`boxes/` is uploaded as it stands — the paths are the keys the signed documents already point to. +`channels/` is separate because a channel outlives any one version. See +[Distributing Boxes](/v2/guides/distributing-boxes). + +## `verify` + +Run the format checks a consumer can repeat against a signed release document and its archive, +before anything is published. + +```sh +scrollcase verify [--archive ] [--self-test] [--env-report] [--env-report-values] +scrollcase verify --extracted [--env-report] [--env-report-values] +``` + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--archive` | `.zip` next to the release document | The archive to check | +| `--self-test` | off | Extract to a temporary directory and import the declared modules with the box's own interpreter. Only runs on a matching native host | +| `--extracted` | off | Verify an existing extracted payload against the signed payload digest. Cannot be combined with `--archive` or `--self-test` | +| `--env-report` | off | Expand the diagnostic from the compact relevant subset to every resolved variable name; inherited host values remain masked | +| `--env-report-values` | off | Expand the report and reveal inherited host values. Use deliberately: logs may contain secrets | +| `--public-key` | `/signing-public.json` | Trusted key file (a single key, or a `{ "keys": [...] }` bundle) | + +Checks, in order: envelope payload hash and at least one trusted signature; release kind; coherent +target and entry point; archive size and SHA-256; safe entry names; recursively equal shared +`box.json` fields (identity/version, full target, entry point, cache subdirectory, declared +environment, consumer self-test, weights/assets, and provenance); and the declared interpreter. `--self-test` +additionally requires a matching native host, extracts to a temporary directory, checks logical +payload size, and runs the signed import check. It does not repeat scroll-only `pythonCode` or file +assertions, which are builder-only checks. + +After validating the command arguments, `verify` prints a blank line and `Verifying box` (or +`Verifying extracted payload`) before it starts reading and hashing the supplied bytes, so a long +verification gives immediate terminal feedback. + +Every verification result carries a structured environment snapshot in the library. The CLI stays +silent on a plain verification unless a report flag is present; `--self-test` prints the compact +report automatically when the release declares variables, conflicts exist, or inherited variables +such as `PYTHONPATH`, `PYTHONHOME`, `PYTHONSTARTUP`, `PYTHONBREAKPOINT`, `LD_PRELOAD`, or +`DYLD_INSERT_LIBRARIES` can change which code runs. The snapshot is diagnostic output from this +consumer and this process, not evidence signed into the box. + +`--extracted` takes the archive path out of this flow and delegates to the Node consumer's payload +verification operation. It verifies the signed release document and the payload list carried by +``, then checks every file and symbolic link named by that list. Files added after installation +are ignored. This is an explicit integrity check at one moment; it does not attach or execute the +box, and it does not protect the directory from later changes. +Modes and modification times are outside the commitment. The build collector also excludes +`__pycache__` directories and `*.pyc` files, so this command makes no assertion about compiled +Python caches. + +## `run` + +Verify and execute one caller-supplied local release through `scrollcase/consumer`: + +```sh +scrollcase run [--archive ] [--env-report] [--env-report-values] -- [application args] +``` + +The command prints a blank line and `Preparing box for execution` immediately after validating its +arguments, then performs the same signature, schema, archive, safe-entry, manifest-agreement, +installed-size, interpreter, and execution checks as the Node consumer. It extracts into a private +temporary directory and prints the signed box ID, version, target and execution kind after another +blank separator, then attaches terminal stdio and runs the declared script or module with the box's +own Python. Every status write is flushed before the interpreter starts, so it cannot appear after +the box's own output. Signed `defaultArgs` come first; every string after `--` follows unchanged, +without a shell. + +The compact environment report appears automatically when it has something relevant to say. +`--env-report` expands it to every variable name and provenance source while keeping inherited host +values masked; `--env-report-values` explicitly reveals those values and implies the full report. +The signed declaration wins over inherited and caller values. Nothing is filtered. + +The child exit code becomes the Scrollcase exit code. `SIGINT`, `SIGTERM`, and `SIGHUP` are forwarded +to the child; after the child terminates, the temporary box is removed and the CLI terminates by the +same signal. Temporary cleanup also runs after verification failure, spawn failure, normal exit, and +non-zero exit. + +`run` is intentionally local: + +- it never selects a channel or downloads an archive; +- `--archive` names bytes already present on disk, defaulting to the content hash beside the release; +- `--public-key` uses the same trusted key file or bundle as `verify`; +- it refuses library-only releases, non-native targets, and missing or mismatched on-demand assets; +- it never installs persistently, updates an existing box, or owns application lifecycle policy. + +Because one-shot `run` does not download or provide an asset-materialization step, an on-demand box +with required assets fails clearly. A caller that already owns those bytes uses +`verifyAndExtractBox`, materializes each signed descriptor under the prepared root, and then calls +`runExtractedBox`. + +## Tool discovery {#tool-discovery} + +Every command that needs `pixi` or `conda-pack` resolves it the same way, highest precedence +first: + +1. **The explicit flag** — `--pixi `, `--conda-pack `. +2. **The environment** — `SCROLLCASE_PIXI`, `SCROLLCASE_CONDA_PACK`. +3. **The project's own toolchain** — `/bin/`, if the executable is there. This is where + `init` installs, which is why nothing has to be added to `PATH` afterwards. +4. **`PATH`** — the bare `pixi` / `conda-pack` name. + +`build` and `lock` additionally require pixi to be at the exact version the scroll pins; a +different version is an error rather than a silent substitution. + +## Environment variables + +| Variable | Meaning | +| --- | --- | +| `SCROLLCASE_PIXI` | Path to the pixi executable, when not on `PATH`. A `--pixi` flag wins over it | +| `SCROLLCASE_CONDA_PACK` | Path to the conda-pack executable. A `--conda-pack` flag wins over it | diff --git a/docs/v2/reference/configuration.md b/docs/v2/reference/configuration.md new file mode 100644 index 0000000..d283243 --- /dev/null +++ b/docs/v2/reference/configuration.md @@ -0,0 +1,137 @@ +--- +title: Workspace Configuration +description: scrollcase.config.json — where scrolls live and where builds, artefacts and keys go. +--- + +# Workspace Configuration + +Paths come from the project, not from Scrollcase. A `scrollcase.config.json` at the project root +declares where scrolls live and where Scrollcase writes what it builds; the CLI discovers it by +walking up from the working directory, so every command works from anywhere inside the project. + +A project that declares nothing gets sensible defaults — the config file exists for projects that +already keep their scrolls elsewhere, or that adopted Scrollcase after building their own +convention. + +## The file + +```json +{ + "version": 1, + "paths": { + "scrolls": "scrolls", + "build": ".scrollcase/build", + "dist": ".scrollcase/dist", + "keys": ".scrollcase/keys", + "toolchain": ".scrollcase/toolchain" + } +} +``` + +| Key | Default | What lives there | +| --- | --- | --- | +| `paths.scrolls` | `scrolls` | One directory per box, then one per target; each target holds `scroll.json`, `pixi.toml`, and the committed `pixi.lock` | +| `paths.build` | `.scrollcase/build` | Payload scratch space, wiped and regenerated on every build | +| `paths.dist` | `.scrollcase/dist` | Built artefacts under publish-ready `boxes/` and `channels/` trees | +| `paths.keys` | `.scrollcase/keys` | Local signing keys (`signing-private.pem`, `signing-public.json`) | +| `paths.toolchain` | `.scrollcase/toolchain` | `pixi` and `conda-pack`, when `init` installed them for the project | + +Rules: + +- `version` is optional, but when present must be `1`. +- Every `paths` entry is optional; an omitted entry falls back to its default. +- Unknown `paths` keys, non-string values, and malformed JSON are hard errors — a typo fails + loudly rather than being silently ignored behind defaults. +- Relative paths in the config resolve **against the project root** (the config's directory), so + the file is portable across machines and checkouts. + +Commit the config and the scrolls; never commit `.scrollcase/` (build state and artefacts are +regenerated, and the private key must not enter history — `init` writes the ignore rules for +you). + +## Discovery and precedence + +The project root is chosen with this precedence, highest first: + +1. `--project-root ` — treat this directory as the root. +2. The directory of an explicit `--config `. A named config that does not exist is a hard + error. +3. The nearest `scrollcase.config.json` found walking up from the working directory. +4. The working directory itself (defaults apply). + +Each individual path is then resolved with its own precedence, highest first: + +1. **CLI flag** — `--scrolls-dir`, `--build-dir`, `--out-dir`, `--keys-dir`, + `--toolchain-dir`. Flag values resolve + against the **current working directory**, which is what a shell user expects. +2. **Config value** — resolves against the **project root**. +3. **Built-in default** — resolves against the project root. + +| Flag | Overrides | +| --- | --- | +| `--config ` | Use this workspace config explicitly | +| `--project-root ` | Treat this directory as the project root | +| `--scrolls-dir ` | `paths.scrolls` | +| `--build-dir ` | `paths.build` | +| `--out-dir ` | `paths.dist` | +| `--keys-dir ` | `paths.keys` | +| `--toolchain-dir ` | `paths.toolchain` | + +## Examples + +Run against the example scrolls shipped in the Scrollcase repository, from your own project: + +```sh +scrollcase build hello-box/macos-aarch64-metal --scrolls-dir ../scrollcase/examples +``` + +A monorepo that keeps packaging assets under `packaging/`: + +```jsonc +{ + "version": 1, + "paths": { + "scrolls": "packaging/scrolls", + "build": "packaging/.build", + "dist": "packaging/dist", + "keys": "packaging/keys" + } +} +``` + +Note that the git checkout the build records its provenance from is the **project root** — the +box's `builderRevision` is the HEAD of the repository the workspace resolves to. + +## The toolchain pin {#toolchain} + +When [`init` installs the toolchain](/v2/reference/cli#the-toolchain-step) it adds a `toolchain` +block recording the digest it verified: + +```jsonc +{ + "version": 1, + "paths": { "…": "…" }, + "toolchain": { + "pixi": { + "version": "0.73.0", + "assets": { + "pixi-aarch64-apple-darwin.tar.gz": "63e7cc91ef10eda71765c42e951362a084b2cbcbc93fb55c375c4f3acbfd7d00" + } + }, + "condaPack": { + "version": "0.9.2" + } + } +} +``` + +**Commit this block.** The pixi entry records the release digest: the first install trusts the +checksum published beside the release, and every install after it is checked against the value +recorded here. One pixi asset entry accumulates per host, so a mixed-platform team gets one digest +per host asset. The conda-pack entry records the exact package release the managed installer asks +pixi to install. + +Scrollcase writes the block itself; you never have to author it. To change Pixi intentionally, +edit the scroll's `pixiVersion`, install that exact release with consent, relock, rerun the licence +audit, review the changes, and rebuild. Removing `.scrollcase/toolchain/` only removes managed +executables; it does not erase the committed pin or make a floating resolver acceptable. diff --git a/docs/v2/reference/index.md b/docs/v2/reference/index.md new file mode 100644 index 0000000..12aefc9 --- /dev/null +++ b/docs/v2/reference/index.md @@ -0,0 +1,15 @@ +--- +title: Reference +description: Technical specifications for the Scrollcase CLI, workspace declarations, contract schemas, and programmatic Node APIs. +aside: false +next: false +prev: false +--- + +# Reference + +Complete technical specifications and API documentation for Scrollcase CLI flags, workspace manifests, scroll contracts, internal box archive formats, and programmatic Node bindings. + +--- + + diff --git a/docs/v2/reference/schemas.md b/docs/v2/reference/schemas.md new file mode 100644 index 0000000..3ab20e8 --- /dev/null +++ b/docs/v2/reference/schemas.md @@ -0,0 +1,82 @@ +--- +title: JSON Schemas +description: Public schema URLs, package imports, offline registration, generated types, and compatibility. +--- + +# JSON Schemas + +The shipped JSON Schemas are available both from the package and at stable public URLs: + +```text +https://scrollcase.dev/schema/v2/target.schema.json +https://scrollcase.dev/schema/v2/scroll.schema.json +https://scrollcase.dev/schema/v2/box-manifest.schema.json +https://scrollcase.dev/schema/v2/release-manifest.schema.json +https://scrollcase.dev/schema/v2/channel-manifest.schema.json +https://scrollcase.dev/schema/v2/revocations-manifest.schema.json +https://scrollcase.dev/schema/v2/signed-document.schema.json +``` + +The documentation build fails unless these public files are byte-identical to +`src/contract/schema/`, so the npm package remains the single source. + +## Package imports + +Node can import an individual schema through the package export: + +```js +import scrollSchema from 'scrollcase/contract/schema/scroll.schema.json' + with { type: 'json' }; +``` + +Or resolve a shipped file without relying on JSON module syntax: + +```js +import { readFile } from 'node:fs/promises'; +import { schemaUrl } from 'scrollcase/contract'; + +const scrollSchema = JSON.parse(await readFile(schemaUrl('scroll'), 'utf8')); +``` + +## Offline validation + +Absolute `$id` and `$ref` values identify the same schemas whether validation is online or offline. +An offline validator must register every referenced document locally under its published `$id`; +it must not fetch the network during validation. + +```js +import Ajv2020 from 'ajv/dist/2020.js'; +import targetSchema from 'scrollcase/contract/schema/target.schema.json' + with { type: 'json' }; +import scrollSchema from 'scrollcase/contract/schema/scroll.schema.json' + with { type: 'json' }; + +const ajv = new Ajv2020({ strict: true, allErrors: true }); +ajv.addSchema(targetSchema); +const validateScroll = ajv.compile(scrollSchema); + +if (!validateScroll(scroll)) throw new Error(ajv.errorsText(validateScroll.errors)); +``` + +Register `release-manifest.schema.json` before `box-manifest.schema.json`, because the latter +references the release provenance definition. Fragment references after `#` resolve inside the +registered document. + +## Generated TypeScript types + +`scrollcase/contract/types` contains declarations generated from these schemas. Contributors run: + +```sh +npm run types +npm test +``` + +Never hand-edit `src/contract/types/index.d.ts`; the drift test checks the generated output. + +## Compatibility + +All active schemas describe `schemaVersion: 2`. A v2 verifier rejects v1 rather than interpreting +it through the new contract; historical v1 boxes remain usable with the immutable Scrollcase +versions that produced them. Target IDs, document-kind strings, payload encoding, signature +algorithm, and golden fixtures do not change silently at an existing `$id`. A future breaking +change requires another schema version. diff --git a/docs/v2/reference/scroll.md b/docs/v2/reference/scroll.md new file mode 100644 index 0000000..e0a4dd6 --- /dev/null +++ b/docs/v2/reference/scroll.md @@ -0,0 +1,520 @@ +--- +title: The Scroll (scroll.json) +description: Every field of a Scrollcase scroll — identity, target, dependencies, assets, self-test, parity. +--- + +# The Scroll + +A scroll is the only input a build accepts. It is a `scroll.json` checked into your repository, +next to the `pixi.toml` that declares its dependencies and the `pixi.lock` that pins them: + +```text +scrolls/ +└── my-model/ + └── macos-aarch64-metal/ + ├── scroll.json # this document + ├── pixi.toml # dependency declaration, solved by `scrollcase lock` + └── pixi.lock # the pinned result — committed, reviewed, and installed verbatim +``` + +The parent directory is the scroll's declared `boxId`; the child is the canonical ID computed from +its declared `target`. Scrollcase checks both, so the path cannot mislabel the scroll, but neither +value is written twice inside `scroll.json`. Flat source directories are not accepted in v2. + +The machine-readable definition is [`scroll.schema.json`](/schema/v2/scroll.schema.json), also shipped +through the package export. See [JSON Schemas](/v2/reference/schemas). + +Create a new target-specific input with `scrollcase new scroll`. Interactively it asks four +things — the target, the box id, the upstream revision, and where boxes will be published — and +derives everything else, generating the matching `pixi.toml` and a starter `self_test.py`. It +refuses to overwrite an existing scroll. + +## The shortest scroll that builds + +Anything the target or the identity already determines may be left out; it is filled in when the +scroll is read. Write the decisions, not the restatements: + +```json +{ + "$schema": "https://scrollcase.dev/schema/v2/scroll.schema.json", + "schemaVersion": 2, + "boxId": "hello-box", + "modelId": "example-org-hello", + "runtimeId": "hello-box-runtime", + "version": "1.0.0", + "sourceRevision": "example-hello-v1", + "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, + "pythonVersion": "3.14", + "pixiVersion": "0.73.0", + "assetBaseUrl": "https://assets.example.org/boxes", + "selfTest": { "imports": ["json", "sqlite3"] } +} +``` + +## The same scroll, fully spelled out + +Identical in every respect — this is what the file above becomes when it is read. Declaring a +derived field is never wrong; `pythonEntryPoint` is still checked against the target either way. + +```json +{ + "$schema": "https://scrollcase.dev/schema/v2/scroll.schema.json", + "schemaVersion": 2, + "scrollVersion": "1.0.0", + "boxId": "hello-box", + "modelId": "example-org-hello", + "runtimeId": "hello-box-runtime", + "version": "1.0.0", + "sourceRevision": "example-hello-v1", + "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, + "compatibility": { "minHostAppVersion": "1.0.0", "minMacosVersion": "13.0", "minRamGb": 1 }, + "pythonVersion": "3.14", + "pixiVersion": "0.73.0", + "pythonEntryPoint": "venv/bin/python", + "modelCacheSubdir": "model-cache/hello-box", + "environment": { "MODEL_ROOT": "model-cache/hello-box", "HF_HUB_OFFLINE": "1" }, + "assetBaseUrl": "https://assets.example.org/boxes", + "assets": [], + "selfTest": { "imports": ["json", "sqlite3"], "files": [] } +} +``` + +## Identity + +| Field | Required | Meaning | +| --- | --- | --- | +| `schemaVersion` | yes | Always `2`. See [versioning](/v2/reference/box-format#versioning) | +| `scrollId` | no | Provenance identity. When omitted, Scrollcase derives `-` | +| `scrollVersion` | no | Version of the scroll itself — bump it when you change how the box is built. Defaults to `1.0.0` | +| `boxId` | yes | Identity of the box across versions. Appears in archive names, object keys, and the channel pointer | +| `modelId` | yes | Identity of what the box packages — a model, a library, an application | +| `runtimeId` | yes | Identity of the runtime environment the box provides | +| `version` | yes | Version of the box this scroll produces, as it appears in the release manifest | +| `sourceRevision` | yes | Upstream revision of the packaged source, recorded verbatim into provenance | +| `extends` | no | `"../scroll.json"`, marking this file as one target's half of a [split scroll](#one-box-several-targets) | + +"Required" here means required of the scroll a build reads. In a split scroll that is the two halves +joined, so either half may carry any given field. + +`boxId`, `modelId` and `runtimeId` are lowercase identifiers (`^[a-z0-9]+(?:[-.][a-z0-9]+)*$`) in +the published manifests — keep them to that shape. + +An explicit `scrollId` lets a project choose its source identity. It may be omitted because `boxId` +and `target` already contain the meaningful identity and the derived value is deterministic. + +::: tip Three identifiers, three questions +`boxId` answers *which artefact is this a version of?*, `modelId` answers *what is inside?*, and +`runtimeId` answers *what environment does it provide?* Several boxes may package the same payload +with different runtimes, or the same runtime for different payloads; keeping the three separate is +what lets a consumer reason about that. + +The name is historical: the first boxes carried models. What it identifies is whatever the box +packages, and a box that packages a library or an application names that. A project with nothing +to distinguish there sets it to the `boxId` — which is what `scrollcase new scroll` does when +`--model-id` is not passed, so it is a field most scrolls never think about. +::: + +## Target + +```jsonc +"target": { "platform": "linux", "arch": "x86_64", "accelerator": "cuda", "cudaVersion": "12.4" } +``` + +The supported combinations are closed — a target outside this matrix has no defined identifier +and cannot be built, signed, or routed: + +| `platform` | `arch` | `accelerator` | +| --- | --- | --- | +| `macos` | `aarch64` | `metal`, `cpu` | +| `linux` | `x86_64` | `cpu`, `cuda` | +| `windows` | `x86_64` | `cpu`, `cuda` | + +`cudaVersion` (a `major.minor` string) is **required for and only for** CUDA targets: it is part +of the box's identity, so a CUDA 12.4 build can never be mistaken for a 12.8 one. See +[The Box Format](/v2/reference/box-format#targets) for the resulting target IDs, and +[Packaging CUDA Boxes](/v2/guides/packaging-cuda) for what a CUDA scroll declares. + +Every scroll a build reads declares a target. The one exception is the base of a split scroll, +below, which holds what its targets share and so names none of them. + +## One box, several targets + +The targets of a box agree about almost everything and differ in a handful of lines. Repeating the +agreement in each of them means every change has to be made three times, correctly, and a +divergence nobody intended is invisible until a user finds it. + +A scroll may therefore be split. `scrolls//scroll.json` holds what the targets share, and +each `scrolls///scroll.json` declares `extends` plus its own differences: + +```text +scrolls/hello-box/ + scroll.json ← everything the targets share + macos-aarch64-metal/scroll.json ← extends + what this target changes + linux-x86_64-cpu/scroll.json + windows-x86_64-cpu/scroll.json +``` + +```json +{ + "extends": "../scroll.json", + "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, + "compatibility": { "minMacosVersion": "13.0" }, + "condaDependencyLicenseAudit": "scrolls/hello-box/macos-aarch64-metal/conda-licenses.json" +} +``` + +`extends` has exactly one legal value, `"../scroll.json"`. A base is always the box directory's own +`scroll.json` — there is no path to get wrong, nothing to point outside the workspace, and no chain +of bases to follow. A base declares no `target` (it holds what its targets share) and no `extends` +of its own (joining is one level, not a hierarchy); both are refused. + +Only `scroll.json` is split. `pixi.toml` and `pixi.lock` stay in each target directory, because the +solved environment is what differs most between targets. + +### How the two halves are joined + +The rule is stated per field, because a single blanket rule is wrong in both directions. Replacing +everything would make a fragment that adds one asset lose the shared ones; merging everything would +leave `execution` half from each half, producing a `python-script` that inherited a `module`. + +| Fields | Rule | +| --- | --- | +| Scalars, and the cohesive objects `target`, `execution`, `parity` | The fragment replaces the base | +| `assets`, `assetArchives`, `localFiles` | Joined, base entries first. Two entries claiming one `relativePath` is an **error** | +| `prunePaths`, `uncompressedPaths`, `selfTest.imports`, `selfTest.files` | Joined, base first, repeats dropped | +| `compatibility`, `environment` | Joined key by key; on a shared key the fragment wins | +| `selfTest.pythonCode` / `selfTest.pythonFile` | One slot: a fragment naming either replaces both | +| `extends` | Dropped — the joined scroll extends nothing | + +The distinction between the two list rules is deliberate. A repeated prune path or import is the +same instruction twice and is harmless, so it is dropped. A repeated `relativePath` means two +different sources claiming one file in the box, which is a conflict — the second would silently +overwrite the first — so it is refused rather than resolved by a precedence rule nobody would +remember. That is why a file each target ships its own copy of belongs in the **fragments**, not the +base: three sources for `entrypoint.py` cannot be one declaration. + +Order is declaration order, base first — for the joined lists and for a joined map's keys. Nothing +is sorted, and nothing needs to be: one pair of files always produces one result, which is what +rebuilding byte-identically requires. A split scroll and a hand-written whole one hold the same +entries; a joined map may serialise its keys in a different order. + +The result of the join is the **effective scroll** — the object schema validation runs against, the +build reads, and provenance records. Nothing downstream can tell which half a value came from. + +## Environment + +| Field | Required | Meaning | +| --- | --- | --- | +| `pythonVersion` | yes | Python version the box carries, recorded into provenance | +| `pixiVersion` | yes | The exact pixi release used to solve and install. `lock` and `build` refuse any other version | +| `pythonEntryPoint` | no | Interpreter path relative to the box root. Fixed per target: `venv/bin/python` on macOS and Linux, `venv/python.exe` on Windows. Derived from the target when omitted, and a mismatch is still rejected when declared | +| `modelCacheSubdir` | no | Directory relative to the box root holding model assets. Defaults to `model-cache/` | +| `environment` | no | String environment variables required whenever Scrollcase runs the box interpreter | +| `condaDependencyLicenseAudit` | no | Path (from the project root) to the reviewed licence inventory, written and declared by [`audit --write`](/v2/reference/cli#audit). When declared, the build fails if the lock no longer matches what was reviewed | + +The dependencies themselves live in `pixi.toml`, not here: + +```toml +[workspace] +name = "hello-box-macos-aarch64-metal" +channels = ["conda-forge"] +platforms = ["osx-arm64"] + +[dependencies] +python = "3.14.*" +``` + +`platforms` must equal the target's conda subdirectory — `osx-arm64`, `linux-64`, or `win-64` — +or the solve produces an environment that cannot run on the machine the box is for. + +[`scrollcase add dep `](/v2/reference/cli#add) writes into every target's manifest at once, +so they cannot drift apart, and `--from-requirements` imports an existing pip file. + +### Declared runtime environment + +`environment` is a map of names to string values, one per +[`scrollcase add env NAME=VALUE`](/v2/reference/cli#add): + +```jsonc +"environment": { + "MODEL_ROOT": "model-cache/hello", + "HF_HUB_OFFLINE": "1" +} +``` + +The builder copies the map unchanged into `box.json` and the signed release. It applies the same +values to the build self-test and every parity run, so a bad path or offline setting fails before a +box reaches a user. Consumers apply the declaration when they run the box or repeat its self-test. +On a name conflict, the signed release wins over the inherited host environment and caller-supplied +`env`; the small target-validation map still wins for its own accelerator controls, so a declared +variable cannot silently turn a CUDA or Metal check into a CPU check. + +A variable only one target needs belongs in that target's fragment, because `extends` joins +`environment` key by key. That is how a `cpu` target switches off an accelerator backend its packed +library ships anyway — `"GGML_METAL_DEVICES": "0"` for llama.cpp on macOS. See +[Running a box](/v2/guides/troubleshooting#running-a-box) for the failure that prevents. + +This does **not** replace or filter the host environment. A box inherits it exactly as before. +Scrollcase reports the resulting provenance through its CLI and Node/Python consumer APIs; see +[Environment reports](/v2/reference/api#environment-reports). Names must be non-empty and contain +neither `=` nor NUL; values are strings and may be empty but cannot contain NUL. + +## Execution intent + +`execution` records how a consumer may start the box: + +```jsonc +"execution": { + "kind": "python-script", + "script": "app/main.py", + "defaultArgs": ["--serve"] +} +``` + +or: + +```jsonc +"execution": { + "kind": "python-module", + "module": "example_model.main", + "defaultArgs": [] +} +``` + +Omit `execution` for a library-only box. `scrollcase new scroll` presents the three authoring +choices as `python-script`, `python-module`, and `library-only`; the last one deliberately emits no +execution object. + +Script authoring either hashes an existing regular project file or generates a minimal starter. +The exact SHA-256 is recorded in `localFiles`, the payload path is traversal-checked, and neither an +existing source nor an existing scroll is overwritten. + +The builder copies this object unchanged into both the signed release and `box.json`, then checks +that the script or module is present after staging and pruning. A script must be a safe relative +regular-file path. A module must use strict dotted syntax and resolve to runnable module content in +the box root or the target's Python environment; discovery inspects files and never imports the +application. `verify` repeats the schema, agreement, interpreter, and archive-presence checks +before an optional self-test can run box code. + +## Compatibility + +Constraints the installing host must satisfy. The whole object is optional — declaring no +constraint is a legitimate answer, and Scrollcase will not invent one on your behalf. What you do +declare is copied into the release manifest **verbatim and never interpreted**, so a project may +add its own fields alongside these: + +| Field | Meaning | +| --- | --- | +| `minHostAppVersion` | Lowest version of the installing application this box supports | +| `maxHostAppVersionExclusive` | Upper bound, exclusive | +| `minMacosVersion` | Minimum macOS version | +| `minRamGb` | Installed memory in decimal GB (1 GB = 1,000,000,000 bytes) | +| `minNvidiaDriverVersion` | Minimum NVIDIA driver version | + +A consumer that cannot evaluate a constraint must refuse the box rather than assume it passes. + +## Assets and payload contents + +### `assets` + +Files fetched over the network during the build. Every entry is size- and hash-checked before it +enters the payload, so a moved or replaced upstream file fails the build instead of quietly +producing a different box under the same version. The list is optional and defaults to empty. + +Do not write these by hand: [`scrollcase add asset `](/v2/reference/cli#add) fetches the URL +once and records the `sizeBytes` and `sha256` it found, which are the only two fields here that +cannot be known without downloading the file. + +```jsonc +"assets": [ + { + "url": "https://huggingface.co/example-org/model/resolve/main/model.safetensors", + "relativePath": "model-cache/hello/model.safetensors", + "sizeBytes": 438012416, + "sha256": "9f2b…c1" + } +] +``` + +Retries inside one download operation resume from a partial file, and a partial transfer is +renamed into place only after its size and hash match. The build scratch tree is recreated at +process start, so there is no cross-process cache. See [Managing Model +Weights](/v2/guides/managing-weights). + +### `assetArchives` + +Downloaded archives to expand into the payload. Extraction preserves files already present and +refuses to overwrite them. + +```jsonc +"assetArchives": [ + { + "relativePath": "model-cache/hello/weights.tar.gz", + "format": "tar.gz", + "destination": "model-cache/hello", + "stripComponents": 1, + "removeAfterExtract": true + } +] +``` + +`format` is `zip` or `tar.gz`. Archives are expanded at build time, so they **cannot be combined +with `on-demand` weights** — the build fails rather than declaring a layout that never +materialises. + +### `localFiles` + +Files copied from your own repository into the payload. Added and removed with +[`scrollcase add file`](/v2/reference/cli#add) and [`remove file`](/v2/reference/cli#remove). + +```jsonc +"localFiles": [ + { "sourcePath": "runtime/entrypoint.py", "relativePath": "entrypoint.py" }, + { "sourcePath": "legal/MODEL_NOTICE.md", "relativePath": "THIRD_PARTY_NOTICES/MODEL_NOTICE.md", "sha256": "4c7e…9a" } +] +``` + +`sha256` is an optional **pin**: when it is present, the build refuses a file whose contents no +longer match. Leave it off the files you are still writing — a script you edit every day would +otherwise fail its own build until you recomputed a digest by hand. Add it to the files that must +not change without review, such as a licence notice or a reviewed runtime shim. Either way, what +ships is hashed into the signed release, so the box's contents are always accounted for. + +[`scrollcase refresh`](/v2/reference/cli#refresh) recomputes a pin after a reviewed change, so keeping +one does not mean recomputing digests by hand. + +A pin covers the file's bytes, so anything that rewrites them between commit and build breaks it. +The usual culprit is Git's line-ending conversion: on Windows a text file is checked out with CRLF +by default and no longer matches the hash the scroll declares, and the build stops on a checkout +that looks clean. Mark the pinned paths in `.gitattributes` so they are never converted: + +```text +legal/MODEL_NOTICE.md -text +``` + +### `prunePaths` + +Payload paths deleted before packing, to keep the box to what it actually needs at run time — a +box is a multi-gigabyte download for an end user, so pruning is a user-facing concern rather than +tidiness. + +```jsonc +"prunePaths": ["venv/share/doc", "venv/lib/python3.11/site-packages/numpy/tests"] +``` + +Each entry is a **literal path** removed recursively — there is no glob support, and a path that +matches nothing is skipped silently. Over-pruning is caught by `selfTest.files`, below. + +### `uncompressedPaths` + +Payload paths stored in the archive rather than deflated, because their bytes are already +compressed — re-compressing them costs build time and makes the archive marginally larger. + +```jsonc +"uncompressedPaths": ["model-cache/hello", "corpora/images"] +``` + +An entry matches that path **and everything beneath it**, so one line can name a weights file or +the directory an `assetArchives` entry expanded into. Every path declared in `assets` is stored +automatically and does not need repeating here. + +The decision is taken from the scroll alone — nothing opens the file or reads its extension — which +is what keeps two builds of the same commit byte-identical. + +::: warning Payload paths +Every path inside the payload (`relativePath`, `destination`, `prunePaths`, `uncompressedPaths`, +`selfTest.files`) is relative and may never escape the payload root. Absolute paths, `..` segments, +and drive letters are rejected. +::: + +### `assetBaseUrl` + +Base URL the built archive and its objects are published under. It is what the signed release and +channel documents point at. Required unless passed per build with `--asset-base-url`. + +## Self-test + +Builder checks run with the payload's **own interpreter** before the box is archived. Schema +version 2 signs the import subset for a consumer to repeat; it does not carry the richer file or +`pythonCode` assertions. + +```jsonc +"selfTest": { + "imports": ["torch", "transformers"], + "files": ["model-cache/hello/model.safetensors"], + "pythonFile": "scrolls/hello-box/macos-aarch64-metal/self_test.py" +} +``` + +| Field | Required | Meaning | +| --- | --- | --- | +| `imports` | yes | One or more modules imported with the box's interpreter, added with [`add import`](/v2/reference/cli#add). These names are signed and repeated by `verify --self-test` | +| `files` | no | Files that must still exist after pruning — this is what stops an over-aggressive prune from shipping a broken box. Defaults to empty | +| `pythonFile` | no | Project path to a Python file run after the imports succeed | +| `pythonCode` | no | The same thing inline, for a single assertion. Mutually exclusive with `pythonFile` | + +Prefer `pythonFile` for anything longer than one line. A self-test is real code and deserves an +editor that knows it: in a file it keeps its syntax highlighting, its linter, and a readable diff, +where inline it is a JSON string with escaped newlines. `scrollcase new scroll` generates one next +to the scroll and points the field at it. + +The target's own platform assertion is prepended automatically, and the run happens under the +accelerator's validation environment. The file is read at build time and executed from the payload +root, so it can read what the box ships and import what it packs. A file listed in `files` that is +a deliberately deferred on-demand asset is not required to be present. After pruning, the builder +checks required files, then runs the target assertion, imports, the extra Python, and finally +optional parity. A consumer runs the target assertion and signed imports only. + +## Weights mode + +```jsonc +"weights": "embed" +``` + +`embed` (the default) packs assets into the archive: the box installs with no network and works +air-gapped, at the cost of a large artefact. `on-demand` leaves them out and carries their URL, +path, size and SHA-256 in the signed release. A caller must materialize those files; the local +consumers verify them before execution and do not download them. A build may override this with +`--weights`. See +[Managing Model Weights](/v2/guides/managing-weights). + +## Parity (optional) + +An optional numerical gate: run a check inside the box on more than one accelerator and require +the results to agree. + +```jsonc +"parity": { + "script": "checks/parity.py", + "accelerators": ["cpu", "metal"], + "tolerances": { "absolute": 1e-4, "relative": 1e-3, "minimumCosine": 0.9999 } +} +``` + +| Field | Meaning | +| --- | --- | +| `script` | Path inside the box, run with the box's own interpreter. Must print a JSON array of numbers, or an object with a `values` array | +| `accelerators` | At least two, each run under its target's validation environment. **The first is the reference** the others are compared against — conventionally `cpu` | +| `tolerances` | At least one bound. `absolute` and `relative` are finite numbers greater than zero; `minimumCosine` is finite and at most 1 | + +Every declared threshold is enforced conjunctively: passing one never excuses breaching another. +Scrollcase runs the check and enforces the thresholds; what the check computes, and what closeness +is acceptable, belong to your project. Full treatment in +[Accelerator Parity](/v2/guides/accelerator-parity). + +## Validation summary + +Validation is ordered so malformed input cannot trigger a process, fetch, or build-directory +mutation: + +1. Parse `scroll.json` and, when it declares `extends`, read and join its base first — neither half + of a split scroll is a complete document, so validating one alone would report the other half's + fields as missing. Validate the joined result against the shipped scroll and target schemas. +2. For a nested scroll, require a target, and require the parent and child directories to match + `boxId` and the canonical target; reject invalid target/entry-point combinations and default + on-demand weights with `assetArchives`. +3. Resolve a build-time `--weights` override and repeat the archive/policy check. +4. Require a matching native host, discover the exact tools, and require `pixi.lock`. +5. Record Git provenance and reject a dirty tree unless `--allow-dirty` was explicit. +6. Only then recreate build state, install from the lock, download verified assets, and enforce + semantic checks whose inputs appear later, including lock/audit agreement. diff --git a/docs/v2/white-paper.md b/docs/v2/white-paper.md new file mode 100644 index 0000000..b9ad607 --- /dev/null +++ b/docs/v2/white-paper.md @@ -0,0 +1,6348 @@ +--- +title: Technical White Paper +description: A complete, self-contained technical description of Scrollcase — its vocabulary, boundary, substrate, contract, build pipeline, signing, consumers, invariants and tests. +outline: [2, 3] +next: false +prev: false +--- + +# Scrollcase — Technical White Paper + +Scrollcase turns a declarative **scroll** into a **box**: a portable, locked, self-contained Python +environment for one operating system and one accelerator, packed so that it runs somewhere other +than where it was built, signed so that whoever receives it can prove what they received, and +accompanied by a dependency licence inventory. + +This document describes how that is done, module by module, together with the specifications of the +substrate it is built on and a glossary of every technical term it uses. It is written to be +**studied end to end** rather than consulted in spots. + +
+ +::: info The canonical copy +This document is a single Markdown file with no external assets. It is meant to be downloadable and +studiable offline. + +- Read or download the source: + [`docs/white-paper.md`](https://github.com/suffro/scrollcase/blob/main/docs/white-paper.md) on GitHub +- Direct raw download: + [`raw.githubusercontent.com/suffro/scrollcase/main/docs/white-paper.md`](https://raw.githubusercontent.com/suffro/scrollcase/main/docs/white-paper.md) +- PDF download: `print → PDF` + +::: + +
+ +
+ +### Document map + +The order is deliberate: vocabulary first, then the format, then what produces it, then what +consumes it, then the properties that hold across all of it. + +| Section | Subject | +| --- | --- | +| [1. How to read this document](#_1-how-to-read-this-document) | Audience, conventions, self-containment | +| [2. Glossary](#_2-glossary) | Every technical term used here, canonical and domain | +| [3. The problem and the boundary](#_3-the-problem-and-the-boundary) | What Scrollcase is, and what it deliberately is not | +| [4. The substrate](#_4-the-substrate) | pixi, conda-pack, conda-forge, and the three runtime dependencies | +| [5. The contract](#_5-the-contract) | `src/contract/`: targets, envelopes, links, schemas, fixtures, types | +| [6. The build pipeline](#_6-the-build-pipeline) | `src/build/`: the ordered steps and every module that serves them | +| [7. Signing and custody](#_7-signing-and-custody) | `src/sign/`: keys, local signing, external signers, verification | +| [8. The consumers](#_8-the-consumers) | Node, Python, and Rust, side by side, and their shared conformance fixtures | +| [9. The command line](#_9-the-command-line) | The nine verbs and where the thin-CLI boundary runs | +| [10. The invariants](#_10-the-invariants) | Determinism, provenance, verify-never-trust, and the paths that break silently | +| [11. Test map](#_11-test-map) | Which test proves which behaviour | +| [12. Appendices](#_12-appendices) | Module summary, index of public exports | + +Every section is present in this copy; the document is complete and self-contained. + +
+ +## 1. How to read this document + +
+ +### 1.1 Audience + +
+ +This paper addresses three kinds of reader, and assumes nothing about which one you are beyond +general software engineering literacy: + +- **Engineers integrating Scrollcase** — building against its Node or Python surfaces, or driving + its command line from a pipeline, and needing to know exactly what each call guarantees. +- **Engineers auditing Scrollcase** — establishing what a box commits to, what is checked before + anything executes, where trust begins and where it ends. +- **Contributors** — changing the code, and needing the reasoning that makes a given shape the right + one before replacing it with another. + +No prior familiarity with conda, pixi, or the box format is assumed. Every term is defined here. + +
+ +### 1.2 The self-containment rule + + +Everything this document relies on is explained inside it. There is no prerequisite reading and no +further reading: if an explanation would have to happen elsewhere, it happens here instead. + +
+ +What does appear are **provenance references** — citations telling you where the code just described +lives, so that a claim can be checked against its implementation: + +- source paths, written as `src/build/box.mjs` +- schema names, written as `release-manifest.schema.json` +- test files, written as `tests/unit/archive-security.test.mjs` + +These are citations, not redirections. Nothing in this paper requires you to open them. + +**Rejected:** the more usual documentation style of linking each concept to a page that develops it. +For a document meant to be read linearly and offline, an outbound link is a hole: it either breaks +the reading or breaks the download. The cost is deliberate duplication with the rest of the +documentation site, which describes the same system for a different purpose — how to use it, rather +than how it is built. + +
+ +### 1.3 Conventions + + +**Canonical terms.** Scrollcase has a small controlled vocabulary — box, scroll, target, payload, +release, channel, revocations, self-test, parity. These words mean exactly one thing each, +everywhere: in the code, in error messages, in the schemas, and here. On first significant use a +canonical term links to its glossary entry, like [box](#box). Everything else in the glossary is +linked the same way. + +
+ +**Casing is functional.** *Scrollcase* is the project; `scrollcase` lowercase is an identifier — the +command, the npm package, the exported subpaths, `scrollcase.config.json`, the default +`scrollcase.box` document namespace. Where this document writes one or the other, it means that one. + +**Decisions carry their rejected alternative.** A design statement without the option it displaced +is an assertion, not a decision. Where a choice was genuinely contested, the discarded alternative +is recorded with it — as in the paragraph above. + +**Code excerpts** are illustrative and abridged. The implementation is the authority; each excerpt +names the file it came from. + +**Diagrams are plain text.** Every diagram here is drawn inside a code block rather than rendered by +a diagramming library, so that it survives being printed to PDF and stays readable in the raw +Markdown source. The rest of the site uses rendered diagrams; this document deliberately does not. + +**Two words are overloaded**, unavoidably, because the surrounding ecosystems already claimed them. +*Payload* means both the tree assembled before archiving and the encoded contents of a signed +envelope; *channel* means both a Scrollcase release pointer and a conda package source. Both pairs +have separate glossary entries, and this document always makes clear which is meant. + +
+ +### 1.4 What this document describes + + +| Scrollcase version | 0.6.0 | +| --- | --- | +| Box format | `schemaVersion: 2` | +| Substrate | pixi + conda-pack + conda-forge | +| Runtime dependencies | `tar`, `yauzl`, `yazl` | +| Node engine | >= 20 | +| Licence | Apache-2.0 | + +
+ +Version 2 is a clean break from version 1. A v2 verifier rejects a `schemaVersion: 1` document with +an explicit unsupported-version error rather than reinterpreting it, and this paper describes v2 +only. Published v1 artefacts remain usable with the Scrollcase versions that produced them, and are +otherwise out of scope here. + +## 2. Glossary + +Every term this document uses in a specialised sense is defined here, in one place, so that a reader +can start from any section and resolve unfamiliar vocabulary without leaving the page. Entries are +grouped — canonical Scrollcase vocabulary first, then the packaging, filesystem, cryptography and +distribution terms the design borrows from elsewhere — and alphabetised within each group. + +
+ +### 2.1 Canonical vocabulary + +These are the words Scrollcase controls. Each means exactly one thing, and the code, the schemas and +the error messages use no synonym for any of them. + +
+ +
+ +#### Box + +The built artefact: a single ZIP archive containing a complete, relocated Python environment, a +self-describing manifest, any embedded assets, and — when the [scroll](#scroll) declares a reviewed +licence audit — a dependency licence inventory. A box is built +for exactly one [target](#target). It is never called an image or a container — it is neither, it +carries no operating system and no isolation boundary, and borrowing either word would import +expectations Scrollcase does not meet. + +Reference: `src/build/box.mjs`, `docs/reference/box-format.md`. + +
+ +
+ +#### Box ID + +The stable identifier of the thing being packaged, declared by the [scroll](#scroll) as `boxId` and +carried into `box.json` and the [release](#release). One box ID spans every version and every target +of that box. It is distinct from `modelId` and `runtimeId`, which are the publishing project's own +identifiers for what the box contains and what runs it; Scrollcase stores and transports all three +without interpreting them. + +
+ +
+ +#### `box.json` + +The manifest packed **inside** the archive, at its root, so that an extracted box is +self-describing: whoever holds the directory but not the [release](#release) document can still +determine what it is, which target it is for, which interpreter to use, and how it was built. +Verification compares it field by field against the signed release; the two disagreeing is a +failure, not a merge. + +
+ +
+ +#### Channel + +One of the three signed document types, and a mutable pointer: it names which [release](#release) is +current for a given box on a given track. The tracks are `nightly`, `beta` and `stable`, ordered +from least to most stable. A channel document is signed exactly like a release, so moving a pointer +is an act somebody has to authorise cryptographically. + +Not to be confused with a [conda channel](#conda-channel), which is a package source. + +Reference: `CHANNELS` in `src/contract/document-shape.mjs`, `channel-manifest.schema.json`. + +
+ +
+ +#### Document namespace + +The dotted lowercase prefix on every signed document's `kind` discriminator — `scrollcase.box` by +default, giving `scrollcase.box.release`, `scrollcase.box.channel`, `scrollcase.box.revocations`. +The namespace belongs to the **publishing project**, not to the tool: a project with boxes already +installed in the field keeps emitting the namespace its clients recognise, and Scrollcase never +hard-codes one. + +Reference: `documentKinds()` in `src/contract/document-shape.mjs`. + +
+ +
+ +#### Parity + +The optional cross-accelerator numerical gate. A scroll may declare a check script inside the box, +the accelerators to run it under, and tolerances (`absolute`, `relative`, `minimumCosine`); +Scrollcase runs the check once per accelerator, compares every run against the first, and fails the +build on a breach. It answers a packaging question — *did this box get the wrong wheels, a CPU-only +build shipped as CUDA, a broken BLAS?* — and never a scientific one. The threshold is the project's +to declare; enforcing it is Scrollcase's to do. + +Reference: `src/build/parity.mjs`. + +
+ +
+ +#### Payload + +The tree assembled on disk before archiving: `box.json`, the packed environment under `venv/`, any +embedded assets, and `THIRD_PARTY_NOTICES/`. What the archive contains is exactly this tree, so +statements about "what a payload may contain" — which links, which entry names — are statements +about the archive too. + +Distinct from a [document payload](#document-payload), which is the encoded content of a signed +envelope. + +
+ +
+ +#### Release + +One of the three signed document types, and the immutable description of one built box: identity, +target, compatibility requirements, where the archive lives, its size and SHA-256, the import subset +a consumer should repeat, the asset policy, and [provenance](#provenance). A release is never edited +after signing; a correction ships as a new version. + +Reference: `release-manifest.schema.json`. + +
+ +
+ +#### Revocations + +One of the three signed document types: a signed statement that named releases must no longer be +used. Scrollcase defines the format and can verify such a document; publishing one, and acting on +it, belong to whoever distributes boxes. + +Reference: `revocations-manifest.schema.json`. + +
+ +
+ +#### Scroll + +The declarative input, stored as `scroll.json`, and the only input a build accepts. It states box +identity, the target, versions, the dependencies to solve, asset declarations, weights policy, the +self-test, execution intent, and optional compatibility and parity blocks. Scrolls live under +`scrolls///` by default, so that every target variant of one box is visible +together. + +Reference: `scroll.schema.json`, `src/build/scroll.mjs`. + +
+ +
+ +#### Self-test + +The import check run with the box's **own** interpreter — not the host's — as the last step before a +payload is allowed to become an archive. The builder runs a platform assertion, the declared +imports, optional scroll-only Python code, and file assertions. The release signs the import subset, +which is the part a consumer can repeat after extraction; the scroll-only assertions stay builder +checks, because they are not part of the signed release and pretending otherwise would claim a +consumer verified something it never saw. + +
+ +
+ +#### Signed document + +The single envelope every Scrollcase document travels in: `schemaVersion`, `payloadEncoding`, +`payloadBase64`, `payloadSha256`, and a non-empty array of signatures. It is a container, not a +type — the type is inside, discriminated by the payload's `kind`. + +Reference: `signed-document.schema.json`, `src/contract/documents.mjs`. + +
+ +
+ +#### Target + +The `(platform, arch, accelerator)` triple a box is built for, plus a CUDA ABI version when the +accelerator is CUDA. The supported matrix is closed: `macos/aarch64` with `metal` or `cpu`, +`linux/x86_64` with `cpu` or `cuda`, `windows/x86_64` with `cpu` or `cuda`. A box is built for one +target and makes no claim about any other. + +Reference: `src/contract/targets.mjs`. + +
+ +
+ +#### Target adapter + +What a target implies for the built payload: the Python layout inside the box (`venv/bin/python` +versus `venv/python.exe`), the scripts directory, the executable suffix, the launcher kind, the +pinned archive backend, how native libraries are inspected on that platform, the environment that +forces a run onto one accelerator, and the platform assertion prepended to every self-test. Adapters +are part of the format rather than an implementation detail, because a consumer unpacking a box +relies on that layout. + +
+ +
+ +#### Target ID + +The canonical slug a target reduces to: `--`, except CUDA, which +appends the version with no separator — `linux-x86_64-cuda12.4`. It appears in archive names, object +keys, routes and directory names, so every implementation of the format must produce it character +for character. `cudaVersion` is required for CUDA and forbidden everywhere else, so a slug is never +ambiguous. + +Reference: `boxTargetId()` in `src/contract/targets.mjs`; +golden cases in `fixtures/target-id-contract.json`. + +
+ +
+ +#### Workspace + +The project layout Scrollcase operates in, declared by a `scrollcase.config.json` at the project +root and discovered by walking up from the working directory. It resolves where scrolls, build +directories, distribution output, keys and the project-local toolchain live. Defaults are `scrolls/` +and `.scrollcase/{build,dist,keys}`. Paths come from the project, never from the tool's own location +on disk. + +Reference: `src/build/workspace.mjs`. + +
+ +
+ +### 2.2 Packaging and environment terms + + +
+ +#### ABI + +*Application Binary Interface* — the binary-level contract between compiled artefacts: calling +conventions, symbol names, struct layouts. Two builds of a library with the same version can be ABI +incompatible, which is why a CUDA target pins a CUDA ABI version (`12.4`) rather than trusting a +package version to describe compatibility. + +
+ +
+ +
+ +#### Accelerator + +The compute backend a box is built to use: `cpu`, `metal` (Apple's GPU API, reached from Python +through MPS), or `cuda` (NVIDIA, with an ABI version). The accelerator is part of the target because +it changes which packages the solver selects — a CPU build and a CUDA build of the same library can +differ in nothing but that. + +
+ +
+ +#### Air-gapped + +Describes an installation environment with no network access at all. A box built with embedded +weights installs and runs air-gapped, because everything it needs is inside the archive. This is the +property `embed` exists to preserve. + +
+ +
+ +#### Asset + +A file the box needs that is not a Python package — model weights, tokenizers, fixtures. Assets are +declared in the scroll with a URL or local path, a size and a SHA-256, and are size- and +hash-checked before they enter the payload. Under `embed` they are packed into the archive; under +`on-demand` they are left out and their descriptors travel in the signed release. + +Reference: `src/build/assets.mjs`. + +
+ +
+ +#### conda + +A package manager and package format originating in the scientific Python world, whose distinguishing +property is that it packages **compiled artefacts** — shared libraries, compilers, CUDA runtimes — +alongside Python code, rather than assuming they are already present on the host. + +
+ +
+ +#### conda channel + +A source of conda packages: an indexed repository of package archives organised by +[subdir](#conda-subdir). Scrollcase's generated manifests pin exactly one, `conda-forge`. Distinct +from a Scrollcase [channel](#channel), which is a signed release pointer. + +
+ +
+ +#### conda-forge + +The community-maintained conda channel Scrollcase builds from. It matters here for three reasons: it +distributes native libraries as a coherent, mutually compatible package set rather than as +independently built wheels; it covers the scientific and machine-learning stack including +accelerator builds; and it annotates every package with an SPDX licence, which is what makes a +derived licence inventory possible without inspecting package contents. + +
+ +
+ +#### conda-meta + +The directory inside a conda prefix holding one JSON record per installed package, written by the +installer. Scrollcase rewrites these records into a canonical form — keeping only `name`, `version`, +`build` and `license` — because as written they carry per-file hashes that differ between two +installs of the same lock, and absolute paths into the build machine's package cache. + +Reference: `canonicalizeCondaRecords()` in `src/build/pixi.mjs`. + +
+ +
+ +#### conda-pack + +The tool that turns an installed conda [prefix](#prefix) into a relocatable archive. It collects the +prefix's contents and replaces the build-time prefix string inside text files with a neutral +placeholder, so the result can be extracted anywhere. + +
+ +
+ +#### conda subdir + +The conda ecosystem's platform identifier — `osx-arm64`, `linux-64`, `win-64`. It is the value of +`platforms` in the generated pixi manifest, and it must equal the one implied by the box's target, +or the solve produces an environment that cannot run on the machine the box is for. + +
+ +
+ +#### Lockfile + +A file recording the exact resolved set of packages — names, versions, builds, sources and hashes — +that a manifest's constraints produced at solve time. Scrollcase commits `pixi.lock` and installs +from it without re-resolving, which is what makes two builds of one commit produce the same +environment. It also carries an SPDX licence per package, which is the input to the licence +inventory. + +
+ +
+ +#### Manifest + +Two distinct files carry this name and this document distinguishes them explicitly: the **pixi +manifest** (`pixi.toml`) states dependency constraints, channels and platforms as *input to a +solve*; the **box manifest** ([`box.json`](#box-json)) describes a built box as *output*. Where the +word appears alone it means whichever the surrounding sentence is about. + +
+ +
+ +#### Prefix + +A conda installation root: a directory containing `bin/` (or `Scripts/` on Windows), `lib/`, +`conda-meta/` and everything else an environment consists of. Every path inside a prefix is +meaningful relative to it — which is what makes moving one non-trivial, and what relocation is +about. Inside a box the prefix is `venv/`. + +
+ +
+ +#### PyPI + +The Python Package Index, the default source for `pip` and the home of [wheels](#wheel). Scrollcase +does not build from it: the guarantees it needs about native libraries are properties of a +coherently built channel, not of independently published wheels. PyPI appears in the tool only as an +option for how a *consumer's* project installs the Scrollcase Python consumer package itself. + +
+ +
+ +#### Relocation + +Making an environment work at a directory other than the one it was built in. The problem is that +build-time absolute paths get written into scripts, service files and package metadata. Scrollcase's +model is: pack with conda-pack (which replaces the build prefix with a placeholder), extract into +the box, delete the service files that still carry the prefix, canonicalise the package records, +settle the symbolic links, and rewrite generated console scripts to resolve Python next to +themselves. A box then needs no relocation step at install time. + +
+ +
+ +#### Solve + +The act of turning dependency constraints into an exact package set that satisfies all of them +simultaneously, for one platform. Scrollcase separates solving (`lock`, run by a human when +dependencies change, producing a reviewable diff) from installing (`build`, which consumes the +committed result and never re-resolves). + +
+ +
+ +#### SPDX + +A standard vocabulary of licence identifiers — `MIT`, `Apache-2.0`, `BSD-3-Clause` — used by +conda-forge package metadata and therefore by the lock. Scrollcase's licence inventory reports these +identifiers as declared; a package declaring no licence fails the parse outright, because an +unlicensed dependency is a legal problem rather than a reporting gap. + +
+ +
+ +#### Wheel + +The binary distribution format of the PyPI ecosystem (`.whl`), the unit `pip install` normally +consumes. Wheels can and do contain compiled code, but each is built independently by its own +publisher, so a set of wheels is not automatically a mutually compatible set of native libraries. +That difference is why the substrate is conda-based. + +
+ +
+ +#### `venv/` + +The directory inside a box holding the packed environment — the relocated conda prefix, complete +with its interpreter, libraries and canonicalised `conda-meta/`. The name is conventional; the +contents are a conda prefix, not a Python `venv` in the standard-library sense, and nothing inside a +box ever runs `conda`. + +
+ +
+ +### 2.3 Filesystem and archive terms + + +
+ +#### Content-addressed + +Named by the hash of its own contents rather than by an assigned name. A box archive's object key +contains its SHA-256, so the name changes if a single byte does; a signed release can therefore +commit to exactly the bytes it describes, and a consumer can verify what it received against the +name it fetched. + +
+ +
+ +
+ +#### Digest + +The fixed-length output of a hash function over some bytes — here always SHA-256, written as 64 +lowercase hexadecimal characters. Used throughout: for the archive, for assets, for the downloaded +toolchain, for the lockfile, and for the encoded payload of every signed document. + +
+ +
+ +#### Materialise + +To replace a symbolic link with a real copy of what it points at. Scrollcase materialises every link +it cannot prove safe to carry, which keeps the payload correct at the cost of duplicated bytes, and +is the reason the link rule described in section 6 exists at all. + +
+ +
+ +#### Path traversal + +An archive entry whose name escapes the directory it is extracted into — `../../etc/passwd`, an +absolute path, a Windows drive letter. Every entry name a Scrollcase archive contains is validated +against this on the way out, by the builder, by `verify`, and independently by each consumer. + +Reference: `safeRelativePath()` in `src/build/filesystem.mjs`. + +
+ +
+ +#### Shebang + +The `#!` first line of an executable text file, telling the operating system which interpreter to +run it with. Generated console scripts in a conda prefix embed the **build machine's** absolute +interpreter path there, which is one of the concrete things relocation has to erase. + +
+ +
+ +#### Soname + +The versioned name convention for shared libraries on Linux — `libfoo.so` → `libfoo.so.6` → +`libfoo.so.6.2.1`, the first two usually being symbolic links to the third. It is the single largest +reason a conda prefix is dense with links, and the reason materialising all of them was measured to +make roughly 60% of an extracted Linux box duplicates of its own bytes. + +
+ +
+ +#### Symbolic link + +A filesystem entry whose content is a path to another entry. Links are the classic way an archive +writes outside the directory it was extracted into, so Scrollcase carries one only when it can prove +— purely lexically, so that every host answers identically — that its target is relative, stays +inside the payload after `..` is applied segment by segment, and ends at a regular file. Everything +else is materialised. Windows boxes carry no links at all. + +Reference: `src/contract/links.mjs`. + +
+ +
+ +#### Zip64 + +The ZIP extension that lifts the format's 4 GiB and 65,535-entry limits. Box archives are Zip64 +*capable* — a packed scientific environment routinely exceeds the entry count, and embedded weights +routinely exceed the size — while the writer emits Zip64 structures only where they are needed, so +small boxes stay readable by the widest range of tools. + +
+ +
+ +### 2.4 Cryptography and distribution terms + + +
+ +#### base64 + +The encoding that represents arbitrary bytes as ASCII text. A signed document's payload is +base64-encoded UTF-8 JSON — `payloadEncoding: "base64-json-utf8"` — which is what makes a signature +verifiable by hashing bytes as transmitted. + +
+ +
+ +
+ +#### Canonical JSON + +Any scheme for reducing a JSON value to one unambiguous byte sequence, so that two implementations +signing the same value sign the same bytes. Scrollcase deliberately does **not** use one: the +payload travels as exact base64 of the bytes that were signed. + +**Rejected:** canonicalisation. It requires every client, in every language, to implement identical +key ordering, number formatting and string escaping — historically the richest source of +cross-language signature bugs. Transmitting the exact bytes removes the problem instead of solving +it repeatedly. + +
+ +
+ +#### Detached signature + +A signature stored separately from the artefact it covers, so the artefact itself is left byte for +byte unmodified. A Scrollcase [release](#release) is detached in exactly this sense with respect to +the **archive**: the archive is never rewritten to hold a signature, and the signed document commits +to it through its size and SHA-256. With respect to the release *metadata*, the signature is not +detached — payload and signatures travel together inside one envelope. + +
+ +
+ +#### Document payload + +The content of a signed envelope: UTF-8 JSON, base64-encoded into `payloadBase64`, with its SHA-256 +in `payloadSha256`. Decoding checks that hash before the contents are read at all, which catches a +truncated or edited document before anything acts on it. Distinct from the box +[payload](#payload) tree. + +
+ +
+ +#### ed25519 + +The digital signature scheme Scrollcase uses, and the only one the format defines. It is chosen for +small keys and signatures, fast verification, and the absence of parameter choices that can be got +wrong. Keys are generated with Node's own `crypto` module; there is no cryptographic dependency to +audit. + +
+ +
+ +#### Envelope + +The outer, type-agnostic structure of a signed document — the fields that let a verifier check +integrity and authenticity before it knows or cares what kind of document it holds. + +
+ +
+ +#### Key ID + +A short stable identifier for a signing key, carried by each signature so that a verifier can tell +which key produced it. A document is accepted when **any one** of its signatures verifies against a +trusted key, which is what allows a key to be rotated without reissuing every document already +published. + +
+ +
+ +#### Trust key + +A public key a consumer has decided to trust, supplied to verification by the caller. Scrollcase +verifies against the keys it is given; deciding which keys those are is the caller's policy, not the +tool's. + +
+ +
+ +### 2.5 Process and guarantee terms + + +
+ +#### Determinism + +The property that rebuilding the same commit produces a byte-identical archive. It is maintained by +construction: fixed archive timestamps, stable entry ordering, modes derived from the target +adapter, the build time taken from the commit rather than the clock, canonicalised package records, +and no random value anywhere. Determinism is what makes an independent rebuild a meaningful check on +a published box. + +
+ +
+ +
+ +#### Dirty tree + +A working tree with uncommitted changes, including untracked files, while respecting Git ignore +rules. Building from one requires `--allow-dirty` and is recorded in the box as +`sourceTreeDirty: true`, because a build that cannot be reproduced from its recorded revision has to +say so. + +
+ +
+ +#### Injection seam + +A dependency deliberately passed in rather than reached for, so that a test can substitute it. +Scrollcase has two of consequence: subprocess execution, which goes through `run` / `runResult`, and +network access, which goes through an injectable `fetch`. They are the reason the unit suite can +exercise the real pipeline without a toolchain, without the network, and without writing outside a +temporary directory. + +
+ +
+ +#### Provenance + +The record of where a box came from: which scroll and scroll version, the 40-hex commit of the +source tree that built it, whether that tree was dirty, the upstream revision of the packaged model +source as the scroll declared it, the Python and pixi versions, the SHA-256 of the lock the +environment was solved from, and the build timestamp taken from the commit. It is recorded from +observed state and never accepted from caller input. + +
+ +
+ +#### Weights mode + +Whether a box carries its assets or refers to them: `embed` packs them into the archive, so the box +installs air-gapped at the cost of a large artefact; `on-demand` leaves them out and carries their +URL, path, size and SHA-256 in the signed release and in `box.json`. The declared hash is what makes +deferring safe — the release commits to exactly which bytes are expected, whatever host serves them. + +
+ +## 3. The problem and the boundary + +
+ +### 3.1 The problem + + +A scientific or machine-learning Python environment is mostly not Python. Underneath the imports sit +compiled libraries — BLAS and LAPACK implementations, image and audio codecs, compression libraries, +accelerator runtimes — each with its own ABI, its own build flags, and its own expectations about +what else is present on the machine. The Python code on top is a thin veneer over that, and it is +the veneer that the usual packaging tools describe well. + +
+ +This produces a specific, recurring failure. An environment works on the machine where it was +assembled, and then: + +- **it cannot be reproduced.** Constraints re-resolve months later to a different set, or a + dependency's newest build changes an ABI, and the environment that installs today is not the one + that was tested. +- **it cannot be moved.** Absolute paths from the build machine are baked into scripts, service + files and metadata; a directory copied elsewhere is a directory that no longer runs. +- **it cannot be verified.** The receiver has a set of files and a hope. Nothing states what the + bytes should have been, and nothing signed says who produced them. +- **it cannot be inventoried.** Somebody eventually asks what is inside and under which licences, + and the answer has to be reconstructed by inspection. + +Each of these has partial answers in isolation. A lockfile addresses reproducibility but not +relocation. A container addresses relocation but assumes a container runtime, which a desktop +application, an offline workstation, or a locked-down laboratory machine may not have. A signature +addresses verification but only if something upstream produced a stable artefact worth signing. What +is missing is a single artefact that is all four at once. + +
+ +### 3.2 What Scrollcase is + + +Scrollcase produces exactly that artefact, and defines the format so it can be verified by anyone. + +
+ +```text + scroll.json the declarative input, the only input a build accepts + | + v + pixi.lock solved once by a human, committed, reviewed + | + v + installed conda prefix materialised from the lock, never re-resolved + | + v + payload tree box.json | venv/ | assets | licence notices + | + v + deterministic ZIP content-addressed: its name is its own digest + | + v + signed release + a signed channel pointer to it + | + v + any consumer verify, extract, run +``` + +Read as guarantees rather than as steps, that pipeline says: + +1. **One declarative input.** A build accepts a [scroll](#scroll) and nothing else. There is no + imperative build script, no hook, and no place for a build to acquire behaviour that is not + written down in a reviewable file. +2. **Locked, not resolved.** The environment is solved once by a human running `lock`, and the + result is committed. `build` installs the locked set without re-resolving. +3. **Packed for elsewhere.** The prefix is packed and repaired so that the box runs from any + directory on any machine matching its target, with no install-time fixer, no activation script, + and no build-machine path anywhere inside it. +4. **Proved before it ships.** The box's own interpreter imports the declared modules before the + payload is allowed to become an archive. +5. **Deterministic.** The same commit rebuilds to the same bytes, so an independent rebuild is a + real check. +6. **Signed and self-describing.** A release commits to the archive's size and SHA-256; the archive + carries a manifest that must agree with it. +7. **Inventoried.** When the scroll declares a reviewed licence audit, the dependency licence + inventory is derived from the lock, so it is a property of what was solved rather than of + somebody's notes. + +Scrollcase is **a library as well as a command line**. Its Node surfaces are `scrollcase/contract`, +`scrollcase/contract/browser`, `scrollcase/contract/types`, `scrollcase/build`, `scrollcase/sign` +and `scrollcase/consumer`, plus the published schemas and fixtures; the Python consumer package is +imported as `scrollcase_consumer`. The nine command-line verbs — `init`, `new`, `doctor`, `keygen`, +`lock`, `audit`, `build`, `verify`, `run` — are a thin layer over those surfaces, not a separate +implementation. + +It is open source under Apache-2.0 and vendor-neutral: it carries no reference to any specific +consuming project, anywhere. + +
+ +### 3.3 What Scrollcase is not + + +This boundary is the point of the project, and it is the thing most likely to erode, because every +individual crossing of it looks convenient at the time. + +
+ +**Not a distribution system.** Scrollcase may prepare and execute a caller-supplied local box, but +it does not select channels, download boxes, update installations, promote, revoke, publish, or +serve. The consumer APIs operate on release documents, archives, trust keys and destinations that +the caller supplies. + +**Not a CI system.** No model catalogue, no runner allocation, no cost policy, no build-evidence +records for somebody else's pipeline. + +**Not a scientific validator.** Scrollcase *enforces* a numerical tolerance its user declared — see +[parity](#parity) — and never decides what is scientifically correct or what a fixture means. + +**Not tied to any consuming project.** No consumer's name appears in identifiers, error messages, +environment variables, default paths, wire strings, or examples. A project-specific value is +declared by the project, in its config, its scroll, or a flag, and Scrollcase stays ignorant of what +it means. + +
+ +### 3.4 Why the boundary is drawn there + + +The boundary is not modesty about scope. It is what keeps the guarantees provable. + +
+ +Every guarantee in section 3.2 is a statement about a local, closed operation: these inputs produce +these bytes; these bytes hash to this value; this signature verifies against this key. Each can be +checked by rerunning it. A registry, a promotion policy, or an update mechanism introduces +guarantees of an entirely different kind — about availability, about rollout, about what a fleet of +installed clients believes at a given moment — and those cannot be checked by rerunning anything. +A tool that offered both would have to keep proving both sets at once, on every release, and the +weaker set would set the pace. + +There is also a compositional argument. Most projects that need signed environments already have a +distribution mechanism: an object store, a CDN, an internal artefact service, an application updater +they have already threat-modelled. A tool that stops at "here is a signed artefact and here is how +to verify it" composes with all of them. A tool that ships its own registry composes with none. + +**Rejected:** folding download, channel selection, update and lifecycle policy into the tool — which +is the obvious next feature request, and was declined for the reasons above. What Scrollcase does +instead is define the formats those layers need: a [release](#release) that commits to an archive, a +[channel](#channel) that points at a release, a [revocations](#revocations) document that withdraws +one. The formats are specified and verifiable; the policies that use them belong to whoever owns the +fleet. + +::: warning A note for contributors +A change that crosses this boundary is wrong even when it would be convenient, and even when it is +small. The characteristic shape of such a change is a helper that fetches something, a default that +encodes somebody's rollout policy, or a field that only makes sense to one consumer. +::: + +## 4. The substrate + +Scrollcase supports exactly one dependency backend: **pixi + conda-pack + conda-forge**. pixi solves +and installs, conda-pack relocates the resulting prefix, conda-forge supplies the packages. + +**Rejected:** a second backend for projects already standardised on a wheel-based tool such as +`uv`. A packaging tool's product is its guarantees — that an environment installs, relocates, +self-tests and is reproducible from a lock. Two backends means proving every guarantee twice, on +every platform, for every release, and the guarantees are the product. Projects on another tool +convert their scrolls once; Scrollcase avoids a permanent double burden. + +The conda-forge path also solves the problem a wheel-based one structurally cannot, which is the +subject of the next subsection. + +
+ +### 4.1 conda-forge + + +conda-forge is a community-maintained [conda channel](#conda-channel): an indexed repository of +packages, organised by [conda subdir](#conda-subdir), built by a shared infrastructure against a +shared set of pinned base libraries. + +
+ +Three properties make it the substrate rather than a substrate. + +**It distributes native code as a coherent set.** A conda package can contain anything a prefix +needs — a shared library, a compiler runtime, a CUDA toolkit component — and the channel's packages +are built against each other's pinned versions. A set of wheels is not this: each wheel is built +independently by its publisher, each vendors or expects native libraries on its own terms, and their +mutual compatibility is a coincidence that usually holds. For a stack that is mostly compiled code, +"usually holds" is exactly the failure mode Scrollcase exists to remove. + +**It covers the accelerator matrix.** CPU, CUDA and Metal builds of the major scientific and machine +learning packages exist in the channel, selected by the solver from the constraints the scroll +declares. This is what lets a target's accelerator be a *solve input* rather than a post-hoc +substitution. + +**Every package declares an SPDX licence.** That metadata is recorded per package in the +[lockfile](#lockfile), which is what makes the dependency licence inventory a pure function of the +lock — computable without building anything, and reviewable when dependencies change rather than at +the end of a multi-gigabyte build. + +Generated pixi manifests pin exactly one channel: + +```toml +[workspace] +name = "example-model" +channels = ["conda-forge"] +platforms = ["osx-arm64"] +``` + +Reference: `pixiManifest()` in `src/build/authoring.mjs`. + +The single-channel pin is deliberate. Channel priority across multiple sources is one of the classic +ways a conda environment becomes irreproducible: the same constraints resolve differently depending +on which channel wins, and which channel wins depends on configuration that is easy to leave out of +version control. One channel, named in the committed manifest, removes the question. + +Note also `platforms`: it is a single-element list holding the target's conda subdir. The manifest +pins the channels **and** the single target platform, so resolution is host-independent and no +per-invocation platform flag is needed anywhere in Scrollcase's argument vectors. + +
+ +### 4.2 pixi + + +pixi is a conda-ecosystem workspace manager: it reads a `pixi.toml` manifest of constraints, +channels and platforms, solves them into a `pixi.lock`, and installs that lock into a prefix under +`.pixi/envs/`. Scrollcase uses it for exactly two things — solving and installing — plus one +auxiliary use, installing conda-pack itself. + +
+ +
+ +#### The three invocations + +Scrollcase's argument vectors are small, explicit, and constructed by pure functions so that they +can be asserted in tests without running anything. + +**Solve.** `lock` resolves a scroll's manifest into its committed lockfile without installing: + +```js +// src/build/pixi.mjs +export function pixiLockArguments(manifestPath) { + return ['lock', '--manifest-path', manifestPath]; +} +``` + +This is run by a human when dependencies change. The lock is committed and reviewed, and the diff is +the artefact a reviewer actually reads. + +**Install.** `build` materialises the environment from the committed lock, never re-resolving: + +```js +// src/build/pixi.mjs +export function pixiInstallArguments(manifestPath) { + return ['install', '--manifest-path', manifestPath, '--frozen']; +} +``` + +`--frozen` is the load-bearing flag: it installs exactly the locked packages without touching or +re-checking the lock, so what ships is byte for byte what was reviewed. Install-from-lock, +never-resolve. Whether the lock is still fresh with respect to its manifest is a separate concern +belonging to a project's CI, not to a build that is about to spend minutes and gigabytes. + +**Auxiliary install.** `init --install-toolchain` uses the project's own pixi to install conda-pack +into the project's own toolchain directory: + +```js +// src/build/toolchain.mjs +run(pixi, ['global', 'install', `conda-pack==${CONDA_PACK_VERSION}`], { + env: { PIXI_HOME: toolchainDir }, +}); +``` + +`PIXI_HOME` is what keeps the result inside the project instead of in the user's home directory. +Integrity here is conda-forge's to provide: conda-pack is resolved and verified by pixi exactly as +any other package is. + +
+ +
+ +#### The build workspace + +`build` never installs into the tracked scroll directory. It stages the manifest and the lock +side by side into a build-local workspace and installs there, so that pixi's `.pixi/envs/` tree +lands in the build directory and is removed afterwards: + +```js +// src/build/pixi.mjs — installAndPackPixiEnvironment +const workspace = join(buildDir, 'pixi-workspace'); +await copyFile(manifestPath, join(workspace, 'pixi.toml')); +await copyFile(lockPath, join(workspace, 'pixi.lock')); +run(pixi, pixiInstallArguments(join(workspace, 'pixi.toml'))); +const prefix = join(workspace, '.pixi', 'envs', 'default'); +``` + +The multi-gigabyte workspace and the intermediate packed tarball are both removed before the payload +is archived. + +
+ +
+ +#### Version pinning and discovery + +A scroll pins the pixi release it was solved against, and `build` refuses to proceed with a +different one: + +```js +// src/build/pixi.mjs — findPixi +if (found.version !== requiredVersion) fail(`Scroll requires pixi ${requiredVersion}, found ${found.version}.`); +``` + +The reason is direct: a different resolver version can select different packages, and a box that +silently differs from the one that was tested is exactly what the whole pipeline exists to prevent. + +Discovery follows a fixed precedence, highest first: + +| Rank | Source | Mechanism | +| --- | --- | --- | +| 1 | Explicit flag | `--pixi ` / `--conda-pack ` | +| 2 | Environment override | `SCROLLCASE_PIXI` / `SCROLLCASE_CONDA_PACK` | +| 3 | Project-local toolchain | `/bin/pixi`, if it exists | +| 4 | `PATH` | the bare name | + +The project-local toolchain is *looked up* rather than configured, which is what makes +`init --install-toolchain` sufficient on its own: nothing has to be added to `PATH` for the next +command to find what was just installed. This is also one of the four paths that break silently — +discovery behaves differently with and without an installed project toolchain, and a change to it +must be checked both ways. + +Reference: `toolCandidate()` in `src/build/pixi.mjs`; `tests/unit/toolchain.test.mjs`. + +Two probe functions sit beside the strict finders and answer a weaker question — *is there a pixi at +all, and at what version?* — which is what `doctor` and `init` need before they can report or offer +anything. `probePixi()` parses the version from `pixi --version`; `probeCondaPack()` only confirms +that conda-pack runs, because its own `--version` reports `0.0.0` and is unusable as a pin. + +
+ +
+ +#### Installing the toolchain, and verifying it + +`init` prepares a workspace without touching the network. When pixi or conda-pack is missing it +*offers* to install them and downloads nothing until an explicit yes. Without a terminal to answer +the question — CI, a pipe — nothing is installed at all: silence is not consent. + +The install sequence for pixi, in `installPixi()`: + +1. Select the release asset for this host from a frozen table keyed by `platform/arch`. A host + outside the table is not a failure — it means the toolchain has to be installed by hand. +2. Determine the expected digest: the value the project has already recorded, when it has one; + otherwise the checksum pixi publishes beside the archive, which is then returned so the caller + can pin it. +3. Download the archive into an OS temporary staging directory. +4. Hash the bytes on disk and compare. A mismatch is a hard failure and **nothing is installed**. +5. Unpack through the same guarded extractor the payload uses, so that even a known publisher's + archive cannot write outside the staging directory. +6. Move the binary into `/bin/`, falling back to a copy when staging and destination + are on different volumes, which is routine on Windows and on CI runners. + +| Host | Published asset | Format | +| --- | --- | --- | +| `darwin/arm64` | `pixi-aarch64-apple-darwin.tar.gz` | `tar.gz` | +| `darwin/x64` | `pixi-x86_64-apple-darwin.tar.gz` | `tar.gz` | +| `linux/x64` | `pixi-x86_64-unknown-linux-musl.tar.gz` | `tar.gz` | +| `linux/arm64` | `pixi-aarch64-unknown-linux-musl.tar.gz` | `tar.gz` | +| `win32/x64` | `pixi-x86_64-pc-windows-msvc.zip` | `zip` | +| `win32/arm64` | `pixi-aarch64-pc-windows-msvc.zip` | `zip` | + +The digest, once verified, is recorded in the project's config. Every later install — a teammate's +machine, a CI runner — is then checked against a value the project reviewed rather than against +whatever the server serves that day. + +**Rejected:** installing silently, and the `curl | sh` convention it would imitate. A packaging tool +whose entire product is verified artefacts cannot begin by running unverified bytes it fetched +without being asked. The consent requirement is also what makes `init` safe to re-run, which is a +property worth more than the keystroke it costs. + +Nothing is placed on `PATH`, nothing is installed system-wide, and deleting the toolchain directory +undoes the whole thing. + +Reference: `src/build/toolchain.mjs`; `tests/unit/toolchain.test.mjs`. + +
+ +
+ +### 4.3 conda-pack + + +conda-pack turns an installed prefix into a relocatable archive. Scrollcase invokes it with four +arguments and no others: + +
+ +```js +// src/build/pixi.mjs +export function condaPackArguments(prefix, outputPath) { + return ['-p', prefix, '-o', outputPath, '--format', 'tar.gz']; +} +``` + +The version is pinned to `0.9.2` in a single exported constant, beside the code that depends on its +output: + +```js +// src/build/toolchain.mjs +export const CONDA_PACK_VERSION = '0.9.2'; +``` + +conda-pack changes the bytes staged into a box, so letting a resolver pick a newer release would +make the same Scrollcase version produce different payloads over time. Changing the pin is a +reviewed Scrollcase release, not an incidental upgrade. + +The resulting tarball is extracted into the payload as `venv/`. conda-pack emits the prefix contents +at the tar root, so extracting into `venv/` yields the conda layout — `bin/`, `lib/`, `conda-meta/` +— directly beneath it. + +
+ +#### Why conda-unpack is deliberately never run + +conda-pack embeds a fixer, `conda-unpack`, intended to be run once at the destination to rewrite the +placeholder prefix into the real installation path. Scrollcase removes it instead of running it. + +The measurement that settled this was taken on a probe environment: **zero files carried the build +prefix before running the fixer, and thirty-six after.** Running it at build time would stamp the +build machine's absolute paths into dozens of files that then ship to users — leaking a developer's +directory layout while still being wrong at the user's install location. And running it at the +user's location is not available either, because that would make installation a step that executes +code from inside the box before anything has verified it. + +What Scrollcase does instead: + +- **Delete the service files that carry the build prefix**: `conda-meta/pixi_env_prefix`, + `conda-meta/pixi`, `bin/conda-unpack`, `Scripts/conda-unpack.exe`, + `Scripts/conda-unpack-script.py`. +- **Canonicalise `conda-meta/`** to the four fields that are properties of the package as published + — `name`, `version`, `build`, `license` — dropping per-file hashes that differ between installs + and absolute paths into the build machine's package cache. The rule is an allowlist rather than a + denylist of known-volatile fields, deliberately: a field a future pixi release starts writing + cannot reintroduce the drift, because it was never eligible to be copied. +- **Settle the symbolic links**, keeping only those the payload rule can prove safe. +- **Repair generated launchers**, rewriting console-script shebangs to resolve Python next to + themselves rather than at a build path. + +A conda-forge prefix imports and runs from any location with no activation environment and no +relocation fixer — proven cold on macOS and Windows, on CPU and GPU, before any of this repair +existed. The repair is therefore about removing leaked build paths, not about making the environment +work. + +**Rejected:** `pixi-pack`, which ships packages rather than a tree and needs a per-user install step +plus a bundled unpacker at the other end. The slow step is compression, and it is better paid once +by whoever builds than on every install by everyone. + +
+ +
+ +### 4.4 The three runtime dependencies + + +The published package depends on three libraries and nothing else: + +| Package | Version | Role | +| --- | --- | --- | +| `tar` | 7.5.22 | Reads the conda-pack tarball into the payload; validates and extracts `tar.gz` scroll assets and toolchain archives | +| `yauzl` | 3.4.0 | Reads and validates box ZIP archives | +| `yazl` | 3.3.1 | Writes the deterministic box ZIP archive | + +
+ +The rule behind that list is: reach for a Node built-in before adding a package. Node covers hashing +and signing (`node:crypto`), HTTP (`fetch`), streaming, filesystem work and subprocesses; what it +does not cover is ZIP, in either direction, and TAR. Those three libraries fill exactly that gap. + +Two consequences are worth stating explicitly. + +**Archive behaviour is a pinned property, not a host property.** Scrollcase never shells out to the +host's `tar`, `unzip`, or PowerShell expansion. A box therefore reads and writes identically on +macOS, Linux and Windows, and a build has exactly the external dependencies `doctor` reports — +pixi and conda-pack — rather than an invisible dependence on whatever archive tools happen to be +installed. + +**The archive backend is part of the format.** Each [target adapter](#target-adapter) carries an +`archive` descriptor naming the format, the writer, the reader, the reader used for scroll asset +tarballs, and Zip64 capability. It is declared in the contract rather than inferred, because a +consumer reading a box needs to know what produced it. + +::: info Development-only dependencies +`ajv` and `ajv-formats` (schema validation in tests), `json-schema-to-typescript` (type generation), +`typescript`, `vitest` and `@types/node` are development dependencies. None ships to a consumer, and +none is loaded at runtime. Runtime schema validation inside Scrollcase is done by a dependency-free +internal validator reading the shipped schemas — see section 6. +::: + +
+ +#### How each is used + +**`yazl` — writing.** Entries are added in a stable collected order, with a fixed timestamp +(`2000-01-01T00:00:00Z`), a mode derived from the target adapter rather than from the filesystem, +DOS timestamps forced so no local timezone leaks in, and deflate at a fixed compression level — +except for the paths a scroll declared as already compressed, which are stored instead. +Symbolic links are written as a small entry whose content is the target string and whose mode +carries the symbolic-link type bits — the same two facts every ZIP implementation reads a link back +from. Zip64 structures are emitted only where needed. + +**`yauzl` — reading.** Archives are opened with strict file names, entry-size validation, string +decoding and lazy entries. Every entry is classified and validated *before* anything is extracted: +encrypted entries are refused, special entries are refused, names are checked against path +traversal, duplicates and file/directory collisions are refused, link targets over 1024 bytes are +refused, and every link is judged against the payload link rule as received rather than as intended. +Link targets are read once during validation and reused during extraction, so a concurrently +rewritten archive cannot pass the check with one value and extract with another. + +**`tar` — reading only.** Used to extract the conda-pack output into the payload, and to validate +and extract `tar.gz` scroll assets and toolchain archives. TAR entries are validated before +extraction and the accepted types are `File`, `OldFile` and `Directory` only: links and special +entries in a TAR are refused outright. + +One subtlety belongs here because it is not obvious from the code's shape. When extracting the +conda-pack tarball, links are deliberately extracted in a **second pass**, after every regular entry +is on disk. The extractor refuses to create a link whose target passes through another link, and a +conda prefix trips that condition routinely — a package can ship `current -> ` and then +`pkgdata.inc -> current/pkgdata.inc`, which arrives in a plain Python environment that never asked +for that package and made the whole box unbuildable. Deferring link creation resolves it without +weakening anything: creating a link is not traversing one, and the targets are resolved and checked +immediately afterwards, with anything leaving the tree dropped. + +Reference: `src/build/archive.mjs`, `installAndPackPixiEnvironment()` in `src/build/pixi.mjs`; +`tests/unit/archive-security.test.mjs`. + +
+ +## 5. The contract + +`src/contract/` is the single source of truth for what a [box](#box) *is*: which targets exist, how +a target is named, what layout the payload has, which symbolic links it may carry, and the shape of +every document a build emits. Everything else in the repository — the builder, the signer, both +consumers, the command line — is an implementation that must satisfy it. + +It is the smallest part of the system and the one with the strictest rules, because it is the part +that other people's code depends on. + +
+ +### 5.1 Three artefacts that must never disagree + + +The contract ships three descriptions of the same rules, in three forms, each for a different kind +of consumer: + +| Artefact | Location | What it is | Who uses it | +| --- | --- | --- | --- | +| Reference implementation | `src/contract/*.mjs` | The rules as executable code | JavaScript callers, and the builder itself | +| JSON Schemas | `src/contract/schema/*.json`, published at `/schema/v2/*.json` | The machine-readable specification | Validators, editors, any language with a schema library | +| Golden fixtures | `src/contract/fixtures/*.json` | What "agreeing" means, concretely | Implementations in other languages, proving themselves | + +
+ +The relationship between them is a rule, not a convention: + +**A client written in another language does not import the code. It mirrors the rules and proves the +mirror against the fixtures.** + +```text + src/contract/ + the reference implementation + | + +---------------------+---------------------+ + | | | + schema/*.json fixtures/*.json imported directly by + machine-readable golden cases the builder and the + specification | Node consumer + | | + v +--------> Node consumer + generated types | + src/contract/types/ +--------> Python consumer + scrollcase_consumer +``` + +**Rejected:** publishing a shared runtime that every implementation links against. It would make one +language's package manager a dependency of every other language's client, and it would make the +format's rules unavailable to anyone unwilling to take that dependency. Fixtures cost more to +maintain and are the only mechanism that lets two independent implementations be checked against +each other rather than against each other's assumptions. + +Two helpers exist so that a caller never has to guess where those artefacts live inside an installed +package: + +```js +// src/contract/index.mjs +export function schemaUrl(name) { return new URL(`./schema/${name}.schema.json`, import.meta.url); } +export function fixtureUrl(name) { return new URL(`./fixtures/${name}.json`, import.meta.url); } +``` + +Both return a `URL` resolved against the module's own location, so they keep working under any +install layout, in a bundler, or from a global installation. + +
+ +#### The two entry points + +The contract is exposed twice, and the split is load-bearing: + +- **`scrollcase/contract`** — the complete surface, including payload decoding, which needs Node's + `crypto` for hashing. +- **`scrollcase/contract/browser`** — target identity, document naming, the constants, and the + structural envelope guard. No Node built-in is reachable from it, so it loads in a browser, in a + Worker, and in Node alike. + +A test walks the browser entry point's entire import graph and fails if any module in it reaches a +Node built-in (`tests/unit/package-surface.test.mjs`). The reason is practical: a client that only +needs to compute a [target ID](#target-id) or recognise a document `kind` should not have to bundle +a hashing implementation to do it. + +
+ +
+ +### 5.2 The target model — `targets.mjs` + + +
+ +#### A closed matrix + +A [target](#target) is `(platform, arch, accelerator)`, plus a CUDA ABI version when the accelerator +is CUDA. The supported combinations are enumerated, not derived: + +```js +// src/contract/targets.mjs +const TARGET_ACCELERATORS = { + macos: { aarch64: ['metal', 'cpu'] }, + linux: { x86_64: ['cpu', 'cuda'] }, + windows: { x86_64: ['cpu', 'cuda'] }, +}; +``` + +A target outside this matrix has no defined identifier and cannot be built, signed, or routed. That +is a deliberate refusal rather than a gap: every entry in the matrix implies a +[target adapter](#target-adapter), a conda subdir, a validated payload layout and a tested +relocation path. A combination nobody has proven those for would produce a box whose guarantees +nobody can state. + +**Rejected:** accepting an arbitrary triple and failing later, at build time. Failing at identity +time means the failure happens before a scroll is authored, before a lock is solved, and before +anything is downloaded. + +
+ +
+ +
+ +#### Target identity + +`boxTargetId()` reduces a target to the canonical slug that appears in archive names, object keys, +routes and directory names: + +```text +macos-aarch64-metal macos-aarch64-cpu +linux-x86_64-cpu linux-x86_64-cuda12.4 +windows-x86_64-cpu windows-x86_64-cuda12.4 +``` + +The rule is `--`, except CUDA, which appends the version with no +separator. Validation happens in a fixed order, and each step exists to make one class of ambiguity +impossible: + +1. **The value is an object.** A string or `null` is rejected with a type error rather than + producing `undefined-undefined-undefined`. +2. **The triple is in the matrix.** The accelerator is looked up through platform and arch, so an + accelerator valid on one platform is not silently accepted on another. +3. **CUDA carries a version**, matching `^[1-9][0-9]*\.[0-9]+$` — major and minor, no prefix, no + leading zero in the major component. +4. **Nothing else carries one.** `cudaVersion` on a CPU or Metal target is an error, not an ignored + field. + +Steps 3 and 4 together are what make the slug injective: exactly one target maps to +`linux-x86_64-cuda12.4`, and a target with an irrelevant CUDA version cannot masquerade as a +different one. + +
+ +
+ +#### Target adapters + +An adapter states what a target implies for the built payload. It is part of the format rather than +an implementation detail, because a consumer unpacking a box relies on that layout to find the +interpreter. + +| Field | `macos-aarch64` | `linux-x86_64` | `windows-x86_64` | +| --- | --- | --- | --- | +| `host.platform` / `host.arch` | `darwin` / `arm64` | `linux` / `x64` | `win32` / `x64` | +| `condaSubdir` | `osx-arm64` | `linux-64` | `win-64` | +| `python.payloadRoot` | `venv` | `venv` | `venv` | +| `python.entryPoint` | `venv/bin/python` | `venv/bin/python` | `venv/python.exe` | +| `python.scriptsDirectory` | `venv/bin` | `venv/bin` | `venv/Scripts` | +| `python.executableSuffix` | *(empty)* | *(empty)* | `.exe` | +| `python.launcherKind` | `posix-polyglot` | `posix-polyglot` | `uv-windows-pe` | +| `nativeLibraryInspection` | `otool -L`, `.dylib` `.so` | `ldd`, `.so` | `dumpbin /DEPENDENTS`, `.dll` `.pyd` | +| `validationEnvironments` | `cpu`, `metal` | `cpu`, `cuda` | `cpu`, `cuda` | +| `executionAffectingEnvironmentVariables` | Python controls + `DYLD_INSERT_LIBRARIES` | Python controls + `LD_PRELOAD` | Python controls | +| `selfTestPython` | `assert sys.platform == 'darwin'` | `assert sys.platform.startswith('linux')` | `assert sys.platform == 'win32'` | +| `archive` | shared backend descriptor | shared backend descriptor | shared backend descriptor | + +Every adapter is deeply frozen, and `boxTargetAdapters()` hands out a fresh array, so a caller +cannot mutate the format for everyone else in the process. + +Three details deserve their own note. + +**`validationEnvironments` are how an accelerator is forced.** Each is a small environment map +applied to validation runs — `CUDA_VISIBLE_DEVICES: ''` to force CPU, `CUDA_VISIBLE_DEVICES: '0'` to +force CUDA, `PYTORCH_ENABLE_MPS_FALLBACK: '0'` so a Metal run fails loudly instead of quietly +falling back to CPU. Without that last one, a [parity](#parity) check comparing Metal against CPU +could pass by comparing CPU against itself. + +**`executionAffectingEnvironmentVariables` drives diagnostics, not policy.** The shared Python set +is `PYTHONPATH`, `PYTHONHOME`, `PYTHONSTARTUP`, and `PYTHONBREAKPOINT`; the two POSIX loaders add +their platform-specific injection variable. Their presence is reported because it can change which +code runs. No adapter filters them. + +**`selfTestPython` is prepended to every self-test**, so the check begins by asserting it is running +on the platform the box claims. A box that somehow reached the wrong operating system fails at the +first line rather than at an import that happens to exist on both. + +**`launcherKind: 'uv-windows-pe'` is a frozen wire string.** It reads like a reference to a tool this +project does not use, and it is: the value is inert, and only names a launcher shape. It is recorded +here because it is the single most likely thing in the contract for a well-meaning cleanup to +"correct", and changing it would change the format for every client that already reads it. + +
+ +
+ +#### Host and layout assertions + +Two guards are exported beside the model, and both are refusals rather than conveniences. + +`assertNativeHost(adapter, host)` refuses to build or lock a target on a machine that is not the one +it ships for. There is no cross-compilation: the environment being packed contains native code +solved and installed for one platform, and a self-test run on the wrong host would prove nothing +about the box. + +`assertPythonEntryPoint(adapter, entryPoint)` refuses a scroll whose declared interpreter path +disagrees with the adapter's layout. The entry point is not free-form input — it is a fact about the +target — and accepting a disagreement would produce a signed release whose `pythonEntryPoint` +pointed at nothing. + +
+ +
+ +#### Two accessors the builder needs + +`condaSubdir(target)` maps a validated target to its conda platform subdir, which becomes the single +entry in the generated manifest's `platforms` list. + +`pixiAccelerator(scroll)` returns the accelerator descriptor a scroll selects, rejecting target +drift: `metal` and `cpu` need no extra conda knobs — the osx-arm64 build ships MPS support and CPU +is the default build — while `cuda` returns the version that pins a `cuda-version` and declares the +system requirement that makes the solver select GPU builds. + +Reference: `tests/unit/contract-targets.test.mjs`. + +
+ +
+ +### 5.3 The envelope — `document-shape.mjs` and `documents.mjs` + + +Every signed document Scrollcase emits travels in one envelope. The envelope is a container, not a +type: the type is inside it, discriminated by the payload's `kind`. + +
+ +```jsonc +{ + "schemaVersion": 2, + "payloadEncoding": "base64-json-utf8", + "payloadBase64": "eyJraW5kIjoic2Nyb2xsY2FzZS5ib3gucmVsZWFzZSIsIn0=", + "payloadSha256": "7d2c9a41…", + "signatures": [ + { "algorithm": "ed25519", "keyId": "scrollcase-9f2b7c1e04a83d56", "signatureBase64": "…" } + ] +} +``` + +
+ +#### Why the split across two modules + +`document-shape.mjs` holds everything that has no reason to depend on Node: the constants, the +namespacing functions, and the structural guard. `documents.mjs` re-exports all of it and adds the +one function that does need Node — payload decoding, which hashes bytes. That is what lets the +browser entry point offer document naming and envelope recognition without pulling in a hashing +implementation. + +
+ +
+ +#### The constants are the format + +```js +// src/contract/document-shape.mjs +export const BOX_SCHEMA_VERSION = 2; +export const PAYLOAD_ENCODING = 'base64-json-utf8'; +export const SIGNATURE_ALGORITHM = 'ed25519'; +export const DEFAULT_DOCUMENT_NAMESPACE = 'scrollcase.box'; +export const CHANNELS = Object.freeze(['nightly', 'beta', 'stable']); +``` + +Each is a single point of truth rather than a literal repeated across modules, and each is a value +the wire format commits to. Changing any of them is a `schemaVersion` change, not an edit. + +`CHANNELS` is ordered from least to most stable, and the ordering is meaningful: it is the +vocabulary a channel document's `channel` field is closed to, in both the code and the schema, and a +test asserts the two lists are identical. + +
+ +
+ +#### Namespacing + +```js +// src/contract/document-shape.mjs +const DOCUMENT_TYPES = Object.freeze(['release', 'channel', 'revocations']); +const NAMESPACE_PATTERN = /^[a-z0-9]+(?:[.-][a-z0-9]+)*$/; +``` + +`documentKinds(namespace)` returns the three `kind` discriminators under a namespace, frozen: +`.release`, `.channel`, `.revocations`. `parseDocumentKind(kind)` +inverts it, splitting at the **last** dot — so a namespace may itself contain dots — and returning +`null` for anything that is not a document kind at all rather than throwing, because asking "is this +one of ours?" is a legitimate question with a legitimate negative answer. + +An invalid namespace is a `TypeError`, not a sanitised value. Emitting a document under a namespace +that is nearly what was asked for would produce documents a project's own clients silently ignore. + +The reason the namespace is a parameter at all: a project that already publishes boxes owns its +namespace, and its installed clients recognise documents by it. A tool that renamed those documents +underneath a publisher would break every client in the field. So `scrollcase.box` is only the +default for a project with no published history to preserve. + +
+ +
+ +#### Recognising an envelope, and decoding it + +`isSignedBoxDocument(value)` is a **shape check**, and its documentation is emphatic about what it +does not mean: it says the document is worth attempting to verify, never that its signature is good. +It checks the version and encoding constants, the presence and type of the payload fields, and that +the signature array is non-empty and every entry names the right algorithm with a string key ID and +signature. + +`decodeDocumentPayload(document)` does three things in a fixed order, and the order is the point: + +1. **Refuse `schemaVersion: 1` explicitly**, with the remedy in the message — + `Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.` A v1 document is not + reinterpreted, and it is not rejected as merely malformed either; it is named. +2. **Refuse anything that fails the shape check.** +3. **Hash the decoded bytes and compare against `payloadSha256`** *before* parsing them as JSON. A + truncated or edited document is caught before its contents are read at all. + +Only then is the payload parsed and returned — and the return is documented as *still unverified*. +Decoding is not verification: no signature has been checked at this point. Verification is section +7's subject, and it is a separate call that the consumers make before anything acts on a payload. + +
+ +
+ +#### Why the payload is base64, not canonical JSON + +The payload travels as exact base64-encoded UTF-8 JSON. Verifying a signature therefore means +hashing bytes that were transmitted verbatim. + +**Rejected:** canonical JSON. Canonicalisation requires every implementation, in every language, to +agree on key ordering, number formatting, Unicode escaping and whitespace — historically the richest +source of cross-language signature bugs, and a class of bug that surfaces as an unverifiable +document at a user's machine rather than as a test failure. Transmitting the exact bytes removes the +problem instead of solving it once per language. The cost is a slightly larger document and a +payload that is not human-readable without a decode step, which is a fair price for a signature that +means the same thing everywhere. + +Reference: `tests/unit/contract-schema.test.mjs`, `tests/unit/v2-migration.test.mjs`. + +
+ +
+ +### 5.4 The link rule — `links.mjs` + + +A conda prefix is dense with symbolic links, and what a payload does with them is a security +question dressed as a size question. This module is the whole answer, and it consults nothing but +its arguments. + +
+ +
+ +#### The five rules + +1. A target is **relative** — never absolute, never a drive letter, never containing a backslash or + a NUL byte. +2. Resolved against the link's own directory, it **stays inside the payload**: `..` is allowed + exactly as far as it cannot escape. +3. A link resolves to a **regular file**, never to a directory. +4. **No entry may have a link as a path prefix**, so nothing is ever written *through* a link. +5. Chains **terminate**, within a small bound, without a cycle. + +Rule 3 is what keeps the rest small. A directory link is legitimate in a conda prefix — +`lib/python3.1` → `python3.11` is real, and worth about one duplicated standard library — but it is +also the only way an entry can be written through a link and land somewhere its own name does not +describe. Refusing directory links removes an entire class of escape, and leaves rule 4 as a second +lock on a door rule 3 already welded shut. + +**Rejected:** carrying directory links, and with them the last few dozen megabytes. Keeping the rule +small enough to state in five lines and prove in two languages was worth more. + +**Rejected:** a `schemaVersion` bump when links were first allowed into payloads. The signed +document did not change; only what the archive may contain grew. A consumer predating the rule +rejects a link entry with a clear error rather than misreading it, which is the only thing a version +bump would have bought. + +
+ +
+ +#### The implementation + +```js +// src/contract/links.mjs +export const MAX_PAYLOAD_LINK_DEPTH = 8; +``` + +Real prefixes use one or two hops. A longer chain has no legitimate source and is the cheap way to +make resolution expensive, so the bound is part of the rule rather than an implementation limit. + +| Function | Question it answers | +| --- | --- | +| `isRelativeLinkTarget(target)` | Is this target *shaped* like one a payload may carry, before resolving anything? | +| `resolvePayloadLinkTarget(linkPath, target)` | Where does this link point, relative to the payload root — or `null` if it may not be carried? | +| `findEntryThroughLink(entries)` | Does any entry in this set have a link as a path prefix? | +| `findUnresolvableLink(entries)` | Does every link chain in this set end at a regular file present in the same set? | +| `targetCarriesLinks(platform)` | May this target's payload contain links at all? | + +`resolvePayloadLinkTarget` walks the target segment by segment, popping for `..` and refusing on +underflow. Checking per segment rather than on the final result is what catches a target that +climbs out of the payload and back in — a path that looks contained once resolved but escaped on the +way. It also refuses a link that resolves to itself. + +`findUnresolvableLink` follows each chain against the entry set, and refuses it at a directory, at +nothing, at itself, or past the depth bound. One subtlety: a directory can exist *implicitly*, +through its children, without an entry of its own, so the set of directories is computed from every +entry's path prefixes before any chain is followed. Checking only for an explicit directory entry +would let a link to an implicit directory through. + +`targetCarriesLinks(platform)` returns false for Windows. Creating a symbolic link there needs +Developer Mode or elevation, so a Windows box materialises every link rather than producing an +archive that fails to extract on an ordinary machine. + +
+ +
+ +#### Applied three times + +The same rule runs at three points, against three different sources of truth: + +| Where | Against what | Why again | +| --- | --- | --- | +| Builder, `src/build/pixi.mjs` | The real filesystem, with `realpath` | It can, and a materialised link is cheaper to produce than to reject | +| Archive writer, `src/build/archive.mjs` | The entry set about to be written | Shipping a box a consumer must reject is worse than not building one | +| Every consumer | The archive **as received** | No consumer trusts the builder; a box assembled by hand gets no benefit of the doubt | + +That nothing here touches the filesystem is what makes three applications possible. A rule that +consulted the disk would give three different answers on three machines; a purely lexical rule gives +the same answer everywhere, which is what lets the builder, the Node consumer and the Python +consumer apply one rule rather than three approximations of it. + +Reference: `tests/unit/contract-links.test.mjs`, `tests/unit/archive-security.test.mjs`. + +
+ +
+ +### 5.5 The payload digest — `payload-digest.mjs` + +A release commits to `archive.sha256`, which proves every payload byte — for as long as the archive +exists. An application that installs a box once and runs it for months has deleted it, and +`installedSizeBytes` is a free-space figure rather than an identity. So a box also carries a list. + +
+ +
+ +#### A list, not a snapshot + +The payload holds one file, `payload-digest.v1`, with one record per entry: its path, whether it is a +file or a [link](#symbolic-link), and the SHA-256 of its content. The release signs the SHA-256 of +that list, so the signed document grows by one field rather than by the megabytes a per-file table +would add to a prefix holding twenty thousand files. + +That indirection is also what makes verification a closed question. A verifier walks the **list**, +never the directory, so anything the list does not name is never visited: the `__pycache__` Python +writes on first import, the model cache a caller fills after extraction, a file an application writes +into its own working directory. Those are invisible by construction, not by an exclusion list that +would have to be guessed at and kept in step. + +**Rejected:** hashing a walk of the installed tree into a single root value, with no list at all. It +reads as the same guarantee for a smaller format, but the directory is then the input, so every one +of those legitimate extra files makes an honest box fail. + +
+ +
+ +#### What a record leaves out + +| Omitted | Why | +| --- | --- | +| Mode | `archiveFileMode` synthesises `0o755`/`0o644` from the target and the path instead of preserving what the packed prefix carried, so observed modes could never match an extracted tree and canonical ones would hash what the release already states. Windows extraction skips `chmod` entirely | +| Modification time | The payload is stamped with one fixed instant before archiving, but no extractor restores it; installed files carry the wall-clock of their install | +| Directories | Neither the entry collector nor the archive writer represents one, so an empty directory is already lost between build and install | + +A link is hashed by its target string rather than opened. Following it would record the target's +bytes a second time under the link's name, and would make a link indistinguishable from a copy — +which is the distinction the record's kind byte exists to keep. + +Records are sorted by their own bytes rather than by their paths compared as strings. The two are +the same ordering, because a path cannot contain NUL and NUL sorts below every byte a path can hold. +Only one of them is unambiguous across languages: comparing strings asks each implementation to agree +on what a string is, and this repository's own two already disagree above the Basic Multilingual +Plane, where JavaScript orders by UTF-16 code unit and Python by code point. + +Reference: `tests/unit/contract-payload-digest.test.mjs`, and the shared vectors at +`src/contract/fixtures/payload-digest-contract.json`. + +
+ +
+ +### 5.6 The eight schemas + + +The schemas are the machine-readable specification, written against JSON Schema draft 2020-12. + +
+ +| Schema | Title | Describes | +| --- | --- | --- | +| `target.schema.json` | Box target | The `(platform, arch, accelerator)` triple and its CUDA rule | +| `execution.schema.json` | Box execution | The optional, shell-free application entry point | +| `scroll.schema.json` | Box scroll | The declarative build input | +| `box-manifest.schema.json` | Box manifest (`box.json`) | The manifest packed inside the archive | +| `release-manifest.schema.json` | Box release manifest | The immutable description of one built box | +| `channel-manifest.schema.json` | Box channel manifest | The mutable pointer from a channel to releases | +| `revocations-manifest.schema.json` | Box revocations manifest | The signed withdrawal list | +| `signed-document.schema.json` | Signed box document | The envelope all three document types travel in | + +
+ +#### How they reference each other + +```text + signed-document ....> release-manifest + ....> channel-manifest + ....> revocations-manifest + + target -----> scroll, release-manifest, box-manifest, + channel-manifest, revocations-manifest + + execution --> scroll, release-manifest, box-manifest + + release-manifest --> box-manifest ($defs: provenance, sha256) + + ----> a real $ref + ....> not a $ref: an opaque base64 payload, resolved after decoding +``` + +The reference from `box-manifest` into `release-manifest`'s `$defs` is deliberate: `box.json` +carries the *same* provenance block as the release it belongs to, and defining it once is what makes +that literally true rather than approximately true. + +The envelope's relationship to the three payload types is dotted because it is not a `$ref`: the +envelope describes an opaque base64 string, and which payload schema applies is decided by the +`kind` inside it after decoding. That indirection is what lets a verifier check integrity before it +knows what it is holding. + +
+ +
+ +#### Conventions shared by all of them + +**`additionalProperties: false` everywhere, with one deliberate exception.** An unknown field is a +misunderstanding, and accepting it silently would let a typo'd key look like a working +configuration. The exception is `compatibility`, which is open on purpose: a project may declare its +own constraints alongside the defined ones, and the builder copies them through verbatim without +ever interpreting them. The schema states the counterpart obligation — *a consumer that cannot +evaluate a constraint must refuse the box rather than assume it passes.* + +**Three reused patterns.** + +| `$def` | Pattern | Used for | +| --- | --- | --- | +| `identifier` | `^[a-z0-9]+(?:[-.][a-z0-9]+)*$` | `boxId`, `modelId`, `runtimeId` | +| `sha256` | `^[a-f0-9]{64}$` | Every digest, lowercase hex only | +| `payloadPath` | a negative-lookahead chain | Any path inside the payload | + +The `payloadPath` pattern is worth reading in full, because it encodes the path-safety rule at the +schema level rather than leaving it to code: + +```text +^(?!/)(?![A-Za-z]:)(?!.*\\)(?!.*(?:^|/)\.\.(?:/|$))(?!.*//).+$ +``` + +Not absolute, not a drive letter, no backslash anywhere, no `..` segment, no empty segment, +non-empty. A path that fails this never reaches the code that would have to reject it. + +**`schemaVersion` is `const: 2` in every document schema.** Not a minimum, not a range: a v1 +document fails schema validation with the same finality as the code rejects it. + +**`weights` and `assets` are paired by `dependentRequired`** in both the release and the box +manifest. Declaring one without the other is a contradiction — on-demand weights with no asset +descriptors, or asset descriptors on a box claiming to be self-contained — and the schema refuses +both directions. + +
+ +
+ +#### The scroll + +The largest schema, and the only one describing *input* rather than output. Nine fields are +required: `schemaVersion`, `boxId`, `modelId`, `runtimeId`, `version`, `sourceRevision`, +`pythonVersion`, `pixiVersion` and `selfTest`. A tenth, `target`, is required of every scroll a +build reads but not by the schema, because the base of a split scroll legitimately has none; the +reader enforces it, so a base file still validates in an editor. + +That list is shorter than the format needs, because a scroll is a file someone writes by hand and +several of its fields were only ever restatements of others. `pythonEntryPoint` is the clearest +case: the target adapter admits exactly one value and the reader rejected any other, so requiring it +obliged the author to type the single string that was already implied. Those fields are now derived +when the scroll is read, in one place, so every consumer of a scroll still sees a complete object: + +| Field | Derived value | +| --- | --- | +| `scrollVersion` | `1.0.0` | +| `compatibility` | `{}` — declaring no constraint is an answer, and inventing one would be a claim the project never made | +| `pythonEntryPoint` | The target adapter's interpreter path; still checked against the target when declared | +| `modelCacheSubdir` | `model-cache/` | +| `assets` | `[]` | +| `selfTest.files` | `[]` | + +The optional fields are where a scroll expresses intent: + +| Field | Purpose | +| --- | --- | +| `$schema` | Associates the file with the published schema, for editor validation and hover help | +| `extends` | `../scroll.json`, marking this file as one target's half of a split scroll | +| `scrollId` | Provenance identity; derived deterministically as `-` when omitted | +| `condaDependencyLicenseAudit` | Path to the reviewed licence inventory the build must still match | +| `assetBaseUrl` | Base URL the built archive and its objects are published under | +| `assetArchives` | Downloaded archives to expand into the payload, with `stripComponents` and `removeAfterExtract` | +| `localFiles` | Files copied from the project's own repository, optionally pinned to a declared hash | +| `prunePaths` | Payload paths deleted before packing | +| `weights` | `embed` (default) or `on-demand` | +| `execution` | The application entry point | +| `parity` | The cross-accelerator numerical gate | + +Three of these carry a rule worth stating explicitly. `assets` may be empty, but every entry is +size- and hash-checked before use, so a moved or replaced upstream file fails the build instead of +silently changing the box. `localFiles` may carry the same pin applied inward, and here it is +**optional**: an asset arrives over a network nobody controls, whereas a local file comes out of the +project's own checkout, where git already records what changed, and what ships is hashed into the +signed release either way. Making the pin mandatory did not buy a guarantee so much as a chore — +every edit to a generated entry point failed the next build until its digest was recomputed by hand +— so a project now pins what it wants frozen, such as a licence notice or a reviewed shim, and +leaves the pin off what it is still writing. And `selfTest.files` lists what must still exist +*after* pruning, which is what stops an over-aggressive `prunePaths` from shipping a broken box. + +`selfTest` carries one more choice: the extra Python it runs after the imports may be given inline +as `pythonCode` or, mutually exclusively, as `pythonFile` — a path to a file in the project, read at +build time and executed from the payload root. A self-test that is worth writing outgrows a JSON +string almost immediately, and in a file it keeps its syntax highlighting, its linter and a readable +diff. + +`parity` requires a script, at least two accelerators, and at least one tolerance. The first +accelerator listed is the reference the others are compared against — conventionally `cpu`, being +the one available everywhere. The tolerances are `absolute`, `relative` and `minimumCosine`, and the +schema explains why more than one exists: absolute guards entries near zero where relative error is +meaningless, and cosine similarity catches a result that drifted in direction rather than magnitude. + +
+ +
+ +#### Execution + +A closed union of exactly two shapes, both requiring `defaultArgs`: + +```jsonc +{ "kind": "python-script", "script": "entrypoint.py", "defaultArgs": [] } +{ "kind": "python-module", "module": "example_model.main", "defaultArgs": ["--serve"] } +``` + +A script is a `payloadPath` — a regular file inside the box. A module is a strict Python dotted +name, `^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*$`, which admits no command-line syntax +and no shell fragment. `defaultArgs` are placed before caller-supplied arguments and every item is +passed directly, without a shell. + +Absence of the whole block means the box is **intentionally library-only** — a positive statement, +not an omission. + +**Rejected:** a shell command. A shell changes what an argument means depending on its contents, and +creates an injection surface at exactly the point where caller-supplied arguments meet signed +metadata. A closed union of two shapes cannot be talked into running something else. + +
+ +
+ +#### The release manifest + +Thirteen required fields describing one built box, of which three groups matter most. + +**`archive`** is `{ format: "zip", url, sha256, sizeBytes }`, all four required. Size *and* hash, +not hash alone: the size is checkable before a byte is read, so a consumer can refuse an +implausible download before spending on it. + +**`selfTest`** is `{ pythonImports, timeoutSeconds }` — the import subset a consumer can repeat +after extraction. The schema states plainly that the builder also ran the scroll's Python-code and +file assertions and that those are builder-only, rather than implying the signed check covers them. + +**`provenance`** requires all nine of its fields: `scrollId`, `scrollVersion`, `builderRevision` +(exactly 40 hex characters), `sourceTreeDirty`, `sourceRevision`, `pythonVersion`, `pixiVersion`, +`dependencyLockSha256` and `builtAt`. `sourceTreeDirty` is a required boolean rather than an +optional flag, so "clean" is always an assertion somebody made and never the absence of one. + +`installedSizeBytes` is optional and is a free-space estimate, not an integrity identity. At build +it is the sum of the logical sizes of every payload file and link, including `payload-digest.v1`; +preparation compares an extracted tree with it, while an attached receipt reports a fresh +measurement of the directory and deliberately does not compare that measurement with the signed +figure. A caller still needs headroom for the archive, temporary copies, allocation units and +filesystem metadata. + +
+ +
+ +#### The box manifest + +Deliberately a near-copy of the release, minus what only makes sense to a distributor — no `kind`, +no `archive`, no `compatibility` — and sharing the release's `provenance` and `sha256` definitions +by reference. + +One asymmetry is worth knowing, because it looks like an oversight and is not: `box.json` constrains +its identifiers only as non-empty strings, while the scroll and the release constrain them to the +`identifier` pattern. Nothing is lost, because verification compares the two documents field by +field — an identifier that passed the release's pattern is the one that must appear in `box.json`. +The narrower check happens where the value originates. + +
+ +
+ +#### The channel and revocations manifests + +A **channel** is small and mutable: `channel`, `boxId`, `target`, `updatedAt`, `cohortSalt` and a +non-empty `releases` array of `{ version, releaseManifestUrl, rolloutPercentage }`, evaluated in +order, a client taking the first entry whose cohort it falls into. It is signed independently from +releases, so promoting a build never requires re-signing it. `cohortSalt` makes cohort assignment +stable per client and unpredictable across channels, so a staged rollout cannot be gamed by +reinstalling. + +A **revocations** document lists `{ boxId, version, target?, reason, revokedAt }`, with `target` +omitted when every target of a version is withdrawn. Its array **may be empty**, and the schema says +why: an empty signed list is a positive statement that nothing is revoked, which a client can +distinguish from a missing or withheld document. That distinction is the difference between "nothing +is revoked" and "somebody prevented you from finding out". + +Scrollcase defines and can verify both. Publishing them, and acting on them, belong to whoever +distributes boxes — this is the boundary of section 3, expressed as a format. + +
+ +
+ +#### Publication is checked, not assumed + +Each schema's `$id` is an absolute `https://scrollcase.dev/schema/v2/` URL, and byte-identical +copies are published under `docs/public/schema/v2/`. Two tests enforce it: one compares every +published file against its source byte for byte, and one asserts that every `$id` and every absolute +`$ref` resolves to a schema that is actually published. A schema referencing a sibling that never +shipped would validate locally and fail for everyone else. + +Generation never depends on that host being reachable — the type generator resolves +`scrollcase.dev` URLs back to the files in the tree. + +Reference: `tests/unit/contract-schema.test.mjs`, `tests/unit/docs-contract.test.mjs`. + +
+ +
+ +### 5.7 The golden fixtures + + +Fixtures are the contract's answer to a question specifications cannot answer on their own: *does +your implementation agree with mine?* + +
+ +
+ +#### `target-id-contract.json` + +Six valid cases and seven invalid ones, each named. The valid cases pair a target with the exact +slug it must produce; the invalid ones are the combinations that must be rejected: + +- CUDA without a version, and CPU or Metal **with** one +- a CUDA version carrying a prefix (`cuda12.4`) or missing its minor component (`12`) +- targets outside the matrix — macOS on Intel, Linux on arm64 + +This is what other-language implementations validate their mirrors against, and it is why the +seven invalid cases matter more than the six valid ones. Agreeing about what works is easy; +agreeing about what must fail is where independent implementations drift. + +
+ +
+ +#### `consumer-conformance.json` + +Sixty-seven language-neutral semantic cases shared by the Node, Python and Rust consumers, plus +twenty-eight error patterns each case's failure message must match. The cases cover valid preparation +under both signing paths, a project's own `compatibility` constraint carried rather than refused, +every tampering scenario, a v1 document refused by name, unsafe archive +entries, extraction collisions, per-platform entry points, attachment across process restarts, +installed-payload verification, argument ordering, stream forwarding, exit codes and signals, +temporary-directory cleanup, on-demand asset failures, signed environment agreement, precedence, +masking, explicit value reveal, and report parity across preparation, attachment, payload +verification, and execution. + +Error *patterns* rather than exact strings, deliberately: two languages should agree on what went +wrong without being forced to phrase it identically. Section 8 covers the cases in detail — they +describe consumer behaviour, and are listed here because the fixture is part of the contract rather +than of either consumer. + +
+ +
+ +#### `fixtures/examples/` + +Seven complete, valid documents — a scroll, a scroll on the pixi substrate, a box manifest, a +release manifest, a channel manifest, a signed release, and the public key that signed it. They +serve as schema conformance evidence and as a starting point for an implementer. + +The signed example earns its keep twice over: one test decodes it and validates the payload against +the release schema, and another **verifies its actual ed25519 signature against the shipped public +key**. A fixture that merely parsed would prove nothing about signing; this one fails if the +signature scheme, the payload encoding or the key format ever changes underneath it. + +
+ +
+ +### 5.8 Generated types + + +Two surfaces are generated, both by `npm run types`, and neither is ever hand-edited. + +
+ +**Contract types** — `scripts/generate-contract-types.mjs` compiles the eight schemas into +`src/contract/types/index.d.ts`: + +| Schema | Generated type | +| --- | --- | +| `target.schema.json` | `BoxTarget` | +| `execution.schema.json` | `BoxExecution` | +| `scroll.schema.json` | `BoxScroll` | +| `box-manifest.schema.json` | `BoxManifest` | +| `release-manifest.schema.json` | `BoxReleaseManifest` | +| `channel-manifest.schema.json` | `BoxChannelManifest` | +| `revocations-manifest.schema.json` | `BoxRevocationsManifest` | +| `signed-document.schema.json` | `SignedBoxDocument` | + +The type names are declared explicitly rather than derived from file names, so that renaming a +schema file cannot silently rename a type somebody imports. + +**Runtime declarations** — `scripts/generate-runtime-types.mjs` runs TypeScript over the JSDoc +already reviewed beside each function, emitting `.d.mts` files for the complete dependency closure +of the five public entry points. It compiles a declaration-free staging copy first: otherwise +TypeScript would see the previously committed declarations beside the JavaScript and treat them as +inputs, making regeneration depend on the output it is meant to replace. + +Both outputs are **committed**, which is why the package needs no build step and why `npm publish` +ships only reviewed files. Both generators have a `--check` mode, run by the test suite: a schema +change or an API change that was not accompanied by a regeneration fails the suite instead of +shipping stale types. + +The principle is the same one behind the licence inventory. Types are a *projection* of the schemas, +never a second definition of the format, exactly as the inventory is a projection of the lock rather +than a document maintained beside it. Anything maintained in two places is eventually maintained in +one. + +::: danger Do not hand-edit +`src/contract/types/index.d.ts` and every `src/**/*.d.mts` are generated. Regenerate with +`npm run types`. An edit to either survives exactly until the next regeneration, and fails the suite +in the meantime. +::: + +Reference: `tests/unit/package-surface.test.mjs`. + +
+ +### 5.9 What the contract deliberately does not contain + + +The absences are as designed as the contents: + +- **No network access, and no filesystem access** beyond resolving `URL`s for schemas and fixtures. + Every rule here is a pure function of its arguments. +- **No policy.** The contract says what a valid channel document looks like; it does not say which + channel to follow. It says what a revocation is; it does not act on one. It defines + `compatibility` as an open object; it never evaluates a constraint. +- **No cryptographic dependency.** Hashing and signing use Node's own `crypto`, and the browser + entry point reaches neither. +- **No default that encodes anyone's deployment.** The one default it does carry — the + `scrollcase.box` namespace — exists precisely so a project can replace it. + +
+ +Everything the contract omits is something a consumer supplies. That is what makes the same format +usable by a desktop application, an internal artefact service and an air-gapped installer without +any of them inheriting the others' assumptions. + +## 6. The build pipeline + +`src/build/` is where a [scroll](#scroll) becomes a [box](#box). It is the largest layer in the +repository, and almost all of it exists to make one command — `build` — produce an artefact whose +properties can be stated without qualification. + +This section walks the pipeline in execution order first, then every module that serves it. + +
+ +### 6.1 The ordered stages of `build` + + +`buildBox()` in `src/build/box.mjs` is the whole pipeline, written as one linear function. That is +deliberate: the order *is* the design, and a reader who wants to know what happens before the +interpreter first runs should be able to see it without following a call graph. + +
+ +```text + validate assemble prove publish + +----------+ +------------+ +------------+ +-------------+ + | 1 2 3 | ----> | 4 5 6 7 | ---> | 8 9 10 11 | --> | 12 13 14 15 | + +----------+ +------------+ +------------+ +-------------+ + nothing has the payload nothing that the archive is + been mutated is built and failed a check sealed, signed + yet pruned can ship and staged +``` + +| # | Stage | Module | State and files touched | +| --- | --- | --- | --- | +| 1 | Read and validate the scroll | `scroll.mjs` | Reads `scrolls///scroll.json`; resolves the adapter | +| 2 | Validate the build options | `box.mjs` | Channel in `CHANNELS`; weights mode; on-demand refuses `assetArchives` | +| 3 | Refuse an unusable host, toolchain or tree | `targets.mjs`, `pixi.mjs`, `scroll.mjs` | `assertNativeHost`; pinned pixi and conda-pack located; `pixi.lock` present and hashed; git revision read, dirty tree refused | +| 4 | Prepare the build tree | `box.mjs` | Removes and recreates `//payload/`; clears the target's object directory under `dist/` | +| 5 | Solve, pack and relocate | `pixi.mjs`, `launchers.mjs` | Installs into a build-local pixi workspace, packs it, extracts into `payload/venv/`, repairs it, deletes the workspace and tarball | +| 6 | Stage assets | `assets.mjs` | Downloads verified assets, copies verified local files, expands asset archives — the downloads and the archives only when weights are embedded, the local files always | +| 7 | Prune | `box.mjs` | Deletes each `prunePaths` entry from the payload | +| 8 | Licence inventory | `licenses.mjs` | When the scroll declares a reviewed audit: recomputes from the lock, compares against it, writes `payload/THIRD_PARTY_NOTICES/conda-distributions.json` | +| 9 | Post-prune integrity | `box.mjs`, `execution.mjs` | Every `selfTest.files` entry still exists, except an asset deferred by `on-demand`; execution names a real script or discoverable module | +| 10 | Describe | `box.mjs` | Writes `payload/box.json`, so the self-test runs against the payload the box will ship — an application that reads its own manifest to find its files can then be exercised by it | +| 11 | Self-test | `box.mjs` | Runs the adapter's own entry point — `payload/venv/bin/python -c …`, or `venv/python.exe` on Windows — with the target's validation environment | +| 12 | Parity | `parity.mjs` | Runs the declared check once per accelerator and enforces the tolerances | +| 13 | Commit, normalise, measure | `box.mjs`, `filesystem.mjs` | Writes `payload-digest.v1` without listing the list itself, records its hash for the release, stamps every entry with the fixed mtime, and sums the installed size | +| 14 | Archive | `archive.mjs` | Writes `/.zip` deterministically; hashes and measures it | +| 15 | Sign | `sign/index.mjs` | Signs the release, hashes the signed document, signs a channel pointer at 100% | +| 16 | Publish-ready move | `assets.mjs` | Moves archive and release into `dist/boxes////`, writes `dist/channels///.json` | + +Several properties of that order are load-bearing. + +**Everything that can refuse the build does so before anything expensive.** Stages 1 to 3 cost +milliseconds and can each end the build; stage 5 costs minutes and gigabytes. A wrong host, a +missing lock, a pixi at the wrong version and a dirty tree are all caught before a single package is +downloaded. + +**The build tree is destroyed before it is used.** Leftovers from a previous build would otherwise +end up inside the archive, which is both a correctness bug and a determinism bug: + +```js +// src/build/box.mjs +await rm(buildDir, { recursive: true, force: true }); +await rm(objectDir, { recursive: true, force: true }); +await mkdir(payloadDir, { recursive: true }); +``` + +**Pruning happens before every check that could catch an over-prune.** Stage 9 asks whether the +files the box needs at run time are still present, and stage 11 asks whether it can still import +what it claims. Neither would mean anything if pruning came after them. + +**The self-test runs before the payload can become an archive.** This is the step that earns the box +its name: the modules are imported by the payload's *own* interpreter, in the payload directory, +under the target's validation environment. +The scroll's declared environment is present too; target validation is layered last so it cannot be +disabled by a declaration. + +**Parity runs after the self-test, never before.** There is no point comparing accelerators in a box +that cannot import its dependencies in the first place. + +**Nothing is written twice.** The archive and the release document are *moved* into the directory a +publisher uploads, not copied, so the only copy that exists is the one that gets published and there +is no second name for the same bytes. + +
+ +#### The distribution tree + +`build` writes into two places under the workspace's `dist/` directory, and the split is deliberate: + +```text +.scrollcase/dist/ +├── boxes/example-box/1.0.0/macos-aarch64-metal/ +│ ├── .zip +│ └── .release.json +└── channels/example-box/beta/ + └── macos-aarch64-metal.json +``` + +`boxes/` is the tree that goes under the asset base URL verbatim — the same prefix the signed +documents write into their own URLs, so uploading it is a copy rather than a mapping. `channels/` is +separate because a channel is not part of any one version: it is a pointer that moves to the next +one, and filing it under `1.0.0` would leave a stale copy claiming to be current the moment `1.0.1` +ships. + +Both objects are [content-addressed](#content-addressed) by their own hashes, which makes the whole +chain verifiable end to end: + +```text +channel document → release document (by its SHA-256) → archive (by its SHA-256) +``` + +Content addressing also makes publishing idempotent, and makes it impossible to replace an object +with different bytes under the same URL — the URL contains the hash of the bytes it serves. + +
+ +
+ +#### The channel a build emits + +A freshly built channel document goes out at `rolloutPercentage: 100`. A staged rollout is arranged +by editing that document, not by the builder: choosing who receives a release is distribution +policy, and section 3 is why it lives outside this tool. + +Its `cohortSalt` is derived rather than random: + +```js +// src/build/box.mjs +cohortSalt: sha256Hex(Buffer.from(`${scroll.boxId}:${scroll.version}`)).slice(0, 32), +``` + +A random salt would reshuffle which users receive a release every time the same commit was rebuilt, +which is precisely the class of per-run variation [determinism](#determinism) forbids. Deriving it +from box and version keeps cohort assignment stable across rebuilds while still differing between +releases. + +
+ +
+ +### 6.2 Reading a scroll — `scroll.mjs` + + +A scroll is the only input a build accepts, so it is validated completely before anything is +installed. + +
+ +`readExactScroll()` performs seven checks in order: + +1. **The reference is well formed**: exactly `/`, screened by `safeRelativePath`. +2. **The document validates** against the scroll, target and execution schemas, using the internal + validator described in 6.4 — after a split scroll has been joined with its base, so what is + validated is what the build will read. +3. **A target is declared.** Required of the joined scroll rather than by the schema, so that the + base of a split scroll still validates on its own. +4. **Weights and archives are compatible**: `on-demand` with `assetArchives` is refused, because + those archives are expanded at build time and cannot be deferred. +5. **Every declared path is safe.** One sweep screens `modelCacheSubdir`, every asset path, both + ends of every asset archive, both ends of every local file, every prune path, every self-test + file, the self-test Python file, the execution script, the parity script and the licence audit + path. +6. **The directory names agree with the declarations.** The parent directory must equal `boxId` and + the child must equal the canonical [target ID](#target-id). +7. **The entry point agrees with the adapter**, via `assertPythonEntryPoint`. + +Check 6 deserves its reasoning. The layout is `scrolls///`, and the directory names +are *checked context*, not identity: the scroll declares both facts, and the filesystem is required +to agree. That makes every target variant of one box visible together without making a directory +name the source of the box's identity. + +`scrollId` follows from the same principle. It is optional input; when a scroll omits it, provenance +derives it deterministically: + +```js +// src/build/scroll.mjs +scrollId: scroll.scrollId ?? `${scroll.boxId}-${targetId}`, +``` + +**Rejected:** requiring `scrollId` to repeat the directory name. That made the filesystem a second +identity layer, and encouraged product-plus-machine directory names even though the scroll already +declares both facts. + +
+ +#### One effective scroll + +Reading is also where a scroll becomes complete. `effectiveScroll()` runs between validation and the +path sweep, filling in every field the target or the identity already determines — the interpreter +path, `model-cache/`, a `scrollVersion` of `1.0.0`, and the empty collections. Everything +downstream, including the provenance record, sees that one object and never has to ask whether a +field was written down. + +A split scroll is completed the same way, one step earlier. `joinScrollFragment()` runs *before* +validation, because neither half of a split scroll is a complete document: validating the fragment +alone would report every field the base holds as missing. The joined result is what the schema sees, +what the build reads, and what provenance records. + +The join rule is stated per field rather than as one blanket behaviour, and that is the whole +substance of the feature: + +| Fields | Rule | +| --- | --- | +| Scalars, and the cohesive objects `target`, `execution`, `parity` | The fragment replaces the base | +| `assets`, `assetArchives`, `localFiles` | Joined base-first; a repeated `relativePath` is an error | +| `prunePaths`, `uncompressedPaths`, `selfTest.imports`, `selfTest.files` | Joined base-first, repeats dropped | +| `compatibility`, `environment` | Joined key by key, the fragment winning a shared key | +| `selfTest.pythonCode` / `selfTest.pythonFile` | One slot; a fragment naming either replaces both | +| `extends` | Dropped — the joined scroll extends nothing | + +Each row is a rejection of the two obvious alternatives. Replace-everything would make a fragment +that adds one asset lose the shared ones. Merge-everything would leave `execution` half from each +half — a `python-script` kind carrying a `module` inherited from the base, which no author wrote and +the schema would then have to catch. The two list rules differ for the same reason: a repeated prune +path is one instruction twice and is dropped, while a repeated `relativePath` is two sources +claiming one file in the box, which is refused rather than settled by a precedence rule nobody would +remember. + +Order is declaration order, base first, in both the joined lists and a joined map's keys. Nothing is +sorted, because determinism asks only that one pair of files always produce one result. The +consequence is stated rather than hidden: a split scroll and a hand-written whole one hold the same +entries, and a joined map may serialise its keys in a different order. Sorting instead would change +the bytes of every box whose map was not already alphabetical, to fix nothing. + +`extends` takes exactly one value, `../scroll.json`. A path parameter would have invited traversal +screening, base chains, and a scroll that reaches outside its workspace; a fixed value costs nothing +and forecloses all three. A base declares no `target` — it holds what its targets share — and no +`extends` of its own. Both are checked in the reader, which is the only place that sees either file +on its own. + +Deriving in the reader rather than at each use is the whole point. The alternative — a `??` at every +call site — spreads the definition of "what this field means when absent" across the builder, where +two of them eventually disagree. Here there is one place to read, and a scroll that spells a derived +field out explicitly produces exactly the same object as one that omits it; `assertPythonEntryPoint` +still runs either way, so declaring the wrong interpreter is as much an error as it ever was. + +
+ +
+ +#### Selecting a scroll + +`scrollCandidates(name)` accepts three shapes: an exact `/` reference loads one +scroll; a bare box name expands to that box's target scrolls; omitting the name discovers every +nested scroll in the workspace. Every candidate is validated *before* it is offered, so a misleading +directory never becomes a selectable target, and every directory listing is sorted with +`compareStableStrings` so the order does not depend on the filesystem. + +`readScroll()` sits above it and **fails on ambiguity**. A box with more than one target scroll is a +hard error at the library level: + +```js +// src/build/scroll.mjs +fail(`Box ${name} has multiple scroll targets (…); use / or select a target explicitly.`); +``` + +Only the CLI edge is allowed to ask a person which one they meant. A library that prompted would be +a library that hangs in CI. + +
+ +
+ +#### Provenance from git + +Two functions read the state a box records about where it came from. + +```js +// src/build/scroll.mjs +export function sourceBuildState(cwd) { + const revision = runResult('git', ['rev-parse', 'HEAD'], { capture: true, cwd }); + if (revision.status !== 0) return null; + const status = runResult('git', ['status', '--porcelain', '--untracked-files=all'], { capture: true, cwd }); + return { revision: revision.stdout.trim(), dirty: status.stdout.trim().length > 0 }; +} +``` + +`--untracked-files=all` is what makes "dirty" mean what a reader expects: a file that exists but was +never committed makes a build unreproducible exactly as an edited one does, while Git's ignore rules +still keep generated state out of the answer. Returning `null` outside a checkout forces the caller +to handle it explicitly rather than inventing a revision. + +`sourceBuildTime(cwd)` takes the build timestamp from the HEAD commit rather than the clock, and +falls back to the Unix epoch outside a checkout — deliberately a constant, since a wall-clock +fallback would reintroduce exactly the nondeterminism this avoids. + +
+ +
+ +### 6.3 The workspace — `workspace.mjs` + + +Where a project keeps its scrolls, and where the tool writes what it builds, is the project's +decision. + +
+ +A [workspace](#workspace) is declared by `scrollcase.config.json` at the project root, +discovered by walking up from the working directory. + +| Path | Default | Holds | +| --- | --- | --- | +| `scrolls` | `scrolls` | Authored scrolls, `pixi.toml` and `pixi.lock` | +| `build` | `.scrollcase/build` | Scratch: the pixi workspace, the payload, the staged archive | +| `dist` | `.scrollcase/dist` | Publishable output: `boxes/` and `channels/` | +| `keys` | `.scrollcase/keys` | Local signing keys | +| `toolchain` | `.scrollcase/toolchain` | The project's own pixi and conda-pack | + +Resolution has two precedence rules that are easy to conflate and are not the same. + +**Root selection**, highest first: `--project-root`, the directory of an explicit `--config`, the +nearest `scrollcase.config.json` above the working directory, and finally the working directory +itself. An explicitly named config that does not exist is a hard error — silently ignoring it would +hide a typo behind the defaults. + +**Path resolution**, highest first: a CLI flag, the config's `paths` entry, the built-in default. +The subtlety is what each resolves *against*: + +- a **flag** resolves against the current working directory, because it was typed by a person + standing in some directory and that is what a shell user expects; +- a **config value** resolves against the project root, so that a config file is portable and means + the same thing from any working directory. + +The resolved workspace is frozen, and installed once per process by `configureWorkspace()`. Modules +read it through `getWorkspace()` rather than at import time, which is what lets an entry point +configure paths from flags before anything downstream observes them. `resetWorkspace()` exists as +the test seam. + +A config is shape-checked on read: a malformed JSON document, a non-object, an unknown `paths` key +or a non-string path each fail with the file named. An unknown key is refused rather than ignored, +for the same reason `additionalProperties: false` is the schema default — a typo that looks like it +worked is worse than an error. + +**Rejected:** deriving paths from the tool's own location on disk. That only works while the tool +lives inside the project it serves, and Scrollcase must run from anywhere against any project that +declares a workspace. + +
+ +### 6.4 Runtime schema validation — `schema-validation.mjs` + + +Scroll structure is validated at runtime, from the shipped schemas, before tool discovery or any +build-directory mutation. The validator is written from scratch, in 194 lines, and implements the +subset of JSON Schema 2020-12 the shipped schemas use: `$ref` (local pointers and absolute +registered `$id`s), `const`, `enum`, `type`, `minLength`, `pattern`, numeric bounds, `minItems`, +`items`, `minProperties`, `required`, `properties`, `additionalProperties: false`, +`dependentRequired`, `allOf`, `oneOf`, `if`/`then`/`else` and `not`. + +
+ +**Rejected:** taking Ajv as a fourth runtime dependency. Ajv is excellent and remains a *development* +dependency, used in the test suite where a second opinion about the schemas is worth having. Adding +it to the runtime would widen the installed surface of every consumer for one narrow job. + +The module's most important property is stated in its own header: *the schemas remain the source of +truth; this module deliberately contains no scroll field list.* A validator that enumerated fields +would be a second definition of the format, and the two would drift. + +It returns the **first** disagreement as a human-readable string with a JSON-pointer-like path +(`$.assets[2].sha256 does not match the required pattern`), rather than a list. A scroll author +fixing one problem at a time is better served by one clear message than by a cascade caused by the +first. + +
+ +### 6.5 Building the environment — `pixi.mjs` + + +Section 4 covered the three pixi invocations, the conda-pack arguments and the refusal to run +`conda-unpack`. What remains is what happens to the extracted tree, which is where relocation +actually becomes true. + +
+ +`installAndPackPixiEnvironment()` runs seven steps: + +1. **Stage a build-local pixi workspace.** The manifest and lock are copied side by side into + `/pixi-workspace/`, so the resulting `.pixi/envs/default` prefix is build-local and + never lands inside the tracked scroll directory. +2. **Install and pack.** `pixi install --frozen`, then `conda-pack -p -o + --format tar.gz`. +3. **Extract into `payload/venv/`**, using the pinned Node `tar` implementation, with links + deferred to a second pass. +4. **Delete the service files** that carry the build prefix. +5. **Canonicalise `conda-meta/`.** +6. **Settle every symbolic link.** +7. **Repair the generated launchers**, then delete the multi-gigabyte workspace and the tarball. + +The order of the last three matters and is documented in the code: links are settled *before* +launcher repair, so the repair walks a tree whose shape is final and rewrites each script's bytes +exactly once, under its own name rather than once per alias. + +
+ +#### Deferred link extraction + +Links cannot be created during extraction. The extractor refuses a link whose target passes through +another link — a defence against writing content through a link, and not negotiable — and a conda +prefix trips that condition routinely. One package ships `current -> ` and then +`pkgdata.inc -> current/pkgdata.inc`, and it arrives in a plain Python environment that never asked +for it, which made the whole box unbuildable. + +So links are collected during extraction and created in a second pass, once every regular entry is +already on disk: + +```js +// src/build/pixi.mjs +const deferredLinks = []; +await tar.x({ + file: packPath, cwd: venvDir, gzip: true, preservePaths: false, strict: true, + filter: (entryPath, entry) => { + if (entry.type !== 'SymbolicLink') return true; + deferredLinks.push({ path: safeRelativePath(entryPath), target: String(entry.linkpath) }); + return false; + }, +}); +``` + +Creating a link is not traversing one, and every deferred target is resolved and checked immediately +afterwards. The pass is sorted, so the tree is built identically whatever order the tar happened to +list its entries in, and a regular entry already occupying a path wins over a link to it: content +beats an alias. + +
+ +
+ +#### Canonicalising `conda-meta/` + +Per-package records are written by the *installer*, not by the package, and two installs of the +identical lock do not produce identical ones. They also carry absolute paths into the build +machine's package cache. Scrollcase keeps four fields and discards everything else: + +```js +// src/build/pixi.mjs +const CONDA_RECORD_FIELDS = Object.freeze(['name', 'version', 'build', 'license']); +``` + +`build` earns its place because name and version do not identify a conda binary: one version is +published in many builds, and a CPU and a CUDA build of the same library can differ in nothing else. +All four are properties of the package *as published* rather than of the install that placed it, +which is what makes them stable across rebuilds. + +The rule is an **allowlist**, deliberately, rather than a list of known-volatile fields to strip. A +field a future pixi release starts writing then cannot reintroduce the drift, because it was never +eligible to be copied in the first place. Anything in the directory that is not a record — conda's +`history` log — is removed entirely. + +Nothing inside a box reads any of this: conda is never shipped inside one, and package versions stay +readable from `site-packages` where a Python tool actually looks. + +
+ +
+ +#### Settling the links + +`settleSymlinksInPlace()` walks the tree in sorted order and, for each link, supplies the filesystem +facts the contract rule needs and applies its answer: + +| Situation | Action | +| --- | --- | +| Dangling, or unstattable | Removed | +| Resolves outside the prefix | Removed — it would drag a host file into the box | +| Target is a directory | [Materialised](#materialise) recursively, then walked again | +| Target is a file and the rule permits carrying it | Kept as a link | +| Anything else | Materialised as a copy, preserving the mode | + +The walk is sorted because whether a link may be kept can depend on what an earlier entry became, +and `readdir` order is the filesystem's business — two builds must settle the tree identically. + +`keepsAsLink()` then asks the decisive question, and asks it twice: + +```js +// src/build/pixi.mjs +const resolved = resolvePayloadLinkTarget(relativeLink, rawTarget); +if (resolved === null) return false; +// The lexical answer and the filesystem's answer must agree. +``` + +The **raw** target is what gets archived, so it is what must satisfy the lexical rule. The +filesystem is then asked whether following it really lands inside the prefix, at a regular file. +The two can disagree when the target is reached through another link — precisely the case a purely +lexical check cannot see — and both must say yes. + +
+ +
+ +### 6.6 Repairing launchers — `launchers.mjs` + + +Console scripts generated at solve time (`tqdm`, `isympy`, `f2py`, …) carry the build machine's +absolute interpreter path in their [shebang](#shebang). That path means nothing on a user's machine, +and shipping it leaks a developer's directory layout. + +
+ +The repair rewrites each affected script to resolve Python next to itself: + +```sh +#!/bin/sh +'''exec' "$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)/python" "$0" "$@" +' ''' +``` + +This shape is a **shell trampoline**: `/bin/sh` runs the second line, which re-executes the file with +the interpreter sitting beside it, and Python then reads the same two lines as a triple-quoted string +and ignores them. It exists because a direct absolute shebang can exceed the POSIX length limit, and +because there is no way to write "the interpreter next to this file" in a shebang at all. + +Two details in the implementation are easy to get wrong and are handled explicitly: + +- **The whole file is searched for a build path, not just line one.** A trampoline hides the path + below the first line, so a first-line-only check would miss exactly the scripts most likely to + carry one. The forbidden set is the prefix, the pixi workspace and the payload directory. +- **An existing trampoline is unwrapped before rewriting.** The header closes its quote either on + its own `' '''` line or at the end of the same line, so the parser scans forward to whichever line + closes it and keeps only the Python body. + +Only POSIX launchers are repaired. Windows console scripts are executables, not text with a shebang, +and are handled by the launcher shape the adapter records. + +
+ +### 6.7 Staging assets — `assets.mjs` + + +Every [asset](#asset) is declared with a size and a SHA-256 in the scroll, and nothing enters the +payload before both match. That is what makes a box reproducible even though its inputs live on +servers outside anyone's control: if an upstream file is moved, replaced or silently re-uploaded, +the build fails instead of quietly producing a different box under the same version. + +
+ +
+ +#### `downloadVerified` + +The interesting part is resumption, because model weights are large enough that a connection reset +near the end of a multi-gigabyte transfer is a real event rather than a hypothetical one. + +```text +destination exists? → reuse only if size AND hash match + ↓ +loop (max 5 attempts): + resumeAt = size of .part, or 0 + fetch with Range: bytes=- when resumeAt > 0 + append only if the server answered 206; a 200 overwrites + on network error: wait 2000 × attempt ms, retry + ↓ +size must equal sizeBytes → hash must equal sha256 → rename .part into place +``` + +Four decisions are worth naming. + +**The `.part` file is renamed into place only after the hash matches.** An interrupted or corrupted +transfer therefore can never masquerade as a finished asset — a completed file at the destination +path is a verified file, always. + +**A 200 response overwrites rather than appends.** A server that ignores `Range` replies with the +whole body and status 200; appending that to a partial would produce a file of the right length made +of the wrong bytes, which is exactly the failure the hash exists to catch and exactly the failure +that is cheapest to avoid. + +**A failed HTTP status is not retried.** A 404 or a 403 is a hard error, not a transient drop; +retrying it five times with backoff only delays the message. + +**A full-size partial with the wrong digest is deleted.** It cannot be resumed — asking for bytes +after its end would either fail forever or append unrelated data — so removing it lets the next +build start from byte zero and recover from a corrupt mirror response. + +This is deliberately **not** a cross-process cache. The build scratch tree is recreated at process +start, so resumption is scoped to one download operation, and the documentation says so rather than +implying a persistence that does not exist. + +
+ +
+ +#### `copyVerifiedLocalFile` and `expandAssetArchive` + +Local files are copied from the project's own repository and hashed against the scroll's declaration +first, so a licence notice or a runtime shim cannot drift from what was reviewed. + +Asset archives are listed and validated before extraction — the archive-slip defence described in +6.12 — and `stripComponents` insists on finding exactly one top-level directory to strip, so a +surprising layout fails loudly rather than producing a wrong tree. The compressed original is +removed after expansion unless the scroll asks otherwise, since it is dead weight inside the payload +once unpacked. + +
+ +
+ +#### `moveIntoPlace` + +A box archive is measured in gigabytes, so publishing renames rather than copies: on one filesystem +the bytes never move at all. The copy-and-remove fallback exists for a project that points its build +and dist directories at different volumes, where `rename` cannot work. + +
+ +
+ +### 6.8 The licence inventory — `licenses.mjs` and `audit.mjs` + + +The inventory is derived from the committed [lockfile](#lockfile) rather than from the installed +tree. The lock already carries an [SPDX](#spdx) licence per package, and `pixi install --frozen` +guarantees the installed set equals it. So the audit is a pure function of a file a human reviews, +computable without a built prefix, and unable to drift from what was approved. + +
+ +That is what lets `audit` run in a second, with no toolchain and no network, so licence review +happens when dependencies change rather than at the end of a multi-gigabyte build. + +
+ +#### Parsing the lock + +`lockedCondaDistributions()` scans the lock's `packages:` section directly rather than taking a +transitive YAML dependency — the structure is regular and machine-generated, a list of +`- conda: ` or `- pypi: ` items each followed by indented `key: value` fields. + +For conda entries, name and version come from the package filename rather than from the record: + +```js +// src/build/licenses.mjs — parseCondaPackageReference +// conda names may contain '-', but version and build never do, so they are the last two segments. +``` + +Two rules keep the result honest. A package whose licence is absent or literally `UNKNOWN` +**fails the parse outright** — an unlicensed dependency is a legal problem, not a reporting gap. +And names are kept raw rather than normalised, because conda filenames already carry the canonical +name and normalising would mangle legitimate leading-underscore names such as `_openmp_mutex`. + +The result is sorted by name then version, which is what makes the inventory itself deterministic. + +
+ +
+ +#### The audit document + +`createCondaDependencyLicenseAudit()` produces: + +```jsonc +{ + "schemaVersion": 2, + "kind": "scrollcase.box.dependency-license-audit", + "targetId": "macos-aarch64-metal", + "dependencyLockSha256": "…", + "packages": [{ "name": "…", "version": "…", "declaredLicense": "MIT", "source": "conda" }] +} +``` + +It carries the lock's hash, so the inventory names the exact input it was derived from, and its +`kind` is namespaced like every other document — a project keeps its own namespace here too. + +`validateCondaDependencyLicenseAudit()` compares a reviewed copy against a freshly computed one by +exact JSON equality. During a build, that comparison happens before the inventory is written into +`THIRD_PARTY_NOTICES/conda-distributions.json`, so a box can only ship an inventory somebody +reviewed. + +
+ +
+ +#### `audit.mjs` + +The `audit` verb runs the same functions the build runs, so a reviewed audit and the one a build +produces cannot disagree by construction. It adds a summary — package count and licences ranked by +frequency — and one policy decision: + +**Writing is explicit.** `--write` overwrites the reviewed file; the default compares and fails on +any difference. Making writing the default is precisely how an unreviewed licence change would slip +through. + +
+ +
+ +### 6.9 Execution prerequisites — `execution.mjs` + + +Execution metadata names either one regular payload file or one dotted Python module. Proving that +name resolves must not involve running anything, and this module is how. + +
+ +For a **script**, the check is set membership: the path, screened by `safeRelativePath`, must be a +regular entry in the payload. + +For a **module**, the check enumerates the places Python would find it and asks whether any exists +as a regular file: + +```js +// src/build/execution.mjs +const relativeCandidates = [`${modulePath}.py`, `${modulePath}/__main__.py`]; +const standardLibrary = adapter.platform === 'windows' + ? 'venv/Lib' + : `venv/lib/python${pythonMajorMinor(pythonVersion)}`; +const roots = ['', standardLibrary, `${standardLibrary}/site-packages`]; +``` + +Both `foo/bar.py` and `foo/bar/__main__.py` are accepted, since `python -m` runs either. The roots +are the payload root, the standard library and `site-packages`, and the Windows standard library +lives at `venv/Lib` rather than under a version-named directory — one of the three-target +differences that no single-host test suite can catch. + +**Rejected:** proving a module by importing it. Importing runs `__init__.py`, which is application +code, and would turn validation into execution before the trust chain has finished. The whole point +of a static check is that the same function can be used by the builder *and* by a consumer that has +not yet decided to trust the box. + +That shared use is why the module takes a `files` set rather than a directory: the builder passes +`collectFiles()` output, the verifier passes the ZIP entry classification, and both are the same +representation. + +
+ +### 6.10 Self-test and parity + + +
+ +#### The self-test + +```js +// src/build/box.mjs +run(interpreter, ['-c', code], { + cwd: payloadDir, + env: mergeEnvironmentLayers( + adapter.platform, + scroll.environment ?? {}, + adapter.validationEnvironments[scroll.target.accelerator], + ), +}); +``` + +Three things make it meaningful. The interpreter is the payload's own, so what is proven is the box +rather than the host. The working directory is the payload, so relative resolution behaves as it +will after extraction. And the environment is the target's validation environment, so a Metal box is +tested with the MPS fallback disabled rather than quietly falling back to CPU. The signed +environment declaration is applied here too, before the target controls, so a bad runtime path fails +the build while accelerator validation remains authoritative. + +The code that runs is the adapter's platform assertion, then the declared imports, then the scroll's +optional extra Python. Only the import subset reaches the signed release, with +`timeoutSeconds: 180` recorded as the bound a consumer should apply when repeating it. The scroll's +Python-code and file assertions stay builder-only because they are not part of the signed release, +and claiming otherwise would tell a consumer it had verified something it never saw. + +
+ +
+ +
+ +#### Parity — `parity.mjs` + +The [parity](#parity) gate runs the declared script once per accelerator, using the declared box +environment followed by each accelerator's validation environment, and compares every run against +the first. + +The check script must print a JSON array of numbers, or an object with a `values` array. Three +refusals happen before any arithmetic: + +- output that is not JSON, reported with the first 200 characters so the failure is diagnosable; +- an empty or missing `values` array; +- **any non-finite value.** A `NaN` or an infinity is the classic symptom of a broken accelerator + build, so it is reported as such rather than being allowed to poison the comparison. + +`compareValues()` computes three quantities in one pass: the maximum absolute difference, the +maximum relative difference, and the cosine similarity of the two vectors. The relative error is +accumulated only where the reference entry has magnitude: + +```js +// src/build/parity.mjs +if (Math.abs(expected) > 0) maximumRelative = Math.max(maximumRelative, absolute / Math.abs(expected)); +``` + +Relative error is meaningless around zero — dividing by a reference of `0` yields infinity for any +discrepancy at all — so the absolute bound is what guards near-zero entries, and cosine similarity +catches a result that drifted in direction rather than in magnitude. That is why the schema allows +three tolerances and requires at least one: they answer different questions about the same +comparison. + +`breachedTolerance()` reports which declared bound was exceeded, by how much, and against what. The +first accelerator listed is the reference — conventionally `cpu`, being the one available everywhere +and the least likely to be wrong — and the measurements are returned even when nothing failed, so +they can be recorded as evidence. + +The division of labour is the point, and it is the boundary of section 3 applied to numbers: +Scrollcase owns the mechanism and enforces the declared threshold; the project owns the check +script, the fixture, and what closeness means for its model. + +
+ +
+ +### 6.11 Determinism primitives — `filesystem.mjs` + + +Two invariants live in this module: every payload tree is enumerated in one stable order and stamped +with one fixed timestamp, and every relative path that will be joined to a directory is screened +against traversal. + +
+ +```js +// src/build/filesystem.mjs +export const FIXED_ARCHIVE_TIME = new Date('2000-01-01T00:00:00.000Z'); +``` + +Any fixed instant would do; this one is a recognisable round date safely past the 1980 floor of +DOS/ZIP timestamps. + +| Function | Role | +| --- | --- | +| `compareStableStrings` | Ordering by code unit, independent of host locale and ICU data | +| `safeRelativePath` | The [path-traversal](#path-traversal) screen, normalised to forward slashes | +| `collectEntries` | The canonical sorted enumeration: files and links, with link targets | +| `collectFiles` | Every payload path, links included | +| `collectRegularFiles` | Only paths backed by their own bytes | +| `payloadSize` | What a box occupies once extracted | +| `validateExtractedTree` | Refuses links and special nodes in an extracted tree | +| `normalizeTree` | Stamps the fixed mtime on every entry | +| `sha256File` | Streaming hash, so a multi-gigabyte archive is never buffered | + +Several of these carry a decision that is invisible until it bites. + +**Ordering is by code unit, not by locale.** `localeCompare` would make archive entry order depend +on the machine's ICU data, which is a per-host variable and therefore a determinism bug waiting for +a differently configured CI runner. + +**Links are classified before directories.** A symbolic link to a directory reports +`isDirectory() === false` from `lstat` but would be walked into by any check that stats rather than +lstats, so the classification order is what keeps the walk from following a link out of the tree. + +**`collectFiles` and `collectRegularFiles` are two functions on purpose.** A caller asking "is this +path in the box?" wants a link to count, because a linked path is a path that resolves. A caller +that rewrites bytes must not see links, because writing through one would edit the target twice — +once under its own name and once under the link's. Launcher repair uses the second kind of question. + +**`payloadSize` uses `lstat`, not `stat`.** A link costs its own few bytes, not the size of what it +points at. Counting the target would restore on paper exactly the duplication that carrying links +removes from disk, and this number is what a consumer checks free space against. + +**`normalizeTree` uses `lutimes`, not `utimes`.** Stamping through a link would stamp its target +once under its own name and again through every link pointing at it. + +**Three names are skipped during enumeration**: `__pycache__`, `.DS_Store` and any `.pyc`. Bytecode +caches are written by whichever interpreter happens to run first and are the classic source of a +non-reproducible tree; the third is a macOS artefact that has no business inside a box. + +Anything that is neither a regular file nor a permitted link — a socket, a device, a fifo — fails +the enumeration outright, because nothing else can be archived, hashed or relocated meaningfully. + +
+ +### 6.12 Archiving — `archive.mjs` + + +The writing side is where [determinism](#determinism) becomes bytes. + +
+ +```js +// src/build/archive.mjs +const compressionLevel = isDeclaredUncompressed(entry.path, uncompressedPaths) ? 0 : 6; +zip.addFile(join(payloadDir, ...entry.path.split('/')), entry.path, { + compress: compressionLevel !== 0, + compressionLevel, + mtime: FIXED_ARCHIVE_TIME, + mode: archiveFileMode(adapter, entry.path), + forceDosTimestamp: true, +}); +``` + +Every variable that could differ between two runs has been removed. Entry order comes from the +sorted enumeration; the timestamp is fixed and forced into DOS form so no local timezone leaks in; +the compression level is pinned per path, because a different level produces different bytes for +identical input; and the mode comes from the **target adapter** rather than from the filesystem: + +| Path | Mode | Reason | +| --- | --- | --- | +| Windows target, anything | `0644` | Windows has no executable bit to preserve | +| The interpreter entry point | `0755` | It must be executable after extraction | +| Anything under the scripts directory | `0755` | Console scripts are executables | +| Everything else | `0644` | | + +Reading the mode from disk instead would make the archive depend on the umask of whoever ran the +build. + +**Already-compressed paths are stored rather than deflated.** Model weights arrive compressed — +GGUF, safetensors — and deflating them buys nothing while costing real time: measured on +incompressible bytes, level 6 runs at 47 MB/s and produces a result 0.03% *larger* than its input, +and dropping to level 1 recovers 4 MB/s because the search fails either way. Lowering the level is +therefore not a fix; only not compressing is. Which paths those are comes from the scroll and never +from the file: every declared asset is stored automatically, and `uncompressedPaths` names anything +else the project knows to be compressed already, matching a path itself and everything beneath it. +Nothing here opens the file or reads its extension, so the decision depends only on the scroll and +the path — which is what keeps two builds of the same commit byte-identical. + +Symbolic links are written as a small entry whose *content* is the target string, under a mode +carrying the symbolic-link type bits — the same two facts every ZIP implementation reads a link back +from. Before any of this, `assertPayloadLinksAreCarryable()` re-applies the contract rule to the +entry set about to be written: a failure there is a bug in Scrollcase rather than bad input, but +shipping a box a consumer must reject is worse than not building one. + +Zip64 is emitted only where needed (`zip.end({ forceZip64Format: false })`), so a small box stays +readable by the widest range of tools while a large one is still correct. + +
+ +#### The reading side + +Nothing inside an archive is trusted before validation. `listZipEntries()` walks every entry and +refuses, in this order: encrypted entries, unsafe names, special entries, link targets over 1024 +bytes, duplicate paths, file/directory collisions, links that do not resolve to a file inside the +payload, and any entry that would be written through a link. + +Two details matter more than they look. + +**Link targets are read during validation and reused during extraction.** Reading the target twice +would let a concurrently rewritten archive pass the check with one value and extract with another — +a time-of-check-to-time-of-use gap in the one place it would be most rewarding to exploit. + +**A collision check runs before extraction, not during it.** Two entries with the same path, or a +file where another entry expects a directory, are refused up front rather than discovered when the +second write fails. + +TAR is handled more strictly still: only `File`, `OldFile` and `Directory` entries are accepted, so +links and special entries in a scroll asset archive are refused outright. Those archives come from +outside, and the copy that follows extraction would write through any link they contained. + +Reference: `tests/unit/archive-security.test.mjs`, `tests/unit/assets.test.mjs`. + +
+ +
+ +### 6.13 Naming — `identity.mjs` + + +Three small functions decide where a release's artefacts live relative to everything else: + +
+ +```js +// src/build/identity.mjs +boxReleaseStem(release) // -- +boxReleaseObjectPrefix(release) // boxes/// +builderVersionFields(source) // { pixiVersion } +``` + +Both names are derived from the release's identity fields alone, so the archive, its release +document and the staged objects agree on their location without any of them recording the others' +paths. Whatever a project uses to serve boxes, laying storage out under this prefix means the URLs +inside the signed documents already point at the right objects. + +
+ +### 6.14 Signing, from the builder's side + + +The builder signs twice, through one call each, and treats signing as a service it consumes rather +than a concern it implements: + +```js +// src/build/box.mjs +const signing = { signerCommand, privatePath, publicPath }; +await signDocument(release, signing); +await signDocument(channelDocument, signing); +``` + +
+ +Everything about how that signature is produced — local key or external signer, the mandatory +payload echo, local re-verification — is section 7's subject. What matters here is the ordering: the +release is signed after the archive exists and has been hashed, and the channel is signed after the +release document exists and has been hashed. Neither document can commit to something that has not +been measured. + +
+ +### 6.15 Verifying locally — `verify.mjs` + + +`verify` re-runs a consumer's install-time checks locally, before anything is published. The point +is that a box which would fail on a user's machine fails here instead. + +
+ +The trust chain is split without being duplicated. `inspectReleaseDocument()` performs the part +that needs only a release document and trust key; `inspectBoxArchive()` calls it and continues with +the archive. Attachment and installed-payload verification therefore reuse the same interpretation +of signature, schema, kind and target without inventing an option that makes one function's return +shape conditional. + +The two functions perform the complete read-only chain in this fixed order: + +1. Refuse `schemaVersion: 1` explicitly. +2. Validate the **signed envelope** against its schema. +3. **Verify the signature** against the trusted key. +4. Refuse a payload that is not `schemaVersion: 2`. +5. Validate the **release manifest** against its schema. +6. Confirm the document's `kind` parses as a *release*. +7. Resolve the target adapter and check the entry point against it. +8. Sanity-check `installedSizeBytes` if present. + +Those eight steps are `inspectReleaseDocument()`. `inspectBoxArchive()` then continues: + +9. Locate the archive — beside the release document, under the hash that document commits to. +10. Check the archive's **size**, then its **SHA-256**. +11. List and validate **every archive entry**. +12. Read `box.json` out of the archive and validate it against its schema. +13. Assert **agreement** between `box.json` and the signed release. +14. Confirm the declared interpreter path resolves inside the archive. +15. Confirm execution metadata names a real script or a discoverable module. + +Nothing in that list executes anything from inside the box. Every step is a read, and the expensive +ones come after the cheap ones that could have ended the check. + +
+ +#### The agreement check + +```js +// src/build/verify.mjs +const AGREEMENT_FIELDS = [ + 'schemaVersion', 'boxId', 'modelId', 'runtimeId', 'version', 'target', + 'pythonEntryPoint', 'modelCacheSubdir', 'environment', 'selfTest', 'execution', + 'weights', 'assets', 'provenance', +]; +``` + +Each is compared with `isDeepStrictEqual`, so nested objects — the target, the self-test, the whole +provenance block, the environment map, the asset descriptor list — must agree recursively rather than merely being +present. This is what binds the archive's contents to its signed metadata: the release commits to +the archive by hash, and the archive's own description of itself must match the release. + +Only fields that exist in *both* schema-version-2 documents belong in that list. Release-only +transport data — `kind`, `archive`, `compatibility`, `installedSizeBytes`, `payloadDigest` — has no +counterpart in `box.json`, and demanding one would be demanding agreement about a field that does +not exist. The list itself already names and hashes `box.json`; placing its commitment inside that +file would create a recursive value. + +
+ +
+ +#### Two entry sets, deliberately + +```js +// src/build/verify.mjs +const files = new Set(entries.filter((entry) => entry.kind === 'file').map((entry) => entry.path)); +const resolvablePaths = new Set(entries + .filter((entry) => entry.kind === 'file' || entry.kind === 'link') + .map((entry) => entry.path)); +``` + +`box.json` is read *out of* the archive, so it must be an entry with its own bytes. The interpreter +path and the execution target only need to *resolve*, and a link does resolve — to a file inside +this same payload, because the link rule allowed nothing else. Using one set for both questions +would either reject a legitimate interpreter alias or accept a manifest that was only a link. + +
+ +
+ +#### Verifying with `--self-test` + +With `--self-test`, verification extracts the archive into a temporary directory, checks that the +extracted payload size matches the signed `installedSizeBytes`, recomputes the payload digest when +the release carries one, and only then runs the box's own interpreter against the signed import +subset. This is the pre-publication point that proves the build's list describes what its archive +actually extracts to. The temporary directory is removed in a `finally` block whether or not the +check passed. + +It requires a matching native host, through `assertNativeHost`. That is a deliberate limitation +rather than an oversight: running a Linux box's interpreter on macOS proves nothing, and pretending +otherwise would make the strongest check in the tool the least trustworthy. + +
+ +
+ +### 6.16 Setting up a project + + +Three modules cover everything before a build: workspace scaffolding, scroll authoring, and the +optional dependencies of the generated consumer templates. + +
+ +
+ +#### `project.mjs` — `init` and `doctor` + +`initProject()` writes four things and **never overwrites**: `scrollcase.config.json`, a short +`SCROLLCASE.md` project guide, the scrolls directory, and an appended `.gitignore` block. Existing +files are recorded as skipped, so a half-configured workspace can be completed by running the +command again without touching authored input. + +The `.gitignore` block is matched by a marker comment: + +```js +// src/build/project.mjs +const GITIGNORE_MARKER = '# scrollcase build state'; +``` + +Changing that string would make an already-scaffolded project look unmarked and append the rules a +second time. It is a small thing that a blanket capitalisation pass has already broken once. + +`ensureToolchain()` is where consent lives — as an **injected function**, not a terminal read: + +```js +if (!await confirm(missing)) return { installed: [], missing, declined: true }; +``` + +The CLI asks a human, a scripted setup passes a flag, and CI without a terminal answers no. Nothing +is downloaded before that call returns true. A present pixi at the *wrong* version counts as missing +when a version was requested, because resolver versions are part of the scroll's reproducibility +contract and `--pixi-version` must install what it promises. + +What it records in the project config is the other half of the design: + +```jsonc +{ + "toolchain": { + "pixi": { "version": "0.73.0", "assets": { "pixi-aarch64-apple-darwin.tar.gz": "" } }, + "condaPack": { "version": "0.9.2" } + } +} +``` + +The first install trusts the checksum published beside the release; every later one is checked +against the value the project committed. A teammate or a CI runner therefore cannot silently receive +different bytes than the ones somebody reviewed. + +`diagnose()` — the `doctor` verb — checks the workspace, the scrolls directory, the git checkout, +pixi and conda-pack. Every check **reports rather than throws**, so a user with neither tool learns +both in one run instead of one per attempt, and every failing check carries its remedy. + +
+ +
+ +#### `authoring.mjs` — `new scroll` + +The only command that authors real project identity, target, versions, compatibility, weights and +execution intent. Its guarantees are atomicity and non-destruction: + +- Every material value is validated **before the first write**. A non-terminal call that omits one + fails rather than guessing. +- The scroll is written into a staging directory beside its destination and moved into place with a + single `rename`, so an interrupted run leaves no half-written scroll. +- An existing scroll directory is a hard error, and a generated starter script is written with the + exclusive `wx` flag. +- The generated scroll is validated against the schemas before anything is written at all. + +What it generates is deliberately short. Every field the reader can derive is left out, so the file +reads as the decisions its author made rather than a form they filled in; the generated starter +script is recorded in `localFiles` **without a hash pin**, because the first thing an author does +with a starter is edit it; and the self-test is written as a real `self_test.py` beside the scroll, +with `selfTest.pythonFile` pointing at it. + +Two constants live here rather than in a lookup: `DEFAULT_PYTHON_VERSION`, one minor behind the +newest Python conda-forge publishes, and `LATEST_PYTHON_VERSION`, what `--python-version latest` +resolves to. Both are committed and moved deliberately at release time by +`scripts/bump-python-version.mjs`, which asks conda-forge what it has built. The alternative — +resolving the newest Python on each invocation — would make the same command produce different +scrolls in different months, which is the failure a scroll exists to prevent; and defaulting to the +very newest would hand a first-time user a solve that cannot succeed, because conda-forge builds the +heavy compiled packages for a new minor months after the interpreter lands. `latest` therefore +resolves once, at authoring time, and the resolved number is what the scroll records. + +Execution intent is a closed set at this level too — `python-script`, `python-module` or +`library-only` — and a `library-only` scroll declaring a script, a module or default arguments is +refused rather than silently simplified. + +The weights mode is not one of the decisions `new scroll` asks about. It says where declared assets +live — inside the archive, or beside it for the caller to materialize — and a box that declares no +assets, which is most of them, has nothing for it to decide. `createScroll` defaults it to `embed` +and then leaves it out of the generated file, because that is the schema's own default and a scroll +should read like the decisions its author actually made. `--weights on-demand` states the other +choice for a box whose assets are published separately. + +`ensureExampleScroll()` creates the disposable `example-box` that `init` offers, through the same +validated authoring path as any real scroll. An existing target directory is treated as authored +input and left untouched, including when a user has edited the starter. + +`ensureConsumerTemplates()` writes the three consumer templates, and is deliberately a separate +function called from a separate question. The Rust template is a small Cargo crate with its own +manifest and `/target/` ignore; none of the three is ever overwritten. They were once part of the +example, and declining a throwaway scroll took them with it — which was wrong in the case that +matters most: a project that knows it does not want a demo is a project that has an application to +write, and these are that application's starting point. For the same reason they name no box of +their own, only a placeholder release path the author fills in. + +**Rejected:** treating setup metadata as the project's real scroll, and equally, leaving a newcomer +with an empty directory. The example is explicitly disposable onboarding material; real inputs are +created independently rather than edited from guessed product metadata. + +
+ +
+ +#### `scroll-edit.mjs` — changing a scroll that exists + +`authoring.mjs` creates one scroll from nothing; this module changes one already checked in, which +is a different problem in one specific way. A box may be split across a base and several target +fragments, so every edit answers **which file** before it answers what — and that question has one +answer here rather than one per command. + +Two guarantees cover every edit. It is **atomic**: new bytes go to a staging file beside the +original and move into place with a single rename, so an interrupted run leaves no half-written +scroll. And it is **verified**: afterwards every target of the box is read back through `readScroll`, +the same path a build uses, and the originals are restored if any of them no longer loads. The +verification deliberately covers the whole box rather than the edited file, because a base and its +fragments only mean anything together — an entry added to the base can collide with one a fragment +already declared, which is exactly the case worth catching before it is saved. + +`addAsset` fetches a URL once and records the size and hash it found. Those are the two values a +scroll cannot omit and no author can know without downloading the file, which is what made writing +one by hand a matter of `curl | shasum` and careful pasting. Recording them here weakens nothing: +the guarantee has always been that they are pinned once and checked on every build. + +`addFile` writes no `sha256`, for the reason given in the schema section — the file being added is +usually the one about to be edited. `removeScrollEntry` is the exact inverse of both, `selfTest.files` +line included, and a path that matched nothing is an error rather than a quiet success. + +`refreshScroll` recomputes the pins a project asked for. Its restraint is the interesting part: a +remote asset's hash is what stands between a replaced upstream file and a silently different box, so +re-fetching is opt-in, a difference is reported and refused, and accepting it takes a separate +`repin`. **Rejected:** refreshing remote hashes by default. That would make every upstream +substitution disappear into the next `refresh`, and the build would go green — which is the whole +protection, removed by the command meant to maintain it. + +`editableScrollFields` reads the field list out of the schema rather than keeping one in step by +hand, minus an explicit set the format does not let a person change: structural values, values the +layout or target fixes, and the collections, which have their own commands. + +`setEnvironmentVariable` and `addSelfTestImport` exist because those two were, for a while, the only +parts of a scroll with no command behind them — a map and a list that a single-value prompt cannot +edit, left to a hand edit in a file every other field had been freed from. Each sets one entry and +leaves the rest alone. Removing the last environment variable takes the empty map with it; removing +the last self-test import is refused, because a box has to prove it can import something and writing +a scroll the schema would reject is not a service. + +
+ +
+ +#### `dependencies.mjs` — the `[dependencies]` table + +`pixi.toml` is the second-most tedious part of authoring a box and, unlike the scroll, it has to be +edited once per target. This module changes every manifest of a box at once, so a dependency is one +command rather than three edits that have to agree. + +It edits **text**, not a parsed document. A TOML parser would be a new runtime dependency for a job +whose whole scope is one table of `name = "spec"` lines, and re-emitting would rewrite the comments +and spacing the project chose; the check is that the table's boundaries are found by its header and +the next one, so nothing is written into a `[target.…]` table below it. + +No version is looked up. An added dependency defaults to `*` and the committed `pixi.lock` records +what was actually solved. **Rejected:** asking the network for a "latest" to write into the manifest, +which would put a second, weaker pin beside the real one and leave the two to drift. + +`readRequirements` translates a pip `requirements.txt`. The table of PyPI names whose conda-forge +package is called something else is deliberately short — every entry is one this project can state +with confidence — and **every rename and every skip is reported**, because a name guessed wrongly +produces a lock that resolves and a box that cannot import what it was built for. That failure +arrives long after the command that caused it, which is why the command is loud. + +
+ +
+ +#### `consumer-setup.mjs` + +Optional installation of the generated templates' dependencies, into the *initialised project* — not +into Scrollcase's managed toolchain. Every command runs from the workspace root. + +Three package-manager realities are handled explicitly. On Windows, `npm` is a `.cmd` shim that `spawnSync` +cannot execute directly, so it is invoked through the command interpreter. On a PEP 668 +"externally-managed environment" — the default on modern Linux distributions and Homebrew Python — +`pip install` is retried with `--user --break-system-packages`, which keeps package files out of the +distribution's managed prefix rather than fighting it. Rust dependencies belong to a Cargo +manifest, so the optional Rust setup runs `cargo add` against the generated template crate rather +than attempting a global installation of a library. `isCargoAvailable()` probes the package manager +before the interactive questions: if it is absent, the Rust question and install are skipped while +the generated crate remains available for later setup. + +Consent and the Python package source are chosen at the CLI edge and passed in, never read from a +terminal here. + +
+ +
+ +### 6.17 Process primitives — `process.mjs` + + +Sixty-six lines, two exported behaviours, and both of them are architecture. + +```js +// src/build/process.mjs +export function fail(message) { throw new Error(message); } +``` + +
+ +**Every validation failure in the tool goes through `fail`.** That is what lets the CLI exit +non-zero with exactly one clear line, and it is why there is no second error path to keep consistent. + +`runResult()` wraps `spawnSync` and returns the raw result; `run()` interprets it, distinguishing a +command that could not start from one that exited non-zero, and attaching captured output to the +message when there is any. `mergeEnvironmentLayers()` preserves that inheritance while removing +case-only duplicates on Windows, where `Path` and `PATH` name the same variable; later layers win +deterministically instead of leaving Node's child serializer to choose. That is what lets a +validation environment force an accelerator without discarding everything else, and the +64 MiB buffer is sized for a chatty solver rather than for a prompt. + +Both are the **[injection seam](#injection-seam)** the whole test suite depends on. Passing a fake +runner is how the pipeline tests build boxes with no pixi, no conda-pack and no network, while still +exercising the real orchestration code — which is the only way to test a pipeline whose real +execution costs minutes and gigabytes. + +
+ +### 6.18 The build layer's public surface — `index.mjs` + + +`scrollcase/build` exports the pieces a project might legitimately need to drive or inspect a build +itself: + +| Group | Exports | +| --- | --- | +| Archive | `createDeterministicZip`, `extractZipArchive`, `listZipEntries` | +| Filesystem | `collectFiles`, `fileExists`, `sha256File` | +| Identity | `boxReleaseObjectPrefix`, `boxReleaseStem`, `builderVersionFields` | +| Launchers | `repairPosixLaunchers` | +| Licences | `createCondaDependencyLicenseAudit`, `lockedCondaDistributions`, `parseCondaPackageReference`, `validateCondaDependencyLicenseAudit` | +| pixi | `condaPackArguments`, `findCondaPack`, `findPixi`, `installAndPackPixiEnvironment`, `pixiInstallArguments`, `pixiLockArguments` | +| Process | `fail`, `run`, `runResult` | +| Toolchain | `CONDA_PACK_VERSION` | +| Workspace | `DEFAULT_WORKSPACE_PATHS`, `SCROLLCASE_CONFIG_FILENAME`, `configureWorkspace`, `findWorkspaceConfig`, `getWorkspace`, `resolveWorkspace`, `workspaceOverridesFromArgv`, `workspaceOverridesFromFlags` | + +
+ +What is *not* exported is as informative. `buildBox` itself is reached through the CLI rather than +advertised as a library entry point, and the internal orchestration — asset staging, parity, the +self-test — has no public surface. A change to any name in that table is a change to a public API; +a change inside the modules it comes from is not. + +## 7. Signing and custody + +A box is a file. Anyone can produce a file. What makes one box different from another file with the +same name is that a [release](#release) document commits to its bytes, and that document carries a +signature somebody's key produced. Section 5 described the shape of that document; section 6 +described when the builder asks for it. This section describes the part in between: where a key +comes from, what exactly gets signed, how a signature is checked, and how an operator keeps the +private half somewhere Scrollcase never sees. + +Two modules do all of it. `src/sign/keys.mjs` owns key material and verification; `src/sign/index.mjs` +owns the choice between signing paths. Together they are under three hundred lines, which is +intentional: cryptographic surface area is a liability, and everything here that could have been an +option is a constant instead. + +
+ +### 7.1 What the signature is for + +A signature answers exactly one question — *did the holder of this key assert this payload?* — and +Scrollcase builds its entire trust chain out of that single answer: + +
+ +```text + trusted public key keyId + ed25519 public bytes + | + | verifies + v + signed document payloadBase64 + payloadSha256 + signatures + | + | decodes to + v + release manifest archive.sha256, archive.sizeBytes + | + | commits to + v + the archive one exact byte string + | + | contains + v + box.json must agree with the release, field by field +``` + +Each link is mechanical. The key verifies the document; the document *is* the release; the release +names the archive by [digest](#digest) and size; the archive contains a [`box.json`](#box-json) that +must agree with the release. A consumer that trusts one public key therefore transitively knows +everything about the box, and nothing in the chain requires trusting the transport, the mirror, the +filesystem, or the builder. + +The chain is only as good as its weakest step, which is why the format refuses to let any step be +approximate. A [detached signature](#detached-signature) over "the release, more or less" would be +worthless; the signature is over exact bytes, and those exact bytes are what gets published. Section +5.3 explains why the payload travels as [base64](#base64) rather than as +[canonical JSON](#canonical-json); this section is what consumes that decision. + +
+ +### 7.2 Generating a key — `keys.mjs` + +`scrollcase keygen` produces an [ed25519](#ed25519) pair. There is no algorithm flag, no curve +choice, and no key size: ed25519 is small, fast, deterministic, has no parameter that can be chosen +badly, and is implemented by every runtime a consumer might be written in. + +
+ +
+ +#### The pair, and what is written where + +```js +// src/sign/keys.mjs +const { privateKey, publicKey } = generateKeyPairSync('ed25519'); +const privatePem = privateKey.export({ type: 'pkcs8', format: 'pem' }); +const publicPem = publicKey.export({ type: 'spki', format: 'pem' }); +const publicDer = publicKey.export({ type: 'spki', format: 'der' }); +const rawPublicKey = publicDer.subarray(publicDer.length - 32); +``` + +Two files are written, and they are not symmetrical: + +| File | Default location | Contents | Mode | +| --- | --- | --- | --- | +| Private key | `.scrollcase/keys/signing-private.pem` | PKCS#8 PEM | `0600`, then `chmod` again | +| Public key | `.scrollcase/keys/signing-public.json` | JSON: `algorithm`, `keyId`, `publicKeyBase64`, `publicKeyPem` | default | + +The private key is written with `mode: 0o600` **and** `chmod`ed to `0600` immediately afterwards. +That looks redundant and is not: the mode passed to `writeFile` is masked by the process umask, so a +permissive umask would widen the file at creation. The second call fixes the mode unconditionally. + +The public file carries the key twice on purpose. `publicKeyPem` is what Node's `createPublicKey` +consumes directly; `publicKeyBase64` is the raw 32 bytes, which is the form every non-Node verifier +expects — a Rust client, a browser using WebCrypto, a Python consumer that would otherwise have to +parse PEM to get at the same bytes. An ed25519 SPKI DER is a fixed twelve-byte header followed by +the key, so the raw form is simply the tail of the DER encoding, and both fields are derived from +one export rather than from two independent code paths that could disagree. + +
+ +
+ +#### The key ID + +```js +// src/sign/keys.mjs +const resolvedKeyId = keyId || `scrollcase-${sha256Hex(rawPublicKey).slice(0, 16)}`; +``` + +A [key ID](#key-id) is a lookup label: a document's signature says which key to try, and a verifier +finds that key in its trust file. Deriving the default from the key's own bytes makes it stable +across machines, collision-resistant in practice, and free of any registry that would have to exist +somewhere and be kept correct. `--key-id` overrides it for operators whose custody system already +names keys. + +**The ID is a hint, never an authority.** Verification looks up the key by ID and then verifies the +signature against that key's actual bytes. A document claiming a key ID it was not signed with fails +exactly like a document with a corrupt signature. + +
+ +
+ +#### Why `--force` exists, and why it is dangerous + +```js +// src/sign/keys.mjs +if (await fileExists(privatePath) && !force) { + fail(`Signing key already exists: ${privatePath}. Pass --force to rotate it explicitly.`); +} +``` + +Overwriting a signing key is a legitimate operation — keys get rotated, and a development key gets +replaced by a real one. Doing it *silently* is not, because a key has no record of what it signed. +After an accidental overwrite, every document produced with the previous key still exists, still +looks well-formed, and no longer verifies against anything the operator holds; there is no way to +enumerate the affected documents, and no way to re-sign what has already been distributed. + +::: danger `keygen --force` is not a way to fix a key mismatch +A "private and public signing keys do not match" error means the two files came from different +pairs. Regenerating makes the message go away by making a *new* identity, which invalidates every +document signed with the old one instead of repairing anything. Find the matching public key. +::: + +::: warning Private keys never leave the machine +The private key lives under `.scrollcase/keys/`, which the workspace marks as generated state. It is +never printed, never logged, never included in a box, and never committed. Nothing in Scrollcase +reads it except the local signing path, and the external-signer path never sees a private key at +all. +::: + +
+ +
+ +### 7.3 Signing with a local key + +The local path is what `keygen` makes possible: enough for development, and enough for anyone +content to hold their own key. + +
+ +
+ +#### Reading the pair back + +```js +// src/sign/keys.mjs +const privateKey = createPrivateKey(await readFile(privatePath, 'utf8')); +const publicKey = createPublicKey(privateKey); +const rawPublicKey = publicKey.export({ type: 'spki', format: 'der' }).subarray(-32); +const metadata = JSON.parse(await readFile(publicPath, 'utf8')); +if (metadata.publicKeyBase64 !== rawPublicKey.toString('base64')) { + fail('Private and public signing keys do not match.'); +} +``` + +The public half is *derived* from the private key and compared against the published file. This is +the check that catches a half-restored backup, a copied private key beside somebody else's public +file, or a rotation that replaced one file and not the other — at the start of a build, with a clear +message, rather than three minutes later when a consumer cannot verify what was produced. + +
+ +
+ +#### The envelope the signer produces + +```js +// src/sign/index.mjs +const payloadBytes = Buffer.from(`${JSON.stringify(payload, null, 2)}\n`, 'utf8'); +``` + +```js +// src/sign/keys.mjs +return { + schemaVersion: BOX_SCHEMA_VERSION, + payloadEncoding: PAYLOAD_ENCODING, + payloadBase64: payloadBytes.toString('base64'), + payloadSha256: sha256Hex(payloadBytes), + signatures: [{ + algorithm: 'ed25519', + keyId: metadata.keyId, + signatureBase64: edSign(null, payloadBytes, privateKey).toString('base64'), + }], +}; +``` + +Three properties of those twelve lines carry the whole format: + +- **The payload is serialised exactly once.** The same `payloadBytes` are hashed, signed, and + base64-encoded into the [envelope](#envelope). There is no second serialisation anywhere that could + differ by a space, and therefore no way for the published bytes to drift from the signed bytes. +- **The digest is redundant with the signature, deliberately.** `payloadSha256` lets a reader detect + a truncated or corrupted document, and lets a tool identify a payload, without holding a key. It is + a convenience and an integrity check, never a substitute for verification — which is why the + function that checks it is named for what it does and not for what it does not. +- **`edSign(null, …)`** passes no digest algorithm because ed25519 hashes internally. Passing one + would be a category error the API happens to accept. + +`signatures` is an array from the outset. One key is the normal case, but the field cost nothing to +make plural and is what makes rotation expressible at all. + +
+ +
+ +### 7.4 Verifying a signed document + +Verification lives in the same module as key generation, because a signature nobody checks is +theatre. Every code path that consumes a signed document — `build` re-verifying an external signer, +`verify`, every consumer — arrives at these two functions. + +
+ +
+ +#### Decoding without verifying + +```js +// src/sign/keys.mjs +export function decodeSignedDocument(document) { + if (document?.schemaVersion === 1) { + fail('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); + } + if (document?.schemaVersion !== BOX_SCHEMA_VERSION || document?.payloadEncoding !== PAYLOAD_ENCODING) { + fail('Unsupported signed document.'); + } + const bytes = Buffer.from(document.payloadBase64, 'base64'); + if (sha256Hex(bytes) !== document.payloadSha256) fail('Signed payload SHA-256 mismatch.'); + return { bytes, payload: JSON.parse(bytes.toString('utf8')) }; +} +``` + +Four things happen and one deliberately does not. Version 1 is refused by name rather than being +reinterpreted; the envelope constants must match; the payload is decoded; the checksum must hold. +The signature is **not** checked, and the JSDoc says so in its first line. + +This function exists because reading a document is sometimes legitimate without trusting it — +inspecting a release, displaying what a channel points at, extracting an identifier for a log line. +Making that a separate, honestly named function is safer than a single function with a `verify: false` +option, which is the shape that eventually gets called with the wrong argument. + +
+ +
+ +#### The trust file, and rotation + +```js +// src/sign/keys.mjs +function trustedKeyEntries(value) { + return Array.isArray(value?.keys) ? value.keys : [value]; +} +``` + +A [trust key](#trust-key) file is either a single key object — exactly what `keygen` writes — or a +`{ keys: [...] }` bundle. One shape would have been simpler; two mean a consumer can start with the +file the tool produced and grow into a bundle without any migration, and that a project can ship one +file listing every key it has ever used. + +```js +// src/sign/keys.mjs +const valid = document.signatures?.some((signature) => { + const key = trusted.find((candidate) => candidate.keyId === signature.keyId); + return key?.publicKeyPem + && edVerify(null, bytes, createPublicKey(key.publicKeyPem), Buffer.from(signature.signatureBase64, 'base64')); +}); +if (!valid) fail('Document has no valid signature from a trusted ed25519 key.'); +``` + +**Any one signature verifying against any one trusted key accepts the document.** That is what makes +key rotation survivable: during a rotation, documents are signed with both the outgoing and the +incoming key, holders of either trust file accept them, and the outgoing key is retired once the new +one has propagated. Requiring *all* signatures to verify would mean a consumer who has not yet +learned the new key rejects a document that was signed correctly — turning a rotation into an +outage. + +The permissiveness is bounded in the way that matters: a signature whose key ID is unknown is +ignored rather than trusted, an unparseable or non-ed25519 key contributes nothing, and a document +with no verifying signature at all fails. Adding a signature to a document can never make it *less* +acceptable, and can never make it acceptable to someone who trusts none of the signers. + +
+ +
+ +### 7.5 The external signer + +Production key custody is not Scrollcase's business. An organisation may keep its signing key in an +HSM, a cloud KMS, a signing service behind an approval workflow, or a machine no build ever runs on. +`--signer-command` hands the payload to a command the operator configures and takes back a signed +document. + +
+ +
+ +#### The exchange + +> The command receives the payload bytes on **stdin** and writes the complete signed document as +> **JSON on stdout**. + +That is the entire protocol. Any language, any credential mechanism, no plugin API to keep +compatible, no dynamic loading of somebody's code into the build process. A shell script wrapping a +cloud KMS's sign call satisfies it; so does a compiled binary talking to an HSM. + +```js +// src/sign/index.mjs +const result = runResult(executable, args, { + input: payloadBytes, + capture: true, + maxBuffer: 16 * 1024 * 1024, +}); +``` + +Three failure modes are distinguished, because "the signer did not work" is not an actionable +message: the command could not start (`result.error`), it exited non-zero (the message carries the +exit status and the trimmed stderr), or its stdout did not parse as JSON. The 16 MiB buffer is +generous for a document measured in kilobytes and bounded so that a runaway signer cannot exhaust +memory. `runResult` is the same [injection seam](#injection-seam) every other subprocess goes +through, which is how the external path is tested without an external signer. + +
+ +
+ +#### Parsing the command + +The command may be given as an array, in which case each element must be a string and the array is +used as-is — no parsing, no ambiguity. Given as a string, it is tokenised: single quotes are +literal, double quotes honour `\\` and `\"`, an unquoted backslash escapes whitespace, quotes and +itself, and an unmatched quote fails the build rather than guessing. + +A string form exists at all because the command arrives from a config file or a command-line flag, +where an array is awkward to express. The tokeniser is deliberately not a shell: it does no +expansion, no globbing, no substitution and no pipeline handling, so a payload or an environment +value can never turn into a shell construct. What it produces is an argument vector, which is then +executed without a shell. + +
+ +
+ +#### The three things a signer is not trusted about + +```js +// src/sign/index.mjs +const document = signWithCommand(payloadBytes, signerCommand, runResult); +if (document?.payloadBase64 !== payloadBytes.toString('base64')) { + fail('External signer returned a different payload than the one it was given.'); +} +// Verified against the trust anchor the operator points at, not against the signer's word. +await verifySignedDocument(document, publicPath); +return document; +``` + +1. **What it signed.** The returned `payloadBase64` must equal the base64 of the exact bytes the + signer was handed. A signer that substitutes a payload — through a bug, a re-serialisation, or + malice — fails the build. Without this check, an external signer could return a valid signature + over a *different* release and the builder would publish it. +2. **That the signature is real.** The document is verified locally against the operator's own + trusted public key file before the build continues. The signer's assertion that it signed + something is not evidence. +3. **That the key is the expected one.** Because verification runs against the trust anchor the + operator points at — the same file a consumer would use — a signer that signs with the wrong key + fails here rather than in the field. + +::: tip This is hard rule 7 in its most concentrated form +*Verify, never trust.* The external signer is the one place where Scrollcase asks something outside +itself for a security-critical result, and it is the place with the most checks on the answer. A +signer that fails any of the three produces no box at all, which is strictly better than producing a +box nobody can install. +::: + +
+ +
+ +### 7.6 The CLI edge — `cli-signing.mjs` + +Key *paths* are a project concern, and readiness is checked before anything expensive starts. + +
+ +```js +// src/cli.mjs +function keyPaths(flags) { + const keysDir = getWorkspace().keysDir; + return { + privatePath: resolve(text(flags, 'private-key') || join(keysDir, 'signing-private.pem')), + publicPath: resolve(text(flags, 'public-key') || join(keysDir, 'signing-public.json')), + }; +} +``` + +Both paths default into the [workspace](#workspace)'s key directory and are overridable per +invocation, following the same rule as every other path in the tool: the project declares where +things live, and Scrollcase never derives a location from its own position on disk. + +`ensureBuildSigningKeys` then runs as a **read-only preflight** before `build` does any work: + +| Situation | Result | +| --- | --- | +| External signer, trusted public key present | Proceed | +| External signer, no public key | Fail — the key that verifies the signer is required | +| Local path, both files present | Proceed | +| Local path, exactly one file present | Fail — refuses to replace the existing key | +| Local path, neither present | Fail — run `keygen` first | + +Every failing case is a refusal, never a repair. `build` creating or rotating identity material would +mutate the project before [provenance](#provenance) has even been established, and could silently +change the identity under documents already published. The one-file-present case is the sharpest: +the obvious "helpful" behaviour is to regenerate the missing half, which is impossible for a public +key and catastrophic for a private one. Refusing is the only correct answer. + +
+ +### 7.7 What signing deliberately does not do + +The absences here are the same boundary the rest of the tool draws, applied to cryptography. + +
+ +- **No second algorithm.** One curve, one signature format, one verification path. A second + algorithm would double the surface every consumer implementation must get right, to buy nothing + the first one does not already provide. +- **No revocation checking.** The [revocations](#revocations) document has a defined shape because a + publishing project needs one; Scrollcase never fetches, consults or enforces it. Deciding that a + release is no longer acceptable is a distribution policy, and distribution is outside the boundary. +- **No timestamping or countersignature policy.** No trusted time source, no notary, no threshold + rules about how many signatures constitute acceptance. A project that needs those builds them on + top of an envelope that already carries a signature array. +- **No key distribution.** How a consumer obtains the trusted public key — bundled in an installer, + pinned in an application, fetched from a well-known URL — is the project's decision. Scrollcase + takes a path to a file. +- **No encryption.** Signatures establish authenticity and integrity, not confidentiality. A box + travels in the clear; anyone who wants it private encrypts the transport or the storage. +- **No key escrow, backup or recovery.** Scrollcase writes a key file with restrictive permissions + and stops. Where it is backed up, and who can use it, is custody — which is the operator's, which + is the whole reason the external signer exists. + +## 8. The consumers + +Everything up to here produces a box. This section is about the other end: a machine that holds a +signed [release](#release) document, a trusted public key, and either an archive plus destination or +an already-extracted root, and wants a working Python environment whose identity it can establish. + +Scrollcase ships **three** implementations of that — in Node, in Python, and in Rust — because the +code that consumes a box usually is not the code that built it. They are not an original and its +ports; they are three mirrors of one contract, and they are held to it by the same fixtures. + +
+ +### 8.1 Three implementations, one contract + +
+ +#### The parallel surfaces + +| Concern | Node — `scrollcase/consumer` | Python — `scrollcase_consumer` | Rust — `scrollcase-consumer` | +| --- | --- | --- | --- | +| Verify and prepare | `verifyAndExtractBox()` | `verify_and_extract_box()` | `verify_and_extract_box()` | +| Re-attach an extracted box | `attachExtractedBox()` | `attach_extracted_box()` | `attach_extracted_box()` | +| Verify an installed payload | `verifyExtractedPayload()` | `verify_extracted_payload()` | `verify_extracted_payload()` | +| Execute a prepared box | `runExtractedBox()` | `run_extracted_box()` | `run_extracted_box()` | +| One-shot run | `runBox()` | `run_box()` | `run_box()` | +| Trust source | `publicPath` or `trustedKeys` | `public_key_path` or `trusted_keys` | `TrustAnchors::KeyFile` or `::Keys` | +| Receipt | frozen `PreparedBox` object | frozen `PreparedBox` dataclass | `PreparedBox` with private fields | +| Failure | `fail()` → `Error` | `ScrollcaseConsumerError` | `fail!()` → opaque `Error` | +| Private state binding | `WeakMap` | `weakref.WeakKeyDictionary` | private fields, no public constructor | +| Process seam | `spawn` option | `popen_factory` argument | `SpawnBox` trait | +| Signal seam | `signalSource` option | `signal.signal` on the main thread | a channel the caller owns | +| Schemas | read from `src/contract/schema/` | bundled copies, checked by `sync_schemas.py --check` | bundled copies used by the tests, checked by `sync-assets.mjs --check` | +| Dependencies | `yauzl` for reading archives | `cryptography`, `jsonschema` | `ed25519-dalek`, `zip`, `sha2`, `serde`, `base64` | + +The Python package is distributed separately (`scrollcase-consumer` on PyPI, requiring Python 3.10 or +newer), ships `py.typed`, and is checked under `mypy --strict`. It depends on `cryptography` for +ed25519 and `jsonschema` for schema validation, and on nothing else; ZIP reading uses the standard +library's `zipfile`. + +The crate is distributed separately too (`scrollcase-consumer` on crates.io, requiring Rust 1.88 or +newer). It forbids `unsafe`, is synchronous throughout so an application chooses its own runtime or +none, and — being a library embedded in someone else's process — installs no signal handler of its +own. + +All three take their trusted keys from a file **or** from the caller directly, and for the same +reason: an application that holds its keys in a keyring, an environment variable or a secrets +manager should not have to write key material to disk to check a signature. Node and Python refuse +both sources or neither rather than resolving by preference — a caller that named two has not +decided which keys it trusts — while Rust's `TrustAnchors` enum makes both invalid states +unrepresentable. In every implementation the named source is resolved once, at the entry point, and +everything below it sees one list of keys; supplying them directly is not a second verification path. + +The parser contract is shared too. A trust source is one key object or `{ "keys": [...] }`; every +entry needs a string `keyId`, while `publicKeyPem` may be absent or `null` and otherwise must be a +string. An empty bundle parses and then cannot verify any signature. Malformed JSON, bundle shapes +or entries produce `Invalid trusted ed25519 key file.` in all three; an unusable PEM is skipped and +reaches the common no-valid-signature error, never a raw crypto-library exception. Node and Python +additionally validate directly supplied arrays and report `Invalid trusted ed25519 keys.`; Rust's +typed `Vec` makes the corresponding malformed field types impossible to construct. +An unreadable trust file keeps the same error prefix and adds its path and the I/O detail. These +cases live in `consumer-conformance.json`, not in three independent readings of the rule. + +What differs is what that buys. In Rust it additionally closes a chain the format cannot close from +the inside: the crate is compiled into an application handed to someone else, a trust file beside +that application is editable by whoever holds the machine, and editing it, signing a box with the +substituted key, and having the application accept the result is otherwise a complete attack. +Anchors compiled in with `include_str!` move that decision into the binary. The same trick buys +Node and Python far less, because there is no binary: a hard-coded key sits in a source file the +attacker can edit exactly as easily as the trust file. Where Node and Python validate a release against the canonical schemas at run time, the crate +encodes those schemas as types that refuse an unknown field wherever the schema is closed — and, in +the one object it is not, `compatibility`, keep what they do not recognise instead. `rust/tests/schema.rs` +then proves the types and the schemas still agree, with `jsonschema` as a development dependency that +never reaches a consumer. That equivalence has to be checked in both directions: a typed parse +drifting *stricter* than the schema refuses documents the format defines as valid, which is how the +crate once came to reject a project's own compatibility constraint. + +
+ +
+ +
+ +#### Why three, and not a binding + +A native binding, or a subprocess call into the Node implementation, would make one runtime a +dependency of the others. A Python application that wants to run a box would have to ship Node; a +Node application would have to ship Python; a Rust desktop client would have to ship both to avoid +writing either. All are unacceptable for the situation these boxes exist to serve, where the point +is a self-contained artefact with a short dependency list. + +**Rejected:** a shared native core through FFI. It would replace three readable implementations of a +few hundred lines each with a build matrix, a packaging problem per platform, and a class of bug no +language's tooling can see. What holds them honest is not shared code — it is +[section 8.7](#_8-7-the-shared-conformance-fixture)'s shared fixture, plus schemas copied from one +canonical source by a checked step. + +
+ +
+ +### 8.2 The fixed verification order + +Nothing from inside a box runs until the complete trust chain has passed. The order is not an +implementation detail; it is part of the contract, and every consumer follows it. + +
+ +```text + 1 signed document schema, signature, payload digest + 2 release manifest schema, schemaVersion 2, kind + 3 target adapter resolved, declared interpreter path + 4 archive located, size, SHA-256 + 5 every archive entry safe path, kind, collisions, links + 6 box.json read, schema, agreement with the release + 7 execution script exists, or module is discoverable + ---------------------------------------------------------------- read only + 8 extract into a staging directory beside the destination + 9 on-demand assets verified by size and digest + 10 spawn the box's own interpreter +``` + +Two properties of that order are worth stating outright. **Everything cheap that can reject comes +first** — a bad signature costs a few milliseconds to detect, and there is no reason to hash a +multi-gigabyte archive before finding out. And **extraction is late**: no byte is written outside a +staging directory until every property of the archive has been established from the archive itself. + +
+ +#### Why the Node consumer reuses the builder's inspection + +The Node consumer calls `inspectReleaseDocument()` and `inspectBoxArchive()` from +`src/build/verify.mjs` — the same functions `scrollcase verify` uses, described step by step in +section 6.15. Attachment and payload verification stop after the document half; preparation +continues through the archive half. That is a deliberate coupling. +Adding an execution API must not create a second, subtly different interpretation of a signed +release: the moment there are two, they drift, and the difference between them is a security bug +nobody is looking for. + +The cost is that the consumer module graph reaches into `src/build/`. That was accepted because the +alternative — copying fifteen ordered checks — is exactly the failure this project's contract rules +exist to prevent. + +
+ +
+ +#### The Python mirror — `_contract.py` + +Python cannot import those functions, so `_inspect_release_document` and `_inspect_box_archive` +mirror the same split step for step, and `_contract.py` mirrors the behaviour schemas cannot +express: the three +[target adapters](#target-adapter), the [target ID](#target-id) rule, `safe_relative_path`, static +execution discovery, and the [symbolic link](#symbolic-link) rule with the same +`MAX_PAYLOAD_LINK_DEPTH = 8`. + +The module's docstring states the boundary it lives inside: *schemas stay canonical in +`src/contract`*. What is mirrored in code is only what a schema cannot check at runtime. Everything +else is a **copy** of the canonical schema file, placed by `python/scripts/sync_schemas.py`, and +`--check` fails the test suite when a copy drifts from its source. Five schemas travel this way: +`signed-document`, `release-manifest`, `box-manifest`, `target` and `execution` — the consumer's +half of the eight. + +The mirror is proved rather than asserted. `python/tests/test_contract.py` checks the Python target +rules against `fixtures/target-id-contract.json`, the same golden file every other implementation +answers to. + +
+ +
+ +### 8.3 Preparing a box + +`verifyAndExtractBox` / `verify_and_extract_box` turns a release document, an archive and a +destination into a verified directory on disk — and executes nothing. + +
+ +
+ +#### The staging dance + +```js +// src/consumer/verify-and-extract.mjs +const stageRoot = await mkdtemp(join(parent, `.scrollcase-prepare-${basename(finalRoot)}-`)); +const extractedRoot = join(stageRoot, 'payload'); +``` + +The staging directory is created **beside the final destination**, not in the system temporary +directory. That is what keeps the final `rename` on one filesystem, which is what makes it atomic: +an observer sees either no destination at all or the complete verified tree, never a half-extracted +one. Extracting into `/tmp` and moving would degrade into a copy across a device boundary, and a +copy has an observable middle. + +The sequence inside the `try` block, in order: + +1. Extract the archive into the staging directory. +2. Compare the extracted [payload](#payload) size against the release's `installedSizeBytes`. +3. **Re-hash the source archive** and compare it to the release again. +4. `lstat` the staged root. +5. Check once more that the destination does not exist. +6. `rename` the staged payload onto the destination. +7. `lstat` the destination and require the same device and inode as the staged root. + +Step 3 is not paranoia about the same file twice. The archive was hashed to make the trust decision; +between that moment and the tree landing in the caller's durable destination, a local file can be +replaced. Re-checking closes the window in which a swapped archive would be extracted under a +verification that no longer describes it. + +Step 7 catches the mirror-image problem at the other end: a destination that was substituted between +the last existence check and the rename is no longer the object that was just verified, and is +refused rather than returned as prepared. + +The destination is checked for existence **three times** — before inspection, after its parent is +created, and immediately before the rename. This narrows the race window; it does not eliminate it, +and it is not claimed to. What it does eliminate is the far larger hole of renaming onto whatever +happens to be there. + +A `finally` removes the staging root on every path, so a failure at any step leaves no partial tree +and no temporary directory behind. + +
+ +
+ +#### The opaque receipt + +A prepared box is represented by a receipt that is deliberately *not* reconstructible: + +```js +// src/consumer/verify-and-extract.mjs +const preparedBoxes = new WeakMap(); +``` + +The receipt itself is public and useful — status, identity, version, [target](#target), target ID, +interpreter path, execution metadata, required assets, signing key IDs, the signed-document payload +digest, the archive digest and size, the measured installed size, and a masked environment report +for the verifying process. It is frozen recursively, so a +caller cannot mutate what was verified. `status: 'prepared'` says the directory came from an archive +whose signed hash was checked in this process; `status: 'attached'` says an existing directory was +re-identified without proving its payload bytes. The type carries both values because the two +producers do not make the same assertion. + +What the receipt does *not* contain is the verified release and the identity of the extracted root. +Those live in a `WeakMap` keyed by the exact receipt object, reachable only through +`preparedBoxState()`, which is internal to the consumer module graph and is not re-exported from the +package surface. The consequence is the point: + +::: warning A hand-built object cannot be executed +Passing `{ status: 'prepared', root: '/somewhere/else' }` to `runExtractedBox` fails with *Expected +a PreparedBox returned by verifyAndExtractBox() or attachExtractedBox()*. Execution authority comes +from having gone through one of those checked producers, not from having an object that looks like +their result. +::: + +Python reaches the same property differently, and the difference is instructive: + +```python +# python/src/scrollcase_consumer/models.py +@dataclass(frozen=True, eq=False) +class PreparedBox: +``` + +`eq=False` keeps the default identity-based equality and hashing. A `WeakKeyDictionary` therefore +keys on the *instance*, so a structurally identical copy of a receipt is a different key and carries +no authority. Had the dataclass used the usual field-based equality, a caller could have constructed +an equal object and found it accepted — the exact hole the Node `WeakMap` closes by object identity. +`prepared_box_state` also rejects anything that is not a `PreparedBox` at all. + +
+ +
+ +### 8.4 Executing a prepared box + +Preparation proves what a box is. Execution re-establishes that the box is still what was prepared, +and only then starts a process. + +
+ +
+ +#### What is re-checked before the interpreter starts + +| Check | Why it is repeated here | +| --- | --- | +| Release declares an `execution` entry point | A library-only box prepares successfully and has nothing to run | +| Native host matches the target adapter | A Linux box cannot run on macOS; the message names both | +| Root is a directory with the recorded device and inode | The prepared tree may have been replaced or removed since | +| Interpreter path is present in the extracted tree | The tree on disk, not the archive that produced it | +| Execution script exists, or module is discoverable | Same static rule the builder and `verify` apply | +| Every on-demand [asset](#asset): present, regular file, exact size, exact digest | The caller materialised them; they were never verified in place before | + +The root identity check is the one that is easy to leave out. Without it, preparing a box and +running it later would trust that nothing swapped the directory in between — which on a shared +machine is precisely the assumption an attacker wants. + +::: warning On-demand assets are verified, never fetched +When [weights mode](#weights-mode) is `on-demand`, the release carries signed descriptors and the +receipt exposes them as `requiredAssets`. The caller places those bytes under the box root — often +in an `onPrepared` callback. The consumer then checks each one's size and SHA-256 against the signed +descriptor before spawning anything, and refuses to run if any is missing, is not a regular file, or +does not match. Downloading them is the caller's job, always. +::: + +
+ +
+ +#### Building the argument vector + +```js +// src/consumer/run-extracted.mjs +const executionArgs = release.execution.kind === 'python-script' + ? [join(prepared.root, ...safeRelativePath(release.execution.script).split('/'))] + : ['-m', release.execution.module]; +executionArgs.push(...release.execution.defaultArgs, ...callerArgs); +``` + +The vector is `[interpreter, script | -m module, ...signed default arguments, ...caller arguments]`, +the working directory is the box root, and `shell` is `false` in both implementations. + +Two decisions are encoded in that ordering. **The signed defaults come first**, so a caller can +append to what the publisher declared but cannot displace it. And **no shell is involved**, so a +caller argument containing `$(…)`, `;`, a quote or a newline arrives at the process as one literal +argument. The conformance suite asserts exactly that with an argument of `$(touch never)`: if any +shell ever crept into the path, the case fails and names the file that should not exist. + +The environment keeps three provenance layers: the current process, caller values, and the signed +release declaration, in that precedence order. Nothing is filtered. Windows names are matched +case-insensitively, and the release therefore wins even when the host spells `Path` differently. +The three standard streams default to `inherit` in Node and to the caller's handles in Python. + +The resolver returns the exact child environment and a structured diagnostic. Its compact form +contains signed declarations, inherited variables capable of changing executed code, conflicts and +their winner, and a count of omitted names. `envReport` / `env_report` expands all names; +`envReportValues` / `env_report_values` reveals inherited host values. Verification and attachment +receipts carry a host-plus-release snapshot; execution recalculates it with the caller layer. The +declaration is format. The report is local consumer output and never a box guarantee. + +
+ +
+ +#### Signals and terminal results + +Both implementations forward `SIGINT`, `SIGTERM` and `SIGHUP` to the child while it is alive, and +both undo that at the same point the result settles. + +Node registers handlers on an injectable `signalSource` (defaulting to `process`) and removes every +one of them in a `cleanup` called from both the `error` and `close` paths — a handler that outlived +its child would keep a dead reference and forward a later signal to nothing. Python installs +handlers only when running on the main thread, because installing a signal handler from a worker +thread raises, and restores the previous handlers in a `finally` regardless of how the wait ended. + +The terminal fields are the same in both: `{ exitCode, signal, environmentReport }`, exactly one of +`exitCode` and `signal` being non-null. +Python derives it from the negative return code convention and converts it back to a signal name, so +callers see `SIGTERM` rather than `-15`. + +The result is returned **unchanged**. A non-zero exit is not an error, and a signal is not a +failure — they are the application's terminal semantics, and translating them into a Scrollcase +success/failure convention would destroy information the caller needs. The conformance suite pins +this with a case that expects exit code 23 to arrive as exit code 23. + +
+ +
+ +### 8.5 One-shot execution + +`runBox` / `run_box` is preparation, execution and removal in a single call, for callers who want to +run a box without installing it. + +
+ +```js +// src/consumer/run-box.mjs +const temporaryRoot = await mkdtemp(join(temporaryParent, 'scrollcase-run-')); +try { + const prepared = await verifyAndExtractBox(releaseDocumentPath, { …, destination: join(temporaryRoot, 'box') }); + await options.onPrepared?.(prepared); + return await runExtractedBox(prepared, options); +} finally { + await rm(temporaryRoot, { recursive: true, force: true }); +} +``` + +It composes the two public operations rather than reimplementing either, which is why it is under +fifty lines. The `onPrepared` hook exists for exactly one purpose: an on-demand box needs its assets +placed into the extracted root after preparation and before execution, and that is the only moment +at which the root path is known. + +The `finally` owns cleanup for **every** terminal path — normal exit, non-zero exit, a spawn that +never started, a forwarded signal. Three conformance cases exist solely to prove that the temporary +directory is empty afterwards in the success, spawn-failure and signal cases, because a cleanup that +only runs on the happy path is how temporary directories full of multi-gigabyte environments +accumulate. + +
+ +### 8.6 Defensive extraction on the Python side — `extract.py` + +Section 6.12 covered the archive reader the builder and the Node consumer share. Python needs its +own, and it is written to the same rule: **classify every entry before writing any byte**, and never +delegate a security decision to whichever platform happens to be running. + +
+ +The refusals, in the order they can occur: + +| Condition | Rejected because | +| --- | --- | +| Encryption flag set (`flag_bits & 0x1`) | A box is never encrypted; an encrypted entry is either corrupt or a probe | +| Link target longer than 1024 bytes | A real target is a file name; anything near a path limit is corrupt or hostile | +| Entry name that is absolute, contains `..`, an empty segment, a NUL, or a drive letter | [Path traversal](#path-traversal) | +| Any type that is not file, directory or symlink | Devices, FIFOs and sockets have no place in a payload | +| Duplicate path, entry under a path already seen as a file, or a file at a path used as a parent | An archive that describes two different trees | +| A link that does not resolve to a regular file inside the payload | The link rule, re-applied to the archive as received | +| Any entry whose path passes through a link | The classic way an archive writes outside the directory it was extracted into | + +Extraction then writes with `mkdir(exist_ok=False)` and `open("xb")` — exclusive creation +throughout, so nothing existing is ever overwritten — re-checks each entry's byte count as it +streams, writes symlinks as the validated relative string without ever resolving them, and applies +the recorded mode on POSIX platforms. Afterwards `validate_extracted_tree` walks the result and +rejects any special node that materialised anyway. + +Two smaller decisions in the same file are easy to miss and load-bearing. `read_zip_entry` is +bounded at 1 MiB and checks both the declared size and the bytes actually read, so a lying header +cannot make a metadata read expensive. And `payload_size` uses `lstat`, counting a link as its own +few bytes rather than the size of its target — counting targets would restore on paper exactly the +duplication that carrying links removes from disk, and this number is what a caller checks free +space against. + +
+ +### 8.7 The shared conformance fixture + +Three implementations agreeing today is worth little; what matters is that they cannot silently +diverge tomorrow. `src/contract/fixtures/consumer-conformance.json` is how that is enforced. + +
+ +
+ +#### What is in the file + +Sixty-seven cases and twenty-eight error patterns, in a language-neutral JSON document. Each case is +a small declarative record: + +```json +{ + "id": "shell-metacharacter-preservation", + "action": "run-prepared", + "runtime": { "args": ["$(touch never)", "semi;colon", "quote'\"value"], "exitCode": 0 }, + "expected": { + "outcome": "completed", + "argv": ["$BOX/$NATIVE_PYTHON", "$BOX/app/main.py", "--default", "value with spaces", + "$(touch never)", "semi;colon", "quote'\"value"], + "cwd": "$BOX", + "shell": false + } +} +``` + +`action` is one of `prepare`, `attach`, `verify-payload`, `run-prepared` or `run-box`. `fixture` +selects the box to build (signer, target, execution kind, whether an on-demand asset or payload +digest is declared), `mutation` names a single named corruption to apply, and `runtime` supplies +arguments, an exit code, a signal, stream handling, a spawn failure, an asset state, or the request +to re-attach before a prepared run. Both harnesses reject an unknown action explicitly; it cannot +fall through into `run-box` and pass as the wrong operation. + +`requiresSymlinks` marks the one thing a host may be unable to do: Windows boxes are link-free and +creating a link there needs elevation, so cases that depend on one are skipped rather than +weakened. + +The three tokens keep a case both exact and portable: `$NATIVE_PYTHON` and `$NATIVE_TARGET` expand +to the running host's interpreter path and target ID, and `$BOX` to the prepared root. A case can +therefore assert a complete absolute argument vector without hard-coding a platform or a temporary +path. + +Note that the fixture's own `schemaVersion: 1` is the *fixture format's* version. It is unrelated to +the box format's `schemaVersion: 2`. + +
+ +
+ +#### What the cases cover + +| Group | Cases | What is pinned | +| --- | --- | --- | +| Valid preparation | 4 | Local and external signing paths both produce the same receipt; an interpreter reached through a payload link is accepted; a `compatibility` constraint the format does not define is carried rather than refused | +| Tampering | 6 | Altered signature, altered payload, altered archive bytes, altered size, release/`box.json` disagreement, altered execution metadata | +| Missing pieces | 3 | Absent interpreter, absent script, undiscoverable module | +| Hostile archives | 7 | Traversal, absolute path, escaping link, special entry, encrypted entry, duplicate entry, file/directory collision | +| Destination safety | 1 | An existing destination is refused and left untouched | +| Per-target entry points | 3 | macOS, Linux and Windows receipts, including `venv/python.exe` | +| Execution semantics | 6 | Persistent root survives, argument ordering, shell metacharacters, stream forwarding, non-zero exit, signal forwarding | +| Temporary cleanup | 3 | Empty afterwards on success, on spawn failure, on signal | +| On-demand assets | 3 | Missing, wrong size, wrong digest — each refused before spawning | +| Valid attachment | 5 | Existing roots, attach-then-run, a linked interpreter, materialised assets, and unrelated extra files | +| Attachment refusal | 8 | Missing/file/link roots, missing interpreter or script, foreign target, and missing or wrong-hash assets | +| Installed-payload verification | 11 | Match; extra files, mode and mtime ignored; tampered, deleted or retargeted entries refused; on-demand assets ignored; absent, missing or altered digest commitments refused | + +The three per-target **preparation** cases run on any host because preparation has no native-host +requirement. Attachment and execution do: the foreign-target attachment case pins that distinction. +Keeping those questions separate lets a single machine prove all three interpreter layouts without +pretending it can mint an executable receipt for all three. + +
+ +
+ +#### Error patterns, not error strings + +```json +{ + "errorPatterns": { + "archive-hash": "Archive SHA-256 mismatch", + "unsafe-path": "Unsafe relative path|invalid relative path|absolute path", + "link-entry": "link does not resolve to a file inside the payload|…|link target is too long" + } +} +``` + +The `link-entry` pattern above is abridged; in the file it carries a third alternative, for an entry +written *through* a link. + +A case asserts *which* failure occurred, not how it was phrased. Each harness matches the raised +message case-insensitively against the patterns and reports the matching code; a message that +matches nothing becomes `unclassified: `, which fails the comparison and prints the +text, so an unexpected failure is loud rather than silently mapped onto the wrong code. + +Classification is first-match-wins, so new expressions are checked against the messages all older +cases already produce. The payload-entry expression is anchored at the start, for example, and +therefore cannot steal *Extracted payload size does not match the signed release* from its existing +classification. + +**Rejected:** requiring byte-identical messages in every language. It would force one language's +phrasing on the others, make any wording improvement a cross-language breaking change, and prove +nothing extra — what matters is that all of them refuse the same input for the same reason. + +
+ +
+ +#### The three harnesses + +Nothing is shared between the harnesses but the JSON. `tests/helpers/consumer-conformance.mjs` +builds its fixture box with `yazl` and mutates real ZIP bytes — flipping the encryption bit in both +the local and central headers, rewriting entry names in place under a byte-length constraint so +offsets stay valid. `python/tests/conformance_support.py` builds an equivalent fixture with +`zipfile` and its own `ArchiveEntry` records. `rust/tests/support/mod.rs` builds a third with the +`zip` crate and patches the central directory itself. Each drives its own consumer through its own +fake process factory, then compares its observed result with the *same* expected object. + +That independence is the point. A shared harness would let one bug hide in every language; three +harnesses agreeing on one expectation file is evidence about the contract rather than about the test +code. It has already paid for itself: bringing the Rust consumer to the file surfaced two real +defects — an archive naming one path twice, whose duplicate the `zip` crate collapses before a +reader can see it, and a linked interpreter that was refused on attach and then sized as though it +were nothing. + +A third was found later, and by then only a case in this file could have caught it: the Rust +consumer refused a release carrying a `compatibility` constraint the format does not define, which +the schema allows, the builder copies through, and the other two consumers accept. A divergence in +what an implementation *accepts* leaves no error message behind in the languages that behave, so +nothing but a shared case that expects success can pin it. `unknown-compatibility-constraint` is +that case. + +
+ +
+ +### 8.8 The CLI's `run` — `cli-run.mjs` + +`scrollcase run` is a thin edge over the Node consumer, not a third implementation. + +
+ +```js +// src/cli-run.mjs +if (result.signal) terminate(result.signal); +else setExitCode(result.exitCode ?? 1); +``` + +The CLI dispatch prints a blank separator and `Preparing box for execution` before calling this +module, so verification and extraction never look like a hung command. The module adds two things +and nothing else: another blank separator plus two status lines printed from the `onPrepared` hook, +and a translation of the child's terminal result into this process's own. A child killed by a +signal makes the CLI kill *itself* with the same signal, so a shell sees the real termination cause +rather than an invented exit code; otherwise the child's exit code becomes the CLI's. + +Both lines go to **stderr**, because stdout belongs to the box. Every other verb owns its standard +output; `run` hands it to the application, and a status line written there would land inside +whatever file or process the caller piped that output into, with the box unable to tell. The second +line states what `run` is — a one-shot extraction, deleted on exit — and prints on every run rather +than above a size threshold, because a caller who does not know that reads a repeated +multi-gigabyte extraction as the tool being slow. Each write is awaited before execution begins; +otherwise the parent stderr and the box stdout can be displayed out of order when either stream is +piped. A box kept across runs is `verifyAndExtractBox` once, then `attachExtractedBox` and +`runExtractedBox` from the library, not this verb. + +Verification, extraction, execution, signal forwarding and cleanup all stay in `runBox`. The +injectable `run`, `log`, `setExitCode` and `terminate` parameters exist so the translation can be +tested without terminating the test runner. + +`verify` follows the same launch convention on stdout: a blank line and `Verifying box` — or +`Verifying extracted payload` — are flushed after argument validation and before any potentially +long read or hash. Unlike `run`, it owns stdout because no application process owns that stream. + +
+ +### 8.9 Living past the process that installed the box + +A `PreparedBox` is bound to the process that made it, and that binding is the point: a receipt a +caller could write out and read back would be a forgeable execution capability. But an application +that installs a box once and runs it for months restarts, and re-extracting gigabytes at each launch +is not an answer. So the receipt is not serialised — it is *earned again*. + +
+ +
+ +#### `attachExtractedBox` — a receipt without an archive + +Attachment performs every check that needs no data beyond the signed release: signature and schema, +a target this host can run, the interpreter and execution files present, the signed hashes of +on-demand assets, and the root's device and inode captured for `runExtractedBox` to re-check. It +reads no original payload file contents: it enumerates paths and measures their metadata, so an +embedded five-gigabyte weight does not add five gigabytes of work. Required on-demand assets are the +exception and are hashed in full against their separate signed descriptors. + +Two asymmetries with preparation are deliberate: + +| | `verifyAndExtractBox` | `attachExtractedBox` | +| --- | --- | --- | +| Native host | not required — preparation only writes files | **required** — a receipt minted here exists to be executed | +| Root must | not exist | exist, and be a real directory, never a [symbolic link](#symbolic-link) | +| `status` | `prepared` | `attached` | +| `installedSizeBytes` | compared with the signed figure | measured, never compared | + +The last row is the receipt refusing to overstate itself. An installed tree legitimately grows — +on-demand assets, caches, whatever the application writes in its working directory — so holding it +to the signed figure would fail honest boxes. And `status` distinguishes the two because they do not +prove the same thing: `prepared` means the bytes came from an archive whose signed hash was checked +here; `attached` means a directory was re-identified against a release, and no more. + +
+ +
+ +#### `verifyExtractedPayload` — the separate, opt-in question + +Proving the installed bytes is a different decision with a different cost, so it is a different +call. Nothing invokes it — not attachment, not execution — and section 5.5 describes the list it +reads. Its guarantee is worth stating exactly, because it is narrower than it looks: + +- **It does** bind a directory to a signed release. Without it, an application handed a stale + install, a rollback directory, or a half-deleted tree would get a receipt confidently asserting + the wrong `version`. And it detects ordinary corruption — an interrupted extraction, a full disk, + a quarantined file. +- **It does not** defend against a local attacker. The tree can change between this call and any + later import, and Python imports lazily for the whole life of a process. No consumer library can + close that window; filesystem permissions can, and they belong to the operating system and the + embedding application. Scrollcase does not guard the directory afterwards. +- **It cannot see** `__pycache__` or `*.pyc` at all. The entry collector excludes them by name, so a + stale or hostile compiled module shadowing its source is invisible to the digest permanently, not + merely between check and use. + +A release built before the digest existed is refused by name rather than reported as verified. A +box that carries no commitment must not be mistaken for one that satisfies it. + +The weights mode changes the cost honestly. Embedded weights are payload entries and verification +reads all of them, which can mean tens of gigabytes. On-demand assets were absent when the list was +built and appear later as ignored extras; their integrity is covered separately by the signed +per-file `requiredAssets` descriptors that attachment and execution enforce. Mode and modification +time are also outside the digest, because archive writing synthesises modes and extraction restores +neither the build mode consistently across platforms nor the fixed build timestamp. + +
+ +
+ +### 8.10 What the consumers deliberately do not do + +Every input is local and caller-selected. The list of what that rules out is the boundary from +section 3, restated where it is most tempting to cross. + +
+ +- **No downloading.** Not the archive, not the release document, not the trust key, not on-demand + assets. The consumer verifies bytes it is given. +- **No channel selection.** A [channel](#channel) document points at a release; choosing which one to + install is a distribution decision, and nothing in the consumer reads a channel at all. +- **No revocation lookup.** See section 7.7. +- **No installation lifecycle.** No update, no rollback, no version comparison, no garbage + collection of old boxes, no registry of what is installed. `verifyAndExtractBox` produces one + directory and `attachExtractedBox` re-identifies one it is handed; which directory, and what + becomes of it, stays the caller's. +- **No policy about failure.** A failed verification raises; it does not retry, fall back to a + cached copy, or continue in a degraded mode. There is no "verify if possible" setting, because a + check that can be skipped is not a check. + +::: info The consumer's whole promise +Given caller-supplied local inputs, a consumer either returns the precise prepared, attached, or +payload-verification result the chosen operation promises, or it refuses with a clear error. Box +code runs only from an authentic process-bound receipt, after that execution path's trust checks +have passed. +::: + +## 9. The command line + +
+ +### 9.1 One file, and what it is allowed to do + +`src/cli.mjs` is the whole command line: the `bin` entry `scrollcase` points at it, and it is the +only executable the package ships. Everything it does falls into four categories — parse arguments, +resolve the [workspace](#workspace), ask a human a question, print a line — and everything else is +delegated to a module that could equally be called from a program. + +
+ +That boundary is the reason the tool is usable as a library. A verb is a few lines of glue over a +function that takes explicit inputs: + +```js +// src/cli.mjs +async function verify(path, flags) { + await verifyBox(path, { + publicPath: keyPaths(flags).publicPath, + archive: text(flags, 'archive'), + selfTest: Boolean(flags.get('self-test')), + }); +} +``` + +**Rejected:** letting the modules beneath read `process.argv`, `process.env` or the terminal +directly. It would have removed this file almost entirely, and it would have made every one of those +modules untestable without a fake terminal and unusable from a pipeline that has no terminal at all. +Consent, choice and presentation are injected downward; nothing reaches back up for them. + +
+ +#### The shape of `main()` + +```text + argv + │ + ├─ -v | --version ──────────────► print the package version, exit (no workspace) + │ + ├─ parseArgs(rest) ────────────► { positional, flags, passthrough } + │ + ├─ (no command) | help | --help ► print usage, exit (no workspace) + │ + ├─ configureWorkspace(overridesFromFlags) ◄── every path below comes from here + │ + └─ dispatch on the verb ────────► init | new | doctor | keygen | lock + audit | build | verify | run + │ + unknown ──┴──► fail() + + main().catch(error) ─────────────► "✗ scrollcase: " on stderr, exit code 1 +``` + +Three details in that order are deliberate. + +**The version shortcut runs before anything else**, including argument parsing and workspace +resolution. `scrollcase --version` therefore answers from any directory, including one that contains +no project at all — which is exactly the situation an installer script or a diagnostic report is in +when it asks. `tests/unit/cli-version.test.mjs` runs the real binary from the system temporary +directory for that reason. + +**The workspace is configured once, before any verb touches a path.** `workspaceOverridesFromFlags` +turns `--config`, `--project-root`, `--scrolls-dir`, `--build-dir`, `--out-dir`, `--keys-dir` and +`--toolchain-dir` into overrides, and those beat the project's `scrollcase.config.json`. A verb never +resolves a path itself, so a single invocation cannot half-use two workspaces. + +**There is exactly one failure path.** Every `fail()` in the codebase throws, and every throw lands +in the same handler: + +```js +// src/cli.mjs +main().catch((error) => { + console.error(statusLine('error', `scrollcase: ${…}`, { stream: process.stderr })); + process.exitCode = 1; +}); +``` + +One line on stderr, a non-zero exit code, no stack trace. A shell or a CI step can rely on the status +without parsing anything, and a contributor adding a check does not have to decide how it should be +reported. + +
+ +
+ +### 9.2 The nine verbs + +The set is closed and small. Each verb is one of the phases of a box's life, and the ones that cost +minutes are separated from the ones that cost milliseconds so that a failure is cheap. + +
+ +| Verb | Does | Reads | Writes | Network | +| --- | --- | --- | --- | --- | +| `init` | Scaffold a workspace, optionally an example and the consumer templates, then offer the dependencies | Host, existing files | `scrollcase.config.json`, `scrolls/example-box/…`, `consumer-templates/…`, optionally the toolchain | Only with consent | +| `new scroll` | Author one complete target-specific [scroll](#scroll) | Flags or prompts | `scrolls///` | No | +| `doctor` | Report whether this machine can build | Workspace, git, pixi, conda-pack | Nothing, ever | No | +| `keygen` | Create a local [ed25519](#ed25519) signing key | Existing key files | `signing-private.pem`, `signing-public.json` | No | +| `lock` | Resolve the scroll's pixi manifest | `scroll.json`, `pixi.toml` | `pixi.lock` | Yes — this is where solving happens | +| `audit` | Dependency licence inventory from the lock | `pixi.lock` | Optionally the reviewed audit | No | +| `build` | Solve-free install, self-test, archive, sign | The scroll, the lock, the keys | The [payload](#payload), the archive, the signed documents | Yes — package and asset downloads | +| `verify` | Check an archive, or an existing extracted payload | A [release](#release) document, a [trust key](#trust-key), and either an archive or extracted root | Nothing (a temporary tree with `--self-test`) | No | +| `run` | Verify, extract temporarily, execute | The same three inputs | A temporary extraction, removed afterwards | No | + +Two properties of that table matter more than any individual row. + +**Only `lock` and `build` reach the network**, and both are explicit human actions on a named scroll. +Nothing else in the tool downloads anything without being asked to — `init` only after a terminal +consent, `doctor` never, and the two consumer verbs never at all. + +**`doctor` writes nothing under any circumstance.** It is the verb a user reaches for when something +is already wrong, and a diagnostic that repairs things is a diagnostic whose output cannot be +trusted. `tests/unit/project-surface.test.mjs` runs it in an empty temporary directory and asserts +that the directory is still empty afterwards — no config file, no `scrolls/`. + +
+ +#### `init` — scaffold, then ask + +`init` creates the workspace files, and offers two independent extras: a fixed, disposable +`example-box` scroll for the native host, and the consumer templates. It then offers the optional +installations: the build toolchain, and the dependencies of whichever consumer templates the project +wants. + +It refuses to be used as an authoring command. Passing `--target`, `--platform`, `--accelerator`, +`--cuda-version`, `--box-id`, `--model-id` or `--runtime-id` fails with a pointer to `new scroll`: + +```js +// src/cli.mjs +fail(`init accepts only the fixed example; pass ${…} to scrollcase new scroll.`); +``` + +**Rejected:** letting `init` author the project's first real scroll from flags. The example exists to +be run once and deleted; a scaffolded scroll that looks like a real one invites a project to inherit +identity decisions it never made. Whether to create it is one of the two questions `init` asks first, +both defaulting to yes, and `--no-example` answers it without asking. + +**Also rejected:** one question for both extras. The consumer templates were originally written by +`ensureExampleScroll`, so declining the demo silently declined them too — and the two answer +different needs. The example is disposable; the templates are where a project's own consumer +application starts, in whichever of the three languages it is written in. `--no-templates` declines +them on their own, and passing both flags is what leaves a bare workspace. + +The example's target is chosen by `nativeExampleTarget()` — Metal on macOS, CPU everywhere else — so +the demo never guesses a CUDA ABI version that the host may not have. + +
+ +
+ +#### `new scroll` — one decision, collected completely + +`new scroll` is the authoring verb. `collectNewScrollOptions` in `src/cli-authoring.mjs` gathers +every field — from flags, or from prompts when a terminal is present — and hands `createScroll` one +complete object. The scroll is written atomically after every answer exists, so an abandoned session +leaves nothing behind. + +The positional grammar is checked strictly: `new` accepts exactly the single word `scroll` and +nothing else, because a mistyped subcommand that silently authored something would be worse than an +error. + +
+ +
+ +#### `doctor` — the read-only verb + +`doctor` prints one line per check with its own remedy, and exits non-zero if any failed: + +```text +✓ workspace config /path/to/project/scrollcase.config.json +✓ scrolls /path/to/project/scrolls +✓ git HEAD 6db8803169cc +✗ pixi Scroll requires pixi 0.60.0, found 0.58.0. + → Install pixi 0.60.0 from https://pixi.sh/, or pass --pixi . +✓ conda-pack /path/to/project/.scrollcase/toolchain/bin/conda-pack +``` + +Every check *reports* rather than throwing, so one missing tool does not hide the next problem: +someone whose machine is missing both pixi and conda-pack learns both in one run rather than one per +attempt. The `pixi` row needs a version to check against, and without one it says so explicitly — +`not checked: pass --pixi-version, or run doctor with a scroll` — instead of silently passing. +`--scroll ` takes that version from the scroll itself, which is the form that answers the +question a user actually has: can this machine build *this* box. + +
+ +
+ +#### `keygen` — and the two paths it defaults to + +`keyPaths(flags)` resolves both key locations against the workspace's `keysDir`, so +`--private-key` and `--public-key` are overrides rather than requirements: + +```js +// src/cli.mjs +privatePath: resolve(text(flags, 'private-key') || join(keysDir, 'signing-private.pem')), +publicPath: resolve(text(flags, 'public-key') || join(keysDir, 'signing-public.json')), +``` + +The same function serves `build`, `verify` and `run`, so the key a build signs with and the key a +verification trusts are named by one rule rather than three. Section 7.2 describes what `keygen` +writes and why `--force` is dangerous. + +
+ +
+ +#### `lock` — the only verb that solves + +`lock` reads the scroll, locates the pixi version the scroll pins, and runs the resolver over the +generated `pixi.toml`: + +```js +// src/cli.mjs +const pixi = findPixi({ requiredVersion: scroll.pixiVersion, path: text(flags, 'pixi') }); +run(pixi, pixiLockArguments(join(dir, 'pixi.toml'))); +``` + +It is a human action whose output is committed and reviewed. `build` never solves; it installs from +the committed [lockfile](#lockfile), which is what makes the reviewed set and the shipped set the +same set. Section 4.2 covers the three pixi invocations in full. + +
+ +
+ +#### `audit` — an inventory, without a build + +`audit` derives the licence inventory from the lock alone, prints the counts per licence, and +optionally writes the reviewed copy: + +```text +· 412 packages for example-box-macos-aarch64-metal (macos-aarch64-metal) + 238 BSD-3-Clause + 104 MIT + 41 Apache-2.0 +``` + +The licence rows are sorted by descending count and then by name, so two runs over the same lock +print the same list in the same order. + +Without `--write` it compares against the reviewed audit already on disk and reports agreement; +with `--write` it becomes the reviewed audit. That asymmetry is the whole review mechanism: a build +fails when the two disagree, and making them agree is a deliberate act. Section 6.8 describes the +derivation. + +
+ +
+ +#### `build` — a long pipeline behind one question + +`build` resolves the scroll reference, runs the signing preflight, asks which channel, and then hands +everything to `buildBox`: + +```js +// src/cli.mjs +await ensureBuildSigningKeys(signing); +// Asked at the CLI edge and passed down: buildBox never reads a terminal itself. +const channel = await chooseCliValue('channel', ['beta', …], { flag: text(flags, 'channel') }); +const weights = text(flags, 'weights'); +``` + +The order is the point. The preflight is a read-only check that the keys exist (section 7.6), and it +runs *before* the question, which runs *before* the first expensive stage. A missing key costs a +second, not the twenty minutes it would cost if it were discovered at the signing stage. + +`beta` is listed first so it is the highlighted default in the menu and the value taken when there is +no terminal — the channel a build should land on unless someone deliberately says otherwise. + +The weights mode used to be a second menu, and that was a defect rather than a convenience. It was +preselected on `embed`, so a build of a scroll declaring `on-demand` silently repacked the assets +into the archive for anyone who answered by pressing Enter — the scroll's own declaration overridden +by the menu's default. It is not asked any more: `buildBox` takes the scroll's mode, `--weights` +overrides it deliberately, and the mode in effect is logged rather than negotiated. + +
+ +
+ +#### `verify` and `run` — the consumer at the command line + +Both take a signed release document and trust the key set at `--public-key`. In its archive form, +`verify` finds the archive beside the document or at `--archive`, performs the install-time checks, +and stops; with `--self-test` it additionally extracts, recomputes the payload digest when present, +and imports the signed module list with the box's own interpreter. + +`verify --extracted ` is the other form. It delegates directly to +`verifyExtractedPayload` in `scrollcase/consumer`, needs no archive, and refuses `--archive` or +`--self-test` rather than combining two operations with different meanings. `run` always takes the +archive path: it performs verification, extracts to a temporary directory, executes, forwards +signals, and removes the extraction on every terminal path. + +Neither downloads anything. Both are described from the library side in sections 6.15 and 8. + +
+ +
+ +### 9.3 Parsing arguments — `cli-args.mjs` + +Thirty lines, no dependency, and one rule that matters more than the rest. + +
+ +| Form | Result | +| --- | --- | +| `value` | Appended to `positional` | +| `--name value` | `flags.set('name', 'value')`, and the value is consumed | +| `--name=value` | `flags.set('name', 'value')` | +| `--name` (followed by another `--flag`, or last) | `flags.set('name', true)` | +| everything after `--` | `passthrough`, verbatim | + +**The `--` separator is a hard boundary.** Every string after it belongs to the box application +unchanged, even when it looks like a Scrollcase flag, even when it contains shell metacharacters: + +```bash +scrollcase run release.json -- --public-key 'not; a flag' "$(echo unexpanded)" +``` + +All four of those tokens reach the box application exactly as written. Nothing after `--` is parsed, +rewritten, or interpreted, and — because the consumer spawns without a shell (section 8.4) — nothing +in them is expanded either. `tests/unit/cli-args.test.mjs` asserts that byte-for-byte preservation. + +**Rejected:** an argument-parsing library. The grammar above is the entire surface, the passthrough +rule is the only subtle part of it, and a dependency here would be a dependency in the security path +of every `run` invocation. Section 4.4 states the same reasoning for the runtime dependencies. + +One consequence is worth stating plainly: a flag value that itself begins with `--` cannot be written +in the separated form, because the parser reads the next token as a new flag. Write it as +`--name=--value`. + +
+ +### 9.4 Where the boundary runs + +The CLI owns interaction. Every module below it receives the *answer*, never the question. + +
+ +
+ +#### Consent — `confirm()` + +```js +// src/cli.mjs +if (!process.stdin.isTTY || !process.stdout.isTTY) return false; +``` + +Both ends must be a terminal. There, `confirm(question, hint)` prints the same heading every other +question uses (section 9.5) and the answer line is `[Y/n]`: an empty answer, `y`, or `yes` accepts; +`n`, `no`, or unrecognised input declines. Without a terminal — a CI job, a pipe, a container build +— the answer is still no, because **silence outside an interactive prompt must not be read as +consent**. This is the guard that keeps `init` from downloading a toolchain in an automated +environment that never agreed to one. + +`--install-toolchain` and `--no-install-toolchain` are how an automated caller states the answer it +would have given. They are passed into `ensureToolchain` as a `confirm` callback that ignores the +terminal, so the module below still sees one uniform consent interface. + +
+ +
+ +#### Closed choices — `cli-menu.mjs` + +`selectCliMenu` draws a raw-key menu: arrow keys move, Enter selects, Ctrl-C rejects. It redraws in +place with `\x1b[A`, hides the cursor while it runs, and its `cleanup()` restores the previous raw +mode and shows the cursor again on **every** exit path, including the rejection. A menu that left a +terminal in raw mode would break the shell that invoked it. + +Its title and hint are printed through the shared `promptHeading` and sit **outside** the redrawn +frame, so arrowing through the options never scrolls away the line explaining what is being chosen. + +`chooseCliValue` wraps it with the policy: + +| Situation | Behaviour | +| --- | --- | +| A flag was passed | Validated against the closed list, unless `open` — an unknown value fails | +| A terminal is present | The menu, preselecting the first choice | +| No terminal | The first choice, **and a line saying which default it took** | + +That last row is the interesting one. A silent default in a log is indistinguishable from a decision; +a stated one can be noticed in review. + +**Rejected:** offering free-form values through the menu. A menu implies the list is exhaustive, so +anything open-ended — a path, an identifier, a version — stays an explicit flag or a text prompt. +Safety consent stays out of it for the same reason: a "yes/no" rendered as a list of choices reads as +a preference rather than a decision. + +
+ +
+ +#### Target and scroll selection — `cli-targets.mjs` + +This module owns the one interactive policy that is not a plain menu, because a wrong answer here +packages the wrong thing. + +```text + --target given? ──yes──► must be one of the candidates, or fail + │ + no + ▼ + exactly one candidate? ──yes──► take it + │ + no + ▼ + terminal? ──yes──► menu, preselecting the host default + │ + no + ▼ + a single native candidate, or macOS + Metal? ──yes──► take it, and say so + │ + no + ▼ + fail, listing the candidates and asking for --target +``` + +Scroll selection is deliberately **not** symmetrical with this. When the positional argument is +omitted and there is no terminal, `chooseScroll` fails rather than defaulting: + +```js +// src/cli-targets.mjs +fail('scroll selection requires an interactive terminal; pass / explicitly.'); +``` + +A default target is a fact about the host, and picking it wrong wastes a build. A default *scroll* is +a guess about intent, and picking it wrong locks or packages a different product. Both refuse +ambiguity; they just disagree about what counts as ambiguous. + +Both sort their candidates with `compareStableStrings` — the same raw-string ordering the archive +writer uses (section 6.11) — so the menu is in the same order on every machine, and both refuse +duplicate references outright rather than presenting two indistinguishable rows. + +
+ +
+ +#### Authoring input — `cli-authoring.mjs` + +`collectNewScrollOptions` builds three helpers over the same flag map, and the difference between +them is what happens without a terminal: + +| Helper | With a terminal | Without | +| --- | --- | --- | +| `required(flag, …)` | Prompts, optionally with a default | Fails, naming the missing flag | +| `derived(flag, default)` | Never asks — takes the flag, or settles the default | Identical | +| `finite(flag, …, choices)` | Menu | Fails, naming the flag *and its allowed values* | + +`derived` is the one that decides how long the session is. The wizard once asked nine questions to +produce a file whose answers were nearly all forced — an identity that follows from the box name, a +version whose only sensible starting point is `1.0.0`, a pixi version that `findPixi` will refuse +unless it matches the pixi already installed — and then asked four more optional host constraints +that most projects leave empty. Four questions remain, and each is one nobody else can answer: the +target, the box id, the upstream revision of what is being packaged, and the base URL boxes will be +published under. The rest are flags for the caller who cares. + +`sourceRevision` stays a question for a specific reason. It names the version of the thing being +packaged, it goes verbatim into the box's provenance, and there is nothing to derive it from — a +default there would be a fabricated claim about where a box came from, which the tool refuses to +make anywhere else. + +`promptText` repeats a required question rather than aborting on a blank answer. Aborting discarded +every value already typed and sent the user back to the first question, punishing a slip out of all +proportion to it; the repeat is bounded, so an input stream that only ever yields blank lines ends +in an error rather than a loop nobody can interrupt. The heading is printed once, above the loop: +the retry restates what is required and asks again on a fresh ` ↳ ` line, and repeating the whole +explanation each time would bury the answer being asked for. + +The CUDA ABI version is still asked only after CUDA has been chosen — which is why +`cliTargetFamilies()` lists CUDA without a version and the complete target ID is assembled +afterwards. A question that cannot apply is not asked, rather than asked and discarded. + +`--default-args` is parsed as a JSON array of strings and rejected as a whole if it is anything else, +including an array containing one number. Those strings end up in a signed document and then in an +argument vector; a silently coerced value there would be a signed lie about what the box runs. + +
+ +
+ +#### Where an edit goes — `cli-edit.mjs` + +`add`, `remove`, `edit` and `refresh` all ask one question `new scroll` never has to: which of a +box's scrolls does this change? A box with a base and per-target fragments has two right answers, +and the wrong one is silent — a declaration lands on one target instead of all three, and nothing +complains until a build somewhere is missing a file. + +So it is never guessed. `--target all` writes what the targets share; a target ID writes only that +one; a box with a single target uses it; a box with several asks, with "every target" first in the +menu. Without a terminal and without the flag the command stops. **Rejected:** defaulting to `all`. +It is the commoner intent, which is exactly what makes the rare case — a file specific to one +accelerator — worth a question rather than a silent default. + +`chooseScrollEdit` builds its field menu from the schema, so a name that is not a field cannot be +typed at all. That is a better shape than accepting one and explaining afterwards, and it keeps the +menu honest as the format changes. + +
+ +
+ +#### Ordering optional work — `cli-init.mjs` + +`runInitDependencySetup` exists to enforce one sequence: **every answer is collected before the first +installer runs.** + +```js +// src/cli-init.mjs +const offered = CONSUMER_LANGUAGES.filter((language) => language !== 'rust' || rustAvailable); +const selected = hasTemplates ? await chooseConsumerLanguages(offered) : []; +if (chose('python')) pythonSource = await choosePythonSource(); +const toolchain = await installToolchain(); +const typescript = shouldInstallTypeScript ? installTypeScript() : null; +const python = pythonSource ? installPython(pythonSource) : null; +const rust = shouldInstallRust ? installRust() : null; +``` + +Interleaving them would let a multi-minute download interrupt the remaining questions, leaving a user +who walked away with a half-collected set of choices and a half-installed project. It also makes the +whole interaction reviewable as one block before anything irreversible happens. + +The three languages are **one multi-select menu**, not three consecutive yes/no prompts. They are the +same question asked about three languages, and asked one at a time they became three chances to +answer by reflex; asked together they are a list a person reads once. Nothing is preselected, and an +empty selection is a complete answer rather than an unfinished question. The one list, +`CONSUMER_LANGUAGES`, builds the menu and reads its answer back, so an entry cannot be offered +without an installer behind it. + +`resolveExampleChoice` and `resolveTemplatesChoice` answer the questions that come before all of +those. The templates decide which installs are offered at all; the example decides nothing else. +`--no-example` and `--no-templates` decide without asking, an interactive caller is asked and +defaults to yes, and a caller without a terminal keeps both. That last branch reads backwards next to +the installs, where silence means no, and it is deliberate: writing a disposable scaffold into the +workspace the user just pointed at is not an irreversible act, and a non-interactive `init` +therefore still produces exactly what it produced before there was a question to answer. + +`resolvePythonConsumerSource` handles the one branch that cannot be decided in advance: conda-forge +was chosen but conda is not installed. It offers PyPI, and a declined offer returns `null` — which +skips the Python install rather than silently substituting a different package source. +The CLI similarly probes Cargo before entering this sequence, so a missing optional package manager +leaves Rust out of the menu rather than turning an accepted default into a subprocess failure. +`tests/unit/cli-init.test.mjs` asserts both scaffold questions' three branches, the ordering, the +declined fallback, and the unavailable-Cargo branch. + +
+ +
+ +#### The two remaining edges + +`cli-signing.mjs` is a read-only preflight over the key files, described in section 7.6. It never +creates a key: generating one silently would produce a box signed by a key nobody chose to trust. + +`cli-run.mjs` translates the child process's terminal result into the CLI's own, described in section +8.8. It is the only place in the tool that terminates the current process on purpose. + +
+ +
+ +### 9.5 Presentation — `cli-output.mjs` + +Library modules produce values and messages; this module decides what they look like. Keeping the two +apart is what lets a program embed the build without inheriting a terminal aesthetic. + +
+ +| Kind | Symbol | Used for | +| --- | --- | --- | +| `success` | `✓` | Something exists now that did not before | +| `step` | `→` | A stage starting, or the next command to run | +| `info` | `·` | A path or a fact, subordinate to the line above | +| `warning` | `⚠` | Something was skipped that the user may have wanted | +| `error` | `✗` | The single failure line, on stderr | + +**Only the symbol is coloured**, never the message. Colour is applied when the stream is a terminal, +`NO_COLOR` is absent, and `TERM` is not `dumb`: + +```js +// src/cli-output.mjs +const colourAvailable = (stream, env) => + Boolean(stream.isTTY && !Object.hasOwn(env, 'NO_COLOR') && env.TERM !== 'dumb'); +``` + +`Object.hasOwn` rather than a truthiness test: `NO_COLOR=` with an empty value is still the user +asking for no colour, and reading it as "false" would be a bug in exactly the environment that took +the trouble to set it. The symbols survive redirection, so a captured log is still readable without +any escape sequences in it. + +The same module owns the shape of every question the CLI asks. `promptHeading` and `promptMarker` +are used by the text prompts (`cli-authoring.mjs`), the raw-key menus (`cli-menu.mjs`) and the +yes/no consent questions (`cli.mjs`), so one layout covers all three: + +```text +Upstream revision +Which version of the thing you are packaging this is — a model commit, a release tag. Recorded +verbatim in the box provenance: + ↳ upstream-v1 +``` + +A blank line, the field's name, the line explaining it, then the answer after ` ↳ `. Before this, a +`new scroll` session printed hint, question and answer on adjacent lines nine times running, and the +result was a wall in which the explanations were indistinguishable from the things being asked. The +explanation ends in a colon — replacing its full stop — because it is the line directly above the +answer, and a line that already ends in `?` is left alone, which is what a menu title is. + +**Two palette colours, not two RGB values.** The title is magenta and the marker is bright black, +both ANSI palette entries, so the terminal's own theme supplies them and the result stays legible on +a light scheme and a dark one alike; a hard-coded colour is chosen against exactly one background. +The explanation stays uncoloured because it is prose, not a label, and `NO_COLOR` removes both +without changing the layout — the blank line and the marker are the structure, the colour is the +enhancement. + +`buildDistributionSummary` prints the closing line of a successful build as two paths relative to +`dist/`, with the content hashes left out — those are in the file names, and repeating a 64-character +digest in a terminal line helps nobody. The `build` verb also filters the pipeline's own log, +dropping the lines `buildBox` emits about what to publish and printing this one summary instead, so +the human-facing instruction is written once at the edge that formats it. + +
+ +### 9.6 What the command line deliberately does not do + +Every item below is a thing a packaging CLI is routinely expected to do, and each is left out for the +reason given in section 3. + +
+ +- **No publishing.** There is no `scrollcase publish`, no upload, no credentials, no bucket. `build` + leaves two files under `dist/` and says where they are; copying them somewhere is the project's + job, and the content-addressed layout makes it a copy rather than a mapping. +- **No fetching a box.** `verify` and `run` take local paths. A verb that downloaded a release would + have to decide what to trust before verifying it. +- **No promotion, rollback or revocation.** A channel document is a file; moving a channel to a new + release means signing a new one. +- **No persistent state.** There is no cache directory, no lock file of its own, no daemon, and no + record of what was previously built. Everything the CLI knows comes from the workspace, the scroll + and the arguments of the current invocation. +- **No mutation of the project's configuration** outside `init`. Flags override the workspace for one + invocation and are never written back, so a `--out-dir` used once does not silently become the + project's setting. +- **No interactive fallback below the edge.** No module beneath `src/cli*.mjs` reads `process.stdin`. + A missing answer in a non-interactive environment is an error with the flag that would supply it, + never a prompt that hangs a pipeline forever. + +## 10. The invariants + +
+ +### 10.1 What an invariant is here + +The previous sections described mechanisms. This one states the four properties those mechanisms +exist to hold, names every place each is enforced, and says what would break it. + +
+ +An invariant in this codebase is not an aspiration. It is a property that some specific code refuses +to let fail, and the refusal is the feature. Three of the four are promises made to whoever receives +a [box](#box) — that it can be rebuilt, that it says truthfully where it came from, and that nothing +in it is believed before it is checked. The fourth is a promise made to whoever changes the code: +that a small set of paths, invisible to the test suite on any one machine, are checked by hand. + +::: info The one-sentence form +Rebuilding the same commit produces the same bytes; a box never lies about its origin; nothing is +believed before it is verified; and four paths that the suite cannot exercise are read against every +change that touches them. +::: + +
+ +### 10.2 Determinism + +**Rebuilding the same commit must produce a byte-identical archive.** Not an equivalent one — the +same SHA-256. + +
+ +This is what makes the whole content-addressed chain worth having. If two builds of one commit could +differ, then the hash in a [release](#release) document would identify a particular *run* rather than +a particular *input*, and an auditor could never reproduce what they were asked to trust. + +Determinism is not achieved by a single mechanism. It is achieved by removing, one at a time, every +source of per-run variation that a normal build tool happily lets through: + +| Source of variation | What removes it | Where | +| --- | --- | --- | +| Wall-clock time in file metadata | `FIXED_ARCHIVE_TIME`, stamped with `lutimes` | `filesystem.mjs`, section 6.11 | +| Wall-clock time in the documents | Build time read from the HEAD commit, epoch outside a checkout | `scroll.mjs`, section 6.2 | +| Local timezone in ZIP timestamps | `forceDosTimestamp: true` | `archive.mjs`, section 6.12 | +| Directory enumeration order | One sorted walk, reused by every consumer of the tree | `filesystem.mjs` | +| Host locale and ICU data | `compareStableStrings` — code-unit ordering, never `localeCompare` | `filesystem.mjs` | +| The builder's umask | Entry modes derived from the [target adapter](#target-adapter) | `archive.mjs` | +| Compression settings | One pinned compression level | `archive.mjs` | +| Randomness | `cohortSalt` derived from box and version | `box.mjs`, section 6.1 | +| Interpreter bytecode caches | `__pycache__` and `*.pyc` skipped during enumeration | `filesystem.mjs` | +| Host filesystem artefacts | `.DS_Store` skipped | `filesystem.mjs` | +| Install-specific conda records | `conda-meta/` reduced to an identity allowlist | `pixi.mjs`, section 6.5 | +| Leftovers from a previous build | The build tree and object directory are removed before use | `box.mjs`, stage 4 | +| Dependency drift over time | Installation from the committed [lockfile](#lockfile), never a fresh solve | `pixi.mjs`, section 4.2 | + +Two of those rows are worth reading twice. **The epoch fallback outside a git checkout is deliberate, +not a shortcut** — a wall-clock fallback would silently reintroduce exactly the variation the rest of +the list removes, in the one case where nobody would look. And **`conda-meta/` canonicalisation is +determinism applied to someone else's file format**: those records carry paths, timestamps and link +counts from the machine that installed them, which are facts about a build host rather than about a +package. + +
+ +#### How it is proven + +`tests/unit/build-pipeline.test.mjs` builds the same fixture twice and compares the archives byte for +byte, and separately compares the signed execution metadata across rebuilds. That second assertion +exists because the archive is the obvious thing to check and the documents are the easy thing to +forget: a timestamp leaking into a manifest would leave the archive identical and the release +document different, which breaks the chain just as thoroughly. + +
+ +
+ +#### The exact scope of the promise + +Determinism is claimed **per commit, per target, per pinned toolchain**. Same commit, same host +platform, same pinned pixi and conda-pack, same committed lock, same weights mode — same bytes. + +It is not a claim that two different operating systems produce the same archive: they cannot, because +they package different native code, and section 5.2 is why a box is built on the host it ships for. +It is not a claim about a *re-solve* either. `lock` is where dependency resolution happens and its +output is a reviewed, committed file; `build` never solves. The lock is what turns "the same +dependencies" from a hope about upstream availability into a fact about a file in the repository. + +::: warning What breaks it +Any per-run value introduced anywhere in the build: a clock read, a random number, an unsorted +`readdir`, a mode read from disk, a temporary path that reaches the payload, a compression setting +left to a library default. Each is individually harmless-looking, which is why the list above is +written out rather than left as a principle. +::: + +
+ +
+ +### 10.3 Provenance + +**A box records the commit it was built from and whether that tree was dirty — and never +fabricates either.** + +
+ +Two facts are read from git before anything expensive happens, and both end up inside the signed +release: the revision, and whether the working tree had any modification, staged or not, tracked or +untracked. `--untracked-files=all` is what makes the second answer match what a reader expects, while +Git's own ignore rules keep generated workspace state out of it. + +| Situation | `build` does | The box says | +| --- | --- | --- | +| Clean checkout | Builds | The revision, `sourceTreeDirty: false` | +| Dirty tree, no flag | **Refuses** | — | +| Dirty tree, `--allow-dirty` | Builds | The revision, `sourceTreeDirty: true` | +| Not a git checkout | **Refuses** | — | + +The last row is the one people try to route around, and it is the most important. A box built outside +a checkout has no revision to record; the alternatives would be to invent one, to leave the field +empty, or to refuse. Inventing is a lie. Leaving it empty makes an unverifiable box structurally +indistinguishable from a verifiable one, which is worse than a lie because it is deniable. So the +build refuses, and `doctor` reports the missing checkout with the reason attached. + +The dirty case is allowed *because* it is recorded. A developer testing a change locally has a +legitimate need to build from an edited tree; what they must not be able to do is hand the result to +someone else without that fact travelling with it. `--allow-dirty` is not permission to hide the +state — it is the acknowledgement that turns it into a recorded one. + +::: danger Never downgrade a dirty build +There is no path, flag or environment variable that makes a dirty build report itself as clean, and +adding one would silently invalidate every provenance claim the format makes. This is one of the +repository's hard rules, and it is enforced by the code that fails the build rather than by +convention. +::: + +
+ +### 10.4 Verify, never trust + +**Every byte that enters the system from somewhere else is checked before it is used**, and the +check happens before the byte can have any effect. + +
+ +This is a single rule applied at eight boundaries. The table is the whole invariant; nothing in +Scrollcase accepts external input without appearing in it. + +| Boundary | What arrives | Checked against | On failure | +| --- | --- | --- | --- | +| Toolchain install | A pixi or conda-pack download | The published checksum, or the pinned digest | Nothing is installed | +| Asset staging | A downloaded weight file | Declared size **and** SHA-256 from the [scroll](#scroll) | Nothing is promoted into the payload | +| Asset staging | A local file or asset archive | The same declared hash; TAR entries restricted to files and directories | The build fails | +| Licence audit | The committed lock | The reviewed audit document | The build fails | +| External signing | A signed document from another process | The payload must be echoed exactly, and the signature verifies locally | The build fails | +| Release document | A signed envelope | Shape, payload hash, `schemaVersion`, a signature from a [trust key](#trust-key) | Nothing is extracted | +| Archive | The box bytes | Size and SHA-256 from the release, then per-entry validation | Nothing is extracted | +| Extracted tree | The payload on disk | Manifest agreement, safe entries, installed size, on-demand asset hashes | Nothing is executed | + +Four properties of that list are what make it an invariant rather than a checklist. + +**Order is part of the check.** Verification precedes execution absolutely: no consumer runs a box's +interpreter, script, module or import before signature, payload shape, archive identity, safe-entry +and manifest-agreement checks have all succeeded (section 8.2). A check performed after the thing it +protects is not a check. + +**A check is re-run when the gap between checking and using is exploitable.** The consumer re-hashes +after staging rather than trusting the hash it computed before the move, and the archive reader reads +each link target once and reuses it, because reading it twice would leave a window between the value +that was validated and the value that is used (sections 8.3 and 6.12). + +**No check is optional.** There is no "verify if possible" setting, no `--skip-verify`, no degraded +mode, and no fallback to a cached copy when verification fails. A check that a caller can turn off is +a check an attacker can arrange to have turned off. + +**Being given something is not evidence about it.** An external signer is not trusted about the +payload it signs, a scroll is not trusted about the bytes at a URL, a release is not trusted about +the archive beside it, and an archive is not trusted about its own entries. In each case the party +supplying the input is precisely the party that would benefit from lying about it. + +::: warning For contributors +An inconvenient check is not a check to delete. Every row above has cost someone time; that is what +they are for. If one is genuinely wrong, the fix is a different check, not a missing one. +::: + +
+ +### 10.5 The rules that constrain every change + +Beyond the three guarantees, four rules bound what the project may become. Each has a mechanical +guard, because a rule with no guard survives exactly as long as everyone remembers it. + +
+ +| Rule | Why | Guard | +| --- | --- | --- | +| No consuming project's name anywhere in the tool | It must stay usable by projects with nothing to do with the one that first needed it | `tests/unit/v2-migration.test.mjs` greps the whole tracked tree, content and paths | +| The document namespace belongs to the publishing project | A project with boxes in the field keeps emitting the kinds its clients recognise | `documentKinds(namespace)`; the schemas accept any well-formed namespace and nothing else | +| One substrate, and only one | Two dependency backends means proving every guarantee twice | The absence of any second backend, and section 4's single-substrate description | +| Published v1 is immutable; v2 is a clean break | A reinterpreted old document is a silent wrong answer | `decodeSignedDocument` rejects `schemaVersion: 1` with an explicit remedy | + +The first guard is worth a note for anyone who reads it. It runs `git grep` over every tracked file +*and* every tracked path, and the retired term it searches for is assembled from two string fragments +inside the test so that the test file itself does not contain the word it forbids. That is not +cleverness for its own sake: a guard that trips on itself gets weakened, and a weakened guard is how +the name comes back. + +
+ +### 10.6 The four paths that break silently + +The suite runs on one host, with the toolchain stubbed and the network unavailable. Four things are +therefore true in exactly one configuration during testing and in several in the field. **A green +suite says nothing about them.** + +
+ +
+ +#### 1. The three targets + +macOS, Linux and Windows differ in interpreter layout, scripts directory, launcher repair and native +library inspection. A change to packing, relocation or path handling has to be read against all +three, because the machine running the tests exercises one. + +| | macOS / Linux | Windows | +| --- | --- | --- | +| Interpreter | `venv/bin/python` | `venv/python.exe` | +| Scripts | `venv/bin` | `venv/Scripts` | +| Launcher repair | Rewrites the [shebang](#shebang) into a shell trampoline | None — the launchers are executables, not text | +| Native inspection | `otool -L` / `ldd`, `.dylib` `.so` | `dumpbin /DEPENDENTS`, `.dll` `.pyd` | +| Payload links | Carried when they pass the link rule | None — creating one needs elevation | + +The last row is the one that is easiest to forget, and `tests/unit/contract-links.test.mjs` asserts +it directly: a Windows box is link-free by contract, so any code that assumes a link is present, or +that assumes one is absent on POSIX, is wrong on one of the two. + +
+ +
+ +#### 2. `embed` versus `on-demand` weights + +The two [weights modes](#weights-mode) produce different archives, different release documents and +different consumer behaviour from the same scroll: + +| | `embed` | `on-demand` | +| --- | --- | --- | +| Assets in the archive | Yes | No | +| Descriptors in the signed release | Not needed | Required | +| `assetArchives` | Allowed | **Refused** | +| Air-gapped install | Works | Needs the assets materialised first | +| Consumer before execution | Nothing extra | Verifies every materialised asset against its signed hash | + +Any change to asset staging, to the manifest, or to what the consumer checks before spawning affects +both, and a test that only covers the default mode covers half the behaviour. + +
+ +
+ +#### 3. Local key versus external signer + +The local path signs in-process. The external path hands a payload to another program and gets a +document back — and that document must echo the exact payload it was given, then verify locally +before the build continues (section 7.5). + +The suite covers both, but not equally. `tests/unit/signing.test.mjs` drives the dispatch through an +injected process runner, and the shared conformance fixture carries a `valid-external-signer` case +beside its `valid-local-signer` one, so both kinds of document are consumed for real. What no test +does — deliberately, since a test must never reach the network — is run an actual external signing +command. A change to the exchange itself is therefore proven against a fake process and not against +whatever a cloud KMS or an HSM does with the same bytes. + +
+ +
+ +#### 4. Toolchain from `PATH` versus the project's own + +Discovery order is the four-row table in section 4.2: an explicit flag, then `SCROLLCASE_PIXI` or +`SCROLLCASE_CONDA_PACK`, then the project's installed toolchain, then a bare name left to `PATH`. + +```js +// src/build/pixi.mjs +if (path) return String(path); +const fromEnvironment = process.env[environmentVariable]; +if (fromEnvironment) return String(fromEnvironment); +// … the workspace's toolchain directory, if the executable is there … +return installed && existsSync(installed) ? installed : name; +``` + +A machine with no project toolchain takes a different branch from one that has run +`init --install-toolchain`, and a developer's machine usually has both. Discovery changes must be +checked in both states; the fall-through when no workspace can be resolved at all is a third. + +
+ +
+ +#### What to do about them + +Read the change against each of the four, including the ones that cannot be executed on the machine +at hand. That is not a weaker form of testing — it is the honest description of what the suite +covers, written down so that "the tests pass" is never mistaken for "the four paths are fine". Where +a path *can* be exercised through an [injection seam](#injection-seam) — a fake process factory, a +stubbed host descriptor, an injected `fetch` — the suite already does, and adding a case there is +always better than adding a note here. + +
+ +## 11. Test map + +
+ +### 11.1 Three suites, one contract + +There are three independent test suites, in three languages, and none is authoritative over the +others. They meet at the shared conformance fixture described in section 8.7. + +
+ +| Suite | Runner | Command | Covers | +| --- | --- | --- | --- | +| Node | Vitest | `npm test` | The contract, the build pipeline, signing, the Node consumer, the CLI, the package surface, the docs | +| Python | `unittest` | `python -m unittest discover -s tests -t .` from `python/` | The contract mirror, the Python consumer, the packaging surface | +| Rust | `cargo test` | `cargo test --all-targets` from `rust/` | The contract mirror, the schemas the types stand in for, the Rust consumer | + +The Python suite is also gated by three checks that are not tests but fail the same way: `mypy src` +for static types, `python scripts/sync_schemas.py --check` for the bundled schema copies, and +`python scripts/check_distribution.py dist/*` for what the wheel and sdist actually contain. The +Rust suite is gated the same way by `cargo clippy --all-targets -- -D warnings`, by +`node scripts/sync-assets.mjs --check` for the copied fixtures and schemas, and by `cargo package` +for what the crate would actually publish. None of the three passes `--locked`, because the crate is +a library and ships no committed `Cargo.lock`: a consuming application pins its own versions, and +here the flag would only forbid writing the lockfile each command needs. All three run on Linux, +macOS and Windows, +because the layout differences of section 10.6 are exactly where a consumer breaks. + +
+ +#### The conventions the suite is written to + +**Exercise the real path, not just the import.** A module that loads is not a module that works. An +early refactor dropped a constant the licence parser used; every `audit` invocation threw a +`ReferenceError` while the suite stayed green, because nothing called that function end to end. The +lesson is written into the tests as a habit: assert the observable outcome of a real call, not the +existence of an export. + +**Prefer a behaviour someone depends on.** A tampered archive is rejected; a rebuild is +byte-identical; a dirty tree is refused; a checksum mismatch installs nothing. Each of those is a +sentence a user could have written. Implementation details are asserted only where they *are* the +contract — entry ordering, argument vectors, file modes. + +**Prove a new guard can fail.** A test never seen red is not yet a guard. When one is added, whatever +it protects is broken once, the failure is observed, and then it is restored. + +**Never reach the network, and never write outside a temporary directory.** Every test that needs a +project creates one under the system temp directory and removes it afterwards; every test that would +need a download injects a `fetch` instead. A suite that touches the network fails for reasons that +have nothing to do with the change being tested, and a suite that writes into the repository +eventually deletes something. + +
+ +
+ +### 11.2 The seams that make it testable + +Section 2 defines an [injection seam](#injection-seam) as a dependency passed in rather than reached +for. The suite is the reason they exist, and four of them carry most of the weight. + +
+ +
+ +#### The fake toolchain + +`run` and `runResult` are parameters, so a test can stand in for pixi and conda-pack by +*materialising exactly what each real invocation is contracted to produce*: an environment +[prefix](#prefix) with an interpreter, `conda-meta/` records, and a tarball packed from it. + +```js +// tests/unit/build-pipeline.test.mjs +if (command === 'pixi' && args[0] === 'install') { … writeDeep(prefix, …); } +if (command === 'conda-pack') { … tar.c({ file: output, cwd: prefix, gzip: true }, ['.']); } +// Anything else is the box's own interpreter, running the self-test. +``` + +Solving is the one step that genuinely needs external tools and a network. Everything after it — +asset staging, pruning, `box.json`, the self-test gate, the deterministic archive, signing, the +publish-ready move — is the real implementation running against a real filesystem. That is the line +the suite draws: simulate the substrate, never the code under test. + +The fake prefix is not a tidy stub. It plants the [symbolic link](#symbolic-link) shapes a real conda +prefix carries — icu's `current` → `78.3` directory link and a `pkgdata.inc` pointing *through* it — +because that exact shape once made a plain `python` environment fail to unpack, and a stub made only +of regular files would never have caught it. It also plants a link escaping the tree, next to a real +file at the escape target, so that a regression which followed links would copy a build-machine file +into the box and be caught doing it. + +
+ +
+ +#### The other three + +**An injectable `fetch`**, plus injectable sleep and log functions, let the asset tests assert the +exact `Range` header of a resumed download, the restart after a same-size wrong-hash response, and +the retry behaviour after a dropped connection — none of which could be provoked reliably against a +real server. + +**An injectable host descriptor** (`{ platform, arch }`) lets the target-selection tests assert the +macOS Metal default, the ambiguous-host refusal, and the example target chosen for all three hosts +from any one machine — the closest the suite gets to covering the three-target path. + +**A fake process factory** on both consumer sides lets execution be asserted without spawning a +Python that does not exist: the argument vector, the shell-free invocation, signal forwarding and +listener cleanup are all observable. The fake is not trusted on its own, though. Both languages also +run a *real* child — a shell trampoline that re-executes the test runner's own interpreter, standing +in for the box's Python — and assert that shell metacharacters in the arguments arrive as literal +text and create nothing. Those cases are skipped on Windows, where the trampoline shape does not +exist. + +Every one of these is also how the modules stay usable as a library. A seam added for a test is a +seam an integrator can use. + +
+ +
+ +### 11.3 The Node suite, file by file + +Twenty-seven test files under `tests/unit/`, plus two shared fixtures under `tests/helpers/`. + +
+ +| File | What it proves | +| --- | --- | +| `archive-security.test.mjs` | Traversal spellings are rejected before any join; a traversal entry stops extraction before the destination is created; a link resolving inside the payload is carried, one reaching outside is refused, nothing is written *through* a link, an over-long link target is refused; scroll tarball links are rejected before any extracted asset is copied; entries are ordered by raw path strings, not host collation | +| `assets.test.mjs` | Only bytes matching the declared size **and** hash are written; a resumed download sends the exact `Range` header; a same-size wrong-hash response is never promoted and restarts cleanly; a dropped connection is retried through injected time and logging | +| `build-pipeline.test.mjs` | The pipeline end to end: scroll layout and target resolution, every refusal that must happen before probing or fetching, a full build-sign-verify, signed environment propagation into both manifests and self-tests, platform-correct symlink handling, `conda-meta/` reduced to identity, the `dist/` layout with nothing written twice, a **byte-identical rebuild** of the same commit, dirty-tree and non-checkout refusals, on-demand descriptors instead of packed assets, detection of a tampered archive, rejection of an untrusted key, and manifest agreement field by field | +| `cli-args.test.mjs` | Every application argument after `--` is preserved byte for byte; the inline, separated and bare flag forms still parse as before | +| `cli-init.test.mjs` | `[Y/n]` accepts an empty answer as yes while rejecting unknown input; the example and the templates are answered independently; every consumer and toolchain answer is collected before any installer runs; only what the one menu selected is installed; PyPI is offered when conda-forge was chosen without conda; a declined fallback installs nothing; unavailable Cargo is never offered even if selected; no consumer questions are asked when there are no templates | +| `cli-output.test.mjs` | Symbols survive redirection while ANSI does not; only the symbol is coloured; `NO_COLOR` wins even when empty; the distribution summary is relative and hash-free | +| `cli-run.test.mjs` | Exactly one release path before the separator; `runBox` is called once and the child's exit code is preserved; environment report flags and stderr formatting; a termination signal is re-raised *after* cleanup; the real CLI preserves application arguments and exit status, and forwards Ctrl-C while still removing the temporary box; a library-only release and an unmaterialised on-demand asset are refused; a non-native target is refused before any interpreter is spawned | +| `cli-signing.test.mjs` | The preflight fails clearly when no local keys exist, refuses to overwrite an incomplete pair, and requires the trust key for an external signer without offering to generate one | +| `cli-target-choice.test.mjs` | The whole selection policy: sole host target without a terminal, refusal of an ambiguous non-terminal choice, the macOS Metal default and preselection, the navigable menu, explicit `--target` honoured and validated, scroll selection through the menu and its non-terminal refusal, canonical target parsing including the CUDA ABI, example-scroll creation, `--no-example` and `--no-templates` each on their own and together, non-terminal `new scroll`, the channel menu, and the multi-select menu — Space toggling, an empty confirmation, and a selection outside what was offered | +| `cli-version.test.mjs` | `-v` and `--version` print exactly the package version, run from an unrelated working directory | +| `cli-verify.test.mjs` | `verify --extracted` delegates to the consumer, reports signed identity and entry count, names a tampered path through the CLI failure edge, emits masked and explicitly revealed environment reports, refuses archive/self-test combinations, and requires a directory value | +| `consumer-conformance.test.mjs` | Every case in the shared fixture, through the Node consumer | +| `consumer-setup.test.mjs` | Cargo and Conda detection from the workspace root; npm run through `cmd.exe` on Windows; Cargo adding the Rust dependency to the generated manifest; the PEP 668 user-install fallback; a clear error when conda disappears after selection; an unknown package source rejected before anything runs | +| `consumer.test.mjs` | Preparation, attachment, installed-payload verification, execution and one-shot: immutable process-bound receipts, environment precedence and reports at every surface, root and asset checks, list-not-directory semantics, named corruption failures, shell-free invocation, signals, and cleanup on every terminal path | +| `contract-links.test.mjs` | The link rule: the shapes a real prefix produces are accepted; targets escaping the payload, host-only shapes, cycles, over-long chains, dangling links and directory targets are refused; writing through a directory link is refused while an unused one is fine; and a Windows box is link-free | +| `contract-payload-digest.test.mjs` | The canonical payload entry stream against shared golden vectors, including byte ordering above the BMP, newline framing, link/file discrimination, parsing refusals, and the collector's self-exclusion | +| `contract-schema.test.mjs` | The schemas describe what the builder actually emits — real release, channel, box and scroll documents validate, the channel vocabulary is the same in code and schema, every shipped example scroll validates, the execution union is canonical, release and box manifests carry the same optional execution contract; the namespace defaults to `scrollcase.box`, accepts a project's own, and rejects a malformed one; the envelope rejects a payload-hash mismatch and any missing field; and a shipped signed example verifies against its public key | +| `contract-targets.test.mjs` | Every golden target-ID case; every unsupported target and invalid CUDA combination refused; adapters cover the accepted matrix and describe a layout consumers can rely on; the archive backend names the versions the package actually installs; the conda subdir mapping; the native-host and entry-point assertions | +| `docs-contract.test.mjs` | The documentation is checked against the code: schemas are published byte-identically on the routes their `$id`s claim, the privacy page exists and is linked, no third-party script is loaded, every CLI verb and option appears in the CLI reference, every public runtime export appears in the API reference, every complete JSON example parses and validates, internal routes resolve — and the three white-paper drift cases in section 11.5 | +| `execution-contract.test.mjs` | A Python script must be a regular archive file at its exact safe path; runnable modules are found in both the POSIX and Windows layouts; a module in neither the box root nor its environment is refused | +| `package-surface.test.mjs` | Every advertised subpath exists and resolves; `files` ships everything the exports map points at; the executable ships under the canonical command name; each entry point imports the way a dependent would; the browser graph reaches no Node built-in; a strict TypeScript consumer type-checks every entry point; an `npm pack` dry run contains the complete consumer import closure; the schema and fixture wildcards resolve; and both generated surfaces still match their sources | +| `environment.test.mjs` | Case-aware Windows precedence, inherited-environment preservation, compact selection, host-value masking, explicit reveal, and report formatting | +| `parity.test.mjs` | The metrics themselves — absolute error, relative error, cosine similarity, the zero-reference case, a length mismatch, which bound was breached — and the gate: one run per accelerator under each accelerator environment with the declared box environment applied, a failing build on drift, non-finite output refused, non-numeric output refused, at least two accelerators required, and the whole gate skipped when a scroll declares none | +| `project-surface.test.mjs` | `init` scaffolds the workspace and never overwrites, so re-running is safe; `doctor` reports every problem at once with a remedy each, reports a wrong pixi version as a failure rather than an absence, and **writes nothing**; `audit` summarises straight from the lock, writes the reviewed copy only when asked, and fails when the lock no longer matches it | +| `scroll-authoring.test.mjs` | Every authored shape — library-only, wizard answers with no weights menu among them, a module with default arguments, the Windows interpreter and conda platform derived from the adapter — plus a staged script hashed at a safe payload path, a generated starter whose declared hash matches its bytes, an initialised example left untouched, consumer templates written without an example and never over an edited one, and refusal to overwrite anything | +| `signing.test.mjs` | The external signer: a quoted command with spaces is parsed and its result verified locally; a signer that validly signs a *different* payload is rejected; an invalid signature is rejected even when the payload was echoed exactly | +| `toolchain.test.mjs` | The published asset name and URLs per host, a null rather than a guessed URL for an unsupported host, digest parsing in both checksum-file forms, verified installation including across filesystems, a checksum mismatch installing **nothing**, the pinned digest preferred over the server's, the pinned conda-pack version, and the `init` offer — nothing downloaded on a no, only what is missing asked for, both pins recorded, an unsupported host reported | +| `v2-migration.test.mjs` | Only the canonical v2 scroll schema is published; a v1 signed document is rejected with the migration remedy; the channel vocabulary is closed; the workspace exposes no legacy field; and no retired product terminology survives in tracked content or paths | + +The two helpers are shared rather than duplicated: `consumer-box-fixture.mjs` builds a real signed box +— through `createDeterministicZip`, `signDocument` and the real target adapter — for both the local +and the external signer, and `consumer-conformance.mjs` builds the mutated archives the shared +fixture describes. + +
+ +### 11.4 The Python and Rust suites + +Six test files under `python/tests/`, plus two support modules. + +
+ +| File | What it proves | +| --- | --- | +| `test_contract.py` | The mirror is faithful: every canonical target-ID and payload-digest vector in the shared fixtures, bundled schemas are exact generated copies, and the payload link rule accepts and refuses exactly what the Node implementation does | +| `test_verify.py` | Verification, extraction, attachment and installed-payload checking: immutable typed receipts with honest status, existing/file/link roots handled correctly, native-host and asset checks, list-not-directory semantics, named tampering failures, v1 and invalid signatures refused before extraction, archive/manifest disagreement, installed size, and hostile ZIP entries | +| `test_run.py` | Execution: signed and caller arguments preserved in order without a shell, environment reports from verification, attachment and execution, release precedence and masking, `-m` invocation, a **real** child process preserving shell metacharacters, on-demand assets verified before spawning, replaced roots and forged receipts and library-only boxes refused, a non-native target refused before spawning, signals forwarded with parent handlers restored, one-shot execution removing its temporary bytes on every terminal path, and the real standard streams routed through the box interpreter | +| `test_conformance.py` | Every case in the shared fixture, through the Python consumer | +| `test_public_api.py` | The package exports the five consumer operations and immutable environment report models, with every public name declared in `__all__` | +| `test_release.py` | The release tag check accepts `python-v`, and rejects both the Node tag namespace and a mismatched version — so the two packages cannot be released under each other's tags | + +`support.py` and `conformance_support.py` are the Python counterparts of the Node helpers, and +deliberately share no code with them: `zipfile` and hand-built `ArchiveEntry` records against `yazl` +and mutated ZIP bytes. Three independent harnesses agreeing on one expectation file is evidence +about the contract; one shared harness would only be evidence about itself. + +Seven test files under `rust/tests/`, plus one support module. + +| File | What it proves | +| --- | --- | +| `contract.rs` | The mirror is faithful: every canonical target-ID and payload-digest vector in the shared fixtures, and the link rule accepting and refusing exactly what the other implementations do | +| `schema.rs` | The types the crate parses with and the canonical schemas reach the same verdict on the examples and on mutations chosen where a typed parse and a schema most plausibly drift — an unknown field, a missing required field, a pattern violation, a broken bound, the `weights`/`assets` co-requirement, and the one open object where agreement means accepting rather than refusing | +| `release_document.rs` | The half of the trust chain that needs no archive, over a real release signed the way `signWithLocalKey` signs — so the crate is proved against documents the signer it exists to read produced, not documents it produced itself | +| `archive.rs` | The read-only chain over real archives: each case breaks exactly one thing and asserts *which* check fired, because a check that fires for the wrong reason has stopped working | +| `prepare.rs` | Preparation, attachment and payload verification: the only three ways to obtain the receipt the execution surface accepts | +| `run.rs` | Execution against a really spawned fixture interpreter — the argument vector, the environment, the process lifecycle, and forwarded signals | +| `conformance.rs` | Every case in the shared fixture, through the Rust consumer | + +`support/mod.rs` is the third harness: the `zip` crate, its own `Entry` records, and its own +central-directory patching for the cases that need a hostile archive. `run.rs` is unix-gated, because +its stand-in interpreter is a shell script; the code it exercises is not, and its Windows branches +are read against the same expectations. The rest of the suite does run there, which is how a +Windows-only defect in preparation — a staging path canonicalised after the rename that had moved +it, so that every preparation failed — was caught before the crate was published. + +
+ +### 11.5 The three drift cases for this document + +A white paper describing a codebase is a document that decays silently. Three cases in +`docs-contract.test.mjs` make the decay loud. + +
+ +| Case | Fails when | +| --- | --- | +| Every module under `src/**/*.mjs` appears in the white paper | A module is added, renamed or moved without documenting it | +| Every public export in `package.json` `exports` appears, with every named runtime export | A new export is published without describing it, or one is renamed | +| Every intra-page `](#…)` anchor resolves to a heading that exists | A glossary entry is renamed, or a heading it points at is reworded | + +The third exists because nothing else covers it. VitePress fails a build on a dead link *between* +pages, but an anchor into the same page that matches no heading renders as a link that quietly goes +nowhere — and in a document whose every technical term links to a glossary entry, that would be the +first thing to rot. The check recomputes each heading's slug the way the site generator does, from +the raw Markdown, so it needs no built site to run. + +The first two are what keep the module-by-module promise honest. This document claims to describe +every module and every public surface; the test is what makes that claim falsifiable rather than +aspirational. + +
+ +### 11.6 What the suite deliberately does not prove + +Stated plainly, because "the tests pass" is only meaningful next to this list. + +
+ +- **No real box is ever built.** No pixi solve, no conda-pack, no gigabytes, no network. The build + tests exercise every stage around the substrate, with the substrate simulated (section 11.2). +- **No real toolchain is ever installed.** Downloads, checksums and installation are driven through + injected primitives; the bytes are fabricated in the test. +- **One host at a time.** The build tests target whichever platform the suite is running on, because + the native-host gate rightly refuses anything else. Section 10.6 is the list of what that leaves + uncovered. +- **No real external signer.** The dispatch is proven against an injected process runner. +- **No printed output.** The print layout of this page is verified by a human printing it, not by a + test. +- **The docs build is a separate gate.** `cd docs && npm run build` is what fails on a dead link + between pages; `npm test` does not render the site. + +::: info The escalation ladder +`npm test` after every change. `cd docs && npm run build` when documentation changed. `npm run types` +then `npm test` when a schema changed. The Python suite, `mypy`, the schema check, the wheel build +and the distribution inspection when `python/` changed. `cargo test`, `cargo clippy`, the asset +check and `cargo package` when `rust/` changed. A real build only when a human asks for one. +::: + +## 12. Appendices + +
+ +### 12.1 Module summary + +Every JavaScript module the package ships, with the section that describes it. Four directories, one +responsibility each: the format, what produces it, what signs it, what consumes it — and the command +line over all of them. The Rust crate follows at the end, since it ships separately. + +
+ +
+ +#### `src/contract/` — the format + +| Module | Role | Section | +| --- | --- | --- | +| `src/contract/index.mjs` | The contract entry point: the single source of truth for what a box is | 5.1 | +| `src/contract/browser.mjs` | The same model without any Node built-in, for a browser or an edge runtime | 5.1 | +| `src/contract/targets.mjs` | The [target](#target) model, the identity rule, and the adapter per target | 5.2 | +| `src/contract/document-shape.mjs` | The platform-neutral parts of the [envelope](#envelope): shape checks and namespacing | 5.3 | +| `src/contract/documents.mjs` | The envelope reference implementation, including payload decoding | 5.3 | +| `src/contract/links.mjs` | The rule deciding which [symbolic links](#symbolic-link) a payload may carry | 5.4 | +| `src/contract/payload-digest.mjs` | The canonical entry list a release commits to, so an extracted install can be re-identified | 5.5 | + +
+ +
+ +#### `src/build/` — what produces a box + +| Module | Role | Section | +| --- | --- | --- | +| `src/build/index.mjs` | The build layer's public surface | 6.18 | +| `src/build/box.mjs` | `buildBox` — the pipeline itself, as one ordered function | 6.1 | +| `src/build/scroll.mjs` | Reading a [scroll](#scroll), resolving a reference, and reading git [provenance](#provenance) | 6.2 | +| `src/build/workspace.mjs` | [Workspace](#workspace) discovery and path resolution | 6.3 | +| `src/build/schema-validation.mjs` | Dependency-free runtime validation against the shipped schemas | 6.4 | +| `src/build/pixi.mjs` | Tool discovery, the exact pixi and conda-pack arguments, packing and relocation | 6.5 | +| `src/build/launchers.mjs` | Repairing the console scripts a conda environment generates | 6.6 | +| `src/build/assets.mjs` | Verified download, verified copy, archive expansion, and the publish-ready move | 6.7 | +| `src/build/licenses.mjs` | The [SPDX](#spdx) licence inventory derived from the [lockfile](#lockfile) | 6.8 | +| `src/build/audit.mjs` | `auditScroll` — the inventory as a command, with the reviewed-copy comparison | 6.8 | +| `src/build/execution.mjs` | Static execution prerequisites shared by the builder and the verifier | 6.9 | +| `src/build/parity.mjs` | The optional cross-accelerator numerical gate | 6.10 | +| `src/build/filesystem.mjs` | [Determinism](#determinism) and path-safety primitives | 6.11 | +| `src/build/archive.mjs` | Deterministic ZIP writing, and defensive reading | 6.12 | +| `src/build/identity.mjs` | Where a release's artefacts live relative to everything else | 6.13 | +| `src/build/verify.mjs` | `verifyBox` — a consumer's install-time checks, run locally | 6.15 | +| `src/build/project.mjs` | `init` and `doctor`: scaffolding, and diagnosis that writes nothing | 6.16 | +| `src/build/authoring.mjs` | Atomic creation of one target-specific scroll | 6.16 | +| `src/build/scroll-edit.mjs` | Changing a scroll that exists: which file, atomically, verified | 6.16 | +| `src/build/dependencies.mjs` | The `[dependencies]` table of a box's pixi manifests | 6.16 | +| `src/build/consumer-setup.mjs` | The optional consumer dependencies an initialised project may want | 6.16 | +| `src/build/toolchain.mjs` | Checksum-verified installation of pixi and conda-pack, only on consent | 4.2 | +| `src/build/process.mjs` | `fail`, `run` and `runResult` — the one error path and the subprocess seam | 6.17 | + +
+ +
+ +#### `src/sign/` and `src/consumer/` + +| Module | Role | Section | +| --- | --- | --- | +| `src/environment.mjs` | Case-aware environment precedence, provenance reports, masking and CLI formatting | 6.17, 8.4 | +| `src/sign/index.mjs` | Two signing paths and one envelope: a local key, or an external signer | 7.3, 7.5 | +| `src/sign/keys.mjs` | Key generation, reading a pair back, and signature verification | 7.2, 7.4 | +| `src/consumer/index.mjs` | The local execution surface: the five operations and nothing else | 8.1 | +| `src/consumer/verify-and-extract.mjs` | Preparation, attachment and installed-payload verification, with opaque receipts for the executable paths | 8.3, 8.9 | +| `src/consumer/run-extracted.mjs` | Shell-free execution of a box this process already prepared | 8.4 | +| `src/consumer/run-box.mjs` | One-shot: prepare into a private temporary root, run, remove every byte | 8.5 | + +
+ +
+ +#### `src/cli*.mjs` — the edge + +| Module | Role | Section | +| --- | --- | --- | +| `src/cli.mjs` | Argument dispatch, workspace configuration, and the single failure path | 9.1, 9.2 | +| `src/cli-args.mjs` | The flag grammar and the `--` passthrough boundary | 9.3 | +| `src/cli-menu.mjs` | The raw-key menus, single- and multi-select, and the policy that turns a flag or a terminal into a choice | 9.4 | +| `src/cli-targets.mjs` | Target and scroll selection, including the host defaults | 9.4 | +| `src/cli-authoring.mjs` | Input collection for `new scroll`, from flags or prompts | 9.4 | +| `src/cli-edit.mjs` | Which box, which target, which field: the questions an edit asks | 9.4 | +| `src/cli-init.mjs` | The order of `init`'s questions: the two scaffold questions first, then every answer before any installer | 9.4 | +| `src/cli-signing.mjs` | The read-only signing preflight | 7.6 | +| `src/cli-run.mjs` | Translating a child's terminal result into this process's own | 8.8 | +| `src/cli-output.mjs` | Status symbols, the shared question layout, optional colour, and the distribution summary | 9.5 | + +
+ +
+ +#### `rust/src/` — the crate + +Published separately, and listed here because it implements the same section 8 as the modules above. + +| Module | Role | Section | +| --- | --- | --- | +| `error.rs` | One opaque error type and the `fail!` macro — the single failure path, deliberately not an enum a caller could match on and come to depend on | 8.1 | +| `path.rs` | The path-safety primitive every extraction and attachment goes through | 8.2 | +| `contract/` | The mirror: `targets.rs`, `documents.rs`, `links.rs`, `payload_digest.rs` | 5.2–5.5 | +| `trust.rs` | Trust anchors from either source, key rotation, and strict ed25519 verification | 7.4 | +| `release.rs` | The typed release and box manifests, refusing an unknown field where the others run a schema — except in `compatibility`, the one object the schema leaves open, whose unfamiliar constraints are carried to the caller | 8.1 | +| `archive.rs` | Defensive reading and extraction, including the duplicate-name check the ZIP backend cannot make; that check locates EOCD or EOCD64 and streams only the declared central-directory records, because identical index bytes inside stored nested archives are payload data | 8.2, 8.6 | +| `filesystem.rs` | Walking, sizing and validating an extracted tree, links included | 8.3 | +| `execution.rs` | The static execution prerequisites | 8.4 | +| `environment.rs` | Environment precedence, masking and the report | 8.4 | +| `verify.rs` | Release inspection, manifest agreement, archive inspection | 8.2 | +| `prepare.rs` | `PreparedBox` and the three ways to obtain one | 8.3, 8.9 | +| `run.rs` | Shell-free execution, the `SpawnBox` seam, and caller-owned signal forwarding | 8.4, 8.5 | + +
+ +
+ +### 12.2 Index of public exports + +Anything in this appendix is a public API. Changing a name, a signature or a meaning here is a +change to the package's contract, not an implementation detail. + +
+ +
+ +#### The subpaths + +| Subpath | Provides | +| --- | --- | +| `scrollcase/contract` | The complete contract, including payload decoding and hashing through Node's `crypto` | +| `scrollcase/contract/browser` | The same model with no Node built-in reachable from it | +| `scrollcase/contract/types` | The generated TypeScript definitions for every document the format defines | +| `scrollcase/contract/schema/*.json` | The canonical JSON Schemas, resolvable by a mirror implementation | +| `scrollcase/contract/fixtures/*.json` | The golden fixtures a mirror implementation proves itself against | +| `scrollcase/build` | Building, packing, verifying, auditing and workspace resolution | +| `scrollcase/sign` | Key generation, signing, decoding and verification | +| `scrollcase/consumer` | The five local execution operations | + +There is deliberately **no root export**. Importing `scrollcase` gets nothing; every consumer names +the surface it depends on, which is what lets the browser-safe subset stay browser-safe and lets a +consumer-only dependent avoid the entire build layer. + +
+ +
+ +#### `scrollcase/contract` + +| Export | Kind | Meaning | +| --- | --- | --- | +| `BOX_SCHEMA_VERSION` | constant | `2` — the only format version this release reads or writes | +| `CHANNELS` | constant | The closed vocabulary: `nightly`, `beta`, `stable` | +| `DEFAULT_DOCUMENT_NAMESPACE` | constant | `scrollcase.box`, used when a project declares none | +| `PAYLOAD_ENCODING` | constant | The envelope's payload encoding identifier | +| `SIGNATURE_ALGORITHM` | constant | `ed25519` | +| `boxTargetId` | function | The canonical [target ID](#target-id) for a validated target | +| `boxTargetAdapter` | function | The [target adapter](#target-adapter) for one target | +| `boxTargetAdapters` | function | A fresh array of every adapter the format defines | +| `condaSubdir` | function | The [conda subdir](#conda-subdir) a target maps to | +| `pixiAccelerator` | function | The accelerator descriptor a scroll selects | +| `assertNativeHost` | function | Refuses a build on a host that is not the target it ships for | +| `assertPythonEntryPoint` | function | Refuses an entry point that disagrees with the adapter layout | +| `documentKinds` | function | The three `kind` strings for a publishing project's namespace | +| `parseDocumentKind` | function | Splits a `kind` back into namespace and document type | +| `isSignedBoxDocument` | function | The structural envelope guard — shape only, never trust | +| `decodeDocumentPayload` | function | Decodes an envelope payload after checking its hash | +| `schemaUrl`, `fixtureUrl` | function | Resolve a shipped schema or fixture from a dependent package | + +`scrollcase/contract/browser` exports the same set minus `decodeDocumentPayload`, `schemaUrl` and +`fixtureUrl`. The first needs Node's `crypto` to hash a payload; the other two build no more than a +`URL` against the module's own location, and are left out because what they resolve to is a file on +disk beside the installed package rather than something a browser can fetch. + +
+ +
+ +#### `scrollcase/sign` and `scrollcase/consumer` + +| Export | Meaning | +| --- | --- | +| `generateSigningKey` | Creates a local [ed25519](#ed25519) pair, writing the private and public files | +| `readSigningKey` | Reads a pair back, cross-checking that the two files belong together | +| `signDocument` | Signs a payload with a local key, or through an external signer command | +| `verifySignedDocument` | Verifies an envelope against a [trust key](#trust-key) set, named by path or supplied directly | +| `parseTrustedKeys` | Reads both trust-file shapes from text or bytes rather than from a path | +| `resolveTrustedKeys` | Resolves exactly one named trust source — `publicPath` or `trustedKeys` — into the keys verification runs against | +| `decodeSignedDocument` | Decodes an envelope **without** verifying it — named for what it does not do | +| `verifyAndExtractBox` | Verifies a local box and prepares it at a destination, returning a receipt | +| `attachExtractedBox` | Re-identifies an already-extracted box in a new process, without its archive | +| `verifyExtractedPayload` | Proves an installed tree against the entry list its release commits to | +| `runExtractedBox` | Executes a box this process prepared or attached, shell-free, forwarding signals | +| `runBox` | One-shot: prepare into a temporary root, execute, remove every byte | + +
+ +
+ +#### `scrollcase/build` + +| Export | Meaning | +| --- | --- | +| `CONDA_PACK_VERSION` | The conda-pack version a managed install pins | +| `DEFAULT_WORKSPACE_PATHS`, `SCROLLCASE_CONFIG_FILENAME` | The workspace defaults and the config file name | +| `resolveWorkspace`, `configureWorkspace`, `getWorkspace`, `findWorkspaceConfig` | Workspace resolution and the per-process installed workspace | +| `workspaceOverridesFromArgv`, `workspaceOverridesFromFlags` | Turning caller arguments into workspace overrides | +| `findPixi`, `findCondaPack` | Toolchain discovery, in the documented precedence order | +| `pixiLockArguments`, `pixiInstallArguments`, `condaPackArguments` | The exact argument vectors, as data | +| `installAndPackPixiEnvironment` | Solve-free install, pack, extract and repair into the payload | +| `repairPosixLaunchers` | The [shebang](#shebang) trampoline repair | +| `createDeterministicZip`, `listZipEntries`, `extractZipArchive` | Deterministic writing and defensive reading | +| `collectFiles`, `fileExists`, `sha256File` | Payload enumeration and streaming hashing | +| `payloadDigest` | The canonical entry list of an extracted tree, reduced to one hash | +| `boxReleaseStem`, `boxReleaseObjectPrefix`, `builderVersionFields` | Release naming and builder identity | +| `lockedCondaDistributions`, `parseCondaPackageReference` | Reading the lock into package identities | +| `createCondaDependencyLicenseAudit`, `validateCondaDependencyLicenseAudit` | Producing and checking the licence inventory | +| `run`, `runResult`, `fail` | The subprocess seam and the single error path | + +
+ +
+ +### 12.3 Closing + +
+ +Scrollcase is a small tool with a narrow promise, and almost every design decision in this document +exists to keep the promise narrow. A [box](#box) is a Python environment for one operating system and +one accelerator, built from a reviewed [lockfile](#lockfile), packed so that it runs elsewhere, +signed so that it can be proven, and accompanied by an inventory of what it contains. It is not a +registry, not a scheduler, not a distribution system, and not a judge of whether the thing inside it +is scientifically right. + +::: info The whole thing, in one sentence +Given a scroll and a committed lock, Scrollcase produces bytes that can be rebuilt exactly, proven to +come from a known commit and a known key, and refused entirely when any part of that fails. +::: + + + diff --git a/docs/white-paper.md b/docs/white-paper.md index 8dc122a..12b955d 100644 --- a/docs/white-paper.md +++ b/docs/white-paper.md @@ -8,10 +8,11 @@ prev: false # Scrollcase — Technical White Paper -Scrollcase turns a declarative **scroll** into a **box**: a portable, locked, self-contained Python +Scrollcase turns a declarative **scroll** into a **box**: a portable, locked, self-contained environment for one operating system and one accelerator, packed so that it runs somewhere other than where it was built, signed so that whoever receives it can prove what they received, and -accompanied by a dependency licence inventory. +accompanied by a dependency licence inventory. What runs inside it is the box's **runtime** — +`python`, `node`, or `native`, which starts a compiled binary and no interpreter at all. This document describes how that is done, module by module, together with the specifications of the substrate it is built on and a glossary of every technical term it uses. It is written to be @@ -177,12 +178,13 @@ the error messages use no synonym for any of them. #### Box -The built artefact: a single ZIP archive containing a complete, relocated Python environment, a +The built artefact: a single ZIP archive containing a complete, relocated conda-forge environment, a self-describing manifest, any embedded assets, and — when the [scroll](#scroll) declares a reviewed -licence audit — a dependency licence inventory. A box is built -for exactly one [target](#target). It is never called an image or a container — it is neither, it -carries no operating system and no isolation boundary, and borrowing either word would import -expectations Scrollcase does not meet. +licence audit — a dependency licence inventory. A box is built for exactly one [target](#target) and +declares exactly one [runtime](#runtime), which is what says whether the environment holds a Python +interpreter, a Node one, or no interpreter at all. It is never called an image or a container — it is +neither, it carries no operating system and no isolation boundary, and borrowing either word would +import expectations Scrollcase does not meet. Reference: `src/build/box.mjs`, `docs/reference/box-format.md`. @@ -297,11 +299,27 @@ Reference: `revocations-manifest.schema.json`.
+#### Runtime + +What runs *inside* a box, declared by the scroll and signed into the release: where the interpreter +sits, which execution kinds exist, how a declaration becomes a command line, and which inherited +environment variables can change what that command loads. The format defines `python`, `node` and +`native`, and this build implements all three. A [target](#target) says which machine a box runs on; +a runtime says what runs on it, and keeping them separate is what makes a second runtime an adapter +rather than a fork. + +Reference: `src/contract/runtimes.mjs`, `src/runtimes/`. + +
+ +
+ #### Scroll The declarative input, stored as `scroll.json`, and the only input a build accepts. It states box -identity, the target, versions, the dependencies to solve, asset declarations, weights policy, the -self-test, execution intent, and optional compatibility and parity blocks. Scrolls live under +identity, the target, the runtime, versions, the dependencies to solve, asset declarations with +their per-entry `embed` decision, the self-test, execution intent, and optional compatibility, +licence-declaration and parity blocks. Scrolls live under `scrolls///` by default, so that every target variant of one box is visible together. @@ -432,10 +450,12 @@ property `embed` exists to preserve. #### Asset -A file the box needs that is not a Python package — model weights, tokenizers, fixtures. Assets are -declared in the scroll with a URL or local path, a size and a SHA-256, and are size- and -hash-checked before they enter the payload. Under `embed` they are packed into the archive; under -`on-demand` they are left out and their descriptors travel in the signed release. +A file the box needs that its dependency solve does not provide — model weights, tokenizers, +fixtures, a compiled binary. Assets are declared in the scroll with a URL or local path, a size and +a SHA-256, and are size- and hash-checked before they enter the payload. The decision is per entry: +`embed: true`, the default, packs the file into the archive, while `embed: false` leaves it out and +sends its descriptor in the signed release for the consumer to materialise. One box does both at +once. Reference: `src/build/assets.mjs`. @@ -1931,7 +1951,7 @@ problem instead of solving it once per language. The cost is a slightly larger d payload that is not human-readable without a decode step, which is a fair price for a signature that means the same thing everywhere. -Reference: `tests/unit/contract-schema.test.mjs`, `tests/unit/v2-migration.test.mjs`. +Reference: `tests/unit/contract-schema.test.mjs`, `tests/unit/v3-migration.test.mjs`.
@@ -2171,10 +2191,11 @@ non-empty. A path that fails this never reaches the code that would have to reje **`schemaVersion` is `const: 3` in every document schema.** Not a minimum, not a range: an older document fails schema validation with the same finality as the code rejects it. -**`weights` and `assets` are paired by `dependentRequired`** in both the release and the box -manifest. Declaring one without the other is a contradiction — on-demand weights with no asset -descriptors, or asset descriptors on a box claiming to be self-contained — and the schema refuses -both directions. +**`assets` needs no cross-field companion.** Version 2 paired a box-wide `weights` switch with the +descriptor list through `dependentRequired`, because declaring one without the other was a +contradiction. Version 3 moved the decision onto each asset's own `embed` flag, so the list *is* the +declaration: it holds exactly the deferred entries, an all-embedded box carries none, and there is no +second field left to disagree with. @@ -2211,11 +2232,11 @@ The optional fields are where a scroll expresses intent: | `extends` | `../scroll.json`, marking this file as one target's half of a split scroll | | `scrollId` | Provenance identity; derived deterministically as `-` when omitted | | `condaDependencyLicenseAudit` | Path to the reviewed licence inventory the build must still match | -| `assetBaseUrl` | Base URL the built archive and its objects are published under | +| `bundledLicenseDeclaration` | Path to the reviewed licences of what was linked *inside* a binary the box ships — the half `pixi.lock` cannot see | +| `publishBaseUrl` | Where the built archive and its signed documents will be published, so each can point at the next. Optional: a box built to run locally is never published, and the build then omits both links rather than inventing an address | | `assetArchives` | Downloaded archives to expand into the payload, with `stripComponents` and `removeAfterExtract` | | `localFiles` | Files copied from the project's own repository, optionally pinned to a declared hash | | `prunePaths` | Payload paths deleted before packing | -| `weights` | `embed` (default) or `on-demand` | | `execution` | The application entry point | | `parity` | The cross-accelerator numerical gate | @@ -2523,18 +2544,19 @@ interpreter first runs should be able to see it without following a call graph. | 2 | Validate the build options | `box.mjs` | Channel in `CHANNELS`; the deferred-asset list is read off the scroll | | 3 | Refuse an unusable host, toolchain or tree | `targets.mjs`, `pixi.mjs`, `scroll.mjs` | `assertNativeHost`; pinned pixi and conda-pack located; `pixi.lock` present and hashed; git revision read, dirty tree refused | | 4 | Prepare the build tree | `box.mjs` | Removes and recreates `//payload/`; clears the target's object directory under `dist/` | -| 5 | Solve, pack and relocate | `pixi.mjs`, `runtimes/python/launchers.mjs` | Installs into a build-local pixi workspace, packs it, extracts into `payload/venv/`, repairs it, deletes the workspace and tarball | -| 6 | Stage assets | `assets.mjs` | Downloads verified assets, copies verified local files, expands asset archives — the downloads and the archives only when weights are embedded, the local files always | +| 5 | Solve, pack and relocate | `pixi.mjs`, `runtimes//`, `runtimes/launchers.mjs` | Installs into a build-local pixi workspace, packs it, extracts into `payload/venv/`, repairs it through the runtime's own `repairLaunchers`, deletes the workspace and tarball | +| 6 | Stage assets | `assets.mjs` | Downloads verified assets, copies verified local files, expands asset archives — an asset declared `embed: false` is skipped and travels as a descriptor instead, the local files always copied | | 7 | Prune | `box.mjs` | Deletes each `prunePaths` entry from the payload | -| 8 | Licence inventory | `licenses.mjs` | When the scroll declares a reviewed audit: recomputes from the lock, compares against it, writes `payload/THIRD_PARTY_NOTICES/conda-distributions.json` | -| 9 | Post-prune integrity | `box.mjs`, `execution.mjs` | Every `selfTest.files` entry still exists, except an asset deferred by `on-demand`; execution names a real script or discoverable module | -| 10 | Describe | `box.mjs` | Writes `payload/box.json`, so the self-test runs against the payload the box will ship — an application that reads its own manifest to find its files can then be exercised by it | -| 11 | Self-test | `box.mjs` | Runs the adapter's own entry point — `payload/venv/bin/python -c …`, or `venv/python.exe` on Windows — with the target's validation environment | -| 12 | Parity | `parity.mjs` | Runs the declared check once per accelerator and enforces the tolerances | -| 13 | Commit, normalise, measure | `box.mjs`, `filesystem.mjs` | Writes `payload-digest.v1` without listing the list itself, records its hash for the release, stamps every entry with the fixed mtime, and sums the installed size | -| 14 | Archive | `archive.mjs` | Writes `/.zip` deterministically; hashes and measures it | -| 15 | Sign | `sign/index.mjs` | Signs the release, hashes the signed document, signs a channel pointer at 100% | -| 16 | Publish-ready move | `assets.mjs` | Moves archive and release into `dist/boxes////`, writes `dist/channels///.json` | +| 8 | Runtime payload files | `runtimes//` | Whatever the runtime needs in the payload that nothing declares, through its optional `preparePayload` — for `node`, the box's own `package.json`. After the prunes, so a project cannot prune a file about to be written, and before the payload is read, so what it writes is archived and digested like everything else | +| 9 | Licence inventories | `licenses.mjs` | When the scroll declares a reviewed conda audit: recomputes from the lock, compares against it, writes `payload/THIRD_PARTY_NOTICES/conda-distributions.json`. When it declares `bundledLicenseDeclaration`: validates it against the release schema's own `$defs/bundledLicenses`, checks every `linkedInto` path is a file the box carries, writes `payload/THIRD_PARTY_NOTICES/bundled-dependencies.json`, and returns it for the release | +| 10 | Post-prune integrity | `box.mjs`, `execution.mjs` | Every `selfTest.files` entry still exists, except an asset declared `embed: false`; execution names a real script, discoverable module or carried binary, and whatever the box starts will come out of the archive executable | +| 11 | Describe | `box.mjs` | Writes `payload/box.json`, so the self-test runs against the payload the box will ship — an application that reads its own manifest to find its files can then be exercised by it | +| 12 | Self-test | `box.mjs` | Runs every invocation the runtime's probe implies — `payload/venv/bin/python -c …` for a Python box, `venv/bin/node -e …` for a Node one, the declared binary itself for a `native` one — with the target's validation environment | +| 13 | Parity | `parity.mjs` | Runs the declared check once per accelerator and enforces the tolerances. Refused outright for `native`, which has no interpreter to run the check with | +| 14 | Commit, normalise, measure | `box.mjs`, `filesystem.mjs` | Writes `payload-digest.v1` without listing the list itself, records its hash for the release, stamps every entry with the fixed mtime, and sums the installed size | +| 15 | Archive | `archive.mjs` | Writes `/.zip` deterministically; hashes and measures it | +| 16 | Sign | `sign/index.mjs` | Signs the release, hashes the signed document, signs a channel pointer at 100% | +| 17 | Publish-ready move | `assets.mjs` | Moves archive and release into `dist/boxes////`, writes `dist/channels///.json` | Several properties of that order are load-bearing. @@ -2553,13 +2575,17 @@ await rm(objectDir, { recursive: true, force: true }); await mkdir(payloadDir, { recursive: true }); ``` -**Pruning happens before every check that could catch an over-prune.** Stage 9 asks whether the -files the box needs at run time are still present, and stage 11 asks whether it can still import -what it claims. Neither would mean anything if pruning came after them. +**Pruning happens before every check that could catch an over-prune.** Stage 10 asks whether the +files the box needs at run time are still present, and stage 12 asks whether the box can still answer +what it claims. Neither would mean anything if pruning came after them. Stage 8 sits between the +prune and both checks for the same reason from the other direction: a file the runtime writes for +itself must survive the prune and still be seen by everything that reads the payload. **The self-test runs before the payload can become an archive.** This is the step that earns the box -its name: the modules are imported by the payload's *own* interpreter, in the payload directory, -under the target's validation environment. +its name: the probe is answered by the payload's *own* runtime, in the payload directory, under the +target's validation environment — imports through the interpreter for `python` and `node`, and for +`native`, which has no loader to ask, the box's own declared binary run with the arguments the probe +names. The scroll's declared environment is present too; target validation is layered last so it cannot be disabled by a declaration. @@ -2585,7 +2611,7 @@ is no second name for the same bytes. └── macos-aarch64-metal.json ``` -`boxes/` is the tree that goes under the asset base URL verbatim — the same prefix the signed +`boxes/` is the tree that goes under the publish base URL verbatim — the same prefix the signed documents write into their own URLs, so uploading it is a copy rather than a mapping. `channels/` is separate because a channel is not part of any one version: it is a pointer that moves to the next one, and filing it under `1.0.0` would leave a stale copy claiming to be current the moment `1.0.1` @@ -2635,24 +2661,31 @@ installed. -`readExactScroll()` performs seven checks in order: +`readExactScroll()` performs nine checks in order: 1. **The reference is well formed**: exactly `/`, screened by `safeRelativePath`. 2. **The document validates** against the scroll, target and execution schemas, using the internal validator described in 6.4 — after a split scroll has been joined with its base, so what is validated is what the build will read. -3. **A target is declared.** Required of the joined scroll rather than by the schema, so that the +3. **The runtime is one this build implements.** The wire vocabulary is deliberately wider than the + implemented set, so the schema admits an id there is no adapter for and this is where that + becomes a refusal by name rather than a misreading. +4. **The declaration is internally consistent for that runtime**: the execution kind belongs to it, + a `selfTest.commands` probe has an execution to invoke, and no probe shape is declared that the + runtime cannot answer — an `imports` probe in a `native` box is refused here rather than at + self-test time, after the whole build has been paid for. +5. **A target is declared.** Required of the joined scroll rather than by the schema, so that the base of a split scroll still validates on its own. -4. **Weights and archives are compatible**: `on-demand` with `assetArchives` is refused, because - those archives are expanded at build time and cannot be deferred. -5. **Every declared path is safe.** One sweep screens `cacheSubdir`, every asset path, both - ends of every asset archive, both ends of every local file, every prune path, every self-test - file, the self-test Python file, the execution script, the parity script and the licence audit - path. -6. **The directory names agree with the declarations.** The parent directory must equal `boxId` and +6. **Parity is possible.** Refused when the runtime's layout names no entry point, because the gate + runs a source file with the box's own interpreter and a `native` box has none. +7. **Every declared path is safe, and no two land in the same place.** One sweep screens + `cacheSubdir`, every asset path, both ends of every asset archive, both ends of every local file, + every prune path, every uncompressed path, every self-test file, the self-test script, the + execution script or binary, the parity script and both licence declaration paths. +8. **The directory names agree with the declarations.** The parent directory must equal `boxId` and the child must equal the canonical [target ID](#target-id). -7. **The entry point agrees with the runtime's layout for the target**, via - `assertRuntimeEntryPoint`. +9. **The entry point agrees with the runtime's layout for the target**, via + `assertRuntimeEntryPoint` — which for `native` means refusing one outright. Check 6 deserves its reasoning. The layout is `scrolls///`, and the directory names are *checked context*, not identity: the scroll declares both facts, and the filesystem is required @@ -3528,8 +3561,8 @@ ones come after the cheap ones that could have ended the check. // src/build/verify.mjs const AGREEMENT_FIELDS = [ 'schemaVersion', 'boxId', 'labels', 'version', 'target', - 'runtime', 'cacheSubdir', 'environment', 'selfTest', 'execution', - 'weights', 'assets', 'provenance', + 'runtime', 'cacheSubdir', 'bundledLicenses', 'environment', + 'selfTest', 'execution', 'assets', 'provenance', ]; ``` @@ -3538,7 +3571,13 @@ provenance block, the environment map, the asset descriptor list — must agree present. This is what binds the archive's contents to its signed metadata: the release commits to the archive by hash, and the archive's own description of itself must match the release. -Only fields that exist in *both* schema-version-2 documents belong in that list. Release-only +Two entries there carry a reason of their own. `assets` holds the per-entry `embed` decision by +construction — it lists exactly the deferred entries — so a box that quietly changed its mind about +one asset disagrees with its release. `bundledLicenses` is compared for the same reason it is signed +at all: a licence inventory that could differ between the document a reviewer read and the box a user +installed would be worth nothing. + +Only fields that exist in *both* schema-version-3 documents belong in that list. Release-only transport data — `kind`, `archive`, `compatibility`, `installedSizeBytes`, `payloadDigest` — has no counterpart in `box.json`, and demanding one would be demanding agreement about a field that does not exist. The list itself already names and hashes `box.json`; placing its commitment inside that @@ -3647,7 +3686,7 @@ both in one run instead of one per attempt, and every failing check carries its #### `authoring.mjs` — `new scroll` -The only command that authors real project identity, target, versions, compatibility, weights and +The only command that authors real project identity, target, runtime, versions, compatibility and execution intent. Its guarantees are atomicity and non-destruction: - Every material value is validated **before the first write**. A non-terminal call that omits one @@ -4317,7 +4356,7 @@ The absences here are the same boundary the rest of the tool draws, applied to c Everything up to here produces a box. This section is about the other end: a machine that holds a signed [release](#release) document, a trusted public key, and either an archive plus destination or -an already-extracted root, and wants a working Python environment whose identity it can establish. +an already-extracted root, and wants a working box whose identity it can establish. Scrollcase ships **three** implementations of that — in Node, in Python, and in Rust — because the code that consumes a box usually is not the code that built it. They are not an original and its @@ -4345,12 +4384,14 @@ ports; they are three mirrors of one contract, and they are held to it by the sa | Process seam | `spawn` option | `popen_factory` argument | `SpawnBox` trait | | Signal seam | `signalSource` option | `signal.signal` on the main thread | a channel the caller owns | | Schemas | read from `src/contract/schema/` | bundled copies, checked by `sync_schemas.py --check` | bundled copies used by the tests, checked by `sync-assets.mjs --check` | -| Dependencies | `yauzl` for reading archives | `cryptography`, `jsonschema` | `ed25519-dalek`, `zip`, `sha2`, `serde`, `base64` | +| Dependencies | `yauzl` for reading archives | `cryptography`, `jsonschema`, `referencing` | `ed25519-dalek`, `zip`, `sha2`, `serde`, `base64` | The Python package is distributed separately (`scrollcase-consumer` on PyPI, requiring Python 3.10 or newer), ships `py.typed`, and is checked under `mypy --strict`. It depends on `cryptography` for -ed25519 and `jsonschema` for schema validation, and on nothing else; ZIP reading uses the standard -library's `zipfile`. +ed25519, on `jsonschema` for schema validation and on `referencing` for the registry that resolves +the `$ref`s between the bundled schemas, and on nothing else; ZIP reading uses the standard +library's `zipfile`. `jsonschema` installs `referencing` anyway, but a module this package imports +is a dependency this package declares — a transitive one is another project's decision to change. The crate is distributed separately too (`scrollcase-consumer` on crates.io, requiring Rust 1.88 or newer). It forbids `unsafe`, is synchronous throughout so an application chooses its own runtime or @@ -5870,16 +5911,18 @@ guard, because a rule with no guard survives exactly as long as everyone remembe | Rule | Why | Guard | | --- | --- | --- | -| No consuming project's name anywhere in the tool | It must stay usable by projects with nothing to do with the one that first needed it | `tests/unit/v2-migration.test.mjs` greps the whole tracked tree, content and paths | +| No consuming project's name anywhere in the tool | It must stay usable by projects with nothing to do with the one that first needed it | `tests/unit/v3-migration.test.mjs` greps the whole tracked tree, content and paths | | The document namespace belongs to the publishing project | A project with boxes in the field keeps emitting the kinds its clients recognise | `documentKinds(namespace)`; the schemas accept any well-formed namespace and nothing else | | One substrate, and only one | Two dependency backends means proving every guarantee twice | The absence of any second backend, and section 4's single-substrate description | | Published v1 and v2 are immutable; v3 is a clean break | A reinterpreted old document is a silent wrong answer | `decodeSignedDocument` rejects both by name, each with its own remedy | The first guard is worth a note for anyone who reads it. It runs `git grep` over every tracked file -*and* every tracked path, and the retired term it searches for is assembled from two string fragments -inside the test so that the test file itself does not contain the word it forbids. That is not -cleverness for its own sake: a guard that trips on itself gets weakened, and a weakened guard is how -the name comes back. +*and* every tracked path, once for the name of the project Scrollcase was extracted from and once for +a product term retired before the rename. Each word is assembled from two string fragments inside the +test so that the test file itself does not contain what it forbids. That is not cleverness for its +own sake: a guard that trips on itself gets weakened, and a weakened guard is how the name comes +back. Both searches must find nothing — `git grep` exiting 1 — and an invocation that fails for any +other reason fails the test rather than passing quietly.
@@ -6142,7 +6185,7 @@ Twenty-seven test files under `tests/unit/`, plus two shared fixtures under `tes | `scroll-authoring.test.mjs` | Every authored shape — library-only, wizard answers with no weights menu among them, a module with default arguments, the Windows interpreter and conda platform derived from the adapter — plus a staged script hashed at a safe payload path, a generated starter whose declared hash matches its bytes, an initialised example left untouched, consumer templates written without an example and never over an edited one, and refusal to overwrite anything | | `signing.test.mjs` | The external signer: a quoted command with spaces is parsed and its result verified locally; a signer that validly signs a *different* payload is rejected; an invalid signature is rejected even when the payload was echoed exactly | | `toolchain.test.mjs` | The published asset name and URLs per host, a null rather than a guessed URL for an unsupported host, digest parsing in both checksum-file forms, verified installation including across filesystems, a checksum mismatch installing **nothing**, the pinned digest preferred over the server's, the pinned conda-pack version, and the `init` offer — nothing downloaded on a no, only what is missing asked for, both pins recorded, an unsupported host reported | -| `v2-migration.test.mjs` | Only the canonical v2 scroll schema is published; a v1 signed document is rejected with the migration remedy; the channel vocabulary is closed; the workspace exposes no legacy field; and no retired product terminology survives in tracked content or paths | +| `v3-migration.test.mjs` | Only the canonical v3 scroll schema is published; a v1 and a v2 signed document are each rejected with the migration remedy; the channel vocabulary is closed; the workspace exposes no legacy field; and neither the extracted-from project's name nor retired product terminology survives in tracked content or paths | The two helpers are shared rather than duplicated: `consumer-box-fixture.mjs` builds a real signed box — through `createDeterministicZip`, `signDocument` and the real target adapter — for both the local @@ -6176,7 +6219,7 @@ Seven test files under `rust/tests/`, plus one support module. | File | What it proves | | --- | --- | | `contract.rs` | The mirror is faithful: every canonical target-ID and payload-digest vector in the shared fixtures, and the link rule accepting and refusing exactly what the other implementations do | -| `schema.rs` | The types the crate parses with and the canonical schemas reach the same verdict on the examples and on mutations chosen where a typed parse and a schema most plausibly drift — an unknown field, a missing required field, a pattern violation, a broken bound, the `weights`/`assets` co-requirement, and the one open object where agreement means accepting rather than refusing | +| `schema.rs` | The types the crate parses with and the canonical schemas reach the same verdict on the examples and on mutations chosen where a typed parse and a schema most plausibly drift — an unknown field, a missing required field, a pattern violation, a broken bound, and the one open object where agreement means accepting rather than refusing | | `release_document.rs` | The half of the trust chain that needs no archive, over a real release signed the way `signWithLocalKey` signs — so the crate is proved against documents the signer it exists to read produced, not documents it produced itself | | `archive.rs` | The read-only chain over real archives: each case breaks exactly one thing and asserts *which* check fired, because a check that fires for the wrong reason has stopped working | | `prepare.rs` | Preparation, attachment and payload verification: the only three ways to obtain the receipt the execution surface accepts | @@ -6357,6 +6400,7 @@ what a box's runtime is allowed to need that another runtime would not. | `src/cli-init.mjs` | The order of `init`'s questions: the two scaffold questions first, then every answer before any installer | 9.4 | | `src/cli-signing.mjs` | The read-only signing preflight | 7.6 | | `src/cli-run.mjs` | Translating a child's terminal result into this process's own | 8.8 | +| `src/cli-docs.mjs` | The one place the documentation site's URL is written, and the section each interactive question points at | 9.4 | | `src/cli-output.mjs` | Status symbols, the shared question layout, optional colour, and the distribution summary | 9.5 |
@@ -6502,8 +6546,8 @@ disk beside the installed package rather than something a browser can fetch. Scrollcase is a small tool with a narrow promise, and almost every design decision in this document -exists to keep the promise narrow. A [box](#box) is a Python environment for one operating system and -one accelerator, built from a reviewed [lockfile](#lockfile), packed so that it runs elsewhere, +exists to keep the promise narrow. A [box](#box) is one [runtime](#runtime)'s environment for one +operating system and one accelerator, built from a reviewed [lockfile](#lockfile), packed so that it runs elsewhere, signed so that it can be proven, and accompanied by an inventory of what it contains. It is not a registry, not a scheduler, not a distribution system, and not a judge of whether the thing inside it is scientifically right. diff --git a/examples/README.md b/examples/README.md index 5dc8227..29a7658 100644 --- a/examples/README.md +++ b/examples/README.md @@ -97,7 +97,7 @@ declares — the build stops with a mismatch on a checkout that looks perfectly marks the affected paths in [`.gitattributes`](../.gitattributes); a project declaring its own `localFiles` needs the same for the files it names. -## `node-box` +## `hello-box-node` The same thing as `hello-box`, one runtime over: a bare Node 22 environment from conda-forge, a `node-script` entry point, and nothing to download beyond the runtime itself. One target @@ -118,10 +118,10 @@ builder writes one so the walk stops inside the box, and leaves it alone if the one. Ship your own as a `localFile` if you want ESM. ```sh -node src/cli.mjs build node-box/macos-aarch64-metal --scrolls-dir examples +node src/cli.mjs build hello-box-node/macos-aarch64-metal --scrolls-dir examples ``` -## `native-box` +## `hello-box-native` A box with **no interpreter at all**. It packs conda-forge's `zstd` and runs `venv/bin/zstd` directly: the binary is the command line, `runtime.version` and `runtime.entryPoint` are absent, and @@ -139,10 +139,147 @@ that cannot start fails the build rather than the user. The environment is small (`zstd` and `libzlib`) and the licence audit is derived from the lock as usual — `native` means "no interpreter", not "no dependencies". +It is also the one example that declares **no `publishBaseUrl`**, deliberately: nothing here is ever +published, so its release names no address for its archive and its channel names none for its +release. Everything else is unchanged — the archive is hashed, both documents are signed, `verify +--self-test` passes, and `run` works. Compare its release with any other example's to see exactly +what a publish location adds, and what it does not. + +```sh +node src/cli.mjs build hello-box-native/macos-aarch64-metal --scrolls-dir examples +``` + +## `codon-demo` + +What a `node` box is actually for, rather than what it minimally is: a reference table and the tool +that queries it, shipped and signed together. The recipient needs neither Node, nor npm, nor a +database — the box carries its own interpreter, its own data, and the code that joins them. + +The table is the standard genetic code (NCBI translation table 1). `run` with no arguments prints +what the box carries; `run -- ATG` answers forward; `run -- Leucine` answers backwards; an unknown +term exits 1. RNA is accepted too, so `UUG` and `TTG` give the same answer. + +```sh +node src/cli.mjs build codon-demo/macos-aarch64-metal --scrolls-dir examples +node src/cli.mjs run .scrollcase/dist/boxes/codon-demo/1.0.0/macos-aarch64-metal/*.release.json -- Leucine +``` + +```text +Leucine (Leu) is encoded by 6 codons: CTA, CTC, CTG, CTT, TTA, TTG +``` + +Three things in it are worth reading: + +**No npm.** Scrollcase solves from conda-forge and nothing else, so a `node` box cannot declare an +npm dependency. The tool loads its table with `node:sqlite`, which is part of Node itself, and the +JavaScript enters through `localFiles` like any other project file. That is the shape a `node` box +has: conda-forge supplies the runtime and the native libraries, the project supplies the code. + +**Node 26, deliberately.** `node:sqlite` needs a recent Node to be usable without a flag, and a box +that needed `--experimental-sqlite` could not say so: `execution.defaultArgs` land *after* the script +path, never before it. Pinning the runtime was the fix; the scroll is the place that decides. + +**The data is pinned by hash.** `codons.csv` carries its SHA-256 in `localFiles`, so reference data +cannot change without the build stopping. Appending one fabricated row is refused by name — +`Local box file SHA-256 mismatch` — before anything is packed or signed. That is the point of a +signed box carrying data rather than fetching it. + +## `transcode-demo` + +What a `native` box is actually for: ffmpeg, pinned, with everything it links against, signed. The +recipient transcodes with the exact build that was tested, on a machine that has no ffmpeg and needs +no compiler. 121 MB archived, 391 MB extracted, 90 packages in the lock — which is the honest cost +of "just install ffmpeg" made visible. + ```sh -node src/cli.mjs build native-box/macos-aarch64-metal --scrolls-dir examples +node src/cli.mjs build transcode-demo/macos-aarch64-metal --scrolls-dir examples +r=.scrollcase/dist/boxes/transcode-demo/1.0.0/macos-aarch64-metal/*.release.json +node src/cli.mjs run $r -- -version +node src/cli.mjs run $r -- -f lavfi -i "testsrc=duration=2:size=640x480:rate=25" \ + -c:v libx264 -pix_fmt yuv420p /tmp/out.mp4 ``` +The second command writes a real MP4 outside the box, which is worth noticing: `run` extracts to a +temporary directory and deletes it on exit, so anything the box produces has to be written somewhere +the caller names. An application that runs a box repeatedly extracts it durably through a consumer +instead. + +Three things in it are worth reading: + +**No glue.** A `native` box starts one binary with the arguments the scroll fixed, and nothing else. +`defaultArgs` is `["-hide_banner"]`, so every invocation is that plus whatever the caller adds. +There is no script in between, because a box that needed one would be a `node` or `python` box. + +**The self-test proves a real encode.** Not just `-version`: the second probe generates a test +pattern with `lavfi`, encodes it through `libx264` and discards the output. A box whose codecs did +not load fails the build. No media file ships to make that possible — ffmpeg synthesises its own +input, which is the trick that keeps the example free of a sample video. + +**`expectExitCode` is 254, and that is not a typo.** The third probe points ffmpeg at a file that is +not there. ffmpeg reports the negative C error number, `ENOENT` is 2, and a process exit status is +one byte — so `-2` surfaces as 254. It is in the scroll because a self-test asserts the binary's +*real* contract rather than a convention someone assumed: the value was measured against the built +payload, not guessed, after the first build failed expecting 1. + +**The licence inventory earns its place here.** 21 of the 90 packages are GPL-family, including +ffmpeg itself, `x264` and `x265` at GPL-2.0-or-later. Anyone redistributing this box needs to know +that before they ship it, not after — and `scrollcase audit` derives it from the lock rather than +asking anyone to remember. + +## `dataset-demo` + +The second `native` box, and a different kind of program from `transcode-demo`: the HDF5 +command-line tools, which read the format most scientific instrument data and model weights are +stored in. 36 MB archived. It ships a small dataset and the reader together. + +The case it answers is not "I cannot install this" but **"we must all read this file the same +way"**. A signed box fixes the reader, so an inspection somebody publishes is one anybody can +repeat. + +```sh +node src/cli.mjs build dataset-demo/macos-aarch64-metal --scrolls-dir examples +r=.scrollcase/dist/boxes/dataset-demo/1.0.0/macos-aarch64-metal/*.release.json +node src/cli.mjs run $r -- -H readings.h5 +node src/cli.mjs run $r -- -d /measurements/monthly readings.h5 +``` + +```text +GROUP "measurements" { + DATASET "monthly" { + DATATYPE H5T_IEEE_F64LE + DATASPACE SIMPLE { ( 12, 3 ) / ( 12, 3 ) } +``` + +`readings.h5` is pinned by SHA-256 and generated rather than committed blind: `readings.txt` and +`readings.conf` sit beside it, and `h5import readings.txt -c readings.conf -o readings.h5` rebuilds +it. The numbers are a synthetic seasonal series — the point is the format and the reader, not the +measurement. + +### Why not a bioinformatics tool + +That was the intent, and conda-forge is the reason it is not. **Almost every bioinformatics package +lives on bioconda**, a second channel: `samtools`, `bwa`, `seqkit`, `minimap2`, `hmmer`, `diamond`, +`blast`, `muscle` and `fasttree` are all absent from conda-forge. Adding a channel to one example +would demonstrate something this project does not claim, so it was not done. + +`mafft` is the exception that is present — and it fails as a `native` box, instructively. Its +`venv/bin/mafft` is a shell wrapper that finds its helper binaries through a path compiled into it: + +```text +prefix=/Users/runner/miniforge3/conda-bld/mafft_.../_h_env_placehold_placehold_.../libexec/mafft +``` + +That is the machine that *built the conda package*, and Scrollcase does not repair a binary's — or a +script's — recorded paths. The package offers `MAFFT_BINARIES` as an override, but it must be +absolute, and a box is extracted to a different temporary directory on every run: the signed +`environment` is a fixed string map with no substitution, so there is nothing correct to put in it. +The self-test caught it before anything was signed. + +This is the second instance of the limitation `hello-box-native` documents, in a new shape — there a +dylib re-exported through an unrewritten path, here a wrapper script. **Before choosing a program +for a `native` box, check what it actually is.** `file venv/bin/` answering "shell script" +is the warning; `Mach-O 64-bit executable` or an ELF binary is what relocates cleanly. + ## `sentiment-demo` The same pipeline carrying a real model: DistilBERT SST-2 quantised to INT8 in ONNX form, with the diff --git a/examples/codon-demo/macos-aarch64-metal/codons.csv b/examples/codon-demo/macos-aarch64-metal/codons.csv new file mode 100644 index 0000000..e7f4a39 --- /dev/null +++ b/examples/codon-demo/macos-aarch64-metal/codons.csv @@ -0,0 +1,65 @@ +codon,symbol,abbrev,name,role +TTT,F,Phe,Phenylalanine, +TTC,F,Phe,Phenylalanine, +TTA,L,Leu,Leucine, +TTG,L,Leu,Leucine,start +TCT,S,Ser,Serine, +TCC,S,Ser,Serine, +TCA,S,Ser,Serine, +TCG,S,Ser,Serine, +TAT,Y,Tyr,Tyrosine, +TAC,Y,Tyr,Tyrosine, +TAA,*,Ter,Stop,stop +TAG,*,Ter,Stop,stop +TGT,C,Cys,Cysteine, +TGC,C,Cys,Cysteine, +TGA,*,Ter,Stop,stop +TGG,W,Trp,Tryptophan, +CTT,L,Leu,Leucine, +CTC,L,Leu,Leucine, +CTA,L,Leu,Leucine, +CTG,L,Leu,Leucine,start +CCT,P,Pro,Proline, +CCC,P,Pro,Proline, +CCA,P,Pro,Proline, +CCG,P,Pro,Proline, +CAT,H,His,Histidine, +CAC,H,His,Histidine, +CAA,Q,Gln,Glutamine, +CAG,Q,Gln,Glutamine, +CGT,R,Arg,Arginine, +CGC,R,Arg,Arginine, +CGA,R,Arg,Arginine, +CGG,R,Arg,Arginine, +ATT,I,Ile,Isoleucine, +ATC,I,Ile,Isoleucine, +ATA,I,Ile,Isoleucine, +ATG,M,Met,Methionine,start +ACT,T,Thr,Threonine, +ACC,T,Thr,Threonine, +ACA,T,Thr,Threonine, +ACG,T,Thr,Threonine, +AAT,N,Asn,Asparagine, +AAC,N,Asn,Asparagine, +AAA,K,Lys,Lysine, +AAG,K,Lys,Lysine, +AGT,S,Ser,Serine, +AGC,S,Ser,Serine, +AGA,R,Arg,Arginine, +AGG,R,Arg,Arginine, +GTT,V,Val,Valine, +GTC,V,Val,Valine, +GTA,V,Val,Valine, +GTG,V,Val,Valine, +GCT,A,Ala,Alanine, +GCC,A,Ala,Alanine, +GCA,A,Ala,Alanine, +GCG,A,Ala,Alanine, +GAT,D,Asp,Aspartic acid, +GAC,D,Asp,Aspartic acid, +GAA,E,Glu,Glutamic acid, +GAG,E,Glu,Glutamic acid, +GGT,G,Gly,Glycine, +GGC,G,Gly,Glycine, +GGA,G,Gly,Glycine, +GGG,G,Gly,Glycine, diff --git a/examples/codon-demo/macos-aarch64-metal/conda-licenses.json b/examples/codon-demo/macos-aarch64-metal/conda-licenses.json new file mode 100644 index 0000000..7ef1ff2 --- /dev/null +++ b/examples/codon-demo/macos-aarch64-metal/conda-licenses.json @@ -0,0 +1,104 @@ +{ + "schemaVersion": 2, + "kind": "scrollcase.box.dependency-license-audit", + "targetId": "macos-aarch64-metal", + "dependencyLockSha256": "fbe564fdaebc65b1cb2d030c2d1a55bc538ae954e34ea5148cddc91a1ddd11af", + "packages": [ + { + "name": "c-ares", + "version": "1.34.8", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "ca-certificates", + "version": "2026.7.22", + "declaredLicense": "ISC", + "source": "conda" + }, + { + "name": "icu", + "version": "78.3", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libabseil", + "version": "20260526.0", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libbrotlicommon", + "version": "1.2.0", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libbrotlidec", + "version": "1.2.0", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libbrotlienc", + "version": "1.2.0", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libcxx", + "version": "23.1.0", + "declaredLicense": "Apache-2.0 WITH LLVM-exception", + "source": "conda" + }, + { + "name": "libev", + "version": "4.33", + "declaredLicense": "BSD-2-Clause OR GPL-2.0-or-later", + "source": "conda" + }, + { + "name": "libnghttp2", + "version": "1.68.1", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libsqlite", + "version": "3.53.4", + "declaredLicense": "blessing", + "source": "conda" + }, + { + "name": "libuv", + "version": "1.52.1", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libzlib", + "version": "1.3.2", + "declaredLicense": "Zlib", + "source": "conda" + }, + { + "name": "nodejs", + "version": "26.6.0", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "openssl", + "version": "3.6.4", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "zstd", + "version": "1.5.7", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + } + ] +} diff --git a/examples/codon-demo/macos-aarch64-metal/entrypoint.js b/examples/codon-demo/macos-aarch64-metal/entrypoint.js new file mode 100644 index 0000000..ea2deeb --- /dev/null +++ b/examples/codon-demo/macos-aarch64-metal/entrypoint.js @@ -0,0 +1,88 @@ +// A reference table and the tool that queries it, shipped together in one signed box. +// +// This is the shape a `node` box is actually for: the recipient needs neither Node nor npm nor a +// database — the box carries its own interpreter, its own data, and the code that joins them. The +// data is loaded into SQLite through `node:sqlite`, which is part of Node itself, so the whole tool +// has no dependency that conda-forge did not install. +// +// The table is the standard genetic code (NCBI translation table 1), generated from its canonical +// published form rather than typed out, and pinned by hash in the scroll: reference data that +// changed silently would make every answer below wrong without anything failing. + +const { readFileSync } = require('node:fs'); +const { join } = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); + +/** Loads the shipped CSV into an in-memory table. Sixty-four rows: reading it all is the fast path. */ +function openTable() { + const csv = readFileSync(join(__dirname, 'codons.csv'), 'utf8').trim().split('\n'); + const [, ...rows] = csv; + const db = new DatabaseSync(':memory:'); + db.exec('CREATE TABLE codon (codon TEXT PRIMARY KEY, symbol TEXT, abbrev TEXT, name TEXT, role TEXT)'); + const insert = db.prepare('INSERT INTO codon VALUES (?, ?, ?, ?, ?)'); + for (const row of rows) insert.run(...row.split(',')); + return db; +} + +/** What the box says about itself when asked nothing: the shape of the data it carries. */ +function summarise(db) { + const { codons, aminoAcids } = db.prepare( + 'SELECT COUNT(*) AS codons, COUNT(DISTINCT symbol) AS aminoAcids FROM codon', + ).get(); + console.log(`${codons} codons, ${aminoAcids} distinct outcomes (20 amino acids and stop).`); + const starts = db.prepare("SELECT codon FROM codon WHERE role = 'start' ORDER BY codon").all(); + console.log(`Start codons: ${starts.map(({ codon }) => codon).join(', ')}`); + console.log(''); + console.log('Codons per amino acid, most redundant first:'); + const grouped = db.prepare(` + SELECT abbrev, name, COUNT(*) AS n, GROUP_CONCAT(codon, ' ') AS codons + FROM codon GROUP BY symbol ORDER BY n DESC, abbrev + `).all(); + for (const { abbrev, name, n, codons: list } of grouped) { + console.log(` ${abbrev.padEnd(4)} ${String(n).padStart(2)} ${name.padEnd(14)} ${list}`); + } +} + +/** A three-letter DNA codon: the forward question, one row out. */ +function lookupCodon(db, codon) { + const row = db.prepare('SELECT * FROM codon WHERE codon = ?').get(codon); + if (!row) return false; + const role = row.role ? ` (${row.role} codon)` : ''; + console.log(`${row.codon} → ${row.abbrev} (${row.symbol}) ${row.name}${role}`); + return true; +} + +/** A symbol, abbreviation or name: the reverse question, every codon that encodes it. */ +function lookupAminoAcid(db, term) { + const rows = db.prepare(` + SELECT codon, abbrev, name FROM codon + WHERE symbol = ?1 COLLATE NOCASE OR abbrev = ?1 COLLATE NOCASE OR name = ?1 COLLATE NOCASE + ORDER BY codon + `).all(term); + if (rows.length === 0) return false; + const [{ abbrev, name }] = rows; + const plural = rows.length === 1 ? 'codon' : 'codons'; + console.log(`${name} (${abbrev}) is encoded by ${rows.length} ${plural}: ${rows.map((r) => r.codon).join(', ')}`); + return true; +} + +function main(argv) { + const db = openTable(); + if (argv.length === 0) { + summarise(db); + return 0; + } + const term = argv[0].trim(); + // A codon and an amino acid are told apart by shape, not by a flag: three DNA bases can only be + // the first, and anything else can only be the second. + const asCodon = /^[ACGTUacgtu]{3}$/.test(term); + const found = asCodon + ? lookupCodon(db, term.toUpperCase().replaceAll('U', 'T')) + : lookupAminoAcid(db, term); + if (found) return 0; + console.error(`Not in the standard genetic code: ${term}`); + console.error('Give a codon (TTG), or an amino acid by symbol, abbreviation or name (L, Leu, Leucine).'); + return 1; +} + +process.exitCode = main(process.argv.slice(2)); diff --git a/examples/codon-demo/macos-aarch64-metal/pixi.lock b/examples/codon-demo/macos-aarch64-metal/pixi.lock new file mode 100644 index 0000000..c3d2b4a --- /dev/null +++ b/examples/codon-demo/macos-aarch64-metal/pixi.lock @@ -0,0 +1,249 @@ +version: 7 +platforms: +- name: osx-arm64 + virtual-packages: + - __unix=0=0 + - __osx=13.0 + - __archspec=0=m1 +environments: + default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + packages: + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.8-h74c22ad_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-py310h579977c_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libabseil-20260526.0-cxx17_h4c27e2a_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlicommon-1.2.0-h1dcdb26_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlidec-1.2.0-h5295a6a_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlienc-1.2.0-h2ddc9cb_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-23.1.0-h55c6f16_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libev-4.33-h1a92334_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libnghttp2-1.68.1-h435687b_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.53.4-hca69786_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libuv-1.52.1-h1a92334_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/nodejs-26.6.0-h00e74ec_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.4-h55eecbc_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda +packages: +- conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + sha256: 0a0544cf95f64394fe4959286f5c71f5444ad58feb0602e53becb27448d24da6 + md5: 0f51e2391ade309db462a55611263e9c + depends: + - __unix + license: ISC + run_exports: {} + size: 131780 + timestamp: 1784754889428 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.8-h74c22ad_2.conda + sha256: 82bdd5a3fcada6afef6255a9bf41172f07b362f36e04a354dc2abc0a29bfb756 + md5: 4cc01a374215de38387d45e19c8c5c9c + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - c-ares >=1.34.8,<2.0a0 + size: 197819 + timestamp: 1787169996553 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-py310h579977c_2.conda + sha256: 6cdb5dee54c72e56ab189fb3ad33cb28533553d42590e7e831160248f4416a43 + md5: a5efc0b42bb8b42e97d0a29ae3e3c187 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - icu >=78.3,<79.0a0 + size: 14070242 + timestamp: 1786545847761 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libabseil-20260526.0-cxx17_h4c27e2a_2.conda + sha256: d9c62dae9d8f5bebadf72fcf369c00cc353183cb2521f9fffbf4c2f70b58bef9 + md5: 2aa5e7dc7b5d218908effd2a8c70a8a8 + depends: + - __osx >=11.0 + - libcxx >=19 + constrains: + - abseil-cpp =20260526.0 + - libabseil-static =20260526.0=cxx17* + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - libabseil >=20260526.0,<20260527.0a0 + - libabseil =*=cxx17* + size: 1277537 + timestamp: 1787217005681 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlicommon-1.2.0-h1dcdb26_3.conda + sha256: 5e62b856b2e77ce98db44133bacab1281b42c1040dcbd69dbacfb80890cff5b0 + md5: b457450ba3f27c4749783c0204bd17b0 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - libbrotlicommon >=1.2.0,<1.3.0a0 + size: 80027 + timestamp: 1786622846050 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlidec-1.2.0-h5295a6a_3.conda + sha256: 510c9fce0d9ffcf41741dd96fc05db381672109d5665ef002c2e58f0d4ca0118 + md5: e07a99c6fdd984f4d588060f2f936bf4 + depends: + - __osx >=11.0 + - libbrotlicommon 1.2.0 h1dcdb26_3 + license: MIT + license_family: MIT + run_exports: + weak: + - libbrotlidec >=1.2.0,<1.3.0a0 + size: 29935 + timestamp: 1786622857695 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlienc-1.2.0-h2ddc9cb_3.conda + sha256: eac412417eee2e93c62e9d53559f1f9c14f40b6c41e2c432af8b517596898dd1 + md5: 954c78a9f591bfb12c79beaff7338ec8 + depends: + - __osx >=11.0 + - libbrotlicommon 1.2.0 h1dcdb26_3 + license: MIT + license_family: MIT + run_exports: + weak: + - libbrotlienc >=1.2.0,<1.3.0a0 + size: 295650 + timestamp: 1786622868044 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-23.1.0-h55c6f16_0.conda + sha256: d3e5f0b767964af25ef81edffc002f8e822b1a5c55330da1ae3a857f70c4e4ee + md5: 5303ba06fab927399ed8dfb3227b0af8 + depends: + - __osx >=11.0 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + run_exports: {} + size: 575940 + timestamp: 1787698697875 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libev-4.33-h1a92334_3.conda + sha256: c0100064506ae8abb432c5a506d474f10af2cf48c33d62bc221fb28b6d6ff6ac + md5: 19e86c8a6a47e92bb2e70ca12e758c5c + depends: + - __osx >=11.0 + license: BSD-2-Clause OR GPL-2.0-or-later + license_family: GPL + run_exports: + weak: + - libev >=4.33,<4.34.0a0 + size: 39991 + timestamp: 1785917376956 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libnghttp2-1.68.1-h435687b_1.conda + sha256: 445440d8cccccb6585f71d1eb7c949d64a2af2ced5481d924621424ff3b6a398 + md5: ac9902dcbe0417f50cf95ab2bde062fa + depends: + - __osx >=11.0 + - c-ares >=1.34.8,<2.0a0 + - libcxx >=21 + - libev >=4.33,<4.34.0a0 + - libev >=4.33,<5.0a0 + - libzlib >=1.3.2,<2.0a0 + - openssl >=3.5.7,<4.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - libnghttp2 >=1.68.1,<2.0a0 + size: 567024 + timestamp: 1787177422467 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.53.4-hca69786_1.conda + sha256: 839b31d4830e896b4d315b551d27f2bb08026bc946df04bcc360d8627c3ba2cd + md5: fde96d40ebe9a9cb34e4122380189ddb + depends: + - __osx >=11.0 + - icu >=78.3,<79.0a0 + - libzlib >=1.3.2,<2.0a0 + license: blessing + run_exports: + weak: + - libsqlite >=3.53.4,<4.0a0 + size: 942754 + timestamp: 1787051243846 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libuv-1.52.1-h1a92334_1.conda + sha256: 4f47de9de1990efd998edbbd6793f89c8f02ffde987ae8120b1c006acefd2a04 + md5: de09bd0f175611e94f21b28f8c708e80 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - libuv >=1.52.1,<2.0a0 + size: 122729 + timestamp: 1785914645797 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + sha256: a18fa5d5bac452401459f966cf0d872224e8080c4ff93c77e168d43ab42ef9d7 + md5: f39288f0ea63ae962e1a2e4f355a0d75 + depends: + - __osx >=11.0 + constrains: + - zlib 1.3.2 *_3 + license: Zlib + license_family: Other + run_exports: + weak: + - libzlib >=1.3.2,<2.0a0 + size: 47822 + timestamp: 1785277049190 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/nodejs-26.6.0-h00e74ec_0.conda + sha256: 379ba26eb11cccb4d9a10989d9afef3f8bc4c9039cbe371dec3e3ee24594597b + md5: f0ba635c2293dff6fb86fc2150c8c110 + depends: + - libcxx >=19 + - __osx >=12.0 + - libnghttp2 >=1.68.1,<2.0a0 + - libzlib >=1.3.2,<2.0a0 + - libbrotlicommon >=1.2.0,<1.3.0a0 + - libbrotlienc >=1.2.0,<1.3.0a0 + - libbrotlidec >=1.2.0,<1.3.0a0 + - libuv >=1.52.1,<2.0a0 + - icu >=78.3,<79.0a0 + - openssl >=3.5.7,<4.0a0 + - libabseil >=20260526.0,<20260527.0a0 + - libabseil * cxx17* + - zstd >=1.5.7,<1.6.0a0 + - c-ares >=1.34.8,<2.0a0 + - libsqlite >=3.53.4,<4.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - nodejs >=26.6.0,<27.0a0 + size: 18228083 + timestamp: 1785852758781 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.4-h55eecbc_0.conda + sha256: f23239eacd75c4c50705e68fae1aa3292da473e6a3a4abe2330f1e6afa680704 + md5: ae71ab40048c19a389a7dcccb86c2481 + depends: + - __osx >=11.0 + - ca-certificates + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - openssl >=3.6.4,<4.0a0 + size: 3110142 + timestamp: 1787698648639 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda + sha256: da867f5092eb0cb746d353694f0098031fd9817a4ce7d5743121209ae0f406ca + md5: 4ec2684c73812cc2c3d78379384a39cc + depends: + - __osx >=11.0 + - libzlib >=1.3.2,<2.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - zstd >=1.5.7,<1.6.0a0 + size: 433687 + timestamp: 1786599629846 diff --git a/examples/codon-demo/macos-aarch64-metal/pixi.toml b/examples/codon-demo/macos-aarch64-metal/pixi.toml new file mode 100644 index 0000000..017e828 --- /dev/null +++ b/examples/codon-demo/macos-aarch64-metal/pixi.toml @@ -0,0 +1,16 @@ +# A `node` box that ships reference data and the tool that queries it. +# +# Node and nothing else. The tool reads its table with `node:sqlite`, which is part of Node itself, +# so the box has no dependency conda-forge did not install — and none from npm, which is not a +# channel Scrollcase solves from. JavaScript that a project writes enters through `localFiles`. +# +# Pinned to 26 rather than the authoring default: `node:sqlite` is only usable without a flag from +# a recent Node, and a box that needed `--experimental-sqlite` could not declare that through +# `execution.defaultArgs`, which land after the script rather than before it. +[workspace] +name = "codon-demo" +channels = ["conda-forge"] +platforms = ["osx-arm64"] + +[dependencies] +nodejs = "26.*" diff --git a/examples/codon-demo/macos-aarch64-metal/scroll.json b/examples/codon-demo/macos-aarch64-metal/scroll.json new file mode 100644 index 0000000..881c0b5 --- /dev/null +++ b/examples/codon-demo/macos-aarch64-metal/scroll.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "schemaVersion": 3, + "boxId": "codon-demo", + "version": "1.0.0", + "sourceRevision": "ncbi-translation-table-1", + "target": { + "platform": "macos", + "arch": "aarch64", + "accelerator": "metal" + }, + "compatibility": { + "minHostAppVersion": "1.0.0", + "minMacosVersion": "13.0" + }, + "runtime": { + "id": "node", + "version": "26" + }, + "pixiVersion": "0.73.0", + "condaDependencyLicenseAudit": "examples/codon-demo/macos-aarch64-metal/conda-licenses.json", + "cacheSubdir": "cache/codon-demo", + "publishBaseUrl": "https://assets.example.org/boxes", + "selfTest": { + "imports": [ + "node:sqlite", + "node:fs" + ], + "files": [ + "entrypoint.js", + "codons.csv" + ], + "commands": [ + { "args": [] }, + { "args": ["ATG"] }, + { "args": ["Leucine"] }, + { "args": ["ZZZ"], "expectExitCode": 1 } + ] + }, + "localFiles": [ + { + "sourcePath": "examples/codon-demo/macos-aarch64-metal/entrypoint.js", + "relativePath": "entrypoint.js" + }, + { + "sourcePath": "examples/codon-demo/macos-aarch64-metal/codons.csv", + "relativePath": "codons.csv", + "sha256": "6e33318966fc91da6281e4bcf245f5451c91cf6c0eb64dbc8bb9e07fcd829ae1" + } + ], + "execution": { + "kind": "node-script", + "script": "entrypoint.js", + "defaultArgs": [] + } +} diff --git a/examples/dataset-demo/macos-aarch64-metal/conda-licenses.json b/examples/dataset-demo/macos-aarch64-metal/conda-licenses.json new file mode 100644 index 0000000..d894245 --- /dev/null +++ b/examples/dataset-demo/macos-aarch64-metal/conda-licenses.json @@ -0,0 +1,200 @@ +{ + "schemaVersion": 2, + "kind": "scrollcase.box.dependency-license-audit", + "targetId": "macos-aarch64-metal", + "dependencyLockSha256": "a7586aac10bfec4c3def51b92c3865aa5d59f2b32cc2beef3660b5af48a80d1a", + "packages": [ + { + "name": "_openmp_mutex", + "version": "4.5", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "aws-c-auth", + "version": "0.10.4", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "aws-c-cal", + "version": "0.9.15", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "aws-c-common", + "version": "0.14.3", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "aws-c-compression", + "version": "0.3.2", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "aws-c-http", + "version": "0.11.0", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "aws-c-io", + "version": "0.27.5", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "aws-c-s3", + "version": "0.13.1", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "aws-c-sdkutils", + "version": "0.2.7", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "aws-checksums", + "version": "0.2.10", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "c-ares", + "version": "1.34.8", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "ca-certificates", + "version": "2026.7.22", + "declaredLicense": "ISC", + "source": "conda" + }, + { + "name": "hdf5", + "version": "2.2.0", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "icu", + "version": "78.3", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "krb5", + "version": "1.22.2", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libaec", + "version": "1.1.5", + "declaredLicense": "BSD-2-Clause", + "source": "conda" + }, + { + "name": "libcurl", + "version": "8.21.0", + "declaredLicense": "curl", + "source": "conda" + }, + { + "name": "libcxx", + "version": "23.1.0", + "declaredLicense": "Apache-2.0 WITH LLVM-exception", + "source": "conda" + }, + { + "name": "libedit", + "version": "3.1.20250104", + "declaredLicense": "BSD-2-Clause", + "source": "conda" + }, + { + "name": "libev", + "version": "4.33", + "declaredLicense": "BSD-2-Clause OR GPL-2.0-or-later", + "source": "conda" + }, + { + "name": "libgcc", + "version": "16.2.0", + "declaredLicense": "GPL-3.0-only WITH GCC-exception-3.1", + "source": "conda" + }, + { + "name": "libgfortran", + "version": "16.2.0", + "declaredLicense": "GPL-3.0-only WITH GCC-exception-3.1", + "source": "conda" + }, + { + "name": "libgfortran5", + "version": "16.2.0", + "declaredLicense": "GPL-3.0-only WITH GCC-exception-3.1", + "source": "conda" + }, + { + "name": "libnghttp2", + "version": "1.68.1", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libpsl", + "version": "0.23.1", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libssh2", + "version": "1.11.1", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "libzlib", + "version": "1.3.2", + "declaredLicense": "Zlib", + "source": "conda" + }, + { + "name": "llvm-openmp", + "version": "23.1.0", + "declaredLicense": "Apache-2.0 WITH LLVM-exception", + "source": "conda" + }, + { + "name": "ncurses", + "version": "6.6", + "declaredLicense": "X11 AND BSD-3-Clause", + "source": "conda" + }, + { + "name": "openssl", + "version": "3.6.4", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "s2n", + "version": "1.7.6", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "zstd", + "version": "1.5.7", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + } + ] +} diff --git a/examples/dataset-demo/macos-aarch64-metal/pixi.lock b/examples/dataset-demo/macos-aarch64-metal/pixi.lock new file mode 100644 index 0000000..730b2c3 --- /dev/null +++ b/examples/dataset-demo/macos-aarch64-metal/pixi.lock @@ -0,0 +1,491 @@ +version: 7 +platforms: +- name: osx-arm64 + virtual-packages: + - __unix=0=0 + - __osx=13.0 + - __archspec=0=m1 +environments: + default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + packages: + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/_openmp_mutex-4.5-8_kmp_llvm.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-auth-0.10.4-h436dbe7_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-cal-0.9.15-hdbc54b2_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-common-0.14.3-h84a0fba_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-compression-0.3.2-he8604bc_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-http-0.11.0-hc1b69ad_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-io-0.27.5-h8fb7669_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-s3-0.13.1-h1fbe2f8_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-sdkutils-0.2.7-he8604bc_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-checksums-0.2.10-he8604bc_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.8-h74c22ad_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/hdf5-2.2.0-nompi_h7eff4e6_100.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-py310h579977c_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/krb5-1.22.2-h34f8a20_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libaec-1.1.5-h8664d51_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcurl-8.21.0-h6651222_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-23.1.0-h55c6f16_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libedit-3.1.20250104-pl5321h26f1114_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libev-4.33-h1a92334_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgcc-16.2.0-h3cf6597_4.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran-16.2.0-h07b0088_4.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran5-16.2.0-hdb7a957_4.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libnghttp2-1.68.1-h435687b_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libpsl-0.23.1-hdb0c161_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libssh2-1.11.1-hbf2880b_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/llvm-openmp-23.1.0-hdb3d66b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.6-he64c551_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.4-h55eecbc_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/s2n-1.7.6-h1b28cb6_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda +packages: +- conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + sha256: 0a0544cf95f64394fe4959286f5c71f5444ad58feb0602e53becb27448d24da6 + md5: 0f51e2391ade309db462a55611263e9c + depends: + - __unix + license: ISC + run_exports: {} + size: 131780 + timestamp: 1784754889428 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/_openmp_mutex-4.5-8_kmp_llvm.conda + build_number: 8 + sha256: c7c98ce74f6a7ff25aaa4706d06fa41d63a333add4d697b73aa4d8d2d7ad7651 + md5: a2d706b4a7d603524133037c34f7fb76 + depends: + - llvm-openmp >=9.0.1 + license: BSD-3-Clause + run_exports: + weak: + - _openmp_mutex >=4.5 + size: 8016 + timestamp: 1788046437162 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-auth-0.10.4-h436dbe7_2.conda + sha256: bea3d014ebc03803cd18b1df1c7cbeb2fb7cf260126030fe7dc18a6bf674d131 + md5: b9494d6793762d9f9838420c8e73c8d9 + depends: + - __osx >=11.0 + - aws-c-cal >=0.9.15,<0.9.16.0a0 + - aws-c-sdkutils >=0.2.7,<0.2.8.0a0 + - aws-c-common >=0.14.3,<0.14.4.0a0 + - aws-c-io >=0.27.5,<0.27.6.0a0 + - aws-c-http >=0.11.0,<0.11.1.0a0 + license: Apache-2.0 + license_family: APACHE + run_exports: + weak: + - aws-c-auth >=0.10.4,<0.10.5.0a0 + size: 117346 + timestamp: 1785199104682 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-cal-0.9.15-hdbc54b2_1.conda + sha256: f60b9b4b9693ce86e335c268638c299804ca9cfd7ffb3b96415dcac52c1bd13b + md5: 70d1d0405910b78d06d74a8bba0013d2 + depends: + - __osx >=11.0 + - aws-c-common >=0.14.3,<0.14.4.0a0 + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - aws-c-cal >=0.9.15,<0.9.16.0a0 + size: 46154 + timestamp: 1785158287452 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-common-0.14.3-h84a0fba_0.conda + sha256: e46cbb8c24626da9792f7eebae86b3e8fa611e039945be15a78a70015eeae17e + md5: 0097da38976713a53acb6332cc9de888 + depends: + - __osx >=11.0 + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - aws-c-common >=0.14.3,<0.14.4.0a0 + size: 228913 + timestamp: 1784596525019 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-compression-0.3.2-he8604bc_5.conda + sha256: 857df6c5048c206e4c424100f1a649b7ef5ee3b01cf6207ff3ae35fe61642478 + md5: 139a1ac73d9e111cb3f73d07958a1130 + depends: + - __osx >=11.0 + - aws-c-common >=0.14.3,<0.14.4.0a0 + license: Apache-2.0 + license_family: APACHE + run_exports: + weak: + - aws-c-compression >=0.3.2,<0.3.3.0a0 + size: 21777 + timestamp: 1785157595687 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-http-0.11.0-hc1b69ad_5.conda + sha256: 1d8dbd1cea250a1d2d932294bf7e9d5c9e5efc9e09993dc883885e0ff2447fd3 + md5: d7d04ddaf9dae601a9a77c547e7d09ab + depends: + - __osx >=11.0 + - aws-c-cal >=0.9.15,<0.9.16.0a0 + - aws-c-compression >=0.3.2,<0.3.3.0a0 + - aws-c-common >=0.14.3,<0.14.4.0a0 + - aws-c-io >=0.27.5,<0.27.6.0a0 + license: Apache-2.0 + license_family: APACHE + run_exports: + weak: + - aws-c-http >=0.11.0,<0.11.1.0a0 + size: 177804 + timestamp: 1785186207384 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-io-0.27.5-h8fb7669_1.conda + sha256: 7b346b8bd32c1f013d231922a9413d45c152d5c533c994bb4fb76c5f4e1ad20b + md5: 4fa643897c9481edc41aca4a7597a653 + depends: + - __osx >=11.0 + - aws-c-common >=0.14.3,<0.14.4.0a0 + - s2n >=1.7.6,<1.7.7.0a0 + - aws-c-cal >=0.9.15,<0.9.16.0a0 + license: Apache-2.0 + license_family: APACHE + run_exports: + weak: + - aws-c-io >=0.27.5,<0.27.6.0a0 + size: 190778 + timestamp: 1785170135436 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-s3-0.13.1-h1fbe2f8_1.conda + sha256: fe906dcd9cfac79c61bf60927f9e9dad328c9fbb9a2204ffb0b3c955c33817f0 + md5: bbdbec4b0952af625b066672514c0eb7 + depends: + - __osx >=11.0 + - aws-checksums >=0.2.10,<0.2.11.0a0 + - aws-c-io >=0.27.5,<0.27.6.0a0 + - aws-c-common >=0.14.3,<0.14.4.0a0 + - aws-c-http >=0.11.0,<0.11.1.0a0 + - aws-c-cal >=0.9.15,<0.9.16.0a0 + - aws-c-auth >=0.10.4,<0.10.5.0a0 + license: Apache-2.0 + license_family: APACHE + run_exports: + weak: + - aws-c-s3 >=0.13.1,<0.13.2.0a0 + size: 134318 + timestamp: 1785351362781 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-c-sdkutils-0.2.7-he8604bc_3.conda + sha256: 16b3cd17f3806866c74fe7fe7badaf511293766f1356b4f491f88eb98ff4da16 + md5: b87573f533751acc5de162eced05e3b1 + depends: + - __osx >=11.0 + - aws-c-common >=0.14.3,<0.14.4.0a0 + license: Apache-2.0 + license_family: APACHE + run_exports: + weak: + - aws-c-sdkutils >=0.2.7,<0.2.8.0a0 + size: 60776 + timestamp: 1785159351292 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/aws-checksums-0.2.10-he8604bc_5.conda + sha256: 1f301d04c67c63b5e31b3ac5deaa20749cb86b6eadff04fdf9c79599ce9dd3d0 + md5: 3864eab4cfa7f8d524d558c52db32708 + depends: + - __osx >=11.0 + - aws-c-common >=0.14.3,<0.14.4.0a0 + license: Apache-2.0 + license_family: APACHE + run_exports: + weak: + - aws-checksums >=0.2.10,<0.2.11.0a0 + size: 92236 + timestamp: 1785159288972 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.8-h74c22ad_2.conda + sha256: 82bdd5a3fcada6afef6255a9bf41172f07b362f36e04a354dc2abc0a29bfb756 + md5: 4cc01a374215de38387d45e19c8c5c9c + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - c-ares >=1.34.8,<2.0a0 + size: 197819 + timestamp: 1787169996553 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/hdf5-2.2.0-nompi_h7eff4e6_100.conda + sha256: f4628ccb83c2e89baa55c4edbef4c6c8a1b6ff14e3661017796efaaf49c51c6c + md5: abd52e74327b5b8145c950326432d0b2 + depends: + - __osx >=11.0 + - aws-c-auth >=0.10.4,<0.10.5.0a0 + - aws-c-common >=0.14.3,<0.14.4.0a0 + - aws-c-http >=0.11.0,<0.11.1.0a0 + - aws-c-io >=0.27.5,<0.27.6.0a0 + - aws-c-s3 >=0.13.1,<0.13.2.0a0 + - aws-c-sdkutils >=0.2.7,<0.2.8.0a0 + - libaec >=1.1.5,<2.0a0 + - libcurl >=8.21.0,<9.0a0 + - libcxx >=19 + - libgfortran + - libgfortran5 >=14.4.0 + - libzlib >=1.3.2,<2.0a0 + - openssl >=3.5.7,<4.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - hdf5 >=2.2.0,<3.0a0 + size: 3731030 + timestamp: 1786234076033 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-py310h579977c_2.conda + sha256: 6cdb5dee54c72e56ab189fb3ad33cb28533553d42590e7e831160248f4416a43 + md5: a5efc0b42bb8b42e97d0a29ae3e3c187 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - icu >=78.3,<79.0a0 + size: 14070242 + timestamp: 1786545847761 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/krb5-1.22.2-h34f8a20_2.conda + sha256: aaea1d42b07769920db2af7471ece9399d2613448863da64ad8272998db5db34 + md5: 15235dd10450d67bc25ccafd5b46d2bc + depends: + - __osx >=11.0 + - libcxx >=21 + - libedit >=3.1.20250104,<3.2.0a0 + - libedit >=3.1.20250104,<4.0a0 + - openssl >=3.5.7,<4.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - krb5 >=1.22.2,<1.23.0a0 + size: 1165740 + timestamp: 1786762145768 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libaec-1.1.5-h8664d51_0.conda + sha256: af9cd8db11eb719e38a3340c88bb4882cf19b5b4237d93845224489fc2a13b46 + md5: 13e6d9ae0efbc9d2e9a01a91f4372b41 + depends: + - __osx >=11.0 + - libcxx >=19 + license: BSD-2-Clause + license_family: BSD + run_exports: + weak: + - libaec >=1.1.5,<2.0a0 + size: 30390 + timestamp: 1769222133373 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcurl-8.21.0-h6651222_5.conda + sha256: 59dd850def9473e7f63ed72afc750bba9670316f279c0bd409572cd47569ed5f + md5: 5ae67c128838b686894b4a3aef015c29 + depends: + - __osx >=11.0 + - krb5 >=1.22.2,<1.23.0a0 + - libnghttp2 >=1.68.1,<2.0a0 + - libpsl >=0.23.1,<0.24.0a0 + - libssh2 >=1.11.1,<2.0a0 + - libzlib >=1.3.2,<2.0a0 + - openssl >=3.5.7,<4.0a0 + - zstd >=1.5.7,<1.6.0a0 + license: curl + license_family: MIT + run_exports: + weak: + - libcurl >=8.21.0,<9.0a0 + size: 409852 + timestamp: 1787183686192 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-23.1.0-h55c6f16_0.conda + sha256: d3e5f0b767964af25ef81edffc002f8e822b1a5c55330da1ae3a857f70c4e4ee + md5: 5303ba06fab927399ed8dfb3227b0af8 + depends: + - __osx >=11.0 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + run_exports: {} + size: 575940 + timestamp: 1787698697875 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libedit-3.1.20250104-pl5321h26f1114_1.conda + sha256: 257c926f19e32bdb981fc674c76966049b6fc73f706bae58e9fed8757ad1da70 + md5: 843ef89082f368cb889305084d3b483c + depends: + - ncurses + - __osx >=11.0 + - ncurses >=6.6,<7.0a0 + license: BSD-2-Clause + license_family: BSD + run_exports: + weak: + - libedit >=3.1.20250104,<3.2.0a0 + size: 107742 + timestamp: 1786616721640 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libev-4.33-h1a92334_3.conda + sha256: c0100064506ae8abb432c5a506d474f10af2cf48c33d62bc221fb28b6d6ff6ac + md5: 19e86c8a6a47e92bb2e70ca12e758c5c + depends: + - __osx >=11.0 + license: BSD-2-Clause OR GPL-2.0-or-later + license_family: GPL + run_exports: + weak: + - libev >=4.33,<4.34.0a0 + size: 39991 + timestamp: 1785917376956 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgcc-16.2.0-h3cf6597_4.conda + sha256: 00b8d07ea98344c72c6e332a09b8060f4176849d5fd0eb78dd5b30176ad97ca4 + md5: 408af8d9d5226ff45ff04756ff1aa91e + depends: + - _openmp_mutex + constrains: + - libgcc-ng ==16.2.0=*_4 + - libgomp 16.2.0 4 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + run_exports: {} + size: 365758 + timestamp: 1787617459249 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran-16.2.0-h07b0088_4.conda + sha256: 7582efbbffc6964093afd80e01c2ac60d89aa4e404cac664544675b9d4d1c5e7 + md5: aa6c627acd0f5709d9c79bf708a7b81b + depends: + - libgfortran5 16.2.0 hdb7a957_4 + constrains: + - libgfortran-ng ==16.2.0=*_4 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + run_exports: {} + size: 99624 + timestamp: 1787617544212 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran5-16.2.0-hdb7a957_4.conda + sha256: 902719c38c7a390d305435a362dc83893754c3f933569a38347c434cd1c12540 + md5: d54390644d893d50e739b19145aae45f + depends: + - libgcc >=16.2.0 + constrains: + - libgfortran 16.2.0 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + run_exports: {} + size: 554806 + timestamp: 1787617465010 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libnghttp2-1.68.1-h435687b_1.conda + sha256: 445440d8cccccb6585f71d1eb7c949d64a2af2ced5481d924621424ff3b6a398 + md5: ac9902dcbe0417f50cf95ab2bde062fa + depends: + - __osx >=11.0 + - c-ares >=1.34.8,<2.0a0 + - libcxx >=21 + - libev >=4.33,<4.34.0a0 + - libev >=4.33,<5.0a0 + - libzlib >=1.3.2,<2.0a0 + - openssl >=3.5.7,<4.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - libnghttp2 >=1.68.1,<2.0a0 + size: 567024 + timestamp: 1787177422467 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libpsl-0.23.1-hdb0c161_1.conda + sha256: c8c3153df39abedf3413483f672151ad70f1adb0c06c192bdf7ec432c3616b07 + md5: 5d270f716a2b4bc13df9af8612071b17 + depends: + - __osx >=11.0 + - libcxx >=21 + - icu >=78.3,<79.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - libpsl >=0.23.1,<0.24.0a0 + size: 72673 + timestamp: 1786970928205 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libssh2-1.11.1-hbf2880b_1.conda + sha256: a056e518c3d2d09fbb1778cea80c0c95338fb24adf1a817bbf90fb484850a304 + md5: d957a8eda98c49b073e8dcfd677c127a + depends: + - libzlib >=1.3.2,<2.0a0 + - openssl >=3.5.7,<4.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - libssh2 >=1.11.1,<2.0a0 + size: 280492 + timestamp: 1786714226814 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + sha256: a18fa5d5bac452401459f966cf0d872224e8080c4ff93c77e168d43ab42ef9d7 + md5: f39288f0ea63ae962e1a2e4f355a0d75 + depends: + - __osx >=11.0 + constrains: + - zlib 1.3.2 *_3 + license: Zlib + license_family: Other + run_exports: + weak: + - libzlib >=1.3.2,<2.0a0 + size: 47822 + timestamp: 1785277049190 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/llvm-openmp-23.1.0-hdb3d66b_0.conda + sha256: 996d3a9de61c8ccc3fcb265938f9845f11868251f38e6db4e7190de3f9a971a2 + md5: 0affff5130a421d11d4d77ffbb00dad7 + depends: + - __osx >=11.0 + constrains: + - intel-openmp <0.0a0 + - openmp 23.1.0|23.1.0.* + license: Apache-2.0 WITH LLVM-exception + license_family: APACHE + run_exports: + strong: + - llvm-openmp >=23.1.0 + size: 287808 + timestamp: 1787723323710 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.6-he64c551_1.conda + sha256: 7024a48c8c0d0114ed4ab53c76bf9275d50e91ba7cea367a9aead638d3c29c68 + md5: 3dfa0d0316dc246cd44937a557de4501 + depends: + - __osx >=11.0 + license: X11 AND BSD-3-Clause + run_exports: + weak: + - ncurses >=6.6,<7.0a0 + size: 804298 + timestamp: 1786355189145 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.4-h55eecbc_0.conda + sha256: f23239eacd75c4c50705e68fae1aa3292da473e6a3a4abe2330f1e6afa680704 + md5: ae71ab40048c19a389a7dcccb86c2481 + depends: + - __osx >=11.0 + - ca-certificates + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - openssl >=3.6.4,<4.0a0 + size: 3110142 + timestamp: 1787698648639 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/s2n-1.7.6-h1b28cb6_0.conda + sha256: d2133326625aabcd20a7908bd2783cd207adf1a042c0512b493e49599fd5315a + md5: aa98fac113d87d7501704ad35ed071f2 + depends: + - __osx >=11.0 + - openssl >=3.5.7,<4.0a0 + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - s2n >=1.7.6,<1.7.7.0a0 + size: 277238 + timestamp: 1784674786218 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda + sha256: da867f5092eb0cb746d353694f0098031fd9817a4ce7d5743121209ae0f406ca + md5: 4ec2684c73812cc2c3d78379384a39cc + depends: + - __osx >=11.0 + - libzlib >=1.3.2,<2.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - zstd >=1.5.7,<1.6.0a0 + size: 433687 + timestamp: 1786599629846 diff --git a/examples/dataset-demo/macos-aarch64-metal/pixi.toml b/examples/dataset-demo/macos-aarch64-metal/pixi.toml new file mode 100644 index 0000000..63c6e12 --- /dev/null +++ b/examples/dataset-demo/macos-aarch64-metal/pixi.toml @@ -0,0 +1,16 @@ +# A second `native` box, and a different kind of program from `transcode-demo`. +# +# The HDF5 command-line tools: small compiled binaries that read the format most scientific +# instrument data and model weights are stored in. The case this answers is not "I cannot install +# it" but "we must all read this file the same way" — a signed box fixes the reader, so an +# inspection somebody publishes is one anybody can repeat byte for byte. +# +# conda-forge, like every other example. Most bioinformatics tooling lives on bioconda instead, and +# an example reaching for a second channel would demonstrate something this project does not claim. +[workspace] +name = "dataset-demo" +channels = ["conda-forge"] +platforms = ["osx-arm64"] + +[dependencies] +hdf5 = "2.*" diff --git a/examples/dataset-demo/macos-aarch64-metal/readings.conf b/examples/dataset-demo/macos-aarch64-metal/readings.conf new file mode 100644 index 0000000..4df9b54 --- /dev/null +++ b/examples/dataset-demo/macos-aarch64-metal/readings.conf @@ -0,0 +1,8 @@ +PATH /measurements/monthly +INPUT-CLASS TEXTFP +INPUT-SIZE 64 +RANK 2 +DIMENSION-SIZES 12 3 +OUTPUT-CLASS FP +OUTPUT-SIZE 64 +OUTPUT-ARCHITECTURE NATIVE diff --git a/examples/dataset-demo/macos-aarch64-metal/readings.h5 b/examples/dataset-demo/macos-aarch64-metal/readings.h5 new file mode 100644 index 0000000000000000000000000000000000000000..1e96bb7547bf4e37c806a34fef8de174531efe33 GIT binary patch literal 2336 zcmeD5aB<`1lHy|G;9!7(|4^X72@x@XO6*q)GIMbNUC+!2m1IzWxqbnPGX^C``mKBIpmf-%qeequGz3ONV1$N%jswKw zst~%t&_QuSQO3l#1`bXMPcJCl)`IYJO`z(GptLSjy(t0pnC3HpfeO?dDJZQCr6r&= z3zSxN_*Iv+r(`mRgCWNnnW&W8`)ln@<92z%^f5ry{n*a{24V!%W10`qhv^;wDgTl> literal 0 HcmV?d00001 diff --git a/examples/dataset-demo/macos-aarch64-metal/readings.txt b/examples/dataset-demo/macos-aarch64-metal/readings.txt new file mode 100644 index 0000000..ce614cf --- /dev/null +++ b/examples/dataset-demo/macos-aarch64-metal/readings.txt @@ -0,0 +1,12 @@ +14.000 10.500 17.500 +16.928 13.428 20.428 +18.000 14.500 21.500 +16.928 13.428 20.428 +14.000 10.500 17.500 +10.000 6.500 13.500 +6.000 2.500 9.500 +3.072 -0.428 6.572 +2.000 -1.500 5.500 +3.072 -0.428 6.572 +6.000 2.500 9.500 +10.000 6.500 13.500 diff --git a/examples/dataset-demo/macos-aarch64-metal/scroll.json b/examples/dataset-demo/macos-aarch64-metal/scroll.json new file mode 100644 index 0000000..ebad1cc --- /dev/null +++ b/examples/dataset-demo/macos-aarch64-metal/scroll.json @@ -0,0 +1,60 @@ +{ + "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "schemaVersion": 3, + "boxId": "dataset-demo", + "version": "1.0.0", + "sourceRevision": "hdf5-2-conda-forge", + "target": { + "platform": "macos", + "arch": "aarch64", + "accelerator": "metal" + }, + "compatibility": { + "minHostAppVersion": "1.0.0", + "minMacosVersion": "13.0" + }, + "runtime": { + "id": "native" + }, + "pixiVersion": "0.73.0", + "condaDependencyLicenseAudit": "examples/dataset-demo/macos-aarch64-metal/conda-licenses.json", + "cacheSubdir": "cache/dataset-demo", + "publishBaseUrl": "https://assets.example.org/boxes", + "selfTest": { + "files": [ + "readings.h5" + ], + "commands": [ + { + "args": [ + "--version" + ] + }, + { + "args": [ + "-H", + "readings.h5" + ] + }, + { + "args": [ + "-d", + "/measurements/monthly", + "readings.h5" + ] + } + ] + }, + "localFiles": [ + { + "sourcePath": "examples/dataset-demo/macos-aarch64-metal/readings.h5", + "relativePath": "readings.h5", + "sha256": "1e87f417207c0f3359ec3238adacb7795665bc7bf4af8abaeb6e20f33772dfcc" + } + ], + "execution": { + "kind": "native-binary", + "binary": "venv/bin/h5dump", + "defaultArgs": [] + } +} diff --git a/examples/native-box/macos-aarch64-metal/conda-licenses.json b/examples/hello-box-native/macos-aarch64-metal/conda-licenses.json similarity index 100% rename from examples/native-box/macos-aarch64-metal/conda-licenses.json rename to examples/hello-box-native/macos-aarch64-metal/conda-licenses.json diff --git a/examples/native-box/macos-aarch64-metal/pixi.lock b/examples/hello-box-native/macos-aarch64-metal/pixi.lock similarity index 100% rename from examples/native-box/macos-aarch64-metal/pixi.lock rename to examples/hello-box-native/macos-aarch64-metal/pixi.lock diff --git a/examples/native-box/macos-aarch64-metal/pixi.toml b/examples/hello-box-native/macos-aarch64-metal/pixi.toml similarity index 95% rename from examples/native-box/macos-aarch64-metal/pixi.toml rename to examples/hello-box-native/macos-aarch64-metal/pixi.toml index c361a5f..b78628b 100644 --- a/examples/native-box/macos-aarch64-metal/pixi.toml +++ b/examples/hello-box-native/macos-aarch64-metal/pixi.toml @@ -5,7 +5,7 @@ # was linked against, and conda-forge's `ncurses` carries an unrewritten build-machine path to # `libtinfo` that no relocation step here fixes. See the native runtime's stated limitation. [workspace] -name = "native-box" +name = "hello-box-native" channels = ["conda-forge"] platforms = ["osx-arm64"] diff --git a/examples/native-box/macos-aarch64-metal/scroll.json b/examples/hello-box-native/macos-aarch64-metal/scroll.json similarity index 78% rename from examples/native-box/macos-aarch64-metal/scroll.json rename to examples/hello-box-native/macos-aarch64-metal/scroll.json index 9bcd4d4..35ced19 100644 --- a/examples/native-box/macos-aarch64-metal/scroll.json +++ b/examples/hello-box-native/macos-aarch64-metal/scroll.json @@ -1,7 +1,7 @@ { "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", "schemaVersion": 3, - "boxId": "native-box", + "boxId": "hello-box-native", "version": "1.0.0", "sourceRevision": "example-native-v1", "target": { @@ -17,9 +17,8 @@ "id": "native" }, "pixiVersion": "0.73.0", - "condaDependencyLicenseAudit": "examples/native-box/macos-aarch64-metal/conda-licenses.json", - "cacheSubdir": "cache/native-box", - "assetBaseUrl": "https://assets.example.org/boxes", + "condaDependencyLicenseAudit": "examples/hello-box-native/macos-aarch64-metal/conda-licenses.json", + "cacheSubdir": "cache/hello-box-native", "selfTest": { "commands": [ { diff --git a/examples/node-box/macos-aarch64-metal/conda-licenses.json b/examples/hello-box-node/macos-aarch64-metal/conda-licenses.json similarity index 100% rename from examples/node-box/macos-aarch64-metal/conda-licenses.json rename to examples/hello-box-node/macos-aarch64-metal/conda-licenses.json diff --git a/examples/node-box/macos-aarch64-metal/entrypoint.js b/examples/hello-box-node/macos-aarch64-metal/entrypoint.js similarity index 100% rename from examples/node-box/macos-aarch64-metal/entrypoint.js rename to examples/hello-box-node/macos-aarch64-metal/entrypoint.js diff --git a/examples/node-box/macos-aarch64-metal/pixi.lock b/examples/hello-box-node/macos-aarch64-metal/pixi.lock similarity index 100% rename from examples/node-box/macos-aarch64-metal/pixi.lock rename to examples/hello-box-node/macos-aarch64-metal/pixi.lock diff --git a/examples/node-box/macos-aarch64-metal/pixi.toml b/examples/hello-box-node/macos-aarch64-metal/pixi.toml similarity index 88% rename from examples/node-box/macos-aarch64-metal/pixi.toml rename to examples/hello-box-node/macos-aarch64-metal/pixi.toml index c44d839..9b9e642 100644 --- a/examples/node-box/macos-aarch64-metal/pixi.toml +++ b/examples/hello-box-node/macos-aarch64-metal/pixi.toml @@ -1,6 +1,6 @@ # Minimal Scrollcase example: a bare Node environment from conda-forge, packed as a box. [workspace] -name = "node-box" +name = "hello-box-node" channels = ["conda-forge"] platforms = ["osx-arm64"] diff --git a/examples/node-box/macos-aarch64-metal/scroll.json b/examples/hello-box-node/macos-aarch64-metal/scroll.json similarity index 71% rename from examples/node-box/macos-aarch64-metal/scroll.json rename to examples/hello-box-node/macos-aarch64-metal/scroll.json index 95aecc7..5366ed1 100644 --- a/examples/node-box/macos-aarch64-metal/scroll.json +++ b/examples/hello-box-node/macos-aarch64-metal/scroll.json @@ -1,7 +1,7 @@ { "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", "schemaVersion": 3, - "boxId": "node-box", + "boxId": "hello-box-node", "version": "1.0.0", "sourceRevision": "example-node-v1", "target": { @@ -18,9 +18,9 @@ "version": "22" }, "pixiVersion": "0.73.0", - "condaDependencyLicenseAudit": "examples/node-box/macos-aarch64-metal/conda-licenses.json", - "cacheSubdir": "cache/node-box", - "assetBaseUrl": "https://assets.example.org/boxes", + "condaDependencyLicenseAudit": "examples/hello-box-node/macos-aarch64-metal/conda-licenses.json", + "cacheSubdir": "cache/hello-box-node", + "publishBaseUrl": "https://assets.example.org/boxes", "selfTest": { "imports": [ "node:fs", @@ -37,7 +37,7 @@ }, "localFiles": [ { - "sourcePath": "examples/node-box/macos-aarch64-metal/entrypoint.js", + "sourcePath": "examples/hello-box-node/macos-aarch64-metal/entrypoint.js", "relativePath": "entrypoint.js" } ], diff --git a/examples/hello-box/scroll.json b/examples/hello-box/scroll.json index 1db6fea..fc6c2d4 100644 --- a/examples/hello-box/scroll.json +++ b/examples/hello-box/scroll.json @@ -15,7 +15,7 @@ "pixiVersion": "0.73.0", "cacheSubdir": "cache/hello", "environment": {}, - "assetBaseUrl": "https://assets.example.org/boxes", + "publishBaseUrl": "https://assets.example.org/boxes", "assets": [], "selfTest": { "imports": [ diff --git a/examples/llm-demo/README.md b/examples/llm-demo/README.md index f9f14bb..76cfe36 100644 --- a/examples/llm-demo/README.md +++ b/examples/llm-demo/README.md @@ -51,8 +51,8 @@ declares, and embeds them. Most of the archive is that one file. **The whole model is one asset.** A GGUF holds the weights, the tokenizer *and* the chat template in a single container, so this scroll declares one file where `sentiment-demo` declares three — and there is no tokenizer that can drift out of step with the weights it belongs to. It is pinned to an -immutable upstream revision with its size and SHA-256, and `weights: embed` puts it in the archive, -so the box installs and runs air-gapped. +immutable upstream revision with its size and SHA-256, and it carries no `embed: false`, so the +default applies and the file is packed into the archive — the box installs and runs air-gapped. **The environment declares no offline flag.** `sentiment-demo` sets `HF_HUB_OFFLINE=1` and two siblings because its stack really does contain a Hugging Face client. @@ -79,7 +79,7 @@ it would win over the value the person debugging supplies, and weld the switch s **The self-test has to generate, not just import.** `selfTest.imports` is the part the signed release carries, which is why `verify --self-test` can repeat it later with the box's own -interpreter. `files` and `pythonFile` stay builder-only: `shared/self_test.py` loads the gigabyte and +interpreter. `files` and `script` stay builder-only: `shared/self_test.py` loads the gigabyte and asserts that the answer to *What is the capital of Italy?* contains `rome`, so a box that cannot generate is never signed. It asserts a substring rather than a sentence — greedy decoding is reproducible, but a llama.cpp point release may reword prose without anything being wrong. diff --git a/examples/llm-demo/scroll.json b/examples/llm-demo/scroll.json index ecb850b..d0aa4da 100644 --- a/examples/llm-demo/scroll.json +++ b/examples/llm-demo/scroll.json @@ -22,7 +22,7 @@ "environment": { "PYTHONDONTWRITEBYTECODE": "1" }, - "assetBaseUrl": "https://assets.example.org/boxes", + "publishBaseUrl": "https://assets.example.org/boxes", "assets": [ { "url": "https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF/resolve/2d4a76a30b4af41ecd395c35725ac11688d4cfe4/smollm2-1.7b-instruct-q4_k_m.gguf", diff --git a/examples/sentiment-demo/README.md b/examples/sentiment-demo/README.md index 6145cfd..5781d46 100644 --- a/examples/sentiment-demo/README.md +++ b/examples/sentiment-demo/README.md @@ -46,8 +46,8 @@ declares, and embeds them. Most of the archive is those weights and onnxruntime. ## What is worth reading in the scroll **The model is three pinned assets, not a download.** Each declares a URL at an immutable upstream -revision, its size and its SHA-256; the build fails if a byte moved. `weights: embed` puts them in -the archive, so the box installs and runs air-gapped. +revision, its size and its SHA-256; the build fails if a byte moved. None declares `embed: false`, +so all three are packed into the archive and the box installs and runs air-gapped. **The environment is signed, not merely set.** `HF_HUB_OFFLINE=1`, `TRANSFORMERS_OFFLINE=1` and `TOKENIZERS_PARALLELISM=false` are carried in the release and override the host. This is defence in diff --git a/examples/sentiment-demo/scroll.json b/examples/sentiment-demo/scroll.json index e20f22c..daa74c8 100644 --- a/examples/sentiment-demo/scroll.json +++ b/examples/sentiment-demo/scroll.json @@ -24,7 +24,7 @@ "TRANSFORMERS_OFFLINE": "1", "TOKENIZERS_PARALLELISM": "false" }, - "assetBaseUrl": "https://assets.example.org/boxes", + "publishBaseUrl": "https://assets.example.org/boxes", "assets": [ { "url": "https://huggingface.co/onnx-community/distilbert-base-uncased-finetuned-sst-2-english-ONNX/resolve/fd49941c1b822846cb14970cdf430a7cfbe0f5b9/onnx/model_int8.onnx", diff --git a/examples/transcode-demo/macos-aarch64-metal/conda-licenses.json b/examples/transcode-demo/macos-aarch64-metal/conda-licenses.json new file mode 100644 index 0000000..69c8e7b --- /dev/null +++ b/examples/transcode-demo/macos-aarch64-metal/conda-licenses.json @@ -0,0 +1,548 @@ +{ + "schemaVersion": 2, + "kind": "scrollcase.box.dependency-license-audit", + "targetId": "macos-aarch64-metal", + "dependencyLockSha256": "22bb7d708a8dc8ed91cfa38c69ee5237077f6543241a2f38a7663b0e76c7c442", + "packages": [ + { + "name": "aom", + "version": "3.14.1", + "declaredLicense": "BSD-2-Clause", + "source": "conda" + }, + { + "name": "bzip2", + "version": "1.0.8", + "declaredLicense": "bzip2-1.0.6", + "source": "conda" + }, + { + "name": "ca-certificates", + "version": "2026.7.22", + "declaredLicense": "ISC", + "source": "conda" + }, + { + "name": "cairo", + "version": "1.18.4", + "declaredLicense": "LGPL-2.1-only or MPL-1.1", + "source": "conda" + }, + { + "name": "dav1d", + "version": "1.2.1", + "declaredLicense": "BSD-2-Clause", + "source": "conda" + }, + { + "name": "dbus", + "version": "1.16.2", + "declaredLicense": "AFL-2.1 OR GPL-2.0-or-later", + "source": "conda" + }, + { + "name": "ffmpeg", + "version": "9.0.1", + "declaredLicense": "GPL-2.0-or-later", + "source": "conda" + }, + { + "name": "font-ttf-dejavu-sans-mono", + "version": "2.37", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "font-ttf-inconsolata", + "version": "3.000", + "declaredLicense": "OFL-1.1", + "source": "conda" + }, + { + "name": "font-ttf-source-code-pro", + "version": "2.038", + "declaredLicense": "OFL-1.1", + "source": "conda" + }, + { + "name": "font-ttf-ubuntu", + "version": "0.83", + "declaredLicense": "LicenseRef-Ubuntu-Font-Licence-Version-1.0", + "source": "conda" + }, + { + "name": "fontconfig", + "version": "2.18.3", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "fonts-conda-ecosystem", + "version": "1", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "fonts-conda-forge", + "version": "1", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "freetype", + "version": "2.14.3", + "declaredLicense": "GPL-2.0-only OR FTL", + "source": "conda" + }, + { + "name": "fribidi", + "version": "1.0.16", + "declaredLicense": "LGPL-2.1-or-later", + "source": "conda" + }, + { + "name": "gdk-pixbuf", + "version": "2.44.8", + "declaredLicense": "LGPL-2.1-or-later", + "source": "conda" + }, + { + "name": "glslang", + "version": "16.5.0", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "gmp", + "version": "6.3.0", + "declaredLicense": "GPL-2.0-or-later OR LGPL-3.0-or-later", + "source": "conda" + }, + { + "name": "graphite2", + "version": "1.3.15", + "declaredLicense": "LGPL-2.0-or-later", + "source": "conda" + }, + { + "name": "harfbuzz", + "version": "14.4.0", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "icu", + "version": "78.3", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "lame", + "version": "4.0", + "declaredLicense": "LGPL-2.0-only", + "source": "conda" + }, + { + "name": "lcms2", + "version": "2.19.1", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "lerc", + "version": "4.2.0", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libabseil", + "version": "20260526.0", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libass", + "version": "0.17.5", + "declaredLicense": "ISC", + "source": "conda" + }, + { + "name": "libbrotlicommon", + "version": "1.2.0", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libbrotlidec", + "version": "1.2.0", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libbrotlienc", + "version": "1.2.0", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libcxx", + "version": "23.1.0", + "declaredLicense": "Apache-2.0 WITH LLVM-exception", + "source": "conda" + }, + { + "name": "libdeflate", + "version": "1.25", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libdovi", + "version": "3.4.0", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libexpat", + "version": "2.8.1", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libffi", + "version": "3.7.0", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libfreetype", + "version": "2.14.3", + "declaredLicense": "GPL-2.0-only OR FTL", + "source": "conda" + }, + { + "name": "libfreetype6", + "version": "2.14.3", + "declaredLicense": "GPL-2.0-only OR FTL", + "source": "conda" + }, + { + "name": "libglib", + "version": "2.88.3", + "declaredLicense": "LGPL-2.1-or-later", + "source": "conda" + }, + { + "name": "libharfbuzz", + "version": "14.4.0", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libharfbuzz-devel", + "version": "14.4.0", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libhwloc", + "version": "2.13.0", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "libhwy", + "version": "1.4.0", + "declaredLicense": "Apache-2.0 OR BSD-3-Clause", + "source": "conda" + }, + { + "name": "libiconv", + "version": "1.18", + "declaredLicense": "LGPL-2.1-only", + "source": "conda" + }, + { + "name": "libintl", + "version": "0.25.1", + "declaredLicense": "LGPL-2.1-or-later", + "source": "conda" + }, + { + "name": "libjpeg-turbo", + "version": "3.2.0", + "declaredLicense": "IJG AND BSD-3-Clause AND Zlib", + "source": "conda" + }, + { + "name": "libjxl", + "version": "0.12.0", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "liblzma", + "version": "5.8.3", + "declaredLicense": "0BSD", + "source": "conda" + }, + { + "name": "libogg", + "version": "1.3.5", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "libopenvino", + "version": "2026.3.1", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libopenvino-arm-cpu-plugin", + "version": "2026.3.1", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libopenvino-auto-batch-plugin", + "version": "2026.3.1", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libopenvino-auto-plugin", + "version": "2026.3.1", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libopenvino-hetero-plugin", + "version": "2026.3.1", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libopenvino-ir-frontend", + "version": "2026.3.1", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libopenvino-onnx-frontend", + "version": "2026.3.1", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libopenvino-paddle-frontend", + "version": "2026.3.1", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libopenvino-pytorch-frontend", + "version": "2026.3.1", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libopenvino-tensorflow-frontend", + "version": "2026.3.1", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libopenvino-tensorflow-lite-frontend", + "version": "2026.3.1", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libopus", + "version": "1.6.1", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "libplacebo", + "version": "7.360.1", + "declaredLicense": "LGPL-2.1-or-later", + "source": "conda" + }, + { + "name": "libpng", + "version": "1.6.58", + "declaredLicense": "zlib-acknowledgement", + "source": "conda" + }, + { + "name": "libprotobuf", + "version": "7.35.1", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "librsvg", + "version": "2.62.3", + "declaredLicense": "LGPL-2.1-or-later", + "source": "conda" + }, + { + "name": "libtiff", + "version": "4.7.2", + "declaredLicense": "HPND", + "source": "conda" + }, + { + "name": "libusb", + "version": "1.0.29", + "declaredLicense": "LGPL-2.1-or-later", + "source": "conda" + }, + { + "name": "libvorbis", + "version": "1.3.7", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "libvpx", + "version": "1.17.0", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "libvulkan-loader", + "version": "1.4.357.0", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "libwebp-base", + "version": "1.6.0", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "libxml2", + "version": "2.15.3", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libxml2-16", + "version": "2.15.3", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "libzlib", + "version": "1.3.2", + "declaredLicense": "Zlib", + "source": "conda" + }, + { + "name": "mpg123", + "version": "1.33.7", + "declaredLicense": "LGPL-2.1-only", + "source": "conda" + }, + { + "name": "openh264", + "version": "2.6.0", + "declaredLicense": "BSD-2-Clause", + "source": "conda" + }, + { + "name": "openssl", + "version": "3.6.4", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "pango", + "version": "1.58.2", + "declaredLicense": "LGPL-2.1-or-later", + "source": "conda" + }, + { + "name": "pcre2", + "version": "10.47", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "pixman", + "version": "0.46.4", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "pugixml", + "version": "1.15", + "declaredLicense": "MIT", + "source": "conda" + }, + { + "name": "sdl2", + "version": "2.32.56", + "declaredLicense": "Zlib", + "source": "conda" + }, + { + "name": "sdl3", + "version": "3.4.14", + "declaredLicense": "Zlib", + "source": "conda" + }, + { + "name": "shaderc", + "version": "2026.3", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "snappy", + "version": "1.2.2", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + }, + { + "name": "spirv-tools", + "version": "2026.3", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "svt-av1", + "version": "4.2.0", + "declaredLicense": "BSD-2-Clause", + "source": "conda" + }, + { + "name": "tbb", + "version": "2023.0.0", + "declaredLicense": "Apache-2.0", + "source": "conda" + }, + { + "name": "x264", + "version": "1!164.3095", + "declaredLicense": "GPL-2.0-or-later", + "source": "conda" + }, + { + "name": "x265", + "version": "3.5", + "declaredLicense": "GPL-2.0-or-later", + "source": "conda" + }, + { + "name": "zstd", + "version": "1.5.7", + "declaredLicense": "BSD-3-Clause", + "source": "conda" + } + ] +} diff --git a/examples/transcode-demo/macos-aarch64-metal/pixi.lock b/examples/transcode-demo/macos-aarch64-metal/pixi.lock new file mode 100644 index 0000000..4100861 --- /dev/null +++ b/examples/transcode-demo/macos-aarch64-metal/pixi.lock @@ -0,0 +1,1374 @@ +version: 7 +platforms: +- name: osx-arm64 + virtual-packages: + - __unix=0=0 + - __osx=13.0 + - __archspec=0=m1 +environments: + default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + packages: + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-hc364b38_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/aom-3.14.1-pl5321hf79ea98_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h4e30115_10.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cairo-1.18.4-hdbf5564_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/dav1d-1.2.1-hb547adb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/dbus-1.16.2-h3ff7a7c_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ffmpeg-9.0.1-gpl_h7659707_101.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/fontconfig-2.18.3-h81aa574_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/freetype-2.14.3-hce30654_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/fribidi-1.0.16-h84a0fba_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/gdk-pixbuf-2.44.8-h5867e2c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/glslang-16.5.0-h1a33c25_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/gmp-6.3.0-hf5e55b1_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/graphite2-1.3.15-h784d473_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/harfbuzz-14.4.0-hce30654_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-py310h579977c_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/lame-4.0-hef9b5c2_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/lcms2-2.19.1-hdfa7624_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/lerc-4.2.0-h1eee2c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libabseil-20260526.0-cxx17_h4c27e2a_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libass-0.17.5-h3245dfc_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlicommon-1.2.0-h1dcdb26_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlidec-1.2.0-h5295a6a_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlienc-1.2.0-h2ddc9cb_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-23.1.0-h55c6f16_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libdeflate-1.25-he7e0567_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libdovi-3.4.0-h78f8ca3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.8.1-hf6b4638_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.7.0-h47dc5ef_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libfreetype-2.14.3-hce30654_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libfreetype6-2.14.3-h2ed5691_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libglib-2.88.3-h89cd0c0_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libharfbuzz-14.4.0-hcda0f7c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libharfbuzz-devel-14.4.0-hcda0f7c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libhwloc-2.13.0-default_ha97f43a_1000.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libhwy-1.4.0-hd2bdd19_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-he4c29f2_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libintl-0.25.1-h493aca8_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libjpeg-turbo-3.2.0-h84a0fba_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libjxl-0.12.0-hb71b141_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.3-h8088a28_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libogg-1.3.5-h48c0fde_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-2026.3.1-h036fd89_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-arm-cpu-plugin-2026.3.1-h036fd89_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-auto-batch-plugin-2026.3.1-h4771a85_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-auto-plugin-2026.3.1-h4771a85_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-hetero-plugin-2026.3.1-h7561220_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-ir-frontend-2026.3.1-h7561220_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-onnx-frontend-2026.3.1-h3fa9d4a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-paddle-frontend-2026.3.1-h3fa9d4a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-pytorch-frontend-2026.3.1-h3df7365_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-tensorflow-frontend-2026.3.1-h5e3894e_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-tensorflow-lite-frontend-2026.3.1-h3df7365_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopus-1.6.1-h74c22ad_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libplacebo-7.360.1-hca394fb_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libpng-1.6.58-hf5e6511_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libprotobuf-7.35.1-h391e224_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/librsvg-2.62.3-he8aa2a2_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libtiff-4.7.2-hf67920b_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libusb-1.0.29-hbc156a2_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libvorbis-1.3.7-h81086ad_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libvpx-1.17.0-h484c67d_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libvulkan-loader-1.4.357.0-hfbe1efa_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libwebp-base-1.6.0-h202fb40_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libxml2-16-2.15.3-h5ef1a60_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libxml2-2.15.3-h5654f7c_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/mpg123-1.33.7-h0e3f465_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openh264-2.6.0-hf8510ef_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.4-h55eecbc_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pango-1.58.2-hf80efc4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pcre2-10.47-he63d830_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pixman-0.46.4-h784d473_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pugixml-1.15-hd3d436d_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/sdl2-2.32.56-h784d473_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/sdl3-3.4.14-h6fa9c73_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/shaderc-2026.3-hfd4ff19_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/snappy-1.2.2-hada39a4_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/spirv-tools-2026.3-hc0b298b_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/svt-av1-4.2.0-h484c67d_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tbb-2023.0.0-he0260a5_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/x264-1!164.3095-h57fd34a_2.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/x265-3.5-hbc6ce65_3.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda +packages: +- conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + sha256: 0a0544cf95f64394fe4959286f5c71f5444ad58feb0602e53becb27448d24da6 + md5: 0f51e2391ade309db462a55611263e9c + depends: + - __unix + license: ISC + run_exports: {} + size: 131780 + timestamp: 1784754889428 +- conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 + sha256: 58d7f40d2940dd0a8aa28651239adbf5613254df0f75789919c4e6762054403b + md5: 0c96522c6bdaed4b1566d11387caaf45 + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 397370 + timestamp: 1566932522327 +- conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 + sha256: c52a29fdac682c20d252facc50f01e7c2e7ceac52aa9817aaf0bb83f7559ec5c + md5: 34893075a5c9e55cdafac56607368fc6 + license: OFL-1.1 + license_family: Other + run_exports: {} + size: 96530 + timestamp: 1620479909603 +- conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 + sha256: 00925c8c055a2275614b4d983e1df637245e19058d79fc7dd1a93b8d9fb4b139 + md5: 4d59c254e01d9cde7957100457e2d5fb + license: OFL-1.1 + license_family: Other + run_exports: {} + size: 700814 + timestamp: 1620479612257 +- conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda + sha256: 2821ec1dc454bd8b9a31d0ed22a7ce22422c0aef163c59f49dfdf915d0f0ca14 + md5: 49023d73832ef61042f6a237cb2687e7 + license: LicenseRef-Ubuntu-Font-Licence-Version-1.0 + license_family: Other + run_exports: {} + size: 1620504 + timestamp: 1727511233259 +- conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 + sha256: a997f2f1921bb9c9d76e6fa2f6b408b7fa549edd349a77639c9fe7a23ea93e61 + md5: fee5683a3f04bd15cbd8318b096a27ab + depends: + - fonts-conda-forge + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 3667 + timestamp: 1566974674465 +- conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-hc364b38_1.conda + sha256: 54eea8469786bc2291cc40bca5f46438d3e062a399e8f53f013b6a9f50e98333 + md5: a7970cd949a077b7cb9696379d338681 + depends: + - font-ttf-ubuntu + - font-ttf-inconsolata + - font-ttf-dejavu-sans-mono + - font-ttf-source-code-pro + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 4059 + timestamp: 1762351264405 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/aom-3.14.1-pl5321hf79ea98_2.conda + sha256: 292b3cfab77d5db222b53add0ec3b5e7e5c04ce2cb8cf8a7237cbe0643b3e6c1 + md5: 36c4372d5ec4e878999789360e1ad533 + depends: + - __osx >=11.0 + - libcxx >=21 + license: BSD-2-Clause + license_family: BSD + run_exports: + weak: + - aom >=3.14.1,<3.15.0a0 + size: 2808232 + timestamp: 1787256153818 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h4e30115_10.conda + sha256: 8ec22f0ba25cbfc2e64d70cf29459eccd7ffdf6436f6a6ff15bbfef799f7d4f6 + md5: b50612e7d190b8061ab4e7dc119cf4d5 + depends: + - __osx >=11.0 + license: bzip2-1.0.6 + license_family: BSD + run_exports: + weak: + - bzip2 >=1.0.8,<2.0a0 + size: 124965 + timestamp: 1785906749812 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/cairo-1.18.4-hdbf5564_2.conda + sha256: 3d368a359c3b526082eb8b8c4a07371f0c1e40e6dadfeef326dbec7956c149d0 + md5: ae0e0e8ea304c6ff1608d7918703ba29 + depends: + - __osx >=11.0 + - fontconfig >=2.18.3,<3.0a0 + - fonts-conda-ecosystem + - icu >=78.3,<79.0a0 + - libcxx >=21 + - libexpat >=2.8.1,<3.0a0 + - libfreetype >=2.14.3 + - libfreetype6 >=2.14.3 + - libglib >=2.88.3,<3.0a0 + - libpng >=1.6.58,<1.7.0a0 + - libzlib >=1.3.2,<2.0a0 + - pixman >=0.46.4,<1.0a0 + license: LGPL-2.1-only or MPL-1.1 + run_exports: + weak: + - cairo >=1.18.4,<2.0a0 + size: 841724 + timestamp: 1787926737310 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/dav1d-1.2.1-hb547adb_0.conda + sha256: 93e077b880a85baec8227e8c72199220c7f87849ad32d02c14fb3807368260b8 + md5: 5a74cdee497e6b65173e10d94582fae6 + license: BSD-2-Clause + license_family: BSD + run_exports: + weak: + - dav1d >=1.2.1,<1.2.2.0a0 + size: 316394 + timestamp: 1685695959391 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/dbus-1.16.2-h3ff7a7c_1.conda + sha256: a8207751ed261764061866880da38e4d3063e167178bfe85b6db9501432462ba + md5: 5a3506971d2d53023c1c4450e908a8da + depends: + - libcxx >=19 + - __osx >=11.0 + - libglib >=2.86.2,<3.0a0 + - libzlib >=1.3.1,<2.0a0 + - libexpat >=2.7.3,<3.0a0 + license: AFL-2.1 OR GPL-2.0-or-later + run_exports: + weak: + - dbus >=1.16.2,<2.0a0 + size: 393811 + timestamp: 1764536084131 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/ffmpeg-9.0.1-gpl_h7659707_101.conda + sha256: 7ad506378018ad343c63e56480d296e1a1d268e7bba141faccd94a6c9455cead + md5: 6c2a7b88f81f28af7acd1aa0cc326434 + depends: + - __osx >=11.0 + - aom >=3.14.1,<3.15.0a0 + - bzip2 >=1.0.8,<2.0a0 + - dav1d >=1.2.1,<1.2.2.0a0 + - fontconfig >=2.18.3,<3.0a0 + - fonts-conda-ecosystem + - gmp >=6.3.0,<7.0a0 + - lame >=4.0,<4.1.0a0 + - libass >=0.17.5,<0.17.6.0a0 + - libcxx >=21 + - libexpat >=2.8.1,<3.0a0 + - libfreetype >=2.14.3 + - libfreetype6 >=2.14.3 + - libharfbuzz >=14.4.0 + - libiconv >=1.18,<2.0a0 + - libjxl >=0.12.0,<0.13.0a0 + - liblzma >=5.8.3,<6.0a0 + - libopenvino >=2026.3.1,<2026.3.2.0a0 + - libopenvino-arm-cpu-plugin >=2026.3.1,<2026.3.2.0a0 + - libopenvino-auto-batch-plugin >=2026.3.1,<2026.3.2.0a0 + - libopenvino-auto-plugin >=2026.3.1,<2026.3.2.0a0 + - libopenvino-hetero-plugin >=2026.3.1,<2026.3.2.0a0 + - libopenvino-ir-frontend >=2026.3.1,<2026.3.2.0a0 + - libopenvino-onnx-frontend >=2026.3.1,<2026.3.2.0a0 + - libopenvino-paddle-frontend >=2026.3.1,<2026.3.2.0a0 + - libopenvino-pytorch-frontend >=2026.3.1,<2026.3.2.0a0 + - libopenvino-tensorflow-frontend >=2026.3.1,<2026.3.2.0a0 + - libopenvino-tensorflow-lite-frontend >=2026.3.1,<2026.3.2.0a0 + - libopus >=1.6.1,<2.0a0 + - libplacebo >=7.360.1,<7.361.0a0 + - librsvg >=2.62.3,<3.0a0 + - libvorbis >=1.3.7,<1.4.0a0 + - libvpx >=1.17.0,<1.18.0a0 + - libvulkan-loader >=1.4.357.0,<2.0a0 + - libwebp-base >=1.6.0,<2.0a0 + - libxml2 + - libxml2-16 >=2.15.3 + - libzlib >=1.3.2,<2.0a0 + - openh264 >=2.6.0,<2.6.1.0a0 + - openssl >=3.5.8,<4.0a0 + - sdl2 >=2.32.56,<3.0a0 + - svt-av1 >=4.2.0,<4.2.1.0a0 + - x264 >=1!164.3095,<1!165 + - x265 >=3.5,<3.6.0a0 + license: GPL-2.0-or-later + license_family: GPL + run_exports: + weak: + - ffmpeg >=9.0.1,<10.0a0 + size: 10701402 + timestamp: 1788012196120 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/fontconfig-2.18.3-h81aa574_1.conda + sha256: 004570d35fb0eff73ce3ae49b209a47622d0c2e580dd0dc4c61d59626b386a09 + md5: 8ac8cc3fe744b484de4387703134d8b2 + depends: + - __osx >=11.0 + - libexpat >=2.8.1,<3.0a0 + - libfreetype >=2.14.3 + - libfreetype6 >=2.14.3 + - libintl >=0.25.1,<1.0a0 + - libzlib >=1.3.2,<2.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - fontconfig >=2.18.3,<3.0a0 + - fonts-conda-ecosystem + size: 265659 + timestamp: 1786667932256 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/freetype-2.14.3-hce30654_2.conda + sha256: 90681f1d0658cb2fd49a074f644eb4774272499290487a4bebb1c4df01d6997b + md5: 2f4cc7dc2e6622591735c49dda1b92aa + depends: + - libfreetype 2.14.3 hce30654_2 + - libfreetype6 2.14.3 h2ed5691_2 + license: GPL-2.0-only OR FTL + run_exports: + weak: + - libfreetype >=2.14.3 + - libfreetype6 >=2.14.3 + size: 175388 + timestamp: 1786641016583 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/fribidi-1.0.16-h84a0fba_1.conda + sha256: 6dd18694340b84290bb1906cc1359502b974aada6a8476cfe6dec3ce0e860af8 + md5: 2bb7d7dd91116b8c85e805b0e08cc67b + depends: + - __osx >=11.0 + license: LGPL-2.1-or-later + run_exports: + weak: + - fribidi >=1.0.16,<2.0a0 + size: 60230 + timestamp: 1785912572097 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/gdk-pixbuf-2.44.8-h5867e2c_0.conda + sha256: 0e91dc3531a3a7c4938abc115d7a49be61284a9a8650b404a37b2123fe5cf7dd + md5: aa0e0c67d3e89789959e0d77c8361b67 + depends: + - __osx >=11.0 + - libglib >=2.88.3,<3.0a0 + - libintl >=0.25.1,<1.0a0 + - libjpeg-turbo >=3.2.0,<4.0a0 + - liblzma >=5.8.3,<6.0a0 + - libpng >=1.6.58,<1.7.0a0 + - libtiff >=4.7.2,<4.8.0a0 + license: LGPL-2.1-or-later + license_family: LGPL + run_exports: + weak: + - gdk-pixbuf >=2.44.8,<3.0a0 + size: 551879 + timestamp: 1786715345711 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/glslang-16.5.0-h1a33c25_2.conda + sha256: c161660cc0a1bd3808365081624babe07fd8104ec67441f3a21d75ab45fe7118 + md5: f6a0ee2a26f4c3faa7aad24c097cbc2d + depends: + - __osx >=11.0 + - libcxx >=21 + - spirv-tools >=2026,<2027.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - glslang >=16,<17.0a0 + size: 897203 + timestamp: 1787687123081 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/gmp-6.3.0-hf5e55b1_3.conda + sha256: 980cb1bbc3656d6f923d44297a2c4ab75f20482b8930c70537fd7f1ef2378894 + md5: bbfa5bd261b68536ddcda39e03d3407f + depends: + - libcxx >=19 + - __osx >=11.0 + license: GPL-2.0-or-later OR LGPL-3.0-or-later + run_exports: + weak: + - gmp >=6.3.0,<7.0a0 + size: 399307 + timestamp: 1786629210135 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/graphite2-1.3.15-h784d473_1.conda + sha256: 471f34a187fdb4f2df33e26f2e471b16c239a5b277903f643d9c3a8c9a9f44ec + md5: 0c7b78d9ffff4f5a6ca28d346f734d8f + depends: + - libcxx >=19 + - __osx >=11.0 + license: LGPL-2.0-or-later + license_family: LGPL + run_exports: + weak: + - graphite2 >=1.3.15,<2.0a0 + size: 86493 + timestamp: 1786118637573 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/harfbuzz-14.4.0-hce30654_0.conda + sha256: 1cb2c173e80c2c166589e2e43d822db4ce84385d54800a108de39f240e0c24c6 + md5: f71b69fa28637827ec618fe783b7ec11 + depends: + - libharfbuzz-devel 14.4.0 hcda0f7c_0 + license: MIT + license_family: MIT + run_exports: + weak: + - libharfbuzz >=14.4.0 + size: 11020 + timestamp: 1787795107813 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-py310h579977c_2.conda + sha256: 6cdb5dee54c72e56ab189fb3ad33cb28533553d42590e7e831160248f4416a43 + md5: a5efc0b42bb8b42e97d0a29ae3e3c187 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - icu >=78.3,<79.0a0 + size: 14070242 + timestamp: 1786545847761 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/lame-4.0-hef9b5c2_1.conda + sha256: 98b9195b315777c9cf1d785b5d43f223bafadbb2c5467116d688bd96305fa353 + md5: 5f74847ab1fbc7ffdcfe9d25269eeff6 + depends: + - __osx >=11.0 + - mpg123 >=1.33.7,<1.34.0a0 + license: LGPL-2.0-only + license_family: LGPL + run_exports: + weak: + - lame >=4.0,<4.1.0a0 + size: 296576 + timestamp: 1786293154234 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/lcms2-2.19.1-hdfa7624_1.conda + sha256: ccb5598fad3694e79bf54f0eb812e3b3c3dd63d1497e631f5978800eadb9bcc4 + md5: d2f2c7c10e2957647d45589b7701a453 + depends: + - __osx >=11.0 + - libjpeg-turbo >=3.1.4.1,<4.0a0 + - libtiff >=4.7.1,<4.8.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - lcms2 >=2.19.1,<3.0a0 + size: 213747 + timestamp: 1780212240694 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/lerc-4.2.0-h1eee2c3_0.conda + sha256: c97aa17d16d2ac332ba9f184e82ce8f72dcb10e9a10c5f299030be2d44e191b9 + md5: e429aec4037d5cb8fd34ded9f5dadd39 + depends: + - __osx >=11.0 + - libcxx >=19 + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - lerc >=4.2.0,<5.0a0 + size: 166477 + timestamp: 1785036480092 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libabseil-20260526.0-cxx17_h4c27e2a_2.conda + sha256: d9c62dae9d8f5bebadf72fcf369c00cc353183cb2521f9fffbf4c2f70b58bef9 + md5: 2aa5e7dc7b5d218908effd2a8c70a8a8 + depends: + - __osx >=11.0 + - libcxx >=19 + constrains: + - abseil-cpp =20260526.0 + - libabseil-static =20260526.0=cxx17* + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - libabseil >=20260526.0,<20260527.0a0 + - libabseil =*=cxx17* + size: 1277537 + timestamp: 1787217005681 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libass-0.17.5-h3245dfc_0.conda + sha256: b006fccaa3e4d122188bd3db71d630d4d3a7d2f99fbcdef5ac53b3299c65909d + md5: baae8eafd053d3e6bd88c6d053c6f624 + depends: + - __osx >=11.0 + - libfreetype >=2.14.3 + - libfreetype6 >=2.14.3 + - libiconv >=1.18,<2.0a0 + - harfbuzz >=14.2.1 + - fontconfig >=2.18.1,<3.0a0 + - fonts-conda-ecosystem + - libzlib >=1.3.2,<2.0a0 + - fribidi >=1.0.16,<2.0a0 + license: ISC + run_exports: + weak: + - libass >=0.17.5,<0.17.6.0a0 + size: 139969 + timestamp: 1782299036301 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlicommon-1.2.0-h1dcdb26_3.conda + sha256: 5e62b856b2e77ce98db44133bacab1281b42c1040dcbd69dbacfb80890cff5b0 + md5: b457450ba3f27c4749783c0204bd17b0 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - libbrotlicommon >=1.2.0,<1.3.0a0 + size: 80027 + timestamp: 1786622846050 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlidec-1.2.0-h5295a6a_3.conda + sha256: 510c9fce0d9ffcf41741dd96fc05db381672109d5665ef002c2e58f0d4ca0118 + md5: e07a99c6fdd984f4d588060f2f936bf4 + depends: + - __osx >=11.0 + - libbrotlicommon 1.2.0 h1dcdb26_3 + license: MIT + license_family: MIT + run_exports: + weak: + - libbrotlidec >=1.2.0,<1.3.0a0 + size: 29935 + timestamp: 1786622857695 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlienc-1.2.0-h2ddc9cb_3.conda + sha256: eac412417eee2e93c62e9d53559f1f9c14f40b6c41e2c432af8b517596898dd1 + md5: 954c78a9f591bfb12c79beaff7338ec8 + depends: + - __osx >=11.0 + - libbrotlicommon 1.2.0 h1dcdb26_3 + license: MIT + license_family: MIT + run_exports: + weak: + - libbrotlienc >=1.2.0,<1.3.0a0 + size: 295650 + timestamp: 1786622868044 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-23.1.0-h55c6f16_0.conda + sha256: d3e5f0b767964af25ef81edffc002f8e822b1a5c55330da1ae3a857f70c4e4ee + md5: 5303ba06fab927399ed8dfb3227b0af8 + depends: + - __osx >=11.0 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + run_exports: {} + size: 575940 + timestamp: 1787698697875 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libdeflate-1.25-he7e0567_1.conda + sha256: d896f4aa4ce4c590c2838678cb1917356fdb461d2a189991c0280c818c362172 + md5: 78650d671cb56909bb3e5c13bce310f9 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - libdeflate >=1.25,<1.26.0a0 + size: 55727 + timestamp: 1785909153744 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libdovi-3.4.0-h78f8ca3_0.conda + sha256: 601623820de052084831278e9fce93aa47fc9afb571a84b806688e46920a276b + md5: 39f3bc34f80d45b03176c600f1d3c9f5 + depends: + - __osx >=11.0 + constrains: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - libdovi >=3.4.0,<4.0a0 + size: 357852 + timestamp: 1784281788521 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.8.1-hf6b4638_1.conda + sha256: 5af74261101e3c777399c6294b2b5d290e508153268eb2e9ff99c4d69834612f + md5: a915151d5d3c5bf039f5ccc8402a436f + depends: + - __osx >=11.0 + constrains: + - expat 2.8.1.* + license: MIT + license_family: MIT + run_exports: {} + size: 69362 + timestamp: 1781203631990 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.7.0-h47dc5ef_1.conda + sha256: 2783389a7b9dda04c62cc54c6bbeb03dd3cef24b3fc642715d02a3fa19464a9a + md5: 216bbcc23c11e9695b3db4a8771d76eb + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - libffi >=3.7.0,<3.8.0a0 + size: 43637 + timestamp: 1787753403688 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libfreetype-2.14.3-hce30654_2.conda + sha256: 19ce8bd320414cb6b9889ecbf48a3ded848b0d1068b76ff6bea099bd7387d6f3 + md5: ee1fc5bba400ff0cae27fbf141a1ae0c + depends: + - libfreetype6 >=2.14.3 + license: GPL-2.0-only OR FTL + run_exports: {} + size: 8367 + timestamp: 1786641013393 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libfreetype6-2.14.3-h2ed5691_2.conda + sha256: d9b203d6484ad491b5b5f33d49a9d7a91189d776a9adae53d2b63af18ec11e6a + md5: 881cfb44ea02cc9849f612725a959660 + depends: + - __osx >=11.0 + - libpng >=1.6.58,<1.7.0a0 + - libzlib >=1.3.2,<2.0a0 + constrains: + - freetype >=2.14.3 + license: GPL-2.0-only OR FTL + run_exports: {} + size: 340923 + timestamp: 1786641012794 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libglib-2.88.3-h89cd0c0_2.conda + sha256: 9c25def7c7f9ab98b265690c80012e917d080538875f9679b46ccd0c042e6201 + md5: 14ef48eb322e0bfb1e5fe31e047359e3 + depends: + - __osx >=11.0 + - libintl >=0.25.1,<1.0a0 + - libzlib >=1.3.2,<2.0a0 + - pcre2 >=10.47,<10.48.0a0 + - libiconv >=1.18,<2.0a0 + - libffi >=3.7.0,<3.8.0a0 + constrains: + - glib >2.66 + license: LGPL-2.1-or-later + run_exports: + weak: + - libglib >=2.88.3,<3.0a0 + size: 4443052 + timestamp: 1787884141804 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libharfbuzz-14.4.0-hcda0f7c_0.conda + sha256: d152fe1ade504acaa7d4167f74565ea1dfd85d88cc9085e076e0e7010068e894 + md5: c931e6d5af7f8f824fab6c474f3a1e94 + depends: + - __osx >=11.0 + - cairo >=1.18.4,<2.0a0 + - graphite2 >=1.3.15,<2.0a0 + - icu >=78.3,<79.0a0 + - libcxx >=21 + - libfreetype >=2.14.3 + - libfreetype6 >=2.14.3 + - libglib >=2.88.3,<3.0a0 + - libpng >=1.6.58,<1.7.0a0 + - libzlib >=1.3.2,<2.0a0 + license: MIT + license_family: MIT + run_exports: {} + size: 972888 + timestamp: 1787795084686 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libharfbuzz-devel-14.4.0-hcda0f7c_0.conda + sha256: 1c98121d5e12ffbb86a77b1477d3bd7eca6f5f22dab3eb50ba57bf2fc98234e8 + md5: 6f23309d5939e5c7edcf57095daaa99a + depends: + - __osx >=11.0 + - cairo >=1.18.4,<2.0a0 + - freetype + - graphite2 >=1.3.15,<2.0a0 + - icu >=78.3,<79.0a0 + - libcxx >=21 + - libfreetype >=2.14.3 + - libfreetype6 >=2.14.3 + - libglib >=2.88.3,<3.0a0 + - libharfbuzz 14.4.0 hcda0f7c_0 + - libpng >=1.6.58,<1.7.0a0 + - libzlib >=1.3.2,<2.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - libharfbuzz >=14.4.0 + size: 1508756 + timestamp: 1787795102730 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libhwloc-2.13.0-default_ha97f43a_1000.conda + sha256: d47c3c030671d196ff1cdd343e93eb2ae0d7b665cb79f8164cc91488796db437 + md5: fed55ddd65a830cb62e78f07cfffcd41 + depends: + - __osx >=11.0 + - libcxx >=19 + - libxml2 + - libxml2-16 >=2.14.6 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - libhwloc >=2.13.0,<2.13.1.0a0 + size: 2339152 + timestamp: 1770953916323 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libhwy-1.4.0-hd2bdd19_1.conda + sha256: d965f815878a176fb75329d02718f449230e687e100a081762688f763990abf2 + md5: 0fd99c6f562545ace82194d79e697ad3 + depends: + - __osx >=11.0 + - libcxx >=21 + license: Apache-2.0 OR BSD-3-Clause + run_exports: + weak: + - libhwy >=1.4.0,<1.5.0a0 + size: 612619 + timestamp: 1787282995626 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-he4c29f2_3.conda + sha256: 689a14968267f2f97c07112fca5636e7d756036b9aee969911267c20bd3eea1f + md5: 17f4744c0873e9f77ac9b5cd0a27c187 + depends: + - __osx >=11.0 + license: LGPL-2.1-only + run_exports: + weak: + - libiconv >=1.18,<2.0a0 + size: 750816 + timestamp: 1787033961086 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libintl-0.25.1-h493aca8_0.conda + sha256: 99d2cebcd8f84961b86784451b010f5f0a795ed1c08f1e7c76fbb3c22abf021a + md5: 5103f6a6b210a3912faf8d7db516918c + depends: + - __osx >=11.0 + - libiconv >=1.18,<2.0a0 + license: LGPL-2.1-or-later + run_exports: + weak: + - libintl >=0.25.1,<1.0a0 + size: 90957 + timestamp: 1751558394144 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libjpeg-turbo-3.2.0-h84a0fba_1.conda + sha256: 05006418f9392c9b723e8428808db106de0350a497832f95f921b85ff1072310 + md5: b2f8c8e5a7651a1d7c05404c3a159517 + depends: + - __osx >=11.0 + constrains: + - jpeg <0.0.0a + license: IJG AND BSD-3-Clause AND Zlib + run_exports: + weak: + - libjpeg-turbo >=3.2.0,<4.0a0 + size: 558459 + timestamp: 1785896382474 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libjxl-0.12.0-hb71b141_2.conda + sha256: 1f02661543be1722b619548b5207d6eaceab5bf95983fd6c9487f52bd8246a7a + md5: f62de5ae42f3c1f54f87ae2b303f6e90 + depends: + - libcxx >=21 + - __osx >=11.0 + - libhwy >=1.4.0,<1.5.0a0 + - libbrotlienc >=1.2.0,<1.3.0a0 + - libbrotlidec >=1.2.0,<1.3.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - libjxl >=0.12.0,<0.13.0a0 + size: 1054154 + timestamp: 1786691545529 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.3-h8088a28_1.conda + sha256: 23d0630046a3e8b164d8f80f2b74ed2605af2e7050ab9913018056402fae4311 + md5: 8ab10323068b107661a4b9a4af84f3b5 + depends: + - __osx >=11.0 + constrains: + - xz 5.8.3.* + license: 0BSD + run_exports: + weak: + - liblzma >=5.8.3,<6.0a0 + size: 91720 + timestamp: 1786348695846 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libogg-1.3.5-h48c0fde_1.conda + sha256: 28bd1fe20fe43da105da41b95ac201e95a1616126f287985df8e86ddebd1c3d8 + md5: 29b8b11f6d7e6bd0e76c029dcf9dd024 + depends: + - __osx >=11.0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - libogg >=1.3.5,<1.4.0a0 + size: 216719 + timestamp: 1745826006052 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-2026.3.1-h036fd89_0.conda + sha256: 422b0059e92917f5ea80c641f556023362f19bd3fa72cb5d50c2ab77dc2091fb + md5: 37ca8dbe3ad77aadf43d8ee33035f7f6 + depends: + - __osx >=12.0 + - libcxx >=21 + - pugixml >=1.15,<1.16.0a0 + - tbb >=2023.0.0 + license: Apache-2.0 + run_exports: + weak: + - libopenvino >=2026.3.1,<2026.3.2.0a0 + size: 4772558 + timestamp: 1787936017668 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-arm-cpu-plugin-2026.3.1-h036fd89_0.conda + sha256: 3fa44e84c427927fe5040cb4ab642a2cb4efe46c54baa47f09a182ec5b1e0037 + md5: 1fe6514109d05f8696631f04f174340a + depends: + - __osx >=12.0 + - libcxx >=21 + - libopenvino 2026.3.1 h036fd89_0 + - pugixml >=1.15,<1.16.0a0 + - tbb >=2023.0.0 + license: Apache-2.0 + run_exports: {} + size: 8765631 + timestamp: 1787936034648 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-auto-batch-plugin-2026.3.1-h4771a85_0.conda + sha256: 086a6f4efeda3077a833f63b6195d9af468b157e175643c64b4e794d25c683ba + md5: b99f89b3bc1b046cc526643daaf11f5d + depends: + - __osx >=12.0 + - libcxx >=21 + - libopenvino 2026.3.1 h036fd89_0 + - tbb >=2023.0.0 + license: Apache-2.0 + run_exports: {} + size: 105606 + timestamp: 1787936060570 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-auto-plugin-2026.3.1-h4771a85_0.conda + sha256: 9f00e5d401781bfe7d90729ecf67809c24f4067320846ff50bc7c3508d4f9d1c + md5: 170fd8a3f36f8ff24b60d7109956c452 + depends: + - __osx >=12.0 + - libcxx >=21 + - libopenvino 2026.3.1 h036fd89_0 + - tbb >=2023.0.0 + license: Apache-2.0 + run_exports: {} + size: 214275 + timestamp: 1787936070167 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-hetero-plugin-2026.3.1-h7561220_0.conda + sha256: d8d4e98a893e8a85090e90392ebeb0ed25855c565c2e3d3646da14ebc6b7033f + md5: 4750fb35e614f95b5e5ea3060719184a + depends: + - __osx >=12.0 + - libcxx >=21 + - libopenvino 2026.3.1 h036fd89_0 + - pugixml >=1.15,<1.16.0a0 + license: Apache-2.0 + run_exports: {} + size: 194811 + timestamp: 1787936079061 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-ir-frontend-2026.3.1-h7561220_0.conda + sha256: 1f12a149a9559df50c46a083c30ac120e0dd39fe88845dccb87e44928715f8f3 + md5: 3087697f4bbed2025ef11f00d7cfe443 + depends: + - __osx >=12.0 + - libcxx >=21 + - libopenvino 2026.3.1 h036fd89_0 + - pugixml >=1.15,<1.16.0a0 + license: Apache-2.0 + run_exports: + weak: + - libopenvino-ir-frontend >=2026.3.1,<2026.3.2.0a0 + size: 182564 + timestamp: 1787936087769 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-onnx-frontend-2026.3.1-h3fa9d4a_0.conda + sha256: 4046ab001e471992594f05179a9caca40d094713257fa145bcabe7f5d1a56a5f + md5: ab7f8de8b0f366501b09febc2cc096e4 + depends: + - __osx >=12.0 + - libabseil * cxx17* + - libabseil >=20260526.0,<20260527.0a0 + - libcxx >=21 + - libopenvino 2026.3.1 h036fd89_0 + - libprotobuf >=7.35.1,<7.35.2.0a0 + license: Apache-2.0 + run_exports: + weak: + - libopenvino-onnx-frontend >=2026.3.1,<2026.3.2.0a0 + size: 1584621 + timestamp: 1787936097749 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-paddle-frontend-2026.3.1-h3fa9d4a_0.conda + sha256: d91e96b093dcb5d5046aff6b78679a592c183f75f248616a67ef01378f1cb4b6 + md5: ba60ede3e72cc9d44813b9c36b9fdd9c + depends: + - __osx >=12.0 + - libabseil * cxx17* + - libabseil >=20260526.0,<20260527.0a0 + - libcxx >=21 + - libopenvino 2026.3.1 h036fd89_0 + - libprotobuf >=7.35.1,<7.35.2.0a0 + license: Apache-2.0 + run_exports: + weak: + - libopenvino-paddle-frontend >=2026.3.1,<2026.3.2.0a0 + size: 442356 + timestamp: 1787936109482 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-pytorch-frontend-2026.3.1-h3df7365_0.conda + sha256: 2dc0bca7bb5c618e743de86d5898eb74aae63db23b3c31b0871a53af8dffae1d + md5: 2d8d5024c8707ac062c78c5695bc20c3 + depends: + - __osx >=12.0 + - libcxx >=21 + - libopenvino 2026.3.1 h036fd89_0 + license: Apache-2.0 + run_exports: + weak: + - libopenvino-pytorch-frontend >=2026.3.1,<2026.3.2.0a0 + size: 862812 + timestamp: 1787936118796 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-tensorflow-frontend-2026.3.1-h5e3894e_0.conda + sha256: 1f3e0d994b9fed6229284c9fbd76d309fe3e9105b9841dc3f953eef79e8a1187 + md5: 3c5bf585804b80dd8900e675844c2e1f + depends: + - __osx >=12.0 + - libabseil * cxx17* + - libabseil >=20260526.0,<20260527.0a0 + - libcxx >=21 + - libopenvino 2026.3.1 h036fd89_0 + - libprotobuf >=7.35.1,<7.35.2.0a0 + - snappy >=1.2.2,<1.3.0a0 + license: Apache-2.0 + run_exports: + weak: + - libopenvino-tensorflow-frontend >=2026.3.1,<2026.3.2.0a0 + size: 922165 + timestamp: 1787936128534 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenvino-tensorflow-lite-frontend-2026.3.1-h3df7365_0.conda + sha256: 071d331f73232f5a0dd03a7463f058c35d360eafb3feb119a1ba80519ed27462 + md5: f117412cd84b786deebefc31abd7d97a + depends: + - __osx >=12.0 + - libcxx >=21 + - libopenvino 2026.3.1 h036fd89_0 + license: Apache-2.0 + run_exports: + weak: + - libopenvino-tensorflow-lite-frontend >=2026.3.1,<2026.3.2.0a0 + size: 409061 + timestamp: 1787936138040 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopus-1.6.1-h74c22ad_1.conda + sha256: f5fd2e1b1fec6da5d9218177b7d989af15f5aa14a84a1b1491dc3572e2457f1f + md5: 66c2ba320f3687ee66d53cc5546ded5f + depends: + - __osx >=11.0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - libopus >=1.6.1,<2.0a0 + size: 317033 + timestamp: 1787247651190 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libplacebo-7.360.1-hca394fb_1.conda + sha256: 7b50ecb8b110540b5b980ba94499fe760947e079bc119a7fac02dfb3df9cfb53 + md5: dff7f8dd3053f3253d79171216aa1db7 + depends: + - __osx >=11.0 + - libcxx >=19 + - libvulkan-loader >=1.4.341.0,<2.0a0 + - shaderc >=2026.3,<2026.4.0a0 + - lcms2 >=2.19.1,<3.0a0 + - libdovi >=3.4.0,<4.0a0 + license: LGPL-2.1-or-later + run_exports: + weak: + - libplacebo >=7.360.1,<7.361.0a0 + size: 529560 + timestamp: 1784287976080 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libpng-1.6.58-hf5e6511_1.conda + sha256: f88da60b348ee1f3e1a9c20fe02142fdafe889f08da0b1cc33c1f3f14460239d + md5: e0466c58fd07db190b9a81b5e58539f2 + depends: + - __osx >=11.0 + - libzlib >=1.3.2,<2.0a0 + license: zlib-acknowledgement + run_exports: + weak: + - libpng >=1.6.58,<1.7.0a0 + size: 290219 + timestamp: 1786616561185 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libprotobuf-7.35.1-h391e224_3.conda + sha256: 4129c13739e30dfbab8c20594acd8c182de2008f4dfc94d54f52b556eb39050c + md5: 7666c238f885d0c7927607ebfc163f97 + depends: + - __osx >=12.0 + - libabseil * cxx17* + - libabseil >=20260526.0,<20260527.0a0 + - libcxx >=21 + - libzlib >=1.3.2,<2.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - libprotobuf >=7.35.1,<7.35.2.0a0 + size: 2877232 + timestamp: 1787657252942 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/librsvg-2.62.3-he8aa2a2_0.conda + sha256: f5b4fb7b6f13bbfca59613bff2e70b5a398e80727b9d0f814837ffcbc34185e1 + md5: 6973724fadafe66ac6e4f1c55c191407 + depends: + - __osx >=11.0 + - cairo >=1.18.4,<2.0a0 + - fontconfig >=2.18.0,<3.0a0 + - fonts-conda-ecosystem + - gdk-pixbuf >=2.44.6,<3.0a0 + - harfbuzz >=14.2.0 + - libglib >=2.88.1,<3.0a0 + - libxml2-16 >=2.14.6 + - pango >=1.56.4,<2.0a0 + constrains: + - __osx >=11.0 + license: LGPL-2.1-or-later + run_exports: + weak: + - librsvg >=2.62.3,<3.0a0 + size: 2397567 + timestamp: 1780452232118 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libtiff-4.7.2-hf67920b_1.conda + sha256: 847d22a86ae28f66d3888702698254d25bb4eac3a0f64c1fabfb1842ac00be4c + md5: b6b191267455bf202af79f973690ed5d + depends: + - __osx >=11.0 + - lerc >=4.2.0,<5.0a0 + - libcxx >=21 + - libdeflate >=1.25,<1.26.0a0 + - libjpeg-turbo >=3.2.0,<4.0a0 + - liblzma >=5.8.3,<6.0a0 + - libwebp-base >=1.6.0,<2.0a0 + - libzlib >=1.3.2,<2.0a0 + - zstd >=1.5.7,<1.6.0a0 + license: HPND + run_exports: + weak: + - libtiff >=4.7.2,<4.8.0a0 + size: 380645 + timestamp: 1787756067271 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libusb-1.0.29-hbc156a2_0.conda + sha256: 5eee9a2bf359e474d4548874bcfc8d29ebad0d9ba015314439c256904e40aaad + md5: f6654e9e96e9d973981b3b2f898a5bfa + depends: + - __osx >=11.0 + license: LGPL-2.1-or-later + run_exports: + weak: + - libusb >=1.0.29,<2.0a0 + size: 83849 + timestamp: 1748856224950 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libvorbis-1.3.7-h81086ad_2.conda + sha256: 95768e4eceaffb973081fd986d03da15d93aa10609ed202e6fd5ca1e490a3dce + md5: 719e7653178a09f5ca0aa05f349b41f7 + depends: + - libogg + - libcxx >=19 + - __osx >=11.0 + - libogg >=1.3.5,<1.4.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - libvorbis >=1.3.7,<1.4.0a0 + size: 259122 + timestamp: 1753879389702 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libvpx-1.17.0-h484c67d_0.conda + sha256: 2413c6959f78ed821aaa74f596ea3678ef54a63d89c8517f1146dfbdac3f798d + md5: 10b1026148bced752958b3c217cb054a + depends: + - __osx >=11.0 + - libcxx >=21 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - libvpx >=1.17.0,<1.18.0a0 + size: 1234768 + timestamp: 1787764200133 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libvulkan-loader-1.4.357.0-hfbe1efa_2.conda + sha256: aaf22c7f0bbd2bac3a070a49e03c0c4baedf16ce22ad80ca697a7c15422c5de8 + md5: 054a1eb01ab7f0f9aa3bdbe1594e0366 + depends: + - __osx >=11.0 + - libcxx >=21 + constrains: + - libvulkan-headers 1.4.357.0.* + license: Apache-2.0 + license_family: APACHE + run_exports: + weak: + - libvulkan-loader >=1.4.357.0,<2.0a0 + size: 186735 + timestamp: 1787491990282 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libwebp-base-1.6.0-h202fb40_1.conda + sha256: 0ff54650d470c7e54cbeffdd53a8c063e055a7bcf784388c0385ce5c4741b0f4 + md5: 168a13e329259710b28277abc1395b8e + depends: + - __osx >=11.0 + constrains: + - libwebp 1.6.0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - libwebp-base >=1.6.0,<2.0a0 + size: 294522 + timestamp: 1785955350410 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libxml2-16-2.15.3-h5ef1a60_1.conda + sha256: 2ca61d8287339c726b151fa612e83d2afa1418b89fa17694c873dc231bc9b077 + md5: f86c0b8ac5c866049717b55df1049c2c + depends: + - __osx >=11.0 + - icu >=78.3,<79.0a0 + - libiconv >=1.18,<2.0a0 + - liblzma >=5.8.3,<6.0a0 + - libzlib >=1.3.2,<2.0a0 + constrains: + - libxml2 2.15.3 + license: MIT + license_family: MIT + run_exports: {} + size: 466188 + timestamp: 1787237580766 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libxml2-2.15.3-h5654f7c_1.conda + sha256: ff1b70683ccb82f8aa7f3eda53405a2036d849daba4e4e622f330dcd49d37700 + md5: 06a3f8eb5cdde56b2d10b49ae3337954 + depends: + - __osx >=11.0 + - icu >=78.3,<79.0a0 + - libiconv >=1.18,<2.0a0 + - liblzma >=5.8.3,<6.0a0 + - libxml2-16 2.15.3 h5ef1a60_1 + - libzlib >=1.3.2,<2.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - libxml2 + - libxml2-16 >=2.15.3 + size: 41264 + timestamp: 1787237585903 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + sha256: a18fa5d5bac452401459f966cf0d872224e8080c4ff93c77e168d43ab42ef9d7 + md5: f39288f0ea63ae962e1a2e4f355a0d75 + depends: + - __osx >=11.0 + constrains: + - zlib 1.3.2 *_3 + license: Zlib + license_family: Other + run_exports: + weak: + - libzlib >=1.3.2,<2.0a0 + size: 47822 + timestamp: 1785277049190 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/mpg123-1.33.7-h0e3f465_1.conda + sha256: 36cdae1076fc8d92b030af4e8d02e32629aaa31dbe32233040a04d495815b0bb + md5: 84ddf9a511461763c80703597bb91875 + depends: + - __osx >=11.0 + - libcxx >=21 + license: LGPL-2.1-only + license_family: LGPL + run_exports: + weak: + - mpg123 >=1.33.7,<1.34.0a0 + size: 371102 + timestamp: 1787312032517 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/openh264-2.6.0-hf8510ef_2.conda + sha256: abbf05ef3f338eb70623400c6ecf3318fe3a71fbf56b1099d7b3fddd13a0a0aa + md5: 9cafae3c7258971bbf051eeb0e78a628 + depends: + - __osx >=11.0 + - libcxx >=21 + license: BSD-2-Clause + license_family: BSD + run_exports: + weak: + - openh264 >=2.6.0,<2.6.1.0a0 + size: 604653 + timestamp: 1787274245300 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.4-h55eecbc_0.conda + sha256: f23239eacd75c4c50705e68fae1aa3292da473e6a3a4abe2330f1e6afa680704 + md5: ae71ab40048c19a389a7dcccb86c2481 + depends: + - __osx >=11.0 + - ca-certificates + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - openssl >=3.6.4,<4.0a0 + size: 3110142 + timestamp: 1787698648639 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pango-1.58.2-hf80efc4_0.conda + sha256: c300fba11c4cd7cdd7609b6f165981af24d2181d40280ca6af420710c4c7d42e + md5: 83c3d3d895dd96f87b0a1916e2c41f70 + depends: + - __osx >=11.0 + - cairo >=1.18.4,<2.0a0 + - fontconfig >=2.18.2,<3.0a0 + - fonts-conda-ecosystem + - fribidi >=1.0.16,<2.0a0 + - libexpat >=2.8.1,<3.0a0 + - libfreetype >=2.14.3 + - libfreetype6 >=2.14.3 + - libglib >=2.88.3,<3.0a0 + - libharfbuzz >=14.3.0 + - libpng >=1.6.58,<1.7.0a0 + - libzlib >=1.3.2,<2.0a0 + license: LGPL-2.1-or-later + run_exports: + weak: + - pango >=1.58.2,<2.0a0 + size: 444011 + timestamp: 1786108209790 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pcre2-10.47-he63d830_1.conda + sha256: f8c415329b542e1fca1ff2917b80e687c18302dfa0353530668b90cb225b494f + md5: 5ab90033816cbe4097e8a297e5179f67 + depends: + - __osx >=11.0 + - bzip2 >=1.0.8,<2.0a0 + - libzlib >=1.3.2,<2.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - pcre2 >=10.47,<10.48.0a0 + size: 851704 + timestamp: 1787294559157 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pixman-0.46.4-h784d473_3.conda + sha256: 4779cd57231ce2e96fac643fcbdd4a6f51a57986264545445574e9c4acf526d3 + md5: 9a99c0b60efe41c194d01c182d000733 + depends: + - __osx >=11.0 + - libcxx >=19 + license: MIT + license_family: MIT + run_exports: + weak: + - pixman >=0.46.4,<1.0a0 + size: 198717 + timestamp: 1786106922508 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pugixml-1.15-hd3d436d_0.conda + sha256: 5ad8d036040b095f85d23c70624d3e5e1e4c00bc5cea97831542f2dcae294ec9 + md5: b9a4004e46de7aeb005304a13b35cb94 + depends: + - __osx >=11.0 + - libcxx >=18 + license: MIT + license_family: MIT + run_exports: + weak: + - pugixml >=1.15,<1.16.0a0 + size: 91283 + timestamp: 1736601509593 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/sdl2-2.32.56-h784d473_0.conda + sha256: 595db3f62eec1b86aad03ad8c7e4943e78a18e112228c91adf3f3ada4e959a5c + md5: 81a4c982e9ac52e620eb810f463de9ad + depends: + - __osx >=11.0 + - libcxx >=19 + - sdl3 >=3.4.12,<4.0a0 + license: Zlib + run_exports: + weak: + - sdl2 >=2.32.56,<3.0a0 + size: 543557 + timestamp: 1783451926011 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/sdl3-3.4.14-h6fa9c73_0.conda + sha256: d8b6805b8b1011afcad6c857ddd9fd38335f32dcf369152a8ec0615518f6331a + md5: 1e0f2089d4efd60b4407466b0f4cf81a + depends: + - __osx >=11.0 + - libcxx >=19 + - libusb >=1.0.29,<2.0a0 + - libvulkan-loader >=1.4.357.0,<2.0a0 + - dbus >=1.16.2,<2.0a0 + license: Zlib + run_exports: + weak: + - sdl3 >=3.4.14,<4.0a0 + size: 1569006 + timestamp: 1785816152396 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/shaderc-2026.3-hfd4ff19_1.conda + sha256: 1fbea82ad781f388ed21306e967a864895f56cb3f1debb41017f05dc18f5477f + md5: bb6a0172ab548a4de87805182f3afb0b + depends: + - __osx >=11.0 + - glslang >=16,<17.0a0 + - libcxx >=21 + - spirv-tools >=2026,<2027.0a0 + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - shaderc >=2026.3,<2026.4.0a0 + size: 112461 + timestamp: 1787711281322 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/snappy-1.2.2-hada39a4_1.conda + sha256: cb9305ede19584115f43baecdf09a3866bfcd5bcca0d9e527bd76d9a1dbe2d8d + md5: fca4a2222994acd7f691e57f94b750c5 + depends: + - libcxx >=19 + - __osx >=11.0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - snappy >=1.2.2,<1.3.0a0 + size: 38883 + timestamp: 1762948066818 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/spirv-tools-2026.3-hc0b298b_1.conda + sha256: 962119a27ea90e229d8294dab4d21b221b630898a31cbc0c90cfb9f874e6c132 + md5: 92f682d75053c1fe0743ded97193d6e5 + depends: + - __osx >=11.0 + - libcxx >=21 + constrains: + - spirv-headers >=1.4.357.0,<1.4.357.1.0a0 + license: Apache-2.0 + license_family: APACHE + run_exports: + weak: + - spirv-tools >=2026,<2027.0a0 + size: 1683131 + timestamp: 1787525925536 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/svt-av1-4.2.0-h484c67d_1.conda + sha256: 9122d11ed85053b1223ea0645aeac0ca583adc7482e60e3266c37e70d5242195 + md5: 253103cc53f925c8a19fd07a000d7b78 + depends: + - __osx >=11.0 + - libcxx >=21 + license: BSD-2-Clause + license_family: BSD + run_exports: + weak: + - svt-av1 >=4.2.0,<4.2.1.0a0 + size: 1539599 + timestamp: 1787257109113 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/tbb-2023.0.0-he0260a5_2.conda + sha256: 6f72a2984052444b9381020fa329b83dace95f335573dc21199f1b1d1a5f5473 + md5: 440c0a36cc20db1f28877a69afbb5e88 + depends: + - __osx >=11.0 + - libcxx >=19 + - libhwloc >=2.13.0,<2.13.1.0a0 + license: Apache-2.0 + license_family: APACHE + run_exports: {} + size: 122303 + timestamp: 1778675142610 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/x264-1!164.3095-h57fd34a_2.tar.bz2 + sha256: debdf60bbcfa6a60201b12a1d53f36736821db281a28223a09e0685edcce105a + md5: b1f6dccde5d3a1f911960b6e567113ff + license: GPL-2.0-or-later + license_family: GPL + run_exports: + weak: + - x264 >=1!164.3095,<1!165 + size: 717038 + timestamp: 1660323292329 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/x265-3.5-hbc6ce65_3.tar.bz2 + sha256: 2fed6987dba7dee07bd9adc1a6f8e6c699efb851431bcb6ebad7de196e87841d + md5: b1f7f2780feffe310b068c021e8ff9b2 + depends: + - libcxx >=12.0.1 + license: GPL-2.0-or-later + license_family: GPL + run_exports: + weak: + - x265 >=3.5,<3.6.0a0 + size: 1832744 + timestamp: 1646609481185 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda + sha256: da867f5092eb0cb746d353694f0098031fd9817a4ce7d5743121209ae0f406ca + md5: 4ec2684c73812cc2c3d78379384a39cc + depends: + - __osx >=11.0 + - libzlib >=1.3.2,<2.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - zstd >=1.5.7,<1.6.0a0 + size: 433687 + timestamp: 1786599629846 diff --git a/examples/transcode-demo/macos-aarch64-metal/pixi.toml b/examples/transcode-demo/macos-aarch64-metal/pixi.toml new file mode 100644 index 0000000..a920c10 --- /dev/null +++ b/examples/transcode-demo/macos-aarch64-metal/pixi.toml @@ -0,0 +1,16 @@ +# A `native` box: no interpreter at all, and a binary nobody wants to build themselves. +# +# ffmpeg is the archetype the `native` runtime exists for — a large compiled program with a long +# tail of codec libraries, where "just install it" means a different version on every machine and a +# different answer from each. The box pins one, carries everything it links against, and is signed, +# so the transcode a user runs is the transcode that was tested. +# +# There is no `[dependencies]` entry for a runtime here, and that is the point: `native` installs no +# interpreter. What the lock holds is what the binary needs to load. +[workspace] +name = "transcode-demo" +channels = ["conda-forge"] +platforms = ["osx-arm64"] + +[dependencies] +ffmpeg = "9.*" diff --git a/examples/transcode-demo/macos-aarch64-metal/scroll.json b/examples/transcode-demo/macos-aarch64-metal/scroll.json new file mode 100644 index 0000000..cd91ac9 --- /dev/null +++ b/examples/transcode-demo/macos-aarch64-metal/scroll.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "schemaVersion": 3, + "boxId": "transcode-demo", + "version": "1.0.0", + "sourceRevision": "ffmpeg-9-conda-forge", + "target": { + "platform": "macos", + "arch": "aarch64", + "accelerator": "metal" + }, + "compatibility": { + "minHostAppVersion": "1.0.0", + "minMacosVersion": "13.0" + }, + "runtime": { + "id": "native" + }, + "pixiVersion": "0.73.0", + "condaDependencyLicenseAudit": "examples/transcode-demo/macos-aarch64-metal/conda-licenses.json", + "cacheSubdir": "cache/transcode-demo", + "publishBaseUrl": "https://assets.example.org/boxes", + "selfTest": { + "commands": [ + { "args": ["-version"] }, + { + "args": [ + "-f", "lavfi", + "-i", "testsrc=duration=1:size=320x240:rate=10", + "-c:v", "libx264", + "-f", "null", + "-" + ] + }, + { + "args": ["-i", "no-such-input.mp4", "-f", "null", "-"], + "expectExitCode": 254 + } + ] + }, + "execution": { + "kind": "native-binary", + "binary": "venv/bin/ffmpeg", + "defaultArgs": ["-hide_banner"] + } +} diff --git a/python/pyproject.toml b/python/pyproject.toml index 801c085..123badd 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -25,6 +25,7 @@ classifiers = [ dependencies = [ "cryptography>=42,<47", "jsonschema>=4.21,<5", + "referencing>=0.28.4,<1", ] [project.urls] diff --git a/python/src/scrollcase_consumer/schemas/release-manifest.schema.json b/python/src/scrollcase_consumer/schemas/release-manifest.schema.json index 24df66c..7c4abae 100644 --- a/python/src/scrollcase_consumer/schemas/release-manifest.schema.json +++ b/python/src/scrollcase_consumer/schemas/release-manifest.schema.json @@ -84,7 +84,6 @@ "additionalProperties": false, "required": [ "format", - "url", "sha256", "sizeBytes" ], @@ -94,7 +93,8 @@ }, "url": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "Where the archive is published, for the distribution layer that has to fetch it. Absent when the box was built without a publish base URL, which is what a box built to run locally is.\n\nNothing verifies this value and no Scrollcase consumer reads one: an archive is resolved beside its release document and identified by sha256, so a wrong URL here would break a download and no check at all. That is why an absent one is preferred to an invented one \u2014 a false address inside a signed, immutable document stays false forever." }, "sha256": { "$ref": "#/$defs/sha256" diff --git a/python/tests/conformance_support.py b/python/tests/conformance_support.py index bc37e60..72e6f2a 100644 --- a/python/tests/conformance_support.py +++ b/python/tests/conformance_support.py @@ -245,6 +245,16 @@ def _mutate_fixture( fixture.release["environment"] = {"SCROLLCASE_CHANGED_AFTER_BUILD": "1"} fixture.sign() return + if mutation == "strip-release-archive-url": + # A box built without a publish base URL: it was never published, so its release names no + # address for the archive. Every consumer must prepare it exactly as it prepares any other, because + # the URL was never part of the trust chain — the archive is found beside the release document and + # identified by its sha256. + archive = dict(fixture.release["archive"]) + archive.pop("url", None) + fixture.release["archive"] = archive + fixture.sign() + return if mutation == "alter-release-bundled-licenses": # A licence inventory added to the signed release after the box was built. It is signed, so # the signature still verifies; what refuses it is that box.json says something else, which diff --git a/python/tests/test_dependencies.py b/python/tests/test_dependencies.py new file mode 100644 index 0000000..1955be0 --- /dev/null +++ b/python/tests/test_dependencies.py @@ -0,0 +1,113 @@ +"""Every third-party module the package imports must be a declared dependency. + +`referencing` was imported for the schema registry and never declared: it arrived only because +`jsonschema` happens to depend on it today. That works until the day it does not, and a conda-forge +submission — where the declared run requirements are what the solver builds an environment from — +makes the omission a packaging bug rather than a latent one. This walks the shipped source instead +of trusting the list to stay correct by hand. +""" + +from __future__ import annotations + +import ast +import re +import sys +import unittest +from importlib.metadata import packages_distributions +from pathlib import Path + +PACKAGE = "scrollcase_consumer" + + +def normalize(name: str) -> str: + """PEP 503 name normalization, so `types-jsonschema` and `types_jsonschema` are one name.""" + + return re.sub(r"[-_.]+", "-", name).lower() + + +def declared_dependencies(pyproject: str) -> set[str]: + """The normalized names in `[project] dependencies`, read as text. + + The runtime list is a flat array of literals, and reading it this way keeps the test free of a + TOML parser the package itself does not need on Python 3.10. + """ + + lines = iter(pyproject.splitlines()) + for line in lines: + if line.startswith("dependencies = ["): + break + else: # pragma: no cover - the array is what this module exists to read. + raise AssertionError("pyproject.toml declares no [project] dependencies array") + + names: set[str] = set() + for line in lines: + if line.startswith("]"): + return names + requirement = line.strip().strip(",").strip('"') + if requirement: + # Split on the first character that can follow a name in a PEP 508 requirement. + names.add(normalize(re.split(r"[<>=!~;\[ ]", requirement, maxsplit=1)[0])) + raise AssertionError("the dependencies array is unterminated") + + +def imported_roots(source: Path) -> set[str]: + """The absolute top-level modules one file imports, relative imports excluded.""" + + roots: set[str] = set() + tree = ast.parse(source.read_text(encoding="utf-8"), filename=str(source)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + roots.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + roots.add(node.module.split(".")[0]) + return roots + + +class DeclaredDependencyTests(unittest.TestCase): + def setUp(self) -> None: + self.python_root = Path(__file__).resolve().parents[1] + self.declared = declared_dependencies( + (self.python_root / "pyproject.toml").read_text(encoding="utf-8") + ) + self.sources = sorted((self.python_root / "src" / PACKAGE).rglob("*.py")) + + def third_party_imports(self) -> dict[str, set[Path]]: + """Third-party module roots the package imports, each with the files that import it.""" + + # An installed distribution can publish a module under a different name; ask the metadata + # rather than assume `import x` means a distribution called `x`. + distributions = packages_distributions() + found: dict[str, set[Path]] = {} + for source in self.sources: + for root in imported_roots(source): + if root in sys.stdlib_module_names or root in {"__future__", PACKAGE}: + continue + for distribution in distributions.get(root, [root]): + found.setdefault(normalize(distribution), set()).add(source) + return found + + def test_finds_the_modules_it_is_meant_to_check(self) -> None: + # Guard the guard: a walk that silently found nothing would pass every assertion below. + self.assertGreater(len(self.sources), 1) + self.assertIn("jsonschema", self.third_party_imports()) + + def test_every_third_party_import_is_declared(self) -> None: + for distribution, sources in sorted(self.third_party_imports().items()): + with self.subTest(distribution=distribution): + where = ", ".join( + sorted(str(source.relative_to(self.python_root)) for source in sources) + ) + self.assertIn( + distribution, + self.declared, + f"{distribution} is imported by {where} but not in [project] dependencies", + ) + + def test_declares_referencing_rather_than_inheriting_it(self) -> None: + # The specific omission the conda-forge review caught, named so a future edit that drops + # it again fails with the reason attached. + self.assertIn("referencing", self.declared) + + +if __name__ == "__main__": + unittest.main() diff --git a/rust/fixtures/consumer-conformance.json b/rust/fixtures/consumer-conformance.json index 87fc105..9aeaa4c 100644 --- a/rust/fixtures/consumer-conformance.json +++ b/rust/fixtures/consumer-conformance.json @@ -59,6 +59,23 @@ } } }, + { + "id": "release-with-no-published-location", + "action": "prepare", + "mutation": "strip-release-archive-url", + "expected": { + "outcome": "prepared", + "receipt": { + "status": "prepared", + "boxId": "consumer-fixture", + "executionKind": "python-script", + "requiredAssetCount": 0, + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", + "targetId": "$NATIVE_TARGET" + } + } + }, { "id": "trusted-single-key-in-memory", "action": "prepare", diff --git a/rust/src/contract/schema/release-manifest.schema.json b/rust/src/contract/schema/release-manifest.schema.json index 24df66c..7c4abae 100644 --- a/rust/src/contract/schema/release-manifest.schema.json +++ b/rust/src/contract/schema/release-manifest.schema.json @@ -84,7 +84,6 @@ "additionalProperties": false, "required": [ "format", - "url", "sha256", "sizeBytes" ], @@ -94,7 +93,8 @@ }, "url": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "Where the archive is published, for the distribution layer that has to fetch it. Absent when the box was built without a publish base URL, which is what a box built to run locally is.\n\nNothing verifies this value and no Scrollcase consumer reads one: an archive is resolved beside its release document and identified by sha256, so a wrong URL here would break a download and no check at all. That is why an absent one is preferred to an invented one \u2014 a false address inside a signed, immutable document stays false forever." }, "sha256": { "$ref": "#/$defs/sha256" diff --git a/rust/src/release.rs b/rust/src/release.rs index 2517381..46c8d38 100644 --- a/rust/src/release.rs +++ b/rust/src/release.rs @@ -78,8 +78,12 @@ pub struct Compatibility { pub struct Archive { /// Always `zip`. pub format: String, - /// Where the archive was published. This crate never fetches it. - pub url: String, + /// Where the archive was published, for the distribution layer that has to fetch it. Absent + /// when the box was built without a publish base URL, which is what a box built to run locally + /// is. This crate never reads it: an archive is resolved beside its release document and + /// identified by [`Self::sha256`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url: Option, /// Lowercase hex SHA-256 of the archive bytes. pub sha256: String, /// Exact archive size. @@ -525,15 +529,16 @@ impl ReleaseManifest { } } self.runtime.validate()?; - for (label, value) in [ - ("version", &self.version), - ("cacheSubdir", &self.cache_subdir), - ("archive.url", &self.archive.url), - ] { + for (label, value) in [("version", &self.version), ("cacheSubdir", &self.cache_subdir)] { if value.is_empty() { fail!("Invalid release manifest: {label} must not be empty."); } } + // Optional, but an empty string is not the way to say "absent": that is a field the + // publisher filled in with nothing, which no reader can act on. + if self.archive.url.as_ref().is_some_and(String::is_empty) { + fail!("Invalid release manifest: archive.url must not be empty."); + } if self.archive.format != "zip" { fail!("Invalid release manifest: archive format must be zip."); } diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index d4bc7a6..2dc5975 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -707,6 +707,17 @@ fn mutate_fixture(fixture: &mut Fixture, mutation: &str, destination: &Path) { // A licence inventory added to the signed release after the box was built. It is signed, // so the signature still verifies; what refuses it is that box.json says something else, // which is the whole reason the inventory is compared field by field rather than carried. + // A box built without a publish base URL: it was never published, so its release names no + // address for the archive. Every consumer must prepare it exactly as it prepares any other, because + // the URL was never part of the trust chain — the archive is found beside the release document and + // identified by its sha256. + "strip-release-archive-url" => { + fixture.release["archive"] + .as_object_mut() + .expect("archive is an object") + .remove("url"); + fixture.sign(); + } "alter-release-bundled-licenses" => { fixture.release["bundledLicenses"] = json!([{"name": "zlib", "version": "1.3.1", "declaredLicense": "Zlib", "linkedInto": ["box.json"]}]); fixture.sign(); @@ -1269,7 +1280,7 @@ fn the_shared_consumer_conformance_suite_passes() { let suite: Value = serde_json::from_str(SUITE).unwrap(); let patterns = suite["errorPatterns"].as_object().unwrap(); let cases = suite["cases"].as_array().unwrap(); - assert_eq!(cases.len(), 85, "the suite changed size"); + assert_eq!(cases.len(), 86, "the suite changed size"); let mut failures: Vec = Vec::new(); let mut ran = 0usize; diff --git a/scripts/verify-built-docs.mjs b/scripts/verify-built-docs.mjs index b1d3ea6..d1eb3c3 100644 --- a/scripts/verify-built-docs.mjs +++ b/scripts/verify-built-docs.mjs @@ -23,8 +23,32 @@ async function requireFile(path, label) { } } +/** Every rendered page, as `[route, file]`. The sitemap is the yardstick for what the site puts + * forward; this is the yardstick for what it actually serves, deprecated documentation included. */ +async function builtPages(directory, prefix = '') { + const found = []; + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + found.push(...await builtPages(path, `${prefix}/${entry.name}`)); + } else if (entry.name === 'index.html') { + found.push([`${prefix}/`, path]); + } else if (entry.name.endsWith('.html')) { + found.push([`${prefix}/${entry.name.slice(0, -'.html'.length)}`, path]); + } + } + return found; +} + await requireFile(join(distDir, 'privacy.html'), '/privacy'); +// `Number`, because package.json carries this as the string "3" while the format itself writes it +// as the integer 3. The comparison below is about the version, not about which of the two spellings +// a given file happens to use. +const packageSchemaVersion = Number(JSON.parse( + (await requireFile(join(root, 'package.json'), 'package.json')).toString('utf8'), +).schemaVersion); + const schemaNames = (await readdir(schemaSource)) .filter((name) => name.endsWith('.schema.json')) .sort(); @@ -94,7 +118,61 @@ for (const url of routes) { if (!markdown.includes(`\nsource: ${url}\n`)) { throw new Error(`${twin} does not name ${url} as its source.`); } + if (markdown.includes('\ndeprecated: true\n')) { + throw new Error(`${twin} is marked deprecated, but ${url} is the current documentation.`); + } +} + +// The deprecated pages' own twins, which the loop above cannot reach: those routes are kept out of +// the sitemap on purpose. They are also the reason this check exists. The banner a person sees is a +// Vue component living in the rendered HTML, so it never reaches these files — leaving the audience +// most likely to mistake a superseded manual for the current one as the only audience told nothing. +// The notice has to be in the frontmatter, where a parser finds it, and in the text, where +// everything that does not parse frontmatter finds it. +// `/404` is excluded: VitePress renders it without a navbar at all, so it has no chrome to check. +const pages = (await builtPages(distDir)).filter(([route]) => route !== '/404'); + +let deprecatedTwins = 0; +for (const [route] of pages) { + if (!route.startsWith('/v2/')) continue; + const twin = markdownPathFor(route); + const markdown = (await requireFile(join(distDir, twin), `the Markdown twin of ${route}`)).toString('utf8'); + const [frontmatter, ...rest] = markdown.split('\n---\n'); + if (!frontmatter.includes('\ndeprecated: true')) { + throw new Error(`${twin} serves version 2 documentation without a deprecated flag a parser can read.`); + } + + // The two schema versions, which are the fact a consumer can act on: it holds a box carrying + // `schemaVersion`, and comparing that number is how it works out which manual describes what it + // has. So they are checked as numbers, against the package rather than against a literal — the + // day schema version 4 ships, a `current-schema-version: 3` left behind here is a lie told to + // every machine that reads it, and nothing else in the build would notice. + const field = (name) => frontmatter.match(new RegExp(`\\n${name}: (\\S+)`))?.[1]; + const was = field('schema-version'); + const now = field('current-schema-version'); + if (was !== '2') { + throw new Error(`${twin} says it documents schema version ${was ?? 'nothing'}; these pages document 2.`); + } + if (Number(now) !== packageSchemaVersion) { + throw new Error(`${twin} names ${now ?? 'no'} as the current schema version; package.json says ${packageSchemaVersion}.`); + } + if (was === now) { + throw new Error(`${twin} says the deprecated and current schema versions are both ${now}.`); + } + if (!rest.join('\n---\n').trimStart().startsWith('> **DEPRECATED.**')) { + throw new Error(`${twin} does not open by saying it is deprecated, so anything reading the prose is not told.`); + } + // `current` claims a replacement page exists. Emitted only when one does, and it has to be built. + const current = frontmatter.match(/\ncurrent: (\S+)/)?.[1]; + if (current) { + const path = new URL(current).pathname; + if (path === '/') throw new Error(`${twin} names the home page as its replacement, which replaces nothing.`); + const file = path.endsWith('/') ? `${path}index.html` : `${path}.html`; + await requireFile(join(distDir, file), `the replacement page ${current} named by ${twin}`); + } + deprecatedTwins += 1; } +if (!deprecatedTwins) throw new Error('No deprecated Markdown twins were checked; this guard read nothing.'); const llmsIndex = (await requireFile(join(distDir, 'llms.txt'), '/llms.txt')).toString('utf8'); const llmsFull = (await requireFile(join(distDir, 'llms-full.txt'), '/llms-full.txt')).toString('utf8'); @@ -145,6 +223,58 @@ if (JSON.stringify(cataloguedSchemas) !== JSON.stringify(schemaNames)) { throw new Error('The API catalogue and the shipped contract disagree about which schemas exist.'); } +// The version switch is computed by Vue from each page's own route, so neither of its two links is +// written down anywhere the dead-link pass can read. That is what makes it worth checking: a link +// pointing at a route the build never emitted looks perfectly normal on the page, and 404s only for +// the reader who used it. +const switchBlock =/
/; +const switchTarget = /href="([^"]*)"[^>]*>(v2|v3)<\/a>/g; +let checkedSwitches = 0; +for (const [route, file] of pages) { + const html = (await readFile(file)).toString('utf8'); + const block = html.match(switchBlock)?.[0]; + if (!block) throw new Error(`${route} renders no version switch; a reader there cannot change version.`); + const targets = [...block.matchAll(switchTarget)].map((match) => match[1]); + if (targets.length !== 2) { + throw new Error(`${route} offers ${targets.length} versions to switch between rather than two.`); + } + for (const target of targets) { + const path = target.endsWith('/') ? `${target}index.html` : `${target}.html`; + await requireFile(join(distDir, path), `the version switch target ${target} on ${route}`); + } + + // The navbar menu comes from the locale the page belongs to, and getting that wrong is silent: + // the menu renders, every link works, and it walks the reader into the other version's manual + // without saying so. Whichever set a page is in, its menu has to stay in that set. + const deprecated = route.startsWith('/v2/'); + + // The standing deprecation notice, on exactly the deprecated pages. It is the only thing telling + // a reader who arrived from a search result that they are in a superseded manual, and it is + // rendered by a component that decides for itself — so losing it everywhere, or gaining it on the + // current documentation, are both a silent one-line change away. + const notified = html.includes('deprecation-notice'); + if (notified !== deprecated) { + throw new Error(notified + ? `${route} carries the deprecation notice, but it is the current documentation.` + : `${route} is deprecated documentation and says so nowhere on the page.`); + } + + const menus = [...html.matchAll(/]*class="VPNav(?:Bar|Screen)Menu[\s\S]*?<\/nav>/g)] + .map((match) => match[0]); + // Found by writing this check against the wrong attribute order and watching it pass on markup it + // had never matched: a menu the regex misses is an empty string, and an empty string strays nowhere. + if (!menus.length) throw new Error(`${route} renders no navbar menu for this check to read.`); + for (const menu of menus) { + const strays = [...menu.matchAll(/href="(\/[^"]*)"/g)] + .map((match) => match[1]) + .filter((href) => href.startsWith('/v2/') !== deprecated); + if (strays.length) { + throw new Error(`${route} has a navbar menu pointing at the other version: ${strays.join(', ')}`); + } + } + checkedSwitches += 1; +} + const platformHtml = (await requireFile( join(distDir, 'guides', 'platform-examples.html'), 'the platform examples page', @@ -181,6 +311,8 @@ if (panelTags.filter((tag) => !tag.includes('style="display:none;"')).length !== console.log( `Verified built privacy route, ${schemaNames.length} schemas, platform tab semantics, ` - + `the API catalogue, and canonical, Markdown twin, llms.txt and llms-full.txt coverage ` + + `the API catalogue, the version switch, navbar menu and deprecation notice on ` + + `${checkedSwitches} built pages, ` + + `and canonical, Markdown twin, llms.txt and llms-full.txt coverage ` + `of ${routes.length} pages.`, ); diff --git a/src/build/authoring.mjs b/src/build/authoring.mjs index c597e64..960ae9c 100644 --- a/src/build/authoring.mjs +++ b/src/build/authoring.mjs @@ -324,6 +324,39 @@ async function validateScroll(scroll) { if (error) fail(`Generated scroll is invalid: ${error}.`); } +/** + * What a box id may look like, in words rather than as a regular expression. + * + * Shown in the prompt and repeated when an answer is refused. A user meeting the tool does not read + * `^[a-z0-9]+(?:[-.][a-z0-9]+)*$` and know what to type. + */ +export const BOX_ID_SHAPE = + 'lower-case letters and digits, separated by single hyphens or dots — for example my-model or acme.my-model'; + +let identifierPattern; + +/** + * Why a box id is unacceptable, or null when it is fine. + * + * The rule comes out of the schema rather than being restated here: two statements of one pattern + * are two things that can disagree, and the whole point of checking early is that the early answer + * matches the late one. `validateScroll` still has the last word — this only moves the *first* word + * to the prompt that produced the value, because the schema's own report arrives after every other + * question has been answered and says only that the value "does not match the required pattern". + * + * @param {unknown} value + * @returns {Promise} + */ +export async function boxIdProblem(value) { + identifierPattern ??= readFile(scrollSchemaUrl, 'utf8') + .then((text) => JSON.parse(text).$defs.identifier.pattern); + const source = await identifierPattern; + if (typeof value !== 'string' || value.trim() === '') return 'Box ID is required.'; + const trimmed = value.trim(); + if (new RegExp(source).test(trimmed)) return null; + return `${trimmed} is not a usable box ID. Use ${BOX_ID_SHAPE}.`; +} + /** * Creates one nested `/` scroll without overwriting any authored file. * @@ -343,9 +376,12 @@ export async function createScroll({ runtimeVersion, pixiVersion, compatibility = {}, - assetBaseUrl, + publishBaseUrl, executionKind, scriptSourcePath = null, + // A payload-relative path to an entry point the dependency solve already provides, instead of a + // project file to copy in. Mutually exclusive with the two above. + environmentPath = null, generateScript = false, generatedScriptSourcePath = null, scriptRelativePath = null, @@ -371,9 +407,15 @@ export async function createScroll({ ? null : requiredText(resolvedRuntimeVersion, 'runtimeVersion'), pixiVersion: requiredText(pixiVersion, 'pixiVersion'), - // Required whether or not any asset is deferred: the release manifest names the archive's own - // published URL, not just the assets'. - assetBaseUrl: requiredText(assetBaseUrl, 'assetBaseUrl'), + // Optional, exactly as the schema has it. A build does need one — the release manifest names the + // archive's own published URL, not just the assets' — but it may arrive later, from `edit scroll` + // or from `build --publish-base-url`, and a box that never gets one is simply local: its release + // and channel carry no links. Demanding it here instead made the one field nobody knows on day + // one block writing a scroll at all, and invited a placeholder URL into a document whose whole + // value is that it is true. + publishBaseUrl: publishBaseUrl === undefined || publishBaseUrl === null || String(publishBaseUrl).trim() === '' + ? null + : String(publishBaseUrl).trim(), }; if (!compatibility || typeof compatibility !== 'object' || Array.isArray(compatibility)) { fail('compatibility must be an object.'); @@ -414,7 +456,24 @@ export async function createScroll({ let generatedScriptPath = null; let generatedSource = null; const fileKind = FILE_KINDS[executionKind]; - if (fileKind) { + if (fileKind && environmentPath) { + // The entry point is already in the payload because a package put it there — conda-forge's + // `venv/bin/ffmpeg`, a console script the solve generated. Nothing is staged and no `localFiles` + // entry appears: there is no file of the author's to copy, and inventing one would claim the + // project ships something it does not. + // + // This case existed in the format from the start — every `native` example in this repository + // uses it — but not in the authoring surface, which assumed a file-naming execution always + // pointed at a project file. Writing one meant editing `scroll.json` by hand. + if (scriptSourcePath || generateScript) { + fail('Choose either a file from the environment or one from this project, not both.'); + } + execution = { + kind: executionKind, + [fileKind.field]: safeRelativePath(environmentPath), + defaultArgs: [...defaultArgs], + }; + } else if (fileKind) { if (generateScript && scriptSourcePath) { fail('Choose either an existing file or --generate-script, not both.'); } @@ -463,7 +522,7 @@ export async function createScroll({ module: requiredText(module, 'module'), defaultArgs: [...defaultArgs], }; - } else if (module || scriptSourcePath || generateScript || defaultArgs.length > 0) { + } else if (module || scriptSourcePath || environmentPath || generateScript || defaultArgs.length > 0) { fail('library-only execution cannot declare a script, module, or default arguments.'); } @@ -497,7 +556,7 @@ export async function createScroll({ ...(identity.runtimeVersion === null ? {} : { version: identity.runtimeVersion }), }, pixiVersion: identity.pixiVersion, - assetBaseUrl: identity.assetBaseUrl, + ...(identity.publishBaseUrl === null ? {} : { publishBaseUrl: identity.publishBaseUrl }), selfTest: { ...probe, ...(localFile ? { files: [localFile.relativePath] } : {}), @@ -579,7 +638,7 @@ export async function ensureExampleScroll({ sourceRevision: 'example-source-1.0.0', pixiVersion, compatibility: { minHostAppVersion: '1.0.0' }, - assetBaseUrl: 'https://example.org/boxes', + publishBaseUrl: 'https://example.org/boxes', executionKind: 'python-script', generateScript: true, }), diff --git a/src/build/box.mjs b/src/build/box.mjs index 88fdb7e..6a5eb4d 100644 --- a/src/build/box.mjs +++ b/src/build/box.mjs @@ -157,7 +157,7 @@ export async function buildBox(name, options = {}) { const { allowDirty = false, channel = 'beta', - assetBaseUrl: assetBaseUrlOverride = null, + publishBaseUrl: publishBaseUrlOverride = null, namespace, signerCommand = null, privatePath, @@ -181,6 +181,12 @@ export async function buildBox(name, options = {}) { // nothing to ask and nothing to override: a build-time override would silently repack a box under // an identity that no longer describes it. What the scroll decided is reported, not negotiated. const deferred = scroll.assets.filter((asset) => asset.embed === false); + // Where this box will be published, if it will be. Absent is a legitimate answer and the common + // one: a box built to run on the machine that built it is never published, so there is nowhere + // for its documents to point and no value here would be true. The build then omits the two links + // rather than inventing an address — nothing verifies them, and a false address inside a signed, + // immutable document stays false forever. + const publishBaseUrl = String(publishBaseUrlOverride || scroll.publishBaseUrl || '').replace(/\/$/, ''); log(`Runtime: ${scroll.runtime.id}${scroll.runtime.version ? ` ${scroll.runtime.version}` : ''}`); log(`Assets: ${scroll.assets.length - deferred.length} embedded, ${deferred.length} on demand`); // Wheels, native libraries, and the interpreter are proven on the exact OS/architecture they ship for. @@ -453,8 +459,6 @@ export async function buildBox(name, options = {}) { // Content-addressed: the object is named after its own hash, so publishing is idempotent and an // object can never be replaced with different bytes under the same URL. const archiveObject = `${objectPrefix}/${archiveSha}.zip`; - const assetBaseUrl = String(assetBaseUrlOverride || scroll.assetBaseUrl || '').replace(/\/$/, ''); - if (!assetBaseUrl) fail('No asset base URL: declare assetBaseUrl in the scroll or pass --asset-base-url.'); const kinds = documentKinds(namespace); const signing = { signerCommand, privatePath, publicPath }; log('Signing release and channel'); @@ -465,7 +469,12 @@ export async function buildBox(name, options = {}) { ...identity, target: scroll.target, compatibility: scroll.compatibility, - archive: { format: 'zip', url: `${assetBaseUrl}/${archiveObject}`, sha256: archiveSha, sizeBytes: archiveSize }, + archive: { + format: 'zip', + ...(publishBaseUrl ? { url: `${publishBaseUrl}/${archiveObject}` } : {}), + sha256: archiveSha, + sizeBytes: archiveSize, + }, installedSizeBytes, payloadDigest: payloadDigestValue, runtime: scroll.runtime, @@ -482,7 +491,9 @@ export async function buildBox(name, options = {}) { await writeFile(stagedReleasePath, `${JSON.stringify(await signDocument(release, signing), null, 2)}\n`); // The channel points at the release document by *its* hash too, so the whole chain is - // content-addressed: channel -> release document -> archive. + // content-addressed: channel -> release document -> archive. An unpublished box has no chain to + // build, and a channel that still says which version is current is more use than one carrying an + // address that resolves nowhere. const releaseDocumentSha = await sha256File(stagedReleasePath); const channelDocument = { schemaVersion: BOX_SCHEMA_VERSION, @@ -498,7 +509,9 @@ export async function buildBox(name, options = {}) { // rather than by the builder. releases: [{ version: scroll.version, - releaseManifestUrl: `${assetBaseUrl}/${objectPrefix}/${releaseDocumentSha}.release.json`, + ...(publishBaseUrl + ? { releaseManifestUrl: `${publishBaseUrl}/${objectPrefix}/${releaseDocumentSha}.release.json` } + : {}), rolloutPercentage: 100, }], }; @@ -520,11 +533,21 @@ export async function buildBox(name, options = {}) { log(`Release: ${publishedRelease}`); log(`Channel: ${channelPath}`); log(''); - log(`Publish: upload ${join(workspace.distDir, 'boxes')} under ${assetBaseUrl}, keeping its paths,`); - log(' then publish the channel document where your clients look for it.'); + if (publishBaseUrl) { + log(`Publish: upload ${join(workspace.distDir, 'boxes')} under ${publishBaseUrl}, keeping its paths,`); + log(' then publish the channel document where your clients look for it.'); + } else { + // Said plainly rather than left to be discovered from a missing field: this box is complete and + // runnable, and the one thing it cannot do is tell a downloader where to find itself. + log('Local: this box names no publish location, so its documents point nowhere.'); + log(` Run it from here, or rebuild with --publish-base-url to publish it.`); + } return { archivePath: publishedArchive, releasePath: publishedRelease, + // Whether this box has a publish location, so the CLI's closing line does not tell the author + // to distribute documents that point nowhere. + published: Boolean(publishBaseUrl), channelPath, archiveSha256: archiveSha, installedSizeBytes, diff --git a/src/build/project.mjs b/src/build/project.mjs index 16c0d1f..239fb6b 100644 --- a/src/build/project.mjs +++ b/src/build/project.mjs @@ -34,7 +34,8 @@ const PROJECT_GUIDE = `[Scrollcase documentation](https://scrollcase.dev/) # Scrollcase in this project Scrollcase turns a declarative [scroll](https://scrollcase.dev/reference/scroll) into a signed, -portable [box](https://scrollcase.dev/reference/box-format) for one [target](https://scrollcase.dev/reference/box-format#targets). +portable [box](https://scrollcase.dev/reference/box-format) for one [target](https://scrollcase.dev/reference/box-format#targets) and one +[runtime](https://scrollcase.dev/reference/scroll#choosing-a-runtime) — \`python\`, \`node\`, or \`native\`, which carries no interpreter. A workspace holds many boxes; \`new scroll\` asks per box. ## Usual workflow @@ -50,7 +51,7 @@ Run \`npm install scrollcase\` to install Scrollcase CLI. Then: See the [CLI reference](https://scrollcase.dev/reference/cli) and [signing guidance](https://scrollcase.dev/guides/signing-and-custody). The \`consumer-templates/\` -files demonstrate the [consumer APIs](https://scrollcase.dev/reference/api) against local releases. +files demonstrate the [consumer APIs](https://scrollcase.dev/reference/api) — your application's language, not your box's runtime. ## Node consumer @@ -130,6 +131,20 @@ export async function initProject({ }; } +/** + * The newest published pixi, or null when it cannot be found out. + * + * Advisory only, so every failure — offline, a rate limit, a changed feed — is swallowed rather than + * turned into an error. Nothing downstream depends on the answer. + */ +async function newestPixiVersion({ fetchImpl }) { + try { + return await latestPixiVersion({ fetchImpl }); + } catch { + return null; + } +} + /** Reads the project config back, so a toolchain pin is added to it rather than replacing it. */ async function readConfig(configPath) { if (!await fileExists(configPath)) return { version: 1, paths: { ...DEFAULT_WORKSPACE_PATHS } }; @@ -155,6 +170,9 @@ async function readConfig(configPath) { export async function ensureToolchain({ workspace, pixiVersion = null, + // Whether this run may ask the network which pixi is newest. Off by default: the caller decides, + // and a scaffold that reached out on its own would be doing something nobody asked for. + checkLatest = false, confirm, host = process, fetchImpl = fetch, @@ -175,6 +193,11 @@ export async function ensureToolchain({ installed: [], missing: [], pixiVersion: pixi.version, + // Only when the caller says the network is fair game. The lookup is one request to a + // rate-limited public API — it answers 403 on an unauthenticated CI runner — and it is + // advisory, so any failure means the line is simply not printed. `init` still installs + // nothing without consent; this only decides whether it can say a newer pixi exists. + newestPixiVersion: checkLatest ? await newestPixiVersion({ fetchImpl }) : null, condaPackVersion: CONDA_PACK_VERSION, declined: false, }; diff --git a/src/build/scroll-edit.mjs b/src/build/scroll-edit.mjs index e56f62d..b3ad2f3 100644 --- a/src/build/scroll-edit.mjs +++ b/src/build/scroll-edit.mjs @@ -278,7 +278,7 @@ export async function addAsset({ * @param {{ boxId: string, target: string, sourcePath: string, to?: string | null, * executable?: boolean }} options */ -export async function addFile({ boxId, target, sourcePath, to = null, executable = false }) { +export async function addFile({ boxId, target, sourcePath, to = null, executable = false, pin = false }) { const source = safeRelativePath(sourcePath); const absolute = join(getWorkspace().root, ...source.split('/')); let details; @@ -293,7 +293,16 @@ export async function addFile({ boxId, target, sourcePath, to = null, executable const relativePath = safeRelativePath(to ?? basename(source)); // The source file's own mode is deliberately not read: it varies with the umask of whoever // checked the project out, and a build that copied it would not rebuild byte-identically. - const entry = { sourcePath: source, relativePath, ...(executable ? { executable: true } : {}) }; + // Pinning is opt-in because most added files are about to be edited, and a hash recorded now would + // fail the next build. Reference data is the other case: a file the box answers from, where a + // changed byte should stop the build rather than ship a different answer under the same signature. + const sha256 = pin ? await sha256File(absolute) : null; + const entry = { + sourcePath: source, + relativePath, + ...(executable ? { executable: true } : {}), + ...(sha256 ? { sha256 } : {}), + }; const { written } = await updateScrollFiles(boxId, target, (scroll) => ({ ...scroll, localFiles: [...(scroll.localFiles ?? []), entry], @@ -451,6 +460,64 @@ export async function removeSelfTestImport({ boxId, target, module }) { return { written, module }; } +/** + * `add command` — records one invocation of the box's own execution as a self-test probe. + * + * The counterpart of `add import` for a runtime with no module system. A `native` box can only + * prove itself by running what it declares, so without this its probes could not be authored at + * all — the scroll had to be edited by hand, which is exactly what every other declaration here + * exists to avoid. + * + * Argument lists are compared as lists, so the same flags in a different order are a different + * probe. `new scroll` writes an empty one as a placeholder; adding a real probe replaces it, since + * "run it with no arguments" stops being a claim anyone made once a real one exists. + * + * @param {{ boxId: string, target: string, args: string[], expectExitCode?: number }} options + */ +export async function addSelfTestCommand({ boxId, target, args, expectExitCode = 0 }) { + if (!Array.isArray(args) || args.some((value) => typeof value !== 'string')) { + fail('A self-test command is a list of arguments.'); + } + if (!Number.isInteger(expectExitCode) || expectExitCode < 0 || expectExitCode > 255) { + fail('--expect-exit-code must be a whole number between 0 and 255.'); + } + const probe = { args: [...args], ...(expectExitCode === 0 ? {} : { expectExitCode }) }; + const same = (candidate) => + JSON.stringify(candidate.args ?? []) === JSON.stringify(probe.args) + && (candidate.expectExitCode ?? 0) === expectExitCode; + let added = 0; + const { written } = await updateScrollFiles(boxId, target, (scroll) => { + const commands = scroll.selfTest?.commands ?? []; + if (commands.some(same)) return null; + added += 1; + // The placeholder `new scroll` leaves behind: an empty argument list, asserting only that the + // box starts. A real probe supersedes it rather than sitting beside it. + const kept = commands.filter((candidate) => (candidate.args ?? []).length > 0); + return { ...scroll, selfTest: { ...scroll.selfTest, commands: [...kept, probe] } }; + }); + if (added === 0) fail(`${boxId} already runs that self-test command.`); + return { written, probe }; +} + +/** + * `remove command` — drops one self-test probe, matched on its exact argument list. + * + * @param {{ boxId: string, target: string, args: string[] }} options + */ +export async function removeSelfTestCommand({ boxId, target, args }) { + const wanted = JSON.stringify(args); + let removed = 0; + const { written } = await updateScrollFiles(boxId, target, (scroll) => { + const commands = scroll.selfTest?.commands ?? []; + const kept = commands.filter((candidate) => JSON.stringify(candidate.args ?? []) !== wanted); + if (kept.length === commands.length) return null; + removed += 1; + return { ...scroll, selfTest: { ...scroll.selfTest, commands: kept } }; + }); + if (removed === 0) fail(`${boxId} does not run that self-test command.`); + return { written, args }; +} + /** * Fields `edit scroll` refuses, and why each one is not an edit. * diff --git a/src/cli-authoring.mjs b/src/cli-authoring.mjs index c48e725..9c40fd5 100644 --- a/src/cli-authoring.mjs +++ b/src/cli-authoring.mjs @@ -15,13 +15,16 @@ import { createInterface } from 'node:readline/promises'; import { runtimeAdapters } from './contract/runtimes.mjs'; import { + BOX_ID_SHAPE, DEFAULT_RUNTIME_ID, EXAMPLE_PIXI_VERSION, authoredExecutionKinds, + boxIdProblem, resolveRuntimeVersion, } from './build/authoring.mjs'; import { probePixi } from './build/pixi.mjs'; import { fail } from './build/process.mjs'; +import { questionDocs } from './cli-docs.mjs'; import { chooseCliValue } from './cli-menu.mjs'; import { promptHeading, promptMarker } from './cli-output.mjs'; import { chooseTarget, cliTargetFamilies, parseCliTarget } from './cli-targets.mjs'; @@ -31,7 +34,7 @@ const MAX_PROMPT_ATTEMPTS = 5; /** * One line of prose per question, printed above it. * - * A field name is a label, not an explanation. `sourceRevision` and `assetBaseUrl` in particular + * A field name is a label, not an explanation. `sourceRevision` and `publishBaseUrl` in particular * mean nothing to someone meeting the tool for the first time, and a wrong answer to either is * recorded in a signed document. Kept to a single line each: a paragraph in front of every prompt * is skipped as reliably as no help at all. @@ -39,17 +42,62 @@ const MAX_PROMPT_ATTEMPTS = 5; const HINTS = Object.freeze({ target: 'The OS, architecture and accelerator this box is built for. One box, one target.', cudaVersion: 'The CUDA ABI to build against. It is part of the box identity, so 12.4 and 12.8 are different boxes.', - boxId: 'Name of the box across all its versions. Used in its directory, its archives and its channel pointer.', + boxId: `Name of the box across all its versions. Used in its directory, its archives and its channel pointer. ${BOX_ID_SHAPE}.`, sourceRevision: 'Which version of the thing you are packaging this is — a model commit, a release tag. Recorded verbatim in the box provenance.', - assetBaseUrl: 'Where you will publish built boxes. The signed release points at it; it does not have to exist yet.', + publishBaseUrl: 'Where you will publish built boxes, so the signed documents can point at each other. Optional — press Enter, and a box you only run locally never needs one.', runtime: 'What runs inside the box. python and node bring an interpreter; native runs a binary you compiled yourself.', - execution: 'What `scrollcase run` starts inside the box: a file, an importable module, or nothing at all.', - scriptSource: 'Point at a file you already have, or start from a generated stub.', + scriptSource: 'Start from a generated stub, or point at a file you already have.', + binarySource: 'A program a package installs into the box, or one you compiled and keep in this project.', + environmentPath: 'Path inside the box to the program the dependency solve installs, such as venv/bin/ffmpeg.', scriptPath: 'Path from the project root to the file the box should run.', binaryPath: 'Path from the project root to the compiled executable the box should run.', module: 'Dotted name of a module importable inside the box, run with python -m.', }); +/** + * What each execution kind means, in the words someone choosing between them needs. + * + * The hint above the menu is assembled from the kinds actually offered rather than written once for + * every runtime, because the offered set differs per runtime and a fixed sentence went stale the + * moment a second runtime existed: a `node` box was told it could pick "an importable module", which + * is not one of its options and never has been. + * + * Each entry says what the box is *for*, as a gerund phrase, in the menu's own order — so the items + * are parallel, the list has a visible end, and every option carries a reason to pick it. + * + * The framing is the part that took three tries. Asking what `scrollcase run` starts has no honest + * answer for `library-only` except "nothing", which describes an absence and reads as an option that + * does not do anything — so nobody would choose it, and the one thing it is actually for goes + * unsaid. Saying more inside that framing failed twice: hung off the last item it gave "…, or + * nothing at all, for a box other code imports rather than runs", where the list has no visible end + * and "a box other" sends the reader down the wrong parse; moved to the front it gave "Whether this + * box has an entry point `scrollcase run` can start", a relative clause with its "that" dropped and + * a code span inside it. `promptHeading` renders a hint as one lead-in line and appends the colon, + * so there is room for a list and nothing else. Asking what the box is for makes `library-only` an + * answer in its own right rather than the absence of one. + * + * "Imported" is exact, not loose: `authoredExecutionKinds` offers `library-only` only to a runtime + * with an import probe, so it never reaches `native`, which has no module system to be imported by. + */ +const EXECUTION_KIND_MEANINGS = Object.freeze({ + 'python-script': 'running a script file', + 'python-module': 'running an importable module', + 'node-script': 'running a script file', + 'native-binary': 'running the compiled binary you supply', + 'library-only': 'being imported by another application as a library', +}); + +/** The one-line explanation printed above the execution menu, for the kinds this runtime offers. */ +function executionHint(kinds) { + const meanings = kinds.map((kind) => EXECUTION_KIND_MEANINGS[kind] ?? kind); + const listed = meanings.length > 1 + ? `${meanings.slice(0, -1).join(', ')}, or ${meanings.at(-1)}` + : meanings[0]; + // An em dash, not a colon: the renderer adds a colon of its own at the end, and two in one line + // reads as two questions. + return `What this box is for — ${listed}`; +} + /** Host-constraint flags, and the `compatibility` field each one sets. */ const COMPATIBILITY_FLAGS = Object.freeze({ 'min-host-app-version': 'minHostAppVersion', @@ -76,6 +124,11 @@ export async function promptText(question, { defaultValue = null, hint = null, optional = false, + // Returns why an answer is unacceptable, or null when it is fine. Checking here rather than after + // the whole session means a malformed value is refused on the line that produced it, while the + // user still remembers what they typed and has lost nothing else. + validate = null, + docs = null, input = process.stdin, output = process.stdout, } = {}) { @@ -86,17 +139,25 @@ export async function promptText(question, { // The name and its explanation are printed once, above the loop: a field name alone rarely says // what the field is for, and repeating the whole explanation on every retry would bury the // answer the user is being asked for. A retry restates what is required and re-marks the line. - output.write(promptHeading(question, { hint, stream: output })); + output.write(promptHeading(question, { hint, docs, stream: output })); // Bounded, so an input stream that only ever yields blank lines ends in an error rather than a // loop nobody can interrupt. + let lastProblem = null; for (let attempt = 0; attempt < MAX_PROMPT_ATTEMPTS; attempt += 1) { const value = String(await readline.question(`${marker}${suffix}`)).trim(); - if (value) return value; - if (defaultValue !== null) return defaultValue; + const answer = value || defaultValue; + if (answer !== null && answer !== '') { + if (!validate) return answer; + const problem = await validate(answer); + if (!problem) return answer; + lastProblem = problem; + output.write(`${problem}\n`); + continue; + } if (optional) return null; output.write(`${question} is required.\n`); } - fail(`${question} is required.`); + fail(lastProblem ?? `${question} is required.`); } finally { readline.close(); } @@ -122,16 +183,29 @@ function parseLabels(value) { return parsed; } +/** + * The arguments a box always passes to its entry point, in either of the two forms people write. + * + * A JSON array for several — `'["-a", "-b"]'` — and the bare argument for one, because quoting a + * one-element JSON array to say `-hide_banner` is a tax on the common case. The two are told apart + * by the leading bracket rather than by trying JSON first and falling back: a malformed array would + * otherwise become a single literal argument that looks almost right, which is a worse failure than + * being told the array is malformed. + * + * The quotes around the array are the shell's requirement, not this tool's — `[...]` unquoted is a + * glob pattern and never reaches the process. + */ function parseDefaultArgs(value) { if (value === null) return []; + if (!value.trimStart().startsWith('[')) return [value]; let parsed; try { parsed = JSON.parse(value); } catch { - fail('--default-args must be a JSON array of strings.'); + fail('--default-args looks like a JSON array but is not valid JSON. Use \'["-a", "-b"]\', or pass a single argument unquoted.'); } if (!Array.isArray(parsed) || parsed.some((item) => typeof item !== 'string')) { - fail('--default-args must be a JSON array of strings.'); + fail('--default-args must be a JSON array of strings, or a single argument.'); } return parsed; } @@ -144,9 +218,10 @@ async function collectTarget(flags, { terminal, ask, chooseTargetValue }) { const selected = await chooseTargetValue(cliTargetFamilies(), { terminal: true, hint: HINTS.target, + docs: questionDocs('target'), }); if (selected.target.accelerator !== 'cuda') return parseCliTarget(selected.targetId); - const cudaVersion = await ask('CUDA version (major.minor)', { hint: HINTS.cudaVersion }); + const cudaVersion = await ask('CUDA version (major.minor)', { hint: HINTS.cudaVersion, docs: questionDocs('cudaVersion') }); return parseCliTarget(`${selected.targetId}${cudaVersion}`); } @@ -164,20 +239,20 @@ export async function collectNewScrollOptions(flags, { chooseTargetValue = chooseTarget, probe = probePixi, } = {}) { - const required = async (flag, question, hint) => { + const required = async (flag, question, hint, docsKey) => { const supplied = flagText(flags, flag); if (supplied !== null) return supplied; if (!terminal) fail(`new scroll requires --${flag} without a terminal.`); - return ask(question, { hint }); + return ask(question, { hint, docs: questionDocs(docsKey) }); }; /** A value with a defensible default: taken from the flag, otherwise settled without asking. */ const derived = (flag, defaultValue) => flagText(flags, flag) ?? defaultValue; - const finite = async (flag, question, choices, hint) => { + const finite = async (flag, question, choices, hint, docsKey) => { const supplied = flagText(flags, flag); if (!supplied && !terminal) { fail(`new scroll requires --${flag} <${choices.join('|')}> without a terminal.`); } - return choose(question, choices, { flag: supplied, hint, terminal }); + return choose(question, choices, { flag: supplied, hint, docs: questionDocs(docsKey), terminal }); }; const target = await collectTarget(flags, { terminal, ask, chooseTargetValue }); @@ -192,12 +267,25 @@ export async function collectNewScrollOptions(flags, { const runtimeId = await choose('runtime', runtimeIds, { flag: flagText(flags, 'runtime'), hint: HINTS.runtime, + docs: questionDocs('runtime'), terminal, }); - const boxId = await required('box-id', 'Box ID', HINTS.boxId); + // Checked against the schema's own pattern as it is typed. It used to be accepted here and refused + // by `validateScroll` at the very end — after the revision, the URL and the execution kind had all + // been answered — with a message that named neither the value nor the shape it needed. + const boxId = await (async () => { + const supplied = flagText(flags, 'box-id'); + if (supplied !== null) { + const problem = await boxIdProblem(supplied); + if (problem) fail(problem); + return supplied; + } + if (!terminal) fail('new scroll requires --box-id without a terminal.'); + return ask('Box ID', { hint: HINTS.boxId, validate: boxIdProblem, docs: questionDocs('boxId') }); + })(); // The upstream revision is the one identity nothing here can supply: it names the version of the // thing being packaged, and inventing it would put a false claim into the box's provenance. - const sourceRevision = await required('source-revision', 'Upstream revision', HINTS.sourceRevision); + const sourceRevision = await required('source-revision', 'Upstream revision', HINTS.sourceRevision, 'sourceRevision'); const version = derived('version', '1.0.0'); const scrollVersion = derived('scroll-version', undefined); const runtimeVersion = resolveRuntimeVersion(runtimeId, flagText(flags, 'runtime-version')); @@ -218,18 +306,23 @@ export async function collectNewScrollOptions(flags, { if (!Number.isFinite(minRamGb) || minRamGb <= 0) fail('--min-ram-gb must be a positive number.'); compatibility.minRamGb = minRamGb; } - // Not derivable and not optional: the signed release names the URL the archive itself is published - // under, so a build has nowhere to point without it. - const assetBaseUrl = await required('asset-base-url', 'Asset base URL', HINTS.assetBaseUrl); + // Optional, because most boxes never need one: a box built to run where it was built is never + // published, so there is nowhere for its documents to point. Skipping writes no `publishBaseUrl` + // rather than a placeholder — a made-up URL in a signed release is a false statement about where + // the box is published, and the build simply omits the links instead. + const publishBaseUrl = flagText(flags, 'publish-base-url') + ?? (terminal ? await ask('Publish base URL', { hint: HINTS.publishBaseUrl, optional: true, docs: questionDocs('publishBaseUrl') }) : null); // Free-form annotations, flags only and empty by default. Scrollcase reads none of them, so // prompting for one would be asking the author to fill in a field on the tool's behalf. const labels = parseLabels(flagText(flags, 'labels')); - const executionKind = await finite( - 'execution', - 'execution kind', - authoredExecutionKinds(runtimeId), - HINTS.execution, - ); + // A menu of one is not a question. `native` defines exactly one authored kind — it has no module + // system, so there is no `library-only` for it either — and being asked to pick from a single + // option reads as though something else was expected to be there. + const executionKinds = authoredExecutionKinds(runtimeId); + const suppliedKind = flagText(flags, 'execution'); + const executionKind = executionKinds.length === 1 && suppliedKind === null + ? executionKinds[0] + : await finite('execution', 'execution kind', executionKinds, executionHint(executionKinds), 'execution'); const defaultArgs = parseDefaultArgs(flagText(flags, 'default-args')); const result = { @@ -243,39 +336,61 @@ export async function collectNewScrollOptions(flags, { runtimeVersion, pixiVersion, compatibility, - assetBaseUrl, + publishBaseUrl, executionKind, defaultArgs, }; if (executionKind === 'python-module') { - result.module = await required('module', 'Python module', HINTS.module); + result.module = await required('module', 'Python module', HINTS.module, 'module'); } else if (executionKind !== 'library-only') { // Every remaining kind names a payload file. Whether Scrollcase can write a starter for it is // the runtime's answer: it generates source, and it does not generate compiled binaries. const generable = executionKind !== 'native-binary'; const noun = generable ? 'script' : 'binary'; const existing = flagText(flags, 'script'); + const fromEnvironment = flagText(flags, 'from-environment'); const generateScript = Boolean(flags.get('generate-script')); - if (existing && generateScript) { - fail('Choose either --script or --generate-script, not both.'); + if ([existing, fromEnvironment, generateScript || null].filter(Boolean).length > 1) { + fail('Choose one of --script , --from-environment or --generate-script.'); } if (generateScript && !generable) { fail(`Scrollcase cannot generate a ${noun}; point --script at the one you built.`); } - if (existing) result.scriptSourcePath = existing; + if (fromEnvironment) result.environmentPath = fromEnvironment; + else if (existing) result.scriptSourcePath = existing; else if (generateScript) result.generateScript = true; else if (!terminal) { - fail(`${executionKind} execution requires --script ${generable ? ' or --generate-script' : ''} without a terminal.`); + fail(`${executionKind} execution requires --from-environment or --script ${generable ? ' or --generate-script' : ''} without a terminal.`); } else if (!generable) { - result.scriptSourcePath = await ask('Binary path', { hint: HINTS.binaryPath }); + // A compiled binary has two origins and they are not the same question. Most `native` boxes + // package a program conda-forge already installs — `venv/bin/ffmpeg` — and nothing of the + // project's is copied in at all; the other case is a binary the project built itself. Only the + // second was ever askable, which meant the common one needed `scroll.json` edited by hand. + const origin = await choose( + 'binary source', + ['a program the environment provides', 'a compiled binary in this project'], + { hint: HINTS.binarySource, docs: questionDocs('binarySource'), terminal: true }, + ); + if (origin === 'a program the environment provides') { + result.environmentPath = await ask('Path inside the box', { + hint: HINTS.environmentPath, + docs: questionDocs('environmentPath'), + }); + } else { + result.scriptSourcePath = await ask('Binary path', { hint: HINTS.binaryPath, docs: questionDocs('binaryPath') }); + } } else { + // The generated stub first, because it is the preselected answer and the one that works with + // nothing else in place: a first scroll can be built and run immediately, and the stub is a + // file to edit rather than a file to go and find. Pointing at an existing script assumes the + // author already wrote one, which is the later case, not the first. const source = await choose( 'script source', - ['existing project script', 'generate starter script'], - { hint: HINTS.scriptSource, terminal: true }, + ['generate starter script', 'existing project script'], + { hint: HINTS.scriptSource, docs: questionDocs('scriptSource'), terminal: true }, ); if (source === 'existing project script') { - result.scriptSourcePath = await ask('Script path', { hint: HINTS.scriptPath }); + result.scriptSourcePath = await ask('Script path', { hint: HINTS.scriptPath, docs: questionDocs('scriptPath') }); } else result.generateScript = true; } const destination = flagText(flags, 'script-destination'); diff --git a/src/cli-docs.mjs b/src/cli-docs.mjs new file mode 100644 index 0000000..0ae6d94 --- /dev/null +++ b/src/cli-docs.mjs @@ -0,0 +1,76 @@ +/** + * Where each interactive question is explained in full, and the one place those URLs are written. + * + * A prompt has room for a single lead-in line, which is enough to say what a field is and never + * enough to say why it exists or what the alternatives cost. Rather than grow the prompts — + * a paragraph in front of every question is skipped as reliably as no help at all — each one points + * at the section that already answers it. The reader who knows the field ignores the line; the + * reader who does not gets the page instead of a search. + * + * Two rules keep this from becoming a list of dead links, which is worse than no links because a + * terminal cannot show a 404 the way a browser can: + * + * - **One base URL.** `docsUrl()` builds every link here and nothing composes one by hand. + * - **Every entry is asserted.** `cli-docs.test.mjs` resolves each path against `docs/` and each + * fragment against that page's own headings, so a renamed section fails the suite rather than + * printing a URL that goes nowhere. + */ + +/** The published documentation site. The `docs/` sources in this repository are what it serves. */ +export const DOCS_BASE_URL = 'https://scrollcase.dev'; + +/** + * A site-relative documentation path as an absolute URL. + * + * @param {string} path a route beginning with `/`, optionally with a `#fragment` + * @returns {string} + */ +export function docsUrl(path) { + if (!path.startsWith('/')) throw new TypeError(`Documentation path must start with /: ${path}`); + return `${DOCS_BASE_URL}${path}`; +} + +/** + * The section that explains each question the CLI asks, keyed by the question it belongs to. + * + * Only questions with somewhere to send the reader appear. A prompt whose answer is entirely local + * to the session — a path on this machine, a yes/no about this directory — has no section to name, + * and inventing one would send a reader to a page that does not discuss their question. + */ +export const DOCS_LINKS = Object.freeze({ + // `new scroll` + target: '/reference/scroll#target', + cudaVersion: '/reference/scroll#target', + boxId: '/reference/scroll#identity', + sourceRevision: '/reference/scroll#identity', + publishBaseUrl: '/reference/scroll#publishbaseurl', + runtime: '/reference/scroll#choosing-a-runtime', + execution: '/reference/scroll#why-declare-an-execution', + scriptSource: '/reference/scroll#execution-intent', + scriptPath: '/reference/scroll#execution-intent', + binaryPath: '/reference/scroll#execution-intent', + binarySource: '/reference/scroll#execution-intent', + environmentPath: '/reference/scroll#execution-intent', + module: '/reference/scroll#execution-intent', + + // `init` + example: '/getting-started/quickstart', + consumerTemplates: '/reference/api/', + consumerDependencies: '/reference/api/', + pythonConsumerSource: '/reference/api/', + toolchain: '/getting-started/installation', + + // `build` + channel: '/reference/box-format#channel-manifest', +}); + +/** + * The link for one question, absolute and ready to print, or null when it has no section. + * + * @param {keyof DOCS_LINKS | string} question + * @returns {string | null} + */ +export function questionDocs(question) { + const path = DOCS_LINKS[question]; + return path === undefined ? null : docsUrl(path); +} diff --git a/src/cli-init.mjs b/src/cli-init.mjs index b510fd6..e129004 100644 --- a/src/cli-init.mjs +++ b/src/cli-init.mjs @@ -96,3 +96,52 @@ export async function runInitDependencySetup({ rust, }; } + +/** + * What `init` says about the build toolchain, as lines the CLI edge renders. + * + * Four outcomes, and every one of them reports. The last used to be silent: `init` looks for pixi + * and conda-pack on every run and asks only when something is missing, so on a machine that already + * had both, the question a reader had been told to expect never appeared and nothing said why — + * silence there is indistinguishable from never having looked. + * + * Extracted from `cli.mjs` so the four branches can be asserted without a host that happens to have + * the tools installed, which is the reason the silent one went unnoticed. + * + * @param {object} toolchain the result of `ensureToolchain` + * @param {{ toolchainDir: string }} options + * @returns {Array<[('success'|'info'|'warning'), string]>} + */ +export function toolchainReportLines(toolchain, { toolchainDir }) { + if (toolchain.installed.length > 0) { + const lines = [ + ['success', `Installed ${toolchain.installed.join(' and ')} into ${toolchainDir}`], + ['info', 'Nothing was added to PATH; scrollcase finds them there on its own.'], + ]; + if (toolchain.configPath) { + lines.push(['success', `Recorded the toolchain pins in ${toolchain.configPath}`]); + } + return lines; + } + if (toolchain.unsupportedHost) { + return [['warning', `pixi publishes no build for ${toolchain.unsupportedHost}; install ${toolchain.missing.join(' and ')} manually.`]]; + } + if (toolchain.missing.length > 0) { + return [ + ['warning', `Skipped installing ${toolchain.missing.join(' and ')}.`], + ['info', 'Install them yourself, or re-run with --install-toolchain. `scrollcase doctor` reports what is missing.'], + ]; + } + if (toolchain.pixiVersion) { + const lines = [['info', `Found pixi ${toolchain.pixiVersion} and conda-pack; nothing to install.`]]; + // Worth saying because of what happens next rather than as general news: `new scroll` records + // the pixi it finds, and `build` refuses any other version for that scroll. Being behind here + // means every scroll written from now on pins the old resolver. + if (toolchain.newestPixiVersion && toolchain.newestPixiVersion !== toolchain.pixiVersion) { + lines.push(['warning', `pixi ${toolchain.newestPixiVersion} is the newest release.`]); + lines.push(['info', `A scroll created now would pin ${toolchain.pixiVersion}; pass --pixi-version to choose another.`]); + } + return lines; + } + return []; +} diff --git a/src/cli-menu.mjs b/src/cli-menu.mjs index 9841ae3..b732b99 100644 --- a/src/cli-menu.mjs +++ b/src/cli-menu.mjs @@ -19,6 +19,7 @@ import { promptHeading } from './cli-output.mjs'; */ export function selectCliMenu(question, choices, { hint = null, + docs = null, initialIndex = null, input = process.stdin, output = process.stdout, @@ -74,7 +75,7 @@ export function selectCliMenu(question, choices, { input.on('keypress', onKeypress); input.setRawMode(true); input.resume(); - output.write(promptHeading(`Which ${question}?`, { hint, stream: output })); + output.write(promptHeading(`Which ${question}?`, { hint, docs, stream: output })); output.write('\x1b[?25l'); render(); }); @@ -90,6 +91,7 @@ export function selectCliMenu(question, choices, { */ export function selectCliMultiMenu(question, choices, { hint = null, + docs = null, input = process.stdin, output = process.stdout, } = {}) { @@ -149,7 +151,7 @@ export function selectCliMultiMenu(question, choices, { input.on('keypress', onKeypress); input.setRawMode(true); input.resume(); - output.write(promptHeading(question, { hint, stream: output })); + output.write(promptHeading(question, { hint, docs, stream: output })); output.write('\x1b[?25l'); render(); }); @@ -163,6 +165,7 @@ export function selectCliMultiMenu(question, choices, { export async function chooseCliValue(question, choices, { flag = null, hint = null, + docs = null, open = false, terminal = Boolean(process.stdin.isTTY && process.stdout.isTTY), menu = selectCliMenu, @@ -179,7 +182,7 @@ export async function chooseCliValue(question, choices, { log(`scrollcase: no terminal to ask which ${question}; using ${fallback}.`); return fallback; } - const selectedIndex = await menu(question, choices, { hint, initialIndex: 0 }); + const selectedIndex = await menu(question, choices, { hint, docs, initialIndex: 0 }); if (!Number.isInteger(selectedIndex) || selectedIndex < 0 || selectedIndex >= choices.length) { fail(`${question} menu returned an invalid selection.`); } @@ -196,11 +199,12 @@ export async function chooseCliValue(question, choices, { */ export async function chooseCliValues(question, choices, { hint = null, + docs = null, terminal = Boolean(process.stdin.isTTY && process.stdout.isTTY), menu = selectCliMultiMenu, } = {}) { if (choices.length === 0 || !terminal) return []; - const selectedIndices = await menu(question, choices, { hint }); + const selectedIndices = await menu(question, choices, { hint, docs }); if (!Array.isArray(selectedIndices) || selectedIndices.some((index) => !Number.isInteger(index) || index < 0 diff --git a/src/cli-output.mjs b/src/cli-output.mjs index b671522..dec6a2a 100644 --- a/src/cli-output.mjs +++ b/src/cli-output.mjs @@ -71,11 +71,17 @@ export function statusLine(kind, message, { */ export function promptHeading(title, { hint = null, + docs = null, stream = process.stdout, env = process.env, } = {}) { const heading = paint(hint ? title : asLead(title), promptStyles.title, stream, env); - return hint ? `\n${heading}\n${asLead(hint)}\n` : `\n${heading}\n`; + const lines = hint ? `\n${heading}\n${asLead(hint)}\n` : `\n${heading}\n`; + // Its own line, below the lead-in, and muted. A URL inside the explanation would sit between the + // reader and the answer they are being asked for, and the lead-in ends in the colon the answer + // replies to — there is nowhere in it for a link to go. Bright black is the same weight as the + // answer marker: present for whoever wants it, invisible to whoever already knows the field. + return docs ? `${lines}${paint(` ${styles.step.symbol} ${docs}`, promptStyles.marker, stream, env)}\n` : lines; } /** The soft marker an answer is typed after, so the answer reads as the reply to the line above. */ @@ -97,9 +103,18 @@ export function commandTip(command, placeholder, { + `${paint(placeholder, tipStyles.placeholder, stream, env)}`; } -/** Builds the concise, relative distribution instruction printed after a successful build. */ -export function buildDistributionSummary({ archivePath, channelPath }, distDir) { +/** + * The concise, relative closing line printed after a successful build. + * + * Two wordings, because there are two outcomes. A box built with a publish location has documents + * that point at each other and is ready to upload. One built without has neither, and telling its + * author to "distribute" it would send them to publish files whose URLs are missing — the build + * already said so above, and this line must not contradict it. + */ +export function buildDistributionSummary({ archivePath, channelPath, published }, distDir) { const displayPath = (path) => relative(distDir, path).split(sep).join('/'); - return `Build complete — you can distribute the 2 files under ${displayPath(dirname(archivePath))}/ ` - + `and ${displayPath(channelPath)}`; + const files = `${displayPath(dirname(archivePath))}/ and ${displayPath(channelPath)}`; + return published + ? `Build complete — you can distribute the 2 files under ${files}` + : `Build complete — the box and its documents are under ${files}`; } diff --git a/src/cli-targets.mjs b/src/cli-targets.mjs index cedb6be..188c922 100644 --- a/src/cli-targets.mjs +++ b/src/cli-targets.mjs @@ -119,11 +119,12 @@ export async function chooseScroll(candidates, { /** Shows a raw-key target menu and resolves to the selected index. */ export function selectTargetMenu(targetIds, { hint = null, + docs = null, initialIndex = null, input = process.stdin, output = process.stdout, } = {}) { - return selectCliMenu('target', targetIds, { hint, initialIndex, input, output }); + return selectCliMenu('target', targetIds, { hint, docs, initialIndex, input, output }); } /** @@ -140,6 +141,7 @@ export function selectTargetMenu(targetIds, { export async function chooseTarget(candidates, { requested = null, hint = null, + docs = null, terminal = Boolean(process.stdin.isTTY && process.stdout.isTTY), host = { platform: process.platform, arch: process.arch }, menu = selectTargetMenu, @@ -186,6 +188,7 @@ export async function chooseTarget(candidates, { const selectedIndex = await menu(choices.map(({ targetId }) => targetId), { hint, + docs, initialIndex: fallback ? choices.indexOf(fallback) : null, }); if (!Number.isInteger(selectedIndex) || selectedIndex < 0 || selectedIndex >= choices.length) { diff --git a/src/cli.mjs b/src/cli.mjs index 6b397cd..b2dfede 100755 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -43,12 +43,14 @@ import { ALL_TARGETS, addAsset, addFile, + addSelfTestCommand, addSelfTestImport, editableScrollFields, readScrollFamily, refreshScroll, removeEnvironmentVariable, removeScrollEntry, + removeSelfTestCommand, removeSelfTestImport, setEnvironmentVariable, setScrollField, @@ -69,7 +71,9 @@ import { resolvePythonConsumerSource, resolveTemplatesChoice, runInitDependencySetup, + toolchainReportLines, } from './cli-init.mjs'; +import { DOCS_BASE_URL, questionDocs } from './cli-docs.mjs'; import { chooseCliValue, chooseCliValues } from './cli-menu.mjs'; import { buildDistributionSummary, @@ -152,11 +156,11 @@ async function lock(name, flags) { * explanation: consent questions are laid out the same way as the rest, so a person reading a * session sees one shape throughout. */ -async function confirm(question, hint = null) { +async function confirm(question, hint = null, docs = null) { if (!process.stdin.isTTY || !process.stdout.isTTY) return false; const readline = createInterface({ input: process.stdin, output: process.stdout }); try { - process.stdout.write(promptHeading(question, { hint })); + process.stdout.write(promptHeading(question, { hint, docs })); return defaultYesConfirmation(await readline.question(`${promptMarker()}[Y/n] `)); } finally { readline.close(); @@ -200,6 +204,7 @@ async function init(flags) { confirmExample: () => confirm( 'Include the runnable example?', 'A disposable example-box scroll for trying the whole workflow once.', + questionDocs('example'), ), }); const wantsTemplates = await resolveTemplatesChoice({ @@ -208,6 +213,7 @@ async function init(flags) { confirmTemplates: () => confirm( 'Include the consumer templates?', 'Working Node, Python and Rust starting points for the application that runs your boxes.', + questionDocs('consumerTemplates'), ), }); const exampleTarget = wantsExample ? nativeExampleTarget() : null; @@ -265,6 +271,7 @@ async function init(flags) { { hint: `What consumer-templates/ needs to run, installed in ${workspace.root}. ` + 'Selecting none is a valid answer.', + docs: questionDocs('consumerDependencies'), }, ); return offered.filter((language) => chosen.includes(labels[language])); @@ -273,6 +280,7 @@ async function init(flags) { const selectedSource = await chooseCliValue( 'Python consumer package source', ['PyPI with pip', 'conda-forge with conda'], + { docs: questionDocs('pythonConsumerSource') }, ); const source = selectedSource.startsWith('PyPI') ? 'pypi' : 'conda-forge'; return resolvePythonConsumerSource({ @@ -281,18 +289,23 @@ async function init(flags) { confirmPyPIFallback: () => confirm( 'Install scrollcase-consumer from PyPI with pip instead?', 'Conda is not installed, so the conda-forge package cannot be installed here.', + questionDocs('pythonConsumerSource'), ), }); }, installToolchain: () => ensureToolchain({ workspace, pixiVersion, + // Only when a person is watching and has not said to leave the toolchain alone. The lookup is + // one request to a rate-limited public API, and nothing depends on its answer. + checkLatest: interactive && !never, confirm: async (missing) => { if (never) return false; if (always) return true; return confirm( `Install ${missing.join(' and ')} into ${workspace.toolchainDir}?`, `This project needs ${missing.length > 1 ? 'them' : 'it'} to build a box.`, + questionDocs('toolchain'), ); }, }), @@ -305,15 +318,9 @@ async function init(flags) { }); const { toolchain } = setup; - if (toolchain.installed.length > 0) { - success(`Installed ${toolchain.installed.join(' and ')} into ${workspace.toolchainDir}`); - info('Nothing was added to PATH; scrollcase finds them there on its own.'); - if (toolchain.configPath) success(`Recorded the toolchain pins in ${toolchain.configPath}`); - } else if (toolchain.unsupportedHost) { - warning(`pixi publishes no build for ${toolchain.unsupportedHost}; install ${toolchain.missing.join(' and ')} manually.`); - } else if (toolchain.missing.length > 0) { - warning(`Skipped installing ${toolchain.missing.join(' and ')}.`); - info('Install them yourself, or re-run with --install-toolchain. `scrollcase doctor` reports what is missing.'); + const report = { success, info, warning }; + for (const [level, message] of toolchainReportLines(toolchain, { toolchainDir: workspace.toolchainDir })) { + report[level](message); } if (setup.typescript) { @@ -369,8 +376,9 @@ const reportWritten = (written) => { }; /** `add asset|file|dep` — record something in a scroll, or in a box's pixi manifests. */ -async function add(kind, positional, flags) { +async function add(kind, positional, flags, passthrough = []) { const [name, value] = positional; + if (kind === 'command') return addCommand(name, passthrough, flags); if (kind === 'dep') return addDep(name, value, flags); if (kind === 'env' || kind === 'import') return addDeclaration(kind, name, value, flags); if (!value) fail(`Usage: scrollcase add ${kind} <${kind === 'asset' ? 'url' : 'path'}> [--to ] [--on-demand] [--executable] [--target |all]`); @@ -387,12 +395,33 @@ async function add(kind, positional, flags) { executable, log: (message) => step(message), }) - : await addFile({ boxId, target, sourcePath: value, to, executable }); + : await addFile({ boxId, target, sourcePath: value, to, executable, pin: Boolean(flags.get('pin')) }); success(`Added ${result.entry.relativePath} to ${boxId}${target === ALL_TARGETS ? '' : `/${target}`}`); - if (kind === 'asset') info(`${result.entry.sizeBytes} bytes, sha256 ${result.entry.sha256}`); + if (result.entry.sha256) info(`sha256 ${result.entry.sha256}${kind === 'asset' ? '' : ' (pinned)'}`); + if (kind === 'asset') info(`${result.entry.sizeBytes} bytes`); reportWritten(result.written); } +/** + * `add command -- ` — one invocation of the box's own execution, as a self-test probe. + * + * The arguments come after `--` rather than as a quoted list, because they *are* a command line and + * the parser already preserves everything after that boundary byte for byte. It is also the only + * shape that survives flags of its own: `-version` would otherwise be read as Scrollcase's. + */ +async function addCommand(name, args, flags) { + if (args.length === 0) { + fail('Usage: scrollcase add command [--expect-exit-code ] -- '); + } + const requested = text(flags, 'expect-exit-code'); + const expectExitCode = requested === null ? 0 : Number(requested); + const { boxId, target } = await editScope(name, flags); + const result = await addSelfTestCommand({ boxId, target, args, expectExitCode }); + const status = expectExitCode === 0 ? '' : `, expecting exit ${expectExitCode}`; + success(`Added the self-test command \`${result.probe.args.join(' ')}\`${status}`); + return reportWritten(result.written); +} + /** `add env NAME=VALUE` and `add import ` — the two declarations that are not a file. */ async function addDeclaration(kind, name, value, flags) { if (!value) { @@ -458,8 +487,18 @@ async function addDep(name, dependency, flags) { } /** `remove asset|file` — the inverse of `add`, so leaving is as easy as arriving. */ -async function remove(kind, positional, flags) { - const [name, value] = positional; +async function remove(kind, positional, flags, passthrough = []) { + const [name] = positional; + if (kind === 'command') { + if (passthrough.length === 0) { + fail('Usage: scrollcase remove command -- [--target |all]'); + } + const { boxId, target } = await editScope(name, flags); + const result = await removeSelfTestCommand({ boxId, target, args: passthrough }); + success(`Removed the self-test command \`${result.args.join(' ')}\` from ${boxId}`); + return reportWritten(result.written); + } + const [, value] = positional; if (!value) { const argument = { env: 'NAME', import: 'module' }[kind] ?? 'payload path'; fail(`Usage: scrollcase remove ${kind} <${argument}> [--target |all]`); @@ -565,7 +604,7 @@ async function build(name, flags) { const channel = await chooseCliValue( 'channel', ['beta', ...CHANNELS.filter((value) => value !== 'beta')], - { flag: text(flags, 'channel') }, + { flag: text(flags, 'channel'), docs: questionDocs('channel') }, ); // Whether an asset ships inside the archive is a per-entry scroll declaration with no build-time // override. There was one, and a menu preselected on `embed` in front of every build turned out to @@ -576,7 +615,7 @@ async function build(name, flags) { ...signing, allowDirty: Boolean(flags.get('allow-dirty')), channel, - assetBaseUrl: text(flags, 'asset-base-url'), + publishBaseUrl: text(flags, 'publish-base-url'), namespace: text(flags, 'namespace') || undefined, pixiPath: text(flags, 'pixi'), condaPackPath: text(flags, 'conda-pack'), @@ -653,7 +692,7 @@ function usage() { Commands: init Initialize a workspace with a runnable example new scroll Create one guided target-specific scroll - add asset|file|dep|env|import + add asset|file|dep|env|import|command Record a remote file with the size and hash it has, a file from this project, a dependency in the box's pixi manifests, an environment variable (NAME=VALUE), or a self-test import @@ -693,7 +732,8 @@ New scroll options: --runtime python, node or native (default python) --box-id Box identity --source-revision Upstream source revision recorded in provenance - --asset-base-url Base URL used in built release documents + --publish-base-url Where built boxes will be published, so the signed documents can + point at each other. Omit it for a box you only run locally --labels JSON object of free-form annotations carried into the signed release. Scrollcase reads none of them. --version Box version (default 1.0.0) @@ -704,12 +744,15 @@ New scroll options: --min-host-app-version Minimum compatible host application version --execution The runtime's own kinds, plus library-only where the box can still prove something without an entry point - --script Existing project file the box runs + --from-environment Payload path to an entry point a package already installs into the + box, such as venv/bin/ffmpeg. Nothing of the project is copied in + --script Existing project file the box runs, copied into the box --generate-script Generate a minimal starter instead, where the runtime has one --script-destination Payload path for that file (default: the runtime's own name) --generated-script-path Project path for a generated starter --module Dotted module name for python-module - --default-args JSON array of default application arguments + --default-args Arguments the box always passes to its entry point. One argument as + itself, several as a JSON array: --default-args '["-a", "-b"]' --max-host-app-version-exclusive --min-macos-version --min-ram-gb @@ -726,6 +769,10 @@ Add, remove and edit options: the root. --on-demand For add asset: leave this file out of the archive and carry its descriptor in the signed release for the caller to materialize. + --expect-exit-code For add command: the status the probe must exit with (default 0) + --pin For add file: record the file's SHA-256, so a changed byte fails the + build. For reference data the box answers from, not for a file you are + about to edit --executable Mark the added file as one that needs the executable bit. A download and a copy both arrive without one. --version Version constraint for add dep (default *, letting the lock pin it) @@ -757,7 +804,8 @@ Build options: --target Select a target when names a box --channel Channel the signed pointer names (nightly, beta, or stable; default beta) - --asset-base-url Override the scroll's published base URL + --publish-base-url Override the scroll's publishBaseUrl. Without either, the release + and channel carry no links and the box is local-only --namespace Document kind namespace (default scrollcase.box) --allow-dirty Permit a build from an uncommitted source tree --pixi Use this pixi executable @@ -806,6 +854,10 @@ Workspace: --out-dir Built artefacts (default .scrollcase/dist) --keys-dir Local signing keys (default .scrollcase/keys) --toolchain-dir Project-local pixi/conda-pack (default .scrollcase/toolchain) + +Documentation: + ${DOCS_BASE_URL} + Every interactive question also prints the section that explains it in full. `); } @@ -828,17 +880,17 @@ async function main() { } if (command === 'add') { const [kind, ...rest2] = positional; - if (!['asset', 'file', 'dep', 'env', 'import'].includes(kind)) { - fail('Usage: scrollcase add asset|file|dep|env|import [options]'); + if (!['asset', 'file', 'dep', 'env', 'import', 'command'].includes(kind)) { + fail('Usage: scrollcase add asset|file|dep|env|import|command [options]'); } - return add(kind, rest2, flags); + return add(kind, rest2, flags, passthrough); } if (command === 'remove') { const [kind, ...rest2] = positional; - if (!['asset', 'file', 'env', 'import'].includes(kind)) { - fail('Usage: scrollcase remove asset|file|env|import [options]'); + if (!['asset', 'file', 'env', 'import', 'command'].includes(kind)) { + fail('Usage: scrollcase remove asset|file|env|import|command [options]'); } - return remove(kind, rest2, flags); + return remove(kind, rest2, flags, passthrough); } if (command === 'edit') return editScroll(positional, flags); if (command === 'refresh') return refresh(positional[0], flags); diff --git a/src/contract/fixtures/consumer-conformance.json b/src/contract/fixtures/consumer-conformance.json index 87fc105..9aeaa4c 100644 --- a/src/contract/fixtures/consumer-conformance.json +++ b/src/contract/fixtures/consumer-conformance.json @@ -59,6 +59,23 @@ } } }, + { + "id": "release-with-no-published-location", + "action": "prepare", + "mutation": "strip-release-archive-url", + "expected": { + "outcome": "prepared", + "receipt": { + "status": "prepared", + "boxId": "consumer-fixture", + "executionKind": "python-script", + "requiredAssetCount": 0, + "runtimeId": "python", + "entryPoint": "$NATIVE_ENTRY_POINT", + "targetId": "$NATIVE_TARGET" + } + } + }, { "id": "trusted-single-key-in-memory", "action": "prepare", diff --git a/src/contract/fixtures/examples/scroll-pixi.example.json b/src/contract/fixtures/examples/scroll-pixi.example.json index b098a4b..59aabf0 100644 --- a/src/contract/fixtures/examples/scroll-pixi.example.json +++ b/src/contract/fixtures/examples/scroll-pixi.example.json @@ -28,7 +28,7 @@ }, "condaDependencyLicenseAudit": "legal/audits/example-model-linux-x86_64-cuda12.9.json", "cacheSubdir": "cache/example-model", - "assetBaseUrl": "https://assets.example.org/boxes", + "publishBaseUrl": "https://assets.example.org/boxes", "assets": [ { "url": "https://assets.example.org/example-model/weights.safetensors", diff --git a/src/contract/fixtures/examples/scroll.example.json b/src/contract/fixtures/examples/scroll.example.json index e62d4ff..a1be966 100644 --- a/src/contract/fixtures/examples/scroll.example.json +++ b/src/contract/fixtures/examples/scroll.example.json @@ -26,7 +26,7 @@ "entryPoint": "venv/bin/python" }, "cacheSubdir": "cache/example-model", - "assetBaseUrl": "https://assets.example.org/boxes", + "publishBaseUrl": "https://assets.example.org/boxes", "assets": [ { "url": "https://assets.example.org/example-model/weights.safetensors", diff --git a/src/contract/schema/channel-manifest.schema.json b/src/contract/schema/channel-manifest.schema.json index 0e04054..7050fac 100644 --- a/src/contract/schema/channel-manifest.schema.json +++ b/src/contract/schema/channel-manifest.schema.json @@ -56,7 +56,6 @@ "additionalProperties": false, "required": [ "version", - "releaseManifestUrl", "rolloutPercentage" ], "properties": { @@ -66,7 +65,8 @@ }, "releaseManifestUrl": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "Where the signed release document is published. Absent when the box was built without a publish base URL: there is then nothing for this pointer to point at, and a channel that still says which version is current is more use than one carrying an address that does not resolve." }, "rolloutPercentage": { "type": "integer", diff --git a/src/contract/schema/release-manifest.schema.json b/src/contract/schema/release-manifest.schema.json index 24df66c..7c4abae 100644 --- a/src/contract/schema/release-manifest.schema.json +++ b/src/contract/schema/release-manifest.schema.json @@ -84,7 +84,6 @@ "additionalProperties": false, "required": [ "format", - "url", "sha256", "sizeBytes" ], @@ -94,7 +93,8 @@ }, "url": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "Where the archive is published, for the distribution layer that has to fetch it. Absent when the box was built without a publish base URL, which is what a box built to run locally is.\n\nNothing verifies this value and no Scrollcase consumer reads one: an archive is resolved beside its release document and identified by sha256, so a wrong URL here would break a download and no check at all. That is why an absent one is preferred to an invented one \u2014 a false address inside a signed, immutable document stays false forever." }, "sha256": { "$ref": "#/$defs/sha256" diff --git a/src/contract/schema/scroll.schema.json b/src/contract/schema/scroll.schema.json index 74a6464..819ef0e 100644 --- a/src/contract/schema/scroll.schema.json +++ b/src/contract/schema/scroll.schema.json @@ -127,10 +127,13 @@ "pattern": "^[^\\u0000]*$" } }, - "assetBaseUrl": { + "publishBaseUrl": { "type": "string", "minLength": 1, - "description": "Base URL of the mirror the built archive and its objects are published under." + "description": "Base URL the built archive and its signed documents will be published under, so each can point at the next: the channel names the release document, and the release names the archive. It says nothing about the box's own assets \u2014 those carry a URL each \u2014 and nothing about what the box does at run time.\n\nOptional, and genuinely so. A box you build to run locally is never published, so there is nowhere for these documents to point and no value here would be true; the build omits both links rather than inventing an address. Nothing verifies this URL and no Scrollcase consumer reads one: an archive is found beside its release document and identified by its SHA-256.", + "examples": [ + "https://boxes.example.org" + ] }, "assets": { "type": "array", diff --git a/src/contract/types/index.d.ts b/src/contract/types/index.d.ts index 03b8e79..bb36493 100644 --- a/src/contract/types/index.d.ts +++ b/src/contract/types/index.d.ts @@ -176,9 +176,11 @@ export interface BoxScroll { [k: string]: string; }; /** - * Base URL of the mirror the built archive and its objects are published under. + * Base URL the built archive and its signed documents will be published under, so each can point at the next: the channel names the release document, and the release names the archive. It says nothing about the box's own assets — those carry a URL each — and nothing about what the box does at run time. + * + * Optional, and genuinely so. A box you build to run locally is never published, so there is nowhere for these documents to point and no value here would be true; the build omits both links rather than inventing an address. Nothing verifies this URL and no Scrollcase consumer reads one: an archive is found beside its release document and identified by its SHA-256. */ - assetBaseUrl?: string; + publishBaseUrl?: string; /** * Files fetched during the build. Every entry is size- and hash-checked before use, so a moved or replaced upstream file fails the build instead of silently changing the box. May be empty, and defaults to empty. */ @@ -479,7 +481,12 @@ export interface BoxReleaseManifest { }; archive: { format: 'zip'; - url: string; + /** + * Where the archive is published, for the distribution layer that has to fetch it. Absent when the box was built without a publish base URL, which is what a box built to run locally is. + * + * Nothing verifies this value and no Scrollcase consumer reads one: an archive is resolved beside its release document and identified by sha256, so a wrong URL here would break a download and no check at all. That is why an absent one is preferred to an invented one — a false address inside a signed, immutable document stays false forever. + */ + url?: string; sha256: Sha256; sizeBytes: number; }; @@ -536,12 +543,18 @@ export interface BoxChannelManifest { releases: [ { version: string; - releaseManifestUrl: string; + /** + * Where the signed release document is published. Absent when the box was built without a publish base URL: there is then nothing for this pointer to point at, and a channel that still says which version is current is more use than one carrying an address that does not resolve. + */ + releaseManifestUrl?: string; rolloutPercentage: number; }, ...{ version: string; - releaseManifestUrl: string; + /** + * Where the signed release document is published. Absent when the box was built without a publish base URL: there is then nothing for this pointer to point at, and a channel that still says which version is current is more use than one carrying an address that does not resolve. + */ + releaseManifestUrl?: string; rolloutPercentage: number; }[] ]; diff --git a/tests/helpers/consumer-conformance.mjs b/tests/helpers/consumer-conformance.mjs index 8c853e3..9bb4a98 100644 --- a/tests/helpers/consumer-conformance.mjs +++ b/tests/helpers/consumer-conformance.mjs @@ -205,6 +205,16 @@ async function mutateFixture(fixture, mutation, destination) { await writeSignedRelease(fixture, fixture.release); return; } + if (mutation === 'strip-release-archive-url') { + // A box built without a publish base URL: it was never published, so its release names no + // address for the archive. Every consumer must prepare it exactly as it prepares any other, because + // the URL was never part of the trust chain — the archive is found beside the release document and + // identified by its sha256. + const { url: _unpublished, ...archive } = fixture.release.archive; + fixture.release.archive = archive; + await writeSignedRelease(fixture, fixture.release); + return; + } if (mutation === 'alter-release-bundled-licenses') { // A licence inventory added to the signed release after the box was built. It is signed, so the // signature still verifies; what refuses it is that box.json says something else, which is the diff --git a/tests/unit/build-pipeline.test.mjs b/tests/unit/build-pipeline.test.mjs index 0ee1c96..e44849e 100644 --- a/tests/unit/build-pipeline.test.mjs +++ b/tests/unit/build-pipeline.test.mjs @@ -47,7 +47,7 @@ const SCROLL = { runtime: { id: RUNTIME_ID, version: PYTHON_VERSION, entryPoint: HOST_LAYOUT.entryPoint }, pixiVersion: '0.73.0', cacheSubdir: 'cache/example-model', - assetBaseUrl: 'https://assets.example.org/boxes', + publishBaseUrl: 'https://assets.example.org/boxes', assets: [], selfTest: { imports: ['json'], files: [] }, }; @@ -580,6 +580,57 @@ describe('the build pipeline', () => { .rejects.toThrow(/native-binary does not belong to the python runtime/); }); + it('builds a box that names no publish location, and signs no link to nowhere', async () => { + // The common case, not an edge one: a box built to run where it was built is never published, + // so there is nowhere for its documents to point. Every guarantee still holds — the archive is + // hashed, the documents are signed, `verify` passes — because none of them was ever the URL. + const { keys, payloadDir } = await makeProject({ ...SCROLL, publishBaseUrl: undefined }); + const built = await buildBox(SCROLL_REF, { + ...keys, + ...fakeToolchain(payloadDir), + log: () => {}, + }); + + const release = decodeDocumentPayload(JSON.parse(await readFile(built.releasePath, 'utf8'))); + // Absent, not empty and not a placeholder: a false address inside a signed, immutable document + // stays false forever, and nothing here would have caught it. + expect(release.archive.url).toBeUndefined(); + expect(release.archive.sha256).toMatch(/^[a-f0-9]{64}$/); + expect(release.archive.sizeBytes).toBeGreaterThan(0); + + const channel = decodeDocumentPayload(JSON.parse(await readFile(built.channelPath, 'utf8'))); + expect(channel.releases[0].releaseManifestUrl).toBeUndefined(); + // The channel still says which version is current, which is the half that does not need a URL. + expect(channel.releases[0]).toMatchObject({ version: SCROLL.version, rolloutPercentage: 100 }); + + await expect(verifyBox(built.releasePath, { publicPath: keys.publicPath, log: () => {} })) + .resolves.toMatchObject({ status: 'passed' }); + }); + + it('tells the author a local box points nowhere, rather than leaving it to be discovered', async () => { + const { keys, payloadDir } = await makeProject({ ...SCROLL, publishBaseUrl: undefined }); + const lines = []; + await buildBox(SCROLL_REF, { + ...keys, + ...fakeToolchain(payloadDir), + log: (message) => lines.push(String(message)), + }); + expect(lines.join('\n')).toMatch(/names no publish location/); + expect(lines.join('\n')).toMatch(/--publish-base-url/); + }); + + it('accepts the same scroll once the URL arrives from the build flag', async () => { + const { keys, payloadDir } = await makeProject({ ...SCROLL, publishBaseUrl: undefined }); + const built = await buildBox(SCROLL_REF, { + ...keys, + ...fakeToolchain(payloadDir), + publishBaseUrl: 'https://assets.example.org/boxes', + log: () => {}, + }); + const release = decodeDocumentPayload(JSON.parse(await readFile(built.releasePath, 'utf8'))); + expect(release.archive.url).toMatch(/^https:\/\/assets\.example\.org\/boxes\//); + }); + it('refuses a command probe with no execution to invoke', async () => { await makeProject({ ...SCROLL, @@ -1060,7 +1111,7 @@ describe('the build pipeline', () => { // stands puts every object exactly where its own URL already says it is. const release = decodeDocumentPayload(JSON.parse(await readFile(built.releasePath, 'utf8'))); const objectKey = relative(dist, built.archivePath).split(sep).join('/'); - expect(release.archive.url).toBe(`${SCROLL.assetBaseUrl}/${objectKey}`); + expect(release.archive.url).toBe(`${SCROLL.publishBaseUrl}/${objectKey}`); // And a release verifies where it lands, without being told where its archive is. const receipt = await verifyBox(built.releasePath, { publicPath: keys.publicPath, log: () => {} }); diff --git a/tests/unit/cli-docs.test.mjs b/tests/unit/cli-docs.test.mjs new file mode 100644 index 0000000..2ac1343 --- /dev/null +++ b/tests/unit/cli-docs.test.mjs @@ -0,0 +1,90 @@ +/** + * Every documentation link the CLI can print, resolved against the pages in this repository. + * + * A dead link in a browser shows a 404; a dead link in a terminal shows nothing at all, because the + * person who followed it is somewhere else by the time it fails. Nothing else in the suite would + * notice a renamed heading — VitePress checks the links inside `docs/`, not the ones the CLI holds — + * so this is the only thing standing between a section rename and a prompt that sends people + * nowhere. + */ + +import { execFileSync } from 'node:child_process'; +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { DOCS_BASE_URL, DOCS_LINKS, docsUrl, questionDocs } from '../../src/cli-docs.mjs'; + +const root = fileURLToPath(new URL('../..', import.meta.url)); +const docsDir = join(root, 'docs'); + +/** + * VitePress's own slug rule, narrowed to what these headings use: lower-case, punctuation dropped, + * spaces to hyphens. Kept beside the assertion rather than imported, because the point is to agree + * with the published anchor and not with our own helper. + */ +const slug = (heading) => heading + .toLowerCase() + .replace(/`/g, '') + .replace(/[^\w\- ]+/g, '') + .trim() + .replace(/\s+/g, '-'); + +/** + * The Markdown behind a route, resolved the way VitePress resolves one. + * + * A route is either a page or a section: `/reference/scroll` is `reference/scroll.md`, while + * `/reference/api` is `reference/api/index.md`. Trying only the first spelling made this guard + * reject a link that works — which it did, the first time it ran, because the API reference had + * already become a directory. + */ +async function pageSource(page) { + const candidates = [join(docsDir, `${page}.md`), join(docsDir, page, 'index.md')]; + const settled = await Promise.allSettled(candidates.map((path) => readFile(path, 'utf8'))); + const found = settled.find(({ status }) => status === 'fulfilled'); + if (!found) throw new Error(`No page backs /${page}; tried ${candidates.join(' and ')}`); + return found.value; +} + +async function headingsOf(page) { + // Fenced blocks first: a `#` inside a shell sample is a comment, not a heading. + const prose = (await pageSource(page)).replace(/^```[\s\S]*?^```$/gm, ''); + return new Set([...prose.matchAll(/^#{1,6}\s+(.+?)\s*$/gm)].map((match) => slug(match[1]))); +} + +describe('the documentation links the CLI prints', () => { + it('names a real page, and a real heading on it, for every question', async () => { + // Not vacuous: an empty map, or one whose entries stopped being reachable from the prompts, + // would assert nothing at all. + expect(Object.keys(DOCS_LINKS).length).toBeGreaterThan(10); + + for (const [question, path] of Object.entries(DOCS_LINKS)) { + const [route, fragment] = path.split('#'); + const page = route.replace(/^\//, '').replace(/\/$/, ''); + const label = `${question} → ${path}`; + + await expect(pageSource(page), label).resolves.toBeTruthy(); + if (fragment) { + expect([...await headingsOf(page)], label).toContain(fragment); + } + } + }); + + it('builds every link from one base URL', () => { + for (const question of Object.keys(DOCS_LINKS)) { + expect(questionDocs(question)).toBe(`${DOCS_BASE_URL}${DOCS_LINKS[question]}`); + } + // A question with no section returns nothing rather than the bare site, which would send a + // reader to a home page that does not discuss what they were asked. + expect(questionDocs('no-such-question')).toBeNull(); + expect(() => docsUrl('reference/cli')).toThrow(/must start with/); + }); + + it('points `help` at the documentation site', () => { + const help = execFileSync(process.execPath, ['src/cli.mjs', 'help'], { + cwd: root, + encoding: 'utf8', + }); + expect(help).toContain(DOCS_BASE_URL); + }); +}); diff --git a/tests/unit/cli-init.test.mjs b/tests/unit/cli-init.test.mjs index 99ae903..15126bb 100644 --- a/tests/unit/cli-init.test.mjs +++ b/tests/unit/cli-init.test.mjs @@ -5,6 +5,7 @@ import { resolvePythonConsumerSource, resolveTemplatesChoice, runInitDependencySetup, + toolchainReportLines, } from '../../src/cli-init.mjs'; describe('init example choice', () => { @@ -228,3 +229,50 @@ describe('init dependency setup', () => { expect(result).toMatchObject({ rustAvailable: false, installRust: false, rust: null }); }); }); + +/** + * Every outcome of the toolchain step says something. + * + * The "already there" branch used to say nothing at all: `init` looks for pixi and conda-pack on + * every run and asks only when one is missing, so on a machine that had both, the question a reader + * had been told to expect never appeared and nothing explained why. Silence there is + * indistinguishable from never having looked, and it was reported as a bug for exactly that reason. + */ +describe('the toolchain report', () => { + const lines = (toolchain) => + toolchainReportLines(toolchain, { toolchainDir: '/p/.scrollcase/toolchain' }); + const text = (toolchain) => lines(toolchain).map(([, message]) => message).join('\n'); + + it('says so when nothing was missing', () => { + const reported = lines({ installed: [], missing: [], pixiVersion: '0.77.0' }); + expect(reported.length).toBeGreaterThan(0); + expect(text({ installed: [], missing: [], pixiVersion: '0.77.0' })) + .toContain('Found pixi 0.77.0 and conda-pack'); + }); + + it('names the newer pixi, and what pinning it would mean', () => { + const reported = lines({ + installed: [], missing: [], pixiVersion: '0.73.0', newestPixiVersion: '0.78.0', + }); + expect(reported.some(([level]) => level === 'warning')).toBe(true); + // The consequence, not the news: `new scroll` records the pixi it finds and `build` refuses any + // other for that scroll, so being behind decides what every scroll written next pins. + expect(reported.map(([, message]) => message).join('\n')).toContain('would pin 0.73.0'); + }); + + it('stays quiet about the newest release when it is current or unknown', () => { + for (const newestPixiVersion of ['0.77.0', null, undefined]) { + const reported = lines({ installed: [], missing: [], pixiVersion: '0.77.0', newestPixiVersion }); + expect(reported.some(([level]) => level === 'warning'), String(newestPixiVersion)).toBe(false); + } + }); + + it('still reports an install, an unsupported host and a decline', () => { + expect(text({ installed: ['pixi 0.78.0'], missing: [], configPath: '/p/scrollcase.config.json' })) + .toContain('Installed pixi 0.78.0 into /p/.scrollcase/toolchain'); + expect(text({ installed: [], missing: ['pixi'], unsupportedHost: 'aix/ppc64' })) + .toContain('publishes no build for aix/ppc64'); + expect(text({ installed: [], missing: ['pixi', 'conda-pack'], declined: true })) + .toContain('Skipped installing pixi and conda-pack'); + }); +}); diff --git a/tests/unit/cli-output.test.mjs b/tests/unit/cli-output.test.mjs index 819dd77..ed2c6ae 100644 --- a/tests/unit/cli-output.test.mjs +++ b/tests/unit/cli-output.test.mjs @@ -73,9 +73,23 @@ describe('CLI output presentation', () => { expect(buildDistributionSummary({ archivePath: join(distDir, 'boxes', 'demo', '1.2.3', 'macos-aarch64-metal', 'abc123.zip'), channelPath: join(distDir, 'channels', 'demo', 'beta', 'macos-aarch64-metal.json'), + published: true, }, distDir)).toBe( 'Build complete — you can distribute the 2 files under boxes/demo/1.2.3/macos-aarch64-metal/ ' + 'and channels/demo/beta/macos-aarch64-metal.json', ); }); + + it('does not tell the author to distribute a box whose documents point nowhere', () => { + // The build already said the box names no publish location. A closing line inviting the author + // to distribute it would contradict that, and send them to upload documents with no URLs in. + const distDir = join(process.cwd(), '.scrollcase', 'dist'); + const summary = buildDistributionSummary({ + archivePath: join(distDir, 'boxes', 'demo', '1.2.3', 'macos-aarch64-metal', 'abc123.zip'), + channelPath: join(distDir, 'channels', 'demo', 'beta', 'macos-aarch64-metal.json'), + published: false, + }, distDir); + expect(summary).not.toMatch(/distribute/); + expect(summary).toContain('boxes/demo/1.2.3/macos-aarch64-metal/'); + }); }); diff --git a/tests/unit/cli-target-choice.test.mjs b/tests/unit/cli-target-choice.test.mjs index 1ed5735..32371d5 100644 --- a/tests/unit/cli-target-choice.test.mjs +++ b/tests/unit/cli-target-choice.test.mjs @@ -62,7 +62,7 @@ describe('CLI target selection', () => { expect(selected.targetId).toBe('macos-aarch64-cpu'); expect(menu).toHaveBeenCalledWith( ['macos-aarch64-cpu', 'macos-aarch64-metal'], - { hint: null, initialIndex: 1 }, + { hint: null, docs: null, initialIndex: 1 }, ); }); @@ -343,7 +343,7 @@ describe('CLI target selection', () => { '--python-version', '3.11.15', '--pixi-version', '0.73.0', '--min-host-app-version', '1.0.0', - '--asset-base-url', 'https://assets.example.org', + '--publish-base-url', 'https://assets.example.org', '--weights', 'embed', '--execution', 'library-only', ], { encoding: 'utf8' }); @@ -389,7 +389,7 @@ describe('CLI build choices', () => { expect(menu).toHaveBeenCalledWith( 'channel', ['beta', 'stable', 'nightly'], - { hint: null, initialIndex: 0 }, + { hint: null, docs: null, initialIndex: 0 }, ); }); diff --git a/tests/unit/docs-contract.test.mjs b/tests/unit/docs-contract.test.mjs index fb725b0..5c52fef 100644 --- a/tests/unit/docs-contract.test.mjs +++ b/tests/unit/docs-contract.test.mjs @@ -154,7 +154,16 @@ describe('public documentation routes', () => { }); it('documents every public runtime export', async () => { - const reference = await readFile(join(root, 'docs', 'reference', 'api.md'), 'utf8'); + // The API reference is a section, not a page: `reference/api/` carries an index plus one file + // per consumer language. Every export still has to appear somewhere in it, so the whole section + // is the corpus — pinning this to the single file it used to be made a page split look like an + // undocumented export. + const apiDir = join(root, 'docs', 'reference', 'api'); + const pages = (await readdir(apiDir)).filter((name) => name.endsWith('.md')).sort(); + expect(pages.length).toBeGreaterThan(0); + const reference = (await Promise.all( + pages.map((name) => readFile(join(apiDir, name), 'utf8')), + )).join('\n'); for (const [subpath, exports] of Object.entries({ contract, build, diff --git a/tests/unit/docs-markdown-negotiation.test.mjs b/tests/unit/docs-markdown-negotiation.test.mjs index c0cfbac..131bb14 100644 --- a/tests/unit/docs-markdown-negotiation.test.mjs +++ b/tests/unit/docs-markdown-negotiation.test.mjs @@ -8,6 +8,7 @@ * two have to agree, and neither can prove it alone. */ +import { readFile } from 'node:fs/promises'; import { describe, expect, it } from 'vitest'; import { markdownPathFor, onRequest, prefersMarkdown } from '../../docs/functions/_middleware.js'; @@ -129,3 +130,72 @@ describe('onRequest', () => { expect(await response.text()).toBe(''); }); }); + +/** + * The deprecated documentation's HTTP signals. + * + * These matter to exactly the readers who cannot see the banner on the page: a crawler deciding + * whether to index, and a bot that fetched the Markdown twin. Both representations carry them, and + * pages of the current documentation carry none — a `noindex` leaking onto those would quietly + * remove the site from search with nothing visibly wrong. + */ +describe('deprecated documentation headers', () => { + it('keeps the middleware prefix in step with the one the site is built from', async () => { + const { PREFIX } = await import('../../docs/.vitepress/versions.mjs'); + const source = await readFile( + new URL('../../docs/functions/_middleware.js', import.meta.url), 'utf8'); + expect(source).toContain(`const DEPRECATED_PREFIX = '${PREFIX}'`); + }); + + it.each([ + ['a page', 'https://scrollcase.dev/v2/reference/cli', { page: 'v2' }], + ['its Markdown twin', 'https://scrollcase.dev/v2/reference/cli.md', { page: '# CLI\n' }], + ['the landing page', 'https://scrollcase.dev/v2/', { page: 'v2' }], + ])('tells a crawler not to index %s', async (_what, url, options) => { + const response = await onRequest(context(url, options)); + + expect(response.headers.get('X-Robots-Tag')).toBe('noindex, follow'); + expect(response.headers.get('Link')).toContain( + '; rel="deprecation"'); + }); + + it('marks a negotiated Markdown response too', async () => { + const response = await onRequest(context('https://scrollcase.dev/v2/reference/cli', { + accept: 'text/markdown', + twin: '---\ndeprecated: true\n---\n', + })); + + expect(response.headers.get('Content-Type')).toBe('text/markdown; charset=utf-8'); + expect(response.headers.get('X-Robots-Tag')).toBe('noindex, follow'); + }); + + it('leaves the current documentation indexable', async () => { + for (const url of [ + 'https://scrollcase.dev/reference/cli', + 'https://scrollcase.dev/reference/cli.md', + 'https://scrollcase.dev/', + ]) { + const response = await onRequest(context(url, { page: 'a page' })); + expect(response.headers.get('X-Robots-Tag'), url).toBeNull(); + expect(response.headers.get('Link') ?? '', url).not.toContain('rel="deprecation"'); + } + }); + + // RFC 9745's field is a Date and nothing else, so it ships only once there is a real one. A + // header naming a deprecation date that never happened is a claim no client can check. + it('omits the Deprecation field until a date is declared', async () => { + const source = await readFile( + new URL('../../docs/functions/_middleware.js', import.meta.url), 'utf8'); + const declared = source.match(/const DEPRECATED_SINCE = (.+);/)[1]; + const response = await onRequest(context('https://scrollcase.dev/v2/reference/cli', { + page: 'v2', + })); + + if (declared === 'null') { + expect(response.headers.get('Deprecation')).toBeNull(); + } else { + expect(Number(declared), 'a Unix timestamp in seconds').toBeGreaterThan(0); + expect(response.headers.get('Deprecation')).toBe(`@${declared}`); + } + }); +}); diff --git a/tests/unit/project-surface.test.mjs b/tests/unit/project-surface.test.mjs index 3f2f64a..36629ce 100644 --- a/tests/unit/project-surface.test.mjs +++ b/tests/unit/project-surface.test.mjs @@ -56,7 +56,11 @@ describe('setting a project up', () => { expect(lines.length).toBeLessThan(50); expect(lines[0]).toBe('[Scrollcase documentation](https://scrollcase.dev/)'); expect(lines.at(-1)).toBe('[Scrollcase documentation](https://scrollcase.dev/)'); - expect(guide.match(/https:\/\/scrollcase\.dev\//g)).toHaveLength(8); + // Nine, not eight: the runtime reference joined them. A workspace holds many boxes and each + // picks its own runtime, which nothing else in this file said — and the guide is the only + // documentation a scaffolded project starts with. + expect(guide.match(/https:\/\/scrollcase\.dev\//g)).toHaveLength(9); + expect(guide).toContain('https://scrollcase.dev/reference/scroll#choosing-a-runtime'); expect(guide).toContain('https://scrollcase.dev/reference/scroll'); expect(guide).toContain('https://scrollcase.dev/reference/box-format'); expect(guide).toContain('https://scrollcase.dev/reference/box-format#targets'); diff --git a/tests/unit/scroll-authoring.test.mjs b/tests/unit/scroll-authoring.test.mjs index 7a165a6..155ab77 100644 --- a/tests/unit/scroll-authoring.test.mjs +++ b/tests/unit/scroll-authoring.test.mjs @@ -6,10 +6,12 @@ import { PassThrough } from 'node:stream'; import { afterEach, describe, expect, it } from 'vitest'; import { copyVerifiedLocalFile } from '../../src/build/assets.mjs'; import { + BOX_ID_SHAPE, DEFAULT_NODE_VERSION, DEFAULT_PYTHON_VERSION, LATEST_NODE_VERSION, LATEST_PYTHON_VERSION, + boxIdProblem, createScroll, ensureConsumerTemplates, ensureExampleScroll, @@ -22,6 +24,8 @@ import { configureWorkspace, getWorkspace, resetWorkspace } from '../../src/buil import { collectNewScrollOptions, promptText } from '../../src/cli-authoring.mjs'; const TARGET = { platform: 'macos', arch: 'aarch64', accelerator: 'metal' }; +// Every runtime that offers a choice offers `library-only` last, so the hint's list ends on it. +const EXECUTION_KIND_TAIL = 'being imported by another application as a library'; const BASE = { boxId: 'example-model', target: TARGET, @@ -32,7 +36,7 @@ const BASE = { runtimeVersion: '3.11.15', pixiVersion: '0.73.0', compatibility: { minHostAppVersion: '1.0.0' }, - assetBaseUrl: 'https://assets.example.org', + publishBaseUrl: 'https://assets.example.org', }; describe('scroll authoring', () => { @@ -156,7 +160,7 @@ describe('scroll authoring', () => { const answers = new Map([ ['Box ID', BASE.boxId], ['Upstream revision', BASE.sourceRevision], - ['Asset base URL', BASE.assetBaseUrl], + ['Publish base URL', BASE.publishBaseUrl], ['Python module', 'example_model.main'], ]); const asked = []; @@ -169,7 +173,7 @@ describe('scroll authoring', () => { throw new Error(`unexpected question: ${question}`); } // Every question carries one line saying what the field is: a label alone does not explain - // `sourceRevision` or `assetBaseUrl` to someone meeting the tool for the first time. + // `sourceRevision` or `publishBaseUrl` to someone meeting the tool for the first time. expect(promptOptions.hint).toEqual(expect.any(String)); return answers.get(question); }, @@ -210,7 +214,7 @@ describe('scroll authoring', () => { it('pins the pixi that is installed, since a build refuses any other', async () => { const options = await collectNewScrollOptions( new Map([['box-id', 'example-model'], ['source-revision', 'upstream-v1'], - ['asset-base-url', 'https://assets.example.org'], + ['publish-base-url', 'https://assets.example.org'], ['execution', 'library-only'], ['target', 'macos-aarch64-metal']]), { terminal: false, probe: () => ({ path: 'pixi', version: '9.9.9' }) }, ); @@ -218,6 +222,237 @@ describe('scroll authoring', () => { expect(options.pixiVersion).toBe('9.9.9'); }); + it('points a native box at a binary the environment provides, staging nothing', async () => { + const current = await workspace(); + const result = await createScroll({ + workspace: current, + ...BASE, + runtimeId: 'native', + runtimeVersion: undefined, + executionKind: 'native-binary', + environmentPath: 'venv/bin/ffmpeg', + defaultArgs: ['-hide_banner'], + }); + + expect(result.scroll.execution).toEqual({ + kind: 'native-binary', + binary: 'venv/bin/ffmpeg', + defaultArgs: ['-hide_banner'], + }); + // Nothing of the project is copied in, so there is no `localFiles` entry to invent. Claiming one + // would say the project ships a file it does not have. + expect(result.scroll.localFiles).toBeUndefined(); + await expect(readScroll(result.scrollRef)).resolves.toBeTruthy(); + }); + + it('refuses a binary that is both in the environment and in the project', async () => { + const current = await workspace(); + await expect(createScroll({ + workspace: current, + ...BASE, + runtimeId: 'native', + runtimeVersion: undefined, + executionKind: 'native-binary', + environmentPath: 'venv/bin/ffmpeg', + scriptSourcePath: 'tool', + })).rejects.toThrow(/either a file from the environment or one from this project/); + }); + + it('collects --from-environment without asking for a project file', async () => { + const options = await collectNewScrollOptions( + new Map([['target', 'macos-aarch64-metal'], ['runtime', 'native'], + ['box-id', 'transcode-demo'], ['source-revision', 'ffmpeg-9'], + ['from-environment', 'venv/bin/ffmpeg']]), + { terminal: false, probe: () => ({ path: 'pixi', version: BASE.pixiVersion }) }, + ); + + expect(options.environmentPath).toBe('venv/bin/ffmpeg'); + expect(options.scriptSourcePath).toBeUndefined(); + expect(options.generateScript).toBeUndefined(); + }); + + it('asks a native box where its binary comes from, the environment first', async () => { + let offered = null; + const asked = []; + const options = await collectNewScrollOptions( + new Map([['target', 'macos-aarch64-metal'], ['runtime', 'native'], + ['box-id', 'transcode-demo'], ['source-revision', 'ffmpeg-9']]), + { + terminal: true, + ask: async (question) => { asked.push(question); return 'venv/bin/ffmpeg'; }, + choose: async (question, choices) => { + if (question === 'binary source') offered = choices; + return question === 'runtime' ? 'native' : choices[0]; + }, + chooseTargetValue: async () => ({ target: TARGET, targetId: 'macos-aarch64-metal' }), + probe: () => ({ path: 'pixi', version: BASE.pixiVersion }), + }, + ); + + // Most native boxes package a program conda-forge installs, and that answer needs nothing to + // exist yet — so it is the preselected one, as the menu's first entry. + expect(offered).toEqual(['a program the environment provides', 'a compiled binary in this project']); + expect(options.environmentPath).toBe('venv/bin/ffmpeg'); + expect(options.scriptSourcePath).toBeUndefined(); + expect(asked).toContain('Path inside the box'); + expect(asked).not.toContain('Binary path'); + }); + + it('offers each runtime only the execution kinds it has, and explains those and no others', async () => { + const seen = new Map(); + const collect = async (runtimeId) => collectNewScrollOptions( + new Map([['runtime', runtimeId], ['box-id', 'example-model'], + ['source-revision', 'upstream-v1'], ['publish-base-url', 'https://assets.example.org'], + ['script', 'app/entrypoint'], ['target', 'macos-aarch64-metal']]), + { + terminal: true, + ask: async () => 'example_model.main', + choose: async (question, choices, options = {}) => { + if (question === 'execution kind') seen.set(runtimeId, { choices, hint: options.hint }); + return question === 'runtime' ? runtimeId : choices[0]; + }, + chooseTargetValue: async () => ({ target: TARGET, targetId: 'macos-aarch64-metal' }), + probe: () => ({ path: 'pixi', version: BASE.pixiVersion }), + }, + ); + + await collect('python'); + await collect('node'); + // A module is a Python idea. Offering a node author "an importable module" describes a choice + // that is not on their menu, which is how the shared sentence read before it was derived from + // the kinds actually offered. + expect(seen.get('python').hint).toContain('an importable module'); + expect(seen.get('node').hint).not.toContain('an importable module'); + for (const [runtimeId, { choices, hint }] of seen) { + expect(choices.length, runtimeId).toBeGreaterThan(1); + // `promptHeading` renders a hint as one lead-in line, stripping a trailing period and adding + // a colon of its own. So: no second sentence, because it has nowhere to go; no colon inside, + // because two in one line read as two questions; and the line ends where the list ends, + // because an explanation hung off the last item hides the list's boundary — which is exactly + // how "or nothing at all, for a box other code imports rather than runs" became unreadable. + expect(hint, runtimeId).not.toMatch(/\.\s/); + expect(hint, runtimeId).not.toContain(':'); + expect(hint.endsWith(EXECUTION_KIND_TAIL), `${runtimeId}: ${hint}`).toBe(true); + // No option may be described as an absence. `library-only` under a "what does run start" + // framing can only be "nothing at all", which tells the reader what they would not get + // instead of what the choice is for, and reads as an option that does nothing. Every entry + // has to answer why someone would pick it. + expect(hint, runtimeId).not.toMatch(/\bnothing\b|\bnone\b|\bno entry point\b/); + } + + // native defines one authored kind, so there is nothing to choose between and no menu is shown. + const native = await collect('native'); + expect(seen.has('native')).toBe(false); + expect(native.executionKind).toBe('native-binary'); + }); + + it('preselects the generated starter, which is the one that works with nothing else in place', async () => { + let offered = null; + const options = await collectNewScrollOptions( + new Map([['box-id', 'example-model'], ['source-revision', 'upstream-v1'], + ['publish-base-url', 'https://assets.example.org'], ['target', 'macos-aarch64-metal']]), + { + terminal: true, + ask: async () => 'entrypoint.py', + choose: async (question, choices) => { + if (question === 'script source') offered = choices; + if (question === 'runtime') return 'python'; + if (question === 'execution kind') return 'python-script'; + // The menu's first entry is what a preselected answer takes, so the order is the default. + return choices[0]; + }, + chooseTargetValue: async () => ({ target: TARGET, targetId: 'macos-aarch64-metal' }), + probe: () => ({ path: 'pixi', version: BASE.pixiVersion }), + }, + ); + + expect(offered[0]).toBe('generate starter script'); + expect(options.generateScript).toBe(true); + expect(options.scriptSourcePath).toBeUndefined(); + }); + + it('refuses a malformed box ID at the prompt, naming the shape, and asks again', { timeout: 5000 }, async () => { + const input = new PassThrough(); + const output = new PassThrough(); + let rendered = ''; + let asked = 0; + output.on('data', (chunk) => { + rendered += String(chunk); + if (!String(chunk).endsWith('↳ ')) return; + asked += 1; + input.write(asked === 1 ? 'Example Model\n' : 'example-model\n'); + }); + + const answer = await promptText('Box ID', { + hint: `Name of the box. ${BOX_ID_SHAPE}.`, + validate: boxIdProblem, + input, + output, + }); + + expect(answer).toBe('example-model'); + // The value that was refused, and the shape it needed — not "does not match the required + // pattern", and not after every other question in the session had been answered. + expect(rendered).toContain('Example Model is not a usable box ID'); + expect(rendered).toContain('lower-case letters and digits'); + }); + + it('refuses a malformed --box-id before asking anything else', async () => { + const asked = []; + await expect(collectNewScrollOptions( + new Map([['box-id', 'Example Model'], ['target', 'macos-aarch64-metal']]), + { + terminal: true, + ask: async (question) => { asked.push(question); return 'x'; }, + choose: async (question, choices) => (question === 'runtime' ? 'python' : choices[0]), + chooseTargetValue: async () => ({ target: TARGET, targetId: 'macos-aarch64-metal' }), + probe: () => ({ path: 'pixi', version: BASE.pixiVersion }), + }, + )).rejects.toThrow(/Example Model is not a usable box ID/); + expect(asked).toEqual([]); + }); + + it('writes a scroll with no publishBaseUrl when the author skips it, rather than inventing one', async () => { + const current = await workspace(); + const { publishBaseUrl: _skipped, ...withoutUrl } = BASE; + const result = await createScroll({ + workspace: current, + ...withoutUrl, + executionKind: 'library-only', + }); + + // Absent, not a placeholder: a made-up URL in a signed release is a false statement about where + // the box is published, and the scroll schema does not require the field either. + expect(result.scroll.publishBaseUrl).toBeUndefined(); + expect(JSON.parse(await readFile(join(result.scrollDir, 'scroll.json'), 'utf8'))) + .not.toHaveProperty('publishBaseUrl'); + // Still a valid scroll a build can read; `build` is what refuses, by name, and says how to + // supply the URL it needs. + await expect(readScroll(result.scrollRef)).resolves.toBeTruthy(); + }); + + it('lets the wizard skip the publish base URL instead of blocking the session on it', async () => { + let optionalAsk = null; + const options = await collectNewScrollOptions( + new Map([['box-id', 'example-model'], ['source-revision', 'upstream-v1'], + ['target', 'macos-aarch64-metal']]), + { + terminal: true, + ask: async (question, promptOptions = {}) => { + if (question !== 'Publish base URL') return 'value'; + optionalAsk = promptOptions.optional; + return null; + }, + choose: async (question) => (question === 'runtime' ? 'python' : 'library-only'), + chooseTargetValue: async () => ({ target: TARGET, targetId: 'macos-aarch64-metal' }), + probe: () => ({ path: 'pixi', version: BASE.pixiVersion }), + }, + ); + + expect(optionalAsk).toBe(true); + expect(options.publishBaseUrl).toBeNull(); + }); + // Short timeout on purpose: an abort on the first blank answer leaves the prompt waiting for input // nobody will send, so the failure must arrive quickly rather than stall the suite. it('repeats a required question instead of throwing the session away', { timeout: 5000 }, async () => { @@ -503,4 +738,21 @@ describe('scroll authoring', () => { current.root, )).rejects.toThrow(/SHA-256 mismatch/); }); + + it('takes default arguments as one argument or as a JSON array', async () => { + const collect = async (value) => (await collectNewScrollOptions( + new Map([['target', 'macos-aarch64-metal'], ['runtime', 'native'], + ['box-id', 'transcode-demo'], ['source-revision', 'ffmpeg-9'], + ['from-environment', 'venv/bin/ffmpeg'], ['default-args', value]]), + { terminal: false, probe: () => ({ path: 'pixi', version: BASE.pixiVersion }) }, + )).defaultArgs; + + // The common case is one argument, and quoting a one-element JSON array to say it is a tax. + expect(await collect('-hide_banner')).toEqual(['-hide_banner']); + expect(await collect('["-hide_banner", "-nostats"]')).toEqual(['-hide_banner', '-nostats']); + // A value that opens like an array is held to being one: falling back to a literal would turn a + // malformed array into a single argument that looks almost right. + await expect(collect('["-a"')).rejects.toThrow(/not valid JSON/); + await expect(collect('[1, 2]')).rejects.toThrow(/JSON array of strings, or a single argument/); + }); }); diff --git a/tests/unit/scroll-editing.test.mjs b/tests/unit/scroll-editing.test.mjs index 8153ad1..0471f74 100644 --- a/tests/unit/scroll-editing.test.mjs +++ b/tests/unit/scroll-editing.test.mjs @@ -19,6 +19,8 @@ import { ALL_TARGETS, addAsset, addFile, + addSelfTestCommand, + removeSelfTestCommand, addSelfTestImport, editableScrollFields, refreshScroll, @@ -46,7 +48,7 @@ const SHARED = { sourceRevision: 'upstream-v1', runtime: { id: 'python', version: '3.14' }, pixiVersion: '0.73.0', - assetBaseUrl: 'https://assets.example.org/boxes', + publishBaseUrl: 'https://assets.example.org/boxes', selfTest: { imports: ['json'] }, }; @@ -280,7 +282,7 @@ describe('editing an existing scroll', () => { const names = (await editableScrollFields()).map(({ name }) => name); expect(names).toContain('version'); - expect(names).toContain('assetBaseUrl'); + expect(names).toContain('publishBaseUrl'); for (const excluded of ['boxId', 'target', 'runtime', 'schemaVersion', 'extends', 'assets']) { expect(names, excluded).not.toContain(excluded); } @@ -368,6 +370,67 @@ describe('editing an existing scroll', () => { expect(await chooseEditTarget({ boxId: 'example-model', terminal: false })).toBe(TARGET_ID); }); + + /** + * A `native` box's only probe shape is a command, so without these its self-test could not be + * authored at all — the scroll had to be edited by hand, which every other command here exists to + * avoid. `pin` closes the last hand edit: recording a hash used to mean opening the file. + */ + it('authors a native self-test without touching the scroll by hand', async () => { + const { boxDir } = await splitBox({ + base: { + ...SHARED, + runtime: { id: 'native' }, + execution: { kind: 'native-binary', binary: 'venv/bin/ffmpeg', defaultArgs: [] }, + selfTest: { commands: [{ args: [] }] }, + }, + }); + const read = async () => JSON.parse(await readFile(join(boxDir, 'scroll.json'), 'utf8')); + + await addSelfTestCommand({ boxId: 'example-model', target: ALL_TARGETS, args: ['-version'] }); + await addSelfTestCommand({ + boxId: 'example-model', target: ALL_TARGETS, args: ['-i', 'missing.mp4'], expectExitCode: 254, + }); + + // The placeholder `new scroll` leaves — "run it with no arguments" — stops being a claim anyone + // made once a real probe exists, so it is replaced rather than kept beside them. + expect((await read()).selfTest.commands).toEqual([ + { args: ['-version'] }, + { args: ['-i', 'missing.mp4'], expectExitCode: 254 }, + ]); + + await expect(addSelfTestCommand({ + boxId: 'example-model', target: ALL_TARGETS, args: ['-version'], + })).rejects.toThrow(/already runs that self-test command/); + await expect(addSelfTestCommand({ + boxId: 'example-model', target: ALL_TARGETS, args: ['-x'], expectExitCode: 999, + })).rejects.toThrow(/between 0 and 255/); + + await removeSelfTestCommand({ boxId: 'example-model', target: ALL_TARGETS, args: ['-version'] }); + expect((await read()).selfTest.commands).toEqual([ + { args: ['-i', 'missing.mp4'], expectExitCode: 254 }, + ]); + await expect(removeSelfTestCommand({ + boxId: 'example-model', target: ALL_TARGETS, args: ['-nope'], + })).rejects.toThrow(/does not run that self-test command/); + }); + + it('records a file hash only when asked to pin it', async () => { + const { root, boxDir } = await splitBox(); + await writeFile(join(root, 'data.csv'), 'a,b\n1,2\n'); + const read = async () => JSON.parse(await readFile(join(boxDir, 'scroll.json'), 'utf8')); + + await addFile({ boxId: 'example-model', target: ALL_TARGETS, sourcePath: 'data.csv' }); + expect((await read()).localFiles.at(-1).sha256).toBeUndefined(); + + await addFile({ + boxId: 'example-model', target: ALL_TARGETS, sourcePath: 'data.csv', to: 'pinned.csv', pin: true, + }); + // Opt-in on purpose: most added files are about to be edited, and a hash recorded then would + // fail the very next build. Reference data is the case that wants it. + expect((await read()).localFiles.at(-1).sha256).toMatch(/^[a-f0-9]{64}$/); + }); + }); describe('a box pixi manifest', () => { diff --git a/tests/unit/scroll-extends.test.mjs b/tests/unit/scroll-extends.test.mjs index 1797f5a..063493c 100644 --- a/tests/unit/scroll-extends.test.mjs +++ b/tests/unit/scroll-extends.test.mjs @@ -27,7 +27,7 @@ const BASE = { sourceRevision: 'upstream-v1', runtime: { id: 'python', version: '3.14' }, pixiVersion: '0.73.0', - assetBaseUrl: 'https://assets.example.org/boxes', + publishBaseUrl: 'https://assets.example.org/boxes', selfTest: { imports: ['json'] }, }; diff --git a/tests/unit/v3-migration.test.mjs b/tests/unit/v3-migration.test.mjs index b652aad..90a7c13 100644 --- a/tests/unit/v3-migration.test.mjs +++ b/tests/unit/v3-migration.test.mjs @@ -53,13 +53,32 @@ describe('canonical scroll workspace names', () => { expect(workspace).not.toHaveProperty(legacyField); }); - it('keeps retired product terminology out of tracked content and paths', () => { - const retired = ['re', 'cipe'].join(''); + /** + * Two words, forbidden for two different reasons. + * + * The first is the name of the project Scrollcase was extracted from. Hard rule 1 says it appears + * nowhere — not in identifiers, error messages, environment variables, default paths, wire strings + * or examples — because the tool must stay usable by projects that have nothing to do with the one + * that first needed it. It has come back twice already, both times inside files moved after a + * clean grep, so the grep is a test rather than a habit. The second is a retired product term from + * before the rename. + * + * Each is assembled from fragments so that this file does not contain the word it forbids. That is + * not cleverness for its own sake: a guard that trips on itself gets weakened, and a weakened guard + * is how the name comes back. + */ + it.each([ + ['the name of the project this tool was extracted from', ['lia', 'tir']], + ['retired product terminology', ['re', 'cipe']], + ])('keeps %s out of tracked content and paths', (_reason, fragments) => { + const forbidden = fragments.join(''); const contentSearch = spawnSync( 'git', - ['grep', '-I', '-i', '--name-only', retired, '--', '.'], + ['grep', '-I', '-i', '--name-only', forbidden, '--', '.'], { cwd: root, encoding: 'utf8' }, ); + // `git grep` exits 1 for "found nothing", which is the only acceptable answer. Anything else is + // either a hit or a broken invocation, and both must fail rather than pass quietly. expect(contentSearch.status, contentSearch.stderr).toBe(1); expect(contentSearch.stdout).toBe(''); @@ -67,6 +86,6 @@ describe('canonical scroll workspace names', () => { cwd: root, encoding: 'utf8', }).split('\0').filter(Boolean); - expect(trackedPaths.filter((path) => path.toLowerCase().includes(retired))).toEqual([]); + expect(trackedPaths.filter((path) => path.toLowerCase().includes(forbidden))).toEqual([]); }); }); From 67e0a40937c488ab3c553080510a3ff9100e023c Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:26:26 +0200 Subject: [PATCH 20/22] Fix the two Windows failures the first CI run found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both are defects in the tests, not in the product, and both had been invisible because this branch had never been pushed and CI runs on main and pull requests only. Windows carries no POSIX modes: archiveFileMode writes 0644 for every entry of a Windows box, and an extracted file reports no mode to read back. The umask conformance case asserted 755 and 644 on every platform. It is now gated with requiresPosixModes, honoured by all three drivers exactly as requiresSymlinks already is — the case is not weaker on Windows, it is inapplicable there. Proved by forcing the gate closed: 85 run, 1 skipped. The phase C archive-executable test asserted a refusal on every target. On a Windows target there is no bit to be missing, so the build correctly does not refuse. It now asserts both halves of the rule and says which target each applies to. --- python/tests/test_conformance.py | 5 +++++ rust/fixtures/consumer-conformance.json | 1 + rust/tests/conformance.rs | 5 +++++ .../fixtures/consumer-conformance.json | 1 + tests/unit/build-pipeline.test.mjs | 19 ++++++++++++++++--- tests/unit/consumer-conformance.test.mjs | 9 ++++++++- 6 files changed, 36 insertions(+), 4 deletions(-) diff --git a/python/tests/test_conformance.py b/python/tests/test_conformance.py index 03e83c7..2ffc145 100644 --- a/python/tests/test_conformance.py +++ b/python/tests/test_conformance.py @@ -14,6 +14,7 @@ # one cannot run on this host. Skipped rather than weakened: the rule it proves is real on the two # platforms that carry links. SYMLINKS_SUPPORTED = sys.platform != "win32" +POSIX_MODES_SUPPORTED = sys.platform != "win32" class SharedConsumerConformanceTests(unittest.TestCase): @@ -22,6 +23,10 @@ def test_every_shared_semantic_case(self) -> None: for test_case in suite["cases"]: if test_case.get("requiresSymlinks") and not SYMLINKS_SUPPORTED: continue + # Windows carries no POSIX modes at all, so a case asserting one is inapplicable there + # rather than weaker — the same reason a link case is skipped. + if test_case.get("requiresPosixModes") and not POSIX_MODES_SUPPORTED: + continue with self.subTest(case=test_case["id"]): actual, expected, root = run_python_conformance_case( test_case, diff --git a/rust/fixtures/consumer-conformance.json b/rust/fixtures/consumer-conformance.json index 9aeaa4c..9d3d787 100644 --- a/rust/fixtures/consumer-conformance.json +++ b/rust/fixtures/consumer-conformance.json @@ -1402,6 +1402,7 @@ { "id": "declared-executable-survives-a-restrictive-umask", "action": "prepare", + "requiresPosixModes": true, "fixture": { "executableAsset": true }, diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index 2dc5975..3dbb6ca 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -1289,6 +1289,11 @@ fn the_shared_consumer_conformance_suite_passes() { if case.get("requiresSymlinks").is_some() && cfg!(not(unix)) { continue; } + // Windows carries no POSIX modes at all, so a case asserting one is inapplicable there + // rather than weaker — the same reason a link case is skipped. + if case.get("requiresPosixModes").is_some() && cfg!(not(unix)) { + continue; + } ran += 1; let Outcome { actual, expected } = run_case(case, patterns); if actual != expected { diff --git a/src/contract/fixtures/consumer-conformance.json b/src/contract/fixtures/consumer-conformance.json index 9aeaa4c..9d3d787 100644 --- a/src/contract/fixtures/consumer-conformance.json +++ b/src/contract/fixtures/consumer-conformance.json @@ -1402,6 +1402,7 @@ { "id": "declared-executable-survives-a-restrictive-umask", "action": "prepare", + "requiresPosixModes": true, "fixture": { "executableAsset": true }, diff --git a/tests/unit/build-pipeline.test.mjs b/tests/unit/build-pipeline.test.mjs index e44849e..f3d61e2 100644 --- a/tests/unit/build-pipeline.test.mjs +++ b/tests/unit/build-pipeline.test.mjs @@ -504,22 +504,35 @@ describe('the build pipeline', () => { }); it('refuses a box that runs a file the archive would not mark executable', async () => { + // Both halves of one rule, and which half applies is the *target's* business rather than the + // build host's. A POSIX box carries modes, so a file nothing declared executable comes out + // 0644 and the box could never start: refused. A Windows box carries no modes at all — + // `archiveFileMode` writes 0644 for every entry there and Windows decides executability by + // extension — so there is no bit to be missing and nothing to refuse. Asserting the refusal + // unconditionally is what made this pass on macOS and fail on Windows, found by the first CI + // run this branch ever had. const { keys, payloadDir } = await makeProject({ ...SCROLL, runtime: { id: 'native' }, - // Declared without `executable`, so the archive would ship it 0644 and nothing could start it. + // Declared without `executable`, so on a POSIX target the archive would ship it 0644. localFiles: [{ sourcePath: 'tool', relativePath: 'bin/tool' }], execution: { kind: 'native-binary', binary: 'bin/tool', defaultArgs: [] }, selfTest: { commands: [{ args: [] }], files: [] }, }, { projectFiles: { tool: '#!/bin/sh\nexit 0\n' } }); - await expect(buildBox(SCROLL_REF, { + const build = () => buildBox(SCROLL_REF, { ...keys, ...fakeToolchain(payloadDir, { interpreter: false, selfTestCommand: join(payloadDir, 'bin', 'tool'), }), log: () => {}, - })).rejects.toThrow(/runs bin\/tool, which the archive would not mark executable/); + }); + + if (HOST_ADAPTER.host.platform === 'win32') { + await expect(build()).resolves.toMatchObject({ archiveSha256: expect.any(String) }); + return; + } + await expect(build()).rejects.toThrow(/runs bin\/tool, which the archive would not mark executable/); }); it('refuses a native scroll that declares a runtime entry point', async () => { diff --git a/tests/unit/consumer-conformance.test.mjs b/tests/unit/consumer-conformance.test.mjs index 0964168..c15a2e9 100644 --- a/tests/unit/consumer-conformance.test.mjs +++ b/tests/unit/consumer-conformance.test.mjs @@ -12,9 +12,16 @@ const suite = await loadConsumerConformanceSuite(); // platforms that carry links. const symlinksSupported = process.platform !== 'win32'; +// Windows carries no POSIX modes at all — `archiveFileMode` writes 0644 for every entry of a +// Windows box, and an extracted file reports no mode to read back. A case asserting one is not +// weaker there, it is inapplicable, so it is skipped for the same reason a link case is. +const posixModesSupported = process.platform !== 'win32'; + describe('shared consumer conformance — Node', () => { for (const testCase of suite.cases) { - it.skipIf(testCase.requiresSymlinks && !symlinksSupported)(testCase.id, async () => { + const inapplicable = (testCase.requiresSymlinks && !symlinksSupported) + || (testCase.requiresPosixModes && !posixModesSupported); + it.skipIf(inapplicable)(testCase.id, async () => { const result = await runNodeConformanceCase({ ...testCase, suite }); try { expect(result.actual).toEqual(result.expected); From b222eba2b5ce6dac76458f79d8ceaaf7a8550f10 Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:29:53 +0200 Subject: [PATCH 21/22] Drop a redundant unit return type from the Windows umask stub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clippy's unused_unit, on the one function only Windows compiles. It had never been linted: this branch reached Windows CI for the first time today. Checked for real rather than by analogy — cargo clippy --target x86_64-pc-windows-msvc compiles the cfg(not(unix)) branch on this Mac, and is what should have caught it before the push. --- rust/tests/conformance.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index 3dbb6ca..3a976fd 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -516,8 +516,10 @@ impl Drop for UmaskGuard { } } +/// No umask on Windows: the platform carries no POSIX modes for one to mask. The stub keeps the +/// call site free of `cfg` branches, and the cases that actually assert a mode are skipped there. #[cfg(not(unix))] -fn set_umask(_octal: &str) -> () {} +fn set_umask(_octal: &str) {} fn receipt_value(prepared: &PreparedBox, expected: &Value, names: &[String]) -> Value { let mut receipt = json!({ From f4defc56b0a3f2939d7149c63a8e259cf23931bf Mon Sep 17 00:00:00 2001 From: "Lorenzo S." <46894435+Suffro@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:51:03 +0200 Subject: [PATCH 22/22] Fix browser export, add v2-to-v3 guide Restores `scrollcase/contract/browser` by replacing the stale `assertPythonEntryPoint` re-export with runtime-model exports from `runtimes.mjs` (including `assertRuntimeEntryPoint`), so the browser entry point links correctly again. Also adds a new docs guide at `/guides/migrating-from-v2`, wires it into the sidebar and v2 deprecation notice, and updates related references (Node API docs, box-format docs, white paper, changelog). The package-surface test now links each exported subpath in real Node processes via the `exports` map to catch unresolved re-exports that Vitest can miss. --- CHANGELOG.md | 34 ++- docs/.vitepress/config.mts | 1 + docs/.vitepress/theme/DeprecationNotice.vue | 2 +- docs/guides/migrating-from-v2.md | 236 ++++++++++++++++++++ docs/reference/api/node.md | 15 +- docs/reference/box-format.md | 42 +++- docs/white-paper.md | 23 +- src/contract/browser.d.mts | 1 + src/contract/browser.mjs | 25 ++- tests/unit/package-surface.test.mjs | 55 ++++- tests/unit/project-surface.test.mjs | 2 +- 11 files changed, 398 insertions(+), 38 deletions(-) create mode 100644 docs/guides/migrating-from-v2.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c4af273..875dbaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -151,8 +151,9 @@ dual-read path anywhere, and no migration tool: a box is rebuilt from its scroll `DYLD_INSERT_LIBRARIES` on macOS, `LD_PRELOAD` on Linux, nothing on Windows. The runtime contributes the `PYTHON*` half, and `executionAffectingVariables(runtimeId, adapter)` joins the two in the order a diagnostic report prints them. The Rust `BoxTargetAdapter` and the Python - `TargetAdapter` lost the same fields, for the same reason. `assertPythonEntryPoint` keeps its - published name and signature in all three, and delegates to the runtime rule. + `TargetAdapter` lost the same fields, for the same reason. `assertPythonEntryPoint` is gone from + all three, replaced by `assertRuntimeEntryPoint(runtimeId, adapter, entryPoint)`, which asks the + runtime rule rather than the target for the layout it judges against. - The builder-side half of a runtime now lives under `src/runtimes//`. `repairPosixLaunchers` moved from `src/build/launchers.mjs` to `src/runtimes/python/launchers.mjs` — the conda shebang @@ -470,8 +471,37 @@ dual-read path anywhere, and no migration tool: a box is rebuilt from its scroll into a document whose whole value is that it is true. Press Enter to skip; supply it later with `edit scroll`, or per build with `--asset-base-url`. +### Added — one page for the v2 → v3 move + +- **`/guides/migrating-from-v2` is the field-by-field mapping, in one place.** Every renamed scroll + field, every renamed field in the signed documents, every renamed or removed CLI flag, the three + additions version 2 had no equivalent for — `assets[].executable`, `bundledLicenseDeclaration`, + an optional `archive.url` — a before-and-after scroll, and the order to do it in. All of it was + derivable before, from three separate subsections of this changelog and a table in the box-format + reference, which is not the same as being findable by someone holding a v2 scroll. That table is + now the *why* and links here for the *what*, so the mapping has one home rather than two. + +- The deprecation notice on every `/v2/` page links to it, since a reader who arrived there from an + old link or a search result is exactly the person it is for. + ### Fixed +- **`scrollcase/contract/browser` was a dead entry point for the whole of the version 3 work.** It + re-exported `assertPythonEntryPoint` from `targets.mjs`, which the runtime split had renamed and + moved, so linking the module under `node` was a `SyntaxError` before a single statement ran. It + now exports `assertRuntimeEntryPoint` and the rest of the runtime model — `runtimes.mjs` reads no + file, joins no host path and starts no process, so the browser-safe rule is unchanged and the + entry point is once again everything `scrollcase/contract` has except `decodeDocumentPayload`, + `schemaUrl` and `fixtureUrl`. + + **Three things should have caught it and all three looked elsewhere**, which is the part worth + recording. The package-surface import test runs under Vitest, whose resolver forgave the missing + export. The import-closure check is a regular expression over source text and never evaluates a + module. And `tsc` omits an unresolvable re-export from the generated `.d.mts` without a word, so + `types:check` reported no drift. The suite now links all five published entry points in separate + `node` processes, through the `exports` map, which is the walk a dependent's own `import` + performs. + - **A build with no asset base URL is refused before it solves anything.** The URL is needed only when the release document is written, which is after the environment solve, the self-test and the archive — so a scroll that never named one paid for the entire build before being told. It is now diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index c92a561..f15b595 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -107,6 +107,7 @@ const sidebar = [ { text: 'Offline / Air-Gapped Installs', link: '/guides/offline-airgap' }, { text: 'Distributing Boxes', link: '/guides/distributing-boxes' }, { text: 'Platform Examples', link: '/guides/platform-examples' }, + { text: 'Migrating from v2', link: '/guides/migrating-from-v2' }, { text: 'Troubleshooting', link: '/guides/troubleshooting' } ] }, diff --git a/docs/.vitepress/theme/DeprecationNotice.vue b/docs/.vitepress/theme/DeprecationNotice.vue index 1c83e93..842aa1c 100644 --- a/docs/.vitepress/theme/DeprecationNotice.vue +++ b/docs/.vitepress/theme/DeprecationNotice.vue @@ -41,7 +41,7 @@ const mirrored = computed(() => current.value !== '/') refuses a version 2 box by name rather than reading it. {{ mirrored ? 'Read this page in the current version' : 'Go to the current documentation' - }}. + }}, or see how to migrate a box to v3.

diff --git a/docs/guides/migrating-from-v2.md b/docs/guides/migrating-from-v2.md new file mode 100644 index 0000000..e501790 --- /dev/null +++ b/docs/guides/migrating-from-v2.md @@ -0,0 +1,236 @@ +--- +title: Migrating from v2 +description: Every version 2 field, flag and document field, and what it becomes in version 3. +--- + +# Migrating from v2 + +Version 3 is a clean break, and the only one planned. There is no dual-read path and no migration +tool: **a box is rebuilt from its scroll**. Migrating therefore means editing one `scroll.json` per +box, running `scrollcase build` again, and publishing the result — the work is in the scroll, and +this page is the whole of it. + +Everything below is a mapping. Nothing was renamed for tidiness; the reasoning behind each change is +in [what version 3 changed](/reference/box-format#what-version-3-changed) and, at length, in +[design decisions](/concepts/design-decisions). + +## What does not migrate + +**A published v2 box stays a v2 box.** Version 3 refuses one by name rather than reinterpreting it: + +```text +Unsupported schemaVersion 2; rebuild this box with Scrollcase v3. +``` + +A v1 box is refused the same way, naming version 1. Whoever holds either is entitled to know which +rebuild is ahead of them, which is why neither is guessed at. If you need to keep reading an old +box rather than rebuilding it, keep the Scrollcase version that produced it — the +[version 2 documentation](/v2/) stays published for exactly that reason. + +**Your signing keys carry over untouched.** The envelope changed only in its `schemaVersion`: the +payload encoding is still `base64-json-utf8`, the signature algorithm is still `ed25519`, and the +`kind` strings are still `.release`, `.channel` and `.revocations` under the namespace +your project already publishes. There is no key rotation implied by this upgrade, and +`keygen --force` is never part of one — it would silently invalidate every document the old key +signed. See [signing and key custody](/guides/signing-and-custody). + +**`payload-digest.v1` is untouched**, so an extracted v2 installation still identifies itself the +same way. + +## The scroll, field by field + +| Version 2 | Version 3 | Notes | +| --- | --- | --- | +| `pythonVersion` | `runtime.version` | Inside the new required `runtime` block | +| `pythonEntryPoint` | `runtime.entryPoint` | Still derived when omitted, still checked against the target when declared. Absent for `native`, which has no interpreter, and **declaring one there is refused** rather than ignored | +| — | `runtime.id` | New and required: `python`, `node` or `native`. A v2 scroll becomes `"id": "python"`. A version 2 box said *where its Python was* and never *that it was Python*, so a reader had to infer the runtime from the shape of a path | +| `modelId` (required) | `labels`, optional | Free-form and never read by Scrollcase. `"modelId": "example-org/m"` becomes `"labels": { "model": "example-org/m" }`, or nothing at all — both were required and neither was ever read by any code path, so a box packaging a library still had to name a model | +| `runtimeId` (required) | `labels`, optional | Same. It never named a runtime in the version 3 sense — that is `runtime.id` | +| `modelCacheSubdir` | `cacheSubdir` | Same meaning, same default shape (`cache/`). The directory holds whatever the box's large files are, which need not be a model | +| `assetBaseUrl` | `publishBaseUrl`, and now optional | Renamed for what it does: it never touched an asset, only the two links between the signed documents. See [`publishBaseUrl`](/reference/scroll#publishbaseurl) | +| `weights: "embed"` | nothing — `embed` defaults to `true` | Delete the field | +| `weights: "on-demand"` | `"embed": false` on **each** asset entry | Per entry now, so one box may embed a small entry point and defer a large dataset — the case the box-wide switch existed for and could not serve | +| `selfTest.pythonCode` | `selfTest.code` | Extra source in the runtime's own language, still builder-only | +| `selfTest.pythonFile` | `selfTest.script` | Still a project path, still read at build time | +| `selfTest.imports` | `selfTest.imports` | Unchanged in the scroll — but see below, it is the *signed* subset that was renamed | +| `execution.kind: python-script` / `python-module` | unchanged | Joined by `node-script` and `native-binary` | + +Everything not listed is unchanged: `boxId`, `scrollId`, `scrollVersion`, `version`, +`sourceRevision`, `target`, `compatibility`, `environment`, `assetArchives`, `prunePaths`, +`uncompressedPaths`, `pixiVersion`, `parity`, `condaDependencyLicenseAudit`, `extends`, and every +existing field of an `assets[]`, `localFiles[]` or `execution` entry. The three collections gained +fields rather than losing any: `embed` and `executable` on an asset, `executable` on a local file, +and two more `execution.kind` values. + +Two removals with no replacement, because the case they existed for became impossible rather than +renamed: + +- **`assetArchives` has no `embed`.** An archive is expanded at build time, so deferring one would + name nothing that could happen. Version 2 refused that combination across fields; version 3 cannot + express it. +- **`--weights` is gone from the CLI**, not renamed. See the flag table below. + +## The signed documents + +These are what a build emits, not what you write — they are here so a consumer reading a release +manifest or a `box.json` knows what moved. The two carry the same fields under the same names and +changed together; only the release has an `archive` block. + +| Version 2 | Version 3 | Notes | +| --- | --- | --- | +| `pythonEntryPoint` | `runtime.entryPoint` | Inside `runtime`, beside `id` and `version` | +| `modelId`, `runtimeId` | `labels` | Optional, free-form, carried through untouched | +| `modelCacheSubdir` | `cacheSubdir` | | +| `weights` | `assets[].embed` | A deferred asset carries its descriptor in the release, exactly as before | +| `selfTest.pythonImports` | `selfTest.probe.imports` | `selfTest.probe` carries `imports`, `commands`, or both. The old name put Python syntax in the wire format and gave a runtime with no module system no way to state a check at all | +| — | `selfTest.probe.commands` | New: invocations of the box's own declared execution, each with the exit status it must produce. The only probe shape a `native` box can answer | +| `provenance.pythonVersion` | `provenance.runtimeVersion` | May be absent, for a runtime with no version to record | +| `archive.url` (required) | `archive.url` (optional) | See below | +| — | `bundledLicenses` | See below | + +Every schema `$id` and `$ref` moved from `/schema/v2/` to `/schema/v3/`, so anything validating a +document against a pinned URL needs that URL changed too. The published copies are listed in +[JSON Schemas](/reference/schemas). + +## The CLI + +Five flags were renamed or removed. Nothing else that existed in version 2 changed; the rest of the +difference is flags that are new. + +| Version 2 | Version 3 | Notes | +| --- | --- | --- | +| `new scroll --python-version ` | `new scroll --runtime-version ` | Refused for `--runtime native`, which installs no interpreter | +| `new scroll --model-id `, `--runtime-id ` | `new scroll --labels '{"model":"…"}'` | One JSON object in place of two required identities Scrollcase never read | +| `new scroll --asset-base-url ` | `new scroll --publish-base-url ` | Optional now: press Enter past the prompt, and a box you only run locally never needs one | +| `build --asset-base-url ` | `build --publish-base-url ` | Overrides the scroll's `publishBaseUrl` | +| `new scroll --weights`, `build --weights` | **removed, not renamed** | A build-time override of a per-asset declaration repacks a box under an identity that no longer describes it. The scroll's `assets[].embed` is what a build uses, and the only place the choice is stated. At authoring time, `add asset --on-demand` writes `"embed": false` for one asset | + +`new scroll` also gained `--runtime `, which decides which execution kinds are offered, which +starter files are written and which dependency the generated `pixi.toml` declares; and `add` gained +`--executable`, `--pin`, and `--expect-exit-code` for the new declarations below. The full current +surface is the [CLI reference](/reference/cli). + +## Three things version 2 could not say + +These have no version 2 equivalent to map from. They are the reason a v2 scroll sometimes needs more +than a rename. + +### `assets[].executable` and `localFiles[].executable` + +Version 2 gave a payload file the executable bit through a `venv/bin` heuristic, which was the only +mechanism there was. HTTP carries content and not permissions, so a downloaded asset arrived with +none, and a local file is copied rather than moved — a box simply could not ship an asset that runs. +Declare it now: + +```jsonc +"assets": [ + { + "url": "https://tools.example.org/mytool-1.4.0-linux-x86_64", + "relativePath": "bin/mytool", + "sizeBytes": 8421376, + "sha256": "…", + "executable": true + } +] +``` + +The bit is *synthesised* into the archive from this declaration, never read off the build machine, +so two builds of one commit stay byte-identical whatever umask each ran under. Whatever a box +actually starts is checked for it before the archive is written — a `native-binary` a scroll brought +in therefore needs `"executable": true`, or the build refuses. + +### `bundledLicenseDeclaration` + +`pixi.lock` declares a licence per conda package, but it cannot see what was linked *inside* a +binary your scroll supplies, and reading the binary would be guessing. So that half is declared: + +```jsonc +"bundledLicenseDeclaration": "legal/bundled-dependencies.json" +``` + +pointing at a reviewed JSON array of `{ name, version, declaredLicense, linkedInto }` entries. The +build checks that every `linkedInto` path is a file the box really carries, signs the list into the +release and `box.json` as `bundledLicenses`, and writes it to +`THIRD_PARTY_NOTICES/bundled-dependencies.json`. It travels in the release because a licence +decision is made before an archive is downloaded. Its absence means the project declared none, never +that the box has none. See [bundled licences](/reference/scroll#bundled-licences). + +### `archive.url` is optional + +Version 2 required a URL in every release, so an author who only wanted to run a box on their own +machine had to invent an address — while Scrollcase declined to invent one itself, on the grounds +that a placeholder in a signed release is a false statement. Both could not be right. + +A build given no publish location now omits `archive.url` and the channel entry's +`releaseManifestUrl` instead of refusing. Nothing is lost but the address: an archive is verified by +`sha256` and size, and all three consumers find it beside its release document rather than by +following a link. What an unpublished box gives up is the chain a downloader follows, which it has +no use for. + +## A worked example + +The shortest v2 scroll that built: + +```json +{ + "$schema": "https://scrollcase.dev/schema/v2/scroll.schema.json", + "schemaVersion": 2, + "boxId": "hello-box", + "modelId": "example-org-hello", + "runtimeId": "hello-box-runtime", + "version": "1.0.0", + "sourceRevision": "example-hello-v1", + "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, + "pythonVersion": "3.14", + "pixiVersion": "0.73.0", + "assetBaseUrl": "https://assets.example.org/boxes", + "selfTest": { "imports": ["json", "sqlite3"] } +} +``` + +The same box in version 3. Two fields are gone rather than renamed, because nothing ever read them: + +```json +{ + "$schema": "https://scrollcase.dev/schema/v3/scroll.schema.json", + "schemaVersion": 3, + "boxId": "hello-box", + "version": "1.0.0", + "sourceRevision": "example-hello-v1", + "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, + "runtime": { "id": "python", "version": "3.14" }, + "pixiVersion": "0.73.0", + "publishBaseUrl": "https://assets.example.org/boxes", + "selfTest": { "imports": ["json", "sqlite3"] } +} +``` + +If `modelId` was carrying something a person needs — the upstream model a box packages, the team +that owns it — put it in `labels`, where it is signed into the release and still never read by +Scrollcase: + +```jsonc +"labels": { "model": "example-org/hello", "owner": "platform-team" } +``` + +## The order to do it in + +1. **Edit the scroll by hand**, using the first table. `edit scroll` is not the tool for this: it + changes one existing field from a menu built out of the schema, and the fields you are moving + away from are not in the v3 schema to be offered. It is worth running afterwards, though, for + anything you now want to change. +2. **`scrollcase audit `** — the cheapest command that reads the scroll. Every command that + reads one validates it against the v3 schema first and names the field it cannot accept, and + `audit` derives its inventory from the committed lock without building anything or touching the + network. Catching the renames here beats discovering them one build at a time. +3. **`scrollcase lock `**, if the dependency solve changed. It usually has not: version 3 + changed the format, not the substrate. +4. **`scrollcase audit --write`**, if you carry a reviewed licence audit — the inventory is + derived from the lock, and a reviewed audit must still match it. +5. **`scrollcase build `**, then **`scrollcase verify`** on the result. A box that built under + version 2 and refuses to build now is worth reading carefully rather than working around: the + refusals version 3 added — a declared entry point on a `native` box, an `imports` probe on a + runtime with no module system, a missing executable bit — each name something the box could not + have done anyway. +6. **Publish, and leave the v2 documents where they are.** They are immutable and still valid for + the Scrollcase version that produced them. diff --git a/docs/reference/api/node.md b/docs/reference/api/node.md index bfeaf57..9cf10ec 100644 --- a/docs/reference/api/node.md +++ b/docs/reference/api/node.md @@ -278,26 +278,29 @@ boxTargetId({ platform: 'linux', arch: 'x86_64', accelerator: 'cuda', cudaVersio | `schemaUrl` | `(name) => URL` | Absolute URL of a shipped JSON Schema | | `fixtureUrl` | `(name) => URL` | Absolute URL of a shipped fixture | -Constants: `BOX_SCHEMA_VERSION` (`2`), `PAYLOAD_ENCODING` (`'base64-json-utf8'`), +Constants: `BOX_SCHEMA_VERSION` (`3`), `PAYLOAD_ENCODING` (`'base64-json-utf8'`), `SIGNATURE_ALGORITHM` (`'ed25519'`), `DEFAULT_DOCUMENT_NAMESPACE` (`'scrollcase.box'`), `CHANNELS` (`['nightly', 'beta', 'stable']`). ## `scrollcase/contract/browser` The platform-neutral subset of the contract for browsers, Workers, and Node. It exports the target -helpers plus document constants, namespacing helpers, and `isSignedBoxDocument`. Its complete module -graph contains no Node built-ins. +helpers, the whole runtime model, the document constants, the namespacing helpers, and +`isSignedBoxDocument`. Its complete module graph contains no Node built-ins. ```js import { boxTargetId, + runtimeAdapter, isSignedBoxDocument, } from 'scrollcase/contract/browser'; ``` -The full `scrollcase/contract` entry point remains the Node surface and additionally exports -`decodeDocumentPayload`, `schemaUrl`, and `fixtureUrl`. Cryptographic verification remains under -`scrollcase/sign`; the browser guard checks envelope shape only and never establishes trust. +The full `scrollcase/contract` entry point remains the Node surface and exports exactly three things +more: `decodeDocumentPayload`, which needs Node's `crypto` to hash a payload, and `schemaUrl` and +`fixtureUrl`, which resolve a file beside the installed package rather than anything a browser can +fetch. Cryptographic verification remains under `scrollcase/sign`; the browser guard checks envelope +shape only and never establishes trust. ::: warning Decoding is not verifying `decodeDocumentPayload` catches a truncated or edited document, because the payload hash must diff --git a/docs/reference/box-format.md b/docs/reference/box-format.md index a383757..cc8dc42 100644 --- a/docs/reference/box-format.md +++ b/docs/reference/box-format.md @@ -400,14 +400,34 @@ payload encoding, signature algorithm, or golden fixture. ### What version 3 changed -| Version 2 | Version 3 | Why | -| --- | --- | --- | -| `modelId`, `runtimeId` (both required) | `labels`, optional and free-form | Neither was ever read by any code path. They were a consumer's vocabulary in the format: a box packaging a library still had to name a model | -| `pythonVersion`, `pythonEntryPoint` | `runtime: { id, version, entryPoint }` | A box said *where its Python was* and never *that it was Python*. A reader had to infer the runtime from the shape of a path | -| `provenance.pythonVersion` | `provenance.runtimeVersion` | Same reason, and it may now be absent for a runtime that has no version to record | -| `modelCacheSubdir` | `cacheSubdir` | The directory holds whatever the box's large files are | -| `weights: embed \| on-demand` | `assets[].embed`, per entry | A box-wide switch could not ship a small entry point and defer a large dataset. `--weights` went with it: a build-time override of a per-asset declaration repacks a box under an identity that no longer describes it | -| `selfTest.pythonImports` | `selfTest.probe` with `imports` and `commands` | Python syntax in the wire format. A runtime with no module system could not state a check at all | -| Executable bit from a `venv/bin` heuristic | `assets[].executable`, `localFiles[].executable` | A downloaded file arrives with no permissions, so a box could not ship one that runs | -| `python-script`, `python-module` | plus `node-script`, `native-binary` | Named once, so implementing the runtimes was code rather than another wire break — which is exactly how `node` and `native` arrived | -| — | `bundledLicenses`, optional | The licences of what was linked *inside* a binary the box ships. `pixi.lock` cannot see them, so they are declared rather than derived, and signed so a licence decision can be made before an archive is downloaded | +Six things, and one addition: + +1. **A box declares its runtime.** `runtime: { id, version, entryPoint }` replaces `pythonVersion` + and `pythonEntryPoint`. A version 2 box said *where its Python was* and never *that it was + Python*, so a reader had to infer the runtime from the shape of a path. Fixing the vocabulary — + `python`, `node`, `native` — is what let the other two runtimes arrive as code rather than as a + second wire break. +2. **`modelId` and `runtimeId` became `labels`**, optional and free-form. Both were required and + neither was ever read by any code path: they were a consumer's vocabulary written into the + format, so a box packaging a library still had to name a model. `modelCacheSubdir` became + `cacheSubdir` for the same reason — the directory holds whatever the box's large files are. +3. **`weights` became `assets[].embed`, per entry.** A box-wide switch could not ship a small entry + point inside the archive and defer a large dataset beside it, which is the case it existed for. +4. **The self-test generalised.** `selfTest.pythonImports` put Python syntax in the wire format and + gave a runtime with no module system no way to state a check at all. The signed subset is + `selfTest.probe`, carrying `imports`, `commands`, or both. +5. **The executable bit is declared**, through `assets[].executable` and `localFiles[].executable`, + rather than inferred from a `venv/bin` heuristic. A downloaded file arrives with no permissions, + so a box could not ship one that runs. +6. **Publishing became optional.** `assetBaseUrl` is `publishBaseUrl` — it never touched an asset, + only the links between the signed documents — and `archive.url` and a channel entry's + `releaseManifestUrl` may now be absent, because a placeholder address in a signed, immutable + document stays false forever. + +The addition is `bundledLicenses`: the licences of what was linked *inside* a binary the box ships. +`pixi.lock` cannot see them, so they are declared rather than derived, and signed so a licence +decision can be made before an archive is downloaded. + +**[Migrating from v2](/guides/migrating-from-v2) is the field-by-field mapping** — the scroll, the +signed documents, the CLI flags, and the order to do it in. It is kept in one place, so this section +says what moved and that page says what to type. diff --git a/docs/white-paper.md b/docs/white-paper.md index 12b955d..01efe69 100644 --- a/docs/white-paper.md +++ b/docs/white-paper.md @@ -1501,12 +1501,21 @@ The contract is exposed twice, and the split is load-bearing: - **`scrollcase/contract`** — the complete surface, including payload decoding, which needs Node's `crypto` for hashing. -- **`scrollcase/contract/browser`** — target identity, document naming, the constants, and the - structural envelope guard. No Node built-in is reachable from it, so it loads in a browser, in a - Worker, and in Node alike. - -A test walks the browser entry point's entire import graph and fails if any module in it reaches a -Node built-in (`tests/unit/package-surface.test.mjs`). The reason is practical: a client that only +- **`scrollcase/contract/browser`** — target identity, the [runtime](#runtime) model, document + naming, the constants, and the structural envelope guard. No Node built-in is reachable from it, + so it loads in a browser, in a Worker, and in Node alike. + +The split is subtractive, which is what gives a new contract export somewhere obvious to go: the +browser entry point carries everything the full one does except payload decoding and the two helpers +that resolve a file beside the installed package. Everything else the contract states is a statement +about names, and answers the same wherever it is asked. + +Two tests hold that line. One walks the browser entry point's entire import graph and fails if any +module in it reaches a Node built-in; the other links all five published entry points in a real Node +process, because that graph walk reads source text without evaluating it and the test runner's own +resolver forgives a re-export naming a symbol that no longer exists — which is how this entry point +spent the whole of the version 3 work pointing at a function the runtime split had renamed +(`tests/unit/package-surface.test.mjs`). The reason for the split is practical: a client that only needs to compute a [target ID](#target-id) or recognise a document `kind` should not have to bundle a hashing implementation to do it. @@ -6464,7 +6473,7 @@ consumer-only dependent avoid the entire build layer. | Export | Kind | Meaning | | --- | --- | --- | -| `BOX_SCHEMA_VERSION` | constant | `2` — the only format version this release reads or writes | +| `BOX_SCHEMA_VERSION` | constant | `3` — the only format version this release reads or writes | | `CHANNELS` | constant | The closed vocabulary: `nightly`, `beta`, `stable` | | `DEFAULT_DOCUMENT_NAMESPACE` | constant | `scrollcase.box`, used when a project declares none | | `PAYLOAD_ENCODING` | constant | The envelope's payload encoding identifier | diff --git a/src/contract/browser.d.mts b/src/contract/browser.d.mts index d47c09b..3ae414f 100644 --- a/src/contract/browser.d.mts +++ b/src/contract/browser.d.mts @@ -1,2 +1,3 @@ export { assertNativeHost, condaSubdir, pixiAccelerator, boxTargetAdapter, boxTargetAdapters, boxTargetId } from "./targets.mjs"; +export { RUNTIME_IDS, assertRuntimeEntryPoint, executionAffectingVariables, isExecutablePayloadPath, isImplementedRuntime, runtimeAdapter, runtimeAdapters, unimplementedRuntimeMessage } from "./runtimes.mjs"; export { CHANNELS, DEFAULT_DOCUMENT_NAMESPACE, PAYLOAD_ENCODING, BOX_SCHEMA_VERSION, SIGNATURE_ALGORITHM, documentKinds, isSignedBoxDocument, parseDocumentKind } from "./document-shape.mjs"; diff --git a/src/contract/browser.mjs b/src/contract/browser.mjs index 1141613..deb539a 100644 --- a/src/contract/browser.mjs +++ b/src/contract/browser.mjs @@ -1,14 +1,20 @@ /** * Browser-safe reference helpers for the Scrollcase contract. * - * The full `scrollcase/contract` entry point also decodes and hashes signed payloads through Node's - * crypto implementation. Consumers that only need target identity, document names, constants, or - * the structural envelope guard can use this entry point in browsers, Workers, and Node alike. + * The rule for what belongs here is subtractive, so a new contract export has somewhere obvious to + * go: this entry point carries everything `scrollcase/contract` does *except* what needs Node's + * crypto to hash a payload, and except the two helpers that resolve a file beside the installed + * package. Everything else in the contract — target identity, the runtime model, document naming, + * the constants, the structural envelope guard — is a statement about names, and answers the same + * in a browser, a Worker, or Node. + * + * The runtime model is here for that reason rather than by parity: `runtimes.mjs` reads no file, + * joins no host path and starts no process, and a UI validating a box document has the same + * questions to ask of it as the builder does. */ export { assertNativeHost, - assertPythonEntryPoint, condaSubdir, pixiAccelerator, boxTargetAdapter, @@ -16,6 +22,17 @@ export { boxTargetId, } from './targets.mjs'; +export { + RUNTIME_IDS, + assertRuntimeEntryPoint, + executionAffectingVariables, + isExecutablePayloadPath, + isImplementedRuntime, + runtimeAdapter, + runtimeAdapters, + unimplementedRuntimeMessage, +} from './runtimes.mjs'; + export { CHANNELS, DEFAULT_DOCUMENT_NAMESPACE, diff --git a/tests/unit/package-surface.test.mjs b/tests/unit/package-surface.test.mjs index 3731f53..35b2e94 100644 --- a/tests/unit/package-surface.test.mjs +++ b/tests/unit/package-surface.test.mjs @@ -1,16 +1,18 @@ /** * What a consumer of the published package actually gets. * - * Three failures this catches that nothing else does. First, an `exports` map that names a file which + * Four failures this catches that nothing else does. First, an `exports` map that names a file which * moved or was never shipped: every other test imports by relative path, so the package could be - * broken for everyone installing it while the suite stayed green. Second, generated types drifting - * from the schemas they are a projection of — the schemas are the source of truth, and a type that - * disagrees with them is worse than no type at all. Third, runtime declarations that exist but - * silently widen the JavaScript API to `any`. + * broken for everyone installing it while the suite stayed green. Second, an entry point that + * resolves but will not *link* — a re-export naming a symbol its source no longer has, which this + * runner forgives and `node` refuses. Third, generated types drifting from the schemas they are a + * projection of — the schemas are the source of truth, and a type that disagrees with them is worse + * than no type at all. Fourth, runtime declarations that exist but silently widen the JavaScript + * API to `any`. */ import { createRequire } from 'node:module'; -import { execFileSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join, relative } from 'node:path'; @@ -106,6 +108,47 @@ describe('the package surface', () => { expect(typeof sign.verifySignedDocument).toBe('function'); }); + /** + * The same five entry points, loaded by Node itself. + * + * The case above imports them too, and it is not enough: it runs inside Vitest, whose resolver + * links a module graph its own way and forgave a re-export naming a symbol that no longer existed. + * `scrollcase/contract/browser` re-exported `assertPythonEntryPoint` from `targets.mjs` for the + * whole of the version 3 work — the runtime split had moved it and renamed it — and every one of + * the three things that should have caught it looked elsewhere. The import above resolved. The + * closure scan below is a regular expression over source text, which reads specifiers and never + * evaluates a module. And `tsc` omits an unresolvable re-export from the generated `.d.mts` + * without a word, so `types:check` reported no drift. Under `node`, linking that module is a + * SyntaxError before a single statement runs, which is what a dependent would have met. + * + * A separate process per subpath, resolved through the package name so the `exports` map is what + * decides which file is loaded — the same walk `import 'scrollcase/…'` performs for a dependent. + */ + it('links every entry point in a real Node process, through the exports map', () => { + const specifiers = Object.entries(packageJson.exports) + .filter(([, entry]) => typeof entry === 'object' && entry.import) + .map(([subpath]) => `scrollcase${subpath.slice(1)}`); + expect(specifiers).toEqual([ + 'scrollcase/contract', + 'scrollcase/contract/browser', + 'scrollcase/build', + 'scrollcase/consumer', + 'scrollcase/sign', + ]); + + for (const specifier of specifiers) { + // An entry point that links but exports nothing is the other half of the same failure: it + // resolves, it evaluates, and a dependent gets an empty namespace object. + const source = `const module = await import(${JSON.stringify(specifier)}); + if (Object.keys(module).length === 0) throw new Error('${specifier} exports nothing');`; + const result = spawnSync(process.execPath, ['--input-type=module', '--eval', source], { + cwd: repoRootPath, + encoding: 'utf8', + }); + expect(result.status, `${specifier} failed to link under node:\n${result.stderr}`).toBe(0); + } + }); + it('ships a browser-safe contract helper graph with no Node built-ins', async () => { await expect(moduleClosure('src/contract/browser.mjs')).resolves.toBeInstanceOf(Set); }); diff --git a/tests/unit/project-surface.test.mjs b/tests/unit/project-surface.test.mjs index 36629ce..40c381f 100644 --- a/tests/unit/project-surface.test.mjs +++ b/tests/unit/project-surface.test.mjs @@ -182,7 +182,7 @@ describe('auditing dependency licences', () => { runtimeVersion: '3.11.15', pixiVersion: '0.73.0', compatibility: { minHostAppVersion: '1.0.0' }, - assetBaseUrl: 'https://assets.example.org', + publishBaseUrl: 'https://assets.example.org', executionKind: 'library-only', }); const scroll = JSON.parse(await readFile(join(result.scrollDir, 'scroll.json'), 'utf8'));