diff --git a/.github/workflows/layered-workspaces.yml b/.github/workflows/layered-workspaces.yml index a1287094..4684b5cf 100644 --- a/.github/workflows/layered-workspaces.yml +++ b/.github/workflows/layered-workspaces.yml @@ -14,6 +14,9 @@ on: - "scripts/verify-windows-command-recipe-sandbox.ps1" - "scripts/verify-macos-nfs-framework-layers.sh" - "scripts/verify-environment-adapter-plugin.sh" + - "scripts/verify-artifact-adapter-conformance.sh" + - "scripts/verify-artifact-real-tool-gates.sh" + - "scripts/verify-artifact-native-cow-matrix.py" - "scripts/verify-windows-environment-adapter-plugin.ps1" - "trail-environment-adapter-sdk/**" - ".github/workflows/layered-workspaces.yml" @@ -46,10 +49,34 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 + - run: cargo test -p trail-environment-adapter-sdk --locked - run: cargo test -p trail view_core_ --no-default-features - if: ${{ runner.os != 'Windows' }} run: cargo test -p trail workspace_layer --no-default-features + artifact-common-contract: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: scripts/verify-artifact-adapter-conformance.sh + env: + CARGO_TARGET_DIR: ${{ runner.temp }}/trail-artifact-common-contract-target + + artifact-real-tool-gates: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: sudo apt-get update && sudo apt-get install -y fuse3 + - run: sudo chmod 666 /dev/fuse + - run: node --version && npm --version && cmake --version && python3 --version + - run: scripts/verify-artifact-real-tool-gates.sh + env: + TRAIL_RUN_FUSE_COW_TESTS: "1" + fuse-conformance: runs-on: ubuntu-latest steps: @@ -62,6 +89,9 @@ jobs: - run: cargo test -p trail fuse_adapter_runs_shared_mounted_view_suite -- --nocapture env: TRAIL_RUN_FUSE_COW_TESTS: "1" + - run: cargo test -p trail fuse_mount_reads_artifact_manifest_without_layer_materialization -- --nocapture + env: + TRAIL_RUN_FUSE_COW_TESTS: "1" - run: cargo test -p trail real_cmake_configure_build_and_clean_stay_lane_private -- --nocapture env: TRAIL_RUN_FUSE_COW_TESTS: "1" @@ -77,6 +107,19 @@ jobs: - run: cargo test -p trail sync_all_initializes_nested_python_components_at_final_lane_paths -- --nocapture env: TRAIL_RUN_FUSE_COW_TESTS: "1" + - name: Qualify CAS-backed FUSE artifact lifecycle + run: >- + python3 scripts/verify-artifact-native-cow-matrix.py + --output "${{ runner.temp }}/artifact-native-cow/fuse-linux.json" + env: + TRAIL_RUN_FUSE_COW_TESTS: "1" + CARGO_TARGET_DIR: ${{ runner.temp }}/trail-artifact-native-cow-target + - uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: artifact-native-cow-fuse-linux + path: ${{ runner.temp }}/artifact-native-cow/fuse-linux.json + if-no-files-found: error rust-cache-isolation: runs-on: ubuntu-latest @@ -132,6 +175,9 @@ jobs: - run: cargo test -p trail nfs_adapter_runs_shared_mounted_view_suite -- --nocapture env: TRAIL_RUN_NFS_COW_TESTS: "1" + - run: cargo test -p trail nfs_mount_reads_artifact_manifest_without_layer_materialization -- --nocapture + env: + TRAIL_RUN_NFS_COW_TESTS: "1" - run: cargo test -p trail real_nfs_mount_records_new_modified_and_renamed_files -- --nocapture env: TRAIL_RUN_NFS_COW_TESTS: "1" @@ -163,6 +209,19 @@ jobs: - run: cargo test -p trail sync_all_initializes_nested_python_components_at_final_lane_paths -- --nocapture env: TRAIL_RUN_NFS_COW_TESTS: "1" + - name: Qualify CAS-backed NFS artifact lifecycle + run: >- + python3 scripts/verify-artifact-native-cow-matrix.py + --output "${{ runner.temp }}/artifact-native-cow/nfs-macos.json" + env: + TRAIL_RUN_NFS_COW_TESTS: "1" + CARGO_TARGET_DIR: ${{ runner.temp }}/trail-artifact-native-cow-target + - uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: artifact-native-cow-nfs-macos + path: ${{ runner.temp }}/artifact-native-cow/nfs-macos.json + if-no-files-found: error nfs-framework-bench: if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_macos_framework_bench }} @@ -185,6 +244,7 @@ jobs: run: | $env:TRAIL_RUN_DOKAN_COW_TESTS = "1" cargo test -p trail dokan_adapter_runs_shared_mounted_view_suite -- --nocapture + cargo test -p trail dokan_mount_reads_artifact_manifest_without_layer_materialization -- --nocapture cargo test -p trail foreground_dokan_mount_stops_through_a_separate_trail_handle -- --nocapture cargo test -p trail daemon_owned_dokan_mount_returns_ready_and_unmounts_asynchronously -- --nocapture cargo test -p trail real_windows_cmake_build_and_clean_stay_lane_private -- --nocapture @@ -194,12 +254,39 @@ jobs: cargo test -p trail sync_all_initializes_nested_python_components_at_final_lane_paths -- --nocapture - run: ./scripts/verify-windows-command-recipe-sandbox.ps1 - run: ./scripts/verify-windows-environment-adapter-plugin.ps1 + - name: Qualify CAS-backed Dokan artifact lifecycle + shell: pwsh + run: >- + python scripts/verify-artifact-native-cow-matrix.py + --output "${{ runner.temp }}/artifact-native-cow/dokan-windows.json" + env: + TRAIL_RUN_DOKAN_COW_TESTS: "1" + CARGO_TARGET_DIR: ${{ runner.temp }}/trail-artifact-native-cow-target + - uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: artifact-native-cow-dokan-windows + path: ${{ runner.temp }}/artifact-native-cow/dokan-windows.json + if-no-files-found: error million-path-scale: if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_scale }} runs-on: ubuntu-latest + env: + CARGO_TARGET_DIR: /Volumes/Workspace/crabbuild-target/trail-layered-scale + TRAIL_SCALE_EVIDENCE_DIR: /Volumes/Workspace/trail-layered-scale-evidence steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 + - name: Prepare isolated scale storage + run: | + sudo mkdir -p "$CARGO_TARGET_DIR" "$TRAIL_SCALE_EVIDENCE_DIR" + sudo chown -R "$(id -u):$(id -g)" /Volumes/Workspace - run: scripts/verify-layered-lane-scale.sh + - uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: layered-lane-scale-evidence + path: /Volumes/Workspace/trail-layered-scale-evidence + if-no-files-found: error diff --git a/CHANGELOG.md b/CHANGELOG.md index ce7af967..1a76b12b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,178 @@ All notable changes to Trail are documented in this file. Trail follows ## [Unreleased] +### Added + +- Added deterministic 10k/100k/1M artifact and source scale matrices for + 1/5/20 lanes, fail-closed JSON evidence validation, and compositional + owning-host NFS/FUSE/Dokan qualification that distinguishes mounted backend + behavior from shared CAS lifecycle checks and leaves unavailable platforms + explicitly unverified. +- MCP resource completion now returns bounded artifact-envelope and quarantine + identities for the artifact resource templates. + +- Added one behavior-based artifact conformance fixture for reviewed built-ins, local + protocol-v3 plugins, and repository-v2 producers, plus a deterministic evidence-only + JSON certification report covering discovery through last-reference collection. +- Added one blocking Linux real-tool artifact gate covering Cargo and npm resolution, + compiled seed reuse, framework composition, Python/CMake private state, external + metadata, custom repository pipelines, and guarded source export. +- Managed exec, test, eval, terminal-agent, and materialized ACP execution now fail + before launch when an environment requires explicit resolution, pin exact source, + snapshot, generation, and artifact-binding identities, and return additive + preparation/finalization receipts with deterministic sealing and cleanup decisions. +- Added HTTP/OpenAPI and MCP tool/resource parity for environment resolution, + artifact inspection/verification/reachability/accounting, quarantine + list/show/resolve, and explicit generated-source export. All transports use + the shared Rust reports, and MCP operations declare read-only, destructive, + or open-world risk consistently. +- Added `trail env resolve all|component`, artifact inspect/verify/quarantine, + and explicit source-export CLI workflows with exact discovery recovery argv, + deterministic human/plain rendering, JSON/NDJSON reports, and stable failure + exits. Reviewed built-in resolvers execute exact offline argv in isolated + staging and retain fenced failure evidence; restricted repository/plugin + resolver launch still fails closed pending its native sandbox integration. +- Added shared Rust artifact operations and serializable reports for envelope inspection, + attach/sample/full/reproducibility-evidence verification, generation bindings, + quarantine list/show/resolve, bounded content reachability, and CAS-aware workspace and + per-envelope storage accounting. Existing resolution and source-export operations use + the same public model family. +- Protocol-v2 plugins can now declare framework-neutral `verified_external` store + identities by provider, safe reference, SHA-256 digest, and platform without creating + layers or cleanup ownership; repository-v2 conformance fixtures cover JVM-like + dependency/private-state and unknown custom pipelines without new core framework modes. + +- Normalized environment plans now carry a host-owned protocol-v3 identity-contract + digest alongside the exact legacy workspace-layer key. Go cache contracts exclude + machine-local cache paths, CMake remains layer-free and lane-private, OCI/runtime + declarations remain external metadata, and repository v2 keeps its independent + desired-key v2 identity. +- Python environments can now bind an optional uv-generated, hash-bearing requirements + snapshot, warm a performance-only wheel/download cache, and still keep `.venv`, its + bytecode, and embedded path state entirely lane-private. +- `trail.environment/v2` framework fixtures now compose Next.js and Vite build/state + components over the Node dependency component: `.next` and `.vite` remain lane-private, + while validated Vite `dist` content can use an independently keyed immutable layer. +- Manifest-only Node components can now resolve manager-specific npm, pnpm, Yarn, or + Bun lock snapshots into Trail metadata, then reuse the existing frozen-install + dependency seed and performance-only content cache with lane-private COW writes. +- Cargo components without a source-tracked `Cargo.lock` are now reported as resolvable; + a verified Trail-managed lock snapshot can be projected into isolated staging for + real `cargo build --locked --offline` target-seed construction without entering source. +- Artifact-v2 outputs can now replace a workspace layer's compatibility CAS shadow + with desired-key authority, activate through ordinary lane generations, remain + isolated through copy-on-write execution, export their immutable source, and release + generation bindings during retirement and collection. +- Added Rust library artifact resolution component/batch operations with durable fenced + attempts, content-addressed snapshot reuse, explicit-only refresh, bounded redacted + evidence, and deterministic reports. +- Workspace-layer singleflight now records durable generation-fenced owner phases and + waiter outcomes, and only recovers a lock when the exact PID/start identity is proven + dead or mismatched. +- Workspace open now recovers dead artifact constructors and exact owned staging; + doctor and fsck validate raw CAS objects, snapshots, envelopes, attempt coherence, + legacy/CAS layouts, and orphan materializations with repair guidance. +- Backup/restore validation now treats omitted materialization caches as disposable, + rebases restored layer paths before publication, and parallel environment builders + use a bounded SQLite wait during short WAL publication overlap. +- Environment discovery now reports marker-recognized plugins that do not support the + current host as typed `unsupported` proposals without launching plugin code. +- Native lane views now resolve verified immutable artifact manifests lazily, read only + requested blob/chunk ranges, and materialize only touched files during copy-up while + preserving shared FUSE, NFS, and Dokan upper/whiteout semantics. +- Real-directory artifact consumers now reuse tree-root/backend-keyed verified + materialization caches that rebuild from authoritative CAS, restore immutable + permissions on reuse, and clone/reflink or independently copy into mutable state. +- Lane forks now inherit only individually verified CAS-backed outputs after desired-key, + envelope/tree, current adapter package, scope, portability, and backend checks, while + allocating fresh artifact bindings and private workspace identities. +- Portable backups now retain source uppers and authoritative artifact snapshots, + objects, envelopes, attestations, historical generations, and exact bindings while + reporting omitted materializations and performance caches as rebuildable. +- Object GC now traces artifact envelopes through deterministic directory, file, + blob, chunk-list, and chunk edges from generation, attempt, snapshot, attestation, + quarantine, hold, layer, and materialization roots, then reclaims last-reference + content in restartable deterministic batches. +- Lane-space and cache-GC reports now expose artifact logical, unique authoritative, + cross-artifact shared, materialized, lane-private, persisted-prefetch, + demand-loaded, reclaimable, and unknown byte accounting without counting a CAS + object more than once. +- Object GC now orders unreachable artifact DAGs parent-before-child across + transaction batches, allowing an interrupted collection to reopen and resume + without leaving the remaining CAS graph invalid. +- Artifact validations now distinguish structural, loadability, framework, + policy, gate, and reproducibility declarations and produce deterministic, + secret-rejected receipts bound to the exact desired identity and tree. +- Workspace-layer publication now rechecks exact construction pins, freezes and + rescans Trail-owned candidate output, and requires structural and policy host-seal + receipts before a ready artifact envelope can be published or attached. +- Artifact producers now use a host-selected phase/trust-tier capability ceiling for + reviewed built-ins, certified signed plugins, locally trusted plugins, and repository + declarations; signatures authenticate origin without implicitly elevating authority. +- Secret-consuming artifact producers now carry typed non-secret taint evidence; + resolver candidates stay out of shared CAS, runtime-secret generations cannot promote + private output, and producer receipts are rejected if tainted or sensitive while + bounded failure evidence remains exact-value redacted. +- Ready artifact envelopes now receive deterministic content-addressed host attestations + with typed producer, capability, policy, validation, portability, and taint evidence; + inspection and attachment verification detect state/signature tampering and recheck + current plugin package and publisher revocation. +- Resolver plans now fail before attempt publication when paths, arguments, or declared + resource limits exceed host ceilings; native command-recipe tests also prove nested + child execution remains denied. +- Repository environment parsing now recognizes an explicit `trail.environment/v2` + header without changing v1 command semantics, and rejects mixed schema versions across + one local include/profile graph. +- Version-2 repository documents now retain typed resolver, action-phase, validation, + capability, heterogeneous-output, and source-export declarations with strict nested + unknown-field rejection; v1 documents cannot opt into those fields implicitly. +- Repository v2 pipelines now compile into Trail's shared discovery, resolution, + component-graph, desired-key v2, output, validation, and source-export models instead + of introducing a parallel framework-specific execution representation. +- Repository v2 loading now bounds and canonicalizes argv, inputs, authorities, actions, + validations, and exports, and rejects shells/control flow, indirect child launchers, + absolute host paths, raw secrets, provider sockets, forbidden executable phases, + capability escalation, compatible reuse, and host-wide reuse before tool resolution. +- Source-export planning now pins the lane/source and active generation, desired and + artifact identities, exact file-or-directory subtree, destination content state, + collision policy, validation and gate receipts, and explicit authorization without + writing source or materializing the artifact. +- Source-export execution now revalidates every plan pin, reads bounded regular files + directly from CAS, and applies fail/replace semantics through one normal guarded lane + patch so ignore, secret, path, collision, diff, checkpoint, and Git-handoff behavior + stays identical to ordinary source changes; artifact mounting is never a write path. +- Repository-pipeline compatibility coverage now snapshots the v2 source-export wire + contract while exercising v1 planning, include/profile cycles, unsafe and + secret-capable declarations, stale and conflicting destinations, ignored paths, a + custom command framework, and visibility of exported source in the normal lane diff. +- The environment-adapter SDK now defines separate bounded protocol-v3 request, + response, proposal, resolution, typed-phase, validation, capability, identity, + source-export, attestation-requirement, secret-taint, and quarantine-evidence types + without changing v1/v2 wire layouts or granting adapters host mutation authority. +- Adapter protocol negotiation now selects the highest exact mutual identity, and + canonical v1/v2 projections keep every v3-only authority absent. The SDK adds a v3 + pipeline builder, explicit deny-by-default package capability declarations, detailed + validation errors, and an artifact-pipeline example adapter. +- Trail now repeats canonical protocol-v3 validation at the plugin trust boundary, + rejecting oversized or unknown data, duplicate IDs, non-normalized paths, input and + host pin drift, invalid graph/phase combinations, secret-taint underclaims, and any + package/protocol/capability/certification mismatch before normalization. +- Plugin inspection, installation, catalog/trust, and removal reports now expose one + shared protocol-capability record with selected protocol, resolution/export and host + evidence flags, certification ceiling, content policy, and host-attestation policy; + the HTTP/OpenAPI adapter catalog projects the same typed fields. +- Protocol-v3 compatibility now has checked-in length-prefixed CBOR request/response + frames, fail-closed truncation and size-limit tests, property-based exact-negotiation + coverage, hostile host-validation cases, and an SDK contract gate on Linux, macOS, + and Windows. + ### Changed +- Cargo, Node, and Python managed-resolution consumers now use one host-owned snapshot + verifier for proposal/source/component/adapter/format, verification state, secret taint, + and content loading. Production adapters continue to publish only through the common + CAS sealing and atomic generation-activation path; existing environment synchronization + entry points remain compatibility wrappers. - Changed omitted lane workdir mode to lazy qualified transparent `auto`. - Replaced the old environment sync spellings with `trail env sync all` and `trail env sync component`. diff --git a/Cargo.lock b/Cargo.lock index f0731039..e5187150 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -699,6 +699,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fastcdc" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af40d8a8dadb92dc178569a5f5edb5f3056e98255c2de48ab5d59a52892e0c" + [[package]] name = "fastrand" version = "2.4.1" @@ -2938,6 +2944,7 @@ dependencies = [ "dokan", "dokan-sys", "ed25519-dalek", + "fastcdc", "flate2", "fsevent-sys", "fuser", @@ -2959,6 +2966,7 @@ dependencies = [ "rusqlite", "rustix", "serde", + "serde_bytes", "serde_cbor", "serde_json", "sha2", @@ -2978,6 +2986,7 @@ dependencies = [ "walkdir", "widestring 0.4.3", "winapi", + "xattr", "zip", ] @@ -2987,6 +2996,7 @@ version = "0.2.0" dependencies = [ "ed25519-dalek", "hex", + "proptest", "serde", "serde_bytes", "serde_cbor", diff --git a/Cargo.toml b/Cargo.toml index 956a0109..84da0857 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ axoupdater = { version = "0.10.0", features = ["blocking"] } bzip2 = "0.5" ed25519-dalek = "2.2" flate2 = "1.0" +fastcdc = "4.0.1" rayon = "1.10" rocksdb = "0.22" reqwest = { version = "0.12.28", default-features = false, features = ["blocking", "rustls-tls"] } @@ -100,6 +101,7 @@ terminal_size = "0.4.4" anstyle = "1.0.14" time = { version = "=0.3.36", features = ["formatting"] } walkdir = "2.5" +xattr = "1.6" zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } uniffi = "=0.31.0" wasm-bindgen = "0.2.126" diff --git a/README.md b/README.md index ae85bf64..5a3bd20b 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,43 @@ trail lane readiness fix-login trail lane merge fix-login --into main --dry-run ``` +### Source changes and disposable framework artifacts + +A layered lane keeps two kinds of state separate: + +- Source changes are normal repository-relative edits. `lane record`, review, + readiness, merge, and Git export can preserve them. +- Framework artifacts are dependency installs, compiler targets, bundles, + generated caches, and other tool output. Trail discovers them through Cargo, + Node, Next.js, Vite, repository v2, or adapter-v3 contracts; they do not enter + source history unless an explicit declared source export is authorized. + +The reusable artifact path is content addressed. Discovery reads source markers +without running tools. An incomplete proposal can be resolved to an immutable +resolution snapshot. Trail combines the pinned source closure, snapshot, tool +and action identities, platform, output policy, and validation contract into a +desired key. Successful construction produces a content root and verified +artifact envelope. Lanes attach that immutable lower by identity and receive +fresh writable private uppers, so sibling lanes share unchanged bytes while +copy-up writes and whiteouts remain isolated. Materializations are reconstructible +CAS projections, not a second source of truth. + +```sh +trail env discover fix-login +trail env plan fix-login +# Run only when discovery reports a resolvable component. +trail env resolve all fix-login +trail env sync all fix-login +trail env generation fix-login +``` + +`immutable_shared` and `immutable_seed_private` outputs can be reused under +their declared scope. `writable_private` and `disposable` output stays lane +private and is removed with the lane unless a declared, validated promotion is +authorized. A source export is different from promotion: it writes an exact +artifact subtree through ordinary source guardrails, checkpoints the resulting +source change, and makes it eligible for review and merge. + ## Why This Matters for AI Agents AI coding agents produce more than final diffs. They create attempts, diff --git a/docs/agent/troubleshooting.md b/docs/agent/troubleshooting.md index 2009ec09..0ab5db89 100644 --- a/docs/agent/troubleshooting.md +++ b/docs/agent/troubleshooting.md @@ -77,6 +77,41 @@ trail agent start codex --workdir-mode nfs-cow See [Spawn and materialize workdirs](../lanes/spawn-and-materialize-workdirs.md) for platform requirements and mount behavior. +## Environment Sync Says Resolution Is Missing + +Inspect side-effect-free discovery and use its exact recovery command: + +```sh +trail env discover +trail --format json env status +trail env resolve all +trail env sync all +``` + +Managed execution never resolves implicitly. A missing lockfile may produce a +Trail-managed immutable snapshot rather than a source edit. `--refresh` is an +explicit request to rerun resolution; omit it to reuse the current pinned +snapshot. Repository/plugin resolution fails closed when the host cannot enforce +its declared sandbox or authority profile. + +## An Artifact Is Quarantined or Fails Verification + +Do not remove CAS files or edit `.trail/index` manually. Preserve and inspect +the evidence: + +```sh +trail env artifact inspect +trail env artifact verify --level full +trail env artifact quarantine list +trail env artifact quarantine show +``` + +Quarantine means two producers claimed different content for the same desired +key/trust scope, or retained evidence is otherwise unresolved. Use +`trail env artifact quarantine resolve` only after choosing the explicit policy +shown by `--help`. If the envelope is valid but a projection is missing, rerun +`trail env sync all ` to rebuild the materialization. + ## Changes Are Missing First verify which task workdir the agent used: diff --git a/docs/concepts/storage-indexes-and-backups.md b/docs/concepts/storage-indexes-and-backups.md index b7284ce9..71c65559 100644 --- a/docs/concepts/storage-indexes-and-backups.md +++ b/docs/concepts/storage-indexes-and-backups.md @@ -59,7 +59,13 @@ trail doctor trail fsck ``` -`doctor` checks operational readiness, schema version, current branch, `.trailignore` defaults, runtime integration state, and pending approvals. `fsck` verifies structural integrity. +`doctor` checks operational readiness, schema version, current branch, +`.trailignore` defaults, runtime integration state, pending approvals, and +artifact/materialization health. `fsck` verifies structural integrity including +raw artifact object identity and edges, resolution snapshots, envelopes, +construction evidence, and owned versus orphan materializations. Reopening Trail +recovers only staging owned by a provably dead exact process fence; unknown +materializations are reported for review instead of deleted automatically. ## Backups @@ -71,11 +77,21 @@ trail backup verify /tmp/trail-backup trail backup restore /tmp/trail-backup ``` -Restore rewrites materialized lane workdir paths so they point inside the restored workspace, -re-secures the private `.trail` and `.trail/index` directories, and rotates the -changed-path filesystem identity. The next daemon-backed command rebinds the -observer to the restored host and reconciles the workspace before trusting its -incremental ledger again. +Backups retain source uppers and recovery journals plus authoritative artifact +snapshots, objects, envelopes, attestations, historical generations, and exact +generation bindings. They omit mounted projections, generated/scratch uppers, +artifact materializations, and performance caches. Create, verify, and restore +reports expose retained private bytes and the count/known bytes of omitted state +as rebuildable. Verification seals the retained private tree by normalized path, +entry type, symlink target, and file content. + +Restore rewrites lane workdir and retained-view paths so they point inside the +restored workspace, re-secures the private `.trail` and `.trail/index` +directories, retires copied active environment pointers, and rotates the +changed-path filesystem identity. The next environment sync reconstructs +materializations and caches from the retained authority. The next daemon-backed +command rebinds the observer to the restored host and reconciles the workspace +before trusting its incremental ledger again. ## Garbage Collection @@ -86,6 +102,45 @@ trail gc --dry-run trail gc ``` +Object GC now understands artifact CAS graphs. It retains content reachable +from generation bindings, layer shadows and pins, durable attempts and +resolution snapshots, attestations, quarantines, active holds, and recorded +materialization leases. It follows envelope, tree, directory, file, blob, +chunk-list, and chunk edges, so a chunk shared by several artifacts is removed +only after the last retained graph disappears. + +Collection is deterministic and restartable: unreachable artifact DAGs are +ordered parent-before-child with object-ID tie breaking, and live deletion uses +256-object transactions. Every committed batch leaves the uncollected graph +valid, so an interrupted process can reopen and resume. Corrupt or ambiguous +reachability stops the operation without treating missing evidence as +permission to delete. Run +the cache collector before object GC when you also want an unused verified +materialization to stop retaining its reconstructible tree: + +```sh +trail cache gc +trail gc +``` + +`trail lane space` and the structured `trail cache gc` report include an +`artifact_storage` object. It separates logical artifact content, authoritative +CAS bytes unique to one envelope, CAS bytes shared across envelopes, physical +materializations, lane-private allocation, demand-loaded projections, +persisted prefetch allocation, reclaimable bytes, and allocation that Trail +cannot safely attribute. These are multiple views of storage, not values to +sum: reclaimable bytes can also be materialized or demand-loaded, and logical +bytes are independent of both CAS encoding and filesystem allocation. + +Trail deduplicates authoritative bytes by object ID and logical bytes by tree +root. Hot-set prefetch currently performs bounded reads into the operating +system page cache and therefore reports zero persisted prefetch bytes. Native +clone/reflink reports leave filesystem extents under `unknown_bytes` unless the +platform can prove their ownership; they do not invent shared savings. + +Backup archives are self-contained and are created under the workspace write +lock; they do not pin the source workspace after publication. + ## Code Facts Used - Storage schema: `trail/src/db/storage/schema` diff --git a/docs/design/architecture.md b/docs/design/architecture.md index de904b87..e42e62de 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -122,6 +122,37 @@ Trail uses a hybrid storage model: The design separates durable object truth from derived indexes. If derived indexes are damaged or stale, `index rebuild` reconstructs them from reachable operation and message objects. +Framework artifacts use the same authority split rather than a parallel cache +database. Source roots and environment proposals pin repository state; +resolution snapshots preserve dependency selection as environment metadata; +desired keys identify the complete correctness contract; artifact Merkle roots +identify produced bytes; and verified envelopes bind content to output policy, +validation, trust, attestation, and provenance. Generation bindings attach ready +envelopes to lanes. Filesystem layer directories are reconstructible +materializations, while generated, scratch, runtime, and seeded writable uppers +remain lane private. + +```mermaid +flowchart LR + Source["Pinned source root"] --> Proposal["Proposal / optional resolution snapshot"] + Proposal --> Desired["Desired key"] + Desired --> Build["Host-owned construction and validation"] + Build --> Tree["CAS content root"] + Tree --> Envelope["Ready artifact envelope"] + Envelope --> Generation["Environment generation binding"] + Generation --> Lower["Lazy or verified immutable materialization"] + Lower --> Upper["Fresh lane-private upper"] + Upper -. "explicit promotion" .-> Build + Envelope -. "declared source export" .-> Source +``` + +Adapters plan this graph but never become an execution or publication +authority. Trail resolves tools, narrows capabilities, owns subprocesses and +attempt fences, validates candidates, publishes CAS objects/envelopes, activates +generations, and performs recovery and collection. NFS, FUSE, and Dokan adapt +ordinary filesystem requests to the shared manifest/upper core; they do not own +artifact identity or lifecycle decisions. + ```mermaid flowchart LR Refs["Refs
branches and agents"] --> Ops["Operation objects
parents, before_root, after_root"] @@ -176,6 +207,13 @@ Advisory leases are separate from the write lock. The write lock protects databa - Materialized workdirs should not be trusted if their manifest is missing or dirty without recording/sync. - Ignored and internal paths should not be recorded accidentally. - Derived indexes may be rebuilt, but object history and refs are the durable source of truth. +- A ready artifact envelope points to a complete validated content graph; a + materialization is never its only copy. +- A lane generation binds exact desired, envelope, tree, and output identities; + private copy-up bytes and whiteouts never alter the immutable lower. +- Resolution and construction attempts are fenced by exact owner identity. + Recovery may activate the exact prepared successor or leave collectable + unbound content, but it cannot invent a partial active generation. ## Failure Modes diff --git a/docs/design/data-model.md b/docs/design/data-model.md index 0e1f27cd..4bcfb711 100644 --- a/docs/design/data-model.md +++ b/docs/design/data-model.md @@ -199,7 +199,9 @@ Reports are deliberately not the same as storage rows. Examples: - `LaneReadinessReport` aggregates branch status, workdir state, approvals, conflicts, and gate status. - `LaneHandoffReport` bundles readiness, current session context, recent events, spans, operations, and next steps. -- `BackupRestoreReport` includes restore effects such as rewritten workdirs. +- Backup reports distinguish retained private view bytes from omitted + rebuildable materializations/performance caches; `BackupRestoreReport` also + reports rewritten workdirs and restored private views. - `GuardrailCheckReport` includes decision, reasons, path checks, approvals, and optional approval request instructions. This keeps command/API consumers from having to join internal tables themselves. diff --git a/docs/design/environment-adapter-contract.md b/docs/design/environment-adapter-contract.md index ae324e24..f231211c 100644 --- a/docs/design/environment-adapter-contract.md +++ b/docs/design/environment-adapter-contract.md @@ -26,9 +26,11 @@ generation activation. The first built-ins are: - `trail/python-venv@1`: recognizes `pyproject.toml` and the common uv, Poetry, PDM, Pipenv, and requirements lock/manifest files, provisions a layer-free lane-private `.venv`, and keys it by every present dependency file plus the resolved Python - executable. Trail automatically creates the virtual environment through an ephemeral - candidate view at the lane's final mountpoint, so scripts and prefix metadata embed - the correct absolute path without exposing partial state; + executable. An optional uv-generated hash-bearing requirements snapshot remains Trail + metadata and warms a shared performance-only wheel/download cache. Trail automatically + creates the virtual environment through an ephemeral candidate view at the lane's final + mountpoint, so scripts, bytecode, and prefix metadata stay private and embed the correct + absolute path without exposing partial state; - `trail/oci-image@1`: reads `trail.oci.toml`, accepts only lowercase SHA-256 digest-pinned OCI references with an explicit platform, and records provider-owned image identities without commands, caches, mounts, or manufactured directories; @@ -46,6 +48,22 @@ generation activation. The first built-ins are: metadata-only external-artifact declarations while the host retains mount authority and atomic generation activation. +Every normalized plan now also receives a host-owned `AdapterIdentityContractV3` +projection. Its digest enters the component key beside, rather than replacing, the exact +`WorkspaceLayerKeyV1` compatibility/storage identity. The projection records adapter +provenance, source-closure status, stable action/output policy, cache protocol and +compatibility, external/runtime declarations, platform/ABI, portability, and trust +scope. Host cache storage paths are rewritten to logical cache names before hashing, so +moving `.trail` storage cannot change artifact correctness identity. This projection +does not advertise or invoke plugin protocol v3 and grants no v3-only capability. + +The mapping deliberately preserves adapter semantics: Go's module/build stores remain +performance-only while its vendor output remains an immutable private seed; CMake's +path-bound build tree remains writable-private with no layer; OCI images and services +remain provider-owned metadata with lane-specific allocation IDs; v1 command recipes +remain conservative legacy plans, while `trail.environment/v2` recipes retain their +independently compiled `ArtifactDesiredKeyV2` and repository trust ceiling. + `trail env adapters` returns the compiled and installed adapter catalog, including versioned identity, selectors, component kind, discovery markers, implementation provenance, stability, and description without probing the repository or host tools. Discovery obtains candidate @@ -59,6 +77,25 @@ reports logical component identity separately from adapter identity. CLI, HTTP/O MCP, and Rust APIs share this state. Existing `trail deps` behavior remains a Node compatibility surface. +The Rust library and CLI share the common resolution-snapshot boundary: +`resolve_artifact_component` and `resolve_all_artifact_components` accept normalized +executor candidates, validate pinned source/tool/policy identity, reuse snapshots until +an explicit refresh, and own durable attempt evidence and publication. The public +`trail env resolve` operation selects the exact discovered component and runs reviewed +built-in offline, credential-free plans in host-owned isolated staging before entering +that publication boundary. Restricted repository/plugin resolver execution and aligned +HTTP/MCP operations remain planned. Discovery reports unsupported installed plugins +from their pinned marker and package metadata without launching the plugin. + +Snapshot consumption also crosses one host-owned verification boundary. Before a +built-in can plan construction from a stored snapshot, Trail verifies the exact proposal +key, source root, component and adapter identity, snapshot format, verified state, and +clear secret taint, then loads the content object. Ecosystem code supplies only its +deterministic proposal/format identity and content parser. Cargo, Node, and Python no +longer duplicate storage lookup, provenance, verification-state, or taint decisions. +Their planners still revalidate the current resolver executable and policy identity so a +tool or authority change cannot reuse an otherwise well-formed snapshot. + Command recipes and v1/v2 plugins may declare stable logical component dependencies. `sync all` validates missing nodes, duplicate/self edges, complete cycles, and mount collisions before running a command; it then runs a bounded deterministic ready queue, @@ -116,11 +153,29 @@ an offline build; MCP and HTTP classify synchronization as open-world execution repository-controlled Cargo build scripts and proc macros still execute. Read-only status reads persisted state and never invokes package managers or compilers. +A repository containing `Cargo.toml` but no `Cargo.lock` remains visible as a resolvable +component. Its resolver plan is pinned to the complete source root and exact Cargo +executable/policy. Once verified, the lock snapshot remains Trail environment metadata: +the host projects it into isolated staging only after the immutable source root and +refuses to replace a tracked path. Target-seed construction then uses the conventional +`Cargo.lock` location with `--locked --offline`; generation keys retain both the complete +source root and snapshot authority. Source discovery and planning never write a lockfile +back to the repository. + The initial Node adapter deliberately rejects workspace-root installs, local file/link dependencies, pnpm workspace roots, and Yarn Berry/PnP rather than publishing an empty, escaping, or incorrectly keyed layer. Those forms require explicit workspace-graph and link contracts before they can graduate. +Without a source lockfile, `packageManager` selects an npm, pnpm, Yarn, or Bun resolution +contract (npm is the deterministic default). Each manager has its own snapshot format, +candidate path, and fixed lock-only argv. The verified snapshot is projected into a +complete pinned-source staging tree and participates in layer identity by both content +digest and snapshot authority. Frozen installation still produces the ordinary +`immutable_seed_private` `node_modules` lower, while downloads remain in the existing +performance-only content-store namespace and every lane receives a private writable +upper. Unsupported managers remain visible as blocked proposals. + The first public Rust protocol crate is `trail-environment-adapter-sdk`. Local executable packages remain `experimental`: Trail verifies and content-addresses them, records append-only activation/tombstone history, revalidates their executable before every use, @@ -194,6 +249,64 @@ that needs semantic code can become an isolated plugin; a widely used, fully cer plugin can be proposed as a built-in. Component and adapter identities change explicitly when semantics change, so promotion never silently reinterprets an existing generation. +Repository documents at `trail.environment.toml` or `.trail/environment.toml` now +recognize both `trail.environment/v1` and the explicitly opted-in +`trail.environment/v2` header. Version 1 retains its existing paths, defaults, +includes/profiles, validation, and normalized identity. Every file in one include graph +must declare the same version. The v2 header initially accepts the existing restricted +command shape and adds strictly typed `component.resolve`, repeated `component.action`, +repeated `component.validation`, `component.capabilities`, heterogeneous +`component.output`, and repeated `component.source_export` sections. Resolver and action +commands remain argv arrays. Network authority lists, candidate snapshot format/path, +phase names, capability requests, validation requirements, output policies, and export +collision intent are represented as separate fields. Unknown fields fail deserialization, +and a v1 document cannot gain any of these authorities by spelling a v2-only section. +Compilation and execution still pass through the host-owned normalized models and +capability checks; parsing a declaration never grants authority by itself. + +Before discovery returns a v2 proposal, Trail also applies the repository-declaration +ceiling without resolving a tool or running repository code. Every command is a bounded +fixed argv vector naming one direct non-shell executable. Shell interpolation and control +operators, indirect process launchers, absolute host paths, raw secret-bearing values, +and provider-socket references are rejected. Resolver authorities are exact, bounded, +sorted, and deduplicated; construction and validation remain offline. Mounted-execution +and executable source-export phases are unavailable to repository declarations. Input +declarations and their deterministic expansion are separately bounded, and repository +outputs may request only exact workspace reuse or lane-private non-reuse—not compatible +or host-wide reuse. These checks narrow authoring early; the host repeats its normal +sandbox, plan, sealing, and output-contract checks at execution boundaries. + +The v2 compiler does not create a second repository execution graph. It projects one +validated document into the existing `EnvironmentDiscoveredComponentReport`, optional +`ArtifactResolutionPlanV1`, `WorkspaceEnvironmentPlan`, `ArtifactDesiredKeyMaterialV2` +and desired key, `ArtifactOutputContractV2`, `ArtifactValidationV1`, and +`ArtifactSourceExportContractV2` types. Exact executable identities, selected pinned +inputs, normalized output policy, platform identity, policy identity, and requested +network authorities therefore participate in the same host-owned reports and identities +as built-in and plugin adapters. A declared resolver produces a `resolvable` proposal; +it does not run during discovery or compilation. + +Source-export declarations compile into a separate, read-only planning operation before +any generated bytes can enter source. The plan pins the canonical lane and source root, +active environment generation and its source root, desired identity, ready envelope, +complete tree root, selected directory/file object, output-relative artifact path, +repository-relative destination and its current content digest, collision mode, exact +passed validation receipt, optional successful gate receipt, and explicit-user +authorization. `fail` refuses an existing destination during planning; `replace` merely +permits execution to consider the pinned destination and does not bypass later source +guardrails. Planning reads CAS objects directly and never materializes or mounts the +artifact into source. + +Execution repeats the complete plan and rejects any mismatch before mutation. It reads +the pinned directory/file objects directly from CAS with finite file/count/byte limits, +rejects symlinked export trees, computes `fail` or pinned `replace` edits, and submits one +ordinary structured lane patch. That shared path enforces repository-relative path and +case safety, `.trailignore`, hard private-path guardrails, patch/file limits, secret +scanning, exact lane-head compare-and-swap, atomic root publication, changed-path diff, +and workdir projection. The resulting operation is the source checkpoint; its report +includes the normal changed paths and a `trail lane merge ... --dry-run` Git-handoff +command. Artifact mounts and generated uppers are never used as the write mechanism. + ### Add a repository adapter without Rust Use `trail/command@1` when the adapter can be represented as pinned byte inputs, an argv @@ -319,6 +432,23 @@ copy, while removal remains available for recovery. Trust revocation also fails packages closed immediately. Unsigned packages are visibly `local-experimental`; a signature authenticates origin but does not grant stable certification. +Package inspection, installation, the adapter catalog/trust view, and removal reports +share `protocol_capabilities`. It records the currently selected host protocol (or no +mutual protocol), advertised resolution/source-export and host-evidence capability, +certification ceiling, CAS content policy, and host-attestation policy. The report keeps +package capability separate from selection: while v3 normalization is disabled, a +v3/v2 package reports its v3 capability but selects v2, and a v3-only package reports no +selected protocol. Removal captures the last readable package policy before appending +the tombstone; inability to inspect corrupt retained bytes never prevents revocation. + +The host classifies current publisher-authenticated experimental packages and unsigned +local packages as `locally_trusted_plugin` for execution. Only a separate durable +conformance result may select `certified_signed_plugin`. That host-selected tier is +combined with the action phase to produce `ArtifactCapabilityCeilingV1`; the normalized +plan is rejected if its inferred reads, writes, caches, processes, provider use, mounted +execution, network, or secret channels exceed that ceiling. No tier can request direct +publication authority. + The adapter catalog reports each package's planner protocols, supported operating systems, and architectures. Unsupported plugins remain inspectable but are not auto-discovered or executed on the current host. `lane readiness` replans every installed component from @@ -358,6 +488,43 @@ filesystem, network, shell, and child-process sandbox. Ed25519 publisher signatu the host trust store authenticate local packages today. Signed catalogs, independent certification attestations, and WASI transport remain distribution work. +The framework-neutral artifact fixture runs reviewed-built-in, local protocol-v3 +plugin, and repository-v2 producer profiles through the same host-owned stages: +discovery normalization, explicit resolution, desired identity, validation, sealing, +COW isolation, reopen recovery, invalidation, source export, lane retirement, and +last-reference collection. It emits `trail.artifact-adapter-certification/v1` JSON with +one canonical check per stage. A required skip or failure keeps the report unverified; +the report always carries `authority_effect = "evidence_only"` and cannot elevate a +plugin trust tier or grant publication capability. + +Run the deterministic fixture and write its machine-readable report with: + +```sh +CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/trail- \ + scripts/verify-artifact-adapter-conformance.sh /absolute/path/certification.json +``` + +This synthetic common-contract evidence complements, but does not replace, real-tool +and native NFS/FUSE/Dokan qualification. Those unavailable gates remain explicitly +unverified. + +The blocking Linux real-tool gate maps representative shapes to executable tests: + +| Shape | Gate evidence | +| --- | --- | +| Dependency resolution/install | Cargo resolver snapshot and npm managed-lock frozen install | +| Compiled incremental tree | Cargo target seed builds once and is reused | +| Framework/bundler composition | Node dependencies plus distinct Next.js/Vite build and private-cache components | +| Path-bound private state | Real Python virtual environments and CMake configure/build trees | +| External metadata | Bazel/Nix-like verified external-store normalization without fake Trail layers | +| Custom pipeline | Repository-v2 Maven/Gradle-like and unknown generator components | +| Source export | CAS subtree export through normal guarded source checkpointing | + +Run all gates with `scripts/verify-artifact-real-tool-gates.sh`. A missing +required executable fails the gate; it is never converted into passing evidence. Native +mount, multi-lane, and real Next.js/Vite build evidence remains in the owning-platform +qualification jobs. + ## Design principles 1. One contract covers package managers, compilers, build systems, containers, services, @@ -902,6 +1069,30 @@ Adapters declare which command argument or output fields may echo credentials so host can add structured redaction. This declaration supplements, but never replaces, host-wide exact-value redaction and secret scanning. +Receiving a secret value taints the producer output; a non-secret provider/version +reference alone does not. Taint metadata contains only canonical channel classes, never +provider names, handles, paths, or values. Tainted candidates may remain in lane-private +storage, but the host rejects resolution-snapshot publication, shared artifact sealing, +private-output promotion, and source export. Bounded attempt evidence is exact-value +redacted before storage. In the current resolver API a declared credential handle cannot +be distinguished from one actually consumed, so shared resolution fails closed. A future +private resolver-output store may preserve such output without weakening this rule. + +Resolver declarations are also capped by host ceilings before an attempt record or +subprocess can be created: one hour of wall time, 16 MiB for each captured stream, +1 GiB and 1,000,000 entries for the candidate, and 256 child processes. Declared limits +must be non-zero and may narrow these ceilings, never expand them. Repository-relative +path and control-character validation runs in the same pre-attempt boundary. + +Adapters do not author or publish artifact attestations. After host sealing, Trail creates +the deterministic attestation from validated host evidence and stores it as a separate +content-addressed object so signatures and trust observations do not alter artifact tree +identity. Attachment requires an exact statement/envelope match and current producer +trust. For plugin producers, the installed distribution digest, publisher, and publisher +key must still match; tombstoned packages and removed publisher trust fail closed. +Optional signatures cover the canonical attestation statement and are never interpreted +as adapter certification or publication authority. + ## Plugin protocol and capabilities The implemented v1/v2 transport is a natively sandboxed subprocess using one @@ -914,6 +1105,49 @@ the same models. Local plugins are installed by digest and declare: - timeout and response-size limits; - one executable digest and experimental stability. +The SDK's protocol-v3 wire layer is separate from v1/v2 and carries explicit host +ceilings. It models incomplete proposals, resolution plans and verified snapshot input, +typed artifact phases, input roles, validation and capability contracts, identity +declarations, explicit source exports, attestation requirements, secret taint, and +quarantine policy. Only Trail-authored object references can appear as attestation or +quarantine evidence in an adapter request. Adapter responses cannot publish those +objects or acquire mutation authority. Both request and response expose bounded +validation before semantic host normalization; exact protocol negotiation and complete +host-side validation remain mandatory trust boundaries. + +Negotiation compares exact identities and chooses the highest mutual value from v3, +v2, then v1; caller ordering and unknown version-like prefixes have no effect. Until the +host completes v3 normalization, its advertised host set remains v1/v2, so a multi-version +package deterministically falls back while a v3-only package is not invoked. Canonical +v1/v2 compatibility projections preserve their existing commands, outputs, caches, and +external/runtime declarations but carry a structurally empty v3-only section. Missing +legacy fields can therefore never imply proposal, resolver, validation, capability, +identity-certification, export, attestation, or quarantine semantics. + +The SDK v3 builder provides early authoring validation and deterministic set ordering. +Package metadata separately declares resolution/export and host-evidence capabilities +plus a certification ceiling; omission is deny-all legacy behavior. These declarations +are maximum requests, not grants: package trust, host policy, protocol selection, and +native enforcement may only narrow them. + +Trail's v3 exchange validator does not trust builder use. It rechecks request and +response ceilings, exact protocol/request/package/source/host pins, canonical proposal +evidence, complete typed-input equality, normalized paths, duplicate component/action/ +validation/output/export identities, graph and phase/output compatibility, conservative +secret taint, package resolution/export/evidence declarations, and certification +ceiling. Unknown required fields and enum values fail CBOR decoding. Multi-version +packages continue negotiating v2 until the v3 invocation/normalization path is enabled, +so merely installing a v3 declaration cannot bypass this validator. + +The SDK freezes representative proposal request/response frames as length-prefixed CBOR +hex fixtures and decodes them on every test run. Contract tests also exercise truncated +headers and bodies, declared and encoded size overruns, legacy-only negotiation, and +property-generated protocol order/duplication/unknown-version combinations. The layered +workspace CI matrix runs these SDK contracts on Linux, macOS, and Windows. Trail's host +tests separately repeat malicious response, package mismatch, capability ceiling, +platform-identity, publisher-signature, and revocation checks; native plugin scripts own +the operating-system sandbox evidence. + The planner capability set is deliberately fixed rather than open-ended: bounded pinned bytes in, discovery or plan data out, no direct repository reads or writes, no child process, no network, no shell, no secrets, no database, and no mount/publication @@ -961,13 +1195,20 @@ The following mappings show how one policy vocabulary fits very different toolch Next.js and Vite are profiles composed over the Node adapter. They do not duplicate package installation logic. +The executable conformance fixture represents Next.js state as a writable-private +component depending on `node`. Vite is split into an immutable validated `dist` +component and an independently keyed writable-private optimizer-cache component, both +depending on `node`. Splitting outputs with different lifecycles avoids relabeling a +path-bound cache as reusable merely because a sibling build output is portable. These +are repository-v2 compositions, not framework names or new COW modes in Trail core. + ### Rust and Cargo | Concern | Inputs | Policy/binding | | --- | --- | --- | | Rust toolchain | `rust-toolchain.toml`, distribution digest, target components | `immutable_shared` toolchain | | Registry crates and Git checkouts | Cargo version, source registry, checksums | `cache_shared_content` with host-mediated indexes | -| Compiled dependency seed | `Cargo.lock`, features, profiles, rustc identity, target, relevant build-script environment | optional `immutable_seed_private` lower for `target` | +| Compiled dependency seed | source or Trail-managed `Cargo.lock` snapshot, complete source root, features, profiles, rustc identity, target, relevant build-script environment | optional `immutable_seed_private` lower for `target` | | Active target directory | source and commands continuously mutate it | `writable_private` at `target` | | Compiler cache | compiler identity, target, flags | `cache_shared_compiler` through sccache protocol | | Credentials | registry token | `secret_runtime`, injected only into Cargo command | @@ -991,6 +1232,20 @@ CMake adapters must account for absolute paths in `CMakeCache.txt`. A private bu cannot be promoted to a portable immutable artifact merely because compilation succeeded. +### Framework-neutral conformance shapes + +Executable fixtures intentionally avoid adding Maven, Gradle, Bazel, Nix, or arbitrary +custom framework modes to Trail core. Maven/Gradle-like dependency checksum output and +lane-private build/daemon state are separate `trail.environment/v2` command components. +An unknown custom generator uses the same component, validation, desired-key v2, and +explicit source-export contracts. + +Bazel/Nix-like provider stores use a protocol-v2 plugin metadata component containing +sorted `verified_external` entries. Each entry binds a provider token, opaque bounded +reference, SHA-256 digest, and platform identity. It has no action, cache, output, +runtime allocation, or Trail-owned cleanup. OCI runtime declarations remain restricted +to `oci_image`, so a generic store reference cannot be relabeled as a container image. + The implemented `trail/cmake-build@1` adapter covers the safe first slice: discovery, deterministic host/tool compatibility identity, atomic lane-private build-directory ownership, generation provenance, mounted-view classification, and crash recovery. @@ -1055,9 +1310,13 @@ compatible re-sync preserves the private environment and creates no shared layer Additional native fixtures cover multi-component `env sync all`, initializer failure, undeclared source writes, kill-point recovery, and abandoned-candidate cleanup. -Sharing interpreter distributions and wheel/download content safely remains a separate -typed-cache slice; Trail must not represent a path-bearing virtual environment itself as -a portable immutable artifact without relocation validation. +An optional resolver plan pins the complete source root, exact uv executable, offline +hash-generation policy, and `pyproject.toml`, then stores the resulting +`requirements.lock` as verified environment metadata. The host projects it only into +attempt staging and runs hash-required `pip download` into a host-owned +`cache_shared_content` namespace. Evicting that cache affects performance only. Trail +still must not represent a path-bearing virtual environment, its bytecode, or embedded +scripts as a portable immutable artifact without relocation validation. | Concern | Inputs | Policy/binding | | --- | --- | --- | diff --git a/docs/design/guardrails-security-and-redaction.md b/docs/design/guardrails-security-and-redaction.md index 84719c51..2a382823 100644 --- a/docs/design/guardrails-security-and-redaction.md +++ b/docs/design/guardrails-security-and-redaction.md @@ -29,6 +29,27 @@ The safety model is layered: 7. Redact sensitive metadata before storing trace/guardrail payloads. 8. Require daemon auth by default. +The common artifact pipeline adds four untrusted boundaries: repository and +plugin declarations, resolver/constructor subprocesses, candidate filesystem +trees, and imported attestation metadata. Environment adapters are planners, +not executors. Trail repeats validation at the host boundary and controls the +resolved executable, fixed argv, working directory, readable inputs, writable +candidate/cache/temp roots, network authorities, environment roles, child and +byte limits, cancellation, capture redaction, and publication decision. + +Discovery never runs repository code or tools. Resolver, constructor, +validator, and mounted execution capabilities are separate and deny by default. +Repository v2 and adapter v3 requests may narrow a certified profile but cannot +widen it. V1/v2 adapters keep their legacy authority and cannot obtain v3 +resolution, source-export, or attestation privileges through omitted fields. + +Secret bytes are never key material. A phase that receives a secret is tainted; +its output must remain lane private and cannot enter shared CAS, a reusable +materialization, an attestation, or a source export. Candidate ingestion also +rejects credential-like content, protected paths, escaping links, special +files, unsafe modes/xattrs, case collisions, concurrent mutation, and declared +entry/byte/depth limit violations before publishing an envelope. + ```mermaid flowchart TB Input["User, agent, CLI, HTTP, or MCP request"] @@ -266,6 +287,14 @@ CLI daemon routing can read the token from `--daemon-token`, `TRAIL_DAEMON_TOKEN - Guardrail blocked decision: returned in report; callers decide whether to proceed. - Unauthorized daemon request: HTTP 401 and daemon exit code category 11. - Patch rejected: `PATCH_REJECTED`, exit 7. +- Missing capability enforcement rejects the action; it does not silently run + an untrusted resolver or constructor unsandboxed. +- Secret-tainted output remains private and non-promotable even when its content + would otherwise match a shared desired key. +- Divergent ready content for one desired key and trust scope quarantines both + claims and retains immutable evidence until explicit resolution. +- Stale source, snapshot, desired, validation, destination, or authorization + pins reject promotion/source export before ordinary source mutation. ## Code Facts Used diff --git a/docs/design/layered-lane-workspaces.md b/docs/design/layered-lane-workspaces.md index 717ea929..7d7ac8b1 100644 --- a/docs/design/layered-lane-workspaces.md +++ b/docs/design/layered-lane-workspaces.md @@ -51,10 +51,10 @@ Trail already has the right starting points: - Persistent lane uppers, whiteouts, workdir manifests, sessions, turns, checkpoints, gates, readiness, merge queues, and safe Git apply. -The current overlay implementations expose one Trail root as a lower layer and -one lane directory as an upper layer. They do not expose ignored dependencies or -generated artifacts, eagerly load the root file map, and implement overlapping -filesystem behavior separately for FUSE, NFS, and Dokan. +The shared `ViewCore` now exposes a lazy Trail root plus immutable environment +artifact manifests as lower layers and class-specific lane directories as +uppers. FUSE, NFS, and Dokan are protocol adapters over those common lookup, +ranged-read, copy-up, whiteout, rename, and directory semantics. This design evolves that foundation into a **layered lane workspace**: @@ -112,6 +112,11 @@ Generated and dependency directories frequently dominate the source checkout: - Python environments, Java/Gradle caches, generated SDKs, and editor indexes add similar amplification. +Python keeps the path-bearing virtual environment and bytecode in the lane-private +generated upper. A hash-bearing managed requirements snapshot may warm a shared, +evictable wheel/download content cache, but that cache never becomes environment +authority and `.venv` is never promoted as a portable layer without relocation proof. + Filesystem reflinks reduce data-block duplication, but do not remove directory entry and inode cost, do not help on every filesystem, and do not by themselves provide a safe shared model for mutable generated directories. @@ -227,23 +232,38 @@ The transparent COW modes already establish several correct invariants: - Mount failure is explicit rather than silently becoming a full copy. - macOS NFS mount state is persisted for stale-mount recovery. -The principal gaps are: - -1. The lower layer includes only files in the Trail root. Default-ignored paths - such as `node_modules` and `target` are neither visible nor shared. -2. Overlay mounts load the complete root into memory rather than using lazy - lookup and directory iteration. -3. Lower-file reads materialize complete file contents into memory. -4. FUSE, NFS, and Dokan contain separate implementations of lookup, copy-up, - whiteout, rename, and directory behavior. -5. A single upper mixes source changes with generated and dependency changes. -6. Checkpoint detection still relies partly on manifests or scans instead of an +CAS-backed artifacts also have a reconstructible real-directory cache under +`.trail/cache/artifact-materializations`. Its identity combines the verified +artifact tree root with the real-directory backend compatibility key. Trail +stages and verifies complete content before atomic publication, revalidates and +re-seals entries on reuse, and rebuilds missing or corrupt cache directories +from authoritative CAS objects. Copies into mutable layer or lane state prefer +native clone/reflink support and fall back to independent file copies; Trail +does not use hard links that could alias mutable bytes back into this cache. + +When one lane forks another, immutable environment outputs are inherited only +after output-level CAS compatibility checks. Trail revalidates the exact desired +key, ready envelope, complete tree, current adapter implementation/distribution +and package trust, sharing and portability scopes, and the child view backend. +Rejected sibling outputs are not copied merely because another output from the +same component is reusable. Every accepted child artifact binding is new, as +are the child source, generated, scratch, and seeded/writable private uppers. + +The remaining principal gaps are: + +1. Verified real-directory materialization caches participate in deterministic + cache eviction and reachability/accounting, but do not yet have a separate + per-artifact quota policy. +2. Git-root lower-file reads still use a complete-file projection cache; CAS + artifact files use direct bounded blob/chunk ranges and materialize only the + selected file during copy-up. +3. Checkpoint detection still relies partly on manifests or scans instead of an explicit per-view mutation set. -7. Mount lifetime is tied mainly to a terminal agent process; editor workflows +4. Mount lifetime is tied mainly to a terminal agent process; editor workflows need daemon-owned or foreground persistent mounts. -8. The view has no Git compatibility metadata. -9. There is no layer cache lifecycle, key model, quota, pinning, accounting, or - integrity verification. +5. The view has no Git compatibility metadata. +6. Layer cache lifecycle and accounting are implemented; finer per-component + quota policy remains incomplete. ## Architectural Principles @@ -609,17 +629,21 @@ case-colliding paths are rejected. Layer files become read-only before publish. ### Build and publish protocol -1. Acquire a lease for the cache key in one SQLite transaction. +1. Acquire the cache-key file lock and reserve a durable, generation-fenced attempt. 2. If a complete layer already exists, pin and use it. 3. Build in `.trail/cache/staging/`. 4. Run guardrail and approval checks before network access or lifecycle scripts. 5. Validate the resulting tree and build the manifest. 6. Sync files and metadata according to durability policy. 7. Atomically rename staging to the immutable layer location. -8. Mark the layer complete and wake waiters. +8. Mark the attempt complete and release durable waiters. -Concurrent callers for the same key wait for one builder. A dead builder lease -is recoverable; incomplete staging directories are never mounted. +Concurrent callers for the same key record waiter evidence and reuse one builder's +result. Attempts expose `reserved`, `building`, `validating`, `publishing`, and +`completed` phases. Every mutation is fenced by attempt ID, owner generation, PID, and +process-start token. Trail replaces an owner only when that exact process identity is +proven dead or mismatched; unknown liveness remains owned. Incomplete staging +directories are never mounted. ### Node adapter @@ -661,6 +685,20 @@ deletion remains correct but is intentionally not the optimized control path. Trail must not infer that a lockfile update is valid merely because an agent mutated `node_modules`. The lockfile and adapter build remain authoritative. +When a leaf package intentionally omits a lockfile, Trail may resolve one manager-specific +snapshot selected by `packageManager` (defaulting to npm). npm, pnpm, Yarn, and Bun retain +distinct lock names, formats, and frozen-install argv, but all use the common verified +snapshot projection. The snapshot stays outside source and Git; the resulting +`node_modules` seed, package download cache, COW upper, and invalidation behavior are the +same as for a tracked lockfile. Until a narrower package input closure is certified, the +managed snapshot is pinned to the complete source root. + +The repository-v2 framework fixture composes Next.js and Vite over that Node component. +Next owns lane-private `.next`; Vite splits lane-private `.vite` optimizer state from a +separately validated immutable `dist` component. Every framework component has a typed +`build_requires` edge to Node, so dependency resolution and `node_modules` COW remain +centralized while each generated path follows its own reuse and disposal policy. + ### Cargo adapter Cargo needs a different strategy because `target` mixes reusable compiler @@ -678,6 +716,14 @@ The safe default is: base root, lockfile, toolchain, target triple, feature/profile set, Cargo config, and relevant flags. +Library repositories that intentionally omit `Cargo.lock` use the same construction +path after an explicit resolution operation records the lock bytes as Trail metadata. +The verified snapshot is projected only into the attempt-owned source staging tree; it +is never added to the lane source upper or Git. Until certified Cargo input-closure +analysis exists, both its proposal and target seed remain keyed by the complete source +root, so even an unrelated source change requires a new lock snapshot rather than +silently reusing dependency selection from another root. + `cargo clean` creates lane-local whiteouts over a seed; it never deletes the shared layer. Workspace-crate incremental state stays private. Trail may promote a verified target seed after a clean successful build, but must never @@ -1159,7 +1205,7 @@ Cache GC works from pins and policy: - Blob projections are independently reclaimable and rematerializable. - Generated uppers follow per-lane retention and archive policy. -`trail lane space` and `trail cache list` should distinguish: +`trail lane space` and structured cache-GC reports distinguish: ```text logical visible bytes @@ -1168,15 +1214,33 @@ lane-exclusive physical bytes reflink/shared-extent bytes when the platform can report them reclaimable cache bytes uncheckpointed source bytes +artifact logical bytes +unique authoritative CAS bytes +cross-artifact shared CAS bytes +artifact materialized bytes +lane-private bytes +persisted prefetched bytes +demand-loaded projection bytes +artifact/cache reclaimable bytes +unknown or unattributable bytes ``` Ordinary `du` on mountpoints is insufficient and may report logical bytes multiple times. Trail reports should use layer manifests plus platform block or -extent accounting where available and clearly label estimates. - -`fsck` verifies layer manifests and references. Normal Trail object GC and -workspace-cache GC remain separate commands because cache loss is recoverable -while object-history loss is not. +extent accounting where available and clearly label estimates. Logical, +authoritative, physical, and reclaimable values are separate axes and must not +be summed. Unique and cross-artifact shared bytes partition selected encoded +CAS objects without double counting; reclaimable bytes can overlap a physical +cache classification. Operating-system page-cache warming is not persisted and +is explicitly excluded from prefetched bytes. + +`fsck` verifies layer manifests and references, raw CAS object identities and +edges, snapshots, envelopes, attempt state, and materialization ownership. It +labels missing legacy layers as rebuild-required and missing CAS-backed layers +as reconstructible. Orphan layer/restore directories are reported with review +guidance rather than deleted without durable ownership evidence. Normal Trail +object GC and workspace-cache GC remain separate commands because cache loss is +recoverable while object-history loss is not. ## Performance Architecture and Targets @@ -1451,8 +1515,11 @@ Existing lanes using the current mode vocabulary remain valid: - New materialized selections default to `auto`; mounted modes remain explicit. Backups from older schemas restore without views. New backups restore source -uppers but rebuild caches. JSON reports add fields compatibly where possible; -new report types are preferred over changing unrelated existing shapes. +uppers and authenticated journals but rebuild generated/scratch state, +materializations, and performance caches. Authoritative artifact objects and +generation bindings survive even though copied active generations are retired +until a fresh sync. JSON reports identify retained private bytes and omitted +rebuildable state explicitly. ## Alternatives Considered diff --git a/docs/design/storage-and-indexing.md b/docs/design/storage-and-indexing.md index 55f03e9e..7c9cd3f9 100644 --- a/docs/design/storage-and-indexing.md +++ b/docs/design/storage-and-indexing.md @@ -4,10 +4,41 @@ Environment layer manifests and their sorted page objects are authoritative content-addressed objects rooted by `workspace_layers`. Object GC preserves the root and every page. `workspace_layer_publications` records private-output promotion phases and producer evidence; doctor, fsck, backup, restore, and -schema validation treat those rows as authoritative schema-v1 state. Because -backup archives exclude workspace cache bytes and mounted views, restore keeps -publication history but closes attachable attempts as `recovered` and clears -their layer/generation links. +schema validation treat those rows as authoritative schema-v1 state. Backup +archives exclude mountpoints, generated/scratch uppers, and workspace cache +bytes while retaining source uppers and their authenticated recovery journals. +Restore keeps publication history, closes attachable attempts as `recovered`, +and clears only machine-local layer/runtime links. + +Artifact CAS rows are verified against the raw `objects` bytes rather than the +process-local object cache. `fsck` checks object metadata/content identity, +directory/file/chunk edges, complete tree roots, resolution snapshots, ready +envelopes, construction-attempt coherence, and layer materialization ownership. +It distinguishes a legacy layer whose directory is authoritative from a +CAS-backed layer whose directory can be reconstructed, and reports different +repair guidance for each. + +Workspace-open recovery terminalizes only construction attempts whose exact +PID/start identity is proven dead or mismatched. It removes only the lock and +staging directory derived from that owner fence. Missing CAS-backed layer +materializations are reconstructed through a private staging directory; +completed restore staging is removed idempotently. Unknown layer directories or +restore staging are never deleted implicitly and remain visible to doctor and +fsck as bounded orphan diagnostics. + +Private backup-verification stages validate durable CAS without requiring the +intentionally omitted materialization cache. Resolution snapshots, artifact +objects, envelopes, attestations, historical generations, and exact generation +bindings remain in the portable SQLite snapshot. Host-local materialization +rows, performance-cache namespaces, active runtime resources, and generation +activation pointers do not. Restore first recovers retained source uppers from +private staged paths, then rebases them to the destination `.trail` directory +immediately before atomic publication. Reports distinguish retained private +bytes from omitted rebuildable materializations and caches. Verification checks +a deterministic retained-private tree digest before opening the staged +database. Legacy cache absence remains rebuildable through environment +synchronization rather than making an otherwise valid database-only backup +unverifiable. This design section is advanced/internal. It describes the current storage architecture and index maintenance paths. @@ -213,8 +244,65 @@ Garbage collection works from reachability: - Operations reference roots, parents, messages, conflict sets, and event payload objects. - Roots reference file entries and text/blob content. - Lane events and coordination records can reference object IDs. - -`gc --dry-run` reports without pruning. Normal GC deletes unreachable known objects while preserving reachable roots and references. +- Artifact generation bindings and workspace-layer shadows root their exact + envelope and tree identities. Construction/resolution attempts, resolution + snapshots, attestations, quarantines, active holds, and in-progress layer + publications root their durable object evidence. +- Artifact envelopes traverse to tree roots, resolution snapshots, and + typed deterministic validation receipts. Tree roots traverse directory nodes; + directory nodes + traverse directories/files; file nodes traverse blobs or chunk lists; and + chunk lists traverse chunks. Shared nodes remain live while any rooted graph + reaches them. +- Ready compatibility workspace-layer envelopes require exact structural and policy + host-seal receipts. Their deterministic input digests bind the declaration, desired + identity, tree root, validator identity, outcome, and bounded evidence; object or + reference tampering therefore fails attachment and fsck instead of weakening the + publication boundary. +- A recorded real-directory artifact materialization acts as a conservative + local cache lease. Cache eviction removes that row independently; a later + object GC may collect the CAS graph if no durable authority remains. +- Portable backups do not add roots to the source database: backup creation is + serialized by the workspace write lock, and the completed archive contains + its own authoritative object graph. + +An unbound artifact envelope row is an index, not an independent retention +root. GC may therefore collect a ready envelope left between publication and +generation activation when no generation, attempt, attestation, quarantine, +shadow, or hold retains it. + +`gc --dry-run` reports without pruning. Normal GC validates artifact identities +and edges before deleting anything, orders unreachable artifact DAGs +parent-before-child with object-ID tie breaking, and commits batches of at most +256 objects. Each committed batch leaves the remaining artifact graph valid: +interruption preserves earlier batches and rolls back the current batch, while +a later invocation recomputes reachability and resumes. Missing roots, +invalid edges, corrupt identities, unsupported active hold targets, or foreign +key disagreement fail closed rather than risking reachable content. + +Artifact storage reports use distinct accounting axes rather than one total: + +- `logical_bytes` sums file sizes once per distinct artifact tree in scope. +- `unique_authoritative_bytes` and `cross_artifact_shared_bytes` partition the + encoded content-addressed object bytes in scope. An object is counted once; + it is shared when more than one artifact envelope references it. +- `materialized_bytes`, `lane_private_bytes`, `demand_loaded_bytes`, and + `unknown_bytes` classify measured filesystem allocation. CAS-backed artifact + directories and layers are materialized; source/generated/scratch uppers are + lane-private; Git-root blob projections are demand-loaded; legacy layers, + tool namespaces, and unattributable native-clone extents remain unknown. +- `prefetched_bytes` counts persisted storage created only for prefetch. The + current hot-set implementation warms the operating-system page cache, so it + reports zero and excludes that volatile cache explicitly. +- `reclaimable_bytes` is an independent policy/disposition axis and can overlap + materialized or cache classifications. A lane-scoped report includes only + its rebuildable artifact materializations; the legacy top-level workspace + reclaimable field remains workspace-wide. + +Callers must not add logical, authoritative, physical, and reclaimable axes +together. Workspace-cache GC captures these values before deletion and sets +its reclaimable field from the exact ordered candidates selected by retention, +quota, and free-space policy. ## Backup and Restore diff --git a/docs/design/universal-lane-environments.md b/docs/design/universal-lane-environments.md index 215ee18f..f1518d20 100644 --- a/docs/design/universal-lane-environments.md +++ b/docs/design/universal-lane-environments.md @@ -278,6 +278,16 @@ embedded in source URLs. Publication is an atomic rename or equivalent transacti a private staging location. Published immutable content is never repaired in place: a bad artifact becomes `corrupt`, references are quarantined, and a new artifact is built. +Validation declarations are framework-neutral and typed as `structural`, +`loadability`, `framework`, `policy`, `gate`, or `reproducibility`; the legacy +`ecosystem` wire value remains inspectable under its original meaning. A host-run +validation emits a canonical `ArtifactValidationReceiptV1` bound to the exact +desired identity, artifact tree, declaration, validator identity, and validated +input digest. Receipt evidence is bounded, deterministically ordered, and +secret-rejected. Wall-clock time and local paths are excluded from receipt +identity. Only passing receipts with exact desired/tree bindings may enter a +ready envelope, and attachment revalidates those object identities and bindings. + ## Environment generations An environment generation is an immutable selection of component instances and @@ -386,6 +396,21 @@ compiler, lockfile, dependency tree, service image, and policy were active. Dire entering a lane shell remains possible, but commands outside `env exec` receive weaker reproducibility evidence and readiness can report that distinction. +Managed preparation is fail-closed and does not perform resolution as a side effect. +It records `missing_resolution_policy = explicit`, the exact source/view/generation, +the status and verified snapshot identity of every discovered component, and the exact +artifact envelope/tree binding for every active reusable output. Any non-ready proposal +terminates preparation with deterministic resolver recovery argv before graph sync or +command launch. + +Finalization records checkpoint, runtime-disposal, and unmount status together with the +policy-derived decision for each output (`dispose`, retain private state, preserve a +verified artifact, await a declared publication trigger, or replan). A source-root +change invalidates every pending seal or promotion decision; it never silently attaches +the candidate to the post-command source identity. Exec, test, eval, terminal-agent, +and materialized ACP paths share these typed receipts rather than defining transport- +specific lifecycle semantics. + ## Filesystem bindings Bindings are typed rather than encoded as opaque mount arguments: @@ -570,6 +595,41 @@ but identical component keys converge to one artifact. For example: - CMake targets can share a toolchain and immutable install prefix while keeping configuration-specific build trees private. +For a Cargo component without a tracked lockfile, Trail can bind a verified resolution +snapshot to the complete source root and project those bytes only into isolated build +staging. Cargo still observes a normal `Cargo.lock` and runs `--locked --offline`, while +the repository, lane source upper, and later Git export remain unchanged. A different +source root is a different proposal until a narrower input closure is certified. + +Node uses the same metadata projection with ecosystem policy left in adapter data: +`packageManager` selects an npm, pnpm, Yarn, or Bun snapshot format and fixed resolver +argv, while the common host validates source/snapshot identity and prevents source +replacement. All managers converge on a frozen dependency seed plus a performance-only +download cache and lane-private COW upper; Trail core does not gain manager-specific +workspace or storage modes. + +Framework composition follows the same rule. The repository-v2 Next fixture binds +`.next` as writable-private state over a typed Node dependency. Vite uses separate +components for lane-private `.vite` optimizer state and a validated immutable `dist` +artifact. Separating those lifecycles prevents a portable build output from granting +reuse to a path-bound cache, without adding Next.js or Vite branches to Trail core. + +The remaining shipped adapters use the same identity abstraction without changing +their storage policy. Host normalization derives a protocol-v3 identity-contract digest +for Go, CMake, OCI/runtime, and command plans while retaining the exact legacy layer key +as a compatibility/storage identity. Logical cache protocol and compatibility enter the +projection, but machine-local cache paths do not. Go vendor content can therefore remain +an immutable seed backed by performance caches; CMake remains a layer-free private build +tree; OCI remains provider-owned metadata with lane-private runtime allocation; and +repository-v2 commands keep their independent desired-key v2 contract. + +Conformance fixtures also cover ecosystems that have no built-in adapter. A +Maven/Gradle-like checksum graph plus private build state and an unknown custom generator +compile through repository v2. Bazel/Nix-like content stores compile through the generic +plugin `verified_external` identity: Trail records provider/reference/digest/platform +metadata but creates no layer, cache, runtime, or cleanup claim. These are compositions +of common contracts, not framework switches in the lane backend. + Each lane pins a generation independently. Syncing lane A cannot change the active generation, private upper, services, or secret handles of lane B. A new artifact may be built once and become available to both, but each lane attaches it through its own @@ -590,10 +650,64 @@ Trail uses distinct leases for: - cache maintenance; - garbage collection. -Build waiters may stream progress and either reuse the published result or retry after a -failed lease. Leases contain owner operation, heartbeat, expiry, and recovery metadata. -No lease alone proves safety: publication also requires staging ownership and manifest -validation, while activation also requires a lane transaction. +The cache-key file lock remains the fast exclusion boundary. Each lock owner also records +a durable construction attempt with a monotonically increasing owner generation, exact +PID/start identity, source-root pin, heartbeat, and `reserved`, `building`, `validating`, +`publishing`, or `completed` phase. Contenders record durable waiter rows. Completion +releases those waiters; cancellation records their terminal state without invoking a +second builder. Trail only replaces an owner when process liveness proves the exact +PID/start identity dead or mismatched—an indeterminate identity is never stolen. + +No lock or attempt row alone proves safety: publication also requires staging ownership, +content sealing, and manifest validation, while activation also requires a lane +transaction. The bounded component-DAG scheduler remains independent of per-key +singleflight, so unrelated ready components can build concurrently. + +All production adapter publication now enters through this common host path. Built-ins, +plugins, and repository declarations return plans and candidates; they do not reserve +layers, ingest CAS objects, publish manifests, or activate generations themselves. The +older single-component synchronization library wrapper remains as a compatibility entry +point, but delegates to the generation-oriented common synchronizer and grants no legacy +publication shortcut. + +The host seal is taken only after the synchronous producer returns and the output has +been copied into an attempt-owned staging tree that the producer does not control. Trail +then revalidates the exact construction owner, source root, desired key, phase, and +cancellation fence; enforces configured byte quotas; validates normalized contained +paths, regular file metadata, confined links, and secret policy; ingests the complete +bounded tree into CAS; and rescans the staging tree to reject mutation during sealing. +Only after the staged tree is immutable and agrees with its CAS manifest does Trail +publish the ready envelope. + +The built-in migration bridge keeps a workspace layer only as a compatibility +materialization. After Trail verifies that an artifact-v2 envelope names the same tree, +output contract, and portability, that desired-key envelope replaces the legacy CAS +shadow as the durable authority. Ordinary activation records an exact generation +binding to that envelope. Native lane views continue to read the immutable lower while +copy-up writes remain private to the lane; source export reads the authoritative CAS +tree rather than the private upper. Retirement removes the generation binding before +workspace-layer and object garbage collection independently reclaim the disposable +materialization and unreachable authoritative objects. + +Every compatibility workspace-layer envelope carries two required deterministic host +receipts. The structural seal records path/content/limit/secret/tree checks; the policy +seal records producer termination or disconnection, unchanged desired pins, and local +host authorization. Both receipts are bound to the exact desired identity and tree, +and attachment rejects missing, duplicate, failed, stale-digest, or mismatched evidence. +Package-specific trust and revocation remain additional attachment requirements rather +than being inferred from content safety. + +The same publication savepoint now creates one deterministic `ArtifactAttestationV1`. +Its identity covers the envelope and desired/tree/source/snapshot/upstream identities, +producer tier and package/publisher identity, executable/tool identities, host platform, +phase capability ceiling, sandbox/network/script policies, output/share contract, +validation receipts, and clear secret-taint result. It excludes wall-clock observations +and local storage paths. Inspection returns the immutable statement and current database +state; verification recomputes content identity and envelope binding. Optional Ed25519 +signature envelopes are parsed and verified when present, including local signing-key +revocation. Unsigned local host attestations remain valid but are labeled `unsigned`. +Attachment rechecks current plugin distribution digest and publisher key, so publisher +trust removal or package removal blocks reuse without mutating prior evidence. Garbage collection considers active generations, retained predecessors, checkpoints, running operations, open backend references, and leases. Logical and physical byte @@ -601,14 +715,54 @@ accounting are reported separately so shared-space savings remain visible. ## Adapter trust model -Trail supports three adapter tiers: +Trail distinguishes four producer trust tiers: 1. **Built-in adapters** run as reviewed Trail code with narrowly scoped host APIs. -2. **Declarative recipes** describe probes, commands, inputs, outputs, and policies; the +2. **Certified signed plugins** have both authenticated package provenance and a separate + durable conformance certification. +3. **Locally trusted plugins** include unsigned local packages and publisher-authenticated + packages that have not completed conformance certification. A signature authenticates + origin; it does not certify behavior or widen execution authority. +4. **Repository declarations** describe probes, commands, inputs, outputs, and policies; the Trail host executes them in a sandbox. -3. **Capability-constrained plugins** use a versioned WASI/component interface or - equivalent isolated protocol. They request filesystem, process, network, secret, and - runtime capabilities explicitly. + +The host computes a ceiling from producer tier and phase before considering any adapter +request. The current framework-neutral matrix is: + +| Phase | Reviewed built-in | Certified/local plugin | Repository declaration | +| --- | --- | --- | --- | +| Discovery/planning | no process, network, writes, or secrets | same | same | +| Resolution | exact authorities, pinned inputs, opaque credential handles | exact authorities, declared inputs, isolated candidate | same plugin ceiling | +| Construction | reviewed built-in process graph and managed legacy network; candidate plus host caches | declared executable, offline, declared inputs, candidate plus certified host caches | declared executable, offline, declared inputs, candidate only | +| Validation | offline candidate read and receipt-only write | same, with declared executable | same | +| Mounted execution | lane view and declared bindings; runtime injection is host-owned | deny-by-default native sandbox over the lane view and bindings | unavailable | +| Source export | artifact candidate read and one confined destination | same | same | + +No executable tier or phase has publication authority. Adapters may request less than +their ceiling, never more. Repository plans cannot request mounted execution, provider +authority, complete source projection, or host cache writes. Plugin plans remain under +the local-plugin ceiling until a future durable certification record says otherwise. +Reviewed built-ins retain an explicitly identified managed-network compatibility ceiling +while their ecosystem actions migrate to separate resolver and offline-constructor phases. +Repository v2 documents are rejected during side-effect-free loading when fixed argv +contains a shell, interpolation/control token, indirect launcher, absolute host path, +secret-bearing value, or provider socket. Their exact resolver authorities and input +expansion are finite and canonical, and their outputs cannot claim compatible or +host-wide reuse. Native enforcement remains mandatory later; early validation is not a +substitute for the execution sandbox. + +Secret consumption is a one-way trust transition. The host represents it with a typed +`clear` or `tainted` result containing canonical, non-secret channel classes only. A +resolver candidate with exact-value redactions is terminated as +`secret_tainted_output_private_only`: Trail may retain bounded redacted failure evidence, +but it does not store the snapshot candidate or publish a shared envelope. Because the +current resolver request cannot prove that a declared credential handle was unused, +credential-capable resolver publication fails closed before shared snapshot reuse or +creation. Runtime secret access similarly taints the active environment generation; +writable-private outputs from that generation cannot be promoted. Producer receipts are +checked for both typed taint and accidental sensitive fields before they become durable. +Source export remains a planned v2 operation and must apply the same taint check before +reading candidate bytes or writing repository source. Adapters never receive raw database access, shared artifact mutation, arbitrary host paths, or undeclared secrets. They return plans and observations. The host validates @@ -732,6 +886,19 @@ policy decisions, stale reasons, operation links, and redaction rules. Long-runn build, verification, and runtime operations use Trail operations with progress events and cancellation rather than blocking opaque requests. +The shared Rust operation layer now exposes artifact inspection, attach/sample/full/ +reproducibility-evidence verification, quarantine list/show/resolve, bounded content +reachability, workspace/envelope CAS accounting, resolution reports, and source-export +reports. CLI, HTTP/OpenAPI, and MCP adapters must project these models directly; they may +not reinterpret trust, quarantine, reachability, or byte-accounting state. The public +transport wiring uses those shared operations and report models. + +`ManagedExecutionLifecycleReport` includes additive `preparation` and `finalization` +objects. OpenAPI describes the same resolution pins, output pins, sealing decisions, +and cleanup status fields emitted by Rust, CLI, and HTTP-facing reports. ACP retains the +same receipt in its durable prompt-completion or interruption event without changing +ACP wire frames. + ## Readiness, claims, and Git handoff Environment readiness is one input to lane readiness; it does not replace task claims @@ -834,9 +1001,10 @@ backend: - `trail/cmake-build@1` provisions a layer-free private build tree and defers configure to the mounted lane so absolute cache paths remain valid; - `trail/python-venv@1` provisions a layer-free private `.venv`, keys it by dependency - manifests/locks and interpreter identity, and automatically initializes it through an - ephemeral candidate view at the final mountpoint so embedded prefix paths remain valid - without weakening atomic generation activation; + manifests/locks and interpreter identity, optionally binds a hash-bearing managed + requirements snapshot to a performance-only wheel/download cache, and automatically + initializes it through an ephemeral candidate view at the final mountpoint so embedded + prefixes and bytecode stay lane-private without weakening atomic generation activation; - existing layer references are imported into an initial environment generation without copying tree content. diff --git a/docs/getting-started/first-lane-workflow.md b/docs/getting-started/first-lane-workflow.md index 0ad798dd..dff8b868 100644 --- a/docs/getting-started/first-lane-workflow.md +++ b/docs/getting-started/first-lane-workflow.md @@ -20,6 +20,29 @@ trail lane spawn docs-lane --from main trail lane status docs-lane ``` +Inspect the framework environment before launching tools. Discovery and planning +are side-effect free: + +```sh +trail env discover docs-lane +trail env plan docs-lane +``` + +If discovery reports a component as `resolvable`, run the exact recovery command +from its report. To resolve every such component and prewarm the resulting +environment: + +```sh +trail env resolve all docs-lane +trail env sync all docs-lane +trail env generation docs-lane +``` + +Resolution is explicit: managed execution never contacts a resolver on your +behalf. A current immutable snapshot is reused; `--refresh` deliberately reruns +the resolver. A project with no detected environment adapter may continue with +source-only lane work and should not run `env sync`. + Open the lane workdir in your editor or point an external coding agent at it: ```sh @@ -59,6 +82,44 @@ Remove the lane after the work is merged or intentionally abandoned: trail lane rm docs-lane --force ``` +Removal retires the lane's environment generation and deletes its private and +disposable artifact state. Shared immutable CAS content remains only while it is +reachable or retained by cache policy. + +## What Trail Reuses and What It Records + +Trail records source and framework output through different pipelines: + +| State | Identity and storage | Lane behavior | Git behavior | +| --- | --- | --- | --- | +| Source | Pinned source root plus Trail file/line identity | Copy-on-write source upper; checkpointed by `lane record` or managed finalization | Review, merge, then explicit Git handoff | +| Resolution snapshot | Immutable environment metadata | Reused when proposal and resolver pins match | Never source unless a declared source export says so | +| Shared artifact | Desired key → content root → verified artifact envelope | One immutable lower can serve sibling lanes | Not merged or committed | +| Private/disposable output | Lane, generation, component, and output binding | Fresh writable upper; copy-up and whiteouts stay private | Not merged or committed | +| Declared source export | Exact envelope/subtree, validation, destination, and authorization pins | Writes through normal source guardrails and checkpoints | Becomes an ordinary reviewable source change | + +The desired key explains *why* an output may be reused. The content root says +*which bytes* were produced. The artifact envelope binds those identities to +validation, trust, output policy, and provenance. A filesystem materialization +is only a reconstructible projection of the authoritative CAS graph. + +Use the artifact ID returned by structured environment reports to inspect or +verify that evidence: + +```sh +trail --format json env generation docs-lane +trail env artifact inspect +trail env artifact verify --level full +``` + +Generated source is not a fifth output policy. A repository or adapter must +declare the export, then a user explicitly runs: + +```sh +trail env source export docs-lane --component --export +trail lane diff docs-lane --patch +``` + ## Spawn Options Create a virtual lane without a filesystem checkout: diff --git a/docs/guides/maintenance-and-recovery.md b/docs/guides/maintenance-and-recovery.md index 0ca38eef..2f7e4548 100644 --- a/docs/guides/maintenance-and-recovery.md +++ b/docs/guides/maintenance-and-recovery.md @@ -10,6 +10,12 @@ trail doctor Doctor reports operational health across workspace state and integrations. Tests verify it through the CLI, HTTP API, and MCP tool. +Artifact diagnostics distinguish authoritative CAS corruption from a missing or +rebuildable materialization. Doctor reports active quarantine, incomplete/dead +construction owners, stale generation bindings, unknown materialization state, +and native backend prerequisites without treating an unavailable platform gate +as passed. + ## Fsck ```sh @@ -18,6 +24,11 @@ trail fsck Use `fsck` to verify structural repository integrity. +Artifact fsck recomputes object identities and follows snapshot, envelope, tree, +directory, file, blob/chunk, validation, attestation, generation, quarantine, +hold, and materialization edges. Corrupt or ambiguous reachability fails closed; +it never becomes permission to collect bytes. + ## Index Rebuild ```sh @@ -46,6 +57,13 @@ trail backup restore /tmp/trail-backup Use `--overwrite` when creating over an existing backup path and `--force` when restoring over an existing workspace. +Backups retain authoritative artifact snapshots, CAS graphs, envelopes, +attestations, historical generations, bindings, source uppers, and exact-owner +recovery journals. They omit mounted projections, generated/scratch uppers, +verified materializations, and performance caches and report those omissions as +rebuildable. Restore retires copied active generation pointers and requires a +fresh `trail env sync all ` before execution. + ## Garbage Collection ```sh @@ -55,9 +73,66 @@ trail gc Garbage collection prunes unreachable known objects while preserving reachable roots and referenced objects. +Run cache GC before object GC when reclaiming artifact projections: + +```sh +trail cache gc --dry-run +trail cache gc +trail gc --dry-run +trail gc +``` + +Cache GC removes only unpinned reconstructible materializations. Object GC then +traces all live artifact authorities and deletes unreachable DAGs in restartable +bounded batches. Active generations, views, attempts, quarantines, holds, +attestations, and leases remain roots. + +## Artifact Recovery + +Start with read-only evidence: + +```sh +trail doctor +trail fsck +trail --format json env status +trail --format json env generation +trail env artifact inspect +trail env artifact verify --level full +trail env artifact quarantine list +``` + +- For `resolvable`, run the exact `trail env resolve ...` recovery command in + discovery/status output, then sync. Do not add a generated lock snapshot to + source unless a declared source export authorizes it. +- For a missing or corrupt materialization with a valid envelope, rerun sync; + Trail reconstructs it from CAS. +- For a dead exact-owner construction attempt, reopen the workspace so recovery + fences and resumes or abandons that attempt before retrying sync. +- For quarantine, inspect both content identities and choose an explicit + `retain-private`, `accept-incumbent`, `accept-candidate`, or `retire-all` + resolution. Never delete rows or CAS files manually. +- After restore, rerun sync for each retained lane before managed execution. + +## Schema-v1 Upgrade and Rollback Boundary + +Trail still accepts exactly SQLite schema v1 and has no in-place database +migration path. The artifact pipeline extends the fresh schema-v1 contract; an +older schema-v1 workspace whose tables or validators no longer match fails +closed. Before installing a build with a changed storage contract: + +1. Create and verify a backup with the old binary. +2. Preserve that backup and the old binary together. +3. Install the new binary and open a disposable restored copy first. +4. If Trail requests reinitialization, export accepted source to Git, preserve + the backup, and use `trail init --force` rather than editing SQLite. + +Rolling the binary back is safe only with a workspace/backup created under the +older binary's exact storage contract. A newer workspace is not downgraded in +place. Artifact CAS files, rows, refs, and generation bindings must never be +manually copied between the two states. + ## Code Facts Used - Maintenance CLI args: `trail/src/cli/command/maintenance_args.rs` - Maintenance handlers: `trail/src/cli/command/handler/maintenance.rs` - Tests: `doctor_reports_operational_health_across_cli_api_and_mcp`, `backup_create_verify_and_restore_roundtrip`, `gc_prunes_unreachable_known_objects_and_preserves_reachable_roots` - diff --git a/docs/guides/performance-and-scale-benchmarks.md b/docs/guides/performance-and-scale-benchmarks.md index b3146395..261d393e 100644 --- a/docs/guides/performance-and-scale-benchmarks.md +++ b/docs/guides/performance-and-scale-benchmarks.md @@ -3,8 +3,46 @@ Run `scripts/verify-layered-lane-scale.sh` with a unique external `CARGO_TARGET_DIR` and `TRAIL_SCALE_EVIDENCE_DIR` beneath `/Volumes/Workspace`. It executes deterministic 10k/1-lane, 100k/5-lane, and 1M/20-lane fixtures and -writes one JSON evidence file per experiment. A platform whose native backend -gate was not run remains unverified; a skipped result is never passing evidence. +writes both a source-root and an artifact-CAS JSON evidence file per experiment. +`scripts/check-layered-lane-scale-evidence.py` fails the run when any matrix row, +measurement, or explicit skipped-backend disposition is absent. + +The artifact experiment constructs one immutable Merkle tree, binds the same +tree to every lane, writes one distinct private delta per lane, and deliberately +leaves the tree lazy. Its evidence records logical-to-authoritative content +reuse, the naive per-lane copy baseline, actual copied/projected/prefetched and +materialized bytes, private allocated bytes, phase latency, and CAS object +counts. `backend=cas-lazy-unmounted` qualifies only the portable CAS/lazy lookup +path. It sets `native_backend_qualified=false` and lists NFS, FUSE, and Dokan as +skipped; native qualification belongs to the owning-host matrix and cannot be +inferred from this experiment. A skipped result is never passing evidence. + +The million-entry cases are intentionally release-scale rather than pull-request +smoke tests. Run them through the `Layered Workspace Native Gates` workflow's +`run_scale` dispatch or on a machine where the external target and evidence +directories have enough space and time. CI uploads the six checked JSON reports +even when a later correctness gate fails. + +For native artifact qualification, run the owning-host matrix with the native +backend gate enabled. For example, macOS uses: + +```sh +TRAIL_RUN_NFS_COW_TESTS=1 \ +CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/trail-native-cow \ +python3 scripts/verify-artifact-native-cow-matrix.py \ + --output /Volumes/Workspace/trail-native-cow-evidence/nfs-macos.json +``` + +The report is deliberately compositional. The platform adapter owns mounted +immutable lookup, copy-up, and whiteout semantics; Trail's shared CAS lifecycle +owns inheritance, verified materialization/repair, promotion, source export, +retirement, recovery, and accounting. Every check runs on the same owning host, +and the report labels these scopes separately. It passes only when exactly one +test executes for every required check. The other platform backends are emitted +as `unverified` with `reason=not_owning_platform`; reports from their owning-host +jobs are required before claiming those backends. The layered-workspaces CI +workflow uploads Linux FUSE and macOS NFS reports on pull requests and an +explicitly dispatched Dokan report on Windows. Use the CLI scale benchmark to verify large-repo and agent orchestration behavior before treating a change as production ready. diff --git a/docs/integrations/openapi.md b/docs/integrations/openapi.md index e5c65037..ee115545 100644 --- a/docs/integrations/openapi.md +++ b/docs/integrations/openapi.md @@ -26,7 +26,8 @@ curl -H "Authorization: Bearer $TOKEN" \ The OpenAPI path builder groups routes as: - Core: health, OpenAPI, doctor, status, record, diff, timeline, why, history, code-from, config, ignore, guardrails. -- Lanes: list/spawn/show/remove, status, review, contribution, gates, readiness, merge, handoff, workdir, diff, read-file, sync-workdir, record, rewind, tests, evals, patches. +- Lanes: list/spawn/show/remove, status, review, contribution, gates, readiness, merge, handoff, workdir, diff, read-file, sync-workdir, record, rewind, tests, evals, patches, environment resolution, and source export. +- Artifacts: workspace space, envelope inspection and reachability, verification, quarantine list/show/resolve, and the shared artifact report schemas. - Collaboration: sessions, approvals, leases, claims, anchors, merge queue, conflicts, and `POST /v1/lanes/{lane}/merge` (used by `trail lane merge`). - Turns and traces: turns, messages, events, spans, runs. diff --git a/docs/integrations/rust-library.md b/docs/integrations/rust-library.md index 8f92ce30..68859767 100644 --- a/docs/integrations/rust-library.md +++ b/docs/integrations/rust-library.md @@ -33,9 +33,40 @@ pub mod prelude { Use `Trail::init`, `Trail::init_with_text_policy`, `Trail::open`, or discovery/open methods to create a handle, then call typed methods for record, status, agent lifecycle, sessions, patches, merge queue, conflicts, backups, and maintenance. +Artifact-resolution executors can hand a normalized `ArtifactResolutionRequestV1` to +`Trail::resolve_artifact_component`, or a same-source-root set to +`Trail::resolve_all_artifact_components`. The operation validates source and executable +pins, reuses the current snapshot unless refresh is explicit, fences a durable attempt, +redacts bounded diagnostics, and publishes the content-addressed snapshot. Supplying a +candidate is an executor boundary; CLI/HTTP/MCP resolver execution is not exposed yet. + +Artifact lifecycle inspection uses the same serializable reports intended for the other +interfaces: + +```rust +let artifact = db.inspect_artifact(&envelope_id)?; +let verification = db.verify_artifact( + &envelope_id, + ArtifactVerificationLevelV1::Full, +)?; +let reachable = db.artifact_content_reachability(&envelope_id)?; +let quarantines = db.artifact_quarantine_list_report()?; +let space = db.workspace_artifact_space()?; +``` + +`Attach`, `Sample`, and `Full` progressively validate attachment evidence, a +deterministic object sample, or the complete reachable object graph. `Reproduce` +performs full verification and requires a passed durable reproducibility receipt; it +does not silently execute a producer. Quarantine show/resolve and source-export +plan/execute are also public `Trail` operations. Every collection is deterministically +ordered and bounded; limit exhaustion is an error rather than an empty report. + ## Data Types -Reports and model types are serializable with Serde. The CLI, HTTP API, MCP tools, and Rust API share many of the same report structs. +Reports and durable model types are serializable with Serde. Ephemeral resolver request +and candidate types deliberately are not serializable because candidate redaction bytes +must never become a wire or storage contract. The CLI, HTTP API, MCP tools, and Rust API +share many of the same report structs. ## Code Facts Used @@ -43,4 +74,3 @@ Reports and model types are serializable with Serde. The CLI, HTTP API, MCP tool - Public methods: `trail/src/db` - Public models: `trail/src/model` - Test: `prolly_is_importable_through_trail_namespaces` - diff --git a/docs/lanes/large-repository-environments.md b/docs/lanes/large-repository-environments.md index fb5b3b29..9fb0c3c6 100644 --- a/docs/lanes/large-repository-environments.md +++ b/docs/lanes/large-repository-environments.md @@ -101,6 +101,74 @@ Use a separate component for each ownership policy. A shared immutable SDK, a seeded consumer-mutable dependency tree, a persistent private build tree, and disposable test scratch must not be collapsed into one live mutable directory. +## Next.js and Vite composition with repository v2 + +Frameworks compose over dependency components; framework names do not receive +special sharing authority. This tested `trail.environment/v2` shape keeps +Next.js state private while allowing a validated Vite distribution to be +shared. Built-in Node discovery supplies the `node` dependency component. + +```toml +schema = "trail.environment/v2" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "web.next-build" +adapter = "trail/command@1" +kind = "generated" +depends_on = ["node"] +inputs = [{ path = "next-source.js", role = "identity", format = "bytes" }] +outputs = [{ name = "next-state", source = "next-output", target = ".next", policy = "writable_private", reuse = "none", scope = "lane", publish = "manual", portability = "host" }] +[component.build] +command = ["cp", "next-source.js", "next-output/server.js"] +cwd = "." +network = "deny" +scripts = "deny" + +[[component]] +id = "web.vite-build" +adapter = "trail/command@1" +kind = "generated" +depends_on = ["node"] +inputs = [{ path = "vite-source.js", role = "identity", format = "bytes" }] +outputs = [{ name = "dist", source = "dist", target = "dist", policy = "immutable_shared", reuse = "exact", scope = "workspace", publish = "on_sync", portability = "host" }] +[component.build] +command = ["cp", "vite-source.js", "dist/app.js"] +cwd = "." +network = "deny" +scripts = "deny" + +[[component.validation]] +name = "dist-path-contract" +kind = "path_contract" +path = "dist" +required = true +parameters = { maximum_entries = "1000" } +``` + +Create the two declared input files, record them, then verify the graph and +execute the exact component closure: + +```sh +trail lane spawn web-a --from main +trail env discover web-a +trail env graph web-a +trail env plan web-a --component web.vite-build +trail env sync component web.vite-build --lane web-a +trail lane exec web-a -- test -f dist/app.js +trail lane spawn web-b --from web-a +trail lane exec web-b -- test -f dist/app.js +``` + +Both lanes can attach the Vite content root. Their `.next`, `.vite`, incremental +compiler, and daemon state remains in fresh lane-private uppers. If generated +client code must become source, add a `[[component.source_export]]` declaration +and invoke `trail env source export`; do not point a shared output directly into +the repository tree. + Every key includes adapter provenance, declared byte/Merkle identities (or the complete source root when closure is not certified), identity-bearing upstream keys, argv/cwd, tools, identity environment, output policy, platform, @@ -133,8 +201,16 @@ pinned; callers can also use the Rust `pin_workspace_layer` API for an explicit time-bounded or indefinite evidence pin. Private uppers, source, runtime, and secrets are never cache candidates. +Use `trail --format json lane space ` and +`trail --format json cache gc --dry-run` for artifact-aware accounting. The +`artifact_storage` fields keep logical, authoritative CAS, physical, and +reclaimable axes separate. Cache-GC accounting describes the pre-deletion +snapshot and its `reclaimable_bytes` is the exact selected candidate set. + Successful managed commands record only bounded immutable-layer path accesses. A later execution with the exact command fingerprint, component keys, generation, and manifest identities may prefetch that authenticated hot set. Prefetch is advisory and cancellable; lifecycle receipts report its entry and -byte limits, match state, cancellation, and bytes actually read. +byte limits, match state, cancellation, and bytes actually read. Those reads +warm the operating-system page cache without creating a persisted prefetch +store, so storage accounting reports `prefetched_bytes: 0`. diff --git a/docs/lanes/overview.md b/docs/lanes/overview.md index d50c50f5..53fc0540 100644 --- a/docs/lanes/overview.md +++ b/docs/lanes/overview.md @@ -127,6 +127,19 @@ Each phase emits a `managed_execution_phase` lane event. Command failure does not skip source checkpointing or cleanup. Dependency, generated, scratch, secret, and Trail-internal paths stay outside the recorded source change. +The structured lifecycle report also retains two exact receipts. `preparation` +pins the source root, layered view and generation, explicit missing-resolution +policy, each component proposal/snapshot, and each active output binding. A +component that is `resolvable`, `blocked`, `unsupported`, or `ambiguous` stops +before synchronization and returns its exact `trail env resolve ...` recovery +argv; managed execution never resolves implicitly. `finalization` reports the +checkpoint, runtime disposal, and unmount outcomes plus one sealing decision per +output. If execution changes identity-bearing source, a pending publication +decision becomes `replan_required` instead of sealing bytes against stale pins. +The same receipt shape is used by exec, test, eval, terminal-agent, and +materialized ACP execution; older serialized lifecycle reports remain valid +with both receipts absent. + Omitting `--workdir-mode` creates a lazy `auto` layered lane on a qualified native transparent backend. Spawn does not copy source or execute ecosystem tools. The first managed command converges the desired environment; exact diff --git a/docs/reference/cli/integrations-and-maintenance.md b/docs/reference/cli/integrations-and-maintenance.md index fd36180f..a4b03be5 100644 --- a/docs/reference/cli/integrations-and-maintenance.md +++ b/docs/reference/cli/integrations-and-maintenance.md @@ -187,6 +187,41 @@ trail env sync component --lane trail env promote ``` +When discovery reports a component as `resolvable`, produce its immutable +dependency snapshot before synchronization: + +```sh +trail env resolve all [] [--path ] [--refresh] +trail env resolve component [--lane ] [--path ] [--refresh] +``` + +Resolution runs the exact host-validated argv against an isolated projection of +the lane's pinned source. A current snapshot is reused without launching the +tool; `--refresh` is the only way to rerun it. Discovery returns the exact +`resolve component` recovery argv, including `--lane`. Reviewed built-in +resolvers currently require an offline, credential-free plan. Repository and +plugin resolver execution fails closed until its restricted native resolver +sandbox is available. + +Inspect and verify shared artifacts or handle divergent-producer evidence with: + +```sh +trail env artifact inspect +trail env artifact verify --level attach|sample|full|reproduce +trail env artifact quarantine list +trail env artifact quarantine show +trail env artifact quarantine resolve \ + --resolution retain-private|accept-incumbent|accept-candidate|retire-all +trail env source export --component --export +``` + +These lifecycle reports support `--format json` and one-record +`--format ndjson` in addition to deterministic plain and human rendering. +Artifact verification exits nonzero when its returned report has `valid=false`. +Source export always replans and revalidates the pinned artifact, generation, +gate, source root, destination, collision policy, and explicit authorization +before making ordinary guarded source edits. + Inside exactly one mounted or managed lane, the lane argument to `sync all` and `--lane` for `sync component` may be omitted. Trail never guesses from the most recently used lane. Managed execution performs the same convergence @@ -534,6 +569,13 @@ trail backup restore [--force] | `backup verify` | Check that a backup is readable | | `backup restore` | Restore from a backup | +Backup output reports `retained_private_views`/`retained_private_bytes` and +`rebuildable_materializations`/`rebuildable_materialization_bytes` plus +`rebuildable_performance_caches`. Restore additionally reports +`restored_private_views`. Rebuildable counts describe deliberately omitted +host-local projections; authoritative artifact objects and bindings remain in +the backup. + ## Fsck ```text diff --git a/docs/reference/cli/lanes.md b/docs/reference/cli/lanes.md index 66f6e392..98155a3d 100644 --- a/docs/reference/cli/lanes.md +++ b/docs/reference/cli/lanes.md @@ -103,10 +103,26 @@ is quiescent. | Review readiness | `review`, `contribution`, `readiness`, `refresh-preview`, `handoff` | | Coordinate work | `claim`, `message` | | Workdir editing | `record`, `watch`, `read`, `hydrate`, `workdir`, `sync-workdir`, `checkout` | +| Storage accounting | `space` | | Patches and history | `apply-patch`, `diff`, `timeline`, `rewind` | | Validation | `test`, `eval`, `gates` | | Advanced records | `turn`, `run`, `events`, `trace` | +### Inspect lane storage + +```sh +trail lane space +trail --format json lane space +``` + +The structured report's `artifact_storage` object separates logical artifact +bytes, unique and cross-artifact-shared authoritative CAS bytes, physical +materializations, lane-private state, persisted prefetch, demand-loaded +projections, reclaimable data, and unknown allocation. Do not sum those fields: +logical, authoritative, physical, and reclaimable values are separate axes. +The human report presents the same counters. Native COW extents remain unknown +unless Trail has platform evidence proving their ownership. + ## Create and Inspect Lanes ```text diff --git a/docs/reference/config.md b/docs/reference/config.md index aba70b93..27bbeded 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -31,6 +31,17 @@ Use `trail config list`, `get`, and `set` to inspect and edit workspace config. | `lane.max_trace_payload_bytes` | u64 | no | Unsigned integer, zero disables the limit. | | `lane.worktrees_dir` | path | no | Relative path normalized inside workspace. | | `lane.merge_strategy` | string | no | `conservative`. | +| `workspace_views.upper_logical_bytes` | u64 | no | Per-view upper logical-byte limit; zero disables. | +| `workspace_views.upper_file_count` | u64 | no | Per-view upper file-count limit; zero disables. | +| `workspace_views.single_file_bytes` | u64 | no | Per-file upper limit; zero disables. | +| `workspace_views.journal_bytes` | u64 | no | Mutation-journal byte limit; zero disables. | +| `workspace_views.cache_build_bytes` | u64 | no | Per-construction candidate-byte limit; zero disables. | +| `workspace_views.concurrent_cache_builders` | u64 | no | Maximum concurrent builders; must be greater than zero. | +| `workspace_views.cache_retention_secs` | u64 | no | Minimum unreferenced materialization retention; zero allowed. | +| `workspace_views.cache_max_bytes` | u64 | no | Cache pressure ceiling; zero disables. | +| `workspace_views.cache_min_free_bytes` | u64 | no | Minimum filesystem free-space reserve; zero disables. | +| `workspace_views.prefetch_max_bytes` | u64 | no | Authenticated hot-set read ceiling; zero disables. | +| `workspace_views.prefetch_max_entries` | u64 | no | Authenticated hot-set entry ceiling; zero disables. | | `git.export_trailers` | bool | no | Boolean parser values. | | `guardrails.policy` | policy | no | `decision:scope:pattern` rules. | @@ -102,6 +113,15 @@ trail config set lane.max_event_payload_bytes 65536 trail config set lane.max_trace_payload_bytes 65536 ``` +## Workspace Artifact Limits + +The `workspace_views.*` keys bound private uppers, mutation journals, +construction candidates, verified materialization cache pressure, and advisory +hot-set reads. They do not change desired keys or grant publication authority. +Reaching a limit is an explicit error; it is not reported as an empty artifact +or successful cache hit. Prefetch warms the operating-system page cache and +does not create correctness-bearing bytes. + ## Guardrail Policy Grammar Rules are separated by semicolons or newlines: diff --git a/docs/reference/data-types.md b/docs/reference/data-types.md index d7942948..6dcb6186 100644 --- a/docs/reference/data-types.md +++ b/docs/reference/data-types.md @@ -58,6 +58,24 @@ This page summarizes public types used across CLI JSON, HTTP API, MCP structured - `LaneTestReport` - `LaneGateHistoryReport` +## Environment Artifact Reports + +- `ArtifactResolutionComponentReportV1` and `ArtifactResolutionBatchReportV1` +- `ArtifactInspectionReportV1` +- `ArtifactVerificationReportV1` and `ArtifactVerificationLevelV1` +- `ArtifactContentReachabilityReportV1` and `ArtifactReachabilityKindReportV1` +- `ArtifactAttestationReportV1` and `ArtifactAttestationVerificationReportV1` +- `ArtifactQuarantineListReportV1` and `ArtifactQuarantineResolutionReportV1` +- `ArtifactSourceExportPlanV1` and `ArtifactSourceExportExecutionReportV1` +- `ArtifactSpaceReportV1` and `ArtifactStorageAccountingReport` + +Inspection and verification reports carry the exact desired, tree, envelope, trust, +quarantine, binding, reachability, and storage identities applicable to the operation. +The accounting axes distinguish logical bytes, unique authoritative bytes, bytes shared +across artifacts, reconstructible materializations, lane-private bytes, and explicitly +unknown or reclaimable bytes. A zero value is used only when the axis is known to be zero; +the `accounting` field states the scope and attribution boundary. + ## Coordination Types - `LaneSession` @@ -189,6 +207,23 @@ This page summarizes public types used across CLI JSON, HTTP API, MCP structured - `BackupRestoreReport` - `ExternalMutationAuditRecord` +Backup reports separate retained source-view private state from omitted +rebuildable materializations and performance caches. Restore reports the number +of private views successfully staged and rebased. + +## Artifact Resolution Types + +- `ArtifactResolutionPlanV1`: pinned executable, argv, source inputs, authority and + policy bounds, candidate contract, limits, and validations. +- `ArtifactResolutionSnapshotV1`: immutable verified snapshot identity and provenance. +- `ArtifactResolutionAttemptReportV1`: durable fenced attempt, bounded evidence, and + success/failure/recovery state. +- `ArtifactResolutionRequestV1` and `ArtifactResolutionCandidateV1`: ephemeral Rust-only + executor handoff; these are intentionally not serializable. +- `ArtifactResolutionDecisionV1` and `ArtifactResolutionComponentReportV1`: one typed + `resolved`, `reused`, or deliberately `refreshed` decision. +- `ArtifactResolutionBatchReportV1`: deterministically ordered same-source-root result. + ## Code Facts Used - IDs: `trail/src/ids.rs` diff --git a/docs/reference/http-api.md b/docs/reference/http-api.md index ea1ae42b..c184077e 100644 --- a/docs/reference/http-api.md +++ b/docs/reference/http-api.md @@ -4,6 +4,9 @@ Environment sync reports include per-component cache decisions, storage identity, rebuild reason, exact changed identity edges, and byte accounting. `POST /v1/lanes/{lane_or_id}/environment/promote` accepts `component` and `output` and returns the durable publication and successor-generation report. +Artifact resolution, inspection, verification, quarantine, reachability, +workspace accounting, and source export routes serialize the same public Rust +report types used by CLI JSON/NDJSON and MCP structured content. Lane spawn reports include the shared layered-backend prerequisite report. The daemon serves JSON HTTP routes under `/v1`. @@ -115,6 +118,9 @@ x-trail-token: | POST | `/v1/lanes/{lane_or_id}/tests` | Run test gate. | | POST | `/v1/lanes/{lane_or_id}/evals` | Run eval gate. | | POST | `/v1/lanes/{lane_or_id}/patches` | Apply lane patch. | +| POST | `/v1/lanes/{lane_or_id}/environment/resolve` | Resolve or reuse one pinned component snapshot. Body: `component`, optional `path` and `refresh`. | +| POST | `/v1/lanes/{lane_or_id}/environment/resolve-all` | Resolve or reuse every incomplete component snapshot. Body: optional `path` and `refresh`. | +| POST | `/v1/lanes/{lane_or_id}/environment/source-export` | Export one declared generated-source subtree through normal lane source writes. Body: `component` and `export`. | Patch requests accept either native `edits` or compatibility `files`; provide one non-empty array, not both. @@ -158,6 +164,23 @@ branch requires `direct=true`; otherwise enqueue with `POST /v1/lanes/merges/queue` and run the queue. The former branch-scoped `merge-lane` endpoint was removed. +## Artifact Routes + +| Method | Path | Purpose | +| --- | --- | --- | +| GET | `/v1/artifacts/space` | CAS-aware workspace artifact accounting. | +| GET | `/v1/artifacts/{artifact_id}` | Inspect one artifact envelope and its bindings, attestations, quarantine, reachability, and storage evidence. | +| GET | `/v1/artifacts/{artifact_id}/reachability` | Summarize authoritative objects reachable from one envelope. | +| POST | `/v1/artifacts/{artifact_id}/verify` | Verify at `attach`, `sample`, `full`, or `reproduce` level. | +| GET | `/v1/artifact-quarantines` | List active and resolved nondeterminism quarantines. | +| GET | `/v1/artifact-quarantines/{quarantine_id}` | Show retained divergence evidence. | +| POST | `/v1/artifact-quarantines/{quarantine_id}/resolve` | Apply `retain_private`, `accept_incumbent`, `accept_candidate`, or `retire_all`. | + +Resolution executes only a host-owned plan with pinned source and tool +identity. `refresh: false` reuses an existing valid snapshot; wall-clock time +alone never advances dependencies. Source export is an explicit guarded source +mutation and is distinct from artifact promotion or mounting. + ## Collaboration Routes | Method | Path | Purpose | diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 455a52ad..c56143b8 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -268,6 +268,26 @@ before non-dry-run apply. - `trail.lane_rewind` - `trail.lane_remove` +## Lane Environments and Artifacts + +- `trail.env_resolve` +- `trail.env_resolve_all` +- `trail.artifact_space` +- `trail.artifact_inspect` +- `trail.artifact_reachability` +- `trail.artifact_verify` +- `trail.artifact_quarantine_list` +- `trail.artifact_quarantine_show` +- `trail.artifact_quarantine_resolve` +- `trail.env_source_export` + +Resolve tools are open-world writes because a reviewed host-owned resolver may +execute an exact external package-manager command. Artifact space, inspection, +reachability, verification, and quarantine queries are guarded read-only calls. +Quarantine resolution and source export are destructive writes because they +change retained artifact policy or normal lane source state. Every tool returns +the same typed report serialized by Rust, CLI JSON/NDJSON, and HTTP. + ## Sessions, Approvals, Runs, Leases, Anchors - `trail.session_start` @@ -387,18 +407,30 @@ table. ## Resources -Static resources include workspace status, doctor, lanes, merge queue, +Static resources include workspace status, doctor, lanes, artifact space, +artifact quarantines, merge queue, conflicts, OpenAPI, documentation, and agent task dashboard resources: +`trail://workspace/artifacts/space`, +`trail://workspace/artifact-quarantines`, `trail://workspace/agent-tasks`, `trail://workspace/agent-tasks/latest/review`, `trail://workspace/agent-tasks/latest/review-data`, `trail://workspace/agent-tasks/latest/changes`, `trail://workspace/agent-tasks/latest/files`, and `trail://workspace/agent-tasks/latest/focus`. Resource templates cover -individual agent task review-data/review/changes/file/report/focus dashboards, lanes, lane +individual artifact envelopes at +`trail://workspace/artifacts/{artifact_id}`, individual quarantines at +`trail://workspace/artifact-quarantines/{quarantine_id}`, agent task +review-data/review/changes/file/report/focus dashboards, lanes, lane review packets, sessions, turns, conflicts, approvals, run states, and trace spans. +`completion/complete` supplies sorted, bounded candidates for `artifact_id` +and `quarantine_id` on those two templates. Candidates come from durable local +envelope/quarantine rows and use the same prefix-first, substring-second, +100-value limit as other Trail resource completions. An empty workspace returns +an empty completion, not an unknown-template error. + ## Code Facts Used - Tools: `trail/src/mcp/tools` diff --git a/openspec/changes/generalize-framework-artifact-pipeline/.openspec.yaml b/openspec/changes/generalize-framework-artifact-pipeline/.openspec.yaml new file mode 100644 index 00000000..913564e4 --- /dev/null +++ b/openspec/changes/generalize-framework-artifact-pipeline/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-08 diff --git a/openspec/changes/generalize-framework-artifact-pipeline/design.md b/openspec/changes/generalize-framework-artifact-pipeline/design.md new file mode 100644 index 00000000..2cbc8a45 --- /dev/null +++ b/openspec/changes/generalize-framework-artifact-pipeline/design.md @@ -0,0 +1,515 @@ +## Context + +`optimize-large-repo-lanes` is now implemented on the current Trail branch. This follow-up therefore starts from shipped behavior, not from the earlier design assumption that output policy, COW inheritance, promotion, singleflight, lazy layer access, or user-defined command recipes still need to be invented. + +### Audited baseline + +| Shipped foundation | Current ownership/evidence | Consequence for this change | +| --- | --- | --- | +| `immutable_shared`, `immutable_seed_private`, `writable_private`, and `disposable` output policies | `EnvironmentOutputPolicy`, SDK `AdapterOutputPolicy`, schema output rows | Preserve these values and extend their backing identity/storage; do not replace the policy model. | +| Reuse modes, sharing scopes, and `never`/`manual`/`on_sync`/`successful_gate` publication | report models, SDK, recipes, generation outputs | Reuse existing policy fields and promotion authorization. | +| Typed component DAG and `build_requires`, `invalidates_with`, `runtime_requires`, `binds_after` edges | environment planner, command recipes, protocol v2 | Extend key inputs and discovery outcomes; do not add another graph scheduler. | +| Bounded parallel preparation and per-key singleflight | `workspace_environment.rs`, `workspace_layer.rs` | Upgrade ownership and evidence for content sealing; keep scheduling semantics. | +| Journaled promotion into a successor generation | `workspace_layer_publications` and `env promote` | Reuse the publication fence, quiescence, and activation transaction. | +| Immutable lower plus fresh private COW state, component-granular inheritance | workspace layers/views and native NFS/FUSE/Dokan tests | Change only how immutable lower content is stored/materialized. | +| Paged layer manifests, verification stamps, prefetch, quota, GC, and lane space | `workspace_layer.rs`, cache CLI/reports | Make these CAS-aware; retain the operations and reports. | +| `trail.environment.toml` v1 includes/profiles and restricted `trail/command@1` components | `workspace_recipe.rs`, adapter documentation | Add a v2 schema in the same file; do not create `trail.artifacts.toml`. | +| Adapter protocols v1/v2 and signed package trust/revocation | adapter SDK and plugin host | Add explicit v3 negotiation and conversion. | +| CLI/HTTP/MCP/Rust parity for environment operations | shared lane reports and interface tests | Extend shared models; do not implement transport-specific domain behavior. | + +### Remaining gaps proven by the audit + +1. Cargo discovery requires both `Cargo.toml` and `Cargo.lock`; Node discovery similarly requires `package.json` plus a supported lock. A manifest-only component disappears instead of reporting resolution state. +2. `WorkspaceLayerKeyV1` describes desired inputs, and `layer_id` is derived from its digest. Produced byte identity is not independent from the request identity. +3. Publication copies a complete output directory to `.trail/cache/layers/`. Manifest pages store file hashes, but file bytes are authoritative only in that copied tree. Two desired keys producing equal files therefore duplicate those bytes. +4. The current layer manifest includes desired-key and publication fields such as layer ID and creation time, so it cannot serve as a pure deterministic content-tree identity. +5. There is no durable resolution snapshot, artifact nondeterminism quarantine, or artifact-specific attestation. +6. `trail.environment/v1` command recipes are intentionally restricted to the shipped subset. They cannot express controlled resolution, typed validations, several action phases, or explicit generated-source export. + +The product boundary remains: adapters understand frameworks; Trail understands artifact lifecycle. Git remains the publication system for source. `.trail/` contains private operational state. Adapters plan but do not execute, mount, publish, update SQLite, resolve raw secrets, or collect storage. + +### Deliverables + +| Deliverable | Increment beyond the merged baseline | Acceptance evidence | +| --- | --- | --- | +| Visible incomplete discovery | Proposal statuses and recovery actions without executing tools | Cargo/Node manifest-only fixtures and side-effect assertions | +| Resolution snapshots | Pinned, content-addressed environment metadata with controlled authority | resolver success/failure/reuse/refresh/recovery tests | +| Identity v2 | separate desired, tree-content, artifact, and binding identities | canonicalization, invalidation, divergent-output, reopen tests | +| Artifact CAS | deterministic directory/file/chunk objects and reconstructible layer materializations | equal-content and successor deduplication tests | +| Existing lifecycle integration | CAS-backed sync/promotion/inheritance/GC using current generation and COW machinery | crash matrix plus native sibling isolation | +| Repository contract v2 | extensions to `trail.environment.toml`, including source export | parser/adversarial/custom-pipeline E2E tests | +| Adapter protocol v3 | incomplete discovery, resolution, validation, capability, and identity wire types | SDK golden/compatibility/malicious-frame tests | +| Trust evidence | capability profiles, secret taint, quarantine, deterministic attestation | sandbox, redaction, revocation, tamper tests | + +## Goals / Non-Goals + +**Goals:** + +- Represent recognized but incomplete framework components honestly. +- Resolve dependency state explicitly without requiring every generated lock snapshot to enter Git. +- Reuse unchanged artifact bytes across lanes, successor builds, and different desired keys. +- Preserve existing immutable-lower/private-upper isolation and atomic generation activation. +- Make every reuse, rebuild, quarantine, rejection, export, and collection decision explainable. +- Let repository authors compose the same normalized pipeline through an extension of the existing environment file. +- Keep unproven framework output private and make generated-source publication explicit. +- Produce portable content identities and attestations without accepting remote content automatically. + +**Non-Goals:** + +- Replacing package managers or framework build schedulers. +- Sharing a live writable target, `node_modules`, `.next`, virtual environment, database, process, secret, or cache without a separately certified concurrent protocol. +- Inferring correctness inputs from filesystem observation or syscall tracing. +- Automatically resolving dependencies during managed execution unless workspace policy explicitly opts in for the adapter and authority set. +- Automatically writing resolution snapshots or generated output into source history. +- Adding remote cache transport, remote execution, registry hosting, or distributed leases. +- Migrating incompatible `.trail/` database shapes; Trail retains schema-v1 hard-cutover behavior. + +## Decisions + +### 1. Extend the shipped environment pipeline + +The host operation remains desired-state convergence: + +```text +discover pinned source facts (read-only) + -> resolve missing dependency identity (optional and policy-controlled) + -> finalize the existing typed component DAG + -> compare desired keys with active generation/artifacts + -> construct unresolved nodes through current bounded scheduling/singleflight + -> validate and seal deterministic content + -> activate one successor environment generation + -> mount existing immutable lowers plus lane-private uppers + -> execute and checkpoint source + -> retire bindings and collect unreachable content +``` + +`resolve`, `construct`, `validate`, and `seal` have separate durable evidence, but they feed the current environment sync, promotion, generation, mount, retirement, and GC operations. There is no second artifact scheduler. + +Alternatives rejected: + +- A parallel artifact engine would duplicate generation and recovery authority. +- Framework-specific cache managers would repeat COW, identity, trust, and collection logic. +- A generic shell lifecycle would make argv, capabilities, inputs, outputs, and recovery ambiguous. + +### 2. Discovery returns proposals, including incomplete components + +Discovery is marker-driven and side-effect-free. A recognized manifest produces `ArtifactComponentProposalV1` even when a lock snapshot, tool, permission, or platform capability is missing. + +| Status | Meaning | +| --- | --- | +| `ready` | All identity-bearing planning inputs are present and verified. | +| `resolvable` | An adapter supplied a resolver plan and current policy can authorize it. | +| `blocked` | The component is understood but a tool, authority, credential reference, platform feature, or approval is unavailable. | +| `unsupported` | The framework form is recognized but cannot be represented safely. | +| `ambiguous` | Multiple component roots/adapters claim one identity and explicit selection is required. | + +The proposal contains stable reason codes and exact recovery commands. Discovery never invokes a package manager, compiler, repository action, network endpoint, runtime provider, or secret provider. + +### 3. Resolution snapshots are environment metadata + +`ArtifactResolutionPlanV1` pins: + +- proposal and source-root identity; +- exact resolver executable and argv; +- readable manifest/config inputs; +- writable candidate path; +- allowed network authorities and opaque credential handles; +- script policy, environment-name roles, resource limits, and snapshot format; +- structural and ecosystem validation rules. + +The resolver runs in a separate deny-by-default profile. On success Trail ingests a deterministic `ArtifactResolutionSnapshotV1` containing resolved identities, checksums, source proposal key, resolver/tool/policy identity, contacted authority names, predecessor, and format version. Credential bytes and authorization headers are never stored. + +Snapshots remain pinned until their identity inputs change or the user requests `--refresh`. Time alone never advances dependency selection. Managed execution fails with the required `trail env resolve ...` command unless policy explicitly permits automatic resolution for the adapter and exact authority set. + +A snapshot can be exported to a conventional lockfile only through the source-export operation in Decision 10. + +### 4. Keep four independent identities + +The domain model separates: + +```text +ArtifactDesiredKeyV2 + what output is correct for a normalized plan and its declared inputs + +ArtifactTreeRootV1 + exact deterministic directory/file/chunk content and relevant metadata + +ArtifactEnvelopeV1 / ArtifactId + desired key + tree root + output contract + portability/trust + attestation + +ArtifactGenerationBinding + lane view + generation + component/output + artifact or private storage +``` + +`ArtifactDesiredKeyV2` uses canonical CBOR and SHA-256 over: + +- contract, adapter implementation, package distribution, and selected protocol; +- resolution snapshot identity; +- declared source/input closure and semantic normalizer versions; +- identity-bearing upstream desired/artifact identities; +- exact action executable identities, argv, working directories, and phase order; +- output, validation, source-export, network, script, secret-taint, sandbox, and publication policy; +- identity-affecting non-secret environment; +- target, platform, architecture, ABI, portability, reuse, and trust scope. + +Maps and sets are sorted; paths are normalized NFC relative paths; absent and empty fields remain distinct; all enum and codec versions are explicit. When an adapter cannot prove a complete closure, the complete pinned Trail source root and conservative host dimensions enter the desired key, or reusable publication is refused. + +The existing `WorkspaceLayerKeyV1` remains readable only for explicitly supported v1/v2 compatibility projections. New publications use v2 identity. A v1 key is never reinterpreted as v2. + +### 5. Split pure content manifests from artifact provenance + +The current layer manifest mixes tree hashes with layer/request/publication metadata. The new object graph is: + +```text +ArtifactEnvelopeV1 + -> DirectoryNodeV1 root + -> DirectoryNodeV1 children + -> FileNodeV1 + -> BlobObjectV1 (small file) + -> ChunkListV1 -> ChunkObjectV1 (large file) + -> ArtifactAttestationV1 +``` + +Pure content nodes exclude desired keys, adapter names, timestamps, storage paths, layer IDs, generation IDs, and publication attempt IDs. They include normalized entry names, kind, executable/portable mode bits, safe symlink targets, file size, complete-file SHA-256, and child object identities. Equivalent trees produce the same root regardless of traversal order or producing component. + +Artifact envelopes associate the pure root with the desired key, output contract, portability, trust scope, verification policy, and attestation. Different desired keys may point at the same tree root without sharing correctness identity. + +### 6. Use versioned whole-file and content-defined chunk objects + +Files at or below 1 MiB use one immutable blob object. Larger files use `fastcdc-v1` with minimum 256 KiB, target 1 MiB, and maximum 4 MiB chunks. Every file retains a full SHA-256 independent of chunking, and every chunk is independently SHA-256 addressed. Chunk profile and thresholds are versioned in `FileNodeV1`; changing them creates a new file-node identity without relabeling old objects. + +Ingestion is streaming and bounded by declared entry, byte, depth, path, chunk, and time limits. It rejects absolute or traversing paths, non-NFC names, case collisions, escaping links, unsupported special files, prohibited metadata/xattrs, concurrent mutation, and secret-policy violations before publication. + +Alternative rejected: per-layer full directory copies are simple but duplicate equal files and make layer directories the only authoritative bytes. Fixed-size chunks are simpler but amplify changes after insertions in large archives and package indexes. + +### 7. CAS objects are authoritative; layer directories become materializations + +Immutable content objects use Trail's existing object publication and validation boundary. SQLite stores durable references and coordination; the object graph stores authoritative content. A materialized layer under the current cache hierarchy is reconstructible and never the only copy of a ready artifact. + +Backends consume content in two ways: + +- lazy NFS/FUSE/Dokan projections resolve manifest entries and materialize requested file content on demand; +- backends requiring real directories use a verified materialization cache keyed by tree root and backend compatibility. + +Implementation status: the shared backend-neutral workspace core resolves +verified artifact-tree bindings one directory object at a time, serves file +ranges from only the overlapping blob/chunks, and materializes only a selected +file for copy-up. It treats the CAS manifest as authoritative over any legacy +layer directory and preserves existing upper/whiteout behavior. Native FUSE, +NFS, and Dokan acceptance tests exercise the same nonexistent-materialization +fixture; owning-host gates remain the authority for platform qualification. +Backends that require real paths now reuse a verified real-directory cache keyed +by the authoritative tree root and an operating-system/architecture/backend +compatibility identity. Publication projects and verifies CAS content in an +attempt-owned stage, records durable `building`/`verified`/`failed` state, and +atomically publishes an immutable root. Reuse revalidates content and restores +read-only permissions; missing or corrupt cache bytes rebuild from CAS. Private +projections prefer native clone/reflink and fall back to file copies without +hard-linking mutable consumers to the authoritative cache. Cache eviction, +reachability accounting, and quotas remain tasks 6.6-6.8. + +Materialization can use safe platform clone/reflink facilities or immutable content copying. Hard links are allowed only where mode and ownership invariants cannot mutate the content object. Materialization amplification is reported separately and is reclaimable without invalidating an artifact. + +The current `.trail/cache/layers/` layout remains an implementation compatibility surface during rollout, but after CAS activation it contains only verified materialization state and sidecars. Doctor/fsck must detect legacy authoritative-layer layouts versus CAS-backed layouts exactly. + +### 8. Reuse existing publication, promotion, and generation fences + +The shipped publication/promotion operation already pins source/generation/output, quiesces private state, validates containment, publishes an immutable layer, and activates a successor generation. This change inserts content sealing into that path: + +```text +reserved -> constructing -> candidate_closed -> validating + -> sealing_objects -> envelope_ready -> activating -> activated +``` + +Terminal alternatives are `failed`, `cancelled`, `quarantined`, and `repair_required`. Objects become durable before an envelope becomes ready. The existing activation transaction then binds the ready artifact to a successor generation. A crash can leave an unbound ready artifact, never a partially active generation. + +Promotion never mutates or deletes the live private upper. Source, desired key, generation, private-output journal, gate, or validation changes after the fence make the attempt stale and prevent activation. + +Fork inheritance now makes an output-level decision rather than trusting a +component-level layer hit. Before reuse, Trail verifies the child desired key, +the ready artifact envelope and complete content tree, the currently installed +adapter implementation/package identity and trust, output sharing policy, +portability, and native backend support. Only individually accepted immutable +outputs are copied into the successor generation, each with a fresh generation +binding identity. Lane-private source, generated, scratch, seeded, writable, +and disposable state is never inherited by identity. + +### 9. Upgrade singleflight and quarantine nondeterminism + +The existing key lock remains the fast mutual-exclusion boundary, but durable attempt rows record owner token/generation, PID/start identity, heartbeat, pins, current phase, candidate root, waiters, error, and recovery command. At most one live reusable constructor owns `(trust_scope, desired_key)`. + +If two structurally valid constructions for one desired key and trust scope produce different tree roots, Trail: + +1. marks the desired key quarantined; +2. retains bounded references to both candidates and their attestations; +3. prevents automatic shared attachment or publication for that key; +4. reports the differing roots, adapter/tool/policy identities, and available deterministic comparison; +5. optionally permits a policy-controlled lane-private rebuild/retention that is never reported as a shared hit. + +Quarantine resolution never relabels bytes. A user can revoke a producer, narrow policy, replace the desired key contract, or clear a demonstrably corrupt candidate through an explicit audited operation. + +### 10. Generated source uses explicit source export + +`publishable_source` is not an artifact output policy. Artifact output stays under one of the four shipped storage policies. A separate `[[component.source_export]]` declaration identifies a validated candidate subtree and a repository-relative destination. + +Export requires an explicit user operation unless organization policy authorizes a named deterministic gate. Trail revalidates the pinned desired key, artifact/tree identity, source root, destination containment, ignore and guardrail policy, collision behavior, secret policy, and current lane state. It then writes through the normal source-operation path so the result appears in lane diff/review and can be checkpointed and merged through Git. + +Artifact promotion and source export are intentionally different: + +| Operation | Destination | Git-visible | Automatic default | +| --- | --- | --- | --- | +| artifact promotion | immutable Trail artifact + generation | no | policy-controlled | +| source export | lane source upper | yes | never | + +### 11. Extend `trail.environment.toml` with schema v2 + +Trail retains both supported source paths, `trail.environment.toml` and `.trail/environment.toml`. New behavior uses `schema = "trail.environment/v2"`. Version 1 keeps its exact current parsing and restrictions; v2 is opt-in and never changes v1 defaults silently. + +Representative v2 composition: + +```toml +schema = "trail.environment/v2" + +[environment] +default_network = "deny" +default_scripts = "deny" +missing_resolution = "explicit" + +[[component]] +id = "web.dependencies" +root = "apps/web" +adapter = "trail/node@2" + +[[component.input]] +path = "apps/web/package.json" +role = "identity" +format = "bytes" + +[component.resolve] +command = ["pnpm", "install", "--lockfile-only", "--ignore-scripts"] +cwd = "apps/web" +network = { authorities = ["registry.npmjs.org"] } +snapshot = "pnpm-lock.yaml" +format = "application/vnd.pnpm.lock+yaml" + +[[component.action]] +phase = "construct" +command = ["pnpm", "install", "--frozen-lockfile", "--offline", "--ignore-scripts"] +cwd = "apps/web" + +[[component.output]] +name = "modules" +source = "node_modules" +target = "apps/web/node_modules" +policy = "immutable_seed_private" +reuse = "exact" +scope = "workspace" +publish = "on_sync" + +[[component.validation]] +kind = "path_contract" +path = "node_modules" + +[[component]] +id = "web.client" +root = "apps/web" +adapter = "trail/command@2" + +[[component.edge]] +component = "web.dependencies" +type = "build_requires" + +[[component.action]] +phase = "construct" +command = ["pnpm", "exec", "vite", "build"] +cwd = "apps/web" + +[[component.output]] +name = "dist" +source = "dist" +target = ".trail-generated/web-dist" +policy = "immutable_shared" +publish = "successful_gate" +gate = "build" + +[[component.source_export]] +from_output = "dist" +source = "generated-client" +target = "apps/web/src/generated-client" +mode = "explicit" +``` + +The v2 parser has strict unknown-field handling, bounded includes/profiles, fixed argv, normalized paths, deterministic pattern expansion, and capability narrowing. Repository-authored plans cannot request shells, raw secrets, host-wide reuse, provider sockets, arbitrary child processes, or compatible semantic reuse unless a certified profile supplies that authority. + +### 12. Add adapter protocol v3 as a delta + +`trail.environment-adapter/v3` adds only concepts absent from v2: + +- proposal status, missing requirements, and recovery actions; +- resolution-plan and snapshot schemas; +- input roles and complete-closure certification; +- multiple typed action phases and validations; +- desired/content/envelope identity evidence; +- effective capability profile and secret-taint result; +- attestation and quarantine descriptors; +- explicit source-export declarations. + +Negotiation selects the highest exact mutually supported version. V1/v2 responses go through their existing compatibility conversion and cannot request v3-only resolution, source export, attestation, or compatibility certification. Package digest and selected protocol participate in the desired key. SDK builders reject invalid combinations; the host repeats full validation at the trust boundary. + +### 13. Separate resolver, constructor, validator, and execution trust + +Capability profiles are deny-by-default and phase-specific: + +| Profile | Network | Writes | Secrets | Publication authority | +| --- | --- | --- | --- | --- | +| discovery/planning | none | none | none | none | +| resolver | exact authorized authorities | isolated snapshot candidate | opaque handles only | none | +| constructor | offline by default | declared candidate/cache/temp | none by default | none | +| validator | none by default | receipt only | none | none | +| mounted execution | declared lane bindings | lane-private/source as authorized | runtime injection | none | + +Reviewed built-ins, certified signed plugins, locally trusted plugins, and repository declarations have distinct maximum profiles. Host/workspace policy can narrow but never widen them. Unsupported native enforcement fails closed for untrusted actions. + +Any producer receiving secret bytes is tainted and may create only private, non-promotable output in this version. Secret bytes never enter keys, snapshots, objects, manifests, logs, reports, shared caches, attestations, or future remote requests. + +### 14. Attest sealed artifacts without polluting content identity + +`ArtifactAttestationV1` deterministically records: + +- desired key, tree root, artifact envelope, source root, resolution snapshot, and upstream identities; +- adapter implementation/package/publisher/protocol and revocation state at publication; +- executable identities, argv, platform/ABI, sandbox enforcement, network/script policy, and non-secret environment roles; +- output contract, validations, gates, attempt identity, and producer trust tier; +- portability and allowed sharing scope. + +Wall-clock timestamps and local paths are stored as non-identity observation fields or separate attempt metadata. Artifact attachment verifies content, scope, producer trust/revocation, and required evidence. Portable identity does not imply remote trust; imported content remains untrusted until a future transport policy verifies signatures and local acceptance. + +### 15. Extend existing public operations + +Existing `env discover`, `graph`, `plan`, `explain`, `sync`, `promote`, `lane space`, and `cache gc` retain their grammar. Add: + +```text +trail env resolve all [] [--refresh] +trail env resolve component [--lane ] [--refresh] +trail env artifact inspect +trail env artifact verify --level attach|sample|full|reproduce +trail env artifact quarantine list|show|resolve +trail env source export --component --export +``` + +All library, CLI, HTTP/OpenAPI, and MCP surfaces use shared report types. Reports include proposal state, desired/tree/artifact/binding identities, decision source, invalidating edges, attempt phase, verification/trust/quarantine state, logical/unique/shared/materialized/private bytes, and exact recovery commands. + +### 16. Extend schema-v1 durable truth coherently + +Fresh schema creation and validation add records for: + +| Record | Durable purpose | +| --- | --- | +| resolution snapshots/attempts | source pins, snapshot object, resolver evidence, state, owner/recovery | +| artifact trees/envelopes | desired key, content root, manifest object, trust/verification/quarantine state | +| construction attempts/waiters | fenced ownership, phase, candidate, errors, bounded consumers | +| attestations | immutable statement object and optional signature metadata | +| quarantines/holds | divergent candidates, reason, retention, audited resolution | +| generation outputs | exact artifact envelope or private storage binding | +| content reachability index | rebuildable traversal/accounting acceleration, never sole truth | + +Directory/file/chunk/snapshot/envelope/attestation objects are reachable from durable refs and generation bindings. GC traces object edges and retains active/retained generations, live/recoverable attempts, leases, quarantines, backups, and pins. Materialization and prefetch caches are independently reclaimable. + +No database migration is added. An incompatible existing workspace fails before mutation with backup and `trail init --force` guidance. Backup/restore includes authoritative objects and retained private state, while explicitly reporting omitted reconstructible materializations and performance caches. + +### 17. Frameworks are conformance compositions + +| Shape | Resolution/content composition | Required conservative behavior | +| --- | --- | --- | +| Cargo | manifest proposal -> generated/source lock snapshot -> target seed/private upper | complete source-root fallback until workspace/features/build-script closure is certified | +| npm/pnpm/Yarn/Bun | package proposal -> frozen graph -> dependency seed/content cache | lifecycle-script writes isolated; unsupported lock form remains visible | +| Next.js | Node dependency component -> framework/config/build component | `.next/cache` private; output shareable only with relocatability validation | +| Vite | Node dependency component -> plugin/config/mode build | immutable `dist` or explicit source export; optimizer state private/seeded only when certified | +| Python | project proposal -> hash-bearing resolution -> wheels/downloads -> private `.venv` | path-bound environment and bytecode remain private | +| Go | module graph/checksum snapshot -> vendor/external module content | build cache uses certified cache protocol, not artifact authority | +| Maven/Gradle | dependency checksum graph -> immutable dependency objects | daemons and build directories remain private | +| CMake | toolchain/config inputs -> private configure/build tree | only certified compiler cache/content is shared | +| Bazel/Nix | imported verified external-store identities | avoid duplicating stores; workspace output remains private unless exported | +| custom | `trail.environment/v2` fixed graph/actions | unproven closure or portability narrows to private/exact workspace reuse | + +Trail core contains storage and lifecycle concepts, not `nextjs`, `vite`, or package-manager-specific COW modes. + +### 18. Verification and performance gates are evidence-based + +The acceptance matrix must prove: + +- discovery performs no process/network/provider side effects; +- equal desired keys singleflight to one verified artifact; +- different desired keys producing equal files reuse identical content objects; +- changing a bounded subset of a successor tree creates only affected directory/file/chunk objects; +- divergent content for one desired key quarantines all shared candidates; +- 1/5/20 sibling lanes share immutable content while every mutation/whiteout remains private; +- source export is the only path by which declared generated source enters a lane diff; +- retirement/GC preserves chunks reachable from any artifact and reclaims last-reference content deterministically; +- doctor, fsck, backup, and restore preserve or reconstruct the correct authority; +- 10k/100k/1M-entry artifacts remain bounded in memory, manifest paging, startup, and reporting; +- NFS, FUSE, and Dokan evidence is reported only from owning native hosts; skipped evidence remains unverified. + +## Risks / Trade-offs + +- **CAS object count and SQLite reachability can become large** -> page manifests, stream traversal, batch indexed lookups, incremental GC, and bound every operation. +- **Content-defined chunking adds CPU and dependency risk** -> version the profile, benchmark against whole-file storage, retain full-file hashes, and use whole blobs for small files. +- **Materialization caches can temporarily amplify disk use** -> report authoritative versus materialized bytes separately, enforce quotas, and reclaim materializations independently. +- **Resolution can introduce network drift** -> require pinned proposal inputs, explicit authority, immutable snapshots, manual refresh, and offline construction where supported. +- **Repository schema v2 can become a build language** -> keep fixed argv and finite typed phases, reject control flow/shell interpolation, and route semantic logic to adapters. +- **Source export can overwrite user work** -> require pinned source identity, explicit conflict policy, normal source guardrails, reviewable diff, and atomic confined writes. +- **Desired-key v2 invalidates existing cache hits** -> preserve read-only v1 inspection but never reinterpret identities; communicate hard-cutover and allow rebuild from source. +- **Native backend behavior differs** -> keep semantics in the host model and require backend-specific lower-integrity, whiteout, crash, and lazy-read evidence. + +## Migration Plan + +### Phase 0: Freeze the merged baseline + +- Archive/synchronize `optimize-large-repo-lanes` separately when release workflow permits. +- Pin current v1/v2 wire fixtures, `trail.environment/v1`, output enums, promotion reports, schema shape, and native COW behavior. +- Add failing tests that demonstrate manifest-only discovery omission, desired/content conflation, and duplicate whole-directory bytes. + +### Phase 1: Read-only proposals, identities, and planning + +- Add proposal statuses, resolution plans, desired-key v2, tree/envelope models, and report projections without executing resolution or changing layer storage. +- Add `trail.environment/v2` parsing and graph normalization behind an explicit schema tag. +- Keep existing sync/publication behavior authoritative. + +### Phase 2: Resolution snapshots + +- Add resolver attempts, capability enforcement, snapshot objects, explicit CLI/API operations, recovery, and source materialization/export. +- Migrate Cargo and Node manifest-only discovery first. + +### Phase 3: CAS shadow publication + +- Build deterministic tree/file/chunk objects alongside current layer publication. +- Verify new content roots against current copied trees without attaching from CAS. +- Measure object count, chunking CPU, logical/physical bytes, and successor reuse. + +### Phase 4: CAS-backed attachment and promotion + +- Make new artifacts authoritative through envelope objects and materialization projections. +- Route existing sync, promotion, inheritance, verification, backup/restore, doctor/fsck, and GC through CAS-aware operations. +- Preserve old active generations only under the explicit hard-cutover/reinitialization contract. + +### Phase 5: Quarantine, attestations, and protocol v3 + +- Activate nondeterminism detection, trust-scoped attestations, SDK v3, plugin negotiation, and malicious-input fixtures. +- Keep v1/v2 behavior exact and deny v3-only features to compatibility projections. + +### Phase 6: Adapter and framework qualification + +- Migrate a fixture adapter, Cargo, Node, Next.js/Vite composition, Python, and representative custom/external shapes. +- Run native multi-lane, scale, recovery, deduplication, and real-tool gates. + +### Phase 7: Public hard cutover and release + +- Update every public interface and reference document. +- Require backup/reinitialization guidance for incompatible schema-v1 state. +- Release only with exact passed, failed, and skipped platform evidence. + +Rollback before CAS-backed activation disables new planning/resolution and leaves existing storage authoritative. After the schema-v1 hard cutover, rollback requires restoring a pre-cutover backup or reinitializing from Git; Trail does not attempt an in-place downgrade. + +## Open Questions + +None block implementation. The design fixes these choices: extend `trail.environment.toml`; keep four artifact output policies; model generated source as explicit source export; use desired-key v2 plus independent content roots; use SHA-256 and versioned FastCDC; preserve existing COW/promotion/generation machinery; negotiate protocol v3 explicitly; and defer remote transport. diff --git a/openspec/changes/generalize-framework-artifact-pipeline/proposal.md b/openspec/changes/generalize-framework-artifact-pipeline/proposal.md new file mode 100644 index 00000000..9f3e44ea --- /dev/null +++ b/openspec/changes/generalize-framework-artifact-pipeline/proposal.md @@ -0,0 +1,43 @@ +## Why + +The merged `optimize-large-repo-lanes` change now provides Trail's framework-neutral output policies, layered COW isolation, typed component graphs, singleflight construction, explicit promotion, fork inheritance, cache management, and `trail.environment.toml` authoring foundation. The remaining limitation is narrower but fundamental: manifest-only repositories are not represented when lock state is absent, desired keys still double as layer identities, and whole directory layers duplicate equal files across successor artifacts and independently keyed builds. + +Trail should extend the shipped environment model with explicit resolution snapshots and a content-addressed artifact pipeline. Adapters continue to describe framework semantics, while the existing Trail host remains the sole authority for execution, validation, publication, generation binding, recovery, and collection. + +## What Changes + +- Extend side-effect-free discovery with typed `ready`, `resolvable`, `blocked`, `unsupported`, and `ambiguous` proposals so Cargo, Node, Python, Java, and custom components remain visible when lock or resolution state is missing. +- Add explicit, policy-controlled resolution actions that produce immutable Trail-managed snapshots as environment metadata; snapshots never become Git source unless a user invokes a separate source-materialization operation. +- Extend the existing `trail.environment.toml` schema instead of introducing a second repository artifact file. Add resolution, validation, multi-action, capability, and explicit source-export declarations that compile to the same host-owned component graph. +- Separate the desired component key, deterministic produced-content identity, artifact envelope/attestation identity, and lane-generation binding. Preserve the existing component graph and generation model while making those identities independently inspectable. +- Replace whole-directory duplication beneath published layers with deterministic directory/file/chunk content objects and reconstructible materializations. Existing immutable lowers and private COW uppers remain the lane isolation mechanism. +- Retrofit the existing publication and promotion state machines so they seal content objects, validate deterministic manifests, activate successor generations atomically, and recover without mutating a live private upper. +- Add nondeterminism quarantine when the same desired key produces different verified content roots; permit only an explicitly reported lane-private fallback while the key is quarantined. +- Add deterministic artifact attestations and distinct resolver, constructor, validator, and mounted-execution capability profiles. Secret-consuming producers remain private and non-promotable. +- Add adapter protocol v3 and SDK types for incomplete discovery, resolution, content identities, validation, capabilities, and attestations while negotiating v1/v2 packages explicitly. +- Migrate representative built-ins and custom pipelines through the common path, with native COW/CAS evidence for dependency trees, compiled output, framework builds, path-bound private state, and source export. +- Keep artifact transport local-first. Portable identities and attestations are required, but remote cache exchange and remote execution remain deferred. + +## Capabilities + +### New Capabilities + +- `artifact-pipeline-contract`: Incomplete discovery, resolution snapshots, extensions to `trail.environment.toml`, normalized host orchestration, and explicit source export. +- `artifact-identity-invalidation`: Separate desired, content, artifact-envelope, and generation-binding identities; precise invalidation; nondeterminism quarantine; and explainability. +- `artifact-cow-storage`: Deterministic Merkle directory/file/chunk objects integrated with existing immutable layers, promotion, COW bindings, recovery, retention, and accounting. +- `artifact-trust-execution`: Resolver/build capability separation, secret taint, sealing, attestations, trust-scoped attachment, and future-portable identities. +- `artifact-adapter-conformance`: Protocol v3, SDK and repository-schema safety, behavior-based certification, representative real-tool fixtures, and native multi-lane qualification. + +### Modified Capabilities + +None. The merged `optimize-large-repo-lanes` change is still present as a completed change rather than a synchronized capability baseline. These new capabilities normatively depend on and preserve its shipped output-policy, COW-isolation, promotion, generation, singleflight, inheritance, and cache-safety contracts. + +## Impact + +- Environment domain: extend current discovery, planning, graph, sync, managed execution, generation binding, promotion, inheritance, and report models rather than creating parallel orchestration. +- Storage: extend fresh schema-v1 creation/validation with resolution snapshots, content manifests and objects, artifact envelopes, attempts, quarantines, attestations, and reachability. Incompatible stored shapes follow the existing backup plus `trail init --force` hard cutover; no migration framework is added. +- Layer storage: retain `workspace_layers`, `workspace_layer_publications`, generation outputs, mounts, private uppers, and native backends while replacing authoritative duplicated layer bytes with verified content objects plus reconstructible materializations. +- Repository authoring: evolve `trail.environment.toml` and its includes/profiles; do not introduce `trail.artifacts.toml`. +- Adapter surfaces: add explicitly negotiated `trail.environment-adapter/v3` wire types and SDK builders while preserving deterministic v1/v2 conversion. +- Public interfaces: extend the shared Rust reports and their CLI JSON/NDJSON, HTTP/OpenAPI, and MCP projections with resolution, content identity, quarantine, attestation, source export, verification, and CAS accounting. +- Qualification: retain the merged native isolation and scale gates, then add content-deduplication, resolution, nondeterminism, source-export, and real-framework evidence. diff --git a/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-adapter-conformance/spec.md b/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-adapter-conformance/spec.md new file mode 100644 index 00000000..9f5bb440 --- /dev/null +++ b/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-adapter-conformance/spec.md @@ -0,0 +1,54 @@ +## ADDED Requirements + +### Requirement: Protocol v3 expresses the incremental common pipeline +Trail SHALL negotiate a bounded `trail.environment-adapter/v3` protocol for incomplete proposals, resolution plans/snapshots, typed action phases, validations, desired/content/artifact identities, capability profiles, source exports, attestations, and quarantine evidence. V1/v2 packages SHALL retain their exact existing meanings and MUST NOT obtain v3-only authority through missing-field defaults. + +#### Scenario: Multi-version package +- **WHEN** host and package support v3 +- **THEN** they select v3 explicitly and include selected protocol/package identity in planning identity + +#### Scenario: V2 compatibility +- **WHEN** a package supports only v2 +- **THEN** Trail uses its deterministic v2 conversion and rejects requests for resolution, source export, or v3 certification + +### Requirement: SDK and repository schema reject invalid plans early +The SDK v3 builders and `trail.environment/v2` parser SHALL enforce canonical ordering, bounded collections, normalized paths, fixed argv, graph validity, phase/output compatibility, capability ceilings, and source-export constraints. The host MUST repeat validation for every untrusted response. + +#### Scenario: Invalid authoring combination +- **WHEN** a declaration combines secret-tainted construction with shared publication or source export +- **THEN** authoring validation rejects it before serialization or execution + +### Requirement: Certification is behavior-based +Trail SHALL certify built-in, plugin, and repository-defined producers against the same applicable discovery, resolution, identity, validation, sealing, COW, recovery, invalidation, export, retirement, and collection fixtures. Framework names alone MUST NOT grant reuse, portability, trust, or sandbox capabilities. + +#### Scenario: Custom framework passes the contract +- **WHEN** a repository-defined pipeline satisfies the applicable fixtures under repository trust +- **THEN** it receives only the reuse and execution behavior permitted by that trust tier without core framework-specific lifecycle code + +### Requirement: Representative artifact shapes receive real-tool evidence +Qualification SHALL cover at least a dependency-tree resolver/installer, compiled incremental tree, bundler/framework build, path-bound private environment, metadata-only external artifact, and repository-defined custom pipeline. Every gate SHALL prove actual tool execution, identities, reuse/invalidation, lane isolation, source-diff classification, disposal, and storage accounting. + +#### Scenario: Framework composition +- **WHEN** a Vite- or Next.js-like build depends on a Node-like resolution/dependency component +- **THEN** the report shows distinct graph nodes and rebuilds application output without reinstalling unchanged dependencies + +### Requirement: Native evidence covers CAS-backed COW +Every claimed NFS, FUSE, or Dokan backend SHALL prove CAS-backed immutable lower integrity, private writes/whiteouts, fork inheritance, materialization recovery, promotion, source export, retirement, and 1/5/20-lane storage behavior on the owning platform. A skipped gate MUST remain unverified. + +#### Scenario: Sibling isolation under native backend +- **WHEN** sibling lanes share one CAS-backed seed and make conflicting mutations +- **THEN** each lane sees its own result, the lower content root stays valid, and evidence quantifies shared/materialized/private bytes + +### Requirement: Public interfaces share the extended typed reports +Rust, CLI JSON/NDJSON, HTTP/OpenAPI, and MCP SHALL project the same resolution, identity, verification, quarantine, attestation, export, recovery, and accounting models. Human rendering SHALL provide a readable summary while structured output remains the automation contract. + +#### Scenario: Resolve report parity +- **WHEN** equivalent resolution is invoked through supported interfaces +- **THEN** proposal, snapshot, attempt, authority, identity, decision, and recovery fields have the same semantics and deterministic ordering + +### Requirement: Protocol and artifact inputs are bounded and adversarially tested +Trail SHALL bound frames, graphs, patterns, expanded inputs, actions, output entries/bytes/depth, snapshots, objects, chunks, validation output, subprocesses, concurrency, pagination, and diagnostic retention. Limit exhaustion SHALL return explicit failure or truncation and MUST NOT become an empty successful result. + +#### Scenario: Oversized artifact manifest +- **WHEN** a producer exceeds declared entry, byte, depth, path, or chunk limits +- **THEN** Trail aborts sealing, retains no ready envelope, and reports the exact exceeded limit diff --git a/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-cow-storage/spec.md b/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-cow-storage/spec.md new file mode 100644 index 00000000..0785856b --- /dev/null +++ b/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-cow-storage/spec.md @@ -0,0 +1,63 @@ +## ADDED Requirements + +### Requirement: Pure content manifests are deterministic +Trail SHALL represent sealed artifact content as versioned deterministic directory, file, blob, chunk-list, and chunk objects. Pure content identity MUST exclude desired keys, layer IDs, generation IDs, attempt IDs, storage paths, timestamps, and producer provenance. + +#### Scenario: Equivalent trees +- **WHEN** two producers seal equivalent normalized trees in different traversal orders +- **THEN** they produce the same tree-content root + +#### Scenario: Unsafe entry +- **WHEN** a candidate contains traversal, non-normalized names, a case collision, escaping link, prohibited metadata, or unsupported file type +- **THEN** Trail rejects sealing before any ready artifact envelope is reachable + +### Requirement: File content is deduplicated independently of artifacts +Trail SHALL store small files as immutable whole-blob objects and large files through a versioned content-defined chunk profile plus complete-file digest. Artifact envelopes and successor manifests SHALL reference existing unchanged objects instead of copying their bytes. + +#### Scenario: Different keys share files +- **WHEN** artifacts with different desired keys contain equal files or chunks +- **THEN** Trail stores one authoritative content object for each equal identity and both manifests reference it + +#### Scenario: Small successor change +- **WHEN** a successor modifies a bounded subset of files or chunks +- **THEN** Trail creates only affected directory/file/chunk objects plus metadata and reuses all unchanged object identities + +### Requirement: CAS objects are authoritative and materializations are rebuildable +Ready artifacts SHALL remain reconstructible from durable manifests and content objects. Layer directories, backend projections, prefetch data, and verified materialization caches MUST NOT be the only copy of artifact bytes and SHALL be independently reclaimable. + +#### Scenario: Materialization is removed +- **WHEN** GC removes an unmounted artifact materialization while its envelope remains retained +- **THEN** a later attachment reconstructs or lazily serves the same verified content root without rebuilding the framework component + +#### Scenario: Missing content object +- **WHEN** an envelope references a missing or corrupt content object +- **THEN** Trail marks the artifact corrupt, refuses new attachment atomically, and reports repair or rebuild guidance without serving partial surviving bytes + +### Requirement: CAS sealing integrates with existing promotion and COW bindings +Trail SHALL publish content objects and a ready artifact envelope before the existing successor-generation activation transaction. Existing immutable lower, seeded private upper, whiteout, promotion-quiescence, fork-inheritance, and rollback behavior SHALL remain authoritative. + +#### Scenario: Promoted private output +- **WHEN** a declared private output is quiesced, validated, and promoted +- **THEN** Trail seals its content into CAS, activates a successor generation through the existing publication fence, and leaves the live private upper unchanged + +#### Scenario: Sibling mutation +- **WHEN** sibling lanes attach one CAS-backed immutable seed and mutate or delete the same lower path +- **THEN** each sees only its private mutation/whiteout and full verification preserves the common content root + +### Requirement: Publication and recovery preserve exact ownership +Resolution, construction, validation, object sealing, envelope publication, generation activation, materialization, retirement, and collection SHALL use durable fenced state and confined idempotent operations. Recovery MUST delete or resume only state owned by the exact attempt. + +#### Scenario: Stop after envelope publication +- **WHEN** Trail stops after a ready envelope is durable but before generation activation +- **THEN** recovery either completes the exact fenced activation or leaves an unbound collectable artifact while preserving the prior generation + +### Requirement: Reachability and accounting operate over shared objects +Trail SHALL retain every object reachable from active or retained generations, live/recoverable attempts, leases, quarantines, backups, and explicit holds. Accounting SHALL distinguish logical artifact bytes, unique authoritative bytes, bytes shared with other artifacts, materialization bytes, lane-private bytes, and reclaimable bytes without double counting. + +#### Scenario: Last artifact reference expires +- **WHEN** the final envelope and hold referencing a content object expire +- **THEN** deterministic GC can reclaim that object while preserving every object reachable from another artifact + +#### Scenario: Twenty lanes share one tree +- **WHEN** twenty lanes attach one artifact and make small private changes +- **THEN** accounting reports one authoritative shared tree, any reclaimable materializations, and twenty lane-private deltas rather than twenty authoritative copies diff --git a/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-identity-invalidation/spec.md b/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-identity-invalidation/spec.md new file mode 100644 index 00000000..021074f9 --- /dev/null +++ b/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-identity-invalidation/spec.md @@ -0,0 +1,55 @@ +## ADDED Requirements + +### Requirement: Desired, content, artifact, and binding identities are distinct +Trail SHALL model a desired component key, deterministic tree-content root, artifact envelope identity, and lane-generation binding as separate immutable identities. A key change MUST NOT relabel published bytes, and equal content roots MUST remain reusable as storage even when correctness envelopes differ. + +#### Scenario: Equal desired keys +- **WHEN** compatible consumers request the same desired key and trust scope +- **THEN** they converge through existing singleflight on one verified artifact envelope or one fenced construction attempt + +#### Scenario: Different keys produce equal content +- **WHEN** two valid desired keys produce byte-identical trees +- **THEN** their artifact envelopes remain distinct while their manifests reference the same content objects + +### Requirement: Desired-key v2 covers the complete correctness boundary +Trail SHALL derive `ArtifactDesiredKeyV2` from the contract, adapter implementation/package/protocol, resolution snapshot, declared source closure, identity-bearing upstream identities, action executable identities and argv, working directories, outputs, validations, source-export contract, non-secret build environment, target/platform/ABI, portability, trust scope, and network/script/sandbox policy. Unproven closure MUST include the complete pinned source root or prevent reusable publication. + +#### Scenario: Certified irrelevant edit +- **WHEN** a certified complete input closure excludes a changed source path +- **THEN** the desired key remains stable and policy permits reuse of the current verified artifact + +#### Scenario: Unknown influence +- **WHEN** the adapter cannot prove whether source, tool, plugin, script, or environment state affects output +- **THEN** Trail uses the conservative source-root/host dimensions or makes the output private + +### Requirement: Identity versions are never reinterpreted +Trail SHALL encode desired keys and content nodes with explicit canonical codec and algorithm versions. Existing `WorkspaceLayerKeyV1` and adapter v1/v2 projections SHALL remain inspectable only under their exact legacy meaning and MUST NOT be interpreted as v2 identities. + +#### Scenario: Legacy generation inspection +- **WHEN** a retained generation references a v1 layer key +- **THEN** Trail reports its original version and evidence without claiming v2 completeness or content identity + +### Requirement: Managed executions and sealing use pinned identities +Every managed execution SHALL retain one pinned source root and active generation from mount through command exit. A reusable sealing attempt SHALL additionally pin desired key, candidate journal, validation/gate evidence, and resolution/content inputs; any change SHALL make that attempt stale rather than attaching it to new state. + +#### Scenario: Agent edits source while building +- **WHEN** an execution changes identity-bearing source and also produces private generated bytes +- **THEN** Trail checkpoints eligible source, retains generated bytes privately, and requires replanning before shared sealing + +### Requirement: Divergent content for one desired key is quarantined +If structurally valid constructions for the same desired key and trust scope produce different tree roots, Trail SHALL quarantine the key and candidates, disable automatic shared attachment, and retain bounded diagnostic provenance. Trail MUST NOT choose a candidate arbitrarily. + +#### Scenario: Reproducibility divergence +- **WHEN** a second verified construction differs from the existing content root for one desired key +- **THEN** Trail holds both candidates, reports their exact roots and producer evidence, and attaches neither as a normal shared hit + +#### Scenario: Private fallback +- **WHEN** workspace policy permits continued work for a quarantined key +- **THEN** Trail permits only explicitly reported lane-private output and preserves the quarantine + +### Requirement: Identity decisions remain explainable +Every discovery, resolution, plan, sync, inheritance, promotion, export, execution, and verification report SHALL expose applicable desired, content, artifact, and binding identities; decision source; invalidating edges; trust and quarantine state; and known logical, unique, shared, materialized, and private bytes. + +#### Scenario: Tool-only invalidation +- **WHEN** source and resolution inputs are unchanged but an identity-bearing tool changes +- **THEN** Trail reports that tool dimension as the first rebuild cause rather than a generic cache miss diff --git a/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-pipeline-contract/spec.md b/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-pipeline-contract/spec.md new file mode 100644 index 00000000..ec4c0160 --- /dev/null +++ b/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-pipeline-contract/spec.md @@ -0,0 +1,67 @@ +## ADDED Requirements + +### Requirement: Discovery reports recognized incomplete components +Trail SHALL return a typed proposal from side-effect-free source markers even when lock state, tools, platform capabilities, or policy are incomplete. Each proposal SHALL be `ready`, `resolvable`, `blocked`, `unsupported`, or `ambiguous` and SHALL contain stable reasons and recovery actions. + +#### Scenario: Manifest without lock state +- **WHEN** a recognized dependency manifest exists without a source or Trail-managed resolution snapshot +- **THEN** discovery reports the component as `resolvable` or `blocked` and does not silently omit it + +#### Scenario: Discovery remains read-only +- **WHEN** Trail discovers built-in, plugin, or repository-defined components +- **THEN** Trail invokes no package manager, compiler, repository action, network endpoint, runtime provider, or secret provider + +### Requirement: Resolution snapshots are explicit environment metadata +Trail SHALL support an optional resolver phase that converts a pinned proposal into an immutable content-addressed snapshot. Snapshot bytes and provenance SHALL remain environment metadata and MUST NOT become source unless a user authorizes a normal source export/materialization operation. + +#### Scenario: Explicit resolution +- **WHEN** a resolvable proposal lacks a snapshot and the user runs its reported resolve operation +- **THEN** Trail executes the bounded authorized resolver, validates one snapshot, stores it outside source history, and replans against its identity + +#### Scenario: Refresh is deliberate +- **WHEN** a valid snapshot exists and neither its identity inputs nor an explicit `--refresh` request changes +- **THEN** Trail reuses it and does not select newer dependencies merely because time passed + +### Requirement: Resolution and construction extend existing convergence +Trail SHALL feed proposals, snapshots, desired keys, validated artifacts, and private outputs into the existing component DAG, environment generation, promotion, COW binding, managed execution, retirement, and collection operations. It MUST NOT introduce a parallel generation or mount authority. + +#### Scenario: First resolved execution +- **WHEN** every missing snapshot is available and the active generation is incomplete +- **THEN** existing environment synchronization constructs or reuses required nodes and atomically activates one complete successor generation before command launch + +#### Scenario: Phase failure +- **WHEN** resolution, construction, validation, sealing, or activation fails +- **THEN** the previous active generation remains authoritative and the report identifies the exact failed phase and safe recovery action + +### Requirement: Repository pipelines extend trail.environment.toml +Trail SHALL add the common pipeline through an explicitly versioned extension of `trail.environment.toml` and `.trail/environment.toml`. Version-1 documents SHALL retain their existing interpretation; version-2 documents SHALL compile resolution, typed actions, validations, outputs, capabilities, and exports to the same normalized host graph as built-in and plugin adapters. + +#### Scenario: Version-2 custom framework +- **WHEN** a repository declares a bounded version-2 resolver, constructor, validation, artifact output, and source export +- **THEN** Trail validates and plans them without framework-specific lifecycle code or shell interpolation + +#### Scenario: Version-1 compatibility +- **WHEN** Trail reads an existing `trail.environment/v1` document +- **THEN** it preserves the shipped v1 defaults and restrictions and does not infer v2 resolution or export authority + +### Requirement: Existing artifact policies remain the storage vocabulary +Pipeline artifact outputs SHALL use the shipped `immutable_shared`, `immutable_seed_private`, `writable_private`, or `disposable` policy. Caches, runtimes, secrets, and external resources SHALL remain separately typed, and generated source SHALL use a source-export declaration rather than a fifth artifact policy. + +#### Scenario: Seeded mutable output +- **WHEN** an output is `immutable_seed_private` +- **THEN** Trail attaches its verified immutable artifact through the existing lower-layer binding and routes all mutations to fresh lane-private state + +#### Scenario: Generated source +- **WHEN** a component generates bytes intended for Git review +- **THEN** those bytes remain artifact/private output until an authorized source export writes them through normal source guardrails + +### Requirement: Source export is explicit and reviewable +Trail SHALL export generated source only from a declared validated candidate or artifact subtree into a normalized repository-relative destination. Export MUST pin source, desired, content, validation, and destination state and SHALL apply ignore, secret, collision, guardrail, diff, checkpoint, and Git-handoff semantics. + +#### Scenario: Successful source export +- **WHEN** a user authorizes a current declared export and the destination passes all source checks +- **THEN** Trail writes the result into the lane source upper and exposes it in normal lane diff and review + +#### Scenario: Stale or conflicting export +- **WHEN** the source root, artifact identity, validation receipt, destination, or existing source content changes after export planning +- **THEN** Trail writes nothing and reports the stale or conflicting evidence diff --git a/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-trust-execution/spec.md b/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-trust-execution/spec.md new file mode 100644 index 00000000..a78a25d5 --- /dev/null +++ b/openspec/changes/generalize-framework-artifact-pipeline/specs/artifact-trust-execution/spec.md @@ -0,0 +1,51 @@ +## ADDED Requirements + +### Requirement: New pipeline phases preserve host authority +Adapters and repository declarations SHALL return bounded proposals, resolver plans, action graphs, validations, and export declarations. Only the Trail host SHALL resolve tools, allocate storage, execute actions, access providers, seal objects, publish envelopes, activate bindings, export source, recover attempts, or collect content. + +#### Scenario: Adapter requests direct authority +- **WHEN** a v3 plan requests an absolute host path, database operation, artifact ID assignment, direct mount, provider socket, or publication mutation +- **THEN** Trail rejects it before execution and identifies the responsible package and field + +### Requirement: Resolution and construction use separate capability profiles +Resolver actions SHALL receive network access only when explicitly authorized to exact authorities and SHALL record redacted resolution evidence. Constructors and validators SHALL run offline by default against pinned snapshots and managed content; every exception MUST be an identity-bearing, trust-bounded policy input. + +#### Scenario: Controlled resolution +- **WHEN** an authorized resolver needs a configured registry +- **THEN** Trail grants only that authority, records its non-secret identity and checksums, and excludes credentials from durable state + +#### Scenario: Offline constructor connects +- **WHEN** a constructor declared offline attempts a network connection +- **THEN** the sandbox denies it, construction fails, and no artifact envelope becomes ready + +### Requirement: Every phase is deny-by-default and bounded +Trail SHALL enforce normalized bounds for readable inputs, writable candidate/private paths, executable/child graph, environment names, network, runtime, duration, output bytes/entries/depth, captured output, and secret channels. Missing native enforcement MUST fail closed for untrusted resolver, constructor, validator, or export actions. + +#### Scenario: Undeclared write +- **WHEN** a repository action writes outside its candidate, isolated home/temp, approved cache, or source-export destination +- **THEN** Trail denies or detects the mutation and publishes no artifact or source change + +### Requirement: Secret consumption taints output +Secret bytes MUST NOT enter desired keys, snapshots, content objects, manifests, logs, reports, caches, attestations, source exports, or future remote requests. A producer receiving secret bytes SHALL create only lane-private non-promotable and non-exportable output. + +#### Scenario: Signing key used by a producer +- **WHEN** an action receives a signing key through an approved private channel +- **THEN** Trail redacts its evidence and rejects shared sealing or source export of that action's output + +### Requirement: Sealing requires host validation and attestation +Before publishing an artifact envelope, Trail SHALL confirm producer termination, unchanged pins, declared-path containment, safe normalized content, bounds, secret policy, complete content identity, required validations/gates, and producer trust. It SHALL publish a deterministic attestation referencing the desired key, tree root, adapter/package/protocol, tools, platform, capability enforcement, policies, and validation receipts. + +#### Scenario: Successful command leaves unsafe output +- **WHEN** a producer exits successfully but leaves an escaping link, special file, secret-tainted content, undeclared path, or stale validation input +- **THEN** sealing fails and no ready artifact envelope is published + +#### Scenario: Attestation tampering +- **WHEN** stored attestation bytes or references do not match their content identity +- **THEN** attachment fails closed while prior provenance remains inspectable as corrupt evidence + +### Requirement: Portable identity does not grant remote trust +Artifact content and attestation identities SHALL avoid machine-specific storage paths and support explicit portability scopes. Matching desired/content identities from another host MUST NOT authorize attachment without a future transport policy validating origin, signatures, manifest integrity, portability, revocation, and local trust. + +#### Scenario: Untrusted imported object graph +- **WHEN** matching objects arrive outside the local trust scope +- **THEN** Trail rejects them or retains them as untrusted candidates and does not bind them to a generation diff --git a/openspec/changes/generalize-framework-artifact-pipeline/tasks.md b/openspec/changes/generalize-framework-artifact-pipeline/tasks.md new file mode 100644 index 00000000..8714a7a7 --- /dev/null +++ b/openspec/changes/generalize-framework-artifact-pipeline/tasks.md @@ -0,0 +1,122 @@ +## 1. Freeze the Merged Baseline and Failure Evidence + +- [x] 1.1 Record a design-to-code map for the shipped output policies, component DAG, generation rows, sync attempts, layer manifests, publication journal, COW bindings, inheritance, cache GC, `trail.environment/v1`, protocol v1/v2, and public reports +- [x] 1.2 Pin serialization and CLI/API fixtures for the current `WorkspaceLayerKeyV1`, layer/generation/output reports, SDK v1/v2 frames, recipe schema, and promotion reports so later changes cannot reinterpret them +- [x] 1.3 Add failing Cargo and Node fixtures proving manifest-only projects are currently omitted and discovery must return typed incomplete proposals without executing tools +- [x] 1.4 Add a failing storage fixture proving two distinct desired keys with equal file content currently allocate duplicated authoritative layer bytes +- [x] 1.5 Add a failing reproducibility fixture proving current key-derived layer identity cannot represent two different content roots for one desired key safely +- [x] 1.6 Verify `/Volumes/Workspace` and create one checkout-specific `/Volumes/Workspace/crabbuild-target/trail-` target; set it explicitly on every compiling Cargo invocation + +## 2. Proposal and Resolution Domain Model + +- [x] 2.1 Add typed component proposal/status/reason/recovery models for `ready`, `resolvable`, `blocked`, `unsupported`, and `ambiguous` with deterministic ordering and bounds +- [x] 2.2 Refactor built-in, plugin, and repository discovery to return proposals from pinned markers without requiring lock state or invoking tools, network, providers, or repository code +- [x] 2.3 Add `ArtifactResolutionPlanV1` with proposal/source pins, exact executable/argv, readable inputs, candidate output, authority set, script/environment roles, limits, and validation contract +- [x] 2.4 Add deterministic `ArtifactResolutionSnapshotV1` object encoding, content identity, provenance, predecessor, verification state, and proposal-key lookup +- [x] 2.5 Add durable resolver attempts with fenced owner identity, heartbeat, cancellation, bounded output, redacted authority evidence, failure receipts, and orphan recovery +- [x] 2.6 Implement explicit resolve-all and resolve-component library operations with deliberate refresh and no time-based dependency advancement +- [x] 2.7 Add resolver success, snapshot reuse, refresh, stale source, tool/authority mismatch, credential redaction, process death, malformed output, and size-limit tests + +## 3. Desired, Content, Artifact, and Binding Identities + +- [x] 3.1 Add validated ID types for `ArtifactDesiredKeyV2`, tree/file/blob/chunk objects, artifact envelopes, attestations, attempts, quarantines, and generation bindings +- [x] 3.2 Implement canonical CBOR desired-key v2 over adapter/package/protocol, resolution, source closure, upstream identities, actions/tools, output/validation/export contracts, non-secret environment, platform/ABI, and policy +- [x] 3.3 Preserve exact `WorkspaceLayerKeyV1` inspection and v1/v2 conversion while preventing legacy identities from being interpreted as v2-complete +- [x] 3.4 Extend graph diffing to return deterministic first and complete invalidating edges across resolution, tool, action, output, validation, export, trust, and sandbox dimensions +- [x] 3.5 Enforce complete-source-root/private fallback when an adapter lacks certified input closure or portability evidence +- [x] 3.6 Add canonical-order, absent-versus-empty, Unicode/path, semantic-normalizer-version, one-dimension-change, irrelevant-edit, and property tests for identity stability + +## 4. Deterministic Artifact Content Store + +- [x] 4.1 Define versioned deterministic CBOR codecs for directory nodes, file nodes, whole blobs, chunk lists, chunks, tree roots, artifact envelopes, and their object-edge validation +- [x] 4.2 Implement streaming normalized tree ingestion with sorted entries and bounded paths, depth, entries, bytes, links, modes, metadata, and concurrent-mutation detection +- [x] 4.3 Implement whole-blob storage through 1 MiB and `fastcdc-v1` chunking above it using 256 KiB/1 MiB/4 MiB bounds plus complete-file SHA-256 +- [x] 4.4 Reject absolute/traversing/non-NFC paths, case collisions, escaping links, unsupported file types, prohibited modes/xattrs, excessive sparse content, and secret-policy failures before reachability +- [x] 4.5 Implement atomic object publication and validation that safely handles pre-existing equal objects, corrupt collisions, interruption, reopen, and concurrent publishers +- [x] 4.6 Add equal-tree/different-order, equal-files/different-keys, successor-small-change, large-file-boundary, symlink/hardlink, Unicode/case, interruption, and corruption tests +- [x] 4.7 Benchmark whole-file versus chunked ingestion and record CPU, object count, logical bytes, unique bytes, and successor reuse without weakening correctness thresholds + +## 5. Retrofit Existing Layer Publication and Promotion + +- [x] 5.1 Extend the fresh schema creator/validator with artifact trees/envelopes, construction attempts/waiters, resolution records, attestations, quarantines, holds, and exact generation bindings while preserving schema-v1 hard-cutover rules +- [x] 5.2 Add a CAS shadow-publication mode that seals and verifies content roots beside current layer copies without using CAS for attachment +- [x] 5.3 Compare shadow content roots against current full-tree verification and fail publication on any path, metadata, size, or digest disagreement +- [x] 5.4 Change new layer publication to make content objects/envelopes authoritative and treat `.trail/cache/layers` bytes as verified reconstructible materializations +- [x] 5.5 Insert CAS sealing into the current manual/on-sync/successful-gate publication path without changing quiescence, source/generation/gate pins, or successor activation semantics +- [x] 5.6 Extend current singleflight with durable owner/waiter phase evidence and exact dead-owner fencing while retaining bounded DAG scheduling and cancellation +- [x] 5.7 Add crash points from reservation through object publication, envelope readiness, materialization, and activation; prove prior generations and live private uppers remain intact + +## 6. CAS-Backed COW, Recovery, Reachability, and Space + +- [x] 6.1 Add manifest-backed lazy lookup/materialization to NFS, FUSE, and Dokan integration without changing existing lower/upper/whiteout semantics +- [x] 6.2 Add verified materialization caches keyed by tree root/backend compatibility with safe clone/reflink/copy policy and no mutable alias to authoritative content +- [x] 6.3 Extend fork compatibility to verify desired key, artifact envelope, tree integrity, adapter/package trust, portability, scope, and backend support while preserving fresh mutable identities +- [x] 6.4 Extend workspace-open recovery, doctor, and fsck for incomplete CAS attempts, corrupt/missing objects, orphan materializations, legacy versus CAS-backed layouts, and exact repair guidance +- [x] 6.5 Extend backup/restore so authoritative snapshots, envelopes, attestations, objects, bindings, and retained private state survive while omitted materializations/caches are reported as rebuildable +- [x] 6.6 Implement object-graph reachability and incremental deterministic GC across generations, attempts, leases, quarantines, backups, holds, directories, files, blobs, chunk lists, and chunks +- [x] 6.7 Extend lane space/cache reports with logical, unique authoritative, cross-artifact shared, materialized, lane-private, prefetched, demand-loaded, reclaimable, and unknown bytes without double counting +- [x] 6.8 Add last-reference, shared-chunk retention, materialization eviction/rebuild, interrupted GC, parent removal, remount, lower deletion, and 1/5/20-lane accounting tests + +## 7. Validation, Nondeterminism, Trust, Secrets, and Attestations + +- [x] 7.1 Add typed structural, loadability, framework, policy, gate, and reproducibility validation declarations and deterministic receipts +- [x] 7.2 Implement required host sealing checks for producer termination, unchanged pins, path containment, safe content, limits, secret policy, complete tree identity, validations, and producer trust +- [x] 7.3 Detect differing tree roots for one `(trust_scope, desired_key)`, create durable quarantine/holds, block shared attachment, and retain bounded comparison provenance +- [x] 7.4 Implement explicit quarantine list/show/resolve operations and policy-controlled lane-private fallback without relabeling candidates or clearing evidence implicitly +- [x] 7.5 Define and enforce phase-specific capability ceilings for reviewed built-ins, certified signed plugins, locally trusted plugins, and repository declarations +- [x] 7.6 Add secret-taint propagation so secret-consuming producers are private, non-promotable, and non-exportable and secret bytes never enter identities, objects, logs, reports, or caches +- [x] 7.7 Implement deterministic `ArtifactAttestationV1` creation, storage, inspection, attachment validation, optional signature fields, and package/publisher revocation checks +- [x] 7.8 Add divergent-producer, malicious-plan, undeclared-write, denied-network, child-process, secret-leak, unsafe-output, revoked-package, unsupported-sandbox, and attestation-tamper tests + +## 8. trail.environment/v2 and Source Export + +- [x] 8.1 Extend the existing repository parser with explicit `trail.environment/v2` while preserving exact v1 paths, includes/profiles, defaults, validation, and errors +- [x] 8.2 Add v2 resolution, multiple typed action phases, validations, capability declarations, heterogeneous outputs, and source-export sections with strict unknown-field rejection +- [x] 8.3 Compile v2 documents to the same proposal, resolution, graph, desired-key, output, and report models used by built-in and plugin adapters +- [x] 8.4 Preserve fixed argv and bounded deterministic input expansion; reject shell interpolation, control flow, absolute host paths, forbidden child execution, raw secrets, provider sockets, and over-broad reuse +- [x] 8.5 Implement source-export planning with artifact/subtree identity, destination, collision mode, validation/gate, source pin, and explicit authorization +- [x] 8.6 Execute source export through confined normal source writes with ignore, guardrail, secret, collision, diff, checkpoint, and Git-handoff behavior; never through artifact mounting +- [x] 8.7 Add v1 compatibility, parser snapshots, include/profile cycles, adversarial v2 documents, stale/conflicting export, ignored destination, secret-taint, custom framework, and source-diff E2E tests + +## 9. Adapter Protocol v3 and SDK + +- [x] 9.1 Define bounded `trail.environment-adapter/v3` request/response types for proposals, resolution, inputs, typed phases, validations, capabilities, identities, exports, attestations, and quarantine evidence +- [x] 9.2 Implement exact highest-mutual-version negotiation and deterministic v1/v2 conversions that cannot obtain v3 semantics from absent fields +- [x] 9.3 Add SDK v3 builders with canonical collections, validation errors, finite limits, example adapters, package capability declarations, and documentation +- [x] 9.4 Repeat complete host validation for every v3 response and reject duplicate IDs, bad normalization, invalid graph/phase combinations, oversized data, unsupported required fields, and package/protocol mismatch +- [x] 9.5 Extend plugin inspect/install/trust/revocation reports with selected protocol, resolution/export capability, certification ceiling, and content/attestation policy +- [x] 9.6 Add golden frames, CBOR round trips, compatibility fixtures, truncation/over-limit, fuzz/property, malicious response, signature/revocation, and Linux/macOS/Windows enforcement tests +- [x] 9.7 Run SDK unit/doc tests and native plugin verification scripts with the checkout-specific external `CARGO_TARGET_DIR` + +## 10. Built-In Migration and Framework Composition + +- [x] 10.1 Migrate a fixture-only adapter through proposal, resolution, desired key v2, CAS sealing, generation activation, COW execution, export, retirement, and collection before changing real ecosystems +- [x] 10.2 Change Cargo discovery to report `Cargo.toml` without `Cargo.lock`, add a Trail-managed lock snapshot, and qualify real `--locked --offline` construction with conservative source-root identity +- [x] 10.3 Change Node discovery to report `package.json` without a supported lock, add package-manager-specific frozen snapshots, and preserve dependency-seed plus content-cache isolation +- [x] 10.4 Express Vite and Next.js fixtures as build components over Node resolution/dependencies, keeping framework caches/path-bound output private unless validation certifies reuse +- [x] 10.5 Migrate Python resolution/download artifacts while retaining path-bound virtual environments and bytecode/tool caches as private or performance-only +- [x] 10.6 Map Go, CMake, OCI/runtime, and existing command recipes to v2/v3 identities without replacing their shipped private/external/cache semantics +- [x] 10.7 Add Maven/Gradle-like, Bazel/Nix-like, and unknown custom fixtures through plugins or repository v2 rather than framework names in Trail core +- [x] 10.8 Remove adapter-specific resolution/publication shortcuts only after common-path correctness, compatibility, native isolation, and performance evidence passes + +## 11. Public Operations and Conformance + +- [x] 11.1 Add shared library operations/reports for resolve, artifact inspect/verify/quarantine, source export, content reachability, and CAS-aware space while retaining existing env grammar +- [x] 11.2 Add CLI help, deterministic plain output, human rendering, JSON/NDJSON, exit categories, ambiguity behavior, and exact recovery commands for new operations +- [x] 11.3 Add aligned HTTP routes/OpenAPI schemas and MCP tools/resources/risk annotations backed by the same library operations and reports +- [x] 11.4 Extend managed exec/test/eval/agent/ACP preparation with explicit missing-resolution policy, pinned identities, sealing decisions, and finalization receipts +- [x] 11.5 Build behavior-based conformance fixtures shared by built-ins, plugins, and repository v2 for discovery through collection and emit machine-readable certification reports +- [x] 11.6 Add real-tool gates for dependency resolution, compiled incremental output, framework/bundler composition, path-bound private state, external metadata, custom pipeline, and source export +- [x] 11.7 Add deterministic 10k/100k/1M-entry and 1/5/20-lane experiments measuring content reuse, materialization amplification, private deltas, phase latency, object count, and skipped evidence +- [x] 11.8 Run native CAS-backed isolation/materialization/promotion/export/recovery matrices on NFS, FUSE, and Dokan owning hosts; leave unavailable gates explicitly unverified + +## 12. Documentation, Compatibility, and Completion Gates + +- [x] 12.1 Update README and first-lane workflow with source, resolution snapshot, desired key, content root, artifact envelope, materialization, private upper, promotion, and source-export semantics +- [x] 12.2 Update architecture, data model, universal environments, adapter contract, layered workspaces, storage/indexing, guardrail/security, cache/GC, and performance documents to match implementation +- [x] 12.3 Update `trail.environment/v2`, SDK v3, CLI, HTTP, MCP, troubleshooting, doctor/fsck/backup/restore, native prerequisites, and framework composition guidance with executable examples +- [x] 12.4 Update `CHANGELOG.md`, configuration/completions, protocol compatibility, schema-v1 backup/reinitialization, rollback, and release-evidence policy +- [x] 12.5 Run `cargo fmt --all -- --check`, workspace check/test, and Clippy with `--locked`, applicable features, and the checkout-specific external `CARGO_TARGET_DIR` +- [x] 12.6 Run schema-v1, storage/rebuild/backup/restore, managed execution, environment/inheritance/retirement, CLI/terminal, HTTP/MCP, SDK, changed-path, and applicable native gates +- [x] 12.7 Run critical construction, quarantine, publication, activation, and GC tests with both `RUST_TEST_THREADS=1` and the default scheduler +- [x] 12.8 Inspect final diff/status for only intended files, deterministic/bounded contracts, no secret or machine path, no local build artifacts, no external-checkout mutation, and no unsupported platform/performance claim diff --git a/scripts/check-layered-lane-scale-evidence.py b/scripts/check-layered-lane-scale-evidence.py new file mode 100644 index 00000000..f5769364 --- /dev/null +++ b/scripts/check-layered-lane-scale-evidence.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Fail closed when layered-lane scale evidence is missing or overstated.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + + +MATRIX = ((10_000, 1), (100_000, 5), (1_000_000, 20)) +NATIVE_GATES = {"nfs_macos", "fuse_linux", "dokan_windows"} + + +def fail(message: str) -> None: + raise SystemExit(f"layered-lane scale evidence: {message}") + + +def load(path: Path) -> dict[str, Any]: + if not path.is_file(): + fail(f"missing {path}") + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + fail(f"cannot read {path}: {error}") + if not isinstance(value, dict): + fail(f"{path} must contain a JSON object") + return value + + +def require_nonnegative(value: Any, field: str, path: Path) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + fail(f"{path}: {field} must be a non-negative integer") + return value + + +def check_source(path: Path, entries: int, lanes: int) -> None: + report = load(path) + if report.get("schema") != "trail.layered-lane-scale/v1": + fail(f"{path}: unexpected source scale schema") + if report.get("tracked_paths") != entries or report.get("lanes") != lanes: + fail(f"{path}: source scale matrix identity is incorrect") + if report.get("indexed_paths_per_view") != 1: + fail(f"{path}: source views did not remain lazy") + require_nonnegative(report.get("exclusive_physical_bytes"), "exclusive_physical_bytes", path) + skipped = report.get("skipped_native_gates") + if not isinstance(skipped, list) or not set(skipped).issubset(NATIVE_GATES): + fail(f"{path}: skipped_native_gates is invalid") + + +def check_artifact(path: Path, entries: int, lanes: int) -> None: + report = load(path) + if report.get("schema") != "trail.artifact-lane-scale/v1": + fail(f"{path}: unexpected artifact scale schema") + if report.get("artifact_entries") != entries or report.get("lanes") != lanes: + fail(f"{path}: artifact scale matrix identity is incorrect") + if report.get("backend") != "cas-lazy-unmounted" or report.get("backend_qualified") is not True: + fail(f"{path}: portable CAS-lazy experiment was not qualified") + if report.get("native_backend_qualified") is not False: + fail(f"{path}: unmounted experiment must not qualify a native backend") + if set(report.get("skipped_native_gates", [])) != NATIVE_GATES: + fail(f"{path}: every native backend must remain explicitly skipped") + + reuse = report.get("content_reuse") + if not isinstance(reuse, dict): + fail(f"{path}: missing content_reuse") + logical = require_nonnegative(reuse.get("logical_bytes"), "content_reuse.logical_bytes", path) + authoritative = require_nonnegative( + reuse.get("authoritative_encoded_bytes"), + "content_reuse.authoritative_encoded_bytes", + path, + ) + if logical <= authoritative or reuse.get("shared_tree_roots") != 1: + fail(f"{path}: immutable content was not reused from one authoritative tree") + if reuse.get("lane_bindings") != lanes: + fail(f"{path}: not every lane was bound to the shared tree") + + amplification = report.get("materialization_amplification") + if not isinstance(amplification, dict): + fail(f"{path}: missing materialization_amplification") + if amplification.get("materialization_count") != 0 or amplification.get("materialized_physical_bytes") != 0: + fail(f"{path}: CAS-lazy attachment unexpectedly materialized the artifact") + if amplification.get("naive_per_lane_logical_bytes") != logical * lanes: + fail(f"{path}: naive per-lane amplification baseline is incorrect") + for field in ("copied_bytes", "projected_bytes", "prefetched_bytes"): + if amplification.get(field) != 0: + fail(f"{path}: {field} must be zero for the unmounted lazy experiment") + + private = report.get("private_deltas") + if not isinstance(private, dict) or private.get("lane_count") != lanes: + fail(f"{path}: private delta lane count is incorrect") + if require_nonnegative(private.get("minimum_physical_bytes"), "private_deltas.minimum_physical_bytes", path) == 0: + fail(f"{path}: lane-private deltas were not observed") + phases = report.get("phase_latencies_ms") + if not isinstance(phases, dict): + fail(f"{path}: missing phase latency evidence") + for phase in ("artifact_build", "envelope_publish", "lane_attach", "private_write", "accounting"): + require_nonnegative(phases.get(phase), f"phase_latencies_ms.{phase}", path) + objects = report.get("object_count") + if not isinstance(objects, dict) or require_nonnegative(objects.get("published"), "object_count.published", path) == 0: + fail(f"{path}: artifact object publication was not measured") + + +def main() -> None: + if len(sys.argv) != 2: + fail("usage: check-layered-lane-scale-evidence.py EVIDENCE_DIR") + directory = Path(sys.argv[1]) + for entries, lanes in MATRIX: + check_source(directory / f"paths-{entries}-lanes-{lanes}.json", entries, lanes) + check_artifact(directory / f"artifacts-{entries}-lanes-{lanes}.json", entries, lanes) + print(f"verified layered-lane scale evidence in {directory}") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify-artifact-adapter-conformance.sh b/scripts/verify-artifact-adapter-conformance.sh new file mode 100755 index 00000000..d6737cb2 --- /dev/null +++ b/scripts/verify-artifact-adapter-conformance.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${repo_root}" + +: "${CARGO_TARGET_DIR:?set a checkout-specific CARGO_TARGET_DIR beneath /Volumes/Workspace/crabbuild-target}" +if [[ "${GITHUB_ACTIONS:-}" != "true" && "${CARGO_TARGET_DIR}" != /Volumes/Workspace/crabbuild-target/* ]]; then + printf '%s\n' 'CARGO_TARGET_DIR must be beneath /Volumes/Workspace/crabbuild-target' >&2 + exit 2 +fi + +output="${1:-}" +evidence_dir="$(mktemp -d)" +cleanup() { + rm -rf -- "${evidence_dir}" +} +trap cleanup EXIT +report="${evidence_dir}/artifact-adapter-certification.json" + +TRAIL_ARTIFACT_CERTIFICATION_OUTPUT="${report}" \ + cargo test -p trail --lib \ + db::lane::workspace_layer::tests::builtins_plugins_and_repository_v2_share_artifact_pipeline_conformance \ + --locked -- --exact >&2 + +python3 - "${report}" "${output}" <<'PY' +import json +import os +import pathlib +import sys +import tempfile + +source = pathlib.Path(sys.argv[1]) +destination = sys.argv[2] +reports = json.loads(source.read_text(encoding="utf-8")) +families = [report.get("producer_family") for report in reports] +if families != ["builtin", "plugin_v3", "repository_v2"]: + raise SystemExit(f"unexpected producer families: {families!r}") +expected_stages = [ + "discovery", "resolution", "identity", "validation", "sealing", "cow", + "recovery", "invalidation", "export", "retirement", "collection", +] +for report in reports: + if report.get("schema") != "trail.artifact-adapter-certification/v1": + raise SystemExit("unsupported certification report schema") + if report.get("status") != "passed" or report.get("authority_effect") != "evidence_only": + raise SystemExit("certification report did not pass as evidence-only") + checks = report.get("checks") + if not isinstance(checks, list) or [check.get("stage") for check in checks] != expected_stages: + raise SystemExit("certification report stages are incomplete or unordered") + if any(check.get("status") != "passed" or not check.get("applicable") for check in checks): + raise SystemExit("required certification stage was skipped or failed") + +encoded = json.dumps(reports, indent=2, sort_keys=True) + "\n" +if destination: + target = pathlib.Path(destination) + target.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix=f".{target.name}.", dir=target.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, target) + except BaseException: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise +else: + sys.stdout.write(encoded) +PY diff --git a/scripts/verify-artifact-native-cow-matrix.py b/scripts/verify-artifact-native-cow-matrix.py new file mode 100644 index 00000000..91baa4b3 --- /dev/null +++ b/scripts/verify-artifact-native-cow-matrix.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""Run one owning-host native COW qualification and record unavailable backends.""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + + +BACKENDS = { + "nfs_macos": { + "platform": "darwin", + "flag": "TRAIL_RUN_NFS_COW_TESTS", + "native_test": "nfs_mount_reads_artifact_manifest_without_layer_materialization", + }, + "fuse_linux": { + "platform": "linux", + "flag": "TRAIL_RUN_FUSE_COW_TESTS", + "native_test": "fuse_mount_reads_artifact_manifest_without_layer_materialization", + }, + "dokan_windows": { + "platform": "win32", + "flag": "TRAIL_RUN_DOKAN_COW_TESTS", + "native_test": "dokan_mount_reads_artifact_manifest_without_layer_materialization", + }, +} + +COMMON_CHECKS = ( + ( + "fork_inheritance", + "integration", + "lane_environment_inheritance", + "lane_fork_inherits_verified_immutable_layer_with_fresh_private_uppers", + ), + ( + "materialization_recovery", + "lib", + "", + "artifact_materialization_cache_is_tree_keyed_verified_and_copy_safe", + ), + ( + "promotion", + "integration", + "e2e", + "manual_private_output_promotion_uses_native_cli_sandbox_and_preserves_private_bytes", + ), + ( + "source_export", + "lib", + "", + "source_export_execution_checkpoints_normal_source_and_reports_git_handoff", + ), + ( + "retirement", + "lib", + "", + "builtins_plugins_and_repository_v2_share_artifact_pipeline_conformance", + ), + ( + "publication_recovery", + "lib", + "", + "singleflight_waiters_cancel_and_dead_owners_are_fenced", + ), + ( + "lane_storage_1_5_20", + "lib", + "", + "artifact_accounting_does_not_multiply_shared_authority_across_1_5_20_lanes", + ), +) + + +def fail(message: str) -> None: + raise SystemExit(f"artifact native COW matrix: {message}") + + +def host_platform() -> str: + if sys.platform.startswith("win"): + return "win32" + if sys.platform == "darwin": + return "darwin" + if sys.platform.startswith("linux"): + return "linux" + return sys.platform + + +def command_for(kind: str, target: str, test_name: str) -> list[str]: + command = ["cargo", "test", "-p", "trail"] + if kind == "lib": + command.append("--lib") + else: + command.extend(("--test", target)) + command.extend((test_name, "--locked", "--", "--nocapture")) + return command + + +def run_check(stage: str, command: list[str], scope: str) -> dict[str, Any]: + started = time.monotonic() + result = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + elapsed_ms = int((time.monotonic() - started) * 1000) + sys.stdout.write(result.stdout) + passed = ( + result.returncode == 0 + and "running 1 test" in result.stdout + and "1 passed; 0 failed" in result.stdout + ) + return { + "stage": stage, + "status": "passed" if passed else "failed", + "scope": scope, + "command": command, + "test_count": 1 if passed else 0, + "duration_ms": elapsed_ms, + "exit_code": result.returncode, + } + + +def tool_output(*command: str) -> str | None: + result = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + if result.returncode != 0: + return None + value = result.stdout.strip() + return value or None + + +def validate(report: dict[str, Any], owning_backend: str) -> None: + if report.get("schema") != "trail.artifact-native-cow-matrix/v1": + fail("internal report schema mismatch") + backends = report.get("backends") + if not isinstance(backends, list) or [row.get("backend") for row in backends] != list(BACKENDS): + fail("backend rows are missing, duplicated, or unordered") + for row in backends: + backend = row["backend"] + if backend == owning_backend: + if row.get("status") != "passed" or row.get("qualification_kind") != "composed_owning_host": + fail(f"{backend} did not produce owning-host qualification") + checks = row.get("checks") + expected_stages = ["immutable_lower_private_whiteout", *[check[0] for check in COMMON_CHECKS]] + if not isinstance(checks, list) or [check.get("stage") for check in checks] != expected_stages: + fail(f"{backend} check matrix is incomplete or unordered") + if any(check.get("status") != "passed" or check.get("test_count") != 1 for check in checks): + fail(f"{backend} contains a failed or skipped check") + if checks[0].get("scope") != "native_mounted_backend": + fail(f"{backend} lacks mounted-backend evidence") + if any(check.get("scope") != "shared_cas_lifecycle_on_owning_host" for check in checks[1:]): + fail(f"{backend} lifecycle evidence has an unsupported scope") + elif row.get("status") != "unverified" or row.get("reason") != "not_owning_platform": + fail(f"{backend} must remain explicitly unverified") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + + current = host_platform() + owning = [name for name, config in BACKENDS.items() if config["platform"] == current] + if len(owning) != 1: + fail(f"unsupported owning platform {current!r}") + owning_backend = owning[0] + config = BACKENDS[owning_backend] + if os.environ.get(str(config["flag"])) != "1": + fail(f"set {config['flag']}=1; refusing a native test that could silently skip") + + native_command = command_for("lib", "", str(config["native_test"])) + checks = [ + run_check( + "immutable_lower_private_whiteout", + native_command, + "native_mounted_backend", + ) + ] + for stage, kind, target, test_name in COMMON_CHECKS: + checks.append( + run_check( + stage, + command_for(kind, target, test_name), + "shared_cas_lifecycle_on_owning_host", + ) + ) + + rows = [] + for backend, backend_config in BACKENDS.items(): + if backend == owning_backend: + rows.append( + { + "backend": backend, + "platform": backend_config["platform"], + "status": "passed" if all(check["status"] == "passed" for check in checks) else "failed", + "qualification_kind": "composed_owning_host", + "composition": { + "native_adapter_scope": "immutable lower lookup plus private copy-up and whiteout", + "shared_authority_scope": "inheritance, materialization, promotion, export, retirement, recovery, and accounting", + }, + "checks": checks, + } + ) + else: + rows.append( + { + "backend": backend, + "platform": backend_config["platform"], + "status": "unverified", + "reason": "not_owning_platform", + "checks": [], + } + ) + + report = { + "schema": "trail.artifact-native-cow-matrix/v1", + "host": { + "platform": current, + "operating_system": platform.platform(), + "architecture": platform.machine(), + }, + "toolchain": { + "trail_commit": tool_output("git", "rev-parse", "HEAD"), + "rustc": tool_output("rustc", "--version"), + }, + "backends": rows, + } + validate(report, owning_backend) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"verified {owning_backend} artifact COW matrix: {args.output}") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify-artifact-real-tool-gates.sh b/scripts/verify-artifact-real-tool-gates.sh new file mode 100755 index 00000000..676a4695 --- /dev/null +++ b/scripts/verify-artifact-real-tool-gates.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${repo_root}" + +if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + : "${RUNNER_TEMP:?GitHub Actions did not provide RUNNER_TEMP}" + CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-${RUNNER_TEMP}/trail-artifact-real-tool-target}" + export CARGO_TARGET_DIR +else + : "${CARGO_TARGET_DIR:?set a checkout-specific CARGO_TARGET_DIR beneath /Volumes/Workspace/crabbuild-target}" + if [[ "${CARGO_TARGET_DIR}" != /Volumes/Workspace/crabbuild-target/* ]]; then + printf '%s\n' 'CARGO_TARGET_DIR must be beneath /Volumes/Workspace/crabbuild-target' >&2 + exit 2 + fi +fi + +for tool in cargo git node npm cmake python3; do + if ! command -v "${tool}" >/dev/null 2>&1; then + printf 'required real-tool gate dependency is unavailable: %s\n' "${tool}" >&2 + exit 2 + fi +done + +case "$(uname -s)" in + Linux) + if [[ "${TRAIL_RUN_FUSE_COW_TESTS:-}" != "1" ]]; then + printf '%s\n' 'real path-bound gates require TRAIL_RUN_FUSE_COW_TESTS=1 on Linux' >&2 + exit 2 + fi + ;; + Darwin) + if [[ "${TRAIL_RUN_NFS_COW_TESTS:-}" != "1" ]]; then + printf '%s\n' 'real path-bound gates require TRAIL_RUN_NFS_COW_TESTS=1 on macOS' >&2 + exit 2 + fi + ;; + *) + printf '%s\n' 'portable artifact real-tool gate currently requires Linux or macOS' >&2 + exit 2 + ;; +esac + +tests=( + db::lane::workspace_environment::tests::host_resolver_executes_cargo_in_isolated_staging_and_reuses_snapshot + db::lane::workspace_node::tests::manifest_only_npm_uses_managed_lock_and_preserves_seed_cache_isolation + db::lane::workspace_cargo::tests::cargo_adapter_builds_once_and_reuses_one_immutable_target_seed + db::lane::workspace_python::tests::real_python_venvs_embed_lane_paths_and_remain_isolated + db::lane::workspace_cmake::tests::real_cmake_configure_build_and_clean_stay_lane_private + db::lane::workspace_plugin::tests::protocol_v2_bazel_nix_like_stores_remain_metadata_only_after_host_normalization + db::lane::workspace_recipe::tests::maven_gradle_like_and_unknown_custom_shapes_use_repository_v2_components + db::lane::source_export::tests::source_export_execution_checkpoints_normal_source_and_reports_git_handoff +) + +for test_name in "${tests[@]}"; do + cargo test -p trail --lib "${test_name}" --locked -- --exact --nocapture +done + +cargo test -p trail --test e2e \ + next_and_vite_v2_components_compose_through_native_cli_sandbox \ + --locked -- --exact --nocapture + +printf '%s\n' 'artifact real-tool gates: passed' diff --git a/scripts/verify-layered-lane-scale.sh b/scripts/verify-layered-lane-scale.sh index 74ad264f..b44c914b 100755 --- a/scripts/verify-layered-lane-scale.sh +++ b/scripts/verify-layered-lane-scale.sh @@ -20,8 +20,14 @@ for experiment in "10000 1" "100000 5" "1000000 20"; do TRAIL_SCALE_PATHS="$paths" \ TRAIL_SCALE_LANES="$lanes" \ cargo test -p trail --lib large_path_multi_view_scale_acceptance --locked -- --nocapture + TRAIL_RUN_ARTIFACT_SCALE_TEST=1 \ + TRAIL_SCALE_ARTIFACT_ENTRIES="$paths" \ + TRAIL_SCALE_LANES="$lanes" \ + cargo test -p trail --lib large_artifact_multi_lane_scale_acceptance --locked -- --nocapture done +python3 scripts/check-layered-lane-scale-evidence.py "$TRAIL_SCALE_EVIDENCE_DIR" + # Complement the path/lane matrix with the correctness experiments whose # runtime is independent of synthetic path count. These cover reusable # parent/child layers, private-output promotion, interrupted publication, diff --git a/trail-environment-adapter-sdk/Cargo.toml b/trail-environment-adapter-sdk/Cargo.toml index da48275a..e4879a33 100644 --- a/trail-environment-adapter-sdk/Cargo.toml +++ b/trail-environment-adapter-sdk/Cargo.toml @@ -19,6 +19,7 @@ thiserror.workspace = true [dev-dependencies] ed25519-dalek.workspace = true hex.workspace = true +proptest = "=1.8.0" sha2.workspace = true toml.workspace = true diff --git a/trail-environment-adapter-sdk/README.md b/trail-environment-adapter-sdk/README.md index f4f5bda3..9e9b7033 100644 --- a/trail-environment-adapter-sdk/README.md +++ b/trail-environment-adapter-sdk/README.md @@ -132,6 +132,56 @@ wire types. Select it in `trail-adapter.toml`: protocols = ["trail.environment-adapter/v2"] ``` +Protocol v3 is a separate, strictly bounded wire surface so existing v1/v2 request, +response, and plan layouts remain unchanged. `AdapterRequestV3` carries exact host +ceilings and either pinned proposal inputs or typed planning inputs plus an optional +verified resolution snapshot. `AdapterResponseV3` returns an incomplete/ready proposal +or an `AdapterPipelineV3` containing resolution, typed phases, validations, capability +profiles, identity declarations, outputs, explicit source exports, attestation +requirements, secret taint, and quarantine policy. + +Call `validate_bounds()` before writing a v3 frame. It enforces hard limits for input +bytes/files, actions, validations, outputs, authorities, exports, maps, strings, resolver +captures, output trees, and child processes. Trail repeats this validation and the full +semantic/trust validation at the host boundary. Host evidence in a v3 request is tagged +and object-referenced; an adapter cannot mint an attestation, resolve a quarantine, +publish an artifact, mount a lane, or write exported source. + +Protocol-v3 negotiation and host normalization are explicit. Merely adding v3 types to +an adapter does not cause a v1/v2 package to receive resolution, export, attestation, or +compatibility authority. + +Start v3 authoring from +[`examples/artifact-pipeline-adapter.rs`](examples/artifact-pipeline-adapter.rs). Use +`AdapterPipelineV3::builder(...)` rather than constructing an unvalidated response. The +builder canonicalizes set-like dependencies, inputs, authorities, validations, outputs, +and exports; rejects duplicates, unsafe paths and shell-style commands; verifies export +references; and prevents secret-tainted output from becoming reusable or exportable. +It returns `AdapterPipelineV3BuildError` for local authoring errors. The host still +repeats complete validation because serialized plugin output is untrusted. + +A v3 package declares its maximum semantics explicitly; omitted capability metadata is +the deny-all legacy value and is not serialized into v1/v2 packages: + +```toml +[adapter] +protocols = ["trail.environment-adapter/v3", "trail.environment-adapter/v2"] + +[adapter.capabilities] +resolution = true +source_exports = true +host_attestation_evidence = false +host_quarantine_evidence = false +certification_ceiling = "local_artifact" +``` + +`negotiate_highest_mutual_protocol` compares exact protocol identities and selects v3, +then v2, then v1 only when both peers list that exact value. Unknown future strings, +aliases, prefixes, and package list order cannot change the result. The canonical legacy +projection retains v1/v2 data while `v3_only` remains entirely absent, so conversion +cannot infer resolution, validation, certification, export, attestation, or quarantine +authority from missing fields. + A mounted-only plan is authored as: ```rust @@ -228,6 +278,33 @@ contract in the component key, persists it with each generation, and treats clea provider-owned. Registry access, tag resolution, credentials, and runtime allocation are deliberately outside the planner. +The same metadata-only contract represents an already verified provider store without +copying it into a Trail layer. This is suitable for content-addressed build stores, +remote action caches, and other immutable provider objects; it is not a request for +Trail to fetch or execute against the reference: + +```rust +use trail_environment_adapter_sdk::{AdapterExternalArtifact, AdapterPlanV2}; + +let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +let plan = AdapterPlanV2::builder("external-stores", "external") + .identity_input("stores.lock") + .external_artifact(AdapterExternalArtifact::verified_external( + "dependency-store", + "local-store", + "store://objects/example-package", + digest, + "linux/x86_64", + )) + .stale_reason("verified provider identity changed") + .build()?; +# Ok::<(), trail_environment_adapter_sdk::AdapterPlanBuildError>(()) +``` + +`verified_external` accepts a bounded provider token, opaque safe reference, SHA-256 +digest, and exact platform identity (or `any`). It cannot back an OCI runtime resource; +container declarations still require a digest-pinned `oci_image` in the same plan. + An external plan may bind a pinned image to a lane-private service declaration. The adapter still performs no provider calls and receives no Docker socket, network, port, volume, or cleanup authority: diff --git a/trail-environment-adapter-sdk/examples/artifact-pipeline-adapter.rs b/trail-environment-adapter-sdk/examples/artifact-pipeline-adapter.rs new file mode 100644 index 00000000..11d60e48 --- /dev/null +++ b/trail-environment-adapter-sdk/examples/artifact-pipeline-adapter.rs @@ -0,0 +1,134 @@ +use std::collections::BTreeMap; + +use trail_environment_adapter_sdk::{ + denied_capabilities_v3, serve_once_v3, AdapterActionLimitsV3, AdapterActionPhaseV3, + AdapterActionV3, AdapterComponentProposalV3, AdapterIdentityContractV3, AdapterInputRoleV3, + AdapterInputV3, AdapterOperationV3, AdapterOutput, AdapterPipelineV3, AdapterPortability, + AdapterProcessCapabilityV3, AdapterProposalStatusV3, AdapterQuarantinePolicyV3, + AdapterResponseV3, AdapterResultV3, AdapterSourceExportAuthorizationV3, + AdapterSourceExportCollisionV3, AdapterSourceExportV3, AdapterValidationKindV3, + AdapterValidationV3, PROTOCOL_V3, +}; + +fn main() -> Result<(), Box> { + serve_once_v3(|request| { + let result = if request.protocol != PROTOCOL_V3 + || request.adapter_identity != "example/artifact-pipeline@1" + { + AdapterResultV3::Error { + code: "unsupported_request".into(), + message: "expected the example protocol-v3 adapter identity".into(), + recovery_actions: Vec::new(), + } + } else { + match &request.operation { + AdapterOperationV3::Propose { + component_root, + files, + } => AdapterResultV3::Proposed { + component: files + .iter() + .any(|file| file.path == "schema.json") + .then(|| AdapterComponentProposalV3 { + component_id: "example.generated-client".into(), + component_root: component_root.clone(), + kind: "generated".into(), + status: AdapterProposalStatusV3::Ready, + proposal_key: format!("{}:schema", request.source_root), + missing_requirements: Vec::new(), + recovery_actions: Vec::new(), + }), + }, + AdapterOperationV3::Plan { + proposal, files, .. + } => { + let mut builder = AdapterPipelineV3::builder( + (**proposal).clone(), + AdapterIdentityContractV3 { + normalizer_version: "trail-path-v1".into(), + source_closure_complete: true, + semantic_identities: BTreeMap::from([( + "generator_mode".into(), + "client".into(), + )]), + target: "host".into(), + platform: request.host.operating_system.clone(), + architecture: request.host.architecture.clone(), + abi: "host".into(), + portability: AdapterPortability::Host, + portability_certified: false, + portability_scope: "workspace".into(), + trust_scope: "local_plugin".into(), + }, + ); + for pinned in files { + builder = builder.input(AdapterInputV3 { + role: AdapterInputRoleV3::Identity, + required: true, + ..pinned.input.clone() + }); + } + let pipeline = builder + .action(AdapterActionV3 { + name: "construct".into(), + phase: AdapterActionPhaseV3::Construct, + program: "example-codegen".into(), + argv: vec!["build".into(), "--output".into(), "generated".into()], + working_directory: ".".into(), + environment: BTreeMap::new(), + capabilities: + trail_environment_adapter_sdk::AdapterCapabilityProfileV3 { + process: AdapterProcessCapabilityV3::DeclaredExecutable, + ..denied_capabilities_v3() + }, + limits: AdapterActionLimitsV3 { + timeout_ms: 30_000, + stdout_bytes: 1024 * 1024, + stderr_bytes: 1024 * 1024, + output_entries: 10_000, + output_bytes: 64 * 1024 * 1024, + child_processes: 0, + }, + }) + .validation(AdapterValidationV3 { + name: "path-contract".into(), + kind: AdapterValidationKindV3::PathContract, + path: "generated".into(), + required: true, + parameters: BTreeMap::new(), + }) + .output(AdapterOutput::immutable_seed_private( + "generated", + "generated", + ".trail-generated/generated", + )) + .source_export(AdapterSourceExportV3 { + name: "client".into(), + output_name: "generated".into(), + artifact_subpath: "client".into(), + destination: "src/generated".into(), + collision: AdapterSourceExportCollisionV3::Fail, + required_validation: "path-contract".into(), + required_gate: None, + authorization: AdapterSourceExportAuthorizationV3::ExplicitUser, + }) + .quarantine_policy(AdapterQuarantinePolicyV3::FailClosed) + .stale_reason("schema, generator, or host identity changed") + .build(); + match pipeline { + Ok(pipeline) => AdapterResultV3::Planned { + pipeline: Box::new(pipeline), + }, + Err(error) => AdapterResultV3::Error { + code: "invalid_plan".into(), + message: error.to_string(), + recovery_actions: Vec::new(), + }, + } + } + } + }; + AdapterResponseV3::for_request(&request, result) + })?; + Ok(()) +} diff --git a/trail-environment-adapter-sdk/src/lib.rs b/trail-environment-adapter-sdk/src/lib.rs index 2454906e..c2c61c84 100644 --- a/trail-environment-adapter-sdk/src/lib.rs +++ b/trail-environment-adapter-sdk/src/lib.rs @@ -10,16 +10,42 @@ use std::io::{self, Read, Write}; use serde::{Deserialize, Serialize}; +mod protocol_v3; +pub use protocol_v3::*; + pub const PROTOCOL_V1: &str = "trail.environment-adapter/v1"; /// Adds host-sandboxed actions that execute against Trail's ephemeral mounted /// candidate view. V1 remains the default for packages that do not declare a /// protocol list, so existing adapters keep their exact behavior. pub const PROTOCOL_V2: &str = "trail.environment-adapter/v2"; +/// Adds incomplete proposals, explicit resolution, typed artifact phases, +/// validation/capability contracts, generated-source exports, and host-owned +/// attestation/quarantine evidence without changing v1/v2 wire meanings. +pub const PROTOCOL_V3: &str = "trail.environment-adapter/v3"; +/// Exact protocol identities in descending semantic version order. +pub const ADAPTER_PROTOCOLS_HIGHEST_FIRST: [&str; 3] = [PROTOCOL_V3, PROTOCOL_V2, PROTOCOL_V1]; pub const PACKAGE_SCHEMA_V1: &str = "trail.environment-adapter-package/v1"; pub const PACKAGE_SIGNATURE_SCHEMA_V1: &str = "trail.environment-adapter-signature/v1"; pub const TRUSTED_PUBLISHER_KEY_SCHEMA_V1: &str = "trail.environment-adapter-publisher-key/v1"; pub const MAX_FRAME_BYTES: usize = 16 * 1024 * 1024; +/// Select the highest exact protocol identity supported by both peers. +/// Prefixes, aliases, unknown future versions, and list order never influence +/// the result. +pub fn negotiate_highest_mutual_protocol( + host_protocols: &[&str], + package_protocols: &[String], +) -> Option<&'static str> { + ADAPTER_PROTOCOLS_HIGHEST_FIRST + .into_iter() + .find(|candidate| { + host_protocols.contains(candidate) + && package_protocols + .iter() + .any(|protocol| protocol == candidate) + }) +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct AdapterPackageManifest { @@ -46,6 +72,10 @@ pub struct AdapterMetadata { skip_serializing_if = "is_default_v1_protocols" )] pub protocols: Vec, + /// V3-only declarations. Missing metadata grants no v3 semantics and is + /// omitted from legacy package serialization. + #[serde(default, skip_serializing_if = "AdapterPackageCapabilities::is_denied")] + pub capabilities: AdapterPackageCapabilities, #[serde(default = "default_supported_operating_systems")] pub supported_operating_systems: Vec, #[serde(default = "default_supported_architectures")] @@ -58,6 +88,41 @@ fn default_adapter_protocols() -> Vec { vec![PROTOCOL_V1.to_string()] } +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct AdapterPackageCapabilities { + pub resolution: bool, + pub source_exports: bool, + pub host_attestation_evidence: bool, + pub host_quarantine_evidence: bool, + pub certification_ceiling: AdapterCertificationCeiling, +} + +impl AdapterPackageCapabilities { + pub fn is_denied(&self) -> bool { + self == &Self::default() + } +} + +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum AdapterCertificationCeiling { + #[default] + LegacyExact, + LocalArtifact, + PortableArtifact, +} + +impl AdapterCertificationCeiling { + pub fn as_str(self) -> &'static str { + match self { + Self::LegacyExact => "legacy_exact", + Self::LocalArtifact => "local_artifact", + Self::PortableArtifact => "portable_artifact", + } + } +} + fn is_default_v1_protocols(protocols: &[String]) -> bool { protocols == [PROTOCOL_V1] } @@ -318,6 +383,27 @@ impl AdapterExternalArtifact { cleanup_owner: "external".to_string(), } } + + /// A provider-owned immutable store entry that Trail records by exact + /// reference and SHA-256 digest without manufacturing a filesystem layer. + /// The provider remains responsible for storage and cleanup. + pub fn verified_external( + name: impl Into, + provider: impl Into, + reference: impl Into, + digest: impl Into, + platform: impl Into, + ) -> Self { + Self { + name: name.into(), + artifact_type: "verified_external".to_string(), + provider: provider.into(), + reference: reference.into(), + digest: digest.into(), + platform: platform.into(), + cleanup_owner: "external".to_string(), + } + } } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -1076,8 +1162,9 @@ fn validate_adapter_external_artifacts( .bytes() .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) }); - let valid_reference = - artifact + let valid_oci = artifact.artifact_type == "oci_image" + && artifact.provider == "oci" + && artifact .reference .rsplit_once('@') .is_some_and(|(repository, digest)| { @@ -1087,22 +1174,24 @@ fn validate_adapter_external_artifacts( && repository .bytes() .all(|byte| byte.is_ascii_alphanumeric() || b"._-/:".contains(&byte)) - }); + }) + && matches!( + artifact.platform.as_str(), + "linux/amd64" | "linux/arm64" | "windows/amd64" | "windows/arm64" + ); + let valid_verified_external = artifact.artifact_type == "verified_external" + && valid_external_identity_token(&artifact.provider) + && valid_external_reference(&artifact.reference) + && valid_external_platform(&artifact.platform); if artifact.name.is_empty() || artifact.name.len() > 128 || !artifact.name.chars().all(|character| { character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') }) || index > 0 && artifacts[index - 1].name == artifact.name - || artifact.artifact_type != "oci_image" - || artifact.provider != "oci" || artifact.cleanup_owner != "external" || !valid_digest - || !valid_reference - || !matches!( - artifact.platform.as_str(), - "linux/amd64" | "linux/arm64" | "windows/amd64" | "windows/arm64" - ) + || !(valid_oci || valid_verified_external) { return Err(AdapterPlanBuildError::InvalidExternalArtifact { artifact: artifact.name.clone(), @@ -1112,6 +1201,55 @@ fn validate_adapter_external_artifacts( Ok(()) } +fn valid_external_identity_token(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._-".contains(&byte) + }) +} + +fn valid_external_reference(value: &str) -> bool { + let lowered = value.to_ascii_lowercase(); + !value.is_empty() + && value.len() <= 2048 + && !value.chars().any(char::is_control) + && ![ + "authorization", + "password", + "passwd", + "secret", + "token", + "api_key", + "api-key", + "apikey", + "private_key", + "private-key", + "bearer", + ] + .iter() + .any(|needle| lowered.contains(needle)) + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"._-/:+@=".contains(&byte)) +} + +fn valid_external_platform(value: &str) -> bool { + value == "any" + || (!value.is_empty() + && value.len() <= 128 + && !value.starts_with('/') + && !value.ends_with('/') + && value.split('/').all(|segment| { + !segment.is_empty() + && segment != "." + && segment != ".." + && segment.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._-".contains(&byte) + }) + })) +} + fn validate_adapter_runtime_resources( resources: &mut [AdapterRuntimeResource], artifacts: &[AdapterExternalArtifact], @@ -1123,8 +1261,13 @@ fn validate_adapter_runtime_resources( } let artifact_names = artifacts .iter() - .map(|artifact| artifact.name.as_str()) - .collect::>(); + .map(|artifact| { + ( + artifact.name.as_str(), + artifact.artifact_type == "oci_image" && artifact.provider == "oci", + ) + }) + .collect::>(); resources.sort_by(|left, right| left.name.cmp(&right.name)); for resource in resources.iter_mut() { resource @@ -1155,7 +1298,9 @@ fn validate_adapter_runtime_resources( || index > 0 && resources[index - 1].name == resource.name || resource.runtime_type != "container" || resource.provider != "oci" - || !artifact_names.contains(resource.artifact_name.as_str()) + || !artifact_names + .get(resource.artifact_name.as_str()) + .is_some_and(|is_oci_image| *is_oci_image) || resource.container_port == 0 || resource.protocol != "tcp" || resource.health_type != "tcp" @@ -1659,10 +1804,47 @@ pub fn serve_once( write_frame(&mut io::stdout().lock(), &response, MAX_FRAME_BYTES) } +pub fn serve_once_v3( + handler: impl FnOnce(AdapterRequestV3) -> AdapterResponseV3, +) -> Result<(), ProtocolError> { + let request = read_frame(&mut io::stdin().lock(), MAX_FRAME_BYTES)?; + let response = handler(request); + write_frame(&mut io::stdout().lock(), &response, MAX_FRAME_BYTES) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn protocol_negotiation_selects_highest_exact_mutual_identity() { + let package = vec![ + PROTOCOL_V1.to_string(), + "trail.environment-adapter/v30".to_string(), + PROTOCOL_V3.to_string(), + PROTOCOL_V2.to_string(), + ]; + assert_eq!( + negotiate_highest_mutual_protocol(&[PROTOCOL_V1, PROTOCOL_V2, PROTOCOL_V3], &package), + Some(PROTOCOL_V3) + ); + assert_eq!( + negotiate_highest_mutual_protocol(&[PROTOCOL_V1, PROTOCOL_V2], &package), + Some(PROTOCOL_V2) + ); + assert_eq!( + negotiate_highest_mutual_protocol(&[PROTOCOL_V1], &package), + Some(PROTOCOL_V1) + ); + assert_eq!( + negotiate_highest_mutual_protocol( + &[PROTOCOL_V3], + &["trail.environment-adapter/v3-preview".to_string()] + ), + None + ); + } + #[test] fn framed_protocol_round_trips_binary_pinned_files() { let request = AdapterRequest { @@ -1865,6 +2047,62 @@ mod tests { ); } + #[test] + fn v2_builder_supports_bazel_nix_like_external_stores_without_framework_modes() { + let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let plan = AdapterPlanV2::builder("external-stores", "external") + .identity_input("stores.lock") + .external_artifact(AdapterExternalArtifact::verified_external( + "content-addressed-store", + "local-store", + "store://objects/example-package", + digest, + "linux/x86_64", + )) + .external_artifact(AdapterExternalArtifact::verified_external( + "remote-action-cache", + "remote-cas", + "cas://objects/example-action", + digest, + "any", + )) + .stale_reason("verified external store identities changed") + .build() + .unwrap(); + + assert!(plan.actions.is_empty()); + assert!(plan.outputs.is_empty()); + assert!(plan.caches.is_empty()); + assert_eq!(plan.external_artifacts.len(), 2); + assert!(plan + .external_artifacts + .iter() + .all(|artifact| artifact.artifact_type == "verified_external")); + let decoded: AdapterPlanV2 = + serde_cbor::from_slice(&serde_cbor::to_vec(&plan).unwrap()).unwrap(); + assert_eq!(decoded, plan); + + let runtime = AdapterPlanV2::builder("invalid-runtime", "external") + .external_artifact(AdapterExternalArtifact::verified_external( + "store", + "local-store", + "store://objects/example-package", + digest, + "linux/x86_64", + )) + .runtime_resource(AdapterRuntimeResource::oci_container( + "invalid-container", + "store", + 8080, + )) + .stale_reason("invalid external runtime") + .build(); + assert!(matches!( + runtime, + Err(AdapterPlanBuildError::InvalidRuntimeResource { .. }) + )); + } + #[test] fn v2_builder_preserves_typed_dependency_semantics() { let plan = AdapterPlanV2::builder("application", "runtime") @@ -1993,6 +2231,7 @@ sha256 = "sha256:00" ) .unwrap(); assert_eq!(manifest.adapter.protocols, [PROTOCOL_V1]); + assert!(manifest.adapter.capabilities.is_denied()); let encoded = serde_cbor::to_vec(&manifest).unwrap(); let value: serde_cbor::Value = serde_cbor::from_slice(&encoded).unwrap(); let serde_cbor::Value::Map(package) = value else { @@ -2005,6 +2244,46 @@ sha256 = "sha256:00" panic!("adapter metadata did not encode as a CBOR map"); }; assert!(!adapter.contains_key(&serde_cbor::Value::Text("protocols".to_string()))); + assert!(!adapter.contains_key(&serde_cbor::Value::Text("capabilities".to_string()))); + } + + #[test] + fn protocol_v3_package_capabilities_are_explicit_and_round_trip() { + let manifest: AdapterPackageManifest = toml::from_str( + r#"schema = "trail.environment-adapter-package/v1" +[adapter] +canonical_identity = "example/test@1" +implementation_version = "1" +selectors = ["example/test@1"] +kind = "generated" +layer_adapter_name = "test" +discovery_markers = ["test.adapter"] +protocols = ["trail.environment-adapter/v3"] +stability = "experimental" +description = "test" + +[adapter.capabilities] +resolution = true +source_exports = true +host_attestation_evidence = true +host_quarantine_evidence = true +certification_ceiling = "local_artifact" + +[executable] +path = "adapter" +sha256 = "sha256:00" +"#, + ) + .unwrap(); + assert!(manifest.adapter.capabilities.resolution); + assert!(manifest.adapter.capabilities.source_exports); + assert_eq!( + manifest.adapter.capabilities.certification_ceiling, + AdapterCertificationCeiling::LocalArtifact + ); + let decoded: AdapterPackageManifest = + serde_cbor::from_slice(&serde_cbor::to_vec(&manifest).unwrap()).unwrap(); + assert_eq!(decoded, manifest); } #[test] diff --git a/trail-environment-adapter-sdk/src/protocol_v3.rs b/trail-environment-adapter-sdk/src/protocol_v3.rs new file mode 100644 index 00000000..42b6a51a --- /dev/null +++ b/trail-environment-adapter-sdk/src/protocol_v3.rs @@ -0,0 +1,2182 @@ +//! Protocol-v3 wire types. +//! +//! These types deliberately describe plans and host evidence, not execution +//! authority. An adapter cannot mount a lane, publish an artifact, mint a +//! Trail attestation, resolve a quarantine, or write generated source. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +use crate::{ + AdapterAction, AdapterCache, AdapterDependency, AdapterDependencyType, AdapterExternalArtifact, + AdapterHost, AdapterOutput, AdapterOutputPolicy, AdapterPlan, AdapterPlanV2, + AdapterPortability, AdapterRuntimeResource, PinnedFile, PROTOCOL_V1, PROTOCOL_V2, PROTOCOL_V3, +}; + +pub const MAX_V3_PROPOSALS: usize = 1_024; +pub const MAX_V3_REQUIREMENTS: usize = 256; +pub const MAX_V3_RECOVERY_ACTIONS: usize = 64; +pub const MAX_V3_INPUTS: usize = 100_000; +pub const MAX_V3_INPUT_BYTES: u64 = 8 * 1024 * 1024; +pub const MAX_V3_ACTIONS: usize = 64; +pub const MAX_V3_VALIDATIONS: usize = 64; +pub const MAX_V3_OUTPUTS: usize = 32; +pub const MAX_V3_SOURCE_EXPORTS: usize = 32; +pub const MAX_V3_AUTHORITIES: usize = 128; +pub const MAX_V3_IDENTITIES: usize = 1_024; +pub const MAX_V3_ATTESTATION_REFERENCES: usize = 256; +pub const MAX_V3_QUARANTINE_REFERENCES: usize = 64; +pub const MAX_V3_STRING_BYTES: usize = 4_096; +pub const MAX_V3_ARGV: usize = 256; +pub const MAX_V3_MAP_ENTRIES: usize = 1_024; +pub const MAX_V3_ACTION_TIMEOUT_MS: u64 = 60 * 60 * 1_000; +pub const MAX_V3_CAPTURE_BYTES: u64 = 16 * 1024 * 1024; +pub const MAX_V3_OUTPUT_ENTRIES: u64 = 1_000_000; +pub const MAX_V3_OUTPUT_BYTES: u64 = 1024 * 1024 * 1024; +pub const MAX_V3_CHILD_PROCESSES: u32 = 256; + +/// Host ceilings carried on every v3 request. A package may narrow these +/// values in its declaration, but an adapter response cannot widen them. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterProtocolLimitsV3 { + pub max_input_files: u32, + pub max_input_bytes: u64, + pub max_actions: u16, + pub max_validations: u16, + pub max_outputs: u16, + pub max_source_exports: u16, + pub max_authorities: u16, + pub max_response_bytes: u64, +} + +impl Default for AdapterProtocolLimitsV3 { + fn default() -> Self { + Self { + max_input_files: 4_096, + max_input_bytes: MAX_V3_INPUT_BYTES, + max_actions: MAX_V3_ACTIONS as u16, + max_validations: MAX_V3_VALIDATIONS as u16, + max_outputs: MAX_V3_OUTPUTS as u16, + max_source_exports: MAX_V3_SOURCE_EXPORTS as u16, + max_authorities: MAX_V3_AUTHORITIES as u16, + max_response_bytes: 4 * 1024 * 1024, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterRequestV3 { + pub protocol: String, + pub request_id: String, + pub adapter_identity: String, + pub distribution_digest: String, + pub host: AdapterHost, + pub source_root: String, + pub limits: AdapterProtocolLimitsV3, + pub operation: AdapterOperationV3, +} + +impl AdapterRequestV3 { + pub fn new( + request_id: impl Into, + adapter_identity: impl Into, + distribution_digest: impl Into, + host: AdapterHost, + source_root: impl Into, + operation: AdapterOperationV3, + ) -> Self { + Self { + protocol: PROTOCOL_V3.to_string(), + request_id: request_id.into(), + adapter_identity: adapter_identity.into(), + distribution_digest: distribution_digest.into(), + host, + source_root: source_root.into(), + limits: AdapterProtocolLimitsV3::default(), + operation, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "method", rename_all = "snake_case", deny_unknown_fields)] +pub enum AdapterOperationV3 { + Propose { + component_root: String, + files: Vec, + }, + Plan { + proposal: Box, + files: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + resolution_snapshot: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + host_evidence: Option>, + }, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterResponseV3 { + pub protocol: String, + pub request_id: String, + pub result: AdapterResultV3, +} + +impl AdapterResponseV3 { + pub fn for_request(request: &AdapterRequestV3, result: AdapterResultV3) -> Self { + Self { + protocol: request.protocol.clone(), + request_id: request.request_id.clone(), + result, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "result", rename_all = "snake_case", deny_unknown_fields)] +pub enum AdapterResultV3 { + Proposed { + component: Option, + }, + Planned { + pipeline: Box, + }, + Error { + code: String, + message: String, + #[serde(default)] + recovery_actions: Vec, + }, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterProposalStatusV3 { + Ready, + Resolvable, + Blocked, + Unsupported, + Ambiguous, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(deny_unknown_fields)] +pub struct AdapterMissingRequirementV3 { + pub code: String, + pub message: String, + pub requirement_type: AdapterRequirementTypeV3, + pub resolvable: bool, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterRequirementTypeV3 { + SourceInput, + ResolutionSnapshot, + Tool, + PlatformCapability, + PolicyApproval, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(deny_unknown_fields)] +pub struct AdapterRecoveryActionV3 { + pub code: String, + pub message: String, + pub operation: AdapterRecoveryOperationV3, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterRecoveryOperationV3 { + Resolve, + InstallTool, + SelectComponent, + GrantPolicy, + EditSource, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterComponentProposalV3 { + pub component_id: String, + pub component_root: String, + pub kind: String, + pub status: AdapterProposalStatusV3, + pub proposal_key: String, + #[serde(default)] + pub missing_requirements: Vec, + #[serde(default)] + pub recovery_actions: Vec, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterInputRoleV3 { + Identity, + Resolution, + Construction, + Validation, + Runtime, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(deny_unknown_fields)] +pub struct AdapterInputV3 { + pub path: String, + pub content_hash: String, + pub size_bytes: u64, + pub executable: bool, + pub role: AdapterInputRoleV3, + pub format: String, + pub required: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterPinnedInputV3 { + pub input: AdapterInputV3, + #[serde(with = "serde_bytes")] + pub content: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterResolutionPlanV3 { + pub name: String, + pub program: String, + pub argv: Vec, + pub working_directory: String, + pub readable_inputs: Vec, + pub candidate_output: String, + pub snapshot_format: String, + #[serde(default)] + pub allowed_authorities: Vec, + #[serde(default)] + pub credential_handles: Vec, + pub capabilities: AdapterCapabilityProfileV3, + pub limits: AdapterActionLimitsV3, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterResolutionSnapshotV3 { + pub snapshot_id: String, + pub proposal_key: String, + pub format: String, + pub content_sha256: String, + #[serde(with = "serde_bytes")] + pub content: Vec, + #[serde(default)] + pub resolved_identities: BTreeMap, + #[serde(default)] + pub checksums: BTreeMap, + pub verified: bool, + pub secret_taint: AdapterSecretTaintV3, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterActionPhaseV3 { + Construct, + Validate, + Finalize, + MountedInitialization, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterActionV3 { + pub name: String, + pub phase: AdapterActionPhaseV3, + pub program: String, + pub argv: Vec, + pub working_directory: String, + #[serde(default)] + pub environment: BTreeMap, + pub capabilities: AdapterCapabilityProfileV3, + pub limits: AdapterActionLimitsV3, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterActionLimitsV3 { + pub timeout_ms: u64, + pub stdout_bytes: u64, + pub stderr_bytes: u64, + pub output_entries: u64, + pub output_bytes: u64, + pub child_processes: u32, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterNetworkCapabilityV3 { + Deny, + ExactAuthorities, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterFilesystemCapabilityV3 { + Deny, + DeclaredInputs, + IsolatedCandidate, + DeclaredOutputs, + LanePrivateBindings, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterProcessCapabilityV3 { + Deny, + DeclaredExecutable, + ReviewedBuiltinGraph, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterSecretCapabilityV3 { + Deny, + OpaqueHandles, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterPublicationCapabilityV3 { + Deny, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterCapabilityProfileV3 { + pub network: AdapterNetworkCapabilityV3, + #[serde(default)] + pub network_authorities: Vec, + pub filesystem_read: AdapterFilesystemCapabilityV3, + pub filesystem_write: AdapterFilesystemCapabilityV3, + pub process: AdapterProcessCapabilityV3, + pub child_processes: u32, + pub secrets: AdapterSecretCapabilityV3, + pub publication: AdapterPublicationCapabilityV3, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterValidationKindV3 { + PathContract, + Checksum, + Command, + Relocatability, + Reproducibility, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(deny_unknown_fields)] +pub struct AdapterValidationV3 { + pub name: String, + pub kind: AdapterValidationKindV3, + pub path: String, + pub required: bool, + #[serde(default)] + pub parameters: BTreeMap, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterIdentityContractV3 { + pub normalizer_version: String, + pub source_closure_complete: bool, + #[serde(default)] + pub semantic_identities: BTreeMap, + pub target: String, + pub platform: String, + pub architecture: String, + pub abi: String, + pub portability: AdapterPortability, + pub portability_certified: bool, + pub portability_scope: String, + pub trust_scope: String, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterSourceExportCollisionV3 { + Fail, + Replace, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(deny_unknown_fields)] +pub struct AdapterSourceExportV3 { + pub name: String, + pub output_name: String, + pub artifact_subpath: String, + pub destination: String, + pub collision: AdapterSourceExportCollisionV3, + pub required_validation: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub required_gate: Option, + pub authorization: AdapterSourceExportAuthorizationV3, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterSourceExportAuthorizationV3 { + ExplicitUser, +} + +/// Adapter-declared evidence requirements. Trail alone constructs and signs +/// the resulting attestation after sealing an artifact. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterAttestationRequirementsV3 { + #[serde(default)] + pub required_validations: Vec, + pub require_sandbox_evidence: bool, + pub require_executable_identities: bool, + pub signature_policy: AdapterAttestationSignaturePolicyV3, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterAttestationSignaturePolicyV3 { + OptionalLocal, + RequiredTrustedPublisher, +} + +/// Host-authored evidence that may be supplied back to a planner. The issuer +/// tag is not proof by itself; Trail validates every referenced object before +/// including this structure in a request. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterAttestationEvidenceV3 { + pub issuer: AdapterEvidenceIssuerV3, + pub attestation_id: String, + pub envelope_id: String, + pub desired_key: String, + pub tree_root_id: String, + pub verification_state: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterQuarantineEvidenceV3 { + pub issuer: AdapterEvidenceIssuerV3, + pub quarantine_id: String, + pub desired_key: String, + pub incumbent_envelope_id: Option, + pub candidate_envelope_id: String, + pub reason_code: String, + pub state: String, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterEvidenceIssuerV3 { + TrailHost, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterHostEvidenceV3 { + #[serde(default)] + pub attestations: Vec, + #[serde(default)] + pub quarantines: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterPipelineV3 { + pub proposal: AdapterComponentProposalV3, + #[serde(default)] + pub dependencies: Vec, + pub inputs: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolution: Option, + pub actions: Vec, + pub validations: Vec, + pub capabilities: AdapterCapabilityProfileV3, + pub identity: AdapterIdentityContractV3, + pub outputs: Vec, + #[serde(default)] + pub source_exports: Vec, + pub attestation: AdapterAttestationRequirementsV3, + pub secret_taint: AdapterSecretTaintV3, + pub quarantine_policy: AdapterQuarantinePolicyV3, + pub stale_reason: String, +} + +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterSecretTaintV3 { + #[default] + Clear, + Credential, + RuntimeSecret, + Unknown, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterQuarantinePolicyV3 { + FailClosed, + RetainPrivate, +} + +/// Canonical compatibility projection for v1/v2 plans. The nested v3-only +/// fields are always absent; no default can grant resolution, validation, +/// certification, export, attestation, or quarantine semantics. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterLegacyPlanProjectionV3 { + pub source_protocol: String, + pub component_id: String, + pub kind: String, + pub dependencies: Vec, + pub identity_inputs: Vec, + pub semantic_inputs: BTreeMap, + pub caches: Vec, + pub external_artifacts: Vec, + pub runtime_resources: Vec, + pub actions: Vec, + pub outputs: Vec, + pub portability: AdapterPortability, + pub stale_reason: String, + pub v3_only: AdapterV3OnlySemantics, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterV3OnlySemantics { + pub proposal_status: Option, + pub resolution: Option, + pub typed_inputs: Vec, + pub validations: Vec, + pub capabilities: Option, + pub identity: Option, + pub source_exports: Vec, + pub attestation: Option, + pub quarantine_policy: Option, +} + +impl AdapterV3OnlySemantics { + pub fn is_absent(&self) -> bool { + self == &Self::default() + } +} + +impl AdapterLegacyPlanProjectionV3 { + pub fn from_v1(plan: AdapterPlan) -> Self { + let mut dependencies = plan + .dependencies + .into_iter() + .map(|component_id| { + AdapterDependency::new(component_id, AdapterDependencyType::BuildRequires) + }) + .collect::>(); + dependencies.sort(); + let mut identity_inputs = plan.identity_inputs; + identity_inputs.sort(); + let mut outputs = plan.outputs; + outputs.sort_by(|left, right| left.name.cmp(&right.name)); + Self { + source_protocol: PROTOCOL_V1.into(), + component_id: plan.component_id, + kind: plan.kind, + dependencies, + identity_inputs, + semantic_inputs: plan.semantic_inputs, + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + actions: vec![AdapterAction::Staging(plan.command)], + outputs, + portability: plan.portability, + stale_reason: plan.stale_reason, + v3_only: AdapterV3OnlySemantics::default(), + } + } + + pub fn from_v2(plan: AdapterPlanV2) -> Self { + let mut dependencies = plan + .dependencies + .into_iter() + .map(|component_id| { + AdapterDependency::new(component_id, AdapterDependencyType::BuildRequires) + }) + .chain(plan.dependency_edges) + .collect::>(); + dependencies.sort(); + let mut identity_inputs = plan.identity_inputs; + identity_inputs.sort(); + let mut caches = plan.caches; + caches.sort_by(|left, right| left.name.cmp(&right.name)); + let mut external_artifacts = plan.external_artifacts; + external_artifacts.sort_by(|left, right| left.name.cmp(&right.name)); + let mut runtime_resources = plan.runtime_resources; + runtime_resources.sort_by(|left, right| left.name.cmp(&right.name)); + let mut outputs = plan.outputs; + outputs.sort_by(|left, right| left.name.cmp(&right.name)); + Self { + source_protocol: PROTOCOL_V2.into(), + component_id: plan.component_id, + kind: plan.kind, + dependencies, + identity_inputs, + semantic_inputs: plan.semantic_inputs, + caches, + external_artifacts, + runtime_resources, + actions: plan.actions, + outputs, + portability: plan.portability, + stale_reason: plan.stale_reason, + v3_only: AdapterV3OnlySemantics::default(), + } + } +} + +impl AdapterPipelineV3 { + pub fn builder( + proposal: AdapterComponentProposalV3, + identity: AdapterIdentityContractV3, + ) -> AdapterPipelineV3Builder { + AdapterPipelineV3Builder::new(proposal, identity) + } + + /// Rebuild this untrusted wire value through the validating builder and + /// require byte-semantic canonical equality. Hosts call this even when an + /// adapter used the SDK because a plugin may serialize arbitrary values. + pub fn validate_canonical(&self) -> Result<(), AdapterPipelineV3BuildError> { + let rebuilt = AdapterPipelineV3Builder { + proposal: self.proposal.clone(), + dependencies: self.dependencies.clone(), + inputs: self.inputs.clone(), + resolution: self.resolution.clone(), + actions: self.actions.clone(), + validations: self.validations.clone(), + capabilities: self.capabilities.clone(), + identity: self.identity.clone(), + outputs: self.outputs.clone(), + source_exports: self.source_exports.clone(), + attestation: self.attestation.clone(), + secret_taint: self.secret_taint, + quarantine_policy: self.quarantine_policy, + stale_reason: Some(self.stale_reason.clone()), + } + .build()?; + if rebuilt != *self { + return Err(AdapterPipelineV3BuildError::NonCanonical); + } + Ok(()) + } +} + +#[derive(Clone, Debug)] +pub struct AdapterPipelineV3Builder { + proposal: AdapterComponentProposalV3, + dependencies: Vec, + inputs: Vec, + resolution: Option, + actions: Vec, + validations: Vec, + capabilities: AdapterCapabilityProfileV3, + identity: AdapterIdentityContractV3, + outputs: Vec, + source_exports: Vec, + attestation: AdapterAttestationRequirementsV3, + secret_taint: AdapterSecretTaintV3, + quarantine_policy: AdapterQuarantinePolicyV3, + stale_reason: Option, +} + +impl AdapterPipelineV3Builder { + pub fn new(proposal: AdapterComponentProposalV3, identity: AdapterIdentityContractV3) -> Self { + Self { + proposal, + dependencies: Vec::new(), + inputs: Vec::new(), + resolution: None, + actions: Vec::new(), + validations: Vec::new(), + capabilities: denied_capabilities_v3(), + identity, + outputs: Vec::new(), + source_exports: Vec::new(), + attestation: AdapterAttestationRequirementsV3 { + required_validations: Vec::new(), + require_sandbox_evidence: true, + require_executable_identities: true, + signature_policy: AdapterAttestationSignaturePolicyV3::OptionalLocal, + }, + secret_taint: AdapterSecretTaintV3::Clear, + quarantine_policy: AdapterQuarantinePolicyV3::FailClosed, + stale_reason: None, + } + } + + pub fn dependency(mut self, dependency: AdapterDependency) -> Self { + self.dependencies.push(dependency); + self + } + + pub fn input(mut self, input: AdapterInputV3) -> Self { + self.inputs.push(input); + self + } + + pub fn resolution(mut self, resolution: AdapterResolutionPlanV3) -> Self { + self.resolution = Some(resolution); + self + } + + pub fn action(mut self, action: AdapterActionV3) -> Self { + self.actions.push(action); + self + } + + pub fn validation(mut self, validation: AdapterValidationV3) -> Self { + self.validations.push(validation); + self + } + + pub fn capabilities(mut self, capabilities: AdapterCapabilityProfileV3) -> Self { + self.capabilities = capabilities; + self + } + + pub fn output(mut self, output: AdapterOutput) -> Self { + self.outputs.push(output); + self + } + + pub fn source_export(mut self, source_export: AdapterSourceExportV3) -> Self { + self.source_exports.push(source_export); + self + } + + pub fn attestation(mut self, attestation: AdapterAttestationRequirementsV3) -> Self { + self.attestation = attestation; + self + } + + pub fn secret_taint(mut self, secret_taint: AdapterSecretTaintV3) -> Self { + self.secret_taint = secret_taint; + self + } + + pub fn quarantine_policy(mut self, policy: AdapterQuarantinePolicyV3) -> Self { + self.quarantine_policy = policy; + self + } + + pub fn stale_reason(mut self, stale_reason: impl Into) -> Self { + self.stale_reason = Some(stale_reason.into()); + self + } + + pub fn build(mut self) -> Result { + self.proposal + .missing_requirements + .sort_by(|left, right| left.code.cmp(&right.code)); + reject_duplicate_by( + &self.proposal.missing_requirements, + "proposal.missing_requirements", + |requirement| requirement.code.as_str(), + )?; + self.proposal + .recovery_actions + .sort_by(|left, right| left.code.cmp(&right.code)); + reject_duplicate_by( + &self.proposal.recovery_actions, + "proposal.recovery_actions", + |action| action.code.as_str(), + )?; + if self.proposal.status == AdapterProposalStatusV3::Ready + && !self.proposal.missing_requirements.is_empty() + { + return Err(AdapterPipelineV3BuildError::InvalidCombination( + "a ready proposal cannot retain missing requirements", + )); + } + if matches!( + self.proposal.status, + AdapterProposalStatusV3::Blocked + | AdapterProposalStatusV3::Unsupported + | AdapterProposalStatusV3::Ambiguous + ) { + return Err(AdapterPipelineV3BuildError::InvalidCombination( + "blocked, unsupported, or ambiguous proposals cannot produce a pipeline", + )); + } + if self.proposal.status == AdapterProposalStatusV3::Resolvable && self.resolution.is_none() + { + return Err(AdapterPipelineV3BuildError::MissingField { + field: "resolution", + }); + } + if self.inputs.is_empty() { + return Err(AdapterPipelineV3BuildError::MissingField { field: "inputs" }); + } + if self.outputs.is_empty() { + return Err(AdapterPipelineV3BuildError::MissingField { field: "outputs" }); + } + let stale_reason = + self.stale_reason + .take() + .ok_or(AdapterPipelineV3BuildError::MissingField { + field: "stale_reason", + })?; + + self.dependencies.sort(); + reject_duplicate_by(&self.dependencies, "dependencies", |dependency| { + dependency.component_id.as_str() + })?; + if self + .dependencies + .iter() + .any(|dependency| dependency.component_id == self.proposal.component_id) + { + return Err(AdapterPipelineV3BuildError::InvalidCombination( + "a component cannot depend on itself", + )); + } + self.inputs.sort(); + reject_duplicate_by(&self.inputs, "inputs", |input| input.path.as_str())?; + for input in &self.inputs { + validate_relative_path_v3(&input.path, "inputs.path")?; + if input.size_bytes > MAX_V3_INPUT_BYTES { + return Err(AdapterPipelineV3BuildError::InvalidCombination( + "an input exceeds the v3 byte ceiling", + )); + } + } + + if let Some(resolution) = &mut self.resolution { + canonical_strings_v3( + &mut resolution.readable_inputs, + "resolution.readable_inputs", + )?; + canonical_strings_v3( + &mut resolution.allowed_authorities, + "resolution.allowed_authorities", + )?; + canonical_strings_v3( + &mut resolution.credential_handles, + "resolution.credential_handles", + )?; + canonical_strings_v3( + &mut resolution.capabilities.network_authorities, + "resolution.capabilities.network_authorities", + )?; + validate_command_v3( + &resolution.program, + &resolution.argv, + &resolution.working_directory, + "resolution", + )?; + validate_relative_path_v3(&resolution.candidate_output, "resolution.candidate_output")?; + } + + self.actions.sort_by(|left, right| { + left.phase + .cmp(&right.phase) + .then_with(|| left.name.cmp(&right.name)) + }); + reject_duplicate_by(&self.actions, "actions", |action| action.name.as_str())?; + for action in &mut self.actions { + canonical_strings_v3( + &mut action.capabilities.network_authorities, + "actions.capabilities.network_authorities", + )?; + validate_command_v3( + &action.program, + &action.argv, + &action.working_directory, + "actions", + )?; + } + + self.validations.sort(); + reject_duplicate_by(&self.validations, "validations", |validation| { + validation.name.as_str() + })?; + for validation in &self.validations { + if !validation.path.is_empty() { + validate_relative_path_v3(&validation.path, "validations.path")?; + } + } + + self.outputs + .sort_by(|left, right| left.name.cmp(&right.name)); + reject_duplicate_by(&self.outputs, "outputs.name", |output| output.name.as_str())?; + let mut output_targets = BTreeSet::new(); + for output in &self.outputs { + validate_relative_path_v3(&output.source, "outputs.source")?; + validate_relative_path_v3(&output.target, "outputs.target")?; + if !output_targets.insert(output.target.as_str()) { + return Err(AdapterPipelineV3BuildError::Duplicate { + field: "outputs.target", + value: output.target.clone(), + }); + } + } + + self.source_exports.sort(); + reject_duplicate_by(&self.source_exports, "source_exports", |export| { + export.name.as_str() + })?; + let output_names = self + .outputs + .iter() + .map(|output| output.name.as_str()) + .collect::>(); + let validation_names = self + .validations + .iter() + .map(|validation| validation.name.as_str()) + .collect::>(); + for export in &self.source_exports { + validate_relative_path_v3(&export.artifact_subpath, "source_exports.artifact_subpath")?; + validate_relative_path_v3(&export.destination, "source_exports.destination")?; + if !output_names.contains(export.output_name.as_str()) + || !validation_names.contains(export.required_validation.as_str()) + { + return Err(AdapterPipelineV3BuildError::InvalidCombination( + "a source export must reference one declared output and validation", + )); + } + } + canonical_strings_v3( + &mut self.attestation.required_validations, + "attestation.required_validations", + )?; + if self + .attestation + .required_validations + .iter() + .any(|name| !validation_names.contains(name.as_str())) + { + return Err(AdapterPipelineV3BuildError::InvalidCombination( + "attestation requirements must reference declared validations", + )); + } + + let has_mounted_action = self + .actions + .iter() + .any(|action| action.phase == AdapterActionPhaseV3::MountedInitialization); + if has_mounted_action + && self.outputs.iter().any(|output| { + !matches!( + output.policy, + AdapterOutputPolicy::WritablePrivate | AdapterOutputPolicy::Disposable + ) + }) + { + return Err(AdapterPipelineV3BuildError::InvalidCombination( + "mounted initialization requires lane-private or disposable outputs", + )); + } + if self.secret_taint != AdapterSecretTaintV3::Clear + && (!self.source_exports.is_empty() + || self.outputs.iter().any(|output| { + matches!( + output.policy, + AdapterOutputPolicy::ImmutableShared + | AdapterOutputPolicy::ImmutableSeedPrivate + ) + })) + { + return Err(AdapterPipelineV3BuildError::InvalidCombination( + "secret-tainted pipelines cannot publish reusable artifacts or source exports", + )); + } + canonical_strings_v3( + &mut self.capabilities.network_authorities, + "capabilities.network_authorities", + )?; + + let pipeline = AdapterPipelineV3 { + proposal: self.proposal, + dependencies: self.dependencies, + inputs: self.inputs, + resolution: self.resolution, + actions: self.actions, + validations: self.validations, + capabilities: self.capabilities, + identity: self.identity, + outputs: self.outputs, + source_exports: self.source_exports, + attestation: self.attestation, + secret_taint: self.secret_taint, + quarantine_policy: self.quarantine_policy, + stale_reason, + }; + pipeline.validate_bounds(&AdapterProtocolLimitsV3::default())?; + Ok(pipeline) + } +} + +pub fn denied_capabilities_v3() -> AdapterCapabilityProfileV3 { + AdapterCapabilityProfileV3 { + network: AdapterNetworkCapabilityV3::Deny, + network_authorities: Vec::new(), + filesystem_read: AdapterFilesystemCapabilityV3::Deny, + filesystem_write: AdapterFilesystemCapabilityV3::Deny, + process: AdapterProcessCapabilityV3::Deny, + child_processes: 0, + secrets: AdapterSecretCapabilityV3::Deny, + publication: AdapterPublicationCapabilityV3::Deny, + } +} + +#[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] +pub enum AdapterPipelineV3BuildError { + #[error("protocol-v3 pipeline field `{field}` is required")] + MissingField { field: &'static str }, + #[error("protocol-v3 pipeline field `{field}` repeats `{value}`")] + Duplicate { field: &'static str, value: String }, + #[error("protocol-v3 pipeline field `{field}` has invalid relative path `{path}`")] + InvalidPath { field: &'static str, path: String }, + #[error("protocol-v3 pipeline {field} must use fixed, non-shell argv")] + UnsafeCommand { field: &'static str }, + #[error("protocol-v3 pipeline has an invalid combination: {0}")] + InvalidCombination(&'static str), + #[error("protocol-v3 pipeline collections are not in canonical order")] + NonCanonical, + #[error(transparent)] + Bounds(#[from] AdapterProtocolV3BoundsError), +} + +fn canonical_strings_v3( + values: &mut [String], + field: &'static str, +) -> Result<(), AdapterPipelineV3BuildError> { + values.sort(); + if let Some(pair) = values.windows(2).find(|pair| pair[0] == pair[1]) { + return Err(AdapterPipelineV3BuildError::Duplicate { + field, + value: pair[0].clone(), + }); + } + Ok(()) +} + +fn reject_duplicate_by<'a, T>( + values: &'a [T], + field: &'static str, + key: impl Fn(&'a T) -> &'a str, +) -> Result<(), AdapterPipelineV3BuildError> { + if let Some(pair) = values + .windows(2) + .find(|pair| key(&pair[0]) == key(&pair[1])) + { + return Err(AdapterPipelineV3BuildError::Duplicate { + field, + value: key(&pair[0]).to_string(), + }); + } + Ok(()) +} + +fn validate_command_v3( + program: &str, + argv: &[String], + working_directory: &str, + field: &'static str, +) -> Result<(), AdapterPipelineV3BuildError> { + let program_name = program.rsplit(['/', '\\']).next().unwrap_or(program); + let shell = matches!( + program_name.to_ascii_lowercase().as_str(), + "sh" | "bash" | "dash" | "zsh" | "fish" | "cmd" | "cmd.exe" | "powershell" | "pwsh" + ); + if program.is_empty() + || program.contains(['/', '\\']) + || shell + || argv.len() > MAX_V3_ARGV + || argv.iter().any(|argument| { + argument.is_empty() + || argument.contains('\0') + || argument.contains("$(") + || argument.contains('`') + || argument.chars().any(char::is_control) + }) + { + return Err(AdapterPipelineV3BuildError::UnsafeCommand { field }); + } + validate_relative_path_v3(working_directory, field) +} + +fn validate_relative_path_v3( + path: &str, + field: &'static str, +) -> Result<(), AdapterPipelineV3BuildError> { + if path == "." { + return Ok(()); + } + let invalid = path.is_empty() + || path.starts_with('/') + || path.contains('\\') + || path.chars().any(char::is_control) + || path + .split('/') + .any(|segment| segment.is_empty() || segment == "." || segment == ".."); + if invalid { + Err(AdapterPipelineV3BuildError::InvalidPath { + field, + path: path.to_string(), + }) + } else { + Ok(()) + } +} + +#[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] +pub enum AdapterProtocolV3BoundsError { + #[error("protocol-v3 field `{field}` is required")] + Empty { field: &'static str }, + #[error("protocol-v3 field `{field}` is {actual} bytes; maximum is {maximum}")] + StringTooLong { + field: &'static str, + actual: usize, + maximum: usize, + }, + #[error("protocol-v3 collection `{field}` has {actual} entries; maximum is {maximum}")] + CollectionTooLarge { + field: &'static str, + actual: usize, + maximum: usize, + }, + #[error("protocol-v3 byte collection `{field}` has {actual} bytes; maximum is {maximum}")] + BytesTooLarge { + field: &'static str, + actual: u64, + maximum: u64, + }, + #[error("protocol-v3 request declares invalid host limit `{field}` = {actual}; maximum is {maximum}")] + InvalidLimit { + field: &'static str, + actual: u64, + maximum: u64, + }, + #[error("protocol-v3 input `{path}` declares {declared} bytes but carries {actual}")] + InputSizeMismatch { + path: String, + declared: u64, + actual: u64, + }, + #[error("expected protocol `{PROTOCOL_V3}`, received `{actual}`")] + Protocol { actual: String }, +} + +impl AdapterRequestV3 { + /// Enforce allocation and collection ceilings before an adapter receives + /// a request. Semantic and trust validation remains host-owned. + pub fn validate_bounds(&self) -> Result<(), AdapterProtocolV3BoundsError> { + check_protocol(&self.protocol)?; + check_string(&self.request_id, "request_id")?; + check_string(&self.adapter_identity, "adapter_identity")?; + check_string(&self.distribution_digest, "distribution_digest")?; + check_string(&self.source_root, "source_root")?; + self.limits.validate()?; + match &self.operation { + AdapterOperationV3::Propose { + component_root, + files, + } => { + check_string_allow_empty(component_root, "component_root")?; + validate_pinned_files(files, &self.limits)?; + } + AdapterOperationV3::Plan { + proposal, + files, + resolution_snapshot, + host_evidence, + } => { + proposal.validate_bounds()?; + check_count( + files.len(), + "inputs", + (self.limits.max_input_files as usize).min(MAX_V3_INPUTS), + )?; + let mut total = 0u64; + for pinned in files { + pinned.input.validate_bounds()?; + let actual = pinned.content.len() as u64; + if actual != pinned.input.size_bytes { + return Err(AdapterProtocolV3BoundsError::InputSizeMismatch { + path: pinned.input.path.clone(), + declared: pinned.input.size_bytes, + actual, + }); + } + total = total.saturating_add(actual); + } + check_bytes( + total, + "inputs", + self.limits.max_input_bytes.min(MAX_V3_INPUT_BYTES), + )?; + if let Some(snapshot) = resolution_snapshot { + snapshot.validate_bounds()?; + } + if let Some(evidence) = host_evidence { + evidence.validate_bounds()?; + } + } + } + Ok(()) + } +} + +impl AdapterResponseV3 { + /// Enforce the request's negotiated ceilings before host normalization. + pub fn validate_bounds( + &self, + limits: &AdapterProtocolLimitsV3, + ) -> Result<(), AdapterProtocolV3BoundsError> { + check_protocol(&self.protocol)?; + check_string(&self.request_id, "request_id")?; + limits.validate()?; + match &self.result { + AdapterResultV3::Proposed { component } => { + if let Some(component) = component { + component.validate_bounds()?; + } + } + AdapterResultV3::Planned { pipeline } => pipeline.validate_bounds(limits)?, + AdapterResultV3::Error { + code, + message, + recovery_actions, + } => { + check_string(code, "error.code")?; + check_string(message, "error.message")?; + check_count( + recovery_actions.len(), + "error.recovery_actions", + MAX_V3_RECOVERY_ACTIONS, + )?; + for action in recovery_actions { + action.validate_bounds()?; + } + } + } + Ok(()) + } +} + +impl AdapterProtocolLimitsV3 { + fn validate(&self) -> Result<(), AdapterProtocolV3BoundsError> { + check_limit( + self.max_input_files as u64, + "max_input_files", + MAX_V3_INPUTS as u64, + )?; + check_limit(self.max_input_bytes, "max_input_bytes", MAX_V3_INPUT_BYTES)?; + check_limit( + self.max_actions as u64, + "max_actions", + MAX_V3_ACTIONS as u64, + )?; + check_limit( + self.max_validations as u64, + "max_validations", + MAX_V3_VALIDATIONS as u64, + )?; + check_limit( + self.max_outputs as u64, + "max_outputs", + MAX_V3_OUTPUTS as u64, + )?; + check_limit( + self.max_source_exports as u64, + "max_source_exports", + MAX_V3_SOURCE_EXPORTS as u64, + )?; + check_limit( + self.max_authorities as u64, + "max_authorities", + MAX_V3_AUTHORITIES as u64, + )?; + check_limit( + self.max_response_bytes, + "max_response_bytes", + crate::MAX_FRAME_BYTES as u64, + ) + } +} + +impl AdapterComponentProposalV3 { + fn validate_bounds(&self) -> Result<(), AdapterProtocolV3BoundsError> { + check_string(&self.component_id, "proposal.component_id")?; + check_string_allow_empty(&self.component_root, "proposal.component_root")?; + check_string(&self.kind, "proposal.kind")?; + check_string(&self.proposal_key, "proposal.proposal_key")?; + check_count( + self.missing_requirements.len(), + "proposal.missing_requirements", + MAX_V3_REQUIREMENTS, + )?; + for requirement in &self.missing_requirements { + check_string(&requirement.code, "requirement.code")?; + check_string(&requirement.message, "requirement.message")?; + } + check_count( + self.recovery_actions.len(), + "proposal.recovery_actions", + MAX_V3_RECOVERY_ACTIONS, + )?; + for action in &self.recovery_actions { + action.validate_bounds()?; + } + Ok(()) + } +} + +impl AdapterRecoveryActionV3 { + fn validate_bounds(&self) -> Result<(), AdapterProtocolV3BoundsError> { + check_string(&self.code, "recovery_action.code")?; + check_string(&self.message, "recovery_action.message") + } +} + +impl AdapterInputV3 { + fn validate_bounds(&self) -> Result<(), AdapterProtocolV3BoundsError> { + check_string(&self.path, "input.path")?; + check_string(&self.content_hash, "input.content_hash")?; + check_string(&self.format, "input.format") + } +} + +impl AdapterResolutionSnapshotV3 { + fn validate_bounds(&self) -> Result<(), AdapterProtocolV3BoundsError> { + check_string(&self.snapshot_id, "resolution_snapshot.snapshot_id")?; + check_string(&self.proposal_key, "resolution_snapshot.proposal_key")?; + check_string(&self.format, "resolution_snapshot.format")?; + check_string(&self.content_sha256, "resolution_snapshot.content_sha256")?; + check_bytes( + self.content.len() as u64, + "resolution_snapshot.content", + MAX_V3_INPUT_BYTES, + )?; + check_map( + &self.resolved_identities, + "resolution_snapshot.resolved_identities", + )?; + check_map(&self.checksums, "resolution_snapshot.checksums") + } +} + +impl AdapterHostEvidenceV3 { + fn validate_bounds(&self) -> Result<(), AdapterProtocolV3BoundsError> { + check_count( + self.attestations.len(), + "host_evidence.attestations", + MAX_V3_ATTESTATION_REFERENCES, + )?; + check_count( + self.quarantines.len(), + "host_evidence.quarantines", + MAX_V3_QUARANTINE_REFERENCES, + )?; + for attestation in &self.attestations { + check_string(&attestation.attestation_id, "attestation.attestation_id")?; + check_string(&attestation.envelope_id, "attestation.envelope_id")?; + check_string(&attestation.desired_key, "attestation.desired_key")?; + check_string(&attestation.tree_root_id, "attestation.tree_root_id")?; + check_string( + &attestation.verification_state, + "attestation.verification_state", + )?; + } + for quarantine in &self.quarantines { + check_string(&quarantine.quarantine_id, "quarantine.quarantine_id")?; + check_string(&quarantine.desired_key, "quarantine.desired_key")?; + if let Some(envelope) = &quarantine.incumbent_envelope_id { + check_string(envelope, "quarantine.incumbent_envelope_id")?; + } + check_string( + &quarantine.candidate_envelope_id, + "quarantine.candidate_envelope_id", + )?; + check_string(&quarantine.reason_code, "quarantine.reason_code")?; + check_string(&quarantine.state, "quarantine.state")?; + } + Ok(()) + } +} + +impl AdapterPipelineV3 { + fn validate_bounds( + &self, + limits: &AdapterProtocolLimitsV3, + ) -> Result<(), AdapterProtocolV3BoundsError> { + self.proposal.validate_bounds()?; + check_count( + self.dependencies.len(), + "pipeline.dependencies", + MAX_V3_IDENTITIES, + )?; + for dependency in &self.dependencies { + check_string(&dependency.component_id, "pipeline.dependencies")?; + } + check_count(self.inputs.len(), "pipeline.inputs", MAX_V3_INPUTS)?; + check_count( + self.actions.len(), + "pipeline.actions", + (limits.max_actions as usize).min(MAX_V3_ACTIONS), + )?; + check_count( + self.validations.len(), + "pipeline.validations", + (limits.max_validations as usize).min(MAX_V3_VALIDATIONS), + )?; + check_count( + self.outputs.len(), + "pipeline.outputs", + (limits.max_outputs as usize).min(MAX_V3_OUTPUTS), + )?; + check_count( + self.source_exports.len(), + "pipeline.source_exports", + (limits.max_source_exports as usize).min(MAX_V3_SOURCE_EXPORTS), + )?; + for input in &self.inputs { + input.validate_bounds()?; + } + if let Some(resolution) = &self.resolution { + validate_resolution_bounds(resolution, limits)?; + } + for action in &self.actions { + check_string(&action.name, "action.name")?; + check_string(&action.program, "action.program")?; + check_string(&action.working_directory, "action.working_directory")?; + check_string_slice(&action.argv, "action.argv", MAX_V3_ARGV)?; + check_map(&action.environment, "action.environment")?; + validate_capability_bounds(&action.capabilities, limits)?; + validate_action_limits(&action.limits)?; + } + for validation in &self.validations { + check_string(&validation.name, "validation.name")?; + check_string_allow_empty(&validation.path, "validation.path")?; + check_map(&validation.parameters, "validation.parameters")?; + } + validate_capability_bounds(&self.capabilities, limits)?; + check_map( + &self.identity.semantic_identities, + "identity.semantic_identities", + )?; + check_string( + &self.identity.normalizer_version, + "identity.normalizer_version", + )?; + check_string(&self.identity.target, "identity.target")?; + check_string(&self.identity.platform, "identity.platform")?; + check_string(&self.identity.architecture, "identity.architecture")?; + check_string(&self.identity.abi, "identity.abi")?; + check_string( + &self.identity.portability_scope, + "identity.portability_scope", + )?; + check_string(&self.identity.trust_scope, "identity.trust_scope")?; + for export in &self.source_exports { + check_string(&export.name, "source_export.name")?; + check_string(&export.output_name, "source_export.output_name")?; + check_string(&export.artifact_subpath, "source_export.artifact_subpath")?; + check_string(&export.destination, "source_export.destination")?; + check_string( + &export.required_validation, + "source_export.required_validation", + )?; + if let Some(gate) = &export.required_gate { + check_string(gate, "source_export.required_gate")?; + } + } + for output in &self.outputs { + check_string(&output.name, "output.name")?; + check_string(&output.source, "output.source")?; + check_string(&output.target, "output.target")?; + if let Some(gate) = &output.gate { + check_string(gate, "output.gate")?; + } + } + check_string_slice( + &self.attestation.required_validations, + "attestation.required_validations", + MAX_V3_ATTESTATION_REFERENCES, + )?; + check_string(&self.stale_reason, "pipeline.stale_reason") + } +} + +fn validate_resolution_bounds( + resolution: &AdapterResolutionPlanV3, + limits: &AdapterProtocolLimitsV3, +) -> Result<(), AdapterProtocolV3BoundsError> { + check_string(&resolution.name, "resolution.name")?; + check_string(&resolution.program, "resolution.program")?; + check_string( + &resolution.working_directory, + "resolution.working_directory", + )?; + check_string(&resolution.candidate_output, "resolution.candidate_output")?; + check_string(&resolution.snapshot_format, "resolution.snapshot_format")?; + check_string_slice(&resolution.argv, "resolution.argv", MAX_V3_ARGV)?; + check_string_slice( + &resolution.readable_inputs, + "resolution.readable_inputs", + MAX_V3_INPUTS, + )?; + check_string_slice( + &resolution.allowed_authorities, + "resolution.allowed_authorities", + (limits.max_authorities as usize).min(MAX_V3_AUTHORITIES), + )?; + check_string_slice( + &resolution.credential_handles, + "resolution.credential_handles", + MAX_V3_AUTHORITIES, + )?; + validate_capability_bounds(&resolution.capabilities, limits)?; + validate_action_limits(&resolution.limits) +} + +fn validate_capability_bounds( + capability: &AdapterCapabilityProfileV3, + limits: &AdapterProtocolLimitsV3, +) -> Result<(), AdapterProtocolV3BoundsError> { + check_string_slice( + &capability.network_authorities, + "capabilities.network_authorities", + (limits.max_authorities as usize).min(MAX_V3_AUTHORITIES), + )?; + check_limit_allow_zero( + capability.child_processes as u64, + "capabilities.child_processes", + MAX_V3_CHILD_PROCESSES as u64, + ) +} + +fn validate_action_limits( + limits: &AdapterActionLimitsV3, +) -> Result<(), AdapterProtocolV3BoundsError> { + check_limit( + limits.timeout_ms, + "action_limits.timeout_ms", + MAX_V3_ACTION_TIMEOUT_MS, + )?; + check_limit( + limits.stdout_bytes, + "action_limits.stdout_bytes", + MAX_V3_CAPTURE_BYTES, + )?; + check_limit( + limits.stderr_bytes, + "action_limits.stderr_bytes", + MAX_V3_CAPTURE_BYTES, + )?; + check_limit( + limits.output_entries, + "action_limits.output_entries", + MAX_V3_OUTPUT_ENTRIES, + )?; + check_limit( + limits.output_bytes, + "action_limits.output_bytes", + MAX_V3_OUTPUT_BYTES, + )?; + check_limit_allow_zero( + limits.child_processes as u64, + "action_limits.child_processes", + MAX_V3_CHILD_PROCESSES as u64, + ) +} + +fn validate_pinned_files( + files: &[PinnedFile], + limits: &AdapterProtocolLimitsV3, +) -> Result<(), AdapterProtocolV3BoundsError> { + check_count( + files.len(), + "pinned_files", + (limits.max_input_files as usize).min(MAX_V3_INPUTS), + )?; + let mut total = 0u64; + for file in files { + check_string(&file.path, "pinned_file.path")?; + check_string(&file.content_hash, "pinned_file.content_hash")?; + total = total.saturating_add(file.content.len() as u64); + } + check_bytes( + total, + "pinned_files", + limits.max_input_bytes.min(MAX_V3_INPUT_BYTES), + ) +} + +fn check_protocol(protocol: &str) -> Result<(), AdapterProtocolV3BoundsError> { + if protocol == PROTOCOL_V3 { + Ok(()) + } else { + Err(AdapterProtocolV3BoundsError::Protocol { + actual: protocol.to_string(), + }) + } +} + +fn check_limit( + actual: u64, + field: &'static str, + maximum: u64, +) -> Result<(), AdapterProtocolV3BoundsError> { + if actual == 0 || actual > maximum { + Err(AdapterProtocolV3BoundsError::InvalidLimit { + field, + actual, + maximum, + }) + } else { + Ok(()) + } +} + +fn check_limit_allow_zero( + actual: u64, + field: &'static str, + maximum: u64, +) -> Result<(), AdapterProtocolV3BoundsError> { + if actual > maximum { + Err(AdapterProtocolV3BoundsError::InvalidLimit { + field, + actual, + maximum, + }) + } else { + Ok(()) + } +} + +fn check_count( + actual: usize, + field: &'static str, + maximum: usize, +) -> Result<(), AdapterProtocolV3BoundsError> { + if actual > maximum { + Err(AdapterProtocolV3BoundsError::CollectionTooLarge { + field, + actual, + maximum, + }) + } else { + Ok(()) + } +} + +fn check_bytes( + actual: u64, + field: &'static str, + maximum: u64, +) -> Result<(), AdapterProtocolV3BoundsError> { + if actual > maximum { + Err(AdapterProtocolV3BoundsError::BytesTooLarge { + field, + actual, + maximum, + }) + } else { + Ok(()) + } +} + +fn check_string(value: &str, field: &'static str) -> Result<(), AdapterProtocolV3BoundsError> { + if value.is_empty() { + return Err(AdapterProtocolV3BoundsError::Empty { field }); + } + check_string_allow_empty(value, field) +} + +fn check_string_allow_empty( + value: &str, + field: &'static str, +) -> Result<(), AdapterProtocolV3BoundsError> { + if value.len() > MAX_V3_STRING_BYTES { + Err(AdapterProtocolV3BoundsError::StringTooLong { + field, + actual: value.len(), + maximum: MAX_V3_STRING_BYTES, + }) + } else { + Ok(()) + } +} + +fn check_string_slice( + values: &[String], + field: &'static str, + maximum: usize, +) -> Result<(), AdapterProtocolV3BoundsError> { + check_count(values.len(), field, maximum)?; + for value in values { + check_string(value, field)?; + } + Ok(()) +} + +fn check_map( + values: &BTreeMap, + field: &'static str, +) -> Result<(), AdapterProtocolV3BoundsError> { + check_count(values.len(), field, MAX_V3_MAP_ENTRIES)?; + for (key, value) in values { + check_string(key, field)?; + check_string(value, field)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{AdapterDependencyType, AdapterOutput, AdapterPortability, AdapterSharingScope}; + + fn proposal() -> AdapterComponentProposalV3 { + AdapterComponentProposalV3 { + component_id: "example.codegen".into(), + component_root: ".".into(), + kind: "generated".into(), + status: AdapterProposalStatusV3::Ready, + proposal_key: "sha256:proposal".into(), + missing_requirements: Vec::new(), + recovery_actions: Vec::new(), + } + } + + fn limits() -> AdapterActionLimitsV3 { + AdapterActionLimitsV3 { + timeout_ms: 30_000, + stdout_bytes: 1024 * 1024, + stderr_bytes: 1024 * 1024, + output_entries: 10_000, + output_bytes: 64 * 1024 * 1024, + child_processes: 1, + } + } + + fn capabilities() -> AdapterCapabilityProfileV3 { + AdapterCapabilityProfileV3 { + network: AdapterNetworkCapabilityV3::Deny, + network_authorities: Vec::new(), + filesystem_read: AdapterFilesystemCapabilityV3::DeclaredInputs, + filesystem_write: AdapterFilesystemCapabilityV3::IsolatedCandidate, + process: AdapterProcessCapabilityV3::DeclaredExecutable, + child_processes: 1, + secrets: AdapterSecretCapabilityV3::Deny, + publication: AdapterPublicationCapabilityV3::Deny, + } + } + + fn identity() -> AdapterIdentityContractV3 { + AdapterIdentityContractV3 { + normalizer_version: "trail-path-v1".into(), + source_closure_complete: true, + semantic_identities: BTreeMap::from([("mode".into(), "production".into())]), + target: "host".into(), + platform: "linux".into(), + architecture: "x86_64".into(), + abi: "gnu".into(), + portability: AdapterPortability::Platform, + portability_certified: false, + portability_scope: "workspace".into(), + trust_scope: "local_plugin".into(), + } + } + + fn input() -> AdapterInputV3 { + AdapterInputV3 { + path: "schema.json".into(), + content_hash: "sha256:input".into(), + size_bytes: 3, + executable: false, + role: AdapterInputRoleV3::Identity, + format: "application/json".into(), + required: true, + } + } + + fn pinned_input() -> AdapterPinnedInputV3 { + AdapterPinnedInputV3 { + input: input(), + content: b"{}\n".to_vec(), + } + } + + #[test] + fn v3_plan_request_round_trips_resolution_and_host_evidence() { + let request = AdapterRequestV3::new( + "request-v3", + "example/codegen@1", + "sha256:distribution", + AdapterHost { + operating_system: "linux".into(), + architecture: "x86_64".into(), + }, + "root:v3", + AdapterOperationV3::Plan { + proposal: Box::new(proposal()), + files: vec![pinned_input()], + resolution_snapshot: Some(Box::new(AdapterResolutionSnapshotV3 { + snapshot_id: "object:snapshot".into(), + proposal_key: "sha256:proposal".into(), + format: "application/vnd.example.lock".into(), + content_sha256: "sha256:snapshot".into(), + content: b"locked\n".to_vec(), + resolved_identities: BTreeMap::from([("dependency".into(), "1.0.0".into())]), + checksums: BTreeMap::from([("dependency".into(), "sha256:dep".into())]), + verified: true, + secret_taint: AdapterSecretTaintV3::Clear, + })), + host_evidence: Some(Box::new(AdapterHostEvidenceV3 { + attestations: vec![AdapterAttestationEvidenceV3 { + issuer: AdapterEvidenceIssuerV3::TrailHost, + attestation_id: "attestation:one".into(), + envelope_id: "envelope:one".into(), + desired_key: "desired:one".into(), + tree_root_id: "tree:one".into(), + verification_state: "verified".into(), + }], + quarantines: vec![AdapterQuarantineEvidenceV3 { + issuer: AdapterEvidenceIssuerV3::TrailHost, + quarantine_id: "quarantine:one".into(), + desired_key: "desired:other".into(), + incumbent_envelope_id: None, + candidate_envelope_id: "envelope:other".into(), + reason_code: "divergent_content".into(), + state: "open".into(), + }], + })), + }, + ); + request.validate_bounds().unwrap(); + let decoded: AdapterRequestV3 = + serde_cbor::from_slice(&serde_cbor::to_vec(&request).unwrap()).unwrap(); + assert_eq!(decoded, request); + } + + #[test] + fn v3_pipeline_response_covers_common_artifact_contract() { + let pipeline = AdapterPipelineV3 { + proposal: proposal(), + dependencies: vec![AdapterDependency::new( + "example.compiler", + AdapterDependencyType::BuildRequires, + )], + inputs: vec![input()], + resolution: Some(AdapterResolutionPlanV3 { + name: "lock".into(), + program: "resolver".into(), + argv: vec!["resolve".into(), "--output".into(), "generated.lock".into()], + working_directory: ".".into(), + readable_inputs: vec!["schema.json".into()], + candidate_output: "generated.lock".into(), + snapshot_format: "application/vnd.example.lock".into(), + allowed_authorities: vec!["registry.example:443".into()], + credential_handles: Vec::new(), + capabilities: AdapterCapabilityProfileV3 { + network: AdapterNetworkCapabilityV3::ExactAuthorities, + network_authorities: vec!["registry.example:443".into()], + ..capabilities() + }, + limits: limits(), + }), + actions: vec![AdapterActionV3 { + name: "construct".into(), + phase: AdapterActionPhaseV3::Construct, + program: "generator".into(), + argv: vec!["build".into()], + working_directory: ".".into(), + environment: BTreeMap::new(), + capabilities: capabilities(), + limits: limits(), + }], + validations: vec![AdapterValidationV3 { + name: "path-contract".into(), + kind: AdapterValidationKindV3::PathContract, + path: "generated".into(), + required: true, + parameters: BTreeMap::new(), + }], + capabilities: capabilities(), + identity: identity(), + outputs: vec![AdapterOutput::immutable_seed_private( + "generated", + "generated", + ".trail-generated/generated", + )], + source_exports: vec![AdapterSourceExportV3 { + name: "client".into(), + output_name: "generated".into(), + artifact_subpath: "client".into(), + destination: "src/generated".into(), + collision: AdapterSourceExportCollisionV3::Fail, + required_validation: "path-contract".into(), + required_gate: None, + authorization: AdapterSourceExportAuthorizationV3::ExplicitUser, + }], + attestation: AdapterAttestationRequirementsV3 { + required_validations: vec!["path-contract".into()], + require_sandbox_evidence: true, + require_executable_identities: true, + signature_policy: AdapterAttestationSignaturePolicyV3::OptionalLocal, + }, + secret_taint: AdapterSecretTaintV3::Clear, + quarantine_policy: AdapterQuarantinePolicyV3::FailClosed, + stale_reason: "input, resolver, generator, or platform changed".into(), + }; + assert_eq!(pipeline.outputs[0].scope, AdapterSharingScope::Workspace); + let response = AdapterResponseV3 { + protocol: PROTOCOL_V3.into(), + request_id: "request-v3".into(), + result: AdapterResultV3::Planned { + pipeline: Box::new(pipeline), + }, + }; + response + .validate_bounds(&AdapterProtocolLimitsV3::default()) + .unwrap(); + let decoded: AdapterResponseV3 = + serde_cbor::from_slice(&serde_cbor::to_vec(&response).unwrap()).unwrap(); + assert_eq!(decoded, response); + } + + #[test] + fn v3_bounds_reject_limit_widening_and_input_mismatch() { + let mut request = AdapterRequestV3::new( + "request-v3", + "example/codegen@1", + "sha256:distribution", + AdapterHost { + operating_system: "linux".into(), + architecture: "x86_64".into(), + }, + "root:v3", + AdapterOperationV3::Plan { + proposal: Box::new(proposal()), + files: vec![pinned_input()], + resolution_snapshot: None, + host_evidence: None, + }, + ); + request.limits.max_actions = (MAX_V3_ACTIONS + 1) as u16; + assert!(matches!( + request.validate_bounds(), + Err(AdapterProtocolV3BoundsError::InvalidLimit { + field: "max_actions", + .. + }) + )); + + request.limits = AdapterProtocolLimitsV3::default(); + let AdapterOperationV3::Plan { files, .. } = &mut request.operation else { + unreachable!(); + }; + files[0].input.size_bytes += 1; + assert!(matches!( + request.validate_bounds(), + Err(AdapterProtocolV3BoundsError::InputSizeMismatch { .. }) + )); + } + + #[test] + fn v3_builder_canonicalizes_collections_and_rejects_v3_authority_conflicts() { + let validation = AdapterValidationV3 { + name: "path-contract".into(), + kind: AdapterValidationKindV3::PathContract, + path: "generated".into(), + required: true, + parameters: BTreeMap::new(), + }; + let pipeline = AdapterPipelineV3::builder(proposal(), identity()) + .dependency(AdapterDependency::new( + "z-runtime", + AdapterDependencyType::RuntimeRequires, + )) + .dependency(AdapterDependency::new( + "a-compiler", + AdapterDependencyType::BuildRequires, + )) + .input(AdapterInputV3 { + path: "z.json".into(), + ..input() + }) + .input(input()) + .action(AdapterActionV3 { + name: "construct".into(), + phase: AdapterActionPhaseV3::Construct, + program: "generator".into(), + argv: vec!["build".into()], + working_directory: ".".into(), + environment: BTreeMap::new(), + capabilities: capabilities(), + limits: limits(), + }) + .validation(validation.clone()) + .output(AdapterOutput::immutable_seed_private( + "generated", + "generated", + ".trail-generated/generated", + )) + .source_export(AdapterSourceExportV3 { + name: "client".into(), + output_name: "generated".into(), + artifact_subpath: "client".into(), + destination: "src/generated".into(), + collision: AdapterSourceExportCollisionV3::Fail, + required_validation: validation.name.clone(), + required_gate: None, + authorization: AdapterSourceExportAuthorizationV3::ExplicitUser, + }) + .stale_reason("source, tool, or platform changed") + .build() + .unwrap(); + assert_eq!(pipeline.dependencies[0].component_id, "a-compiler"); + assert_eq!(pipeline.inputs[0].path, "schema.json"); + pipeline.validate_canonical().unwrap(); + let mut noncanonical = pipeline.clone(); + noncanonical.inputs.reverse(); + assert_eq!( + noncanonical.validate_canonical(), + Err(AdapterPipelineV3BuildError::NonCanonical) + ); + + let secret_export = AdapterPipelineV3::builder(proposal(), identity()) + .input(input()) + .action(AdapterActionV3 { + name: "construct".into(), + phase: AdapterActionPhaseV3::Construct, + program: "generator".into(), + argv: vec!["build".into()], + working_directory: ".".into(), + environment: BTreeMap::new(), + capabilities: capabilities(), + limits: limits(), + }) + .validation(validation) + .output(AdapterOutput::immutable_shared( + "generated", + "generated", + ".trail-generated/generated", + )) + .secret_taint(AdapterSecretTaintV3::Credential) + .stale_reason("source changed") + .build(); + assert!(matches!( + secret_export, + Err(AdapterPipelineV3BuildError::InvalidCombination( + "secret-tainted pipelines cannot publish reusable artifacts or source exports" + )) + )); + + let shell = AdapterPipelineV3::builder(proposal(), identity()) + .input(input()) + .action(AdapterActionV3 { + name: "construct".into(), + phase: AdapterActionPhaseV3::Construct, + program: "sh".into(), + argv: vec!["-c".into(), "build".into()], + working_directory: ".".into(), + environment: BTreeMap::new(), + capabilities: capabilities(), + limits: limits(), + }) + .output(AdapterOutput::writable_private( + "generated", + "generated", + ".trail-generated/generated", + )) + .stale_reason("source changed") + .build(); + assert!(matches!( + shell, + Err(AdapterPipelineV3BuildError::UnsafeCommand { .. }) + )); + } + + #[test] + fn legacy_plan_projections_are_canonical_and_never_infer_v3_semantics() { + let v1 = AdapterPlan::builder("legacy", "generated") + .dependencies(["z-tool", "a-source"]) + .identity_inputs(["z.lock", "a.toml"]) + .command(crate::AdapterCommand::new("generator", ["build"])) + .outputs([ + AdapterOutput::writable_private("z-state", "z", "z"), + AdapterOutput::immutable_seed_private("a-seed", "a", "a"), + ]) + .stale_reason("legacy inputs changed") + .build() + .unwrap(); + let projection = AdapterLegacyPlanProjectionV3::from_v1(v1); + assert_eq!(projection.source_protocol, PROTOCOL_V1); + assert_eq!(projection.dependencies[0].component_id, "a-source"); + assert_eq!(projection.identity_inputs, ["a.toml", "z.lock"]); + assert_eq!(projection.outputs[0].name, "a-seed"); + assert!(projection.v3_only.is_absent()); + + let v2 = AdapterPlanV2::builder("legacy-v2", "generated") + .runtime_requires("z-runtime") + .build_requires("a-compiler") + .identity_inputs(["z.lock", "a.toml"]) + .mounted_command(crate::AdapterCommand::new("initializer", ["prepare"])) + .output(AdapterOutput::writable_private("state", "state", "state")) + .stale_reason("legacy v2 inputs changed") + .build() + .unwrap(); + let projection = AdapterLegacyPlanProjectionV3::from_v2(v2); + assert_eq!(projection.source_protocol, PROTOCOL_V2); + assert_eq!(projection.dependencies[0].component_id, "a-compiler"); + assert!(projection.v3_only.is_absent()); + + let decoded: AdapterLegacyPlanProjectionV3 = + serde_cbor::from_slice(&serde_cbor::to_vec(&projection).unwrap()).unwrap(); + assert_eq!(decoded, projection); + assert!(decoded.v3_only.is_absent()); + } + + #[test] + fn v3_wire_rejects_unknown_required_fields_and_enum_values() { + #[derive(Serialize)] + struct InputWithUnknownField { + path: String, + content_hash: String, + size_bytes: u64, + executable: bool, + role: AdapterInputRoleV3, + format: String, + required: bool, + future_required_authority: bool, + } + let encoded = serde_cbor::to_vec(&InputWithUnknownField { + path: "input.txt".into(), + content_hash: "sha256:input".into(), + size_bytes: 1, + executable: false, + role: AdapterInputRoleV3::Identity, + format: "bytes".into(), + required: true, + future_required_authority: true, + }) + .unwrap(); + assert!(serde_cbor::from_slice::(&encoded).is_err()); + assert!(serde_cbor::from_slice::( + &serde_cbor::to_vec(&"future_phase").unwrap() + ) + .is_err()); + } +} diff --git a/trail-environment-adapter-sdk/tests/fixtures/legacy-v1-package-without-protocols.toml b/trail-environment-adapter-sdk/tests/fixtures/legacy-v1-package-without-protocols.toml new file mode 100644 index 00000000..d0506ddc --- /dev/null +++ b/trail-environment-adapter-sdk/tests/fixtures/legacy-v1-package-without-protocols.toml @@ -0,0 +1,15 @@ +schema = "trail.environment-adapter-package/v1" + +[adapter] +canonical_identity = "fixture/legacy-v1@1" +implementation_version = "1" +selectors = ["fixture-v1"] +kind = "generated" +layer_adapter_name = "fixture-v1" +discovery_markers = ["fixture.v1"] +stability = "experimental" +description = "Legacy v1 package with no protocol or capability metadata" + +[executable] +path = "fixture-adapter" +sha256 = "sha256:0000000000000000000000000000000000000000000000000000000000000000" diff --git a/trail-environment-adapter-sdk/tests/fixtures/legacy-v2-package-without-capabilities.toml b/trail-environment-adapter-sdk/tests/fixtures/legacy-v2-package-without-capabilities.toml new file mode 100644 index 00000000..242e9710 --- /dev/null +++ b/trail-environment-adapter-sdk/tests/fixtures/legacy-v2-package-without-capabilities.toml @@ -0,0 +1,16 @@ +schema = "trail.environment-adapter-package/v1" + +[adapter] +canonical_identity = "fixture/legacy-v2@1" +implementation_version = "1" +selectors = ["fixture-v2"] +kind = "generated" +layer_adapter_name = "fixture-v2" +discovery_markers = ["fixture.v2"] +protocols = ["trail.environment-adapter/v2", "trail.environment-adapter/v1"] +stability = "experimental" +description = "Legacy v2 package with no v3 capability metadata" + +[executable] +path = "fixture-adapter" +sha256 = "sha256:1111111111111111111111111111111111111111111111111111111111111111" diff --git a/trail-environment-adapter-sdk/tests/fixtures/protocol-v3-propose-request.frame.hex b/trail-environment-adapter-sdk/tests/fixtures/protocol-v3-propose-request.frame.hex new file mode 100644 index 00000000..8a750ee9 --- /dev/null +++ b/trail-environment-adapter-sdk/tests/fixtures/protocol-v3-propose-request.frame.hex @@ -0,0 +1 @@ +000001fca86870726f746f636f6c781c747261696c2e656e7669726f6e6d656e742d616461707465722f76336a726571756573745f69646f666978747572652d7265717565737470616461707465725f6964656e7469747971666978747572652f636f646567656e403173646973747269627574696f6e5f646967657374781b7368613235363a666978747572652d646973747269627574696f6e64686f7374a2706f7065726174696e675f73797374656d656c696e75786c617263686974656374757265667838365f36346b736f757263655f726f6f746c726f6f743a66697874757265666c696d697473a86f6d61785f696e7075745f66696c65731910006f6d61785f696e7075745f62797465731a008000006b6d61785f616374696f6e7318406f6d61785f76616c69646174696f6e7318406b6d61785f6f7574707574731820726d61785f736f757263655f6578706f72747318206f6d61785f617574686f7269746965731880726d61785f726573706f6e73655f62797465731a00400000696f7065726174696f6ea3666d6574686f646770726f706f73656e636f6d706f6e656e745f726f6f74612e6566696c657381a464706174686b736368656d612e6a736f6e6c636f6e74656e745f68617368747368613235363a666978747572652d696e7075746a65786563757461626c65f467636f6e74656e74437b7d0a diff --git a/trail-environment-adapter-sdk/tests/fixtures/protocol-v3-propose-response.frame.hex b/trail-environment-adapter-sdk/tests/fixtures/protocol-v3-propose-response.frame.hex new file mode 100644 index 00000000..0a5bcd2f --- /dev/null +++ b/trail-environment-adapter-sdk/tests/fixtures/protocol-v3-propose-response.frame.hex @@ -0,0 +1 @@ +000000fda36870726f746f636f6c781c747261696c2e656e7669726f6e6d656e742d616461707465722f76336a726571756573745f69646f666978747572652d7265717565737466726573756c74a266726573756c746870726f706f73656469636f6d706f6e656e74a76c636f6d706f6e656e745f69646f666978747572652e636f646567656e6e636f6d706f6e656e745f726f6f74612e646b696e646967656e657261746564667374617475736572656164796c70726f706f73616c5f6b6579777368613235363a666978747572652d70726f706f73616c746d697373696e675f726571756972656d656e747380707265636f766572795f616374696f6e7380 diff --git a/trail-environment-adapter-sdk/tests/protocol_v3_contract.rs b/trail-environment-adapter-sdk/tests/protocol_v3_contract.rs new file mode 100644 index 00000000..1837e36a --- /dev/null +++ b/trail-environment-adapter-sdk/tests/protocol_v3_contract.rs @@ -0,0 +1,205 @@ +use std::collections::BTreeSet; + +use proptest::prelude::*; +use trail_environment_adapter_sdk::{ + negotiate_highest_mutual_protocol, read_frame, write_frame, AdapterComponentProposalV3, + AdapterHost, AdapterOperationV3, AdapterPackageCapabilities, AdapterPackageManifest, + AdapterProposalStatusV3, AdapterRequestV3, AdapterResponseV3, AdapterResultV3, PinnedFile, + ProtocolError, PROTOCOL_V1, PROTOCOL_V2, PROTOCOL_V3, +}; + +const GOLDEN_REQUEST_FRAME: &str = include_str!("fixtures/protocol-v3-propose-request.frame.hex"); +const GOLDEN_RESPONSE_FRAME: &str = include_str!("fixtures/protocol-v3-propose-response.frame.hex"); + +fn proposal() -> AdapterComponentProposalV3 { + AdapterComponentProposalV3 { + component_id: "fixture.codegen".into(), + component_root: ".".into(), + kind: "generated".into(), + status: AdapterProposalStatusV3::Ready, + proposal_key: "sha256:fixture-proposal".into(), + missing_requirements: Vec::new(), + recovery_actions: Vec::new(), + } +} + +fn request() -> AdapterRequestV3 { + AdapterRequestV3::new( + "fixture-request", + "fixture/codegen@1", + "sha256:fixture-distribution", + AdapterHost { + operating_system: "linux".into(), + architecture: "x86_64".into(), + }, + "root:fixture", + AdapterOperationV3::Propose { + component_root: ".".into(), + files: vec![PinnedFile { + path: "schema.json".into(), + content_hash: "sha256:fixture-input".into(), + executable: false, + content: b"{}\n".to_vec(), + }], + }, + ) +} + +fn response() -> AdapterResponseV3 { + AdapterResponseV3::for_request( + &request(), + AdapterResultV3::Proposed { + component: Some(proposal()), + }, + ) +} + +fn framed(value: &T) -> Vec { + let mut bytes = Vec::new(); + write_frame(&mut bytes, value, 1024 * 1024).unwrap(); + bytes +} + +fn golden_bytes(fixture: &str) -> Vec { + hex::decode(fixture.split_whitespace().collect::()).unwrap() +} + +#[test] +fn protocol_v3_golden_frames_are_stable_and_round_trip() { + let request = request(); + let request_frame = framed(&request); + assert_eq!( + request_frame, + golden_bytes(GOLDEN_REQUEST_FRAME), + "request frame changed: {}", + hex::encode(&request_frame) + ); + let decoded: AdapterRequestV3 = read_frame( + &mut golden_bytes(GOLDEN_REQUEST_FRAME).as_slice(), + 1024 * 1024, + ) + .unwrap(); + assert_eq!(decoded, request); + + let response = response(); + let response_frame = framed(&response); + assert_eq!( + response_frame, + golden_bytes(GOLDEN_RESPONSE_FRAME), + "response frame changed: {}", + hex::encode(&response_frame) + ); + let decoded: AdapterResponseV3 = read_frame( + &mut golden_bytes(GOLDEN_RESPONSE_FRAME).as_slice(), + 1024 * 1024, + ) + .unwrap(); + assert_eq!(decoded, response); +} + +#[test] +fn protocol_v3_frames_fail_closed_on_truncation_and_declared_over_limit() { + let complete = framed(&request()); + for length in [0, 1, 3, 4, complete.len() - 1] { + let mut truncated = &complete[..length]; + assert!(matches!( + read_frame::(&mut truncated, 1024 * 1024), + Err(ProtocolError::Truncated) + )); + } + + let body_length = u32::from_be_bytes(complete[..4].try_into().unwrap()) as usize; + assert!(matches!( + read_frame::(&mut complete.as_slice(), body_length - 1), + Err(ProtocolError::FrameTooLarge { + actual, + maximum + }) if actual == body_length && maximum == body_length - 1 + )); + assert!(matches!( + write_frame(&mut Vec::new(), &request(), body_length - 1), + Err(ProtocolError::FrameTooLarge { + actual, + maximum + }) if actual == body_length && maximum == body_length - 1 + )); +} + +#[test] +fn legacy_protocol_lists_negotiate_without_v3_authority() { + let legacy_v1 = vec![PROTOCOL_V1.to_string()]; + let legacy_v2 = vec![PROTOCOL_V1.to_string(), PROTOCOL_V2.to_string()]; + assert_eq!( + negotiate_highest_mutual_protocol(&[PROTOCOL_V3, PROTOCOL_V2, PROTOCOL_V1], &legacy_v1), + Some(PROTOCOL_V1) + ); + assert_eq!( + negotiate_highest_mutual_protocol(&[PROTOCOL_V3, PROTOCOL_V2, PROTOCOL_V1], &legacy_v2), + Some(PROTOCOL_V2) + ); +} + +#[test] +fn legacy_package_fixtures_preserve_exact_v1_v2_meaning() { + let v1: AdapterPackageManifest = toml::from_str(include_str!( + "fixtures/legacy-v1-package-without-protocols.toml" + )) + .unwrap(); + assert_eq!(v1.adapter.protocols, [PROTOCOL_V1]); + assert_eq!( + v1.adapter.capabilities, + AdapterPackageCapabilities::default() + ); + + let v2: AdapterPackageManifest = toml::from_str(include_str!( + "fixtures/legacy-v2-package-without-capabilities.toml" + )) + .unwrap(); + assert_eq!(v2.adapter.protocols, [PROTOCOL_V2, PROTOCOL_V1]); + assert_eq!( + v2.adapter.capabilities, + AdapterPackageCapabilities::default() + ); + assert!(!v2 + .adapter + .protocols + .iter() + .any(|value| value == PROTOCOL_V3)); +} + +proptest! { + #[test] + fn negotiation_is_exact_order_independent_and_selects_the_highest_mutual( + host in prop::collection::vec(0u8..6, 0..12), + package in prop::collection::vec(0u8..6, 0..12), + ) { + const IDENTITIES: [&str; 6] = [ + PROTOCOL_V1, + PROTOCOL_V2, + PROTOCOL_V3, + "trail.environment-adapter/v3-preview", + "trail.environment-adapter/v30", + "unknown", + ]; + let host = host.into_iter().map(|index| IDENTITIES[index as usize]).collect::>(); + let package = package + .into_iter() + .map(|index| IDENTITIES[index as usize].to_string()) + .collect::>(); + let host_set = host.iter().copied().collect::>(); + let package_set = package.iter().map(String::as_str).collect::>(); + let expected = [PROTOCOL_V3, PROTOCOL_V2, PROTOCOL_V1] + .into_iter() + .find(|candidate| host_set.contains(candidate) && package_set.contains(candidate)); + prop_assert_eq!(negotiate_highest_mutual_protocol(&host, &package), expected); + + let mut reversed_host = host; + let mut reversed_package = package; + reversed_host.reverse(); + reversed_package.reverse(); + prop_assert_eq!( + negotiate_highest_mutual_protocol(&reversed_host, &reversed_package), + expected + ); + } +} diff --git a/trail/Cargo.toml b/trail/Cargo.toml index 4fc9a722..232f9c58 100644 --- a/trail/Cargo.toml +++ b/trail/Cargo.toml @@ -23,6 +23,7 @@ bzip2.workspace = true clap.workspace = true ed25519-dalek.workspace = true flate2.workspace = true +fastcdc.workspace = true getrandom.workspace = true hex.workspace = true ignore.workspace = true @@ -36,6 +37,7 @@ reqwest.workspace = true rusqlite.workspace = true rustix.workspace = true serde.workspace = true +serde_bytes.workspace = true serde_cbor.workspace = true serde_json.workspace = true globset.workspace = true @@ -72,6 +74,9 @@ tokio = { workspace = true, features = ["io-util", "macros", "net", "sync"] } [target.'cfg(target_os = "windows")'.dependencies] winapi = { version = "0.3", features = ["errhandlingapi", "fileapi", "handleapi", "jobapi2", "minwinbase", "ntdef", "ntstatus", "processthreadsapi", "sddl", "securitybaseapi", "synchapi", "userenv", "winbase", "winerror", "winnt"] } +[target.'cfg(unix)'.dependencies] +xattr.workspace = true + [target.'cfg(all(target_os = "windows", target_arch = "x86_64"))'.dependencies] dokan = "0.3.1" dokan-sys = "0.3.1" diff --git a/trail/src/acp.rs b/trail/src/acp.rs index 0ae65e5e..cefc8e89 100644 --- a/trail/src/acp.rs +++ b/trail/src/acp.rs @@ -2150,7 +2150,8 @@ impl CaptureCoordinator { "stop_reason": stop_reason(message), "error": message.get("error").cloned(), "checkpoint_error": checkpoint_error, - "managed_execution_id": lifecycle.as_ref().map(|receipt| &receipt.execution_id) + "managed_execution_id": lifecycle.as_ref().map(|receipt| &receipt.execution_id), + "managed_execution": lifecycle.as_ref() }))), None, None, @@ -2923,9 +2924,10 @@ impl CaptureCoordinator { "acp_managed_execution_interrupted", Some(serde_json::json!({ "finish_reason": format!("{reason:?}"), - "execution_id": lifecycle.execution_id, - "checkpoint_error": lifecycle.checkpoint_error, - "disposal_error": lifecycle.disposal_error + "execution_id": &lifecycle.execution_id, + "checkpoint_error": &lifecycle.checkpoint_error, + "disposal_error": &lifecycle.disposal_error, + "managed_execution": &lifecycle })), None, None, @@ -5172,6 +5174,18 @@ mod tests { payload["surface"] == "acp_prompt" && payload["phase"] == "checkpoint" }) })); + let finished = turn + .events + .iter() + .find(|event| event.event_type == "acp_prompt_finished") + .expect("managed ACP completion must retain its lifecycle receipt"); + let managed = &finished.payload.as_ref().unwrap()["managed_execution"]; + assert_eq!( + managed["preparation"]["missing_resolution_policy"], + "explicit" + ); + assert_eq!(managed["finalization"]["checkpoint_status"], "failed"); + assert_eq!(managed["finalization"]["complete"], false); } #[test] @@ -5259,7 +5273,20 @@ mod tests { .flat_map(|turn| turn.events) .find(|event| event.event_type == "acp_managed_execution_interrupted") .expect("ACP interruption must be durable"); - assert_eq!(interruption.payload.unwrap()["finish_reason"], "EditorEof"); + let payload = interruption.payload.unwrap(); + assert_eq!(payload["finish_reason"], "EditorEof"); + assert_eq!( + payload["managed_execution"]["preparation"]["missing_resolution_policy"], + "explicit" + ); + assert_eq!( + payload["managed_execution"]["finalization"]["checkpoint_status"], + "succeeded" + ); + assert_eq!( + payload["managed_execution"]["finalization"]["complete"], + true + ); let branch = db.lane_branch(lane).unwrap(); assert_ne!(branch.head_change, branch.base_change); } diff --git a/trail/src/cli/command.rs b/trail/src/cli/command.rs index 61103126..c6b335a5 100644 --- a/trail/src/cli/command.rs +++ b/trail/src/cli/command.rs @@ -403,6 +403,67 @@ mod tests { assert!(Cli::try_parse_from(["trail", "env", "sync-all", "lane-a"]).is_err()); } + #[test] + fn parses_environment_artifact_lifecycle_commands() { + let cli = Cli::try_parse_from([ + "trail", + "env", + "resolve", + "component", + "cargo-target:root", + "--lane", + "lane-a", + "--refresh", + ]) + .expect("environment resolve component should parse"); + let Command::Env(EnvironmentCommand { + command: + EnvironmentSubcommand::Resolve(EnvironmentResolveCommand { + command: EnvironmentResolveSubcommand::Component(args), + }), + }) = cli.command + else { + panic!("expected environment resolve component command"); + }; + assert_eq!(args.component, "cargo-target:root"); + assert_eq!(args.lane.as_deref(), Some("lane-a")); + assert!(args.refresh); + + Cli::try_parse_from([ + "trail", + "env", + "artifact", + "verify", + "artifact_envelope_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "--level", + "full", + ]) + .expect("environment artifact verify should parse"); + Cli::try_parse_from([ + "trail", + "env", + "artifact", + "quarantine", + "resolve", + "artifact_quarantine_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "--resolution", + "retain-private", + ]) + .expect("environment quarantine resolution should parse"); + Cli::try_parse_from([ + "trail", + "env", + "source", + "export", + "lane-a", + "--component", + "generated.types", + "--export", + "types", + ]) + .expect("environment source export should parse"); + } + #[test] fn parses_environment_runtime_lifecycle_commands() { for (action, expected) in [ diff --git a/trail/src/cli/command/environment_args.rs b/trail/src/cli/command/environment_args.rs index 0328b99b..dd1a6cd1 100644 --- a/trail/src/cli/command/environment_args.rs +++ b/trail/src/cli/command/environment_args.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use clap::{Args, Subcommand}; +use clap::{Args, Subcommand, ValueEnum}; #[derive(Args)] pub(super) struct DepsCommand { @@ -52,14 +52,157 @@ pub(super) enum EnvironmentSubcommand { Explain(EnvironmentExplainArgs), /// Preview the normalized component key, actions, outputs, and capabilities without executing. Plan(EnvironmentSyncArgs), + /// Produce or deliberately refresh immutable dependency-resolution snapshots. + Resolve(EnvironmentResolveCommand), /// Converge all desired components or one named component. Sync(EnvironmentSyncCommand), + /// Inspect and verify immutable artifacts or resolve quarantine evidence. + Artifact(EnvironmentArtifactCommand), + /// Explicitly write a declared generated-source export into lane source. + Source(EnvironmentSourceCommand), /// Publish one quiesced manual private output as a reusable immutable layer. Promote(EnvironmentPromoteArgs), /// Inspect, reconcile, or stop lane-private runtime services. Runtime(EnvironmentRuntimeCommand), } +#[derive(Args)] +pub(super) struct EnvironmentResolveCommand { + #[command(subcommand)] + pub(super) command: EnvironmentResolveSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum EnvironmentResolveSubcommand { + /// Resolve every incomplete component in deterministic discovery order. + All(EnvironmentResolveAllArgs), + /// Resolve one stable component ID from `trail env discover`. + Component(EnvironmentResolveComponentArgs), +} + +#[derive(Args)] +pub(super) struct EnvironmentResolveAllArgs { + /// Lane name. Omit only inside exactly one mounted or managed lane context. + pub(super) lane: Option, + /// Restrict discovery to one source-relative component root. + #[arg(long, value_name = "ROOT")] + pub(super) path: Option, + /// Deliberately rerun the resolver and supersede its current snapshot. + #[arg(long)] + pub(super) refresh: bool, +} + +#[derive(Args)] +pub(super) struct EnvironmentResolveComponentArgs { + /// Stable logical component ID from `trail env discover`. + pub(super) component: String, + /// Lane name. Omit only inside exactly one mounted or managed lane context. + #[arg(long)] + pub(super) lane: Option, + /// Select one source-relative root when the component ID is ambiguous. + #[arg(long, value_name = "ROOT")] + pub(super) path: Option, + /// Deliberately rerun the resolver and supersede its current snapshot. + #[arg(long)] + pub(super) refresh: bool, +} + +#[derive(Args)] +pub(super) struct EnvironmentArtifactCommand { + #[command(subcommand)] + pub(super) command: EnvironmentArtifactSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum EnvironmentArtifactSubcommand { + /// Inspect one artifact envelope and its bindings, trust, and storage evidence. + Inspect(EnvironmentArtifactIdArgs), + /// Verify one artifact at an explicit evidence level. + Verify(EnvironmentArtifactVerifyArgs), + /// Manage durable divergent-producer quarantine evidence. + Quarantine(EnvironmentArtifactQuarantineCommand), +} + +#[derive(Args)] +pub(super) struct EnvironmentArtifactIdArgs { + /// Content-derived artifact envelope ID. + pub(super) artifact: String, +} + +#[derive(Clone, Copy, Debug, ValueEnum)] +pub(super) enum EnvironmentArtifactVerificationLevelArg { + Attach, + Sample, + Full, + Reproduce, +} + +#[derive(Args)] +pub(super) struct EnvironmentArtifactVerifyArgs { + /// Content-derived artifact envelope ID. + pub(super) artifact: String, + /// Evidence depth: attach, sample, full, or reproduce. + #[arg(long, value_enum)] + pub(super) level: EnvironmentArtifactVerificationLevelArg, +} + +#[derive(Args)] +pub(super) struct EnvironmentArtifactQuarantineCommand { + #[command(subcommand)] + pub(super) command: EnvironmentArtifactQuarantineSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum EnvironmentArtifactQuarantineSubcommand { + /// List active and resolved quarantine records deterministically. + List, + /// Show one quarantine record and its immutable evidence identity. + Show(EnvironmentArtifactQuarantineIdArgs), + /// Resolve one active quarantine through an explicit policy choice. + Resolve(EnvironmentArtifactQuarantineResolveArgs), +} + +#[derive(Args)] +pub(super) struct EnvironmentArtifactQuarantineIdArgs { + pub(super) quarantine: String, +} + +#[derive(Clone, Copy, Debug, ValueEnum)] +pub(super) enum EnvironmentArtifactQuarantineResolutionArg { + RetainPrivate, + AcceptIncumbent, + AcceptCandidate, + RetireAll, +} + +#[derive(Args)] +pub(super) struct EnvironmentArtifactQuarantineResolveArgs { + pub(super) quarantine: String, + #[arg(long, value_enum)] + pub(super) resolution: EnvironmentArtifactQuarantineResolutionArg, +} + +#[derive(Args)] +pub(super) struct EnvironmentSourceCommand { + #[command(subcommand)] + pub(super) command: EnvironmentSourceSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum EnvironmentSourceSubcommand { + /// Export one declared artifact subtree through ordinary guarded source writes. + Export(EnvironmentSourceExportArgs), +} + +#[derive(Args)] +pub(super) struct EnvironmentSourceExportArgs { + pub(super) lane: String, + #[arg(long, value_name = "ID")] + pub(super) component: String, + #[arg(long, value_name = "NAME")] + pub(super) export: String, +} + #[derive(Args)] pub(super) struct EnvironmentPromoteArgs { pub(super) lane: String, diff --git a/trail/src/cli/command/handler.rs b/trail/src/cli/command/handler.rs index 45d79329..18700762 100644 --- a/trail/src/cli/command/handler.rs +++ b/trail/src/cli/command/handler.rs @@ -5,7 +5,9 @@ use clap::Parser; use super::{render::*, *}; use trail::{ - acp::AcpRelayOptions, Actor, Error, InitImportMode, LaneGateOptions, OperationKind, + acp::AcpRelayOptions, Actor, ArtifactEnvelopeId, ArtifactQuarantineId, + ArtifactQuarantineResolutionV1, ArtifactSourceExportAuthorizationV1, + ArtifactVerificationLevelV1, Error, InitImportMode, LaneGateOptions, OperationKind, PatchDocument, RecordOptions, Result, StructuredErrorEnvelope, Trail, }; @@ -118,7 +120,7 @@ fn run(cli: Cli) -> Result<()> { let command = cli.command; if matches!(ctx.format, OutputFormat::Ndjson) && !supports_ndjson(&command) { return Err(Error::InvalidInput( - "--format ndjson is available only for streaming watch commands; use --format json for a single report" + "--format ndjson is available for streaming watch commands and environment artifact lifecycle reports; use --format json for other single reports" .to_string(), )); } @@ -229,6 +231,12 @@ fn supports_ndjson(command: &Command) -> bool { Command::Lane(LaneCommand { command: LaneSubcommand::Watch(args), }) => !args.once, + Command::Env(EnvironmentCommand { + command: + EnvironmentSubcommand::Resolve(_) + | EnvironmentSubcommand::Artifact(_) + | EnvironmentSubcommand::Source(_), + }) => true, _ => false, } } @@ -238,11 +246,15 @@ fn render_specialist( title: &str, report: &T, ) -> Result<()> { - render_semantic_report(title, report, ctx.json, &ctx.render) + if matches!(ctx.format, OutputFormat::Ndjson) { + render_ndjson(report) + } else { + render_semantic_report(title, report, ctx.json, &ctx.render) + } } fn handle_environment_command(ctx: &RuntimeContext, environment: EnvironmentCommand) -> Result<()> { - let db = open_db(ctx)?; + let mut db = open_db(ctx)?; match environment.command { EnvironmentSubcommand::Adapters => render_specialist( ctx, @@ -328,6 +340,33 @@ fn handle_environment_command(ctx: &RuntimeContext, environment: EnvironmentComm args.component.as_deref(), )?, ), + EnvironmentSubcommand::Resolve(resolve) => match resolve.command { + EnvironmentResolveSubcommand::All(args) => { + let lane = resolve_environment_sync_lane(&db, args.lane.as_deref())?; + render_specialist( + ctx, + "Resolved environments", + &db.resolve_all_workspace_environment_components( + &lane, + args.path.as_deref(), + args.refresh, + )?, + ) + } + EnvironmentResolveSubcommand::Component(args) => { + let lane = resolve_environment_sync_lane(&db, args.lane.as_deref())?; + render_specialist( + ctx, + "Resolved environment", + &db.resolve_workspace_environment_component( + &lane, + &args.component, + args.path.as_deref(), + args.refresh, + )?, + ) + } + }, EnvironmentSubcommand::Sync(sync) => match sync.command { EnvironmentSyncSubcommand::All(args) => { let lane = resolve_environment_sync_lane(&db, args.lane.as_deref())?; @@ -351,6 +390,98 @@ fn handle_environment_command(ctx: &RuntimeContext, environment: EnvironmentComm ) } }, + EnvironmentSubcommand::Artifact(artifact) => match artifact.command { + EnvironmentArtifactSubcommand::Inspect(args) => { + let artifact = ArtifactEnvelopeId::parse(args.artifact).map_err(|error| { + Error::InvalidInput(format!("invalid artifact envelope ID: {error}")) + })?; + render_specialist(ctx, "Artifact", &db.inspect_artifact(&artifact)?) + } + EnvironmentArtifactSubcommand::Verify(args) => { + let artifact = ArtifactEnvelopeId::parse(args.artifact).map_err(|error| { + Error::InvalidInput(format!("invalid artifact envelope ID: {error}")) + })?; + let level = match args.level { + EnvironmentArtifactVerificationLevelArg::Attach => { + ArtifactVerificationLevelV1::Attach + } + EnvironmentArtifactVerificationLevelArg::Sample => { + ArtifactVerificationLevelV1::Sample + } + EnvironmentArtifactVerificationLevelArg::Full => { + ArtifactVerificationLevelV1::Full + } + EnvironmentArtifactVerificationLevelArg::Reproduce => { + ArtifactVerificationLevelV1::Reproduce + } + }; + let report = db.verify_artifact(&artifact, level)?; + render_specialist(ctx, "Artifact verification", &report)?; + if report.valid { + Ok(()) + } else { + Err(Error::InvalidInput(format!( + "artifact `{artifact}` failed `{}` verification", + level.as_str() + ))) + } + } + EnvironmentArtifactSubcommand::Quarantine(quarantine) => match quarantine.command { + EnvironmentArtifactQuarantineSubcommand::List => render_specialist( + ctx, + "Artifact quarantines", + &db.artifact_quarantine_list_report()?, + ), + EnvironmentArtifactQuarantineSubcommand::Show(args) => { + let quarantine = + ArtifactQuarantineId::parse(args.quarantine).map_err(|error| { + Error::InvalidInput(format!("invalid quarantine ID: {error}")) + })?; + render_specialist( + ctx, + "Artifact quarantine", + &db.artifact_quarantine(&quarantine)?, + ) + } + EnvironmentArtifactQuarantineSubcommand::Resolve(args) => { + let quarantine = + ArtifactQuarantineId::parse(args.quarantine).map_err(|error| { + Error::InvalidInput(format!("invalid quarantine ID: {error}")) + })?; + let resolution = match args.resolution { + EnvironmentArtifactQuarantineResolutionArg::RetainPrivate => { + ArtifactQuarantineResolutionV1::RetainPrivate + } + EnvironmentArtifactQuarantineResolutionArg::AcceptIncumbent => { + ArtifactQuarantineResolutionV1::AcceptIncumbent + } + EnvironmentArtifactQuarantineResolutionArg::AcceptCandidate => { + ArtifactQuarantineResolutionV1::AcceptCandidate + } + EnvironmentArtifactQuarantineResolutionArg::RetireAll => { + ArtifactQuarantineResolutionV1::RetireAll + } + }; + render_specialist( + ctx, + "Resolved artifact quarantine", + &db.resolve_artifact_quarantine_report(&quarantine, resolution)?, + ) + } + }, + }, + EnvironmentSubcommand::Source(source) => match source.command { + EnvironmentSourceSubcommand::Export(args) => { + let plan = db.plan_artifact_source_export( + &args.lane, + &args.component, + &args.export, + ArtifactSourceExportAuthorizationV1::ExplicitUser, + )?; + let report = db.execute_artifact_source_export(plan)?; + render_specialist(ctx, "Exported artifact source", &report) + } + }, EnvironmentSubcommand::Promote(args) => render_specialist( ctx, "Promoted environment output", diff --git a/trail/src/cli/command/handler/daemon_start.rs b/trail/src/cli/command/handler/daemon_start.rs index a8c46d60..e112a39f 100644 --- a/trail/src/cli/command/handler/daemon_start.rs +++ b/trail/src/cli/command/handler/daemon_start.rs @@ -92,6 +92,8 @@ struct SocketLeafIdentity { inode: u64, ctime_sec: i64, ctime_nsec: i64, + stable_creation_sec: i64, + stable_creation_nsec: i64, } #[derive(Clone, Debug)] @@ -1358,6 +1360,16 @@ fn socket_leaf_creation_timestamp(metadata: &libc::stat) -> (i64, i64) { (metadata.st_ctime, metadata.st_ctime_nsec) } +#[cfg(target_os = "macos")] +fn socket_leaf_stable_creation_timestamp(metadata: &libc::stat) -> (i64, i64) { + (metadata.st_birthtime, metadata.st_birthtime_nsec) +} + +#[cfg(not(target_os = "macos"))] +fn socket_leaf_stable_creation_timestamp(_metadata: &libc::stat) -> (i64, i64) { + (0, 0) +} + fn socket_ctime_identity( socket_ctime_sec: Option, socket_ctime_nsec: Option, @@ -1390,11 +1402,15 @@ fn verify_socket_leaf_owner( )); } let (ctime_sec, ctime_nsec) = socket_leaf_creation_timestamp(&metadata); + let (stable_creation_sec, stable_creation_nsec) = + socket_leaf_stable_creation_timestamp(&metadata); Ok(SocketLeafIdentity { device: metadata.st_dev as u64, inode: metadata.st_ino as u64, ctime_sec, ctime_nsec, + stable_creation_sec, + stable_creation_nsec, }) } @@ -1421,6 +1437,46 @@ fn verify_secure_socket_leaf_identity( Ok(identity) } +fn socket_leaf_identity_matches( + expected: SocketLeafIdentity, + captured: SocketLeafIdentity, +) -> bool { + expected == captured +} + +fn socket_leaf_identity_from_file(file: &File) -> Result { + let metadata = file.metadata()?; + if metadata.mode() & u32::from(libc::S_IFMT) != u32::from(libc::S_IFSOCK) { + return Err(Error::DaemonUnavailable( + "workspace daemon socket identity handle changed type".into(), + )); + } + Ok(SocketLeafIdentity { + device: metadata.dev(), + inode: metadata.ino(), + ctime_sec: metadata.ctime(), + ctime_nsec: metadata.ctime_nsec(), + stable_creation_sec: 0, + stable_creation_nsec: 0, + }) +} + +#[cfg(target_os = "linux")] +fn open_socket_leaf_identity_handle(parent: &File, leaf: &str) -> std::io::Result> { + openat_file( + parent, + leaf, + libc::O_PATH | libc::O_NOFOLLOW | libc::O_CLOEXEC, + 0, + ) + .map(Some) +} + +#[cfg(not(target_os = "linux"))] +fn open_socket_leaf_identity_handle(_parent: &File, _leaf: &str) -> std::io::Result> { + Ok(None) +} + fn publish_owner_file(authority: &SecureAuthority, name: &str, bytes: &[u8]) -> Result<()> { let tmp = format!(".{name}.{}.tmp", random_hex(12)?); let mut file = openat_file( @@ -1548,6 +1604,11 @@ fn remove_socket_leaf_if_identity( missing_ok: bool, run_test_boundary: bool, ) -> Result<()> { + let identity_handle = match open_socket_leaf_identity_handle(&authority.trail_directory, leaf) { + Ok(handle) => handle, + Err(error) if missing_ok && error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(Error::Io(error)), + }; let expected = match socket_leaf_stat(&authority.trail_directory, leaf) { Ok(_) => verify_secure_socket_leaf_identity( &authority.trail_directory, @@ -1559,6 +1620,13 @@ fn remove_socket_leaf_if_identity( Err(error) if missing_ok && error.kind() == std::io::ErrorKind::NotFound => return Ok(()), Err(error) => return Err(Error::Io(error)), }; + if let Some(handle) = identity_handle.as_ref() + && !socket_leaf_identity_matches(expected, socket_leaf_identity_from_file(handle)?) + { + return Err(Error::DaemonUnavailable( + "workspace daemon socket identity changed while acquiring cleanup authority".into(), + )); + } #[cfg(debug_assertions)] if run_test_boundary { @@ -1582,7 +1650,19 @@ fn remove_socket_leaf_if_identity( authority.trail_directory.sync_all()?; let captured = verify_socket_leaf_owner(&authority.trail_directory, &quarantine, Some(0o600))?; - if captured.device != expected.device || captured.inode != expected.inode { + // Linux keeps the original inode alive through O_PATH so it cannot be + // recycled during the rename race. macOS socket handles are unavailable, + // so APFS birth time supplies the stable post-rename identity dimension. + let captured_matches = if let Some(handle) = identity_handle.as_ref() { + socket_leaf_identity_matches(socket_leaf_identity_from_file(handle)?, captured) + } else { + captured.device == expected.device + && captured.inode == expected.inode + && (!cfg!(target_os = "macos") + || (captured.stable_creation_sec, captured.stable_creation_nsec) + == (expected.stable_creation_sec, expected.stable_creation_nsec)) + }; + if !captured_matches { let restore = renameat_noreplace(&authority.trail_directory, &quarantine, leaf); let _ = authority.trail_directory.sync_all(); return match restore { @@ -1834,3 +1914,31 @@ pub(super) fn workspace_from_context(ctx: &RuntimeContext) -> Result { .or_else(|| std::env::current_dir().ok()) .ok_or_else(|| Error::InvalidInput("workspace path is unavailable".into())) } + +#[cfg(test)] +mod tests { + use super::{socket_leaf_identity_matches, SocketLeafIdentity}; + + #[test] + fn socket_leaf_identity_rejects_creation_time_reuse() { + let expected = SocketLeafIdentity { + device: 7, + inode: 11, + ctime_sec: 13, + ctime_nsec: 17, + stable_creation_sec: 23, + stable_creation_nsec: 29, + }; + let substituted = SocketLeafIdentity { + ctime_nsec: 19, + ..expected + }; + let recycled = SocketLeafIdentity { + stable_creation_nsec: 31, + ..expected + }; + + assert!(!socket_leaf_identity_matches(expected, substituted)); + assert!(!socket_leaf_identity_matches(expected, recycled)); + } +} diff --git a/trail/src/cli/command/render/lane/work.rs b/trail/src/cli/command/render/lane/work.rs index 02974d0a..ce2070da 100644 --- a/trail/src/cli/command/render/lane/work.rs +++ b/trail/src/cli/command/render/lane/work.rs @@ -100,6 +100,42 @@ pub(crate) fn render_workspace_space( "Lane-exclusive".to_string(), byte_count(report.lane_exclusive_physical_bytes), ), + ( + "Artifact logical".to_string(), + byte_count(report.artifact_storage.logical_bytes), + ), + ( + "Artifact authoritative unique".to_string(), + byte_count(report.artifact_storage.unique_authoritative_bytes), + ), + ( + "Artifact authoritative shared".to_string(), + byte_count(report.artifact_storage.cross_artifact_shared_bytes), + ), + ( + "Artifact materialized".to_string(), + byte_count(report.artifact_storage.materialized_bytes), + ), + ( + "Artifact lane-private".to_string(), + byte_count(report.artifact_storage.lane_private_bytes), + ), + ( + "Artifact prefetched".to_string(), + byte_count(report.artifact_storage.prefetched_bytes), + ), + ( + "Artifact demand-loaded".to_string(), + byte_count(report.artifact_storage.demand_loaded_bytes), + ), + ( + "Artifact reclaimable".to_string(), + byte_count(report.artifact_storage.reclaimable_bytes), + ), + ( + "Artifact unknown".to_string(), + byte_count(report.artifact_storage.unknown_bytes), + ), ("Backend".to_string(), report.backend.clone()), ( "Logical files".to_string(), diff --git a/trail/src/cli/command/render/maintenance.rs b/trail/src/cli/command/render/maintenance.rs index a3bf275f..47bbe413 100644 --- a/trail/src/cli/command/render/maintenance.rs +++ b/trail/src/cli/command/render/maintenance.rs @@ -66,6 +66,21 @@ pub(crate) fn render_backup_create( "SQLite".to_string(), format!("{} bytes", report.sqlite_bytes), ), + ( + "Retained private views".to_string(), + report.retained_private_views.to_string(), + ), + ( + "Rebuildable materializations".to_string(), + format!( + "{} ({} bytes omitted)", + report.rebuildable_materializations, report.rebuildable_materialization_bytes + ), + ), + ( + "Rebuildable performance caches".to_string(), + report.rebuildable_performance_caches.to_string(), + ), ])); if !report.fsck_errors.is_empty() { document = document.block(UiBlock::Checklist( @@ -108,6 +123,14 @@ pub(crate) fn render_backup_verify( ("Refs".to_string(), report.checked_refs.to_string()), ("Roots".to_string(), report.checked_roots.to_string()), ("Text objects".to_string(), report.checked_texts.to_string()), + ( + "Retained private views".to_string(), + report.retained_private_views.to_string(), + ), + ( + "Rebuildable materializations".to_string(), + report.rebuildable_materializations.to_string(), + ), ])); if !report.errors.is_empty() { document = document.block(UiBlock::Checklist( @@ -142,6 +165,14 @@ pub(crate) fn render_backup_restore( "Rewritten workdirs".to_string(), report.rewritten_workdirs.to_string(), ), + ( + "Restored private views".to_string(), + report.restored_private_views.to_string(), + ), + ( + "Rebuildable materializations".to_string(), + report.rebuildable_materializations.to_string(), + ), ])), options, ) diff --git a/trail/src/db/change_ledger/activation.rs b/trail/src/db/change_ledger/activation.rs index dd970ad2..c8c426bb 100644 --- a/trail/src/db/change_ledger/activation.rs +++ b/trail/src/db/change_ledger/activation.rs @@ -4,14 +4,14 @@ use sha2::{Digest, Sha256}; const APPROVED_PRODUCER_INVENTORY_SHA256: &str = "af2cca0566976a6d6f6cea00e99fe5089c91e357ca1d0a50fd5397edcda32833"; const APPROVED_RAW_MUTATION_INVENTORY_SHA256: &str = - "4fa1ed52b64a9d4a33f09a2915063a695890aaa80af5aa352a7f0d8815c2f1c4"; + "a6708f5ae489be0126aef47d8ec4e9b6e687d8a3f2908bff657ea87b5988e5d1"; const APPROVED_ACTIVATION_AUDIT_SHA256: &str = - "5bc402bb1f51cd7f9e2b48dc85cbd84f5ca5cee0deb33b4dab7b97b5432df74a"; + "b09c7edc5e2c45d6a53826175553fa83701cf02c5fc69820e762108c7971b01e"; const ACTIVATION_AUDIT_MANIFEST: &str = concat!( "trail-changed-path-activation-v1\n", "schema=1\n", "producer=af2cca0566976a6d6f6cea00e99fe5089c91e357ca1d0a50fd5397edcda32833\n", - "raw=4fa1ed52b64a9d4a33f09a2915063a695890aaa80af5aa352a7f0d8815c2f1c4\n", + "raw=a6708f5ae489be0126aef47d8ec4e9b6e687d8a3f2908bff657ea87b5988e5d1\n", "linux_suite=changed_path_ledger_linux\n", "macos_suite=changed_path_ledger_macos\n", "recovery_suite=changed_path_ledger_recovery\n", diff --git a/trail/src/db/core/backup/create.rs b/trail/src/db/core/backup/create.rs index a5b2f1d2..88a34e7b 100644 --- a/trail/src/db/core/backup/create.rs +++ b/trail/src/db/core/backup/create.rs @@ -3,6 +3,7 @@ use crate::db::change_ledger::mark_backup_scopes_untrusted; use crate::db::core::backup::publication::{ publish_staged_tree, remove_any, remove_retained_tree, sibling_stage, sync_file_for_publication, }; +use crate::db::lane::ViewMutationBarrier; impl Trail { pub fn create_backup( @@ -61,11 +62,27 @@ impl Trail { fs::copy(trailignore, output.join(".trailignore"))?; } + let retained_views = retained_private_views(self)?; + let mut view_barriers = Vec::with_capacity(retained_views.len()); + for view in &retained_views { + view_barriers.push(ViewMutationBarrier::exclusive(&view.meta_dir)?); + } + let (retained_private_views, retained_private_bytes) = + copy_retained_private_views(&retained_views, output)?; + let (sealed_private_bytes, retained_private_sha256) = + portable_tree_digest(&output.join("views"))?; + if sealed_private_bytes != retained_private_bytes { + return Err(Error::Conflict( + "retained private view bytes changed while creating the backup".into(), + )); + } + let sqlite_path = output.join(DB_RELATIVE_PATH); let sqlite_path_text = sqlite_path.to_string_lossy().to_string(); self.conn .execute("VACUUM main INTO ?1", params![sqlite_path_text])?; let backup_conn = Connection::open(&sqlite_path)?; + let rebuildable = sanitize_portable_backup_database(&backup_conn)?; mark_backup_scopes_untrusted(&backup_conn)?; let checkpoint_busy: i64 = backup_conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |row| row.get(0))?; @@ -75,6 +92,7 @@ impl Trail { )); } drop(backup_conn); + drop(view_barriers); sync_file_for_publication(&sqlite_path)?; let (sqlite_bytes, sqlite_sha256) = file_digest(&sqlite_path)?; @@ -103,6 +121,12 @@ impl Trail { sqlite_bytes, sqlite_sha256: sqlite_sha256.clone(), worktree_bytes, + retained_private_views, + retained_private_bytes, + retained_private_sha256, + rebuildable_materializations: rebuildable.materializations, + rebuildable_materialization_bytes: rebuildable.materialization_bytes, + rebuildable_performance_caches: rebuildable.performance_caches, }; let manifest_path = backup_manifest_path(output); fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)?; @@ -119,7 +143,167 @@ impl Trail { sqlite_bytes, sqlite_sha256, worktree_bytes, + retained_private_views, + retained_private_bytes, + rebuildable_materializations: rebuildable.materializations, + rebuildable_materialization_bytes: rebuildable.materialization_bytes, + rebuildable_performance_caches: rebuildable.performance_caches, fsck_errors: fsck.errors, }) } } + +#[derive(Debug)] +struct RetainedPrivateView { + view_id: String, + source_upper: PathBuf, + meta_dir: PathBuf, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(super) struct RebuildableBackupState { + pub(super) materializations: u64, + pub(super) materialization_bytes: u64, + pub(super) performance_caches: u64, +} + +fn retained_private_views(db: &Trail) -> Result> { + let mut statement = db + .conn + .prepare("SELECT view_id,source_upper,meta_dir FROM workspace_views ORDER BY view_id")?; + let rows = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + PathBuf::from(row.get::<_, String>(1)?), + PathBuf::from(row.get::<_, String>(2)?), + )) + })? + .collect::, _>>()?; + let mut retained = Vec::new(); + for (view_id, source_upper, meta_dir) in rows { + let mut components = Path::new(&view_id).components(); + if !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() { + return Err(Error::Corrupt(format!( + "workspace view ID `{view_id}` is not a confined backup path" + ))); + } + let view_dir = db.db_dir.join("views").join(&view_id); + let expected_source = view_dir.join("source-upper"); + let expected_meta = view_dir.join("meta"); + if !source_upper.is_dir() { + continue; + } + if !meta_dir.is_dir() { + return Err(Error::Corrupt(format!( + "workspace view `{view_id}` source upper has no recovery metadata" + ))); + } + if fs::canonicalize(&source_upper)? != fs::canonicalize(&expected_source)? + || fs::canonicalize(&meta_dir)? != fs::canonicalize(&expected_meta)? + { + return Err(Error::Corrupt(format!( + "workspace view `{view_id}` has noncanonical private-state paths" + ))); + } + retained.push(RetainedPrivateView { + view_id, + source_upper, + meta_dir, + }); + } + Ok(retained) +} + +fn copy_retained_private_views(views: &[RetainedPrivateView], output: &Path) -> Result<(u64, u64)> { + let mut bytes = 0_u64; + for view in views { + let destination = output.join("views").join(&view.view_id); + bytes = bytes.saturating_add(copy_dir_recursive( + &view.source_upper, + &destination.join("source-upper"), + )?); + bytes = bytes.saturating_add(copy_dir_recursive( + &view.meta_dir, + &destination.join("meta"), + )?); + bytes = bytes.saturating_sub(scrub_ephemeral_view_metadata(&destination.join("meta"))?); + } + Ok((views.len() as u64, bytes)) +} + +fn scrub_ephemeral_view_metadata(meta_dir: &Path) -> Result { + if !meta_dir.is_dir() { + return Ok(0); + } + let mut removed_bytes = 0_u64; + for entry in fs::read_dir(meta_dir)? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with("checkpoint-barrier.") + || matches!( + name.as_ref(), + "mount.json" | "unmount-request.json" | "view.json" + ) + { + let metadata = fs::symlink_metadata(entry.path())?; + if metadata.is_file() { + removed_bytes = removed_bytes.saturating_add(metadata.len()); + } + remove_any(&entry.path())?; + } + } + Ok(removed_bytes) +} + +pub(super) fn sanitize_portable_backup_database( + conn: &Connection, +) -> Result { + conn.execute_batch("BEGIN IMMEDIATE;")?; + let result = (|| -> Result { + let (artifact_count, artifact_bytes): (i64, i64) = conn.query_row( + "SELECT COUNT(*),COALESCE(SUM(COALESCE(physical_bytes,logical_bytes)),0) + FROM artifact_materializations", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + let (layer_count, layer_bytes): (i64, i64) = conn.query_row( + "SELECT COUNT(*),COALESCE(SUM(COALESCE(physical_bytes,logical_bytes)),0) + FROM workspace_layers", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + let performance_caches: i64 = conn.query_row( + "SELECT COUNT(*) FROM environment_cache_namespaces", + [], + |row| row.get(0), + )?; + conn.execute_batch( + "DELETE FROM artifact_materializations; + DELETE FROM environment_hot_access_sessions; + DELETE FROM environment_hot_sets; + DELETE FROM environment_generation_caches; + DELETE FROM environment_component_caches; + DELETE FROM environment_cache_namespaces;", + )?; + Ok(RebuildableBackupState { + materializations: u64::try_from(artifact_count.saturating_add(layer_count)) + .map_err(|_| Error::Corrupt("negative rebuildable materialization count".into()))?, + materialization_bytes: u64::try_from(artifact_bytes.saturating_add(layer_bytes)) + .map_err(|_| Error::Corrupt("negative rebuildable materialization bytes".into()))?, + performance_caches: u64::try_from(performance_caches) + .map_err(|_| Error::Corrupt("negative rebuildable cache count".into()))?, + }) + })(); + match result { + Ok(summary) => { + conn.execute_batch("COMMIT;")?; + Ok(summary) + } + Err(error) => { + let _ = conn.execute_batch("ROLLBACK;"); + Err(error) + } + } +} diff --git a/trail/src/db/core/backup/restore.rs b/trail/src/db/core/backup/restore.rs index df239123..f1c39587 100644 --- a/trail/src/db/core/backup/restore.rs +++ b/trail/src/db/core/backup/restore.rs @@ -54,7 +54,7 @@ impl Trail { sibling_stage(&db_dir, "restore-stage"), )?; - let restore_result = (|| -> Result<(u64, crate::model::FsckReport)> { + let restore_result = (|| -> Result<(u64, u64, crate::model::FsckReport)> { restore_step( "create restore stage structure", (|| -> Result<()> { @@ -76,6 +76,7 @@ impl Trail { &backup_path.join("worktrees"), &temp_dir.join("worktrees"), )?; + copy_dir_recursive(&backup_path.join("views"), &temp_dir.join("views"))?; secure_restored_db_dir(&temp_dir) })(), )?; @@ -111,6 +112,9 @@ impl Trail { )?; rewritten }; + let restored_private_views: u64 = + db.conn + .query_row("SELECT COUNT(*) FROM workspace_views", [], |row| row.get(0))?; test_crash_point("restore_after_staged_workdir_rewrite"); restore_step("recover restore stage", db.recover_after_open())?; test_crash_point("restore_after_staged_recovery"); @@ -121,6 +125,10 @@ impl Trail { fsck.errors.join("; ") ))); } + restore_step( + "finalize restored workspace view paths", + db.finalize_restored_workspace_view_paths(), + )?; let checkpoint_busy: i64 = db.conn .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |row| row.get(0))?; @@ -133,9 +141,9 @@ impl Trail { test_crash_point("restore_after_staged_checkpoint"); restore_step("sync restore stage", sync_tree_bottom_up(&temp_dir))?; test_crash_point("restore_after_staged_sync"); - Ok((rewritten_workdirs, fsck)) + Ok((rewritten_workdirs, restored_private_views, fsck)) })(); - let (rewritten_workdirs, fsck) = match restore_result { + let (rewritten_workdirs, restored_private_views, fsck) = match restore_result { Ok(prepared) => prepared, Err(err) => { let _ = remove_any(&temp_dir); @@ -162,6 +170,11 @@ impl Trail { replaced_existing, restored_trailignore, rewritten_workdirs, + restored_private_views, + retained_private_bytes: manifest.retained_private_bytes, + rebuildable_materializations: manifest.rebuildable_materializations, + rebuildable_materialization_bytes: manifest.rebuildable_materialization_bytes, + rebuildable_performance_caches: manifest.rebuildable_performance_caches, checked_refs: fsck.checked_refs, checked_roots: fsck.checked_roots, checked_texts: fsck.checked_texts, diff --git a/trail/src/db/core/backup/verify.rs b/trail/src/db/core/backup/verify.rs index efa9e1c8..360b9c0f 100644 --- a/trail/src/db/core/backup/verify.rs +++ b/trail/src/db/core/backup/verify.rs @@ -14,6 +14,11 @@ impl Trail { let mut checked_texts = 0; let mut sqlite_bytes = None; let mut sqlite_sha256 = None; + let mut retained_private_views = 0; + let mut retained_private_bytes = 0; + let mut rebuildable_materializations = 0; + let mut rebuildable_materialization_bytes = 0; + let mut rebuildable_performance_caches = 0; let manifest = match read_backup_manifest(&path) { Ok(manifest) => { @@ -25,6 +30,11 @@ impl Trail { } workspace_id = Some(manifest.workspace_id.clone()); branch = Some(manifest.branch.clone()); + retained_private_views = manifest.retained_private_views; + retained_private_bytes = manifest.retained_private_bytes; + rebuildable_materializations = manifest.rebuildable_materializations; + rebuildable_materialization_bytes = manifest.rebuildable_materialization_bytes; + rebuildable_performance_caches = manifest.rebuildable_performance_caches; Some(manifest) } Err(err) => { @@ -63,6 +73,41 @@ impl Trail { errors.push(format!("missing SQLite store `{}`", DB_RELATIVE_PATH)); } + if let Some(manifest) = &manifest + && !manifest.retained_private_sha256.is_empty() + { + match portable_tree_digest(&path.join("views")) { + Ok((bytes, digest)) => { + if bytes != manifest.retained_private_bytes { + errors.push(format!( + "retained private byte size mismatch: manifest {}, actual {bytes}", + manifest.retained_private_bytes + )); + } + if digest != manifest.retained_private_sha256 { + errors.push("retained private SHA-256 mismatch".to_string()); + } + let view_count = fs::read_dir(path.join("views")) + .map(|entries| { + entries + .filter_map(std::result::Result::ok) + .filter(|entry| entry.path().is_dir()) + .count() as u64 + }) + .unwrap_or(0); + if view_count != manifest.retained_private_views { + errors.push(format!( + "retained private view count mismatch: manifest {}, actual {view_count}", + manifest.retained_private_views + )); + } + } + Err(error) => { + errors.push(format!("could not verify retained private state: {error}")) + } + } + } + if path.join(CONFIG_FILE).is_file() && path.join(HEAD_FILE).is_file() && sqlite_path.is_file() @@ -82,37 +127,49 @@ impl Trail { fs::copy(path.join(CONFIG_FILE), verify_dir.join(CONFIG_FILE))?; fs::copy(path.join(HEAD_FILE), verify_dir.join(HEAD_FILE))?; fs::copy(&sqlite_path, verify_dir.join(DB_RELATIVE_PATH))?; + copy_dir_recursive(&path.join("views"), &verify_dir.join("views"))?; super::open_staged_copy(&verify_dir, &verify_dir) })(); match verify_open { - Ok(db) => match db.fsck() { - Ok(fsck) => { - checked_refs = fsck.checked_refs; - checked_roots = fsck.checked_roots; - checked_texts = fsck.checked_texts; - errors.extend(fsck.errors); - workspace_id.get_or_insert_with(|| db.config.workspace.id.clone()); - branch.get_or_insert(db.current_branch()?); - let trusted_scopes: i64 = db.conn.query_row( - "SELECT COUNT(*) FROM changed_path_scopes + Ok(mut db) => { + let rewrite = (|| -> Result<()> { + let _lock = db.acquire_write_lock()?; + db.rewrite_restored_lane_workdir_paths()?; + Ok(()) + })(); + if let Err(err) = rewrite { + errors.push(format!("could not prepare portable backup view: {err}")); + } + match db.fsck() { + Ok(fsck) => { + checked_refs = fsck.checked_refs; + checked_roots = fsck.checked_roots; + checked_texts = fsck.checked_texts; + errors.extend(fsck.errors); + workspace_id.get_or_insert_with(|| db.config.workspace.id.clone()); + branch.get_or_insert(db.current_branch()?); + let trusted_scopes: i64 = db.conn.query_row( + "SELECT COUNT(*) FROM changed_path_scopes WHERE retired_at IS NULL AND trust_state='trusted'", - [], - |row| row.get(0), - )?; - let live_observers: i64 = db.conn.query_row( - "SELECT (SELECT COUNT(*) FROM changed_path_observer_owners) + [], + |row| row.get(0), + )?; + let live_observers: i64 = db.conn.query_row( + "SELECT (SELECT COUNT(*) FROM changed_path_observer_owners) + (SELECT COUNT(*) FROM changed_path_observer_segments)", - [], - |row| row.get(0), - )?; - if trusted_scopes != 0 || live_observers != 0 { - errors.push( - "changed-path backup is not fenced or marked untrusted".to_string(), - ); + [], + |row| row.get(0), + )?; + if trusted_scopes != 0 || live_observers != 0 { + errors.push( + "changed-path backup is not fenced or marked untrusted" + .to_string(), + ); + } } + Err(err) => errors.push(format!("fsck failed: {err}")), } - Err(err) => errors.push(format!("fsck failed: {err}")), - }, + } Err(err) => errors.push(format!("could not open backup store: {err}")), } let _ = fs::remove_dir_all(&verify_dir); @@ -128,6 +185,11 @@ impl Trail { checked_texts, sqlite_bytes, sqlite_sha256, + retained_private_views, + retained_private_bytes, + rebuildable_materializations, + rebuildable_materialization_bytes, + rebuildable_performance_caches, errors, }) } diff --git a/trail/src/db/core/doctor_storage.rs b/trail/src/db/core/doctor_storage.rs index 7850472d..9e2de798 100644 --- a/trail/src/db/core/doctor_storage.rs +++ b/trail/src/db/core/doctor_storage.rs @@ -257,6 +257,8 @@ pub(super) fn push_workspace_views_check(db: &Trail, checks: &mut Vec, _>>()? }; + let views = { + let mut stmt = self.conn.prepare( + "SELECT v.view_id,v.lane_id,a.name,v.checkpoint_seq,v.generation + FROM workspace_views v JOIN lanes a ON a.lane_id=v.lane_id + ORDER BY v.view_id", + )?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?.max(0) as u64, + row.get::<_, i64>(4)?.max(0) as u64, + )) + })?; + rows.collect::, _>>()? + }; self.conn.execute_batch("BEGIN IMMEDIATE;")?; let rewrite = (|| -> Result { @@ -38,17 +55,14 @@ impl Trail { rewritten += 1; } - // Backups do not contain `.trail/views`. Invalidate every view and - // its derived environment state before normal open recovery can - // inspect source-workspace absolute paths from the copied SQLite - // store. Backups also exclude performance cache bytes, so retain - // publication attempts as non-attachable recovery provenance and - // discard layer/cache authorities. A lane can create a fresh view - // and generation in the restored workspace. + // Cache and runtime projections are host-local. Preserve durable + // artifact objects, snapshots, envelopes, attestations, historical + // generations, and exact generation bindings, but retire active + // pointers and remove every omitted projection before recovery. self.conn.execute_batch( "UPDATE workspace_layer_publications SET phase='recovered', successor_generation_id=NULL, - manifest_object_id=NULL, layer_id=NULL, + layer_id=NULL, error_code='backup_restore_cache_invalidated', error_message='backup restore excluded workspace cache bytes', updated_at=unixepoch(), finished_at=unixepoch() @@ -66,14 +80,12 @@ impl Trail { WHERE generation_id IN (SELECT generation_id FROM environment_generations); DELETE FROM environment_generation_caches WHERE generation_id IN (SELECT generation_id FROM environment_generations); - DELETE FROM environment_generation_edges - WHERE generation_id IN (SELECT generation_id FROM environment_generations); - DELETE FROM environment_generation_outputs - WHERE generation_id IN (SELECT generation_id FROM environment_generations); - DELETE FROM environment_generation_components - WHERE generation_id IN (SELECT generation_id FROM environment_generations); + UPDATE environment_generation_outputs SET layer_id=NULL; + UPDATE environment_generation_components SET layer_id=NULL; DELETE FROM environment_view_generations; - DELETE FROM environment_generations; + UPDATE environment_generations + SET state='retired', retired_at=COALESCE(retired_at,unixepoch()) + WHERE state='active'; DELETE FROM environment_sync_attempts; DELETE FROM environment_component_runtime_secrets; DELETE FROM environment_component_runtime_resources; @@ -85,10 +97,62 @@ impl Trail { DELETE FROM environment_component_states; DELETE FROM workspace_environment_states; DELETE FROM workspace_view_layers; + DELETE FROM workspace_layer_artifact_shadows; + DELETE FROM artifact_materializations; + DELETE FROM environment_cache_namespaces; DELETE FROM workspace_layers; - DELETE FROM workspace_git_shadows; - DELETE FROM workspace_views;", + DELETE FROM workspace_git_shadows;", )?; + + for (view_id, lane_id, lane_name, checkpoint_seq, generation) in views { + let mut components = Path::new(&view_id).components(); + let confined = matches!(components.next(), Some(Component::Normal(_))) + && components.next().is_none(); + let staged_view = self.db_dir.join("views").join(&view_id); + let staged_source = staged_view.join("source-upper"); + let staged_meta = staged_view.join("meta"); + if !confined || !staged_source.is_dir() || !staged_meta.is_dir() { + self.conn.execute( + "DELETE FROM workspace_views WHERE view_id=?1", + params![view_id], + )?; + continue; + } + + let staged_generated = staged_view.join("generated-upper"); + let staged_scratch = staged_view.join("scratch-upper"); + fs::create_dir_all(&staged_generated)?; + fs::create_dir_all(&staged_scratch)?; + scrub_restored_view_metadata(&staged_meta)?; + let mut barrier = ViewMutationBarrier::exclusive(&staged_meta)?; + barrier.record_checkpoint_cut(checkpoint_seq, generation)?; + + let staged_database_view = self.db_dir.join("views").join(&view_id); + let mountpoint = self.default_lane_workdir_path(&lane_name)?; + self.conn.execute( + "UPDATE workspace_views + SET mountpoint=?1,source_upper=?2,generated_upper=?3,scratch_upper=?4, + meta_dir=?5,journal_path=?6,status='recovered',owner_pid=NULL, + owner_start_token=NULL,heartbeat_at=NULL,updated_at=?7 + WHERE view_id=?8 AND lane_id=?9", + params![ + mountpoint.to_string_lossy(), + staged_database_view.join("source-upper").to_string_lossy(), + staged_database_view + .join("generated-upper") + .to_string_lossy(), + staged_database_view.join("scratch-upper").to_string_lossy(), + staged_database_view.join("meta").to_string_lossy(), + staged_database_view + .join("meta") + .join("mutation-journal.jsonl") + .to_string_lossy(), + now_ts(), + view_id, + lane_id, + ], + )?; + } Ok(rewritten) })(); match rewrite { @@ -106,6 +170,50 @@ impl Trail { } } + pub(crate) fn finalize_restored_workspace_view_paths(&mut self) -> Result<()> { + let view_ids = { + let mut statement = self + .conn + .prepare("SELECT view_id FROM workspace_views ORDER BY view_id")?; + statement + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()? + }; + let final_db_dir = self.workspace_root.join(".trail"); + self.conn.execute_batch("BEGIN IMMEDIATE;")?; + let update = (|| -> Result<()> { + for view_id in view_ids { + let final_view = final_db_dir.join("views").join(&view_id); + self.conn.execute( + "UPDATE workspace_views + SET source_upper=?1,generated_upper=?2,scratch_upper=?3, + meta_dir=?4,journal_path=?5,updated_at=?6 + WHERE view_id=?7", + params![ + final_view.join("source-upper").to_string_lossy(), + final_view.join("generated-upper").to_string_lossy(), + final_view.join("scratch-upper").to_string_lossy(), + final_view.join("meta").to_string_lossy(), + final_view + .join("meta") + .join("mutation-journal.jsonl") + .to_string_lossy(), + now_ts(), + view_id, + ], + )?; + } + Ok(()) + })(); + match update { + Ok(()) => self.conn.execute_batch("COMMIT;").map_err(Error::from), + Err(error) => { + let _ = self.conn.execute_batch("ROLLBACK;"); + Err(error) + } + } + } + pub fn lane_details(&self, lane: &str) -> Result { match self.lane_branch(lane) { Ok(branch) => { @@ -244,3 +352,26 @@ impl Trail { }) } } + +fn scrub_restored_view_metadata(meta_dir: &Path) -> Result<()> { + for entry in fs::read_dir(meta_dir)? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with("checkpoint-barrier.") + || matches!( + name.as_ref(), + "mount.json" | "unmount-request.json" | "view.json" + ) + { + let path = entry.path(); + let metadata = fs::symlink_metadata(&path)?; + if metadata.is_dir() && !metadata.file_type().is_symlink() { + fs::remove_dir_all(path)?; + } else { + fs::remove_file(path)?; + } + } + } + Ok(()) +} diff --git a/trail/src/db/lane/managed_execution.rs b/trail/src/db/lane/managed_execution.rs index ac35e678..7bedd420 100644 --- a/trail/src/db/lane/managed_execution.rs +++ b/trail/src/db/lane/managed_execution.rs @@ -22,6 +22,8 @@ pub struct ManagedExecutionContext { pub workdir: PathBuf, pub environment: Vec<(String, String)>, pub environment_generation: Option, + preparation: ManagedExecutionPreparationReceipt, + sealing_decisions: Vec, mount: Option>, phases: Vec, #[cfg(test)] @@ -133,6 +135,38 @@ impl Trail { return Err(error); } }; + let resolution_pins = self.managed_execution_resolution_pins(&discovered)?; + if let Some(unresolved) = resolution_pins + .iter() + .find(|pin| pin.status != EnvironmentComponentProposalStatus::Ready) + { + let recovery = unresolved + .recovery_command + .as_ref() + .map(|command| command.join(" ")) + .unwrap_or_else(|| format!("trail env discover {lane}")); + let error = Error::InvalidInput(format!( + "managed execution requires explicit resolution for environment component `{}` ({}); run `{recovery}`", + unresolved.component_id, + unresolved.status.as_str() + )); + self.push_managed_execution_phase( + &mut phases, + &branch.lane_id, + &execution_id, + surface, + &command_fingerprint, + "discover_plan", + "failed", + Some(&error.to_string()), + Some(serde_json::json!({ + "missing_resolution_policy": ManagedExecutionMissingResolutionPolicy::Explicit, + "resolution_pins": resolution_pins, + "recovery_command": unresolved.recovery_command, + })), + )?; + return Err(error); + } if view.is_none() && !discovered.components.is_empty() { self.push_managed_execution_phase( &mut phases, @@ -207,6 +241,8 @@ impl Trail { "component_count": discovered.components.len(), "graph_nodes": graph.total_nodes, "graph_edges": graph.total_edges, + "missing_resolution_policy": ManagedExecutionMissingResolutionPolicy::Explicit, + "resolution_pins": resolution_pins, })), )?; @@ -285,6 +321,16 @@ impl Trail { } else { None }; + let (output_pins, sealing_decisions) = if let Some(generation) = &active_generation { + let bindings = + self.artifact_generation_bindings_for_generation(&generation.generation_id)?; + ( + managed_execution_output_pins(generation, &bindings)?, + managed_execution_sealing_decisions(generation), + ) + } else { + (Vec::new(), Vec::new()) + }; if let (Some(view), Some(generation)) = (&view, &active_generation) { let cancelled = AtomicBool::new(false); let prefetch = @@ -514,11 +560,25 @@ impl Trail { lane: lane.to_string(), lane_id: branch.lane_id, command_fingerprint, - source_root: head.root_id, - view, + source_root: head.root_id.clone(), + view: view.clone(), workdir, environment, - environment_generation: active_generation.map(|generation| generation.generation_id), + environment_generation: active_generation + .as_ref() + .map(|generation| generation.generation_id.clone()), + preparation: ManagedExecutionPreparationReceipt { + source_root: head.root_id.clone(), + view_id: view.as_ref().map(|view| view.view_id.clone()), + view_generation: view.as_ref().map(|view| view.generation), + missing_resolution_policy: ManagedExecutionMissingResolutionPolicy::Explicit, + resolution_pins, + environment_generation: active_generation + .as_ref() + .map(|generation| generation.generation_id.clone()), + output_pins, + }, + sealing_decisions, mount, phases, #[cfg(test)] @@ -701,16 +761,65 @@ impl Trail { None, ); + let source_root_after = checkpoint + .as_ref() + .map(|checkpoint| checkpoint.root_id.clone()); + let source_changed = source_root_after + .as_ref() + .is_some_and(|root| root != &context.source_root); + let mut sealing_decisions = context.sealing_decisions.clone(); + if source_changed { + for decision in &mut sealing_decisions { + if decision.decision.starts_with("await_") { + decision.decision = "replan_required".to_string(); + decision.reason = + "identity-bearing lane source changed during execution; replan before sealing" + .to_string(); + } + } + } + let checkpoint_status = if checkpoint_error.is_some() { + "failed" + } else { + "succeeded" + }; + let disposal_status = if disposal_error.is_some() { + "failed" + } else if has_runtime { + "succeeded" + } else { + "skipped" + }; + let unmount_status = if had_mount { "succeeded" } else { "skipped" }; + let errors = checkpoint_error + .iter() + .chain(disposal_error.iter()) + .cloned() + .collect::>(); + let finalization = ManagedExecutionFinalizationReceipt { + source_root_before: context.source_root.clone(), + source_root_after, + source_changed, + checkpoint_status: checkpoint_status.to_string(), + disposal_status: disposal_status.to_string(), + unmount_status: unmount_status.to_string(), + complete: errors.is_empty(), + sealing_decisions, + errors, + }; + ManagedExecutionLifecycleReport { execution_id: context.execution_id, surface: context.surface, command_fingerprint: context.command_fingerprint, + preparation: Some(context.preparation), environment_generation: context.environment_generation, checkpoint, checkpoint_error, checkpoint_error_code, disposal_error, recorded, + finalization: Some(finalization), phases: context.phases, } } @@ -1146,6 +1255,37 @@ impl Trail { } Ok(report) } + + fn managed_execution_resolution_pins( + &self, + discovery: &EnvironmentDiscoveryReport, + ) -> Result> { + discovery + .components + .iter() + .map(|component| { + let snapshot = self.artifact_resolution_snapshot_for_component( + &discovery.source_root, + &component.component_id, + &component.adapter_identity, + )?; + let (snapshot_id, proposal_key) = snapshot + .map(|(snapshot_id, snapshot)| (Some(snapshot_id), Some(snapshot.proposal_key))) + .unwrap_or((None, None)); + Ok(ManagedExecutionResolutionPin { + component_id: component.component_id.clone(), + adapter_identity: component.adapter_identity.clone(), + status: component.status.clone(), + proposal_key, + snapshot_id, + recovery_command: component + .recovery_actions + .iter() + .find_map(|action| action.command.clone()), + }) + }) + .collect() + } } fn managed_execution_id(lane: &str, surface: &str, command: &[String]) -> Result { @@ -1177,6 +1317,105 @@ fn managed_environment_is_current( }) } +fn managed_execution_output_pins( + generation: &EnvironmentGenerationReport, + bindings: &[ArtifactGenerationBindingReportV1], +) -> Result> { + let bindings = bindings + .iter() + .map(|binding| { + ( + (binding.component_id.as_str(), binding.output_name.as_str()), + binding, + ) + }) + .collect::>(); + let mut pins = Vec::new(); + for component in &generation.components { + for output in &component.outputs { + let binding = bindings + .get(&(component.component_id.as_str(), output.name.as_str())) + .copied(); + if output.policy.has_immutable_layer() && binding.is_none() { + return Err(Error::Corrupt(format!( + "managed execution cannot pin immutable output `{}/{}` because generation `{}` has no artifact binding", + component.component_id, output.name, generation.generation_id + ))); + } + if let Some(binding) = binding + && binding.desired_key != component.component_key + { + return Err(Error::Corrupt(format!( + "managed execution artifact binding for `{}/{}` disagrees with generation component identity", + component.component_id, output.name + ))); + } + pins.push(ManagedExecutionOutputPin { + component_id: component.component_id.clone(), + output_name: output.name.clone(), + component_key: component.component_key.clone(), + policy: output.policy, + storage_identity: output.storage_identity.clone(), + artifact_binding_id: binding.map(|binding| binding.binding_id.clone()), + artifact_envelope_id: binding.map(|binding| binding.envelope_id.clone()), + artifact_tree_root_id: binding.map(|binding| binding.tree_root_id.clone()), + artifact_binding_identity: binding.map(|binding| binding.binding_identity.clone()), + }); + } + } + Ok(pins) +} + +fn managed_execution_sealing_decisions( + generation: &EnvironmentGenerationReport, +) -> Vec { + generation + .components + .iter() + .flat_map(|component| { + component.outputs.iter().map(|output| { + let (decision, reason) = match (output.policy, output.publish) { + (EnvironmentOutputPolicy::Disposable, _) => { + ("dispose", "disposable output is never sealed or promoted") + } + (_, EnvironmentPublicationTrigger::SuccessfulGate) => ( + "await_successful_gate", + "seal only after the named successful gate revalidates its pins", + ), + (_, EnvironmentPublicationTrigger::OnSync) => ( + "await_sync", + "seal only during a later environment synchronization", + ), + (_, EnvironmentPublicationTrigger::Manual) => ( + "await_manual_promotion", + "retain private changes until explicit promotion", + ), + (EnvironmentOutputPolicy::ImmutableShared, _) => ( + "preserve_verified_artifact", + "mounted immutable content is already sealed and cannot be modified", + ), + (EnvironmentOutputPolicy::ImmutableSeedPrivate, _) => ( + "retain_private_delta", + "writes remain in the lane-private upper", + ), + (EnvironmentOutputPolicy::WritablePrivate, _) => { + ("retain_private", "writable output remains lane-private") + } + }; + ManagedExecutionSealingDecision { + component_id: component.component_id.clone(), + output_name: output.name.clone(), + policy: output.policy, + publication: output.publish, + gate: output.gate.clone(), + decision: decision.to_string(), + reason: reason.to_string(), + } + }) + }) + .collect() +} + fn workspace_checkpoint_from_lane_record(record: LaneRecordReport) -> WorkspaceCheckpointReport { WorkspaceCheckpointReport { view_id: String::new(), @@ -1197,6 +1436,7 @@ fn workspace_checkpoint_from_lane_record(record: LaneRecordReport) -> WorkspaceC #[cfg(test)] mod tests { use super::*; + use crate::{ArtifactEnvelopeId, ArtifactTreeId}; fn hot_generation(manifest: &str) -> EnvironmentGenerationReport { EnvironmentGenerationReport { @@ -1425,5 +1665,139 @@ mod tests { .is_some_and(|error| error.contains("injected cleanup failure")) })); assert!(report.phases.iter().any(|phase| phase.phase == "unmount")); + let preparation = report.preparation.as_ref().unwrap(); + assert_eq!( + preparation.missing_resolution_policy, + ManagedExecutionMissingResolutionPolicy::Explicit + ); + assert!(preparation.resolution_pins.is_empty()); + assert!(preparation.output_pins.is_empty()); + let finalization = report.finalization.as_ref().unwrap(); + assert!(!finalization.complete); + assert_eq!(finalization.disposal_status, "failed"); + assert!(finalization + .errors + .iter() + .any(|error| error.contains("injected cleanup failure"))); + } + + #[test] + fn managed_preparation_requires_explicit_resolution_with_exact_recovery() { + let root = tempfile::tempdir().unwrap(); + std::fs::write( + root.path().join("Cargo.toml"), + "[package]\nname='managed-resolution'\nversion='0.1.0'\nedition='2024'\n", + ) + .unwrap(); + Trail::init(root.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(root.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "needs-resolution", + Some("main"), + LaneWorkdirMode::PortableCopy, + None, + None, + None, + &[], + false, + ) + .unwrap(); + + let error = db + .prepare_managed_lane_execution( + "needs-resolution", + "lane_exec", + &["cargo".into(), "test".into()], + ) + .err() + .unwrap(); + assert!(error.to_string().contains( + "managed execution requires explicit resolution for environment component `cargo-target-seed` (resolvable)" + )); + assert!(error + .to_string() + .contains("trail env resolve component cargo-target-seed --lane needs-resolution")); + } + + #[test] + fn managed_output_pins_and_sealing_decisions_are_exact_and_deterministic() { + let envelope_id = + ArtifactEnvelopeId::parse(format!("artifact_envelope_{}", "a".repeat(64))).unwrap(); + let tree_root_id = + ArtifactTreeId::parse(format!("artifact_tree_{}", "b".repeat(64))).unwrap(); + let generation = EnvironmentGenerationReport { + generation_id: "envgen-test".into(), + view_id: "view-test".into(), + generation_sequence: 1, + source_root: ObjectId("source-root".into()), + specification_digest: "specification".into(), + predecessor_generation_id: None, + state: "active".into(), + components: vec![EnvironmentGenerationComponentReport { + component_id: "fixture".into(), + adapter_identity: "trail/fixture@1".into(), + kind: "generated".into(), + component_key: "desired-key".into(), + layer_id: Some("layer-test".into()), + mount_path: Some("generated".into()), + dependencies: Vec::new(), + outputs: vec![ + EnvironmentGenerationOutputReport { + name: "shared".into(), + policy: EnvironmentOutputPolicy::ImmutableShared, + reuse: EnvironmentReuseMode::Exact, + scope: EnvironmentSharingScope::Workspace, + publish: EnvironmentPublicationTrigger::Never, + gate: None, + storage_identity: "layer-test".into(), + layer_id: Some("layer-test".into()), + manifest_object_id: Some("manifest-test".into()), + publication_id: None, + mount_path: "generated".into(), + layer_subpath: String::new(), + }, + EnvironmentGenerationOutputReport { + name: "private".into(), + policy: EnvironmentOutputPolicy::WritablePrivate, + reuse: EnvironmentReuseMode::None, + scope: EnvironmentSharingScope::Lane, + publish: EnvironmentPublicationTrigger::Manual, + gate: None, + storage_identity: "private-test".into(), + layer_id: None, + manifest_object_id: None, + publication_id: None, + mount_path: "private".into(), + layer_subpath: String::new(), + }, + ], + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + }], + created_at: 1, + activated_at: Some(1), + retired_at: None, + }; + let bindings = vec![ArtifactGenerationBindingReportV1 { + binding_id: "binding-test".into(), + generation_id: generation.generation_id.clone(), + component_id: "fixture".into(), + output_name: "shared".into(), + desired_key: "desired-key".into(), + envelope_id: envelope_id.clone(), + tree_root_id: tree_root_id.clone(), + binding_identity: "artifact-binding-test".into(), + created_at: 1, + }]; + + let pins = managed_execution_output_pins(&generation, &bindings).unwrap(); + assert_eq!(pins.len(), 2); + assert_eq!(pins[0].artifact_envelope_id, Some(envelope_id)); + assert_eq!(pins[0].artifact_tree_root_id, Some(tree_root_id)); + assert!(pins[1].artifact_envelope_id.is_none()); + let decisions = managed_execution_sealing_decisions(&generation); + assert_eq!(decisions[0].decision, "preserve_verified_artifact"); + assert_eq!(decisions[1].decision, "await_manual_promotion"); } } diff --git a/trail/src/db/lane/mod.rs b/trail/src/db/lane/mod.rs index 20b84507..b77538fd 100644 --- a/trail/src/db/lane/mod.rs +++ b/trail/src/db/lane/mod.rs @@ -25,8 +25,15 @@ mod patching; mod readiness; mod retirement; mod rewind; +mod source_export; mod turns; mod workdir; +// Phase-one artifact contracts are intentionally reachable only from their +// qualification tests until explicit resolve and CAS publication operations +// activate them in later OpenSpec tasks. +#[allow(dead_code)] +mod workspace_artifact; +pub(crate) use workspace_artifact::validate_artifact_validation_receipt; mod workspace_cargo; mod workspace_cmake; mod workspace_environment; diff --git a/trail/src/db/lane/retirement.rs b/trail/src/db/lane/retirement.rs index 599bd847..f94fec6e 100644 --- a/trail/src/db/lane/retirement.rs +++ b/trail/src/db/lane/retirement.rs @@ -480,6 +480,9 @@ impl Trail { ) -> Result<()> { if let Some(view) = view { for statement in [ + "DELETE FROM artifact_generation_bindings + WHERE generation_id IN ( + SELECT generation_id FROM environment_generations WHERE view_id=?1)", "DELETE FROM environment_secret_access_audit WHERE generation_id IN ( SELECT generation_id FROM environment_generations WHERE view_id=?1)", diff --git a/trail/src/db/lane/source_export.rs b/trail/src/db/lane/source_export.rs new file mode 100644 index 00000000..9003956c --- /dev/null +++ b/trail/src/db/lane/source_export.rs @@ -0,0 +1,869 @@ +use super::workspace_artifact::ArtifactLazyEntry; +use super::*; +use crate::ids::{ArtifactEnvelopeId, ArtifactTreeId}; + +const ARTIFACT_SOURCE_EXPORT_PLAN_VERSION: u16 = 1; +const MAX_SOURCE_EXPORT_FILES: usize = 10_000; +const MAX_SOURCE_EXPORT_FILE_BYTES: u64 = 16 * 1024 * 1024; +const MAX_SOURCE_EXPORT_TOTAL_BYTES: u64 = 64 * 1024 * 1024; + +impl Trail { + /// Plan one explicit generated-source export without writing source. + /// + /// The returned evidence pins every authority consumed by execution. A + /// later writer must reject the plan if any source, generation, artifact, + /// validation, gate, or destination pin changes. + pub fn plan_artifact_source_export( + &self, + lane: &str, + component_id: &str, + export_name: &str, + authorization: ArtifactSourceExportAuthorizationV1, + ) -> Result { + let lane_details = self.lane_details(lane)?; + let branch = &lane_details.branch; + let generation = self + .active_environment_generation(&lane_details.record.name)? + .ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{}` has no active environment generation to export", + lane_details.record.name + )) + })?; + if generation.state != "active" { + return Err(Error::InvalidInput(format!( + "environment generation `{}` is not active", + generation.generation_id + ))); + } + let contract = self + .command_recipe_source_exports(&generation.source_root, component_id)? + .into_iter() + .find(|contract| contract.name == export_name) + .ok_or_else(|| { + Error::InvalidInput(format!( + "repository component `{component_id}` has no source export named `{export_name}`" + )) + })?; + if contract.authorization_mode != "explicit" + || authorization != ArtifactSourceExportAuthorizationV1::ExplicitUser + { + return Err(Error::InvalidInput(format!( + "source export `{export_name}` requires explicit user authorization" + ))); + } + let collision_mode = match contract.collision_policy.as_str() { + "fail" => ArtifactSourceExportCollisionModeV1::Fail, + "replace" => ArtifactSourceExportCollisionModeV1::Replace, + other => { + return Err(Error::Corrupt(format!( + "source export `{export_name}` retained unsupported collision mode `{other}`" + ))) + } + }; + + let binding = self + .conn + .query_row( + "SELECT b.desired_key,b.envelope_id,b.tree_root_id,o.layer_subpath + FROM artifact_generation_bindings b + JOIN environment_generation_outputs o + ON o.generation_id=b.generation_id + AND o.component_id=b.component_id + AND o.output_name=b.output_name + WHERE b.generation_id=?1 AND b.component_id=?2 AND b.output_name=?3", + params![ + &generation.generation_id, + component_id, + &contract.output_name + ], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + }, + ) + .optional()? + .ok_or_else(|| { + Error::InvalidInput(format!( + "active generation `{}` has no sealed artifact binding for `{component_id}/{}`", + generation.generation_id, contract.output_name + )) + })?; + let (binding_desired_key, envelope_id, tree_root_id, layer_subpath) = binding; + let envelope_id = ArtifactEnvelopeId::parse(envelope_id).map_err(|error| { + Error::Corrupt(format!("invalid source-export envelope ID: {error}")) + })?; + let tree_root_id = ArtifactTreeId::parse(tree_root_id).map_err(|error| { + Error::Corrupt(format!("invalid source-export tree-root ID: {error}")) + })?; + let envelope = + self.verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_root_id)?; + if !envelope.secret_taint.is_clear() { + return Err(Error::InvalidInput(format!( + "source export `{export_name}` cannot use secret-tainted artifact output" + ))); + } + let encoded_desired_key = match &envelope.desired_identity { + ArtifactDesiredIdentityV1::WorkspaceLayerV1 { cache_key, .. } => cache_key, + ArtifactDesiredIdentityV1::ArtifactDesiredV2 { desired_key } => &desired_key.0, + }; + if encoded_desired_key != &binding_desired_key { + return Err(Error::Corrupt(format!( + "source-export artifact binding for `{component_id}/{}` disagrees with envelope desired identity", + contract.output_name + ))); + } + + let artifact_subpath = normalize_relative_path(&join_source_export_path( + &layer_subpath, + &contract.artifact_subpath, + ))?; + let subtree = match self.artifact_tree_lazy_entry(&tree_root_id, &artifact_subpath)? { + Some(ArtifactLazyEntry::Directory { node_id }) => { + ArtifactSourceExportSubtreeV1::Directory { node_id } + } + Some(ArtifactLazyEntry::File { node_id, .. }) => { + ArtifactSourceExportSubtreeV1::File { node_id } + } + Some(ArtifactLazyEntry::Symlink { .. }) => { + return Err(Error::InvalidInput(format!( + "source export `{export_name}` cannot select a symlink as its artifact root" + ))) + } + None => { + return Err(Error::InvalidInput(format!( + "source export `{export_name}` artifact subtree `{}` does not exist", + contract.artifact_subpath + ))) + } + }; + + let mut matching_validation_receipts = Vec::new(); + for receipt_id in &envelope.validation_receipt_ids { + let receipt = self.artifact_validation_receipt(receipt_id)?; + if receipt.declaration.name == contract.required_validation + && receipt.outcome == ArtifactValidationOutcomeV1::Passed + && receipt.desired_identity == envelope.desired_identity + && receipt.tree_root_id == tree_root_id + { + matching_validation_receipts.push(receipt_id.clone()); + } + } + if matching_validation_receipts.len() != 1 { + return Err(Error::InvalidInput(format!( + "source export `{export_name}` requires exactly one passed validation `{}` for the exact artifact", + contract.required_validation + ))); + } + let validation_receipt_id = matching_validation_receipts.remove(0); + + let gate = contract + .required_gate + .as_deref() + .map(|kind| { + let gate = self + .latest_lane_gate(&branch.lane_id, kind)? + .ok_or_else(|| { + Error::InvalidInput(format!( + "source export `{export_name}` requires a recorded `{kind}` gate" + )) + })?; + if !gate.success + || gate.source_root.as_ref() != Some(&branch.head_root) + || gate.view_id.as_deref() != Some(&generation.view_id) + { + return Err(Error::InvalidInput(format!( + "source export `{export_name}` requires a successful `{kind}` gate pinned to the current source and environment view" + ))); + } + Ok(ArtifactSourceExportGatePinV1 { + event_id: gate.event_id, + kind: kind.to_string(), + source_root: branch.head_root.clone(), + view_id: generation.view_id.clone(), + view_generation: gate.view_generation.ok_or_else(|| { + Error::InvalidInput(format!( + "source export `{export_name}` gate `{kind}` lacks a view-generation pin" + )) + })?, + }) + }) + .transpose()?; + + let destination = normalize_relative_path(&contract.destination)?; + let destination_pin = + self.source_export_destination_pin(&branch.head_root, &destination)?; + if collision_mode == ArtifactSourceExportCollisionModeV1::Fail && destination_pin.exists { + return Err(Error::Conflict(format!( + "source export `{export_name}` destination `{destination}` already exists and collision mode is `fail`" + ))); + } + + Ok(ArtifactSourceExportPlanV1 { + version: ARTIFACT_SOURCE_EXPORT_PLAN_VERSION, + lane_id: branch.lane_id.clone(), + lane: lane_details.record.name, + component_id: component_id.to_string(), + export_name: export_name.to_string(), + output_name: contract.output_name, + source_root: branch.head_root.clone(), + generation_id: generation.generation_id, + generation_source_root: generation.source_root, + desired_identity: envelope.desired_identity, + envelope_id, + tree_root_id, + subtree, + artifact_subpath, + destination, + destination_pin, + collision_mode, + validation_receipt_id, + gate, + authorization, + }) + } + + fn source_export_destination_pin( + &self, + source_root: &ObjectId, + destination: &str, + ) -> Result { + let selected = + self.load_root_files_for_selections(source_root, &[destination.to_string()])?; + let logical_bytes = selected.values().try_fold(0u64, |total, entry| { + total.checked_add(entry.size_bytes).ok_or_else(|| { + Error::InvalidInput("source-export destination byte count overflowed".into()) + }) + })?; + let exists = !selected.is_empty(); + let content_digest = exists + .then(|| serde_json::to_vec(&selected).map(|bytes| sha256_hex(&bytes))) + .transpose()?; + Ok(ArtifactSourceExportDestinationPinV1 { + exists, + content_digest, + entry_count: selected.len() as u64, + logical_bytes, + }) + } + + /// Revalidate and apply a pinned export through Trail's ordinary structured + /// source patch path. Artifact storage is read directly; it is never + /// mounted over or aliased into lane source. + pub fn execute_artifact_source_export( + &mut self, + plan: ArtifactSourceExportPlanV1, + ) -> Result { + if plan.version != ARTIFACT_SOURCE_EXPORT_PLAN_VERSION + || plan.authorization != ArtifactSourceExportAuthorizationV1::ExplicitUser + { + return Err(Error::InvalidInput( + "source export plan version or authorization is invalid".into(), + )); + } + let current = self.plan_artifact_source_export( + &plan.lane, + &plan.component_id, + &plan.export_name, + plan.authorization, + )?; + if current != plan { + return Err(Error::InvalidInput(format!( + "source export plan for `{}/{}` is stale; replan against the current source, generation, artifact, validation, gate, and destination state", + plan.component_id, plan.export_name + ))); + } + + let exported_files = self.source_export_patch_files(&plan)?; + if exported_files.is_empty() { + return Err(Error::InvalidInput(format!( + "source export `{}` selected no regular files", + plan.export_name + ))); + } + let existing = self.load_root_files_for_selections( + &plan.source_root, + std::slice::from_ref(&plan.destination), + )?; + let mut edits = Vec::new(); + if plan.collision_mode == ArtifactSourceExportCollisionModeV1::Replace { + edits.extend( + existing + .keys() + .filter(|path| !exported_files.contains_key(*path)) + .cloned() + .map(|path| PatchEdit::Delete { path }), + ); + } + for (path, (bytes, executable)) in exported_files { + match String::from_utf8(bytes) { + Ok(content) if !content.contains('\0') => edits.push(PatchEdit::Write { + path, + content, + executable, + }), + Ok(content) => edits.push(PatchEdit::WriteBytes { + path, + bytes_hex: hex::encode(content.into_bytes()), + executable, + }), + Err(error) => edits.push(PatchEdit::WriteBytes { + path, + bytes_hex: hex::encode(error.into_bytes()), + executable, + }), + } + } + let branch = self.lane_branch(&plan.lane)?; + let patch = PatchDocument { + base_change: Some(branch.head_change.0), + message: Some(format!( + "Export {}/{} from artifact {}", + plan.component_id, plan.export_name, plan.envelope_id + )), + session_id: branch.session_id, + allow_ignored: false, + allow_stale: false, + edits, + }; + let applied = self.apply_lane_patch(&plan.lane, patch)?; + Ok(ArtifactSourceExportExecutionReportV1 { + plan: plan.clone(), + operation: applied.operation, + root_id: applied.root_id, + changed_paths: applied.changed_paths, + checkpointed: true, + git_handoff: vec![ + "trail".into(), + "lane".into(), + "merge".into(), + plan.lane, + "--into".into(), + self.config.workspace.default_branch.clone(), + "--dry-run".into(), + ], + }) + } + + fn source_export_patch_files( + &self, + plan: &ArtifactSourceExportPlanV1, + ) -> Result, bool)>> { + let root_entry = self + .artifact_tree_lazy_entry(&plan.tree_root_id, &plan.artifact_subpath)? + .ok_or_else(|| { + Error::InvalidInput(format!( + "source export `{}` artifact subtree disappeared", + plan.export_name + )) + })?; + let current_subtree = source_export_subtree_identity(&root_entry)?; + if current_subtree != plan.subtree { + return Err(Error::InvalidInput(format!( + "source export `{}` artifact subtree identity changed", + plan.export_name + ))); + } + + let mut pending = vec![(plan.artifact_subpath.clone(), String::new(), root_entry)]; + let mut files = BTreeMap::new(); + let mut total_bytes = 0u64; + while let Some((artifact_path, relative_path, entry)) = pending.pop() { + match entry { + ArtifactLazyEntry::Directory { .. } => { + let mut children = + self.artifact_tree_lazy_children(&plan.tree_root_id, &artifact_path)?; + children.reverse(); + for (name, child) in children { + let child_artifact_path = join_source_export_path(&artifact_path, &name); + let child_relative_path = if relative_path.is_empty() { + name + } else { + join_source_export_path(&relative_path, &name) + }; + pending.push((child_artifact_path, child_relative_path, child)); + } + } + ArtifactLazyEntry::File { + node_id, + mode, + size_bytes, + } => { + if files.len() >= MAX_SOURCE_EXPORT_FILES + || size_bytes > MAX_SOURCE_EXPORT_FILE_BYTES + { + return Err(Error::InvalidInput(format!( + "source export `{}` exceeds file count or per-file byte limits", + plan.export_name + ))); + } + total_bytes = total_bytes.checked_add(size_bytes).ok_or_else(|| { + Error::InvalidInput("source export byte count overflowed".into()) + })?; + if total_bytes > MAX_SOURCE_EXPORT_TOTAL_BYTES { + return Err(Error::InvalidInput(format!( + "source export `{}` exceeds the {MAX_SOURCE_EXPORT_TOTAL_BYTES}-byte limit", + plan.export_name + ))); + } + let count = u32::try_from(size_bytes).map_err(|_| { + Error::InvalidInput("source export file size exceeds read bounds".into()) + })?; + let bytes = self.artifact_file_read_range(&node_id, 0, count)?; + if bytes.len() as u64 != size_bytes { + return Err(Error::Corrupt(format!( + "source export file `{artifact_path}` was not read completely" + ))); + } + let destination = if relative_path.is_empty() { + plan.destination.clone() + } else { + join_source_export_path(&plan.destination, &relative_path) + }; + let destination = normalize_relative_path(&destination)?; + if files + .insert(destination.clone(), (bytes, mode & 0o111 != 0)) + .is_some() + { + return Err(Error::Corrupt(format!( + "source export produced duplicate destination `{destination}`" + ))); + } + } + ArtifactLazyEntry::Symlink { .. } => { + return Err(Error::InvalidInput(format!( + "source export `{}` contains a symlink; generated source exports currently accept regular files and directories only", + plan.export_name + ))); + } + } + } + Ok(files) + } +} + +fn source_export_subtree_identity( + entry: &ArtifactLazyEntry, +) -> Result { + match entry { + ArtifactLazyEntry::Directory { node_id } => Ok(ArtifactSourceExportSubtreeV1::Directory { + node_id: node_id.clone(), + }), + ArtifactLazyEntry::File { node_id, .. } => Ok(ArtifactSourceExportSubtreeV1::File { + node_id: node_id.clone(), + }), + ArtifactLazyEntry::Symlink { .. } => Err(Error::InvalidInput( + "source export cannot select a symlink root".into(), + )), + } +} + +fn join_source_export_path(parent: &str, child: &str) -> String { + if parent.is_empty() { + child.to_string() + } else { + format!("{parent}/{child}") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn setup_source_export_fixture( + collision: &str, + gate: Option<&str>, + existing_destination: bool, + ignore_destination: bool, + ) -> (tempfile::TempDir, Trail) { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("input.txt"), "identity\n").unwrap(); + if ignore_destination { + fs::write( + workspace.path().join(".trailignore"), + "src/generated-client/\n", + ) + .unwrap(); + } + if existing_destination { + fs::create_dir_all(workspace.path().join("src/generated-client")).unwrap(); + fs::write( + workspace.path().join("src/generated-client/old.rs"), + "old\n", + ) + .unwrap(); + } + let gate = gate + .map(|gate| format!("gate = {gate:?}\n")) + .unwrap_or_default(); + fs::write( + workspace.path().join("trail.environment.toml"), + format!( + r#"schema = "trail.environment/v2" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "custom.export" +adapter = "trail/command@1" +inputs = [{{ path = "input.txt" }}] + +[component.build] +command = ["cp", "input.txt", "generated/generated-client/new.rs"] + +[[component.output]] +name = "generated" +source = "generated" +target = ".trail-generated/export" +policy = "immutable_seed_private" +reuse = "exact" +scope = "workspace" +publish = "on_sync" + +[[component.source_export]] +name = "client" +from_output = "generated" +source = "generated-client" +target = "src/generated-client" +mode = "explicit" +collision = "{collision}" +{gate}"# + ), + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "export", + Some("main"), + mode, + None, + None, + None, + &[], + false, + ) + .unwrap(); + + let candidate = tempfile::tempdir().unwrap(); + fs::create_dir_all(candidate.path().join("outputs/0000/generated-client")).unwrap(); + fs::write( + candidate + .path() + .join("outputs/0000/generated-client/new.rs"), + "generated\n", + ) + .unwrap(); + let branch = db.lane_branch("export").unwrap(); + let view = db.lane_workspace_view("export").unwrap().unwrap(); + let layer_key = WorkspaceLayerKeyV1 { + kind: "generated".into(), + adapter: "command".into(), + adapter_version: 1, + inputs: BTreeMap::from([("source_root".into(), branch.head_root.0.clone())]), + tool_versions: BTreeMap::from([("cp".into(), "fixture".into())]), + platform: std::env::consts::OS.into(), + architecture: std::env::consts::ARCH.into(), + portability_scope: "workspace".into(), + strategy: "source-export-fixture".into(), + }; + let cache_key = db.workspace_layer_cache_key(&layer_key).unwrap(); + let _lock = db.acquire_write_lock().unwrap(); + let (tree_root_id, _) = db + .ingest_artifact_tree_under_write_lock(candidate.path()) + .unwrap(); + let envelope_id = db + .put_legacy_artifact_envelope_under_write_lock( + &layer_key, + &cache_key, + tree_root_id.clone(), + ) + .unwrap(); + let generation_id = "env_generation_source_export_fixture"; + db.conn + .execute( + "INSERT INTO environment_generations( + generation_id,view_id,generation_sequence,source_root,specification_digest, + predecessor_generation_id,state,created_at,activated_at,retired_at) + VALUES(?1,?2,1,?3,'source-export-spec',NULL,'active',1,1,NULL)", + params![generation_id, &view.view_id, branch.head_root.0], + ) + .unwrap(); + db.conn + .execute( + "INSERT INTO environment_generation_components( + generation_id,component_id,adapter_identity,kind,component_key,layer_id,mount_path) + VALUES(?1,'custom.export','trail/command@1','generated',?2,NULL,NULL)", + params![generation_id, &cache_key], + ) + .unwrap(); + db.conn + .execute( + "INSERT INTO environment_generation_outputs( + generation_id,component_id,output_name,policy,reuse_mode,sharing_scope, + publication_trigger,storage_identity,layer_id,mount_path,layer_subpath) + VALUES(?1,'custom.export','generated','immutable_seed_private','exact','workspace', + 'on_sync',?2,NULL,'.trail-generated/export','outputs/0000')", + params![generation_id, &envelope_id.0], + ) + .unwrap(); + db.conn + .execute( + "INSERT INTO artifact_generation_bindings( + binding_id,generation_id,component_id,output_name,desired_key,envelope_id, + tree_root_id,binding_identity,created_at) + VALUES('binding_source_export',?1,'custom.export','generated',?2,?3,?4, + 'source-export-binding',1)", + params![generation_id, &cache_key, &envelope_id.0, &tree_root_id.0], + ) + .unwrap(); + db.conn + .execute( + "INSERT INTO environment_view_generations(view_id,generation_id,updated_at) + VALUES(?1,?2,1)", + params![&view.view_id, generation_id], + ) + .unwrap(); + drop(_lock); + (workspace, db) + } + + #[test] + fn source_export_plan_pins_artifact_subtree_destination_and_authorization() { + let (_workspace, db) = setup_source_export_fixture("fail", None, false, false); + let plan = db + .plan_artifact_source_export( + "export", + "custom.export", + "client", + ArtifactSourceExportAuthorizationV1::ExplicitUser, + ) + .unwrap(); + assert_eq!(plan.version, ARTIFACT_SOURCE_EXPORT_PLAN_VERSION); + assert_eq!(plan.export_name, "client"); + assert_eq!(plan.output_name, "generated"); + assert_eq!(plan.artifact_subpath, "outputs/0000/generated-client"); + assert!(matches!( + plan.subtree, + ArtifactSourceExportSubtreeV1::Directory { .. } + )); + assert_eq!(plan.destination, "src/generated-client"); + assert!(!plan.destination_pin.exists); + assert_eq!( + plan.authorization, + ArtifactSourceExportAuthorizationV1::ExplicitUser + ); + assert_eq!( + db.artifact_validation_receipt(&plan.validation_receipt_id) + .unwrap() + .declaration + .name, + super::super::workspace_artifact::HOST_WORKSPACE_LAYER_STRUCTURAL_SEAL + ); + } + + #[test] + fn source_export_plan_rejects_missing_gate_and_fail_collision() { + let (_workspace, db) = setup_source_export_fixture("replace", Some("test"), false, false); + let error = db + .plan_artifact_source_export( + "export", + "custom.export", + "client", + ArtifactSourceExportAuthorizationV1::ExplicitUser, + ) + .unwrap_err(); + assert!(error + .to_string() + .contains("requires a recorded `test` gate")); + + let (_workspace, db) = setup_source_export_fixture("fail", None, true, false); + let error = db + .plan_artifact_source_export( + "export", + "custom.export", + "client", + ArtifactSourceExportAuthorizationV1::ExplicitUser, + ) + .unwrap_err(); + assert!(error.to_string().contains("collision mode is `fail`")); + } + + #[test] + fn source_export_execution_checkpoints_normal_source_and_reports_git_handoff() { + let (_workspace, mut db) = setup_source_export_fixture("fail", None, false, false); + let plan = db + .plan_artifact_source_export( + "export", + "custom.export", + "client", + ArtifactSourceExportAuthorizationV1::ExplicitUser, + ) + .unwrap(); + let report = db.execute_artifact_source_export(plan).unwrap(); + assert!(report.checkpointed); + assert_eq!( + report.git_handoff, + vec![ + "trail", + "lane", + "merge", + "export", + "--into", + "main", + "--dry-run" + ] + ); + assert!(report + .changed_paths + .iter() + .any(|change| change.path == "src/generated-client/new.rs")); + let diff = db.diff_lane("export", false).unwrap(); + assert!(diff + .files + .iter() + .any(|change| change.path == "src/generated-client/new.rs")); + let branch = db.lane_branch("export").unwrap(); + let entry = db + .root_file_entry(&branch.head_root, "src/generated-client/new.rs") + .unwrap() + .unwrap(); + let bytes = db + .materialize_entries_bytes(&BTreeMap::from([( + "src/generated-client/new.rs".into(), + entry, + )])) + .unwrap(); + assert_eq!(bytes["src/generated-client/new.rs"], b"generated\n"); + } + + #[test] + fn source_export_execution_replaces_pinned_content_and_rejects_stale_or_ignored_writes() { + let (_workspace, mut db) = setup_source_export_fixture("replace", None, true, false); + let plan = db + .plan_artifact_source_export( + "export", + "custom.export", + "client", + ArtifactSourceExportAuthorizationV1::ExplicitUser, + ) + .unwrap(); + let report = db.execute_artifact_source_export(plan).unwrap(); + assert!(report + .changed_paths + .iter() + .any(|change| change.path == "src/generated-client/old.rs")); + let branch = db.lane_branch("export").unwrap(); + assert!(db + .root_file_entry(&branch.head_root, "src/generated-client/old.rs") + .unwrap() + .is_none()); + + let (_workspace, mut db) = setup_source_export_fixture("fail", None, false, false); + let plan = db + .plan_artifact_source_export( + "export", + "custom.export", + "client", + ArtifactSourceExportAuthorizationV1::ExplicitUser, + ) + .unwrap(); + let branch = db.lane_branch("export").unwrap(); + db.apply_lane_patch( + "export", + PatchDocument { + base_change: Some(branch.head_change.0), + message: Some("advance source".into()), + session_id: None, + allow_ignored: false, + allow_stale: false, + edits: vec![PatchEdit::Write { + path: "advanced.txt".into(), + content: "advanced\n".into(), + executable: false, + }], + }, + ) + .unwrap(); + let error = db.execute_artifact_source_export(plan).unwrap_err(); + assert!(error.to_string().contains("is stale")); + + let (_workspace, mut db) = setup_source_export_fixture("replace", None, true, false); + let plan = db + .plan_artifact_source_export( + "export", + "custom.export", + "client", + ArtifactSourceExportAuthorizationV1::ExplicitUser, + ) + .unwrap(); + let branch = db.lane_branch("export").unwrap(); + db.apply_lane_patch( + "export", + PatchDocument { + base_change: Some(branch.head_change.0), + message: Some("conflict with planned destination".into()), + session_id: None, + allow_ignored: false, + allow_stale: false, + edits: vec![PatchEdit::Write { + path: "src/generated-client/old.rs".into(), + content: "changed after plan\n".into(), + executable: false, + }], + }, + ) + .unwrap(); + let error = db.execute_artifact_source_export(plan).unwrap_err(); + assert!(error.to_string().contains("is stale")); + let branch = db.lane_branch("export").unwrap(); + assert!(db + .root_file_entry(&branch.head_root, "src/generated-client/new.rs") + .unwrap() + .is_none()); + let entry = db + .root_file_entry(&branch.head_root, "src/generated-client/old.rs") + .unwrap() + .unwrap(); + let bytes = db + .materialize_entries_bytes(&BTreeMap::from([( + "src/generated-client/old.rs".into(), + entry, + )])) + .unwrap(); + assert_eq!( + bytes["src/generated-client/old.rs"], + b"changed after plan\n" + ); + + let (_workspace, mut db) = setup_source_export_fixture("fail", None, false, true); + let plan = db + .plan_artifact_source_export( + "export", + "custom.export", + "client", + ArtifactSourceExportAuthorizationV1::ExplicitUser, + ) + .unwrap(); + let error = db.execute_artifact_source_export(plan).unwrap_err(); + assert!(error.to_string().contains("ignored")); + let branch = db.lane_branch("export").unwrap(); + assert!(db + .root_file_entry(&branch.head_root, "src/generated-client/new.rs") + .unwrap() + .is_none()); + } +} diff --git a/trail/src/db/lane/workdir/dokan.rs b/trail/src/db/lane/workdir/dokan.rs index d672d4e5..79d83e71 100644 --- a/trail/src/db/lane/workdir/dokan.rs +++ b/trail/src/db/lane/workdir/dokan.rs @@ -835,6 +835,58 @@ mod mounted_conformance { drop(mount); } + #[test] + fn dokan_mount_reads_artifact_manifest_without_layer_materialization() { + if std::env::var_os("TRAIL_RUN_DOKAN_COW_TESTS").is_none() { + return; + } + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "baseline\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "dokan-lazy-artifact", + Some("main"), + LaneWorkdirMode::DokanCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let (binding, tree_id, missing_cache) = + lazy_artifact_conformance_binding(&db, temp.path()).unwrap(); + let branch = db.lane_branch("dokan-lazy-artifact").unwrap(); + let source_root = db.get_ref(&branch.ref_name).unwrap().root_id; + let source_upper = PathBuf::from( + db.lane_workspace_view("dokan-lazy-artifact") + .unwrap() + .unwrap() + .source_upper, + ); + let mount = db + .mount_dokan_cow_workdir_for_lane_with_ephemeral_bindings( + "dokan-lazy-artifact", + source_upper, + source_root, + vec![binding], + ) + .unwrap(); + let workdir = PathBuf::from( + db.lane_workdir("dokan-lazy-artifact") + .unwrap() + .workdir + .unwrap(), + ); + run_mounted_lazy_artifact_conformance(&workdir, &missing_cache).unwrap(); + assert!(db + .artifact_tree_lazy_entry(&tree_id, "payload/pkg/tool.js") + .unwrap() + .is_some()); + drop(mount); + } + #[test] fn foreground_dokan_mount_stops_through_a_separate_trail_handle() { if std::env::var_os("TRAIL_RUN_DOKAN_COW_TESTS").is_none() { diff --git a/trail/src/db/lane/workdir/fuse.rs b/trail/src/db/lane/workdir/fuse.rs index 41b0aec9..dcf237d8 100644 --- a/trail/src/db/lane/workdir/fuse.rs +++ b/trail/src/db/lane/workdir/fuse.rs @@ -815,6 +815,58 @@ mod fuse_overlay { drop(mount); } + #[test] + fn fuse_mount_reads_artifact_manifest_without_layer_materialization() { + if std::env::var_os("TRAIL_RUN_FUSE_COW_TESTS").is_none() { + return; + } + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "baseline\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "fuse-lazy-artifact", + Some("main"), + LaneWorkdirMode::FuseCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let (binding, tree_id, missing_cache) = + lazy_artifact_conformance_binding(&db, temp.path()).unwrap(); + let branch = db.lane_branch("fuse-lazy-artifact").unwrap(); + let source_root = db.get_ref(&branch.ref_name).unwrap().root_id; + let source_upper = PathBuf::from( + db.lane_workspace_view("fuse-lazy-artifact") + .unwrap() + .unwrap() + .source_upper, + ); + let mount = db + .mount_fuse_cow_workdir_for_lane_with_ephemeral_bindings( + "fuse-lazy-artifact", + source_upper, + source_root, + vec![binding], + ) + .unwrap(); + let workdir = PathBuf::from( + db.lane_workdir("fuse-lazy-artifact") + .unwrap() + .workdir + .unwrap(), + ); + run_mounted_lazy_artifact_conformance(&workdir, &missing_cache).unwrap(); + assert!(db + .artifact_tree_lazy_entry(&tree_id, "payload/pkg/tool.js") + .unwrap() + .is_some()); + drop(mount); + } + #[test] fn cargo_target_seed_reuses_compiler_results_with_private_writable_targets() { if std::env::var_os("TRAIL_RUN_FUSE_COW_TESTS").is_none() { @@ -861,6 +913,16 @@ mod fuse_overlay { "cargo generate-lockfile failed: {}", String::from_utf8_lossy(&lock.stderr) ); + let nested_lock = std::process::Command::new("cargo") + .args(["generate-lockfile", "--offline"]) + .current_dir(temp.path().join("shared-dep")) + .output() + .unwrap(); + assert!( + nested_lock.status.success(), + "nested cargo generate-lockfile failed: {}", + String::from_utf8_lossy(&nested_lock.stderr) + ); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(temp.path()).unwrap(); diff --git a/trail/src/db/lane/workdir/nfs_overlay.rs b/trail/src/db/lane/workdir/nfs_overlay.rs index d751d1dd..96a3acd9 100644 --- a/trail/src/db/lane/workdir/nfs_overlay.rs +++ b/trail/src/db/lane/workdir/nfs_overlay.rs @@ -1001,6 +1001,58 @@ mod macos { drop(mount); } + #[test] + fn nfs_mount_reads_artifact_manifest_without_layer_materialization() { + if std::env::var_os("TRAIL_RUN_NFS_COW_TESTS").is_none() { + return; + } + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "baseline\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "nfs-lazy-artifact", + Some("main"), + LaneWorkdirMode::NfsCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let (binding, tree_id, missing_cache) = + lazy_artifact_conformance_binding(&db, temp.path()).unwrap(); + let branch = db.lane_branch("nfs-lazy-artifact").unwrap(); + let source_root = db.get_ref(&branch.ref_name).unwrap().root_id; + let source_upper = PathBuf::from( + db.lane_workspace_view("nfs-lazy-artifact") + .unwrap() + .unwrap() + .source_upper, + ); + let mount = db + .mount_nfs_cow_workdir_for_lane_with_ephemeral_bindings( + "nfs-lazy-artifact", + source_upper, + source_root, + vec![binding], + ) + .unwrap(); + let workdir = PathBuf::from( + db.lane_workdir("nfs-lazy-artifact") + .unwrap() + .workdir + .unwrap(), + ); + run_mounted_lazy_artifact_conformance(&workdir, &missing_cache).unwrap(); + assert!(db + .artifact_tree_lazy_entry(&tree_id, "payload/pkg/tool.js") + .unwrap() + .is_some()); + drop(mount); + } + #[test] fn foreground_nfs_mount_stops_through_a_separate_trail_handle() { if std::env::var_os("TRAIL_RUN_NFS_COW_TESTS").is_none() { @@ -1212,6 +1264,7 @@ mod macos { let temp = tempfile::tempdir().unwrap(); fs::create_dir_all(temp.path().join("src")).unwrap(); fs::write(temp.path().join("README.md"), "baseline\n").unwrap(); + fs::write(temp.path().join("delete-after-mount.txt"), "delete me\n").unwrap(); fs::write(temp.path().join("src/old.txt"), "old\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(temp.path()).unwrap(); @@ -1239,6 +1292,7 @@ mod macos { .collect::>(); assert!(root_names.contains(OsStr::new("docs"))); fs::rename(workdir.join("src/old.txt"), workdir.join("src/renamed.txt")).unwrap(); + fs::remove_file(workdir.join("delete-after-mount.txt")).unwrap(); assert!(db.mount_nfs_cow_workdir_for_lane("nfs-test").is_err()); let report = db .record_lane_workdir("nfs-test", Some("NFS checkpoint".to_string())) @@ -1250,7 +1304,12 @@ mod macos { .collect::>(); assert_eq!( paths, - BTreeSet::from(["README.md", "docs/new.txt", "src/renamed.txt"]) + BTreeSet::from([ + "README.md", + "delete-after-mount.txt", + "docs/new.txt", + "src/renamed.txt", + ]) ); drop(mount); assert!(!is_nfs_mount(&workdir)); @@ -1259,6 +1318,7 @@ mod macos { let remount = db.mount_nfs_cow_workdir_for_lane("nfs-test").unwrap(); assert_eq!(fs::read(workdir.join("README.md")).unwrap(), b"changed\n"); assert_eq!(fs::read(workdir.join("docs/new.txt")).unwrap(), b"new\n"); + assert!(!workdir.join("delete-after-mount.txt").exists()); drop(remount); assert!(!is_nfs_mount(&workdir)); } @@ -2031,6 +2091,16 @@ mod macos { "cargo generate-lockfile failed: {}", String::from_utf8_lossy(&lock.stderr) ); + let nested_lock = Command::new("cargo") + .args(["generate-lockfile", "--offline"]) + .current_dir(temp.path().join("shared-dep")) + .output() + .unwrap(); + assert!( + nested_lock.status.success(), + "nested cargo generate-lockfile failed: {}", + String::from_utf8_lossy(&nested_lock.stderr) + ); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(temp.path()).unwrap(); diff --git a/trail/src/db/lane/workdir/view_conformance.rs b/trail/src/db/lane/workdir/view_conformance.rs index bcf70654..1a28f67a 100644 --- a/trail/src/db/lane/workdir/view_conformance.rs +++ b/trail/src/db/lane/workdir/view_conformance.rs @@ -1,10 +1,71 @@ use super::*; +use crate::ids::ArtifactTreeId; #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct ViewConformanceResult { pub(crate) changed_paths: BTreeSet, } +pub(crate) fn lazy_artifact_conformance_binding( + db: &Trail, + fixture_root: &Path, +) -> Result<(WorkspaceLayerBinding, ArtifactTreeId, PathBuf)> { + let source = fixture_root.join("lazy-artifact-input"); + fs::create_dir_all(source.join("payload/pkg"))?; + fs::write(source.join("payload/pkg/index.js"), b"shared artifact\n")?; + fs::write(source.join("payload/pkg/tool.js"), b"tool artifact\n")?; + let tree_id = { + let _lock = db.acquire_write_lock()?; + db.ingest_artifact_tree_under_write_lock(&source)?.0 + }; + let missing_cache = fixture_root.join("never-materialized-artifact-layer"); + Ok(( + WorkspaceLayerBinding { + binding_identity: "lazy-artifact-conformance".into(), + layer_id: Some("lazy-artifact-conformance".into()), + mount_path: "node_modules".into(), + storage_path: Some(missing_cache.clone()), + artifact_tree_id: Some(tree_id.clone()), + artifact_subpath: "payload".into(), + kind: "dependency".into(), + priority: 100, + }, + tree_id, + missing_cache, + )) +} + +pub(crate) fn run_mounted_lazy_artifact_conformance( + root: &Path, + missing_cache: &Path, +) -> Result<()> { + let index = root.join("node_modules/pkg/index.js"); + let tool = root.join("node_modules/pkg/tool.js"); + if fs::read(&index)? != b"shared artifact\n" || fs::read(&tool)? != b"tool artifact\n" { + return Err(Error::InvalidInput( + "mounted lazy artifact baseline is invalid".into(), + )); + } + if missing_cache.exists() { + return Err(Error::InvalidInput( + "lazy artifact backend unexpectedly materialized the complete layer".into(), + )); + } + let file = OpenOptions::new().write(true).open(&index)?; + use std::io::{Seek, SeekFrom, Write}; + let mut file = file; + file.seek(SeekFrom::Start(0))?; + file.write_all(b"private")?; + file.sync_all()?; + fs::remove_file(&tool)?; + if fs::read(&index)? != b"privateartifact\n" || tool.exists() || missing_cache.exists() { + return Err(Error::InvalidInput( + "mounted lazy artifact copy-up or whiteout behavior failed".into(), + )); + } + Ok(()) +} + /// One protocol-independent operation trace used by mounted FUSE, NFS, and /// Dokan acceptance tests. The trace deliberately exercises mixed lower/upper /// directories, ranged writes, metadata, rename, delete, and remount-visible diff --git a/trail/src/db/lane/workdir/view_core.rs b/trail/src/db/lane/workdir/view_core.rs index 86cba47d..72b50d74 100644 --- a/trail/src/db/lane/workdir/view_core.rs +++ b/trail/src/db/lane/workdir/view_core.rs @@ -1,4 +1,6 @@ use super::*; +use crate::db::lane::workspace_artifact::ArtifactLazyEntry; +use crate::ids::ArtifactTreeId; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fs::{self, File, OpenOptions}; #[cfg(unix)] @@ -131,6 +133,22 @@ struct CachedImmutableReadFile { attr: ViewNodeAttr, } +#[derive(Clone, Debug)] +enum LayerEntryResolution { + Artifact { + entry: ArtifactLazyEntry, + tree_id: ArtifactTreeId, + artifact_path: String, + layer_id: Option, + layer_relative_path: String, + }, + Materialized { + path: PathBuf, + layer_id: Option, + layer_relative_path: String, + }, +} + const LAYER_MOUNT_RESET_INTENT_VERSION: u16 = 2; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -418,11 +436,8 @@ impl ViewCore { fn layer_resolution( &self, path: &str, - ) -> std::result::Result, String)>, i32> { + ) -> std::result::Result, i32> { for binding in &self.layers { - let Some(storage_path) = &binding.storage_path else { - continue; - }; let suffix = if path == binding.mount_path { Some("") } else { @@ -432,6 +447,42 @@ impl ViewCore { let Some(suffix) = suffix else { continue; }; + if let Some(tree_id) = &binding.artifact_tree_id { + let artifact_path = match (binding.artifact_subpath.as_str(), suffix) { + ("", "") => String::new(), + (prefix, "") => prefix.to_string(), + ("", suffix) => suffix.to_string(), + (prefix, suffix) => format!("{prefix}/{suffix}"), + }; + if let Some(entry) = self + .db + .artifact_tree_lazy_entry(tree_id, &artifact_path) + .map_err(|_| EIO)? + { + #[cfg(windows)] + let entry = match entry { + ArtifactLazyEntry::Symlink { target } => self + .db + .artifact_tree_lazy_follow_symlink(tree_id, &artifact_path, &target) + .map_err(|_| EIO)? + .ok_or(EIO)?, + entry => entry, + }; + return Ok(Some(LayerEntryResolution::Artifact { + entry, + tree_id: tree_id.clone(), + artifact_path, + layer_id: binding.layer_id.clone(), + layer_relative_path: suffix.to_string(), + })); + } + // A verified CAS shadow is authoritative. Never expose an + // extra path found only in a stale materialization cache. + continue; + } + let Some(storage_path) = &binding.storage_path else { + continue; + }; let candidate = if suffix.is_empty() { storage_path.clone() } else { @@ -449,21 +500,15 @@ impl ViewCore { return Err(EPERM); } } - return Ok(Some(( - candidate, - binding.layer_id.clone(), - suffix.to_string(), - ))); + return Ok(Some(LayerEntryResolution::Materialized { + path: candidate, + layer_id: binding.layer_id.clone(), + layer_relative_path: suffix.to_string(), + })); } Ok(None) } - fn layer_path(&self, path: &str) -> std::result::Result, i32> { - Ok(self - .layer_resolution(path)? - .map(|(candidate, _, _)| candidate)) - } - fn record_hot_layer_access(&mut self, layer_id: &str, path: &str, size_bytes: u64) { let Some(capture) = &self.hot_access else { return; @@ -489,11 +534,19 @@ impl ViewCore { } fn layer_directory_exists(&self, path: &str) -> std::result::Result { - if self - .layer_path(path)? - .is_some_and(|path| fs::metadata(path).is_ok_and(|metadata| metadata.is_dir())) - { - return Ok(true); + if let Some(resolution) = self.layer_resolution(path)? { + match resolution { + LayerEntryResolution::Artifact { + entry: ArtifactLazyEntry::Directory { .. }, + .. + } => return Ok(true), + LayerEntryResolution::Materialized { path, .. } + if fs::metadata(&path).is_ok_and(|metadata| metadata.is_dir()) => + { + return Ok(true); + } + _ => {} + } } let prefix = if path.is_empty() { String::new() @@ -610,15 +663,27 @@ impl ViewCore { } return Ok(None); } - if let Some(layer_path) = self.layer_path(path)? { - let metadata = fs::symlink_metadata(layer_path).map_err(io_errno)?; + if let Some(layer) = self.layer_resolution(path)? { + if let LayerEntryResolution::Artifact { entry, .. } = layer { + return Ok(Some(match entry { + ArtifactLazyEntry::Directory { .. } => ViewNodeKind::Directory, + ArtifactLazyEntry::File { .. } => ViewNodeKind::File, + ArtifactLazyEntry::Symlink { .. } => ViewNodeKind::Symlink, + })); + } + let LayerEntryResolution::Materialized { + path: layer_path, .. + } = layer + else { + unreachable!() + }; + let metadata = fs::symlink_metadata(&layer_path).map_err(io_errno)?; if metadata.file_type().is_symlink() { #[cfg(unix)] return Ok(Some(ViewNodeKind::Symlink)); #[cfg(windows)] { - let metadata = - fs::metadata(self.layer_path(path)?.ok_or(ENOENT)?).map_err(io_errno)?; + let metadata = fs::metadata(&layer_path).map_err(io_errno)?; return Ok(if metadata.is_dir() { Some(ViewNodeKind::Directory) } else if metadata.is_file() { @@ -685,7 +750,40 @@ impl ViewCore { modified: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH), }); } - if let Some(layer_path) = self.layer_path(path)? { + if let Some(layer) = self.layer_resolution(path)? { + if let LayerEntryResolution::Artifact { entry, .. } = layer { + return Ok(match entry { + ArtifactLazyEntry::File { + mode, size_bytes, .. + } => ViewNodeAttr { + ino, + kind, + mode, + size: size_bytes, + modified: SystemTime::UNIX_EPOCH, + }, + ArtifactLazyEntry::Symlink { ref target } => ViewNodeAttr { + ino, + kind, + mode: 0o777, + size: target.len() as u64, + modified: SystemTime::UNIX_EPOCH, + }, + ArtifactLazyEntry::Directory { .. } => ViewNodeAttr { + ino, + kind, + mode: 0o755, + size: 0, + modified: SystemTime::UNIX_EPOCH, + }, + }); + } + let LayerEntryResolution::Materialized { + path: layer_path, .. + } = layer + else { + unreachable!() + }; let metadata = if kind == ViewNodeKind::Symlink { fs::symlink_metadata(layer_path) } else { @@ -770,11 +868,30 @@ impl ViewCore { names.insert(name.to_string()); } } - if let Some(layer_dir) = self.layer_path(&path)? - && let Ok(dir) = fs::read_dir(layer_dir) - { - for entry in dir.flatten() { - names.insert(entry.file_name().to_string_lossy().into_owned()); + if let Some(layer) = self.layer_resolution(&path)? { + match layer { + LayerEntryResolution::Artifact { + entry: ArtifactLazyEntry::Directory { .. }, + tree_id, + artifact_path, + .. + } => { + for (name, _) in self + .db + .artifact_tree_lazy_children(&tree_id, &artifact_path) + .map_err(|_| EIO)? + { + names.insert(name); + } + } + LayerEntryResolution::Materialized { path, .. } => { + if let Ok(dir) = fs::read_dir(path) { + for entry in dir.flatten() { + names.insert(entry.file_name().to_string_lossy().into_owned()); + } + } + } + LayerEntryResolution::Artifact { .. } => {} } } for class in [ @@ -833,34 +950,66 @@ impl ViewCore { let read = read_file_at(&file, &mut bytes, offset).map_err(io_errno)?; bytes.truncate(read); Ok((bytes, offset.saturating_add(read as u64) >= metadata.len())) - } else if let Some((layer_path, layer_id, layer_relative_path)) = - self.layer_resolution(&path)? - { - let metadata = fs::metadata(&layer_path).map_err(io_errno)?; - if !metadata.is_file() { - return Err(EISDIR); - } - let file = File::open(layer_path).map_err(io_errno)?; - let mut bytes = vec![0; count as usize]; - let read = read_file_at(&file, &mut bytes, offset).map_err(io_errno)?; - bytes.truncate(read); - if let Some(layer_id) = layer_id { - self.record_hot_layer_access(&layer_id, &layer_relative_path, metadata.len()); - } - self.immutable_read_files.insert( - ino, - CachedImmutableReadFile { - file, - attr: ViewNodeAttr { + } else if let Some(layer) = self.layer_resolution(&path)? { + match layer { + LayerEntryResolution::Artifact { + entry: + ArtifactLazyEntry::File { + node_id, + size_bytes, + .. + }, + layer_id, + layer_relative_path, + .. + } => { + let bytes = self + .db + .artifact_file_read_range(&node_id, offset, count) + .map_err(|_| EIO)?; + if let Some(layer_id) = layer_id { + self.record_hot_layer_access(&layer_id, &layer_relative_path, size_bytes); + } + let eof = offset.saturating_add(bytes.len() as u64) >= size_bytes; + Ok((bytes, eof)) + } + LayerEntryResolution::Artifact { .. } => Err(EISDIR), + LayerEntryResolution::Materialized { + path: layer_path, + layer_id, + layer_relative_path, + } => { + let metadata = fs::metadata(&layer_path).map_err(io_errno)?; + if !metadata.is_file() { + return Err(EISDIR); + } + let file = File::open(layer_path).map_err(io_errno)?; + let mut bytes = vec![0; count as usize]; + let read = read_file_at(&file, &mut bytes, offset).map_err(io_errno)?; + bytes.truncate(read); + if let Some(layer_id) = layer_id { + self.record_hot_layer_access( + &layer_id, + &layer_relative_path, + metadata.len(), + ); + } + self.immutable_read_files.insert( ino, - kind: ViewNodeKind::File, - mode: copy_up_mode(&metadata), - size: metadata.len(), - modified: SystemTime::UNIX_EPOCH, - }, - }, - ); - Ok((bytes, offset.saturating_add(read as u64) >= metadata.len())) + CachedImmutableReadFile { + file, + attr: ViewNodeAttr { + ino, + kind: ViewNodeKind::File, + mode: copy_up_mode(&metadata), + size: metadata.len(), + modified: SystemTime::UNIX_EPOCH, + }, + }, + ); + Ok((bytes, offset.saturating_add(read as u64) >= metadata.len())) + } + } } else { let entry = self.lower_file(&path)?.ok_or(ENOENT)?; let projection = self.db.project_entry_file(&entry).map_err(|_| EIO)?; @@ -903,8 +1052,16 @@ impl ViewCore { validate_view_symlink_target(&path, &target)?; return Ok(target); } - let layer_path = self.layer_path(&path)?.ok_or(ENOENT)?; - let target = fs::read_link(layer_path).map_err(io_errno)?; + let target = match self.layer_resolution(&path)?.ok_or(ENOENT)? { + LayerEntryResolution::Artifact { + entry: ArtifactLazyEntry::Symlink { target }, + .. + } => PathBuf::from(target), + LayerEntryResolution::Materialized { path, .. } => { + fs::read_link(path).map_err(io_errno)? + } + LayerEntryResolution::Artifact { .. } => return Err(EINVAL), + }; validate_view_symlink_target(&path, &target)?; Ok(target) } @@ -1056,37 +1213,60 @@ impl ViewCore { .map_err(|_| EIO)?; self.ensure_upper_parent(path)?; if created_upper { + let layer = self.layer_resolution(path)?; let visible_size = if truncate { 0 - } else if let Some(layer_path) = self.layer_path(path)? { - fs::metadata(layer_path).map_err(io_errno)?.len() } else { - self.lower_file(path)? - .map(|entry| entry.size_bytes) - .unwrap_or(0) + match &layer { + Some(LayerEntryResolution::Artifact { + entry: ArtifactLazyEntry::File { size_bytes, .. }, + .. + }) => *size_bytes, + Some(LayerEntryResolution::Materialized { path, .. }) => { + fs::metadata(path).map_err(io_errno)?.len() + } + Some(LayerEntryResolution::Artifact { .. }) => return Err(EINVAL), + None => self + .lower_file(path)? + .map(|entry| entry.size_bytes) + .unwrap_or(0), + } }; self.enforce_mutation_quota(path, Some(visible_size), true)?; if !truncate { - if let Some(layer_path) = self.layer_path(path)? { - clone_or_copy_projected_file(&layer_path, &upper).map_err(|_| EIO)?; - let metadata = fs::metadata(&layer_path).map_err(io_errno)?; - set_file_mode(&upper, copy_up_mode(&metadata)).map_err(io_errno)?; - } else { - let entry = self.lower_file(path)?; - if let Some(entry) = entry { - let projection = self.db.project_entry_file(&entry).map_err(|_| EIO)?; - clone_or_copy_projected_file(&projection, &upper).map_err(|_| EIO)?; - set_file_mode( - &upper, - if entry.executable { - 0o755 - } else { - entry.mode & 0o777 - }, - ) - .map_err(io_errno)?; - } else { - File::create(&upper).map_err(io_errno)?; + match layer { + Some(LayerEntryResolution::Artifact { + entry: ArtifactLazyEntry::File { node_id, .. }, + .. + }) => { + self.db + .materialize_artifact_file(&node_id, &upper) + .map_err(|_| EIO)?; + } + Some(LayerEntryResolution::Materialized { + path: layer_path, .. + }) => { + clone_or_copy_projected_file(&layer_path, &upper).map_err(|_| EIO)?; + let metadata = fs::metadata(&layer_path).map_err(io_errno)?; + set_file_mode(&upper, copy_up_mode(&metadata)).map_err(io_errno)?; + } + Some(LayerEntryResolution::Artifact { .. }) => return Err(EINVAL), + None => { + if let Some(entry) = self.lower_file(path)? { + let projection = self.db.project_entry_file(&entry).map_err(|_| EIO)?; + clone_or_copy_projected_file(&projection, &upper).map_err(|_| EIO)?; + set_file_mode( + &upper, + if entry.executable { + 0o755 + } else { + entry.mode & 0o777 + }, + ) + .map_err(io_errno)?; + } else { + File::create(&upper).map_err(io_errno)?; + } } } } else { @@ -1306,7 +1486,7 @@ impl ViewCore { if kind == ViewNodeKind::Directory && !self.children(ino)?.is_empty() { return Err(ENOTEMPTY); } - let hides_lower = self.layer_path(&path)?.is_some() + let hides_lower = self.layer_resolution(&path)?.is_some() || self.layer_directory_exists(&path)? || self.lower_file(&path)?.is_some() || self.lower_directory_exists(&path)?; @@ -1734,12 +1914,27 @@ impl ViewCore { } fn copy_lower_file(&self, source: &str, target: &str) -> std::result::Result<(), i32> { - if let Some(layer_path) = self.layer_path(source)? { + if let Some(layer) = self.layer_resolution(source)? { self.ensure_upper_parent(target)?; let target_path = self.upper_path(target)?; - clone_or_copy_projected_file(&layer_path, &target_path).map_err(|_| EIO)?; - let metadata = fs::metadata(layer_path).map_err(io_errno)?; - set_file_mode(&target_path, copy_up_mode(&metadata)).map_err(io_errno)?; + match layer { + LayerEntryResolution::Artifact { + entry: ArtifactLazyEntry::File { node_id, .. }, + .. + } => { + self.db + .materialize_artifact_file(&node_id, &target_path) + .map_err(|_| EIO)?; + } + LayerEntryResolution::Materialized { + path: layer_path, .. + } => { + clone_or_copy_projected_file(&layer_path, &target_path).map_err(|_| EIO)?; + let metadata = fs::metadata(layer_path).map_err(io_errno)?; + set_file_mode(&target_path, copy_up_mode(&metadata)).map_err(io_errno)?; + } + LayerEntryResolution::Artifact { .. } => return Err(EINVAL), + } self.sync_copied_file(&target_path)?; return Ok(()); } @@ -1850,24 +2045,41 @@ impl ViewCore { fn merge_lower_subtree_into_upper(&self, root: &str) -> std::result::Result<(), i32> { fs::create_dir_all(self.upper_path(root)?).map_err(io_errno)?; - if let Some(layer_root) = self.layer_path(root)? { - for entry in walkdir::WalkDir::new(&layer_root).follow_links(false) { - let entry = entry.map_err(|_| EIO)?; - if entry.path() == layer_root { - continue; + if let Some(layer) = self.layer_resolution(root)? { + match layer { + LayerEntryResolution::Artifact { + entry: ArtifactLazyEntry::Directory { .. }, + tree_id, + artifact_path, + .. + } => { + self.merge_artifact_subtree_into_upper(&tree_id, &artifact_path, root, 0)?; } - let suffix = entry.path().strip_prefix(&layer_root).map_err(|_| EIO)?; - let logical = - normalize_relative_path(&Path::new(root).join(suffix).to_string_lossy()) + LayerEntryResolution::Materialized { + path: layer_root, .. + } => { + for entry in walkdir::WalkDir::new(&layer_root).follow_links(false) { + let entry = entry.map_err(|_| EIO)?; + if entry.path() == layer_root { + continue; + } + let suffix = entry.path().strip_prefix(&layer_root).map_err(|_| EIO)?; + let logical = normalize_relative_path( + &Path::new(root).join(suffix).to_string_lossy(), + ) .map_err(|_| EINVAL)?; - if entry.file_type().is_dir() { - fs::create_dir_all(self.upper_path(&logical)?).map_err(io_errno)?; - } else if entry.file_type().is_file() - || (entry.file_type().is_symlink() - && fs::metadata(entry.path()).is_ok_and(|metadata| metadata.is_file())) - { - self.copy_lower_file(&logical, &logical)?; + if entry.file_type().is_dir() { + fs::create_dir_all(self.upper_path(&logical)?).map_err(io_errno)?; + } else if entry.file_type().is_file() + || (entry.file_type().is_symlink() + && fs::metadata(entry.path()) + .is_ok_and(|metadata| metadata.is_file())) + { + self.copy_lower_file(&logical, &logical)?; + } + } } + LayerEntryResolution::Artifact { .. } => return Err(ENOTDIR), } } for path in self.lower_selection(root)?.into_keys() { @@ -1878,6 +2090,63 @@ impl ViewCore { Ok(()) } + fn merge_artifact_subtree_into_upper( + &self, + tree_id: &ArtifactTreeId, + artifact_root: &str, + logical_root: &str, + depth: usize, + ) -> std::result::Result<(), i32> { + if depth > 256 { + return Err(EIO); + } + let children = self + .db + .artifact_tree_lazy_children(tree_id, artifact_root) + .map_err(|_| EIO)?; + for (name, entry) in children { + let artifact_path = if artifact_root.is_empty() { + name.clone() + } else { + format!("{artifact_root}/{name}") + }; + let logical_path = if logical_root.is_empty() { + name + } else { + format!("{logical_root}/{name}") + }; + if self.is_whiteouted(&logical_path) || self.upper_metadata(&logical_path).is_some() { + continue; + } + let upper = self.upper_path(&logical_path)?; + match entry { + ArtifactLazyEntry::Directory { .. } => { + fs::create_dir_all(&upper).map_err(io_errno)?; + self.merge_artifact_subtree_into_upper( + tree_id, + &artifact_path, + &logical_path, + depth + 1, + )?; + } + ArtifactLazyEntry::File { node_id, .. } => { + self.ensure_upper_parent(&logical_path)?; + self.db + .materialize_artifact_file(&node_id, &upper) + .map_err(|_| EIO)?; + self.sync_copied_file(&upper)?; + } + ArtifactLazyEntry::Symlink { target } => { + let target = PathBuf::from(target); + validate_view_symlink_target(&logical_path, &target)?; + self.ensure_upper_parent(&logical_path)?; + create_view_symlink(&target, &upper).map_err(io_errno)?; + } + } + } + Ok(()) + } + fn move_upper_subtree(&self, old: &str, new: &str) -> std::result::Result<(), i32> { let mut directories = BTreeSet::new(); let mut leaves = Vec::new(); @@ -2517,6 +2786,92 @@ mod tests { assert!(!eof); } + #[test] + fn artifact_manifest_layer_is_lazy_cow_without_materialized_directory() { + let (temp, db, root, upper) = fixture(); + let source = temp.path().join("artifact-source"); + fs::create_dir_all(source.join("payload/pkg")).unwrap(); + fs::write(source.join("payload/pkg/index.js"), b"shared artifact\n").unwrap(); + fs::write(source.join("payload/pkg/tool.sh"), b"#!/bin/sh\n").unwrap(); + fs::set_permissions( + source.join("payload/pkg/tool.sh"), + fs::Permissions::from_mode(0o755), + ) + .unwrap(); + std::os::unix::fs::symlink("index.js", source.join("payload/pkg/current.js")).unwrap(); + let tree_id = { + let _lock = db.acquire_write_lock().unwrap(); + db.ingest_artifact_tree_under_write_lock(&source).unwrap().0 + }; + let missing_cache = temp.path().join("never-materialized-layer"); + let binding = WorkspaceLayerBinding { + binding_identity: "lazy-artifact-layer".into(), + layer_id: Some("lazy-artifact-layer".into()), + mount_path: "node_modules".into(), + storage_path: Some(missing_cache.clone()), + artifact_tree_id: Some(tree_id.clone()), + artifact_subpath: "payload".into(), + kind: "dependency".into(), + priority: 100, + }; + let mut view = ViewCore::new_lazy_with_ephemeral_bindings( + Trail::open(db.workspace_root()).unwrap(), + upper.clone(), + root.clone(), + vec![binding.clone()], + ) + .unwrap(); + + let modules = view.lookup(VIEW_ROOT_INO, "node_modules").unwrap(); + let pkg = view.lookup(modules, "pkg").unwrap(); + let index = view.lookup(pkg, "index.js").unwrap(); + assert_eq!(view.read(index, 7, 8).unwrap().0, b"artifact"); + assert!(view.lookup(pkg, "tool.sh").is_ok()); + assert_eq!(view.attr("node_modules/pkg/tool.sh").unwrap().mode, 0o755); + let link = view.lookup(pkg, "current.js").unwrap(); + assert_eq!(view.readlink(link).unwrap(), PathBuf::from("index.js")); + assert!(!missing_cache.exists()); + + view.write(index, 0, b"private").unwrap(); + assert_eq!( + fs::read( + ViewUpperLayout::from_source_upper(upper.clone()) + .generated_upper + .join("node_modules/pkg/index.js") + ) + .unwrap(), + b"privateartifact\n" + ); + view.remove(pkg, "tool.sh").unwrap(); + assert!(view + .node_kind("node_modules/pkg/tool.sh") + .unwrap() + .is_none()); + assert!(matches!( + db.artifact_tree_lazy_entry(&tree_id, "payload/pkg/tool.sh") + .unwrap(), + Some(ArtifactLazyEntry::File { .. }) + )); + drop(view); + + let mut reopened = ViewCore::new_lazy_with_ephemeral_bindings( + Trail::open(db.workspace_root()).unwrap(), + upper, + root, + vec![binding], + ) + .unwrap(); + let modules = reopened.lookup(VIEW_ROOT_INO, "node_modules").unwrap(); + let pkg = reopened.lookup(modules, "pkg").unwrap(); + let index = reopened.lookup(pkg, "index.js").unwrap(); + assert_eq!(reopened.read(index, 0, 64).unwrap().0, b"privateartifact\n"); + assert!(reopened + .node_kind("node_modules/pkg/tool.sh") + .unwrap() + .is_none()); + assert!(!missing_cache.exists()); + } + #[test] fn interrupted_layer_reset_restores_private_upper_and_whiteouts_when_binding_did_not_commit() { let (_temp, db, root, upper) = fixture(); @@ -2581,6 +2936,8 @@ mod tests { layer_id: None, mount_path: "build".to_string(), storage_path: None, + artifact_tree_id: None, + artifact_subpath: String::new(), kind: "generated".to_string(), priority: 100, }]; diff --git a/trail/src/db/lane/workspace_artifact.rs b/trail/src/db/lane/workspace_artifact.rs new file mode 100644 index 00000000..691c19b7 --- /dev/null +++ b/trail/src/db/lane/workspace_artifact.rs @@ -0,0 +1,10289 @@ +use super::*; +use crate::ids::{ + ArtifactAttemptId, ArtifactAttestationId, ArtifactBlobId, ArtifactChunkId, ArtifactChunkListId, + ArtifactDesiredKeyV2, ArtifactEnvelopeId, ArtifactFileId, ArtifactQuarantineId, ArtifactTreeId, +}; +use ed25519_dalek::{Signature, Verifier, VerifyingKey}; + +const MAX_RESOLUTION_INPUTS: usize = 16_384; +const MAX_RESOLUTION_ARGV: usize = 1_024; +const MAX_RESOLUTION_AUTHORITIES: usize = 256; +const MAX_RESOLUTION_CREDENTIAL_HANDLES: usize = 64; +const MAX_RESOLUTION_ENVIRONMENT_NAMES: usize = 256; +const MAX_RESOLUTION_VALIDATIONS: usize = 256; +const MAX_RESOLUTION_PREDECESSORS: usize = 16_384; +const MAX_RESOLUTION_TEXT_BYTES: usize = 4 * 1024; +const MAX_RESOLUTION_TIMEOUT_MS: u64 = 60 * 60 * 1_000; +const MAX_RESOLUTION_CAPTURE_BYTES: u64 = 16 * 1024 * 1024; +const MAX_RESOLUTION_CANDIDATE_BYTES: u64 = 1024 * 1024 * 1024; +const MAX_RESOLUTION_CANDIDATE_ENTRIES: u64 = 1_000_000; +const MAX_RESOLUTION_CHILD_PROCESSES: u32 = 256; +const ARTIFACT_DESIRED_KEY_MATERIAL_VERSION: u16 = 2; +const ARTIFACT_WHOLE_BLOB_MAX_BYTES: usize = 1024 * 1024; +const ARTIFACT_CHUNK_MIN_BYTES: usize = 256 * 1024; +const ARTIFACT_CHUNK_AVERAGE_BYTES: usize = 1024 * 1024; +const ARTIFACT_CHUNK_MAX_BYTES: usize = 4 * 1024 * 1024; +const MAX_ARTIFACT_TREE_ENTRIES: u64 = 1_000_000; +const MAX_ARTIFACT_TREE_LOGICAL_BYTES: u64 = 1024 * 1024 * 1024 * 1024; +const MAX_ARTIFACT_TREE_DEPTH: usize = 256; +const MAX_PUBLIC_ARTIFACT_REPORT_ITEMS: usize = 10_000; +const MAX_PUBLIC_ARTIFACT_OBJECT_REFERENCES: usize = 10_000_000; +const HOST_WORKSPACE_LAYER_SEAL_VALIDATOR: &str = "trail.host/workspace-layer-sealer@1"; +pub(crate) const HOST_WORKSPACE_LAYER_STRUCTURAL_SEAL: &str = + "trail.host.workspace-layer.structural-seal/v1"; +const HOST_WORKSPACE_LAYER_POLICY_SEAL: &str = "trail.host.workspace-layer.policy-seal/v1"; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ArtifactResolutionAttemptFence { + pub(crate) attempt_id: ArtifactAttemptId, + pub(crate) owner_generation: u64, + pub(crate) owner_pid: u32, + pub(crate) owner_start_token: String, +} + +pub(crate) struct ArtifactResolutionAttemptFailure<'a> { + pub(crate) code: &'a str, + pub(crate) message: &'a str, + pub(crate) contacted_authorities: Vec, + pub(crate) stdout: &'a [u8], + pub(crate) stderr: &'a [u8], + pub(crate) stdout_original_bytes: Option, + pub(crate) stderr_original_bytes: Option, + pub(crate) redactions: &'a [Vec], + pub(crate) cancelled: bool, +} + +pub(crate) struct ArtifactResolutionExecutorFailure { + pub(crate) code: String, + pub(crate) message: String, + pub(crate) contacted_authorities: Vec, + pub(crate) stdout: Vec, + pub(crate) stderr: Vec, + pub(crate) stdout_original_bytes: u64, + pub(crate) stderr_original_bytes: u64, + pub(crate) redactions: Vec>, + pub(crate) cancelled: bool, +} + +pub(crate) type ArtifactResolutionExecutorResult = + std::result::Result>; + +impl ArtifactResolutionExecutorFailure { + pub(crate) fn from_error(code: &str, error: impl std::fmt::Display) -> Box { + Box::new(Self { + code: code.to_string(), + message: error.to_string(), + contacted_authorities: Vec::new(), + stdout: Vec::new(), + stderr: Vec::new(), + stdout_original_bytes: 0, + stderr_original_bytes: 0, + redactions: Vec::new(), + cancelled: false, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ArtifactFlatEntry { + pub(crate) kind: &'static str, + pub(crate) mode: u32, + pub(crate) size_bytes: u64, + pub(crate) content_hash: Option, + pub(crate) symlink_target: Option, +} + +/// One path resolved directly from an immutable artifact manifest. Directory +/// and file identities remain available so callers can continue traversal or +/// read/copy only the selected file without projecting the complete tree. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum ArtifactLazyEntry { + Directory { + node_id: ArtifactTreeId, + }, + File { + node_id: ArtifactFileId, + mode: u32, + size_bytes: u64, + }, + Symlink { + target: String, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ArtifactMaterializationReport { + pub(crate) materialization_id: String, + pub(crate) tree_root_id: ArtifactTreeId, + pub(crate) backend_compatibility: String, + pub(crate) storage_path: PathBuf, + pub(crate) logical_bytes: u64, + pub(crate) physical_bytes: u64, + pub(crate) entry_count: u64, + pub(crate) reused: bool, +} + +impl Trail { + /// Publish or deliberately refresh one resolver-produced snapshot. + /// + /// The executor remains a separate capability boundary: this operation + /// owns pin validation, attempt fencing, bounded/redacted evidence, + /// content-addressed snapshot publication, and reuse. A current snapshot + /// is reused without inspecting candidate bytes unless `refresh` is true; + /// wall-clock time never advances dependency selection. + pub fn resolve_artifact_component( + &self, + request: ArtifactResolutionRequestV1, + refresh: bool, + ) -> Result { + let ArtifactResolutionRequestV1 { plan, candidate } = request; + self.resolve_artifact_component_with_executor(plan, refresh, |_, _| Ok(candidate)) + } + + /// Run one host-owned resolver inside the same durable attempt that later + /// validates and publishes its candidate. Process launch and candidate + /// production failures therefore retain the same fenced recovery evidence + /// as malformed or rejected candidates. + pub(crate) fn resolve_artifact_component_with_executor( + &self, + mut plan: ArtifactResolutionPlanV1, + refresh: bool, + executor: F, + ) -> Result + where + F: FnOnce( + &ArtifactResolutionPlanV1, + &ArtifactResolutionAttemptFence, + ) -> ArtifactResolutionExecutorResult, + { + normalize_artifact_resolution_plan(&mut plan)?; + self.validate_artifact_resolution_plan_pins(&plan)?; + if !plan.credential_handles.is_empty() { + return Err(Error::InvalidInput( + "artifact resolution declares credential access; secret-influenced resolver output must remain lane-private and cannot enter shared CAS" + .into(), + )); + } + + if let Some((snapshot_id, snapshot)) = + self.artifact_resolution_snapshot_for_proposal(&plan.proposal_key)? + && !refresh + { + if snapshot.source_root != plan.source_root + || snapshot.component_id != plan.component_id + || snapshot.adapter_identity != plan.adapter_identity + || snapshot.snapshot_format != plan.snapshot_format + || snapshot.resolver_executable_identity != plan.executable_identity + || snapshot.policy_identity != plan.policy_identity + { + return Err(Error::InvalidInput(format!( + "artifact proposal key `{}` resolves to stale or incompatible snapshot {}; change the proposal key or request an explicit refresh", + plan.proposal_key, snapshot_id + ))); + } + return Ok(ArtifactResolutionComponentReportV1 { + component_id: plan.component_id, + proposal_key: plan.proposal_key, + source_root: plan.source_root, + snapshot_id, + snapshot, + decision: ArtifactResolutionDecisionV1::Reused, + refresh_requested: false, + attempt: None, + }); + } + + let (fence, _) = self.begin_artifact_resolution_attempt(plan.clone())?; + let candidate = match executor(&plan, &fence) { + Ok(candidate) => candidate, + Err(failure) => { + let attempt = self.finish_artifact_resolution_attempt_failure( + &fence, + ArtifactResolutionAttemptFailure { + code: &failure.code, + message: &failure.message, + contacted_authorities: failure.contacted_authorities, + stdout: &failure.stdout, + stderr: &failure.stderr, + stdout_original_bytes: Some(failure.stdout_original_bytes), + stderr_original_bytes: Some(failure.stderr_original_bytes), + redactions: &failure.redactions, + cancelled: failure.cancelled, + }, + )?; + return Err(Error::InvalidInput(format!( + "artifact resolution attempt `{}` failed during resolver execution: {}", + attempt.attempt_id, failure.message + ))); + } + }; + let ArtifactResolutionCandidateV1 { + snapshot_bytes, + resolved_identities, + checksums, + contacted_authorities, + stdout, + stderr, + redactions, + } = candidate; + if redactions.iter().any(|secret| !secret.is_empty()) { + let message = "resolver consumed secret material; its output is private, non-promotable, and cannot enter shared CAS"; + let attempt = self.finish_artifact_resolution_attempt_failure( + &fence, + ArtifactResolutionAttemptFailure { + code: "secret_tainted_output_private_only", + message, + contacted_authorities, + stdout: &stdout, + stderr: &stderr, + stdout_original_bytes: None, + stderr_original_bytes: None, + redactions: &redactions, + cancelled: false, + }, + )?; + return Err(Error::InvalidInput(format!( + "artifact resolution attempt `{}` failed: {message}", + attempt.attempt_id + ))); + } + if snapshot_bytes.is_empty() { + let message = "resolver produced an empty or malformed snapshot candidate"; + let attempt = self.finish_artifact_resolution_attempt_failure( + &fence, + ArtifactResolutionAttemptFailure { + code: "malformed_resolution_candidate", + message, + contacted_authorities: Vec::new(), + stdout: &stdout, + stderr: &stderr, + stdout_original_bytes: None, + stderr_original_bytes: None, + redactions: &redactions, + cancelled: false, + }, + )?; + return Err(Error::InvalidInput(format!( + "artifact resolution attempt `{}` failed: {message}", + attempt.attempt_id + ))); + } + if stdout.len() as u64 > plan.limits.stdout_bytes + || stderr.len() as u64 > plan.limits.stderr_bytes + { + let message = "resolver output exceeded its declared capture limit"; + let attempt = self.finish_artifact_resolution_attempt_failure( + &fence, + ArtifactResolutionAttemptFailure { + code: "captured_output_limit_exceeded", + message, + contacted_authorities: Vec::new(), + stdout: &stdout, + stderr: &stderr, + stdout_original_bytes: None, + stderr_original_bytes: None, + redactions: &redactions, + cancelled: false, + }, + )?; + return Err(Error::InvalidInput(format!( + "artifact resolution attempt `{}` failed: {message}", + attempt.attempt_id + ))); + } + let publication = self.put_artifact_resolution_snapshot( + plan.clone(), + snapshot_bytes, + resolved_identities, + checksums, + contacted_authorities.clone(), + refresh, + ); + let (snapshot_id, snapshot) = match publication { + Ok(snapshot) => snapshot, + Err(error) => { + let message = error.to_string(); + if let Err(finish_error) = self.finish_artifact_resolution_attempt_failure( + &fence, + ArtifactResolutionAttemptFailure { + code: "resolution_candidate_rejected", + message: &message, + contacted_authorities: Vec::new(), + stdout: &stdout, + stderr: &stderr, + stdout_original_bytes: None, + stderr_original_bytes: None, + redactions: &redactions, + cancelled: false, + }, + ) { + return Err(Error::Corrupt(format!( + "artifact resolution candidate was rejected ({message}) and attempt `{}` could not record failure: {finish_error}", + fence.attempt_id + ))); + } + return Err(error); + } + }; + let attempt = self.finish_artifact_resolution_attempt_success( + &fence, + &snapshot_id, + contacted_authorities, + &stdout, + &stderr, + &redactions, + )?; + if attempt.status != ArtifactResolutionAttemptStatusV1::Succeeded { + return Err(Error::InvalidInput(format!( + "artifact resolution attempt `{}` failed while recording bounded output", + attempt.attempt_id + ))); + } + Ok(ArtifactResolutionComponentReportV1 { + component_id: plan.component_id, + proposal_key: plan.proposal_key, + source_root: plan.source_root, + snapshot_id, + snapshot, + decision: if refresh { + ArtifactResolutionDecisionV1::Refreshed + } else { + ArtifactResolutionDecisionV1::Resolved + }, + refresh_requested: refresh, + attempt: Some(attempt), + }) + } + + /// Resolve a deterministic set of component requests for one pinned root. + pub fn resolve_all_artifact_components( + &self, + mut requests: Vec, + refresh: bool, + ) -> Result { + if requests.is_empty() { + return Err(Error::InvalidInput( + "artifact resolve-all requires at least one component request".into(), + )); + } + requests.sort_by(|left, right| { + (&left.plan.component_id, &left.plan.proposal_key) + .cmp(&(&right.plan.component_id, &right.plan.proposal_key)) + }); + if requests.windows(2).any(|pair| { + pair[0].plan.component_id == pair[1].plan.component_id + || pair[0].plan.proposal_key == pair[1].plan.proposal_key + }) { + return Err(Error::InvalidInput( + "artifact resolve-all contains a duplicate component or proposal key".into(), + )); + } + let source_root = requests[0].plan.source_root.clone(); + if requests + .iter() + .any(|request| request.plan.source_root != source_root) + { + return Err(Error::InvalidInput( + "artifact resolve-all requests must pin one source root".into(), + )); + } + let components = requests + .into_iter() + .map(|request| self.resolve_artifact_component(request, refresh)) + .collect::>>()?; + Ok(ArtifactResolutionBatchReportV1 { + source_root, + refresh_requested: refresh, + components, + }) + } + + pub(crate) fn begin_artifact_resolution_attempt( + &self, + mut plan: ArtifactResolutionPlanV1, + ) -> Result<( + ArtifactResolutionAttemptFence, + ArtifactResolutionAttemptReportV1, + )> { + let _lock = self.acquire_write_lock()?; + normalize_artifact_resolution_plan(&mut plan)?; + self.validate_artifact_resolution_plan_pins(&plan)?; + self.recover_artifact_resolution_attempts_under_write_lock()?; + + if let Some((attempt_id, owner_pid)) = self + .conn + .query_row( + "SELECT attempt_id, owner_pid FROM artifact_resolution_attempts + WHERE proposal_key=?1 AND status='running'", + params![plan.proposal_key], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)), + ) + .optional()? + { + return Err(Error::InvalidInput(format!( + "artifact proposal `{}` is already resolving in attempt `{attempt_id}` owned by process {owner_pid}", + plan.proposal_key + ))); + } + + let plan_object_id = self.put_object( + ARTIFACT_RESOLUTION_PLAN_KIND, + ARTIFACT_RESOLUTION_PLAN_VERSION, + &plan, + )?; + let owner_generation = self.conn.query_row( + "SELECT COALESCE(MAX(owner_generation), 0) + 1 + FROM artifact_resolution_attempts WHERE proposal_key=?1", + params![plan.proposal_key], + |row| row.get::<_, i64>(0), + )?; + let owner_generation = u64::try_from(owner_generation).map_err(|_| { + Error::Corrupt( + "artifact resolution owner generation is outside the supported range".into(), + ) + })?; + let owner_pid = std::process::id(); + let owner_start_token = current_process_start_token(); + let attempt_id = ArtifactAttemptId::new( + format!( + "resolution\0{}\0{}\0{owner_generation}\0{owner_pid}\0{owner_start_token}\0{}", + plan.proposal_key, + plan.source_root, + now_nanos() + ) + .as_bytes(), + ); + let authority_evidence = ArtifactResolutionAuthorityEvidenceV1 { + allowed_authorities: plan.allowed_authorities.clone(), + contacted_authorities: Vec::new(), + credential_handles: plan.credential_handles.clone(), + credential_values_redacted: true, + }; + let authority_evidence_json = serde_json::to_vec(&authority_evidence)?; + self.conn.execute( + "INSERT INTO artifact_resolution_attempts( + attempt_id, proposal_key, source_root, plan_object_id, + owner_generation, owner_pid, owner_start_token, status, + cancel_requested, authority_evidence_json, stdout_object_id, + stderr_object_id, snapshot_id, failure_receipt_object_id, + failure_code, failure_message, started_at, heartbeat_at, finished_at + ) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, 'running', 0, ?8, + NULL, NULL, NULL, NULL, NULL, NULL, ?9, ?9, NULL)", + params![ + attempt_id.0, + plan.proposal_key, + plan.source_root.0, + plan_object_id.0, + i64::try_from(owner_generation).map_err(|_| Error::InvalidInput( + "artifact resolution owner generation exceeds SQLite range".into() + ))?, + i64::from(owner_pid), + owner_start_token, + authority_evidence_json, + now_ts(), + ], + )?; + let fence = ArtifactResolutionAttemptFence { + attempt_id: attempt_id.clone(), + owner_generation, + owner_pid, + owner_start_token, + }; + let report = self.artifact_resolution_attempt(&attempt_id)?; + Ok((fence, report)) + } + + pub(crate) fn heartbeat_artifact_resolution_attempt( + &self, + fence: &ArtifactResolutionAttemptFence, + ) -> Result { + let _lock = self.acquire_write_lock()?; + let updated = self.conn.execute( + "UPDATE artifact_resolution_attempts SET heartbeat_at=?1 + WHERE attempt_id=?2 AND owner_generation=?3 AND owner_pid=?4 + AND owner_start_token=?5 AND status='running' AND cancel_requested=0", + params![ + now_ts(), + fence.attempt_id.0, + i64::try_from(fence.owner_generation).map_err(|_| Error::InvalidInput( + "artifact resolution owner generation exceeds SQLite range".into() + ))?, + i64::from(fence.owner_pid), + fence.owner_start_token, + ], + )?; + Ok(updated == 1) + } + + pub(crate) fn cancel_artifact_resolution_attempt( + &self, + attempt_id: &ArtifactAttemptId, + ) -> Result { + let _lock = self.acquire_write_lock()?; + let updated = self.conn.execute( + "UPDATE artifact_resolution_attempts + SET cancel_requested=1, heartbeat_at=?1 + WHERE attempt_id=?2 AND status='running'", + params![now_ts(), attempt_id.0], + )?; + if updated == 0 { + let existing = self.artifact_resolution_attempt(attempt_id)?; + if existing.status == ArtifactResolutionAttemptStatusV1::Running { + return Err(Error::Corrupt(format!( + "artifact resolution attempt `{attempt_id}` could not be cancelled" + ))); + } + return Ok(existing); + } + self.artifact_resolution_attempt(attempt_id) + } + + pub(crate) fn finish_artifact_resolution_attempt_success( + &self, + fence: &ArtifactResolutionAttemptFence, + snapshot_id: &ObjectId, + contacted_authorities: Vec, + stdout: &[u8], + stderr: &[u8], + redactions: &[Vec], + ) -> Result { + let _lock = self.acquire_write_lock()?; + let plan = self.artifact_resolution_plan_for_fence(fence)?; + self.validate_artifact_resolution_plan_pins(&plan)?; + let snapshot = self.get_object::( + ARTIFACT_RESOLUTION_SNAPSHOT_KIND, + snapshot_id, + )?; + validate_artifact_resolution_snapshot(&snapshot)?; + if snapshot.proposal_key != plan.proposal_key || snapshot.source_root != plan.source_root { + return Err(Error::InvalidInput( + "artifact resolution snapshot does not match the fenced proposal/source pins" + .into(), + )); + } + let authority_evidence = + normalized_resolution_authority_evidence(&plan, contacted_authorities)?; + let (stdout_object_id, stdout_truncated) = + self.put_artifact_resolution_capture(stdout, plan.limits.stdout_bytes, redactions)?; + let (stderr_object_id, stderr_truncated) = + self.put_artifact_resolution_capture(stderr, plan.limits.stderr_bytes, redactions)?; + if stdout_truncated || stderr_truncated { + let secret_tainted = resolution_is_secret_tainted(&plan, redactions); + return self.finish_artifact_resolution_attempt_failure_under_write_lock( + fence, + "captured_output_limit_exceeded", + "resolver output exceeded its declared capture limit", + authority_evidence, + stdout_object_id, + stderr_object_id, + secret_tainted, + ArtifactResolutionAttemptStatusV1::Failed, + ); + } + let updated = self.conn.execute( + "UPDATE artifact_resolution_attempts + SET status='succeeded', authority_evidence_json=?1, + stdout_object_id=?2, stderr_object_id=?3, snapshot_id=?4, + heartbeat_at=?5, finished_at=?5 + WHERE attempt_id=?6 AND owner_generation=?7 AND owner_pid=?8 + AND owner_start_token=?9 AND status='running' AND cancel_requested=0", + params![ + serde_json::to_vec(&authority_evidence)?, + stdout_object_id.as_ref().map(|id| id.0.as_str()), + stderr_object_id.as_ref().map(|id| id.0.as_str()), + snapshot_id.0, + now_ts(), + fence.attempt_id.0, + i64::try_from(fence.owner_generation).map_err(|_| Error::InvalidInput( + "artifact resolution owner generation exceeds SQLite range".into() + ))?, + i64::from(fence.owner_pid), + fence.owner_start_token, + ], + )?; + if updated != 1 { + return Err(Error::InvalidInput(format!( + "artifact resolution attempt `{}` lost its owner fence or was cancelled", + fence.attempt_id + ))); + } + self.artifact_resolution_attempt(&fence.attempt_id) + } + + pub(crate) fn finish_artifact_resolution_attempt_failure( + &self, + fence: &ArtifactResolutionAttemptFence, + failure: ArtifactResolutionAttemptFailure<'_>, + ) -> Result { + let _lock = self.acquire_write_lock()?; + let plan = self.artifact_resolution_plan_for_fence(fence)?; + validate_resolution_text(failure.code, "failure code")?; + validate_resolution_text(failure.message, "failure message")?; + let authority_evidence = + normalized_resolution_authority_evidence(&plan, failure.contacted_authorities)?; + let (stdout_object_id, _) = self.put_artifact_resolution_capture_observed( + failure.stdout, + failure + .stdout_original_bytes + .unwrap_or_else(|| u64::try_from(failure.stdout.len()).unwrap_or(u64::MAX)), + plan.limits.stdout_bytes, + failure.redactions, + )?; + let (stderr_object_id, _) = self.put_artifact_resolution_capture_observed( + failure.stderr, + failure + .stderr_original_bytes + .unwrap_or_else(|| u64::try_from(failure.stderr.len()).unwrap_or(u64::MAX)), + plan.limits.stderr_bytes, + failure.redactions, + )?; + let redacted_message = String::from_utf8_lossy(&redact_resolution_bytes( + failure.message.as_bytes(), + failure.redactions, + )) + .into_owned(); + let secret_tainted = resolution_is_secret_tainted(&plan, failure.redactions); + self.finish_artifact_resolution_attempt_failure_under_write_lock( + fence, + failure.code, + &redacted_message, + authority_evidence, + stdout_object_id, + stderr_object_id, + secret_tainted, + if failure.cancelled { + ArtifactResolutionAttemptStatusV1::Cancelled + } else { + ArtifactResolutionAttemptStatusV1::Failed + }, + ) + } + + pub(crate) fn artifact_resolution_attempt( + &self, + attempt_id: &ArtifactAttemptId, + ) -> Result { + let row = self.conn.query_row( + "SELECT proposal_key, source_root, plan_object_id, owner_generation, + owner_pid, status, cancel_requested, authority_evidence_json, + stdout_object_id, stderr_object_id, snapshot_id, + failure_receipt_object_id, failure_code, failure_message, + started_at, heartbeat_at, finished_at + FROM artifact_resolution_attempts WHERE attempt_id=?1", + params![attempt_id.0], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, String>(5)?, + row.get::<_, bool>(6)?, + row.get::<_, Vec>(7)?, + row.get::<_, Option>(8)?, + row.get::<_, Option>(9)?, + row.get::<_, Option>(10)?, + row.get::<_, Option>(11)?, + row.get::<_, Option>(12)?, + row.get::<_, Option>(13)?, + row.get::<_, i64>(14)?, + row.get::<_, i64>(15)?, + row.get::<_, Option>(16)?, + )) + }, + )?; + let plan: ArtifactResolutionPlanV1 = + self.get_object(ARTIFACT_RESOLUTION_PLAN_KIND, &ObjectId(row.2.clone()))?; + Ok(ArtifactResolutionAttemptReportV1 { + attempt_id: attempt_id.clone(), + proposal_key: row.0, + source_root: ObjectId(row.1), + plan_object_id: ObjectId(row.2), + owner_generation: u64::try_from(row.3).map_err(|_| { + Error::Corrupt("artifact resolution owner generation is invalid".into()) + })?, + owner_pid: u32::try_from(row.4) + .map_err(|_| Error::Corrupt("artifact resolution owner PID is invalid".into()))?, + status: parse_artifact_resolution_attempt_status(&row.5)?, + cancel_requested: row.6, + authority_evidence: serde_json::from_slice(&row.7).map_err(|error| { + Error::Corrupt(format!( + "invalid artifact resolution authority evidence: {error}" + )) + })?, + stdout_object_id: row.8.map(ObjectId), + stderr_object_id: row.9.map(ObjectId), + snapshot_id: row.10.map(ObjectId), + failure_receipt_object_id: row.11.map(ObjectId), + failure_code: row.12, + failure_message: row.13, + started_at: row.14, + heartbeat_at: row.15, + finished_at: row.16, + recovery_command: vec![ + "trail".into(), + "env".into(), + "resolve".into(), + "component".into(), + plan.component_id, + ], + }) + } + + pub(crate) fn artifact_resolution_attempts( + &self, + ) -> Result> { + let mut stmt = self.conn.prepare( + "SELECT attempt_id FROM artifact_resolution_attempts + ORDER BY started_at, attempt_id", + )?; + let ids = stmt + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + ids.into_iter() + .map(|id| { + ArtifactAttemptId::parse(id) + .map_err(Error::Corrupt) + .and_then(|id| self.artifact_resolution_attempt(&id)) + }) + .collect() + } + + pub(crate) fn recover_artifact_resolution_attempts_under_write_lock(&self) -> Result<()> { + let running = { + let mut stmt = self.conn.prepare( + "SELECT attempt_id, owner_generation, owner_pid, owner_start_token, + cancel_requested + FROM artifact_resolution_attempts WHERE status='running' + ORDER BY started_at, attempt_id", + )?; + stmt.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, String>(3)?, + row.get::<_, bool>(4)?, + )) + })? + .collect::, _>>()? + }; + for (attempt_id, owner_generation, owner_pid, owner_start_token, cancel_requested) in + running + { + let Ok(owner_pid_u32) = u32::try_from(owner_pid) else { + continue; + }; + if process_start_token_match(owner_pid_u32, &owner_start_token) + != ProcessIdentityMatch::DeadOrMismatch + { + continue; + } + let fence = ArtifactResolutionAttemptFence { + attempt_id: ArtifactAttemptId::parse(attempt_id).map_err(Error::Corrupt)?, + owner_generation: u64::try_from(owner_generation).map_err(|_| { + Error::Corrupt("artifact resolution owner generation is invalid".into()) + })?, + owner_pid: owner_pid_u32, + owner_start_token, + }; + let plan = self.artifact_resolution_plan_for_fence(&fence)?; + let evidence = normalized_resolution_authority_evidence(&plan, Vec::new())?; + let secret_tainted = !plan.credential_handles.is_empty(); + self.finish_artifact_resolution_attempt_failure_under_write_lock( + &fence, + if cancel_requested { + "resolver_cancelled" + } else { + "resolver_owner_lost" + }, + if cancel_requested { + "resolver cancellation was recovered after its owner process exited" + } else { + "resolver owner process exited before publishing a snapshot; retry the reported resolution command" + }, + evidence, + None, + None, + secret_tainted, + if cancel_requested { + ArtifactResolutionAttemptStatusV1::Cancelled + } else { + ArtifactResolutionAttemptStatusV1::Abandoned + }, + )?; + } + Ok(()) + } + + /// Validate artifact CAS identities, edges, snapshots, envelopes, and + /// durable attempt state without mutating repairable evidence. + pub(crate) fn validate_artifact_cas_integrity(&self) -> Result> { + let mut errors = Vec::new(); + let objects = { + let mut statement = self.conn.prepare( + "SELECT a.artifact_id,a.kind,a.version,a.logical_bytes,a.object_id, + o.kind,o.version,o.codec,o.hash_alg,o.size_bytes,o.bytes + FROM artifact_objects a LEFT JOIN objects o ON o.object_id=a.object_id + ORDER BY a.artifact_id", + )?; + statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, String>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, Option>(8)?, + row.get::<_, Option>(9)?, + row.get::<_, Option>>(10)?, + )) + })? + .collect::, _>>()? + }; + for ( + artifact_id, + kind, + version, + logical_bytes, + object_id, + object_kind, + object_version, + codec, + hash_alg, + size_bytes, + bytes, + ) in objects + { + let result = (|| -> Result<()> { + let bytes = bytes.as_deref().ok_or_else(|| { + Error::Corrupt(format!("backing object {object_id} is missing")) + })?; + let encoded_version = u16::try_from(version).map_err(|_| { + Error::Corrupt(format!("artifact object version {version} is invalid")) + })?; + if object_kind.as_deref() != Some(kind.as_str()) + || object_version != Some(version) + || codec.as_deref() != Some("cbor") + || hash_alg.as_deref() != Some("sha256") + || size_bytes != i64::try_from(bytes.len()).ok() + || ObjectId::for_bytes(&kind, encoded_version, bytes).0 != object_id + { + return Err(Error::Corrupt(format!( + "backing object {object_id} metadata or content identity is invalid" + ))); + } + self.validate_artifact_cas_object( + &artifact_id, + &kind, + version, + logical_bytes, + bytes, + ) + })(); + if let Err(error) = result { + errors.push(format!( + "artifact object {artifact_id} is corrupt: {error}; rebuild the owning environment component before reattaching it" + )); + } + } + + let validation_receipts = { + let mut statement = self + .conn + .prepare("SELECT object_id FROM objects WHERE kind=?1 ORDER BY object_id")?; + statement + .query_map(params![ARTIFACT_VALIDATION_RECEIPT_KIND], |row| { + row.get::<_, String>(0) + })? + .collect::, _>>()? + }; + for receipt_id in validation_receipts { + if let Err(error) = self.artifact_validation_receipt(&ObjectId(receipt_id.clone())) { + errors.push(format!( + "artifact validation receipt {receipt_id} is corrupt: {error}; rerun the exact validator before publishing or attaching the artifact" + )); + } + } + + let snapshots = { + let mut statement = self.conn.prepare( + "SELECT snapshot_id,proposal_key,source_root,component_id,adapter_identity, + content_object_id,verification_state,state + FROM artifact_resolution_snapshots ORDER BY snapshot_id", + )?; + statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, String>(6)?, + row.get::<_, String>(7)?, + )) + })? + .collect::, _>>()? + }; + for ( + snapshot_id, + proposal_key, + source_root, + component_id, + adapter_identity, + content_object_id, + verification_state, + state, + ) in snapshots + { + let result = (|| -> Result<()> { + let snapshot_bytes = self.validated_raw_object_bytes( + ARTIFACT_RESOLUTION_SNAPSHOT_KIND, + &ObjectId(snapshot_id.clone()), + ARTIFACT_RESOLUTION_SNAPSHOT_VERSION, + )?; + let snapshot: ArtifactResolutionSnapshotV1 = from_cbor(&snapshot_bytes)?; + validate_artifact_resolution_snapshot(&snapshot)?; + if snapshot.proposal_key != proposal_key + || snapshot.source_root.0 != source_root + || snapshot.component_id != component_id + || snapshot.adapter_identity != adapter_identity + || snapshot.content_object_id.0 != content_object_id + || verification_state != "verified" + || !matches!(state.as_str(), "current" | "superseded") + { + return Err(Error::Corrupt( + "snapshot database identity disagrees with its object".into(), + )); + } + let content_bytes = self.validated_raw_object_bytes( + ARTIFACT_RESOLUTION_CONTENT_KIND, + &snapshot.content_object_id, + ARTIFACT_RESOLUTION_SNAPSHOT_VERSION, + )?; + let content: ArtifactResolutionContentV1 = from_cbor(&content_bytes)?; + if content.version != ARTIFACT_RESOLUTION_SNAPSHOT_VERSION + || content.content_sha256 != snapshot.content_sha256 + || sha256_hex(&content.bytes) != snapshot.content_sha256 + { + return Err(Error::Corrupt( + "snapshot content failed identity verification".into(), + )); + } + Ok(()) + })(); + if let Err(error) = result { + errors.push(format!( + "artifact resolution snapshot {snapshot_id} is corrupt: {error}; run explicit component resolution with refresh after restoring or reinitializing the workspace" + )); + } + } + + let envelopes = { + let mut statement = self.conn.prepare( + "SELECT e.envelope_id,e.tree_root_id,e.object_id,a.object_id, + e.state,e.verification_state + FROM artifact_envelopes e + LEFT JOIN artifact_objects a ON a.artifact_id=e.envelope_id + ORDER BY e.envelope_id", + )?; + statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + )) + })? + .collect::, _>>()? + }; + for (envelope_id, tree_id, object_id, artifact_object_id, state, verification) in envelopes + { + let result = (|| -> Result<()> { + if artifact_object_id.as_deref() != Some(object_id.as_str()) { + return Err(Error::Corrupt( + "envelope object mapping is missing or cross-wired".into(), + )); + } + let envelope_id = + ArtifactEnvelopeId::parse(envelope_id.clone()).map_err(Error::Corrupt)?; + let tree_id = ArtifactTreeId::parse(tree_id).map_err(Error::Corrupt)?; + if state == "ready" && verification == "verified" { + self.verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id)?; + self.artifact_tree_flat_entries(&tree_id)?; + } + Ok(()) + })(); + if let Err(error) = result { + errors.push(format!( + "artifact envelope {envelope_id} is corrupt: {error}; detach affected generations and rebuild the component before reuse" + )); + } + } + + errors.extend(self.validate_artifact_attempt_integrity()?); + Ok(errors) + } + + fn validated_raw_object_bytes( + &self, + kind: &'static str, + object_id: &ObjectId, + version: u16, + ) -> Result> { + let row = self + .conn + .query_row( + "SELECT kind,version,codec,hash_alg,size_bytes,bytes + FROM objects WHERE object_id=?1", + params![object_id.0], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, Vec>(5)?, + )) + }, + ) + .optional()?; + let Some((stored_kind, stored_version, codec, hash_alg, size_bytes, bytes)) = row else { + return Err(Error::ObjectNotFound { + kind, + id: object_id.0.clone(), + }); + }; + if stored_kind != kind + || stored_version != i64::from(version) + || codec != "cbor" + || hash_alg != "sha256" + || i64::try_from(bytes.len()).ok() != Some(size_bytes) + || ObjectId::for_bytes(kind, version, &bytes) != *object_id + { + return Err(Error::Corrupt(format!( + "object {object_id} metadata or content identity is invalid" + ))); + } + Ok(bytes) + } + + fn validate_artifact_cas_object( + &self, + artifact_id: &str, + kind: &str, + version: i64, + logical_bytes: i64, + bytes: &[u8], + ) -> Result<()> { + if logical_bytes < 0 { + return Err(Error::Corrupt("logical byte count is negative".into())); + } + let expected_version = match kind { + ARTIFACT_DIRECTORY_NODE_KIND => ARTIFACT_DIRECTORY_NODE_VERSION, + ARTIFACT_FILE_NODE_KIND => ARTIFACT_FILE_NODE_VERSION, + ARTIFACT_BLOB_KIND => ARTIFACT_BLOB_VERSION, + ARTIFACT_CHUNK_LIST_KIND => ARTIFACT_CHUNK_LIST_VERSION, + ARTIFACT_CHUNK_KIND => ARTIFACT_CHUNK_VERSION, + ARTIFACT_TREE_ROOT_KIND => ARTIFACT_TREE_ROOT_VERSION, + ARTIFACT_ENVELOPE_KIND => ARTIFACT_ENVELOPE_VERSION, + _ => { + return Err(Error::Corrupt(format!( + "unknown artifact object kind `{kind}`" + ))) + } + }; + if version != i64::from(expected_version) { + return Err(Error::Corrupt(format!( + "stored version {version} does not match {kind} version {expected_version}" + ))); + } + let actual_id = match kind { + ARTIFACT_DIRECTORY_NODE_KIND => { + let node: ArtifactDirectoryNodeV1 = from_cbor(bytes)?; + for entry in &node.entries { + match &entry.target { + ArtifactDirectoryEntryTargetV1::Directory { node_id } => { + let _: ArtifactDirectoryNodeV1 = self.get_artifact_cas_object( + &node_id.0, + ARTIFACT_DIRECTORY_NODE_KIND, + ARTIFACT_DIRECTORY_NODE_VERSION, + )?; + } + ArtifactDirectoryEntryTargetV1::File { node_id } => { + let _: ArtifactFileNodeV1 = self.get_artifact_cas_object( + &node_id.0, + ARTIFACT_FILE_NODE_KIND, + ARTIFACT_FILE_NODE_VERSION, + )?; + } + ArtifactDirectoryEntryTargetV1::Symlink { .. } => {} + } + } + encode_artifact_directory_node(node)?.0 .0 + } + ARTIFACT_FILE_NODE_KIND => { + let node: ArtifactFileNodeV1 = from_cbor(bytes)?; + self.verify_artifact_file_content(&node)?; + if logical_bytes as u64 != node.size_bytes { + return Err(Error::Corrupt( + "file logical byte count disagrees with its node".into(), + )); + } + encode_artifact_file_node(node)?.0 .0 + } + ARTIFACT_BLOB_KIND => { + let blob: ArtifactBlobV1 = from_cbor(bytes)?; + if logical_bytes as u64 != blob.bytes.len() as u64 { + return Err(Error::Corrupt( + "blob logical byte count disagrees with its bytes".into(), + )); + } + encode_artifact_blob(blob)?.0 .0 + } + ARTIFACT_CHUNK_LIST_KIND => { + let list: ArtifactChunkListV1 = from_cbor(bytes)?; + if logical_bytes as u64 != list.file_size_bytes { + return Err(Error::Corrupt( + "chunk-list logical byte count disagrees with its file size".into(), + )); + } + for chunk in &list.chunks { + let value: ArtifactChunkV1 = self.get_artifact_cas_object( + &chunk.chunk_id.0, + ARTIFACT_CHUNK_KIND, + ARTIFACT_CHUNK_VERSION, + )?; + if value.bytes.len() as u64 != chunk.size_bytes { + return Err(Error::Corrupt( + "chunk-list edge size disagrees with its chunk".into(), + )); + } + } + encode_artifact_chunk_list(list)?.0 .0 + } + ARTIFACT_CHUNK_KIND => { + let chunk: ArtifactChunkV1 = from_cbor(bytes)?; + if logical_bytes as u64 != chunk.bytes.len() as u64 { + return Err(Error::Corrupt( + "chunk logical byte count disagrees with its bytes".into(), + )); + } + encode_artifact_chunk(chunk)?.0 .0 + } + ARTIFACT_TREE_ROOT_KIND => { + let tree: ArtifactTreeRootV1 = from_cbor(bytes)?; + if logical_bytes as u64 != tree.logical_bytes { + return Err(Error::Corrupt( + "tree logical byte count disagrees with its root".into(), + )); + } + encode_artifact_tree_root(tree)?.0 .0 + } + ARTIFACT_ENVELOPE_KIND => { + let envelope: ArtifactEnvelopeV1 = from_cbor(bytes)?; + if logical_bytes != 0 { + return Err(Error::Corrupt( + "artifact envelope logical byte count must be zero".into(), + )); + } + encode_artifact_envelope(envelope)?.0 .0 + } + _ => unreachable!(), + }; + if actual_id != artifact_id { + return Err(Error::Corrupt(format!( + "encoded identity is {actual_id}, not {artifact_id}" + ))); + } + Ok(()) + } + + fn validate_artifact_attempt_integrity(&self) -> Result> { + let mut errors = Vec::new(); + let mut attempts = self.conn.prepare( + "SELECT attempt_id,source_root,owner_pid,owner_start_token,phase,status, + finished_at + FROM artifact_construction_attempts ORDER BY attempt_id", + )?; + for row in attempts.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, Option>(6)?, + )) + })? { + let (attempt_id, source_root, owner_pid, owner_token, phase, status, finished_at) = + row?; + if self + .get_object::(WORKTREE_ROOT_KIND, &ObjectId(source_root)) + .is_err() + { + errors.push(format!( + "artifact construction attempt {attempt_id} has a missing source root; restore from backup or reinitialize the workspace" + )); + } + if status == "running" { + let owner_dead = u32::try_from(owner_pid).map_or(true, |pid| { + process_start_token_match(pid, &owner_token) + == ProcessIdentityMatch::DeadOrMismatch + }); + if phase == "completed" || finished_at.is_some() || owner_dead { + errors.push(format!( + "artifact construction attempt {attempt_id} is incomplete or has a dead owner; reopen Trail to run exact-owner recovery" + )); + } + } else if phase != "completed" || finished_at.is_none() { + errors.push(format!( + "artifact construction attempt {attempt_id} has incoherent terminal phase evidence; restore from backup or reinitialize the workspace" + )); + } + } + drop(attempts); + + let mut waiters = self.conn.prepare( + "SELECT w.waiter_id,w.status,a.status + FROM artifact_construction_waiters w + JOIN artifact_construction_attempts a ON a.attempt_id=w.attempt_id + ORDER BY w.waiter_id", + )?; + for row in waiters.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + })? { + let (waiter_id, waiter_status, attempt_status) = row?; + if waiter_status == "waiting" && attempt_status != "running" { + errors.push(format!( + "artifact construction waiter {waiter_id} remains waiting on a terminal attempt; reopen Trail to recover it" + )); + } + } + Ok(errors) + } + + fn validate_artifact_resolution_plan_pins( + &self, + plan: &ArtifactResolutionPlanV1, + ) -> Result<()> { + let _: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, &plan.source_root)?; + let resolved_program = Path::new(&plan.resolved_program); + if !resolved_program.is_absolute() { + return Err(Error::InvalidInput( + "artifact resolver resolved program must be an absolute host path".into(), + )); + } + let actual_identity = + super::workspace_environment::workspace_tool_identity_for_path(resolved_program)?; + if actual_identity != plan.executable_identity { + return Err(Error::InvalidInput(format!( + "artifact resolver executable `{}` changed after planning", + plan.program + ))); + } + for input in &plan.readable_inputs { + let entry = self + .root_file_entry(&plan.source_root, &input.source_path)? + .ok_or_else(|| { + Error::InvalidInput(format!( + "artifact resolver input `{}` is absent from pinned source root {}", + input.source_path, plan.source_root + )) + })?; + if entry.content_hash != input.content_hash || entry.size_bytes != input.size_bytes { + return Err(Error::InvalidInput(format!( + "artifact resolver input `{}` changed after planning", + input.source_path + ))); + } + } + Ok(()) + } + + fn artifact_resolution_plan_for_fence( + &self, + fence: &ArtifactResolutionAttemptFence, + ) -> Result { + let plan_object_id = self + .conn + .query_row( + "SELECT plan_object_id FROM artifact_resolution_attempts + WHERE attempt_id=?1 AND owner_generation=?2 AND owner_pid=?3 + AND owner_start_token=?4 AND status='running'", + params![ + fence.attempt_id.0, + i64::try_from(fence.owner_generation).map_err(|_| Error::InvalidInput( + "artifact resolution owner generation exceeds SQLite range".into() + ))?, + i64::from(fence.owner_pid), + fence.owner_start_token, + ], + |row| row.get::<_, String>(0), + ) + .optional()? + .ok_or_else(|| { + Error::InvalidInput(format!( + "artifact resolution attempt `{}` lost its exact owner fence", + fence.attempt_id + )) + })?; + self.get_object(ARTIFACT_RESOLUTION_PLAN_KIND, &ObjectId(plan_object_id)) + } + + fn put_artifact_resolution_capture( + &self, + bytes: &[u8], + limit: u64, + redactions: &[Vec], + ) -> Result<(Option, bool)> { + self.put_artifact_resolution_capture_observed( + bytes, + u64::try_from(bytes.len()).unwrap_or(u64::MAX), + limit, + redactions, + ) + } + + fn put_artifact_resolution_capture_observed( + &self, + bytes: &[u8], + original_bytes: u64, + limit: u64, + redactions: &[Vec], + ) -> Result<(Option, bool)> { + if bytes.is_empty() { + return Ok((None, false)); + } + let redacted = redact_resolution_bytes(bytes, redactions); + let limit = usize::try_from(limit).unwrap_or(usize::MAX); + let truncated = original_bytes > limit as u64 || redacted.len() > limit; + let capture = ArtifactResolutionCaptureV1 { + version: ARTIFACT_RESOLUTION_PLAN_VERSION, + original_bytes, + truncated, + bytes: redacted[..redacted.len().min(limit)].to_vec(), + }; + Ok(( + Some(self.put_object( + ARTIFACT_RESOLUTION_CAPTURE_KIND, + ARTIFACT_RESOLUTION_PLAN_VERSION, + &capture, + )?), + truncated, + )) + } + + #[allow(clippy::too_many_arguments)] + fn finish_artifact_resolution_attempt_failure_under_write_lock( + &self, + fence: &ArtifactResolutionAttemptFence, + code: &str, + message: &str, + authority_evidence: ArtifactResolutionAuthorityEvidenceV1, + stdout_object_id: Option, + stderr_object_id: Option, + secret_tainted: bool, + status: ArtifactResolutionAttemptStatusV1, + ) -> Result { + if !matches!( + status, + ArtifactResolutionAttemptStatusV1::Failed + | ArtifactResolutionAttemptStatusV1::Cancelled + | ArtifactResolutionAttemptStatusV1::Abandoned + ) { + return Err(Error::InvalidInput( + "artifact resolution failure must use a terminal failure status".into(), + )); + } + let plan = self.artifact_resolution_plan_for_fence(fence)?; + let receipt = ArtifactResolutionFailureReceiptV1 { + version: ARTIFACT_RESOLUTION_PLAN_VERSION, + attempt_id: fence.attempt_id.clone(), + proposal_key: plan.proposal_key, + source_root: plan.source_root, + code: code.to_string(), + message: message.to_string(), + authority_evidence: authority_evidence.clone(), + secret_taint: artifact_secret_taint(secret_tainted, "resolver_credential"), + stdout_object_id: stdout_object_id.clone(), + stderr_object_id: stderr_object_id.clone(), + }; + let receipt_id = self.put_object( + ARTIFACT_RESOLUTION_FAILURE_KIND, + ARTIFACT_RESOLUTION_PLAN_VERSION, + &receipt, + )?; + let status_text = artifact_resolution_attempt_status_str(status); + let updated = self.conn.execute( + "UPDATE artifact_resolution_attempts + SET status=?1, authority_evidence_json=?2, stdout_object_id=?3, + stderr_object_id=?4, failure_receipt_object_id=?5, + failure_code=?6, failure_message=?7, heartbeat_at=?8, finished_at=?8 + WHERE attempt_id=?9 AND owner_generation=?10 AND owner_pid=?11 + AND owner_start_token=?12 AND status='running'", + params![ + status_text, + serde_json::to_vec(&authority_evidence)?, + stdout_object_id.as_ref().map(|id| id.0.as_str()), + stderr_object_id.as_ref().map(|id| id.0.as_str()), + receipt_id.0, + code, + message, + now_ts(), + fence.attempt_id.0, + i64::try_from(fence.owner_generation).map_err(|_| Error::InvalidInput( + "artifact resolution owner generation exceeds SQLite range".into() + ))?, + i64::from(fence.owner_pid), + fence.owner_start_token, + ], + )?; + if updated != 1 { + return Err(Error::InvalidInput(format!( + "artifact resolution attempt `{}` lost its exact owner fence", + fence.attempt_id + ))); + } + self.artifact_resolution_attempt(&fence.attempt_id) + } + + fn get_artifact_cas_object( + &self, + artifact_id: &str, + kind: &'static str, + version: u16, + ) -> Result { + let (object_id, stored_kind, stored_version) = self.conn.query_row( + "SELECT object_id, kind, version FROM artifact_objects WHERE artifact_id=?1", + params![artifact_id], + |row| { + Ok(( + ObjectId(row.get::<_, String>(0)?), + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + }, + )?; + if stored_kind != kind || stored_version != i64::from(version) { + return Err(Error::Corrupt(format!( + "artifact object `{artifact_id}` has kind/version {stored_kind}/{stored_version}, expected {kind}/{version}" + ))); + } + self.get_object(kind, &object_id) + } + + fn put_artifact_cas_object( + &self, + artifact_id: &str, + kind: &'static str, + version: u16, + logical_bytes: u64, + value: &T, + ) -> Result { + let canonical_bytes = cbor(value)?; + let logical_bytes = i64::try_from(logical_bytes).map_err(|_| { + Error::InvalidInput("artifact logical byte count exceeds SQLite range".into()) + })?; + self.conn.execute_batch("SAVEPOINT trail_artifact_object")?; + let publication = (|| -> Result { + let object_id = self.put_object(kind, version, value)?; + let stored = self.conn.query_row( + "SELECT kind, version, bytes FROM objects WHERE object_id=?1", + params![object_id.0], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, Vec>(2)?, + )) + }, + )?; + if stored.0 != kind || stored.1 != i64::from(version) || stored.2 != canonical_bytes { + return Err(Error::Corrupt(format!( + "content-addressed object {} conflicts with artifact `{artifact_id}`", + object_id + ))); + } + let existing = self + .conn + .query_row( + "SELECT object_id, kind, version, logical_bytes + FROM artifact_objects WHERE artifact_id=?1", + params![artifact_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + )) + }, + ) + .optional()?; + if let Some((existing_object, existing_kind, existing_version, existing_bytes)) = + existing + { + if existing_object != object_id.0 + || existing_kind != kind + || existing_version != i64::from(version) + || existing_bytes != logical_bytes + { + return Err(Error::Corrupt(format!( + "artifact ID `{artifact_id}` resolves to conflicting object evidence" + ))); + } + return Ok(object_id); + } + self.conn.execute( + "INSERT INTO artifact_objects( + artifact_id, object_id, kind, version, logical_bytes, created_at + ) VALUES(?1, ?2, ?3, ?4, ?5, ?6)", + params![ + artifact_id, + object_id.0, + kind, + i64::from(version), + logical_bytes, + now_ts(), + ], + )?; + Ok(object_id) + })(); + match publication { + Ok(object_id) => { + self.conn + .execute_batch("RELEASE SAVEPOINT trail_artifact_object")?; + Ok(object_id) + } + Err(error) => { + let _ = self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT trail_artifact_object; + RELEASE SAVEPOINT trail_artifact_object", + ); + Err(error) + } + } + } + + fn ingest_artifact_file_bytes(&self, bytes: &[u8], mode: u32) -> Result { + self.ingest_artifact_file_bytes_with_path(bytes, mode, None) + } + + fn ingest_artifact_file_bytes_with_path( + &self, + bytes: &[u8], + mode: u32, + relative_path: Option<&str>, + ) -> Result { + if mode & !0o777 != 0 { + return Err(Error::InvalidInput(format!( + "artifact file mode {mode:o} contains unsupported bits" + ))); + } + validate_artifact_secret_policy(bytes, relative_path)?; + let complete_hash = sha256_hex(bytes); + let content = if bytes.len() <= ARTIFACT_WHOLE_BLOB_MAX_BYTES { + let blob = ArtifactBlobV1 { + version: ARTIFACT_BLOB_VERSION, + content_sha256: complete_hash.clone(), + bytes: bytes.to_vec(), + }; + let (blob_id, _) = encode_artifact_blob(blob.clone())?; + self.put_artifact_cas_object( + &blob_id.0, + ARTIFACT_BLOB_KIND, + ARTIFACT_BLOB_VERSION, + bytes.len() as u64, + &blob, + )?; + ArtifactFileContentV1::Blob { blob_id } + } else { + let mut chunks = Vec::new(); + for boundary in fastcdc::v2020::FastCDC::new( + bytes, + ARTIFACT_CHUNK_MIN_BYTES, + ARTIFACT_CHUNK_AVERAGE_BYTES, + ARTIFACT_CHUNK_MAX_BYTES, + ) { + let end = boundary + .offset + .checked_add(boundary.length) + .ok_or_else(|| { + Error::InvalidInput("artifact chunk boundary overflow".into()) + })?; + let chunk_bytes = bytes.get(boundary.offset..end).ok_or_else(|| { + Error::Corrupt("FastCDC returned an out-of-range artifact chunk".into()) + })?; + let chunk = ArtifactChunkV1 { + version: ARTIFACT_CHUNK_VERSION, + content_sha256: sha256_hex(chunk_bytes), + bytes: chunk_bytes.to_vec(), + }; + let (chunk_id, _) = encode_artifact_chunk(chunk.clone())?; + self.put_artifact_cas_object( + &chunk_id.0, + ARTIFACT_CHUNK_KIND, + ARTIFACT_CHUNK_VERSION, + chunk_bytes.len() as u64, + &chunk, + )?; + chunks.push(ArtifactChunkRefV1 { + chunk_id, + size_bytes: chunk_bytes.len() as u64, + }); + } + let chunk_list = ArtifactChunkListV1 { + version: ARTIFACT_CHUNK_LIST_VERSION, + algorithm: "fastcdc-v1".into(), + file_size_bytes: bytes.len() as u64, + file_sha256: complete_hash.clone(), + chunks, + }; + let (chunk_list_id, _) = encode_artifact_chunk_list(chunk_list.clone())?; + self.put_artifact_cas_object( + &chunk_list_id.0, + ARTIFACT_CHUNK_LIST_KIND, + ARTIFACT_CHUNK_LIST_VERSION, + bytes.len() as u64, + &chunk_list, + )?; + ArtifactFileContentV1::Chunks { chunk_list_id } + }; + let file = ArtifactFileNodeV1 { + version: ARTIFACT_FILE_NODE_VERSION, + mode, + executable: mode & 0o111 != 0, + size_bytes: bytes.len() as u64, + content_sha256: complete_hash, + content, + }; + let (file_id, _) = encode_artifact_file_node(file.clone())?; + self.put_artifact_cas_object( + &file_id.0, + ARTIFACT_FILE_NODE_KIND, + ARTIFACT_FILE_NODE_VERSION, + bytes.len() as u64, + &file, + )?; + Ok(file_id) + } + + fn ingest_artifact_file_path( + &self, + path: &Path, + relative_path: &str, + mode: u32, + ) -> Result { + let before = fs::symlink_metadata(path)?; + if !before.is_file() { + return Err(Error::InvalidPath { + path: path.to_string_lossy().into_owned(), + reason: "artifact file input changed type during ingestion".into(), + }); + } + if before.len() <= ARTIFACT_WHOLE_BLOB_MAX_BYTES as u64 { + let bytes = fs::read(path)?; + let after = fs::symlink_metadata(path)?; + ensure_artifact_file_unchanged(path, &before, &after, bytes.len() as u64)?; + return self.ingest_artifact_file_bytes_with_path(&bytes, mode, Some(relative_path)); + } + + let mut complete_hasher = Sha256::new(); + let mut chunks = Vec::new(); + for item in fastcdc::v2020::StreamCDC::new( + File::open(path)?, + ARTIFACT_CHUNK_MIN_BYTES, + ARTIFACT_CHUNK_AVERAGE_BYTES, + ARTIFACT_CHUNK_MAX_BYTES, + ) { + let boundary = item.map_err(|error| { + Error::InvalidInput(format!( + "cannot chunk artifact file `{}`: {error}", + path.display() + )) + })?; + validate_artifact_secret_policy(&boundary.data, Some(relative_path))?; + complete_hasher.update(&boundary.data); + let chunk = ArtifactChunkV1 { + version: ARTIFACT_CHUNK_VERSION, + content_sha256: sha256_hex(&boundary.data), + bytes: boundary.data, + }; + let (chunk_id, _) = encode_artifact_chunk(chunk.clone())?; + self.put_artifact_cas_object( + &chunk_id.0, + ARTIFACT_CHUNK_KIND, + ARTIFACT_CHUNK_VERSION, + chunk.bytes.len() as u64, + &chunk, + )?; + chunks.push(ArtifactChunkRefV1 { + chunk_id, + size_bytes: chunk.bytes.len() as u64, + }); + } + let after = fs::symlink_metadata(path)?; + let streamed_bytes = chunks + .iter() + .try_fold(0u64, |total, chunk| total.checked_add(chunk.size_bytes)); + ensure_artifact_file_unchanged( + path, + &before, + &after, + streamed_bytes + .ok_or_else(|| Error::InvalidInput("artifact file size overflow".into()))?, + )?; + let complete_hash = hex::encode(complete_hasher.finalize()); + let chunk_list = ArtifactChunkListV1 { + version: ARTIFACT_CHUNK_LIST_VERSION, + algorithm: "fastcdc-v1".into(), + file_size_bytes: before.len(), + file_sha256: complete_hash.clone(), + chunks, + }; + let (chunk_list_id, _) = encode_artifact_chunk_list(chunk_list.clone())?; + self.put_artifact_cas_object( + &chunk_list_id.0, + ARTIFACT_CHUNK_LIST_KIND, + ARTIFACT_CHUNK_LIST_VERSION, + before.len(), + &chunk_list, + )?; + let node = ArtifactFileNodeV1 { + version: ARTIFACT_FILE_NODE_VERSION, + mode, + executable: mode & 0o111 != 0, + size_bytes: before.len(), + content_sha256: complete_hash, + content: ArtifactFileContentV1::Chunks { chunk_list_id }, + }; + let (file_id, _) = encode_artifact_file_node(node.clone())?; + self.put_artifact_cas_object( + &file_id.0, + ARTIFACT_FILE_NODE_KIND, + ARTIFACT_FILE_NODE_VERSION, + before.len(), + &node, + )?; + Ok(file_id) + } + + fn ingest_artifact_tree(&self, source: &Path) -> Result<(ArtifactTreeId, ArtifactTreeRootV1)> { + let _lock = self.acquire_write_lock()?; + self.ingest_artifact_tree_under_write_lock(source) + } + + pub(crate) fn ingest_artifact_tree_under_write_lock( + &self, + source: &Path, + ) -> Result<(ArtifactTreeId, ArtifactTreeRootV1)> { + let root_before = fs::symlink_metadata(source)?; + if root_before.file_type().is_symlink() || !root_before.is_dir() { + return Err(Error::InvalidPath { + path: source.to_string_lossy().into_owned(), + reason: "artifact tree source must be a real directory".into(), + }); + } + let mut directories = BTreeMap::>::new(); + directories.insert(String::new(), Vec::new()); + let mut entry_count = 0u64; + let mut logical_bytes = 0u64; + let mut case_paths = Vec::new(); + + for entry in walkdir::WalkDir::new(source) + .follow_links(false) + .max_depth(MAX_ARTIFACT_TREE_DEPTH + 1) + { + let entry = entry.map_err(|error| Error::InvalidInput(error.to_string()))?; + if entry.depth() == 0 { + continue; + } + if entry.depth() > MAX_ARTIFACT_TREE_DEPTH { + return Err(Error::InvalidInput(format!( + "artifact tree exceeds maximum depth {MAX_ARTIFACT_TREE_DEPTH}" + ))); + } + entry_count = entry_count + .checked_add(1) + .ok_or_else(|| Error::InvalidInput("artifact tree entry count overflow".into()))?; + if entry_count > MAX_ARTIFACT_TREE_ENTRIES { + return Err(Error::InvalidInput(format!( + "artifact tree exceeds {MAX_ARTIFACT_TREE_ENTRIES} entries" + ))); + } + let relative = entry + .path() + .strip_prefix(source) + .map_err(|_| Error::InvalidPath { + path: entry.path().to_string_lossy().into_owned(), + reason: "artifact walk escaped its source root".into(), + })?; + let relative = relative.to_str().ok_or_else(|| Error::InvalidPath { + path: relative.to_string_lossy().into_owned(), + reason: "artifact paths must be valid Unicode".into(), + })?; + let relative = normalize_relative_path(relative)?; + case_paths.push(relative.clone()); + let (parent, name) = relative.rsplit_once('/').unwrap_or(("", &relative)); + validate_artifact_entry_name(name)?; + directories.entry(parent.to_string()).or_default(); + let file_type = entry.file_type(); + validate_artifact_metadata_policy(entry.path(), &fs::symlink_metadata(entry.path())?)?; + if file_type.is_dir() { + directories.entry(relative).or_default(); + } else if file_type.is_file() { + let metadata = fs::symlink_metadata(entry.path())?; + logical_bytes = logical_bytes.checked_add(metadata.len()).ok_or_else(|| { + Error::InvalidInput("artifact tree logical byte count overflow".into()) + })?; + if logical_bytes > MAX_ARTIFACT_TREE_LOGICAL_BYTES { + return Err(Error::InvalidInput(format!( + "artifact tree exceeds {MAX_ARTIFACT_TREE_LOGICAL_BYTES} logical bytes" + ))); + } + let file_id = self.ingest_artifact_file_path( + entry.path(), + &relative, + normalized_artifact_file_mode(&metadata), + )?; + directories + .get_mut(parent) + .unwrap() + .push(ArtifactDirectoryEntryV1 { + name: name.to_string(), + target: ArtifactDirectoryEntryTargetV1::File { node_id: file_id }, + }); + } else if file_type.is_symlink() { + let target = fs::read_link(entry.path())?; + let target = target.to_str().ok_or_else(|| Error::InvalidPath { + path: entry.path().to_string_lossy().into_owned(), + reason: "artifact symlink targets must be valid Unicode".into(), + })?; + validate_artifact_symlink_within_tree(parent, target)?; + directories + .get_mut(parent) + .unwrap() + .push(ArtifactDirectoryEntryV1 { + name: name.to_string(), + target: ArtifactDirectoryEntryTargetV1::Symlink { + target: target.to_string(), + }, + }); + } else { + return Err(Error::InvalidPath { + path: entry.path().to_string_lossy().into_owned(), + reason: "artifact trees support only directories, regular files, and confined symlinks".into(), + }); + } + } + validate_no_case_fold_collisions(&case_paths)?; + + let mut paths = directories.keys().cloned().collect::>(); + paths.sort_by(|left, right| { + right + .split('/') + .count() + .cmp(&left.split('/').count()) + .then_with(|| left.cmp(right)) + }); + for path in paths.into_iter().filter(|path| !path.is_empty()) { + let node = canonical_artifact_directory_node(ArtifactDirectoryNodeV1 { + version: ARTIFACT_DIRECTORY_NODE_VERSION, + entries: directories.remove(&path).unwrap_or_default(), + })?; + let (node_id, _) = encode_artifact_directory_node(node.clone())?; + self.put_artifact_cas_object( + &node_id.0, + ARTIFACT_DIRECTORY_NODE_KIND, + ARTIFACT_DIRECTORY_NODE_VERSION, + 0, + &node, + )?; + let (parent, name) = path.rsplit_once('/').unwrap_or(("", path.as_str())); + directories + .get_mut(parent) + .unwrap() + .push(ArtifactDirectoryEntryV1 { + name: name.to_string(), + target: ArtifactDirectoryEntryTargetV1::Directory { node_id }, + }); + } + let root_node = canonical_artifact_directory_node(ArtifactDirectoryNodeV1 { + version: ARTIFACT_DIRECTORY_NODE_VERSION, + entries: directories.remove("").unwrap_or_default(), + })?; + let (root_directory_id, _) = encode_artifact_directory_node(root_node.clone())?; + self.put_artifact_cas_object( + &root_directory_id.0, + ARTIFACT_DIRECTORY_NODE_KIND, + ARTIFACT_DIRECTORY_NODE_VERSION, + logical_bytes, + &root_node, + )?; + let tree = ArtifactTreeRootV1 { + version: ARTIFACT_TREE_ROOT_VERSION, + root_directory_id, + logical_bytes, + entry_count, + path_normalizer: "trail-paths/v1".into(), + }; + let (tree_id, _) = encode_artifact_tree_root(tree.clone())?; + self.put_artifact_cas_object( + &tree_id.0, + ARTIFACT_TREE_ROOT_KIND, + ARTIFACT_TREE_ROOT_VERSION, + logical_bytes, + &tree, + )?; + let root_after = fs::symlink_metadata(source)?; + if !same_artifact_metadata(&root_before, &root_after) { + return Err(Error::InvalidInput( + "artifact tree root changed during ingestion".into(), + )); + } + Ok((tree_id, tree)) + } + + pub(crate) fn artifact_tree_flat_entries( + &self, + tree_id: &ArtifactTreeId, + ) -> Result> { + let tree: ArtifactTreeRootV1 = self.get_artifact_cas_object( + &tree_id.0, + ARTIFACT_TREE_ROOT_KIND, + ARTIFACT_TREE_ROOT_VERSION, + )?; + let (actual_tree_id, _) = encode_artifact_tree_root(tree.clone())?; + if &actual_tree_id != tree_id { + return Err(Error::Corrupt(format!( + "artifact tree root `{tree_id}` has conflicting encoded identity" + ))); + } + let mut entries = BTreeMap::new(); + let mut visiting = BTreeSet::new(); + self.flatten_artifact_directory( + &tree.root_directory_id, + "", + 0, + &mut visiting, + &mut entries, + )?; + let logical_bytes = entries + .values() + .try_fold(0u64, |total, entry| total.checked_add(entry.size_bytes)); + if entries.len() as u64 != tree.entry_count || logical_bytes != Some(tree.logical_bytes) { + return Err(Error::Corrupt(format!( + "artifact tree `{tree_id}` count or logical-byte edge is invalid" + ))); + } + Ok(entries) + } + + pub(crate) fn artifact_tree_object_ids( + &self, + tree_id: &ArtifactTreeId, + ) -> Result> { + let tree: ArtifactTreeRootV1 = self.get_artifact_cas_object( + &tree_id.0, + ARTIFACT_TREE_ROOT_KIND, + ARTIFACT_TREE_ROOT_VERSION, + )?; + let mut objects = + BTreeSet::from([self.artifact_backing_object_id(&tree_id.0, ARTIFACT_TREE_ROOT_KIND)?]); + let mut visited_directories = BTreeSet::new(); + self.collect_artifact_directory_object_ids( + &tree.root_directory_id, + 0, + &mut visited_directories, + &mut objects, + )?; + Ok(objects) + } + + pub(crate) fn artifact_envelope_object_ids( + &self, + envelope_id: &ArtifactEnvelopeId, + ) -> Result> { + let envelope: ArtifactEnvelopeV1 = self.get_artifact_cas_object( + &envelope_id.0, + ARTIFACT_ENVELOPE_KIND, + ARTIFACT_ENVELOPE_VERSION, + )?; + let mut objects = self.artifact_tree_object_ids(&envelope.tree_root_id)?; + objects.insert(self.artifact_backing_object_id(&envelope_id.0, ARTIFACT_ENVELOPE_KIND)?); + objects.extend( + envelope + .validation_receipt_ids + .into_iter() + .map(|object_id| object_id.0), + ); + if let Some(snapshot_id) = envelope.resolution_snapshot_id { + self.collect_artifact_resolution_snapshot_object_ids(&snapshot_id, &mut objects)?; + } + let mut statement = self.conn.prepare( + "SELECT object_id FROM artifact_attestations + WHERE envelope_id=?1 ORDER BY attestation_id", + )?; + for row in statement.query_map(params![envelope_id.0], |row| row.get::<_, String>(0))? { + objects.insert(row?); + } + Ok(objects) + } + + fn artifact_backing_object_id(&self, artifact_id: &str, expected_kind: &str) -> Result { + let (object_id, kind) = self + .conn + .query_row( + "SELECT object_id,kind FROM artifact_objects WHERE artifact_id=?1", + params![artifact_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .optional()? + .ok_or_else(|| { + Error::Corrupt(format!( + "artifact object `{artifact_id}` is missing its backing object" + )) + })?; + if kind != expected_kind { + return Err(Error::Corrupt(format!( + "artifact object `{artifact_id}` has kind {kind}, expected {expected_kind}" + ))); + } + Ok(object_id) + } + + fn collect_artifact_directory_object_ids( + &self, + directory_id: &ArtifactTreeId, + depth: usize, + visited_directories: &mut BTreeSet, + objects: &mut BTreeSet, + ) -> Result<()> { + if depth > MAX_ARTIFACT_TREE_DEPTH { + return Err(Error::Corrupt( + "artifact accounting exceeded the directory-depth bound".into(), + )); + } + if !visited_directories.insert(directory_id.clone()) { + return Ok(()); + } + objects.insert( + self.artifact_backing_object_id(&directory_id.0, ARTIFACT_DIRECTORY_NODE_KIND)?, + ); + let directory: ArtifactDirectoryNodeV1 = self.get_artifact_cas_object( + &directory_id.0, + ARTIFACT_DIRECTORY_NODE_KIND, + ARTIFACT_DIRECTORY_NODE_VERSION, + )?; + for entry in directory.entries { + match entry.target { + ArtifactDirectoryEntryTargetV1::Directory { node_id } => { + self.collect_artifact_directory_object_ids( + &node_id, + depth + 1, + visited_directories, + objects, + )?; + } + ArtifactDirectoryEntryTargetV1::File { node_id } => { + objects.insert( + self.artifact_backing_object_id(&node_id.0, ARTIFACT_FILE_NODE_KIND)?, + ); + let file: ArtifactFileNodeV1 = self.get_artifact_cas_object( + &node_id.0, + ARTIFACT_FILE_NODE_KIND, + ARTIFACT_FILE_NODE_VERSION, + )?; + match file.content { + ArtifactFileContentV1::Blob { blob_id } => { + objects.insert( + self.artifact_backing_object_id(&blob_id.0, ARTIFACT_BLOB_KIND)?, + ); + } + ArtifactFileContentV1::Chunks { chunk_list_id } => { + objects.insert(self.artifact_backing_object_id( + &chunk_list_id.0, + ARTIFACT_CHUNK_LIST_KIND, + )?); + let list: ArtifactChunkListV1 = self.get_artifact_cas_object( + &chunk_list_id.0, + ARTIFACT_CHUNK_LIST_KIND, + ARTIFACT_CHUNK_LIST_VERSION, + )?; + for chunk in list.chunks { + objects.insert(self.artifact_backing_object_id( + &chunk.chunk_id.0, + ARTIFACT_CHUNK_KIND, + )?); + } + } + } + } + ArtifactDirectoryEntryTargetV1::Symlink { .. } => {} + } + } + Ok(()) + } + + fn collect_artifact_resolution_snapshot_object_ids( + &self, + snapshot_id: &ObjectId, + objects: &mut BTreeSet, + ) -> Result<()> { + let mut pending = BTreeSet::from([snapshot_id.clone()]); + let mut visited = BTreeSet::new(); + while let Some(snapshot_id) = pending.pop_first() { + if !visited.insert(snapshot_id.clone()) { + continue; + } + if visited.len() > MAX_RESOLUTION_PREDECESSORS { + return Err(Error::Corrupt( + "artifact resolution predecessor graph exceeds its bound".into(), + )); + } + let snapshot: ArtifactResolutionSnapshotV1 = + self.get_object(ARTIFACT_RESOLUTION_SNAPSHOT_KIND, &snapshot_id)?; + objects.insert(snapshot_id.0); + objects.insert(snapshot.content_object_id.0); + pending.extend(snapshot.predecessor_snapshot_id); + } + Ok(()) + } + + pub(crate) fn artifact_storage_accounting( + &self, + view_id: Option<&str>, + lane_private_bytes: u64, + demand_loaded_bytes: u64, + reclaimable_bytes: u64, + measured_unknown_bytes: u64, + ) -> Result { + const MAX_ACCOUNTING_ENVELOPES: usize = 10_000; + const MAX_ACCOUNTING_TREES: usize = 10_000; + const MAX_ACCOUNTING_OBJECTS: usize = 10_000_000; + const MAX_ACCOUNTING_OBJECT_REFERENCES: usize = 10_000_000; + + let integrity_errors = self.validate_artifact_cas_integrity()?; + if !integrity_errors.is_empty() { + return Err(Error::Corrupt(integrity_errors.join("; "))); + } + let envelopes = { + let mut statement = self.conn.prepare( + "SELECT envelope_id,tree_root_id FROM artifact_envelopes + ORDER BY envelope_id", + )?; + statement + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()? + }; + if envelopes.len() > MAX_ACCOUNTING_ENVELOPES { + return Err(Error::InvalidInput(format!( + "artifact accounting contains {} envelopes; maximum is {MAX_ACCOUNTING_ENVELOPES}", + envelopes.len() + ))); + } + + let mut envelope_graphs = BTreeMap::>::new(); + let mut object_reference_counts = BTreeMap::::new(); + let mut reference_count = 0_usize; + for (envelope_id, _) in &envelopes { + let envelope_id = ArtifactEnvelopeId::parse(envelope_id.clone()).map_err(|error| { + Error::Corrupt(format!("invalid artifact envelope ID: {error}")) + })?; + let graph = self.artifact_envelope_object_ids(&envelope_id)?; + reference_count = reference_count.checked_add(graph.len()).ok_or_else(|| { + Error::InvalidInput("artifact accounting reference count overflowed".into()) + })?; + if reference_count > MAX_ACCOUNTING_OBJECT_REFERENCES { + return Err(Error::InvalidInput(format!( + "artifact accounting contains more than {MAX_ACCOUNTING_OBJECT_REFERENCES} object references" + ))); + } + for object_id in &graph { + let count = object_reference_counts + .entry(object_id.clone()) + .or_default(); + *count = count.saturating_add(1); + } + envelope_graphs.insert(envelope_id.0, graph); + } + + let (selected_envelopes, mut selected_trees) = if let Some(view_id) = view_id { + let mut statement = self.conn.prepare( + "SELECT b.envelope_id,b.tree_root_id + FROM environment_view_generations v + JOIN artifact_generation_bindings b ON b.generation_id=v.generation_id + WHERE v.view_id=?1 + UNION + SELECT s.envelope_id,s.tree_root_id + FROM workspace_view_layers l + JOIN workspace_layer_artifact_shadows s ON s.layer_id=l.layer_id + WHERE l.view_id=?1 + ORDER BY envelope_id,tree_root_id", + )?; + let rows = statement + .query_map(params![view_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()?; + ( + rows.iter() + .map(|(envelope_id, _)| envelope_id.clone()) + .collect::>(), + rows.into_iter() + .map(|(_, tree_root_id)| tree_root_id) + .collect::>(), + ) + } else { + ( + envelopes + .iter() + .map(|(envelope_id, _)| envelope_id.clone()) + .collect::>(), + envelopes + .iter() + .map(|(_, tree_root_id)| tree_root_id.clone()) + .collect::>(), + ) + }; + + let mut selected_objects = BTreeSet::::new(); + for envelope_id in &selected_envelopes { + let graph = envelope_graphs.get(envelope_id).ok_or_else(|| { + Error::Corrupt(format!( + "selected artifact envelope `{envelope_id}` is missing its accounting graph" + )) + })?; + selected_objects.extend(graph.iter().cloned()); + } + if view_id.is_none() { + let mut statement = self + .conn + .prepare("SELECT object_id FROM artifact_objects ORDER BY object_id")?; + for row in statement.query_map([], |row| row.get::<_, String>(0))? { + selected_objects.insert(row?); + } + let mut statement = self.conn.prepare( + "SELECT object_id FROM objects WHERE kind IN ( + ?1,?2,?3,?4,?5,?6,?7 + ) ORDER BY object_id", + )?; + for row in statement.query_map( + params![ + ARTIFACT_RESOLUTION_SNAPSHOT_KIND, + ARTIFACT_RESOLUTION_CONTENT_KIND, + ARTIFACT_RESOLUTION_PLAN_KIND, + ARTIFACT_RESOLUTION_CAPTURE_KIND, + ARTIFACT_RESOLUTION_FAILURE_KIND, + ARTIFACT_DIVERGENCE_EVIDENCE_KIND, + ARTIFACT_VALIDATION_RECEIPT_KIND, + ], + |row| row.get::<_, String>(0), + )? { + selected_objects.insert(row?); + } + let mut statement = self.conn.prepare( + "SELECT artifact_id FROM artifact_objects + WHERE kind=?1 ORDER BY artifact_id", + )?; + for row in statement.query_map(params![ARTIFACT_TREE_ROOT_KIND], |row| { + row.get::<_, String>(0) + })? { + selected_trees.insert(row?); + } + } + if selected_trees.len() > MAX_ACCOUNTING_TREES { + return Err(Error::InvalidInput(format!( + "artifact accounting contains {} selected trees; maximum is {MAX_ACCOUNTING_TREES}", + selected_trees.len() + ))); + } + if selected_objects.len() > MAX_ACCOUNTING_OBJECTS { + return Err(Error::InvalidInput(format!( + "artifact accounting contains {} selected objects; maximum is {MAX_ACCOUNTING_OBJECTS}", + selected_objects.len() + ))); + } + + let mut unique_authoritative_bytes = 0_u64; + let mut cross_artifact_shared_bytes = 0_u64; + for chunk in selected_objects.iter().collect::>().chunks(512) { + let placeholders = std::iter::repeat_n("?", chunk.len()) + .collect::>() + .join(","); + let sql = format!( + "SELECT object_id,size_bytes FROM objects WHERE object_id IN ({placeholders})" + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map( + params_from_iter(chunk.iter().map(|object_id| object_id.as_str())), + |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)), + )?; + for row in rows { + let (object_id, size_bytes) = row?; + let size_bytes = u64::try_from(size_bytes).map_err(|_| { + Error::Corrupt(format!( + "artifact accounting object `{object_id}` has negative bytes" + )) + })?; + if object_reference_counts + .get(&object_id) + .copied() + .unwrap_or(1) + > 1 + { + cross_artifact_shared_bytes = + cross_artifact_shared_bytes.saturating_add(size_bytes); + } else { + unique_authoritative_bytes = + unique_authoritative_bytes.saturating_add(size_bytes); + } + } + } + + let mut logical_bytes = 0_u64; + for tree_root_id in &selected_trees { + let bytes = self.conn.query_row( + "SELECT logical_bytes FROM artifact_objects + WHERE artifact_id=?1 AND kind=?2", + params![tree_root_id, ARTIFACT_TREE_ROOT_KIND], + |row| row.get::<_, i64>(0), + )?; + logical_bytes = logical_bytes.saturating_add(u64::try_from(bytes).map_err(|_| { + Error::Corrupt(format!( + "artifact tree `{tree_root_id}` has negative logical bytes" + )) + })?); + } + + let materialized_bytes = self.artifact_materialized_bytes(view_id, &selected_trees)?; + Ok(ArtifactStorageAccountingReport { + logical_bytes, + unique_authoritative_bytes, + cross_artifact_shared_bytes, + materialized_bytes, + lane_private_bytes, + prefetched_bytes: 0, + demand_loaded_bytes, + reclaimable_bytes, + unknown_bytes: measured_unknown_bytes, + accounting: format!( + "scope={};axes=logical|authoritative|physical|disposition;authoritative=encoded-cbor-bytes-deduplicated;filesystem=allocated-blocks-or-file-size-estimate;reclaimable=overlapping-disposition;prefetch=os-page-cache-excluded", + if view_id.is_some() { "lane" } else { "workspace" } + ), + }) + } + + /// Return workspace-wide CAS accounting without requiring a lane view. + pub fn workspace_artifact_space(&self) -> Result { + let envelope_count = + self.conn + .query_row("SELECT COUNT(*) FROM artifact_envelopes", [], |row| { + row.get::<_, i64>(0) + })?; + let active_quarantine_count = self.conn.query_row( + "SELECT COUNT(*) FROM artifact_quarantines WHERE state='active'", + [], + |row| row.get::<_, i64>(0), + )?; + Ok(ArtifactSpaceReportV1 { + scope: "workspace".into(), + envelope_count: u64::try_from(envelope_count) + .map_err(|_| Error::Corrupt("negative artifact envelope count".into()))?, + active_quarantine_count: u64::try_from(active_quarantine_count) + .map_err(|_| Error::Corrupt("negative artifact quarantine count".into()))?, + storage: self.artifact_storage_accounting(None, 0, 0, 0, 0)?, + }) + } + + pub(crate) fn artifact_envelope_ids(&self) -> Result> { + let mut statement = self + .conn + .prepare("SELECT envelope_id FROM artifact_envelopes ORDER BY envelope_id")?; + statement + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>() + .map_err(Error::from) + } + + /// Traverse the durable object graph reachable from one artifact envelope. + /// The report is a bounded summary; object identifiers remain private storage + /// details and are not expanded into an unbounded public payload. + pub fn artifact_content_reachability( + &self, + envelope_id: &ArtifactEnvelopeId, + ) -> Result { + let tree_root_id = self.artifact_envelope_tree_id(envelope_id)?; + let object_ids = self.artifact_envelope_object_ids(envelope_id)?; + let objects = self.artifact_object_storage_rows(&object_ids)?; + let mut kinds = BTreeMap::::new(); + let mut encoded_bytes = 0_u64; + for (_, kind, size_bytes) in objects.values() { + encoded_bytes = encoded_bytes.saturating_add(*size_bytes); + let entry = kinds.entry(kind.clone()).or_default(); + entry.0 = entry.0.saturating_add(1); + entry.1 = entry.1.saturating_add(*size_bytes); + } + let tree: ArtifactTreeRootV1 = self.get_artifact_cas_object( + &tree_root_id.0, + ARTIFACT_TREE_ROOT_KIND, + ARTIFACT_TREE_ROOT_VERSION, + )?; + Ok(ArtifactContentReachabilityReportV1 { + envelope_id: envelope_id.clone(), + tree_root_id, + object_count: object_ids.len() as u64, + encoded_bytes, + logical_bytes: tree.logical_bytes, + by_kind: kinds + .into_iter() + .map( + |(kind, (object_count, encoded_bytes))| ArtifactReachabilityKindReportV1 { + kind, + object_count, + encoded_bytes, + }, + ) + .collect(), + complete: true, + recovery_commands: Vec::new(), + }) + } + + /// Inspect one immutable artifact and all public lifecycle evidence bound to it. + pub fn inspect_artifact( + &self, + envelope_id: &ArtifactEnvelopeId, + ) -> Result { + let (desired_key, trust_scope, tree_root_id, object_id, state, verification_state) = self + .conn + .query_row( + "SELECT desired_key,trust_scope,tree_root_id,object_id,state,verification_state + FROM artifact_envelopes WHERE envelope_id=?1", + params![envelope_id.0], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + )) + }, + ) + .optional()? + .ok_or_else(|| Error::ObjectNotFound { + kind: "artifact envelope", + id: envelope_id.0.clone(), + })?; + let tree_root_id = ArtifactTreeId::parse(tree_root_id) + .map_err(|error| Error::Corrupt(format!("invalid artifact tree ID: {error}")))?; + let object_id = ObjectId(object_id); + let envelope: ArtifactEnvelopeV1 = self.get_artifact_cas_object( + &envelope_id.0, + ARTIFACT_ENVELOPE_KIND, + ARTIFACT_ENVELOPE_VERSION, + )?; + let backing_object_id = + self.artifact_backing_object_id(&envelope_id.0, ARTIFACT_ENVELOPE_KIND)?; + let (actual_envelope_id, _) = encode_artifact_envelope(envelope.clone())?; + let encoded_desired_key = match &envelope.desired_identity { + ArtifactDesiredIdentityV1::WorkspaceLayerV1 { cache_key, .. } => cache_key, + ArtifactDesiredIdentityV1::ArtifactDesiredV2 { desired_key } => &desired_key.0, + }; + if actual_envelope_id != *envelope_id + || envelope.tree_root_id != tree_root_id + || encoded_desired_key != &desired_key + || envelope.trust_scope != trust_scope + || backing_object_id != object_id.0 + { + return Err(Error::Corrupt(format!( + "artifact envelope `{envelope_id}` database identity disagrees with its object" + ))); + } + + let bindings = self.artifact_generation_bindings(envelope_id)?; + let attestations = self.artifact_attestations_for_envelope(envelope_id)?; + let quarantines = self + .list_artifact_quarantines()? + .into_iter() + .filter(|record| { + record.incumbent_envelope_id.as_ref() == Some(envelope_id) + || record.candidate_envelope_id == *envelope_id + }) + .collect::>(); + let active_quarantine = quarantines.iter().find(|record| record.state == "active"); + let quarantine_state = if active_quarantine.is_some() { + "active" + } else if quarantines.is_empty() { + "none" + } else { + "resolved" + }; + let trust_state = if attestations.is_empty() { + "missing" + } else if attestations.iter().all(|report| { + self.verify_artifact_attestation(&report.attestation_id) + .is_ok_and(|verification| verification.valid) + }) { + "trusted" + } else { + "untrusted" + }; + let recovery_commands = active_quarantine + .map(|record| { + vec![format!( + "trail env artifact quarantine show {}", + record.quarantine_id + )] + }) + .unwrap_or_default(); + let reachability = self.artifact_content_reachability(envelope_id)?; + let storage = + self.artifact_envelope_storage_accounting(envelope_id, &tree_root_id, &state)?; + Ok(ArtifactInspectionReportV1 { + envelope_id: envelope_id.clone(), + object_id, + desired_key, + tree_root_id, + state, + verification_state, + trust_state: trust_state.into(), + quarantine_state: quarantine_state.into(), + envelope, + bindings, + attestations, + quarantines, + reachability, + storage, + recovery_commands, + }) + } + + /// Verify one artifact at an explicit evidence level. `reproduce` validates + /// durable reproducibility evidence; executing a fresh producer remains a + /// separate managed construction operation. + pub fn verify_artifact( + &self, + envelope_id: &ArtifactEnvelopeId, + level: ArtifactVerificationLevelV1, + ) -> Result { + let inspection = self.inspect_artifact(envelope_id)?; + let mut diagnostics = Vec::new(); + let content_identity_valid = true; + let validation_receipts_valid = self + .validate_envelope_validation_receipts(&inspection.envelope) + .is_ok(); + if !validation_receipts_valid { + diagnostics.push("artifact validation receipts are missing, failed, or stale".into()); + } + let attestation_verifications = inspection + .attestations + .iter() + .map(|attestation| self.verify_artifact_attestation(&attestation.attestation_id)) + .collect::>>()?; + let attestations_valid = !attestation_verifications.is_empty() + && attestation_verifications + .iter() + .all(|verification| verification.valid); + if !attestations_valid { + diagnostics.push("artifact attestation trust or binding verification failed".into()); + } + + let object_ids = self.artifact_envelope_object_ids(envelope_id)?; + let tree_integrity = match level { + ArtifactVerificationLevelV1::Attach => self + .verified_artifact_tree_root(&inspection.tree_root_id) + .map(|_| ()), + ArtifactVerificationLevelV1::Sample => { + let sorted = object_ids.iter().collect::>(); + let mut sampled = BTreeSet::new(); + if let Some(first) = sorted.first() { + sampled.insert((*first).clone()); + } + if let Some(middle) = sorted.get(sorted.len() / 2) { + sampled.insert((*middle).clone()); + } + if let Some(last) = sorted.last() { + sampled.insert((*last).clone()); + } + self.verify_artifact_object_set(&sampled) + } + ArtifactVerificationLevelV1::Full | ArtifactVerificationLevelV1::Reproduce => { + self.verify_artifact_object_set(&object_ids).and_then(|_| { + self.artifact_tree_flat_entries(&inspection.tree_root_id) + .map(drop) + }) + } + }; + let tree_integrity_valid = tree_integrity.is_ok(); + if let Err(error) = tree_integrity { + diagnostics.push(format!( + "artifact tree integrity verification failed: {error}" + )); + } + + let reproduction_evidence_valid = if level == ArtifactVerificationLevelV1::Reproduce { + let mut found = false; + for receipt_id in &inspection.envelope.validation_receipt_ids { + let receipt = self.artifact_validation_receipt(receipt_id)?; + if receipt.declaration.kind == ArtifactValidationKindV1::Reproducibility + && receipt.outcome == ArtifactValidationOutcomeV1::Passed + { + found = true; + } + } + if !found { + diagnostics.push( + "artifact has no passed reproducibility validation receipt; run a managed reproducibility construction before trusting this level" + .into(), + ); + } + Some(found) + } else { + None + }; + + if inspection.state != "ready" { + diagnostics.push(format!( + "artifact envelope state is `{}`, not `ready`", + inspection.state + )); + } + if inspection.verification_state != "verified" { + diagnostics.push(format!( + "artifact verification state is `{}`, not `verified`", + inspection.verification_state + )); + } + if inspection.quarantine_state == "active" { + diagnostics.push("artifact desired identity is actively quarantined".into()); + } + if inspection.trust_state != "trusted" { + diagnostics.push(format!( + "artifact producer trust state is `{}`", + inspection.trust_state + )); + } + let valid = inspection.state == "ready" + && inspection.verification_state == "verified" + && inspection.quarantine_state != "active" + && inspection.trust_state == "trusted" + && content_identity_valid + && tree_integrity_valid + && validation_receipts_valid + && attestations_valid + && reproduction_evidence_valid.unwrap_or(true); + Ok(ArtifactVerificationReportV1 { + envelope_id: envelope_id.clone(), + level, + desired_key: inspection.desired_key, + tree_root_id: inspection.tree_root_id, + envelope_state: inspection.state, + verification_state: inspection.verification_state, + trust_state: inspection.trust_state, + quarantine_state: inspection.quarantine_state, + content_identity_valid, + tree_integrity_valid, + validation_receipts_valid, + attestations_valid, + reproduction_evidence_valid, + valid, + diagnostics, + recovery_commands: inspection.recovery_commands, + reachability: inspection.reachability, + storage: inspection.storage, + }) + } + + fn artifact_envelope_tree_id( + &self, + envelope_id: &ArtifactEnvelopeId, + ) -> Result { + let tree_root_id = self + .conn + .query_row( + "SELECT tree_root_id FROM artifact_envelopes WHERE envelope_id=?1", + params![envelope_id.0], + |row| row.get::<_, String>(0), + ) + .optional()? + .ok_or_else(|| Error::ObjectNotFound { + kind: "artifact envelope", + id: envelope_id.0.clone(), + })?; + ArtifactTreeId::parse(tree_root_id) + .map_err(|error| Error::Corrupt(format!("invalid artifact tree ID: {error}"))) + } + + fn artifact_generation_bindings( + &self, + envelope_id: &ArtifactEnvelopeId, + ) -> Result> { + let mut statement = self.conn.prepare( + "SELECT binding_id,generation_id,component_id,output_name,desired_key, + envelope_id,tree_root_id,binding_identity,created_at + FROM artifact_generation_bindings WHERE envelope_id=?1 + ORDER BY generation_id,component_id,output_name,binding_id", + )?; + let rows = statement + .query_map(params![envelope_id.0], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, String>(6)?, + row.get::<_, String>(7)?, + row.get::<_, i64>(8)?, + )) + })? + .collect::, _>>()?; + if rows.len() > MAX_PUBLIC_ARTIFACT_REPORT_ITEMS { + return Err(Error::InvalidInput(format!( + "artifact inspection contains {} generation bindings; maximum is {MAX_PUBLIC_ARTIFACT_REPORT_ITEMS}", + rows.len() + ))); + } + rows.into_iter() + .map( + |( + binding_id, + generation_id, + component_id, + output_name, + desired_key, + envelope_id, + tree_root_id, + binding_identity, + created_at, + )| { + Ok(ArtifactGenerationBindingReportV1 { + binding_id, + generation_id, + component_id, + output_name, + desired_key, + envelope_id: ArtifactEnvelopeId::parse(envelope_id) + .map_err(Error::Corrupt)?, + tree_root_id: ArtifactTreeId::parse(tree_root_id) + .map_err(Error::Corrupt)?, + binding_identity, + created_at, + }) + }, + ) + .collect() + } + + pub(crate) fn artifact_generation_bindings_for_generation( + &self, + generation_id: &str, + ) -> Result> { + let mut statement = self.conn.prepare( + "SELECT binding_id,generation_id,component_id,output_name,desired_key, + envelope_id,tree_root_id,binding_identity,created_at + FROM artifact_generation_bindings WHERE generation_id=?1 + ORDER BY component_id,output_name,binding_id", + )?; + let rows = statement + .query_map(params![generation_id], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, String>(6)?, + row.get::<_, String>(7)?, + row.get::<_, i64>(8)?, + )) + })? + .collect::, _>>()?; + if rows.len() > MAX_PUBLIC_ARTIFACT_REPORT_ITEMS { + return Err(Error::InvalidInput(format!( + "environment generation `{generation_id}` contains {} artifact bindings; maximum is {MAX_PUBLIC_ARTIFACT_REPORT_ITEMS}", + rows.len() + ))); + } + rows.into_iter() + .map( + |( + binding_id, + generation_id, + component_id, + output_name, + desired_key, + envelope_id, + tree_root_id, + binding_identity, + created_at, + )| { + Ok(ArtifactGenerationBindingReportV1 { + binding_id, + generation_id, + component_id, + output_name, + desired_key, + envelope_id: ArtifactEnvelopeId::parse(envelope_id) + .map_err(Error::Corrupt)?, + tree_root_id: ArtifactTreeId::parse(tree_root_id) + .map_err(Error::Corrupt)?, + binding_identity, + created_at, + }) + }, + ) + .collect() + } + + fn artifact_object_storage_rows( + &self, + object_ids: &BTreeSet, + ) -> Result> { + let mut objects = BTreeMap::new(); + let ids = object_ids.iter().collect::>(); + for chunk in ids.chunks(512) { + let placeholders = std::iter::repeat_n("?", chunk.len()) + .collect::>() + .join(","); + let sql = format!( + "SELECT object_id,kind,size_bytes FROM objects WHERE object_id IN ({placeholders})" + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map( + params_from_iter(chunk.iter().map(|object_id| object_id.as_str())), + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + }, + )?; + for row in rows { + let (object_id, kind, size_bytes) = row?; + let size_bytes = u64::try_from(size_bytes).map_err(|_| { + Error::Corrupt(format!("artifact object `{object_id}` has negative bytes")) + })?; + objects.insert(object_id.clone(), (ObjectId(object_id), kind, size_bytes)); + } + } + if objects.len() != object_ids.len() { + let missing = object_ids + .iter() + .find(|object_id| !objects.contains_key(*object_id)) + .cloned() + .unwrap_or_else(|| "unknown".into()); + return Err(Error::Corrupt(format!( + "artifact reachability references missing object `{missing}`" + ))); + } + Ok(objects) + } + + fn artifact_envelope_storage_accounting( + &self, + envelope_id: &ArtifactEnvelopeId, + tree_root_id: &ArtifactTreeId, + envelope_state: &str, + ) -> Result { + const MAX_INSPECTION_ENVELOPES: usize = 10_000; + let selected = self.artifact_envelope_object_ids(envelope_id)?; + let selected_rows = self.artifact_object_storage_rows(&selected)?; + let envelope_ids = { + let mut statement = self + .conn + .prepare("SELECT envelope_id FROM artifact_envelopes ORDER BY envelope_id")?; + statement + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()? + }; + if envelope_ids.len() > MAX_INSPECTION_ENVELOPES { + return Err(Error::InvalidInput(format!( + "artifact inspection contains {} envelopes; maximum is {MAX_INSPECTION_ENVELOPES}", + envelope_ids.len() + ))); + } + let mut reference_counts = BTreeMap::::new(); + let mut reference_count = 0_usize; + for id in envelope_ids { + let id = ArtifactEnvelopeId::parse(id).map_err(Error::Corrupt)?; + let reachable = self.artifact_envelope_object_ids(&id)?; + reference_count = reference_count + .checked_add(reachable.len()) + .ok_or_else(|| { + Error::InvalidInput("artifact inspection reference count overflowed".into()) + })?; + if reference_count > MAX_PUBLIC_ARTIFACT_OBJECT_REFERENCES { + return Err(Error::InvalidInput(format!( + "artifact inspection contains more than {MAX_PUBLIC_ARTIFACT_OBJECT_REFERENCES} object references" + ))); + } + for object_id in reachable { + *reference_counts.entry(object_id).or_default() += 1; + } + } + let mut unique_authoritative_bytes = 0_u64; + let mut cross_artifact_shared_bytes = 0_u64; + for (object_id, (_, _, bytes)) in &selected_rows { + if reference_counts.get(object_id).copied().unwrap_or(1) > 1 { + cross_artifact_shared_bytes = cross_artifact_shared_bytes.saturating_add(*bytes); + } else { + unique_authoritative_bytes = unique_authoritative_bytes.saturating_add(*bytes); + } + } + let tree: ArtifactTreeRootV1 = self.get_artifact_cas_object( + &tree_root_id.0, + ARTIFACT_TREE_ROOT_KIND, + ARTIFACT_TREE_ROOT_VERSION, + )?; + let materialized = self.conn.query_row( + "SELECT COALESCE(SUM(COALESCE(physical_bytes,0)),0) + FROM artifact_materializations WHERE tree_root_id=?1", + params![tree_root_id.0], + |row| row.get::<_, i64>(0), + )?; + let layer_materialized = self.conn.query_row( + "SELECT COALESCE(SUM(COALESCE(w.physical_bytes,0)),0) + FROM workspace_layer_artifact_shadows s + JOIN workspace_layers w ON w.layer_id=s.layer_id + WHERE s.envelope_id=?1", + params![envelope_id.0], + |row| row.get::<_, i64>(0), + )?; + let materialized_bytes = u64::try_from(materialized) + .and_then(|left| { + u64::try_from(layer_materialized).map(|right| left.saturating_add(right)) + }) + .map_err(|_| Error::Corrupt("artifact materialization has negative bytes".into()))?; + Ok(ArtifactStorageAccountingReport { + logical_bytes: tree.logical_bytes, + unique_authoritative_bytes, + cross_artifact_shared_bytes, + materialized_bytes, + lane_private_bytes: 0, + prefetched_bytes: 0, + demand_loaded_bytes: 0, + reclaimable_bytes: 0, + unknown_bytes: 0, + accounting: format!( + "scope=artifact;state={envelope_state};axes=logical|authoritative|physical|disposition;authoritative=encoded-cbor-bytes-deduplicated;lane-private=not-applicable;reclaimable=requires-workspace-gc-analysis" + ), + }) + } + + fn verify_artifact_object_set(&self, object_ids: &BTreeSet) -> Result<()> { + for object_id in object_ids { + let (kind, version, codec, hash_alg, size_bytes, bytes, artifact_id, logical_bytes) = + self.conn + .query_row( + "SELECT o.kind,o.version,o.codec,o.hash_alg,o.size_bytes,o.bytes, + a.artifact_id,a.logical_bytes + FROM objects o LEFT JOIN artifact_objects a ON a.object_id=o.object_id + WHERE o.object_id=?1", + params![object_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, Vec>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + )) + }, + ) + .optional()? + .ok_or_else(|| Error::ObjectNotFound { + kind: "artifact reachable object", + id: object_id.clone(), + })?; + let version_u16 = u16::try_from(version).map_err(|_| { + Error::Corrupt(format!("artifact object `{object_id}` has invalid version")) + })?; + if codec != "cbor" + || hash_alg != "sha256" + || size_bytes != i64::try_from(bytes.len()).unwrap_or(-1) + || ObjectId::for_bytes(&kind, version_u16, &bytes).0 != *object_id + { + return Err(Error::Corrupt(format!( + "artifact reachable object `{object_id}` failed content identity verification" + ))); + } + if let (Some(artifact_id), Some(logical_bytes)) = (artifact_id, logical_bytes) { + self.validate_artifact_cas_object( + &artifact_id, + &kind, + version, + logical_bytes, + &bytes, + )?; + } + } + Ok(()) + } + + fn artifact_materialized_bytes( + &self, + view_id: Option<&str>, + selected_trees: &BTreeSet, + ) -> Result { + let materializations = if selected_trees.is_empty() { + 0 + } else { + let tree_ids = selected_trees.iter().collect::>(); + let mut total = 0_u64; + for chunk in tree_ids.chunks(512) { + let placeholders = std::iter::repeat_n("?", chunk.len()) + .collect::>() + .join(","); + let sql = format!( + "SELECT COALESCE(SUM(COALESCE(physical_bytes,0)),0) + FROM artifact_materializations WHERE tree_root_id IN ({placeholders})" + ); + let bytes = self.conn.query_row( + &sql, + params_from_iter(chunk.iter().map(|tree_id| tree_id.as_str())), + |row| row.get::<_, i64>(0), + )?; + total = total.saturating_add(u64::try_from(bytes).map_err(|_| { + Error::Corrupt("artifact materialization has negative physical bytes".into()) + })?); + } + total + }; + let layer_bytes = if let Some(view_id) = view_id { + self.conn.query_row( + "SELECT COALESCE(SUM(COALESCE(w.physical_bytes,0)),0) + FROM workspace_view_layers l + JOIN workspace_layers w ON w.layer_id=l.layer_id + WHERE l.view_id=?1 + AND EXISTS ( + SELECT 1 FROM workspace_layer_artifact_shadows s + WHERE s.layer_id=w.layer_id + )", + params![view_id], + |row| row.get::<_, i64>(0), + )? + } else { + self.conn.query_row( + "SELECT COALESCE(SUM(COALESCE(w.physical_bytes,0)),0) + FROM workspace_layers w + WHERE EXISTS ( + SELECT 1 FROM workspace_layer_artifact_shadows s + WHERE s.layer_id=w.layer_id + )", + [], + |row| row.get::<_, i64>(0), + )? + }; + Ok(materializations.saturating_add( + u64::try_from(layer_bytes).map_err(|_| { + Error::Corrupt("workspace layer has negative physical bytes".into()) + })?, + )) + } + + pub(crate) fn artifact_tree_lazy_entry( + &self, + tree_id: &ArtifactTreeId, + relative_path: &str, + ) -> Result> { + let tree = self.verified_artifact_tree_root(tree_id)?; + if relative_path.is_empty() { + return Ok(Some(ArtifactLazyEntry::Directory { + node_id: tree.root_directory_id, + })); + } + let relative_path = normalize_relative_path(relative_path)?; + let mut directory_id = tree.root_directory_id; + let mut segments = relative_path.split('/').peekable(); + while let Some(segment) = segments.next() { + let directory = self.verified_artifact_directory(&directory_id)?; + let Ok(index) = directory + .entries + .binary_search_by(|entry| entry.name.as_str().cmp(segment)) + else { + return Ok(None); + }; + let target = directory.entries[index].target.clone(); + if segments.peek().is_some() { + match target { + ArtifactDirectoryEntryTargetV1::Directory { node_id } => { + directory_id = node_id; + } + ArtifactDirectoryEntryTargetV1::File { .. } + | ArtifactDirectoryEntryTargetV1::Symlink { .. } => return Ok(None), + } + } else { + return self.artifact_lazy_entry_from_target(target).map(Some); + } + } + Ok(None) + } + + pub(crate) fn artifact_tree_lazy_children( + &self, + tree_id: &ArtifactTreeId, + relative_path: &str, + ) -> Result> { + let Some(ArtifactLazyEntry::Directory { node_id }) = + self.artifact_tree_lazy_entry(tree_id, relative_path)? + else { + return Ok(Vec::new()); + }; + let directory = self.verified_artifact_directory(&node_id)?; + directory + .entries + .into_iter() + .map(|entry| { + Ok(( + entry.name, + self.artifact_lazy_entry_from_target(entry.target)?, + )) + }) + .collect() + } + + #[cfg(windows)] + pub(crate) fn artifact_tree_lazy_follow_symlink( + &self, + tree_id: &ArtifactTreeId, + link_path: &str, + target: &str, + ) -> Result> { + let mut path = resolve_artifact_symlink_path(link_path, target)?; + for _ in 0..40 { + let Some(entry) = self.artifact_tree_lazy_entry(tree_id, &path)? else { + return Ok(None); + }; + match entry { + ArtifactLazyEntry::Symlink { target } => { + path = resolve_artifact_symlink_path(&path, &target)?; + } + entry => return Ok(Some(entry)), + } + } + Err(Error::Corrupt(format!( + "artifact symlink `{link_path}` exceeds the resolution bound" + ))) + } + + pub(crate) fn artifact_file_read_range( + &self, + file_id: &ArtifactFileId, + offset: u64, + count: u32, + ) -> Result> { + let file = self.verified_artifact_file(file_id)?; + if offset >= file.size_bytes || count == 0 { + return Ok(Vec::new()); + } + let end = offset.saturating_add(u64::from(count)).min(file.size_bytes); + match file.content { + ArtifactFileContentV1::Blob { blob_id } => { + let blob: ArtifactBlobV1 = self.get_artifact_cas_object( + &blob_id.0, + ARTIFACT_BLOB_KIND, + ARTIFACT_BLOB_VERSION, + )?; + let (actual, _) = encode_artifact_blob(blob.clone())?; + if actual != blob_id || blob.bytes.len() as u64 != file.size_bytes { + return Err(Error::Corrupt( + "artifact blob identity or file-size edge is invalid".into(), + )); + } + Ok(blob.bytes[offset as usize..end as usize].to_vec()) + } + ArtifactFileContentV1::Chunks { chunk_list_id } => { + let list: ArtifactChunkListV1 = self.get_artifact_cas_object( + &chunk_list_id.0, + ARTIFACT_CHUNK_LIST_KIND, + ARTIFACT_CHUNK_LIST_VERSION, + )?; + let (actual, _) = encode_artifact_chunk_list(list.clone())?; + if actual != chunk_list_id + || list.file_size_bytes != file.size_bytes + || list.file_sha256 != file.content_sha256 + { + return Err(Error::Corrupt( + "artifact chunk-list identity or file edge is invalid".into(), + )); + } + let mut output = Vec::with_capacity((end - offset) as usize); + let mut chunk_start = 0u64; + for chunk_ref in list.chunks { + let chunk_end = chunk_start + .checked_add(chunk_ref.size_bytes) + .ok_or_else(|| Error::Corrupt("artifact chunk range overflow".into()))?; + if chunk_end > offset && chunk_start < end { + let chunk: ArtifactChunkV1 = self.get_artifact_cas_object( + &chunk_ref.chunk_id.0, + ARTIFACT_CHUNK_KIND, + ARTIFACT_CHUNK_VERSION, + )?; + let (actual, _) = encode_artifact_chunk(chunk.clone())?; + if actual != chunk_ref.chunk_id + || chunk.bytes.len() as u64 != chunk_ref.size_bytes + { + return Err(Error::Corrupt( + "artifact chunk identity or size edge is invalid".into(), + )); + } + let selected_start = offset.saturating_sub(chunk_start) as usize; + let selected_end = end.min(chunk_end).saturating_sub(chunk_start) as usize; + output.extend_from_slice(&chunk.bytes[selected_start..selected_end]); + } + chunk_start = chunk_end; + if chunk_start >= end { + break; + } + } + if output.len() as u64 != end - offset { + return Err(Error::Corrupt( + "artifact ranged read did not cover the requested file extent".into(), + )); + } + Ok(output) + } + } + } + + pub(crate) fn materialize_artifact_file( + &self, + file_id: &ArtifactFileId, + destination: &Path, + ) -> Result { + let file = self.verified_artifact_file(file_id)?; + self.verify_artifact_file_content(&file)?; + // Open outside the cleanup scope so a create-new collision never + // removes a destination that this materialization attempt did not + // create. + let mut output = OpenOptions::new() + .write(true) + .create_new(true) + .open(destination)?; + let materialized = (|| -> Result<()> { + // Stream bounded ranges so copy-up never allocates a complete + // large artifact file. The complete digest was verified before + // publication. + let mut offset = 0u64; + while offset < file.size_bytes { + let part = self.artifact_file_read_range(file_id, offset, 4 * 1024 * 1024)?; + if part.is_empty() { + return Err(Error::Corrupt( + "artifact file materialization made no progress".into(), + )); + } + output.write_all(&part)?; + offset += part.len() as u64; + } + output.sync_all()?; + set_artifact_materialized_mode(destination, file.mode)?; + Ok(()) + })(); + if let Err(error) = materialized { + let _ = fs::remove_file(destination); + return Err(error); + } + Ok(file.mode) + } + + pub(crate) fn ensure_artifact_tree_materialization( + &self, + tree_id: &ArtifactTreeId, + ) -> Result { + let _lock = self.acquire_write_lock()?; + self.ensure_artifact_tree_materialization_under_write_lock(tree_id) + } + + pub(crate) fn ensure_artifact_tree_materialization_under_write_lock( + &self, + tree_id: &ArtifactTreeId, + ) -> Result { + let tree = self.verified_artifact_tree_root(tree_id)?; + let backend_compatibility = artifact_materialization_backend_compatibility(); + let identity_seed = format!("{}\0{backend_compatibility}", tree_id.0); + let materialization_id = format!( + "materialization_{}", + crate::ids::short_hash(identity_seed.as_bytes(), 32) + ); + let (materialization_parent, staging_parent) = + self.artifact_materialization_cache_parents()?; + let final_path = materialization_parent.join(&materialization_id); + let final_exists = real_artifact_materialization_directory_exists( + &final_path, + "artifact materialization", + )?; + let existing = self + .conn + .query_row( + "SELECT materialization_id,storage_path,state,logical_bytes, + COALESCE(physical_bytes,0),entry_count + FROM artifact_materializations + WHERE tree_root_id=?1 AND backend_compatibility=?2", + params![tree_id.0, &backend_compatibility], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, i64>(5)?, + )) + }, + ) + .optional()?; + if let Some((stored_id, stored_path, state, logical, physical, entries)) = existing { + if stored_id != materialization_id || Path::new(&stored_path) != final_path { + return Err(Error::Corrupt(format!( + "artifact materialization `{stored_id}` has a non-canonical identity or storage path" + ))); + } + if state == "verified" && final_exists { + match self.verify_artifact_materialization(tree_id, &tree, &final_path) { + Ok(()) => { + self.conn.execute( + "UPDATE artifact_materializations SET last_used_at=?1 + WHERE materialization_id=?2 AND state='verified'", + params![now_ts(), &materialization_id], + )?; + return Ok(ArtifactMaterializationReport { + materialization_id, + tree_root_id: tree_id.clone(), + backend_compatibility, + storage_path: final_path, + logical_bytes: u64::try_from(logical).map_err(|_| { + Error::Corrupt( + "artifact materialization has negative logical bytes".into(), + ) + })?, + physical_bytes: u64::try_from(physical).map_err(|_| { + Error::Corrupt( + "artifact materialization has negative physical bytes".into(), + ) + })?, + entry_count: u64::try_from(entries).map_err(|_| { + Error::Corrupt( + "artifact materialization has negative entry count".into(), + ) + })?, + reused: true, + }); + } + Err(_) => { + super::workspace_layer::make_tree_writable(&final_path); + fs::remove_dir_all(&final_path)?; + } + } + } else if final_exists { + super::workspace_layer::make_tree_writable(&final_path); + fs::remove_dir_all(&final_path)?; + } + self.conn.execute( + "UPDATE artifact_materializations SET state='failed',last_used_at=?1 + WHERE materialization_id=?2", + params![now_ts(), &materialization_id], + )?; + } else if final_exists { + match self.verify_artifact_materialization(tree_id, &tree, &final_path) { + Ok(()) => { + let physical = super::workspace_layer::layer_physical_bytes(&final_path)?; + self.upsert_verified_artifact_materialization( + &materialization_id, + tree_id, + &backend_compatibility, + &final_path, + &tree, + physical, + )?; + return Ok(ArtifactMaterializationReport { + materialization_id, + tree_root_id: tree_id.clone(), + backend_compatibility, + storage_path: final_path, + logical_bytes: tree.logical_bytes, + physical_bytes: physical, + entry_count: tree.entry_count, + reused: true, + }); + } + Err(_) => { + super::workspace_layer::make_tree_writable(&final_path); + fs::remove_dir_all(&final_path)?; + } + } + } + + let staging = staging_parent.join(format!("artifact_{materialization_id}")); + if real_artifact_materialization_directory_exists( + &staging, + "artifact materialization staging", + )? { + super::workspace_layer::make_tree_writable(&staging); + fs::remove_dir_all(&staging)?; + } + let now = now_ts(); + self.conn.execute( + "INSERT INTO artifact_materializations( + materialization_id,tree_root_id,backend_compatibility,storage_path,state, + logical_bytes,physical_bytes,entry_count,last_used_at,created_at + ) VALUES(?1,?2,?3,?4,'building',?5,NULL,?6,?7,?7) + ON CONFLICT(tree_root_id,backend_compatibility) DO UPDATE SET + materialization_id=excluded.materialization_id, + storage_path=excluded.storage_path,state='building', + logical_bytes=excluded.logical_bytes,physical_bytes=NULL, + entry_count=excluded.entry_count,last_used_at=excluded.last_used_at", + params![ + &materialization_id, + tree_id.0, + &backend_compatibility, + final_path.to_string_lossy(), + i64::try_from(tree.logical_bytes).map_err(|_| Error::InvalidInput( + "artifact materialization logical bytes exceed SQLite range".into() + ))?, + i64::try_from(tree.entry_count).map_err(|_| Error::InvalidInput( + "artifact materialization entry count exceeds SQLite range".into() + ))?, + now, + ], + )?; + let publication = (|| -> Result { + self.materialize_artifact_tree_under_write_lock(tree_id, &staging) + .map_err(|error| { + Error::Corrupt(format!( + "artifact materialization content projection failed: {error}" + )) + })?; + let entries = + super::workspace_layer::scan_layer_entries(&staging, true).map_err(|error| { + Error::Corrupt(format!( + "artifact materialization immutable scan failed: {error}" + )) + })?; + super::workspace_layer::verify_artifact_shadow_matches_layer_entries( + &tree, + &self.artifact_tree_flat_entries(tree_id)?, + &entries, + )?; + let physical = + super::workspace_layer::layer_physical_bytes(&staging).map_err(|error| { + Error::Corrupt(format!( + "artifact materialization physical accounting failed: {error}" + )) + })?; + fs::rename(&staging, &final_path).map_err(|error| { + Error::Corrupt(format!( + "artifact materialization atomic publication failed: {error}" + )) + })?; + // macOS rejects renaming a read-only directory even when both + // parents are writable. Child entries are already immutable, so + // publish the root while the workspace write lock is held and + // seal that root before marking the database row verified. + super::workspace_layer::set_layer_read_only(&final_path, true, 0o755).map_err( + |error| { + Error::Corrupt(format!( + "artifact materialization root sealing failed: {error}" + )) + }, + )?; + sync_directory(final_path.parent().unwrap()); + Ok(physical) + })(); + let physical = match publication { + Ok(physical) => physical, + Err(error) => { + super::workspace_layer::make_tree_writable(&staging); + let _ = fs::remove_dir_all(&staging); + super::workspace_layer::make_tree_writable(&final_path); + let _ = fs::remove_dir_all(&final_path); + self.conn.execute( + "UPDATE artifact_materializations SET state='failed',last_used_at=?1 + WHERE materialization_id=?2", + params![now_ts(), &materialization_id], + )?; + return Err(error); + } + }; + self.upsert_verified_artifact_materialization( + &materialization_id, + tree_id, + &backend_compatibility, + &final_path, + &tree, + physical, + )?; + Ok(ArtifactMaterializationReport { + materialization_id, + tree_root_id: tree_id.clone(), + backend_compatibility, + storage_path: final_path, + logical_bytes: tree.logical_bytes, + physical_bytes: physical, + entry_count: tree.entry_count, + reused: false, + }) + } + + fn artifact_materialization_cache_parents(&self) -> Result<(PathBuf, PathBuf)> { + // Reuse the environment executor's descriptor-validated staging + // hierarchy rather than independently trusting `.trail/cache` path + // components. + let staging = self.workspace_environment_staging_parent()?; + let cache = staging.parent().ok_or_else(|| { + Error::Corrupt("artifact materialization staging has no cache parent".into()) + })?; + let materializations = cache.join("artifact-materializations"); + match fs::symlink_metadata(&materializations) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => { + return Err(Error::InvalidPath { + path: materializations.to_string_lossy().into_owned(), + reason: "artifact materialization cache must remain a real directory inside Trail storage" + .into(), + }); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + fs::create_dir(&materializations)?; + } + Err(error) => return Err(Error::Io(error)), + } + let canonical = fs::canonicalize(&materializations)?; + if canonical != materializations { + return Err(Error::InvalidPath { + path: materializations.to_string_lossy().into_owned(), + reason: "artifact materialization cache escaped Trail storage".into(), + }); + } + Ok((canonical, staging)) + } + + fn verify_artifact_materialization( + &self, + tree_id: &ArtifactTreeId, + tree: &ArtifactTreeRootV1, + path: &Path, + ) -> Result<()> { + // Verification also restores immutable permissions. Content identity + // alone is insufficient because a writable cache could become a + // mutable alias of authoritative CAS content after this check. + let entries = super::workspace_layer::scan_layer_entries(path, true)?; + super::workspace_layer::verify_artifact_shadow_matches_layer_entries( + tree, + &self.artifact_tree_flat_entries(tree_id)?, + &entries, + )?; + super::workspace_layer::set_layer_read_only(path, true, 0o755) + } + + fn upsert_verified_artifact_materialization( + &self, + materialization_id: &str, + tree_id: &ArtifactTreeId, + backend_compatibility: &str, + path: &Path, + tree: &ArtifactTreeRootV1, + physical_bytes: u64, + ) -> Result<()> { + let now = now_ts(); + self.conn.execute( + "INSERT INTO artifact_materializations( + materialization_id,tree_root_id,backend_compatibility,storage_path,state, + logical_bytes,physical_bytes,entry_count,last_used_at,created_at + ) VALUES(?1,?2,?3,?4,'verified',?5,?6,?7,?8,?8) + ON CONFLICT(tree_root_id,backend_compatibility) DO UPDATE SET + materialization_id=excluded.materialization_id, + storage_path=excluded.storage_path,state='verified', + logical_bytes=excluded.logical_bytes,physical_bytes=excluded.physical_bytes, + entry_count=excluded.entry_count,last_used_at=excluded.last_used_at", + params![ + materialization_id, + tree_id.0, + backend_compatibility, + path.to_string_lossy(), + i64::try_from(tree.logical_bytes).map_err(|_| Error::InvalidInput( + "artifact materialization logical bytes exceed SQLite range".into() + ))?, + i64::try_from(physical_bytes).map_err(|_| Error::InvalidInput( + "artifact materialization physical bytes exceed SQLite range".into() + ))?, + i64::try_from(tree.entry_count).map_err(|_| Error::InvalidInput( + "artifact materialization entry count exceeds SQLite range".into() + ))?, + now, + ], + )?; + Ok(()) + } + + fn verified_artifact_tree_root(&self, tree_id: &ArtifactTreeId) -> Result { + let tree: ArtifactTreeRootV1 = self.get_artifact_cas_object( + &tree_id.0, + ARTIFACT_TREE_ROOT_KIND, + ARTIFACT_TREE_ROOT_VERSION, + )?; + let (actual, _) = encode_artifact_tree_root(tree.clone())?; + if actual != *tree_id { + return Err(Error::Corrupt(format!( + "artifact tree root `{tree_id}` has conflicting encoded identity" + ))); + } + Ok(tree) + } + + fn verified_artifact_directory( + &self, + directory_id: &ArtifactTreeId, + ) -> Result { + let directory: ArtifactDirectoryNodeV1 = self.get_artifact_cas_object( + &directory_id.0, + ARTIFACT_DIRECTORY_NODE_KIND, + ARTIFACT_DIRECTORY_NODE_VERSION, + )?; + let (actual, canonical) = encode_artifact_directory_node(directory.clone())?; + if actual != *directory_id || from_cbor::(&canonical)? != directory + { + return Err(Error::Corrupt(format!( + "artifact directory `{directory_id}` has conflicting encoded identity" + ))); + } + Ok(directory) + } + + fn verified_artifact_file(&self, file_id: &ArtifactFileId) -> Result { + let file: ArtifactFileNodeV1 = self.get_artifact_cas_object( + &file_id.0, + ARTIFACT_FILE_NODE_KIND, + ARTIFACT_FILE_NODE_VERSION, + )?; + let (actual, _) = encode_artifact_file_node(file.clone())?; + if actual != *file_id { + return Err(Error::Corrupt(format!( + "artifact file `{file_id}` has conflicting encoded identity" + ))); + } + Ok(file) + } + + fn artifact_lazy_entry_from_target( + &self, + target: ArtifactDirectoryEntryTargetV1, + ) -> Result { + match target { + ArtifactDirectoryEntryTargetV1::Directory { node_id } => { + Ok(ArtifactLazyEntry::Directory { node_id }) + } + ArtifactDirectoryEntryTargetV1::File { node_id } => { + let file = self.verified_artifact_file(&node_id)?; + Ok(ArtifactLazyEntry::File { + node_id, + mode: file.mode, + size_bytes: file.size_bytes, + }) + } + ArtifactDirectoryEntryTargetV1::Symlink { target } => { + validate_artifact_symlink_target(&target)?; + Ok(ArtifactLazyEntry::Symlink { target }) + } + } + } + + pub(crate) fn put_legacy_artifact_envelope_under_write_lock( + &self, + layer_key: &WorkspaceLayerKeyV1, + cache_key: &str, + tree_root_id: ArtifactTreeId, + ) -> Result { + validate_resolution_text(cache_key, "legacy layer cache key")?; + let canonical_cache_key = self.workspace_layer_cache_key(layer_key)?; + if canonical_cache_key != cache_key { + return Err(Error::InvalidInput( + "workspace layer key changed before artifact sealing".into(), + )); + } + let tree = self.verified_artifact_tree_root(&tree_root_id)?; + let entries = self.artifact_tree_flat_entries(&tree_root_id)?; + if entries.len() as u64 != tree.entry_count + || entries + .values() + .try_fold(0u64, |total, entry| total.checked_add(entry.size_bytes)) + != Some(tree.logical_bytes) + { + return Err(Error::Corrupt( + "workspace layer artifact tree is not a complete content identity".into(), + )); + } + let desired_identity = ArtifactDesiredIdentityV1::WorkspaceLayerV1 { + cache_key: cache_key.to_string(), + canonical_key: layer_key.clone(), + }; + let validation_receipt_ids = [ + self.put_host_workspace_layer_seal_receipt_under_write_lock( + ArtifactValidationV1 { + name: HOST_WORKSPACE_LAYER_STRUCTURAL_SEAL.into(), + kind: ArtifactValidationKindV1::Structural, + required: true, + parameters: BTreeMap::from([ + ("content_identity".into(), "artifact-tree-v1".into()), + ("path_normalizer".into(), tree.path_normalizer.clone()), + ]), + }, + desired_identity.clone(), + tree_root_id.clone(), + BTreeMap::from([ + ("complete_tree_identity".into(), "passed".into()), + ("declared_path_containment".into(), "passed".into()), + ("entry_count".into(), tree.entry_count.to_string()), + ("limits".into(), "passed".into()), + ("logical_bytes".into(), tree.logical_bytes.to_string()), + ("safe_normalized_content".into(), "passed".into()), + ("secret_policy".into(), "passed".into()), + ]), + )?, + self.put_host_workspace_layer_seal_receipt_under_write_lock( + ArtifactValidationV1 { + name: HOST_WORKSPACE_LAYER_POLICY_SEAL.into(), + kind: ArtifactValidationKindV1::Policy, + required: true, + parameters: BTreeMap::from([ + ("pin_contract".into(), "workspace-layer-key-v1".into()), + ("trust_scope".into(), "workspace-layer-v1".into()), + ]), + }, + desired_identity.clone(), + tree_root_id.clone(), + BTreeMap::from([ + ("desired_pins".into(), "unchanged".into()), + ( + "producer_termination".into(), + "terminated_or_disconnected".into(), + ), + ("producer_trust".into(), "local_host_authorized".into()), + ]), + )?, + ] + .into_iter() + .collect(); + let envelope = ArtifactEnvelopeV1 { + version: ARTIFACT_ENVELOPE_VERSION, + desired_identity, + tree_root_id, + component_id: format!("legacy:{}", layer_key.adapter), + output_name: "legacy-layer".into(), + output_policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + portability_scope: layer_key.portability_scope.clone(), + trust_scope: "workspace-layer-v1".into(), + secret_taint: ArtifactSecretTaintV1::Clear, + resolution_snapshot_id: None, + validation_receipt_ids, + }; + let (envelope_id, quarantined) = self.put_artifact_envelope_under_write_lock(envelope)?; + if quarantined { + return Err(Error::InvalidInput(format!( + "artifact desired identity `{cache_key}` produced divergent content and was quarantined" + ))); + } + Ok(envelope_id) + } + + /// Replace a workspace layer's legacy CAS shadow with a verified desired-key-v2 + /// envelope over the exact same immutable tree. The physical layer remains a + /// compatibility materialization; the envelope becomes artifact authority for + /// activation, inheritance, export, reachability, and collection. + pub(crate) fn bind_workspace_layer_artifact_v2( + &self, + layer_id: &str, + envelope: ArtifactEnvelopeV1, + ) -> Result { + let ArtifactDesiredIdentityV1::ArtifactDesiredV2 { .. } = &envelope.desired_identity else { + return Err(Error::InvalidInput( + "workspace artifact-v2 binding requires a desired-key-v2 envelope".into(), + )); + }; + if !envelope.output_policy.has_immutable_layer() { + return Err(Error::InvalidInput( + "workspace artifact-v2 binding requires an immutable output policy".into(), + )); + } + let _lock = self.acquire_write_lock()?; + let layer = self.verify_workspace_layer_for_attach(layer_id)?; + let (tree_root_id, state) = self.conn.query_row( + "SELECT tree_root_id,state FROM workspace_layer_artifact_shadows WHERE layer_id=?1", + params![layer_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + )?; + if state != "verified" || tree_root_id != envelope.tree_root_id.0 { + return Err(Error::InvalidInput(format!( + "workspace layer `{layer_id}` does not contain the exact artifact-v2 tree" + ))); + } + if layer.portability_scope != envelope.portability_scope { + return Err(Error::InvalidInput(format!( + "workspace layer `{layer_id}` and artifact-v2 envelope have different portability scopes" + ))); + } + let tree_root_id = ArtifactTreeId::parse(tree_root_id) + .map_err(|error| Error::Corrupt(format!("invalid artifact tree ID: {error}")))?; + self.artifact_tree_flat_entries(&tree_root_id)?; + let (envelope_id, quarantined) = self.put_artifact_envelope_under_write_lock(envelope)?; + if quarantined { + return Err(Error::InvalidInput(format!( + "workspace layer `{layer_id}` produced divergent artifact-v2 content and was quarantined" + ))); + } + let updated = self.conn.execute( + "UPDATE workspace_layer_artifact_shadows + SET envelope_id=?1,state='verified',verified_at=?2 + WHERE layer_id=?3 AND tree_root_id=?4 AND state='verified'", + params![envelope_id.0, now_ts(), layer_id, tree_root_id.0], + )?; + if updated != 1 { + return Err(Error::Corrupt(format!( + "workspace layer `{layer_id}` artifact binding changed while publishing v2 authority" + ))); + } + Ok(envelope_id) + } + + pub(crate) fn put_artifact_envelope_under_write_lock( + &self, + mut envelope: ArtifactEnvelopeV1, + ) -> Result<(ArtifactEnvelopeId, bool)> { + envelope.validation_receipt_ids.sort(); + envelope.validation_receipt_ids.dedup(); + let desired_key = match &envelope.desired_identity { + ArtifactDesiredIdentityV1::WorkspaceLayerV1 { cache_key, .. } => cache_key.clone(), + ArtifactDesiredIdentityV1::ArtifactDesiredV2 { desired_key } => desired_key.0.clone(), + }; + validate_resolution_text(&desired_key, "artifact desired key")?; + validate_resolution_text(&envelope.trust_scope, "artifact trust scope")?; + validate_artifact_secret_taint(&envelope.secret_taint)?; + if !envelope.secret_taint.is_clear() { + return Err(Error::InvalidInput( + "secret-tainted artifact output must remain lane-private and cannot enter shared CAS" + .into(), + )); + } + self.validate_envelope_validation_receipts(&envelope)?; + let (envelope_id, _) = encode_artifact_envelope(envelope.clone())?; + let object_id = self.put_artifact_cas_object( + &envelope_id.0, + ARTIFACT_ENVELOPE_KIND, + ARTIFACT_ENVELOPE_VERSION, + 0, + &envelope, + )?; + self.conn + .execute_batch("SAVEPOINT trail_artifact_envelope")?; + let publication = (|| -> Result { + let incumbent = self + .conn + .query_row( + "SELECT envelope_id,tree_root_id FROM artifact_envelopes + WHERE desired_key=?1 AND trust_scope=?2 AND state='ready' + ORDER BY envelope_id LIMIT 1", + params![desired_key, envelope.trust_scope], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .optional()?; + let active_quarantine = self + .conn + .query_row( + "SELECT quarantine_id FROM artifact_quarantines + WHERE desired_key=?1 AND trust_scope=?2 AND state='active' + ORDER BY quarantine_id LIMIT 1", + params![desired_key, envelope.trust_scope], + |row| row.get::<_, String>(0), + ) + .optional()?; + let divergent_incumbent = incumbent + .as_ref() + .filter(|(_, tree_root_id)| tree_root_id != &envelope.tree_root_id.0); + let quarantined = active_quarantine.is_some() || divergent_incumbent.is_some(); + self.conn.execute( + "INSERT INTO artifact_envelopes( + envelope_id, desired_key, trust_scope, tree_root_id, object_id, state, + verification_state, created_at, updated_at + ) VALUES(?1, ?2, ?3, ?4, ?5, ?6, 'verified', ?7, ?7) + ON CONFLICT(envelope_id) DO UPDATE SET updated_at=excluded.updated_at", + params![ + envelope_id.0, + desired_key, + envelope.trust_scope, + envelope.tree_root_id.0, + object_id.0, + if quarantined { "quarantined" } else { "ready" }, + now_ts(), + ], + )?; + if let Some((incumbent_id, incumbent_tree_id)) = divergent_incumbent { + let incumbent_envelope_id = ArtifactEnvelopeId::parse(incumbent_id.clone()) + .map_err(|error| { + Error::Corrupt(format!("invalid incumbent envelope ID: {error}")) + })?; + let incumbent_tree_root_id = ArtifactTreeId::parse(incumbent_tree_id.clone()) + .map_err(|error| { + Error::Corrupt(format!("invalid incumbent artifact tree ID: {error}")) + })?; + let evidence = ArtifactDivergenceEvidenceV1 { + version: ARTIFACT_DIVERGENCE_EVIDENCE_VERSION, + trust_scope: envelope.trust_scope.clone(), + desired_key: desired_key.clone(), + incumbent_envelope_id: incumbent_envelope_id.clone(), + incumbent_tree_root_id, + candidate_envelope_id: envelope_id.clone(), + candidate_tree_root_id: envelope.tree_root_id.clone(), + reason_code: "tree_root_divergence".into(), + }; + let evidence_object = self.put_object( + ARTIFACT_DIVERGENCE_EVIDENCE_KIND, + ARTIFACT_DIVERGENCE_EVIDENCE_VERSION, + &evidence, + )?; + let quarantine_id = crate::ids::ArtifactQuarantineId::new(&cbor(&evidence)?); + self.conn.execute( + "UPDATE artifact_envelopes SET state='quarantined',updated_at=?1 + WHERE envelope_id IN (?2,?3)", + params![now_ts(), incumbent_id, envelope_id.0], + )?; + self.conn.execute( + "INSERT OR IGNORE INTO artifact_quarantines( + quarantine_id,trust_scope,desired_key,incumbent_envelope_id, + candidate_envelope_id,reason_code,evidence_object_id,state,created_at + ) VALUES(?1,?2,?3,?4,?5,'tree_root_divergence',?6,'active',?7)", + params![ + quarantine_id.0, + envelope.trust_scope, + desired_key, + incumbent_id, + envelope_id.0, + evidence_object.0, + now_ts(), + ], + )?; + for held_envelope in [incumbent_id, &envelope_id.0] { + let hold_id = format!( + "hold_{}", + crate::ids::short_hash( + format!("{}:{held_envelope}", quarantine_id.0).as_bytes(), + 32, + ) + ); + self.conn.execute( + "INSERT OR IGNORE INTO artifact_holds( + hold_id,target_kind,target_id,reason,created_at + ) VALUES(?1,'artifact_envelope',?2,?3,?4)", + params![hold_id, held_envelope, quarantine_id.0, now_ts()], + )?; + } + } + self.create_artifact_attestation_under_write_lock(&envelope_id, &envelope)?; + Ok(quarantined) + })(); + match publication { + Ok(quarantined) => { + self.conn + .execute_batch("RELEASE SAVEPOINT trail_artifact_envelope")?; + Ok((envelope_id, quarantined)) + } + Err(error) => { + let _ = self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT trail_artifact_envelope; + RELEASE SAVEPOINT trail_artifact_envelope", + ); + Err(error) + } + } + } + + pub(crate) fn verify_ready_artifact_envelope_under_write_lock( + &self, + envelope_id: &ArtifactEnvelopeId, + expected_tree_id: &ArtifactTreeId, + ) -> Result { + let (desired_key, trust_scope, tree_root_id, state, verification_state) = + self.conn.query_row( + "SELECT desired_key, trust_scope, tree_root_id, state, verification_state + FROM artifact_envelopes WHERE envelope_id=?1", + params![envelope_id.0], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + )) + }, + )?; + if state != "ready" + || verification_state != "verified" + || tree_root_id != expected_tree_id.0 + { + return Err(Error::Corrupt(format!( + "artifact envelope `{envelope_id}` is not ready for tree `{expected_tree_id}`" + ))); + } + let envelope: ArtifactEnvelopeV1 = self.get_artifact_cas_object( + &envelope_id.0, + ARTIFACT_ENVELOPE_KIND, + ARTIFACT_ENVELOPE_VERSION, + )?; + let (actual_id, _) = encode_artifact_envelope(envelope.clone())?; + if actual_id != *envelope_id || envelope.tree_root_id != *expected_tree_id { + return Err(Error::Corrupt(format!( + "artifact envelope `{envelope_id}` has conflicting content identity" + ))); + } + let encoded_desired_key = match &envelope.desired_identity { + ArtifactDesiredIdentityV1::WorkspaceLayerV1 { cache_key, .. } => cache_key, + ArtifactDesiredIdentityV1::ArtifactDesiredV2 { desired_key } => &desired_key.0, + }; + if encoded_desired_key != &desired_key || envelope.trust_scope != trust_scope { + return Err(Error::Corrupt(format!( + "artifact envelope `{envelope_id}` database identity disagrees with its object" + ))); + } + self.validate_envelope_validation_receipts(&envelope) + .map_err(|error| { + Error::Corrupt(format!( + "artifact envelope `{envelope_id}` has invalid validation evidence: {error}" + )) + })?; + self.verify_artifact_attestations_for_attachment(envelope_id, &envelope)?; + Ok(envelope) + } + + pub fn artifact_attestation( + &self, + attestation_id: &ArtifactAttestationId, + ) -> Result { + let (envelope_id, object_id, producer_identity, trust_scope, state) = self + .conn + .query_row( + "SELECT envelope_id,object_id,producer_identity,trust_scope,state + FROM artifact_attestations WHERE attestation_id=?1", + params![attestation_id.0], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + )) + }, + ) + .optional()? + .ok_or_else(|| Error::ObjectNotFound { + kind: "artifact attestation", + id: attestation_id.0.clone(), + })?; + let object_id = ObjectId(object_id); + let attestation: ArtifactAttestationV1 = + self.get_object(ARTIFACT_ATTESTATION_KIND, &object_id)?; + let (actual_id, _) = encode_artifact_attestation(attestation.clone())?; + if actual_id != *attestation_id + || attestation.statement.envelope_id.0 != envelope_id + || attestation.statement.producer_identity != producer_identity + || attestation.statement.trust_scope != trust_scope + { + return Err(Error::Corrupt(format!( + "artifact attestation `{attestation_id}` database identity disagrees with its object" + ))); + } + Ok(ArtifactAttestationReportV1 { + attestation_id: attestation_id.clone(), + object_id, + state, + attestation, + }) + } + + pub fn artifact_attestations_for_envelope( + &self, + envelope_id: &ArtifactEnvelopeId, + ) -> Result> { + let mut statement = self.conn.prepare( + "SELECT attestation_id FROM artifact_attestations + WHERE envelope_id=?1 ORDER BY attestation_id", + )?; + let ids = statement + .query_map(params![envelope_id.0], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + if ids.len() > MAX_PUBLIC_ARTIFACT_REPORT_ITEMS { + return Err(Error::InvalidInput(format!( + "artifact `{envelope_id}` has {} attestations; maximum is {MAX_PUBLIC_ARTIFACT_REPORT_ITEMS}", + ids.len() + ))); + } + ids.into_iter() + .map(|id| { + ArtifactAttestationId::parse(id) + .map_err(Error::Corrupt) + .and_then(|id| self.artifact_attestation(&id)) + }) + .collect() + } + + pub fn verify_artifact_attestation( + &self, + attestation_id: &ArtifactAttestationId, + ) -> Result { + let report = self.artifact_attestation(attestation_id)?; + let statement = &report.attestation.statement; + let mut diagnostics = Vec::new(); + let content_identity_valid = encode_artifact_attestation(report.attestation.clone()) + .is_ok_and(|(actual, _)| actual == *attestation_id); + if !content_identity_valid { + diagnostics.push("attestation content identity mismatch".into()); + } + let envelope_binding_valid = self + .get_artifact_cas_object::( + &statement.envelope_id.0, + ARTIFACT_ENVELOPE_KIND, + ARTIFACT_ENVELOPE_VERSION, + ) + .is_ok_and(|envelope| artifact_attestation_matches_envelope(statement, &envelope)); + if !envelope_binding_valid { + diagnostics.push("attestation does not match its exact artifact envelope".into()); + } + let producer_trusted = + self.artifact_attestation_producer_trusted(statement, &mut diagnostics); + let (signature_status, signature_valid) = + self.verify_artifact_attestation_signature(&report.attestation, &mut diagnostics)?; + if report.state != "valid" { + diagnostics.push(format!("attestation database state is `{}`", report.state)); + } + let valid = report.state == "valid" + && content_identity_valid + && envelope_binding_valid + && producer_trusted + && signature_valid; + Ok(ArtifactAttestationVerificationReportV1 { + attestation_id: attestation_id.clone(), + envelope_id: statement.envelope_id.clone(), + state: report.state, + content_identity_valid, + envelope_binding_valid, + producer_trusted, + signature_status, + valid, + diagnostics, + }) + } + + fn create_artifact_attestation_under_write_lock( + &self, + envelope_id: &ArtifactEnvelopeId, + envelope: &ArtifactEnvelopeV1, + ) -> Result { + let statement = self.artifact_attestation_statement(envelope_id, envelope)?; + let attestation = ArtifactAttestationV1 { + statement, + signature: None, + }; + let (attestation_id, _) = encode_artifact_attestation(attestation.clone())?; + let object_id = self.put_object( + ARTIFACT_ATTESTATION_KIND, + ARTIFACT_ATTESTATION_VERSION, + &attestation, + )?; + self.conn.execute( + "INSERT INTO artifact_attestations( + attestation_id,envelope_id,object_id,producer_identity,trust_scope,state, + created_at,updated_at) + VALUES(?1,?2,?3,?4,?5,'valid',?6,?6) + ON CONFLICT(attestation_id) DO UPDATE SET updated_at=excluded.updated_at", + params![ + attestation_id.0, + envelope_id.0, + object_id.0, + attestation.statement.producer_identity, + attestation.statement.trust_scope, + now_ts(), + ], + )?; + Ok(attestation_id) + } + + fn artifact_attestation_statement( + &self, + envelope_id: &ArtifactEnvelopeId, + envelope: &ArtifactEnvelopeV1, + ) -> Result { + let mut source_root = None; + let mut upstream_identities = BTreeMap::new(); + let mut executable_identities = BTreeMap::new(); + let ( + producer_identity, + producer_trust, + implementation, + distribution, + protocol, + platform, + architecture, + ) = match &envelope.desired_identity { + ArtifactDesiredIdentityV1::WorkspaceLayerV1 { canonical_key, .. } => { + source_root = canonical_key + .inputs + .get("source_root") + .filter(|value| value.starts_with("object_")) + .map(|value| ObjectId(value.clone())); + upstream_identities = canonical_key.inputs.clone(); + executable_identities = canonical_key.tool_versions.clone(); + let plugin = self + .installed_environment_plugins()? + .into_iter() + .find(|plugin| { + plugin.manifest.adapter.canonical_identity == canonical_key.adapter + }); + if let Some(plugin) = plugin { + ( + canonical_key.adapter.clone(), + ArtifactProducerTrustTierV1::LocallyTrustedPlugin, + plugin.manifest.adapter.implementation_version, + plugin.distribution_digest, + "trail.environment-adapter/v2".to_string(), + canonical_key.platform.clone(), + canonical_key.architecture.clone(), + ) + } else { + ( + canonical_key.adapter.clone(), + ArtifactProducerTrustTierV1::ReviewedBuiltin, + canonical_key.adapter_version.to_string(), + format!("builtin:{}", canonical_key.adapter), + "workspace-layer/v1".to_string(), + canonical_key.platform.clone(), + canonical_key.architecture.clone(), + ) + } + } + ArtifactDesiredIdentityV1::ArtifactDesiredV2 { .. } => ( + envelope.component_id.clone(), + ArtifactProducerTrustTierV1::RepositoryDeclaration, + "unspecified".to_string(), + "repository-declaration".to_string(), + "artifact-envelope/v1".to_string(), + std::env::consts::OS.to_string(), + std::env::consts::ARCH.to_string(), + ), + }; + let plugin = self + .installed_environment_plugins()? + .into_iter() + .find(|plugin| plugin.manifest.adapter.canonical_identity == producer_identity); + let (publisher, publisher_key_id) = plugin + .map(|plugin| (plugin.publisher, plugin.publisher_key_id)) + .unwrap_or((None, None)); + let statement = ArtifactAttestationStatementV1 { + version: ARTIFACT_ATTESTATION_VERSION, + envelope_id: envelope_id.clone(), + desired_identity: envelope.desired_identity.clone(), + tree_root_id: envelope.tree_root_id.clone(), + source_root, + resolution_snapshot_id: envelope.resolution_snapshot_id.clone(), + upstream_identities, + producer_identity, + producer_trust, + adapter_implementation_version: implementation, + adapter_distribution_digest: distribution, + adapter_protocol: protocol, + publisher, + publisher_key_id, + executable_identities, + platform, + architecture, + abi: "host-default".into(), + capability_ceiling: ArtifactCapabilityCeilingV1::for_phase( + producer_trust, + ArtifactExecutionPhaseV1::Construct, + ), + sandbox_enforcement: "host-sealed-candidate".into(), + network_policy: "producer-plan-enforced".into(), + script_policy: ArtifactScriptPolicyV1::AllowDeclared, + output_name: envelope.output_name.clone(), + output_policy: envelope.output_policy, + portability_scope: envelope.portability_scope.clone(), + trust_scope: envelope.trust_scope.clone(), + validation_receipt_ids: envelope.validation_receipt_ids.clone(), + secret_taint: envelope.secret_taint.clone(), + }; + validate_artifact_attestation_statement(&statement)?; + Ok(statement) + } + + fn verify_artifact_attestations_for_attachment( + &self, + envelope_id: &ArtifactEnvelopeId, + envelope: &ArtifactEnvelopeV1, + ) -> Result<()> { + let attestations = self.artifact_attestations_for_envelope(envelope_id)?; + if attestations.is_empty() { + return Err(Error::Corrupt(format!( + "artifact envelope `{envelope_id}` has no host attestation" + ))); + } + for attestation in attestations { + if !artifact_attestation_matches_envelope(&attestation.attestation.statement, envelope) + { + return Err(Error::Corrupt(format!( + "artifact attestation `{}` does not match envelope `{envelope_id}`", + attestation.attestation_id + ))); + } + let verification = self.verify_artifact_attestation(&attestation.attestation_id)?; + if !verification.valid { + return Err(Error::Corrupt(format!( + "artifact attestation `{}` cannot authorize attachment: {}", + attestation.attestation_id, + verification.diagnostics.join("; ") + ))); + } + } + Ok(()) + } + + fn artifact_attestation_producer_trusted( + &self, + statement: &ArtifactAttestationStatementV1, + diagnostics: &mut Vec, + ) -> bool { + if !matches!( + statement.producer_trust, + ArtifactProducerTrustTierV1::CertifiedSignedPlugin + | ArtifactProducerTrustTierV1::LocallyTrustedPlugin + ) { + return true; + } + let plugins = match self.installed_environment_plugins() { + Ok(plugins) => plugins, + Err(error) => { + diagnostics.push(format!( + "producer package or publisher trust cannot be verified: {error}" + )); + return false; + } + }; + let Some(plugin) = plugins.into_iter().find(|plugin| { + plugin.manifest.adapter.canonical_identity == statement.producer_identity + }) else { + diagnostics.push("producer package is removed or revoked".into()); + return false; + }; + if plugin.distribution_digest != statement.adapter_distribution_digest { + diagnostics.push("producer package digest no longer matches the attestation".into()); + return false; + } + if plugin.publisher != statement.publisher + || plugin.publisher_key_id != statement.publisher_key_id + { + diagnostics.push("producer publisher identity is removed, revoked, or changed".into()); + return false; + } + true + } + + fn verify_artifact_attestation_signature( + &self, + attestation: &ArtifactAttestationV1, + diagnostics: &mut Vec, + ) -> Result<(String, bool)> { + let Some(signature) = &attestation.signature else { + return Ok(("unsigned".into(), true)); + }; + if signature.algorithm != "ed25519" { + diagnostics.push("unsupported artifact attestation signature algorithm".into()); + return Ok(("unsupported".into(), false)); + } + let public_key = match decode_artifact_attestation_hex::<32>( + &signature.public_key_hex, + "artifact attestation public key", + ) { + Ok(public_key) => public_key, + Err(error) => { + diagnostics.push(error.to_string()); + return Ok(("invalid".into(), false)); + } + }; + let expected_key_id = artifact_attestation_signing_key_id(&public_key); + if signature.key_id != expected_key_id { + diagnostics.push("artifact attestation key ID does not match its public key".into()); + return Ok(("invalid".into(), false)); + } + if self.attestation_key_revocation(&expected_key_id)?.is_some() { + diagnostics.push("artifact attestation signing key is revoked".into()); + return Ok(("revoked".into(), false)); + } + let verifying_key = match VerifyingKey::from_bytes(&public_key) { + Ok(key) => key, + Err(error) => { + diagnostics.push(format!("invalid artifact attestation public key: {error}")); + return Ok(("invalid".into(), false)); + } + }; + let signature_bytes = match decode_artifact_attestation_hex::<64>( + &signature.signature_hex, + "artifact attestation signature", + ) { + Ok(signature) => signature, + Err(error) => { + diagnostics.push(error.to_string()); + return Ok(("invalid".into(), false)); + } + }; + let signature = Signature::from_bytes(&signature_bytes); + let statement_bytes = cbor(&attestation.statement)?; + if verifying_key.verify(&statement_bytes, &signature).is_err() { + diagnostics.push("artifact attestation signature verification failed".into()); + return Ok(("invalid".into(), false)); + } + Ok(("verified".into(), true)) + } + + fn validate_envelope_validation_receipts(&self, envelope: &ArtifactEnvelopeV1) -> Result<()> { + let mut seen_receipts = BTreeSet::new(); + let mut host_structural_seal = false; + let mut host_policy_seal = false; + for receipt_id in &envelope.validation_receipt_ids { + if !seen_receipts.insert(receipt_id.clone()) { + return Err(Error::InvalidInput(format!( + "artifact envelope repeats validation receipt `{receipt_id}`" + ))); + } + let receipt = self.artifact_validation_receipt(receipt_id)?; + if receipt.desired_identity != envelope.desired_identity + || receipt.tree_root_id != envelope.tree_root_id + || receipt.outcome != ArtifactValidationOutcomeV1::Passed + { + return Err(Error::InvalidInput(format!( + "artifact validation receipt `{receipt_id}` does not pass for the envelope desired identity and tree" + ))); + } + if receipt.validator_identity == HOST_WORKSPACE_LAYER_SEAL_VALIDATOR { + match (&receipt.declaration.kind, receipt.declaration.name.as_str()) { + ( + ArtifactValidationKindV1::Structural, + HOST_WORKSPACE_LAYER_STRUCTURAL_SEAL, + ) if receipt.declaration.required => { + host_structural_seal = true; + } + (ArtifactValidationKindV1::Policy, HOST_WORKSPACE_LAYER_POLICY_SEAL) + if receipt.declaration.required => + { + host_policy_seal = true; + } + _ => {} + } + } + } + if matches!( + envelope.desired_identity, + ArtifactDesiredIdentityV1::WorkspaceLayerV1 { .. } + ) && (!host_structural_seal || !host_policy_seal) + { + return Err(Error::InvalidInput( + "workspace layer artifact is missing required host structural or policy seal evidence" + .into(), + )); + } + Ok(()) + } + + pub fn list_artifact_quarantines(&self) -> Result> { + let mut statement = self.conn.prepare( + "SELECT quarantine_id,trust_scope,desired_key,incumbent_envelope_id, + candidate_envelope_id,reason_code,evidence_object_id,state,resolution, + created_at,resolved_at + FROM artifact_quarantines ORDER BY created_at,quarantine_id", + )?; + let rows = statement + .query_map([], artifact_quarantine_tuple_from_row)? + .collect::, _>>()?; + if rows.len() > MAX_PUBLIC_ARTIFACT_REPORT_ITEMS { + return Err(Error::InvalidInput(format!( + "artifact quarantine report contains {} rows; maximum is {MAX_PUBLIC_ARTIFACT_REPORT_ITEMS}", + rows.len() + ))); + } + rows.into_iter().map(artifact_quarantine_record).collect() + } + + pub fn artifact_quarantine( + &self, + quarantine_id: &ArtifactQuarantineId, + ) -> Result { + let row = self + .conn + .query_row( + "SELECT quarantine_id,trust_scope,desired_key,incumbent_envelope_id, + candidate_envelope_id,reason_code,evidence_object_id,state,resolution, + created_at,resolved_at + FROM artifact_quarantines WHERE quarantine_id=?1", + params![quarantine_id.0], + artifact_quarantine_tuple_from_row, + ) + .optional()? + .ok_or_else(|| { + Error::InvalidInput(format!( + "artifact quarantine `{quarantine_id}` does not exist" + )) + })?; + artifact_quarantine_record(row) + } + + pub fn resolve_artifact_quarantine( + &self, + quarantine_id: &ArtifactQuarantineId, + resolution: ArtifactQuarantineResolutionV1, + ) -> Result { + let _lock = self.acquire_write_lock()?; + let record = self.artifact_quarantine(quarantine_id)?; + if record.state != "active" { + return Err(Error::InvalidInput(format!( + "artifact quarantine `{quarantine_id}` is already resolved" + ))); + } + let competing = self.conn.query_row( + "SELECT COUNT(*) FROM artifact_quarantines + WHERE trust_scope=?1 AND desired_key=?2 AND state='active' AND quarantine_id<>?3", + params![record.trust_scope, record.desired_key, quarantine_id.0], + |row| row.get::<_, i64>(0), + )?; + if competing != 0 { + return Err(Error::InvalidInput(format!( + "artifact quarantine `{quarantine_id}` cannot resolve while {competing} related quarantine(s) remain active" + ))); + } + self.conn + .execute_batch("SAVEPOINT trail_quarantine_resolution")?; + let resolved = (|| -> Result<()> { + let accepted = match resolution { + ArtifactQuarantineResolutionV1::RetainPrivate + | ArtifactQuarantineResolutionV1::RetireAll => None, + ArtifactQuarantineResolutionV1::AcceptIncumbent => { + Some(record.incumbent_envelope_id.as_ref().ok_or_else(|| { + Error::InvalidInput(format!( + "artifact quarantine `{quarantine_id}` has no incumbent to accept" + )) + })?) + } + ArtifactQuarantineResolutionV1::AcceptCandidate => { + Some(&record.candidate_envelope_id) + } + }; + if matches!(resolution, ArtifactQuarantineResolutionV1::RetireAll) { + self.conn.execute( + "UPDATE artifact_envelopes SET state='retired',updated_at=?1 + WHERE envelope_id=?2 OR envelope_id=?3", + params![ + now_ts(), + record.incumbent_envelope_id.as_ref().map(|id| &id.0), + record.candidate_envelope_id.0, + ], + )?; + } else if let Some(accepted) = accepted { + self.conn.execute( + "UPDATE artifact_envelopes SET state='retired',updated_at=?1 + WHERE envelope_id=?2 OR envelope_id=?3", + params![ + now_ts(), + record.incumbent_envelope_id.as_ref().map(|id| &id.0), + record.candidate_envelope_id.0, + ], + )?; + self.conn.execute( + "UPDATE artifact_envelopes SET state='ready',updated_at=?1 + WHERE envelope_id=?2 AND verification_state='verified'", + params![now_ts(), accepted.0], + )?; + } + self.conn.execute( + "UPDATE artifact_quarantines SET state='resolved',resolution=?1,resolved_at=?2 + WHERE quarantine_id=?3 AND state='active'", + params![resolution.as_str(), now_ts(), quarantine_id.0], + )?; + self.conn.execute( + "DELETE FROM artifact_holds WHERE reason=?1", + params![quarantine_id.0], + )?; + Ok(()) + })(); + match resolved { + Ok(()) => self + .conn + .execute_batch("RELEASE SAVEPOINT trail_quarantine_resolution")?, + Err(error) => { + let _ = self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT trail_quarantine_resolution; + RELEASE SAVEPOINT trail_quarantine_resolution", + ); + return Err(error); + } + } + self.artifact_quarantine(quarantine_id) + } + + pub fn artifact_quarantine_list_report(&self) -> Result { + let quarantines = self.list_artifact_quarantines()?; + Ok(ArtifactQuarantineListReportV1 { + active_count: quarantines + .iter() + .filter(|record| record.state == "active") + .count() as u64, + resolved_count: quarantines + .iter() + .filter(|record| record.state == "resolved") + .count() as u64, + quarantines, + }) + } + + pub fn resolve_artifact_quarantine_report( + &self, + quarantine_id: &ArtifactQuarantineId, + resolution: ArtifactQuarantineResolutionV1, + ) -> Result { + let quarantine = self.resolve_artifact_quarantine(quarantine_id, resolution)?; + let mut affected_envelopes = quarantine + .incumbent_envelope_id + .iter() + .cloned() + .chain(std::iter::once(quarantine.candidate_envelope_id.clone())) + .collect::>(); + affected_envelopes.sort(); + affected_envelopes.dedup(); + Ok(ArtifactQuarantineResolutionReportV1 { + quarantine, + affected_envelopes, + recovery_commands: Vec::new(), + }) + } + + pub(crate) fn materialize_artifact_tree_under_write_lock( + &self, + tree_id: &ArtifactTreeId, + destination: &Path, + ) -> Result<()> { + if destination.exists() { + return Err(Error::InvalidPath { + path: destination.to_string_lossy().into_owned(), + reason: "artifact materialization destination already exists".into(), + }); + } + let tree: ArtifactTreeRootV1 = self.get_artifact_cas_object( + &tree_id.0, + ARTIFACT_TREE_ROOT_KIND, + ARTIFACT_TREE_ROOT_VERSION, + )?; + let (actual_id, _) = encode_artifact_tree_root(tree.clone())?; + if actual_id != *tree_id { + return Err(Error::Corrupt( + "artifact tree cannot materialize because its identity is invalid".into(), + )); + } + // Verify every edge and complete-file digest before exposing a path. + self.artifact_tree_flat_entries(tree_id)?; + fs::create_dir(destination)?; + let materialized = + self.materialize_artifact_directory(&tree.root_directory_id, destination, 0); + if let Err(error) = materialized { + super::workspace_layer::make_tree_writable(destination); + let _ = fs::remove_dir_all(destination); + return Err(error); + } + super::workspace_layer::sync_layer_tree(destination)?; + Ok(()) + } + + fn materialize_artifact_directory( + &self, + directory_id: &ArtifactTreeId, + destination: &Path, + depth: usize, + ) -> Result<()> { + if depth > MAX_ARTIFACT_TREE_DEPTH { + return Err(Error::Corrupt( + "artifact materialization exceeds the directory-depth bound".into(), + )); + } + let directory: ArtifactDirectoryNodeV1 = self.get_artifact_cas_object( + &directory_id.0, + ARTIFACT_DIRECTORY_NODE_KIND, + ARTIFACT_DIRECTORY_NODE_VERSION, + )?; + for entry in directory.entries { + validate_artifact_entry_name(&entry.name)?; + let path = destination.join(&entry.name); + match entry.target { + ArtifactDirectoryEntryTargetV1::Directory { node_id } => { + fs::create_dir(&path)?; + self.materialize_artifact_directory(&node_id, &path, depth + 1)?; + set_artifact_materialized_mode(&path, 0o755)?; + } + ArtifactDirectoryEntryTargetV1::File { node_id } => { + let file: ArtifactFileNodeV1 = self.get_artifact_cas_object( + &node_id.0, + ARTIFACT_FILE_NODE_KIND, + ARTIFACT_FILE_NODE_VERSION, + )?; + let mut output = OpenOptions::new() + .write(true) + .create_new(true) + .open(&path)?; + match &file.content { + ArtifactFileContentV1::Blob { blob_id } => { + let blob: ArtifactBlobV1 = self.get_artifact_cas_object( + &blob_id.0, + ARTIFACT_BLOB_KIND, + ARTIFACT_BLOB_VERSION, + )?; + output.write_all(&blob.bytes)?; + } + ArtifactFileContentV1::Chunks { chunk_list_id } => { + let list: ArtifactChunkListV1 = self.get_artifact_cas_object( + &chunk_list_id.0, + ARTIFACT_CHUNK_LIST_KIND, + ARTIFACT_CHUNK_LIST_VERSION, + )?; + for chunk_ref in list.chunks { + let chunk: ArtifactChunkV1 = self.get_artifact_cas_object( + &chunk_ref.chunk_id.0, + ARTIFACT_CHUNK_KIND, + ARTIFACT_CHUNK_VERSION, + )?; + output.write_all(&chunk.bytes)?; + } + } + } + output.sync_all()?; + set_artifact_materialized_mode(&path, file.mode)?; + } + ArtifactDirectoryEntryTargetV1::Symlink { target } => { + validate_artifact_symlink_target(&target)?; + #[cfg(unix)] + symlink_file(target, &path)?; + #[cfg(windows)] + std::os::windows::fs::symlink_file(target, &path)?; + } + } + } + Ok(()) + } + + fn flatten_artifact_directory( + &self, + directory_id: &ArtifactTreeId, + prefix: &str, + depth: usize, + visiting: &mut BTreeSet, + output: &mut BTreeMap, + ) -> Result<()> { + if depth > MAX_ARTIFACT_TREE_DEPTH || !visiting.insert(directory_id.clone()) { + return Err(Error::Corrupt( + "artifact directory graph is too deep or cyclic".into(), + )); + } + let directory: ArtifactDirectoryNodeV1 = self.get_artifact_cas_object( + &directory_id.0, + ARTIFACT_DIRECTORY_NODE_KIND, + ARTIFACT_DIRECTORY_NODE_VERSION, + )?; + let (actual_id, canonical) = encode_artifact_directory_node(directory.clone())?; + if actual_id != *directory_id + || from_cbor::(&canonical)? != directory + { + return Err(Error::Corrupt(format!( + "artifact directory `{directory_id}` has conflicting encoded identity" + ))); + } + for entry in directory.entries { + let path = if prefix.is_empty() { + entry.name.clone() + } else { + format!("{prefix}/{}", entry.name) + }; + if output.len() as u64 >= MAX_ARTIFACT_TREE_ENTRIES { + return Err(Error::Corrupt( + "artifact directory graph exceeds its entry bound".into(), + )); + } + match entry.target { + ArtifactDirectoryEntryTargetV1::Directory { node_id } => { + if output + .insert( + path.clone(), + ArtifactFlatEntry { + kind: "directory", + mode: 0o755, + size_bytes: 0, + content_hash: None, + symlink_target: None, + }, + ) + .is_some() + { + return Err(Error::Corrupt("duplicate artifact tree path".into())); + } + self.flatten_artifact_directory(&node_id, &path, depth + 1, visiting, output)?; + } + ArtifactDirectoryEntryTargetV1::File { node_id } => { + let file: ArtifactFileNodeV1 = self.get_artifact_cas_object( + &node_id.0, + ARTIFACT_FILE_NODE_KIND, + ARTIFACT_FILE_NODE_VERSION, + )?; + let (actual_id, _) = encode_artifact_file_node(file.clone())?; + if actual_id != node_id { + return Err(Error::Corrupt( + "artifact file node has conflicting encoded identity".into(), + )); + } + self.verify_artifact_file_content(&file)?; + if output + .insert( + path, + ArtifactFlatEntry { + kind: "file", + mode: file.mode, + size_bytes: file.size_bytes, + content_hash: Some(file.content_sha256), + symlink_target: None, + }, + ) + .is_some() + { + return Err(Error::Corrupt("duplicate artifact tree path".into())); + } + } + ArtifactDirectoryEntryTargetV1::Symlink { target } => { + validate_artifact_symlink_within_tree(prefix, &target)?; + if output + .insert( + path, + ArtifactFlatEntry { + kind: "symlink", + mode: 0o777, + size_bytes: 0, + content_hash: None, + symlink_target: Some(target), + }, + ) + .is_some() + { + return Err(Error::Corrupt("duplicate artifact tree path".into())); + } + } + } + } + visiting.remove(directory_id); + Ok(()) + } + + fn verify_artifact_file_content(&self, file: &ArtifactFileNodeV1) -> Result<()> { + let mut hasher = Sha256::new(); + let mut size = 0u64; + match &file.content { + ArtifactFileContentV1::Blob { blob_id } => { + let blob: ArtifactBlobV1 = self.get_artifact_cas_object( + &blob_id.0, + ARTIFACT_BLOB_KIND, + ARTIFACT_BLOB_VERSION, + )?; + let (actual, _) = encode_artifact_blob(blob.clone())?; + if actual != *blob_id { + return Err(Error::Corrupt( + "artifact blob identity edge is invalid".into(), + )); + } + size = blob.bytes.len() as u64; + hasher.update(blob.bytes); + } + ArtifactFileContentV1::Chunks { chunk_list_id } => { + let list: ArtifactChunkListV1 = self.get_artifact_cas_object( + &chunk_list_id.0, + ARTIFACT_CHUNK_LIST_KIND, + ARTIFACT_CHUNK_LIST_VERSION, + )?; + let (actual, _) = encode_artifact_chunk_list(list.clone())?; + if actual != *chunk_list_id { + return Err(Error::Corrupt( + "artifact chunk-list identity edge is invalid".into(), + )); + } + for chunk_ref in list.chunks { + let chunk: ArtifactChunkV1 = self.get_artifact_cas_object( + &chunk_ref.chunk_id.0, + ARTIFACT_CHUNK_KIND, + ARTIFACT_CHUNK_VERSION, + )?; + let (actual, _) = encode_artifact_chunk(chunk.clone())?; + if actual != chunk_ref.chunk_id + || chunk.bytes.len() as u64 != chunk_ref.size_bytes + { + return Err(Error::Corrupt( + "artifact chunk identity edge is invalid".into(), + )); + } + size = size + .checked_add(chunk_ref.size_bytes) + .ok_or_else(|| Error::Corrupt("artifact file size edge overflow".into()))?; + hasher.update(chunk.bytes); + } + } + } + if size != file.size_bytes || hex::encode(hasher.finalize()) != file.content_sha256 { + return Err(Error::Corrupt( + "artifact file complete size or hash edge is invalid".into(), + )); + } + Ok(()) + } + + pub(crate) fn put_artifact_resolution_snapshot( + &self, + mut plan: ArtifactResolutionPlanV1, + snapshot_bytes: Vec, + resolved_identities: BTreeMap, + checksums: BTreeMap, + mut contacted_authorities: Vec, + refresh: bool, + ) -> Result<(ObjectId, ArtifactResolutionSnapshotV1)> { + let _lock = self.acquire_write_lock()?; + normalize_artifact_resolution_plan(&mut plan)?; + if !plan.credential_handles.is_empty() { + return Err(Error::InvalidInput( + "artifact resolution declares credential access; secret-influenced snapshots cannot enter shared CAS" + .into(), + )); + } + if snapshot_bytes.len() as u64 > plan.limits.candidate_bytes { + return Err(Error::InvalidInput(format!( + "artifact resolution candidate contains {} bytes; maximum is {}", + snapshot_bytes.len(), + plan.limits.candidate_bytes + ))); + } + contacted_authorities.sort(); + contacted_authorities.dedup(); + if contacted_authorities.len() > MAX_RESOLUTION_AUTHORITIES + || contacted_authorities + .iter() + .any(|authority| !plan.allowed_authorities.contains(authority)) + { + return Err(Error::InvalidInput( + "resolver contacted an undeclared or excessive network authority".into(), + )); + } + validate_identity_map(&resolved_identities, "resolved identity")?; + validate_identity_map(&checksums, "snapshot checksum")?; + + let current = self.artifact_resolution_snapshot_for_proposal(&plan.proposal_key)?; + if let Some((current_id, current_snapshot)) = current.as_ref() + && !refresh + { + let candidate_sha256 = sha256_hex(&snapshot_bytes); + if current_snapshot.content_sha256 != candidate_sha256 { + return Err(Error::InvalidInput(format!( + "proposal `{}` already has pinned snapshot {}; use explicit refresh to replace it", + plan.proposal_key, current_id + ))); + } + return Ok((current_id.clone(), current_snapshot.clone())); + } + + let content_sha256 = sha256_hex(&snapshot_bytes); + let content = ArtifactResolutionContentV1 { + version: ARTIFACT_RESOLUTION_SNAPSHOT_VERSION, + content_sha256: content_sha256.clone(), + bytes: snapshot_bytes, + }; + let predecessor_snapshot_id = current.map(|(snapshot_id, _)| snapshot_id); + let snapshot = ArtifactResolutionSnapshotV1 { + version: ARTIFACT_RESOLUTION_SNAPSHOT_VERSION, + proposal_key: plan.proposal_key.clone(), + source_root: plan.source_root.clone(), + component_id: plan.component_id.clone(), + adapter_identity: plan.adapter_identity.clone(), + snapshot_format: plan.snapshot_format.clone(), + content_object_id: self.put_object( + ARTIFACT_RESOLUTION_CONTENT_KIND, + ARTIFACT_RESOLUTION_SNAPSHOT_VERSION, + &content, + )?, + content_sha256, + resolved_identities, + checksums, + resolver_executable_identity: plan.executable_identity, + policy_identity: plan.policy_identity, + contacted_authorities, + predecessor_snapshot_id, + secret_taint: ArtifactSecretTaintV1::Clear, + verification_state: ArtifactResolutionVerificationStateV1::Verified, + }; + validate_artifact_resolution_snapshot(&snapshot)?; + let snapshot_id = self.put_object( + ARTIFACT_RESOLUTION_SNAPSHOT_KIND, + ARTIFACT_RESOLUTION_SNAPSHOT_VERSION, + &snapshot, + )?; + + self.conn + .execute_batch("SAVEPOINT trail_artifact_resolution_snapshot")?; + let publication = (|| -> Result<()> { + if refresh { + self.conn.execute( + "UPDATE artifact_resolution_snapshots + SET state='superseded', superseded_at=?1 + WHERE proposal_key=?2 AND state='current'", + params![now_ts(), plan.proposal_key], + )?; + } + self.conn.execute( + "INSERT INTO artifact_resolution_snapshots( + snapshot_id, proposal_key, source_root, component_id, + adapter_identity, content_object_id, predecessor_snapshot_id, + verification_state, state, created_at, superseded_at + ) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, 'verified', 'current', ?8, NULL)", + params![ + snapshot_id.0, + snapshot.proposal_key, + snapshot.source_root.0, + snapshot.component_id, + snapshot.adapter_identity, + snapshot.content_object_id.0, + snapshot + .predecessor_snapshot_id + .as_ref() + .map(|id| id.0.as_str()), + now_ts(), + ], + )?; + Ok(()) + })(); + match publication { + Ok(()) => self + .conn + .execute_batch("RELEASE SAVEPOINT trail_artifact_resolution_snapshot")?, + Err(error) => { + let _ = self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT trail_artifact_resolution_snapshot; + RELEASE SAVEPOINT trail_artifact_resolution_snapshot", + ); + return Err(error); + } + } + Ok((snapshot_id, snapshot)) + } + + pub(crate) fn put_artifact_validation_receipt( + &self, + receipt: ArtifactValidationReceiptV1, + ) -> Result { + let _lock = self.acquire_write_lock()?; + self.put_artifact_validation_receipt_under_write_lock(receipt) + } + + fn put_artifact_validation_receipt_under_write_lock( + &self, + receipt: ArtifactValidationReceiptV1, + ) -> Result { + validate_artifact_validation_receipt(&receipt)?; + self.put_object( + ARTIFACT_VALIDATION_RECEIPT_KIND, + ARTIFACT_VALIDATION_RECEIPT_VERSION, + &receipt, + ) + } + + fn put_host_workspace_layer_seal_receipt_under_write_lock( + &self, + declaration: ArtifactValidationV1, + desired_identity: ArtifactDesiredIdentityV1, + tree_root_id: ArtifactTreeId, + evidence: BTreeMap, + ) -> Result { + let outcome = ArtifactValidationOutcomeV1::Passed; + let validated_input_digest = artifact_validation_receipt_input_digest( + ARTIFACT_VALIDATION_RECEIPT_VERSION, + &declaration, + &desired_identity, + &tree_root_id, + HOST_WORKSPACE_LAYER_SEAL_VALIDATOR, + outcome, + &evidence, + )?; + self.put_artifact_validation_receipt_under_write_lock(ArtifactValidationReceiptV1 { + version: ARTIFACT_VALIDATION_RECEIPT_VERSION, + declaration, + desired_identity, + tree_root_id, + validator_identity: HOST_WORKSPACE_LAYER_SEAL_VALIDATOR.into(), + validated_input_digest, + outcome, + evidence, + }) + } + + pub(crate) fn artifact_validation_receipt( + &self, + receipt_id: &ObjectId, + ) -> Result { + let receipt = self.get_object(ARTIFACT_VALIDATION_RECEIPT_KIND, receipt_id)?; + validate_artifact_validation_receipt(&receipt).map_err(|error| { + Error::Corrupt(format!( + "artifact validation receipt `{receipt_id}` is invalid: {error}" + )) + })?; + Ok(receipt) + } + + pub(crate) fn artifact_resolution_snapshot_for_proposal( + &self, + proposal_key: &str, + ) -> Result> { + validate_resolution_text(proposal_key, "proposal key")?; + let snapshot_id = self + .conn + .query_row( + "SELECT snapshot_id FROM artifact_resolution_snapshots + WHERE proposal_key=?1 AND state='current'", + params![proposal_key], + |row| row.get::<_, String>(0), + ) + .optional()? + .map(ObjectId); + let Some(snapshot_id) = snapshot_id else { + return Ok(None); + }; + let snapshot = self.get_object(ARTIFACT_RESOLUTION_SNAPSHOT_KIND, &snapshot_id)?; + validate_artifact_resolution_snapshot(&snapshot)?; + Ok(Some((snapshot_id, snapshot))) + } + + pub(crate) fn artifact_resolution_snapshot_for_component( + &self, + source_root: &ObjectId, + component_id: &str, + adapter_identity: &str, + ) -> Result> { + let mut statement = self.conn.prepare( + "SELECT snapshot_id FROM artifact_resolution_snapshots + WHERE source_root=?1 AND component_id=?2 AND adapter_identity=?3 + AND state='current' AND verification_state='verified' + ORDER BY proposal_key LIMIT 2", + )?; + let snapshot_ids = statement + .query_map( + params![source_root.0, component_id, adapter_identity], + |row| row.get::<_, String>(0), + )? + .collect::, _>>()?; + if snapshot_ids.len() > 1 { + return Err(Error::Corrupt(format!( + "environment component `{component_id}` has multiple current resolution snapshots for one source root and adapter" + ))); + } + let Some(snapshot_id) = snapshot_ids.into_iter().next().map(ObjectId) else { + return Ok(None); + }; + let snapshot = self.get_object(ARTIFACT_RESOLUTION_SNAPSHOT_KIND, &snapshot_id)?; + validate_artifact_resolution_snapshot(&snapshot)?; + if snapshot.source_root != *source_root + || snapshot.component_id != component_id + || snapshot.adapter_identity != adapter_identity + { + return Err(Error::Corrupt(format!( + "resolution snapshot `{snapshot_id}` does not match its component lookup" + ))); + } + Ok(Some((snapshot_id, snapshot))) + } + + pub(crate) fn has_current_artifact_resolution_snapshot( + &self, + source_root: &ObjectId, + component_id: &str, + adapter_identity: &str, + ) -> Result { + Ok(self.conn.query_row( + "SELECT EXISTS( + SELECT 1 FROM artifact_resolution_snapshots + WHERE source_root=?1 AND component_id=?2 AND adapter_identity=?3 + AND state='current' AND verification_state='verified' + )", + params![source_root.0, component_id, adapter_identity], + |row| row.get::<_, bool>(0), + )?) + } + + pub(crate) fn artifact_resolution_snapshot_content( + &self, + snapshot: &ArtifactResolutionSnapshotV1, + ) -> Result> { + validate_artifact_resolution_snapshot(snapshot)?; + let content: ArtifactResolutionContentV1 = self.get_object( + ARTIFACT_RESOLUTION_CONTENT_KIND, + &snapshot.content_object_id, + )?; + if content.version != ARTIFACT_RESOLUTION_SNAPSHOT_VERSION + || content.content_sha256 != snapshot.content_sha256 + || sha256_hex(&content.bytes) != snapshot.content_sha256 + { + return Err(Error::Corrupt(format!( + "artifact resolution snapshot content {} failed identity verification", + snapshot.content_object_id + ))); + } + Ok(content.bytes) + } + + pub(crate) fn artifact_resolution_snapshot_content_by_id( + &self, + snapshot_id: &ObjectId, + ) -> Result<(ArtifactResolutionSnapshotV1, Vec)> { + let snapshot: ArtifactResolutionSnapshotV1 = + self.get_object(ARTIFACT_RESOLUTION_SNAPSHOT_KIND, snapshot_id)?; + validate_artifact_resolution_snapshot(&snapshot)?; + let bytes = self.artifact_resolution_snapshot_content(&snapshot)?; + Ok((snapshot, bytes)) + } +} + +fn normalized_resolution_authority_evidence( + plan: &ArtifactResolutionPlanV1, + mut contacted_authorities: Vec, +) -> Result { + normalize_string_set( + &mut contacted_authorities, + MAX_RESOLUTION_AUTHORITIES, + "contacted authority", + )?; + if contacted_authorities + .iter() + .any(|authority| !plan.allowed_authorities.contains(authority)) + { + return Err(Error::InvalidInput( + "resolver contacted an undeclared network authority".into(), + )); + } + Ok(ArtifactResolutionAuthorityEvidenceV1 { + allowed_authorities: plan.allowed_authorities.clone(), + contacted_authorities, + credential_handles: plan.credential_handles.clone(), + credential_values_redacted: true, + }) +} + +fn redact_resolution_bytes(bytes: &[u8], redactions: &[Vec]) -> Vec { + const REDACTED: &[u8] = b"[REDACTED]"; + let mut output = bytes.to_vec(); + for secret in redactions.iter().filter(|secret| !secret.is_empty()) { + let mut cursor = 0usize; + while cursor.saturating_add(secret.len()) <= output.len() { + let Some(offset) = output[cursor..] + .windows(secret.len()) + .position(|window| window == secret) + else { + break; + }; + let start = cursor + offset; + output.splice(start..start + secret.len(), REDACTED.iter().copied()); + cursor = start + REDACTED.len(); + } + } + output +} + +fn artifact_resolution_attempt_status_str( + status: ArtifactResolutionAttemptStatusV1, +) -> &'static str { + match status { + ArtifactResolutionAttemptStatusV1::Running => "running", + ArtifactResolutionAttemptStatusV1::Succeeded => "succeeded", + ArtifactResolutionAttemptStatusV1::Failed => "failed", + ArtifactResolutionAttemptStatusV1::Cancelled => "cancelled", + ArtifactResolutionAttemptStatusV1::Abandoned => "abandoned", + } +} + +fn parse_artifact_resolution_attempt_status( + status: &str, +) -> Result { + match status { + "running" => Ok(ArtifactResolutionAttemptStatusV1::Running), + "succeeded" => Ok(ArtifactResolutionAttemptStatusV1::Succeeded), + "failed" => Ok(ArtifactResolutionAttemptStatusV1::Failed), + "cancelled" => Ok(ArtifactResolutionAttemptStatusV1::Cancelled), + "abandoned" => Ok(ArtifactResolutionAttemptStatusV1::Abandoned), + other => Err(Error::Corrupt(format!( + "invalid artifact resolution attempt status `{other}`" + ))), + } +} + +pub(crate) fn normalize_artifact_resolution_plan( + plan: &mut ArtifactResolutionPlanV1, +) -> Result<()> { + if plan.version != ARTIFACT_RESOLUTION_PLAN_VERSION { + return Err(Error::InvalidInput(format!( + "artifact resolution plan version {} is unsupported", + plan.version + ))); + } + validate_artifact_resolution_capability_ceiling(plan)?; + for (value, field) in [ + (&plan.proposal_key, "proposal key"), + (&plan.component_id, "component id"), + (&plan.adapter_identity, "adapter identity"), + (&plan.policy_identity, "policy identity"), + (&plan.program, "program"), + (&plan.resolved_program, "resolved program"), + (&plan.executable_identity, "executable identity"), + (&plan.snapshot_format, "snapshot format"), + ] { + validate_resolution_text(value, field)?; + } + if plan.argv.is_empty() || plan.argv.len() > MAX_RESOLUTION_ARGV { + return Err(Error::InvalidInput(format!( + "artifact resolver argv must contain between 1 and {MAX_RESOLUTION_ARGV} entries" + ))); + } + for argument in &plan.argv { + validate_resolution_text(argument, "resolver argv")?; + } + validate_resolution_relative_path(&plan.working_directory, "working directory", true)?; + validate_resolution_relative_path(&plan.candidate_output, "candidate output", false)?; + if plan.readable_inputs.is_empty() || plan.readable_inputs.len() > MAX_RESOLUTION_INPUTS { + return Err(Error::InvalidInput(format!( + "artifact resolution plan must contain between 1 and {MAX_RESOLUTION_INPUTS} readable inputs" + ))); + } + for input in &plan.readable_inputs { + validate_resolution_relative_path(&input.source_path, "readable input", false)?; + validate_sha256(&input.content_hash, "readable input hash")?; + } + plan.readable_inputs.sort(); + if plan + .readable_inputs + .windows(2) + .any(|pair| pair[0].source_path == pair[1].source_path) + { + return Err(Error::InvalidInput( + "artifact resolution plan contains duplicate readable input paths".into(), + )); + } + normalize_string_set( + &mut plan.allowed_authorities, + MAX_RESOLUTION_AUTHORITIES, + "allowed authority", + )?; + normalize_string_set( + &mut plan.credential_handles, + MAX_RESOLUTION_CREDENTIAL_HANDLES, + "credential handle", + )?; + if plan.environment_roles.len() > MAX_RESOLUTION_ENVIRONMENT_NAMES { + return Err(Error::InvalidInput(format!( + "artifact resolution plan has too many environment roles; maximum is {MAX_RESOLUTION_ENVIRONMENT_NAMES}" + ))); + } + for name in plan.environment_roles.keys() { + if name.is_empty() + || name.len() > 128 + || !name.chars().enumerate().all(|(index, character)| { + character == '_' + || character.is_ascii_alphanumeric() + && (index > 0 || !character.is_ascii_digit()) + }) + { + return Err(Error::InvalidInput(format!( + "artifact resolution environment name `{name}` is invalid" + ))); + } + } + if plan.limits.timeout_ms == 0 + || plan.limits.stdout_bytes == 0 + || plan.limits.stderr_bytes == 0 + || plan.limits.candidate_bytes == 0 + || plan.limits.candidate_entries == 0 + || plan.limits.child_processes == 0 + { + return Err(Error::InvalidInput( + "artifact resolver limits must all be non-zero".into(), + )); + } + if plan.limits.timeout_ms > MAX_RESOLUTION_TIMEOUT_MS + || plan.limits.stdout_bytes > MAX_RESOLUTION_CAPTURE_BYTES + || plan.limits.stderr_bytes > MAX_RESOLUTION_CAPTURE_BYTES + || plan.limits.candidate_bytes > MAX_RESOLUTION_CANDIDATE_BYTES + || plan.limits.candidate_entries > MAX_RESOLUTION_CANDIDATE_ENTRIES + || plan.limits.child_processes > MAX_RESOLUTION_CHILD_PROCESSES + { + return Err(Error::InvalidInput(format!( + "artifact resolver limits exceed host ceilings: timeout_ms<={MAX_RESOLUTION_TIMEOUT_MS}, capture_bytes<={MAX_RESOLUTION_CAPTURE_BYTES}, candidate_bytes<={MAX_RESOLUTION_CANDIDATE_BYTES}, candidate_entries<={MAX_RESOLUTION_CANDIDATE_ENTRIES}, child_processes<={MAX_RESOLUTION_CHILD_PROCESSES}" + ))); + } + if plan.validations.is_empty() || plan.validations.len() > MAX_RESOLUTION_VALIDATIONS { + return Err(Error::InvalidInput(format!( + "artifact resolution plan must contain between 1 and {MAX_RESOLUTION_VALIDATIONS} validations" + ))); + } + for validation in &plan.validations { + validate_artifact_validation_declaration(validation)?; + } + plan.validations.sort(); + if plan + .validations + .windows(2) + .any(|pair| pair[0].name == pair[1].name) + { + return Err(Error::InvalidInput( + "artifact resolution plan contains duplicate validation names".into(), + )); + } + Ok(()) +} + +fn validate_artifact_resolution_capability_ceiling(plan: &ArtifactResolutionPlanV1) -> Result<()> { + let ceiling = ArtifactCapabilityCeilingV1::for_phase( + ArtifactProducerTrustTierV1::RepositoryDeclaration, + ArtifactExecutionPhaseV1::Resolve, + ); + if ceiling.publication_authority + || ceiling.processes != ArtifactProcessCapabilityV1::DeclaredExecutable + || ceiling.filesystem_read != ArtifactFilesystemReadCapabilityV1::DeclaredInputs + || ceiling.filesystem_write != ArtifactFilesystemWriteCapabilityV1::IsolatedCandidate + || (!plan.allowed_authorities.is_empty() + && ceiling.network != ArtifactNetworkCapabilityV1::ExactAuthorities) + || (!plan.credential_handles.is_empty() + && ceiling.secrets != ArtifactSecretCapabilityV1::OpaqueHandles) + { + return Err(Error::InvalidInput(format!( + "artifact resolver plan `{}` exceeds the repository-declaration resolver capability ceiling", + plan.proposal_key + ))); + } + Ok(()) +} + +pub(crate) fn artifact_desired_key_v2( + mut material: ArtifactDesiredKeyMaterialV2, +) -> Result { + if material.version != ARTIFACT_DESIRED_KEY_MATERIAL_VERSION { + return Err(Error::InvalidInput(format!( + "artifact desired-key material version {} is unsupported", + material.version + ))); + } + for (value, field) in [ + (&material.component_id, "component id"), + (&material.adapter_identity, "adapter identity"), + ( + &material.adapter_implementation_version, + "adapter implementation version", + ), + ( + &material.adapter_distribution_digest, + "adapter distribution digest", + ), + (&material.adapter_protocol, "adapter protocol"), + ( + &material.source_closure.normalizer_version, + "source normalizer", + ), + (&material.target, "target"), + (&material.platform, "platform"), + (&material.architecture, "architecture"), + (&material.abi, "ABI"), + (&material.portability_scope, "portability scope"), + (&material.trust_scope, "trust scope"), + (&material.network_policy, "network policy"), + (&material.sandbox_policy, "sandbox policy"), + ] { + validate_resolution_text(value, field)?; + } + if !material.source_closure.certified_complete + && material.source_closure.complete_source_root.is_none() + { + return Err(Error::InvalidInput( + "uncertified artifact source closure must pin the complete source root".into(), + )); + } + if let Some(snapshot_id) = &material.resolution_snapshot_id { + validate_resolution_text(&snapshot_id.0, "resolution snapshot ID")?; + } + if let Some(source_root) = &material.source_closure.complete_source_root { + validate_resolution_text(&source_root.0, "complete source root")?; + } + if material.source_closure.certified_complete + && material.source_closure.declared_inputs.is_empty() + { + return Err(Error::InvalidInput( + "certified artifact source closure must declare at least one input".into(), + )); + } + if material.source_closure.declared_inputs.len() > MAX_RESOLUTION_INPUTS { + return Err(Error::InvalidInput(format!( + "artifact source closure exceeds {MAX_RESOLUTION_INPUTS} declared inputs" + ))); + } + for input in &material.source_closure.declared_inputs { + validate_resolution_relative_path(&input.source_path, "source closure input", false)?; + validate_sha256(&input.content_hash, "source closure input hash")?; + } + material.source_closure.declared_inputs.sort(); + if material + .source_closure + .declared_inputs + .windows(2) + .any(|pair| pair[0].source_path == pair[1].source_path) + { + return Err(Error::InvalidInput( + "artifact source closure contains duplicate paths".into(), + )); + } + validate_identity_map(&material.upstream_identities, "upstream identity")?; + if material.actions.is_empty() || material.actions.len() > MAX_RESOLUTION_VALIDATIONS { + return Err(Error::InvalidInput( + "artifact desired-key material has an empty or excessive action list".into(), + )); + } + for action in &mut material.actions { + validate_resolution_text(&action.name, "action name")?; + validate_resolution_text(&action.executable_identity, "action executable identity")?; + validate_resolution_relative_path( + &action.working_directory, + "action working directory", + true, + )?; + if action.argv.is_empty() || action.argv.len() > MAX_RESOLUTION_ARGV { + return Err(Error::InvalidInput( + "artifact action argv is empty or excessive".into(), + )); + } + for argument in &action.argv { + validate_resolution_text(argument, "action argv")?; + } + normalize_string_set( + &mut action.environment_names, + MAX_RESOLUTION_ENVIRONMENT_NAMES, + "action environment name", + )?; + } + material.actions.sort(); + if material + .actions + .windows(2) + .any(|pair| pair[0].name == pair[1].name) + { + return Err(Error::InvalidInput( + "artifact desired-key material contains duplicate action names".into(), + )); + } + if material.outputs.is_empty() || material.outputs.len() > MAX_RESOLUTION_VALIDATIONS { + return Err(Error::InvalidInput( + "artifact desired-key material has an empty or excessive output list".into(), + )); + } + for output in &material.outputs { + validate_resolution_text(&output.name, "output name")?; + validate_resolution_relative_path(&output.output_path, "output path", false)?; + validate_resolution_relative_path(&output.mount_path, "output mount path", false)?; + if let Some(gate) = &output.gate { + validate_resolution_text(gate, "output gate")?; + } + if !material.portability_certified + && (output.reuse != EnvironmentReuseMode::None + || output.scope != EnvironmentSharingScope::Lane) + { + return Err(Error::InvalidInput(format!( + "artifact output `{}` lacks portability evidence and must use lane-private, non-reusable policy", + output.name + ))); + } + } + material.outputs.sort(); + for validation in &material.validations { + validate_artifact_validation_declaration(validation)?; + } + material.validations.sort(); + for export in &material.source_exports { + validate_resolution_text(&export.name, "source export name")?; + validate_resolution_text(&export.output_name, "source export output name")?; + validate_resolution_relative_path( + &export.artifact_subpath, + "source export subpath", + false, + )?; + validate_resolution_relative_path(&export.destination, "source export destination", false)?; + validate_resolution_text(&export.collision_policy, "source export collision policy")?; + validate_resolution_text(&export.required_validation, "source export validation")?; + if let Some(gate) = &export.required_gate { + validate_resolution_text(gate, "source export gate")?; + } + validate_resolution_text(&export.authorization_mode, "source export authorization")?; + if !matches!(export.collision_policy.as_str(), "fail" | "replace") + || export.authorization_mode != "explicit" + { + return Err(Error::InvalidInput(format!( + "source export `{}` has an unsupported collision or authorization mode", + export.name + ))); + } + } + material.source_exports.sort(); + if material.build_environment.len() > MAX_RESOLUTION_ENVIRONMENT_NAMES { + return Err(Error::InvalidInput( + "artifact build environment contains too many entries".into(), + )); + } + for (name, value) in &material.build_environment { + validate_resolution_text(name, "build environment name")?; + validate_resolution_text(value, "build environment value")?; + if is_sensitive_json_key(name) || contains_sensitive_text(value) { + return Err(Error::InvalidInput(format!( + "artifact build environment `{name}` may contain secret material" + ))); + } + } + let canonical = cbor(&material)?; + Ok(ArtifactDesiredKeyV2::new(&canonical)) +} + +pub(crate) fn diff_artifact_desired_key_v2( + previous: &ArtifactDesiredKeyMaterialV2, + current: &ArtifactDesiredKeyMaterialV2, +) -> Result { + let previous_key = artifact_desired_key_v2(previous.clone())?; + let current_key = artifact_desired_key_v2(current.clone())?; + let mut edges = Vec::new(); + + diff_artifact_scalar( + "resolution", + "format_version", + &previous.version, + ¤t.version, + &mut edges, + ); + diff_artifact_scalar( + "resolution", + "component_id", + &previous.component_id, + ¤t.component_id, + &mut edges, + ); + diff_artifact_scalar( + "resolution", + "snapshot", + &previous.resolution_snapshot_id, + ¤t.resolution_snapshot_id, + &mut edges, + ); + diff_artifact_scalar( + "resolution", + "source_closure", + &previous.source_closure, + ¤t.source_closure, + &mut edges, + ); + diff_artifact_map( + "resolution", + "upstream", + &previous.upstream_identities, + ¤t.upstream_identities, + &mut edges, + ); + + for (name, left, right) in [ + ( + "adapter_identity", + &previous.adapter_identity, + ¤t.adapter_identity, + ), + ( + "adapter_implementation_version", + &previous.adapter_implementation_version, + ¤t.adapter_implementation_version, + ), + ( + "adapter_distribution_digest", + &previous.adapter_distribution_digest, + ¤t.adapter_distribution_digest, + ), + ( + "adapter_protocol", + &previous.adapter_protocol, + ¤t.adapter_protocol, + ), + ] { + diff_artifact_scalar("tool", name, left, right, &mut edges); + } + let previous_actions = previous + .actions + .iter() + .map(|action| (action.name.as_str(), action)) + .collect::>(); + let current_actions = current + .actions + .iter() + .map(|action| (action.name.as_str(), action)) + .collect::>(); + for name in previous_actions + .keys() + .chain(current_actions.keys()) + .copied() + .collect::>() + { + let left = previous_actions.get(name); + let right = current_actions.get(name); + diff_artifact_scalar( + "tool", + &format!("action:{name}:executable"), + &left.map(|action| &action.executable_identity), + &right.map(|action| &action.executable_identity), + &mut edges, + ); + let left_contract = left.map(|action| { + ( + action.phase, + &action.argv, + &action.working_directory, + &action.environment_names, + ) + }); + let right_contract = right.map(|action| { + ( + action.phase, + &action.argv, + &action.working_directory, + &action.environment_names, + ) + }); + diff_artifact_scalar("action", name, &left_contract, &right_contract, &mut edges); + } + diff_artifact_named_contracts( + "output", + &previous.outputs, + ¤t.outputs, + |output| output.name.as_str(), + &mut edges, + ); + diff_artifact_named_contracts( + "validation", + &previous.validations, + ¤t.validations, + |validation| validation.name.as_str(), + &mut edges, + ); + diff_artifact_named_contracts( + "export", + &previous.source_exports, + ¤t.source_exports, + |export| export.name.as_str(), + &mut edges, + ); + diff_artifact_map( + "trust", + "environment", + &previous.build_environment, + ¤t.build_environment, + &mut edges, + ); + for (name, left, right) in [ + ("target", &previous.target, ¤t.target), + ("platform", &previous.platform, ¤t.platform), + ( + "architecture", + &previous.architecture, + ¤t.architecture, + ), + ("abi", &previous.abi, ¤t.abi), + ( + "portability_scope", + &previous.portability_scope, + ¤t.portability_scope, + ), + ("trust_scope", &previous.trust_scope, ¤t.trust_scope), + ( + "network_policy", + &previous.network_policy, + ¤t.network_policy, + ), + ] { + diff_artifact_scalar("trust", name, left, right, &mut edges); + } + diff_artifact_scalar( + "trust", + "portability_certified", + &previous.portability_certified, + ¤t.portability_certified, + &mut edges, + ); + diff_artifact_scalar( + "trust", + "script_policy", + &previous.script_policy, + ¤t.script_policy, + &mut edges, + ); + diff_artifact_scalar( + "sandbox", + "sandbox_policy", + &previous.sandbox_policy, + ¤t.sandbox_policy, + &mut edges, + ); + edges.sort_by(|left, right| { + artifact_invalidation_dimension_rank(&left.dimension) + .cmp(&artifact_invalidation_dimension_rank(&right.dimension)) + .then_with(|| left.name.cmp(&right.name)) + .then_with(|| left.change.cmp(&right.change)) + }); + edges.dedup(); + Ok(ArtifactDesiredKeyDiffV2 { + previous_key, + current_key, + first: edges.first().cloned(), + edges, + }) +} + +fn artifact_invalidation_dimension_rank(dimension: &str) -> u8 { + match dimension { + "resolution" => 0, + "tool" => 1, + "action" => 2, + "output" => 3, + "validation" => 4, + "export" => 5, + "trust" => 6, + "sandbox" => 7, + _ => u8::MAX, + } +} + +type ArtifactQuarantineTuple = ( + String, + String, + String, + Option, + String, + String, + String, + String, + Option, + i64, + Option, +); + +fn artifact_quarantine_tuple_from_row( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + row.get(6)?, + row.get(7)?, + row.get(8)?, + row.get(9)?, + row.get(10)?, + )) +} + +fn artifact_quarantine_record(row: ArtifactQuarantineTuple) -> Result { + Ok(ArtifactQuarantineRecordV1 { + quarantine_id: ArtifactQuarantineId::parse(row.0) + .map_err(|error| Error::Corrupt(format!("invalid artifact quarantine ID: {error}")))?, + trust_scope: row.1, + desired_key: row.2, + incumbent_envelope_id: row + .3 + .map(ArtifactEnvelopeId::parse) + .transpose() + .map_err(|error| Error::Corrupt(format!("invalid artifact envelope ID: {error}")))?, + candidate_envelope_id: ArtifactEnvelopeId::parse(row.4) + .map_err(|error| Error::Corrupt(format!("invalid artifact envelope ID: {error}")))?, + reason_code: row.5, + evidence_object_id: ObjectId(row.6), + state: row.7, + resolution: row.8, + created_at: row.9, + resolved_at: row.10, + }) +} + +fn diff_artifact_scalar( + dimension: &str, + name: &str, + previous: &T, + current: &T, + edges: &mut Vec, +) { + if previous != current { + edges.push(ArtifactInvalidationEdgeV2 { + dimension: dimension.into(), + name: name.into(), + change: "modified".into(), + }); + } +} + +fn diff_artifact_map( + dimension: &str, + prefix: &str, + previous: &BTreeMap, + current: &BTreeMap, + edges: &mut Vec, +) { + for name in previous + .keys() + .chain(current.keys()) + .collect::>() + { + let change = match (previous.get(name), current.get(name)) { + (None, Some(_)) => Some("added"), + (Some(_), None) => Some("removed"), + (Some(left), Some(right)) if left != right => Some("modified"), + _ => None, + }; + if let Some(change) = change { + edges.push(ArtifactInvalidationEdgeV2 { + dimension: dimension.into(), + name: format!("{prefix}:{name}"), + change: change.into(), + }); + } + } +} + +fn diff_artifact_named_contracts( + dimension: &str, + previous: &[T], + current: &[T], + name: impl Fn(&T) -> &str, + edges: &mut Vec, +) { + let previous = previous + .iter() + .map(|item| (name(item), item)) + .collect::>(); + let current = current + .iter() + .map(|item| (name(item), item)) + .collect::>(); + for item_name in previous + .keys() + .chain(current.keys()) + .copied() + .collect::>() + { + let change = match (previous.get(item_name), current.get(item_name)) { + (None, Some(_)) => Some("added"), + (Some(_), None) => Some("removed"), + (Some(left), Some(right)) if left != right => Some("modified"), + _ => None, + }; + if let Some(change) = change { + edges.push(ArtifactInvalidationEdgeV2 { + dimension: dimension.into(), + name: item_name.into(), + change: change.into(), + }); + } + } +} + +fn encode_artifact_directory_node( + mut node: ArtifactDirectoryNodeV1, +) -> Result<(ArtifactTreeId, Vec)> { + if node.version != ARTIFACT_DIRECTORY_NODE_VERSION { + return Err(Error::InvalidInput( + "artifact directory node has an unsupported version".into(), + )); + } + node.entries.sort(); + for entry in &node.entries { + validate_artifact_entry_name(&entry.name)?; + if let ArtifactDirectoryEntryTargetV1::Symlink { target } = &entry.target { + validate_artifact_symlink_target(target)?; + } + } + if node + .entries + .windows(2) + .any(|pair| pair[0].name == pair[1].name) + { + return Err(Error::InvalidInput( + "artifact directory contains duplicate entry names".into(), + )); + } + let bytes = cbor(&node)?; + Ok(( + artifact_tree_id(ARTIFACT_DIRECTORY_NODE_KIND, &bytes), + bytes, + )) +} + +fn canonical_artifact_directory_node( + node: ArtifactDirectoryNodeV1, +) -> Result { + let (_, bytes) = encode_artifact_directory_node(node)?; + from_cbor(&bytes) +} + +fn decode_artifact_directory_node(bytes: &[u8]) -> Result { + let node: ArtifactDirectoryNodeV1 = from_cbor(bytes)?; + let (_, canonical) = encode_artifact_directory_node(node.clone())?; + if canonical != bytes { + return Err(Error::Corrupt( + "artifact directory node is not canonically ordered".into(), + )); + } + Ok(node) +} + +fn encode_artifact_blob(blob: ArtifactBlobV1) -> Result<(ArtifactBlobId, Vec)> { + if blob.version != ARTIFACT_BLOB_VERSION || sha256_hex(&blob.bytes) != blob.content_sha256 { + return Err(Error::InvalidInput( + "artifact blob version or complete content hash is invalid".into(), + )); + } + let bytes = cbor(&blob)?; + Ok((ArtifactBlobId::new(&blob.bytes), bytes)) +} + +fn decode_artifact_blob(bytes: &[u8], expected: &ArtifactBlobId) -> Result { + let blob: ArtifactBlobV1 = from_cbor(bytes)?; + let (actual, canonical) = encode_artifact_blob(blob.clone())?; + if &actual != expected || canonical != bytes { + return Err(Error::Corrupt( + "artifact blob content identity or canonical encoding is invalid".into(), + )); + } + Ok(blob) +} + +fn encode_artifact_chunk(chunk: ArtifactChunkV1) -> Result<(ArtifactChunkId, Vec)> { + if chunk.version != ARTIFACT_CHUNK_VERSION || sha256_hex(&chunk.bytes) != chunk.content_sha256 { + return Err(Error::InvalidInput( + "artifact chunk version or content hash is invalid".into(), + )); + } + let bytes = cbor(&chunk)?; + Ok((ArtifactChunkId::new(&chunk.bytes), bytes)) +} + +fn encode_artifact_chunk_list(list: ArtifactChunkListV1) -> Result<(ArtifactChunkListId, Vec)> { + if list.version != ARTIFACT_CHUNK_LIST_VERSION + || list.algorithm != "fastcdc-v1" + || list.chunks.is_empty() + || list.chunks.iter().any(|chunk| chunk.size_bytes == 0) + || list + .chunks + .iter() + .try_fold(0u64, |total, chunk| total.checked_add(chunk.size_bytes)) + != Some(list.file_size_bytes) + { + return Err(Error::InvalidInput( + "artifact chunk list has invalid version, algorithm, or size edges".into(), + )); + } + validate_sha256(&list.file_sha256, "chunk-list file hash")?; + let bytes = cbor(&list)?; + Ok(( + ArtifactChunkListId::new(&artifact_identity_seed(ARTIFACT_CHUNK_LIST_KIND, &bytes)), + bytes, + )) +} + +fn encode_artifact_file_node(node: ArtifactFileNodeV1) -> Result<(ArtifactFileId, Vec)> { + if node.version != ARTIFACT_FILE_NODE_VERSION + || node.mode & !0o777 != 0 + || node.executable != (node.mode & 0o111 != 0) + { + return Err(Error::InvalidInput( + "artifact file node has invalid version or normalized mode".into(), + )); + } + validate_sha256(&node.content_sha256, "file content hash")?; + let bytes = cbor(&node)?; + Ok(( + ArtifactFileId::new(&artifact_identity_seed(ARTIFACT_FILE_NODE_KIND, &bytes)), + bytes, + )) +} + +fn encode_artifact_tree_root(root: ArtifactTreeRootV1) -> Result<(ArtifactTreeId, Vec)> { + if root.version != ARTIFACT_TREE_ROOT_VERSION || root.path_normalizer != "trail-paths/v1" { + return Err(Error::InvalidInput( + "artifact tree root has invalid version or path normalizer".into(), + )); + } + let bytes = cbor(&root)?; + Ok((artifact_tree_id(ARTIFACT_TREE_ROOT_KIND, &bytes), bytes)) +} + +fn encode_artifact_envelope( + mut envelope: ArtifactEnvelopeV1, +) -> Result<(ArtifactEnvelopeId, Vec)> { + if envelope.version != ARTIFACT_ENVELOPE_VERSION { + return Err(Error::InvalidInput( + "artifact envelope has an unsupported version".into(), + )); + } + validate_resolution_text(&envelope.component_id, "envelope component id")?; + validate_resolution_text(&envelope.output_name, "envelope output name")?; + validate_resolution_text(&envelope.portability_scope, "envelope portability scope")?; + validate_resolution_text(&envelope.trust_scope, "envelope trust scope")?; + validate_artifact_secret_taint(&envelope.secret_taint)?; + envelope.validation_receipt_ids.sort(); + envelope.validation_receipt_ids.dedup(); + let bytes = cbor(&envelope)?; + Ok(( + ArtifactEnvelopeId::new(&artifact_identity_seed(ARTIFACT_ENVELOPE_KIND, &bytes)), + bytes, + )) +} + +fn encode_artifact_attestation( + attestation: ArtifactAttestationV1, +) -> Result<(ArtifactAttestationId, Vec)> { + validate_artifact_attestation_statement(&attestation.statement)?; + if let Some(signature) = &attestation.signature { + validate_resolution_text(&signature.algorithm, "attestation signature algorithm")?; + validate_resolution_text(&signature.key_id, "attestation signature key id")?; + decode_artifact_attestation_hex::<32>( + &signature.public_key_hex, + "artifact attestation public key", + )?; + decode_artifact_attestation_hex::<64>( + &signature.signature_hex, + "artifact attestation signature", + )?; + } + let bytes = cbor(&attestation)?; + Ok((ArtifactAttestationId::new(&bytes), bytes)) +} + +fn validate_artifact_attestation_statement( + statement: &ArtifactAttestationStatementV1, +) -> Result<()> { + if statement.version != ARTIFACT_ATTESTATION_VERSION { + return Err(Error::InvalidInput( + "artifact attestation has an unsupported version".into(), + )); + } + for (value, field) in [ + ( + &statement.producer_identity, + "attestation producer identity", + ), + ( + &statement.adapter_implementation_version, + "attestation adapter implementation version", + ), + ( + &statement.adapter_distribution_digest, + "attestation adapter distribution digest", + ), + (&statement.adapter_protocol, "attestation adapter protocol"), + (&statement.platform, "attestation platform"), + (&statement.architecture, "attestation architecture"), + (&statement.abi, "attestation ABI"), + ( + &statement.sandbox_enforcement, + "attestation sandbox enforcement", + ), + (&statement.network_policy, "attestation network policy"), + (&statement.output_name, "attestation output name"), + ( + &statement.portability_scope, + "attestation portability scope", + ), + (&statement.trust_scope, "attestation trust scope"), + ] { + validate_resolution_text(value, field)?; + } + for (value, field) in [ + (statement.publisher.as_deref(), "attestation publisher"), + ( + statement.publisher_key_id.as_deref(), + "attestation publisher key id", + ), + ] { + if let Some(value) = value { + validate_resolution_text(value, field)?; + } + } + validate_attestation_identity_map( + &statement.upstream_identities, + "attestation upstream identity", + )?; + validate_attestation_identity_map( + &statement.executable_identities, + "attestation executable identity", + )?; + if statement.validation_receipt_ids.len() > MAX_RESOLUTION_VALIDATIONS + || !statement + .validation_receipt_ids + .windows(2) + .all(|pair| pair[0] < pair[1]) + { + return Err(Error::InvalidInput( + "artifact attestation validation receipts are excessive, duplicated, or not canonical" + .into(), + )); + } + validate_artifact_secret_taint(&statement.secret_taint)?; + if !statement.secret_taint.is_clear() { + return Err(Error::InvalidInput( + "secret-tainted output cannot be attested for shared attachment".into(), + )); + } + if statement.capability_ceiling.publication_authority + || statement.capability_ceiling.producer_trust != statement.producer_trust + { + return Err(Error::InvalidInput( + "artifact attestation capability evidence is inconsistent or grants publication authority" + .into(), + )); + } + Ok(()) +} + +fn validate_attestation_identity_map(values: &BTreeMap, field: &str) -> Result<()> { + if values.len() > MAX_RESOLUTION_INPUTS { + return Err(Error::InvalidInput(format!( + "artifact {field} count exceeds {MAX_RESOLUTION_INPUTS}" + ))); + } + for (name, value) in values { + validate_resolution_text(name, field)?; + if value.len() > MAX_RESOLUTION_TEXT_BYTES + || value + .chars() + .any(|character| character.is_control() && !matches!(character, '\n' | '\r' | '\t')) + { + return Err(Error::InvalidInput(format!( + "artifact {field} value is oversized or contains control characters" + ))); + } + if is_sensitive_json_key(name) || contains_sensitive_text(value) { + return Err(Error::InvalidInput(format!( + "artifact {field} may contain secret material" + ))); + } + } + Ok(()) +} + +fn artifact_attestation_matches_envelope( + statement: &ArtifactAttestationStatementV1, + envelope: &ArtifactEnvelopeV1, +) -> bool { + statement.desired_identity == envelope.desired_identity + && statement.tree_root_id == envelope.tree_root_id + && statement.resolution_snapshot_id == envelope.resolution_snapshot_id + && statement.output_name == envelope.output_name + && statement.output_policy == envelope.output_policy + && statement.portability_scope == envelope.portability_scope + && statement.trust_scope == envelope.trust_scope + && statement.validation_receipt_ids == envelope.validation_receipt_ids + && statement.secret_taint == envelope.secret_taint +} + +fn decode_artifact_attestation_hex(value: &str, field: &str) -> Result<[u8; N]> { + let bytes = hex::decode(value) + .map_err(|error| Error::InvalidInput(format!("invalid {field}: {error}")))?; + bytes + .try_into() + .map_err(|_| Error::InvalidInput(format!("invalid {field} length"))) +} + +fn artifact_attestation_signing_key_id(public_key: &[u8; 32]) -> String { + format!( + "attestation_key_{}", + sha256_hex( + &[ + b"trail-artifact-attestation-key-v1\0".as_slice(), + public_key + ] + .concat() + ) + ) +} + +fn artifact_tree_id(kind: &str, bytes: &[u8]) -> ArtifactTreeId { + ArtifactTreeId::new(&artifact_identity_seed(kind, bytes)) +} + +fn artifact_identity_seed(kind: &str, bytes: &[u8]) -> Vec { + let mut seed = Vec::with_capacity(kind.len() + bytes.len() + 10); + seed.extend_from_slice(kind.as_bytes()); + seed.push(0); + seed.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); + seed.extend_from_slice(bytes); + seed +} + +fn validate_artifact_entry_name(name: &str) -> Result<()> { + let normalized = normalize_relative_path(name)?; + if normalized != name || name.contains('/') { + return Err(Error::InvalidInput(format!( + "artifact directory entry `{name}` is not one normalized path component" + ))); + } + Ok(()) +} + +fn validate_artifact_symlink_target(target: &str) -> Result<()> { + if target.is_empty() + || target.len() > MAX_RESOLUTION_TEXT_BYTES + || Path::new(target).is_absolute() + || target.chars().any(char::is_control) + { + return Err(Error::InvalidInput( + "artifact symlink target is empty, absolute, oversized, or contains controls".into(), + )); + } + Ok(()) +} + +fn validate_artifact_symlink_within_tree(parent: &str, target: &str) -> Result<()> { + let link_path = if parent.is_empty() { + "link".to_string() + } else { + format!("{parent}/link") + }; + resolve_artifact_symlink_path(&link_path, target).map(|_| ()) +} + +fn resolve_artifact_symlink_path(link_path: &str, target: &str) -> Result { + validate_artifact_symlink_target(target)?; + let parent = link_path.rsplit_once('/').map_or("", |(parent, _)| parent); + let mut components = parent + .split('/') + .filter(|component| !component.is_empty()) + .map(str::to_string) + .collect::>(); + for component in Path::new(target).components() { + match component { + Component::Normal(component) => { + let component = component.to_str().ok_or_else(|| Error::InvalidPath { + path: target.into(), + reason: "artifact symlink target must be valid Unicode".into(), + })?; + validate_artifact_entry_name(component)?; + components.push(component.to_string()); + } + Component::CurDir => {} + Component::ParentDir => { + if components.pop().is_none() { + return Err(Error::InvalidPath { + path: target.into(), + reason: "artifact symlink escapes the tree root".into(), + }); + } + } + Component::RootDir | Component::Prefix(_) => { + return Err(Error::InvalidPath { + path: target.into(), + reason: "artifact symlink target must be relative".into(), + }); + } + } + } + Ok(components.join("/")) +} + +fn ensure_artifact_file_unchanged( + path: &Path, + before: &fs::Metadata, + after: &fs::Metadata, + observed_bytes: u64, +) -> Result<()> { + if observed_bytes != before.len() || !same_artifact_metadata(before, after) { + return Err(Error::InvalidInput(format!( + "artifact file `{}` changed during ingestion", + path.display() + ))); + } + Ok(()) +} + +fn validate_artifact_secret_policy(bytes: &[u8], relative_path: Option<&str>) -> Result<()> { + let Ok(text) = std::str::from_utf8(bytes) else { + return Ok(()); + }; + let contains_private_key = { + let upper = text.to_ascii_uppercase(); + upper.contains("-----BEGIN ") && upper.contains("PRIVATE KEY-----") + }; + let sensitive = match relative_path { + Some(path) => { + contains_private_key + || is_secret_bearing_artifact_path(path) && contains_sensitive_text(text) + } + None => contains_sensitive_text(text), + }; + if sensitive { + let path = relative_path + .map(|path| format!(" `{path}`")) + .unwrap_or_default(); + return Err(Error::InvalidInput(format!( + "artifact content{path} may contain secret material and cannot enter shared CAS" + ))); + } + Ok(()) +} + +fn is_secret_bearing_artifact_path(path: &str) -> bool { + let name = path.rsplit('/').next().unwrap_or(path).to_ascii_lowercase(); + name == ".npmrc" + || name == ".pypirc" + || name == ".netrc" + || name == "credentials" + || name == "credentials.json" + || name == "secrets" + || name == "secrets.json" + || name == "id_rsa" + || name == "id_ed25519" + || name == ".env" + || name.starts_with(".env.") + || name.ends_with(".env") + || name.ends_with(".pem") + || name.ends_with(".key") +} + +fn validate_artifact_metadata_policy(path: &Path, metadata: &fs::Metadata) -> Result<()> { + #[cfg(unix)] + { + if metadata.permissions().mode() & 0o7000 != 0 { + return Err(Error::InvalidPath { + path: path.to_string_lossy().into_owned(), + reason: "setuid, setgid, and sticky artifact modes are prohibited".into(), + }); + } + if metadata.is_file() + && metadata.len() > ARTIFACT_WHOLE_BLOB_MAX_BYTES as u64 + && metadata.blocks().saturating_mul(512) < metadata.len() / 2 + { + return Err(Error::InvalidPath { + path: path.to_string_lossy().into_owned(), + reason: "excessively sparse artifact files are prohibited".into(), + }); + } + let mut attributes = xattr::list(path)? + .filter(|attribute| attribute != "com.apple.provenance") + .collect::>(); + attributes.sort(); + if let Some(attribute) = attributes.first() { + return Err(Error::InvalidPath { + path: path.to_string_lossy().into_owned(), + reason: format!( + "artifact extended attribute `{}` is prohibited", + attribute.to_string_lossy() + ), + }); + } + } + Ok(()) +} + +fn same_artifact_metadata(left: &fs::Metadata, right: &fs::Metadata) -> bool { + left.len() == right.len() + && left.file_type() == right.file_type() + && left.modified().ok() == right.modified().ok() +} + +#[cfg(unix)] +fn normalized_artifact_file_mode(metadata: &fs::Metadata) -> u32 { + if metadata.permissions().mode() & 0o111 != 0 { + 0o755 + } else { + 0o644 + } +} + +#[cfg(not(unix))] +fn normalized_artifact_file_mode(_metadata: &fs::Metadata) -> u32 { + 0o644 +} + +#[cfg(unix)] +fn set_artifact_materialized_mode(path: &Path, mode: u32) -> Result<()> { + fs::set_permissions(path, fs::Permissions::from_mode(mode))?; + Ok(()) +} + +#[cfg(not(unix))] +fn set_artifact_materialized_mode(_path: &Path, _mode: u32) -> Result<()> { + Ok(()) +} + +fn validate_artifact_resolution_snapshot(snapshot: &ArtifactResolutionSnapshotV1) -> Result<()> { + if snapshot.version != ARTIFACT_RESOLUTION_SNAPSHOT_VERSION { + return Err(Error::Corrupt(format!( + "artifact resolution snapshot version {} is unsupported", + snapshot.version + ))); + } + validate_resolution_text(&snapshot.proposal_key, "snapshot proposal key")?; + validate_resolution_text(&snapshot.component_id, "snapshot component id")?; + validate_resolution_text(&snapshot.adapter_identity, "snapshot adapter identity")?; + validate_resolution_text(&snapshot.snapshot_format, "snapshot format")?; + validate_resolution_text( + &snapshot.resolver_executable_identity, + "snapshot resolver executable identity", + )?; + validate_resolution_text(&snapshot.policy_identity, "snapshot policy identity")?; + validate_sha256(&snapshot.content_sha256, "snapshot content hash")?; + validate_identity_map(&snapshot.resolved_identities, "resolved identity")?; + validate_identity_map(&snapshot.checksums, "snapshot checksum")?; + validate_artifact_secret_taint(&snapshot.secret_taint)?; + if !snapshot.secret_taint.is_clear() { + return Err(Error::Corrupt( + "secret-tainted artifact resolution snapshot entered shared storage".into(), + )); + } + if snapshot.contacted_authorities.len() > MAX_RESOLUTION_AUTHORITIES + || !snapshot + .contacted_authorities + .windows(2) + .all(|pair| pair[0] < pair[1]) + { + return Err(Error::Corrupt( + "artifact snapshot authorities are excessive, duplicated, or not canonical".into(), + )); + } + Ok(()) +} + +fn resolution_is_secret_tainted(plan: &ArtifactResolutionPlanV1, redactions: &[Vec]) -> bool { + !plan.credential_handles.is_empty() || redactions.iter().any(|secret| !secret.is_empty()) +} + +fn artifact_secret_taint(secret_tainted: bool, channel: &str) -> ArtifactSecretTaintV1 { + if secret_tainted { + ArtifactSecretTaintV1::Tainted { + channels: vec![channel.to_string()], + } + } else { + ArtifactSecretTaintV1::Clear + } +} + +pub(super) fn validate_artifact_secret_taint(taint: &ArtifactSecretTaintV1) -> Result<()> { + let ArtifactSecretTaintV1::Tainted { channels } = taint else { + return Ok(()); + }; + if channels.is_empty() + || channels.len() > MAX_RESOLUTION_ENVIRONMENT_NAMES + || !channels.windows(2).all(|pair| pair[0] < pair[1]) + { + return Err(Error::InvalidInput( + "artifact secret-taint channels are empty, excessive, duplicated, or not canonical" + .into(), + )); + } + for channel in channels { + validate_resolution_text(channel, "secret-taint channel")?; + if contains_sensitive_text(channel) { + return Err(Error::InvalidInput( + "artifact secret-taint metadata may identify a channel but cannot contain secret material" + .into(), + )); + } + } + Ok(()) +} + +fn normalize_string_set(values: &mut Vec, maximum: usize, field: &str) -> Result<()> { + if values.len() > maximum { + return Err(Error::InvalidInput(format!( + "artifact resolution {field} count exceeds {maximum}" + ))); + } + for value in values.iter() { + validate_resolution_text(value, field)?; + } + values.sort(); + values.dedup(); + Ok(()) +} + +fn validate_identity_map(values: &BTreeMap, field: &str) -> Result<()> { + if values.len() > MAX_RESOLUTION_INPUTS { + return Err(Error::InvalidInput(format!( + "artifact {field} count exceeds {MAX_RESOLUTION_INPUTS}" + ))); + } + for (key, value) in values { + validate_resolution_text(key, field)?; + validate_resolution_text(value, field)?; + } + Ok(()) +} + +fn validate_artifact_validation_declaration(validation: &ArtifactValidationV1) -> Result<()> { + validate_resolution_text(&validation.name, "validation name")?; + validate_identity_map(&validation.parameters, "validation parameter") +} + +pub(crate) fn validate_artifact_validation_receipt( + receipt: &ArtifactValidationReceiptV1, +) -> Result<()> { + if receipt.version != ARTIFACT_VALIDATION_RECEIPT_VERSION { + return Err(Error::InvalidInput(format!( + "artifact validation receipt version {} is unsupported", + receipt.version + ))); + } + validate_artifact_validation_declaration(&receipt.declaration)?; + validate_resolution_text(&receipt.validator_identity, "validator identity")?; + validate_sha256(&receipt.validated_input_digest, "validated input digest")?; + validate_identity_map(&receipt.evidence, "validation evidence")?; + for (name, value) in &receipt.evidence { + if is_sensitive_json_key(name) || contains_sensitive_text(value) { + return Err(Error::InvalidInput(format!( + "artifact validation evidence `{name}` may contain secret material" + ))); + } + } + if receipt.validator_identity == HOST_WORKSPACE_LAYER_SEAL_VALIDATOR { + let expected = artifact_validation_receipt_input_digest( + receipt.version, + &receipt.declaration, + &receipt.desired_identity, + &receipt.tree_root_id, + &receipt.validator_identity, + receipt.outcome, + &receipt.evidence, + )?; + if receipt.validated_input_digest != expected { + return Err(Error::InvalidInput( + "host workspace layer validation receipt has a stale input digest".into(), + )); + } + } + Ok(()) +} + +fn artifact_validation_receipt_input_digest( + version: u16, + declaration: &ArtifactValidationV1, + desired_identity: &ArtifactDesiredIdentityV1, + tree_root_id: &ArtifactTreeId, + validator_identity: &str, + outcome: ArtifactValidationOutcomeV1, + evidence: &BTreeMap, +) -> Result { + Ok(sha256_hex(&serde_json::to_vec(&( + version, + declaration, + desired_identity, + tree_root_id, + validator_identity, + outcome, + evidence, + ))?)) +} + +fn validate_resolution_relative_path(value: &str, field: &str, allow_dot: bool) -> Result<()> { + if allow_dot && value == "." { + return Ok(()); + } + let normalized = normalize_relative_path(value)?; + if normalized != value { + return Err(Error::InvalidInput(format!( + "artifact resolution {field} `{value}` is not normalized" + ))); + } + Ok(()) +} + +fn validate_resolution_text(value: &str, field: &str) -> Result<()> { + if value.is_empty() + || value.len() > MAX_RESOLUTION_TEXT_BYTES + || value.chars().any(char::is_control) + { + return Err(Error::InvalidInput(format!( + "artifact resolution {field} is empty, oversized, or contains control characters" + ))); + } + Ok(()) +} + +fn artifact_materialization_backend_compatibility() -> String { + format!( + "trail-real-directory/v1/{}/{}", + std::env::consts::OS, + std::env::consts::ARCH + ) +} + +fn real_artifact_materialization_directory_exists(path: &Path, label: &str) -> Result { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => { + Err(Error::InvalidPath { + path: path.to_string_lossy().into_owned(), + reason: format!("{label} must be a real directory inside Trail storage"), + }) + } + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(Error::Io(error)), + } +} + +fn validate_sha256(value: &str, field: &str) -> Result<()> { + if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(Error::InvalidInput(format!( + "artifact resolution {field} must be a SHA-256 hex digest" + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + use std::time::Instant; + + fn fixture_plan(source_root: ObjectId) -> ArtifactResolutionPlanV1 { + ArtifactResolutionPlanV1 { + version: ARTIFACT_RESOLUTION_PLAN_VERSION, + proposal_key: "proposal_fixture".into(), + source_root, + component_id: "cargo:root".into(), + adapter_identity: "trail.builtin/cargo@1".into(), + policy_identity: "policy_fixture".into(), + program: "cargo".into(), + resolved_program: "/usr/bin/cargo".into(), + executable_identity: "sha256:fixture".into(), + argv: vec!["cargo".into(), "generate-lockfile".into()], + working_directory: ".".into(), + readable_inputs: vec![ArtifactResolutionInputV1 { + source_path: "Cargo.toml".into(), + content_hash: "11".repeat(32), + size_bytes: 12, + }], + candidate_output: "candidate/Cargo.lock".into(), + allowed_authorities: vec!["index.crates.io:443".into()], + credential_handles: Vec::new(), + script_policy: ArtifactScriptPolicyV1::Deny, + environment_roles: BTreeMap::from([( + "CARGO_HOME".into(), + ArtifactEnvironmentRoleV1::Runtime, + )]), + limits: ArtifactActionLimitsV1 { + timeout_ms: 60_000, + stdout_bytes: 64 * 1024, + stderr_bytes: 64 * 1024, + candidate_bytes: 1024 * 1024, + candidate_entries: 1, + child_processes: 8, + }, + snapshot_format: "cargo-lock/v4".into(), + validations: vec![ArtifactValidationV1 { + name: "cargo-lock-structure".into(), + kind: ArtifactValidationKindV1::Structural, + required: true, + parameters: BTreeMap::new(), + }], + } + } + + fn executable_fixture_plan(db: &Trail, source_root: ObjectId) -> ArtifactResolutionPlanV1 { + let executable = std::env::current_exe().unwrap(); + let entry = db + .root_file_entry(&source_root, "Cargo.toml") + .unwrap() + .unwrap(); + let mut plan = fixture_plan(source_root); + plan.program = "trail-test-resolver".into(); + plan.resolved_program = executable.to_string_lossy().into_owned(); + plan.executable_identity = + super::super::workspace_environment::workspace_tool_identity_for_path(&executable) + .unwrap(); + plan.readable_inputs = vec![ArtifactResolutionInputV1 { + source_path: "Cargo.toml".into(), + content_hash: entry.content_hash, + size_bytes: entry.size_bytes, + }]; + plan + } + + fn fixture_candidate(bytes: &[u8]) -> ArtifactResolutionCandidateV1 { + ArtifactResolutionCandidateV1 { + snapshot_bytes: bytes.to_vec(), + resolved_identities: BTreeMap::from([("fixture".into(), "1.0.0".into())]), + checksums: BTreeMap::from([("fixture".into(), sha256_hex(bytes))]), + contacted_authorities: vec!["index.crates.io:443".into()], + stdout: b"resolver completed".to_vec(), + stderr: Vec::new(), + redactions: Vec::new(), + } + } + + fn initialized_resolution_fixture() -> (tempfile::TempDir, Trail, ObjectId) { + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join("Cargo.toml"), + "[package]\nname = \"fixture\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let source_root = db.resolve_refish("main").unwrap().root_id; + (temp, db, source_root) + } + + #[test] + fn malicious_resolution_plans_are_rejected_before_attempt_publication() { + let (_temp, db, source_root) = initialized_resolution_fixture(); + let base = executable_fixture_plan(&db, source_root); + + let mut traversing_output = base.clone(); + traversing_output.proposal_key = "proposal_traversing_output".into(); + traversing_output.candidate_output = "../Cargo.lock".into(); + + let mut absolute_workdir = base.clone(); + absolute_workdir.proposal_key = "proposal_absolute_workdir".into(); + absolute_workdir.working_directory = "/tmp".into(); + + let mut control_argument = base.clone(); + control_argument.proposal_key = "proposal_control_argument".into(); + control_argument + .argv + .push("--config\ncredential=leak".into()); + + let mut excessive_limits = base; + excessive_limits.proposal_key = "proposal_excessive_limits".into(); + excessive_limits.limits.timeout_ms = MAX_RESOLUTION_TIMEOUT_MS + 1; + excessive_limits.limits.stdout_bytes = MAX_RESOLUTION_CAPTURE_BYTES + 1; + excessive_limits.limits.candidate_bytes = MAX_RESOLUTION_CANDIDATE_BYTES + 1; + excessive_limits.limits.candidate_entries = MAX_RESOLUTION_CANDIDATE_ENTRIES + 1; + excessive_limits.limits.child_processes = MAX_RESOLUTION_CHILD_PROCESSES + 1; + + for (plan, expected) in [ + (traversing_output, "path must stay inside the workspace"), + (absolute_workdir, "path must stay inside the workspace"), + (control_argument, "control characters"), + (excessive_limits, "exceed host ceilings"), + ] { + let error = db.begin_artifact_resolution_attempt(plan).unwrap_err(); + assert!( + error.to_string().contains(expected), + "unexpected malicious-plan rejection: {error}" + ); + } + assert!(db.artifact_resolution_attempts().unwrap().is_empty()); + } + + fn fixture_desired_material(source_root: ObjectId) -> ArtifactDesiredKeyMaterialV2 { + ArtifactDesiredKeyMaterialV2 { + version: 2, + component_id: "cargo:root".into(), + adapter_identity: "trail.builtin/cargo@1".into(), + adapter_implementation_version: "1".into(), + adapter_distribution_digest: "builtin:cargo:1".into(), + adapter_protocol: "trail.environment-adapter/builtin-v1".into(), + resolution_snapshot_id: None, + source_closure: ArtifactSourceClosureV2 { + normalizer_version: "source-paths/v1".into(), + certified_complete: false, + complete_source_root: Some(source_root), + declared_inputs: vec![ArtifactResolutionInputV1 { + source_path: "Cargo.toml".into(), + content_hash: "11".repeat(32), + size_bytes: 12, + }], + }, + upstream_identities: BTreeMap::new(), + actions: vec![ArtifactActionIdentityV2 { + name: "build".into(), + phase: ArtifactActionPhaseV2::Construct, + executable_identity: "sha256:cargo".into(), + argv: vec!["cargo".into(), "build".into(), "--locked".into()], + working_directory: ".".into(), + environment_names: vec!["CARGO_TARGET_DIR".into()], + }], + outputs: vec![ArtifactOutputContractV2 { + name: "target".into(), + output_path: "target".into(), + mount_path: "target".into(), + policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + reuse: EnvironmentReuseMode::Exact, + scope: EnvironmentSharingScope::Workspace, + publish: EnvironmentPublicationTrigger::OnSync, + gate: None, + }], + validations: vec![ArtifactValidationV1 { + name: "tree".into(), + kind: ArtifactValidationKindV1::Structural, + required: true, + parameters: BTreeMap::new(), + }], + source_exports: Vec::new(), + build_environment: BTreeMap::from([("RUSTFLAGS".into(), "-Cdebuginfo=0".into())]), + target: "debug".into(), + platform: "darwin".into(), + architecture: "aarch64".into(), + abi: "apple".into(), + portability_certified: true, + portability_scope: "workspace".into(), + trust_scope: "builtin".into(), + network_policy: "deny".into(), + script_policy: ArtifactScriptPolicyV1::Deny, + sandbox_policy: "native-deny-by-default".into(), + } + } + + #[test] + fn desired_key_v2_is_canonical_and_separates_identity_dimensions() { + let source_root = ObjectId("object_source".into()); + let mut left = fixture_desired_material(source_root.clone()); + left.upstream_identities.insert("z".into(), "2".into()); + left.upstream_identities.insert("a".into(), "1".into()); + let mut right = fixture_desired_material(source_root); + right.upstream_identities.insert("a".into(), "1".into()); + right.upstream_identities.insert("z".into(), "2".into()); + let left_key = artifact_desired_key_v2(left.clone()).unwrap(); + assert_eq!(left_key, artifact_desired_key_v2(right).unwrap()); + + left.resolution_snapshot_id = Some(ObjectId("object_snapshot".into())); + assert_ne!(left_key, artifact_desired_key_v2(left).unwrap()); + } + + #[test] + fn desired_key_v2_requires_safe_source_fallback_and_rejects_secrets() { + let mut material = fixture_desired_material(ObjectId("object_source".into())); + material.source_closure.complete_source_root = None; + assert!(artifact_desired_key_v2(material.clone()).is_err()); + + material.source_closure.complete_source_root = Some(ObjectId("object_source".into())); + let first_source = artifact_desired_key_v2(material.clone()).unwrap(); + material.source_closure.complete_source_root = Some(ObjectId("object_other_source".into())); + assert_ne!( + first_source, + artifact_desired_key_v2(material.clone()).unwrap() + ); + + material.portability_certified = false; + assert!(artifact_desired_key_v2(material.clone()).is_err()); + material.outputs[0].reuse = EnvironmentReuseMode::None; + material.outputs[0].scope = EnvironmentSharingScope::Lane; + assert!(artifact_desired_key_v2(material.clone()).is_ok()); + + material + .build_environment + .insert("API_TOKEN".into(), "super-secret".into()); + assert!(artifact_desired_key_v2(material).is_err()); + } + + #[test] + fn desired_key_v2_diff_reports_first_and_complete_ordered_invalidation_edges() { + let previous = fixture_desired_material(ObjectId("object_source".into())); + let mut current = previous.clone(); + current.resolution_snapshot_id = Some(ObjectId("object_snapshot".into())); + current.adapter_implementation_version = "2".into(); + current.actions[0].argv.push("--release".into()); + current.outputs[0].mount_path = "target-v2".into(); + current.validations[0] + .parameters + .insert("profile".into(), "strict".into()); + current.source_exports.push(ArtifactSourceExportContractV2 { + name: "bindings".into(), + output_name: "target".into(), + artifact_subpath: "generated".into(), + destination: "src/generated".into(), + collision_policy: "fail".into(), + required_validation: "tree".into(), + required_gate: None, + authorization_mode: "explicit".into(), + }); + current.trust_scope = "signed-plugin".into(); + current.sandbox_policy = "native-strict".into(); + + let diff = diff_artifact_desired_key_v2(&previous, ¤t).unwrap(); + assert_ne!(diff.previous_key, diff.current_key); + assert_eq!(diff.first.as_ref().unwrap().dimension, "resolution"); + assert_eq!( + diff.edges + .iter() + .map(|edge| edge.dimension.as_str()) + .collect::>(), + BTreeSet::from([ + "resolution", + "tool", + "action", + "output", + "validation", + "export", + "trust", + "sandbox", + ]) + ); + let mut sorted = diff.edges.clone(); + sorted.sort_by(|left, right| { + artifact_invalidation_dimension_rank(&left.dimension) + .cmp(&artifact_invalidation_dimension_rank(&right.dimension)) + .then_with(|| left.name.cmp(&right.name)) + .then_with(|| left.change.cmp(&right.change)) + }); + assert_eq!(diff.edges, sorted); + } + + #[test] + fn desired_key_v2_preserves_absence_unicode_and_normalizer_semantics() { + let material = fixture_desired_material(ObjectId("object_source".into())); + let mut encoded = serde_json::to_value(&material).unwrap(); + encoded.as_object_mut().unwrap().remove("source_exports"); + assert!(serde_json::from_value::(encoded).is_err()); + + let mut empty_snapshot = material.clone(); + empty_snapshot.resolution_snapshot_id = Some(ObjectId(String::new())); + assert!(artifact_desired_key_v2(empty_snapshot).is_err()); + assert!(artifact_desired_key_v2(material.clone()).is_ok()); + + let mut composed = material.clone(); + composed.source_closure.certified_complete = true; + composed.source_closure.complete_source_root = None; + composed.source_closure.declared_inputs[0].source_path = "cafĂ©/Cargo.toml".into(); + let composed_key = artifact_desired_key_v2(composed.clone()).unwrap(); + let mut decomposed = composed.clone(); + decomposed.source_closure.declared_inputs[0].source_path = "cafe\u{301}/Cargo.toml".into(); + assert!(artifact_desired_key_v2(decomposed).is_err()); + + composed.source_closure.normalizer_version = "source-paths/v2".into(); + assert_ne!(composed_key, artifact_desired_key_v2(composed).unwrap()); + } + + #[test] + fn each_desired_key_v2_identity_dimension_invalidates_independently() { + let base = fixture_desired_material(ObjectId("object_source".into())); + let base_key = artifact_desired_key_v2(base.clone()).unwrap(); + let mut variants = Vec::new(); + + let mut value = base.clone(); + value.adapter_distribution_digest = "builtin:cargo:2".into(); + variants.push(("adapter", value)); + let mut value = base.clone(); + value.source_closure.normalizer_version = "source-paths/v2".into(); + variants.push(("source", value)); + let mut value = base.clone(); + value.actions[0].argv.push("--release".into()); + variants.push(("action", value)); + let mut value = base.clone(); + value.outputs[0].mount_path = "target-v2".into(); + variants.push(("output", value)); + let mut value = base.clone(); + value.validations[0].required = false; + variants.push(("validation", value)); + let mut value = base.clone(); + value.source_exports.push(ArtifactSourceExportContractV2 { + name: "generated".into(), + output_name: "target".into(), + artifact_subpath: "generated".into(), + destination: "src/generated".into(), + collision_policy: "fail".into(), + required_validation: "tree".into(), + required_gate: None, + authorization_mode: "explicit".into(), + }); + variants.push(("export", value)); + let mut value = base.clone(); + value.abi = "musl".into(); + variants.push(("platform", value)); + let mut value = base; + value.sandbox_policy = "native-strict".into(); + variants.push(("sandbox", value)); + + for (dimension, variant) in variants { + assert_ne!( + base_key, + artifact_desired_key_v2(variant).unwrap(), + "identity dimension `{dimension}` did not invalidate the desired key" + ); + } + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(32))] + + #[test] + fn desired_key_v2_is_stable_for_arbitrary_declared_input_order( + inputs in prop::collection::btree_map("[a-z]{1,12}", any::(), 0..32) + ) { + let mut forward = fixture_desired_material(ObjectId("object_source".into())); + forward.source_closure.certified_complete = true; + forward.source_closure.complete_source_root = None; + forward.source_closure.declared_inputs = inputs + .iter() + .map(|(name, value)| ArtifactResolutionInputV1 { + source_path: format!("inputs/{name}"), + content_hash: sha256_hex(&value.to_le_bytes()), + size_bytes: 8, + }) + .collect(); + if forward.source_closure.declared_inputs.is_empty() { + forward.source_closure.declared_inputs.push(ArtifactResolutionInputV1 { + source_path: "inputs/empty".into(), + content_hash: sha256_hex(b"empty"), + size_bytes: 0, + }); + } + let mut reverse = forward.clone(); + reverse.source_closure.declared_inputs.reverse(); + prop_assert_eq!( + artifact_desired_key_v2(forward).unwrap(), + artifact_desired_key_v2(reverse).unwrap() + ); + } + } + + #[test] + #[ignore = "qualification benchmark; run explicitly with --nocapture"] + fn artifact_cas_benchmark_records_whole_chunked_and_successor_reuse() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + + let whole_bytes = deterministic_benchmark_bytes(ARTIFACT_WHOLE_BLOB_MAX_BYTES); + let whole_cpu = process_cpu_micros(); + let whole_started = Instant::now(); + let whole_file = db.ingest_artifact_file_bytes(&whole_bytes, 0o644).unwrap(); + let whole_wall_micros = whole_started.elapsed().as_micros() as u64; + let whole_cpu_micros = process_cpu_micros().saturating_sub(whole_cpu); + let whole_node: ArtifactFileNodeV1 = db + .get_artifact_cas_object( + &whole_file.0, + ARTIFACT_FILE_NODE_KIND, + ARTIFACT_FILE_NODE_VERSION, + ) + .unwrap(); + assert!(matches!( + whole_node.content, + ArtifactFileContentV1::Blob { .. } + )); + let whole_counts = artifact_benchmark_storage_counts(&db); + + let chunked_bytes = deterministic_benchmark_bytes(16 * 1024 * 1024); + let chunked_cpu = process_cpu_micros(); + let chunked_started = Instant::now(); + let chunked_file = db + .ingest_artifact_file_bytes(&chunked_bytes, 0o644) + .unwrap(); + let chunked_wall_micros = chunked_started.elapsed().as_micros() as u64; + let chunked_cpu_micros = process_cpu_micros().saturating_sub(chunked_cpu); + let chunked_node: ArtifactFileNodeV1 = db + .get_artifact_cas_object( + &chunked_file.0, + ARTIFACT_FILE_NODE_KIND, + ARTIFACT_FILE_NODE_VERSION, + ) + .unwrap(); + let ArtifactFileContentV1::Chunks { chunk_list_id } = chunked_node.content else { + panic!("large benchmark file did not use FastCDC chunks"); + }; + let initial_list: ArtifactChunkListV1 = db + .get_artifact_cas_object( + &chunk_list_id.0, + ARTIFACT_CHUNK_LIST_KIND, + ARTIFACT_CHUNK_LIST_VERSION, + ) + .unwrap(); + assert!(initial_list.chunks.len() > 1); + let chunked_counts = artifact_benchmark_storage_counts(&db); + + let mut successor_bytes = chunked_bytes; + let midpoint = successor_bytes.len() / 2; + for byte in &mut successor_bytes[midpoint..midpoint + 64] { + *byte ^= 0x5a; + } + let successor_cpu = process_cpu_micros(); + let successor_started = Instant::now(); + let successor_file = db + .ingest_artifact_file_bytes(&successor_bytes, 0o644) + .unwrap(); + let successor_wall_micros = successor_started.elapsed().as_micros() as u64; + let successor_cpu_micros = process_cpu_micros().saturating_sub(successor_cpu); + let successor_node: ArtifactFileNodeV1 = db + .get_artifact_cas_object( + &successor_file.0, + ARTIFACT_FILE_NODE_KIND, + ARTIFACT_FILE_NODE_VERSION, + ) + .unwrap(); + let ArtifactFileContentV1::Chunks { + chunk_list_id: successor_list_id, + } = successor_node.content + else { + panic!("large successor benchmark file did not use FastCDC chunks"); + }; + let successor_list: ArtifactChunkListV1 = db + .get_artifact_cas_object( + &successor_list_id.0, + ARTIFACT_CHUNK_LIST_KIND, + ARTIFACT_CHUNK_LIST_VERSION, + ) + .unwrap(); + let initial_chunks = initial_list + .chunks + .iter() + .map(|chunk| &chunk.chunk_id) + .collect::>(); + let reused_chunks = successor_list + .chunks + .iter() + .filter(|chunk| initial_chunks.contains(&chunk.chunk_id)) + .count(); + assert!(reused_chunks > 0, "successor failed to reuse any CDC chunk"); + let successor_counts = artifact_benchmark_storage_counts(&db); + + println!( + "{}", + serde_json::json!({ + "schema": "trail.artifact-cas-benchmark/v1", + "whole": { + "logical_bytes": whole_bytes.len(), + "cpu_micros": whole_cpu_micros, + "wall_micros": whole_wall_micros, + "object_count": whole_counts.0, + "unique_object_bytes": whole_counts.2, + }, + "chunked": { + "logical_bytes": 16 * 1024 * 1024, + "cpu_micros": chunked_cpu_micros, + "wall_micros": chunked_wall_micros, + "new_object_count": chunked_counts.0 - whole_counts.0, + "new_unique_object_bytes": chunked_counts.2 - whole_counts.2, + "chunk_count": initial_list.chunks.len(), + }, + "successor": { + "logical_bytes": successor_bytes.len(), + "cpu_micros": successor_cpu_micros, + "wall_micros": successor_wall_micros, + "new_object_count": successor_counts.0 - chunked_counts.0, + "new_unique_object_bytes": successor_counts.2 - chunked_counts.2, + "chunk_count": successor_list.chunks.len(), + "reused_chunks": reused_chunks, + } + }) + ); + } + + fn artifact_benchmark_storage_counts(db: &Trail) -> (u64, u64, u64) { + db.conn + .query_row( + "SELECT COUNT(*),COALESCE(SUM(a.logical_bytes),0), + COALESCE(SUM(LENGTH(o.bytes)),0) + FROM artifact_objects a JOIN objects o ON o.object_id=a.object_id", + [], + |row| { + Ok(( + row.get::<_, i64>(0)? as u64, + row.get::<_, i64>(1)? as u64, + row.get::<_, i64>(2)? as u64, + )) + }, + ) + .unwrap() + } + + fn deterministic_benchmark_bytes(size: usize) -> Vec { + let mut state = 0x4d59_5df4_d0f3_3173_u64; + (0..size) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as u8 + }) + .collect() + } + + #[cfg(unix)] + fn process_cpu_micros() -> u64 { + let mut usage = std::mem::MaybeUninit::::uninit(); + // SAFETY: getrusage initializes the supplied rusage on success, and + // the pointer is valid for the duration of this call. + if unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) } != 0 { + return 0; + } + // SAFETY: the success branch above guarantees initialization. + let usage = unsafe { usage.assume_init() }; + timeval_micros(usage.ru_utime).saturating_add(timeval_micros(usage.ru_stime)) + } + + #[cfg(not(unix))] + fn process_cpu_micros() -> u64 { + 0 + } + + #[cfg(unix)] + fn timeval_micros(value: libc::timeval) -> u64 { + u64::try_from(value.tv_sec) + .unwrap_or(0) + .saturating_mul(1_000_000) + .saturating_add(u64::try_from(value.tv_usec).unwrap_or(0)) + } + + #[test] + fn divergent_tree_roots_for_one_desired_key_are_quarantined_and_held() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let first_source = tempfile::tempdir().unwrap(); + let second_source = tempfile::tempdir().unwrap(); + fs::write(first_source.path().join("result"), "first\n").unwrap(); + fs::write(second_source.path().join("result"), "second\n").unwrap(); + let (first_tree, _) = db + .ingest_artifact_tree_under_write_lock(first_source.path()) + .unwrap(); + let (second_tree, _) = db + .ingest_artifact_tree_under_write_lock(second_source.path()) + .unwrap(); + let desired_key = + artifact_desired_key_v2(fixture_desired_material(ObjectId("object_source".into()))) + .unwrap(); + let envelope = |tree_root_id| ArtifactEnvelopeV1 { + version: ARTIFACT_ENVELOPE_VERSION, + desired_identity: ArtifactDesiredIdentityV1::ArtifactDesiredV2 { + desired_key: desired_key.clone(), + }, + tree_root_id, + component_id: "cargo:root".into(), + output_name: "target".into(), + output_policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + portability_scope: "workspace".into(), + trust_scope: "builtin".into(), + secret_taint: ArtifactSecretTaintV1::Clear, + resolution_snapshot_id: None, + validation_receipt_ids: Vec::new(), + }; + + let (first_envelope, first_quarantined) = db + .put_artifact_envelope_under_write_lock(envelope(first_tree.clone())) + .unwrap(); + assert!(!first_quarantined); + assert_eq!( + db.artifact_envelope_ids().unwrap(), + vec![first_envelope.0.clone()] + ); + db.verify_ready_artifact_envelope_under_write_lock(&first_envelope, &first_tree) + .unwrap(); + let inspection = db.inspect_artifact(&first_envelope).unwrap(); + assert_eq!(inspection.state, "ready"); + assert_eq!(inspection.verification_state, "verified"); + assert_eq!(inspection.trust_state, "trusted"); + assert_eq!(inspection.quarantine_state, "none"); + assert_eq!(inspection.tree_root_id, first_tree); + assert!(inspection.reachability.complete); + assert!(inspection.reachability.object_count >= 4); + assert!(inspection.storage.logical_bytes > 0); + assert!( + db.verify_artifact(&first_envelope, ArtifactVerificationLevelV1::Full) + .unwrap() + .valid + ); + let reproduce = db + .verify_artifact(&first_envelope, ArtifactVerificationLevelV1::Reproduce) + .unwrap(); + assert!(!reproduce.valid); + assert_eq!(reproduce.reproduction_evidence_valid, Some(false)); + let space = db.workspace_artifact_space().unwrap(); + assert_eq!(space.envelope_count, 1); + assert_eq!(space.active_quarantine_count, 0); + assert!(space.storage.logical_bytes > 0); + let (second_envelope, second_quarantined) = db + .put_artifact_envelope_under_write_lock(envelope(second_tree.clone())) + .unwrap(); + assert!(second_quarantined); + assert!(db + .verify_ready_artifact_envelope_under_write_lock(&first_envelope, &first_tree) + .is_err()); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM artifact_envelopes + WHERE envelope_id IN (?1,?2) AND state='quarantined'", + params![first_envelope.0, second_envelope.0], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 2 + ); + let (quarantine_id, evidence_object_id) = db + .conn + .query_row( + "SELECT quarantine_id,evidence_object_id FROM artifact_quarantines + WHERE desired_key=?1 AND trust_scope='builtin' AND state='active'", + params![desired_key.0], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(); + assert!(quarantine_id.starts_with("artifact_quarantine_")); + let evidence: ArtifactDivergenceEvidenceV1 = db + .get_object( + ARTIFACT_DIVERGENCE_EVIDENCE_KIND, + &ObjectId(evidence_object_id), + ) + .unwrap(); + assert_eq!(evidence.incumbent_tree_root_id, first_tree); + assert_eq!(evidence.candidate_tree_root_id, second_tree); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM artifact_holds WHERE reason=?1", + params![quarantine_id], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 2 + ); + let quarantine_id = ArtifactQuarantineId::parse(quarantine_id).unwrap(); + let quarantine_list = db.artifact_quarantine_list_report().unwrap(); + assert_eq!(quarantine_list.active_count, 1); + assert_eq!(quarantine_list.resolved_count, 0); + assert_eq!(quarantine_list.quarantines.len(), 1); + assert_eq!( + db.artifact_quarantine(&quarantine_id).unwrap().state, + "active" + ); + let quarantined_inspection = db.inspect_artifact(&first_envelope).unwrap(); + assert_eq!(quarantined_inspection.state, "quarantined"); + assert_eq!(quarantined_inspection.quarantine_state, "active"); + assert!( + !db.verify_artifact(&first_envelope, ArtifactVerificationLevelV1::Attach) + .unwrap() + .valid + ); + let resolution_report = db + .resolve_artifact_quarantine_report( + &quarantine_id, + ArtifactQuarantineResolutionV1::RetainPrivate, + ) + .unwrap(); + let resolved = resolution_report.quarantine; + assert_eq!(resolved.state, "resolved"); + assert_eq!(resolved.resolution.as_deref(), Some("retain_private")); + assert_eq!(resolution_report.affected_envelopes.len(), 2); + assert!(db + .verify_ready_artifact_envelope_under_write_lock(&second_envelope, &second_tree) + .is_err()); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM artifact_holds WHERE reason=?1", + params![quarantine_id.0], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + } + + #[test] + fn legacy_workspace_layer_identity_never_defaults_to_v2() { + let legacy = ArtifactDesiredIdentityV1::WorkspaceLayerV1 { + cache_key: "legacy-cache-key".into(), + canonical_key: WorkspaceLayerKeyV1 { + kind: "dependency".into(), + adapter: "node".into(), + adapter_version: 1, + inputs: BTreeMap::from([("lock".into(), "digest".into())]), + tool_versions: BTreeMap::from([("node".into(), "22".into())]), + platform: "darwin".into(), + architecture: "aarch64".into(), + portability_scope: "platform".into(), + strategy: "npm-ci".into(), + }, + }; + let encoded = serde_json::to_value(&legacy).unwrap(); + assert_eq!(encoded["identity_version"], "workspace_layer_v1"); + let decoded: ArtifactDesiredIdentityV1 = serde_json::from_value(encoded).unwrap(); + assert_eq!(decoded, legacy); + assert!(decoded.desired_key_v2().is_none()); + } + + #[test] + fn artifact_object_codecs_are_canonical_and_validate_edges() { + let blob = ArtifactBlobV1 { + version: ARTIFACT_BLOB_VERSION, + content_sha256: sha256_hex(b"same bytes"), + bytes: b"same bytes".to_vec(), + }; + let (blob_id, blob_bytes) = encode_artifact_blob(blob.clone()).unwrap(); + assert_eq!(decode_artifact_blob(&blob_bytes, &blob_id).unwrap(), blob); + + let file = ArtifactFileNodeV1 { + version: ARTIFACT_FILE_NODE_VERSION, + mode: 0o644, + executable: false, + size_bytes: 10, + content_sha256: sha256_hex(b"same bytes"), + content: ArtifactFileContentV1::Blob { blob_id }, + }; + let (file_id, _) = encode_artifact_file_node(file).unwrap(); + let directory = ArtifactDirectoryNodeV1 { + version: ARTIFACT_DIRECTORY_NODE_VERSION, + entries: vec![ + ArtifactDirectoryEntryV1 { + name: "z".into(), + target: ArtifactDirectoryEntryTargetV1::Symlink { target: "a".into() }, + }, + ArtifactDirectoryEntryV1 { + name: "a".into(), + target: ArtifactDirectoryEntryTargetV1::File { node_id: file_id }, + }, + ], + }; + let (directory_id, canonical) = encode_artifact_directory_node(directory).unwrap(); + let decoded = decode_artifact_directory_node(&canonical).unwrap(); + assert_eq!(decoded.entries[0].name, "a"); + assert_eq!( + directory_id, + encode_artifact_directory_node(decoded).unwrap().0 + ); + + let root = ArtifactTreeRootV1 { + version: ARTIFACT_TREE_ROOT_VERSION, + root_directory_id: directory_id.clone(), + logical_bytes: 10, + entry_count: 2, + path_normalizer: "trail-paths/v1".into(), + }; + assert_ne!(encode_artifact_tree_root(root).unwrap().0, directory_id); + } + + #[test] + fn artifact_object_codecs_reject_noncanonical_or_broken_edges() { + let invalid_directory = ArtifactDirectoryNodeV1 { + version: ARTIFACT_DIRECTORY_NODE_VERSION, + entries: vec![ + ArtifactDirectoryEntryV1 { + name: "duplicate".into(), + target: ArtifactDirectoryEntryTargetV1::Symlink { target: "a".into() }, + }, + ArtifactDirectoryEntryV1 { + name: "duplicate".into(), + target: ArtifactDirectoryEntryTargetV1::Symlink { target: "b".into() }, + }, + ], + }; + assert!(encode_artifact_directory_node(invalid_directory).is_err()); + + let invalid_chunks = ArtifactChunkListV1 { + version: ARTIFACT_CHUNK_LIST_VERSION, + algorithm: "fastcdc-v1".into(), + file_size_bytes: 99, + file_sha256: "11".repeat(32), + chunks: vec![ArtifactChunkRefV1 { + chunk_id: ArtifactChunkId::new(b"chunk"), + size_bytes: 1, + }], + }; + assert!(encode_artifact_chunk_list(invalid_chunks).is_err()); + + let chunk = ArtifactChunkV1 { + version: ARTIFACT_CHUNK_VERSION, + content_sha256: "00".repeat(32), + bytes: b"not zero".to_vec(), + }; + assert!(encode_artifact_chunk(chunk).is_err()); + } + + #[test] + fn validation_receipts_are_deterministic_typed_and_bound_to_the_exact_envelope() { + let (_workspace, mut db, source_root) = initialized_resolution_fixture(); + let candidate = tempfile::tempdir().unwrap(); + fs::write( + candidate.path().join("validated.bin"), + b"validated output\n", + ) + .unwrap(); + let (tree_id, _) = db.ingest_artifact_tree(candidate.path()).unwrap(); + let desired_key = artifact_desired_key_v2(fixture_desired_material(source_root)).unwrap(); + let desired_identity = ArtifactDesiredIdentityV1::ArtifactDesiredV2 { + desired_key: desired_key.clone(), + }; + let receipt = ArtifactValidationReceiptV1 { + version: ARTIFACT_VALIDATION_RECEIPT_VERSION, + declaration: ArtifactValidationV1 { + name: "cargo-metadata-loads".into(), + kind: ArtifactValidationKindV1::Loadability, + required: true, + parameters: BTreeMap::from([("format".into(), "cargo-metadata-v1".into())]), + }, + desired_identity: desired_identity.clone(), + tree_root_id: tree_id.clone(), + validator_identity: "trail.builtin/cargo-validator@1#sha256:fixture".into(), + validated_input_digest: sha256_hex(b"desired+tree+validator+policy"), + outcome: ArtifactValidationOutcomeV1::Passed, + evidence: BTreeMap::from([ + ("checked_entries".into(), "1".into()), + ("result".into(), "loadable".into()), + ]), + }; + let receipt_id = db.put_artifact_validation_receipt(receipt.clone()).unwrap(); + assert_eq!( + db.put_artifact_validation_receipt(receipt.clone()).unwrap(), + receipt_id + ); + assert_eq!( + db.artifact_validation_receipt(&receipt_id).unwrap(), + receipt + ); + + let envelope = ArtifactEnvelopeV1 { + version: ARTIFACT_ENVELOPE_VERSION, + desired_identity: desired_identity.clone(), + tree_root_id: tree_id.clone(), + component_id: "cargo:root".into(), + output_name: "target".into(), + output_policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + portability_scope: "workspace".into(), + trust_scope: "builtin".into(), + secret_taint: ArtifactSecretTaintV1::Clear, + resolution_snapshot_id: None, + validation_receipt_ids: vec![receipt_id.clone()], + }; + let (envelope_id, quarantined) = db + .put_artifact_envelope_under_write_lock(envelope.clone()) + .unwrap(); + assert!(!quarantined); + let attestations = db.artifact_attestations_for_envelope(&envelope_id).unwrap(); + assert_eq!(attestations.len(), 1); + assert_eq!( + attestations[0].attestation.statement.envelope_id, + envelope_id + ); + assert_eq!(attestations[0].attestation.statement.tree_root_id, tree_id); + assert_eq!(attestations[0].attestation.signature, None); + let verification = db + .verify_artifact_attestation(&attestations[0].attestation_id) + .unwrap(); + assert!(verification.valid); + assert_eq!(verification.signature_status, "unsigned"); + let mut invalid_signed = attestations[0].attestation.clone(); + invalid_signed.signature = Some(ArtifactAttestationSignatureV1 { + algorithm: "ed25519".into(), + key_id: "fixture-key".into(), + public_key_hex: "00".repeat(32), + signature_hex: "00".repeat(64), + }); + let mut signature_diagnostics = Vec::new(); + let (_, signature_valid) = db + .verify_artifact_attestation_signature(&invalid_signed, &mut signature_diagnostics) + .unwrap(); + assert!(!signature_valid); + assert!(!signature_diagnostics.is_empty()); + let (same_envelope_id, _) = db + .put_artifact_envelope_under_write_lock(envelope.clone()) + .unwrap(); + assert_eq!(same_envelope_id, envelope_id); + assert_eq!( + db.artifact_attestations_for_envelope(&envelope_id) + .unwrap() + .len(), + 1 + ); + db.conn + .execute( + "UPDATE artifact_attestations SET state='revoked' WHERE attestation_id=?1", + params![&attestations[0].attestation_id.0], + ) + .unwrap(); + assert!( + !db.verify_artifact_attestation(&attestations[0].attestation_id) + .unwrap() + .valid + ); + assert!(db + .verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id) + .unwrap_err() + .to_string() + .contains("database state is `revoked`")); + db.conn + .execute( + "UPDATE artifact_attestations SET state='valid' WHERE attestation_id=?1", + params![&attestations[0].attestation_id.0], + ) + .unwrap(); + assert_eq!( + db.verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id) + .unwrap() + .validation_receipt_ids, + vec![receipt_id.clone()] + ); + assert!(db.validate_artifact_cas_integrity().unwrap().is_empty()); + + let mut tainted_envelope = envelope.clone(); + tainted_envelope.secret_taint = ArtifactSecretTaintV1::Tainted { + channels: vec!["runtime_credential".into()], + }; + let error = db + .put_artifact_envelope_under_write_lock(tainted_envelope) + .unwrap_err(); + assert!(error.to_string().contains("must remain lane-private")); + db.conn + .execute( + "INSERT INTO artifact_holds( + hold_id,target_kind,target_id,reason,created_at) + VALUES('hold_validation_receipt','artifact_envelope',?1,'validation-test',1)", + params![envelope_id.0], + ) + .unwrap(); + db.gc(false).unwrap(); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM objects WHERE object_id=?1", + params![receipt_id.0], + |row| row.get::<_, u64>(0), + ) + .unwrap(), + 1 + ); + + let mut failed_receipt = receipt.clone(); + failed_receipt.outcome = ArtifactValidationOutcomeV1::Failed; + let failed_id = db.put_artifact_validation_receipt(failed_receipt).unwrap(); + let mut failed_envelope = envelope.clone(); + failed_envelope.validation_receipt_ids = vec![failed_id]; + assert!(db + .put_artifact_envelope_under_write_lock(failed_envelope) + .unwrap_err() + .to_string() + .contains("does not pass")); + + let mut secret_receipt = receipt; + secret_receipt.evidence.insert( + "output".into(), + "Authorization: Bearer validator-secret".into(), + ); + assert!(db + .put_artifact_validation_receipt(secret_receipt) + .unwrap_err() + .to_string() + .contains("secret material")); + } + + #[test] + fn attachment_fails_closed_for_tampered_attestation_references_and_bytes() { + let (workspace, db, source_root) = initialized_resolution_fixture(); + let candidate = tempfile::tempdir().unwrap(); + fs::write(candidate.path().join("artifact.bin"), b"attested output\n").unwrap(); + let (tree_id, _) = db.ingest_artifact_tree(candidate.path()).unwrap(); + let desired_key = artifact_desired_key_v2(fixture_desired_material(source_root)).unwrap(); + let (envelope_id, quarantined) = db + .put_artifact_envelope_under_write_lock(ArtifactEnvelopeV1 { + version: ARTIFACT_ENVELOPE_VERSION, + desired_identity: ArtifactDesiredIdentityV1::ArtifactDesiredV2 { desired_key }, + tree_root_id: tree_id.clone(), + component_id: "cargo:root".into(), + output_name: "target".into(), + output_policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + portability_scope: "workspace".into(), + trust_scope: "builtin".into(), + secret_taint: ArtifactSecretTaintV1::Clear, + resolution_snapshot_id: None, + validation_receipt_ids: Vec::new(), + }) + .unwrap(); + assert!(!quarantined); + let attestation = db + .artifact_attestations_for_envelope(&envelope_id) + .unwrap() + .pop() + .unwrap(); + + db.conn + .execute( + "UPDATE artifact_attestations SET producer_identity='tampered-producer' + WHERE attestation_id=?1", + params![attestation.attestation_id.0], + ) + .unwrap(); + let reference_error = db + .verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id) + .unwrap_err(); + assert!(reference_error + .to_string() + .contains("database identity disagrees")); + + db.conn + .execute( + "UPDATE artifact_attestations SET producer_identity=?1 + WHERE attestation_id=?2", + params![ + attestation.attestation.statement.producer_identity, + attestation.attestation_id.0 + ], + ) + .unwrap(); + db.verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id) + .unwrap(); + db.conn + .execute( + "UPDATE objects SET bytes=X'00' WHERE object_id=?1", + params![attestation.object_id.0], + ) + .unwrap(); + drop(db); + + let reopened = Trail::open(workspace.path()).unwrap(); + let byte_error = reopened + .verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id) + .unwrap_err(); + assert!( + byte_error.to_string().contains("attestation") + || byte_error.to_string().contains("object") + || byte_error.to_string().contains("CBOR") + || byte_error.to_string().contains("serialization error"), + "unexpected tampered-attestation rejection: {byte_error}" + ); + } + + #[test] + fn artifact_file_storage_uses_whole_blobs_then_fastcdc_chunks() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let _lock = db.acquire_write_lock().unwrap(); + + let small = vec![b'a'; ARTIFACT_WHOLE_BLOB_MAX_BYTES]; + let small_id = db.ingest_artifact_file_bytes(&small, 0o644).unwrap(); + let small_file: ArtifactFileNodeV1 = db + .get_object( + ARTIFACT_FILE_NODE_KIND, + &artifact_object_id(&db, &small_id.0), + ) + .unwrap(); + assert!(matches!( + small_file.content, + ArtifactFileContentV1::Blob { .. } + )); + + let mut large = Vec::with_capacity(5 * 1024 * 1024); + for index in 0..5 * 1024 * 1024 { + large.push(((index * 31 + index / 97) % 251) as u8); + } + let large_id = db.ingest_artifact_file_bytes(&large, 0o755).unwrap(); + let large_file: ArtifactFileNodeV1 = db + .get_object( + ARTIFACT_FILE_NODE_KIND, + &artifact_object_id(&db, &large_id.0), + ) + .unwrap(); + let ArtifactFileContentV1::Chunks { chunk_list_id } = large_file.content else { + panic!("large artifact file must use chunks"); + }; + let chunk_list: ArtifactChunkListV1 = db + .get_object( + ARTIFACT_CHUNK_LIST_KIND, + &artifact_object_id(&db, &chunk_list_id.0), + ) + .unwrap(); + assert_eq!(chunk_list.algorithm, "fastcdc-v1"); + assert_eq!(chunk_list.file_size_bytes, large.len() as u64); + assert!(chunk_list.chunks.len() >= 2); + assert!(chunk_list + .chunks + .iter() + .all(|chunk| chunk.size_bytes <= ARTIFACT_CHUNK_MAX_BYTES as u64)); + } + + #[test] + fn artifact_manifest_lazy_lookup_and_ranged_reads_do_not_require_full_tree() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source = tempfile::tempdir().unwrap(); + fs::create_dir_all(source.path().join("nested")).unwrap(); + let large = deterministic_benchmark_bytes(6 * 1024 * 1024); + fs::write(source.path().join("nested/large.bin"), &large).unwrap(); + fs::write(source.path().join("small.txt"), b"small artifact\n").unwrap(); + #[cfg(unix)] + std::os::unix::fs::symlink("../small.txt", source.path().join("nested/link")).unwrap(); + let (tree_id, _) = db.ingest_artifact_tree(source.path()).unwrap(); + + assert!(matches!( + db.artifact_tree_lazy_entry(&tree_id, "nested").unwrap(), + Some(ArtifactLazyEntry::Directory { .. }) + )); + let Some(ArtifactLazyEntry::File { + node_id, + mode, + size_bytes, + }) = db + .artifact_tree_lazy_entry(&tree_id, "nested/large.bin") + .unwrap() + else { + panic!("large artifact path must resolve lazily"); + }; + assert_eq!(mode, 0o644); + assert_eq!(size_bytes, large.len() as u64); + assert!(db + .artifact_tree_lazy_entry(&tree_id, "nested/missing") + .unwrap() + .is_none()); + let children = db.artifact_tree_lazy_children(&tree_id, "nested").unwrap(); + assert!(children.iter().any(|(name, _)| name == "large.bin")); + #[cfg(unix)] + assert!(children.iter().any(|(name, entry)| { + name == "link" && matches!(entry, ArtifactLazyEntry::Symlink { .. }) + })); + + let file = db.verified_artifact_file(&node_id).unwrap(); + let ArtifactFileContentV1::Chunks { chunk_list_id } = file.content else { + panic!("large lazy-read fixture must be chunked"); + }; + let list: ArtifactChunkListV1 = db + .get_artifact_cas_object( + &chunk_list_id.0, + ARTIFACT_CHUNK_LIST_KIND, + ARTIFACT_CHUNK_LIST_VERSION, + ) + .unwrap(); + assert!(list.chunks.len() > 1); + let first_size = list.chunks[0].size_bytes; + let last = list.chunks.last().unwrap().chunk_id.clone(); + db.conn + .execute( + "UPDATE artifact_objects SET kind='intentionally-unavailable-test-chunk' \ + WHERE artifact_id=?1", + params![last.0], + ) + .unwrap(); + let count = u32::try_from(first_size.min(64 * 1024)).unwrap(); + assert_eq!( + db.artifact_file_read_range(&node_id, 0, count).unwrap(), + large[..count as usize] + ); + } + + #[test] + fn artifact_file_materialization_never_removes_a_preexisting_destination() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source = tempfile::tempdir().unwrap(); + fs::write(source.path().join("artifact.txt"), b"artifact bytes\n").unwrap(); + let (tree_id, _) = db.ingest_artifact_tree(source.path()).unwrap(); + let Some(ArtifactLazyEntry::File { node_id, .. }) = db + .artifact_tree_lazy_entry(&tree_id, "artifact.txt") + .unwrap() + else { + panic!("artifact fixture must resolve to a file"); + }; + let destination_root = tempfile::tempdir().unwrap(); + let destination = destination_root.path().join("existing.txt"); + fs::write(&destination, b"user-owned bytes\n").unwrap(); + + assert!(db + .materialize_artifact_file(&node_id, &destination) + .is_err()); + assert_eq!(fs::read(destination).unwrap(), b"user-owned bytes\n"); + } + + #[test] + fn equal_artifact_files_reuse_cas_objects_across_desired_keys() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let _lock = db.acquire_write_lock().unwrap(); + let bytes = b"framework-neutral reusable bytes"; + let first = db.ingest_artifact_file_bytes(bytes, 0o644).unwrap(); + let object_count = db + .conn + .query_row("SELECT COUNT(*) FROM artifact_objects", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(); + let second = db.ingest_artifact_file_bytes(bytes, 0o644).unwrap(); + assert_eq!(first, second); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM artifact_objects", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + object_count + ); + } + + #[test] + fn artifact_tree_ingestion_is_order_independent_and_reuses_content() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let left = tempfile::tempdir().unwrap(); + let right = tempfile::tempdir().unwrap(); + for root in [left.path(), right.path()] { + fs::create_dir_all(root.join("nested")).unwrap(); + } + fs::write(left.path().join("z.txt"), "z\n").unwrap(); + fs::write(left.path().join("nested/a.txt"), "a\n").unwrap(); + fs::write(right.path().join("nested/a.txt"), "a\n").unwrap(); + fs::write(right.path().join("z.txt"), "z\n").unwrap(); + + let (left_id, left_tree) = db.ingest_artifact_tree(left.path()).unwrap(); + let object_count = db + .conn + .query_row("SELECT COUNT(*) FROM artifact_objects", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(); + let (right_id, right_tree) = db.ingest_artifact_tree(right.path()).unwrap(); + assert_eq!(left_id, right_id); + assert_eq!(left_tree, right_tree); + assert_eq!(left_tree.entry_count, 3); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM artifact_objects", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + object_count + ); + } + + #[cfg(unix)] + #[test] + fn artifact_tree_ingestion_rejects_escaping_symlinks() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let source = tempfile::tempdir().unwrap(); + std::os::unix::fs::symlink("../outside", source.path().join("escape")).unwrap(); + let error = db.ingest_artifact_tree(source.path()).unwrap_err(); + assert!(error.to_string().contains("escapes")); + } + + #[test] + fn artifact_tree_ingestion_rejects_secret_content() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let source = tempfile::tempdir().unwrap(); + fs::write( + source.path().join("generated.env"), + "API_TOKEN=do-not-store\n", + ) + .unwrap(); + let error = db.ingest_artifact_tree(source.path()).unwrap_err(); + assert!(error.to_string().contains("secret material")); + assert!(error.to_string().contains("generated.env")); + } + + #[test] + fn artifact_tree_secret_policy_allows_dependency_source_literals() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let source = tempfile::tempdir().unwrap(); + fs::write( + source.path().join("client.js"), + "export const example = 'Authorization: Bearer abc123';\n", + ) + .unwrap(); + + db.ingest_artifact_tree(source.path()).unwrap(); + + fs::write( + source.path().join("private.pem"), + "-----BEGIN PRIVATE KEY-----\nkey-material\n-----END PRIVATE KEY-----\n", + ) + .unwrap(); + let error = db.ingest_artifact_tree(source.path()).unwrap_err(); + assert!(error.to_string().contains("private.pem")); + assert!(error.to_string().contains("secret material")); + } + + #[cfg(unix)] + #[test] + fn artifact_tree_ingestion_rejects_privileged_modes_and_xattrs() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + + let privileged = tempfile::tempdir().unwrap(); + let privileged_file = privileged.path().join("tool"); + fs::write(&privileged_file, "tool\n").unwrap(); + let mut permissions = fs::metadata(&privileged_file).unwrap().permissions(); + permissions.set_mode(0o4755); + fs::set_permissions(&privileged_file, permissions).unwrap(); + let error = db.ingest_artifact_tree(privileged.path()).unwrap_err(); + assert!(error.to_string().contains("setuid")); + + let attributed = tempfile::tempdir().unwrap(); + let attributed_file = attributed.path().join("artifact"); + fs::write(&attributed_file, "clean\n").unwrap(); + let attribute = if cfg!(target_os = "macos") { + "com.trail.test" + } else { + "user.trail-test" + }; + xattr::set(&attributed_file, attribute, b"value").unwrap(); + let error = db.ingest_artifact_tree(attributed.path()).unwrap_err(); + assert!(error.to_string().contains("extended attribute")); + } + + #[test] + fn artifact_object_publication_rolls_back_and_retries_after_interruption() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let _lock = db.acquire_write_lock().unwrap(); + db.conn + .execute_batch( + "CREATE TRIGGER fail_artifact_mapping + BEFORE INSERT ON artifact_objects + BEGIN SELECT RAISE(ABORT, 'injected artifact publication failure'); END;", + ) + .unwrap(); + assert!(db + .ingest_artifact_file_bytes(b"atomic bytes", 0o644) + .is_err()); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM objects WHERE kind IN + ('ArtifactBlob','ArtifactFileNode')", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + db.conn + .execute_batch("DROP TRIGGER fail_artifact_mapping") + .unwrap(); + db.ingest_artifact_file_bytes(b"atomic bytes", 0o644) + .unwrap(); + } + + #[test] + fn artifact_object_publication_detects_corrupt_preexisting_bytes() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let _lock = db.acquire_write_lock().unwrap(); + let file_id = db + .ingest_artifact_file_bytes(b"collision evidence", 0o644) + .unwrap(); + let file: ArtifactFileNodeV1 = db + .get_object( + ARTIFACT_FILE_NODE_KIND, + &artifact_object_id(&db, &file_id.0), + ) + .unwrap(); + let ArtifactFileContentV1::Blob { blob_id } = file.content else { + panic!("small fixture must use a whole blob"); + }; + db.conn + .execute( + "UPDATE objects SET bytes=x'00' WHERE object_id=?1", + params![artifact_object_id(&db, &blob_id.0).0], + ) + .unwrap(); + let error = db + .ingest_artifact_file_bytes(b"collision evidence", 0o644) + .unwrap_err(); + assert!(error.to_string().contains("conflicts")); + } + + #[test] + fn concurrent_equal_tree_publishers_converge_and_reopen() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let source = tempfile::tempdir().unwrap(); + fs::write(source.path().join("artifact"), "shared\n").unwrap(); + let source = source.path().to_path_buf(); + let workspace = temp.path().to_path_buf(); + let barrier = Arc::new(std::sync::Barrier::new(2)); + let mut handles = Vec::new(); + for _ in 0..2 { + let source = source.clone(); + let workspace = workspace.clone(); + let barrier = barrier.clone(); + handles.push(std::thread::spawn(move || { + let db = Trail::open(workspace).unwrap(); + barrier.wait(); + Trail::with_write_lock_wait(Duration::from_secs(10), || { + db.ingest_artifact_tree(&source) + }) + .unwrap() + .0 + })); + } + let first = handles.remove(0).join().unwrap(); + let second = handles.remove(0).join().unwrap(); + assert_eq!(first, second); + + let reopened = Trail::open(temp.path()).unwrap(); + assert_eq!( + artifact_object_id(&reopened, &first.0), + artifact_object_id(&reopened, &second.0) + ); + } + + #[test] + fn successor_large_file_reuses_unchanged_fastcdc_chunks() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let _lock = db.acquire_write_lock().unwrap(); + let mut first_bytes = Vec::with_capacity(12 * 1024 * 1024); + for index in 0..12 * 1024 * 1024 { + first_bytes.push(((index * 17 + index / 53 + index / 997) % 251) as u8); + } + let first = db.ingest_artifact_file_bytes(&first_bytes, 0o644).unwrap(); + let mut second_bytes = first_bytes; + second_bytes[6 * 1024 * 1024] ^= 0x5a; + let second = db.ingest_artifact_file_bytes(&second_bytes, 0o644).unwrap(); + let first_chunks = artifact_file_chunk_ids(&db, &first); + let second_chunks = artifact_file_chunk_ids(&db, &second); + assert!(first_chunks.len() >= 3); + assert!( + first_chunks.intersection(&second_chunks).next().is_some(), + "a small successor edit should preserve at least one content-defined chunk" + ); + } + + #[cfg(unix)] + #[test] + fn artifact_tree_normalizes_hardlinks_and_confined_symlinks() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let source = tempfile::tempdir().unwrap(); + fs::write(source.path().join("original"), "same inode\n").unwrap(); + fs::hard_link( + source.path().join("original"), + source.path().join("hardlink"), + ) + .unwrap(); + std::os::unix::fs::symlink("original", source.path().join("symlink")).unwrap(); + let (_, tree) = db.ingest_artifact_tree(source.path()).unwrap(); + assert_eq!(tree.entry_count, 3); + let root: ArtifactDirectoryNodeV1 = db + .get_object( + ARTIFACT_DIRECTORY_NODE_KIND, + &artifact_object_id(&db, &tree.root_directory_id.0), + ) + .unwrap(); + let file_ids = root + .entries + .iter() + .filter_map(|entry| match &entry.target { + ArtifactDirectoryEntryTargetV1::File { node_id } => Some(node_id), + _ => None, + }) + .collect::>(); + assert_eq!(file_ids.len(), 2); + assert_eq!(file_ids[0], file_ids[1]); + assert!(root.entries.iter().any(|entry| matches!( + &entry.target, + ArtifactDirectoryEntryTargetV1::Symlink { target } if target == "original" + ))); + } + + #[test] + fn artifact_tree_rejects_non_nfc_and_case_colliding_paths() { + let non_nfc = ArtifactDirectoryNodeV1 { + version: ARTIFACT_DIRECTORY_NODE_VERSION, + entries: vec![ArtifactDirectoryEntryV1 { + name: "e\u{301}".into(), + target: ArtifactDirectoryEntryTargetV1::Symlink { + target: "safe".into(), + }, + }], + }; + assert!(encode_artifact_directory_node(non_nfc).is_err()); + assert!( + validate_no_case_fold_collisions(&["Node".to_string(), "node".to_string()]).is_err() + ); + } + + #[test] + fn artifact_tree_materializes_from_authoritative_objects() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let source = tempfile::tempdir().unwrap(); + fs::create_dir(source.path().join("nested")).unwrap(); + fs::write(source.path().join("nested/artifact"), "reconstruct me\n").unwrap(); + let (tree_id, _) = db.ingest_artifact_tree(source.path()).unwrap(); + let materialization_parent = tempfile::tempdir().unwrap(); + let destination = materialization_parent.path().join("materialized"); + let _lock = db.acquire_write_lock().unwrap(); + db.materialize_artifact_tree_under_write_lock(&tree_id, &destination) + .unwrap(); + assert_eq!( + fs::read(destination.join("nested/artifact")).unwrap(), + b"reconstruct me\n" + ); + assert_eq!( + db.ingest_artifact_tree_under_write_lock(&destination) + .unwrap() + .0, + tree_id + ); + } + + #[test] + fn artifact_materialization_cache_is_tree_keyed_verified_and_copy_safe() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source = tempfile::tempdir().unwrap(); + fs::create_dir_all(source.path().join("pkg")).unwrap(); + fs::write(source.path().join("pkg/index.js"), b"shared-cache\n").unwrap(); + let (tree_id, _) = db.ingest_artifact_tree(source.path()).unwrap(); + + let first = db.ensure_artifact_tree_materialization(&tree_id).unwrap(); + assert!(!first.reused); + assert_eq!(first.tree_root_id, tree_id); + assert_eq!( + fs::read(first.storage_path.join("pkg/index.js")).unwrap(), + b"shared-cache\n" + ); + let second = db.ensure_artifact_tree_materialization(&tree_id).unwrap(); + assert!(second.reused); + assert_eq!(first.storage_path, second.storage_path); + assert_eq!(first.materialization_id, second.materialization_id); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM artifact_materializations", + [], + |row| { row.get::<_, i64>(0) } + ) + .unwrap(), + 1 + ); + + let private = workspace.path().join("private-copy"); + super::super::workspace_layer::copy_layer_tree(&first.storage_path, &private).unwrap(); + super::super::workspace_layer::make_tree_writable(&private); + fs::write(private.join("pkg/index.js"), b"private\n").unwrap(); + assert_eq!( + fs::read(first.storage_path.join("pkg/index.js")).unwrap(), + b"shared-cache\n" + ); + + super::super::workspace_layer::make_tree_writable(&first.storage_path); + fs::write(first.storage_path.join("pkg/index.js"), b"corrupt\n").unwrap(); + let repaired = db.ensure_artifact_tree_materialization(&tree_id).unwrap(); + assert!(!repaired.reused); + assert_eq!(repaired.storage_path, first.storage_path); + assert_eq!( + fs::read(repaired.storage_path.join("pkg/index.js")).unwrap(), + b"shared-cache\n" + ); + + super::super::workspace_layer::make_tree_writable(&repaired.storage_path); + fs::remove_dir_all(&repaired.storage_path).unwrap(); + let rebuilt = db.ensure_artifact_tree_materialization(&tree_id).unwrap(); + assert!(!rebuilt.reused); + assert_eq!(rebuilt.storage_path, repaired.storage_path); + assert_eq!(rebuilt.logical_bytes, b"shared-cache\n".len() as u64); + assert_eq!(rebuilt.entry_count, 2); + assert!(rebuilt.physical_bytes > 0); + assert!(rebuilt + .backend_compatibility + .starts_with("trail-real-directory/v1/")); + + db.conn + .execute( + "DELETE FROM artifact_materializations WHERE materialization_id=?1", + params![&rebuilt.materialization_id], + ) + .unwrap(); + super::super::workspace_layer::make_tree_writable(&rebuilt.storage_path); + let adopted = db.ensure_artifact_tree_materialization(&tree_id).unwrap(); + assert!(adopted.reused); + assert_eq!(adopted.storage_path, rebuilt.storage_path); + assert!(fs::metadata(&adopted.storage_path) + .unwrap() + .permissions() + .readonly()); + + db.conn + .execute( + "UPDATE artifact_materializations SET state='building' + WHERE materialization_id=?1", + params![&adopted.materialization_id], + ) + .unwrap(); + let recovered = db.ensure_artifact_tree_materialization(&tree_id).unwrap(); + assert!(!recovered.reused); + assert_eq!( + fs::read(recovered.storage_path.join("pkg/index.js")).unwrap(), + b"shared-cache\n" + ); + } + + #[test] + fn artifact_materialization_cache_eviction_releases_object_gc_lease() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let source = tempfile::tempdir().unwrap(); + fs::write(source.path().join("artifact"), b"rebuildable cache\n").unwrap(); + let (tree_id, _) = db.ingest_artifact_tree(source.path()).unwrap(); + let materialization = db.ensure_artifact_tree_materialization(&tree_id).unwrap(); + + assert_eq!(db.gc(false).unwrap().pruned_objects, 0); + let preview = db.workspace_cache_gc(true, Some(0)).unwrap(); + assert!(preview.candidates.iter().any(|candidate| { + candidate.kind == "artifact_materialization" + && candidate.id == materialization.materialization_id + })); + assert_eq!( + preview.artifact_storage.materialized_bytes, + materialization.physical_bytes + ); + assert!(preview.artifact_storage.reclaimable_bytes >= materialization.physical_bytes); + assert_eq!( + preview + .artifact_storage + .materialized_bytes + .saturating_add(preview.artifact_storage.demand_loaded_bytes) + .saturating_add(preview.artifact_storage.unknown_bytes), + preview.cache_physical_bytes_before + ); + assert!(materialization.storage_path.exists()); + + let collected = db.workspace_cache_gc(false, Some(0)).unwrap(); + assert!(collected.deleted.iter().any(|candidate| { + candidate.kind == "artifact_materialization" + && candidate.id == materialization.materialization_id + })); + assert!(!materialization.storage_path.exists()); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM artifact_materializations", + [], + |row| { row.get::<_, u64>(0) } + ) + .unwrap(), + 0 + ); + let after_eviction = db.workspace_cache_gc(true, Some(0)).unwrap(); + assert_eq!(after_eviction.artifact_storage.materialized_bytes, 0); + + let object_gc = db.gc(false).unwrap(); + assert!(object_gc.pruned_objects > 0); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM artifact_objects", [], |row| { + row.get::<_, u64>(0) + }) + .unwrap(), + 0 + ); + } + + #[test] + fn backup_restore_preserves_artifact_authority_and_private_source_state() { + let workspace = tempfile::tempdir().unwrap(); + fs::write( + workspace.path().join("Cargo.toml"), + "[package]\nname='backup-artifact'\nversion='0.1.0'\n", + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "backup-artifact", + Some("main"), + LaneWorkdirMode::Virtual, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let lane = db.lane_details("backup-artifact").unwrap().branch; + let mountpoint = db.default_lane_workdir_path("backup-artifact").unwrap(); + let view = db + .create_workspace_view( + &lane.lane_id, + &lane.head_change, + &lane.head_root, + "test-cow", + &mountpoint, + ) + .unwrap(); + let source_upper = PathBuf::from(&view.source_upper); + let mut journal = super::workdir::ViewMutationJournal::open(&source_upper).unwrap(); + journal + .append( + super::workdir::ViewMutationKind::Create, + "agent-change.rs", + None, + ) + .unwrap(); + fs::write(source_upper.join("agent-change.rs"), "private source\n").unwrap(); + + let source_root = db.resolve_refish("main").unwrap().root_id; + let (snapshot_id, _) = db + .put_artifact_resolution_snapshot( + fixture_plan(source_root.clone()), + b"version = 4\n".to_vec(), + BTreeMap::new(), + BTreeMap::new(), + Vec::new(), + false, + ) + .unwrap(); + let artifact_source = tempfile::tempdir().unwrap(); + fs::create_dir_all(artifact_source.path().join("deps")).unwrap(); + fs::write( + artifact_source.path().join("deps/library.rlib"), + "immutable artifact\n", + ) + .unwrap(); + let (tree_id, _) = db.ingest_artifact_tree(artifact_source.path()).unwrap(); + let mut desired_material = fixture_desired_material(source_root.clone()); + desired_material.resolution_snapshot_id = Some(snapshot_id.clone()); + let desired_key = artifact_desired_key_v2(desired_material).unwrap(); + let (envelope_id, quarantined) = db + .put_artifact_envelope_under_write_lock(ArtifactEnvelopeV1 { + version: ARTIFACT_ENVELOPE_VERSION, + desired_identity: ArtifactDesiredIdentityV1::ArtifactDesiredV2 { + desired_key: desired_key.clone(), + }, + tree_root_id: tree_id.clone(), + component_id: "cargo:root".into(), + output_name: "target".into(), + output_policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + portability_scope: "workspace".into(), + trust_scope: "builtin".into(), + secret_taint: ArtifactSecretTaintV1::Clear, + resolution_snapshot_id: Some(snapshot_id.clone()), + validation_receipt_ids: Vec::new(), + }) + .unwrap(); + assert!(!quarantined); + db.verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id) + .unwrap(); + let materialization = db.ensure_artifact_tree_materialization(&tree_id).unwrap(); + + let generation_id = "envgen_backup_artifact"; + db.conn + .execute( + "INSERT INTO environment_generations( + generation_id,view_id,generation_sequence,source_root,specification_digest, + predecessor_generation_id,state,created_at,activated_at,retired_at) + VALUES(?1,?2,1,?3,'backup-spec',NULL,'active',1,1,NULL)", + params![generation_id, &view.view_id, source_root.0], + ) + .unwrap(); + db.conn + .execute( + "INSERT INTO environment_view_generations(view_id,generation_id,updated_at) + VALUES(?1,?2,1)", + params![&view.view_id, generation_id], + ) + .unwrap(); + let binding_identity = format!( + "artifact_binding_{}", + crate::ids::short_hash( + format!("{generation_id}\0cargo:root\0target\0{envelope_id}").as_bytes(), + 32, + ) + ); + db.conn + .execute( + "INSERT INTO artifact_generation_bindings( + binding_id,generation_id,component_id,output_name,desired_key,envelope_id, + tree_root_id,binding_identity,created_at) + VALUES(?1,?2,'cargo:root','target',?3,?4,?5,?6,1)", + params![ + format!( + "binding_{}", + crate::ids::short_hash(binding_identity.as_bytes(), 32) + ), + generation_id, + desired_key.0, + envelope_id.0, + tree_id.0, + binding_identity, + ], + ) + .unwrap(); + let cache_path = workspace.path().join(".trail/cache/namespaces/backup"); + fs::create_dir_all(&cache_path).unwrap(); + db.conn + .execute( + "INSERT INTO environment_cache_namespaces( + namespace_id,adapter_identity,cache_name,protocol,access,authority,scope, + compatibility_json,storage_path,last_used_at,created_at) + VALUES('cache_backup','trail/test@1','registry','content-v1','read_write', + 'performance_only','workspace',X'7B7D',?1,1,1)", + params![cache_path.to_string_lossy()], + ) + .unwrap(); + + let backup_parent = tempfile::tempdir().unwrap(); + let backup = backup_parent.path().join("portable-backup"); + let created = db.create_backup(&backup, false).unwrap(); + assert_eq!(created.retained_private_views, 1); + assert!(created.retained_private_bytes > 0); + assert!(created.rebuildable_materializations >= 1); + assert!(created.rebuildable_materialization_bytes >= materialization.physical_bytes); + assert_eq!(created.rebuildable_performance_caches, 1); + + let backup_conn = Connection::open(backup.join(DB_RELATIVE_PATH)).unwrap(); + for table in [ + "artifact_resolution_snapshots", + "artifact_envelopes", + "artifact_attestations", + "artifact_generation_bindings", + "environment_generations", + ] { + let count: i64 = backup_conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(count, 1, "backup lost authoritative table {table}"); + } + assert_eq!( + backup_conn + .query_row( + "SELECT COUNT(*) FROM artifact_materializations", + [], + |row| { row.get::<_, i64>(0) } + ) + .unwrap(), + 0 + ); + assert_eq!( + backup_conn + .query_row( + "SELECT COUNT(*) FROM environment_cache_namespaces", + [], + |row| row.get::<_, i64>(0) + ) + .unwrap(), + 0 + ); + let backed_up_private = backup + .join("views") + .join(&view.view_id) + .join("source-upper/agent-change.rs"); + assert_eq!( + fs::read_to_string(&backed_up_private).unwrap(), + "private source\n" + ); + let verified = Trail::verify_backup(&backup).unwrap(); + assert!(verified.valid, "{:?}", verified.errors); + assert_eq!(verified.retained_private_views, 1); + fs::write(&backed_up_private, "tampered source\n").unwrap(); + let tampered = Trail::verify_backup(&backup).unwrap(); + assert!(!tampered.valid); + assert!(tampered + .errors + .iter() + .any(|error| error.contains("retained private SHA-256 mismatch"))); + fs::write(&backed_up_private, "private source\n").unwrap(); + assert!(Trail::verify_backup(&backup).unwrap().valid); + + drop(backup_conn); + drop(db); + let restored = tempfile::tempdir().unwrap(); + let restore = Trail::restore_backup(restored.path(), &backup, false).unwrap(); + assert_eq!(restore.restored_private_views, 1); + assert_eq!( + restore.rebuildable_materializations, + created.rebuildable_materializations + ); + let restored_db = Trail::open(restored.path()).unwrap(); + let restored_view = restored_db + .lane_workspace_view("backup-artifact") + .unwrap() + .unwrap(); + assert_eq!( + fs::read_to_string(Path::new(&restored_view.source_upper).join("agent-change.rs")) + .unwrap(), + "private source\n" + ); + assert!(Path::new(&restored_view.generated_upper).is_dir()); + assert_eq!( + restored_db + .conn + .query_row( + "SELECT COUNT(*) FROM artifact_generation_bindings", + [], + |row| row.get::<_, i64>(0) + ) + .unwrap(), + 1 + ); + assert_eq!( + restored_db + .conn + .query_row( + "SELECT state FROM environment_generations WHERE generation_id=?1", + params![generation_id], + |row| row.get::<_, String>(0) + ) + .unwrap(), + "retired" + ); + assert_eq!( + restored_db + .conn + .query_row( + "SELECT COUNT(*) FROM artifact_materializations", + [], + |row| { row.get::<_, i64>(0) } + ) + .unwrap(), + 0 + ); + let rebuilt = restored_db + .ensure_artifact_tree_materialization(&tree_id) + .unwrap(); + assert!(!rebuilt.reused); + assert_eq!( + fs::read_to_string(rebuilt.storage_path.join("deps/library.rlib")).unwrap(), + "immutable artifact\n" + ); + } + + #[test] + fn artifact_materialization_rejects_noncanonical_database_path_without_touching_it() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source = tempfile::tempdir().unwrap(); + fs::write(source.path().join("artifact"), b"safe\n").unwrap(); + let (tree_id, _) = db.ingest_artifact_tree(source.path()).unwrap(); + let materialization = db.ensure_artifact_tree_materialization(&tree_id).unwrap(); + let external = tempfile::tempdir().unwrap(); + let sentinel = external.path().join("sentinel"); + fs::write(&sentinel, b"keep me\n").unwrap(); + db.conn + .execute( + "UPDATE artifact_materializations SET storage_path=?1 + WHERE materialization_id=?2", + params![ + external.path().to_string_lossy(), + &materialization.materialization_id + ], + ) + .unwrap(); + + let error = db + .ensure_artifact_tree_materialization(&tree_id) + .unwrap_err(); + assert!(matches!(error, Error::Corrupt(_))); + assert_eq!(fs::read(&sentinel).unwrap(), b"keep me\n"); + assert!(external.path().is_dir()); + } + + #[cfg(unix)] + #[test] + fn artifact_materialization_rejects_external_cache_and_root_symlinks() { + use std::os::unix::fs::symlink; + + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source = tempfile::tempdir().unwrap(); + fs::write(source.path().join("artifact"), b"safe\n").unwrap(); + let (tree_id, _) = db.ingest_artifact_tree(source.path()).unwrap(); + let external_parent = tempfile::tempdir().unwrap(); + let parent_sentinel = external_parent.path().join("sentinel"); + fs::write(&parent_sentinel, b"parent safe\n").unwrap(); + let staging = db.workspace_environment_staging_parent().unwrap(); + let materializations = staging.parent().unwrap().join("artifact-materializations"); + symlink(external_parent.path(), &materializations).unwrap(); + + let error = db + .ensure_artifact_tree_materialization(&tree_id) + .unwrap_err(); + assert!(matches!(error, Error::InvalidPath { .. })); + assert_eq!(fs::read(&parent_sentinel).unwrap(), b"parent safe\n"); + assert_eq!(fs::read_dir(external_parent.path()).unwrap().count(), 1); + + fs::remove_file(&materializations).unwrap(); + let materialization = db.ensure_artifact_tree_materialization(&tree_id).unwrap(); + super::super::workspace_layer::make_tree_writable(&materialization.storage_path); + fs::remove_dir_all(&materialization.storage_path).unwrap(); + let external_root = tempfile::tempdir().unwrap(); + let root_sentinel = external_root.path().join("sentinel"); + fs::write(&root_sentinel, b"root safe\n").unwrap(); + symlink(external_root.path(), &materialization.storage_path).unwrap(); + + let error = db + .ensure_artifact_tree_materialization(&tree_id) + .unwrap_err(); + assert!(matches!(error, Error::InvalidPath { .. })); + assert_eq!(fs::read(&root_sentinel).unwrap(), b"root safe\n"); + assert!(!fs::metadata(&root_sentinel) + .unwrap() + .permissions() + .readonly()); + } + + fn artifact_file_chunk_ids(db: &Trail, file_id: &ArtifactFileId) -> BTreeSet { + let file: ArtifactFileNodeV1 = db + .get_object(ARTIFACT_FILE_NODE_KIND, &artifact_object_id(db, &file_id.0)) + .unwrap(); + let ArtifactFileContentV1::Chunks { chunk_list_id } = file.content else { + panic!("large fixture must use chunks"); + }; + let list: ArtifactChunkListV1 = db + .get_object( + ARTIFACT_CHUNK_LIST_KIND, + &artifact_object_id(db, &chunk_list_id.0), + ) + .unwrap(); + list.chunks + .into_iter() + .map(|chunk| chunk.chunk_id) + .collect() + } + + fn artifact_object_id(db: &Trail, artifact_id: &str) -> ObjectId { + ObjectId( + db.conn + .query_row( + "SELECT object_id FROM artifact_objects WHERE artifact_id=?1", + params![artifact_id], + |row| row.get::<_, String>(0), + ) + .unwrap(), + ) + } + + #[test] + fn resolution_plan_is_canonical_and_bounded() { + let mut plan = fixture_plan(ObjectId("object_source".into())); + plan.allowed_authorities = vec![ + "index.crates.io:443".into(), + "crates.io:443".into(), + "index.crates.io:443".into(), + ]; + normalize_artifact_resolution_plan(&mut plan).unwrap(); + assert_eq!( + plan.allowed_authorities, + vec!["crates.io:443", "index.crates.io:443"] + ); + + plan.argv = vec!["cargo".into(); MAX_RESOLUTION_ARGV + 1]; + assert!(normalize_artifact_resolution_plan(&mut plan).is_err()); + } + + #[test] + fn resolution_snapshot_is_content_addressed_reused_and_refreshed() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let source_root = db.resolve_refish("main").unwrap().root_id; + let plan = fixture_plan(source_root); + + let (first_id, first) = db + .put_artifact_resolution_snapshot( + plan.clone(), + b"version = 4\n".to_vec(), + BTreeMap::from([("package:a".into(), "1.0.0".into())]), + BTreeMap::from([("package:a".into(), "22".repeat(32))]), + vec!["index.crates.io:443".into()], + false, + ) + .unwrap(); + let (reused_id, reused) = db + .put_artifact_resolution_snapshot( + plan.clone(), + b"version = 4\n".to_vec(), + BTreeMap::new(), + BTreeMap::new(), + vec![], + false, + ) + .unwrap(); + assert_eq!(reused_id, first_id); + assert_eq!(reused, first); + assert_eq!( + db.artifact_resolution_snapshot_content(&first).unwrap(), + b"version = 4\n" + ); + + let (next_id, next) = db + .put_artifact_resolution_snapshot( + plan, + b"version = 4\n# refreshed\n".to_vec(), + BTreeMap::new(), + BTreeMap::new(), + vec![], + true, + ) + .unwrap(); + assert_ne!(next_id, first_id); + assert_eq!(next.predecessor_snapshot_id, Some(first_id)); + assert_eq!( + db.artifact_resolution_snapshot_for_proposal("proposal_fixture") + .unwrap() + .unwrap() + .0, + next_id + ); + } + + #[test] + fn fsck_reads_raw_resolution_snapshot_bytes_past_the_object_cache() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let source_root = db.resolve_refish("main").unwrap().root_id; + let (_, snapshot) = db + .put_artifact_resolution_snapshot( + fixture_plan(source_root), + b"version = 4\n".to_vec(), + BTreeMap::new(), + BTreeMap::new(), + vec![], + false, + ) + .unwrap(); + assert_eq!( + db.artifact_resolution_snapshot_content(&snapshot).unwrap(), + b"version = 4\n" + ); + + db.conn + .execute( + "UPDATE objects SET bytes=X'00' WHERE object_id=?1", + params![snapshot.content_object_id.0], + ) + .unwrap(); + + let errors = db.validate_artifact_cas_integrity().unwrap(); + assert!(errors.iter().any(|error| { + error.contains("artifact resolution snapshot") + && error.contains("explicit component resolution with refresh") + })); + } + + #[test] + fn resolution_snapshot_rejects_undeclared_authority_and_implicit_drift() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let source_root = db.resolve_refish("main").unwrap().root_id; + let plan = fixture_plan(source_root); + let error = db + .put_artifact_resolution_snapshot( + plan.clone(), + b"lock".to_vec(), + BTreeMap::new(), + BTreeMap::new(), + vec!["evil.example:443".into()], + false, + ) + .unwrap_err(); + assert!(error.to_string().contains("undeclared")); + + db.put_artifact_resolution_snapshot( + plan.clone(), + b"lock".to_vec(), + BTreeMap::new(), + BTreeMap::new(), + vec![], + false, + ) + .unwrap(); + let error = db + .put_artifact_resolution_snapshot( + plan, + b"different lock".to_vec(), + BTreeMap::new(), + BTreeMap::new(), + vec![], + false, + ) + .unwrap_err(); + assert!(error.to_string().contains("explicit refresh")); + } + + #[test] + fn resolve_component_reuses_until_deliberate_refresh() { + let (_temp, db, source_root) = initialized_resolution_fixture(); + let plan = executable_fixture_plan(&db, source_root); + let first = db + .resolve_artifact_component( + ArtifactResolutionRequestV1 { + plan: plan.clone(), + candidate: fixture_candidate(b"version = 4\n"), + }, + false, + ) + .unwrap(); + assert_eq!(first.decision, ArtifactResolutionDecisionV1::Resolved); + assert!(first.attempt.is_some()); + + let reused = db + .resolve_artifact_component( + ArtifactResolutionRequestV1 { + plan: plan.clone(), + candidate: fixture_candidate(b"this candidate must not advance selection"), + }, + false, + ) + .unwrap(); + assert_eq!(reused.decision, ArtifactResolutionDecisionV1::Reused); + assert_eq!(reused.snapshot_id, first.snapshot_id); + assert!(reused.attempt.is_none()); + assert_eq!(db.artifact_resolution_attempts().unwrap().len(), 1); + + let refreshed = db + .resolve_artifact_component( + ArtifactResolutionRequestV1 { + plan, + candidate: fixture_candidate(b"version = 4\n# deliberate refresh\n"), + }, + true, + ) + .unwrap(); + assert_eq!(refreshed.decision, ArtifactResolutionDecisionV1::Refreshed); + assert_ne!(refreshed.snapshot_id, first.snapshot_id); + assert_eq!( + refreshed.snapshot.predecessor_snapshot_id, + Some(first.snapshot_id) + ); + assert_eq!(db.artifact_resolution_attempts().unwrap().len(), 2); + } + + #[test] + fn resolve_all_is_deterministic_and_rejects_mixed_roots() { + let (_temp, db, source_root) = initialized_resolution_fixture(); + let first = executable_fixture_plan(&db, source_root.clone()); + let mut second = first.clone(); + second.component_id = "node:root".into(); + second.proposal_key = "proposal_node_fixture".into(); + second.adapter_identity = "trail.builtin/node@1".into(); + + let report = db + .resolve_all_artifact_components( + vec![ + ArtifactResolutionRequestV1 { + plan: second, + candidate: fixture_candidate(b"node-lock"), + }, + ArtifactResolutionRequestV1 { + plan: first.clone(), + candidate: fixture_candidate(b"cargo-lock"), + }, + ], + false, + ) + .unwrap(); + assert_eq!( + report + .components + .iter() + .map(|component| component.component_id.as_str()) + .collect::>(), + vec!["cargo:root", "node:root"] + ); + + let mut mixed = first; + mixed.component_id = "mixed".into(); + mixed.proposal_key = "proposal_mixed".into(); + mixed.source_root = ObjectId("object_other_root".into()); + let error = db + .resolve_all_artifact_components( + vec![ + ArtifactResolutionRequestV1 { + plan: executable_fixture_plan(&db, source_root), + candidate: fixture_candidate(b"one"), + }, + ArtifactResolutionRequestV1 { + plan: mixed, + candidate: fixture_candidate(b"two"), + }, + ], + false, + ) + .unwrap_err(); + assert!(error.to_string().contains("one source root")); + } + + #[test] + fn resolve_component_records_malformed_and_output_limit_failures() { + let (_temp, db, source_root) = initialized_resolution_fixture(); + let plan = executable_fixture_plan(&db, source_root); + let mut malformed = fixture_candidate(b""); + malformed.stderr = b"malformed resolver output".to_vec(); + let error = db + .resolve_artifact_component( + ArtifactResolutionRequestV1 { + plan: plan.clone(), + candidate: malformed, + }, + false, + ) + .unwrap_err(); + assert!(error.to_string().contains("malformed snapshot")); + assert!(db + .artifact_resolution_snapshot_for_proposal(&plan.proposal_key) + .unwrap() + .is_none()); + + let mut limited_plan = plan; + limited_plan.proposal_key = "proposal_output_limit".into(); + limited_plan.limits.stdout_bytes = 4; + let mut oversized = fixture_candidate(b"valid"); + oversized.stdout = b"too much output".to_vec(); + let error = db + .resolve_artifact_component( + ArtifactResolutionRequestV1 { + plan: limited_plan.clone(), + candidate: oversized, + }, + false, + ) + .unwrap_err(); + assert!(error.to_string().contains("capture limit")); + assert!(db + .artifact_resolution_snapshot_for_proposal(&limited_plan.proposal_key) + .unwrap() + .is_none()); + let attempts = db.artifact_resolution_attempts().unwrap(); + assert_eq!(attempts.len(), 2); + assert!(attempts.iter().all(|attempt| { + attempt.status == ArtifactResolutionAttemptStatusV1::Failed + && attempt.failure_receipt_object_id.is_some() + })); + } + + #[test] + fn resolve_component_keeps_secret_tainted_candidate_out_of_shared_cas() { + let (_temp, db, source_root) = initialized_resolution_fixture(); + let plan = executable_fixture_plan(&db, source_root); + let secret = b"credential-value-never-store".to_vec(); + let mut candidate = fixture_candidate(b"snapshot credential-value-never-store"); + candidate.stdout = b"stdout credential-value-never-store".to_vec(); + candidate.stderr = b"stderr credential-value-never-store".to_vec(); + candidate.redactions = vec![secret.clone()]; + + let error = db + .resolve_artifact_component( + ArtifactResolutionRequestV1 { + plan: plan.clone(), + candidate, + }, + false, + ) + .unwrap_err(); + assert!(error.to_string().contains("private, non-promotable")); + assert!(db + .artifact_resolution_snapshot_for_proposal(&plan.proposal_key) + .unwrap() + .is_none()); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM objects WHERE kind=?1", + params![ARTIFACT_RESOLUTION_CONTENT_KIND], + |row| row.get::<_, u64>(0), + ) + .unwrap(), + 0 + ); + + let attempt = db.artifact_resolution_attempts().unwrap().pop().unwrap(); + assert_eq!( + attempt.failure_code.as_deref(), + Some("secret_tainted_output_private_only") + ); + let receipt: ArtifactResolutionFailureReceiptV1 = db + .get_object( + ARTIFACT_RESOLUTION_FAILURE_KIND, + attempt.failure_receipt_object_id.as_ref().unwrap(), + ) + .unwrap(); + assert_eq!( + receipt.secret_taint, + ArtifactSecretTaintV1::Tainted { + channels: vec!["resolver_credential".into()] + } + ); + for capture_id in [attempt.stdout_object_id, attempt.stderr_object_id] + .into_iter() + .flatten() + { + let capture: ArtifactResolutionCaptureV1 = db + .get_object(ARTIFACT_RESOLUTION_CAPTURE_KIND, &capture_id) + .unwrap(); + assert!(!capture + .bytes + .windows(secret.len()) + .any(|bytes| bytes == secret)); + } + let durable_objects = db + .conn + .prepare("SELECT bytes FROM objects ORDER BY object_id") + .unwrap() + .query_map([], |row| row.get::<_, Vec>(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(durable_objects + .iter() + .all(|bytes| !bytes.windows(secret.len()).any(|window| window == secret))); + } + + #[test] + fn resolve_component_rejects_stale_source_and_tool_identity() { + let (_temp, db, source_root) = initialized_resolution_fixture(); + let plan = executable_fixture_plan(&db, source_root); + + let mut stale_source = plan.clone(); + stale_source.readable_inputs[0].content_hash = "00".repeat(32); + let error = db + .resolve_artifact_component( + ArtifactResolutionRequestV1 { + plan: stale_source, + candidate: fixture_candidate(b"stale"), + }, + false, + ) + .unwrap_err(); + assert!(error.to_string().contains("changed after planning")); + + let mut stale_tool = plan; + stale_tool.executable_identity = "sha256:stale-tool".into(); + let error = db + .resolve_artifact_component( + ArtifactResolutionRequestV1 { + plan: stale_tool, + candidate: fixture_candidate(b"stale"), + }, + false, + ) + .unwrap_err(); + assert!(error.to_string().contains("executable")); + assert!(error.to_string().contains("changed after planning")); + } + + #[test] + fn resolution_attempt_is_fenced_cancelled_and_redacts_credentials() { + let (_temp, db, source_root) = initialized_resolution_fixture(); + let mut plan = executable_fixture_plan(&db, source_root); + plan.credential_handles = vec!["registry_credentials".into()]; + let (fence, started) = db.begin_artifact_resolution_attempt(plan).unwrap(); + assert_eq!(started.status, ArtifactResolutionAttemptStatusV1::Running); + assert!(db.heartbeat_artifact_resolution_attempt(&fence).unwrap()); + + let cancelling = db + .cancel_artifact_resolution_attempt(&fence.attempt_id) + .unwrap(); + assert!(cancelling.cancel_requested); + assert!(!db.heartbeat_artifact_resolution_attempt(&fence).unwrap()); + let secret = b"credential-value-never-store".to_vec(); + let finished = db + .finish_artifact_resolution_attempt_failure( + &fence, + ArtifactResolutionAttemptFailure { + code: "cancelled_by_user", + message: "credential-value-never-store was cancelled", + contacted_authorities: vec!["index.crates.io:443".into()], + stdout: b"stdout credential-value-never-store", + stderr: b"stderr credential-value-never-store", + stdout_original_bytes: None, + stderr_original_bytes: None, + redactions: std::slice::from_ref(&secret), + cancelled: true, + }, + ) + .unwrap(); + assert_eq!( + finished.status, + ArtifactResolutionAttemptStatusV1::Cancelled + ); + assert!(finished + .failure_message + .as_ref() + .unwrap() + .contains("[REDACTED]")); + let stdout: ArtifactResolutionCaptureV1 = db + .get_object( + ARTIFACT_RESOLUTION_CAPTURE_KIND, + finished.stdout_object_id.as_ref().unwrap(), + ) + .unwrap(); + assert!(!stdout + .bytes + .windows(secret.len()) + .any(|bytes| bytes == secret)); + let receipt: ArtifactResolutionFailureReceiptV1 = db + .get_object( + ARTIFACT_RESOLUTION_FAILURE_KIND, + finished.failure_receipt_object_id.as_ref().unwrap(), + ) + .unwrap(); + assert!(receipt.message.contains("[REDACTED]")); + assert_eq!( + receipt.authority_evidence.credential_handles, + vec!["registry_credentials"] + ); + assert!(receipt.authority_evidence.credential_values_redacted); + assert_eq!( + receipt.secret_taint, + ArtifactSecretTaintV1::Tainted { + channels: vec!["resolver_credential".into()] + } + ); + assert!(!serde_json::to_vec(&finished) + .unwrap() + .windows(secret.len()) + .any(|bytes| bytes == secret)); + } + + #[test] + fn resolution_attempt_rejects_stale_fence_and_bounds_capture() { + let (_temp, db, source_root) = initialized_resolution_fixture(); + let mut plan = executable_fixture_plan(&db, source_root); + plan.limits.stdout_bytes = 8; + let (fence, _) = db.begin_artifact_resolution_attempt(plan).unwrap(); + let mut stale = fence.clone(); + stale.owner_generation += 1; + assert!(!db.heartbeat_artifact_resolution_attempt(&stale).unwrap()); + assert!(db + .finish_artifact_resolution_attempt_failure( + &stale, + ArtifactResolutionAttemptFailure { + code: "failed", + message: "failure", + contacted_authorities: vec![], + stdout: b"output", + stderr: b"", + stdout_original_bytes: None, + stderr_original_bytes: None, + redactions: &[], + cancelled: false, + }, + ) + .unwrap_err() + .to_string() + .contains("exact owner fence")); + + let finished = db + .finish_artifact_resolution_attempt_failure( + &fence, + ArtifactResolutionAttemptFailure { + code: "resolver_failed", + message: "resolver failed", + contacted_authorities: vec![], + stdout: b"0123456789abcdef", + stderr: b"", + stdout_original_bytes: None, + stderr_original_bytes: None, + redactions: &[], + cancelled: false, + }, + ) + .unwrap(); + let capture: ArtifactResolutionCaptureV1 = db + .get_object( + ARTIFACT_RESOLUTION_CAPTURE_KIND, + finished.stdout_object_id.as_ref().unwrap(), + ) + .unwrap(); + assert_eq!(capture.bytes, b"01234567"); + assert_eq!(capture.original_bytes, 16); + assert!(capture.truncated); + } + + #[test] + fn resolution_attempt_singleflight_and_open_recovery_are_durable() { + let (temp, db, source_root) = initialized_resolution_fixture(); + let plan = executable_fixture_plan(&db, source_root); + let (fence, _) = db.begin_artifact_resolution_attempt(plan.clone()).unwrap(); + let error = db.begin_artifact_resolution_attempt(plan).unwrap_err(); + assert!(error.to_string().contains("already resolving")); + db.conn + .execute( + "UPDATE artifact_resolution_attempts + SET owner_pid=?1, owner_start_token='dead-owner' + WHERE attempt_id=?2", + params![i64::from(u32::MAX), fence.attempt_id.0], + ) + .unwrap(); + drop(db); + + let reopened = Trail::open(temp.path()).unwrap(); + let recovered = reopened + .artifact_resolution_attempt(&fence.attempt_id) + .unwrap(); + assert_eq!( + recovered.status, + ArtifactResolutionAttemptStatusV1::Abandoned + ); + assert_eq!( + recovered.failure_code.as_deref(), + Some("resolver_owner_lost") + ); + assert!(recovered.failure_receipt_object_id.is_some()); + + let plan = executable_fixture_plan(&reopened, recovered.source_root.clone()); + let (_, successor) = reopened.begin_artifact_resolution_attempt(plan).unwrap(); + assert_eq!(successor.owner_generation, recovered.owner_generation + 1); + } + + #[test] + fn resolution_attempt_success_requires_matching_snapshot_and_authority() { + let (_temp, db, source_root) = initialized_resolution_fixture(); + let plan = executable_fixture_plan(&db, source_root); + let (fence, _) = db.begin_artifact_resolution_attempt(plan.clone()).unwrap(); + let (snapshot_id, _) = db + .put_artifact_resolution_snapshot( + plan, + b"version = 4\n".to_vec(), + BTreeMap::new(), + BTreeMap::new(), + vec!["index.crates.io:443".into()], + false, + ) + .unwrap(); + let error = db + .finish_artifact_resolution_attempt_success( + &fence, + &snapshot_id, + vec!["undeclared.example:443".into()], + b"", + b"", + &[], + ) + .unwrap_err(); + assert!(error.to_string().contains("undeclared")); + let finished = db + .finish_artifact_resolution_attempt_success( + &fence, + &snapshot_id, + vec!["index.crates.io:443".into()], + b"resolved", + b"", + &[], + ) + .unwrap(); + assert_eq!( + finished.status, + ArtifactResolutionAttemptStatusV1::Succeeded + ); + assert_eq!(finished.snapshot_id, Some(snapshot_id)); + } + + #[test] + fn object_gc_collects_unbound_artifact_envelopes_and_content_graphs() { + let (workspace, mut db, source_root) = initialized_resolution_fixture(); + let candidate = tempfile::tempdir().unwrap(); + for index in 0..130 { + fs::write( + candidate.path().join(format!("result-{index:03}.bin")), + format!("unbound artifact {index}\n"), + ) + .unwrap(); + } + let (tree_id, _) = db.ingest_artifact_tree(candidate.path()).unwrap(); + let desired_key = artifact_desired_key_v2(fixture_desired_material(source_root)).unwrap(); + let (envelope_id, quarantined) = db + .put_artifact_envelope_under_write_lock(ArtifactEnvelopeV1 { + version: ARTIFACT_ENVELOPE_VERSION, + desired_identity: ArtifactDesiredIdentityV1::ArtifactDesiredV2 { desired_key }, + tree_root_id: tree_id, + component_id: "cargo:root".into(), + output_name: "target".into(), + output_policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + portability_scope: "workspace".into(), + trust_scope: "builtin".into(), + secret_taint: ArtifactSecretTaintV1::Clear, + resolution_snapshot_id: None, + validation_receipt_ids: Vec::new(), + }) + .unwrap(); + assert!(!quarantined); + let artifact_object_count = db + .conn + .query_row("SELECT COUNT(*) FROM artifact_objects", [], |row| { + row.get::<_, u64>(0) + }) + .unwrap(); + assert!(artifact_object_count > 256); + + let preview = db.gc(true).unwrap(); + assert!(preview.prunable_objects >= artifact_object_count); + assert_eq!(preview.pruned_objects, 0); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM artifact_envelopes WHERE envelope_id=?1", + params![envelope_id.0], + |row| row.get::<_, u64>(0), + ) + .unwrap(), + 1 + ); + + Trail::set_gc_test_failure_after_committed_batches_for_current_thread(Some(1)); + let interrupted = db.gc(false).unwrap_err(); + assert!(interrupted + .to_string() + .contains("injected object-GC interruption")); + let remaining_after_interruption = db + .conn + .query_row("SELECT COUNT(*) FROM artifact_objects", [], |row| { + row.get::<_, u64>(0) + }) + .unwrap(); + assert!(remaining_after_interruption > 0); + assert!(remaining_after_interruption < artifact_object_count); + + drop(db); + let mut db = Trail::open(workspace.path()).unwrap(); + let collected = db.gc(false).unwrap(); + assert!(collected.pruned_objects > 0); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM artifact_objects", [], |row| { + row.get::<_, u64>(0) + }) + .unwrap(), + 0 + ); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM artifact_envelopes", [], |row| { + row.get::<_, u64>(0) + }) + .unwrap(), + 0 + ); + assert_eq!(db.gc(false).unwrap().pruned_objects, 0); + drop(workspace); + } + + #[test] + fn object_gc_preserves_shared_chunks_until_the_last_hold_is_removed() { + let (_workspace, mut db, source_root) = initialized_resolution_fixture(); + let first_candidate = tempfile::tempdir().unwrap(); + let second_candidate = tempfile::tempdir().unwrap(); + let mut shared = vec![0_u8; ARTIFACT_WHOLE_BLOB_MAX_BYTES + 512 * 1024]; + for (index, byte) in shared.iter_mut().enumerate() { + *byte = ((index.wrapping_mul(31)) % 251) as u8; + } + for candidate in [&first_candidate, &second_candidate] { + fs::write(candidate.path().join("shared.bin"), &shared).unwrap(); + } + fs::write(first_candidate.path().join("only-first"), b"first\n").unwrap(); + fs::write(second_candidate.path().join("only-second"), b"second\n").unwrap(); + let (first_tree, _) = db.ingest_artifact_tree(first_candidate.path()).unwrap(); + let (second_tree, _) = db.ingest_artifact_tree(second_candidate.path()).unwrap(); + + let first_key = + artifact_desired_key_v2(fixture_desired_material(source_root.clone())).unwrap(); + let mut second_material = fixture_desired_material(source_root); + second_material.target = "release".into(); + let second_key = artifact_desired_key_v2(second_material).unwrap(); + let envelope = |desired_key, tree_root_id| ArtifactEnvelopeV1 { + version: ARTIFACT_ENVELOPE_VERSION, + desired_identity: ArtifactDesiredIdentityV1::ArtifactDesiredV2 { desired_key }, + tree_root_id, + component_id: "cargo:root".into(), + output_name: "target".into(), + output_policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + portability_scope: "workspace".into(), + trust_scope: "builtin".into(), + secret_taint: ArtifactSecretTaintV1::Clear, + resolution_snapshot_id: None, + validation_receipt_ids: Vec::new(), + }; + let (first_envelope, _) = db + .put_artifact_envelope_under_write_lock(envelope(first_key, first_tree.clone())) + .unwrap(); + let (second_envelope, _) = db + .put_artifact_envelope_under_write_lock(envelope(second_key, second_tree)) + .unwrap(); + let accounting = db.artifact_storage_accounting(None, 13, 7, 5, 11).unwrap(); + let authoritative_bytes = db + .conn + .query_row( + "SELECT COALESCE(SUM(LENGTH(o.bytes)),0) + FROM objects o + WHERE o.object_id IN ( + SELECT object_id FROM artifact_objects + UNION SELECT object_id FROM artifact_attestations + )", + [], + |row| row.get::<_, u64>(0), + ) + .unwrap(); + assert_eq!( + accounting + .unique_authoritative_bytes + .saturating_add(accounting.cross_artifact_shared_bytes), + authoritative_bytes + ); + assert!(accounting.unique_authoritative_bytes > 0); + assert!(accounting.cross_artifact_shared_bytes > 0); + assert_eq!( + accounting.logical_bytes, + (shared.len() * 2 + b"first\n".len() + b"second\n".len()) as u64 + ); + assert_eq!(accounting.lane_private_bytes, 13); + assert_eq!(accounting.demand_loaded_bytes, 7); + assert_eq!(accounting.reclaimable_bytes, 5); + assert_eq!(accounting.unknown_bytes, 11); + assert_eq!(accounting.prefetched_bytes, 0); + + let materialization = db + .ensure_artifact_tree_materialization(&first_tree) + .unwrap(); + let with_materialization = db.artifact_storage_accounting(None, 0, 0, 0, 0).unwrap(); + assert_eq!( + with_materialization.materialized_bytes, + materialization.physical_bytes + ); + db.conn + .execute( + "DELETE FROM artifact_materializations WHERE materialization_id=?1", + params![materialization.materialization_id], + ) + .unwrap(); + super::super::workspace_layer::make_tree_writable(&materialization.storage_path); + fs::remove_dir_all(materialization.storage_path).unwrap(); + for (hold_id, envelope_id) in [ + ("hold_first", &first_envelope), + ("hold_second", &second_envelope), + ] { + db.conn + .execute( + "INSERT INTO artifact_holds( + hold_id,target_kind,target_id,reason,created_at + ) VALUES(?1,'artifact_envelope',?2,'gc-test',?3)", + params![hold_id, envelope_id.0, now_ts()], + ) + .unwrap(); + } + let chunk_objects = { + let mut statement = db + .conn + .prepare( + "SELECT object_id FROM artifact_objects + WHERE kind=?1 ORDER BY object_id", + ) + .unwrap(); + statement + .query_map(params![ARTIFACT_CHUNK_KIND], |row| row.get::<_, String>(0)) + .unwrap() + .collect::, _>>() + .unwrap() + }; + assert!(!chunk_objects.is_empty()); + + assert_eq!(db.gc(false).unwrap().pruned_objects, 0); + db.conn + .execute("DELETE FROM artifact_holds WHERE hold_id='hold_first'", []) + .unwrap(); + let first_collection = db.gc(false).unwrap(); + assert!(first_collection.pruned_objects > 0); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM artifact_envelopes WHERE envelope_id=?1", + params![first_envelope.0], + |row| row.get::<_, u64>(0), + ) + .unwrap(), + 0 + ); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM artifact_envelopes WHERE envelope_id=?1", + params![second_envelope.0], + |row| row.get::<_, u64>(0), + ) + .unwrap(), + 1 + ); + for object_id in &chunk_objects { + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM objects WHERE object_id=?1", + params![object_id], + |row| row.get::<_, u64>(0), + ) + .unwrap(), + 1 + ); + } + + db.conn + .execute("DELETE FROM artifact_holds WHERE hold_id='hold_second'", []) + .unwrap(); + let final_collection = db.gc(false).unwrap(); + assert!(final_collection.pruned_objects > 0); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM artifact_objects", [], |row| { + row.get::<_, u64>(0) + }) + .unwrap(), + 0 + ); + for object_id in chunk_objects { + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM objects WHERE object_id=?1", + params![object_id], + |row| row.get::<_, u64>(0), + ) + .unwrap(), + 0 + ); + } + } + + #[test] + fn artifact_accounting_does_not_multiply_shared_authority_across_1_5_20_lanes() { + for lane_count in [1_usize, 5, 20] { + let (_workspace, mut db, source_root) = initialized_resolution_fixture(); + let artifact_source = tempfile::tempdir().unwrap(); + fs::write( + artifact_source.path().join("shared-output.bin"), + vec![42_u8; 32 * 1024], + ) + .unwrap(); + let (tree_id, tree) = db.ingest_artifact_tree(artifact_source.path()).unwrap(); + let desired_key = + artifact_desired_key_v2(fixture_desired_material(source_root.clone())).unwrap(); + let (envelope_id, quarantined) = db + .put_artifact_envelope_under_write_lock(ArtifactEnvelopeV1 { + version: ARTIFACT_ENVELOPE_VERSION, + desired_identity: ArtifactDesiredIdentityV1::ArtifactDesiredV2 { + desired_key: desired_key.clone(), + }, + tree_root_id: tree_id.clone(), + component_id: "cargo:root".into(), + output_name: "target".into(), + output_policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + portability_scope: "workspace".into(), + trust_scope: "builtin".into(), + secret_taint: ArtifactSecretTaintV1::Clear, + resolution_snapshot_id: None, + validation_receipt_ids: Vec::new(), + }) + .unwrap(); + assert!(!quarantined); + let materialization = db.ensure_artifact_tree_materialization(&tree_id).unwrap(); + + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + let mut lane_reports = Vec::new(); + for index in 0..lane_count { + let lane = format!("accounting-{lane_count}-{index}"); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + &lane, + Some("main"), + mode.clone(), + None, + None, + None, + &[], + false, + ) + .unwrap(); + let view = db.lane_workspace_view(&lane).unwrap().unwrap(); + fs::write( + Path::new(&view.generated_upper).join("lane-private.bin"), + vec![index as u8; 1024 + index], + ) + .unwrap(); + let generation_id = format!("envgen_accounting_{lane_count}_{index}"); + db.conn + .execute( + "INSERT INTO environment_generations( + generation_id,view_id,generation_sequence,source_root, + specification_digest,predecessor_generation_id,state,created_at, + activated_at,retired_at) + VALUES(?1,?2,1,?3,'accounting-spec',NULL,'active',1,1,NULL)", + params![generation_id, view.view_id, source_root.0], + ) + .unwrap(); + db.conn + .execute( + "INSERT INTO environment_view_generations( + view_id,generation_id,updated_at) VALUES(?1,?2,1)", + params![view.view_id, generation_id], + ) + .unwrap(); + db.conn + .execute( + "INSERT INTO artifact_generation_bindings( + binding_id,generation_id,component_id,output_name,desired_key, + envelope_id,tree_root_id,binding_identity,created_at) + VALUES(?1,?2,'cargo:root','target',?3,?4,?5,?6,1)", + params![ + format!("binding_accounting_{lane_count}_{index}"), + generation_id, + desired_key.0, + envelope_id.0, + tree_id.0, + format!("identity_accounting_{lane_count}_{index}"), + ], + ) + .unwrap(); + lane_reports.push(db.lane_workspace_space(&lane).unwrap()); + } + + let workspace_accounting = db.artifact_storage_accounting(None, 0, 0, 0, 0).unwrap(); + assert_eq!(workspace_accounting.logical_bytes, tree.logical_bytes); + assert_eq!(workspace_accounting.cross_artifact_shared_bytes, 0); + assert!(workspace_accounting.unique_authoritative_bytes > 0); + assert_eq!( + workspace_accounting.materialized_bytes, + materialization.physical_bytes + ); + for report in lane_reports { + assert_eq!(report.artifact_storage.logical_bytes, tree.logical_bytes); + assert_eq!( + report.artifact_storage.unique_authoritative_bytes, + workspace_accounting.unique_authoritative_bytes + ); + assert_eq!(report.artifact_storage.cross_artifact_shared_bytes, 0); + assert_eq!( + report.artifact_storage.materialized_bytes, + materialization.physical_bytes + ); + assert_eq!( + report.artifact_storage.lane_private_bytes, + report.lane_exclusive_physical_bytes + ); + assert!(report.artifact_storage.lane_private_bytes > 0); + assert_eq!( + report.artifact_storage.reclaimable_bytes, + materialization.physical_bytes + ); + assert_eq!(report.artifact_storage.prefetched_bytes, 0); + assert_eq!(report.artifact_storage.demand_loaded_bytes, 0); + assert_eq!(report.artifact_storage.unknown_bytes, 0); + } + } + } + + fn synthetic_artifact_tree( + db: &Trail, + entry_count: u64, + ) -> (ArtifactTreeId, ArtifactTreeRootV1) { + const FILES_PER_DIRECTORY: u64 = 1_000; + + assert!(matches!(entry_count, 10_000 | 100_000 | 1_000_000)); + let directory_count = entry_count.div_ceil(FILES_PER_DIRECTORY + 1); + let file_count = entry_count - directory_count; + let required_directories = file_count.div_ceil(FILES_PER_DIRECTORY); + assert!(required_directories <= directory_count); + assert!(directory_count - required_directories <= 1); + + let file_bytes = b"shared immutable artifact content\n"; + let file_id = db.ingest_artifact_file_bytes(file_bytes, 0o644).unwrap(); + let mut root_entries = Vec::with_capacity(directory_count as usize); + let mut remaining_files = file_count; + for directory_index in 0..directory_count { + let files_in_directory = remaining_files.min(FILES_PER_DIRECTORY); + let entries = (0..files_in_directory) + .map(|file_index| ArtifactDirectoryEntryV1 { + name: format!("file-{file_index:04}.bin"), + target: ArtifactDirectoryEntryTargetV1::File { + node_id: file_id.clone(), + }, + }) + .collect(); + let directory = canonical_artifact_directory_node(ArtifactDirectoryNodeV1 { + version: ARTIFACT_DIRECTORY_NODE_VERSION, + entries, + }) + .unwrap(); + let (directory_id, _) = encode_artifact_directory_node(directory.clone()).unwrap(); + db.put_artifact_cas_object( + &directory_id.0, + ARTIFACT_DIRECTORY_NODE_KIND, + ARTIFACT_DIRECTORY_NODE_VERSION, + files_in_directory * file_bytes.len() as u64, + &directory, + ) + .unwrap(); + root_entries.push(ArtifactDirectoryEntryV1 { + name: format!("bucket-{directory_index:04}"), + target: ArtifactDirectoryEntryTargetV1::Directory { + node_id: directory_id, + }, + }); + remaining_files -= files_in_directory; + } + assert_eq!(remaining_files, 0); + + let root_directory = canonical_artifact_directory_node(ArtifactDirectoryNodeV1 { + version: ARTIFACT_DIRECTORY_NODE_VERSION, + entries: root_entries, + }) + .unwrap(); + let (root_directory_id, _) = + encode_artifact_directory_node(root_directory.clone()).unwrap(); + let logical_bytes = file_count * file_bytes.len() as u64; + db.put_artifact_cas_object( + &root_directory_id.0, + ARTIFACT_DIRECTORY_NODE_KIND, + ARTIFACT_DIRECTORY_NODE_VERSION, + logical_bytes, + &root_directory, + ) + .unwrap(); + let tree = ArtifactTreeRootV1 { + version: ARTIFACT_TREE_ROOT_VERSION, + root_directory_id, + logical_bytes, + entry_count, + path_normalizer: "trail-paths/v1".into(), + }; + let (tree_id, _) = encode_artifact_tree_root(tree.clone()).unwrap(); + db.put_artifact_cas_object( + &tree_id.0, + ARTIFACT_TREE_ROOT_KIND, + ARTIFACT_TREE_ROOT_VERSION, + logical_bytes, + &tree, + ) + .unwrap(); + (tree_id, tree) + } + + #[test] + fn large_artifact_multi_lane_scale_acceptance() { + if std::env::var_os("TRAIL_RUN_ARTIFACT_SCALE_TEST").is_none() { + return; + } + let entry_count = std::env::var("TRAIL_SCALE_ARTIFACT_ENTRIES") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(1_000_000); + let lane_count = std::env::var("TRAIL_SCALE_LANES") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(20); + assert!(matches!(entry_count, 10_000 | 100_000 | 1_000_000)); + assert!(matches!(lane_count, 1 | 5 | 20)); + + let (_workspace, mut db, source_root) = initialized_resolution_fixture(); + let object_count_before = db + .conn + .query_row("SELECT COUNT(*) FROM artifact_objects", [], |row| { + row.get::<_, u64>(0) + }) + .unwrap(); + let build_started = Instant::now(); + let (tree_id, tree) = synthetic_artifact_tree(&db, entry_count); + let artifact_build_ms = build_started.elapsed().as_millis(); + + let publish_started = Instant::now(); + let desired_key = + artifact_desired_key_v2(fixture_desired_material(source_root.clone())).unwrap(); + let (envelope_id, quarantined) = db + .put_artifact_envelope_under_write_lock(ArtifactEnvelopeV1 { + version: ARTIFACT_ENVELOPE_VERSION, + desired_identity: ArtifactDesiredIdentityV1::ArtifactDesiredV2 { + desired_key: desired_key.clone(), + }, + tree_root_id: tree_id.clone(), + component_id: "scale:synthetic".into(), + output_name: "large-artifact".into(), + output_policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + portability_scope: "workspace".into(), + trust_scope: "builtin".into(), + secret_taint: ArtifactSecretTaintV1::Clear, + resolution_snapshot_id: None, + validation_receipt_ids: Vec::new(), + }) + .unwrap(); + assert!(!quarantined); + let envelope_publish_ms = publish_started.elapsed().as_millis(); + let reachability = db.artifact_content_reachability(&envelope_id).unwrap(); + let object_count_after_publish = db + .conn + .query_row("SELECT COUNT(*) FROM artifact_objects", [], |row| { + row.get::<_, u64>(0) + }) + .unwrap(); + + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + let attach_started = Instant::now(); + let mut lanes = Vec::with_capacity(lane_count); + for index in 0..lane_count { + let lane = format!("artifact-scale-{lane_count}-{index:02}"); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + &lane, + Some("main"), + mode.clone(), + None, + None, + None, + &[], + false, + ) + .unwrap(); + let view = db.lane_workspace_view(&lane).unwrap().unwrap(); + let generation_id = format!("envgen_artifact_scale_{lane_count}_{index}"); + db.conn + .execute( + "INSERT INTO environment_generations( + generation_id,view_id,generation_sequence,source_root, + specification_digest,predecessor_generation_id,state,created_at, + activated_at,retired_at) + VALUES(?1,?2,1,?3,'artifact-scale',NULL,'active',1,1,NULL)", + params![generation_id, view.view_id, source_root.0], + ) + .unwrap(); + db.conn + .execute( + "INSERT INTO environment_view_generations(view_id,generation_id,updated_at) + VALUES(?1,?2,1)", + params![view.view_id, generation_id], + ) + .unwrap(); + db.conn + .execute( + "INSERT INTO artifact_generation_bindings( + binding_id,generation_id,component_id,output_name,desired_key, + envelope_id,tree_root_id,binding_identity,created_at) + VALUES(?1,?2,'scale:synthetic','large-artifact',?3,?4,?5,?6,1)", + params![ + format!("binding_artifact_scale_{lane_count}_{index}"), + generation_id, + desired_key.0, + envelope_id.0, + tree_id.0, + format!("identity_artifact_scale_{lane_count}_{index}"), + ], + ) + .unwrap(); + lanes.push((lane, PathBuf::from(view.generated_upper))); + } + let lane_attach_ms = attach_started.elapsed().as_millis(); + + let private_write_started = Instant::now(); + for (index, (_, generated_upper)) in lanes.iter().enumerate() { + fs::write( + generated_upper.join("lane-private.bin"), + vec![index as u8; 4_096 + index], + ) + .unwrap(); + } + let private_write_ms = private_write_started.elapsed().as_millis(); + + let accounting_started = Instant::now(); + let workspace_accounting = db.artifact_storage_accounting(None, 0, 0, 0, 0).unwrap(); + let representative_lane = db.lane_workspace_space(&lanes[0].0).unwrap(); + let accounting_ms = accounting_started.elapsed().as_millis(); + let lane_private_bytes = lanes + .iter() + .map(|(_, generated_upper)| { + super::workspace_layer::layer_physical_bytes(generated_upper).unwrap() + }) + .collect::>(); + assert!(lane_private_bytes.iter().all(|bytes| *bytes > 0)); + assert_eq!( + representative_lane.artifact_storage.logical_bytes, + tree.logical_bytes + ); + assert_eq!( + representative_lane + .artifact_storage + .unique_authoritative_bytes, + workspace_accounting.unique_authoritative_bytes + ); + assert_eq!(representative_lane.artifact_storage.materialized_bytes, 0); + let (binding_count, distinct_tree_count) = db + .conn + .query_row( + "SELECT COUNT(*),COUNT(DISTINCT tree_root_id) + FROM artifact_generation_bindings", + [], + |row| Ok((row.get::<_, u64>(0)?, row.get::<_, u64>(1)?)), + ) + .unwrap(); + assert_eq!(binding_count, lane_count as u64); + assert_eq!(distinct_tree_count, 1); + assert_eq!(workspace_accounting.logical_bytes, tree.logical_bytes); + assert_eq!(workspace_accounting.materialized_bytes, 0); + assert!(workspace_accounting.unique_authoritative_bytes < tree.logical_bytes); + let materialization_count = db + .conn + .query_row( + "SELECT COUNT(*) FROM artifact_materializations", + [], + |row| row.get::<_, u64>(0), + ) + .unwrap(); + assert_eq!(materialization_count, 0); + + let skipped_native_gates = vec!["nfs_macos", "fuse_linux", "dokan_windows"]; + let private_total = lane_private_bytes.iter().copied().sum::(); + let evidence = serde_json::json!({ + "schema": "trail.artifact-lane-scale/v1", + "host": { + "platform": std::env::consts::OS, + "architecture": std::env::consts::ARCH, + }, + "backend": "cas-lazy-unmounted", + "backend_qualified": true, + "native_backend_qualified": false, + "artifact_entries": entry_count, + "lanes": lane_count, + "phase_latencies_ms": { + "artifact_build": artifact_build_ms, + "envelope_publish": envelope_publish_ms, + "lane_attach": lane_attach_ms, + "private_write": private_write_ms, + "accounting": accounting_ms, + }, + "content_reuse": { + "logical_bytes": tree.logical_bytes, + "authoritative_encoded_bytes": workspace_accounting.unique_authoritative_bytes, + "reused_logical_bytes": tree.logical_bytes.saturating_sub(workspace_accounting.unique_authoritative_bytes), + "shared_tree_roots": distinct_tree_count, + "lane_bindings": binding_count, + }, + "materialization_amplification": { + "materialization_count": materialization_count, + "materialized_physical_bytes": workspace_accounting.materialized_bytes, + "naive_per_lane_logical_bytes": tree.logical_bytes.saturating_mul(lane_count as u64), + "copied_bytes": 0, + "projected_bytes": 0, + "prefetched_bytes": 0, + }, + "private_deltas": { + "lane_count": lane_private_bytes.len(), + "total_physical_bytes": private_total, + "minimum_physical_bytes": lane_private_bytes.iter().min(), + "maximum_physical_bytes": lane_private_bytes.iter().max(), + }, + "object_count": { + "before": object_count_before, + "after_publish": object_count_after_publish, + "published": object_count_after_publish.saturating_sub(object_count_before), + "reachable": reachability.object_count, + }, + "skipped_native_gates": skipped_native_gates, + }); + println!("{evidence}"); + if let Some(directory) = std::env::var_os("TRAIL_SCALE_EVIDENCE_DIR") { + let directory = PathBuf::from(directory); + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join(format!("artifacts-{entry_count}-lanes-{lane_count}.json")), + serde_json::to_vec_pretty(&evidence).unwrap(), + ) + .unwrap(); + } + } + + #[test] + fn object_gc_traces_resolution_snapshots_and_live_attempt_evidence() { + let (_workspace, mut db, source_root) = initialized_resolution_fixture(); + let plan = executable_fixture_plan(&db, source_root); + let (fence, attempt) = db.begin_artifact_resolution_attempt(plan.clone()).unwrap(); + let (snapshot_id, snapshot) = db + .put_artifact_resolution_snapshot( + plan, + b"version = 4\n".to_vec(), + BTreeMap::new(), + BTreeMap::new(), + vec!["index.crates.io:443".into()], + false, + ) + .unwrap(); + + assert_eq!(db.gc(false).unwrap().pruned_objects, 0); + for object_id in [ + attempt.plan_object_id.0, + snapshot_id.0, + snapshot.content_object_id.0, + ] { + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM objects WHERE object_id=?1", + params![object_id], + |row| row.get::<_, u64>(0), + ) + .unwrap(), + 1 + ); + } + db.cancel_artifact_resolution_attempt(&fence.attempt_id) + .unwrap(); + } +} diff --git a/trail/src/db/lane/workspace_cargo.rs b/trail/src/db/lane/workspace_cargo.rs index adc8e8fd..c926a124 100644 --- a/trail/src/db/lane/workspace_cargo.rs +++ b/trail/src/db/lane/workspace_cargo.rs @@ -1,10 +1,13 @@ use super::workspace_environment::{ resolve_workspace_tool_executable, WorkspaceEnvironmentAdapter, - WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentCacheAccess, - WorkspaceEnvironmentCacheProtocol, WorkspaceEnvironmentCommand, WorkspaceEnvironmentOutput, - WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, WorkspaceEnvironmentSandboxPolicy, + WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentAdapterProposal, + WorkspaceEnvironmentCacheAccess, WorkspaceEnvironmentCacheProtocol, + WorkspaceEnvironmentCommand, WorkspaceEnvironmentOutput, WorkspaceEnvironmentOutputPolicy, + WorkspaceEnvironmentPlan, WorkspaceEnvironmentResolutionInput, + WorkspaceEnvironmentSandboxPolicy, }; use super::*; +use crate::ids::sha256_hex; pub(crate) struct CargoTargetSeedAdapter; @@ -47,10 +50,110 @@ impl WorkspaceEnvironmentAdapter for CargoTargetSeedAdapter { let root = normalize_component_root(component_root)?; Ok(db .root_file_entry(source_root, &join_repo_path(&root, "Cargo.toml"))? + .is_some()) + } + + fn propose( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result> { + let root = normalize_component_root(component_root)?; + if db + .root_file_entry(source_root, &join_repo_path(&root, "Cargo.toml"))? + .is_none() + { + return Ok(None); + } + if db + .root_file_entry(source_root, &join_repo_path(&root, "Cargo.lock"))? .is_some() - && db - .root_file_entry(source_root, &join_repo_path(&root, "Cargo.lock"))? - .is_some()) + { + return Ok(Some(WorkspaceEnvironmentAdapterProposal::ready())); + } + if cargo_resolution_snapshot(db, source_root, &root)?.is_some() { + return Ok(Some(WorkspaceEnvironmentAdapterProposal::ready())); + } + Ok(Some(WorkspaceEnvironmentAdapterProposal::resolvable( + EnvironmentProposalReasonReport { + code: "resolution_snapshot_missing".to_string(), + message: "Cargo.toml is present but no Cargo.lock or Trail-managed resolution snapshot is available".to_string(), + }, + EnvironmentRecoveryActionReport { + code: "resolve_component".to_string(), + description: "Resolve and pin a Trail-managed Cargo.lock snapshot without adding it to source".to_string(), + command: None, + }, + ))) + } + + fn resolution_plan( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result> { + let root = normalize_component_root(component_root)?; + let manifest_path = join_repo_path(&root, "Cargo.toml"); + let Some(manifest) = db.root_file_entry(source_root, &manifest_path)? else { + return Ok(None); + }; + let cargo_tool = resolve_workspace_tool_executable("cargo")?; + let proposal_key = cargo_resolution_proposal_key(source_root, &root); + let policy_identity = sha256_hex( + format!( + "cargo-lock-resolver-v1\0{}\0offline\0scripts-denied", + cargo_tool.identity + ) + .as_bytes(), + ); + Ok(Some(ArtifactResolutionPlanV1 { + version: ARTIFACT_RESOLUTION_PLAN_VERSION, + proposal_key, + source_root: source_root.clone(), + component_id: self.component_id(&root)?, + adapter_identity: self.identity().to_string(), + policy_identity, + program: "cargo".to_string(), + resolved_program: cargo_tool.path.to_string_lossy().into_owned(), + executable_identity: cargo_tool.identity, + argv: vec![ + "cargo".to_string(), + "generate-lockfile".to_string(), + "--offline".to_string(), + ], + working_directory: if root.is_empty() { + ".".to_string() + } else { + root.clone() + }, + readable_inputs: vec![ArtifactResolutionInputV1 { + source_path: manifest_path, + content_hash: manifest.content_hash, + size_bytes: manifest.size_bytes, + }], + candidate_output: join_repo_path(&root, "Cargo.lock"), + allowed_authorities: Vec::new(), + credential_handles: Vec::new(), + script_policy: ArtifactScriptPolicyV1::Deny, + environment_roles: BTreeMap::new(), + limits: ArtifactActionLimitsV1 { + timeout_ms: 5 * 60 * 1_000, + stdout_bytes: 1024 * 1024, + stderr_bytes: 1024 * 1024, + candidate_bytes: 16 * 1024 * 1024, + candidate_entries: 1, + child_processes: 32, + }, + snapshot_format: "cargo-lock-toml-v1".to_string(), + validations: vec![ArtifactValidationV1 { + name: "cargo-lock-structure".to_string(), + kind: ArtifactValidationKindV1::Framework, + required: true, + parameters: BTreeMap::from([("format".to_string(), "toml".to_string())]), + }], + })) } fn plan( @@ -70,12 +173,52 @@ impl WorkspaceEnvironmentAdapter for CargoTargetSeedAdapter { display_component_root(&component_root) )) })?; - let lock = db.root_file_entry(source_root, &lock_path)?.ok_or_else(|| { - Error::InvalidInput(format!( - "Cargo component `{}` has no Cargo.lock; generate and record a lockfile before synchronizing a target seed", - display_component_root(&component_root) - )) - })?; + let source_lock = db.root_file_entry(source_root, &lock_path)?; + let component_id = self.component_id(&component_root)?; + let (lock_content_hash, lock_authority, resolution_inputs) = if let Some(lock) = source_lock + { + (lock.content_hash, "source".to_string(), Vec::new()) + } else { + let resolution_plan = self + .resolution_plan(db, source_root, &component_root)? + .ok_or_else(|| { + Error::Corrupt(format!( + "Cargo component `{}` lost its resolver plan", + display_component_root(&component_root) + )) + })?; + let (snapshot_id, snapshot, bytes) = + cargo_resolution_snapshot(db, source_root, &component_root)?.ok_or_else(|| { + Error::InvalidInput(format!( + "Cargo component `{}` has no Cargo.lock or Trail-managed resolution snapshot; resolve it through Trail's artifact resolution operation before synchronizing", + display_component_root(&component_root) + )) + })?; + if snapshot.resolver_executable_identity != resolution_plan.executable_identity + || snapshot.policy_identity != resolution_plan.policy_identity + { + return Err(Error::InvalidInput(format!( + "Cargo component `{}` resolution snapshot was created by a different Cargo executable or resolver policy; resolve it again for the current toolchain", + display_component_root(&component_root) + ))); + } + validate_cargo_lock_snapshot(&bytes)?; + let size_bytes = u64::try_from(bytes.len()).map_err(|_| { + Error::InvalidInput("Cargo lock snapshot exceeds platform limits".to_string()) + })?; + ( + snapshot.content_sha256.clone(), + format!("snapshot:{}", snapshot_id.0), + vec![WorkspaceEnvironmentResolutionInput { + snapshot_id, + source_root: source_root.clone(), + source_path: lock_path.clone(), + staging_path: format!("project/{lock_path}"), + content_hash: snapshot.content_sha256, + size_bytes, + }], + ) + }; let cargo_version = command_identity("cargo", &["--version"])?; let rustc_identity = command_identity("rustc", &["-vV"])?; let cargo_tool = resolve_workspace_tool_executable("cargo")?; @@ -161,6 +304,13 @@ impl WorkspaceEnvironmentAdapter for CargoTargetSeedAdapter { "SCCACHE_DIR".to_string(), sccache_cache.storage_path.to_string_lossy().into_owned(), ); + // A long-lived sccache server can outlive an attempt-owned temp + // directory. Cache I/O loss must degrade to rustc, not fail the + // deterministic target-seed build. + environment.insert( + "SCCACHE_IGNORE_SERVER_IO_ERROR".to_string(), + "1".to_string(), + ); tool_versions.insert("sccache".to_string(), sccache_version); tool_versions.insert("sccache-executable".to_string(), sccache_tool.identity); cache_names.push(sccache_cache.name.clone()); @@ -190,7 +340,7 @@ impl WorkspaceEnvironmentAdapter for CargoTargetSeedAdapter { format!("{component_root}/target") }; Ok(WorkspaceEnvironmentPlan { - component_id: self.component_id(&component_root)?, + component_id, adapter_identity: self.identity().to_string(), adapter_version: 1, implementation_version: env!("CARGO_PKG_VERSION").to_string(), @@ -205,7 +355,8 @@ impl WorkspaceEnvironmentAdapter for CargoTargetSeedAdapter { inputs: BTreeMap::from([ ("source_root".to_string(), source_root.0.clone()), (manifest_path, manifest.content_hash), - (lock_path, lock.content_hash), + (lock_path, lock_content_hash), + ("lock_authority".to_string(), lock_authority), ( "output_contract".to_string(), format!("immutable-seed-private:{mount_path}"), @@ -237,6 +388,7 @@ impl WorkspaceEnvironmentAdapter for CargoTargetSeedAdapter { ), }, inputs: Vec::new(), + resolution_inputs, source_projection: Some((source_root.clone(), "project".to_string())), pre_commands: vec![WorkspaceEnvironmentCommand { program: "cargo".to_string(), @@ -287,6 +439,56 @@ impl WorkspaceEnvironmentAdapter for CargoTargetSeedAdapter { } } +fn cargo_resolution_proposal_key(source_root: &ObjectId, component_root: &str) -> String { + let identity = format!( + "cargo-lock-resolution-v1\0{}\0{}\0{}", + source_root.0, component_root, CARGO_TARGET_SEED_ADAPTER_METADATA.canonical_identity + ); + format!("cargo_lock_v1_{}", sha256_hex(identity.as_bytes())) +} + +fn cargo_resolution_snapshot( + db: &Trail, + source_root: &ObjectId, + component_root: &str, +) -> Result)>> { + let proposal_key = cargo_resolution_proposal_key(source_root, component_root); + let expected_component = CARGO_TARGET_SEED_ADAPTER.component_id(component_root)?; + db.verified_workspace_environment_resolution_snapshot( + &proposal_key, + source_root, + &expected_component, + CARGO_TARGET_SEED_ADAPTER.identity(), + "cargo-lock-toml-v1", + validate_cargo_lock_snapshot, + ) +} + +fn validate_cargo_lock_snapshot(bytes: &[u8]) -> Result<()> { + let text = std::str::from_utf8(bytes).map_err(|_| { + Error::InvalidInput("Trail-managed Cargo.lock snapshot is not UTF-8".to_string()) + })?; + let document = toml::from_str::(text).map_err(|error| { + Error::InvalidInput(format!( + "Trail-managed Cargo.lock snapshot is malformed TOML: {error}" + )) + })?; + let version = document + .get("version") + .and_then(toml::Value::as_integer) + .ok_or_else(|| { + Error::InvalidInput( + "Trail-managed Cargo.lock snapshot has no integer lockfile version".to_string(), + ) + })?; + if !(1..=4).contains(&version) { + return Err(Error::InvalidInput(format!( + "Trail-managed Cargo.lock snapshot version {version} is unsupported" + ))); + } + Ok(()) +} + fn normalize_component_root(component_root: &str) -> Result { if component_root.trim_matches('/').is_empty() { Ok(String::new()) @@ -336,6 +538,171 @@ fn command_is_available(program: &str) -> bool { mod tests { use super::*; + fn native_cow_mode() -> LaneWorkdirMode { + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + } + } + + #[test] + fn manifest_only_cargo_uses_managed_lock_for_locked_offline_shared_seed() { + if command_identity("cargo", &["--version"]).is_err() + || command_identity("rustc", &["-vV"]).is_err() + { + return; + } + let workspace = tempfile::tempdir().unwrap(); + fs::create_dir_all(workspace.path().join("src")).unwrap(); + fs::write( + workspace.path().join("Cargo.toml"), + "[package]\nname = \"trail-managed-lock-test\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + ) + .unwrap(); + fs::write( + workspace.path().join("src/lib.rs"), + "pub fn managed() -> bool { true }\n", + ) + .unwrap(); + + let resolver = tempfile::tempdir().unwrap(); + fs::create_dir_all(resolver.path().join("src")).unwrap(); + fs::copy( + workspace.path().join("Cargo.toml"), + resolver.path().join("Cargo.toml"), + ) + .unwrap(); + fs::copy( + workspace.path().join("src/lib.rs"), + resolver.path().join("src/lib.rs"), + ) + .unwrap(); + let generated = Command::new("cargo") + .args(["generate-lockfile", "--offline"]) + .current_dir(resolver.path()) + .output() + .unwrap(); + assert!( + generated.status.success(), + "cargo generate-lockfile failed: {}", + String::from_utf8_lossy(&generated.stderr) + ); + let lock_bytes = fs::read(resolver.path().join("Cargo.lock")).unwrap(); + + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let source_root = db.get_ref("refs/branches/main").unwrap().root_id; + assert!(CARGO_TARGET_SEED_ADAPTER + .detect(&db, &source_root, "") + .unwrap()); + let proposal = CARGO_TARGET_SEED_ADAPTER + .propose(&db, &source_root, "") + .unwrap() + .unwrap(); + assert_eq!( + proposal.status, + EnvironmentComponentProposalStatus::Resolvable + ); + let resolution_plan = CARGO_TARGET_SEED_ADAPTER + .resolution_plan(&db, &source_root, "") + .unwrap() + .unwrap(); + let resolved = db + .resolve_artifact_component( + ArtifactResolutionRequestV1 { + plan: resolution_plan, + candidate: ArtifactResolutionCandidateV1 { + snapshot_bytes: lock_bytes, + resolved_identities: BTreeMap::new(), + checksums: BTreeMap::new(), + contacted_authorities: Vec::new(), + stdout: Vec::new(), + stderr: Vec::new(), + redactions: Vec::new(), + }, + }, + false, + ) + .unwrap(); + assert_eq!(resolved.decision, ArtifactResolutionDecisionV1::Resolved); + assert_eq!( + CARGO_TARGET_SEED_ADAPTER + .propose(&db, &source_root, "") + .unwrap() + .unwrap() + .status, + EnvironmentComponentProposalStatus::Ready + ); + + for lane in ["managed-lock-one", "managed-lock-two"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + native_cow_mode(), + None, + None, + None, + &[], + false, + ) + .unwrap(); + } + let first = db + .sync_workspace_environment("managed-lock-one", "cargo", None) + .unwrap(); + let second = db + .sync_workspace_environment("managed-lock-two", "cargo", None) + .unwrap(); + assert_eq!(first.layer_id, second.layer_id); + assert_eq!(first.cache_key, second.cache_key); + assert!(Path::new(&first.storage_path).join("debug").is_dir()); + assert!(!workspace.path().join("Cargo.lock").exists()); + assert!(db + .root_file_entry(&source_root, "Cargo.lock") + .unwrap() + .is_none()); + + fs::write( + workspace.path().join("README.md"), + "unrelated source change\n", + ) + .unwrap(); + db.record( + Some("main"), + Some("change source root".to_string()), + Actor::human(), + false, + ) + .unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "managed-lock-new-root", + Some("main"), + native_cow_mode(), + None, + None, + None, + &[], + false, + ) + .unwrap(); + let discovery = db + .discover_workspace_environment("managed-lock-new-root", None) + .unwrap(); + assert_eq!( + discovery.components[0].status, + EnvironmentComponentProposalStatus::Resolvable + ); + let error = db + .sync_workspace_environment("managed-lock-new-root", "cargo", None) + .unwrap_err(); + assert!(error + .to_string() + .contains("Trail-managed resolution snapshot")); + } + #[test] fn cargo_adapter_builds_once_and_reuses_one_immutable_target_seed() { if command_identity("cargo", &["--version"]).is_err() @@ -368,18 +735,11 @@ mod tests { Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(workspace.path()).unwrap(); - let mode = if cfg!(target_os = "macos") { - LaneWorkdirMode::NfsCow - } else if cfg!(target_os = "windows") { - LaneWorkdirMode::DokanCow - } else { - LaneWorkdirMode::FuseCow - }; for lane in ["cargo-one", "cargo-two"] { db.spawn_lane_with_workdir_mode_paths_and_neighbors( lane, Some("main"), - mode.clone(), + native_cow_mode(), None, None, None, diff --git a/trail/src/db/lane/workspace_cmake.rs b/trail/src/db/lane/workspace_cmake.rs index d9321fcb..6dc67c34 100644 --- a/trail/src/db/lane/workspace_cmake.rs +++ b/trail/src/db/lane/workspace_cmake.rs @@ -109,6 +109,7 @@ impl WorkspaceEnvironmentAdapter for CmakeBuildTreeAdapter { strategy: "cmake-build-tree-private-v1".to_string(), }, inputs: Vec::new(), + resolution_inputs: Vec::new(), source_projection: None, pre_commands: Vec::new(), command: None, @@ -227,6 +228,24 @@ mod tests { EnvironmentOutputPolicy::WritablePrivate ); assert!(plan.tools.contains_key("cmake-executable")); + let source_root = db.resolve_branch_ref("main").unwrap().root_id; + let raw_plan = CMAKE_BUILD_TREE_ADAPTER + .plan(&db, &source_root, "") + .unwrap(); + let identity = super::workspace_environment::workspace_environment_identity_contract_v3( + &raw_plan, + super::workspace_environment::workspace_environment_artifact_contract_digest(&raw_plan) + .unwrap(), + ) + .unwrap(); + assert!(!identity.source_closure_complete); + assert!(!identity.portability_certified); + assert_eq!(identity.trust_scope, "builtin"); + assert_eq!( + raw_plan.outputs[0].policy, + WorkspaceEnvironmentOutputPolicy::WritablePrivate + ); + assert!(raw_plan.caches.is_empty()); let report = db .sync_workspace_environment_component("cmake", "trail/cmake-build@1", None, None) .unwrap(); diff --git a/trail/src/db/lane/workspace_environment.rs b/trail/src/db/lane/workspace_environment.rs index 935c9601..faad4591 100644 --- a/trail/src/db/lane/workspace_environment.rs +++ b/trail/src/db/lane/workspace_environment.rs @@ -4,9 +4,51 @@ use super::*; #[cfg(any(target_os = "linux", target_os = "macos"))] use crate::db::change_ledger::secure_fs::SecureDirectory; use std::ffi::OsString; +use std::io::Read; use std::process::Stdio; use std::thread; +const MAX_ENVIRONMENT_PROPOSALS: usize = 100_000; +const MAX_ENVIRONMENT_PROPOSAL_REASONS: usize = 16; +const MAX_ENVIRONMENT_RECOVERY_ACTIONS: usize = 16; +const MAX_ENVIRONMENT_PROPOSAL_TEXT_BYTES: usize = 4 * 1024; +const MAX_ENVIRONMENT_RECOVERY_COMMAND_ARGS: usize = 64; +const PARALLEL_ENVIRONMENT_SQLITE_WAIT_SECS: u64 = 30; +const MAX_RESOLVER_SOURCE_ENTRIES: u64 = 1_000_000; +const MAX_RESOLVER_SOURCE_BYTES: u64 = 16 * 1024 * 1024 * 1024; +const MAX_ENVIRONMENT_COMMAND_DIAGNOSTIC_BYTES: usize = 64 * 1024; + +struct BoundedResolverPipe { + bytes: Vec, + original_bytes: u64, +} + +struct BoundedEnvironmentCommandDiagnostic { + bytes: Vec, + truncated: bool, +} + +fn spawn_bounded_environment_command_diagnostic( + mut reader: impl Read + Send + 'static, +) -> thread::JoinHandle> { + thread::spawn(move || { + let mut bytes = Vec::with_capacity(MAX_ENVIRONMENT_COMMAND_DIAGNOSTIC_BYTES); + let mut truncated = false; + let mut buffer = [0_u8; 16 * 1024]; + loop { + let read = reader.read(&mut buffer)?; + if read == 0 { + break; + } + let remaining = MAX_ENVIRONMENT_COMMAND_DIAGNOSTIC_BYTES.saturating_sub(bytes.len()); + let retained = remaining.min(read); + bytes.extend_from_slice(&buffer[..retained]); + truncated |= retained != read; + } + Ok(BoundedEnvironmentCommandDiagnostic { bytes, truncated }) + }) +} + /// One repository file that the host projects into an adapter-owned staging /// directory. Adapters describe the mapping; they never receive writable /// access to the lane source view. @@ -17,6 +59,20 @@ pub(crate) struct WorkspaceEnvironmentInput { pub(crate) entry: FileEntry, } +/// One verified resolver snapshot projected into adapter-owned staging. The +/// bytes remain environment metadata in Trail's object store rather than +/// becoming mergeable source merely because a framework requires a lockfile +/// at a conventional repository path while it runs. +#[derive(Clone, Debug)] +pub(crate) struct WorkspaceEnvironmentResolutionInput { + pub(crate) snapshot_id: ObjectId, + pub(crate) source_root: ObjectId, + pub(crate) source_path: String, + pub(crate) staging_path: String, + pub(crate) content_hash: String, + pub(crate) size_bytes: u64, +} + /// A command plan is deliberately argv-based. Trail owns the working /// directory, environment injection, staging tree, and publication boundary. #[derive(Clone, Debug)] @@ -334,6 +390,40 @@ pub(crate) enum WorkspaceEnvironmentSandboxPolicy { RestrictedPluginMounted, } +fn workspace_environment_capability_ceiling( + producer_trust: ArtifactProducerTrustTierV1, + phase: ArtifactExecutionPhaseV1, +) -> ArtifactCapabilityCeilingV1 { + ArtifactCapabilityCeilingV1::for_phase(producer_trust, phase) +} + +fn environment_plugin_producer_trust_tier( + plugin: &super::workspace_plugin::InstalledEnvironmentPlugin, +) -> Result { + match ( + plugin.publisher.as_deref(), + plugin.publisher_key_id.as_deref(), + plugin.trust.as_str(), + ) { + (Some(_), Some(_), "publisher_signed") + if plugin.certification_tier.starts_with("certified-") => + { + Ok(ArtifactProducerTrustTierV1::CertifiedSignedPlugin) + } + (Some(_), Some(_), "publisher_signed") | (None, None, "local_unsigned") => { + // Publisher authentication proves package origin but does not by + // itself certify the adapter's behavior. Until conformance + // certification is durable, both forms retain the local-plugin + // execution ceiling. + Ok(ArtifactProducerTrustTierV1::LocallyTrustedPlugin) + } + _ => Err(Error::Corrupt(format!( + "installed adapter `{}` has inconsistent producer trust evidence", + plugin.manifest.adapter.canonical_identity + ))), + } +} + /// Normalized plan emitted by every built-in environment adapter. /// /// The layer key and output contract are data so the host can validate and @@ -354,6 +444,9 @@ pub(crate) struct WorkspaceEnvironmentPlan { pub(crate) resolved_dependencies: Vec, pub(crate) layer_key: WorkspaceLayerKeyV1, pub(crate) inputs: Vec, + /// Verified resolution metadata projected after the immutable source root. + /// Host validation forbids these entries from replacing tracked source. + pub(crate) resolution_inputs: Vec, /// Optional complete pinned source projection for adapters such as Cargo /// whose build graph can include arbitrary workspace source files. The /// host streams this root in bounded chunks during explicit sync. @@ -416,6 +509,48 @@ pub(crate) struct WorkspaceEnvironmentAdapterMetadata { pub(crate) description: &'static str, } +/// Side-effect-free adapter result for a recognized component root. Planning +/// and resolution stay separate so discovery can report incomplete framework +/// state without invoking ecosystem tooling. +#[derive(Clone, Debug)] +pub(crate) struct WorkspaceEnvironmentAdapterProposal { + pub(crate) status: EnvironmentComponentProposalStatus, + pub(crate) reasons: Vec, + pub(crate) recovery_actions: Vec, +} + +impl WorkspaceEnvironmentAdapterProposal { + pub(crate) fn ready() -> Self { + Self { + status: EnvironmentComponentProposalStatus::Ready, + reasons: Vec::new(), + recovery_actions: Vec::new(), + } + } + + pub(crate) fn blocked( + reason: EnvironmentProposalReasonReport, + recovery_action: EnvironmentRecoveryActionReport, + ) -> Self { + Self { + status: EnvironmentComponentProposalStatus::Blocked, + reasons: vec![reason], + recovery_actions: vec![recovery_action], + } + } + + pub(crate) fn resolvable( + reason: EnvironmentProposalReasonReport, + recovery_action: EnvironmentRecoveryActionReport, + ) -> Self { + Self { + status: EnvironmentComponentProposalStatus::Resolvable, + reasons: vec![reason], + recovery_actions: vec![recovery_action], + } + } +} + /// Ecosystem code is restricted to discovery and deterministic planning. /// Trail remains responsible for executing, validating, publishing, binding, /// persisting, and reporting the resulting environment. @@ -445,6 +580,26 @@ pub(crate) trait WorkspaceEnvironmentAdapter: Sync { fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result; + fn resolution_plan( + &self, + _db: &Trail, + _source_root: &ObjectId, + _component_root: &str, + ) -> Result> { + Ok(None) + } + + fn propose( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result> { + Ok(self + .detect(db, source_root, component_root)? + .then(WorkspaceEnvironmentAdapterProposal::ready)) + } + fn plan( &self, db: &Trail, @@ -513,6 +668,7 @@ impl Trail { .map(|marker| (*marker).to_string()) .collect(), protocols: Vec::new(), + protocol_capabilities: EnvironmentPluginProtocolCapabilitiesReport::default(), supported_operating_systems: metadata .supported_operating_systems .iter() @@ -550,6 +706,8 @@ impl Trail { .collect::>(); for plugin in self.installed_environment_plugins()? { let metadata = plugin.manifest.adapter; + let protocol_capabilities = + super::workspace_plugin::environment_plugin_protocol_capabilities(&metadata); let (namespace, name, contract_major) = super::workspace_plugin::validate_plugin_identity(&metadata.canonical_identity)?; for selector in &metadata.selectors { @@ -576,6 +734,7 @@ impl Trail { layer_adapter_name: metadata.layer_adapter_name, discovery_markers: metadata.discovery_markers, protocols: metadata.protocols, + protocol_capabilities, supported_operating_systems: metadata.supported_operating_systems, supported_architectures: metadata.supported_architectures, source: "plugin".to_string(), @@ -787,7 +946,7 @@ impl Trail { ) -> Result { let branch = self.lane_branch(lane)?; let head = self.get_ref(&branch.ref_name)?; - let mut roots = BTreeSet::new(); + let mut roots = BTreeMap::>::new(); let plugins = self.installed_environment_plugins()?; if !plugins.is_empty() { // This also rejects selectors that collide with built-ins before @@ -805,11 +964,26 @@ impl Trail { .flat_map(|plugin| plugin.manifest.adapter.discovery_markers.iter().cloned()), ); if let Some(component_root) = component_root { - roots.insert(if component_root.trim_matches('/').is_empty() { + let root = if component_root.trim_matches('/').is_empty() { String::new() } else { normalize_relative_path(component_root)? - }); + }; + let mut present = BTreeSet::new(); + for marker in &discovery_markers { + if marker.contains('/') { + continue; + } + let marker_path = if root.is_empty() { + marker.clone() + } else { + format!("{root}/{marker}") + }; + if self.root_file_entry(&head.root_id, &marker_path)?.is_some() { + present.insert(marker.clone()); + } + } + roots.insert(root, present); } else { self.for_each_root_file_chunk(&head.root_id, 1024, |chunk| { for (path, _) in chunk { @@ -817,11 +991,14 @@ impl Trail { if !discovery_markers.contains(file_name) { continue; } - roots.insert( - path.rsplit_once('/') - .map(|(parent, _)| parent.to_string()) - .unwrap_or_default(), - ); + roots + .entry( + path.rsplit_once('/') + .map(|(parent, _)| parent.to_string()) + .unwrap_or_default(), + ) + .or_default() + .insert(file_name.to_string()); } Ok(()) })?; @@ -829,19 +1006,44 @@ impl Trail { let mut components = Vec::new(); let mut conflicts = Vec::new(); - for root in roots { + for (root, present_markers) in roots { for adapter in builtin_environment_adapters() { - if adapter.detect(self, &head.root_id, &root)? { + if !adapter + .metadata() + .discovery_markers + .iter() + .any(|marker| present_markers.contains(*marker)) + { + continue; + } + if let Some(proposal) = adapter.propose(self, &head.root_id, &root)? { components.push(EnvironmentDiscoveredComponentReport { component_id: adapter.component_id(&root)?, component_root: root.clone(), kind: adapter.kind().to_string(), adapter_identity: adapter.identity().to_string(), + status: proposal.status, + reasons: proposal.reasons, + recovery_actions: proposal.recovery_actions, }); } } for plugin in &plugins { + if !plugin + .manifest + .adapter + .discovery_markers + .iter() + .any(|marker| present_markers.contains(marker)) + { + continue; + } if !super::workspace_plugin::environment_plugin_supports_current_host(plugin) { + components.push( + super::workspace_plugin::unsupported_environment_plugin_proposal( + plugin, &root, + )?, + ); continue; } if let Some(component) = @@ -852,6 +1054,12 @@ impl Trail { } } components.extend(self.command_recipe_discovery(&head.root_id, component_root)?); + if components.len() > MAX_ENVIRONMENT_PROPOSALS { + return Err(Error::InvalidInput(format!( + "environment discovery produced {} component proposals; maximum is {MAX_ENVIRONMENT_PROPOSALS}", + components.len() + ))); + } components.sort_by(|left, right| { ( &left.component_root, @@ -864,6 +1072,41 @@ impl Trail { &right.adapter_identity, )) }); + let ambiguous_ids = components + .iter() + .fold(BTreeMap::::new(), |mut counts, component| { + *counts.entry(component.component_id.clone()).or_default() += 1; + counts + }) + .into_iter() + .filter_map(|(component_id, count)| (count > 1).then_some(component_id)) + .collect::>(); + for component in &mut components { + if ambiguous_ids.contains(&component.component_id) { + component.status = EnvironmentComponentProposalStatus::Ambiguous; + component.reasons.push(EnvironmentProposalReasonReport { + code: "component_identity_conflict".to_string(), + message: format!( + "multiple adapters proposed logical component `{}`", + component.component_id + ), + }); + } + for action in &mut component.recovery_actions { + if action.code == "resolve_component" { + action.command = Some(vec![ + "trail".into(), + "env".into(), + "resolve".into(), + "component".into(), + component.component_id.clone(), + "--lane".into(), + lane.to_string(), + ]); + } + } + normalize_environment_component_proposal(component)?; + } for duplicate in components.windows(2) { if duplicate[0].component_id == duplicate[1].component_id { conflicts.push(EnvironmentDiscoveryConflictReport { @@ -886,6 +1129,460 @@ impl Trail { }) } + /// Resolve one discovered component by running its host-validated resolver + /// plan against an isolated projection of the exact lane source root. + pub fn resolve_workspace_environment_component( + &self, + lane: &str, + component_id: &str, + component_root: Option<&str>, + refresh: bool, + ) -> Result { + let discovery = self.discover_workspace_environment(lane, component_root)?; + let matching = discovery + .components + .iter() + .filter(|component| component.component_id == component_id) + .collect::>(); + if matching.len() != 1 + || matching[0].status == EnvironmentComponentProposalStatus::Ambiguous + { + return Err(Error::InvalidInput(format!( + "environment component `{component_id}` is missing or ambiguous; inspect `trail env discover {lane}` and select one component root with `--path`" + ))); + } + let component = matching[0]; + if !matches!( + component.status, + EnvironmentComponentProposalStatus::Resolvable + | EnvironmentComponentProposalStatus::Ready + ) { + return Err(Error::InvalidInput(format!( + "environment component `{component_id}` is `{}` and has no resolver plan; inspect `trail env discover {lane}`", + component.status.as_str() + ))); + } + let plan = if component.adapter_identity + == super::workspace_recipe::COMMAND_RECIPE_ADAPTER_METADATA.canonical_identity + { + self.command_recipe_resolution_plan(&discovery.source_root, component_id)? + } else if let Some(adapter) = builtin_environment_adapters() + .into_iter() + .find(|adapter| adapter.identity() == component.adapter_identity) + { + adapter.resolution_plan(self, &discovery.source_root, &component.component_root)? + } else { + return Err(Error::InvalidInput(format!( + "environment adapter `{}` does not expose a host-executable v3 resolver plan; inspect `trail env discover {lane}`", + component.adapter_identity + ))); + } + .ok_or_else(|| { + Error::InvalidInput(format!( + "environment component `{component_id}` reported resolvable but produced no resolver plan" + )) + })?; + if component.status == EnvironmentComponentProposalStatus::Ready + && self + .artifact_resolution_snapshot_for_proposal(&plan.proposal_key)? + .is_none() + { + return Err(Error::InvalidInput(format!( + "environment component `{component_id}` is ready from source authority and has no Trail-managed resolver snapshot to reuse" + ))); + } + if plan.source_root != discovery.source_root + || plan.component_id != component.component_id + || plan.adapter_identity != component.adapter_identity + { + return Err(Error::Corrupt(format!( + "environment component `{component_id}` resolver plan disagrees with discovery" + ))); + } + let reviewed_builtin = builtin_environment_adapters() + .into_iter() + .any(|adapter| adapter.identity() == component.adapter_identity); + if !reviewed_builtin { + return Err(Error::InvalidInput( + "resolver execution for repository and plugin components requires the restricted native resolver sandbox, which is not yet wired to this operation; the resolver was not launched" + .to_string(), + )); + } + self.resolve_artifact_component_with_executor(plan, refresh, |plan, fence| { + self.execute_reviewed_builtin_resolution_plan(lane, plan, fence) + }) + } + + /// Resolve every currently resolvable component in deterministic discovery + /// order. Ready components need no snapshot; incomplete states fail closed + /// with the discovery command that explains their recovery. + pub fn resolve_all_workspace_environment_components( + &self, + lane: &str, + component_root: Option<&str>, + refresh: bool, + ) -> Result { + let discovery = self.discover_workspace_environment(lane, component_root)?; + if !discovery.conflicts.is_empty() + || discovery.components.iter().any(|component| { + matches!( + component.status, + EnvironmentComponentProposalStatus::Blocked + | EnvironmentComponentProposalStatus::Unsupported + | EnvironmentComponentProposalStatus::Ambiguous + ) + }) + { + return Err(Error::InvalidInput(format!( + "environment resolution is incomplete or ambiguous; inspect `trail env discover {lane}`" + ))); + } + let mut components = Vec::new(); + for component in &discovery.components { + let requires_resolution = component.status + == EnvironmentComponentProposalStatus::Resolvable + || (component.status == EnvironmentComponentProposalStatus::Ready + && self.has_current_artifact_resolution_snapshot( + &discovery.source_root, + &component.component_id, + &component.adapter_identity, + )?); + if !requires_resolution { + continue; + } + components.push(self.resolve_workspace_environment_component( + lane, + &component.component_id, + Some(&component.component_root), + refresh, + )?); + } + Ok(ArtifactResolutionBatchReportV1 { + source_root: discovery.source_root, + refresh_requested: refresh, + components, + }) + } + + fn execute_reviewed_builtin_resolution_plan( + &self, + lane: &str, + plan: &ArtifactResolutionPlanV1, + fence: &super::workspace_artifact::ArtifactResolutionAttemptFence, + ) -> super::workspace_artifact::ArtifactResolutionExecutorResult { + let fail = |code: &str, error: &dyn std::fmt::Display| { + super::workspace_artifact::ArtifactResolutionExecutorFailure::from_error(code, error) + }; + if !plan.allowed_authorities.is_empty() || !plan.credential_handles.is_empty() { + return Err(fail( + "resolver_authority_not_enforceable", + &"reviewed built-in resolution currently permits only offline, credential-free plans", + )); + } + let actual_identity = workspace_tool_identity_for_path(Path::new(&plan.resolved_program)) + .map_err(|error| fail("resolver_tool_identity_failed", &error))?; + if actual_identity != plan.executable_identity { + return Err(fail( + "resolver_tool_identity_changed", + &format!( + "resolver executable identity changed after planning: expected `{}`, found `{actual_identity}`", + plan.executable_identity + ), + )); + } + let branch = self + .lane_branch(lane) + .map_err(|error| fail("resolver_source_pin_failed", &error))?; + if branch.head_root != plan.source_root { + return Err(fail( + "resolver_source_changed", + &"lane source changed after resolver planning", + )); + } + + let staging_parent = self + .workspace_environment_staging_parent() + .map_err(|error| fail("resolver_staging_failed", &error))?; + let staging = tempfile::Builder::new() + .prefix("trail-resolution-") + .tempdir_in(staging_parent) + .map_err(|error| fail("resolver_staging_failed", &error))?; + let project = staging.path().join("project"); + let isolated_home = staging.path().join("home"); + let isolated_tmp = staging.path().join("tmp"); + fs::create_dir_all(&project) + .and_then(|_| fs::create_dir_all(&isolated_home)) + .and_then(|_| fs::create_dir_all(&isolated_tmp)) + .map_err(|error| fail("resolver_staging_failed", &error))?; + + let mut source_entries = 0_u64; + let mut source_bytes = 0_u64; + let mut source_hashes = BTreeMap::new(); + self.for_each_root_file_chunk(&plan.source_root, 1024, |chunk| { + for (path, entry) in chunk { + source_entries = source_entries.saturating_add(1); + source_bytes = source_bytes.saturating_add(entry.size_bytes); + if source_entries > MAX_RESOLVER_SOURCE_ENTRIES + || source_bytes > MAX_RESOLVER_SOURCE_BYTES + { + return Err(Error::InvalidInput(format!( + "resolver source projection exceeds {MAX_RESOLVER_SOURCE_ENTRIES} entries or {MAX_RESOLVER_SOURCE_BYTES} bytes" + ))); + } + self.materialize_workspace_environment_input(&project, &path, &entry)?; + let projected = safe_join(&project, &path)?; + let mut permissions = fs::metadata(&projected)?.permissions(); + permissions.set_readonly(true); + fs::set_permissions(&projected, permissions)?; + source_hashes.insert(path, entry.content_hash); + } + Ok(()) + }) + .map_err(|error| fail("resolver_source_projection_failed", &error))?; + + let candidate_path = safe_join(&project, &plan.candidate_output) + .map_err(|error| fail("resolver_candidate_path_invalid", &error))?; + if candidate_path.exists() { + return Err(fail( + "resolver_candidate_would_replace_source", + &format!( + "resolver candidate `{}` already exists in pinned source", + plan.candidate_output + ), + )); + } + if let Some(parent) = candidate_path.parent() { + fs::create_dir_all(parent) + .map_err(|error| fail("resolver_candidate_parent_failed", &error))?; + } + let working_directory = if plan.working_directory == "." { + project.clone() + } else { + safe_join(&project, &plan.working_directory) + .map_err(|error| fail("resolver_working_directory_invalid", &error))? + }; + if !working_directory.is_dir() { + return Err(fail( + "resolver_working_directory_missing", + &format!( + "resolver working directory `{}` does not exist", + plan.working_directory + ), + )); + } + + let mut command = Command::new(&plan.resolved_program); + command + .args(&plan.argv[1..]) + .current_dir(&working_directory) + .env_clear() + .env("HOME", &isolated_home) + .env("TMPDIR", &isolated_tmp) + .env("TMP", &isolated_tmp) + .env("TEMP", &isolated_tmp) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if let Some(path) = std::env::var_os("PATH") { + command.env("PATH", path); + } + if plan.program == "cargo" { + let rustup_home = std::env::var_os("RUSTUP_HOME").or_else(|| { + std::env::var_os("HOME") + .map(PathBuf::from) + .map(|home| home.join(".rustup").into_os_string()) + }); + if let Some(rustup_home) = rustup_home.filter(|path| Path::new(path).is_dir()) { + command.env("RUSTUP_HOME", rustup_home); + } + } + let mut child = command + .spawn() + .map_err(|error| fail("resolver_process_spawn_failed", &error))?; + let Some(stdout) = child.stdout.take() else { + let _ = child.kill(); + let _ = child.wait(); + return Err(fail( + "resolver_capture_failed", + &"resolver stdout pipe was not created", + )); + }; + let Some(stderr) = child.stderr.take() else { + let _ = child.kill(); + let _ = child.wait(); + return Err(fail( + "resolver_capture_failed", + &"resolver stderr pipe was not created", + )); + }; + let stdout_limit = plan.limits.stdout_bytes; + let stderr_limit = plan.limits.stderr_bytes; + let stdout_reader = thread::spawn(move || read_bounded_resolver_pipe(stdout, stdout_limit)); + let stderr_reader = thread::spawn(move || read_bounded_resolver_pipe(stderr, stderr_limit)); + let started = Instant::now(); + let mut last_heartbeat = started; + let timeout = Duration::from_millis(plan.limits.timeout_ms); + let (status, timed_out, cancelled) = loop { + match child.try_wait() { + Ok(Some(status)) => break (status, false, false), + Ok(None) if started.elapsed() < timeout => { + if last_heartbeat.elapsed() >= Duration::from_secs(1) { + let active = match self.heartbeat_artifact_resolution_attempt(fence) { + Ok(active) => active, + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(fail("resolver_heartbeat_failed", &error)); + } + }; + if !active { + let _ = child.kill(); + let status = child + .wait() + .map_err(|error| fail("resolver_process_wait_failed", &error))?; + break (status, false, true); + } + last_heartbeat = Instant::now(); + } + thread::sleep(Duration::from_millis(25)); + } + Ok(None) => { + let _ = child.kill(); + let status = child + .wait() + .map_err(|error| fail("resolver_process_wait_failed", &error))?; + break (status, true, false); + } + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(fail("resolver_process_wait_failed", &error)); + } + } + }; + let stdout = stdout_reader + .join() + .map_err(|_| { + fail( + "resolver_capture_failed", + &"resolver stdout reader panicked", + ) + })? + .map_err(|error| fail("resolver_capture_failed", &error))?; + let stderr = stderr_reader + .join() + .map_err(|_| { + fail( + "resolver_capture_failed", + &"resolver stderr reader panicked", + ) + })? + .map_err(|error| fail("resolver_capture_failed", &error))?; + if timed_out + || cancelled + || !status.success() + || stdout.original_bytes > stdout_limit + || stderr.original_bytes > stderr_limit + { + return Err(Box::new( + super::workspace_artifact::ArtifactResolutionExecutorFailure { + code: if timed_out { + "resolver_process_timed_out" + } else if cancelled { + "resolver_cancelled" + } else if !status.success() { + "resolver_process_failed" + } else { + "resolver_capture_limit_exceeded" + } + .into(), + message: if timed_out { + format!( + "resolver exceeded its {} ms timeout", + plan.limits.timeout_ms + ) + } else if cancelled { + "resolver was cancelled or lost its owner fence".into() + } else if !status.success() { + format!("resolver exited with status {status}") + } else { + "resolver output exceeded its declared capture limit".into() + }, + contacted_authorities: Vec::new(), + stdout: stdout.bytes, + stderr: stderr.bytes, + stdout_original_bytes: stdout.original_bytes, + stderr_original_bytes: stderr.original_bytes, + redactions: Vec::new(), + cancelled, + }, + )); + } + + let metadata = fs::symlink_metadata(&candidate_path) + .map_err(|error| fail("resolver_candidate_missing", &error))?; + if !metadata.file_type().is_file() || metadata.len() > plan.limits.candidate_bytes { + return Err(fail( + "resolver_candidate_invalid", + &format!( + "resolver candidate `{}` is not a bounded regular file", + plan.candidate_output + ), + )); + } + let snapshot_bytes = fs::read(&candidate_path) + .map_err(|error| fail("resolver_candidate_read_failed", &error))?; + for (path, expected_hash) in &source_hashes { + let projected = safe_join(&project, path) + .map_err(|error| fail("resolver_source_revalidation_failed", &error))?; + let actual_hash = sha256_hex( + &fs::read(&projected) + .map_err(|error| fail("resolver_source_revalidation_failed", &error))?, + ); + if actual_hash != *expected_hash { + return Err(fail( + "resolver_modified_source", + &format!("resolver modified pinned source input `{path}`"), + )); + } + } + for entry in walkdir::WalkDir::new(&project).follow_links(false) { + let entry = entry.map_err(|error| fail("resolver_output_scan_failed", &error))?; + if entry.path() == project || entry.file_type().is_dir() { + continue; + } + let relative = entry + .path() + .strip_prefix(&project) + .map_err(|error| fail("resolver_output_scan_failed", &error))?; + let relative = relative.to_string_lossy().replace('\\', "/"); + if relative != plan.candidate_output && !source_hashes.contains_key(&relative) { + return Err(fail( + "resolver_undeclared_output", + &format!("resolver produced undeclared path `{relative}`"), + )); + } + } + let current = self + .lane_branch(lane) + .map_err(|error| fail("resolver_source_revalidation_failed", &error))?; + if current.head_root != plan.source_root { + return Err(fail( + "resolver_source_changed", + &"lane source changed while the resolver was running", + )); + } + Ok(ArtifactResolutionCandidateV1 { + snapshot_bytes, + resolved_identities: BTreeMap::new(), + checksums: BTreeMap::new(), + contacted_authorities: Vec::new(), + stdout: stdout.bytes, + stderr: stderr.bytes, + redactions: Vec::new(), + }) + } + /// Return the complete desired ordering/invalidation graph without /// executing adapter commands or mutating lane state. pub fn workspace_environment_graph( @@ -1208,6 +1905,16 @@ impl Trail { content_hash: input.entry.content_hash, size_bytes: input.entry.size_bytes, }) + .chain( + plan.resolution_inputs + .into_iter() + .map(|input| EnvironmentPlanInputReport { + source_path: input.source_path, + staging_path: input.staging_path, + content_hash: input.content_hash, + size_bytes: input.size_bytes, + }), + ) .collect(), tools, commands, @@ -1747,8 +2454,12 @@ impl Trail { "building", None, )?; - let prepared = - self.prepare_workspace_environment_artifacts(&view.view_id, &plan, &cache_key); + let prepared = self.prepare_workspace_environment_artifacts( + &view.view_id, + &source_root, + &plan, + &cache_key, + ); let mut prepared = match prepared { Ok(prepared) => prepared, Err(err) => { @@ -1979,9 +2690,11 @@ impl Trail { None, )?; } - let mut prepared = match self - .prepare_workspace_environment_artifacts_parallel(&view.view_id, &planned) - { + let mut prepared = match self.prepare_workspace_environment_artifacts_parallel( + &view.view_id, + &discovery.source_root, + &planned, + ) { Ok(prepared) => prepared, Err(err) => { let reason = format!( @@ -2169,6 +2882,10 @@ impl Trail { plan.component_id ))); } + self.validate_workspace_environment_capability_ceilings( + plan, + ArtifactProducerTrustTierV1::ReviewedBuiltin, + )?; self.validate_workspace_environment_plan_common(plan) } @@ -2205,6 +2922,10 @@ impl Trail { "command recipe `{expected_component_id}` returned an inconsistent identity, policy, or action plan" ))); } + self.validate_workspace_environment_capability_ceilings( + plan, + ArtifactProducerTrustTierV1::RepositoryDeclaration, + )?; self.validate_workspace_environment_plan_common_with_tool_cache(plan, tool_identities) } @@ -2254,9 +2975,120 @@ impl Trail { "plugin component `{expected_component_id}` returned an inconsistent identity, provenance, or policy plan" ))); } + self.validate_workspace_environment_capability_ceilings( + plan, + environment_plugin_producer_trust_tier(plugin)?, + )?; self.validate_workspace_environment_plan_common(plan) } + fn validate_workspace_environment_capability_ceilings( + &self, + plan: &WorkspaceEnvironmentPlan, + producer_trust: ArtifactProducerTrustTierV1, + ) -> Result<()> { + use ArtifactExecutionPhaseV1 as Phase; + use ArtifactFilesystemReadCapabilityV1 as Read; + use ArtifactFilesystemWriteCapabilityV1 as Write; + use ArtifactProcessCapabilityV1 as Process; + use ArtifactProducerTrustTierV1 as Trust; + + let expected_sandbox = match producer_trust { + Trust::ReviewedBuiltin => { + plan.sandbox_policy == WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin + } + Trust::RepositoryDeclaration => { + plan.sandbox_policy == WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + } + Trust::CertifiedSignedPlugin | Trust::LocallyTrustedPlugin => matches!( + plan.sandbox_policy, + WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginStaging + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted + ), + }; + if !expected_sandbox { + return Err(Error::InvalidInput(format!( + "component `{}` requests a sandbox policy above its {:?} producer ceiling", + plan.component_id, producer_trust + ))); + } + + let planning = + workspace_environment_capability_ceiling(producer_trust, Phase::DiscoveryPlanning); + let resolver = workspace_environment_capability_ceiling(producer_trust, Phase::Resolve); + let constructor = + workspace_environment_capability_ceiling(producer_trust, Phase::Construct); + let mounted = + workspace_environment_capability_ceiling(producer_trust, Phase::MountedExecution); + let validator = workspace_environment_capability_ceiling(producer_trust, Phase::Validate); + let exporter = + workspace_environment_capability_ceiling(producer_trust, Phase::SourceExport); + if [ + &planning, + &resolver, + &constructor, + &mounted, + &validator, + &exporter, + ] + .into_iter() + .any(|ceiling| ceiling.publication_authority) + { + return Err(Error::Corrupt( + "artifact capability policy granted publication authority to executable code" + .into(), + )); + } + if !plan.pre_commands.is_empty() + && (resolver.processes == Process::Deny || producer_trust != Trust::ReviewedBuiltin) + { + return Err(Error::InvalidInput(format!( + "component `{}` requests a resolver command above its producer ceiling", + plan.component_id + ))); + } + if plan.command.is_some() && constructor.processes == Process::Deny { + return Err(Error::InvalidInput(format!( + "component `{}` requests constructor process execution above its producer ceiling", + plan.component_id + ))); + } + if !plan.mounted_commands.is_empty() + && (mounted.processes == Process::Deny + || mounted.filesystem_read != Read::LaneView + || mounted.filesystem_write != Write::LaneBindings) + { + return Err(Error::InvalidInput(format!( + "component `{}` requests mounted execution above its producer ceiling", + plan.component_id + ))); + } + if plan.source_projection.is_some() + && constructor.filesystem_read != Read::PinnedSourceClosure + { + return Err(Error::InvalidInput(format!( + "component `{}` requests a complete source projection above its producer ceiling", + plan.component_id + ))); + } + if !plan.caches.is_empty() && constructor.filesystem_write != Write::CandidateAndHostCache { + return Err(Error::InvalidInput(format!( + "component `{}` requests host cache writes above its producer ceiling", + plan.component_id + ))); + } + if producer_trust == Trust::RepositoryDeclaration + && (!plan.external_artifacts.is_empty() || !plan.runtime_resources.is_empty()) + { + return Err(Error::InvalidInput(format!( + "component `{}` requests provider authority above the repository-declaration ceiling", + plan.component_id + ))); + } + Ok(()) + } + fn validate_workspace_environment_plan_common( &self, plan: &WorkspaceEnvironmentPlan, @@ -2265,6 +3097,40 @@ impl Trail { self.validate_workspace_environment_plan_common_with_tool_cache(plan, &mut tool_identities) } + pub(super) fn verified_workspace_environment_resolution_snapshot( + &self, + proposal_key: &str, + source_root: &ObjectId, + component_id: &str, + adapter_identity: &str, + snapshot_format: &str, + validate_content: F, + ) -> Result)>> + where + F: FnOnce(&[u8]) -> Result<()>, + { + let Some((snapshot_id, snapshot)) = + self.artifact_resolution_snapshot_for_proposal(proposal_key)? + else { + return Ok(None); + }; + if snapshot.proposal_key != proposal_key + || snapshot.source_root != *source_root + || snapshot.component_id != component_id + || snapshot.adapter_identity != adapter_identity + || snapshot.snapshot_format != snapshot_format + || snapshot.verification_state != ArtifactResolutionVerificationStateV1::Verified + || !snapshot.secret_taint.is_clear() + { + return Err(Error::Corrupt(format!( + "environment resolution snapshot {snapshot_id} does not match proposal `{proposal_key}`" + ))); + } + let bytes = self.artifact_resolution_snapshot_content(&snapshot)?; + validate_content(&bytes)?; + Ok(Some((snapshot_id, snapshot, bytes))) + } + fn validate_workspace_environment_plan_common_with_tool_cache( &self, plan: &WorkspaceEnvironmentPlan, @@ -2377,10 +3243,16 @@ impl Trail { plan.component_id ))); } - let mut external_names = BTreeSet::new(); + let mut external_names = BTreeMap::new(); for artifact in &plan.external_artifacts { validate_workspace_environment_external_artifact(artifact)?; - if !external_names.insert(&artifact.name) { + if external_names + .insert( + artifact.name.as_str(), + artifact.artifact_type == "oci_image" && artifact.provider == "oci", + ) + .is_some() + { return Err(Error::InvalidInput(format!( "component `{}` repeats external artifact `{}`", plan.component_id, artifact.name @@ -2411,9 +3283,12 @@ impl Trail { plan.component_id, resource.name ))); } - if !external_names.contains(&resource.artifact_name) { + if !external_names + .get(resource.artifact_name.as_str()) + .is_some_and(|is_oci_image| *is_oci_image) + { return Err(Error::InvalidInput(format!( - "component `{}` runtime resource `{}` references missing external artifact `{}`", + "component `{}` runtime resource `{}` references missing or non-OCI external artifact `{}`", plan.component_id, resource.name, resource.artifact_name ))); } @@ -2630,6 +3505,67 @@ impl Trail { ))); } } + for input in &plan.resolution_inputs { + normalize_relative_path(&input.source_path)?; + let staging = normalize_relative_path(&input.staging_path)?; + let Some((projection_root, staging_root)) = &plan.source_projection else { + return Err(Error::InvalidInput(format!( + "component `{}` resolution metadata requires a pinned source projection", + plan.component_id + ))); + }; + let expected_staging = format!("{staging_root}/{}", input.source_path); + if projection_root != &input.source_root || staging != expected_staging { + return Err(Error::InvalidInput(format!( + "component `{}` resolution input `{}` is not bound to its pinned source projection", + plan.component_id, input.source_path + ))); + } + if !plan + .layer_key + .inputs + .values() + .any(|value| value == &input.content_hash) + || !plan + .layer_key + .inputs + .values() + .any(|value| value == &format!("snapshot:{}", input.snapshot_id.0)) + { + return Err(Error::InvalidInput(format!( + "component `{}` resolution input {} is absent from its artifact identity", + plan.component_id, input.snapshot_id + ))); + } + if !staging_paths.insert(staging.clone()) { + return Err(Error::InvalidInput(format!( + "component `{}` maps more than one input to `{staging}`", + plan.component_id + ))); + } + let (snapshot, bytes) = + self.artifact_resolution_snapshot_content_by_id(&input.snapshot_id)?; + if snapshot.source_root != input.source_root + || snapshot.component_id != plan.component_id + || snapshot.adapter_identity != plan.adapter_identity + || snapshot.content_sha256 != input.content_hash + || u64::try_from(bytes.len()).unwrap_or(u64::MAX) != input.size_bytes + { + return Err(Error::InvalidInput(format!( + "component `{}` resolution input {} does not match its pinned plan", + plan.component_id, input.snapshot_id + ))); + } + if self + .root_file_entry(&input.source_root, &input.source_path)? + .is_some() + { + return Err(Error::InvalidInput(format!( + "component `{}` resolution input `{}` would replace tracked source", + plan.component_id, input.source_path + ))); + } + } if let Some((_, staging_root)) = &plan.source_projection { normalize_relative_path(staging_root)?; } @@ -2677,6 +3613,7 @@ impl Trail { fn prepare_workspace_environment_artifacts( &self, view_id: &str, + source_root: &ObjectId, plan: &WorkspaceEnvironmentPlan, component_key: &str, ) -> Result { @@ -2706,10 +3643,11 @@ impl Trail { WorkspaceEnvironmentOutputPolicy::ImmutableShared | WorkspaceEnvironmentOutputPolicy::ImmutableSeedPrivate => { let prior = self.workspace_layer_by_cache_key(component_key)?; - let layer = self - .build_workspace_layer_singleflight(&plan.layer_key, |build_dir| { - self.execute_workspace_environment_plan(plan, build_dir) - })?; + let layer = self.build_workspace_layer_singleflight( + &plan.layer_key, + source_root, + |build_dir| self.execute_workspace_environment_plan(plan, build_dir), + )?; let hit = prior .as_ref() .is_some_and(|candidate| candidate.state == "ready"); @@ -2922,6 +3860,7 @@ impl Trail { fn prepare_workspace_environment_artifacts_parallel( &self, view_id: &str, + source_root: &ObjectId, planned: &[(WorkspaceEnvironmentPlan, String, Option)], ) -> Result> { let concurrency = usize::try_from( @@ -2936,7 +3875,7 @@ impl Trail { return planned .iter() .map(|(plan, key, _)| { - self.prepare_workspace_environment_artifacts(view_id, plan, key) + self.prepare_workspace_environment_artifacts(view_id, source_root, plan, key) }) .collect(); } @@ -2972,6 +3911,7 @@ impl Trail { } let workspace = self.workspace_root().to_path_buf(); let view_id = view_id.to_string(); + let source_root = source_root.clone(); let wave = thread::scope(|scope| { let mut handles = Vec::with_capacity(ready.len()); for (component_id, index) in &ready { @@ -2980,12 +3920,27 @@ impl Trail { let key = planned[*index].1.clone(); let workspace = workspace.clone(); let view_id = view_id.clone(); + let source_root = source_root.clone(); handles.push(( *index, component_id, scope.spawn(move || { let db = Trail::open(&workspace)?; - db.prepare_workspace_environment_artifacts(&view_id, &plan, &key) + // External builds run concurrently, while each + // immutable publication remains serialized by the + // workspace lock. A worker can still meet the + // preceding worker's WAL commit briefly after the + // handoff, so wait within the construction lease + // instead of escaping a transient SQLITE_BUSY. + db.conn.busy_timeout(Duration::from_secs( + PARALLEL_ENVIRONMENT_SQLITE_WAIT_SECS, + ))?; + db.prepare_workspace_environment_artifacts( + &view_id, + &source_root, + &plan, + &key, + ) }), )); } @@ -3237,6 +4192,8 @@ impl Trail { layer_id: layer_id.map(str::to_string), mount_path: output.mount_path.clone(), storage_path, + artifact_tree_id: None, + artifact_subpath: String::new(), kind: plan.kind.clone(), priority: 100, }); @@ -3461,7 +4418,10 @@ impl Trail { .env("HOME", &isolated_home) .env("TMPDIR", &isolated_tmp) .env("TMP", &isolated_tmp) - .env("TEMP", &isolated_tmp); + .env("TEMP", &isolated_tmp) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); if plan.sandbox_policy == WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin { command .env("TRAIL_WORKSPACE", self.workspace_root()) @@ -3783,6 +4743,26 @@ impl Trail { &input.entry, )?; } + for input in &plan.resolution_inputs { + let (snapshot, bytes) = + self.artifact_resolution_snapshot_content_by_id(&input.snapshot_id)?; + if snapshot.source_root != input.source_root + || snapshot.component_id != plan.component_id + || snapshot.adapter_identity != plan.adapter_identity + || snapshot.content_sha256 != input.content_hash + || u64::try_from(bytes.len()).unwrap_or(u64::MAX) != input.size_bytes + { + return Err(Error::InvalidInput(format!( + "component `{}` resolution snapshot {} changed after planning", + plan.component_id, input.snapshot_id + ))); + } + self.materialize_workspace_environment_resolution_input( + build_dir, + &input.staging_path, + &bytes, + )?; + } let mut outputs = Vec::with_capacity(plan.outputs.len()); for declaration in &plan.outputs { let output = safe_join(build_dir, &declaration.output_path)?; @@ -4037,7 +5017,10 @@ impl Trail { .env("HOME", &isolated_home) .env("TMPDIR", &isolated_tmp) .env("TMP", &isolated_tmp) - .env("TEMP", &isolated_tmp); + .env("TEMP", &isolated_tmp) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()); if let Some(path) = std::env::var_os("PATH") { command.env("PATH", path); } @@ -4050,16 +5033,45 @@ impl Trail { for name in &command_plan.remove_environment { command.env_remove(name); } - let status = command.status().map_err(|err| { + let mut child = command.spawn().map_err(|err| { Error::InvalidInput(format!( "failed to launch `{}` for component `{}`: {err}", command_plan.program, plan.component_id )) })?; + let stderr = child.stderr.take().ok_or_else(|| { + Error::Corrupt(format!( + "environment build for component `{}` lost its diagnostic pipe", + plan.component_id + )) + })?; + let diagnostic = spawn_bounded_environment_command_diagnostic(stderr); + let status = child.wait(); + let diagnostic = diagnostic + .join() + .map_err(|_| { + Error::Corrupt(format!( + "environment build diagnostic reader for component `{}` panicked", + plan.component_id + )) + })? + .map_err(Error::Io)?; + let status = status?; if !status.success() { + let truncated = diagnostic.truncated; + let diagnostic_text = + redact_sensitive_text(&String::from_utf8_lossy(&diagnostic.bytes)); + let diagnostic_text = diagnostic_text.trim(); + let details = if diagnostic_text.is_empty() { + String::new() + } else if truncated { + format!(": {diagnostic_text} [truncated]") + } else { + format!(": {diagnostic_text}") + }; return Err(Error::InvalidInput(format!( - "environment build for component `{}` failed with {status}", - plan.component_id + "environment build for component `{}` failed with {status}{details}", + plan.component_id, ))); } Ok(()) @@ -4355,6 +5367,37 @@ impl Trail { Ok(()) } + fn materialize_workspace_environment_resolution_input( + &self, + build_dir: &Path, + staging_path: &str, + bytes: &[u8], + ) -> Result<()> { + let destination = safe_join(build_dir, staging_path)?; + if destination.exists() { + return Err(Error::InvalidInput(format!( + "resolution input `{staging_path}` would replace projected source" + ))); + } + if let Some(parent) = destination.parent() { + fs::create_dir_all(parent)?; + } + let mut output = OpenOptions::new() + .write(true) + .create_new(true) + .open(&destination)?; + output.write_all(bytes)?; + output.sync_all()?; + let mut permissions = output.metadata()?.permissions(); + #[cfg(not(unix))] + permissions.set_readonly(false); + #[cfg(unix)] + permissions.set_mode(0o644); + fs::set_permissions(&destination, permissions)?; + output.set_modified(SystemTime::UNIX_EPOCH)?; + Ok(()) + } + pub fn workspace_environment_status( &self, lane: &str, @@ -5073,6 +6116,105 @@ impl Trail { } } +fn normalize_environment_component_proposal( + component: &mut EnvironmentDiscoveredComponentReport, +) -> Result<()> { + if component.reasons.len() > MAX_ENVIRONMENT_PROPOSAL_REASONS { + return Err(Error::InvalidInput(format!( + "environment component `{}` has {} proposal reasons; maximum is {MAX_ENVIRONMENT_PROPOSAL_REASONS}", + component.component_id, + component.reasons.len() + ))); + } + if component.recovery_actions.len() > MAX_ENVIRONMENT_RECOVERY_ACTIONS { + return Err(Error::InvalidInput(format!( + "environment component `{}` has {} recovery actions; maximum is {MAX_ENVIRONMENT_RECOVERY_ACTIONS}", + component.component_id, + component.recovery_actions.len() + ))); + } + for reason in &component.reasons { + validate_environment_proposal_token(&reason.code, "reason code")?; + validate_environment_proposal_text(&reason.message, "reason message")?; + } + for action in &component.recovery_actions { + validate_environment_proposal_token(&action.code, "recovery action code")?; + validate_environment_proposal_text(&action.description, "recovery action description")?; + if let Some(command) = &action.command { + if command.is_empty() || command.len() > MAX_ENVIRONMENT_RECOVERY_COMMAND_ARGS { + return Err(Error::InvalidInput(format!( + "environment recovery action `{}` command must contain between 1 and {MAX_ENVIRONMENT_RECOVERY_COMMAND_ARGS} arguments", + action.code + ))); + } + for argument in command { + validate_environment_proposal_text(argument, "recovery command argument")?; + } + } + } + component + .reasons + .sort_by(|left, right| (&left.code, &left.message).cmp(&(&right.code, &right.message))); + component.recovery_actions.sort_by(|left, right| { + (&left.code, &left.description, &left.command).cmp(&( + &right.code, + &right.description, + &right.command, + )) + }); + Ok(()) +} + +fn validate_environment_proposal_token(value: &str, field: &str) -> Result<()> { + if value.is_empty() + || value.len() > 128 + || !value + .chars() + .all(|character| character.is_ascii_alphanumeric() || "._-".contains(character)) + { + return Err(Error::InvalidInput(format!( + "environment proposal {field} `{value}` is invalid" + ))); + } + Ok(()) +} + +fn validate_environment_proposal_text(value: &str, field: &str) -> Result<()> { + if value.is_empty() + || value.len() > MAX_ENVIRONMENT_PROPOSAL_TEXT_BYTES + || value.chars().any(|character| character == '\0') + { + return Err(Error::InvalidInput(format!( + "environment proposal {field} is empty, oversized, or contains NUL" + ))); + } + Ok(()) +} + +fn read_bounded_resolver_pipe( + mut reader: R, + limit: u64, +) -> std::io::Result { + let mut bytes = Vec::with_capacity(usize::try_from(limit.min(64 * 1024)).unwrap_or(0)); + let mut original_bytes = 0_u64; + let mut buffer = [0_u8; 16 * 1024]; + loop { + let read = reader.read(&mut buffer)?; + if read == 0 { + break; + } + original_bytes = original_bytes.saturating_add(read as u64); + let retained = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + let remaining = limit.saturating_sub(retained); + let keep = read.min(usize::try_from(remaining).unwrap_or(usize::MAX)); + bytes.extend_from_slice(&buffer[..keep]); + } + Ok(BoundedResolverPipe { + bytes, + original_bytes, + }) +} + fn run_supervised_mounted_plugin_process( command: &mut Command, ) -> Result { @@ -5238,7 +6380,9 @@ fn split_adapter_identity(identity: &str, fallback_major: u32) -> (String, Strin ) } -fn add_host_canonical_environment_identity(plan: &mut WorkspaceEnvironmentPlan) -> Result<()> { +pub(super) fn workspace_environment_artifact_contract_digest( + plan: &WorkspaceEnvironmentPlan, +) -> Result { #[derive(Serialize)] struct CanonicalOutput<'a> { name: &'a str, @@ -5257,7 +6401,14 @@ fn add_host_canonical_environment_identity(plan: &mut WorkspaceEnvironmentPlan) executable_identity: &'a str, args: &'a [String], working_directory: &'a str, - environment: &'a BTreeMap, + environment: BTreeMap, + } + #[derive(Serialize)] + struct CanonicalCache<'a> { + name: &'a str, + protocol: &'a str, + access: &'a str, + compatibility: &'a BTreeMap, } for command in plan .pre_commands @@ -5293,6 +6444,16 @@ fn add_host_canonical_environment_identity(plan: &mut WorkspaceEnvironmentPlan) gate: output.gate.as_deref(), }) .collect::>(); + let cache_paths = plan + .caches + .iter() + .map(|cache| { + ( + cache.storage_path.to_string_lossy().into_owned(), + format!("trail-cache:{}", cache.name), + ) + }) + .collect::>(); let commands = plan .pre_commands .iter() @@ -5309,27 +6470,161 @@ fn add_host_canonical_environment_identity(plan: &mut WorkspaceEnvironmentPlan) executable_identity: &command.executable_identity, args: &command.args, working_directory: &command.working_directory, - environment: &command.environment, + environment: command + .environment + .iter() + .map(|(name, value)| { + ( + name.clone(), + cache_paths + .get(value) + .cloned() + .unwrap_or_else(|| value.clone()), + ) + }) + .collect(), + }) + .collect::>(); + let caches = plan + .caches + .iter() + .map(|cache| CanonicalCache { + name: &cache.name, + protocol: cache.protocol.as_str(), + access: cache.access.as_str(), + compatibility: &cache.compatibility, }) .collect::>(); + let artifact_contract_digest = sha256_hex(&serde_json::to_vec(&serde_json::json!({ + "contract_version": 1, + "adapter_identity": plan.adapter_identity, + "implementation_version": plan.implementation_version, + "distribution_digest": plan.distribution_digest, + "outputs": outputs, + "commands": commands, + "caches": caches, + "external_artifacts": workspace_external_artifacts_identity(&plan.external_artifacts)?, + "runtime_resources": workspace_runtime_resources_identity(&plan.runtime_resources)?, + "validation": format!("sandbox:{:?}", plan.sandbox_policy), + "platform": plan.layer_key.platform, + "architecture": plan.layer_key.architecture, + "portability_scope": plan.layer_key.portability_scope, + }))?); + Ok(artifact_contract_digest) +} + +fn add_host_canonical_environment_identity(plan: &mut WorkspaceEnvironmentPlan) -> Result<()> { + let artifact_contract_digest = workspace_environment_artifact_contract_digest(plan)?; plan.layer_key.inputs.insert( "host:artifact_contract_v1".to_string(), - sha256_hex(&serde_json::to_vec(&serde_json::json!({ - "contract_version": 1, - "adapter_identity": plan.adapter_identity, - "implementation_version": plan.implementation_version, - "distribution_digest": plan.distribution_digest, - "outputs": outputs, - "commands": commands, - "validation": format!("sandbox:{:?}", plan.sandbox_policy), - "platform": plan.layer_key.platform, - "architecture": plan.layer_key.architecture, - "portability_scope": plan.layer_key.portability_scope, - }))?), + artifact_contract_digest.clone(), + ); + let identity = workspace_environment_identity_contract_v3(plan, artifact_contract_digest)?; + plan.layer_key.inputs.insert( + "host:adapter_identity_protocol".to_string(), + trail_environment_adapter_sdk::PROTOCOL_V3.to_string(), + ); + plan.layer_key.inputs.insert( + "host:adapter_identity_v3".to_string(), + sha256_hex(&serde_json::to_vec(&identity)?), ); Ok(()) } +pub(super) fn workspace_environment_identity_contract_v3( + plan: &WorkspaceEnvironmentPlan, + artifact_contract_digest: String, +) -> Result { + use trail_environment_adapter_sdk::{AdapterIdentityContractV3, AdapterPortability}; + + let mut semantic_identities = BTreeMap::from([( + "host_artifact_contract".to_string(), + artifact_contract_digest, + )]); + for cache in &plan.caches { + let contract = serde_json::json!({ + "name": cache.name, + "protocol": cache.protocol.as_str(), + "access": cache.access.as_str(), + "compatibility": cache.compatibility, + }); + semantic_identities.insert( + format!("performance_cache:{}", cache.name), + sha256_hex(&serde_json::to_vec(&contract)?), + ); + } + if !plan.external_artifacts.is_empty() { + semantic_identities.insert( + "external_artifacts".to_string(), + sha256_hex(workspace_external_artifacts_identity(&plan.external_artifacts)?.as_bytes()), + ); + } + if !plan.runtime_resources.is_empty() { + semantic_identities.insert( + "runtime_resources".to_string(), + sha256_hex(workspace_runtime_resources_identity(&plan.runtime_resources)?.as_bytes()), + ); + } + let snapshot_ids = plan + .resolution_inputs + .iter() + .map(|input| input.snapshot_id.0.as_str()) + .collect::>(); + if !snapshot_ids.is_empty() { + semantic_identities.insert( + "resolution_snapshots".to_string(), + sha256_hex( + snapshot_ids + .into_iter() + .collect::>() + .join("\0") + .as_bytes(), + ), + ); + } + + let portability = if plan + .outputs + .iter() + .any(|output| output.scope == EnvironmentSharingScope::Host) + || plan.layer_key.portability_scope.contains("host") + { + AdapterPortability::Host + } else { + AdapterPortability::Platform + }; + let portability_certified = plan.sandbox_policy + == WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin + && !plan.outputs.is_empty() + && plan.outputs.iter().all(|output| { + matches!( + output.policy, + WorkspaceEnvironmentOutputPolicy::ImmutableShared + | WorkspaceEnvironmentOutputPolicy::ImmutableSeedPrivate + ) + }); + let trust_scope = match plan.sandbox_policy { + WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin => "builtin", + WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe => "repository", + WorkspaceEnvironmentSandboxPolicy::RestrictedPluginStaging + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted => "plugin", + }; + + Ok(AdapterIdentityContractV3 { + normalizer_version: "trail-host-environment-plan/v3".to_string(), + source_closure_complete: plan.source_projection.is_some(), + semantic_identities, + target: plan.layer_key.strategy.clone(), + platform: plan.layer_key.platform.clone(), + architecture: plan.layer_key.architecture.clone(), + abi: "host-default".to_string(), + portability, + portability_certified, + portability_scope: plan.layer_key.portability_scope.clone(), + trust_scope: trust_scope.to_string(), + }) +} + fn parse_canonical_adapter_identity(identity: &str) -> Option<(String, String, u32)> { let (namespace, remainder) = identity.split_once('/')?; let (name, major) = remainder.rsplit_once('@')?; @@ -5910,8 +7205,6 @@ pub(super) fn validate_environment_external_artifact_report( || !artifact.name.chars().all(|character| { character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') }) - || artifact.artifact_type != "oci_image" - || artifact.provider != "oci" || artifact.cleanup_owner != "external" { return Err(Error::InvalidInput(format!( @@ -5935,42 +7228,92 @@ pub(super) fn validate_environment_external_artifact_report( artifact.name ))); } - let (repository, reference_digest) = artifact.reference.rsplit_once('@').ok_or_else(|| { - Error::InvalidInput(format!( - "external OCI artifact `{}` must use a digest-pinned reference", - artifact.name - )) - })?; - if repository.is_empty() - || repository.len() > 2048 - || repository.contains('@') - || repository.starts_with('/') - || repository.ends_with('/') - || !repository - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || b"._-/:".contains(&byte)) - || reference_digest != artifact.digest - { - return Err(Error::InvalidInput(format!( - "external OCI artifact `{}` has an invalid or mismatched reference", - artifact.name - ))); - } - let Some((operating_system, architecture)) = artifact.platform.split_once('/') else { - return Err(Error::InvalidInput(format!( - "external OCI artifact `{}` requires an os/architecture platform", + match artifact.artifact_type.as_str() { + "oci_image" if artifact.provider == "oci" => { + let (repository, reference_digest) = + artifact.reference.rsplit_once('@').ok_or_else(|| { + Error::InvalidInput(format!( + "external OCI artifact `{}` must use a digest-pinned reference", + artifact.name + )) + })?; + if repository.is_empty() + || repository.len() > 2048 + || repository.contains('@') + || repository.starts_with('/') + || repository.ends_with('/') + || !repository + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"._-/:".contains(&byte)) + || reference_digest != artifact.digest + { + return Err(Error::InvalidInput(format!( + "external OCI artifact `{}` has an invalid or mismatched reference", + artifact.name + ))); + } + let Some((operating_system, architecture)) = artifact.platform.split_once('/') else { + return Err(Error::InvalidInput(format!( + "external OCI artifact `{}` requires an os/architecture platform", + artifact.name + ))); + }; + if !matches!(operating_system, "linux" | "windows") + || !matches!(architecture, "amd64" | "arm64") + { + return Err(Error::InvalidInput(format!( + "external OCI artifact `{}` uses unsupported platform `{}`", + artifact.name, artifact.platform + ))); + } + Ok(()) + } + "verified_external" + if valid_external_identity_token(&artifact.provider) + && valid_external_reference(&artifact.reference) + && valid_external_platform(&artifact.platform) => + { + Ok(()) + } + _ => Err(Error::InvalidInput(format!( + "external artifact `{}` has an unsupported type, provider, reference, or platform contract", artifact.name - ))); - }; - if !matches!(operating_system, "linux" | "windows") - || !matches!(architecture, "amd64" | "arm64") - { - return Err(Error::InvalidInput(format!( - "external OCI artifact `{}` uses unsupported platform `{}`", - artifact.name, artifact.platform - ))); + ))), } - Ok(()) +} + +fn valid_external_identity_token(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._-".contains(&byte) + }) +} + +fn valid_external_reference(value: &str) -> bool { + !value.is_empty() + && value.len() <= 2048 + && !value.chars().any(char::is_control) + && !may_contain_sensitive_text(value) + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"._-/:+@=".contains(&byte)) +} + +fn valid_external_platform(value: &str) -> bool { + value == "any" + || (!value.is_empty() + && value.len() <= 128 + && !value.starts_with('/') + && !value.ends_with('/') + && value.split('/').all(|segment| { + !segment.is_empty() + && segment != "." + && segment != ".." + && segment.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._-".contains(&byte) + }) + })) } struct WorkspaceEnvironmentCacheUseGuard { @@ -6270,7 +7613,7 @@ pub(super) fn validate_environment_component_identity(component_id: &str) -> Res Ok(()) } -fn workspace_tool_identity_for_path(path: &Path) -> Result { +pub(crate) fn workspace_tool_identity_for_path(path: &Path) -> Result { let canonical = fs::canonicalize(path)?; if !canonical.is_file() { return Err(Error::InvalidInput(format!( @@ -6465,6 +7808,68 @@ pub(super) fn sandbox_profile_escape(path: &Path) -> String { mod tests { use super::*; + #[test] + fn phase_capability_ceilings_are_total_deny_publication_and_do_not_elevate_signatures() { + use ArtifactExecutionPhaseV1 as Phase; + use ArtifactNetworkCapabilityV1 as Network; + use ArtifactProcessCapabilityV1 as Process; + use ArtifactProducerTrustTierV1 as Trust; + + let tiers = [ + Trust::ReviewedBuiltin, + Trust::CertifiedSignedPlugin, + Trust::LocallyTrustedPlugin, + Trust::RepositoryDeclaration, + ]; + let phases = [ + Phase::DiscoveryPlanning, + Phase::Resolve, + Phase::Construct, + Phase::Validate, + Phase::MountedExecution, + Phase::SourceExport, + ]; + for tier in tiers { + for phase in phases { + let ceiling = workspace_environment_capability_ceiling(tier, phase); + assert_eq!(ceiling.producer_trust, tier); + assert_eq!(ceiling.phase, phase); + assert!(!ceiling.publication_authority); + } + } + assert_eq!( + workspace_environment_capability_ceiling(Trust::ReviewedBuiltin, Phase::Construct,) + .network, + Network::ReviewedBuiltinManaged + ); + for tier in [ + Trust::CertifiedSignedPlugin, + Trust::LocallyTrustedPlugin, + Trust::RepositoryDeclaration, + ] { + assert_eq!( + workspace_environment_capability_ceiling(tier, Phase::Construct).network, + Network::Deny + ); + } + assert_eq!( + workspace_environment_capability_ceiling( + Trust::RepositoryDeclaration, + Phase::MountedExecution, + ) + .processes, + Process::Deny + ); + let signed = workspace_environment_capability_ceiling( + Trust::CertifiedSignedPlugin, + Phase::Construct, + ); + let mut local = + workspace_environment_capability_ceiling(Trust::LocallyTrustedPlugin, Phase::Construct); + local.producer_trust = Trust::CertifiedSignedPlugin; + assert_eq!(signed, local); + } + #[test] fn workspace_environment_staging_parent_is_trail_owned() { let workspace = tempfile::tempdir().unwrap(); @@ -6616,6 +8021,7 @@ mod tests { strategy: "cache-test".to_string(), }, inputs: Vec::new(), + resolution_inputs: Vec::new(), source_projection: None, pre_commands: Vec::new(), command: Some(command.clone()), @@ -6640,6 +8046,108 @@ mod tests { (plan, command) } + #[test] + fn host_v3_identity_projection_is_canonical_and_preserves_private_cache_semantics() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let (mut plan, _) = cache_test_plan(&db, WorkspaceEnvironmentCacheAccess::HostExclusive); + + add_host_canonical_environment_identity(&mut plan).unwrap(); + let first_key = db.workspace_layer_cache_key(&plan.layer_key).unwrap(); + assert_eq!( + plan.layer_key + .inputs + .get("host:adapter_identity_protocol") + .map(String::as_str), + Some(trail_environment_adapter_sdk::PROTOCOL_V3) + ); + assert!(plan + .layer_key + .inputs + .get("host:adapter_identity_v3") + .is_some_and(|digest| digest.len() == 64)); + + add_host_canonical_environment_identity(&mut plan).unwrap(); + assert_eq!( + db.workspace_layer_cache_key(&plan.layer_key).unwrap(), + first_key + ); + let contract_digest = workspace_environment_artifact_contract_digest(&plan).unwrap(); + let identity = workspace_environment_identity_contract_v3(&plan, contract_digest).unwrap(); + assert_eq!(identity.trust_scope, "builtin"); + assert!(!identity.source_closure_complete); + assert!(!identity.portability_certified); + assert!(identity + .semantic_identities + .contains_key("performance_cache:test-cache")); + assert_eq!( + plan.caches[0].access, + WorkspaceEnvironmentCacheAccess::HostExclusive + ); + assert_eq!( + plan.outputs[0].policy, + WorkspaceEnvironmentOutputPolicy::WritablePrivate + ); + } + + #[test] + fn inferred_plan_authority_cannot_exceed_the_selected_producer_tier() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let (mut plan, command) = + cache_test_plan(&db, WorkspaceEnvironmentCacheAccess::HostExclusive); + db.validate_workspace_environment_capability_ceilings( + &plan, + ArtifactProducerTrustTierV1::ReviewedBuiltin, + ) + .unwrap(); + assert!(db + .validate_workspace_environment_capability_ceilings( + &plan, + ArtifactProducerTrustTierV1::RepositoryDeclaration, + ) + .unwrap_err() + .to_string() + .contains("sandbox policy above")); + + plan.sandbox_policy = WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe; + assert!(db + .validate_workspace_environment_capability_ceilings( + &plan, + ArtifactProducerTrustTierV1::RepositoryDeclaration, + ) + .unwrap_err() + .to_string() + .contains("host cache writes above")); + plan.caches.clear(); + plan.command = None; + plan.mounted_commands = vec![command]; + assert!(db + .validate_workspace_environment_capability_ceilings( + &plan, + ArtifactProducerTrustTierV1::RepositoryDeclaration, + ) + .unwrap_err() + .to_string() + .contains("mounted execution above")); + } + + #[test] + fn environment_command_diagnostics_are_bounded_while_draining_the_pipe() { + let input = vec![b'x'; MAX_ENVIRONMENT_COMMAND_DIAGNOSTIC_BYTES + 1]; + let diagnostic = spawn_bounded_environment_command_diagnostic(std::io::Cursor::new(input)) + .join() + .unwrap() + .unwrap(); + assert_eq!( + diagnostic.bytes.len(), + MAX_ENVIRONMENT_COMMAND_DIAGNOSTIC_BYTES + ); + assert!(diagnostic.truncated); + } + #[test] fn cache_namespace_identity_is_deterministic_and_compatibility_scoped() { let workspace = tempfile::tempdir().unwrap(); @@ -6711,11 +8219,15 @@ mod tests { fs::write(plan.caches[0].storage_path.join("entry"), "cached\n").unwrap(); let (sender, receiver) = std::sync::mpsc::channel(); - let workspace_path = workspace.path().to_path_buf(); + // Open the competing handle before measuring cache-lock contention. + // Under parallel test load, open recovery may legitimately hold the + // workspace mutation lock for longer than the short cache timeout; + // that is a different serialization boundary from the host-exclusive + // namespace lease this test is intended to exercise. + let concurrent = Trail::open(workspace.path()).unwrap(); let concurrent_plan = plan.clone(); let concurrent_command = command.clone(); let worker = thread::spawn(move || { - let concurrent = Trail::open(workspace_path).unwrap(); let guard = concurrent .acquire_workspace_environment_cache_uses(&concurrent_plan, &concurrent_command) .unwrap(); @@ -6914,6 +8426,7 @@ mod tests { strategy: "mounted-failure-candidate".to_string(), }, inputs: Vec::new(), + resolution_inputs: Vec::new(), source_projection: None, pre_commands: Vec::new(), command: None, @@ -7249,6 +8762,7 @@ mod tests { strategy: "dependency-scale-test".to_string(), }, inputs: Vec::new(), + resolution_inputs: Vec::new(), source_projection: None, pre_commands: Vec::new(), command: None, @@ -7312,6 +8826,7 @@ mod tests { strategy: "typed-edge-test".to_string(), }, inputs: Vec::new(), + resolution_inputs: Vec::new(), source_projection: None, pre_commands: Vec::new(), command: None, @@ -7431,6 +8946,7 @@ mod tests { strategy: "test".to_string(), }, inputs: Vec::new(), + resolution_inputs: Vec::new(), source_projection: None, pre_commands: Vec::new(), command: Some(command.clone()), @@ -7459,6 +8975,190 @@ mod tests { assert!(error.to_string().contains("changed after")); } + #[test] + fn component_proposal_evidence_is_sorted_and_bounded() { + let mut proposal = EnvironmentDiscoveredComponentReport { + component_id: "component".to_string(), + component_root: String::new(), + kind: "dependency".to_string(), + adapter_identity: "test/adapter@1".to_string(), + status: EnvironmentComponentProposalStatus::Blocked, + reasons: vec![ + EnvironmentProposalReasonReport { + code: "z_reason".to_string(), + message: "last".to_string(), + }, + EnvironmentProposalReasonReport { + code: "a_reason".to_string(), + message: "first".to_string(), + }, + ], + recovery_actions: vec![ + EnvironmentRecoveryActionReport { + code: "z_action".to_string(), + description: "last".to_string(), + command: None, + }, + EnvironmentRecoveryActionReport { + code: "a_action".to_string(), + description: "first".to_string(), + command: Some(vec!["trail".to_string(), "env".to_string()]), + }, + ], + }; + normalize_environment_component_proposal(&mut proposal).unwrap(); + assert_eq!(proposal.reasons[0].code, "a_reason"); + assert_eq!(proposal.recovery_actions[0].code, "a_action"); + + proposal.reasons[0].message = "x".repeat(MAX_ENVIRONMENT_PROPOSAL_TEXT_BYTES + 1); + let error = normalize_environment_component_proposal(&mut proposal).unwrap_err(); + assert!(error.to_string().contains("oversized")); + } + + #[test] + fn manifest_only_cargo_and_node_components_are_reported_without_resolution_side_effects() { + let workspace = tempfile::tempdir().unwrap(); + fs::write( + workspace.path().join("Cargo.toml"), + "[package]\nname = \"manifest-only\"\nversion = \"0.1.0\"\nedition = \"2024\"\n", + ) + .unwrap(); + fs::write( + workspace.path().join("package.json"), + r#"{"name":"manifest-only","version":"1.0.0","private":true}"#, + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "manifest-only", + Some("main"), + mode, + None, + None, + None, + &[], + false, + ) + .unwrap(); + + let discovery = db + .discover_workspace_environment("manifest-only", None) + .unwrap(); + assert!(discovery.conflicts.is_empty()); + assert_eq!(discovery.components.len(), 2); + for component in &discovery.components { + assert_eq!( + component.status, + EnvironmentComponentProposalStatus::Resolvable + ); + assert_eq!( + component.recovery_actions[0].command, + Some(vec![ + "trail".into(), + "env".into(), + "resolve".into(), + "component".into(), + component.component_id.clone(), + "--lane".into(), + "manifest-only".into(), + ]) + ); + assert_eq!(component.reasons.len(), 1); + assert_eq!(component.reasons[0].code, "resolution_snapshot_missing"); + assert_eq!(component.recovery_actions.len(), 1); + } + assert!(!workspace.path().join("Cargo.lock").exists()); + assert!(!workspace.path().join("package-lock.json").exists()); + assert!(!workspace.path().join("pnpm-lock.yaml").exists()); + assert!(!workspace.path().join("yarn.lock").exists()); + assert!(!workspace.path().join("bun.lock").exists()); + } + + #[test] + fn host_resolver_executes_cargo_in_isolated_staging_and_reuses_snapshot() { + if resolve_workspace_tool_executable("cargo").is_err() { + return; + } + let workspace = tempfile::tempdir().unwrap(); + fs::create_dir_all(workspace.path().join("src")).unwrap(); + fs::write( + workspace.path().join("Cargo.toml"), + "[package]\nname = \"host-resolver\"\nversion = \"0.1.0\"\nedition = \"2024\"\n", + ) + .unwrap(); + fs::write( + workspace.path().join("src/lib.rs"), + "pub fn value() -> u8 { 1 }\n", + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "resolve-cargo", + Some("main"), + LaneWorkdirMode::Virtual, + None, + None, + None, + &[], + false, + ) + .unwrap(); + + let first = db + .resolve_workspace_environment_component( + "resolve-cargo", + "cargo-target-seed", + None, + false, + ) + .unwrap(); + assert_eq!(first.decision, ArtifactResolutionDecisionV1::Resolved); + assert!(first.attempt.is_some()); + assert!(!workspace.path().join("Cargo.lock").exists()); + let second = db + .resolve_workspace_environment_component( + "resolve-cargo", + "cargo-target-seed", + None, + false, + ) + .unwrap(); + assert_eq!(second.snapshot_id, first.snapshot_id); + assert_eq!(second.decision, ArtifactResolutionDecisionV1::Reused); + assert!(second.attempt.is_none()); + let refreshed = db + .resolve_workspace_environment_component( + "resolve-cargo", + "cargo-target-seed", + None, + true, + ) + .unwrap(); + assert_eq!(refreshed.decision, ArtifactResolutionDecisionV1::Refreshed); + assert_eq!( + refreshed.snapshot.predecessor_snapshot_id, + Some(first.snapshot_id) + ); + assert!(refreshed.attempt.is_some()); + let batch = db + .resolve_all_workspace_environment_components("resolve-cargo", None, false) + .unwrap(); + assert_eq!(batch.components.len(), 1); + assert_eq!( + batch.components[0].decision, + ArtifactResolutionDecisionV1::Reused + ); + } + #[test] fn automatic_detection_rejects_ambiguous_polyglot_roots() { let workspace = tempfile::tempdir().unwrap(); @@ -7592,6 +9292,7 @@ mod tests { strategy: "test".to_string(), }, inputs: Vec::new(), + resolution_inputs: Vec::new(), source_projection: None, pre_commands: Vec::new(), command: Some(WorkspaceEnvironmentCommand { @@ -7684,6 +9385,7 @@ mod tests { strategy: "test".to_string(), }, inputs: Vec::new(), + resolution_inputs: Vec::new(), source_projection: None, pre_commands: Vec::new(), command: Some(command), diff --git a/trail/src/db/lane/workspace_go.rs b/trail/src/db/lane/workspace_go.rs index c1e206f9..2f9734c4 100644 --- a/trail/src/db/lane/workspace_go.rs +++ b/trail/src/db/lane/workspace_go.rs @@ -171,6 +171,7 @@ impl WorkspaceEnvironmentAdapter for GoVendorAdapter { strategy: "go-mod-vendor-v1".to_string(), }, inputs: Vec::new(), + resolution_inputs: Vec::new(), source_projection: Some((source_root.clone(), "project".to_string())), pre_commands: Vec::new(), command: Some(WorkspaceEnvironmentCommand { @@ -289,6 +290,48 @@ mod tests { .unwrap(); Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(workspace.path()).unwrap(); + let source_root = db.resolve_branch_ref("main").unwrap().root_id; + let raw_plan = GO_VENDOR_ADAPTER.plan(&db, &source_root, "").unwrap(); + let contract_digest = + super::workspace_environment::workspace_environment_artifact_contract_digest(&raw_plan) + .unwrap(); + let identity = super::workspace_environment::workspace_environment_identity_contract_v3( + &raw_plan, + contract_digest.clone(), + ) + .unwrap(); + assert!(identity.source_closure_complete); + assert!(identity.portability_certified); + assert_eq!(identity.trust_scope, "builtin"); + assert_eq!(identity.semantic_identities.len(), 3); + assert!(identity + .semantic_identities + .contains_key("performance_cache:module-store")); + assert!(identity + .semantic_identities + .contains_key("performance_cache:build-cache")); + + let mut relocated_plan = raw_plan.clone(); + for cache in &mut relocated_plan.caches { + let old_path = cache.storage_path.to_string_lossy().into_owned(); + cache.storage_path = PathBuf::from(format!("relocated-cache-{}", cache.name)); + let new_path = cache.storage_path.to_string_lossy().into_owned(); + for command in relocated_plan.command.iter_mut() { + for value in command.environment.values_mut() { + if value == &old_path { + *value = new_path.clone(); + } + } + } + } + assert_eq!( + super::workspace_environment::workspace_environment_artifact_contract_digest( + &relocated_plan, + ) + .unwrap(), + contract_digest, + "host cache locations are execution bindings, not artifact identity" + ); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow } else if cfg!(target_os = "windows") { diff --git a/trail/src/db/lane/workspace_layer.rs b/trail/src/db/lane/workspace_layer.rs index 219005eb..85816139 100644 --- a/trail/src/db/lane/workspace_layer.rs +++ b/trail/src/db/lane/workspace_layer.rs @@ -1,10 +1,46 @@ use super::workdir::{classify_view_path, PreparedLayerMountReset, ViewCore, ViewPathClass}; use super::*; +use crate::ids::{ArtifactEnvelopeId, ArtifactTreeId}; use serde::{Deserialize, Serialize}; use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering}; use std::thread; const LAYER_BUILD_LEASE_SECS: i64 = 300; +const CONSTRUCTION_EVIDENCE_LOCK_WAIT_SECS: u64 = 10; + +#[derive(Clone, Debug)] +struct ArtifactConstructionAttemptFence { + attempt_id: String, + owner_generation: u64, + owner_pid: u32, + owner_start_token: String, +} + +#[derive(Clone, Debug)] +struct ArtifactConstructionWaiterFence { + attempt_id: String, + waiter_id: String, + owner_pid: u32, + owner_start_token: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct InheritedArtifactBinding { + output_name: String, + desired_key: String, + envelope_id: String, + tree_root_id: String, +} + +struct ArtifactInheritanceRequest<'a> { + component_id: &'a str, + component_key: &'a str, + output_name: &'a str, + output_policy: EnvironmentOutputPolicy, + child_backend: &'a str, + child_desired_key_was_planned: bool, +} + const WORKSPACE_LAYER_VERIFICATION_STAMP_VERSION: u16 = 1; const WORKSPACE_LAYER_SIDECAR_MAX_BYTES: u64 = 64 * 1024; @@ -25,6 +61,10 @@ fn inheritable_workspace_layer_scope(scope: &str) -> bool { ) } +fn artifact_inheritance_backend_supported(backend: &str) -> bool { + matches!(backend, "fuse" | "nfs" | "dokan") +} + #[derive(Clone, Debug)] pub(crate) struct WorkspaceLayerBinding { /// Durable identity used by filesystem-side activation recovery. For an @@ -35,6 +75,12 @@ pub(crate) struct WorkspaceLayerBinding { pub(crate) layer_id: Option, pub(crate) mount_path: String, pub(crate) storage_path: Option, + /// Verified artifact tree backing this immutable layer. Native workspace + /// backends prefer this manifest-backed view and retain `storage_path` as + /// a compatibility/materialization-cache fallback. + pub(crate) artifact_tree_id: Option, + /// Path inside `artifact_tree_id` mounted at `mount_path`. + pub(crate) artifact_subpath: String, pub(crate) kind: String, #[allow(dead_code)] pub(crate) priority: i64, @@ -119,7 +165,7 @@ fn workspace_layer_manifest_entry_count(manifest: &WorkspaceLayerManifest) -> u6 } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -struct WorkspaceLayerEntry { +pub(crate) struct WorkspaceLayerEntry { kind: String, mode: u32, size_bytes: u64, @@ -240,6 +286,16 @@ impl Trail { .ok_or_else(|| { Error::InvalidInput("lane has no active environment generation".to_string()) })?; + let generation_secret_tainted = self.conn.query_row( + "SELECT EXISTS( + SELECT 1 FROM environment_secret_access_audit + WHERE generation_id=?1 AND status='available' + )", + params![&predecessor], + |row| row.get::<_, bool>(0), + )?; + let producer_receipt_json = + validate_private_output_promotion_taint(generation_secret_tainted, producer_receipt)?; let (policy, reuse, scope, trigger, gate, binding_identity, mount_path, component_key) = self.conn .query_row( @@ -341,7 +397,7 @@ impl Trail { &output_identity, trigger.as_str(), &gate, - producer_receipt.map(serde_json::to_vec).transpose()?, + producer_receipt_json, std::process::id(), current_process_start_token(), staging_relative, @@ -577,6 +633,160 @@ impl Trail { Ok(()) } + /// Rebuild missing verified layer cache directories from authoritative CAS + /// envelopes during workspace-open recovery. This runs under the existing + /// workspace write lock, stages into an attempt-owned directory, validates + /// against the legacy manifest, and only then publishes by atomic rename. + pub(crate) fn recover_workspace_layer_materializations(&self) -> Result<()> { + let private_stage = self.db_dir != self.workspace_root.join(".trail"); + let mut statement = self.conn.prepare( + "SELECT l.layer_id,l.storage_path,l.manifest_object_id, + s.tree_root_id,s.envelope_id + FROM workspace_layers l + JOIN workspace_layer_artifact_shadows s ON s.layer_id=l.layer_id + JOIN artifact_envelopes e ON e.envelope_id=s.envelope_id + WHERE l.state='ready' AND s.state='verified' + AND e.state='ready' AND e.verification_state='verified' + ORDER BY l.layer_id", + )?; + let candidates = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + )) + })? + .collect::, _>>()?; + drop(statement); + + for (layer_id, stored_path, manifest_id, tree_id, envelope_id) in candidates { + if !valid_workspace_layer_id(&layer_id) { + return Err(Error::Corrupt(format!( + "workspace layer ID `{layer_id}` is invalid; restore from backup or reinitialize the workspace" + ))); + } + let final_path = self.db_dir.join("cache/layers").join(&layer_id); + if private_stage { + if !workspace_layer_storage_path_matches_id(&stored_path, &layer_id) { + return Err(Error::Corrupt(format!( + "workspace layer `{layer_id}` has a non-canonical materialization path" + ))); + } + let restored_path = self + .workspace_root + .join(".trail/cache/layers") + .join(&layer_id); + if Path::new(&stored_path) != restored_path { + self.conn.execute( + "UPDATE workspace_layers SET storage_path=?1 WHERE layer_id=?2", + params![restored_path.to_string_lossy(), &layer_id], + )?; + } + } else if Path::new(&stored_path) != final_path { + return Err(Error::Corrupt(format!( + "workspace layer `{layer_id}` has a non-canonical materialization path" + ))); + } + let staging = self + .db_dir + .join("cache/staging") + .join(format!("restore_{layer_id}")); + if final_path.exists() { + remove_exact_recovery_directory( + &staging, + "completed artifact materialization staging", + )?; + continue; + } + let tree_id = crate::ids::ArtifactTreeId::parse(tree_id) + .map_err(|error| Error::Corrupt(format!("invalid artifact tree ID: {error}")))?; + let envelope_id = + crate::ids::ArtifactEnvelopeId::parse(envelope_id).map_err(|error| { + Error::Corrupt(format!("invalid artifact envelope ID: {error}")) + })?; + let envelope = self + .verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id) + .map_err(|error| { + Error::Corrupt(format!( + "CAS-backed workspace layer `{layer_id}` cannot be reconstructed because envelope `{envelope_id}` failed verification: {error}; restore the workspace from backup or reinitialize it and run environment synchronization" + )) + })?; + if !matches!( + envelope.desired_identity, + ArtifactDesiredIdentityV1::WorkspaceLayerV1 { .. } + ) { + return Err(Error::Corrupt(format!( + "workspace layer `{layer_id}` references a non-legacy artifact envelope" + ))); + } + let mut manifest: WorkspaceLayerManifest = self + .get_object( + WORKSPACE_LAYER_MANIFEST_KIND, + &ObjectId(manifest_id.clone()), + ) + .map_err(|error| { + Error::Corrupt(format!( + "CAS-backed workspace layer `{layer_id}` has no valid layer manifest: {error}; restore the workspace from backup or reinitialize it and run environment synchronization" + )) + })?; + self.hydrate_workspace_layer_manifest(&mut manifest) + .map_err(|error| { + Error::Corrupt(format!( + "CAS-backed workspace layer `{layer_id}` has an invalid paged manifest: {error}; restore the workspace from backup or reinitialize it and run environment synchronization" + )) + })?; + remove_exact_recovery_directory(&staging, "artifact materialization staging")?; + fs::create_dir_all(staging.parent().unwrap())?; + let shared_materialization = self + .ensure_artifact_tree_materialization_under_write_lock(&tree_id) + .map_err(|error| { + Error::Corrupt(format!( + "CAS-backed workspace layer `{layer_id}` could not materialize tree `{tree_id}` through its shared cache: {error}; restore the workspace from backup or reinitialize it and run environment synchronization" + )) + })?; + copy_layer_tree(&shared_materialization.storage_path, &staging)?; + test_crash_point("layer_after_cas_materialization"); + // The legacy manifest records the immutable attachment modes + // (0555/0444), so seal the reconstructed children before the + // byte-for-byte manifest comparison. + let actual = scan_layer_entries(&staging, true)?; + if manifest.layer_id != layer_id || manifest.entries != actual { + make_tree_writable(&staging); + fs::remove_dir_all(&staging)?; + return Err(Error::Corrupt(format!( + "workspace layer `{layer_id}` CAS materialization disagrees with its manifest" + ))); + } + fs::create_dir_all(final_path.parent().unwrap())?; + fs::rename(&staging, &final_path)?; + set_layer_read_only( + &final_path, + true, + layer_mode(&fs::symlink_metadata(&final_path)?), + )?; + sync_directory(final_path.parent().unwrap()); + let physical_bytes = layer_physical_bytes(&final_path)?; + self.conn.execute( + "UPDATE workspace_layers SET physical_bytes=?1,last_used_at=?2 + WHERE layer_id=?3 AND state='ready'", + params![physical_bytes as i64, now_ts(), layer_id], + )?; + let report = self + .workspace_layer_by_cache_key(&manifest.cache_key)? + .ok_or_else(|| { + Error::Corrupt(format!( + "workspace layer `{layer_id}` disappeared after CAS materialization" + )) + })?; + write_workspace_layer_publish_marker_from_report(&report, &manifest_id)?; + write_workspace_layer_verification_stamp(&report, &manifest_id)?; + } + Ok(()) + } + pub(crate) fn validate_workspace_layer_publications_integrity(&self) -> Result> { let mut statement = self.conn.prepare( "SELECT p.publication_id,p.view_id,p.predecessor_generation_id, @@ -726,6 +936,324 @@ impl Trail { Ok(errors) } + pub(crate) fn validate_workspace_artifact_layout_integrity(&self) -> Result> { + const MAX_MATERIALIZATION_DIRECTORIES: usize = 100_000; + let mut errors = Vec::new(); + // Backup verification and restore validation open a private SQLite + // stage whose disposable cache tree is intentionally absent. Validate + // durable path identity there, but reserve physical materialization + // checks for an active `.trail` database directory. + let private_stage = self.db_dir != self.workspace_root.join(".trail"); + let layers = { + let mut statement = self.conn.prepare( + "SELECT l.layer_id,l.storage_path,l.state, + CASE WHEN s.layer_id IS NULL THEN 0 ELSE 1 END + FROM workspace_layers l + LEFT JOIN workspace_layer_artifact_shadows s ON s.layer_id=l.layer_id + ORDER BY l.layer_id", + )?; + statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, bool>(3)?, + )) + })? + .collect::, _>>()? + }; + let known_layers = layers + .iter() + .filter(|(layer_id, _, _, _)| valid_workspace_layer_id(layer_id)) + .map(|(layer_id, _, _, _)| layer_id.clone()) + .collect::>(); + for (layer_id, stored_path, state, cas_backed) in &layers { + if !valid_workspace_layer_id(layer_id) { + errors.push(format!( + "workspace layer ID `{layer_id}` is invalid; restore from backup or reinitialize the workspace" + )); + continue; + } + let expected = self.db_dir.join("cache/layers").join(layer_id); + if private_stage { + if !workspace_layer_storage_path_matches_id(stored_path, layer_id) { + errors.push(format!( + "workspace layer {layer_id} has a non-canonical storage path; restore from backup or reinitialize the workspace" + )); + } + continue; + } + if Path::new(stored_path) != expected { + errors.push(format!( + "workspace layer {layer_id} has a non-canonical storage path; restore from backup or reinitialize the workspace" + )); + continue; + } + if state != "ready" { + continue; + } + if !expected.is_dir() { + errors.push(if *cas_backed { + format!( + "CAS-backed workspace layer {layer_id} has no materialization; reopen Trail to reconstruct it from its verified envelope" + ) + } else { + format!( + "legacy workspace layer {layer_id} is missing and has no CAS authority; run environment synchronization to rebuild it" + ) + }); + continue; + } + if let Err(error) = self.verify_workspace_layer(layer_id) { + errors.push(if *cas_backed { + format!( + "CAS-backed workspace layer {layer_id} failed materialization verification: {error}; move the materialization outside `.trail/cache/layers` and reopen Trail" + ) + } else { + format!( + "legacy workspace layer {layer_id} failed verification: {error}; run environment synchronization to rebuild it" + ) + }); + } + } + + let layer_root = self.db_dir.join("cache/layers"); + if layer_root.is_dir() { + for (index, entry) in fs::read_dir(&layer_root)?.enumerate() { + if index >= MAX_MATERIALIZATION_DIRECTORIES { + errors.push(format!( + "workspace layer materialization directory exceeds the {MAX_MATERIALIZATION_DIRECTORIES}-entry diagnostic bound; inspect `.trail/cache/layers` manually" + )); + break; + } + let entry = entry?; + let name = entry.file_name().to_string_lossy().into_owned(); + if workspace_layer_sidecar_owner(&name) + .is_some_and(|layer_id| known_layers.contains(layer_id)) + { + continue; + } + if !known_layers.contains(&name) { + errors.push(format!( + "orphan workspace layer materialization `{name}` has no database owner; move it outside `.trail/cache/layers`, rerun `trail fsck`, then delete it after review" + )); + } + } + } + + let staging_root = self.db_dir.join("cache/staging"); + if staging_root.is_dir() { + for (index, entry) in fs::read_dir(&staging_root)?.enumerate() { + if index >= MAX_MATERIALIZATION_DIRECTORIES { + errors.push(format!( + "workspace layer staging directory exceeds the {MAX_MATERIALIZATION_DIRECTORIES}-entry diagnostic bound; inspect `.trail/cache/staging` manually" + )); + break; + } + let entry = entry?; + let name = entry.file_name().to_string_lossy().into_owned(); + let Some(layer_id) = name.strip_prefix("restore_") else { + continue; + }; + errors.push(if known_layers.contains(layer_id) { + format!( + "workspace layer {layer_id} retains incomplete materialization staging; reopen Trail to recover the exact staged path" + ) + } else { + format!( + "orphan artifact materialization staging `{name}` has no database owner; move it outside `.trail/cache/staging` and rerun `trail fsck`" + ) + }); + } + } + Ok(errors) + } + + fn inheritance_adapter_compatibility( + &self, + parent_view_id: &str, + component_id: &str, + component_key: &str, + adapter_identity: &str, + layer_adapter: &str, + ) -> std::result::Result<(), &'static str> { + let (canonical_identity, contract_major, implementation_version, distribution_digest) = + if let Some(metadata) = + super::workspace_environment::registered_environment_adapter_metadata() + .into_iter() + .find(|metadata| { + metadata.canonical_identity == adapter_identity + || metadata.selectors.contains(&adapter_identity) + }) + { + if metadata.layer_adapter_name != layer_adapter { + return Err("adapter_package_identity_mismatch"); + } + if !metadata + .supported_operating_systems + .contains(&std::env::consts::OS) + || !metadata + .supported_architectures + .contains(&std::env::consts::ARCH) + { + return Err("adapter_host_unsupported"); + } + ( + metadata.canonical_identity.to_string(), + metadata.contract_major, + metadata.implementation_version.to_string(), + metadata.distribution_digest.to_string(), + ) + } else { + let plugin = match self.environment_plugin_for_selector(adapter_identity) { + Ok(Some(plugin)) => plugin, + Ok(None) => return Err("adapter_package_unavailable"), + Err(_) => return Err("adapter_package_trust_failed"), + }; + if plugin.manifest.adapter.layer_adapter_name != layer_adapter { + return Err("adapter_package_identity_mismatch"); + } + if !super::workspace_plugin::environment_plugin_supports_current_host(&plugin) { + return Err("adapter_host_unsupported"); + } + let (_, _, contract_major) = super::workspace_plugin::validate_plugin_identity( + &plugin.manifest.adapter.canonical_identity, + ) + .map_err(|_| "adapter_package_identity_mismatch")?; + ( + plugin.manifest.adapter.canonical_identity, + contract_major, + plugin.manifest.adapter.implementation_version, + plugin.distribution_digest, + ) + }; + let recorded = self + .conn + .query_row( + "SELECT adapter_identity,adapter_version,implementation_version, + COALESCE(distribution_digest,''),attached_key,status + FROM environment_component_states + WHERE view_id=?1 AND component_id=?2", + params![parent_view_id, component_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, u32>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, String>(5)?, + )) + }, + ) + .optional() + .map_err(|_| "adapter_package_evidence_invalid")? + .ok_or("adapter_package_identity_unproven")?; + if recorded.0 != canonical_identity + || recorded.1 != contract_major + || recorded.2 != implementation_version + || recorded.3 != distribution_digest + || recorded.4.as_deref() != Some(component_key) + || recorded.5 != "ready" + { + return Err("adapter_package_identity_mismatch"); + } + Ok(()) + } + + fn verify_artifact_inheritance_binding( + &self, + layer: &WorkspaceLayerReport, + request: ArtifactInheritanceRequest<'_>, + ) -> std::result::Result { + if !artifact_inheritance_backend_supported(request.child_backend) { + return Err("artifact_backend_unsupported"); + } + let shadow = self + .conn + .query_row( + "SELECT tree_root_id,envelope_id,state + FROM workspace_layer_artifact_shadows WHERE layer_id=?1", + params![&layer.layer_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }, + ) + .optional() + .map_err(|_| "artifact_binding_query_failed")? + .ok_or("artifact_binding_missing")?; + if shadow.2 != "verified" { + return Err("artifact_binding_unverified"); + } + let tree_id = ArtifactTreeId::parse(shadow.0.clone()) + .map_err(|_| "artifact_tree_identity_invalid")?; + let envelope_id = crate::ids::ArtifactEnvelopeId::parse(shadow.1.clone()) + .map_err(|_| "artifact_envelope_identity_invalid")?; + let envelope = self + .verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id) + .map_err(|_| "artifact_envelope_verification_failed")?; + self.artifact_tree_flat_entries(&tree_id) + .map_err(|_| "artifact_tree_verification_failed")?; + if envelope.portability_scope != layer.portability_scope + || !inheritable_workspace_layer_scope(&envelope.portability_scope) + { + return Err("artifact_portability_mismatch"); + } + let desired_key = match &envelope.desired_identity { + ArtifactDesiredIdentityV1::WorkspaceLayerV1 { + cache_key, + canonical_key, + } => { + if envelope.component_id != format!("legacy:{}", layer.adapter) + || envelope.output_name != "legacy-layer" + || !envelope.output_policy.has_immutable_layer() + || !request.output_policy.has_immutable_layer() + || envelope.trust_scope != "workspace-layer-v1" + || cache_key != request.component_key + || canonical_key.adapter != layer.adapter + || canonical_key.portability_scope != layer.portability_scope + || canonical_key.portability_scope != "portable" + && (canonical_key.platform != std::env::consts::OS + || canonical_key.architecture != std::env::consts::ARCH) + || self + .workspace_layer_cache_key(canonical_key) + .ok() + .as_deref() + != Some(request.component_key) + { + return Err("artifact_desired_identity_mismatch"); + } + cache_key.clone() + } + ArtifactDesiredIdentityV1::ArtifactDesiredV2 { desired_key } => { + if !request.child_desired_key_was_planned { + return Err("artifact_desired_identity_unproven"); + } + if envelope.component_id != request.component_id + || envelope.output_name != request.output_name + || envelope.output_policy != request.output_policy + { + return Err("artifact_output_contract_mismatch"); + } + if desired_key.0 != request.component_key { + return Err("artifact_desired_identity_mismatch"); + } + desired_key.0.clone() + } + }; + Ok(InheritedArtifactBinding { + output_name: request.output_name.to_string(), + desired_key, + envelope_id: envelope_id.0, + tree_root_id: tree_id.0, + }) + } + pub(crate) fn inherit_workspace_environment_generation( &self, parent_lane_id: &str, @@ -801,6 +1329,7 @@ impl Trail { mount_path: Option, source_path: String, priority: i64, + artifact_bindings: Vec, } let candidates = { @@ -829,6 +1358,7 @@ impl Trail { mount_path: row.get(5)?, source_path: row.get(6)?, priority: row.get(7)?, + artifact_bindings: Vec::new(), }) }, )? @@ -836,7 +1366,7 @@ impl Trail { }; let mut inherited = Vec::new(); let mut decisions = Vec::::new(); - for component in candidates { + for mut component in candidates { let mut component_rejection = None; if let Some(planned) = desired.get(&component.component_id) { if planned.adapter_identity != component.adapter_identity { @@ -852,7 +1382,7 @@ impl Trail { let outputs = { let mut statement = self.conn.prepare( - "SELECT output_name,policy,storage_identity,layer_id + "SELECT output_name,policy,reuse_mode,sharing_scope,storage_identity,layer_id FROM environment_generation_outputs WHERE generation_id=?1 AND component_id=?2 ORDER BY output_name", )?; @@ -863,19 +1393,26 @@ impl Trail { Ok(( row.get::<_, String>(0)?, row.get::<_, EnvironmentOutputPolicy>(1)?, - row.get::<_, String>(2)?, - row.get::<_, Option>(3)?, + row.get::<_, EnvironmentReuseMode>(2)?, + row.get::<_, EnvironmentSharingScope>(3)?, + row.get::<_, String>(4)?, + row.get::<_, Option>(5)?, )) }, )? .collect::, _>>()? }; let mut reusable_output = false; - for (output_name, policy, storage_identity, layer_id) in outputs { + for (output_name, policy, reuse, scope, storage_identity, layer_id) in outputs { let mut rejection = component_rejection; let decision = if !policy.has_immutable_layer() { rejection = Some("fresh_lane_private_upper"); EnvironmentComponentDecision::Private + } else if reuse == EnvironmentReuseMode::None + || scope == EnvironmentSharingScope::Lane + { + rejection = Some("artifact_sharing_policy_rejected"); + EnvironmentComponentDecision::Rejected } else if rejection.is_some() { EnvironmentComponentDecision::Rejected } else if let Some(layer_id) = layer_id.as_deref() { @@ -883,8 +1420,39 @@ impl Trail { Ok(layer) if inheritable_workspace_layer_scope(&layer.portability_scope) => { - reusable_output = true; - EnvironmentComponentDecision::Reused + if let Err(reason) = self.inheritance_adapter_compatibility( + &parent_view.view_id, + &component.component_id, + &component.component_key, + &component.adapter_identity, + &layer.adapter, + ) { + rejection = Some(reason); + EnvironmentComponentDecision::Rejected + } else { + match self.verify_artifact_inheritance_binding( + &layer, + ArtifactInheritanceRequest { + component_id: &component.component_id, + component_key: &component.component_key, + output_name: &output_name, + output_policy: policy, + child_backend: &child_view.backend, + child_desired_key_was_planned: desired + .contains_key(&component.component_id), + }, + ) { + Ok(binding) => { + component.artifact_bindings.push(binding); + reusable_output = true; + EnvironmentComponentDecision::Reused + } + Err(reason) => { + rejection = Some(reason); + EnvironmentComponentDecision::Rejected + } + } + } } Ok(_) => { rejection = Some("unsupported_portability_scope"); @@ -1010,23 +1578,67 @@ impl Trail { &component.component_id ], )?; - self.conn.execute( - "INSERT INTO environment_generation_outputs( - generation_id,component_id,output_name,policy,reuse_mode,sharing_scope, - publication_trigger,publication_gate,storage_identity,layer_id, - manifest_object_id,publication_id,mount_path,layer_subpath) - SELECT ?1,component_id,output_name,policy,reuse_mode,sharing_scope, - publication_trigger,publication_gate,storage_identity,layer_id, - manifest_object_id,publication_id,mount_path,layer_subpath - FROM environment_generation_outputs - WHERE generation_id=?2 AND component_id=?3 - AND policy LIKE 'immutable%' AND layer_id IS NOT NULL", - params![ - &generation_id, - &parent_generation_id, - &component.component_id - ], - )?; + for binding in &component.artifact_bindings { + let inserted = self.conn.execute( + "INSERT INTO environment_generation_outputs( + generation_id,component_id,output_name,policy,reuse_mode,sharing_scope, + publication_trigger,publication_gate,storage_identity,layer_id, + manifest_object_id,publication_id,mount_path,layer_subpath) + SELECT ?1,component_id,output_name,policy,reuse_mode,sharing_scope, + publication_trigger,publication_gate,storage_identity,layer_id, + manifest_object_id,publication_id,mount_path,layer_subpath + FROM environment_generation_outputs + WHERE generation_id=?2 AND component_id=?3 AND output_name=?4 + AND policy LIKE 'immutable%' AND layer_id IS NOT NULL", + params![ + &generation_id, + &parent_generation_id, + &component.component_id, + &binding.output_name + ], + )?; + if inserted != 1 { + return Err(Error::Corrupt(format!( + "inherited artifact output `{}/{}` changed after compatibility verification", + component.component_id, binding.output_name + ))); + } + let binding_identity = format!( + "artifact_binding_{}", + crate::ids::short_hash( + format!( + "{}\0{}\0{}\0{}", + generation_id, + component.component_id, + binding.output_name, + binding.envelope_id + ) + .as_bytes(), + 32, + ) + ); + let binding_id = format!( + "binding_{}", + crate::ids::short_hash(binding_identity.as_bytes(), 32) + ); + self.conn.execute( + "INSERT INTO artifact_generation_bindings( + binding_id,generation_id,component_id,output_name,desired_key, + envelope_id,tree_root_id,binding_identity,created_at) + VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9)", + params![ + binding_id, + &generation_id, + &component.component_id, + &binding.output_name, + &binding.desired_key, + &binding.envelope_id, + &binding.tree_root_id, + binding_identity, + now + ], + )?; + } } self.conn.execute( "INSERT INTO environment_view_generations(view_id,generation_id,updated_at) @@ -1065,20 +1677,529 @@ impl Trail { Ok(sha256_hex(&serde_json::to_vec(key)?)) } - pub(crate) fn build_workspace_layer_singleflight( - &self, - key: &WorkspaceLayerKeyV1, - builder: F, - ) -> Result - where - F: FnOnce(&Path) -> Result, - { - self.build_workspace_layer_singleflight_with_cancel(key, &AtomicBool::new(false), builder) - } - - pub(crate) fn build_workspace_layer_singleflight_with_cancel( + fn begin_artifact_construction_attempt( + &self, + desired_key: &str, + source_root: &ObjectId, + ) -> Result { + let _lock = Self::with_write_lock_wait( + Duration::from_secs(CONSTRUCTION_EVIDENCE_LOCK_WAIT_SECS), + || self.acquire_write_lock(), + )?; + let _: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, source_root)?; + if let Some((attempt_id, generation, owner_pid, owner_start_token)) = self + .conn + .query_row( + "SELECT attempt_id,owner_generation,owner_pid,owner_start_token + FROM artifact_construction_attempts + WHERE desired_key=?1 AND status='running'", + params![desired_key], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, String>(3)?, + )) + }, + ) + .optional()? + { + let owner_pid = u32::try_from(owner_pid).map_err(|_| { + Error::Corrupt(format!( + "artifact construction attempt `{attempt_id}` has invalid owner PID" + )) + })?; + if process_start_token_match(owner_pid, &owner_start_token) + != ProcessIdentityMatch::DeadOrMismatch + { + return Err(Error::InvalidInput(format!( + "artifact desired key `{desired_key}` is still owned by live or indeterminate construction attempt `{attempt_id}`" + ))); + } + let updated = self.conn.execute( + "UPDATE artifact_construction_attempts + SET phase='completed',status='abandoned',reason_code='owner_lost', + reason='construction owner process exited',heartbeat_at=?1,finished_at=?1 + WHERE attempt_id=?2 AND owner_generation=?3 AND owner_pid=?4 + AND owner_start_token=?5 AND status='running'", + params![ + now_ts(), + attempt_id, + generation, + i64::from(owner_pid), + owner_start_token + ], + )?; + if updated != 1 { + return Err(Error::InvalidInput(format!( + "artifact construction attempt `{attempt_id}` changed during dead-owner fencing" + ))); + } + self.conn.execute( + "UPDATE artifact_construction_waiters + SET status='abandoned',updated_at=?1 + WHERE attempt_id=?2 AND status='waiting'", + params![now_ts(), attempt_id], + )?; + } + let owner_generation = self.conn.query_row( + "SELECT COALESCE(MAX(owner_generation),0)+1 + FROM artifact_construction_attempts WHERE desired_key=?1", + params![desired_key], + |row| row.get::<_, i64>(0), + )?; + let owner_generation = u64::try_from(owner_generation).map_err(|_| { + Error::Corrupt("artifact construction generation exceeds supported range".into()) + })?; + let owner_pid = std::process::id(); + let owner_start_token = current_process_start_token(); + let attempt_id = format!( + "construct_{}", + crate::ids::short_hash( + format!( + "{desired_key}\0{}\0{owner_generation}\0{owner_pid}\0{owner_start_token}\0{}", + source_root.0, + now_nanos() + ) + .as_bytes(), + 32 + ) + ); + self.conn.execute( + "INSERT INTO artifact_construction_attempts( + attempt_id,desired_key,source_root,owner_generation,owner_pid, + owner_start_token,phase,status,candidate_journal_object_id,envelope_id, + reason_code,reason,cancel_requested,started_at,heartbeat_at,finished_at) + VALUES(?1,?2,?3,?4,?5,?6,'reserved','running',NULL,NULL,NULL,NULL,0,?7,?7,NULL)", + params![ + attempt_id, + desired_key, + source_root.0, + i64::try_from(owner_generation).map_err(|_| Error::InvalidInput( + "artifact construction generation exceeds SQLite range".into() + ))?, + i64::from(owner_pid), + owner_start_token, + now_ts(), + ], + )?; + Ok(ArtifactConstructionAttemptFence { + attempt_id, + owner_generation, + owner_pid, + owner_start_token, + }) + } + + fn register_artifact_construction_waiter( + &self, + desired_key: &str, + ) -> Result> { + let _lock = Self::with_write_lock_wait( + Duration::from_secs(CONSTRUCTION_EVIDENCE_LOCK_WAIT_SECS), + || self.acquire_write_lock(), + )?; + let Some((attempt_id, owner_pid, owner_start_token)) = self + .conn + .query_row( + "SELECT attempt_id,owner_pid,owner_start_token + FROM artifact_construction_attempts + WHERE desired_key=?1 AND status='running'", + params![desired_key], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + )) + }, + ) + .optional()? + else { + return Ok(None); + }; + let construction_owner_pid = u32::try_from(owner_pid).map_err(|_| { + Error::Corrupt(format!( + "artifact construction attempt `{attempt_id}` has invalid owner PID" + )) + })?; + if process_start_token_match(construction_owner_pid, &owner_start_token) + == ProcessIdentityMatch::DeadOrMismatch + { + return Ok(None); + } + let owner_pid = std::process::id(); + let owner_start_token = current_process_start_token(); + let waiter_id = format!( + "waiter_{}", + crate::ids::short_hash( + format!( + "{attempt_id}\0{owner_pid}\0{owner_start_token}\0{}", + now_nanos() + ) + .as_bytes(), + 32 + ) + ); + self.conn.execute( + "INSERT INTO artifact_construction_waiters( + attempt_id,waiter_id,owner_pid,owner_start_token,status,created_at,updated_at) + VALUES(?1,?2,?3,?4,'waiting',?5,?5)", + params![ + attempt_id, + waiter_id, + i64::from(owner_pid), + owner_start_token, + now_ts() + ], + )?; + Ok(Some(ArtifactConstructionWaiterFence { + attempt_id, + waiter_id, + owner_pid, + owner_start_token, + })) + } + + fn finish_artifact_construction_waiter( + &self, + waiter: &ArtifactConstructionWaiterFence, + status: &str, + ) -> Result<()> { + if !matches!(status, "released" | "cancelled" | "abandoned") { + return Err(Error::InvalidInput(format!( + "invalid artifact construction waiter status `{status}`" + ))); + } + let _lock = Self::with_write_lock_wait( + Duration::from_secs(CONSTRUCTION_EVIDENCE_LOCK_WAIT_SECS), + || self.acquire_write_lock(), + )?; + let updated = self.conn.execute( + "UPDATE artifact_construction_waiters SET status=?1,updated_at=?2 + WHERE attempt_id=?3 AND waiter_id=?4 AND owner_pid=?5 + AND owner_start_token=?6 AND status='waiting'", + params![ + status, + now_ts(), + waiter.attempt_id, + waiter.waiter_id, + i64::from(waiter.owner_pid), + waiter.owner_start_token, + ], + )?; + if updated == 0 { + let existing_status = self + .conn + .query_row( + "SELECT status FROM artifact_construction_waiters + WHERE attempt_id=?1 AND waiter_id=?2 AND owner_pid=?3 + AND owner_start_token=?4", + params![ + waiter.attempt_id, + waiter.waiter_id, + i64::from(waiter.owner_pid), + waiter.owner_start_token, + ], + |row| row.get::<_, String>(0), + ) + .optional()?; + if existing_status + .as_deref() + .is_some_and(|status| matches!(status, "released" | "cancelled" | "abandoned")) + { + return Ok(()); + } + return Err(Error::InvalidInput(format!( + "artifact construction waiter `{}` lost its exact owner fence", + waiter.waiter_id + ))); + } + Ok(()) + } + + fn advance_artifact_construction_attempt( + &self, + attempt: &ArtifactConstructionAttemptFence, + phase: &str, + ) -> Result<()> { + if !matches!(phase, "building" | "validating" | "publishing") { + return Err(Error::InvalidInput(format!( + "invalid artifact construction phase `{phase}`" + ))); + } + let _lock = Self::with_write_lock_wait( + Duration::from_secs(CONSTRUCTION_EVIDENCE_LOCK_WAIT_SECS), + || self.acquire_write_lock(), + )?; + let updated = self.conn.execute( + "UPDATE artifact_construction_attempts SET phase=?1,heartbeat_at=?2 + WHERE attempt_id=?3 AND owner_generation=?4 AND owner_pid=?5 + AND owner_start_token=?6 AND status='running' AND cancel_requested=0", + params![ + phase, + now_ts(), + attempt.attempt_id, + i64::try_from(attempt.owner_generation).map_err(|_| Error::InvalidInput( + "artifact construction generation exceeds SQLite range".into() + ))?, + i64::from(attempt.owner_pid), + attempt.owner_start_token, + ], + )?; + if updated != 1 { + return Err(Error::InvalidInput(format!( + "artifact construction attempt `{}` lost its exact owner fence or was cancelled", + attempt.attempt_id + ))); + } + Ok(()) + } + + fn finish_artifact_construction_attempt( + &self, + attempt: &ArtifactConstructionAttemptFence, + status: &str, + reason_code: Option<&str>, + reason: Option<&str>, + ) -> Result<()> { + if !matches!(status, "succeeded" | "failed" | "cancelled" | "abandoned") { + return Err(Error::InvalidInput(format!( + "invalid artifact construction attempt status `{status}`" + ))); + } + let _lock = Self::with_write_lock_wait( + Duration::from_secs(CONSTRUCTION_EVIDENCE_LOCK_WAIT_SECS), + || self.acquire_write_lock(), + )?; + let updated = self.conn.execute( + "UPDATE artifact_construction_attempts + SET phase='completed',status=?1,reason_code=?2,reason=?3, + heartbeat_at=?4,finished_at=?4 + WHERE attempt_id=?5 AND owner_generation=?6 AND owner_pid=?7 + AND owner_start_token=?8 AND status='running'", + params![ + status, + reason_code, + reason, + now_ts(), + attempt.attempt_id, + i64::try_from(attempt.owner_generation).map_err(|_| Error::InvalidInput( + "artifact construction generation exceeds SQLite range".into() + ))?, + i64::from(attempt.owner_pid), + attempt.owner_start_token, + ], + )?; + if updated != 1 { + return Err(Error::InvalidInput(format!( + "artifact construction attempt `{}` lost its exact owner fence", + attempt.attempt_id + ))); + } + self.conn.execute( + "UPDATE artifact_construction_waiters SET status='released',updated_at=?1 + WHERE attempt_id=?2 AND status='waiting'", + params![now_ts(), attempt.attempt_id], + )?; + Ok(()) + } + + fn verify_artifact_construction_seal_fence( + &self, + attempt: &ArtifactConstructionAttemptFence, + desired_key: &str, + source_root: &ObjectId, + ) -> Result<()> { + let _lock = Self::with_write_lock_wait( + Duration::from_secs(CONSTRUCTION_EVIDENCE_LOCK_WAIT_SECS), + || self.acquire_write_lock(), + )?; + let matched = self + .conn + .query_row( + "SELECT 1 FROM artifact_construction_attempts + WHERE attempt_id=?1 AND desired_key=?2 AND source_root=?3 + AND owner_generation=?4 AND owner_pid=?5 AND owner_start_token=?6 + AND phase='validating' AND status='running' AND cancel_requested=0", + params![ + attempt.attempt_id, + desired_key, + source_root.0, + i64::try_from(attempt.owner_generation).map_err(|_| Error::InvalidInput( + "artifact construction generation exceeds SQLite range".into() + ))?, + i64::from(attempt.owner_pid), + attempt.owner_start_token, + ], + |_| Ok(()), + ) + .optional()? + .is_some(); + if !matched { + return Err(Error::InvalidInput(format!( + "artifact construction attempt `{}` changed its source, desired key, owner, phase, or cancellation pin before sealing", + attempt.attempt_id + ))); + } + let _: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, source_root)?; + Ok(()) + } + + /// Terminalize construction attempts whose exact process owner is proven + /// dead and remove only staging/lock paths derived from that fenced owner. + /// This is called by workspace-open recovery while the workspace write + /// lock is already held. + pub(crate) fn recover_artifact_construction_attempts_under_write_lock(&self) -> Result<()> { + let attempts = { + let mut statement = self.conn.prepare( + "SELECT attempt_id,desired_key,owner_generation,owner_pid, + owner_start_token,cancel_requested + FROM artifact_construction_attempts + WHERE status='running' ORDER BY started_at,attempt_id", + )?; + statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, String>(4)?, + row.get::<_, bool>(5)?, + )) + })? + .collect::, _>>()? + }; + for ( + attempt_id, + desired_key, + owner_generation, + owner_pid, + owner_start_token, + cancel_requested, + ) in attempts + { + if !valid_workspace_cache_key(&desired_key) { + return Err(Error::Corrupt(format!( + "artifact construction attempt `{attempt_id}` has an invalid desired key; restore from backup or reinitialize the workspace" + ))); + } + let owner_pid = u32::try_from(owner_pid).map_err(|_| { + Error::Corrupt(format!( + "artifact construction attempt `{attempt_id}` has invalid owner PID" + )) + })?; + if process_start_token_match(owner_pid, &owner_start_token) + != ProcessIdentityMatch::DeadOrMismatch + { + continue; + } + let owner_token = format!("{owner_pid}:{owner_start_token}"); + let staging = self.db_dir.join("cache/staging").join(format!( + "input_{}", + crate::ids::short_hash(format!("{desired_key}:{owner_token}").as_bytes(), 12) + )); + remove_exact_recovery_directory(&staging, "artifact construction staging")?; + + let lock_path = self + .db_dir + .join("cache/staging/locks") + .join(format!("{desired_key}.lock")); + if fs::read_to_string(&lock_path) + .ok() + .is_some_and(|value| value == owner_token) + { + fs::remove_file(&lock_path)?; + } + + self.conn + .execute_batch("SAVEPOINT trail_construction_recovery")?; + let recovery = (|| -> Result<()> { + let status = if cancel_requested { + "cancelled" + } else { + "abandoned" + }; + let reason_code = if cancel_requested { + "construction_cancelled" + } else { + "owner_lost" + }; + let reason = if cancel_requested { + "construction cancellation was recovered after its owner exited" + } else { + "construction owner exited before artifact publication" + }; + let updated = self.conn.execute( + "UPDATE artifact_construction_attempts + SET phase='completed',status=?1,reason_code=?2,reason=?3, + heartbeat_at=?4,finished_at=?4 + WHERE attempt_id=?5 AND desired_key=?6 AND owner_generation=?7 + AND owner_pid=?8 AND owner_start_token=?9 AND status='running'", + params![ + status, + reason_code, + reason, + now_ts(), + attempt_id, + desired_key, + owner_generation, + i64::from(owner_pid), + owner_start_token, + ], + )?; + if updated != 1 { + return Err(Error::Corrupt(format!( + "artifact construction attempt `{attempt_id}` changed during exact-owner recovery" + ))); + } + self.conn.execute( + "UPDATE artifact_construction_waiters + SET status='abandoned',updated_at=?1 + WHERE attempt_id=?2 AND status='waiting'", + params![now_ts(), attempt_id], + )?; + Ok(()) + })(); + match recovery { + Ok(()) => self + .conn + .execute_batch("RELEASE SAVEPOINT trail_construction_recovery")?, + Err(error) => { + let _ = self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT trail_construction_recovery; + RELEASE SAVEPOINT trail_construction_recovery", + ); + return Err(error); + } + } + } + Ok(()) + } + + pub(crate) fn build_workspace_layer_singleflight( &self, key: &WorkspaceLayerKeyV1, + source_root: &ObjectId, + builder: F, + ) -> Result + where + F: FnOnce(&Path) -> Result, + { + self.build_workspace_layer_singleflight_with_cancel( + key, + source_root, + &AtomicBool::new(false), + builder, + ) + } + + pub(crate) fn build_workspace_layer_singleflight_with_cancel( + &self, + key: &WorkspaceLayerKeyV1, + source_root: &ObjectId, cancelled: &AtomicBool, builder: F, ) -> Result @@ -1098,8 +2219,12 @@ impl Trail { fs::create_dir_all(lock_path.parent().unwrap())?; let token = format!("{}:{}", std::process::id(), current_process_start_token()); let deadline = Instant::now() + Duration::from_secs(LAYER_BUILD_LEASE_SECS as u64); + let mut waiter = None; let guard = loop { if cancelled.load(AtomicOrdering::Acquire) { + if let Some(waiter) = waiter.as_ref() { + self.finish_artifact_construction_waiter(waiter, "cancelled")?; + } return Err(Error::InvalidInput(format!( "cancelled while waiting for workspace layer key {cache_key}" ))); @@ -1121,6 +2246,9 @@ impl Trail { if let Some(layer) = self.workspace_layer_by_cache_key(&cache_key)? && layer.state == "ready" { + if let Some(waiter) = waiter.as_ref() { + self.finish_artifact_construction_waiter(waiter, "released")?; + } return self.verify_workspace_layer_for_attach(&layer.layer_id); } if build_lock_is_stale(&lock_path)? { @@ -1128,12 +2256,21 @@ impl Trail { let _ = fs::rename(&lock_path, stale); continue; } + if waiter.is_none() { + waiter = self.register_artifact_construction_waiter(&cache_key)?; + } if Instant::now() >= deadline { + if let Some(waiter) = waiter.as_ref() { + self.finish_artifact_construction_waiter(waiter, "cancelled")?; + } return Err(Error::InvalidInput(format!( "timed out waiting for workspace layer key {cache_key}" ))); } if cancelled.load(AtomicOrdering::Acquire) { + if let Some(waiter) = waiter.as_ref() { + self.finish_artifact_construction_waiter(waiter, "cancelled")?; + } return Err(Error::InvalidInput(format!( "cancelled while waiting for workspace layer key {cache_key}" ))); @@ -1143,10 +2280,20 @@ impl Trail { Err(err) => return Err(Error::Io(err)), } }; + if let Some(waiter) = waiter.as_ref() { + self.finish_artifact_construction_waiter(waiter, "abandoned")?; + } + let attempt = self.begin_artifact_construction_attempt(&cache_key, source_root)?; let builder_limit = self.config().workspace_views.concurrent_cache_builders; if builder_limit > 0 && active_cache_builder_count(lock_path.parent().unwrap())? > builder_limit { + self.finish_artifact_construction_attempt( + &attempt, + "failed", + Some("builder_quota_exceeded"), + Some("workspace cache builder quota exceeded"), + )?; drop(guard); return Err(Error::InvalidInput(format!( "workspace cache builder quota exceeded (limit {builder_limit})" @@ -1155,6 +2302,7 @@ impl Trail { if let Some(layer) = self.workspace_layer_by_cache_key(&cache_key)? && layer.state == "ready" { + self.finish_artifact_construction_attempt(&attempt, "succeeded", None, None)?; drop(guard); return self.verify_workspace_layer_for_attach(&layer.layer_id); } @@ -1163,40 +2311,96 @@ impl Trail { crate::ids::short_hash(format!("{cache_key}:{token}").as_bytes(), 12) )); if build_dir.exists() { + self.finish_artifact_construction_attempt( + &attempt, + "failed", + Some("staging_path_collision"), + Some("workspace layer build directory already exists"), + )?; return Err(Error::InvalidInput(format!( "workspace layer build directory `{}` already exists", build_dir.display() ))); } - fs::create_dir_all(&build_dir)?; + if let Err(error) = fs::create_dir_all(&build_dir) { + self.finish_artifact_construction_attempt( + &attempt, + "failed", + Some("staging_create_failed"), + Some("workspace layer staging directory could not be created"), + )?; + return Err(Error::Io(error)); + } + self.advance_artifact_construction_attempt(&attempt, "building")?; let output = match builder(&build_dir) { Ok(output) => output, Err(err) => { make_tree_writable(&build_dir); let failed = build_dir.with_extension(format!("failed.{}", now_ts())); let _ = fs::rename(&build_dir, failed); + self.finish_artifact_construction_attempt( + &attempt, + "failed", + Some("builder_failed"), + Some("workspace layer builder failed"), + )?; return Err(err); } }; - self.enforce_workspace_cache_build_quota(&output)?; + self.advance_artifact_construction_attempt(&attempt, "validating")?; + if let Err(error) = self.enforce_workspace_cache_build_quota(&output) { + self.finish_artifact_construction_attempt( + &attempt, + "failed", + Some("output_quota_exceeded"), + Some("workspace layer output failed quota validation"), + )?; + return Err(error); + } + if let Err(error) = + self.verify_artifact_construction_seal_fence(&attempt, &cache_key, source_root) + { + self.finish_artifact_construction_attempt( + &attempt, + "failed", + Some("seal_pin_changed"), + Some("artifact construction pins changed before sealing"), + )?; + return Err(error); + } // Different ready DAG nodes may finish their external builds at the // same time. Publication is intentionally serialized by the // workspace lock, so a builder waits for that short commit boundary // instead of treating another independent builder as a command // conflict. + self.advance_artifact_construction_attempt(&attempt, "publishing")?; let report = Self::with_write_lock_wait(Duration::from_secs(LAYER_BUILD_LEASE_SECS as u64), || { self.publish_workspace_layer_from_directory(key, &output) - })?; + }); + let report = match report { + Ok(report) => report, + Err(error) => { + self.finish_artifact_construction_attempt( + &attempt, + "failed", + Some("publication_failed"), + Some("workspace layer publication failed"), + )?; + return Err(error); + } + }; + self.finish_artifact_construction_attempt(&attempt, "succeeded", None, None)?; make_tree_writable(&build_dir); let _ = fs::remove_dir_all(&build_dir); drop(guard); Ok(report) } - /// Publish a prebuilt directory as an immutable cache layer. The - /// workspace write lock is the singleflight boundary: concurrent callers - /// for the same canonical key observe one completed publish and reuse it. + /// Publish a prebuilt directory as an immutable cache layer. Callers that + /// need construction singleflight use `build_workspace_layer_singleflight`; + /// this operation serializes the short publication transaction and reuses + /// an already-ready canonical key. pub fn publish_workspace_layer_from_directory( &self, key: &WorkspaceLayerKeyV1, @@ -1273,13 +2477,35 @@ impl Trail { now, ], )?; + test_crash_point("layer_after_reservation"); let publish = (|| -> Result { copy_layer_tree(source, &staging)?; - let _validated_entries = scan_layer_entries(&staging, false)?; + let validated_entries = scan_layer_entries(&staging, false)?; sync_layer_tree(&staging)?; test_crash_point("layer_after_staging_sync"); + let (artifact_tree_id, artifact_tree) = + self.ingest_artifact_tree_under_write_lock(&staging)?; + test_crash_point("layer_after_cas_tree"); + let stable_entries = scan_layer_entries(&staging, false)?; + if stable_entries != validated_entries { + return Err(Error::InvalidInput( + "workspace layer candidate changed while Trail was sealing its content".into(), + )); + } + verify_artifact_shadow_matches_layer_entries( + &artifact_tree, + &self.artifact_tree_flat_entries(&artifact_tree_id)?, + &stable_entries, + )?; let entries = scan_layer_entries(&staging, true)?; + sync_layer_tree(&staging)?; + let artifact_envelope_id = self.put_legacy_artifact_envelope_under_write_lock( + key, + &cache_key, + artifact_tree_id.clone(), + )?; + test_crash_point("layer_after_envelope_ready"); let logical_bytes = entries.values().map(|entry| entry.size_bytes).sum(); let entry_count = entries.len() as u64; let mut entry_pages = Vec::new(); @@ -1353,6 +2579,17 @@ impl Trail { )?; sync_directory(final_path.parent().unwrap()); test_crash_point("layer_after_atomic_rename"); + self.conn.execute( + "INSERT OR REPLACE INTO workspace_layer_artifact_shadows( + layer_id, tree_root_id, envelope_id, state, verified_at + ) VALUES(?1, ?2, ?3, 'verified', ?4)", + params![ + layer_id, + artifact_tree_id.0, + artifact_envelope_id.0, + now_ts() + ], + )?; self.conn.execute( "UPDATE workspace_layers SET manifest_object_id = ?1, storage_path = ?2, state = 'ready', logical_bytes = ?3, physical_bytes = ?4, entry_count = ?5, builder_id = NULL, lease_expires_at = NULL, last_used_at = ?6 WHERE cache_key = ?7", params![ @@ -1461,6 +2698,18 @@ impl Trail { "workspace layer `{layer_id}` cannot recover because its published tree is corrupt" ))); } + let (artifact_tree_id, artifact_tree) = + self.ingest_artifact_tree_under_write_lock(final_path)?; + verify_artifact_shadow_matches_layer_entries( + &artifact_tree, + &self.artifact_tree_flat_entries(&artifact_tree_id)?, + &actual, + )?; + let artifact_envelope_id = self.put_legacy_artifact_envelope_under_write_lock( + key, + cache_key, + artifact_tree_id.clone(), + )?; self.conn.execute( "INSERT INTO workspace_layers (layer_id, kind, cache_key, adapter, adapter_version, manifest_object_id, storage_path, state, logical_bytes, physical_bytes, entry_count, portability_scope, builder_id, lease_expires_at, last_used_at, created_at) \ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'ready', ?8, ?9, ?10, ?11, NULL, NULL, ?12, ?12) \ @@ -1480,6 +2729,17 @@ impl Trail { now_ts(), ], )?; + self.conn.execute( + "INSERT OR REPLACE INTO workspace_layer_artifact_shadows( + layer_id, tree_root_id, envelope_id, state, verified_at + ) VALUES(?1, ?2, ?3, 'verified', ?4)", + params![ + layer_id, + artifact_tree_id.0, + artifact_envelope_id.0, + now_ts() + ], + )?; let report = self .workspace_layer_by_cache_key(cache_key)? .ok_or_else(|| { @@ -1560,7 +2820,11 @@ impl Trail { pub(crate) fn workspace_layer_object_roots(&self) -> Result> { let mut statement = self.conn.prepare( "SELECT manifest_object_id FROM workspace_layers - WHERE manifest_object_id IS NOT NULL ORDER BY layer_id", + WHERE manifest_object_id IS NOT NULL + UNION + SELECT manifest_object_id FROM workspace_layer_publications + WHERE manifest_object_id IS NOT NULL + ORDER BY manifest_object_id", )?; let manifests = statement .query_map([], |row| row.get::<_, String>(0))? @@ -1640,9 +2904,41 @@ impl Trail { )?; let blobs = cache_tree_usage(&self.db_dir.join("cache/blobs"))?; let environment_caches = cache_tree_usage(&self.db_dir.join("cache/namespaces"))?; + let artifact_materializations = self.conn.query_row( + "SELECT COALESCE(SUM(COALESCE(physical_bytes, 0)), 0) + FROM artifact_materializations WHERE state != 'building'", + [], + |row| row.get::<_, i64>(0), + )?; Ok((layer_bytes.max(0) as u64) .saturating_add(blobs) - .saturating_add(environment_caches)) + .saturating_add(environment_caches) + .saturating_add(artifact_materializations.max(0) as u64)) + } + + pub(crate) fn workspace_view_reclaimable_artifact_bytes(&self, view_id: &str) -> Result { + let bytes = self.conn.query_row( + "WITH selected_trees(tree_root_id) AS ( + SELECT b.tree_root_id + FROM environment_view_generations v + JOIN artifact_generation_bindings b ON b.generation_id=v.generation_id + WHERE v.view_id=?1 + UNION + SELECT s.tree_root_id + FROM workspace_view_layers l + JOIN workspace_layer_artifact_shadows s ON s.layer_id=l.layer_id + WHERE l.view_id=?1 + ) + SELECT COALESCE(SUM(COALESCE(m.physical_bytes,0)),0) + FROM artifact_materializations m + JOIN selected_trees t ON t.tree_root_id=m.tree_root_id + WHERE m.state != 'building'", + params![view_id], + |row| row.get::<_, i64>(0), + )?; + u64::try_from(bytes).map_err(|_| { + Error::Corrupt("artifact materialization has negative physical bytes".into()) + }) } pub fn workspace_cache_gc( @@ -1655,6 +2951,12 @@ impl Trail { let cutoff = now_ts().saturating_sub(retention_secs.min(i64::MAX as u64) as i64); let cache_root = self.db_dir.join("cache"); let cache_physical_bytes_before = cache_tree_usage(&cache_root)?; + let demand_loaded_bytes = cache_tree_usage(&cache_root.join("blobs"))?; + let mut artifact_storage = + self.artifact_storage_accounting(None, 0, demand_loaded_bytes, 0, 0)?; + artifact_storage.unknown_bytes = cache_physical_bytes_before + .saturating_sub(demand_loaded_bytes) + .saturating_sub(artifact_storage.materialized_bytes); let mut found = Vec::::new(); { let mut stmt = self.conn.prepare( @@ -1745,6 +3047,53 @@ impl Trail { }); } } + { + let mut stmt = self.conn.prepare( + "SELECT materialization_id,storage_path,COALESCE(physical_bytes,0), + last_used_at,state + FROM artifact_materializations + ORDER BY last_used_at ASC,materialization_id ASC", + )?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?.max(0) as u64, + row.get::<_, i64>(3)?, + row.get::<_, String>(4)?, + )) + })?; + for row in rows { + let (materialization_id, storage_path, physical_bytes, last_used_at, state) = row?; + if state == "building" { + continue; + } + let expected = cache_root + .join("artifact-materializations") + .join(&materialization_id); + if Path::new(&storage_path) != expected { + return Err(Error::Corrupt(format!( + "artifact materialization `{materialization_id}` has invalid storage path `{storage_path}`" + ))); + } + found.push(CacheGcCandidate { + entry: WorkspaceCacheGcEntry { + kind: "artifact_materialization".to_string(), + id: materialization_id, + path: storage_path, + physical_bytes, + pinned: false, + reason: if last_used_at <= cutoff { + "artifact_materialization_retention_expired".to_string() + } else { + "artifact_materialization_lru".to_string() + }, + }, + last_used_at, + retention_expired: last_used_at <= cutoff, + }); + } + } let blob_root = cache_root.join("blobs"); if blob_root.exists() { for entry in walkdir::WalkDir::new(&blob_root).follow_links(false) { @@ -1815,6 +3164,7 @@ impl Trail { let reclaimable_bytes = selected .iter() .fold(0_u64, |sum, item| sum.saturating_add(item.physical_bytes)); + artifact_storage.reclaimable_bytes = reclaimable_bytes; if dry_run { return Ok(WorkspaceCacheGcReport { dry_run, @@ -1824,6 +3174,7 @@ impl Trail { reclaimed_bytes: 0, candidates: selected, deleted: Vec::new(), + artifact_storage, }); } @@ -1897,16 +3248,16 @@ impl Trail { path.display() ))); } - if let Err(err) = self.conn.execute( - "DELETE FROM workspace_layers WHERE layer_id = ?1 - AND NOT EXISTS (SELECT 1 FROM workspace_view_layers WHERE layer_id = ?1) - AND NOT EXISTS (SELECT 1 FROM environment_generation_components WHERE layer_id = ?1) - AND NOT EXISTS (SELECT 1 FROM workspace_layer_publications WHERE layer_id = ?1 AND phase IN ('published','activated')) - AND NOT EXISTS (SELECT 1 FROM workspace_layer_pins WHERE layer_id = ?1 AND (expires_at IS NULL OR expires_at > unixepoch()))", - params![candidate.id], - ) { - let _ = fs::rename(&trash_path, &path); - return Err(Error::from(err)); + match self.delete_workspace_layer_row_and_shadow(&candidate.id) { + Ok(true) => {} + Ok(false) => { + let _ = fs::rename(&trash_path, &path); + continue; + } + Err(error) => { + let _ = fs::rename(&trash_path, &path); + return Err(error); + } } make_tree_writable(&trash_path); fs::remove_dir_all(&trash_path).map_err(|err| { @@ -1916,14 +3267,9 @@ impl Trail { )) })?; } else { - self.conn.execute( - "DELETE FROM workspace_layers WHERE layer_id = ?1 - AND NOT EXISTS (SELECT 1 FROM workspace_view_layers WHERE layer_id = ?1) - AND NOT EXISTS (SELECT 1 FROM environment_generation_components WHERE layer_id = ?1) - AND NOT EXISTS (SELECT 1 FROM workspace_layer_publications WHERE layer_id = ?1 AND phase IN ('published','activated')) - AND NOT EXISTS (SELECT 1 FROM workspace_layer_pins WHERE layer_id = ?1 AND (expires_at IS NULL OR expires_at > unixepoch()))", - params![candidate.id], - )?; + if !self.delete_workspace_layer_row_and_shadow(&candidate.id)? { + continue; + } remove_workspace_layer_trash_entries(&trash, &candidate.id)?; } let _ = fs::remove_file(workspace_layer_marker_path(&path)); @@ -1993,6 +3339,85 @@ impl Trail { .join("cache/namespace-leases") .join(&candidate.id), ); + } else if candidate.kind == "artifact_materialization" { + let expected = cache_root + .join("artifact-materializations") + .join(&candidate.id); + if path != expected { + return Err(Error::Corrupt(format!( + "artifact materialization `{}` escaped cache storage", + candidate.id + ))); + } + let state = self + .conn + .query_row( + "SELECT state FROM artifact_materializations + WHERE materialization_id=?1", + params![candidate.id], + |row| row.get::<_, String>(0), + ) + .optional()?; + if state.as_deref() == Some("building") { + continue; + } + let trash_path = trash.join(format!( + "artifact-materialization.{}.{}", + candidate.id, + crate::ids::short_hash( + format!("{}:{}", candidate.id, now_nanos()).as_bytes(), + 12 + ) + )); + if path.exists() { + let metadata = fs::symlink_metadata(&path)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(Error::Corrupt(format!( + "artifact materialization `{}` is not a real directory", + candidate.id + ))); + } + let root_mode = layer_mode(&metadata); + make_tree_writable(&path); + if let Err(error) = fs::rename(&path, &trash_path) { + let _ = set_layer_read_only(&path, true, root_mode); + return Err(Error::InvalidInput(format!( + "failed to quarantine artifact materialization `{}`: {error}", + path.display() + ))); + } + } + let removed = self.conn.execute( + "DELETE FROM artifact_materializations + WHERE materialization_id=?1 AND state!='building'", + params![candidate.id], + ); + match removed { + Ok(1) => {} + Ok(_) => { + if trash_path.exists() { + let _ = fs::rename(&trash_path, &path); + let _ = set_layer_read_only(&path, true, 0o755); + } + continue; + } + Err(error) => { + if trash_path.exists() { + let _ = fs::rename(&trash_path, &path); + let _ = set_layer_read_only(&path, true, 0o755); + } + return Err(Error::from(error)); + } + } + if trash_path.exists() { + make_tree_writable(&trash_path); + fs::remove_dir_all(&trash_path).map_err(|error| { + Error::InvalidInput(format!( + "failed to remove quarantined artifact materialization `{}`: {error}", + trash_path.display() + )) + })?; + } } else { if !path.is_file() { continue; @@ -2029,9 +3454,61 @@ impl Trail { reclaimed_bytes, candidates: selected, deleted, + artifact_storage, }) } + fn delete_workspace_layer_row_and_shadow(&self, layer_id: &str) -> Result { + self.conn + .execute_batch("SAVEPOINT trail_workspace_layer_gc")?; + let deletion = (|| -> Result { + let pinned = self.conn.query_row( + "SELECT (EXISTS(SELECT 1 FROM workspace_view_layers WHERE layer_id=?1) + OR EXISTS(SELECT 1 FROM environment_generation_components WHERE layer_id=?1) + OR EXISTS(SELECT 1 FROM workspace_layer_publications WHERE layer_id=?1 AND phase IN ('published','activated')) + OR EXISTS(SELECT 1 FROM workspace_layer_pins WHERE layer_id=?1 AND (expires_at IS NULL OR expires_at > unixepoch())))", + params![layer_id], + |row| row.get::<_, i64>(0), + )? != 0; + if pinned { + return Ok(false); + } + self.conn.execute( + "DELETE FROM workspace_layer_artifact_shadows WHERE layer_id=?1", + params![layer_id], + )?; + Ok(self.conn.execute( + "DELETE FROM workspace_layers WHERE layer_id=?1 + AND NOT EXISTS (SELECT 1 FROM workspace_view_layers WHERE layer_id=?1) + AND NOT EXISTS (SELECT 1 FROM environment_generation_components WHERE layer_id=?1) + AND NOT EXISTS (SELECT 1 FROM workspace_layer_publications WHERE layer_id=?1 AND phase IN ('published','activated')) + AND NOT EXISTS (SELECT 1 FROM workspace_layer_pins WHERE layer_id=?1 AND (expires_at IS NULL OR expires_at > unixepoch()))", + params![layer_id], + )? != 0) + })(); + match deletion { + Ok(true) => { + self.conn + .execute_batch("RELEASE SAVEPOINT trail_workspace_layer_gc")?; + Ok(true) + } + Ok(false) => { + self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT trail_workspace_layer_gc; + RELEASE SAVEPOINT trail_workspace_layer_gc", + )?; + Ok(false) + } + Err(error) => { + let _ = self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT trail_workspace_layer_gc; + RELEASE SAVEPOINT trail_workspace_layer_gc", + ); + Err(error) + } + } + } + pub(crate) fn workspace_view_layer_reports( &self, view_id: &str, @@ -2360,6 +3837,61 @@ impl Trail { ) } + fn validate_artifact_v2_activation( + &self, + layer: &WorkspaceLayerReport, + activation: &EnvironmentLayerActivation, + ) -> Result { + let (tree_root_id, envelope_id, state) = self.conn.query_row( + "SELECT tree_root_id,envelope_id,state + FROM workspace_layer_artifact_shadows WHERE layer_id=?1", + params![&layer.layer_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }, + )?; + if state != "verified" { + return Err(Error::Corrupt(format!( + "workspace layer `{}` has an unverified artifact binding", + layer.layer_id + ))); + } + let tree_root_id = ArtifactTreeId::parse(tree_root_id) + .map_err(|error| Error::Corrupt(format!("invalid artifact tree ID: {error}")))?; + let envelope_id = ArtifactEnvelopeId::parse(envelope_id) + .map_err(|error| Error::Corrupt(format!("invalid artifact envelope ID: {error}")))?; + let envelope = + self.verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_root_id)?; + let ArtifactDesiredIdentityV1::ArtifactDesiredV2 { desired_key } = + &envelope.desired_identity + else { + return Ok(false); + }; + let matching_outputs = activation + .outputs + .iter() + .filter(|output| { + output.name == envelope.output_name && output.policy == envelope.output_policy + }) + .count(); + if desired_key.0 != activation.expected_key + || envelope.component_id != activation.component_id + || envelope.portability_scope != layer.portability_scope + || activation.outputs.len() != 1 + || matching_outputs != 1 + { + return Err(Error::InvalidInput(format!( + "environment component `{}` does not match its artifact-v2 desired/output contract", + activation.component_id + ))); + } + Ok(true) + } + fn replace_declared_workspace_layers_with_removals_internal( &self, lane: &str, @@ -2453,13 +3985,6 @@ impl Trail { let mut resolved = Vec::with_capacity(requested.len()); let mut mount_paths = Vec::<(String, String)>::new(); for activation in requested { - if self.workspace_layer_cache_key(&activation.canonical_key)? != activation.expected_key - { - return Err(Error::Corrupt(format!( - "environment component `{}` canonical key does not match its expected key", - activation.component_id - ))); - } let mut dependency_ids = BTreeSet::new(); for (dependency, component_key, edge_type) in &activation.dependencies { super::workspace_environment::validate_environment_component_identity(dependency)?; @@ -2574,6 +4099,28 @@ impl Trail { .as_deref() .map(|layer_id| self.verify_workspace_layer_for_attach(layer_id)) .transpose()?; + let canonical_storage_key = + self.workspace_layer_cache_key(&activation.canonical_key)?; + if layer + .as_ref() + .is_some_and(|layer| layer.cache_key != canonical_storage_key) + { + return Err(Error::Corrupt(format!( + "environment component `{}` canonical storage key does not match its layer", + activation.component_id + ))); + } + let artifact_v2 = layer + .as_ref() + .map(|layer| self.validate_artifact_v2_activation(layer, activation)) + .transpose()? + .unwrap_or(false); + if !artifact_v2 && canonical_storage_key != activation.expected_key { + return Err(Error::Corrupt(format!( + "environment component `{}` canonical key does not match its expected key", + activation.component_id + ))); + } if let Some(layer) = &layer && layer.kind != activation.kind { @@ -2709,7 +4256,13 @@ impl Trail { private_seed: output.private_seed.clone(), }); } - resolved.push((activation.clone(), layer, outputs, previous_bindings)); + resolved.push(( + activation.clone(), + layer, + outputs, + previous_bindings, + artifact_v2, + )); } self.validate_environment_batch_mount_ownership( &view.view_id, @@ -2741,7 +4294,7 @@ impl Trail { } } } - for (component, layer, outputs, previous_bindings) in &resolved { + for (component, layer, outputs, previous_bindings, _) in &resolved { for output in outputs { if matches!( output.policy, @@ -2885,34 +4438,36 @@ impl Trail { params![&view.view_id, component_id], )?; } - for (component, layer, outputs, previous_bindings) in &resolved { - self.conn.execute( - "INSERT OR IGNORE INTO environment_component_key_provenance - (component_key, canonical_key_json, created_at) VALUES (?1, ?2, ?3)", - params![ - &component.expected_key, - serde_json::to_vec(&component.canonical_key)?, - now_ts() - ], - )?; - let stored_key = self.conn.query_row( - "SELECT canonical_key_json FROM environment_component_key_provenance - WHERE component_key = ?1", - params![&component.expected_key], - |row| row.get::<_, Vec>(0), - )?; - let stored_key: WorkspaceLayerKeyV1 = - serde_json::from_slice(&stored_key).map_err(|error| { - Error::Corrupt(format!( - "environment key provenance `{}` is malformed: {error}", + for (component, layer, outputs, previous_bindings, artifact_v2) in &resolved { + if !artifact_v2 { + self.conn.execute( + "INSERT OR IGNORE INTO environment_component_key_provenance + (component_key, canonical_key_json, created_at) VALUES (?1, ?2, ?3)", + params![ + &component.expected_key, + serde_json::to_vec(&component.canonical_key)?, + now_ts() + ], + )?; + let stored_key = self.conn.query_row( + "SELECT canonical_key_json FROM environment_component_key_provenance + WHERE component_key = ?1", + params![&component.expected_key], + |row| row.get::<_, Vec>(0), + )?; + let stored_key: WorkspaceLayerKeyV1 = serde_json::from_slice(&stored_key) + .map_err(|error| { + Error::Corrupt(format!( + "environment key provenance `{}` is malformed: {error}", + component.expected_key + )) + })?; + if stored_key != component.canonical_key { + return Err(Error::Corrupt(format!( + "environment key provenance `{}` does not match its content identity", component.expected_key - )) - })?; - if stored_key != component.canonical_key { - return Err(Error::Corrupt(format!( - "environment key provenance `{}` does not match its content identity", - component.expected_key - ))); + ))); + } } for (previous_mount, _, _, _) in previous_bindings { self.conn.execute( @@ -3543,6 +5098,102 @@ impl Trail { Ok(()) } + fn record_artifact_generation_binding( + &self, + generation_id: &str, + component: &EnvironmentGenerationComponentReport, + output: &EnvironmentGenerationOutputReport, + created_at: i64, + ) -> Result<()> { + if !output.policy.has_immutable_layer() { + return Ok(()); + } + let layer_id = output.layer_id.as_deref().ok_or_else(|| { + Error::Corrupt(format!( + "immutable generation output `{}/{}` has no workspace layer", + component.component_id, output.name + )) + })?; + let (tree_root_id, envelope_id, state) = self.conn.query_row( + "SELECT tree_root_id,envelope_id,state + FROM workspace_layer_artifact_shadows WHERE layer_id=?1", + params![layer_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }, + )?; + if state != "verified" { + return Err(Error::Corrupt(format!( + "workspace layer `{layer_id}` has an unverified artifact binding" + ))); + } + let tree_root_id = ArtifactTreeId::parse(tree_root_id) + .map_err(|error| Error::Corrupt(format!("invalid artifact tree ID: {error}")))?; + let envelope_id = ArtifactEnvelopeId::parse(envelope_id) + .map_err(|error| Error::Corrupt(format!("invalid artifact envelope ID: {error}")))?; + let envelope = + self.verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_root_id)?; + let desired_key = match &envelope.desired_identity { + ArtifactDesiredIdentityV1::WorkspaceLayerV1 { cache_key, .. } => cache_key.clone(), + ArtifactDesiredIdentityV1::ArtifactDesiredV2 { desired_key } => { + if desired_key.0 != component.component_key + || envelope.component_id != component.component_id + || envelope.output_name != output.name + || envelope.output_policy != output.policy + { + return Err(Error::Corrupt(format!( + "artifact-v2 envelope does not match generation output `{}/{}`", + component.component_id, output.name + ))); + } + desired_key.0.clone() + } + }; + if desired_key != component.component_key { + return Err(Error::Corrupt(format!( + "artifact desired key does not match generation component `{}`", + component.component_id + ))); + } + let binding_identity = format!( + "artifact_binding_{}", + crate::ids::short_hash( + format!( + "{}\0{}\0{}\0{}", + generation_id, component.component_id, output.name, envelope_id.0 + ) + .as_bytes(), + 32, + ) + ); + let binding_id = format!( + "binding_{}", + crate::ids::short_hash(binding_identity.as_bytes(), 32) + ); + self.conn.execute( + "INSERT INTO artifact_generation_bindings( + binding_id,generation_id,component_id,output_name,desired_key, + envelope_id,tree_root_id,binding_identity,created_at) + VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9)", + params![ + binding_id, + generation_id, + &component.component_id, + &output.name, + desired_key, + envelope_id.0, + tree_root_id.0, + binding_identity, + created_at + ], + )?; + Ok(()) + } + fn record_environment_generation(&self, lane: &str, view_id: &str) -> Result { let branch = self.lane_branch(lane)?; let head = self.get_ref(&branch.ref_name)?; @@ -3750,6 +5401,7 @@ impl Trail { &output.layer_subpath ], )?; + self.record_artifact_generation_binding(&generation_id, component, output, now)?; } for dependency in &component.dependencies { self.conn.execute( @@ -3974,9 +5626,12 @@ impl Trail { source_upper: &Path, ) -> Result> { let mut stmt = self.conn.prepare( - "SELECT l.layer_id, b.mount_path, l.storage_path, b.source_path, l.kind, b.priority \ + "SELECT l.layer_id, b.mount_path, l.storage_path, b.source_path, l.kind, b.priority, \ + s.tree_root_id \ FROM workspace_views v JOIN workspace_view_layers b ON b.view_id = v.view_id \ JOIN workspace_layers l ON l.layer_id = b.layer_id \ + LEFT JOIN workspace_layer_artifact_shadows s \ + ON s.layer_id = l.layer_id AND s.state = 'verified' \ WHERE v.source_upper = ?1 AND b.read_only = 1 AND l.state = 'ready' \ ORDER BY length(b.mount_path) DESC, b.priority DESC", )?; @@ -3989,6 +5644,7 @@ impl Trail { row.get::<_, String>(3)?, row.get::<_, String>(4)?, row.get::<_, i64>(5)?, + row.get::<_, Option>(6)?, )) }) .map_err(Error::from)? @@ -3996,7 +5652,23 @@ impl Trail { let mut bindings = rows .into_iter() .map( - |(layer_id, mount_path, storage_path, source_path, kind, priority)| { + |( + layer_id, + mount_path, + storage_path, + source_path, + kind, + priority, + artifact_tree_id, + )| { + let artifact_tree_id = artifact_tree_id + .map(ArtifactTreeId::parse) + .transpose() + .map_err(|reason| { + Error::Corrupt(format!( + "workspace layer `{layer_id}` has invalid artifact tree identity: {reason}" + )) + })?; let storage_path = if source_path.is_empty() { PathBuf::from(storage_path) } else { @@ -4007,6 +5679,8 @@ impl Trail { layer_id: Some(layer_id), mount_path, storage_path: Some(storage_path), + artifact_tree_id, + artifact_subpath: source_path, kind, priority, }) @@ -4027,6 +5701,8 @@ impl Trail { layer_id: None, mount_path: row.get(1)?, storage_path: None, + artifact_tree_id: None, + artifact_subpath: String::new(), kind: row.get(2)?, priority: 100, }) @@ -4045,6 +5721,48 @@ impl Trail { } } +fn validate_private_output_promotion_taint( + generation_secret_tainted: bool, + producer_receipt: Option<&serde_json::Value>, +) -> Result>> { + if generation_secret_tainted { + return Err(Error::InvalidInput( + "secret-tainted private output is non-promotable and must remain lane-private" + .to_string(), + )); + } + let Some(producer_receipt) = producer_receipt else { + return Ok(None); + }; + let secret_taint = match producer_receipt.get("secret_taint") { + Some(value) => { + serde_json::from_value::(value.clone()).map_err(|error| { + Error::InvalidInput(format!( + "producer receipt has malformed secret-taint evidence: {error}" + )) + })? + } + None => ArtifactSecretTaintV1::Clear, + }; + super::workspace_artifact::validate_artifact_secret_taint(&secret_taint)?; + if !secret_taint.is_clear() { + return Err(Error::InvalidInput( + "secret-tainted private output is non-promotable and must remain lane-private" + .to_string(), + )); + } + let mut non_taint_evidence = producer_receipt.clone(); + if let Some(object) = non_taint_evidence.as_object_mut() { + object.remove("secret_taint"); + } + if contains_sensitive_json(&non_taint_evidence) { + return Err(Error::InvalidInput( + "producer receipt may contain secret material and cannot be stored".to_string(), + )); + } + Ok(Some(serde_json::to_vec(producer_receipt)?)) +} + struct EnvironmentRuntimeAllocationNames { allocation_id: String, container_name: String, @@ -4175,6 +5893,63 @@ struct CacheBuildKeyGuard { token: String, } +fn remove_exact_recovery_directory(path: &Path, purpose: &str) -> Result<()> { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(Error::Io(error)), + }; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(Error::Corrupt(format!( + "{purpose} path `{}` is not a real directory; move it outside `.trail` and reopen Trail", + path.display() + ))); + } + make_tree_writable(path); + fs::remove_dir_all(path)?; + Ok(()) +} + +fn valid_workspace_layer_id(value: &str) -> bool { + value.strip_prefix("layer_").is_some_and(|suffix| { + suffix.len() == 32 + && suffix + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) +} + +fn valid_workspace_cache_key(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn workspace_layer_storage_path_matches_id(value: &str, layer_id: &str) -> bool { + let path = Path::new(value); + path.is_absolute() + && !path.components().any(|component| { + matches!( + component, + std::path::Component::CurDir | std::path::Component::ParentDir + ) + }) + && path.ends_with( + Path::new(".trail") + .join("cache") + .join("layers") + .join(layer_id), + ) +} + +fn workspace_layer_sidecar_owner(value: &str) -> Option<&str> { + let value = value.strip_prefix('.')?; + value + .strip_suffix(".publish.json") + .or_else(|| value.strip_suffix(".verified.json")) +} + impl Drop for CacheBuildKeyGuard { fn drop(&mut self) { if fs::read_to_string(&self.path) @@ -4201,7 +5976,7 @@ fn build_lock_is_stale(path: &Path) -> Result { let Ok(pid) = pid.parse::() else { return malformed_build_lock_is_stale(path); }; - Ok(!process_matches_start_token(pid, token)) + Ok(process_start_token_match(pid, token) == ProcessIdentityMatch::DeadOrMismatch) } fn acquire_environment_cache_maintenance( @@ -4536,7 +6311,7 @@ fn layer_builder_is_alive(layer_id: &str, conn: &Connection) -> Result { Ok(process_matches_start_token(pid, token)) } -fn copy_layer_tree(source: &Path, destination: &Path) -> Result<()> { +pub(crate) fn copy_layer_tree(source: &Path, destination: &Path) -> Result<()> { fs::create_dir_all(destination)?; let mut folded = HashMap::::new(); for entry in walkdir::WalkDir::new(source).follow_links(false) { @@ -4671,7 +6446,7 @@ fn publication_staging_path(db_dir: &Path, publication_id: &str, stored: &str) - safe_join(db_dir, stored) } -fn scan_layer_entries( +pub(crate) fn scan_layer_entries( root: &Path, make_read_only: bool, ) -> Result> { @@ -4756,6 +6531,49 @@ fn scan_layer_entries( Ok(entries) } +pub(crate) fn verify_artifact_shadow_matches_layer_entries( + tree: &ArtifactTreeRootV1, + artifact_entries: &BTreeMap, + layer_entries: &BTreeMap, +) -> Result<()> { + let layer_logical_bytes = layer_entries + .values() + .try_fold(0u64, |total, entry| total.checked_add(entry.size_bytes)); + if tree.entry_count != layer_entries.len() as u64 + || Some(tree.logical_bytes) != layer_logical_bytes + || artifact_entries.len() != layer_entries.len() + { + return Err(Error::Corrupt( + "CAS shadow count or logical bytes disagree with the workspace layer scan".into(), + )); + } + for (path, layer) in layer_entries { + let artifact = artifact_entries.get(path).ok_or_else(|| { + Error::Corrupt(format!( + "CAS shadow is missing workspace layer path `{path}`" + )) + })?; + let normalized_mode = match layer.kind.as_str() { + "directory" => 0o755, + "file" if layer.mode & 0o111 != 0 => 0o755, + "file" => 0o644, + "symlink" => 0o777, + _ => layer.mode, + }; + if artifact.kind != layer.kind + || artifact.mode != normalized_mode + || artifact.size_bytes != layer.size_bytes + || artifact.content_hash != layer.content_hash + || artifact.symlink_target != layer.symlink_target + { + return Err(Error::Corrupt(format!( + "CAS shadow disagrees with workspace layer path `{path}`" + ))); + } + } + Ok(()) +} + fn sha256_layer_file(path: &Path) -> Result { let mut file = fs::File::open(path)?; let mut hasher = Sha256::new(); @@ -4857,7 +6675,7 @@ fn layer_mode(metadata: &fs::Metadata) -> u32 { } #[cfg(unix)] -fn set_layer_read_only(path: &Path, directory: bool, original_mode: u32) -> Result<()> { +pub(crate) fn set_layer_read_only(path: &Path, directory: bool, original_mode: u32) -> Result<()> { use std::os::unix::fs::PermissionsExt; let executable = directory || original_mode & 0o111 != 0; fs::set_permissions( @@ -4877,7 +6695,11 @@ fn immutable_layer_mode(directory: bool, original_mode: u32) -> u32 { } #[cfg(not(unix))] -fn set_layer_read_only(path: &Path, _directory: bool, _original_mode: u32) -> Result<()> { +pub(crate) fn set_layer_read_only( + path: &Path, + _directory: bool, + _original_mode: u32, +) -> Result<()> { let mut permissions = fs::metadata(path)?.permissions(); permissions.set_readonly(true); fs::set_permissions(path, permissions)?; @@ -4893,7 +6715,7 @@ fn immutable_layer_mode(directory: bool, _original_mode: u32) -> u32 { } } -fn sync_layer_tree(root: &Path) -> Result<()> { +pub(crate) fn sync_layer_tree(root: &Path) -> Result<()> { for entry in walkdir::WalkDir::new(root).follow_links(false) { let entry = entry.map_err(|err| Error::InvalidInput(err.to_string()))?; if entry.file_type().is_file() { @@ -4904,7 +6726,7 @@ fn sync_layer_tree(root: &Path) -> Result<()> { Ok(()) } -fn layer_physical_bytes(root: &Path) -> Result { +pub(crate) fn layer_physical_bytes(root: &Path) -> Result { let mut bytes = 0_u64; for entry in walkdir::WalkDir::new(root).follow_links(false) { let entry = entry.map_err(|err| Error::InvalidInput(err.to_string()))?; @@ -4930,6 +6752,7 @@ fn layer_file_physical_bytes(metadata: &fs::Metadata) -> u64 { #[cfg(test)] mod tests { use super::super::workdir::{ViewCore, VIEW_ROOT_INO}; + use super::super::workspace_environment::WorkspaceEnvironmentAdapterProposal; use super::*; use std::process::Stdio; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -4948,6 +6771,610 @@ mod tests { } } + #[derive(Clone, Copy)] + struct ArtifactConformanceProfile { + producer_family: &'static str, + adapter_identity: &'static str, + distribution_digest: &'static str, + adapter_protocol: &'static str, + trust_scope: &'static str, + trust_tier: ArtifactProducerTrustTierV1, + } + + fn run_artifact_pipeline_conformance_fixture( + profile: ArtifactConformanceProfile, + ) -> ArtifactAdapterCertificationReportV1 { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("fixture.input"), "fixture input\n").unwrap(); + fs::write( + workspace.path().join("trail.environment.toml"), + r#"schema = "trail.environment/v2" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "fixture.pipeline" +adapter = "trail/command@1" +inputs = [{ path = "fixture.input" }] + +[component.build] +command = ["fixture-builder", "build"] + +[[component.validation]] +name = "path-contract" +kind = "structural" +required = true + +[[component.output]] +name = "generated" +source = "artifact" +target = ".trail-generated/fixture" +policy = "immutable_seed_private" +reuse = "exact" +scope = "workspace" +publish = "on_sync" + +[[component.source_export]] +name = "client" +from_output = "generated" +source = "generated-client" +target = "src/generated-client" +mode = "explicit" +collision = "fail" +validation = "path-contract" +"#, + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "fixture", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let source_root = db.lane_branch("fixture").unwrap().head_root; + let input = db + .root_file_entry(&source_root, "fixture.input") + .unwrap() + .unwrap(); + let proposal = WorkspaceEnvironmentAdapterProposal { + status: EnvironmentComponentProposalStatus::Resolvable, + reasons: vec![EnvironmentProposalReasonReport { + code: "resolution_snapshot_missing".into(), + message: "fixture adapter requires an immutable resolution snapshot".into(), + }], + recovery_actions: vec![EnvironmentRecoveryActionReport { + code: "resolve_fixture".into(), + description: "resolve the fixture component".into(), + command: None, + }], + }; + assert_eq!( + proposal.status, + EnvironmentComponentProposalStatus::Resolvable + ); + + let resolver = std::env::current_exe().unwrap(); + let resolver_identity = + super::super::workspace_environment::workspace_tool_identity_for_path(&resolver) + .unwrap(); + let resolution = db + .resolve_artifact_component( + ArtifactResolutionRequestV1 { + plan: ArtifactResolutionPlanV1 { + version: ARTIFACT_RESOLUTION_PLAN_VERSION, + proposal_key: "fixture.pipeline:proposal-v1".into(), + source_root: source_root.clone(), + component_id: "fixture.pipeline".into(), + adapter_identity: profile.adapter_identity.into(), + policy_identity: "fixture-policy-v1".into(), + program: "fixture-resolver".into(), + resolved_program: resolver.to_string_lossy().into_owned(), + executable_identity: resolver_identity.clone(), + argv: vec!["fixture-resolver".into(), "resolve".into()], + working_directory: ".".into(), + readable_inputs: vec![ArtifactResolutionInputV1 { + source_path: "fixture.input".into(), + content_hash: input.content_hash.clone(), + size_bytes: input.size_bytes, + }], + candidate_output: "fixture.lock".into(), + allowed_authorities: Vec::new(), + credential_handles: Vec::new(), + script_policy: ArtifactScriptPolicyV1::Deny, + environment_roles: BTreeMap::new(), + limits: ArtifactActionLimitsV1 { + timeout_ms: 30_000, + stdout_bytes: 1024, + stderr_bytes: 1024, + candidate_bytes: 1024, + candidate_entries: 1, + child_processes: 1, + }, + snapshot_format: "fixture-lock/v1".into(), + validations: vec![ArtifactValidationV1 { + name: "fixture-lock-structure".into(), + kind: ArtifactValidationKindV1::Structural, + required: true, + parameters: BTreeMap::new(), + }], + }, + candidate: ArtifactResolutionCandidateV1 { + snapshot_bytes: b"fixture = 1\n".to_vec(), + resolved_identities: BTreeMap::from([("fixture".into(), "1".into())]), + checksums: BTreeMap::from([( + "fixture".into(), + sha256_hex(b"fixture = 1\n"), + )]), + contacted_authorities: Vec::new(), + stdout: b"resolved\n".to_vec(), + stderr: Vec::new(), + redactions: Vec::new(), + }, + }, + false, + ) + .unwrap(); + assert_eq!(resolution.decision, ArtifactResolutionDecisionV1::Resolved); + + let desired_material = ArtifactDesiredKeyMaterialV2 { + version: 2, + component_id: "fixture.pipeline".into(), + adapter_identity: profile.adapter_identity.into(), + adapter_implementation_version: "1".into(), + adapter_distribution_digest: profile.distribution_digest.into(), + adapter_protocol: profile.adapter_protocol.into(), + resolution_snapshot_id: Some(resolution.snapshot_id.clone()), + source_closure: ArtifactSourceClosureV2 { + normalizer_version: "source-paths/v1".into(), + certified_complete: false, + complete_source_root: Some(source_root.clone()), + declared_inputs: vec![ArtifactResolutionInputV1 { + source_path: "fixture.input".into(), + content_hash: input.content_hash, + size_bytes: input.size_bytes, + }], + }, + upstream_identities: BTreeMap::new(), + actions: vec![ArtifactActionIdentityV2 { + name: "construct".into(), + phase: ArtifactActionPhaseV2::Construct, + executable_identity: resolver_identity, + argv: vec!["fixture-builder".into(), "build".into()], + working_directory: ".".into(), + environment_names: Vec::new(), + }], + outputs: vec![ArtifactOutputContractV2 { + name: "generated".into(), + output_path: "artifact".into(), + mount_path: ".trail-generated/fixture".into(), + policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + reuse: EnvironmentReuseMode::Exact, + scope: EnvironmentSharingScope::Workspace, + publish: EnvironmentPublicationTrigger::OnSync, + gate: None, + }], + validations: vec![ArtifactValidationV1 { + name: "path-contract".into(), + kind: ArtifactValidationKindV1::Structural, + required: true, + parameters: BTreeMap::new(), + }], + source_exports: vec![ArtifactSourceExportContractV2 { + name: "client".into(), + output_name: "generated".into(), + artifact_subpath: "generated-client".into(), + destination: "src/generated-client".into(), + collision_policy: "fail".into(), + required_validation: "path-contract".into(), + required_gate: None, + authorization_mode: "explicit".into(), + }], + build_environment: BTreeMap::new(), + target: "fixture".into(), + platform: std::env::consts::OS.into(), + architecture: std::env::consts::ARCH.into(), + abi: "host".into(), + portability_certified: true, + portability_scope: "workspace".into(), + trust_scope: profile.trust_scope.into(), + network_policy: "deny".into(), + script_policy: ArtifactScriptPolicyV1::Deny, + sandbox_policy: "native-deny-by-default".into(), + }; + let desired_key = + super::super::workspace_artifact::artifact_desired_key_v2(desired_material.clone()) + .unwrap(); + let mut invalidated_material = desired_material; + invalidated_material.target = "fixture-invalidated".into(); + let invalidated_key = + super::super::workspace_artifact::artifact_desired_key_v2(invalidated_material) + .unwrap(); + assert_ne!(desired_key, invalidated_key); + + let candidate = tempfile::tempdir().unwrap(); + fs::create_dir_all(candidate.path().join("artifact/generated-client")).unwrap(); + fs::write( + candidate.path().join("artifact/generated-client/new.rs"), + "generated\n", + ) + .unwrap(); + let storage_key = WorkspaceLayerKeyV1 { + kind: "generated".into(), + adapter: "fixture-common-artifact".into(), + adapter_version: 1, + inputs: BTreeMap::from([ + ("source_root".into(), source_root.0.clone()), + ( + "resolution_snapshot".into(), + resolution.snapshot_id.0.clone(), + ), + ]), + tool_versions: BTreeMap::from([("fixture-builder".into(), "1".into())]), + platform: std::env::consts::OS.into(), + architecture: std::env::consts::ARCH.into(), + portability_scope: "workspace".into(), + strategy: "fixture-common-artifact-v2".into(), + }; + let layer = db + .publish_workspace_layer_from_directory(&storage_key, candidate.path()) + .unwrap(); + let tree_root_id = ArtifactTreeId::parse( + db.conn + .query_row( + "SELECT tree_root_id FROM workspace_layer_artifact_shadows WHERE layer_id=?1", + params![&layer.layer_id], + |row| row.get::<_, String>(0), + ) + .unwrap(), + ) + .unwrap(); + let desired_identity = ArtifactDesiredIdentityV1::ArtifactDesiredV2 { + desired_key: desired_key.clone(), + }; + let validation_receipt_id = db + .put_artifact_validation_receipt(ArtifactValidationReceiptV1 { + version: ARTIFACT_VALIDATION_RECEIPT_VERSION, + declaration: ArtifactValidationV1 { + name: "path-contract".into(), + kind: ArtifactValidationKindV1::Structural, + required: true, + parameters: BTreeMap::new(), + }, + desired_identity: desired_identity.clone(), + tree_root_id: tree_root_id.clone(), + validator_identity: "trail.fixture/structural-validator@1".into(), + validated_input_digest: sha256_hex(b"fixture desired+tree+validator"), + outcome: ArtifactValidationOutcomeV1::Passed, + evidence: BTreeMap::from([("structure".into(), "passed".into())]), + }) + .unwrap(); + let envelope_id = db + .bind_workspace_layer_artifact_v2( + &layer.layer_id, + ArtifactEnvelopeV1 { + version: ARTIFACT_ENVELOPE_VERSION, + desired_identity, + tree_root_id: tree_root_id.clone(), + component_id: "fixture.pipeline".into(), + output_name: "generated".into(), + output_policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + portability_scope: "workspace".into(), + trust_scope: profile.trust_scope.into(), + secret_taint: ArtifactSecretTaintV1::Clear, + resolution_snapshot_id: Some(resolution.snapshot_id.clone()), + validation_receipt_ids: vec![validation_receipt_id.clone()], + }, + ) + .unwrap(); + let reopened = Trail::open(workspace.path()).unwrap(); + let reopened_artifact = reopened.inspect_artifact(&envelope_id).unwrap(); + assert_eq!(reopened_artifact.tree_root_id, tree_root_id); + drop(reopened); + db.replace_declared_workspace_layers_at_source( + "fixture", + &[EnvironmentLayerActivation { + layer_id: Some(layer.layer_id.clone()), + outputs: vec![EnvironmentLayerOutputActivation { + name: "generated".into(), + mount_path: ".trail-generated/fixture".into(), + policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + reuse: EnvironmentReuseMode::Exact, + scope: EnvironmentSharingScope::Workspace, + publish: EnvironmentPublicationTrigger::OnSync, + gate: None, + binding_identity: layer.layer_id.clone(), + manifest_object_id: None, + publication_id: None, + private_seed: None, + layer_subpath: "artifact".into(), + }], + component_id: "fixture.pipeline".into(), + adapter_identity: profile.adapter_identity.into(), + adapter_version: 1, + implementation_version: "1".into(), + distribution_digest: profile.distribution_digest.into(), + kind: "generated".into(), + dependencies: Vec::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + expected_key: desired_key.0.clone(), + canonical_key: storage_key, + }], + &source_root, + ) + .unwrap(); + let generation = db + .active_environment_generation("fixture") + .unwrap() + .unwrap(); + assert_eq!( + db.conn + .query_row( + "SELECT desired_key FROM artifact_generation_bindings + WHERE generation_id=?1 AND component_id='fixture.pipeline'", + params![&generation.generation_id], + |row| row.get::<_, String>(0), + ) + .unwrap(), + desired_key.0 + ); + + let branch = db.lane_branch("fixture").unwrap(); + let paths = db.workspace_view_paths_for_lane("fixture").unwrap(); + let mut view = ViewCore::new_lazy( + Trail::open(workspace.path()).unwrap(), + paths.source_upper.clone(), + branch.head_root, + ) + .unwrap(); + let generated = view.lookup(VIEW_ROOT_INO, ".trail-generated").unwrap(); + let fixture = view.lookup(generated, "fixture").unwrap(); + let client = view.lookup(fixture, "generated-client").unwrap(); + let file = view.lookup(client, "new.rs").unwrap(); + assert_eq!(view.read(file, 0, 64).unwrap().0, b"generated\n"); + view.write(file, 0, b"private!!\n").unwrap(); + assert_eq!(view.read(file, 0, 64).unwrap().0, b"private!!\n"); + assert_eq!( + fs::read(Path::new(&layer.storage_path).join("artifact/generated-client/new.rs")) + .unwrap(), + b"generated\n" + ); + drop(view); + + let export = db + .plan_artifact_source_export( + "fixture", + "fixture.pipeline", + "client", + ArtifactSourceExportAuthorizationV1::ExplicitUser, + ) + .unwrap(); + assert_eq!(export.envelope_id, envelope_id); + let exported = db.execute_artifact_source_export(export).unwrap(); + assert!(exported.checkpointed); + let branch = db.lane_branch("fixture").unwrap(); + let exported_entry = db + .root_file_entry(&branch.head_root, "src/generated-client/new.rs") + .unwrap() + .unwrap(); + let exported_bytes = db + .materialize_entries_bytes(&BTreeMap::from([( + "src/generated-client/new.rs".into(), + exported_entry, + )])) + .unwrap(); + assert_eq!( + exported_bytes["src/generated-client/new.rs"], + b"generated\n" + ); + + db.remove_lane("fixture", true).unwrap(); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM artifact_generation_bindings", + [], + |row| { row.get::<_, u64>(0) } + ) + .unwrap(), + 0 + ); + let cache_gc = db.workspace_cache_gc(false, Some(0)).unwrap(); + assert!(cache_gc + .deleted + .iter() + .any(|candidate| candidate.id == layer.layer_id)); + db.gc(false).unwrap(); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM artifact_envelopes WHERE envelope_id=?1", + params![envelope_id.0], + |row| row.get::<_, u64>(0), + ) + .unwrap(), + 0 + ); + let evidence = [ + ("discovery", format!("producer:{}", profile.producer_family)), + ("resolution", "exact_snapshot:published".to_string()), + ("identity", "desired_key:canonical_v2".to_string()), + ("validation", "required_receipt:passed".to_string()), + ("sealing", "envelope_tree:bound_and_reopened".to_string()), + ("cow", "private_upper:lower_unchanged".to_string()), + ("recovery", "reopen:artifact_authority_valid".to_string()), + ("invalidation", "changed_target:changes_key".to_string()), + ("export", "source_export:checkpointed".to_string()), + ("retirement", "generation_bindings:released".to_string()), + ("collection", "artifact_envelope:collected".to_string()), + ]; + let report = ArtifactAdapterCertificationReportV1 { + schema: "trail.artifact-adapter-certification/v1".into(), + producer_family: profile.producer_family.into(), + adapter_identity: profile.adapter_identity.into(), + trust_tier: profile.trust_tier, + status: ArtifactAdapterConformanceStatusV1::Passed, + authority_effect: "evidence_only".into(), + checks: evidence + .into_iter() + .map(|(stage, evidence)| ArtifactAdapterConformanceCheckV1 { + stage: stage.into(), + applicable: true, + status: ArtifactAdapterConformanceStatusV1::Passed, + evidence: vec![evidence], + }) + .collect(), + }; + report.validate().unwrap(); + report + } + + #[test] + fn builtins_plugins_and_repository_v2_share_artifact_pipeline_conformance() { + let reports = [ + ArtifactConformanceProfile { + producer_family: "builtin", + adapter_identity: "trail.fixture/builtin@1", + distribution_digest: "builtin:fixture-v1", + adapter_protocol: "trail.environment-adapter/v3", + trust_scope: "builtin", + trust_tier: ArtifactProducerTrustTierV1::ReviewedBuiltin, + }, + ArtifactConformanceProfile { + producer_family: "plugin_v3", + adapter_identity: "example/fixture-plugin@1", + distribution_digest: "sha256:fixture-plugin-v1", + adapter_protocol: "trail.environment-adapter/v3", + trust_scope: "plugin-local", + trust_tier: ArtifactProducerTrustTierV1::LocallyTrustedPlugin, + }, + ArtifactConformanceProfile { + producer_family: "repository_v2", + adapter_identity: "trail/command@1", + distribution_digest: "repository:trail.environment/v2", + adapter_protocol: "trail.environment/v2", + trust_scope: "repository", + trust_tier: ArtifactProducerTrustTierV1::RepositoryDeclaration, + }, + ] + .map(run_artifact_pipeline_conformance_fixture); + assert_eq!( + reports + .iter() + .map(|report| report.producer_family.as_str()) + .collect::>(), + ["builtin", "plugin_v3", "repository_v2"] + ); + let encoded = serde_json::to_string(&reports).unwrap(); + assert!(!encoded.contains(workspace_secret_probe())); + if let Some(path) = std::env::var_os("TRAIL_ARTIFACT_CERTIFICATION_OUTPUT") { + fs::write(path, format!("{encoded}\n")).unwrap(); + } + } + + fn workspace_secret_probe() -> &'static str { + "credential-value-never-store" + } + + #[test] + fn private_output_promotion_rejects_secret_taint_and_sensitive_receipts() { + let clear = serde_json::json!({ + "gate": "integration", + "secret_taint": { "state": "clear" } + }); + assert!(validate_private_output_promotion_taint(false, Some(&clear)) + .unwrap() + .is_some()); + + let tainted = serde_json::json!({ + "gate": "integration", + "secret_taint": { + "state": "tainted", + "channels": ["runtime_credential"] + } + }); + assert!( + validate_private_output_promotion_taint(false, Some(&tainted)) + .unwrap_err() + .to_string() + .contains("non-promotable") + ); + assert!(validate_private_output_promotion_taint(true, None) + .unwrap_err() + .to_string() + .contains("non-promotable")); + + let sensitive = serde_json::json!({ + "gate": "integration", + "authorization": "Bearer credential-value-never-store" + }); + assert!( + validate_private_output_promotion_taint(false, Some(&sensitive)) + .unwrap_err() + .to_string() + .contains("secret material") + ); + } + + fn artifact_inheritance_request<'a>( + component_key: &'a str, + child_backend: &'a str, + ) -> ArtifactInheritanceRequest<'a> { + ArtifactInheritanceRequest { + component_id: "node", + component_key, + output_name: "dependencies", + output_policy: EnvironmentOutputPolicy::ImmutableSeedPrivate, + child_backend, + child_desired_key_was_planned: false, + } + } + + #[test] + fn workspace_layer_storage_identity_requires_lowercase_canonical_absolute_paths() { + let layer_id = "layer_0123456789abcdef0123456789abcdef"; + let canonical = std::env::temp_dir() + .join(".trail/cache/layers") + .join(layer_id); + assert!(valid_workspace_layer_id(layer_id)); + assert!(workspace_layer_storage_path_matches_id( + &canonical.to_string_lossy(), + layer_id + )); + assert!(!valid_workspace_layer_id( + "layer_0123456789ABCDEF0123456789ABCDEF" + )); + assert!(!workspace_layer_storage_path_matches_id( + &Path::new("relative/.trail/cache/layers") + .join(layer_id) + .to_string_lossy(), + layer_id + )); + assert!(!workspace_layer_storage_path_matches_id( + &std::env::temp_dir() + .join("ignored/../.trail/cache/layers") + .join(layer_id) + .to_string_lossy(), + layer_id + )); + } + #[test] fn canonical_artifact_key_covers_every_declared_identity_dimension() { let workspace = tempfile::tempdir().unwrap(); @@ -5060,6 +7487,139 @@ mod tests { } } + #[test] + fn artifact_inheritance_checks_backend_adapter_identity_envelope_tree_and_scope() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let built = tempfile::tempdir().unwrap(); + fs::write(built.path().join("artifact"), b"shared\n").unwrap(); + let layer = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + db.conn + .execute( + "INSERT INTO environment_component_states( + view_id,component_id,adapter_identity,adapter_version,implementation_version, + distribution_digest,kind,expected_key,attached_key,status,reason,updated_at) + VALUES('parent-view','node','trail/node@1',1,?1,'builtin:node-plan-v1', + 'dependency',?2,?2,'ready',NULL,1)", + params![env!("CARGO_PKG_VERSION"), &layer.cache_key], + ) + .unwrap(); + + assert_eq!( + db.inheritance_adapter_compatibility( + "parent-view", + "node", + &layer.cache_key, + "trail/node@1", + "node" + ), + Ok(()) + ); + assert_eq!( + db.inheritance_adapter_compatibility( + "parent-view", + "node", + &layer.cache_key, + "trail/node@1", + "cargo" + ), + Err("adapter_package_identity_mismatch") + ); + assert_eq!( + db.inheritance_adapter_compatibility( + "parent-view", + "node", + &layer.cache_key, + "missing/adapter@1", + "node" + ), + Err("adapter_package_unavailable") + ); + db.conn + .execute( + "UPDATE environment_component_states SET distribution_digest='changed-package' + WHERE view_id='parent-view' AND component_id='node'", + [], + ) + .unwrap(); + assert_eq!( + db.inheritance_adapter_compatibility( + "parent-view", + "node", + &layer.cache_key, + "trail/node@1", + "node" + ), + Err("adapter_package_identity_mismatch") + ); + db.conn + .execute( + "UPDATE environment_component_states + SET distribution_digest='builtin:node-plan-v1' + WHERE view_id='parent-view' AND component_id='node'", + [], + ) + .unwrap(); + + let native_backend = if cfg!(target_os = "macos") { + "nfs" + } else if cfg!(target_os = "windows") { + "dokan" + } else { + "fuse" + }; + let binding = db + .verify_artifact_inheritance_binding( + &layer, + artifact_inheritance_request(&layer.cache_key, native_backend), + ) + .unwrap(); + assert_eq!(binding.desired_key, layer.cache_key); + assert!(binding.envelope_id.starts_with("artifact_envelope_")); + assert!(binding.tree_root_id.starts_with("artifact_tree_")); + + assert_eq!( + db.verify_artifact_inheritance_binding( + &layer, + artifact_inheritance_request("wrong-key", "virtual"), + ), + Err("artifact_backend_unsupported") + ); + assert_eq!( + db.verify_artifact_inheritance_binding( + &layer, + artifact_inheritance_request("wrong-key", native_backend), + ), + Err("artifact_desired_identity_mismatch") + ); + let mut wrong_scope = layer.clone(); + wrong_scope.portability_scope = "host".to_string(); + assert_eq!( + db.verify_artifact_inheritance_binding( + &wrong_scope, + artifact_inheritance_request(&layer.cache_key, native_backend), + ), + Err("artifact_portability_mismatch") + ); + + db.conn + .execute( + "UPDATE artifact_envelopes SET state='quarantined' WHERE envelope_id=?1", + params![&binding.envelope_id], + ) + .unwrap(); + assert_eq!( + db.verify_artifact_inheritance_binding( + &layer, + artifact_inheritance_request(&layer.cache_key, native_backend), + ), + Err("artifact_envelope_verification_failed") + ); + } + #[test] fn legacy_layer_manifests_without_canonical_keys_remain_readable() { let manifest = WorkspaceLayerManifest { @@ -5513,10 +8073,22 @@ mod tests { panic!("cache publish crash helper passed its requested crash point"); } + #[test] + fn cache_materialization_crash_helper() { + let Some(workspace) = std::env::var_os("TRAIL_TEST_CRASH_WORKSPACE") else { + return; + }; + let _ = Trail::open(PathBuf::from(workspace)); + panic!("cache materialization crash helper passed its requested crash point"); + } + #[test] fn killing_cache_publish_at_each_durable_phase_preserves_source_and_recovers() { for phase in [ + "layer_after_reservation", "layer_after_staging_sync", + "layer_after_cas_tree", + "layer_after_envelope_ready", "layer_after_publish_marker", "layer_after_atomic_rename", "layer_after_ready_state", @@ -5590,45 +8162,351 @@ mod tests { } } - fn wait_for_crash_handshake(child: &mut std::process::Child, ready: &Path, phase: &str) { - for _ in 0..1_000 { - if ready.is_file() { - return; - } - if let Some(status) = child.try_wait().unwrap() { - panic!("crash helper exited at {phase} before handshake: {status}"); - } - std::thread::sleep(Duration::from_millis(10)); - } - let _ = child.kill(); - panic!("timed out waiting for crash helper at {phase}"); + #[test] + fn killing_cas_rematerialization_before_activation_retries_without_private_data_loss() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "restore-source", + Some("main"), + mode, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let source_upper = db + .workspace_view_paths_for_lane("restore-source") + .unwrap() + .source_upper; + fs::write(source_upper.join("uncheckpointed.rs"), "keep me\n").unwrap(); + let built = tempfile::tempdir().unwrap(); + fs::create_dir_all(built.path().join("pkg")).unwrap(); + fs::write(built.path().join("pkg/index.js"), "cached\n").unwrap(); + let layer = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + let storage_path = PathBuf::from(&layer.storage_path); + make_tree_writable(&storage_path); + fs::remove_dir_all(&storage_path).unwrap(); + drop(db); + + let phase = "layer_after_cas_materialization"; + let ready = workspace.path().join(format!("{phase}.ready")); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "db::lane::workspace_layer::tests::cache_materialization_crash_helper", + "--nocapture", + ]) + .env("RUST_TEST_THREADS", "1") + .env("TRAIL_TEST_CRASH_AT", phase) + .env("TRAIL_TEST_CRASH_READY", &ready) + .env("TRAIL_TEST_CRASH_WORKSPACE", workspace.path()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + wait_for_crash_handshake(&mut child, &ready, phase); + child.kill().unwrap(); + let _ = child.wait().unwrap(); + + assert_eq!( + fs::read_to_string(source_upper.join("uncheckpointed.rs")).unwrap(), + "keep me\n" + ); + let reopened = Trail::open(workspace.path()).unwrap(); + reopened.verify_workspace_layer(&layer.layer_id).unwrap(); + assert_eq!( + fs::read_to_string(storage_path.join("pkg/index.js")).unwrap(), + "cached\n" + ); + assert_eq!( + fs::read_to_string(source_upper.join("uncheckpointed.rs")).unwrap(), + "keep me\n" + ); + } + + fn wait_for_crash_handshake(child: &mut std::process::Child, ready: &Path, phase: &str) { + for _ in 0..1_000 { + if ready.is_file() { + return; + } + if let Some(status) = child.try_wait().unwrap() { + panic!("crash helper exited at {phase} before handshake: {status}"); + } + std::thread::sleep(Duration::from_millis(10)); + } + let _ = child.kill(); + panic!("timed out waiting for crash helper at {phase}"); + } + + #[test] + fn identical_layer_keys_publish_once_and_reuse_read_only_tree() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let built = tempfile::tempdir().unwrap(); + fs::create_dir_all(built.path().join("pkg")).unwrap(); + fs::write(built.path().join("pkg/index.js"), "module.exports = 1;\n").unwrap(); + + let first = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + let second = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + assert_eq!(first.layer_id, second.layer_id); + assert_eq!(db.list_workspace_layers().unwrap().len(), 1); + let shadow = db + .conn + .query_row( + "SELECT tree_root_id, envelope_id, state FROM workspace_layer_artifact_shadows + WHERE layer_id=?1", + params![first.layer_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }, + ) + .unwrap(); + assert!(shadow.0.starts_with("artifact_tree_")); + assert!(shadow.1.starts_with("artifact_envelope_")); + assert_eq!(shadow.2, "verified"); + assert_eq!( + db.conn + .query_row( + "SELECT state FROM artifact_envelopes WHERE envelope_id=?1", + params![shadow.1], + |row| row.get::<_, String>(0), + ) + .unwrap(), + "ready" + ); + assert!( + fs::metadata(Path::new(&first.storage_path).join("pkg/index.js")) + .unwrap() + .permissions() + .readonly() + ); + db.verify_workspace_layer(&first.layer_id).unwrap(); + } + + #[test] + fn workspace_open_reconstructs_missing_layer_materialization_from_verified_cas() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let built = tempfile::tempdir().unwrap(); + fs::create_dir_all(built.path().join("pkg")).unwrap(); + fs::write( + built.path().join("pkg/index.js"), + "module.exports = 'cas';\n", + ) + .unwrap(); + let published = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + let storage_path = PathBuf::from(&published.storage_path); + let object_count = db + .conn + .query_row("SELECT COUNT(*) FROM artifact_objects", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(); + make_tree_writable(&storage_path); + fs::remove_dir_all(&storage_path).unwrap(); + drop(db); + + let reopened = Trail::open(workspace.path()).unwrap(); + assert_eq!( + fs::read_to_string(storage_path.join("pkg/index.js")).unwrap(), + "module.exports = 'cas';\n" + ); + assert!(fs::metadata(storage_path.join("pkg/index.js")) + .unwrap() + .permissions() + .readonly()); + reopened + .verify_workspace_layer(&published.layer_id) + .unwrap(); + assert_eq!( + reopened + .conn + .query_row("SELECT COUNT(*) FROM artifact_objects", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + object_count + ); + } + + #[test] + fn workspace_open_removes_completed_materialization_staging() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let built = tempfile::tempdir().unwrap(); + fs::write(built.path().join("artifact"), "ready\n").unwrap(); + let layer = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + let staging = db + .db_dir + .join("cache/staging") + .join(format!("restore_{}", layer.layer_id)); + fs::create_dir_all(&staging).unwrap(); + fs::write(staging.join("partial"), "stale\n").unwrap(); + drop(db); + + let reopened = Trail::open(workspace.path()).unwrap(); + assert!(!staging.exists()); + reopened.verify_workspace_layer(&layer.layer_id).unwrap(); + } + + #[test] + fn workspace_open_reports_exact_guidance_for_corrupt_cas_reconstruction() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let built = tempfile::tempdir().unwrap(); + fs::write(built.path().join("artifact"), "ready\n").unwrap(); + let layer = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + let corrupt_object = db + .conn + .query_row( + "SELECT object_id FROM artifact_objects WHERE kind=?1 ORDER BY artifact_id LIMIT 1", + params![ARTIFACT_BLOB_KIND], + |row| row.get::<_, String>(0), + ) + .unwrap(); + db.conn + .execute( + "UPDATE objects SET bytes=X'00' WHERE object_id=?1", + params![corrupt_object], + ) + .unwrap(); + let layer_path = Path::new(&layer.storage_path); + make_tree_writable(layer_path); + fs::remove_dir_all(layer_path).unwrap(); + drop(db); + + let error = match Trail::open(workspace.path()) { + Ok(_) => panic!("corrupt CAS reconstruction unexpectedly succeeded"), + Err(error) => error, + }; + let message = error.to_string(); + assert!(message.contains("could not materialize tree"), "{message}"); + assert!( + message.contains("restore the workspace from backup"), + "{message}" + ); + assert!( + message.contains("run environment synchronization"), + "{message}" + ); + } + + #[test] + fn cas_shadow_comparison_rejects_path_metadata_and_digest_disagreement() { + let tree = ArtifactTreeRootV1 { + version: ARTIFACT_TREE_ROOT_VERSION, + root_directory_id: crate::ids::ArtifactTreeId::new(b"root"), + logical_bytes: 4, + entry_count: 1, + path_normalizer: "trail-paths/v1".into(), + }; + let artifact = BTreeMap::from([( + "artifact".into(), + super::workspace_artifact::ArtifactFlatEntry { + kind: "file", + mode: 0o644, + size_bytes: 4, + content_hash: Some(sha256_hex(b"good")), + symlink_target: None, + }, + )]); + let mut layer = BTreeMap::from([( + "artifact".into(), + WorkspaceLayerEntry { + kind: "file".into(), + mode: 0o644, + size_bytes: 4, + content_hash: Some(sha256_hex(b"good")), + symlink_target: None, + }, + )]); + verify_artifact_shadow_matches_layer_entries(&tree, &artifact, &layer).unwrap(); + layer.get_mut("artifact").unwrap().content_hash = Some(sha256_hex(b"evil")); + assert!(verify_artifact_shadow_matches_layer_entries(&tree, &artifact, &layer).is_err()); } #[test] - fn identical_layer_keys_publish_once_and_reuse_read_only_tree() { + #[ignore = "known pre-CAS failure: distinct desired keys duplicate equal authoritative trees"] + fn equal_content_under_distinct_keys_reuses_authoritative_bytes() { let workspace = tempfile::tempdir().unwrap(); fs::write(workspace.path().join("README.md"), "root\n").unwrap(); Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let db = Trail::open(workspace.path()).unwrap(); let built = tempfile::tempdir().unwrap(); - fs::create_dir_all(built.path().join("pkg")).unwrap(); - fs::write(built.path().join("pkg/index.js"), "module.exports = 1;\n").unwrap(); + fs::write(built.path().join("artifact"), "identical bytes\n").unwrap(); + let first_key = key(); + let mut second_key = key(); + second_key + .inputs + .insert("feature".into(), "different-desired-key".into()); let first = db - .publish_workspace_layer_from_directory(&key(), built.path()) + .publish_workspace_layer_from_directory(&first_key, built.path()) .unwrap(); let second = db - .publish_workspace_layer_from_directory(&key(), built.path()) + .publish_workspace_layer_from_directory(&second_key, built.path()) .unwrap(); - assert_eq!(first.layer_id, second.layer_id); - assert_eq!(db.list_workspace_layers().unwrap().len(), 1); - assert!( - fs::metadata(Path::new(&first.storage_path).join("pkg/index.js")) - .unwrap() - .permissions() - .readonly() + + assert_eq!( + first.storage_path, second.storage_path, + "equal content should have one authoritative CAS tree" ); - db.verify_workspace_layer(&first.layer_id).unwrap(); + } + + #[test] + #[ignore = "known pre-CAS failure: one desired key silently reuses a divergent content tree"] + fn one_desired_key_rejects_a_divergent_content_root() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let first = tempfile::tempdir().unwrap(); + let divergent = tempfile::tempdir().unwrap(); + fs::write(first.path().join("artifact"), "producer A\n").unwrap(); + fs::write(divergent.path().join("artifact"), "producer B\n").unwrap(); + db.publish_workspace_layer_from_directory(&key(), first.path()) + .unwrap(); + + let error = db + .publish_workspace_layer_from_directory(&key(), divergent.path()) + .expect_err("a divergent content root for one desired key must be quarantined"); + assert!(error.to_string().contains("divergent")); } #[test] @@ -5739,6 +8617,83 @@ mod tests { } } + #[test] + fn host_sealing_precedes_ready_workspace_layer_envelopes() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let built = tempfile::tempdir().unwrap(); + fs::create_dir_all(built.path().join("pkg")).unwrap(); + fs::write(built.path().join("pkg/index.js"), "sealed\n").unwrap(); + + let layer = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + let (tree_id, envelope_id) = db + .conn + .query_row( + "SELECT tree_root_id,envelope_id + FROM workspace_layer_artifact_shadows WHERE layer_id=?1", + params![layer.layer_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(); + let tree_id = ArtifactTreeId::parse(tree_id).unwrap(); + let envelope_id = crate::ids::ArtifactEnvelopeId::parse(envelope_id).unwrap(); + let envelope = db + .verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id) + .unwrap(); + assert_eq!(envelope.validation_receipt_ids.len(), 2); + let receipts = envelope + .validation_receipt_ids + .iter() + .map(|receipt_id| db.artifact_validation_receipt(receipt_id).unwrap()) + .collect::>(); + assert!(receipts.iter().any(|receipt| { + receipt.declaration.name == "trail.host.workspace-layer.structural-seal/v1" + && receipt.declaration.kind == ArtifactValidationKindV1::Structural + && receipt + .evidence + .get("complete_tree_identity") + .map(String::as_str) + == Some("passed") + })); + assert!(receipts.iter().any(|receipt| { + receipt.declaration.name == "trail.host.workspace-layer.policy-seal/v1" + && receipt.declaration.kind == ArtifactValidationKindV1::Policy + && receipt + .evidence + .get("producer_termination") + .map(String::as_str) + == Some("terminated_or_disconnected") + && receipt.evidence.get("desired_pins").map(String::as_str) == Some("unchanged") + && receipt.evidence.get("producer_trust").map(String::as_str) + == Some("local_host_authorized") + })); + + let mut missing_required_seals = envelope.clone(); + missing_required_seals.validation_receipt_ids.clear(); + assert!(db + .put_artifact_envelope_under_write_lock(missing_required_seals) + .unwrap_err() + .to_string() + .contains("missing required host")); + + let mut stale_receipt = receipts + .into_iter() + .find(|receipt| receipt.declaration.name == "trail.host.workspace-layer.policy-seal/v1") + .unwrap(); + stale_receipt + .evidence + .insert("producer_trust".into(), "untrusted".into()); + assert!(db + .put_artifact_validation_receipt(stale_receipt) + .unwrap_err() + .to_string() + .contains("stale input digest")); + } + #[test] fn two_views_share_one_layer_but_copy_writes_to_private_generated_uppers() { let workspace = tempfile::tempdir().unwrap(); @@ -6169,31 +9124,111 @@ mod tests { fs::write(workspace.path().join("README.md"), "root\n").unwrap(); Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let workspace_path = workspace.path().to_path_buf(); + let observer = Trail::open(workspace.path()).unwrap(); + let source_root = observer.resolve_refish("main").unwrap().root_id; + let key = key(); + let cache_key = observer.workspace_layer_cache_key(&key).unwrap(); let builds = Arc::new(AtomicUsize::new(0)); - let mut workers = Vec::new(); - for _ in 0..2 { - let workspace_path = workspace_path.clone(); - let builds = Arc::clone(&builds); - let key = key(); - workers.push(std::thread::spawn(move || { - let db = Trail::open(&workspace_path).unwrap(); - db.build_workspace_layer_singleflight(&key, |build_dir| { - builds.fetch_add(1, Ordering::SeqCst); - std::thread::sleep(Duration::from_millis(150)); - let output = build_dir.join("output"); - fs::create_dir_all(&output).unwrap(); - fs::write(output.join("index.js"), "shared\n").unwrap(); - Ok(output) - }) - .unwrap() - })); + let (started_tx, started_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0); + let owner_workspace = workspace_path.clone(); + let owner_builds = Arc::clone(&builds); + let owner_key = key.clone(); + let owner_root = source_root.clone(); + let owner = std::thread::spawn(move || { + let db = Trail::open(&owner_workspace).unwrap(); + db.build_workspace_layer_singleflight(&owner_key, &owner_root, |build_dir| { + owner_builds.fetch_add(1, Ordering::SeqCst); + started_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + let output = build_dir.join("output"); + fs::create_dir_all(&output).unwrap(); + fs::write(output.join("index.js"), "shared\n").unwrap(); + Ok(output) + }) + .unwrap() + }); + started_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + + let waiter_workspace = workspace_path.clone(); + let waiter_key = key.clone(); + let waiter_root = source_root.clone(); + let waiter = std::thread::spawn(move || { + let db = Trail::open(&waiter_workspace).unwrap(); + db.build_workspace_layer_singleflight(&waiter_key, &waiter_root, |_| { + panic!("a singleflight waiter must not invoke a second builder") + }) + .unwrap() + }); + + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let waiting = observer + .conn + .query_row( + "SELECT COUNT(*) FROM artifact_construction_waiters w + JOIN artifact_construction_attempts a ON a.attempt_id=w.attempt_id + WHERE a.desired_key=?1 AND w.status='waiting'", + params![cache_key], + |row| row.get::<_, i64>(0), + ) + .unwrap(); + if waiting == 1 { + break; + } + assert!( + Instant::now() < deadline, + "waiter did not publish durable waiting evidence" + ); + std::thread::sleep(Duration::from_millis(10)); } - let reports = workers - .into_iter() - .map(|worker| worker.join().unwrap()) - .collect::>(); + let running = observer + .conn + .query_row( + "SELECT phase,status,owner_generation + FROM artifact_construction_attempts WHERE desired_key=?1", + params![cache_key], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + }, + ) + .unwrap(); + assert_eq!(running, ("building".into(), "running".into(), 1)); + + release_tx.send(()).unwrap(); + let owner_report = owner.join().unwrap(); + let waiter_report = waiter.join().unwrap(); assert_eq!(builds.load(Ordering::SeqCst), 1); - assert_eq!(reports[0].layer_id, reports[1].layer_id); + assert_eq!(owner_report.layer_id, waiter_report.layer_id); + let completed = observer + .conn + .query_row( + "SELECT phase,status,owner_generation + FROM artifact_construction_attempts WHERE desired_key=?1", + params![cache_key], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + }, + ) + .unwrap(); + assert_eq!(completed, ("completed".into(), "succeeded".into(), 1)); + let waiter_status = observer + .conn + .query_row( + "SELECT status FROM artifact_construction_waiters", + [], + |row| row.get::<_, String>(0), + ) + .unwrap(); + assert_eq!(waiter_status, "released"); } #[test] @@ -6203,6 +9238,7 @@ mod tests { Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let db = Trail::open(workspace.path()).unwrap(); let key = key(); + let source_root = db.resolve_refish("main").unwrap().root_id; let cache_key = db.workspace_layer_cache_key(&key).unwrap(); let lock_dir = db.db_dir.join("cache/staging/locks"); fs::create_dir_all(&lock_dir).unwrap(); @@ -6214,15 +9250,26 @@ mod tests { .unwrap(); let cancelled = AtomicBool::new(true); let error = db - .build_workspace_layer_singleflight_with_cancel(&key, &cancelled, |_| { + .build_workspace_layer_singleflight_with_cancel(&key, &source_root, &cancelled, |_| { panic!("a cancelled waiter must not invoke the builder") }) .unwrap_err(); assert!(error.to_string().contains("cancelled while waiting")); fs::write(&lock, "4294967295:dead-owner").unwrap(); + db.conn + .execute( + "INSERT INTO artifact_construction_attempts( + attempt_id,desired_key,source_root,owner_generation,owner_pid, + owner_start_token,phase,status,candidate_journal_object_id,envelope_id, + reason_code,reason,cancel_requested,started_at,heartbeat_at,finished_at) + VALUES('construct_dead',?1,?2,1,4294967295,'dead-owner','building', + 'running',NULL,NULL,NULL,NULL,0,?3,?3,NULL)", + params![cache_key, source_root.0, now_ts()], + ) + .unwrap(); let report = db - .build_workspace_layer_singleflight(&key, |build_dir| { + .build_workspace_layer_singleflight(&key, &source_root, |build_dir| { let output = build_dir.join("output"); fs::create_dir_all(&output).unwrap(); fs::write(output.join("artifact"), "recovered\n").unwrap(); @@ -6234,6 +9281,254 @@ mod tests { fs::read_to_string(Path::new(&report.storage_path).join("artifact")).unwrap(), "recovered\n" ); + let attempts = db + .conn + .prepare( + "SELECT owner_generation,phase,status,reason_code + FROM artifact_construction_attempts + WHERE desired_key=?1 ORDER BY owner_generation", + ) + .unwrap() + .query_map(params![cache_key], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + )) + }) + .unwrap() + .collect::>>() + .unwrap(); + assert_eq!( + attempts, + vec![ + ( + 1, + "completed".into(), + "abandoned".into(), + Some("owner_lost".into()) + ), + (2, "completed".into(), "succeeded".into(), None), + ] + ); + } + + #[test] + fn construction_phase_updates_require_the_exact_owner_generation() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source_root = db.resolve_refish("main").unwrap().root_id; + let attempt = db + .begin_artifact_construction_attempt("exact-fence-test", &source_root) + .unwrap(); + let mut stale_fence = attempt.clone(); + stale_fence.owner_generation += 1; + + let error = db + .advance_artifact_construction_attempt(&stale_fence, "building") + .unwrap_err(); + assert!(error.to_string().contains("lost its exact owner fence")); + db.advance_artifact_construction_attempt(&attempt, "building") + .unwrap(); + db.advance_artifact_construction_attempt(&attempt, "validating") + .unwrap(); + db.verify_artifact_construction_seal_fence(&attempt, "exact-fence-test", &source_root) + .unwrap(); + db.conn + .execute( + "UPDATE artifact_construction_attempts SET desired_key='changed-pin' + WHERE attempt_id=?1", + params![attempt.attempt_id], + ) + .unwrap(); + assert!(db + .verify_artifact_construction_seal_fence(&attempt, "exact-fence-test", &source_root,) + .unwrap_err() + .to_string() + .contains("changed its source, desired key")); + db.finish_artifact_construction_attempt( + &attempt, + "abandoned", + Some("test_complete"), + Some("test completed"), + ) + .unwrap(); + } + + #[test] + fn workspace_open_recovers_dead_construction_owner_and_exact_staging() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source_root = db.resolve_refish("main").unwrap().root_id; + let desired_key = db.workspace_layer_cache_key(&key()).unwrap(); + let owner_pid = u32::MAX; + let owner_start_token = "dead-open-owner"; + let owner_token = format!("{owner_pid}:{owner_start_token}"); + let staging = db.db_dir.join("cache/staging").join(format!( + "input_{}", + crate::ids::short_hash(format!("{desired_key}:{owner_token}").as_bytes(), 12) + )); + fs::create_dir_all(&staging).unwrap(); + fs::write(staging.join("partial"), "incomplete\n").unwrap(); + let lock = db + .db_dir + .join("cache/staging/locks") + .join(format!("{desired_key}.lock")); + fs::create_dir_all(lock.parent().unwrap()).unwrap(); + fs::write(&lock, &owner_token).unwrap(); + db.conn + .execute( + "INSERT INTO artifact_construction_attempts( + attempt_id,desired_key,source_root,owner_generation,owner_pid, + owner_start_token,phase,status,candidate_journal_object_id,envelope_id, + reason_code,reason,cancel_requested,started_at,heartbeat_at,finished_at) + VALUES('construct_open_dead',?1,?2,1,?3,?4,'building','running', + NULL,NULL,NULL,NULL,0,?5,?5,NULL)", + params![ + desired_key, + source_root.0, + i64::from(owner_pid), + owner_start_token, + now_ts() + ], + ) + .unwrap(); + db.conn + .execute( + "INSERT INTO artifact_construction_waiters( + attempt_id,waiter_id,owner_pid,owner_start_token,status,created_at,updated_at) + VALUES('construct_open_dead','waiter_open_dead',?1,?2,'waiting',?3,?3)", + params![i64::from(owner_pid), owner_start_token, now_ts()], + ) + .unwrap(); + drop(db); + + let reopened = Trail::open(workspace.path()).unwrap(); + assert!(!staging.exists()); + assert!(!lock.exists()); + let attempt = reopened + .conn + .query_row( + "SELECT phase,status,reason_code FROM artifact_construction_attempts + WHERE attempt_id='construct_open_dead'", + [], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }, + ) + .unwrap(); + assert_eq!( + attempt, + ("completed".into(), "abandoned".into(), "owner_lost".into()) + ); + let waiter = reopened + .conn + .query_row( + "SELECT status FROM artifact_construction_waiters + WHERE waiter_id='waiter_open_dead'", + [], + |row| row.get::<_, String>(0), + ) + .unwrap(); + assert_eq!(waiter, "abandoned"); + } + + #[test] + fn fsck_and_doctor_report_cas_corruption_and_orphan_materializations() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source = tempfile::tempdir().unwrap(); + fs::write(source.path().join("artifact"), "verified\n").unwrap(); + let layer = db + .publish_workspace_layer_from_directory(&key(), source.path()) + .unwrap(); + drop(db); + let db = Trail::open(workspace.path()).unwrap(); + let corrupt_object = db + .conn + .query_row( + "SELECT object_id FROM artifact_objects WHERE kind=?1 ORDER BY artifact_id LIMIT 1", + params![ARTIFACT_BLOB_KIND], + |row| row.get::<_, String>(0), + ) + .unwrap(); + db.conn + .execute( + "UPDATE objects SET bytes=X'00' WHERE object_id=?1", + params![corrupt_object], + ) + .unwrap(); + let layer_path = Path::new(&layer.storage_path); + make_tree_writable(layer_path); + fs::remove_dir_all(layer_path).unwrap(); + let orphan_id = format!("layer_{}", "a".repeat(32)); + fs::create_dir_all(db.db_dir.join("cache/layers").join(&orphan_id)).unwrap(); + + let fsck = db.fsck().unwrap(); + assert!( + fsck.errors.iter().any(|error| { + error.contains("artifact object") && error.contains("rebuild the owning") + }), + "fsck errors: {:?}", + fsck.errors + ); + assert!(fsck.errors.iter().any(|error| { + error.contains("CAS-backed workspace layer") + && error.contains("reopen Trail to reconstruct") + })); + assert!(fsck.errors.iter().any(|error| { + error.contains("orphan workspace layer materialization") + && error.contains("rerun `trail fsck`") + })); + + let doctor = db.doctor().unwrap(); + assert_eq!(doctor.status, "error"); + let doctor_json = serde_json::to_string(&doctor).unwrap(); + assert!(doctor_json.contains("CAS-backed workspace layer")); + assert!(doctor_json.contains("orphan workspace layer materialization")); + } + + #[test] + fn fsck_distinguishes_missing_legacy_layer_without_cas_authority() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source = tempfile::tempdir().unwrap(); + fs::write(source.path().join("artifact"), "legacy\n").unwrap(); + let layer = db + .publish_workspace_layer_from_directory(&key(), source.path()) + .unwrap(); + db.conn + .execute( + "DELETE FROM workspace_layer_artifact_shadows WHERE layer_id=?1", + params![layer.layer_id], + ) + .unwrap(); + let layer_path = Path::new(&layer.storage_path); + make_tree_writable(layer_path); + fs::remove_dir_all(layer_path).unwrap(); + + let fsck = db.fsck().unwrap(); + assert!(fsck.errors.iter().any(|error| { + error.contains("legacy workspace layer") + && error.contains("run environment synchronization to rebuild it") + })); + assert!(!fsck + .errors + .iter() + .any(|error| error.contains("reconstruct it from its verified envelope"))); } #[test] diff --git a/trail/src/db/lane/workspace_node.rs b/trail/src/db/lane/workspace_node.rs index bc5b3bc6..14682127 100644 --- a/trail/src/db/lane/workspace_node.rs +++ b/trail/src/db/lane/workspace_node.rs @@ -1,11 +1,13 @@ use super::workspace_environment::{ resolve_workspace_tool_executable, WorkspaceEnvironmentAdapter, - WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentCacheAccess, - WorkspaceEnvironmentCacheProtocol, WorkspaceEnvironmentCommand, WorkspaceEnvironmentInput, - WorkspaceEnvironmentOutput, WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, - WorkspaceEnvironmentSandboxPolicy, + WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentAdapterProposal, + WorkspaceEnvironmentCacheAccess, WorkspaceEnvironmentCacheProtocol, + WorkspaceEnvironmentCommand, WorkspaceEnvironmentInput, WorkspaceEnvironmentOutput, + WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, + WorkspaceEnvironmentResolutionInput, WorkspaceEnvironmentSandboxPolicy, }; use super::*; +use crate::ids::sha256_hex; pub(crate) struct NodeWorkspaceAdapter; @@ -45,22 +47,135 @@ impl WorkspaceEnvironmentAdapter for NodeWorkspaceAdapter { } fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result { + let root = normalize_package_root(component_root)?; + Ok(db + .root_file_entry(source_root, &join_repo_path(&root, "package.json"))? + .is_some()) + } + + fn propose( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result> { let root = normalize_package_root(component_root)?; if db .root_file_entry(source_root, &join_repo_path(&root, "package.json"))? .is_none() { - return Ok(false); + return Ok(None); } for (name, _) in supported_lockfiles() { if db .root_file_entry(source_root, &join_repo_path(&root, name))? .is_some() { - return Ok(true); + return Ok(Some(WorkspaceEnvironmentAdapterProposal::ready())); + } + } + let spec = match node_resolution_spec(db, source_root, &root) { + Ok(spec) => spec, + Err(error) => { + return Ok(Some(WorkspaceEnvironmentAdapterProposal::blocked( + EnvironmentProposalReasonReport { + code: "node_resolution_unsupported".to_string(), + message: error.to_string(), + }, + EnvironmentRecoveryActionReport { + code: "choose_supported_package_manager".to_string(), + description: + "Set packageManager to npm, pnpm, yarn, or bun and resolve a matching lock snapshot" + .to_string(), + command: None, + }, + ))); } + }; + if node_resolution_snapshot(db, source_root, &root, &spec)?.is_some() { + return Ok(Some(WorkspaceEnvironmentAdapterProposal::ready())); } - Ok(false) + Ok(Some(WorkspaceEnvironmentAdapterProposal::resolvable( + EnvironmentProposalReasonReport { + code: "resolution_snapshot_missing".to_string(), + message: "package.json is present but no supported package-manager lockfile or Trail-managed resolution snapshot is available".to_string(), + }, + EnvironmentRecoveryActionReport { + code: "resolve_component".to_string(), + description: format!("Resolve and pin a Trail-managed {} snapshot without adding `{}` to source", spec.manager, spec.lock_name), + command: None, + }, + ))) + } + + fn resolution_plan( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result> { + let root = normalize_package_root(component_root)?; + let package_path = join_repo_path(&root, "package.json"); + let Some(package) = db.root_file_entry(source_root, &package_path)? else { + return Ok(None); + }; + let spec = node_resolution_spec(db, source_root, &root)?; + let manager_tool = resolve_workspace_tool_executable(spec.manager)?; + let policy_identity = sha256_hex( + format!( + "node-lock-resolver-v1\0{}\0{}\0ignore-scripts", + spec.manager, manager_tool.identity + ) + .as_bytes(), + ); + Ok(Some(ArtifactResolutionPlanV1 { + version: ARTIFACT_RESOLUTION_PLAN_VERSION, + proposal_key: node_resolution_proposal_key(source_root, &root, &spec), + source_root: source_root.clone(), + component_id: self.component_id(&root)?, + adapter_identity: self.identity().to_string(), + policy_identity, + program: spec.manager.to_string(), + resolved_program: manager_tool.path.to_string_lossy().into_owned(), + executable_identity: manager_tool.identity, + argv: node_lock_resolution_argv(&spec) + .into_iter() + .map(str::to_string) + .collect(), + working_directory: if root.is_empty() { + ".".to_string() + } else { + root.clone() + }, + readable_inputs: vec![ArtifactResolutionInputV1 { + source_path: package_path, + content_hash: package.content_hash, + size_bytes: package.size_bytes, + }], + candidate_output: join_repo_path(&root, spec.lock_name), + allowed_authorities: Vec::new(), + credential_handles: Vec::new(), + script_policy: ArtifactScriptPolicyV1::Deny, + environment_roles: BTreeMap::new(), + limits: ArtifactActionLimitsV1 { + timeout_ms: 10 * 60 * 1_000, + stdout_bytes: 1024 * 1024, + stderr_bytes: 1024 * 1024, + candidate_bytes: 64 * 1024 * 1024, + candidate_entries: 1, + child_processes: 64, + }, + snapshot_format: spec.snapshot_format(), + validations: vec![ArtifactValidationV1 { + name: format!("{}-lock-structure", spec.manager), + kind: ArtifactValidationKindV1::Framework, + required: true, + parameters: BTreeMap::from([ + ("manager".to_string(), spec.manager.to_string()), + ("lockfile".to_string(), spec.lock_name.to_string()), + ]), + }], + })) } fn plan( @@ -107,6 +222,9 @@ impl Trail { } )) })?; + let package_projection = self.project_entry_file(&package_entry)?; + let package_text = fs::read_to_string(package_projection)?; + let package_value: serde_json::Value = serde_json::from_str(&package_text)?; let mut selected = None; for (name, manager) in supported_lockfiles() { let path = join_repo_path(&package_root, name); @@ -115,23 +233,80 @@ impl Trail { break; } } - let (lock_path, manager, lock_entry) = selected.ok_or_else(|| { - Error::InvalidInput(format!( - "Node package root `{}` has no supported frozen-install lockfile", - if package_root.is_empty() { - "." - } else { - &package_root - } - )) - })?; + let component_id = NODE_WORKSPACE_ADAPTER.component_id(&package_root)?; + let ( + lock_path, + manager, + lock_content_hash, + lock_authority, + source_lock_entry, + resolution_inputs, + source_projection, + ) = if let Some((lock_path, manager, lock_entry)) = selected { + ( + lock_path, + manager, + lock_entry.content_hash.clone(), + "source".to_string(), + Some(lock_entry), + Vec::new(), + None, + ) + } else { + let spec = node_resolution_spec_from_package(&package_value)?; + let resolution_plan = NODE_WORKSPACE_ADAPTER + .resolution_plan(self, root_id, &package_root)? + .ok_or_else(|| { + Error::Corrupt(format!( + "Node component `{}` lost its resolver plan", + display_package_root(&package_root) + )) + })?; + let (snapshot_id, snapshot, bytes) = node_resolution_snapshot( + self, + root_id, + &package_root, + &spec, + )? + .ok_or_else(|| { + Error::InvalidInput(format!( + "Node component `{}` has no supported lockfile or Trail-managed resolution snapshot; resolve it through Trail's artifact resolution operation before synchronizing", + display_package_root(&package_root) + )) + })?; + if snapshot.resolver_executable_identity != resolution_plan.executable_identity + || snapshot.policy_identity != resolution_plan.policy_identity + { + return Err(Error::InvalidInput(format!( + "Node component `{}` resolution snapshot was created by a different package-manager executable or resolver policy; resolve it again for the current tool", + display_package_root(&package_root) + ))); + } + let size_bytes = u64::try_from(bytes.len()).map_err(|_| { + Error::InvalidInput("Node lock snapshot exceeds platform limits".to_string()) + })?; + let lock_path = join_repo_path(&package_root, spec.lock_name); + ( + lock_path.clone(), + spec.manager.to_string(), + snapshot.content_sha256.clone(), + format!("snapshot:{}", snapshot_id.0), + None, + vec![WorkspaceEnvironmentResolutionInput { + snapshot_id, + source_root: root_id.clone(), + source_path: lock_path.clone(), + staging_path: format!("project/{lock_path}"), + content_hash: snapshot.content_sha256, + size_bytes, + }], + Some((root_id.clone(), "project".to_string())), + ) + }; let manager_version = tool_version(&manager)?; let node_version = tool_version("node")?; let node_tool = resolve_workspace_tool_executable("node")?; let manager_tool = resolve_workspace_tool_executable(&manager)?; - let package_projection = self.project_entry_file(&package_entry)?; - let package_text = fs::read_to_string(package_projection)?; - let package_value: serde_json::Value = serde_json::from_str(&package_text)?; if package_value.get("workspaces").is_some() { return Err(Error::InvalidInput(format!( "Node component `{}` declares workspaces; synchronize a supported leaf package explicitly until the monorepo adapter is enabled", @@ -167,10 +342,10 @@ impl Trail { .to_string(), )); } - let mut files = BTreeMap::from([ - (package_json.clone(), package_entry.clone()), - (lock_path.clone(), lock_entry.clone()), - ]); + let mut files = BTreeMap::from([(package_json.clone(), package_entry.clone())]); + if let Some(lock_entry) = source_lock_entry { + files.insert(lock_path.clone(), lock_entry); + } for name in [ ".npmrc", ".yarnrc", @@ -189,6 +364,8 @@ impl Trail { .iter() .map(|(path, entry)| (path.clone(), entry.content_hash.clone())) .collect::>(); + key_inputs.insert(lock_path.clone(), lock_content_hash); + key_inputs.insert("lock_authority".to_string(), lock_authority); key_inputs.insert( "adapter_implementation".to_string(), implementation_version.clone(), @@ -264,24 +441,28 @@ impl Trail { .into_iter() .map(str::to_string) .collect(); - let inputs = files - .into_iter() - .map(|(source_path, entry)| { - let relative = strip_package_root(&source_path, &package_root)?; - Ok(WorkspaceEnvironmentInput { - source_path, - staging_path: format!("project/{relative}"), - entry, + let inputs = if source_projection.is_some() { + Vec::new() + } else { + files + .into_iter() + .map(|(source_path, entry)| { + let relative = strip_package_root(&source_path, &package_root)?; + Ok(WorkspaceEnvironmentInput { + source_path, + staging_path: format!("project/{relative}"), + entry, + }) }) - }) - .collect::>>()?; + .collect::>>()? + }; let mount_path = if package_root.is_empty() { "node_modules".to_string() } else { format!("{package_root}/node_modules") }; Ok(WorkspaceEnvironmentPlan { - component_id: NODE_WORKSPACE_ADAPTER.component_id(&package_root)?, + component_id, adapter_identity: NODE_WORKSPACE_ADAPTER.identity().to_string(), adapter_version: 1, implementation_version, @@ -291,7 +472,8 @@ impl Trail { resolved_dependencies: Vec::new(), layer_key: key, inputs, - source_projection: None, + resolution_inputs, + source_projection, pre_commands: Vec::new(), command: Some(WorkspaceEnvironmentCommand { program: manager, @@ -326,6 +508,181 @@ impl Trail { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct NodeResolutionSpec { + manager: &'static str, + lock_name: &'static str, +} + +impl NodeResolutionSpec { + fn snapshot_format(self) -> String { + format!("node-lock-{}-{}-v1", self.manager, self.lock_name) + } +} + +fn node_resolution_spec( + db: &Trail, + source_root: &ObjectId, + package_root: &str, +) -> Result { + let package_path = join_repo_path(package_root, "package.json"); + let package = db + .root_file_entry(source_root, &package_path)? + .ok_or_else(|| { + Error::InvalidInput(format!( + "Node component `{}` has no package.json", + display_package_root(package_root) + )) + })?; + let projection = db.project_entry_file(&package)?; + let document: serde_json::Value = serde_json::from_slice(&fs::read(projection)?)?; + node_resolution_spec_from_package(&document) +} + +fn node_resolution_spec_from_package(package: &serde_json::Value) -> Result { + let manager = package + .get("packageManager") + .and_then(serde_json::Value::as_str) + .map(|identity| identity.split_once('@').map_or(identity, |(name, _)| name)) + .unwrap_or("npm"); + match manager { + "npm" => Ok(NodeResolutionSpec { + manager: "npm", + lock_name: "package-lock.json", + }), + "pnpm" => Ok(NodeResolutionSpec { + manager: "pnpm", + lock_name: "pnpm-lock.yaml", + }), + "yarn" => Ok(NodeResolutionSpec { + manager: "yarn", + lock_name: "yarn.lock", + }), + "bun" => Ok(NodeResolutionSpec { + manager: "bun", + lock_name: "bun.lock", + }), + other => Err(Error::InvalidInput(format!( + "Node packageManager `{other}` is unsupported; expected npm, pnpm, yarn, or bun" + ))), + } +} + +fn node_lock_resolution_argv(spec: &NodeResolutionSpec) -> Vec<&'static str> { + match spec.manager { + "npm" => vec![ + "npm", + "install", + "--package-lock-only", + "--ignore-scripts", + "--no-audit", + "--no-fund", + ], + "pnpm" => vec!["pnpm", "install", "--lockfile-only", "--ignore-scripts"], + "yarn" => vec!["yarn", "install", "--ignore-scripts"], + "bun" => vec!["bun", "install", "--lockfile-only", "--ignore-scripts"], + _ => Vec::new(), + } +} + +fn node_resolution_proposal_key( + source_root: &ObjectId, + package_root: &str, + spec: &NodeResolutionSpec, +) -> String { + let identity = format!( + "node-lock-resolution-v1\0{}\0{}\0{}\0{}\0{}", + source_root.0, + package_root, + NODE_WORKSPACE_ADAPTER_METADATA.canonical_identity, + spec.manager, + spec.lock_name + ); + format!("node_lock_v1_{}", sha256_hex(identity.as_bytes())) +} + +fn node_resolution_snapshot( + db: &Trail, + source_root: &ObjectId, + package_root: &str, + spec: &NodeResolutionSpec, +) -> Result)>> { + let proposal_key = node_resolution_proposal_key(source_root, package_root, spec); + let expected_component = NODE_WORKSPACE_ADAPTER.component_id(package_root)?; + let snapshot_format = spec.snapshot_format(); + db.verified_workspace_environment_resolution_snapshot( + &proposal_key, + source_root, + &expected_component, + NODE_WORKSPACE_ADAPTER.identity(), + &snapshot_format, + |bytes| validate_node_lock_snapshot(spec, bytes), + ) +} + +fn validate_node_lock_snapshot(spec: &NodeResolutionSpec, bytes: &[u8]) -> Result<()> { + if bytes.is_empty() { + return Err(Error::InvalidInput(format!( + "Trail-managed {} lock snapshot is empty", + spec.manager + ))); + } + match spec.manager { + "npm" => { + let document: serde_json::Value = serde_json::from_slice(bytes).map_err(|error| { + Error::InvalidInput(format!( + "Trail-managed npm lock snapshot is malformed JSON: {error}" + )) + })?; + let version = document + .get("lockfileVersion") + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| { + Error::InvalidInput( + "Trail-managed npm lock snapshot has no lockfileVersion".to_string(), + ) + })?; + if !(1..=3).contains(&version) { + return Err(Error::InvalidInput(format!( + "Trail-managed npm lockfileVersion {version} is unsupported" + ))); + } + } + "pnpm" => { + let text = std::str::from_utf8(bytes).map_err(|_| { + Error::InvalidInput( + "Trail-managed pnpm lock snapshot is not UTF-8 YAML".to_string(), + ) + })?; + if !text + .lines() + .any(|line| line.trim_start().starts_with("lockfileVersion:")) + { + return Err(Error::InvalidInput( + "Trail-managed pnpm lock snapshot has no lockfileVersion".to_string(), + )); + } + } + "yarn" => { + let text = std::str::from_utf8(bytes).map_err(|_| { + Error::InvalidInput("Trail-managed Yarn lock snapshot is not UTF-8".to_string()) + })?; + if !text.contains("yarn lockfile v1") && !text.contains("__metadata:") { + return Err(Error::InvalidInput( + "Trail-managed Yarn lock snapshot has no recognized format marker".to_string(), + )); + } + } + "bun" => {} + other => { + return Err(Error::InvalidInput(format!( + "Trail-managed Node lock snapshot uses unsupported manager `{other}`" + ))); + } + } + Ok(()) +} + fn supported_lockfiles() -> [(&'static str, &'static str); 6] { [ ("pnpm-lock.yaml", "pnpm"), @@ -411,6 +768,213 @@ fn strip_package_root(path: &str, package_root: &str) -> Result { mod tests { use super::*; + fn native_cow_mode() -> LaneWorkdirMode { + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + } + } + + #[test] + fn package_manager_specific_snapshot_formats_are_distinct_and_validated() { + let cases = [ + ( + NodeResolutionSpec { + manager: "npm", + lock_name: "package-lock.json", + }, + br#"{"name":"fixture","lockfileVersion":3,"packages":{}}"#.as_slice(), + ), + ( + NodeResolutionSpec { + manager: "pnpm", + lock_name: "pnpm-lock.yaml", + }, + b"lockfileVersion: '9.0'\nimporters: {}\n".as_slice(), + ), + ( + NodeResolutionSpec { + manager: "yarn", + lock_name: "yarn.lock", + }, + b"# yarn lockfile v1\n".as_slice(), + ), + ( + NodeResolutionSpec { + manager: "bun", + lock_name: "bun.lock", + }, + b"{\n \"lockfileVersion\": 1\n}\n".as_slice(), + ), + ]; + let mut formats = BTreeSet::new(); + for (spec, bytes) in cases { + validate_node_lock_snapshot(&spec, bytes).unwrap(); + assert!(formats.insert(spec.snapshot_format())); + assert_eq!(node_lock_resolution_argv(&spec)[0], spec.manager); + } + assert_eq!(formats.len(), 4); + assert!(validate_node_lock_snapshot( + &NodeResolutionSpec { + manager: "npm", + lock_name: "package-lock.json" + }, + b"{}" + ) + .unwrap_err() + .to_string() + .contains("lockfileVersion")); + } + + #[test] + fn manifest_only_npm_uses_managed_lock_and_preserves_seed_cache_isolation() { + if !Command::new("npm") + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) + || !Command::new("node") + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) + { + return; + } + let workspace = tempfile::tempdir().unwrap(); + let package = r#"{"name":"trail-managed-node-lock","version":"1.0.0","private":true,"packageManager":"npm@10.0.0"}"#; + fs::write(workspace.path().join("package.json"), package).unwrap(); + + let resolver = tempfile::tempdir().unwrap(); + fs::write(resolver.path().join("package.json"), package).unwrap(); + let generated = Command::new("npm") + .args([ + "install", + "--package-lock-only", + "--ignore-scripts", + "--no-audit", + "--no-fund", + ]) + .current_dir(resolver.path()) + .output() + .unwrap(); + assert!( + generated.status.success(), + "npm lock generation failed: {}", + String::from_utf8_lossy(&generated.stderr) + ); + let lock_bytes = fs::read(resolver.path().join("package-lock.json")).unwrap(); + + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let source_root = db.get_ref("refs/branches/main").unwrap().root_id; + assert!(NODE_WORKSPACE_ADAPTER + .detect(&db, &source_root, "") + .unwrap()); + assert_eq!( + NODE_WORKSPACE_ADAPTER + .propose(&db, &source_root, "") + .unwrap() + .unwrap() + .status, + EnvironmentComponentProposalStatus::Resolvable + ); + let resolution_plan = NODE_WORKSPACE_ADAPTER + .resolution_plan(&db, &source_root, "") + .unwrap() + .unwrap(); + assert_eq!( + resolution_plan.snapshot_format, + "node-lock-npm-package-lock.json-v1" + ); + let resolved = db + .resolve_artifact_component( + ArtifactResolutionRequestV1 { + plan: resolution_plan, + candidate: ArtifactResolutionCandidateV1 { + snapshot_bytes: lock_bytes, + resolved_identities: BTreeMap::new(), + checksums: BTreeMap::new(), + contacted_authorities: Vec::new(), + stdout: Vec::new(), + stderr: Vec::new(), + redactions: Vec::new(), + }, + }, + false, + ) + .unwrap(); + assert_eq!(resolved.decision, ArtifactResolutionDecisionV1::Resolved); + + for lane in ["managed-node-one", "managed-node-two"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + native_cow_mode(), + None, + None, + None, + &[], + false, + ) + .unwrap(); + } + let first = db + .sync_workspace_environment("managed-node-one", "node", None) + .unwrap(); + let second = db + .sync_workspace_environment("managed-node-two", "node", None) + .unwrap(); + assert_eq!(first.layer_id, second.layer_id); + assert_eq!(first.cache_key, second.cache_key); + assert!(!workspace.path().join("package-lock.json").exists()); + assert!(db + .root_file_entry(&source_root, "package-lock.json") + .unwrap() + .is_none()); + let generation = db + .active_environment_generation("managed-node-one") + .unwrap() + .unwrap(); + assert_eq!( + generation.components[0].outputs[0].policy, + EnvironmentOutputPolicy::ImmutableSeedPrivate + ); + assert_eq!( + generation.components[0].caches[0].authority, + "performance_only" + ); + assert_eq!(generation.components[0].caches[0].protocol, "content_store"); + + fs::write(workspace.path().join("README.md"), "new source root\n").unwrap(); + db.record( + Some("main"), + Some("change Node source root".to_string()), + Actor::human(), + false, + ) + .unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "managed-node-new-root", + Some("main"), + native_cow_mode(), + None, + None, + None, + &[], + false, + ) + .unwrap(); + let discovery = db + .discover_workspace_environment("managed-node-new-root", None) + .unwrap(); + assert_eq!( + discovery.components[0].status, + EnvironmentComponentProposalStatus::Resolvable + ); + } + #[test] fn two_lanes_with_identical_node_inputs_reuse_one_real_frozen_install() { if !Command::new("npm") @@ -437,18 +1001,11 @@ mod tests { .unwrap(); Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(workspace.path()).unwrap(); - let mode = if cfg!(target_os = "macos") { - LaneWorkdirMode::NfsCow - } else if cfg!(target_os = "windows") { - LaneWorkdirMode::DokanCow - } else { - LaneWorkdirMode::FuseCow - }; for lane in ["node-one", "node-two"] { db.spawn_lane_with_workdir_mode_paths_and_neighbors( lane, Some("main"), - mode.clone(), + native_cow_mode(), None, None, None, diff --git a/trail/src/db/lane/workspace_oci.rs b/trail/src/db/lane/workspace_oci.rs index daa82082..7b9b627e 100644 --- a/trail/src/db/lane/workspace_oci.rs +++ b/trail/src/db/lane/workspace_oci.rs @@ -313,6 +313,7 @@ impl WorkspaceEnvironmentAdapter for OciImageAdapter { strategy: "pinned-oci-images-v1".to_string(), }, inputs: Vec::new(), + resolution_inputs: Vec::new(), source_projection: None, pre_commands: Vec::new(), command: None, @@ -401,6 +402,21 @@ mod tests { assert_eq!(plan.external_artifacts.len(), 1); assert_eq!(plan.external_artifacts[0].digest, DIGEST); assert_eq!(plan.external_artifacts[0].platform, "linux/amd64"); + let identity = super::workspace_environment::workspace_environment_identity_contract_v3( + &plan, + super::workspace_environment::workspace_environment_artifact_contract_digest(&plan) + .unwrap(), + ) + .unwrap(); + assert!(!identity.source_closure_complete); + assert!(!identity.portability_certified); + assert!(identity + .semantic_identities + .contains_key("external_artifacts")); + assert!(!identity + .semantic_identities + .contains_key("runtime_resources")); + assert_eq!(identity.trust_scope, "builtin"); let after = fs::read_dir(&cache_root) .ok() @@ -529,6 +545,23 @@ mod tests { Some("POSTGRES_PASSWORD_FILE") ); assert!(plan.outputs.is_empty()); + let raw_plan = OCI_IMAGE_ADAPTER + .plan(&db, &db.resolve_branch_ref("main").unwrap().root_id, "") + .unwrap(); + let identity = super::workspace_environment::workspace_environment_identity_contract_v3( + &raw_plan, + super::workspace_environment::workspace_environment_artifact_contract_digest(&raw_plan) + .unwrap(), + ) + .unwrap(); + assert!(identity + .semantic_identities + .contains_key("external_artifacts")); + assert!(identity + .semantic_identities + .contains_key("runtime_resources")); + assert!(raw_plan.outputs.is_empty()); + assert!(raw_plan.caches.is_empty()); let first = db .sync_workspace_environment_component( diff --git a/trail/src/db/lane/workspace_plugin.rs b/trail/src/db/lane/workspace_plugin.rs index 9eb2dcdf..daa46275 100644 --- a/trail/src/db/lane/workspace_plugin.rs +++ b/trail/src/db/lane/workspace_plugin.rs @@ -7,14 +7,16 @@ use ed25519_dalek::{Signature, VerifyingKey}; use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; use serde::{Deserialize, Serialize}; use trail_environment_adapter_sdk::{ - read_frame, write_frame, AdapterAction, AdapterCache, AdapterCacheAccess, AdapterCacheProtocol, - AdapterCommand, AdapterDependencyType, AdapterExternalArtifact, AdapterHost, AdapterOperation, - AdapterOutput, AdapterOutputPolicy, AdapterPackageManifest, AdapterPackageSignature, - AdapterPermissions, AdapterPlan, AdapterPlanV2, AdapterPortability, AdapterPublicationTrigger, - AdapterPublisherKey, AdapterRequest, AdapterResponse, AdapterResult, AdapterReuseMode, - AdapterRuntimeResource, AdapterSharingScope, DiscoveredComponent, PinnedFile, MAX_FRAME_BYTES, - PACKAGE_SCHEMA_V1, PACKAGE_SIGNATURE_SCHEMA_V1, PROTOCOL_V1, PROTOCOL_V2, - TRUSTED_PUBLISHER_KEY_SCHEMA_V1, + negotiate_highest_mutual_protocol, read_frame, write_frame, AdapterAction, AdapterCache, + AdapterCacheAccess, AdapterCacheProtocol, AdapterCommand, AdapterDependencyType, + AdapterExternalArtifact, AdapterHost, AdapterOperation, AdapterOutput, AdapterOutputPolicy, + AdapterPackageManifest, AdapterPackageSignature, AdapterPermissions, AdapterPlan, + AdapterPlanV2, AdapterPortability, AdapterProposalStatusV3, AdapterPublicationTrigger, + AdapterPublisherKey, AdapterRequest, AdapterRequestV3, AdapterResponse, AdapterResponseV3, + AdapterResult, AdapterResultV3, AdapterReuseMode, AdapterRuntimeResource, + AdapterSecretCapabilityV3, AdapterSecretTaintV3, AdapterSharingScope, DiscoveredComponent, + PinnedFile, MAX_FRAME_BYTES, PACKAGE_SCHEMA_V1, PACKAGE_SIGNATURE_SCHEMA_V1, PROTOCOL_V1, + PROTOCOL_V2, PROTOCOL_V3, TRUSTED_PUBLISHER_KEY_SCHEMA_V1, }; use super::workspace_environment::{ @@ -349,6 +351,8 @@ impl Trail { package_directory: impl AsRef, ) -> Result { let prepared = prepare_environment_plugin_package(package_directory)?; + let protocol_capabilities = + environment_plugin_protocol_capabilities(&prepared.package.adapter); let distribution_material = adapter_package_distribution_material( &prepared.payload_material, prepared.signature.as_ref(), @@ -364,6 +368,7 @@ impl Trail { .as_ref() .map(|signature| signature.publisher.clone()), publisher_key_id: prepared.signature.map(|signature| signature.key_id), + protocol_capabilities, }) } @@ -401,6 +406,7 @@ impl Trail { adapter_package_distribution_material(&payload_material, signature.as_ref())?; let distribution_hex = sha256_hex(&distribution_material); let distribution_digest = format!("sha256:{distribution_hex}"); + let protocol_capabilities = environment_plugin_protocol_capabilities(&package.adapter); let executable_file = package.executable.path.clone(); let installed = InstalledPluginManifest { package: package.clone(), @@ -526,6 +532,7 @@ impl Trail { publisher_key_id, trust, certification_tier, + protocol_capabilities, }) } @@ -534,6 +541,12 @@ impl Trail { canonical_identity: &str, ) -> Result { validate_plugin_identity(canonical_identity)?; + let protocol_capabilities = self + .environment_plugin_for_selector(canonical_identity) + .ok() + .flatten() + .map(|plugin| environment_plugin_protocol_capabilities(&plugin.manifest.adapter)) + .unwrap_or_default(); let existing = self .latest_environment_plugin_registry_record(canonical_identity)? .filter(|record| record.action == "install") @@ -548,6 +561,7 @@ impl Trail { Ok(EnvironmentPluginRemoveReport { canonical_identity: canonical_identity.to_string(), removed_distribution_digest: existing, + protocol_capabilities, }) } @@ -981,6 +995,9 @@ impl Trail { component_root: component_root.to_string(), kind: component.kind, adapter_identity: plugin.manifest.adapter.canonical_identity.clone(), + status: EnvironmentComponentProposalStatus::Ready, + reasons: Vec::new(), + recovery_actions: Vec::new(), })) } _ => Err(Error::InvalidInput(format!( @@ -1981,6 +1998,7 @@ impl Trail { .to_string(), }, inputs, + resolution_inputs: Vec::new(), source_projection: None, pre_commands: Vec::new(), command, @@ -2276,6 +2294,240 @@ fn validate_discovered_plugin_component( validate_plugin_component_id(&component.component_id) } +#[cfg_attr( + not(test), + expect( + dead_code, + reason = "v3 exchange validation is staged before v3 invocation/normalization is enabled" + ) +)] +pub(super) fn validate_environment_plugin_v3_exchange( + plugin: &InstalledEnvironmentPlugin, + request: &AdapterRequestV3, + response: AdapterResponseV3, +) -> Result { + if !plugin + .manifest + .adapter + .protocols + .iter() + .any(|protocol| protocol == PROTOCOL_V3) + || request.protocol != PROTOCOL_V3 + || response.protocol != PROTOCOL_V3 + || response.request_id != request.request_id + || request.adapter_identity != plugin.manifest.adapter.canonical_identity + || request.distribution_digest != plugin.distribution_digest + { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned a protocol/package/request mismatch for protocol v3", + plugin.manifest.adapter.canonical_identity + ))); + } + request.validate_bounds().map_err(|error| { + Error::InvalidInput(format!("invalid protocol-v3 adapter request: {error}")) + })?; + response.validate_bounds(&request.limits).map_err(|error| { + Error::InvalidInput(format!("invalid protocol-v3 adapter response: {error}")) + })?; + + let capabilities = &plugin.manifest.adapter.capabilities; + let host_evidence = match &request.operation { + trail_environment_adapter_sdk::AdapterOperationV3::Plan { host_evidence, .. } => { + host_evidence.as_deref() + } + trail_environment_adapter_sdk::AdapterOperationV3::Propose { .. } => None, + }; + if host_evidence.is_some_and(|evidence| { + (!evidence.attestations.is_empty() && !capabilities.host_attestation_evidence) + || (!evidence.quarantines.is_empty() && !capabilities.host_quarantine_evidence) + }) { + return Err(Error::InvalidInput(format!( + "adapter `{}` received host evidence beyond its package declaration", + plugin.manifest.adapter.canonical_identity + ))); + } + + match response.result { + AdapterResultV3::Error { + code, + message, + recovery_actions: _, + } => Err(Error::InvalidInput(format!( + "adapter `{}` rejected protocol-v3 request with {}: {}", + plugin.manifest.adapter.canonical_identity, + redact_sensitive_text(&code), + redact_sensitive_text(&message) + ))), + AdapterResultV3::Proposed { component } => { + let trail_environment_adapter_sdk::AdapterOperationV3::Propose { + component_root, .. + } = &request.operation + else { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned a proposal to a non-proposal request", + plugin.manifest.adapter.canonical_identity + ))); + }; + if let Some(proposal) = &component { + validate_environment_plugin_v3_proposal(plugin, component_root, proposal)?; + } + Ok(AdapterResultV3::Proposed { component }) + } + AdapterResultV3::Planned { pipeline } => { + let trail_environment_adapter_sdk::AdapterOperationV3::Plan { + proposal, files, .. + } = &request.operation + else { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned a plan to a non-plan request", + plugin.manifest.adapter.canonical_identity + ))); + }; + if pipeline.proposal != **proposal { + return Err(Error::InvalidInput(format!( + "adapter `{}` changed the pinned proposal while planning", + plugin.manifest.adapter.canonical_identity + ))); + } + validate_environment_plugin_v3_proposal( + plugin, + &proposal.component_root, + &pipeline.proposal, + )?; + pipeline.validate_canonical().map_err(|error| { + Error::InvalidInput(format!( + "adapter `{}` returned a noncanonical or invalid protocol-v3 pipeline: {error}", + plugin.manifest.adapter.canonical_identity + )) + })?; + + let mut expected_inputs = files + .iter() + .map(|pinned| pinned.input.clone()) + .collect::>(); + expected_inputs.sort(); + if pipeline.inputs != expected_inputs { + return Err(Error::InvalidInput(format!( + "adapter `{}` protocol-v3 plan omitted, added, or changed pinned inputs", + plugin.manifest.adapter.canonical_identity + ))); + } + for input in &pipeline.inputs { + if normalize_relative_path(&input.path)? != input.path { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned non-normalized input `{}`", + plugin.manifest.adapter.canonical_identity, input.path + ))); + } + } + if pipeline.identity.platform != request.host.operating_system + || pipeline.identity.architecture != request.host.architecture + { + return Err(Error::InvalidInput(format!( + "adapter `{}` changed the pinned host identity", + plugin.manifest.adapter.canonical_identity + ))); + } + if pipeline.resolution.is_some() && !capabilities.resolution { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned resolution without package capability", + plugin.manifest.adapter.canonical_identity + ))); + } + if !pipeline.source_exports.is_empty() && !capabilities.source_exports { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned source exports without package capability", + plugin.manifest.adapter.canonical_identity + ))); + } + if pipeline.identity.portability_certified + && capabilities.certification_ceiling + != trail_environment_adapter_sdk::AdapterCertificationCeiling::PortableArtifact + { + return Err(Error::InvalidInput(format!( + "adapter `{}` exceeded its package certification ceiling", + plugin.manifest.adapter.canonical_identity + ))); + } + if pipeline.secret_taint == AdapterSecretTaintV3::Clear + && (pipeline.capabilities.secrets != AdapterSecretCapabilityV3::Deny + || pipeline.actions.iter().any(|action| { + action.capabilities.secrets != AdapterSecretCapabilityV3::Deny + }) + || pipeline.resolution.as_ref().is_some_and(|resolution| { + resolution.capabilities.secrets != AdapterSecretCapabilityV3::Deny + })) + { + return Err(Error::InvalidInput(format!( + "adapter `{}` must conservatively taint a secret-capable protocol-v3 pipeline", + plugin.manifest.adapter.canonical_identity + ))); + } + Ok(AdapterResultV3::Planned { pipeline }) + } + } +} + +#[cfg_attr( + not(test), + expect( + dead_code, + reason = "called by the staged v3 exchange validator before v3 invocation is enabled" + ) +)] +fn validate_environment_plugin_v3_proposal( + plugin: &InstalledEnvironmentPlugin, + expected_root: &str, + proposal: &trail_environment_adapter_sdk::AdapterComponentProposalV3, +) -> Result<()> { + validate_discovered_plugin_component( + plugin, + &DiscoveredComponent { + component_id: proposal.component_id.clone(), + kind: proposal.kind.clone(), + }, + )?; + if normalize_plugin_component_root(&proposal.component_root)? + != normalize_plugin_component_root(expected_root)? + || proposal.proposal_key.trim().is_empty() + || contains_sensitive_text(&proposal.proposal_key) + { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned an invalid protocol-v3 proposal root or key", + plugin.manifest.adapter.canonical_identity + ))); + } + if proposal.status == AdapterProposalStatusV3::Ready + && !proposal.missing_requirements.is_empty() + { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned a ready proposal with missing requirements", + plugin.manifest.adapter.canonical_identity + ))); + } + if !strictly_increasing_by(&proposal.missing_requirements, |requirement| { + requirement.code.as_str() + }) || !strictly_increasing_by(&proposal.recovery_actions, |action| action.code.as_str()) + { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned duplicate or noncanonical proposal evidence", + plugin.manifest.adapter.canonical_identity + ))); + } + Ok(()) +} + +#[cfg_attr( + not(test), + expect( + dead_code, + reason = "called by the staged v3 proposal validator before v3 invocation is enabled" + ) +)] +fn strictly_increasing_by<'a, T>(values: &'a [T], key: impl Fn(&'a T) -> &'a str) -> bool { + values.windows(2).all(|pair| key(&pair[0]) < key(&pair[1])) +} + fn validate_plugin_component_id(component_id: &str) -> Result<()> { if component_id.is_empty() || component_id.len() > 256 @@ -2308,30 +2560,101 @@ pub(super) fn environment_plugin_supports_current_host( .any(|value| value == std::env::consts::ARCH) } +pub(super) fn unsupported_environment_plugin_proposal( + plugin: &InstalledEnvironmentPlugin, + component_root: &str, +) -> Result { + let component_root = normalize_plugin_component_root(component_root)?; + let component_id = if component_root.is_empty() { + format!("plugin:{}", plugin.manifest.adapter.layer_adapter_name) + } else { + format!( + "plugin:{}:{component_root}", + plugin.manifest.adapter.layer_adapter_name + ) + }; + validate_plugin_component_id(&component_id)?; + Ok(EnvironmentDiscoveredComponentReport { + component_id, + component_root, + kind: plugin.manifest.adapter.kind.clone(), + adapter_identity: plugin.manifest.adapter.canonical_identity.clone(), + status: EnvironmentComponentProposalStatus::Unsupported, + reasons: vec![EnvironmentProposalReasonReport { + code: "unsupported_host_platform".to_string(), + message: format!( + "adapter `{}` recognizes this marker but does not support {}/{}", + plugin.manifest.adapter.canonical_identity, + std::env::consts::OS, + std::env::consts::ARCH + ), + }], + recovery_actions: vec![EnvironmentRecoveryActionReport { + code: "use_supported_host".to_string(), + description: format!( + "Use a host matching the adapter's supported operating systems ({}) and architectures ({})", + plugin + .manifest + .adapter + .supported_operating_systems + .join(", "), + plugin + .manifest + .adapter + .supported_architectures + .join(", ") + ), + command: None, + }], + }) +} + pub(super) fn selected_environment_plugin_protocol( plugin: &InstalledEnvironmentPlugin, ) -> Result<&'static str> { - if plugin - .manifest - .adapter - .protocols - .iter() - .any(|protocol| protocol == PROTOCOL_V2) - { - Ok(PROTOCOL_V2) - } else if plugin - .manifest - .adapter - .protocols - .iter() - .any(|protocol| protocol == PROTOCOL_V1) - { - Ok(PROTOCOL_V1) - } else { - Err(Error::InvalidInput(format!( + negotiate_highest_mutual_protocol( + &[PROTOCOL_V2, PROTOCOL_V1], + &plugin.manifest.adapter.protocols, + ) + .ok_or_else(|| { + Error::InvalidInput(format!( "adapter `{}` shares no supported protocol with this Trail host", plugin.manifest.adapter.canonical_identity - ))) + )) + }) +} + +pub(super) fn environment_plugin_protocol_capabilities( + metadata: &trail_environment_adapter_sdk::AdapterMetadata, +) -> EnvironmentPluginProtocolCapabilitiesReport { + let declares_v3 = metadata + .protocols + .iter() + .any(|protocol| protocol == PROTOCOL_V3); + let capabilities = &metadata.capabilities; + EnvironmentPluginProtocolCapabilitiesReport { + selected_protocol: negotiate_highest_mutual_protocol( + &[PROTOCOL_V2, PROTOCOL_V1], + &metadata.protocols, + ) + .map(str::to_string), + resolution_capable: declares_v3 && capabilities.resolution, + source_export_capable: declares_v3 && capabilities.source_exports, + host_attestation_evidence_capable: declares_v3 && capabilities.host_attestation_evidence, + host_quarantine_evidence_capable: declares_v3 && capabilities.host_quarantine_evidence, + certification_ceiling: capabilities.certification_ceiling.as_str().into(), + content_policy: if declares_v3 { + "host_verified_local_cas" + } else { + "legacy_exact_layer" + } + .into(), + attestation_policy: if declares_v3 { + "host_authored_required" + } else { + "legacy_host_evidence" + } + .into(), } } @@ -2736,13 +3059,25 @@ fn canonicalize_and_validate_package(package: &mut AdapterPackageManifest) -> Re .adapter .protocols .iter() - .any(|protocol| !matches!(protocol.as_str(), PROTOCOL_V1 | PROTOCOL_V2)) + .any(|protocol| !matches!(protocol.as_str(), PROTOCOL_V1 | PROTOCOL_V2 | PROTOCOL_V3)) { return Err(Error::InvalidInput(format!( "adapter `{}` must declare one or more host-supported protocols", package.adapter.canonical_identity ))); } + if !package + .adapter + .protocols + .iter() + .any(|protocol| protocol == PROTOCOL_V3) + && !package.adapter.capabilities.is_denied() + { + return Err(Error::InvalidInput(format!( + "adapter `{}` declares protocol-v3 capabilities without protocol v3", + package.adapter.canonical_identity + ))); + } for marker in &package.adapter.discovery_markers { if marker == "." || marker == ".." @@ -2997,8 +3332,387 @@ fn validate_plugin_catalog(plugins: &[InstalledEnvironmentPlugin]) -> Result<()> #[cfg(test)] mod tests { use super::*; + use crate::ids::{ArtifactEnvelopeId, ArtifactTreeId}; use ed25519_dalek::{Signer, SigningKey}; + fn plugin_with_protocols(protocols: &[&str]) -> InstalledEnvironmentPlugin { + InstalledEnvironmentPlugin { + manifest: AdapterPackageManifest { + schema: PACKAGE_SCHEMA_V1.into(), + adapter: trail_environment_adapter_sdk::AdapterMetadata { + canonical_identity: "example/test@1".into(), + implementation_version: "1".into(), + selectors: vec!["example/test@1".into()], + kind: "generated".into(), + layer_adapter_name: "test".into(), + discovery_markers: vec!["test.adapter".into()], + protocols: protocols + .iter() + .map(|protocol| (*protocol).into()) + .collect(), + capabilities: + trail_environment_adapter_sdk::AdapterPackageCapabilities::default(), + supported_operating_systems: vec![std::env::consts::OS.into()], + supported_architectures: vec![std::env::consts::ARCH.into()], + stability: "experimental".into(), + description: "test".into(), + }, + executable: trail_environment_adapter_sdk::AdapterExecutable { + path: "adapter".into(), + sha256: format!("sha256:{}", "0".repeat(64)), + }, + permissions: AdapterPermissions::default(), + }, + distribution_digest: format!("sha256:{}", "1".repeat(64)), + executable_digest: format!("sha256:{}", "2".repeat(64)), + executable_path: PathBuf::from("adapter"), + publisher: None, + publisher_key_id: None, + trust: "local".into(), + certification_tier: "uncertified".into(), + } + } + + fn valid_v3_exchange() -> ( + InstalledEnvironmentPlugin, + AdapterRequestV3, + AdapterResponseV3, + ) { + use trail_environment_adapter_sdk::{ + denied_capabilities_v3, AdapterActionLimitsV3, AdapterActionPhaseV3, AdapterActionV3, + AdapterCertificationCeiling, AdapterComponentProposalV3, AdapterIdentityContractV3, + AdapterInputRoleV3, AdapterInputV3, AdapterOperationV3, AdapterPinnedInputV3, + AdapterPipelineV3, AdapterProcessCapabilityV3, AdapterProtocolLimitsV3, + AdapterQuarantinePolicyV3, AdapterSourceExportAuthorizationV3, + AdapterSourceExportCollisionV3, AdapterSourceExportV3, AdapterValidationKindV3, + AdapterValidationV3, + }; + + let mut plugin = plugin_with_protocols(&[PROTOCOL_V3, PROTOCOL_V2]); + plugin.manifest.adapter.capabilities.source_exports = true; + plugin.manifest.adapter.capabilities.certification_ceiling = + AdapterCertificationCeiling::LocalArtifact; + let proposal = AdapterComponentProposalV3 { + component_id: "example.generated".into(), + component_root: ".".into(), + kind: "generated".into(), + status: AdapterProposalStatusV3::Ready, + proposal_key: "sha256:proposal".into(), + missing_requirements: Vec::new(), + recovery_actions: Vec::new(), + }; + let inputs = [ + AdapterPinnedInputV3 { + input: AdapterInputV3 { + path: "z.json".into(), + content_hash: "sha256:z".into(), + size_bytes: 2, + executable: false, + role: AdapterInputRoleV3::Identity, + format: "application/json".into(), + required: true, + }, + content: b"{}".to_vec(), + }, + AdapterPinnedInputV3 { + input: AdapterInputV3 { + path: "a.json".into(), + content_hash: "sha256:a".into(), + size_bytes: 2, + executable: false, + role: AdapterInputRoleV3::Identity, + format: "application/json".into(), + required: true, + }, + content: b"{}".to_vec(), + }, + ]; + let request = AdapterRequestV3 { + protocol: PROTOCOL_V3.into(), + request_id: "request-v3".into(), + adapter_identity: plugin.manifest.adapter.canonical_identity.clone(), + distribution_digest: plugin.distribution_digest.clone(), + host: AdapterHost { + operating_system: std::env::consts::OS.into(), + architecture: std::env::consts::ARCH.into(), + }, + source_root: "root:v3".into(), + limits: AdapterProtocolLimitsV3::default(), + operation: AdapterOperationV3::Plan { + proposal: Box::new(proposal.clone()), + files: inputs.to_vec(), + resolution_snapshot: None, + host_evidence: None, + }, + }; + let mut action_capabilities = denied_capabilities_v3(); + action_capabilities.process = AdapterProcessCapabilityV3::DeclaredExecutable; + let pipeline = AdapterPipelineV3::builder( + proposal, + AdapterIdentityContractV3 { + normalizer_version: "trail-path-v1".into(), + source_closure_complete: true, + semantic_identities: BTreeMap::new(), + target: "host".into(), + platform: std::env::consts::OS.into(), + architecture: std::env::consts::ARCH.into(), + abi: "host".into(), + portability: AdapterPortability::Host, + portability_certified: false, + portability_scope: "workspace".into(), + trust_scope: "local_plugin".into(), + }, + ) + .input(inputs[0].input.clone()) + .input(inputs[1].input.clone()) + .action(AdapterActionV3 { + name: "construct".into(), + phase: AdapterActionPhaseV3::Construct, + program: "generator".into(), + argv: vec!["build".into()], + working_directory: ".".into(), + environment: BTreeMap::new(), + capabilities: action_capabilities.clone(), + limits: AdapterActionLimitsV3 { + timeout_ms: 30_000, + stdout_bytes: 1024, + stderr_bytes: 1024, + output_entries: 100, + output_bytes: 1024 * 1024, + child_processes: 0, + }, + }) + .action(AdapterActionV3 { + name: "finalize".into(), + phase: AdapterActionPhaseV3::Finalize, + program: "generator".into(), + argv: vec!["finalize".into()], + working_directory: ".".into(), + environment: BTreeMap::new(), + capabilities: action_capabilities, + limits: AdapterActionLimitsV3 { + timeout_ms: 30_000, + stdout_bytes: 1024, + stderr_bytes: 1024, + output_entries: 100, + output_bytes: 1024 * 1024, + child_processes: 0, + }, + }) + .validation(AdapterValidationV3 { + name: "path-contract".into(), + kind: AdapterValidationKindV3::PathContract, + path: "generated".into(), + required: true, + parameters: BTreeMap::new(), + }) + .output(AdapterOutput::immutable_seed_private( + "generated", + "generated", + ".trail-generated/generated", + )) + .source_export(AdapterSourceExportV3 { + name: "client".into(), + output_name: "generated".into(), + artifact_subpath: "client".into(), + destination: "src/generated".into(), + collision: AdapterSourceExportCollisionV3::Fail, + required_validation: "path-contract".into(), + required_gate: None, + authorization: AdapterSourceExportAuthorizationV3::ExplicitUser, + }) + .quarantine_policy(AdapterQuarantinePolicyV3::FailClosed) + .stale_reason("source or tool changed") + .build() + .unwrap(); + let response = AdapterResponseV3 { + protocol: PROTOCOL_V3.into(), + request_id: request.request_id.clone(), + result: AdapterResultV3::Planned { + pipeline: Box::new(pipeline), + }, + }; + (plugin, request, response) + } + + #[test] + fn host_negotiation_falls_back_exactly_until_v3_normalization_is_enabled() { + assert_eq!( + selected_environment_plugin_protocol(&plugin_with_protocols(&[ + PROTOCOL_V1, + PROTOCOL_V3, + PROTOCOL_V2, + ])) + .unwrap(), + PROTOCOL_V2 + ); + assert_eq!( + selected_environment_plugin_protocol(&plugin_with_protocols(&[ + PROTOCOL_V3, + PROTOCOL_V1, + ])) + .unwrap(), + PROTOCOL_V1 + ); + assert!( + selected_environment_plugin_protocol(&plugin_with_protocols(&[PROTOCOL_V3])) + .unwrap_err() + .to_string() + .contains("shares no supported protocol") + ); + + let mut plugin = plugin_with_protocols(&[PROTOCOL_V3, PROTOCOL_V2]); + plugin.manifest.adapter.capabilities.resolution = true; + plugin.manifest.adapter.capabilities.source_exports = true; + plugin.manifest.adapter.capabilities.certification_ceiling = + trail_environment_adapter_sdk::AdapterCertificationCeiling::LocalArtifact; + let report = environment_plugin_protocol_capabilities(&plugin.manifest.adapter); + assert_eq!(report.selected_protocol.as_deref(), Some(PROTOCOL_V2)); + assert!(report.resolution_capable); + assert!(report.source_export_capable); + assert_eq!(report.certification_ceiling, "local_artifact"); + assert_eq!(report.content_policy, "host_verified_local_cas"); + assert_eq!(report.attestation_policy, "host_authored_required"); + } + + #[test] + fn protocol_v3_host_validation_repeats_bounds_canonical_and_package_checks() { + let (plugin, request, response) = valid_v3_exchange(); + assert!(matches!( + validate_environment_plugin_v3_exchange(&plugin, &request, response.clone()).unwrap(), + AdapterResultV3::Planned { .. } + )); + + let mut noncanonical = response.clone(); + let AdapterResultV3::Planned { pipeline } = &mut noncanonical.result else { + unreachable!(); + }; + pipeline.inputs.reverse(); + assert!( + validate_environment_plugin_v3_exchange(&plugin, &request, noncanonical) + .unwrap_err() + .to_string() + .contains("noncanonical") + ); + + let mut too_small = request.clone(); + too_small.limits.max_actions = 1; + assert!( + validate_environment_plugin_v3_exchange(&plugin, &too_small, response.clone()) + .unwrap_err() + .to_string() + .contains("pipeline.actions") + ); + + let mut underdeclared = plugin.clone(); + underdeclared.manifest.adapter.capabilities.source_exports = false; + assert!(validate_environment_plugin_v3_exchange( + &underdeclared, + &request, + response.clone() + ) + .unwrap_err() + .to_string() + .contains("without package capability")); + + let mut duplicate = response.clone(); + let AdapterResultV3::Planned { pipeline } = &mut duplicate.result else { + unreachable!(); + }; + pipeline.actions.push(pipeline.actions[0].clone()); + assert!( + validate_environment_plugin_v3_exchange(&plugin, &request, duplicate) + .unwrap_err() + .to_string() + .contains("repeats") + ); + + let mut invalid_request = request.clone(); + let trail_environment_adapter_sdk::AdapterOperationV3::Plan { files, .. } = + &mut invalid_request.operation + else { + unreachable!(); + }; + files[0].input.path = "src/../escape".into(); + let mut invalid_path = response.clone(); + let AdapterResultV3::Planned { pipeline } = &mut invalid_path.result else { + unreachable!(); + }; + pipeline.inputs[1].path = "src/../escape".into(); + pipeline.inputs.sort(); + assert!( + validate_environment_plugin_v3_exchange(&plugin, &invalid_request, invalid_path) + .unwrap_err() + .to_string() + .contains("invalid relative path") + ); + + let mut invalid_phase = response.clone(); + let AdapterResultV3::Planned { pipeline } = &mut invalid_phase.result else { + unreachable!(); + }; + pipeline.actions[0].phase = + trail_environment_adapter_sdk::AdapterActionPhaseV3::MountedInitialization; + pipeline.actions.sort_by(|left, right| { + left.phase + .cmp(&right.phase) + .then_with(|| left.name.cmp(&right.name)) + }); + assert!( + validate_environment_plugin_v3_exchange(&plugin, &request, invalid_phase) + .unwrap_err() + .to_string() + .contains("mounted initialization") + ); + + let mut mismatched = request; + mismatched.distribution_digest = "sha256:wrong".into(); + assert!( + validate_environment_plugin_v3_exchange(&plugin, &mismatched, response) + .unwrap_err() + .to_string() + .contains("protocol/package/request mismatch") + ); + } + + #[test] + fn protocol_v3_host_identity_is_enforced_for_every_supported_platform() { + for (operating_system, architecture) in [ + ("linux", "x86_64"), + ("macos", "aarch64"), + ("windows", "x86_64"), + ] { + let (mut plugin, mut request, mut response) = valid_v3_exchange(); + plugin.manifest.adapter.supported_operating_systems = + vec![operating_system.to_string()]; + plugin.manifest.adapter.supported_architectures = vec![architecture.to_string()]; + request.host.operating_system = operating_system.into(); + request.host.architecture = architecture.into(); + if let AdapterResultV3::Planned { pipeline } = &mut response.result { + pipeline.identity.platform = operating_system.into(); + pipeline.identity.architecture = architecture.into(); + } else { + unreachable!(); + } + assert!( + validate_environment_plugin_v3_exchange(&plugin, &request, response.clone()) + .is_ok(), + "matching {operating_system}/{architecture} identity must pass" + ); + + let AdapterResultV3::Planned { pipeline } = &mut response.result else { + unreachable!(); + }; + pipeline.identity.architecture = "mismatched".into(); + assert!( + validate_environment_plugin_v3_exchange(&plugin, &request, response) + .unwrap_err() + .to_string() + .contains("changed the pinned host identity") + ); + } + } + #[test] fn protocol_v2_typed_dependencies_normalize_to_host_edge_semantics() { let plan = AdapterPlanV2::builder("application", "generated") @@ -3059,6 +3773,70 @@ mod tests { ); } + #[test] + fn protocol_v2_bazel_nix_like_stores_remain_metadata_only_after_host_normalization() { + let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let plan = AdapterPlanV2::builder("verified-stores", "external") + .identity_input("stores.lock") + .external_artifact(AdapterExternalArtifact::verified_external( + "content-addressed-store", + "local-store", + "store://objects/example-package", + digest, + "linux/x86_64", + )) + .external_artifact(AdapterExternalArtifact::verified_external( + "remote-action-cache", + "remote-cas", + "cas://objects/example-action", + digest, + "any", + )) + .stale_reason("verified external store identities changed") + .build() + .unwrap(); + + let normalized = ProposedPluginPlan::from_v2(plan).unwrap(); + assert_eq!(normalized.component_id, "verified-stores"); + assert!(normalized.outputs.is_empty()); + assert!(normalized.caches.is_empty()); + assert!(normalized.runtime_resources.is_empty()); + assert_eq!(normalized.external_artifacts.len(), 2); + assert!(normalized.external_artifacts.iter().all(|artifact| { + artifact.artifact_type == "verified_external" && artifact.cleanup_owner == "external" + })); + for artifact in &normalized.external_artifacts { + super::workspace_environment::validate_environment_external_artifact_report( + &EnvironmentExternalArtifactReport { + name: artifact.name.clone(), + artifact_type: artifact.artifact_type.clone(), + provider: artifact.provider.clone(), + reference: artifact.reference.clone(), + digest: artifact.digest.clone(), + platform: artifact.platform.clone(), + cleanup_owner: artifact.cleanup_owner.clone(), + }, + ) + .unwrap(); + } + let mut secret_like = normalized.external_artifacts[0].clone(); + secret_like.reference = "store://objects/token=credential".to_string(); + assert!( + super::workspace_environment::validate_environment_external_artifact_report( + &EnvironmentExternalArtifactReport { + name: secret_like.name, + artifact_type: secret_like.artifact_type, + provider: secret_like.provider, + reference: secret_like.reference, + digest: secret_like.digest, + platform: secret_like.platform, + cleanup_owner: secret_like.cleanup_owner, + }, + ) + .is_err() + ); + } + #[test] fn protocol_v2_plugin_caches_are_host_scoped_and_fail_closed() { let workspace = tempfile::tempdir().unwrap(); @@ -3158,6 +3936,26 @@ max_response_bytes = 1048576 .unwrap(); } + fn make_test_package_unsupported_on_current_host(root: &Path) { + let manifest_path = root.join(PLUGIN_PACKAGE_MANIFEST); + let manifest = fs::read_to_string(&manifest_path).unwrap(); + let unsupported_os = if cfg!(target_os = "windows") { + "linux" + } else { + "windows" + }; + fs::write( + manifest_path, + manifest.replace( + "stability = \"experimental\"", + &format!( + "supported_operating_systems = [\"{unsupported_os}\"]\nstability = \"experimental\"" + ), + ), + ) + .unwrap(); + } + fn sign_test_package(root: &Path, signing_key: &SigningKey, publisher: &str) -> PathBuf { let mut package: AdapterPackageManifest = toml::from_str(&fs::read_to_string(root.join(PLUGIN_PACKAGE_MANIFEST)).unwrap()) @@ -3209,6 +4007,10 @@ max_response_bytes = 1048576 .unwrap(); assert_eq!(inspected.canonical_identity, "example/test@1"); assert!(!inspected.signature_present); + assert_eq!( + inspected.protocol_capabilities.selected_protocol.as_deref(), + Some(PROTOCOL_V1) + ); let installed = db .install_environment_adapter_plugin(package.path()) @@ -3219,6 +4021,10 @@ max_response_bytes = 1048576 assert_eq!(installed.trust, "local_unsigned"); assert_eq!(installed.certification_tier, "local-experimental"); assert!(installed.publisher.is_none()); + assert_eq!( + installed.protocol_capabilities.selected_protocol.as_deref(), + Some(PROTOCOL_V1) + ); assert!(Path::new(&installed.package_path).is_dir()); let catalog = db.workspace_environment_adapters().unwrap(); let entry = catalog @@ -3229,6 +4035,10 @@ max_response_bytes = 1048576 assert_eq!(entry.source, "plugin"); assert_eq!(entry.trust, "local_unsigned"); assert_eq!(entry.certification_tier, "local-experimental"); + assert_eq!( + entry.protocol_capabilities.selected_protocol.as_deref(), + Some(PROTOCOL_V1) + ); assert_eq!( entry.identity.distribution_digest.as_deref(), Some(installed.distribution_digest.as_str()) @@ -3248,6 +4058,10 @@ max_response_bytes = 1048576 removed.removed_distribution_digest.as_deref(), Some(installed.distribution_digest.as_str()) ); + assert_eq!( + removed.protocol_capabilities.selected_protocol.as_deref(), + Some(PROTOCOL_V1) + ); assert!(db .workspace_environment_adapters() .unwrap() @@ -3256,6 +4070,52 @@ max_response_bytes = 1048576 .all(|entry| entry.canonical_identity != "example/test@1")); } + #[test] + fn unsupported_plugin_marker_returns_proposal_without_invoking_plugin() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("test.plugin"), "recognized marker\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let package = tempfile::tempdir().unwrap(); + write_test_package(package.path(), "example/unsupported@1"); + make_test_package_unsupported_on_current_host(package.path()); + db.install_environment_adapter_plugin(package.path()) + .unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "unsupported-plugin", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + + // The fixture executable is not a protocol peer. Discovery can only + // succeed if the host reports the pinned marker without launching it. + let discovery = db + .discover_workspace_environment("unsupported-plugin", None) + .unwrap(); + assert_eq!(discovery.components.len(), 1); + let proposal = &discovery.components[0]; + assert_eq!(proposal.component_id, "plugin:test-plugin"); + assert_eq!(proposal.adapter_identity, "example/unsupported@1"); + assert_eq!( + proposal.status, + EnvironmentComponentProposalStatus::Unsupported + ); + assert_eq!(proposal.reasons[0].code, "unsupported_host_platform"); + assert_eq!(proposal.recovery_actions[0].code, "use_supported_host"); + } + #[test] fn signed_plugin_requires_live_publisher_trust_and_reports_authentication() { let workspace = tempfile::tempdir().unwrap(); @@ -3310,6 +4170,40 @@ max_response_bytes = 1048576 let trust = db.environment_adapter_publisher_trust().unwrap(); assert_eq!(trust.keys.len(), 1); assert_eq!(trust.keys[0].key_id, trusted.key_id); + let artifact = tempfile::tempdir().unwrap(); + fs::write(artifact.path().join("plugin-output"), "signed output\n").unwrap(); + let layer = db + .publish_workspace_layer_from_directory( + &WorkspaceLayerKeyV1 { + kind: "generated".into(), + adapter: "example/signed@1".into(), + adapter_version: 1, + inputs: BTreeMap::from([("source_root".into(), "fixture-source".into())]), + tool_versions: BTreeMap::from([( + "adapter".into(), + installed.executable_digest.clone(), + )]), + platform: std::env::consts::OS.into(), + architecture: std::env::consts::ARCH.into(), + portability_scope: "host".into(), + strategy: "signed-plugin-test".into(), + }, + artifact.path(), + ) + .unwrap(); + let (envelope_id, tree_id) = db + .conn + .query_row( + "SELECT envelope_id,tree_root_id FROM workspace_layer_artifact_shadows + WHERE layer_id=?1", + params![layer.layer_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(); + let envelope_id = ArtifactEnvelopeId::parse(envelope_id).unwrap(); + let tree_id = ArtifactTreeId::parse(tree_id).unwrap(); + db.verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id) + .unwrap(); db.remove_environment_adapter_publisher_key(&trusted.key_id) .unwrap(); assert!(db @@ -3317,6 +4211,11 @@ max_response_bytes = 1048576 .unwrap_err() .to_string() .contains("no longer has a valid trusted publisher")); + assert!(db + .verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id) + .unwrap_err() + .to_string() + .contains("publisher trust cannot be verified")); assert!(db .environment_adapter_publisher_trust() .unwrap() @@ -3326,6 +4225,8 @@ max_response_bytes = 1048576 db.trust_environment_adapter_publisher_key(&key_document) .unwrap(); assert!(db.workspace_environment_adapters().is_ok()); + db.verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id) + .unwrap(); let signature_path = package.path().join(PLUGIN_PACKAGE_SIGNATURE); let mut signature: AdapterPackageSignature = toml::from_str(&fs::read_to_string(&signature_path).unwrap()).unwrap(); @@ -3342,6 +4243,11 @@ max_response_bytes = 1048576 .remove_environment_adapter_plugin("example/signed@1") .unwrap(); assert!(removed.removed_distribution_digest.is_some()); + assert!(db + .verify_ready_artifact_envelope_under_write_lock(&envelope_id, &tree_id) + .unwrap_err() + .to_string() + .contains("producer package is removed or revoked")); } #[test] diff --git a/trail/src/db/lane/workspace_python.rs b/trail/src/db/lane/workspace_python.rs index 557a7cf7..77b4713b 100644 --- a/trail/src/db/lane/workspace_python.rs +++ b/trail/src/db/lane/workspace_python.rs @@ -1,10 +1,13 @@ use super::workspace_environment::{ resolve_workspace_tool_executable, workspace_mounted_commands_identity, - WorkspaceEnvironmentAdapter, WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentCommand, - WorkspaceEnvironmentInput, WorkspaceEnvironmentOutput, WorkspaceEnvironmentOutputPolicy, - WorkspaceEnvironmentPlan, WorkspaceEnvironmentSandboxPolicy, + WorkspaceEnvironmentAdapter, WorkspaceEnvironmentAdapterMetadata, + WorkspaceEnvironmentAdapterProposal, WorkspaceEnvironmentCacheAccess, + WorkspaceEnvironmentCacheProtocol, WorkspaceEnvironmentCommand, WorkspaceEnvironmentInput, + WorkspaceEnvironmentOutput, WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, + WorkspaceEnvironmentResolutionInput, WorkspaceEnvironmentSandboxPolicy, }; use super::*; +use crate::ids::sha256_hex; pub(crate) struct PythonVenvAdapter; @@ -20,6 +23,15 @@ const PYTHON_IDENTITY_FILES: [&str; 7] = [ "requirements.txt", ]; +const PYTHON_RESOLUTION_FILES: [&str; 6] = [ + "uv.lock", + "poetry.lock", + "pdm.lock", + "Pipfile.lock", + "requirements.lock", + "requirements.txt", +]; + static PYTHON_VENV_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = WorkspaceEnvironmentAdapterMetadata { canonical_identity: "trail/python-venv@1", @@ -65,6 +77,94 @@ impl WorkspaceEnvironmentAdapter for PythonVenvAdapter { Ok(false) } + fn propose( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result> { + let root = normalize_python_component_root(component_root)?; + if !self.detect(db, source_root, &root)? { + return Ok(None); + } + Ok(Some(WorkspaceEnvironmentAdapterProposal::ready())) + } + + fn resolution_plan( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result> { + let root = normalize_python_component_root(component_root)?; + let pyproject_path = join_python_path(&root, "pyproject.toml"); + let Some(pyproject) = db.root_file_entry(source_root, &pyproject_path)? else { + return Ok(None); + }; + let uv = resolve_workspace_tool_executable("uv")?; + let policy_identity = sha256_hex( + format!( + "python-uv-requirements-resolver-v1\0{}\0offline\0generate-hashes", + uv.identity + ) + .as_bytes(), + ); + Ok(Some(ArtifactResolutionPlanV1 { + version: ARTIFACT_RESOLUTION_PLAN_VERSION, + proposal_key: python_resolution_proposal_key(source_root, &root), + source_root: source_root.clone(), + component_id: self.component_id(&root)?, + adapter_identity: self.identity().to_string(), + policy_identity, + program: "uv".to_string(), + resolved_program: uv.path.to_string_lossy().into_owned(), + executable_identity: uv.identity, + argv: vec![ + "uv".to_string(), + "pip".to_string(), + "compile".to_string(), + "--offline".to_string(), + "--generate-hashes".to_string(), + "--output-file".to_string(), + "requirements.lock".to_string(), + "pyproject.toml".to_string(), + ], + working_directory: if root.is_empty() { + ".".to_string() + } else { + root.clone() + }, + readable_inputs: vec![ArtifactResolutionInputV1 { + source_path: pyproject_path, + content_hash: pyproject.content_hash, + size_bytes: pyproject.size_bytes, + }], + candidate_output: join_python_path(&root, "requirements.lock"), + allowed_authorities: Vec::new(), + credential_handles: Vec::new(), + script_policy: ArtifactScriptPolicyV1::Deny, + environment_roles: BTreeMap::new(), + limits: ArtifactActionLimitsV1 { + timeout_ms: 10 * 60 * 1_000, + stdout_bytes: 1024 * 1024, + stderr_bytes: 1024 * 1024, + candidate_bytes: 64 * 1024 * 1024, + candidate_entries: 1, + child_processes: 64, + }, + snapshot_format: "python-requirements-hashes-v1".to_string(), + validations: vec![ArtifactValidationV1 { + name: "python-requirements-hashes".to_string(), + kind: ArtifactValidationKindV1::Framework, + required: true, + parameters: BTreeMap::from([( + "hash_mode".to_string(), + "required-for-nonempty".to_string(), + )]), + }], + })) + } + fn plan( &self, db: &Trail, @@ -123,25 +223,143 @@ impl WorkspaceEnvironmentAdapter for PythonVenvAdapter { workspace_mounted_commands_identity(std::slice::from_ref(&mounted_command))?, ), ]); + let source_resolution = python_source_resolution(db, source_root, &component_root)?; + let managed_snapshot = if source_resolution.is_none() { + python_resolution_snapshot(db, source_root, &component_root)? + } else { + None + }; + let mut resolution_inputs = Vec::new(); + let mut source_projection = None; + let managed_resolution = managed_snapshot.is_some(); + if let Some((snapshot_id, snapshot, bytes)) = managed_snapshot { + let resolution_plan = self + .resolution_plan(db, source_root, &component_root)? + .ok_or_else(|| { + Error::InvalidInput(format!( + "Python component `{}` has no supported source lock and cannot produce a managed resolution plan", + display_python_root(&component_root) + )) + })?; + if snapshot.resolver_executable_identity != resolution_plan.executable_identity + || snapshot.policy_identity != resolution_plan.policy_identity + { + return Err(Error::InvalidInput(format!( + "Python component `{}` resolution snapshot was created by a different uv executable or resolver policy; resolve it again for the current tool", + display_python_root(&component_root) + ))); + } + let resolution_path = join_python_path(&component_root, "requirements.lock"); + let size_bytes = u64::try_from(bytes.len()).map_err(|_| { + Error::InvalidInput( + "Python resolution snapshot exceeds platform limits".to_string(), + ) + })?; + key_inputs.insert("source_root".to_string(), source_root.0.clone()); + key_inputs.insert( + format!("input:{resolution_path}"), + snapshot.content_sha256.clone(), + ); + key_inputs.insert( + "resolution_authority".to_string(), + format!("snapshot:{}", snapshot_id.0), + ); + resolution_inputs.push(WorkspaceEnvironmentResolutionInput { + snapshot_id, + source_root: source_root.clone(), + source_path: resolution_path.clone(), + staging_path: format!("project/{resolution_path}"), + content_hash: snapshot.content_sha256, + size_bytes, + }); + source_projection = Some((source_root.clone(), "project".to_string())); + } let mut inputs = Vec::new(); for file in PYTHON_IDENTITY_FILES { let path = join_python_path(&component_root, file); if let Some(entry) = db.root_file_entry(source_root, &path)? { key_inputs.insert(format!("input:{path}"), entry.content_hash.clone()); - inputs.push(WorkspaceEnvironmentInput { - source_path: path.clone(), - staging_path: format!("project/{path}"), - entry, - }); + if !managed_resolution { + inputs.push(WorkspaceEnvironmentInput { + source_path: path.clone(), + staging_path: format!("project/{path}"), + entry, + }); + } } } - if inputs.is_empty() { + if inputs.is_empty() && resolution_inputs.is_empty() { return Err(Error::InvalidInput(format!( "Python component `{}` has no supported dependency manifest or lockfile", display_python_root(&component_root) ))); } inputs.sort_by(|left, right| left.source_path.cmp(&right.source_path)); + let download_cache = db.declare_workspace_environment_cache( + self.identity(), + "python-downloads", + WorkspaceEnvironmentCacheProtocol::ContentStore, + WorkspaceEnvironmentCacheAccess::ToolConcurrent, + BTreeMap::from([ + ("python_executable".to_string(), python.identity.clone()), + ("platform".to_string(), std::env::consts::OS.to_string()), + ( + "architecture".to_string(), + std::env::consts::ARCH.to_string(), + ), + ]), + )?; + let pre_commands = if managed_resolution { + let working_directory = if component_root.is_empty() { + "project".to_string() + } else { + format!("project/{component_root}") + }; + vec![WorkspaceEnvironmentCommand { + program: "python".to_string(), + resolved_program: python.path.clone(), + executable_identity: python.identity.clone(), + args: vec![ + "-m".to_string(), + "pip".to_string(), + "download".to_string(), + "--require-hashes".to_string(), + "--no-deps".to_string(), + "--disable-pip-version-check".to_string(), + "--dest".to_string(), + download_cache + .storage_path + .join("wheels") + .to_string_lossy() + .into_owned(), + "-r".to_string(), + "requirements.lock".to_string(), + ], + working_directory, + environment: BTreeMap::from([ + ( + "PIP_CACHE_DIR".to_string(), + download_cache + .storage_path + .join("pip") + .to_string_lossy() + .into_owned(), + ), + ( + "UV_CACHE_DIR".to_string(), + download_cache + .storage_path + .join("uv") + .to_string_lossy() + .into_owned(), + ), + ]), + remove_environment: Vec::new(), + cache_names: vec![download_cache.name.clone()], + }] + } else { + Vec::new() + }; Ok(WorkspaceEnvironmentPlan { component_id, adapter_identity: self.identity().to_string(), @@ -166,14 +384,15 @@ impl WorkspaceEnvironmentAdapter for PythonVenvAdapter { strategy: "python-venv-private-mounted-init-v2".to_string(), }, inputs, - source_projection: None, - pre_commands: Vec::new(), + resolution_inputs, + source_projection, + pre_commands, // Python virtual environments commonly embed absolute interpreter // and prefix paths, so the host initializes this output through // an ephemeral candidate view mounted at the final lane path. command: None, mounted_commands: vec![mounted_command], - caches: Vec::new(), + caches: vec![download_cache], external_artifacts: Vec::new(), runtime_resources: Vec::new(), sandbox_policy: WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin, @@ -195,6 +414,77 @@ impl WorkspaceEnvironmentAdapter for PythonVenvAdapter { } } +fn python_source_resolution( + db: &Trail, + source_root: &ObjectId, + component_root: &str, +) -> Result> { + for file in PYTHON_RESOLUTION_FILES { + let path = join_python_path(component_root, file); + if db.root_file_entry(source_root, &path)?.is_some() { + return Ok(Some(path)); + } + } + Ok(None) +} + +fn python_resolution_proposal_key(source_root: &ObjectId, component_root: &str) -> String { + let identity = format!( + "python-requirements-resolution-v1\0{}\0{}\0{}", + source_root.0, component_root, PYTHON_VENV_ADAPTER_METADATA.canonical_identity + ); + format!("python_requirements_v1_{}", sha256_hex(identity.as_bytes())) +} + +fn python_resolution_snapshot( + db: &Trail, + source_root: &ObjectId, + component_root: &str, +) -> Result)>> { + let proposal_key = python_resolution_proposal_key(source_root, component_root); + let expected_component = PYTHON_VENV_ADAPTER.component_id(component_root)?; + db.verified_workspace_environment_resolution_snapshot( + &proposal_key, + source_root, + &expected_component, + PYTHON_VENV_ADAPTER.identity(), + "python-requirements-hashes-v1", + validate_python_requirements_snapshot, + ) +} + +fn validate_python_requirements_snapshot(bytes: &[u8]) -> Result<()> { + let text = std::str::from_utf8(bytes).map_err(|_| { + Error::InvalidInput("Trail-managed Python requirements snapshot is not UTF-8".to_string()) + })?; + let mut has_requirement = false; + for line in text.lines().map(str::trim) { + if line.is_empty() || line.starts_with('#') || line.starts_with("--hash=sha256:") { + continue; + } + if line.starts_with('-') || line.contains(" @ ") { + return Err(Error::InvalidInput( + "Trail-managed Python requirements snapshot contains an unpinned directive or URL" + .to_string(), + )); + } + has_requirement = true; + if !line.contains("==") { + return Err(Error::InvalidInput( + "Trail-managed Python requirements snapshot contains an unpinned requirement" + .to_string(), + )); + } + } + if has_requirement && !text.contains("--hash=sha256:") { + return Err(Error::InvalidInput( + "Trail-managed Python requirements snapshot contains packages without SHA-256 hashes" + .to_string(), + )); + } + Ok(()) +} + fn resolve_python_executable() -> Result { #[cfg(windows)] let candidates = ["python", "python3"]; @@ -261,6 +551,174 @@ mod tests { panic!("timed out waiting for mounted crash helper at {phase}"); } + #[test] + fn python_requirements_snapshot_requires_pins_and_hashes() { + validate_python_requirements_snapshot( + b"# generated\nexample==1.2.3 --hash=sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n", + ) + .unwrap(); + validate_python_requirements_snapshot(b"# no dependencies\n").unwrap(); + assert!(validate_python_requirements_snapshot(b"example>=1\n") + .unwrap_err() + .to_string() + .contains("unpinned")); + assert!(validate_python_requirements_snapshot(b"example==1\n") + .unwrap_err() + .to_string() + .contains("without SHA-256")); + assert!(validate_python_requirements_snapshot(b"-e ../example\n") + .unwrap_err() + .to_string() + .contains("directive or URL")); + } + + #[test] + fn managed_python_resolution_warms_download_cache_but_keeps_venv_and_bytecode_private() { + let Ok(python) = resolve_python_executable() else { + return; + }; + let Ok(uv) = resolve_workspace_tool_executable("uv") else { + return; + }; + let workspace = tempfile::tempdir().unwrap(); + let pyproject = + "[project]\nname = \"managed-python\"\nversion = \"0.1.0\"\ndependencies = []\n"; + fs::write(workspace.path().join("pyproject.toml"), pyproject).unwrap(); + let resolver = tempfile::tempdir().unwrap(); + fs::write(resolver.path().join("pyproject.toml"), pyproject).unwrap(); + let generated = Command::new(&uv.path) + .args([ + "pip", + "compile", + "--offline", + "--generate-hashes", + "--output-file", + "requirements.lock", + "pyproject.toml", + ]) + .current_dir(resolver.path()) + .output() + .unwrap(); + assert!( + generated.status.success(), + "uv resolution failed: {}", + String::from_utf8_lossy(&generated.stderr) + ); + let requirements = fs::read(resolver.path().join("requirements.lock")).unwrap(); + validate_python_requirements_snapshot(&requirements).unwrap(); + + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let source_root = db.get_ref("refs/branches/main").unwrap().root_id; + let resolution_plan = PYTHON_VENV_ADAPTER + .resolution_plan(&db, &source_root, "") + .unwrap() + .unwrap(); + assert_eq!(resolution_plan.executable_identity, uv.identity); + db.resolve_artifact_component( + ArtifactResolutionRequestV1 { + plan: resolution_plan, + candidate: ArtifactResolutionCandidateV1 { + snapshot_bytes: requirements, + resolved_identities: BTreeMap::new(), + checksums: BTreeMap::new(), + contacted_authorities: Vec::new(), + stdout: Vec::new(), + stderr: Vec::new(), + redactions: Vec::new(), + }, + }, + false, + ) + .unwrap(); + + for lane in ["managed-python-one", "managed-python-two"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + } + let plan = db + .plan_workspace_environment("managed-python-one", "python", None) + .unwrap(); + assert_eq!(plan.inputs.len(), 1); + assert_eq!(plan.inputs[0].source_path, "requirements.lock"); + assert_eq!(plan.caches.len(), 1); + assert_eq!(plan.caches[0].protocol, "content_store"); + assert_eq!(plan.caches[0].authority, "performance_only"); + assert_eq!(plan.commands.len(), 2); + assert_eq!(plan.commands[0].phase, "staging"); + assert_eq!(plan.commands[1].phase, "mounted_initialization"); + assert!(!workspace.path().join("requirements.lock").exists()); + assert!(db + .root_file_entry(&source_root, "requirements.lock") + .unwrap() + .is_none()); + + #[cfg(target_os = "linux")] + if std::env::var_os("TRAIL_RUN_FUSE_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + #[cfg(target_os = "macos")] + if std::env::var_os("TRAIL_RUN_NFS_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + #[cfg(windows)] + if std::env::var_os("TRAIL_RUN_DOKAN_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + let first = db + .sync_workspace_environment_component("managed-python-one", "python", None, None) + .unwrap(); + let second = db + .sync_workspace_environment_component("managed-python-two", "python", None, None) + .unwrap(); + assert!(first.layers.is_empty()); + assert!(second.layers.is_empty()); + assert_eq!( + first.generation.components[0].caches[0].namespace_id, + second.generation.components[0].caches[0].namespace_id + ); + assert_eq!(first.generation.components[0].outputs[0].layer_id, None); + let first_paths = db + .workspace_view_paths_for_lane("managed-python-one") + .unwrap(); + let second_paths = db + .workspace_view_paths_for_lane("managed-python-two") + .unwrap(); + let first_bytecode = first_paths + .generated_upper + .join(".venv/lib/private/__pycache__"); + fs::create_dir_all(&first_bytecode).unwrap(); + fs::write(first_bytecode.join("module.pyc"), b"private bytecode").unwrap(); + assert!(!second_paths + .generated_upper + .join(".venv/lib/private/__pycache__/module.pyc") + .exists()); + assert!(first_paths + .generated_upper + .join(".venv/pyvenv.cfg") + .is_file()); + assert!(second_paths + .generated_upper + .join(".venv/pyvenv.cfg") + .is_file()); + assert!(python.path.is_absolute()); + } + #[test] fn component_selector_resolves_nested_discovered_adapter_without_path() { if resolve_python_executable().is_err() { diff --git a/trail/src/db/lane/workspace_recipe.rs b/trail/src/db/lane/workspace_recipe.rs index 5a6a9d87..3b88e949 100644 --- a/trail/src/db/lane/workspace_recipe.rs +++ b/trail/src/db/lane/workspace_recipe.rs @@ -1,6 +1,8 @@ use globset::{GlobBuilder, GlobSetBuilder}; use serde::{Deserialize, Serialize}; +use crate::ids::ArtifactDesiredKeyV2; + use super::workspace_environment::{ resolve_workspace_tool_executable, validate_environment_output_contract, ResolvedWorkspaceTool, WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentCommand, @@ -9,15 +11,26 @@ use super::workspace_environment::{ }; use super::*; -const RECIPE_SCHEMA: &str = "trail.environment/v1"; +const RECIPE_SCHEMA_V1: &str = "trail.environment/v1"; +const RECIPE_SCHEMA_V2: &str = "trail.environment/v2"; const RECIPE_ADAPTER_IDENTITY: &str = "trail/command@1"; const RECIPE_SPEC_PATHS: [&str; 2] = ["trail.environment.toml", ".trail/environment.toml"]; const MAX_RECIPE_SPEC_BYTES: u64 = 1024 * 1024; const MAX_RECIPE_TOTAL_SPEC_BYTES: u64 = 4 * 1024 * 1024; const MAX_RECIPE_INCLUDE_FILES: usize = 32; const MAX_RECIPE_INCLUDE_DEPTH: usize = 8; +const MAX_RECIPE_INPUT_DECLARATIONS: usize = 4_096; const MAX_RECIPE_INPUT_FILES: usize = 100_000; const MAX_RECIPE_INPUT_BYTES: u64 = 2 * 1024 * 1024 * 1024; +const MAX_RECIPE_ACTIONS: usize = 64; +const MAX_RECIPE_VALIDATIONS: usize = 64; +const MAX_RECIPE_SOURCE_EXPORTS: usize = 32; +const MAX_RECIPE_COMMAND_ARGUMENTS: usize = 1_024; +const MAX_RECIPE_ARGUMENT_BYTES: usize = 128 * 1024; +const MAX_RECIPE_NETWORK_AUTHORITIES: usize = 256; +const MAX_RECIPE_ENVIRONMENT_ENTRIES: usize = 256; +const MAX_RECIPE_VALIDATION_PARAMETERS: usize = 256; +const MAX_RECIPE_CHILD_PROCESSES: u32 = 256; #[cfg(test)] thread_local! { @@ -44,6 +57,7 @@ pub(crate) static COMMAND_RECIPE_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMe #[derive(Clone, Debug)] struct CommandRecipe { + schema: RecipeSchemaVersion, specification_digest: String, specification_sources: BTreeMap, profile_versions: BTreeMap, @@ -51,6 +65,65 @@ struct CommandRecipe { component: RecipeComponent, } +#[derive(Clone, Debug)] +pub(crate) struct CompiledRepositoryArtifactPipelineV2 { + pub(crate) proposal: EnvironmentDiscoveredComponentReport, + pub(crate) resolution_plan: Option, + pub(crate) graph_plan: WorkspaceEnvironmentPlan, + pub(crate) desired_material: ArtifactDesiredKeyMaterialV2, + pub(crate) desired_key: ArtifactDesiredKeyV2, + pub(crate) outputs: Vec, + pub(crate) validations: Vec, + pub(crate) source_exports: Vec, +} + +impl CompiledRepositoryArtifactPipelineV2 { + fn into_graph_plan(self) -> Result { + if self.proposal.component_id != self.graph_plan.component_id + || self.desired_material.component_id != self.graph_plan.component_id + || self + .resolution_plan + .as_ref() + .is_some_and(|plan| plan.component_id != self.graph_plan.component_id) + || self.desired_material.outputs != self.outputs + || self.desired_material.validations != self.validations + || self.desired_material.source_exports != self.source_exports + || super::workspace_artifact::artifact_desired_key_v2(self.desired_material)? + != self.desired_key + { + return Err(Error::Corrupt( + "compiled repository artifact pipeline models disagree".into(), + )); + } + Ok(self.graph_plan) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RecipeSchemaVersion { + V1, + V2, +} + +impl RecipeSchemaVersion { + fn parse(value: &str, path: &str) -> Result { + match value { + RECIPE_SCHEMA_V1 => Ok(Self::V1), + RECIPE_SCHEMA_V2 => Ok(Self::V2), + other => Err(Error::InvalidInput(format!( + "unsupported environment schema `{other}` in `{path}`; expected `{RECIPE_SCHEMA_V1}` or `{RECIPE_SCHEMA_V2}`" + ))), + } + } + + fn as_str(self) -> &'static str { + match self { + Self::V1 => RECIPE_SCHEMA_V1, + Self::V2 => RECIPE_SCHEMA_V2, + } + } +} + #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] struct RecipeSpecification { @@ -71,6 +144,8 @@ struct RecipeEnvironment { name: Option, default_network: String, default_scripts: String, + #[serde(default)] + missing_resolution: Option, } impl Default for RecipeEnvironment { @@ -79,6 +154,7 @@ impl Default for RecipeEnvironment { name: None, default_network: "deny".to_string(), default_scripts: "deny".to_string(), + missing_resolution: None, } } } @@ -105,6 +181,16 @@ struct RecipeComponentDefinition { outputs: Vec, #[serde(default)] build: Option, + #[serde(default, rename = "resolve")] + resolution: Option, + #[serde(default, rename = "action")] + actions: Vec, + #[serde(default, rename = "validation")] + validations: Vec, + #[serde(default)] + capabilities: Option, + #[serde(default, rename = "source_export")] + source_exports: Vec, } #[derive(Clone, Debug, Deserialize)] @@ -127,6 +213,16 @@ struct RecipeProfile { outputs: Vec, #[serde(default)] build: Option, + #[serde(default, rename = "resolve")] + resolution: Option, + #[serde(default, rename = "action")] + actions: Vec, + #[serde(default, rename = "validation")] + validations: Vec, + #[serde(default)] + capabilities: Option, + #[serde(default, rename = "source_export")] + source_exports: Vec, } #[derive(Clone, Debug, Default)] @@ -138,6 +234,11 @@ struct RecipeFragment { inputs: Vec, outputs: Vec, build: Option, + resolution: Option, + actions: Vec, + validations: Vec, + capabilities: Option, + source_exports: Vec, } #[derive(Clone, Debug)] @@ -148,6 +249,7 @@ struct ResolvedRecipeProfile { #[derive(Debug, Default)] struct RecipeDocuments { + schema: Option, defaults: RecipeEnvironment, profiles: BTreeMap, components: Vec, @@ -165,6 +267,11 @@ struct RecipeComponent { inputs: Vec, outputs: Vec, build: RecipeBuild, + resolution: Option, + actions: Vec, + validations: Vec, + capabilities: Option, + source_exports: Vec, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -222,6 +329,117 @@ struct RecipeBuild { environment: BTreeMap, } +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RecipeResolution { + command: Vec, + #[serde(default)] + cwd: Option, + #[serde(default)] + network: Option, + snapshot: String, + format: String, + #[serde(default)] + environment: BTreeMap, + #[serde(default)] + capabilities: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(untagged)] +enum RecipeNetwork { + Policy(String), + Authorities(RecipeNetworkAuthorities), +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RecipeNetworkAuthorities { + authorities: Vec, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum RecipeActionPhase { + Construct, + Validate, + MountedExecution, + SourceExport, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RecipeAction { + #[serde(default)] + name: Option, + phase: RecipeActionPhase, + command: Vec, + #[serde(default)] + cwd: Option, + #[serde(default)] + network: Option, + #[serde(default)] + scripts: Option, + #[serde(default)] + environment: BTreeMap, + #[serde(default)] + capabilities: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RecipeValidation { + #[serde(default)] + name: Option, + kind: String, + #[serde(default)] + path: Option, + #[serde(default)] + command: Vec, + #[serde(default = "default_true")] + required: bool, + #[serde(default)] + parameters: BTreeMap, + #[serde(default)] + gate: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RecipeCapabilities { + #[serde(default)] + network: Option, + #[serde(default)] + filesystem_read: Option, + #[serde(default)] + filesystem_write: Option, + #[serde(default)] + process: Option, + #[serde(default)] + child_processes: Option, + #[serde(default)] + secrets: Option, + #[serde(default)] + publication: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RecipeSourceExport { + #[serde(default)] + name: Option, + from_output: String, + source: String, + target: String, + mode: String, + #[serde(default = "default_fail_collision")] + collision: String, + #[serde(default)] + validation: Option, + #[serde(default)] + gate: Option, +} + fn default_recipe_kind() -> String { "generated".to_string() } @@ -238,392 +456,1504 @@ fn default_host_portability() -> String { "host".to_string() } -impl Trail { - pub(crate) fn command_recipe_discovery( - &self, - source_root: &ObjectId, - component_root: Option<&str>, - ) -> Result> { - let requested_root = component_root - .map(normalize_recipe_path_allow_root) - .transpose()?; - let recipes = self.load_command_recipes(source_root)?; - Ok(recipes - .into_iter() - .filter(|recipe| { - requested_root - .as_ref() - .is_none_or(|root| root == &recipe.component.root) - }) - .map(|recipe| EnvironmentDiscoveredComponentReport { - component_id: recipe.component.id, - component_root: recipe.component.root, - kind: recipe.component.kind, - adapter_identity: RECIPE_ADAPTER_IDENTITY.to_string(), +fn default_true() -> bool { + true +} + +fn default_fail_collision() -> String { + "fail".to_string() +} + +fn compile_recipe_validations( + validations: &[RecipeValidation], +) -> Result> { + let mut compiled = validations + .iter() + .enumerate() + .map(|(index, validation)| { + let kind = match validation.kind.as_str() { + "structural" | "path_contract" => ArtifactValidationKindV1::Structural, + "loadability" => ArtifactValidationKindV1::Loadability, + "framework" => ArtifactValidationKindV1::Framework, + "policy" => ArtifactValidationKindV1::Policy, + "gate" => ArtifactValidationKindV1::Gate, + "reproducibility" => ArtifactValidationKindV1::Reproducibility, + other => { + return Err(Error::InvalidInput(format!( + "unsupported repository validation kind `{other}`" + ))) + } + }; + let mut parameters = validation.parameters.clone(); + if let Some(path) = &validation.path { + parameters.insert("path".into(), normalize_relative_path(path)?); + } + if !validation.command.is_empty() { + parameters.insert( + "command".into(), + serde_json::to_string(&validation.command)?, + ); + } + if let Some(gate) = &validation.gate { + parameters.insert("gate".into(), gate.clone()); + } + Ok(ArtifactValidationV1 { + name: validation + .name + .clone() + .unwrap_or_else(|| format!("validation-{index}")), + kind, + required: validation.required, + parameters, }) - .collect()) + }) + .collect::>>()?; + compiled.sort(); + if compiled.windows(2).any(|pair| pair[0].name == pair[1].name) { + return Err(Error::InvalidInput( + "repository component declares duplicate validation names".into(), + )); } + Ok(compiled) +} - pub(crate) fn command_recipe_plan( - &self, - source_root: &ObjectId, - component_id: &str, - ) -> Result { - let recipes = self.load_command_recipes(source_root)?; - let recipe = recipes - .into_iter() - .find(|recipe| recipe.component.id == component_id) - .ok_or_else(|| { - Error::InvalidInput(format!( - "no `{RECIPE_ADAPTER_IDENTITY}` component named `{component_id}` exists in the pinned environment specification" - )) - })?; - self.plan_command_recipe(source_root, recipe) +fn compile_recipe_source_exports( + component: &RecipeComponent, +) -> Vec { + component + .source_exports + .iter() + .map(|export| ArtifactSourceExportContractV2 { + name: export + .name + .clone() + .unwrap_or_else(|| export.from_output.clone()), + output_name: export.from_output.clone(), + artifact_subpath: export.source.clone(), + destination: export.target.clone(), + collision_policy: export.collision.clone(), + required_validation: export.validation.clone().unwrap_or_else(|| { + super::workspace_artifact::HOST_WORKSPACE_LAYER_STRUCTURAL_SEAL.into() + }), + required_gate: export.gate.clone(), + authorization_mode: export.mode.clone(), + }) + .collect() +} + +fn recipe_network_authorities(network: Option<&RecipeNetwork>) -> Result> { + let mut authorities = match network { + None => Vec::new(), + Some(RecipeNetwork::Policy(policy)) if policy == "deny" => Vec::new(), + Some(RecipeNetwork::Policy(policy)) => Err(Error::InvalidInput(format!( + "repository resolver network policy `{policy}` must be `deny` or an exact authority list" + )))?, + Some(RecipeNetwork::Authorities(authorities)) => authorities.authorities.clone(), + }; + if authorities.len() > MAX_RECIPE_NETWORK_AUTHORITIES { + return Err(Error::InvalidInput(format!( + "repository resolver declares more than {MAX_RECIPE_NETWORK_AUTHORITIES} network authorities" + ))); + } + for authority in &authorities { + validate_recipe_network_authority(authority)?; } + authorities.sort(); + authorities.dedup(); + Ok(authorities) +} - pub(crate) fn command_recipe_plans( - &self, - source_root: &ObjectId, - component_ids: &BTreeSet, - ) -> Result> { - let recipes = self.load_command_recipes(source_root)?; - let mut plans = BTreeMap::new(); - let mut tools = BTreeMap::::new(); - for recipe in recipes { - if component_ids.contains(&recipe.component.id) { - let component_id = recipe.component.id.clone(); - let program = recipe - .component - .build - .command - .first() - .cloned() - .ok_or_else(|| { - Error::InvalidInput(format!( - "command component `{component_id}` has an empty build.command" - )) - })?; - let tool = if let Some(tool) = tools.get(&program) { - tool.clone() - } else { - let tool = resolve_workspace_tool_executable(&program)?; - tools.insert(program, tool.clone()); - tool - }; - plans.insert( - component_id, - self.plan_command_recipe_with_tool(source_root, recipe, Some(tool))?, - ); - } - } - if plans.len() != component_ids.len() { - let missing = component_ids - .iter() - .filter(|component_id| !plans.contains_key(*component_id)) - .cloned() - .collect::>(); +fn recipe_network_policy_identity(network: Option<&RecipeNetwork>) -> Result { + let authorities = recipe_network_authorities(network)?; + if authorities.is_empty() { + Ok("deny".into()) + } else { + Ok(format!("exact:{}", authorities.join(","))) + } +} + +fn merge_recipe_identity_environment( + target: &mut BTreeMap, + source: &BTreeMap, + component_id: &str, +) -> Result<()> { + for (name, value) in source { + validate_recipe_environment(name, value, component_id)?; + if let Some(previous) = target.insert(name.clone(), value.clone()) + && previous != *value + { return Err(Error::InvalidInput(format!( - "pinned environment specification is missing command component(s): {}", - missing.join(", ") + "repository component `{component_id}` declares conflicting identity environment values for `{name}`" ))); } - Ok(plans) } + Ok(()) +} - pub(crate) fn command_recipe_plan_for_root( - &self, - source_root: &ObjectId, - component_root: &str, - ) -> Result { - let component_root = normalize_recipe_path_allow_root(component_root)?; - let mut matching = self - .load_command_recipes(source_root)? - .into_iter() - .filter(|recipe| recipe.component.root == component_root) - .collect::>(); - match matching.len() { - 1 => self.plan_command_recipe(source_root, matching.remove(0)), - 0 => Err(Error::InvalidInput(format!( - "no `{RECIPE_ADAPTER_IDENTITY}` component is declared at `{}`", - display_recipe_root(&component_root) - ))), - count => Err(Error::InvalidInput(format!( - "{count} `{RECIPE_ADAPTER_IDENTITY}` components are declared at `{}`; synchronize all components or give each recipe a distinct root", - display_recipe_root(&component_root) - ))), - } - } +#[derive(Clone, Copy)] +enum RecipeCapabilityPhase { + Resolve, + Construct, + Validate, +} - fn load_command_recipes(&self, source_root: &ObjectId) -> Result> { - #[cfg(test)] - COMMAND_RECIPE_LOAD_COUNT.with(|count| count.set(count.get() + 1)); - let mut found = Vec::new(); - for path in RECIPE_SPEC_PATHS { - if self.root_file_entry(source_root, path)?.is_some() { - found.push(path.to_string()); - } - } - if found.len() > 1 { - return Err(Error::InvalidInput(format!( - "environment specification is ambiguous; keep only one of {}", - RECIPE_SPEC_PATHS.join(", ") - ))); +impl RecipeCapabilityPhase { + fn name(self) -> &'static str { + match self { + Self::Resolve => "resolve", + Self::Construct => "construct", + Self::Validate => "validate", } - let Some(spec_path) = found.pop() else { - return Ok(Vec::new()); - }; + } +} - let mut documents = RecipeDocuments::default(); - let mut visited = BTreeSet::new(); - let mut stack = Vec::new(); - let mut total_bytes = 0u64; - self.collect_recipe_document( - source_root, - &spec_path, - 0, - true, - &mut documents, - &mut visited, - &mut stack, - &mut total_bytes, - )?; +fn validate_recipe_v2_component( + component: &RecipeComponent, + defaults: &RecipeEnvironment, +) -> Result<()> { + if component.inputs.len() > MAX_RECIPE_INPUT_DECLARATIONS { + return Err(Error::InvalidInput(format!( + "repository component `{}` declares more than {MAX_RECIPE_INPUT_DECLARATIONS} inputs", + component.id + ))); + } + if component.actions.len() > MAX_RECIPE_ACTIONS { + return Err(Error::InvalidInput(format!( + "repository component `{}` declares more than {MAX_RECIPE_ACTIONS} actions", + component.id + ))); + } + if component.validations.len() > MAX_RECIPE_VALIDATIONS { + return Err(Error::InvalidInput(format!( + "repository component `{}` declares more than {MAX_RECIPE_VALIDATIONS} validations", + component.id + ))); + } + if component.source_exports.len() > MAX_RECIPE_SOURCE_EXPORTS { + return Err(Error::InvalidInput(format!( + "repository component `{}` declares more than {MAX_RECIPE_SOURCE_EXPORTS} source exports", + component.id + ))); + } - let mut ids = BTreeSet::new(); - let mut targets = BTreeMap::::new(); - let mut profile_cache = BTreeMap::new(); - let mut recipes = Vec::with_capacity(documents.components.len()); - for definition in documents.components { - let (component, profile_versions) = - resolve_recipe_component(definition, &documents.profiles, &mut profile_cache)?; - validate_recipe_component_identity(&component.id)?; - if !ids.insert(component.id.clone()) { - return Err(Error::InvalidInput(format!( - "environment specification declares component `{}` more than once", - component.id - ))); - } - if component.adapter.as_str() != RECIPE_ADAPTER_IDENTITY { - return Err(Error::InvalidInput(format!( - "component `{}` uses unsupported declarative adapter `{}`; this specification host currently accepts only `{RECIPE_ADAPTER_IDENTITY}`", - component.id, component.adapter - ))); - } - if component.kind != "generated" { - return Err(Error::InvalidInput(format!( - "command component `{}` must use kind = \"generated\"", - component.id - ))); - } - if component.outputs.is_empty() || component.outputs.len() > 32 { - return Err(Error::InvalidInput(format!( - "command component `{}` must declare between 1 and 32 outputs", - component.id, - ))); - } - let mut output_names = BTreeSet::new(); - for (index, output) in component.outputs.iter().enumerate() { - let name = output - .name - .clone() - .unwrap_or_else(|| format!("output-{index}")); - validate_recipe_output_name(&name, &component.id)?; - if !output_names.insert(name.clone()) { - return Err(Error::InvalidInput(format!( - "command component `{}` declares output name `{name}` more than once", - component.id - ))); - } - let target = normalize_relative_path(&output.target)?; - if let Some((other_target, other_id)) = recipe_target_overlap(&targets, &target) { - return Err(Error::InvalidInput(format!( - "command component `{}` target `{target}` overlaps component `{other_id}` target `{other_target}`", - component.id - ))); - } - targets.insert(target, format!("{}:{name}", component.id)); - } - let canonical = serde_json::to_vec(&(RECIPE_SCHEMA, &component, &profile_versions))?; - recipes.push(CommandRecipe { - specification_digest: sha256_hex(&canonical), - specification_sources: documents.specification_sources.clone(), - profile_versions, - defaults: documents.defaults.clone(), - component, - }); - } - recipes.sort_by(|left, right| left.component.id.cmp(&right.component.id)); - Ok(recipes) + validate_recipe_fixed_argv( + &component.build.command, + &component.id, + "build.command", + false, + )?; + validate_recipe_phase_cwd( + component.build.cwd.as_deref().unwrap_or(&component.root), + &component.root, + &component.id, + "build.cwd", + )?; + validate_recipe_environment_map(&component.build.environment, &component.id, "build")?; + if component + .build + .network + .as_deref() + .unwrap_or(&defaults.default_network) + != "deny" + || component + .build + .scripts + .as_deref() + .unwrap_or(&defaults.default_scripts) + != "deny" + { + return Err(Error::InvalidInput(format!( + "repository component `{}` build requires network = \"deny\" and scripts = \"deny\"", + component.id + ))); + } + if let Some(capabilities) = &component.capabilities { + validate_recipe_capabilities( + capabilities, + RecipeCapabilityPhase::Construct, + &component.id, + )?; } - #[allow(clippy::too_many_arguments)] - fn collect_recipe_document( - &self, - source_root: &ObjectId, - path: &str, - depth: usize, - is_root: bool, - documents: &mut RecipeDocuments, - visited: &mut BTreeSet, - stack: &mut Vec, - total_bytes: &mut u64, - ) -> Result<()> { - if depth > MAX_RECIPE_INCLUDE_DEPTH { + if let Some(resolution) = &component.resolution { + validate_recipe_fixed_argv(&resolution.command, &component.id, "resolve.command", false)?; + validate_recipe_phase_cwd( + resolution.cwd.as_deref().unwrap_or(&component.root), + &component.root, + &component.id, + "resolve.cwd", + )?; + normalize_relative_path(&resolution.snapshot)?; + if resolution.format.is_empty() + || resolution.format.len() > 512 + || resolution.format.contains(char::is_control) + || contains_sensitive_text(&resolution.format) + { return Err(Error::InvalidInput(format!( - "environment specification include depth exceeds {MAX_RECIPE_INCLUDE_DEPTH} at `{path}`" + "repository component `{}` has an invalid resolution snapshot format", + component.id ))); } - if let Some(index) = stack.iter().position(|candidate| candidate == path) { - let mut cycle = stack[index..].to_vec(); - cycle.push(path.to_string()); + recipe_network_authorities(resolution.network.as_ref())?; + validate_recipe_environment_map(&resolution.environment, &component.id, "resolver")?; + if let Some(capabilities) = &resolution.capabilities { + validate_recipe_capabilities( + capabilities, + RecipeCapabilityPhase::Resolve, + &component.id, + )?; + } + } + + let mut action_names = BTreeSet::new(); + for (index, action) in component.actions.iter().enumerate() { + if matches!( + action.phase, + RecipeActionPhase::MountedExecution | RecipeActionPhase::SourceExport + ) { return Err(Error::InvalidInput(format!( - "environment specification include cycle: {}", - cycle.join(" -> ") + "repository component `{}` action {} requests forbidden phase `{:?}`; mounted execution and source export cannot execute repository commands", + component.id, index, action.phase ))); } - if visited.contains(path) { - return Ok(()); - } - if visited.len().saturating_add(stack.len()) >= MAX_RECIPE_INCLUDE_FILES { + let action_name = action + .name + .clone() + .unwrap_or_else(|| format!("action-{index}")); + validate_recipe_output_name(&action_name, &component.id)?; + if !action_names.insert(action_name.clone()) { return Err(Error::InvalidInput(format!( - "environment specification includes more than {MAX_RECIPE_INCLUDE_FILES} files" + "repository component `{}` declares action name `{action_name}` more than once", + component.id ))); } - let entry = self.root_file_entry(source_root, path)?.ok_or_else(|| { - Error::InvalidInput(format!( - "environment specification include `{path}` does not exist in the pinned source root" - )) - })?; - if entry.size_bytes > MAX_RECIPE_SPEC_BYTES { + validate_recipe_fixed_argv( + &action.command, + &component.id, + &format!("action `{action_name}` command"), + false, + )?; + validate_recipe_phase_cwd( + action.cwd.as_deref().unwrap_or(&component.root), + &component.root, + &component.id, + &format!("action `{action_name}` cwd"), + )?; + if !recipe_network_authorities(action.network.as_ref())?.is_empty() { return Err(Error::InvalidInput(format!( - "environment specification `{path}` is {} bytes; the per-file maximum is {MAX_RECIPE_SPEC_BYTES}", - entry.size_bytes + "repository component `{}` action `{action_name}` must be offline", + component.id ))); } - *total_bytes = total_bytes.checked_add(entry.size_bytes).ok_or_else(|| { - Error::InvalidInput("environment specification size overflowed".to_string()) - })?; - if *total_bytes > MAX_RECIPE_TOTAL_SPEC_BYTES { + if action + .scripts + .as_deref() + .is_some_and(|policy| policy != "deny") + { return Err(Error::InvalidInput(format!( - "environment specifications total more than {MAX_RECIPE_TOTAL_SPEC_BYTES} bytes" + "repository component `{}` action `{action_name}` must deny scripts", + component.id ))); } - let entries = BTreeMap::from([(path.to_string(), entry.clone())]); - let bytes = self - .materialize_entries_bytes(&entries)? - .remove(path) - .ok_or_else(|| Error::Corrupt(format!("failed to read `{path}` from source root")))?; - let text = String::from_utf8(bytes).map_err(|_| { - Error::InvalidInput(format!("environment specification `{path}` must be UTF-8")) - })?; - let specification: RecipeSpecification = toml::from_str(&text).map_err(|err| { - Error::InvalidInput(format!("invalid environment specification `{path}`: {err}")) - })?; - validate_recipe_specification_header(&specification, path)?; - - stack.push(path.to_string()); - for include in &specification.include { - let include_path = resolve_recipe_include_path(path, include)?; - self.collect_recipe_document( - source_root, - &include_path, - depth + 1, - false, - documents, - visited, - stack, - total_bytes, - )?; + validate_recipe_environment_map( + &action.environment, + &component.id, + &format!("action `{action_name}`"), + )?; + if let Some(capabilities) = &action.capabilities { + let phase = if action.phase == RecipeActionPhase::Validate { + RecipeCapabilityPhase::Validate + } else { + RecipeCapabilityPhase::Construct + }; + validate_recipe_capabilities(capabilities, phase, &component.id)?; } - stack.pop(); + } - for (name, profile) in specification.profile { - let canonical_name = canonical_recipe_profile_name(&name)?; - if documents - .profiles - .insert(canonical_name.clone(), profile) - .is_some() + for (index, validation) in component.validations.iter().enumerate() { + if let Some(name) = &validation.name { + validate_recipe_output_name(name, &component.id)?; + } + if let Some(path) = &validation.path { + normalize_relative_path(path)?; + } + validate_recipe_fixed_argv( + &validation.command, + &component.id, + &format!("validation {index} command"), + true, + )?; + if validation.parameters.len() > MAX_RECIPE_VALIDATION_PARAMETERS { + return Err(Error::InvalidInput(format!( + "repository component `{}` validation {index} declares too many parameters", + component.id + ))); + } + for (name, value) in &validation.parameters { + if name.is_empty() + || name.len() > 128 + || value.len() > MAX_RECIPE_ARGUMENT_BYTES + || name.contains(char::is_control) + || value.contains(char::is_control) + || contains_sensitive_text(name) + || contains_sensitive_text(value) + || contains_provider_socket_reference(value) { return Err(Error::InvalidInput(format!( - "environment specifications declare profile `{canonical_name}` more than once" + "repository component `{}` validation {index} has an unsafe parameter `{name}`", + component.id ))); } } - documents.components.extend(specification.components); - if is_root { - documents.defaults = specification.environment; - } - documents - .specification_sources - .insert(path.to_string(), entry.content_hash); - visited.insert(path.to_string()); - Ok(()) - } - - fn plan_command_recipe( - &self, - source_root: &ObjectId, - recipe: CommandRecipe, - ) -> Result { - self.plan_command_recipe_with_tool(source_root, recipe, None) } - fn plan_command_recipe_with_tool( - &self, - source_root: &ObjectId, - recipe: CommandRecipe, - resolved_tool: Option, - ) -> Result { - let component = recipe.component; - let network = component - .build - .network - .as_deref() - .unwrap_or(&recipe.defaults.default_network); - let scripts = component - .build - .scripts - .as_deref() - .unwrap_or(&recipe.defaults.default_scripts); - if network != "deny" || scripts != "deny" { + for output in &component.outputs { + if output.reuse == Some(EnvironmentReuseMode::Compatible) + || output.scope == Some(EnvironmentSharingScope::Host) + { return Err(Error::InvalidInput(format!( - "command component `{}` requires network = \"deny\" and scripts = \"deny\"", + "repository component `{}` cannot request compatible or host-wide artifact reuse", component.id ))); } - if component.build.command.is_empty() { + } + let output_names = component + .outputs + .iter() + .enumerate() + .map(|(index, output)| { + output + .name + .clone() + .unwrap_or_else(|| format!("output-{index}")) + }) + .collect::>(); + for export in &component.source_exports { + let export_name = export.name.as_deref().unwrap_or(&export.from_output); + validate_recipe_output_name(export_name, &component.id)?; + validate_recipe_output_name(&export.from_output, &component.id)?; + if !output_names.contains(&export.from_output) { return Err(Error::InvalidInput(format!( - "command component `{}` has an empty build.command", - component.id + "repository component `{}` source export references unknown output `{}`", + component.id, export.from_output ))); } - if component.build.command.len() > 4096 - || component.build.command.iter().any(|argument| { - argument.len() > 128 * 1024 - || argument.contains('\0') - || contains_sensitive_text(argument) - }) - { + normalize_relative_path(&export.source)?; + normalize_relative_path(&export.target)?; + if export.mode != "explicit" { return Err(Error::InvalidInput(format!( - "command component `{}` exceeds command argument limits", + "repository component `{}` source export mode must be `explicit`", component.id ))); } - let program = &component.build.command[0]; - if program.contains('/') || program.contains('\\') || is_shell_program(program) { + if !matches!(export.collision.as_str(), "fail" | "replace") { return Err(Error::InvalidInput(format!( - "command component `{}` must name a non-shell executable from PATH, not `{program}`", + "repository component `{}` source export `{export_name}` collision mode must be `fail` or `replace`", component.id ))); } - let tool = resolved_tool - .map(Ok) - .unwrap_or_else(|| resolve_workspace_tool_executable(program))?; - validate_recipe_tool_path(self, &tool.path, &component.id)?; - let cwd = normalize_recipe_path_allow_root( - component.build.cwd.as_deref().unwrap_or(&component.root), - )?; - if !component.root.is_empty() - && cwd != component.root + if let Some(validation) = &export.validation { + validate_recipe_output_name(validation, &component.id)?; + } + if let Some(gate) = &export.gate { + validate_recipe_output_name(gate, &component.id)?; + } + } + let mut export_names = component + .source_exports + .iter() + .map(|export| export.name.as_deref().unwrap_or(&export.from_output)) + .collect::>(); + export_names.sort_unstable(); + if export_names.windows(2).any(|pair| pair[0] == pair[1]) { + return Err(Error::InvalidInput(format!( + "repository component `{}` declares duplicate source export names", + component.id + ))); + } + Ok(()) +} + +fn validate_recipe_fixed_argv( + command: &[String], + component_id: &str, + field: &str, + allow_empty: bool, +) -> Result<()> { + if command.is_empty() { + if allow_empty { + return Ok(()); + } + return Err(Error::InvalidInput(format!( + "repository component `{component_id}` has an empty {field}" + ))); + } + if command.len() > MAX_RECIPE_COMMAND_ARGUMENTS { + return Err(Error::InvalidInput(format!( + "repository component `{component_id}` {field} exceeds {MAX_RECIPE_COMMAND_ARGUMENTS} argv entries" + ))); + } + let program = &command[0]; + if program.contains('/') + || program.contains('\\') + || is_shell_program(program) + || is_indirect_process_launcher(program) + { + return Err(Error::InvalidInput(format!( + "repository component `{component_id}` {field} must name one non-shell, non-launcher executable from PATH, not `{program}`" + ))); + } + for argument in command { + if argument.is_empty() + || argument.len() > MAX_RECIPE_ARGUMENT_BYTES + || argument.contains('\0') + || argument.contains('\n') + || argument.contains('\r') + || contains_sensitive_text(argument) + || contains_shell_interpolation(argument) + || contains_provider_socket_reference(argument) + || is_absolute_host_path(argument) + { + return Err(Error::InvalidInput(format!( + "repository component `{component_id}` {field} contains an unsafe or excessive argv entry" + ))); + } + } + Ok(()) +} + +fn validate_recipe_phase_cwd( + cwd: &str, + component_root: &str, + component_id: &str, + field: &str, +) -> Result<()> { + let cwd = normalize_recipe_path_allow_root(cwd)?; + if !component_root.is_empty() + && cwd != component_root + && !cwd.starts_with(&format!("{component_root}/")) + { + return Err(Error::InvalidInput(format!( + "repository component `{component_id}` {field} `{cwd}` escapes component root `{component_root}`" + ))); + } + Ok(()) +} + +fn validate_recipe_environment_map( + environment: &BTreeMap, + component_id: &str, + phase: &str, +) -> Result<()> { + if environment.len() > MAX_RECIPE_ENVIRONMENT_ENTRIES { + return Err(Error::InvalidInput(format!( + "repository component `{component_id}` {phase} environment exceeds {MAX_RECIPE_ENVIRONMENT_ENTRIES} entries" + ))); + } + for (name, value) in environment { + validate_recipe_environment(name, value, component_id)?; + if contains_provider_socket_reference(name) || contains_provider_socket_reference(value) { + return Err(Error::InvalidInput(format!( + "repository component `{component_id}` {phase} environment entry `{name}` requests a provider socket" + ))); + } + } + Ok(()) +} + +fn validate_recipe_capabilities( + capabilities: &RecipeCapabilities, + phase: RecipeCapabilityPhase, + component_id: &str, +) -> Result<()> { + let allowed_network = match phase { + RecipeCapabilityPhase::Resolve => &["deny", "exact_authorities"][..], + RecipeCapabilityPhase::Construct | RecipeCapabilityPhase::Validate => &["deny"][..], + }; + let allowed_read = match phase { + RecipeCapabilityPhase::Resolve | RecipeCapabilityPhase::Construct => { + &["declared_inputs"][..] + } + RecipeCapabilityPhase::Validate => &["artifact_candidate"][..], + }; + let allowed_write = match phase { + RecipeCapabilityPhase::Resolve | RecipeCapabilityPhase::Construct => { + &["isolated_candidate"][..] + } + RecipeCapabilityPhase::Validate => &["validation_receipt"][..], + }; + validate_recipe_capability_value( + capabilities.network.as_deref(), + allowed_network, + component_id, + phase, + "network", + )?; + validate_recipe_capability_value( + capabilities.filesystem_read.as_deref(), + allowed_read, + component_id, + phase, + "filesystem_read", + )?; + validate_recipe_capability_value( + capabilities.filesystem_write.as_deref(), + allowed_write, + component_id, + phase, + "filesystem_write", + )?; + validate_recipe_capability_value( + capabilities.process.as_deref(), + &["declared_executable"], + component_id, + phase, + "process", + )?; + let allowed_secrets = if matches!(phase, RecipeCapabilityPhase::Resolve) { + &["deny", "opaque_handles"][..] + } else { + &["deny"][..] + }; + validate_recipe_capability_value( + capabilities.secrets.as_deref(), + allowed_secrets, + component_id, + phase, + "secrets", + )?; + validate_recipe_capability_value( + capabilities.publication.as_deref(), + &["deny"], + component_id, + phase, + "publication", + )?; + if capabilities + .child_processes + .is_some_and(|limit| limit == 0 || limit > MAX_RECIPE_CHILD_PROCESSES) + { + return Err(Error::InvalidInput(format!( + "repository component `{component_id}` {} child-process limit must be between 1 and {MAX_RECIPE_CHILD_PROCESSES}", + phase.name() + ))); + } + Ok(()) +} + +fn validate_recipe_capability_value( + value: Option<&str>, + allowed: &[&str], + component_id: &str, + phase: RecipeCapabilityPhase, + field: &str, +) -> Result<()> { + if let Some(value) = value + && !allowed.contains(&value) + { + return Err(Error::InvalidInput(format!( + "repository component `{component_id}` {} capability `{field} = {value}` exceeds the repository-declaration ceiling", + phase.name() + ))); + } + Ok(()) +} + +fn validate_recipe_network_authority(authority: &str) -> Result<()> { + if authority.is_empty() + || authority.len() > 512 + || authority.contains(char::is_whitespace) + || authority.contains(char::is_control) + || authority.contains('/') + || authority.contains('\\') + || authority.contains('@') + || authority.contains("//") + || contains_sensitive_text(authority) + || contains_shell_interpolation(authority) + || contains_provider_socket_reference(authority) + { + return Err(Error::InvalidInput(format!( + "repository resolver authority `{authority}` is not an exact non-secret network authority" + ))); + } + Ok(()) +} + +fn contains_shell_interpolation(argument: &str) -> bool { + argument.contains("$(") + || argument.contains("${") + || argument.contains('`') + || matches!( + argument, + "&&" | "||" | ";" | "|" | "&" | ">" | ">>" | "<" | "<<" + ) +} + +fn is_indirect_process_launcher(program: &str) -> bool { + matches!( + program.to_ascii_lowercase().as_str(), + "env" + | "xargs" + | "parallel" + | "nohup" + | "nice" + | "setsid" + | "sudo" + | "su" + | "doas" + | "command" + | "exec" + ) +} + +fn is_absolute_host_path(argument: &str) -> bool { + argument.starts_with('/') + || argument.starts_with("\\\\") + || argument.starts_with("file://") + || argument.as_bytes().get(1) == Some(&b':') + && argument + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphabetic) + && argument + .as_bytes() + .get(2) + .is_some_and(|byte| matches!(byte, b'/' | b'\\')) +} + +fn contains_provider_socket_reference(value: &str) -> bool { + let lower = value.to_ascii_lowercase(); + lower.contains("unix://") + || lower.contains("npipe://") + || lower.contains("/var/run/docker.sock") + || lower.contains("/run/docker.sock") + || lower.contains("/run/containerd/") + || lower.contains("ssh_auth_sock") + || lower.contains("docker_host") + || lower.contains("container_host") + || lower.contains("buildkit_host") +} + +impl Trail { + pub(crate) fn command_recipe_discovery( + &self, + source_root: &ObjectId, + component_root: Option<&str>, + ) -> Result> { + let requested_root = component_root + .map(normalize_recipe_path_allow_root) + .transpose()?; + let recipes = self.load_command_recipes(source_root)?; + Ok(recipes + .into_iter() + .filter(|recipe| { + requested_root + .as_ref() + .is_none_or(|root| root == &recipe.component.root) + }) + .map(|recipe| { + let resolvable = recipe.schema == RecipeSchemaVersion::V2 + && recipe.component.resolution.is_some(); + EnvironmentDiscoveredComponentReport { + component_id: recipe.component.id.clone(), + component_root: recipe.component.root, + kind: recipe.component.kind, + adapter_identity: RECIPE_ADAPTER_IDENTITY.to_string(), + status: if resolvable { + EnvironmentComponentProposalStatus::Resolvable + } else { + EnvironmentComponentProposalStatus::Ready + }, + reasons: if resolvable { + vec![EnvironmentProposalReasonReport { + code: "resolution_snapshot_required".into(), + message: + "repository component declares an explicit resolution snapshot" + .into(), + }] + } else { + Vec::new() + }, + recovery_actions: if resolvable { + vec![EnvironmentRecoveryActionReport { + code: "resolve_component".into(), + description: "resolve and pin the declared component snapshot".into(), + command: Some(vec![ + "trail".into(), + "env".into(), + "resolve".into(), + "--component".into(), + recipe.component.id, + ]), + }] + } else { + Vec::new() + }, + } + }) + .collect()) + } + + pub(crate) fn command_recipe_plan( + &self, + source_root: &ObjectId, + component_id: &str, + ) -> Result { + let recipes = self.load_command_recipes(source_root)?; + let recipe = recipes + .into_iter() + .find(|recipe| recipe.component.id == component_id) + .ok_or_else(|| { + Error::InvalidInput(format!( + "no `{RECIPE_ADAPTER_IDENTITY}` component named `{component_id}` exists in the pinned environment specification" + )) + })?; + if recipe.schema == RecipeSchemaVersion::V2 { + return self + .compile_repository_artifact_pipeline_v2_recipe(source_root, recipe)? + .into_graph_plan(); + } + self.plan_command_recipe(source_root, recipe) + } + + pub(crate) fn command_recipe_resolution_plan( + &self, + source_root: &ObjectId, + component_id: &str, + ) -> Result> { + let recipe = self + .load_command_recipes(source_root)? + .into_iter() + .find(|recipe| recipe.component.id == component_id) + .ok_or_else(|| { + Error::InvalidInput(format!( + "no repository environment component named `{component_id}` exists" + )) + })?; + if recipe.schema != RecipeSchemaVersion::V2 { + return Ok(None); + } + Ok(self + .compile_repository_artifact_pipeline_v2_recipe(source_root, recipe)? + .resolution_plan) + } + + #[cfg(test)] + fn compile_repository_artifact_pipeline_v2( + &self, + source_root: &ObjectId, + component_id: &str, + ) -> Result { + let recipe = self + .load_command_recipes(source_root)? + .into_iter() + .find(|recipe| recipe.component.id == component_id) + .ok_or_else(|| { + Error::InvalidInput(format!( + "no repository environment component named `{component_id}` exists" + )) + })?; + self.compile_repository_artifact_pipeline_v2_recipe(source_root, recipe) + } + + fn compile_repository_artifact_pipeline_v2_recipe( + &self, + source_root: &ObjectId, + recipe: CommandRecipe, + ) -> Result { + if recipe.schema != RecipeSchemaVersion::V2 { + return Err(Error::InvalidInput(format!( + "component `{}` requires `{RECIPE_SCHEMA_V2}` for artifact-pipeline compilation", + recipe.component.id + ))); + } + let graph_plan = self.plan_command_recipe(source_root, recipe.clone())?; + let component = &recipe.component; + let validations = compile_recipe_validations(&component.validations)?; + let resolution_plan = component + .resolution + .as_ref() + .map(|resolution| { + self.compile_recipe_resolution_plan( + source_root, + &recipe, + &graph_plan, + resolution, + &validations, + ) + }) + .transpose()?; + let outputs = graph_plan + .outputs + .iter() + .map(|output| ArtifactOutputContractV2 { + name: output.name.clone(), + output_path: output.output_path.clone(), + mount_path: output.mount_path.clone(), + policy: output.policy, + reuse: output.reuse, + scope: output.scope, + publish: output.publish, + gate: output.gate.clone(), + }) + .collect::>(); + let source_exports = compile_recipe_source_exports(component); + let mut actions = Vec::new(); + actions.push(self.compile_recipe_action_identity( + "build", + ArtifactActionPhaseV2::Construct, + &component.build.command, + component.build.cwd.as_deref().unwrap_or("."), + &component.build.environment, + )?); + if let Some(resolution) = &component.resolution { + actions.push(self.compile_recipe_action_identity( + "resolve", + ArtifactActionPhaseV2::Resolve, + &resolution.command, + resolution.cwd.as_deref().unwrap_or("."), + &resolution.environment, + )?); + } + for (index, action) in component.actions.iter().enumerate() { + let phase = match action.phase { + RecipeActionPhase::Construct => ArtifactActionPhaseV2::Construct, + RecipeActionPhase::Validate => ArtifactActionPhaseV2::Validate, + RecipeActionPhase::MountedExecution | RecipeActionPhase::SourceExport => { + ArtifactActionPhaseV2::Finalize + } + }; + let action_name = action + .name + .clone() + .unwrap_or_else(|| format!("action-{index}")); + actions.push(self.compile_recipe_action_identity( + &action_name, + phase, + &action.command, + action.cwd.as_deref().unwrap_or(&component.root), + &action.environment, + )?); + } + actions.sort(); + + let mut build_environment = component.build.environment.clone(); + if let Some(resolution) = &component.resolution { + merge_recipe_identity_environment( + &mut build_environment, + &resolution.environment, + &component.id, + )?; + } + for action in &component.actions { + merge_recipe_identity_environment( + &mut build_environment, + &action.environment, + &component.id, + )?; + } + let declared_inputs = graph_plan + .inputs + .iter() + .map(|input| ArtifactResolutionInputV1 { + source_path: input.source_path.clone(), + content_hash: input.entry.content_hash.clone(), + size_bytes: input.entry.size_bytes, + }) + .collect::>(); + let desired_material = ArtifactDesiredKeyMaterialV2 { + version: 2, + component_id: component.id.clone(), + adapter_identity: component.adapter.clone(), + adapter_implementation_version: env!("CARGO_PKG_VERSION").into(), + adapter_distribution_digest: "builtin:repository-environment-v2".into(), + adapter_protocol: RECIPE_SCHEMA_V2.into(), + resolution_snapshot_id: None, + source_closure: ArtifactSourceClosureV2 { + normalizer_version: "repository-inputs/v1".into(), + certified_complete: false, + complete_source_root: Some(source_root.clone()), + declared_inputs, + }, + upstream_identities: BTreeMap::new(), + actions, + outputs: outputs.clone(), + validations: validations.clone(), + source_exports: source_exports.clone(), + build_environment, + target: "repository-declared".into(), + platform: std::env::consts::OS.into(), + architecture: std::env::consts::ARCH.into(), + abi: "host-default".into(), + // The host has normalized every output through + // `validate_environment_output_contract`; repository text alone + // never sets this bit and compatible reuse remains unavailable. + portability_certified: true, + portability_scope: "workspace".into(), + trust_scope: "repository".into(), + network_policy: recipe_network_policy_identity( + component + .resolution + .as_ref() + .and_then(|resolution| resolution.network.as_ref()), + )?, + script_policy: ArtifactScriptPolicyV1::Deny, + sandbox_policy: "restricted-repository-pipeline-v2".into(), + }; + let desired_key = + super::workspace_artifact::artifact_desired_key_v2(desired_material.clone())?; + let proposal = EnvironmentDiscoveredComponentReport { + component_id: component.id.clone(), + component_root: component.root.clone(), + kind: component.kind.clone(), + adapter_identity: component.adapter.clone(), + status: if resolution_plan.is_some() { + EnvironmentComponentProposalStatus::Resolvable + } else { + EnvironmentComponentProposalStatus::Ready + }, + reasons: resolution_plan.as_ref().map_or_else(Vec::new, |_| { + vec![EnvironmentProposalReasonReport { + code: "resolution_snapshot_required".into(), + message: "repository component declares an explicit resolution snapshot".into(), + }] + }), + recovery_actions: resolution_plan.as_ref().map_or_else(Vec::new, |_| { + vec![EnvironmentRecoveryActionReport { + code: "resolve_component".into(), + description: "resolve and pin the declared component snapshot".into(), + command: Some(vec![ + "trail".into(), + "env".into(), + "resolve".into(), + "--component".into(), + component.id.clone(), + ]), + }] + }), + }; + Ok(CompiledRepositoryArtifactPipelineV2 { + proposal, + resolution_plan, + graph_plan, + desired_material, + desired_key, + outputs, + validations, + source_exports, + }) + } + + fn compile_recipe_resolution_plan( + &self, + source_root: &ObjectId, + recipe: &CommandRecipe, + graph_plan: &WorkspaceEnvironmentPlan, + resolution: &RecipeResolution, + validations: &[ArtifactValidationV1], + ) -> Result { + let program = resolution.command.first().ok_or_else(|| { + Error::InvalidInput(format!( + "repository component `{}` resolver command is empty", + recipe.component.id + )) + })?; + let tool = resolve_workspace_tool_executable(program)?; + let working_directory = resolution + .cwd + .clone() + .unwrap_or_else(|| recipe.component.root.clone()); + let mut plan = ArtifactResolutionPlanV1 { + version: ARTIFACT_RESOLUTION_PLAN_VERSION, + proposal_key: format!("repository_v2_{}", recipe.specification_digest), + source_root: source_root.clone(), + component_id: recipe.component.id.clone(), + adapter_identity: recipe.component.adapter.clone(), + policy_identity: sha256_hex(&serde_json::to_vec(&( + &resolution.capabilities, + &recipe.component.capabilities, + ))?), + program: program.clone(), + resolved_program: tool.path.to_string_lossy().into_owned(), + executable_identity: tool.identity, + argv: resolution.command.clone(), + working_directory: working_directory.clone(), + readable_inputs: graph_plan + .inputs + .iter() + .map(|input| ArtifactResolutionInputV1 { + source_path: input.source_path.clone(), + content_hash: input.entry.content_hash.clone(), + size_bytes: input.entry.size_bytes, + }) + .collect(), + candidate_output: normalize_relative_path(&join_recipe_path( + &working_directory, + &resolution.snapshot, + ))?, + allowed_authorities: recipe_network_authorities(resolution.network.as_ref())?, + credential_handles: Vec::new(), + script_policy: ArtifactScriptPolicyV1::Deny, + environment_roles: resolution + .environment + .keys() + .map(|name| (name.clone(), ArtifactEnvironmentRoleV1::Identity)) + .collect(), + limits: ArtifactActionLimitsV1 { + timeout_ms: 5 * 60 * 1_000, + stdout_bytes: 1024 * 1024, + stderr_bytes: 1024 * 1024, + candidate_bytes: 256 * 1024 * 1024, + candidate_entries: 100_000, + child_processes: resolution + .capabilities + .as_ref() + .and_then(|capabilities| capabilities.child_processes) + .unwrap_or(1) + .max(1), + }, + snapshot_format: resolution.format.clone(), + validations: if validations.is_empty() { + vec![ArtifactValidationV1 { + name: "snapshot-structure".into(), + kind: ArtifactValidationKindV1::Structural, + required: true, + parameters: BTreeMap::new(), + }] + } else { + validations.to_vec() + }, + }; + super::workspace_artifact::normalize_artifact_resolution_plan(&mut plan)?; + Ok(plan) + } + + fn compile_recipe_action_identity( + &self, + name: &str, + phase: ArtifactActionPhaseV2, + command: &[String], + working_directory: &str, + environment: &BTreeMap, + ) -> Result { + let program = command.first().ok_or_else(|| { + Error::InvalidInput(format!("repository action `{name}` command is empty")) + })?; + let tool = resolve_workspace_tool_executable(program)?; + let normalized_working_directory = normalize_recipe_path_allow_root(working_directory)?; + Ok(ArtifactActionIdentityV2 { + name: name.into(), + phase, + executable_identity: tool.identity, + argv: command.to_vec(), + working_directory: if normalized_working_directory.is_empty() { + ".".into() + } else { + normalized_working_directory + }, + environment_names: environment.keys().cloned().collect(), + }) + } + + pub(crate) fn command_recipe_plans( + &self, + source_root: &ObjectId, + component_ids: &BTreeSet, + ) -> Result> { + let recipes = self.load_command_recipes(source_root)?; + let mut plans = BTreeMap::new(); + let mut tools = BTreeMap::::new(); + for recipe in recipes { + if component_ids.contains(&recipe.component.id) { + let component_id = recipe.component.id.clone(); + if recipe.schema == RecipeSchemaVersion::V2 { + plans.insert( + component_id, + self.compile_repository_artifact_pipeline_v2_recipe(source_root, recipe)? + .into_graph_plan()?, + ); + continue; + } + let program = recipe + .component + .build + .command + .first() + .cloned() + .ok_or_else(|| { + Error::InvalidInput(format!( + "command component `{component_id}` has an empty build.command" + )) + })?; + let tool = if let Some(tool) = tools.get(&program) { + tool.clone() + } else { + let tool = resolve_workspace_tool_executable(&program)?; + tools.insert(program, tool.clone()); + tool + }; + plans.insert( + component_id, + self.plan_command_recipe_with_tool(source_root, recipe, Some(tool))?, + ); + } + } + if plans.len() != component_ids.len() { + let missing = component_ids + .iter() + .filter(|component_id| !plans.contains_key(*component_id)) + .cloned() + .collect::>(); + return Err(Error::InvalidInput(format!( + "pinned environment specification is missing command component(s): {}", + missing.join(", ") + ))); + } + Ok(plans) + } + + pub(crate) fn command_recipe_source_exports( + &self, + source_root: &ObjectId, + component_id: &str, + ) -> Result> { + let recipe = self + .load_command_recipes(source_root)? + .into_iter() + .find(|recipe| recipe.component.id == component_id) + .ok_or_else(|| { + Error::InvalidInput(format!( + "no repository environment component named `{component_id}` exists" + )) + })?; + if recipe.schema != RecipeSchemaVersion::V2 { + return Err(Error::InvalidInput(format!( + "component `{component_id}` requires `{RECIPE_SCHEMA_V2}` for source export" + ))); + } + Ok(compile_recipe_source_exports(&recipe.component)) + } + + pub(crate) fn command_recipe_plan_for_root( + &self, + source_root: &ObjectId, + component_root: &str, + ) -> Result { + let component_root = normalize_recipe_path_allow_root(component_root)?; + let mut matching = self + .load_command_recipes(source_root)? + .into_iter() + .filter(|recipe| recipe.component.root == component_root) + .collect::>(); + match matching.len() { + 1 => { + let recipe = matching.remove(0); + if recipe.schema == RecipeSchemaVersion::V2 { + self.compile_repository_artifact_pipeline_v2_recipe(source_root, recipe)? + .into_graph_plan() + } else { + self.plan_command_recipe(source_root, recipe) + } + } + 0 => Err(Error::InvalidInput(format!( + "no `{RECIPE_ADAPTER_IDENTITY}` component is declared at `{}`", + display_recipe_root(&component_root) + ))), + count => Err(Error::InvalidInput(format!( + "{count} `{RECIPE_ADAPTER_IDENTITY}` components are declared at `{}`; synchronize all components or give each recipe a distinct root", + display_recipe_root(&component_root) + ))), + } + } + + fn load_command_recipes(&self, source_root: &ObjectId) -> Result> { + #[cfg(test)] + COMMAND_RECIPE_LOAD_COUNT.with(|count| count.set(count.get() + 1)); + let mut found = Vec::new(); + for path in RECIPE_SPEC_PATHS { + if self.root_file_entry(source_root, path)?.is_some() { + found.push(path.to_string()); + } + } + if found.len() > 1 { + return Err(Error::InvalidInput(format!( + "environment specification is ambiguous; keep only one of {}", + RECIPE_SPEC_PATHS.join(", ") + ))); + } + let Some(spec_path) = found.pop() else { + return Ok(Vec::new()); + }; + + let mut documents = RecipeDocuments::default(); + let mut visited = BTreeSet::new(); + let mut stack = Vec::new(); + let mut total_bytes = 0u64; + self.collect_recipe_document( + source_root, + &spec_path, + 0, + true, + &mut documents, + &mut visited, + &mut stack, + &mut total_bytes, + )?; + + let mut ids = BTreeSet::new(); + let mut targets = BTreeMap::::new(); + let mut profile_cache = BTreeMap::new(); + let mut recipes = Vec::with_capacity(documents.components.len()); + for definition in documents.components { + let (component, profile_versions) = + resolve_recipe_component(definition, &documents.profiles, &mut profile_cache)?; + validate_recipe_component_identity(&component.id)?; + if !ids.insert(component.id.clone()) { + return Err(Error::InvalidInput(format!( + "environment specification declares component `{}` more than once", + component.id + ))); + } + if component.adapter.as_str() != RECIPE_ADAPTER_IDENTITY { + return Err(Error::InvalidInput(format!( + "component `{}` uses unsupported declarative adapter `{}`; this specification host currently accepts only `{RECIPE_ADAPTER_IDENTITY}`", + component.id, component.adapter + ))); + } + if component.kind != "generated" { + return Err(Error::InvalidInput(format!( + "command component `{}` must use kind = \"generated\"", + component.id + ))); + } + if component.outputs.is_empty() || component.outputs.len() > 32 { + return Err(Error::InvalidInput(format!( + "command component `{}` must declare between 1 and 32 outputs", + component.id, + ))); + } + let mut output_names = BTreeSet::new(); + for (index, output) in component.outputs.iter().enumerate() { + let name = output + .name + .clone() + .unwrap_or_else(|| format!("output-{index}")); + validate_recipe_output_name(&name, &component.id)?; + if !output_names.insert(name.clone()) { + return Err(Error::InvalidInput(format!( + "command component `{}` declares output name `{name}` more than once", + component.id + ))); + } + let target = normalize_relative_path(&output.target)?; + if let Some((other_target, other_id)) = recipe_target_overlap(&targets, &target) { + return Err(Error::InvalidInput(format!( + "command component `{}` target `{target}` overlaps component `{other_id}` target `{other_target}`", + component.id + ))); + } + targets.insert(target, format!("{}:{name}", component.id)); + } + let schema = documents.schema.ok_or_else(|| { + Error::Corrupt("environment specification graph lost its schema version".into()) + })?; + if schema == RecipeSchemaVersion::V2 { + validate_recipe_v2_component(&component, &documents.defaults)?; + } + let canonical = serde_json::to_vec(&(schema.as_str(), &component, &profile_versions))?; + recipes.push(CommandRecipe { + schema, + specification_digest: sha256_hex(&canonical), + specification_sources: documents.specification_sources.clone(), + profile_versions, + defaults: documents.defaults.clone(), + component, + }); + } + recipes.sort_by(|left, right| left.component.id.cmp(&right.component.id)); + Ok(recipes) + } + + #[allow(clippy::too_many_arguments)] + fn collect_recipe_document( + &self, + source_root: &ObjectId, + path: &str, + depth: usize, + is_root: bool, + documents: &mut RecipeDocuments, + visited: &mut BTreeSet, + stack: &mut Vec, + total_bytes: &mut u64, + ) -> Result<()> { + if depth > MAX_RECIPE_INCLUDE_DEPTH { + return Err(Error::InvalidInput(format!( + "environment specification include depth exceeds {MAX_RECIPE_INCLUDE_DEPTH} at `{path}`" + ))); + } + if let Some(index) = stack.iter().position(|candidate| candidate == path) { + let mut cycle = stack[index..].to_vec(); + cycle.push(path.to_string()); + return Err(Error::InvalidInput(format!( + "environment specification include cycle: {}", + cycle.join(" -> ") + ))); + } + if visited.contains(path) { + return Ok(()); + } + if visited.len().saturating_add(stack.len()) >= MAX_RECIPE_INCLUDE_FILES { + return Err(Error::InvalidInput(format!( + "environment specification includes more than {MAX_RECIPE_INCLUDE_FILES} files" + ))); + } + let entry = self.root_file_entry(source_root, path)?.ok_or_else(|| { + Error::InvalidInput(format!( + "environment specification include `{path}` does not exist in the pinned source root" + )) + })?; + if entry.size_bytes > MAX_RECIPE_SPEC_BYTES { + return Err(Error::InvalidInput(format!( + "environment specification `{path}` is {} bytes; the per-file maximum is {MAX_RECIPE_SPEC_BYTES}", + entry.size_bytes + ))); + } + *total_bytes = total_bytes.checked_add(entry.size_bytes).ok_or_else(|| { + Error::InvalidInput("environment specification size overflowed".to_string()) + })?; + if *total_bytes > MAX_RECIPE_TOTAL_SPEC_BYTES { + return Err(Error::InvalidInput(format!( + "environment specifications total more than {MAX_RECIPE_TOTAL_SPEC_BYTES} bytes" + ))); + } + let entries = BTreeMap::from([(path.to_string(), entry.clone())]); + let bytes = self + .materialize_entries_bytes(&entries)? + .remove(path) + .ok_or_else(|| Error::Corrupt(format!("failed to read `{path}` from source root")))?; + let text = String::from_utf8(bytes).map_err(|_| { + Error::InvalidInput(format!("environment specification `{path}` must be UTF-8")) + })?; + let specification: RecipeSpecification = toml::from_str(&text).map_err(|err| { + Error::InvalidInput(format!("invalid environment specification `{path}`: {err}")) + })?; + let schema = validate_recipe_specification_header(&specification, path)?; + if let Some(expected) = documents.schema { + if schema != expected { + return Err(Error::InvalidInput(format!( + "environment specification `{path}` uses schema `{}` but the root document uses `{}`", + schema.as_str(), + expected.as_str() + ))); + } + } else { + documents.schema = Some(schema); + } + + stack.push(path.to_string()); + for include in &specification.include { + let include_path = resolve_recipe_include_path(path, include)?; + self.collect_recipe_document( + source_root, + &include_path, + depth + 1, + false, + documents, + visited, + stack, + total_bytes, + )?; + } + stack.pop(); + + for (name, profile) in specification.profile { + let canonical_name = canonical_recipe_profile_name(&name)?; + if documents + .profiles + .insert(canonical_name.clone(), profile) + .is_some() + { + return Err(Error::InvalidInput(format!( + "environment specifications declare profile `{canonical_name}` more than once" + ))); + } + } + documents.components.extend(specification.components); + if is_root { + documents.defaults = specification.environment; + } + documents + .specification_sources + .insert(path.to_string(), entry.content_hash); + visited.insert(path.to_string()); + Ok(()) + } + + fn plan_command_recipe( + &self, + source_root: &ObjectId, + recipe: CommandRecipe, + ) -> Result { + self.plan_command_recipe_with_tool(source_root, recipe, None) + } + + fn plan_command_recipe_with_tool( + &self, + source_root: &ObjectId, + recipe: CommandRecipe, + resolved_tool: Option, + ) -> Result { + let component = recipe.component; + let network = component + .build + .network + .as_deref() + .unwrap_or(&recipe.defaults.default_network); + let scripts = component + .build + .scripts + .as_deref() + .unwrap_or(&recipe.defaults.default_scripts); + if network != "deny" || scripts != "deny" { + return Err(Error::InvalidInput(format!( + "command component `{}` requires network = \"deny\" and scripts = \"deny\"", + component.id + ))); + } + if component.build.command.is_empty() { + return Err(Error::InvalidInput(format!( + "command component `{}` has an empty build.command", + component.id + ))); + } + if component.build.command.len() > 4096 + || component.build.command.iter().any(|argument| { + argument.len() > 128 * 1024 + || argument.contains('\0') + || contains_sensitive_text(argument) + }) + { + return Err(Error::InvalidInput(format!( + "command component `{}` exceeds command argument limits", + component.id + ))); + } + let program = &component.build.command[0]; + if program.contains('/') || program.contains('\\') || is_shell_program(program) { + return Err(Error::InvalidInput(format!( + "command component `{}` must name a non-shell executable from PATH, not `{program}`", + component.id + ))); + } + let tool = resolved_tool + .map(Ok) + .unwrap_or_else(|| resolve_workspace_tool_executable(program))?; + validate_recipe_tool_path(self, &tool.path, &component.id)?; + let cwd = normalize_recipe_path_allow_root( + component.build.cwd.as_deref().unwrap_or(&component.root), + )?; + if !component.root.is_empty() + && cwd != component.root && !cwd.starts_with(&format!("{}/", component.root)) { return Err(Error::InvalidInput(format!( @@ -822,6 +2152,7 @@ impl Trail { strategy: "restricted-command-recipe-v1".to_string(), }, inputs, + resolution_inputs: Vec::new(), source_projection: None, pre_commands: Vec::new(), command: Some(WorkspaceEnvironmentCommand { @@ -939,647 +2270,1563 @@ impl Trail { } } } - let total_bytes = selected.values().try_fold(0u64, |total, entry| { - total.checked_add(entry.size_bytes).ok_or_else(|| { - Error::InvalidInput(format!( - "command component `{}` input byte count overflowed", - component.id - )) - }) - })?; - if selected.len() > MAX_RECIPE_INPUT_FILES || total_bytes > MAX_RECIPE_INPUT_BYTES { + let total_bytes = selected.values().try_fold(0u64, |total, entry| { + total.checked_add(entry.size_bytes).ok_or_else(|| { + Error::InvalidInput(format!( + "command component `{}` input byte count overflowed", + component.id + )) + }) + })?; + if selected.len() > MAX_RECIPE_INPUT_FILES || total_bytes > MAX_RECIPE_INPUT_BYTES { + return Err(Error::InvalidInput(format!( + "command component `{}` selects {} files and {total_bytes} bytes; limits are {MAX_RECIPE_INPUT_FILES} files and {MAX_RECIPE_INPUT_BYTES} bytes", + component.id, + selected.len() + ))); + } + Ok(selected) + } +} + +fn validate_recipe_specification_header( + specification: &RecipeSpecification, + path: &str, +) -> Result { + let schema = RecipeSchemaVersion::parse(&specification.schema, path)?; + if schema == RecipeSchemaVersion::V1 + && (specification.environment.missing_resolution.is_some() + || specification.profile.values().any(recipe_profile_uses_v2) + || specification + .components + .iter() + .any(recipe_component_definition_uses_v2)) + { + return Err(Error::InvalidInput(format!( + "environment specification `{path}` uses fields that require `{RECIPE_SCHEMA_V2}`" + ))); + } + if specification.environment.default_network != "deny" + || specification.environment.default_scripts != "deny" + { + return Err(Error::InvalidInput(format!( + "environment specification `{path}` must set default_network and default_scripts to `deny`" + ))); + } + if specification + .environment + .missing_resolution + .as_deref() + .is_some_and(|policy| policy != "explicit") + { + return Err(Error::InvalidInput(format!( + "environment specification `{path}` missing_resolution must be `explicit`" + ))); + } + let _environment_name = specification.environment.name.as_deref(); + Ok(schema) +} + +fn recipe_profile_uses_v2(profile: &RecipeProfile) -> bool { + profile.resolution.is_some() + || !profile.actions.is_empty() + || !profile.validations.is_empty() + || profile.capabilities.is_some() + || !profile.source_exports.is_empty() +} + +fn recipe_component_definition_uses_v2(component: &RecipeComponentDefinition) -> bool { + component.resolution.is_some() + || !component.actions.is_empty() + || !component.validations.is_empty() + || component.capabilities.is_some() + || !component.source_exports.is_empty() +} + +fn resolve_recipe_include_path(including_path: &str, include: &str) -> Result { + if include.is_empty() + || include.starts_with('/') + || include.contains("://") + || include.contains('\\') + || contains_glob_meta(include) + || include + .split('/') + .any(|segment| segment.is_empty() || segment == "." || segment == "..") + { + return Err(Error::InvalidInput(format!( + "invalid local environment specification include `{include}` in `{including_path}`" + ))); + } + let parent = including_path + .rsplit_once('/') + .map(|(parent, _)| parent) + .unwrap_or(""); + normalize_relative_path(&join_recipe_path(parent, include)) +} + +fn validate_recipe_profile_name(name: &str) -> Result<()> { + let canonical = name.strip_prefix("profile.").unwrap_or(name); + if canonical.is_empty() + || canonical.len() > 256 + || !canonical + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_' | '/')) + || canonical + .split('/') + .any(|segment| segment.is_empty() || segment == "..") + { + return Err(Error::InvalidInput(format!( + "invalid environment recipe profile name `{name}`" + ))); + } + Ok(()) +} + +fn canonical_recipe_profile_name(name: &str) -> Result { + validate_recipe_profile_name(name)?; + Ok(name.strip_prefix("profile.").unwrap_or(name).to_string()) +} + +fn recipe_profile_fragment(profile: &RecipeProfile) -> RecipeFragment { + RecipeFragment { + adapter: profile.adapter.clone(), + kind: profile.kind.clone(), + dependencies: profile.depends_on.clone(), + edges: profile.edges.clone(), + inputs: profile.inputs.clone(), + outputs: profile.outputs.clone(), + build: profile.build.clone(), + resolution: profile.resolution.clone(), + actions: profile.actions.clone(), + validations: profile.validations.clone(), + capabilities: profile.capabilities.clone(), + source_exports: profile.source_exports.clone(), + } +} + +fn apply_recipe_fragment(target: &mut RecipeFragment, source: &RecipeFragment) { + if source.adapter.is_some() { + target.adapter.clone_from(&source.adapter); + } + if source.kind.is_some() { + target.kind.clone_from(&source.kind); + } + target.dependencies.extend(source.dependencies.clone()); + target.edges.extend(source.edges.clone()); + target.inputs.extend(source.inputs.clone()); + if !source.outputs.is_empty() { + target.outputs.clone_from(&source.outputs); + } + if source.build.is_some() { + target.build.clone_from(&source.build); + } + if source.resolution.is_some() { + target.resolution.clone_from(&source.resolution); + } + target.actions.extend(source.actions.clone()); + target.validations.extend(source.validations.clone()); + if source.capabilities.is_some() { + target.capabilities.clone_from(&source.capabilities); + } + target.source_exports.extend(source.source_exports.clone()); +} + +fn resolve_recipe_profile( + requested_name: &str, + profiles: &BTreeMap, + cache: &mut BTreeMap, + stack: &mut Vec, +) -> Result { + let name = canonical_recipe_profile_name(requested_name)?; + if let Some(resolved) = cache.get(&name) { + return Ok(resolved.clone()); + } + if let Some(index) = stack.iter().position(|candidate| candidate == &name) { + let mut cycle = stack[index..].to_vec(); + cycle.push(name); + return Err(Error::InvalidInput(format!( + "environment recipe profile cycle: {}", + cycle.join(" -> ") + ))); + } + let profile = profiles.get(&name).ok_or_else(|| { + Error::InvalidInput(format!( + "environment recipe references unknown profile `{requested_name}`" + )) + })?; + if profile.version.is_empty() + || profile.version.len() > 128 + || profile.version.contains(char::is_whitespace) + || profile.version.contains('\0') + { + return Err(Error::InvalidInput(format!( + "environment recipe profile `{name}` has invalid version `{}`", + profile.version + ))); + } + + stack.push(name.clone()); + let mut fragment = RecipeFragment::default(); + let mut versions = BTreeMap::new(); + for parent in &profile.extends { + let resolved = resolve_recipe_profile(parent, profiles, cache, stack)?; + apply_recipe_fragment(&mut fragment, &resolved.fragment); + versions.extend(resolved.versions); + } + stack.pop(); + apply_recipe_fragment(&mut fragment, &recipe_profile_fragment(profile)); + versions.insert(name.clone(), profile.version.clone()); + let resolved = ResolvedRecipeProfile { fragment, versions }; + cache.insert(name, resolved.clone()); + Ok(resolved) +} + +fn resolve_recipe_component( + definition: RecipeComponentDefinition, + profiles: &BTreeMap, + cache: &mut BTreeMap, +) -> Result<(RecipeComponent, BTreeMap)> { + validate_recipe_component_identity(&definition.id)?; + let root = normalize_recipe_path_allow_root(&definition.root)?; + let mut fragment = RecipeFragment::default(); + let mut versions = BTreeMap::new(); + let mut stack = Vec::new(); + for profile_name in &definition.extends { + let resolved = resolve_recipe_profile(profile_name, profiles, cache, &mut stack)?; + apply_recipe_fragment(&mut fragment, &resolved.fragment); + versions.extend(resolved.versions); + } + apply_recipe_fragment( + &mut fragment, + &RecipeFragment { + adapter: definition.adapter, + kind: definition.kind, + dependencies: definition.depends_on, + edges: definition.edges, + inputs: definition.inputs, + outputs: definition.outputs, + build: definition.build, + resolution: definition.resolution, + actions: definition.actions, + validations: definition.validations, + capabilities: definition.capabilities, + source_exports: definition.source_exports, + }, + ); + let adapter = fragment.adapter.ok_or_else(|| { + Error::InvalidInput(format!( + "command component `{}` has no adapter after profile expansion", + definition.id + )) + })?; + let mut build = fragment.build.ok_or_else(|| { + Error::InvalidInput(format!( + "command component `{}` has no build declaration after profile expansion", + definition.id + )) + })?; + let mut inputs = fragment.inputs; + let mut outputs = fragment.outputs; + let mut resolution = fragment.resolution; + let mut actions = fragment.actions; + let mut validations = fragment.validations; + let capabilities = fragment.capabilities; + let mut source_exports = fragment.source_exports; + let mut dependencies = fragment.dependencies; + let edges = fragment.edges; + let mut seen_dependencies = BTreeSet::new(); + dependencies.retain(|dependency| seen_dependencies.insert(dependency.clone())); + for dependency in &dependencies { + validate_recipe_component_identity(dependency)?; + if dependency == &definition.id { + return Err(Error::InvalidInput(format!( + "environment component `{}` cannot depend on itself", + definition.id + ))); + } + } + let mut typed_edge_components = BTreeMap::new(); + for edge in edges { + if let Some(previous) = + typed_edge_components.insert(edge.component.clone(), edge.edge_type.clone()) + && previous != edge.edge_type + { + return Err(Error::InvalidInput(format!( + "environment component `{}` declares conflicting edge types `{previous}` and `{}` for `{}`", + definition.id, edge.edge_type, edge.component + ))); + } + } + let edges = typed_edge_components + .into_iter() + .map(|(component, edge_type)| RecipeDependencyEdge { + component, + edge_type, + }) + .collect::>(); + for edge in &edges { + validate_recipe_component_identity(&edge.component)?; + WorkspaceEnvironmentEdgeType::parse(&edge.edge_type)?; + if edge.component == definition.id { return Err(Error::InvalidInput(format!( - "command component `{}` selects {} files and {total_bytes} bytes; limits are {MAX_RECIPE_INPUT_FILES} files and {MAX_RECIPE_INPUT_BYTES} bytes", - component.id, - selected.len() + "environment component `{}` cannot depend on itself", + definition.id ))); } - Ok(selected) + if seen_dependencies.contains(&edge.component) { + return Err(Error::InvalidInput(format!( + "environment component `{}` declares both legacy depends_on and typed edge for `{}`", + definition.id, edge.component + ))); + } + } + for input in &mut inputs { + input.path = expand_recipe_root_template(&input.path, &root); + } + for output in &mut outputs { + output.source = expand_recipe_root_template(&output.source, &root); + output.target = expand_recipe_root_template(&output.target, &root); + } + for argument in &mut build.command { + *argument = expand_recipe_root_template(argument, &root); + } + if let Some(cwd) = &mut build.cwd { + *cwd = expand_recipe_root_template(cwd, &root); + } + for value in build.environment.values_mut() { + *value = expand_recipe_root_template(value, &root); + } + if let Some(resolution) = &mut resolution { + for argument in &mut resolution.command { + *argument = expand_recipe_root_template(argument, &root); + } + if let Some(cwd) = &mut resolution.cwd { + *cwd = expand_recipe_root_template(cwd, &root); + } + resolution.snapshot = expand_recipe_root_template(&resolution.snapshot, &root); + for value in resolution.environment.values_mut() { + *value = expand_recipe_root_template(value, &root); + } + } + for action in &mut actions { + for argument in &mut action.command { + *argument = expand_recipe_root_template(argument, &root); + } + if let Some(cwd) = &mut action.cwd { + *cwd = expand_recipe_root_template(cwd, &root); + } + for value in action.environment.values_mut() { + *value = expand_recipe_root_template(value, &root); + } + } + for validation in &mut validations { + if let Some(path) = &mut validation.path { + *path = expand_recipe_root_template(path, &root); + } + for argument in &mut validation.command { + *argument = expand_recipe_root_template(argument, &root); + } + } + for export in &mut source_exports { + export.source = expand_recipe_root_template(&export.source, &root); + export.target = expand_recipe_root_template(&export.target, &root); } + let mut seen_inputs = BTreeSet::new(); + inputs.retain(|input| { + seen_inputs.insert(( + input.path.clone(), + input.role.clone(), + input.format.clone(), + input.optional, + )) + }); + Ok(( + RecipeComponent { + id: definition.id, + adapter, + root, + kind: fragment.kind.unwrap_or_else(default_recipe_kind), + dependencies, + edges, + inputs, + outputs, + build, + resolution, + actions, + validations, + capabilities, + source_exports, + }, + versions, + )) } -fn validate_recipe_specification_header( - specification: &RecipeSpecification, - path: &str, -) -> Result<()> { - if specification.schema != RECIPE_SCHEMA { +fn expand_recipe_root_template(value: &str, root: &str) -> String { + if root.is_empty() { + value.replace("{root}/", "").replace("{root}", ".") + } else { + value.replace("{root}", root) + } +} + +fn validate_recipe_component_identity(id: &str) -> Result<()> { + if id.is_empty() + || id.len() > 256 + || !id + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_' | ':' | '/')) + || id.starts_with('/') + || id.ends_with('/') + || id + .split('/') + .any(|segment| segment.is_empty() || segment == "..") + { return Err(Error::InvalidInput(format!( - "unsupported environment schema `{}` in `{path}`; expected `{RECIPE_SCHEMA}`", - specification.schema + "invalid command component id `{id}`" ))); } - if specification.environment.default_network != "deny" - || specification.environment.default_scripts != "deny" + Ok(()) +} + +fn validate_recipe_output_name(name: &str, component_id: &str) -> Result<()> { + if name.is_empty() + || name.len() > 128 + || !name + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_')) { return Err(Error::InvalidInput(format!( - "environment specification `{path}` must set default_network and default_scripts to `deny`" + "command component `{component_id}` has invalid output name `{name}`" ))); } - let _environment_name = specification.environment.name.as_deref(); Ok(()) } -fn resolve_recipe_include_path(including_path: &str, include: &str) -> Result { - if include.is_empty() - || include.starts_with('/') - || include.contains("://") - || include.contains('\\') - || contains_glob_meta(include) - || include - .split('/') - .any(|segment| segment.is_empty() || segment == "." || segment == "..") +fn validate_recipe_pattern(pattern: &str) -> Result<()> { + if pattern.is_empty() + || pattern.starts_with('/') + || pattern.contains('\\') + || pattern.split('/').any(|segment| segment == "..") { return Err(Error::InvalidInput(format!( - "invalid local environment specification include `{include}` in `{including_path}`" + "invalid repository-relative recipe input `{pattern}`" ))); } - let parent = including_path - .rsplit_once('/') - .map(|(parent, _)| parent) - .unwrap_or(""); - normalize_relative_path(&join_recipe_path(parent, include)) + normalize_relative_path(pattern).map(|_| ()) } -fn validate_recipe_profile_name(name: &str) -> Result<()> { - let canonical = name.strip_prefix("profile.").unwrap_or(name); - if canonical.is_empty() - || canonical.len() > 256 - || !canonical +fn contains_glob_meta(path: &str) -> bool { + path.bytes() + .any(|byte| matches!(byte, b'*' | b'?' | b'[' | b'{' | b'!')) +} + +pub(super) fn validate_recipe_environment( + name: &str, + value: &str, + component_id: &str, +) -> Result<()> { + let valid_name = !name.is_empty() + && name .chars() - .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_' | '/')) - || canonical - .split('/') - .any(|segment| segment.is_empty() || segment == "..") + .all(|ch| ch == '_' || ch.is_ascii_alphanumeric()); + let upper = name.to_ascii_uppercase(); + let sensitive = [ + "TOKEN", + "SECRET", + "PASSWORD", + "PASSWD", + "CREDENTIAL", + "PRIVATE_KEY", + "AUTH", + ] + .iter() + .any(|needle| upper.contains(needle)); + if !valid_name + || sensitive + || matches!( + upper.as_str(), + "PATH" | "HOME" | "TMP" | "TMPDIR" | "TEMP" | "SHELL" | "DYLD_INSERT_LIBRARIES" + ) + || value.contains('\0') + || value.len() > 128 * 1024 + || contains_sensitive_text(value) { return Err(Error::InvalidInput(format!( - "invalid environment recipe profile name `{name}`" + "command component `{component_id}` has forbidden environment entry `{name}`" ))); } Ok(()) } -fn canonical_recipe_profile_name(name: &str) -> Result { - validate_recipe_profile_name(name)?; - Ok(name.strip_prefix("profile.").unwrap_or(name).to_string()) +pub(super) fn validate_recipe_tool_path(db: &Trail, path: &Path, component_id: &str) -> Result<()> { + let canonical = fs::canonicalize(path)?; + let mut forbidden = vec![db.workspace_root.clone(), db.db_dir.clone()]; + if let Some(home) = std::env::var_os("HOME") { + forbidden.push(PathBuf::from(home)); + } + if forbidden.iter().any(|root| canonical.starts_with(root)) { + return Err(Error::InvalidInput(format!( + "command component `{component_id}` executable `{}` is under a mutable workspace or user home; bind a host-managed toolchain instead", + canonical.display() + ))); + } + Ok(()) } -fn recipe_profile_fragment(profile: &RecipeProfile) -> RecipeFragment { - RecipeFragment { - adapter: profile.adapter.clone(), - kind: profile.kind.clone(), - dependencies: profile.depends_on.clone(), - edges: profile.edges.clone(), - inputs: profile.inputs.clone(), - outputs: profile.outputs.clone(), - build: profile.build.clone(), - } +pub(super) fn is_shell_program(program: &str) -> bool { + matches!( + program.to_ascii_lowercase().as_str(), + "sh" | "bash" + | "zsh" + | "fish" + | "dash" + | "ksh" + | "csh" + | "tcsh" + | "cmd" + | "cmd.exe" + | "powershell" + | "powershell.exe" + | "pwsh" + ) } -fn apply_recipe_fragment(target: &mut RecipeFragment, source: &RecipeFragment) { - if source.adapter.is_some() { - target.adapter.clone_from(&source.adapter); - } - if source.kind.is_some() { - target.kind.clone_from(&source.kind); - } - target.dependencies.extend(source.dependencies.clone()); - target.edges.extend(source.edges.clone()); - target.inputs.extend(source.inputs.clone()); - if !source.outputs.is_empty() { - target.outputs.clone_from(&source.outputs); - } - if source.build.is_some() { - target.build.clone_from(&source.build); +fn normalize_recipe_path_allow_root(path: &str) -> Result { + if path.trim_matches('/').is_empty() || path == "." { + Ok(String::new()) + } else { + normalize_relative_path(path) } } -fn resolve_recipe_profile( - requested_name: &str, - profiles: &BTreeMap, - cache: &mut BTreeMap, - stack: &mut Vec, -) -> Result { - let name = canonical_recipe_profile_name(requested_name)?; - if let Some(resolved) = cache.get(&name) { - return Ok(resolved.clone()); +fn join_recipe_path(root: &str, child: &str) -> String { + if root.is_empty() { + child.to_string() + } else { + format!("{root}/{child}") } - if let Some(index) = stack.iter().position(|candidate| candidate == &name) { - let mut cycle = stack[index..].to_vec(); - cycle.push(name); - return Err(Error::InvalidInput(format!( - "environment recipe profile cycle: {}", - cycle.join(" -> ") - ))); +} + +fn recipe_paths_overlap(left: &str, right: &str) -> bool { + left == right + || left.starts_with(&format!("{right}/")) + || right.starts_with(&format!("{left}/")) +} + +fn recipe_target_overlap<'a>( + targets: &'a BTreeMap, + target: &str, +) -> Option<(&'a str, &'a str)> { + if let Some((stored, owner)) = targets.get_key_value(target) { + return Some((stored, owner)); } - let profile = profiles.get(&name).ok_or_else(|| { - Error::InvalidInput(format!( - "environment recipe references unknown profile `{requested_name}`" - )) - })?; - if profile.version.is_empty() - || profile.version.len() > 128 - || profile.version.contains(char::is_whitespace) - || profile.version.contains('\0') - { - return Err(Error::InvalidInput(format!( - "environment recipe profile `{name}` has invalid version `{}`", - profile.version - ))); + let mut prefix = String::new(); + let mut segments = target.split('/').peekable(); + while let Some(segment) = segments.next() { + if !prefix.is_empty() { + prefix.push('/'); + } + prefix.push_str(segment); + if segments.peek().is_some() + && let Some((stored, owner)) = targets.get_key_value(&prefix) + { + return Some((stored, owner)); + } } + targets + .range(target.to_string()..) + .next() + .filter(|(stored, _)| stored.starts_with(&format!("{target}/"))) + .map(|(stored, owner)| (stored.as_str(), owner.as_str())) +} - stack.push(name.clone()); - let mut fragment = RecipeFragment::default(); - let mut versions = BTreeMap::new(); - for parent in &profile.extends { - let resolved = resolve_recipe_profile(parent, profiles, cache, stack)?; - apply_recipe_fragment(&mut fragment, &resolved.fragment); - versions.extend(resolved.versions); +fn display_recipe_root(root: &str) -> &str { + if root.is_empty() { + "." + } else { + root } - stack.pop(); - apply_recipe_fragment(&mut fragment, &recipe_profile_fragment(profile)); - versions.insert(name.clone(), profile.version.clone()); - let resolved = ResolvedRecipeProfile { fragment, versions }; - cache.insert(name, resolved.clone()); - Ok(resolved) } -fn resolve_recipe_component( - definition: RecipeComponentDefinition, - profiles: &BTreeMap, - cache: &mut BTreeMap, -) -> Result<(RecipeComponent, BTreeMap)> { - validate_recipe_component_identity(&definition.id)?; - let root = normalize_recipe_path_allow_root(&definition.root)?; - let mut fragment = RecipeFragment::default(); - let mut versions = BTreeMap::new(); - let mut stack = Vec::new(); - for profile_name in &definition.extends { - let resolved = resolve_recipe_profile(profile_name, profiles, cache, &mut stack)?; - apply_recipe_fragment(&mut fragment, &resolved.fragment); - versions.extend(resolved.versions); +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ordered_recipe_target_overlap_finds_ancestors_and_descendants() { + let descendants = BTreeMap::from([("generated/nested".to_string(), "child".to_string())]); + assert_eq!( + recipe_target_overlap(&descendants, "generated"), + Some(("generated/nested", "child")) + ); + let ancestors = BTreeMap::from([("generated".to_string(), "parent".to_string())]); + assert_eq!( + recipe_target_overlap(&ancestors, "generated/nested"), + Some(("generated", "parent")) + ); + assert!(recipe_target_overlap(&ancestors, "generated-sibling").is_none()); } - apply_recipe_fragment( - &mut fragment, - &RecipeFragment { - adapter: definition.adapter, - kind: definition.kind, - dependencies: definition.depends_on, - edges: definition.edges, - inputs: definition.inputs, - outputs: definition.outputs, - build: definition.build, - }, - ); - let adapter = fragment.adapter.ok_or_else(|| { - Error::InvalidInput(format!( - "command component `{}` has no adapter after profile expansion", - definition.id - )) - })?; - let mut build = fragment.build.ok_or_else(|| { - Error::InvalidInput(format!( - "command component `{}` has no build declaration after profile expansion", - definition.id - )) - })?; - let mut inputs = fragment.inputs; - let mut outputs = fragment.outputs; - let mut dependencies = fragment.dependencies; - let edges = fragment.edges; - let mut seen_dependencies = BTreeSet::new(); - dependencies.retain(|dependency| seen_dependencies.insert(dependency.clone())); - for dependency in &dependencies { - validate_recipe_component_identity(dependency)?; - if dependency == &definition.id { - return Err(Error::InvalidInput(format!( - "environment component `{}` cannot depend on itself", - definition.id - ))); - } + + fn write_recipe_workspace(workspace: &Path, command: &[&str]) { + write_recipe_workspace_with_policy(workspace, command, "immutable_seed_private"); } - let mut typed_edge_components = BTreeMap::new(); - for edge in edges { - if let Some(previous) = - typed_edge_components.insert(edge.component.clone(), edge.edge_type.clone()) - && previous != edge.edge_type - { - return Err(Error::InvalidInput(format!( - "environment component `{}` declares conflicting edge types `{previous}` and `{}` for `{}`", - definition.id, edge.edge_type, edge.component - ))); - } + + fn write_recipe_workspace_with_policy(workspace: &Path, command: &[&str], policy: &str) { + fs::write(workspace.join("input.txt"), "declared input\n").unwrap(); + let command = command + .iter() + .map(|value| format!("{:?}", value)) + .collect::>() + .join(", "); + fs::write( + workspace.join("trail.environment.toml"), + format!( + r#"schema = "trail.environment/v1" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "generated.copy" +adapter = "trail/command@1" +root = "." +kind = "generated" + +[[component.input]] +path = "*.txt" +role = "identity" +format = "bytes" + +[component.build] +command = [{command}] +cwd = "." +network = "deny" +scripts = "deny" + +[[component.output]] +name = "generated" +source = "generated" +target = ".trail-generated/copy" +policy = "{policy}" +portability = "host" +"# + ), + ) + .unwrap(); } - let edges = typed_edge_components - .into_iter() - .map(|(component, edge_type)| RecipeDependencyEdge { - component, - edge_type, - }) - .collect::>(); - for edge in &edges { - validate_recipe_component_identity(&edge.component)?; - WorkspaceEnvironmentEdgeType::parse(&edge.edge_type)?; - if edge.component == definition.id { - return Err(Error::InvalidInput(format!( - "environment component `{}` cannot depend on itself", - definition.id - ))); - } - if seen_dependencies.contains(&edge.component) { - return Err(Error::InvalidInput(format!( - "environment component `{}` declares both legacy depends_on and typed edge for `{}`", - definition.id, edge.component - ))); + + fn open_recipe_lane(command: &[&str]) -> (tempfile::TempDir, Trail) { + let workspace = tempfile::tempdir().unwrap(); + write_recipe_workspace(workspace.path(), command); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + for lane in ["recipe-a", "recipe-b"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + mode.clone(), + None, + None, + None, + &[], + false, + ) + .unwrap(); } + (workspace, db) } - for input in &mut inputs { - input.path = expand_recipe_root_template(&input.path, &root); - } - for output in &mut outputs { - output.source = expand_recipe_root_template(&output.source, &root); - output.target = expand_recipe_root_template(&output.target, &root); - } - for argument in &mut build.command { - *argument = expand_recipe_root_template(argument, &root); - } - if let Some(cwd) = &mut build.cwd { - *cwd = expand_recipe_root_template(cwd, &root); - } - for value in build.environment.values_mut() { - *value = expand_recipe_root_template(value, &root); - } - let mut seen_inputs = BTreeSet::new(); - inputs.retain(|input| { - seen_inputs.insert(( - input.path.clone(), - input.role.clone(), - input.format.clone(), - input.optional, - )) - }); - Ok(( - RecipeComponent { - id: definition.id, - adapter, - root, - kind: fragment.kind.unwrap_or_else(default_recipe_kind), - dependencies, - edges, - inputs, - outputs, - build, - }, - versions, - )) -} -fn expand_recipe_root_template(value: &str, root: &str) -> String { - if root.is_empty() { - value.replace("{root}/", "").replace("{root}", ".") - } else { - value.replace("{root}", root) + fn open_recipe_graph(specification: &str) -> (tempfile::TempDir, Trail) { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("input.txt"), "graph\n").unwrap(); + fs::write( + workspace.path().join("trail.environment.toml"), + specification, + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "graph", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + (workspace, db) } -} -fn validate_recipe_component_identity(id: &str) -> Result<()> { - if id.is_empty() - || id.len() > 256 - || !id - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_' | ':' | '/')) - || id.starts_with('/') - || id.ends_with('/') - || id - .split('/') - .any(|segment| segment.is_empty() || segment == "..") - { - return Err(Error::InvalidInput(format!( - "invalid command component id `{id}`" - ))); + #[test] + fn command_recipe_discovery_and_plan_are_side_effect_free_and_exact() { + let (_workspace, db) = open_recipe_lane(&["cp", "input.txt", "generated/copied.txt"]); + let discovery = db.discover_workspace_environment("recipe-a", None).unwrap(); + assert_eq!(discovery.components.len(), 1); + assert_eq!(discovery.components[0].component_id, "generated.copy"); + assert_eq!( + discovery.components[0].adapter_identity, + RECIPE_ADAPTER_IDENTITY + ); + let plan = db + .command_recipe_plan(&discovery.source_root, "generated.copy") + .unwrap(); + assert_eq!( + plan.sandbox_policy, + WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + ); + assert_eq!(plan.outputs[0].mount_path, ".trail-generated/copy"); + assert_eq!(plan.inputs.len(), 1); + assert_eq!(plan.inputs[0].source_path, "input.txt"); + let identity = super::workspace_environment::workspace_environment_identity_contract_v3( + &plan, + super::workspace_environment::workspace_environment_artifact_contract_digest(&plan) + .unwrap(), + ) + .unwrap(); + assert!(!identity.source_closure_complete); + assert!(!identity.portability_certified); + assert_eq!(identity.trust_scope, "repository"); + let report = db + .plan_workspace_environment("recipe-a", RECIPE_ADAPTER_IDENTITY, None) + .unwrap(); + assert_eq!(report.component_id, "generated.copy"); + assert_eq!(report.capabilities.network, "deny"); + assert_eq!(report.capabilities.shell, "deny"); + assert_eq!(report.capabilities.scripts, "deny"); + assert_eq!(report.capabilities.secrets, "deny"); + assert_eq!(report.capabilities.filesystem_read, vec!["input.txt"]); + assert_eq!( + report.capabilities.filesystem_write, + vec!["project/generated"] + ); + assert!(db.list_workspace_layers().unwrap().is_empty()); } - Ok(()) -} -fn validate_recipe_output_name(name: &str, component_id: &str) -> Result<()> { - if name.is_empty() - || name.len() > 128 - || !name - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_')) - { - return Err(Error::InvalidInput(format!( - "command component `{component_id}` has invalid output name `{name}`" - ))); - } - Ok(()) -} + #[test] + fn explicit_v2_schema_preserves_v1_command_recipe_planning() { + let workspace = tempfile::tempdir().unwrap(); + write_recipe_workspace( + workspace.path(), + &["cp", "input.txt", "generated/copied.txt"], + ); + let path = workspace.path().join("trail.environment.toml"); + let v1 = fs::read_to_string(&path).unwrap(); + fs::write(&path, v1.replace(RECIPE_SCHEMA_V1, RECIPE_SCHEMA_V2)).unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source_root = db.resolve_branch_ref("main").unwrap().root_id; -fn validate_recipe_pattern(pattern: &str) -> Result<()> { - if pattern.is_empty() - || pattern.starts_with('/') - || pattern.contains('\\') - || pattern.split('/').any(|segment| segment == "..") - { - return Err(Error::InvalidInput(format!( - "invalid repository-relative recipe input `{pattern}`" - ))); + let recipes = db.load_command_recipes(&source_root).unwrap(); + assert_eq!(recipes.len(), 1); + assert_eq!(recipes[0].component.adapter, RECIPE_ADAPTER_IDENTITY); + let plan = db + .command_recipe_plan(&source_root, "generated.copy") + .unwrap(); + assert_eq!( + plan.sandbox_policy, + WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + ); + assert_eq!(plan.outputs[0].mount_path, ".trail-generated/copy"); + assert!(db.list_workspace_layers().unwrap().is_empty()); } - normalize_relative_path(pattern).map(|_| ()) -} -fn contains_glob_meta(path: &str) -> bool { - path.bytes() - .any(|byte| matches!(byte, b'*' | b'?' | b'[' | b'{' | b'!')) -} + #[test] + fn v2_schema_parses_typed_pipeline_sections_and_heterogeneous_outputs() { + let specification = r#"schema = "trail.environment/v2" -pub(super) fn validate_recipe_environment( - name: &str, - value: &str, - component_id: &str, -) -> Result<()> { - let valid_name = !name.is_empty() - && name - .chars() - .all(|ch| ch == '_' || ch.is_ascii_alphanumeric()); - let upper = name.to_ascii_uppercase(); - let sensitive = [ - "TOKEN", - "SECRET", - "PASSWORD", - "PASSWD", - "CREDENTIAL", - "PRIVATE_KEY", - "AUTH", - ] - .iter() - .any(|needle| upper.contains(needle)); - if !valid_name - || sensitive - || matches!( - upper.as_str(), - "PATH" | "HOME" | "TMP" | "TMPDIR" | "TEMP" | "SHELL" | "DYLD_INSERT_LIBRARIES" - ) - || value.contains('\0') - || value.len() > 128 * 1024 - || contains_sensitive_text(value) - { - return Err(Error::InvalidInput(format!( - "command component `{component_id}` has forbidden environment entry `{name}`" - ))); +[environment] +default_network = "deny" +default_scripts = "deny" +missing_resolution = "explicit" + +[[component]] +id = "custom.pipeline" +adapter = "trail/command@1" +kind = "generated" +inputs = [{ path = "input.txt", role = "identity", format = "bytes" }] + +[component.build] +command = ["cp", "input.txt", "generated/result.txt"] +cwd = "." + +[component.resolve] +command = ["cp", "input.txt", "generated.lock"] +cwd = "." +network = { authorities = ["registry.example:443"] } +snapshot = "generated.lock" +format = "application/vnd.example.lock+json" + +[component.resolve.capabilities] +network = "exact_authorities" +filesystem_write = "isolated_candidate" +process = "declared_executable" +child_processes = 4 +secrets = "opaque_handles" +publication = "deny" + +[[component.action]] +name = "construct" +phase = "construct" +command = ["cp", "input.txt", "generated/result.txt"] +cwd = "." +network = "deny" + +[[component.action]] +name = "load-check" +phase = "validate" +command = ["cp", "generated/result.txt", "generated/checked.txt"] + +[[component.validation]] +name = "path-contract" +kind = "path_contract" +path = "generated" +required = true +parameters = { maximum_entries = "1000" } + +[component.capabilities] +network = "deny" +filesystem_read = "declared_inputs" +filesystem_write = "isolated_candidate" +process = "declared_executable" +child_processes = 1 +secrets = "deny" +publication = "deny" + +[[component.output]] +name = "seed" +source = "generated" +target = ".trail-generated/seed" +policy = "immutable_seed_private" +reuse = "exact" +scope = "workspace" +publish = "on_sync" + +[[component.output]] +name = "scratch" +source = "scratch" +target = ".trail-generated/scratch" +policy = "disposable" +reuse = "none" +scope = "lane" +publish = "never" + +[[component.source_export]] +from_output = "seed" +source = "generated-client" +target = "src/generated-client" +mode = "explicit" +collision = "fail" +validation = "path-contract" +"#; + let (_workspace, db) = open_recipe_graph(specification); + let source_root = db.resolve_branch_ref("main").unwrap().root_id; + let recipes = db.load_command_recipes(&source_root).unwrap(); + let component = &recipes[0].component; + + assert_eq!( + component.resolution.as_ref().unwrap().snapshot, + "generated.lock" + ); + assert_eq!(component.actions.len(), 2); + assert_eq!(component.actions[0].phase, RecipeActionPhase::Construct); + assert_eq!(component.actions[1].phase, RecipeActionPhase::Validate); + assert_eq!(component.validations.len(), 1); + assert_eq!(component.outputs.len(), 2); + assert_eq!( + component.outputs[0].policy, + EnvironmentOutputPolicy::ImmutableSeedPrivate + ); + assert_eq!( + component.outputs[1].policy, + EnvironmentOutputPolicy::Disposable + ); + assert_eq!( + component.capabilities.as_ref().unwrap().child_processes, + Some(1) + ); + assert_eq!(component.source_exports.len(), 1); + assert_eq!(component.source_exports[0].target, "src/generated-client"); + + let compiled = db + .compile_repository_artifact_pipeline_v2(&source_root, "custom.pipeline") + .unwrap(); + assert_eq!( + compiled.proposal.status, + EnvironmentComponentProposalStatus::Resolvable + ); + assert_eq!(compiled.graph_plan.component_id, "custom.pipeline"); + assert_eq!( + compiled + .resolution_plan + .as_ref() + .unwrap() + .allowed_authorities, + vec!["registry.example:443"] + ); + assert_eq!(compiled.desired_material.actions.len(), 4); + assert_eq!(compiled.outputs.len(), 2); + assert_eq!(compiled.validations.len(), 1); + assert_eq!(compiled.source_exports.len(), 1); + assert_eq!( + serde_json::to_value(&compiled.source_exports[0]).unwrap(), + serde_json::json!({ + "name": "seed", + "output_name": "seed", + "artifact_subpath": "generated-client", + "destination": "src/generated-client", + "collision_policy": "fail", + "required_validation": "path-contract", + "authorization_mode": "explicit" + }) + ); + assert_eq!( + compiled.desired_key, + super::super::workspace_artifact::artifact_desired_key_v2( + compiled.desired_material.clone() + ) + .unwrap() + ); + assert_eq!( + compiled.desired_material.adapter_protocol, + RECIPE_SCHEMA_V2, + "repository v2 retains its explicit desired-key protocol instead of being relabeled as plugin v3" + ); } - Ok(()) -} -pub(super) fn validate_recipe_tool_path(db: &Trail, path: &Path, component_id: &str) -> Result<()> { - let canonical = fs::canonicalize(path)?; - let mut forbidden = vec![db.workspace_root.clone(), db.db_dir.clone()]; - if let Some(home) = std::env::var_os("HOME") { - forbidden.push(PathBuf::from(home)); - } - if forbidden.iter().any(|root| canonical.starts_with(root)) { - return Err(Error::InvalidInput(format!( - "command component `{component_id}` executable `{}` is under a mutable workspace or user home; bind a host-managed toolchain instead", - canonical.display() - ))); - } - Ok(()) -} + #[test] + fn v2_schema_rejects_non_explicit_missing_resolution_policy() { + let specification = r#"schema = "trail.environment/v2" -pub(super) fn is_shell_program(program: &str) -> bool { - matches!( - program.to_ascii_lowercase().as_str(), - "sh" | "bash" - | "zsh" - | "fish" - | "dash" - | "ksh" - | "csh" - | "tcsh" - | "cmd" - | "cmd.exe" - | "powershell" - | "powershell.exe" - | "pwsh" - ) -} +[environment] +default_network = "deny" +default_scripts = "deny" +missing_resolution = "automatic" -fn normalize_recipe_path_allow_root(path: &str) -> Result { - if path.trim_matches('/').is_empty() || path == "." { - Ok(String::new()) - } else { - normalize_relative_path(path) - } -} +[[component]] +id = "custom.pipeline" +adapter = "trail/command@1" +kind = "generated" +inputs = [{ path = "input.txt", role = "identity", format = "bytes" }] -fn join_recipe_path(root: &str, child: &str) -> String { - if root.is_empty() { - child.to_string() - } else { - format!("{root}/{child}") - } -} +[component.build] +command = ["cp", "input.txt", "generated/result.txt"] +cwd = "." -fn recipe_paths_overlap(left: &str, right: &str) -> bool { - left == right - || left.starts_with(&format!("{right}/")) - || right.starts_with(&format!("{left}/")) -} +[[component.output]] +name = "seed" +source = "generated" +target = ".trail-generated/seed" +policy = "immutable_shared" +reuse = "exact" +scope = "workspace" +publish = "on_sync" +"#; + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("input.txt"), "input\n").unwrap(); + fs::write( + workspace.path().join("trail.environment.toml"), + specification, + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source_root = db.resolve_branch_ref("main").unwrap().root_id; -fn recipe_target_overlap<'a>( - targets: &'a BTreeMap, - target: &str, -) -> Option<(&'a str, &'a str)> { - if let Some((stored, owner)) = targets.get_key_value(target) { - return Some((stored, owner)); + let error = db.load_command_recipes(&source_root).unwrap_err(); + assert_eq!( + error.to_string(), + "invalid input: environment specification `trail.environment.toml` missing_resolution must be `explicit`" + ); } - let mut prefix = String::new(); - let mut segments = target.split('/').peekable(); - while let Some(segment) = segments.next() { - if !prefix.is_empty() { - prefix.push('/'); - } - prefix.push_str(segment); - if segments.peek().is_some() - && let Some((stored, owner)) = targets.get_key_value(&prefix) + + #[cfg(target_os = "macos")] + #[test] + fn next_and_vite_v2_components_compose_over_node_with_private_framework_state() { + if !Command::new("npm") + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) + || !Command::new("node") + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) { - return Some((stored, owner)); + return; + } + let workspace = tempfile::tempdir().unwrap(); + fs::write( + workspace.path().join("package.json"), + r#"{"name":"framework-composition","version":"1.0.0","private":true}"#, + ) + .unwrap(); + fs::write( + workspace.path().join("package-lock.json"), + r#"{"name":"framework-composition","version":"1.0.0","lockfileVersion":3,"requires":true,"packages":{"":{"name":"framework-composition","version":"1.0.0"}}}"#, + ) + .unwrap(); + fs::write(workspace.path().join("next-source.js"), "next fixture\n").unwrap(); + fs::write(workspace.path().join("vite-source.js"), "vite fixture\n").unwrap(); + fs::write( + workspace.path().join("trail.environment.toml"), + r#"schema = "trail.environment/v2" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "web.next-build" +adapter = "trail/command@1" +kind = "generated" +depends_on = ["node"] +inputs = [{ path = "next-source.js", role = "identity", format = "bytes" }] +outputs = [{ name = "next-state", source = "next-output", target = ".next", policy = "writable_private", reuse = "none", scope = "lane", publish = "manual", portability = "host" }] +[component.build] +command = ["cp", "next-source.js", "next-output/server.js"] +cwd = "." +network = "deny" +scripts = "deny" + +[[component]] +id = "web.vite-build" +adapter = "trail/command@1" +kind = "generated" +depends_on = ["node"] +inputs = [{ path = "vite-source.js", role = "identity", format = "bytes" }] +outputs = [ + { name = "dist", source = "dist", target = "dist", policy = "immutable_shared", reuse = "exact", scope = "workspace", publish = "on_sync", portability = "host" } +] +[component.build] +command = ["cp", "vite-source.js", "dist/app.js"] +cwd = "." +network = "deny" +scripts = "deny" + +[[component.validation]] +name = "dist-path-contract" +kind = "path_contract" +path = "dist" +required = true +parameters = { maximum_entries = "1000" } + +[[component]] +id = "web.vite-cache" +adapter = "trail/command@1" +kind = "generated" +depends_on = ["node"] +inputs = [{ path = "vite-source.js", role = "identity", format = "bytes" }] +outputs = [{ name = "vite-cache", source = "vite-cache", target = ".vite", policy = "writable_private", reuse = "none", scope = "lane", publish = "manual", portability = "host" }] +[component.build] +command = ["cp", "vite-source.js", "vite-cache/metadata.json"] +cwd = "." +network = "deny" +scripts = "deny" +"#, + ) + .unwrap(); + + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + for lane in ["framework-one", "framework-two"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + mode.clone(), + None, + None, + None, + &[], + false, + ) + .unwrap(); } + + let graph = db + .workspace_environment_graph("framework-one", None) + .unwrap(); + assert_eq!(graph.nodes.len(), 4); + assert_eq!(graph.edges.len(), 3); + assert!(graph.edges.iter().all(|edge| { + edge.source_component_id == "node" && edge.edge_type == "build_requires" + })); + + let first = db + .sync_all_workspace_environments("framework-one", None) + .unwrap(); + let second = db + .sync_all_workspace_environments("framework-two", None) + .unwrap(); + assert_eq!(first.generation.components.len(), 4); + assert_eq!(second.generation.components.len(), 4); + let next = first + .generation + .components + .iter() + .find(|component| component.component_id == "web.next-build") + .unwrap(); + assert_eq!( + next.outputs[0].policy, + EnvironmentOutputPolicy::WritablePrivate + ); + assert!(next.outputs[0].layer_id.is_none()); + let vite = first + .generation + .components + .iter() + .find(|component| component.component_id == "web.vite-build") + .unwrap(); + assert_eq!(vite.outputs.len(), 1); + assert_eq!( + vite.outputs[0].policy, + EnvironmentOutputPolicy::ImmutableShared + ); + assert!(vite.outputs[0].layer_id.is_some()); + let vite_cache = first + .generation + .components + .iter() + .find(|component| component.component_id == "web.vite-cache") + .unwrap(); + assert_eq!( + vite_cache.outputs[0].policy, + EnvironmentOutputPolicy::WritablePrivate + ); + assert!(vite_cache.outputs[0].layer_id.is_none()); + + let first_view = db.lane_workspace_view("framework-one").unwrap().unwrap(); + let second_view = db.lane_workspace_view("framework-two").unwrap().unwrap(); + let first_generated = Path::new(&first_view.generated_upper); + let second_generated = Path::new(&second_view.generated_upper); + fs::write(first_generated.join(".next/lane.txt"), "one\n").unwrap(); + fs::write(second_generated.join(".next/lane.txt"), "two\n").unwrap(); + fs::write(first_generated.join(".vite/cache.txt"), "one\n").unwrap(); + fs::write(second_generated.join(".vite/cache.txt"), "two\n").unwrap(); + assert_eq!( + fs::read_to_string(first_generated.join(".next/lane.txt")).unwrap(), + "one\n" + ); + assert_eq!( + fs::read_to_string(second_generated.join(".next/lane.txt")).unwrap(), + "two\n" + ); + assert_eq!( + fs::read_to_string(first_generated.join(".vite/cache.txt")).unwrap(), + "one\n" + ); + assert_eq!( + fs::read_to_string(second_generated.join(".vite/cache.txt")).unwrap(), + "two\n" + ); } - targets - .range(target.to_string()..) - .next() - .filter(|(stored, _)| stored.starts_with(&format!("{target}/"))) - .map(|(stored, owner)| (stored.as_str(), owner.as_str())) -} -fn display_recipe_root(root: &str) -> &str { - if root.is_empty() { - "." - } else { - root + #[test] + fn v2_pipeline_sections_reject_unknown_fields_and_v1_cannot_opt_in_implicitly() { + for (schema, extra, expected) in [ + ( + RECIPE_SCHEMA_V2, + "[[component.action]]\nphase = \"construct\"\ncommand = [\"tool\"]\nshell = true\n", + "unknown field", + ), + ( + RECIPE_SCHEMA_V1, + "[[component.action]]\nphase = \"construct\"\ncommand = [\"tool\"]\n", + RECIPE_SCHEMA_V2, + ), + ] { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("input.txt"), "strict\n").unwrap(); + fs::write( + workspace.path().join("trail.environment.toml"), + format!( + "schema = {schema:?}\n[[component]]\nid = \"strict\"\nadapter = \"trail/command@1\"\ninputs = [{{ path = \"input.txt\" }}]\noutputs = [{{ source = \"generated\", target = \"generated\" }}]\n[component.build]\ncommand = [\"tool\"]\n{extra}" + ), + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source_root = db.resolve_branch_ref("main").unwrap().root_id; + let error = db.load_command_recipes(&source_root).unwrap_err(); + assert!( + error.to_string().contains(expected), + "unexpected strict-v2 parser error: {error}" + ); + } } -} -#[cfg(test)] -mod tests { - use super::*; + #[test] + fn v2_repository_pipeline_rejects_unsafe_authority_and_reuse_requests() { + struct Case { + name: &'static str, + command: &'static [&'static str], + before_output: &'static str, + reuse: &'static str, + scope: &'static str, + after_output: &'static str, + expected: &'static str, + } + + let cases = [ + Case { + name: "shell interpolation", + command: &["cp", "$(read-secret)", "generated/result.txt"], + before_output: "", + reuse: "exact", + scope: "workspace", + after_output: "", + expected: "unsafe or excessive argv", + }, + Case { + name: "shell control flow", + command: &["cp", "input.txt", "&&", "generated/result.txt"], + before_output: "", + reuse: "exact", + scope: "workspace", + after_output: "", + expected: "unsafe or excessive argv", + }, + Case { + name: "absolute host path", + command: &["cp", "/etc/passwd", "generated/result.txt"], + before_output: "", + reuse: "exact", + scope: "workspace", + after_output: "", + expected: "unsafe or excessive argv", + }, + Case { + name: "indirect child launcher", + command: &["env", "cp", "input.txt", "generated/result.txt"], + before_output: "", + reuse: "exact", + scope: "workspace", + after_output: "", + expected: "non-shell, non-launcher executable", + }, + Case { + name: "raw secret environment", + command: &["cp", "input.txt", "generated/result.txt"], + before_output: "[component.build.environment]\nAPI_TOKEN = \"sk-live-secret\"\n", + reuse: "exact", + scope: "workspace", + after_output: "", + expected: "forbidden environment entry", + }, + Case { + name: "provider socket", + command: &["cp", "input.txt", "generated/result.txt"], + before_output: "[component.build.environment]\nDOCKER_HOST = \"unix:///var/run/docker.sock\"\n", + reuse: "exact", + scope: "workspace", + after_output: "", + expected: "requests a provider socket", + }, + Case { + name: "forbidden process graph", + command: &["cp", "input.txt", "generated/result.txt"], + before_output: "[component.capabilities]\nprocess = \"reviewed_builtin_graph\"\n", + reuse: "exact", + scope: "workspace", + after_output: "", + expected: "exceeds the repository-declaration ceiling", + }, + Case { + name: "secret-tainted constructor", + command: &["cp", "input.txt", "generated/result.txt"], + before_output: "[component.capabilities]\nsecrets = \"opaque_handles\"\n", + reuse: "exact", + scope: "workspace", + after_output: "", + expected: "exceeds the repository-declaration ceiling", + }, + Case { + name: "excessive child processes", + command: &["cp", "input.txt", "generated/result.txt"], + before_output: "[component.capabilities]\nchild_processes = 257\n", + reuse: "exact", + scope: "workspace", + after_output: "", + expected: "child-process limit", + }, + Case { + name: "host-wide reuse", + command: &["cp", "input.txt", "generated/result.txt"], + before_output: "", + reuse: "exact", + scope: "host", + after_output: "", + expected: "host-wide artifact reuse", + }, + Case { + name: "compatible reuse", + command: &["cp", "input.txt", "generated/result.txt"], + before_output: "", + reuse: "compatible", + scope: "workspace", + after_output: "", + expected: "compatible or host-wide artifact reuse", + }, + Case { + name: "mounted repository action", + command: &["cp", "input.txt", "generated/result.txt"], + before_output: "", + reuse: "exact", + scope: "workspace", + after_output: "[[component.action]]\nphase = \"mounted_execution\"\ncommand = [\"cp\", \"input.txt\", \"generated/mounted.txt\"]\n", + expected: "requests forbidden phase", + }, + Case { + name: "online constructor", + command: &["cp", "input.txt", "generated/result.txt"], + before_output: "", + reuse: "exact", + scope: "workspace", + after_output: "[[component.action]]\nphase = \"construct\"\ncommand = [\"cp\", \"input.txt\", \"generated/online.txt\"]\nnetwork = { authorities = [\"registry.example:443\"] }\n", + expected: "must be offline", + }, + ]; + + for case in cases { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("input.txt"), "strict\n").unwrap(); + let command = serde_json::to_string(case.command).unwrap(); + fs::write( + workspace.path().join("trail.environment.toml"), + format!( + r#"schema = "trail.environment/v2" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "unsafe.pipeline" +adapter = "trail/command@1" +inputs = [{{ path = "input.txt" }}] + +[component.build] +command = {command} +cwd = "." +{before_output} +[[component.output]] +name = "generated" +source = "generated" +target = ".trail-generated/unsafe" +policy = "immutable_seed_private" +reuse = "{reuse}" +scope = "{scope}" +publish = "on_sync" +{after_output}"#, + before_output = case.before_output, + reuse = case.reuse, + scope = case.scope, + after_output = case.after_output, + ), + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source_root = db.resolve_branch_ref("main").unwrap().root_id; + let error = db.load_command_recipes(&source_root).unwrap_err(); + assert!( + error.to_string().contains(case.expected), + "{} produced unexpected error: {error}", + case.name + ); + } + } #[test] - fn ordered_recipe_target_overlap_finds_ancestors_and_descendants() { - let descendants = BTreeMap::from([("generated/nested".to_string(), "child".to_string())]); - assert_eq!( - recipe_target_overlap(&descendants, "generated"), - Some(("generated/nested", "child")) - ); - let ancestors = BTreeMap::from([("generated".to_string(), "parent".to_string())]); + fn v2_repository_authorities_are_bounded_sorted_and_deduplicated() { + let network = RecipeNetwork::Authorities(RecipeNetworkAuthorities { + authorities: vec![ + "registry.z.example:443".into(), + "registry.a.example:443".into(), + "registry.z.example:443".into(), + ], + }); assert_eq!( - recipe_target_overlap(&ancestors, "generated/nested"), - Some(("generated", "parent")) + recipe_network_authorities(Some(&network)).unwrap(), + vec!["registry.a.example:443", "registry.z.example:443"] ); - assert!(recipe_target_overlap(&ancestors, "generated-sibling").is_none()); - } - fn write_recipe_workspace(workspace: &Path, command: &[&str]) { - write_recipe_workspace_with_policy(workspace, command, "immutable_seed_private"); + let excessive = RecipeNetwork::Authorities(RecipeNetworkAuthorities { + authorities: (0..=MAX_RECIPE_NETWORK_AUTHORITIES) + .map(|index| format!("registry-{index}.example:443")) + .collect(), + }); + let error = recipe_network_authorities(Some(&excessive)).unwrap_err(); + assert!(error.to_string().contains("more than")); } - fn write_recipe_workspace_with_policy(workspace: &Path, command: &[&str], policy: &str) { - fs::write(workspace.join("input.txt"), "declared input\n").unwrap(); - let command = command - .iter() - .map(|value| format!("{:?}", value)) - .collect::>() - .join(", "); + #[test] + fn v2_repository_input_declarations_are_bounded_and_expansion_is_sorted() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("z.txt"), "z\n").unwrap(); + fs::write(workspace.path().join("a.txt"), "a\n").unwrap(); fs::write( - workspace.join("trail.environment.toml"), - format!( - r#"schema = "trail.environment/v1" - -[environment] -default_network = "deny" -default_scripts = "deny" + workspace.path().join("trail.environment.toml"), + r#"schema = "trail.environment/v2" [[component]] -id = "generated.copy" +id = "sorted.inputs" adapter = "trail/command@1" -root = "." -kind = "generated" - -[[component.input]] -path = "*.txt" -role = "identity" -format = "bytes" +inputs = [{ path = "*.txt" }] [component.build] -command = [{command}] -cwd = "." -network = "deny" -scripts = "deny" +command = ["cp", "a.txt", "generated/result.txt"] [[component.output]] name = "generated" source = "generated" -target = ".trail-generated/copy" -policy = "{policy}" -portability = "host" +target = ".trail-generated/sorted" +policy = "immutable_seed_private" +reuse = "exact" +scope = "workspace" +publish = "on_sync" +"#, + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source_root = db.resolve_branch_ref("main").unwrap().root_id; + let recipes = db.load_command_recipes(&source_root).unwrap(); + let selected = db + .expand_recipe_inputs(&source_root, &recipes[0].component) + .unwrap(); + assert_eq!( + selected.keys().cloned().collect::>(), + vec!["a.txt", "z.txt"] + ); + + let excessive_workspace = tempfile::tempdir().unwrap(); + let inputs = (0..=MAX_RECIPE_INPUT_DECLARATIONS) + .map(|index| format!("{{ path = \"missing/{index}.txt\", optional = true }}")) + .collect::>() + .join(","); + fs::write( + excessive_workspace.path().join("trail.environment.toml"), + format!( + r#"schema = "trail.environment/v2" +[[component]] +id = "excessive.inputs" +adapter = "trail/command@1" +inputs = [{inputs}] +[component.build] +command = ["cp", "input.txt", "generated/result.txt"] +[[component.output]] +source = "generated" +target = ".trail-generated/excessive" +policy = "immutable_seed_private" +reuse = "exact" +scope = "workspace" +publish = "on_sync" "# ), ) .unwrap(); + Trail::init( + excessive_workspace.path(), + "main", + InitImportMode::WorkingTree, + false, + ) + .unwrap(); + let excessive_db = Trail::open(excessive_workspace.path()).unwrap(); + let excessive_root = excessive_db.resolve_branch_ref("main").unwrap().root_id; + let error = excessive_db + .load_command_recipes(&excessive_root) + .unwrap_err(); + assert!(error.to_string().contains("declares more than")); } - fn open_recipe_lane(command: &[&str]) -> (tempfile::TempDir, Trail) { + #[test] + fn repository_document_graph_rejects_mixed_schema_versions() { let workspace = tempfile::tempdir().unwrap(); - write_recipe_workspace(workspace.path(), command); + fs::create_dir_all(workspace.path().join("config")).unwrap(); + fs::write( + workspace.path().join("trail.environment.toml"), + format!("schema = {RECIPE_SCHEMA_V2:?}\ninclude = [\"config/profile.toml\"]\n"), + ) + .unwrap(); + fs::write( + workspace.path().join("config/profile.toml"), + format!("schema = {RECIPE_SCHEMA_V1:?}\n"), + ) + .unwrap(); Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); - let mut db = Trail::open(workspace.path()).unwrap(); - let mode = if cfg!(target_os = "macos") { - LaneWorkdirMode::NfsCow - } else if cfg!(target_os = "windows") { - LaneWorkdirMode::DokanCow - } else { - LaneWorkdirMode::FuseCow - }; - for lane in ["recipe-a", "recipe-b"] { - db.spawn_lane_with_workdir_mode_paths_and_neighbors( - lane, - Some("main"), - mode.clone(), - None, - None, - None, - &[], - false, - ) - .unwrap(); - } - (workspace, db) + let db = Trail::open(workspace.path()).unwrap(); + let source_root = db.resolve_branch_ref("main").unwrap().root_id; + + let error = db.load_command_recipes(&source_root).unwrap_err(); + assert!(error.to_string().contains("config/profile.toml")); + assert!(error.to_string().contains(RECIPE_SCHEMA_V1)); + assert!(error.to_string().contains(RECIPE_SCHEMA_V2)); } - fn open_recipe_graph(specification: &str) -> (tempfile::TempDir, Trail) { + #[test] + fn repository_document_rejects_unsupported_schema_with_supported_versions() { let workspace = tempfile::tempdir().unwrap(); - fs::write(workspace.path().join("input.txt"), "graph\n").unwrap(); fs::write( workspace.path().join("trail.environment.toml"), - specification, + "schema = \"trail.environment/v3\"\n", ) .unwrap(); Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); - let mut db = Trail::open(workspace.path()).unwrap(); - db.spawn_lane_with_workdir_mode_paths_and_neighbors( - "graph", - Some("main"), - if cfg!(target_os = "macos") { - LaneWorkdirMode::NfsCow - } else if cfg!(target_os = "windows") { - LaneWorkdirMode::DokanCow - } else { - LaneWorkdirMode::FuseCow - }, - None, - None, - None, - &[], - false, - ) - .unwrap(); - (workspace, db) + let db = Trail::open(workspace.path()).unwrap(); + let source_root = db.resolve_branch_ref("main").unwrap().root_id; + + let error = db.load_command_recipes(&source_root).unwrap_err(); + assert!(error.to_string().contains(RECIPE_SCHEMA_V1)); + assert!(error.to_string().contains(RECIPE_SCHEMA_V2)); } #[test] - fn command_recipe_discovery_and_plan_are_side_effect_free_and_exact() { - let (_workspace, db) = open_recipe_lane(&["cp", "input.txt", "generated/copied.txt"]); + fn command_recipe_discovery_does_not_require_or_execute_declared_tool() { + let (_workspace, db) = open_recipe_lane(&[ + "trail-fixture-tool-that-does-not-exist", + "input.txt", + "generated/copied.txt", + ]); + let discovery = db.discover_workspace_environment("recipe-a", None).unwrap(); assert_eq!(discovery.components.len(), 1); assert_eq!(discovery.components[0].component_id, "generated.copy"); assert_eq!( - discovery.components[0].adapter_identity, - RECIPE_ADAPTER_IDENTITY - ); - let plan = db - .command_recipe_plan(&discovery.source_root, "generated.copy") - .unwrap(); - assert_eq!( - plan.sandbox_policy, - WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe - ); - assert_eq!(plan.outputs[0].mount_path, ".trail-generated/copy"); - assert_eq!(plan.inputs.len(), 1); - assert_eq!(plan.inputs[0].source_path, "input.txt"); - let report = db - .plan_workspace_environment("recipe-a", RECIPE_ADAPTER_IDENTITY, None) - .unwrap(); - assert_eq!(report.component_id, "generated.copy"); - assert_eq!(report.capabilities.network, "deny"); - assert_eq!(report.capabilities.shell, "deny"); - assert_eq!(report.capabilities.scripts, "deny"); - assert_eq!(report.capabilities.secrets, "deny"); - assert_eq!(report.capabilities.filesystem_read, vec!["input.txt"]); - assert_eq!( - report.capabilities.filesystem_write, - vec!["project/generated"] + discovery.components[0].status, + EnvironmentComponentProposalStatus::Ready ); assert!(db.list_workspace_layers().unwrap().is_empty()); + + let error = db + .command_recipe_plan(&discovery.source_root, "generated.copy") + .unwrap_err(); + assert!(error + .to_string() + .contains("trail-fixture-tool-that-does-not-exist")); } #[test] @@ -1657,6 +3904,121 @@ scripts = "deny" assert!(db.list_workspace_layers().unwrap().is_empty()); } + #[test] + fn maven_gradle_like_and_unknown_custom_shapes_use_repository_v2_components() { + let specification = r#"schema = "trail.environment/v2" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "jvm.dependencies" +adapter = "trail/command@1" +kind = "generated" +inputs = [{ path = "input.txt", role = "identity", format = "bytes" }] +outputs = [{ name = "dependencies", source = "dependencies", target = ".trail-generated/jvm-dependencies", policy = "immutable_seed_private", reuse = "exact", scope = "workspace", publish = "on_sync", portability = "host" }] +[component.build] +command = ["cp", "input.txt", "dependencies/checksums.lock"] +cwd = "." +network = "deny" +scripts = "deny" + +[[component.validation]] +name = "dependency-checksum-graph" +kind = "path_contract" +path = "dependencies" +required = true +parameters = { maximum_entries = "10000" } + +[[component]] +id = "jvm.private-build-state" +adapter = "trail/command@1" +kind = "generated" +depends_on = ["jvm.dependencies"] +inputs = [{ path = "input.txt", role = "identity", format = "bytes" }] +outputs = [{ name = "build-state", source = "build-state", target = ".trail-generated/jvm-build", policy = "writable_private", reuse = "none", scope = "lane", publish = "manual", portability = "host" }] +[component.build] +command = ["cp", "input.txt", "build-state/task-state.bin"] +cwd = "." +network = "deny" +scripts = "deny" + +[[component]] +id = "custom.codegen" +adapter = "trail/command@1" +kind = "generated" +inputs = [{ path = "input.txt", role = "identity", format = "bytes" }] +outputs = [{ name = "generated-api", source = "generated-api", target = ".trail-generated/custom-api", policy = "immutable_seed_private", reuse = "exact", scope = "workspace", publish = "on_sync", portability = "host" }] +[component.build] +command = ["cp", "input.txt", "generated-api/client.txt"] +cwd = "." +network = "deny" +scripts = "deny" + +[[component.validation]] +name = "generated-api-contract" +kind = "path_contract" +path = "generated-api" +required = true + +[[component.source_export]] +from_output = "generated-api" +source = "client.txt" +target = "src/generated/client.txt" +mode = "explicit" +collision = "fail" +validation = "generated-api-contract" +"#; + let (_workspace, db) = open_recipe_graph(specification); + let source_root = db.resolve_branch_ref("main").unwrap().root_id; + + let dependencies = db + .compile_repository_artifact_pipeline_v2(&source_root, "jvm.dependencies") + .unwrap(); + let build_state = db + .compile_repository_artifact_pipeline_v2(&source_root, "jvm.private-build-state") + .unwrap(); + let custom = db + .compile_repository_artifact_pipeline_v2(&source_root, "custom.codegen") + .unwrap(); + + for compiled in [&dependencies, &build_state, &custom] { + assert_eq!( + compiled.graph_plan.adapter_identity, + RECIPE_ADAPTER_IDENTITY + ); + assert_eq!(compiled.desired_material.adapter_protocol, RECIPE_SCHEMA_V2); + assert_eq!(compiled.desired_material.trust_scope, "repository"); + assert_eq!(compiled.desired_material.network_policy, "deny"); + } + assert_eq!( + dependencies.outputs[0].policy, + EnvironmentOutputPolicy::ImmutableSeedPrivate + ); + assert_eq!( + dependencies.validations[0].name, + "dependency-checksum-graph" + ); + assert_eq!( + build_state.outputs[0].policy, + EnvironmentOutputPolicy::WritablePrivate + ); + assert_eq!(build_state.outputs[0].reuse, EnvironmentReuseMode::None); + assert_eq!( + build_state.graph_plan.dependencies, + [WorkspaceEnvironmentDependency::build_requires( + "jvm.dependencies" + )] + ); + assert_eq!(custom.source_exports.len(), 1); + assert_eq!( + custom.source_exports[0].destination, + "src/generated/client.txt" + ); + assert!(db.list_workspace_layers().unwrap().is_empty()); + } + #[test] fn component_dependencies_finalize_in_topological_order_and_fail_closed() { let chain = r#"schema = "trail.environment/v1" @@ -2746,6 +5108,22 @@ portability = "host" assert!(db.list_workspace_layers().unwrap().is_empty()); } + #[cfg(target_os = "macos")] + #[test] + fn restricted_command_recipe_denies_child_process_execution() { + let (_workspace, db) = + open_recipe_lane(&["env", "cp", "input.txt", "generated/copied-by-child.txt"]); + let error = db + .sync_workspace_environment("recipe-a", RECIPE_ADAPTER_IDENTITY, None) + .unwrap_err(); + assert!(error.to_string().contains("failed with")); + assert!(db.list_workspace_layers().unwrap().is_empty()); + assert!(db + .active_environment_generation("recipe-a") + .unwrap() + .is_none()); + } + #[cfg(target_os = "macos")] #[test] fn restricted_command_recipe_never_publishes_an_escaping_symlink() { diff --git a/trail/src/db/lane/workspace_view.rs b/trail/src/db/lane/workspace_view.rs index a5817d0c..e14c0d63 100644 --- a/trail/src/db/lane/workspace_view.rs +++ b/trail/src/db/lane/workspace_view.rs @@ -1193,6 +1193,25 @@ impl Trail { .physical_bytes .saturating_add(generated.physical_bytes) .saturating_add(scratch.physical_bytes); + let reclaimable_cache_bytes = self.workspace_reclaimable_cache_bytes()?; + let unknown_layer_bytes = self.conn.query_row( + "SELECT COALESCE(SUM(COALESCE(w.physical_bytes,0)),0) + FROM workspace_view_layers l + JOIN workspace_layers w ON w.layer_id=l.layer_id + LEFT JOIN workspace_layer_artifact_shadows s ON s.layer_id=w.layer_id + WHERE l.view_id=?1 AND s.layer_id IS NULL", + params![&view.view_id], + |row| row.get::<_, i64>(0), + )?; + let unknown_layer_bytes = u64::try_from(unknown_layer_bytes) + .map_err(|_| Error::Corrupt("workspace layer has negative physical bytes".into()))?; + let artifact_storage = self.artifact_storage_accounting( + Some(&view.view_id), + lane_exclusive_physical_bytes, + 0, + self.workspace_view_reclaimable_artifact_bytes(&view.view_id)?, + unknown_layer_bytes, + )?; Ok(WorkspaceSpaceReport { view_id: view.view_id, logical_visible_bytes: root @@ -1203,7 +1222,7 @@ impl Trail { shared_physical_bytes: blobs.physical_bytes.saturating_add(layers.physical_bytes), lane_exclusive_physical_bytes, shared_extent_bytes: None, - reclaimable_cache_bytes: self.workspace_reclaimable_cache_bytes()?, + reclaimable_cache_bytes, uncheckpointed_source_bytes: source.logical_bytes, generated_upper_bytes: generated.logical_bytes, scratch_upper_bytes: scratch.logical_bytes, @@ -1224,6 +1243,7 @@ impl Trail { physical_sharing: PhysicalSharing::Unknown, physical_sharing_evidence: "layered_workspace_uses_separate_shared_and_exclusive_accounting".to_string(), + artifact_storage, }) } @@ -1233,6 +1253,7 @@ impl Trail { let usage = materialized_workdir_usage(&context.workdir)?; let changed_since_baseline_bytes = self.native_cow_changed_since_baseline_bytes(&branch, &context.workdir)?; + let reclaimable_cache_bytes = self.workspace_reclaimable_cache_bytes()?; Ok(WorkspaceSpaceReport { view_id: context.initialization_id, logical_visible_bytes: usage.logical_bytes, @@ -1242,7 +1263,7 @@ impl Trail { // from both ownership projections. lane_exclusive_physical_bytes: 0, shared_extent_bytes: None, - reclaimable_cache_bytes: self.workspace_reclaimable_cache_bytes()?, + reclaimable_cache_bytes, uncheckpointed_source_bytes: changed_since_baseline_bytes, generated_upper_bytes: 0, scratch_upper_bytes: 0, @@ -1259,6 +1280,15 @@ impl Trail { physical_sharing: PhysicalSharing::Unknown, physical_sharing_evidence: "allocated_blocks_do_not_prove_apfs_extent_sharing" .to_string(), + artifact_storage: ArtifactStorageAccountingReport { + lane_private_bytes: 0, + reclaimable_bytes: 0, + unknown_bytes: usage.physical_bytes, + accounting: + "scope=lane;native-clone-extents=unattributed;artifact-bindings=unavailable;workspace-reclaimable-excluded" + .into(), + ..ArtifactStorageAccountingReport::default() + }, }) } @@ -2079,6 +2109,14 @@ mod tests { ); assert_eq!(space.changed_since_baseline_bytes, None); assert_eq!(space.physical_sharing, PhysicalSharing::Unknown); + assert_eq!( + space.artifact_storage.lane_private_bytes, + space.lane_exclusive_physical_bytes + ); + assert_eq!(space.artifact_storage.logical_bytes, 0); + assert_eq!(space.artifact_storage.materialized_bytes, 0); + assert_eq!(space.artifact_storage.reclaimable_bytes, 0); + assert_eq!(space.artifact_storage.prefetched_bytes, 0); drop(db); let reopened = Trail::open(temp.path()).unwrap(); diff --git a/trail/src/db/mod.rs b/trail/src/db/mod.rs index 11cf089b..ddf02d1b 100644 --- a/trail/src/db/mod.rs +++ b/trail/src/db/mod.rs @@ -2827,6 +2827,18 @@ pub(crate) struct BackupManifest { sqlite_bytes: u64, sqlite_sha256: String, worktree_bytes: u64, + #[serde(default)] + retained_private_views: u64, + #[serde(default)] + retained_private_bytes: u64, + #[serde(default)] + retained_private_sha256: String, + #[serde(default)] + rebuildable_materializations: u64, + #[serde(default)] + rebuildable_materialization_bytes: u64, + #[serde(default)] + rebuildable_performance_caches: u64, } #[derive(Debug)] diff --git a/trail/src/db/storage/lifecycle/gc.rs b/trail/src/db/storage/lifecycle/gc.rs index cc27d24d..be9c4487 100644 --- a/trail/src/db/storage/lifecycle/gc.rs +++ b/trail/src/db/storage/lifecycle/gc.rs @@ -1,7 +1,39 @@ use super::*; use crate::db::change_ledger::{ledger_gc_roots, IntentGcRoot}; +const GC_DELETE_BATCH_SIZE: usize = 256; + +#[cfg(test)] +thread_local! { + static GC_FAIL_AFTER_COMMITTED_BATCHES: std::cell::Cell> = + const { std::cell::Cell::new(None) }; +} + +#[cfg(test)] +fn fail_gc_after_committed_batch_if_requested() -> Result<()> { + GC_FAIL_AFTER_COMMITTED_BATCHES.with(|remaining| match remaining.get() { + Some(0 | 1) => { + remaining.set(None); + Err(Error::InvalidInput( + "injected object-GC interruption after committed batch".into(), + )) + } + Some(count) => { + remaining.set(Some(count - 1)); + Ok(()) + } + None => Ok(()), + }) +} + impl Trail { + #[cfg(test)] + pub(crate) fn set_gc_test_failure_after_committed_batches_for_current_thread( + batches: Option, + ) { + GC_FAIL_AFTER_COMMITTED_BATCHES.with(|remaining| remaining.set(batches)); + } + pub fn gc(&mut self, dry_run: bool) -> Result { let _lock = self.acquire_write_lock()?; // Capture roots before recovery terminalizes an intent. This makes the @@ -27,12 +59,17 @@ impl Trail { if known_kinds.contains(kind.as_str()) { total_known += 1; if !reachable.contains(&object_id) { - prunable.push(object_id); + prunable.push((object_id, kind)); } } else { preserved_unknown += 1; } } + // Every committed batch must leave the uncollected artifact graph + // internally valid so a later GC can restart after process loss. + // Therefore artifact parents are deleted before their children rather + // than relying on unrelated content-hash order. + let prunable = self.order_gc_prunable_objects(prunable)?; let mut report = GcReport { dry_run, total_known_objects: total_known, @@ -43,19 +80,251 @@ impl Trail { errors: Vec::new(), }; if !dry_run { - for object_id in &prunable { - self.conn.execute( - "DELETE FROM objects WHERE object_id = ?1", - params![object_id], - )?; - report.pruned_objects += 1; + for batch in prunable.chunks(GC_DELETE_BATCH_SIZE) { + report.pruned_objects += self.delete_gc_object_batch(batch)?; + #[cfg(test)] + fail_gc_after_committed_batch_if_requested()?; } + self.object_cache + .lock() + .expect("object cache poisoned") + .clear(); let rebuild = self.rebuild_indexes_unlocked()?; report.errors.extend(rebuild.errors); } Ok(report) } + fn delete_gc_object_batch(&self, object_ids: &[String]) -> Result { + self.conn.execute_batch("SAVEPOINT trail_gc_object_batch")?; + let deletion = (|| -> Result { + let mut deleted = 0_u64; + for object_id in object_ids { + // An envelope row is a lookup/index record, not an independent + // retention root. Durable consumers (generation bindings, + // attempts, attestations, quarantines, shadows, and holds) are + // traced before this point and their foreign keys fail closed + // if a new root type is ever omitted. + self.conn.execute( + "DELETE FROM artifact_attestations WHERE object_id=?1", + params![object_id], + )?; + self.conn.execute( + "DELETE FROM artifact_envelopes WHERE object_id=?1", + params![object_id], + )?; + self.conn.execute( + "DELETE FROM artifact_objects WHERE object_id=?1", + params![object_id], + )?; + let removed = self + .conn + .execute("DELETE FROM objects WHERE object_id=?1", params![object_id])?; + if removed != 1 { + return Err(Error::Corrupt(format!( + "garbage-collection candidate `{object_id}` disappeared during its fenced batch" + ))); + } + deleted = deleted.saturating_add(1); + } + Ok(deleted) + })(); + match deletion { + Ok(deleted) => { + self.conn + .execute_batch("RELEASE SAVEPOINT trail_gc_object_batch")?; + Ok(deleted) + } + Err(error) => { + let _ = self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT trail_gc_object_batch; + RELEASE SAVEPOINT trail_gc_object_batch", + ); + Err(error) + } + } + } + + fn order_gc_prunable_objects(&self, prunable: Vec<(String, String)>) -> Result> { + let candidate_kinds = prunable.into_iter().collect::>(); + let candidate_ids = candidate_kinds.keys().cloned().collect::>(); + let mut artifact_ids = BTreeMap::::new(); + let mut artifact_rows = Vec::<(String, String, Vec)>::new(); + { + let mut statement = self.conn.prepare( + "SELECT a.artifact_id,a.object_id,a.kind,o.bytes + FROM artifact_objects a + JOIN objects o ON o.object_id=a.object_id + ORDER BY a.artifact_id", + )?; + for row in statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Vec>(3)?, + )) + })? { + let (artifact_id, object_id, kind, bytes) = row?; + artifact_ids.insert(artifact_id, object_id.clone()); + if candidate_ids.contains(&object_id) { + artifact_rows.push((object_id, kind, bytes)); + } + } + } + + let mut edges = BTreeMap::>::new(); + let mut add_artifact_edge = |parent: &str, artifact_id: &str| -> Result<()> { + let child = artifact_ids.get(artifact_id).ok_or_else(|| { + Error::Corrupt(format!( + "garbage-collection artifact edge `{artifact_id}` has no object mapping" + )) + })?; + if candidate_ids.contains(child) { + edges + .entry(parent.to_string()) + .or_default() + .insert(child.clone()); + } + Ok(()) + }; + for (object_id, kind, bytes) in artifact_rows { + match kind.as_str() { + ARTIFACT_ENVELOPE_KIND => { + let envelope: ArtifactEnvelopeV1 = from_cbor(&bytes)?; + add_artifact_edge(&object_id, &envelope.tree_root_id.0)?; + } + ARTIFACT_TREE_ROOT_KIND => { + let tree: ArtifactTreeRootV1 = from_cbor(&bytes)?; + add_artifact_edge(&object_id, &tree.root_directory_id.0)?; + } + ARTIFACT_DIRECTORY_NODE_KIND => { + let directory: ArtifactDirectoryNodeV1 = from_cbor(&bytes)?; + for entry in directory.entries { + match entry.target { + ArtifactDirectoryEntryTargetV1::Directory { node_id } => { + add_artifact_edge(&object_id, &node_id.0)?; + } + ArtifactDirectoryEntryTargetV1::File { node_id } => { + add_artifact_edge(&object_id, &node_id.0)?; + } + ArtifactDirectoryEntryTargetV1::Symlink { .. } => {} + } + } + } + ARTIFACT_FILE_NODE_KIND => { + let file: ArtifactFileNodeV1 = from_cbor(&bytes)?; + match file.content { + ArtifactFileContentV1::Blob { blob_id } => { + add_artifact_edge(&object_id, &blob_id.0)?; + } + ArtifactFileContentV1::Chunks { chunk_list_id } => { + add_artifact_edge(&object_id, &chunk_list_id.0)?; + } + } + } + ARTIFACT_CHUNK_LIST_KIND => { + let list: ArtifactChunkListV1 = from_cbor(&bytes)?; + for chunk in list.chunks { + add_artifact_edge(&object_id, &chunk.chunk_id.0)?; + } + } + ARTIFACT_BLOB_KIND | ARTIFACT_CHUNK_KIND => {} + other => { + return Err(Error::Corrupt(format!( + "garbage-collection encountered unsupported artifact kind `{other}`" + ))) + } + } + } + { + let mut statement = self.conn.prepare( + "SELECT a.object_id,e.object_id + FROM artifact_attestations a + JOIN artifact_envelopes e ON e.envelope_id=a.envelope_id + ORDER BY a.attestation_id", + )?; + for row in statement.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? { + let (attestation_object, envelope_object) = row?; + if candidate_ids.contains(&attestation_object) + && candidate_ids.contains(&envelope_object) + { + edges + .entry(attestation_object) + .or_default() + .insert(envelope_object); + } + } + } + + let mut incoming = candidate_ids + .iter() + .map(|object_id| (object_id.clone(), 0_usize)) + .collect::>(); + for children in edges.values() { + for child in children { + let count = incoming.get_mut(child).ok_or_else(|| { + Error::Corrupt(format!( + "garbage-collection dependency `{child}` is not a candidate" + )) + })?; + *count = count.checked_add(1).ok_or_else(|| { + Error::Corrupt("garbage-collection dependency count overflowed".into()) + })?; + } + } + let mut ready = incoming + .iter() + .filter(|(_, count)| **count == 0) + .map(|(object_id, _)| { + ( + !is_artifact_gc_kind( + candidate_kinds + .get(object_id) + .map(String::as_str) + .unwrap_or_default(), + ), + object_id.clone(), + ) + }) + .collect::>(); + let mut ordered = Vec::with_capacity(candidate_ids.len()); + while let Some((_, object_id)) = ready.pop_first() { + ordered.push(object_id.clone()); + if let Some(children) = edges.get(&object_id) { + for child in children { + let count = incoming.get_mut(child).ok_or_else(|| { + Error::Corrupt(format!( + "garbage-collection dependency `{child}` disappeared" + )) + })?; + *count = count.checked_sub(1).ok_or_else(|| { + Error::Corrupt("garbage-collection dependency count underflowed".into()) + })?; + if *count == 0 { + ready.insert(( + !is_artifact_gc_kind( + candidate_kinds + .get(child) + .map(String::as_str) + .unwrap_or_default(), + ), + child.clone(), + )); + } + } + } + } + if ordered.len() != candidate_ids.len() { + return Err(Error::Corrupt( + "garbage-collection artifact dependency graph contains a cycle".into(), + )); + } + Ok(ordered) + } + fn reachable_object_ids_with_intent_roots( &self, intent_roots: &[IntentGcRoot], @@ -150,6 +419,8 @@ impl Trail { reachable.insert(object_id); } + self.collect_artifact_gc_reachable(&mut reachable, &mut errors)?; + let mut stmt = self.conn.prepare("SELECT object_id FROM anchors")?; let rows = stmt.query_map([], |row| row.get::<_, String>(0))?; for row in rows { @@ -164,6 +435,481 @@ impl Trail { Ok(reachable) } + fn collect_artifact_gc_reachable( + &self, + reachable: &mut HashSet, + errors: &mut Vec, + ) -> Result<()> { + let integrity_errors = self.validate_artifact_cas_integrity()?; + if !integrity_errors.is_empty() { + errors.extend(integrity_errors); + return Ok(()); + } + + let mut pending_artifacts = BTreeSet::<(String, String)>::new(); + let mut pending_objects = BTreeSet::::new(); + + { + let mut statement = self.conn.prepare( + "SELECT envelope_id,tree_root_id FROM artifact_generation_bindings + ORDER BY binding_id", + )?; + for row in statement.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? { + let (envelope_id, tree_root_id) = row?; + pending_artifacts.insert((envelope_id, ARTIFACT_ENVELOPE_KIND.into())); + pending_artifacts.insert((tree_root_id, ARTIFACT_TREE_ROOT_KIND.into())); + } + } + { + let mut statement = self.conn.prepare( + "SELECT envelope_id,tree_root_id FROM workspace_layer_artifact_shadows + ORDER BY layer_id", + )?; + for row in statement.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? { + let (envelope_id, tree_root_id) = row?; + pending_artifacts.insert((envelope_id, ARTIFACT_ENVELOPE_KIND.into())); + pending_artifacts.insert((tree_root_id, ARTIFACT_TREE_ROOT_KIND.into())); + } + } + { + // A verified real-directory cache is a conservative local lease. + // Cache eviction removes this row independently; the next object + // GC can then reclaim the authoritative graph if no durable root + // remains. + let mut statement = self.conn.prepare( + "SELECT tree_root_id FROM artifact_materializations + ORDER BY materialization_id", + )?; + for row in statement.query_map([], |row| row.get::<_, String>(0))? { + pending_artifacts.insert((row?, ARTIFACT_TREE_ROOT_KIND.into())); + } + } + { + let mut statement = self.conn.prepare( + "SELECT source_root,candidate_journal_object_id,envelope_id + FROM artifact_construction_attempts ORDER BY attempt_id", + )?; + for row in statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + )) + })? { + let (source_root, journal, envelope) = row?; + pending_objects.insert(source_root); + pending_objects.extend(journal); + if let Some(envelope) = envelope { + pending_artifacts.insert((envelope, ARTIFACT_ENVELOPE_KIND.into())); + } + } + } + { + let mut statement = self.conn.prepare( + "SELECT source_root,plan_object_id,stdout_object_id,stderr_object_id, + snapshot_id,failure_receipt_object_id + FROM artifact_resolution_attempts ORDER BY attempt_id", + )?; + for row in statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?, + )) + })? { + let (source, plan, stdout, stderr, snapshot, failure) = row?; + pending_objects.insert(source); + pending_objects.insert(plan); + pending_objects.extend(stdout); + pending_objects.extend(stderr); + pending_objects.extend(snapshot); + pending_objects.extend(failure); + } + } + { + let mut statement = self.conn.prepare( + "SELECT snapshot_id,source_root,content_object_id,predecessor_snapshot_id + FROM artifact_resolution_snapshots ORDER BY snapshot_id", + )?; + for row in statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + )) + })? { + let (snapshot, source, content, predecessor) = row?; + pending_objects.insert(snapshot); + pending_objects.insert(source); + pending_objects.insert(content); + pending_objects.extend(predecessor); + } + } + { + let mut statement = self.conn.prepare( + "SELECT incumbent_envelope_id,candidate_envelope_id,evidence_object_id + FROM artifact_quarantines ORDER BY quarantine_id", + )?; + for row in statement.query_map([], |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + })? { + let (incumbent, candidate, evidence) = row?; + if let Some(incumbent) = incumbent { + pending_artifacts.insert((incumbent, ARTIFACT_ENVELOPE_KIND.into())); + } + pending_artifacts.insert((candidate, ARTIFACT_ENVELOPE_KIND.into())); + pending_objects.insert(evidence); + } + } + { + let now = now_ts(); + let mut statement = self.conn.prepare( + "SELECT hold_id,target_kind,target_id FROM artifact_holds + WHERE expires_at IS NULL OR expires_at>?1 ORDER BY hold_id", + )?; + for row in statement.query_map(params![now], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + })? { + let (hold_id, target_kind, target_id) = row?; + match target_kind.as_str() { + "artifact_envelope" => { + pending_artifacts.insert((target_id, ARTIFACT_ENVELOPE_KIND.into())); + } + "artifact_tree" => { + pending_artifacts.insert((target_id, ARTIFACT_TREE_ROOT_KIND.into())); + } + "artifact_object" => { + let kind = self + .conn + .query_row( + "SELECT kind FROM artifact_objects WHERE artifact_id=?1", + params![target_id], + |row| row.get::<_, String>(0), + ) + .optional()?; + match kind { + Some(kind) => { + pending_artifacts.insert((target_id, kind)); + } + None => errors.push(format!( + "artifact hold {hold_id} references missing artifact object {target_id}" + )), + } + } + "object" | "resolution_snapshot" => { + pending_objects.insert(target_id); + } + _ => errors.push(format!( + "artifact hold {hold_id} has unsupported target kind `{target_kind}`" + )), + } + } + } + { + // Publication rows and external backups are different durability + // boundaries: an in-progress publication remains in this database + // and roots its pins, while a completed backup is a self-contained + // archive created under the same workspace write lock. + let mut statement = self.conn.prepare( + "SELECT source_root,manifest_object_id FROM workspace_layer_publications + ORDER BY publication_id", + )?; + for row in statement.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) + })? { + let (source_root, manifest) = row?; + pending_objects.insert(source_root); + pending_objects.extend(manifest); + } + } + + let mut visited_artifacts = HashSet::::new(); + let mut visited_objects = HashSet::::new(); + while !pending_artifacts.is_empty() || !pending_objects.is_empty() { + while let Some((artifact_id, expected_kind)) = pending_artifacts.pop_first() { + let mapping = self + .conn + .query_row( + "SELECT object_id,kind FROM artifact_objects WHERE artifact_id=?1", + params![artifact_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .optional()?; + let Some((object_id, actual_kind)) = mapping else { + errors.push(format!( + "reachable artifact object {artifact_id} is missing its CAS mapping" + )); + continue; + }; + if actual_kind != expected_kind { + errors.push(format!( + "reachable artifact object {artifact_id} has kind {actual_kind}, expected {expected_kind}" + )); + continue; + } + if visited_artifacts.insert(artifact_id) { + pending_objects.insert(object_id); + } + } + + let Some(object_id) = pending_objects.pop_first() else { + continue; + }; + if !visited_objects.insert(object_id.clone()) { + continue; + } + let Some((kind, version, bytes)) = self.validated_gc_object(&object_id, errors)? else { + continue; + }; + reachable.insert(object_id.clone()); + + let edge_result = (|| -> Result<()> { + match kind.as_str() { + WORKTREE_ROOT_KIND => { + self.collect_root_reachable( + &ObjectId(object_id.clone()), + reachable, + errors, + ); + } + ARTIFACT_DIRECTORY_NODE_KIND => { + require_gc_object_version(&kind, version, ARTIFACT_DIRECTORY_NODE_VERSION)?; + let node: ArtifactDirectoryNodeV1 = from_cbor(&bytes)?; + for entry in node.entries { + match entry.target { + ArtifactDirectoryEntryTargetV1::Directory { node_id } => { + pending_artifacts + .insert((node_id.0, ARTIFACT_DIRECTORY_NODE_KIND.into())); + } + ArtifactDirectoryEntryTargetV1::File { node_id } => { + pending_artifacts + .insert((node_id.0, ARTIFACT_FILE_NODE_KIND.into())); + } + ArtifactDirectoryEntryTargetV1::Symlink { .. } => {} + } + } + } + ARTIFACT_FILE_NODE_KIND => { + require_gc_object_version(&kind, version, ARTIFACT_FILE_NODE_VERSION)?; + let node: ArtifactFileNodeV1 = from_cbor(&bytes)?; + match node.content { + ArtifactFileContentV1::Blob { blob_id } => { + pending_artifacts.insert((blob_id.0, ARTIFACT_BLOB_KIND.into())); + } + ArtifactFileContentV1::Chunks { chunk_list_id } => { + pending_artifacts + .insert((chunk_list_id.0, ARTIFACT_CHUNK_LIST_KIND.into())); + } + } + } + ARTIFACT_CHUNK_LIST_KIND => { + require_gc_object_version(&kind, version, ARTIFACT_CHUNK_LIST_VERSION)?; + let list: ArtifactChunkListV1 = from_cbor(&bytes)?; + for chunk in list.chunks { + pending_artifacts + .insert((chunk.chunk_id.0, ARTIFACT_CHUNK_KIND.into())); + } + } + ARTIFACT_TREE_ROOT_KIND => { + require_gc_object_version(&kind, version, ARTIFACT_TREE_ROOT_VERSION)?; + let tree: ArtifactTreeRootV1 = from_cbor(&bytes)?; + pending_artifacts.insert(( + tree.root_directory_id.0, + ARTIFACT_DIRECTORY_NODE_KIND.into(), + )); + } + ARTIFACT_ENVELOPE_KIND => { + require_gc_object_version(&kind, version, ARTIFACT_ENVELOPE_VERSION)?; + let envelope: ArtifactEnvelopeV1 = from_cbor(&bytes)?; + pending_artifacts + .insert((envelope.tree_root_id.0, ARTIFACT_TREE_ROOT_KIND.into())); + pending_objects + .extend(envelope.resolution_snapshot_id.into_iter().map(|id| id.0)); + pending_objects + .extend(envelope.validation_receipt_ids.into_iter().map(|id| id.0)); + let mut statement = self.conn.prepare( + "SELECT a.object_id FROM artifact_attestations a + JOIN artifact_envelopes e ON e.envelope_id=a.envelope_id + WHERE e.object_id=?1 ORDER BY a.attestation_id", + )?; + for object_id in statement + .query_map(params![&object_id], |row| row.get::<_, String>(0))? + { + pending_objects.insert(object_id?); + } + } + ARTIFACT_ATTESTATION_KIND => { + require_gc_object_version(&kind, version, ARTIFACT_ATTESTATION_VERSION)?; + let attestation: ArtifactAttestationV1 = from_cbor(&bytes)?; + pending_artifacts.insert(( + attestation.statement.envelope_id.0, + ARTIFACT_ENVELOPE_KIND.into(), + )); + } + ARTIFACT_RESOLUTION_PLAN_KIND => { + require_gc_object_version( + &kind, + version, + ARTIFACT_RESOLUTION_PLAN_VERSION, + )?; + let plan: ArtifactResolutionPlanV1 = from_cbor(&bytes)?; + pending_objects.insert(plan.source_root.0); + } + ARTIFACT_RESOLUTION_SNAPSHOT_KIND => { + require_gc_object_version( + &kind, + version, + ARTIFACT_RESOLUTION_SNAPSHOT_VERSION, + )?; + let snapshot: ArtifactResolutionSnapshotV1 = from_cbor(&bytes)?; + pending_objects.insert(snapshot.source_root.0); + pending_objects.insert(snapshot.content_object_id.0); + pending_objects + .extend(snapshot.predecessor_snapshot_id.into_iter().map(|id| id.0)); + } + ARTIFACT_RESOLUTION_FAILURE_KIND => { + require_gc_object_version( + &kind, + version, + ARTIFACT_RESOLUTION_PLAN_VERSION, + )?; + let receipt: ArtifactResolutionFailureReceiptV1 = from_cbor(&bytes)?; + pending_objects.insert(receipt.source_root.0); + pending_objects.extend(receipt.stdout_object_id.into_iter().map(|id| id.0)); + pending_objects.extend(receipt.stderr_object_id.into_iter().map(|id| id.0)); + } + ARTIFACT_DIVERGENCE_EVIDENCE_KIND => { + require_gc_object_version( + &kind, + version, + ARTIFACT_DIVERGENCE_EVIDENCE_VERSION, + )?; + let evidence: ArtifactDivergenceEvidenceV1 = from_cbor(&bytes)?; + pending_artifacts.insert(( + evidence.incumbent_envelope_id.0, + ARTIFACT_ENVELOPE_KIND.into(), + )); + pending_artifacts.insert(( + evidence.incumbent_tree_root_id.0, + ARTIFACT_TREE_ROOT_KIND.into(), + )); + pending_artifacts.insert(( + evidence.candidate_envelope_id.0, + ARTIFACT_ENVELOPE_KIND.into(), + )); + pending_artifacts.insert(( + evidence.candidate_tree_root_id.0, + ARTIFACT_TREE_ROOT_KIND.into(), + )); + } + ARTIFACT_BLOB_KIND => { + require_gc_object_version(&kind, version, ARTIFACT_BLOB_VERSION)?; + let _: ArtifactBlobV1 = from_cbor(&bytes)?; + } + ARTIFACT_CHUNK_KIND => { + require_gc_object_version(&kind, version, ARTIFACT_CHUNK_VERSION)?; + let _: ArtifactChunkV1 = from_cbor(&bytes)?; + } + ARTIFACT_RESOLUTION_CONTENT_KIND => { + require_gc_object_version( + &kind, + version, + ARTIFACT_RESOLUTION_SNAPSHOT_VERSION, + )?; + let _: ArtifactResolutionContentV1 = from_cbor(&bytes)?; + } + ARTIFACT_RESOLUTION_CAPTURE_KIND => { + require_gc_object_version( + &kind, + version, + ARTIFACT_RESOLUTION_PLAN_VERSION, + )?; + let _: ArtifactResolutionCaptureV1 = from_cbor(&bytes)?; + } + ARTIFACT_VALIDATION_RECEIPT_KIND => { + require_gc_object_version( + &kind, + version, + ARTIFACT_VALIDATION_RECEIPT_VERSION, + )?; + let receipt: ArtifactValidationReceiptV1 = from_cbor(&bytes)?; + crate::db::lane::validate_artifact_validation_receipt(&receipt)?; + } + _ => {} + } + Ok(()) + })(); + if let Err(error) = edge_result { + errors.push(format!( + "failed to traverse reachable object {object_id} ({kind}): {error}" + )); + } + } + Ok(()) + } + + fn validated_gc_object( + &self, + object_id: &str, + errors: &mut Vec, + ) -> Result)>> { + let row = self + .conn + .query_row( + "SELECT kind,version,codec,hash_alg,size_bytes,bytes + FROM objects WHERE object_id=?1", + params![object_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, Vec>(5)?, + )) + }, + ) + .optional()?; + let Some((kind, version, codec, hash_alg, size_bytes, bytes)) = row else { + errors.push(format!("reachable object {object_id} is missing")); + return Ok(None); + }; + let Ok(version) = u16::try_from(version) else { + errors.push(format!( + "reachable object {object_id} has an invalid version" + )); + return Ok(None); + }; + if codec != "cbor" + || hash_alg != "sha256" + || i64::try_from(bytes.len()).ok() != Some(size_bytes) + || ObjectId::for_bytes(&kind, version, &bytes).0 != object_id + { + errors.push(format!( + "reachable object {object_id} has invalid metadata or content identity" + )); + return Ok(None); + } + Ok(Some((kind, version, bytes))) + } + pub(crate) fn collect_lane_event_object_refs( &self, reachable: &mut HashSet, @@ -217,3 +963,26 @@ impl Trail { } } } + +fn is_artifact_gc_kind(kind: &str) -> bool { + matches!( + kind, + ARTIFACT_DIRECTORY_NODE_KIND + | ARTIFACT_FILE_NODE_KIND + | ARTIFACT_BLOB_KIND + | ARTIFACT_CHUNK_LIST_KIND + | ARTIFACT_CHUNK_KIND + | ARTIFACT_TREE_ROOT_KIND + | ARTIFACT_ENVELOPE_KIND + | ARTIFACT_ATTESTATION_KIND + ) +} + +fn require_gc_object_version(kind: &str, actual: u16, expected: u16) -> Result<()> { + if actual != expected { + return Err(Error::Corrupt(format!( + "object kind {kind} has version {actual}, expected {expected}" + ))); + } + Ok(()) +} diff --git a/trail/src/db/storage/schema/ddl.rs b/trail/src/db/storage/schema/ddl.rs index 05da034c..166f47d4 100644 --- a/trail/src/db/storage/schema/ddl.rs +++ b/trail/src/db/storage/schema/ddl.rs @@ -108,6 +108,169 @@ CREATE TABLE conflict_sets ( details_json TEXT, created_at INTEGER NOT NULL ); +CREATE TABLE artifact_resolution_snapshots ( + snapshot_id TEXT PRIMARY KEY REFERENCES objects(object_id), + proposal_key TEXT NOT NULL, + source_root TEXT NOT NULL REFERENCES objects(object_id), + component_id TEXT NOT NULL, + adapter_identity TEXT NOT NULL, + content_object_id TEXT NOT NULL REFERENCES objects(object_id), + predecessor_snapshot_id TEXT REFERENCES objects(object_id), + verification_state TEXT NOT NULL + CHECK (verification_state IN ('verified','rejected')), + state TEXT NOT NULL CHECK (state IN ('current','superseded')), + created_at INTEGER NOT NULL, + superseded_at INTEGER + ); +CREATE UNIQUE INDEX artifact_resolution_snapshots_current_idx + ON artifact_resolution_snapshots(proposal_key) + WHERE state = 'current'; +CREATE INDEX artifact_resolution_snapshots_content_idx + ON artifact_resolution_snapshots(content_object_id); +CREATE TABLE artifact_objects ( + artifact_id TEXT PRIMARY KEY, + object_id TEXT NOT NULL REFERENCES objects(object_id), + kind TEXT NOT NULL, + version INTEGER NOT NULL, + logical_bytes INTEGER NOT NULL, + created_at INTEGER NOT NULL, + UNIQUE(object_id) + ); +CREATE INDEX artifact_objects_kind_idx ON artifact_objects(kind, artifact_id); +CREATE TABLE artifact_envelopes ( + envelope_id TEXT PRIMARY KEY, + desired_key TEXT NOT NULL, + trust_scope TEXT NOT NULL, + tree_root_id TEXT NOT NULL REFERENCES artifact_objects(artifact_id), + object_id TEXT NOT NULL UNIQUE REFERENCES objects(object_id), + state TEXT NOT NULL CHECK (state IN ('candidate','ready','quarantined','retired')), + verification_state TEXT NOT NULL + CHECK (verification_state IN ('pending','verified','rejected')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); +CREATE INDEX artifact_envelopes_desired_idx + ON artifact_envelopes(desired_key, state, envelope_id); +CREATE TABLE artifact_construction_attempts ( + attempt_id TEXT PRIMARY KEY, + desired_key TEXT NOT NULL, + source_root TEXT NOT NULL REFERENCES objects(object_id), + owner_generation INTEGER NOT NULL CHECK (owner_generation > 0), + owner_pid INTEGER NOT NULL CHECK (owner_pid > 0), + owner_start_token TEXT NOT NULL, + phase TEXT NOT NULL CHECK (phase IN + ('reserved','building','validating','publishing','completed')), + status TEXT NOT NULL CHECK (status IN + ('running','succeeded','failed','cancelled','abandoned')), + candidate_journal_object_id TEXT REFERENCES objects(object_id), + envelope_id TEXT REFERENCES artifact_envelopes(envelope_id), + reason_code TEXT, + reason TEXT, + cancel_requested INTEGER NOT NULL DEFAULT 0 CHECK (cancel_requested IN (0,1)), + started_at INTEGER NOT NULL, + heartbeat_at INTEGER NOT NULL, + finished_at INTEGER + ); +CREATE UNIQUE INDEX artifact_construction_attempts_running_idx + ON artifact_construction_attempts(desired_key) + WHERE status = 'running'; +CREATE TABLE artifact_construction_waiters ( + attempt_id TEXT NOT NULL REFERENCES artifact_construction_attempts(attempt_id), + waiter_id TEXT NOT NULL, + owner_pid INTEGER NOT NULL CHECK (owner_pid > 0), + owner_start_token TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('waiting','released','cancelled','abandoned')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (attempt_id, waiter_id) + ); +CREATE TABLE artifact_resolution_attempts ( + attempt_id TEXT PRIMARY KEY, + proposal_key TEXT NOT NULL, + source_root TEXT NOT NULL REFERENCES objects(object_id), + plan_object_id TEXT NOT NULL REFERENCES objects(object_id), + owner_generation INTEGER NOT NULL CHECK (owner_generation > 0), + owner_pid INTEGER NOT NULL CHECK (owner_pid > 0), + owner_start_token TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN + ('running','succeeded','failed','cancelled','abandoned')), + cancel_requested INTEGER NOT NULL DEFAULT 0 CHECK (cancel_requested IN (0,1)), + authority_evidence_json BLOB NOT NULL, + stdout_object_id TEXT REFERENCES objects(object_id), + stderr_object_id TEXT REFERENCES objects(object_id), + snapshot_id TEXT REFERENCES artifact_resolution_snapshots(snapshot_id), + failure_receipt_object_id TEXT REFERENCES objects(object_id), + failure_code TEXT, + failure_message TEXT, + started_at INTEGER NOT NULL, + heartbeat_at INTEGER NOT NULL, + finished_at INTEGER + ); +CREATE UNIQUE INDEX artifact_resolution_attempts_running_idx + ON artifact_resolution_attempts(proposal_key) + WHERE status = 'running'; +CREATE TABLE artifact_attestations ( + attestation_id TEXT PRIMARY KEY, + envelope_id TEXT NOT NULL REFERENCES artifact_envelopes(envelope_id), + object_id TEXT NOT NULL UNIQUE REFERENCES objects(object_id), + producer_identity TEXT NOT NULL, + trust_scope TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('valid','revoked','invalid')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); +CREATE TABLE artifact_quarantines ( + quarantine_id TEXT PRIMARY KEY, + trust_scope TEXT NOT NULL, + desired_key TEXT NOT NULL, + incumbent_envelope_id TEXT REFERENCES artifact_envelopes(envelope_id), + candidate_envelope_id TEXT NOT NULL REFERENCES artifact_envelopes(envelope_id), + reason_code TEXT NOT NULL, + evidence_object_id TEXT NOT NULL REFERENCES objects(object_id), + state TEXT NOT NULL CHECK (state IN ('active','resolved')), + resolution TEXT, + created_at INTEGER NOT NULL, + resolved_at INTEGER + ); +CREATE INDEX artifact_quarantines_desired_idx + ON artifact_quarantines(trust_scope, desired_key, state); +CREATE TABLE artifact_holds ( + hold_id TEXT PRIMARY KEY, + target_kind TEXT NOT NULL, + target_id TEXT NOT NULL, + reason TEXT NOT NULL, + expires_at INTEGER, + created_at INTEGER NOT NULL + ); +CREATE INDEX artifact_holds_target_idx + ON artifact_holds(target_kind, target_id, expires_at); +CREATE TABLE artifact_generation_bindings ( + binding_id TEXT PRIMARY KEY, + generation_id TEXT NOT NULL REFERENCES environment_generations(generation_id), + component_id TEXT NOT NULL, + output_name TEXT NOT NULL, + desired_key TEXT NOT NULL, + envelope_id TEXT NOT NULL REFERENCES artifact_envelopes(envelope_id), + tree_root_id TEXT NOT NULL REFERENCES artifact_objects(artifact_id), + binding_identity TEXT NOT NULL, + created_at INTEGER NOT NULL, + UNIQUE(generation_id, component_id, output_name) + ); +CREATE TABLE artifact_materializations ( + materialization_id TEXT PRIMARY KEY, + tree_root_id TEXT NOT NULL REFERENCES artifact_objects(artifact_id), + backend_compatibility TEXT NOT NULL, + storage_path TEXT NOT NULL UNIQUE, + state TEXT NOT NULL CHECK (state IN ('building','verified','failed')), + logical_bytes INTEGER NOT NULL, + physical_bytes INTEGER, + entry_count INTEGER NOT NULL, + last_used_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + UNIQUE(tree_root_id, backend_compatibility) + ); +CREATE INDEX artifact_materializations_lru_idx + ON artifact_materializations(state, last_used_at, materialization_id); CREATE TABLE environment_cache_namespaces ( namespace_id TEXT PRIMARY KEY, adapter_identity TEXT NOT NULL, @@ -947,6 +1110,13 @@ CREATE TABLE workspace_layers ( last_used_at INTEGER NOT NULL, created_at INTEGER NOT NULL ); +CREATE TABLE workspace_layer_artifact_shadows ( + layer_id TEXT PRIMARY KEY REFERENCES workspace_layers(layer_id), + tree_root_id TEXT NOT NULL REFERENCES artifact_objects(artifact_id), + envelope_id TEXT NOT NULL REFERENCES artifact_envelopes(envelope_id), + state TEXT NOT NULL CHECK (state IN ('verified','failed')), + verified_at INTEGER NOT NULL + ); CREATE TABLE workspace_layer_publications ( publication_id TEXT PRIMARY KEY, view_id TEXT NOT NULL, diff --git a/trail/src/db/util/backup_paths.rs b/trail/src/db/util/backup_paths.rs index e139de8d..0bdd5f14 100644 --- a/trail/src/db/util/backup_paths.rs +++ b/trail/src/db/util/backup_paths.rs @@ -36,3 +36,84 @@ pub(crate) fn file_digest(path: &Path) -> Result<(u64, String)> { } Ok((bytes, hex::encode(hasher.finalize()))) } + +pub(crate) fn portable_tree_digest(path: &Path) -> Result<(u64, String)> { + let mut hasher = Sha256::new(); + let mut bytes = 0_u64; + if !path.exists() { + return Ok((bytes, hex::encode(hasher.finalize()))); + } + let entries = walkdir::WalkDir::new(path) + .follow_links(false) + .min_depth(1) + .into_iter() + .collect::, _>>() + .map_err(|error| Error::Io(error.into()))?; + let mut entries = entries + .into_iter() + .map(|entry| { + let relative = entry.path().strip_prefix(path).map_err(|_| { + Error::Corrupt(format!( + "backup private path `{}` escaped its tree", + entry.path().display() + )) + })?; + Ok((portable_relative_path_bytes(relative)?, entry)) + }) + .collect::>>()?; + entries.sort_by(|left, right| left.0.cmp(&right.0)); + for (relative, entry) in entries { + hasher.update((relative.len() as u64).to_be_bytes()); + hasher.update(&relative); + let metadata = fs::symlink_metadata(entry.path())?; + if metadata.is_dir() { + hasher.update(b"directory\0"); + } else if metadata.is_file() { + hasher.update(b"file\0"); + hasher.update(metadata.len().to_be_bytes()); + let (file_bytes, digest) = file_digest(entry.path())?; + if file_bytes != metadata.len() { + return Err(Error::Conflict(format!( + "backup private file `{}` changed while being sealed", + entry.path().display() + ))); + } + bytes = bytes.saturating_add(file_bytes); + hasher.update(digest.as_bytes()); + } else if metadata.file_type().is_symlink() { + hasher.update(b"symlink\0"); + let target = fs::read_link(entry.path())?; + let target = target.as_os_str().as_encoded_bytes(); + hasher.update((target.len() as u64).to_be_bytes()); + hasher.update(target); + } else { + return Err(Error::InvalidInput(format!( + "backup private path `{}` has an unsupported file type", + entry.path().display() + ))); + } + } + Ok((bytes, hex::encode(hasher.finalize()))) +} + +fn portable_relative_path_bytes(path: &Path) -> Result> { + let mut encoded = Vec::new(); + for component in path.components() { + let Component::Normal(component) = component else { + return Err(Error::Corrupt(format!( + "backup private path `{}` is not relative and normalized", + path.display() + ))); + }; + if !encoded.is_empty() { + encoded.push(b'/'); + } + encoded.extend_from_slice(component.as_encoded_bytes()); + } + if encoded.is_empty() { + return Err(Error::Corrupt( + "backup private tree contains an empty relative path".into(), + )); + } + Ok(encoded) +} diff --git a/trail/src/db/util/gc.rs b/trail/src/db/util/gc.rs index 18f453f4..ca1ea4e3 100644 --- a/trail/src/db/util/gc.rs +++ b/trail/src/db/util/gc.rs @@ -8,6 +8,23 @@ pub(crate) fn known_gc_object_kinds() -> HashSet<&'static str> { BLOB_KIND, MESSAGE_KIND, ANCHOR_KIND, + WORKSPACE_LAYER_MANIFEST_KIND, + WORKSPACE_LAYER_MANIFEST_PAGE_KIND, + ARTIFACT_RESOLUTION_SNAPSHOT_KIND, + ARTIFACT_RESOLUTION_CONTENT_KIND, + ARTIFACT_RESOLUTION_PLAN_KIND, + ARTIFACT_RESOLUTION_CAPTURE_KIND, + ARTIFACT_RESOLUTION_FAILURE_KIND, + ARTIFACT_DIRECTORY_NODE_KIND, + ARTIFACT_FILE_NODE_KIND, + ARTIFACT_BLOB_KIND, + ARTIFACT_CHUNK_LIST_KIND, + ARTIFACT_CHUNK_KIND, + ARTIFACT_TREE_ROOT_KIND, + ARTIFACT_ENVELOPE_KIND, + ARTIFACT_DIVERGENCE_EVIDENCE_KIND, + ARTIFACT_VALIDATION_RECEIPT_KIND, + ARTIFACT_ATTESTATION_KIND, ] .into_iter() .collect() diff --git a/trail/src/ids.rs b/trail/src/ids.rs index 7883bd54..e0d8309f 100644 --- a/trail/src/ids.rs +++ b/trail/src/ids.rs @@ -15,6 +15,87 @@ pub(crate) const ANCHOR_ID_PREFIX: &str = "anchor_"; pub(crate) const CHECKPOINT_ALIAS_PREFIX: &str = "checkpoint_"; pub(crate) const LINE_ALIAS_PREFIX: &str = "line_"; +macro_rules! artifact_id_type { + ($name:ident, $prefix:literal, $domain:literal) => { + #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] + pub struct $name(pub String); + + impl $name { + pub fn new(seed: &[u8]) -> Self { + let mut hasher = Sha256::new(); + hasher.update(b"TRAIL-ARTIFACT-ID-V1\0"); + hasher.update($domain.as_bytes()); + hasher.update((seed.len() as u64).to_le_bytes()); + hasher.update(seed); + Self(format!("{}{}", $prefix, hex::encode(hasher.finalize()))) + } + + pub fn parse(value: impl Into) -> std::result::Result { + let value = value.into(); + let Some(digest) = value.strip_prefix($prefix) else { + return Err(format!("expected an ID with prefix `{}`", $prefix)); + }; + if digest.len() != 64 + || !digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(format!( + "expected `{}` followed by 64 lowercase hexadecimal characters", + $prefix + )); + } + Ok(Self(value)) + } + } + + impl Serialize for $name { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.0) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::parse(value).map_err(serde::de::Error::custom) + } + } + + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } + } + }; +} + +artifact_id_type!(ArtifactDesiredKeyV2, "artifact_desired_", "desired-v2"); +artifact_id_type!(ArtifactTreeId, "artifact_tree_", "tree"); +artifact_id_type!(ArtifactFileId, "artifact_file_", "file"); +artifact_id_type!(ArtifactBlobId, "artifact_blob_", "blob"); +artifact_id_type!(ArtifactChunkListId, "artifact_chunks_", "chunk-list"); +artifact_id_type!(ArtifactChunkId, "artifact_chunk_", "chunk"); +artifact_id_type!(ArtifactEnvelopeId, "artifact_envelope_", "envelope"); +artifact_id_type!( + ArtifactAttestationId, + "artifact_attestation_", + "attestation" +); +artifact_id_type!(ArtifactAttemptId, "artifact_attempt_", "attempt"); +artifact_id_type!(ArtifactQuarantineId, "artifact_quarantine_", "quarantine"); +artifact_id_type!( + ArtifactGenerationBindingId, + "artifact_binding_", + "generation-binding" +); + #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct WorkspaceId(pub String); @@ -258,4 +339,18 @@ mod tests { assert!(!is_change_id(&format!("ch_{digest}"))); assert!(!is_object_id("obj_legacy")); } + + #[test] + fn artifact_ids_are_domain_separated_and_validate_deserialization() { + let desired = ArtifactDesiredKeyV2::new(b"same-seed"); + let tree = ArtifactTreeId::new(b"same-seed"); + assert_ne!(desired.0, tree.0); + assert_eq!( + serde_json::from_str::(&serde_json::to_string(&desired).unwrap()) + .unwrap(), + desired + ); + assert!(serde_json::from_str::("\"artifact_desired_BAD\"").is_err()); + assert!(ArtifactTreeId::parse(desired.0).is_err()); + } } diff --git a/trail/src/lib.rs b/trail/src/lib.rs index 615bd3d3..7fb8debe 100644 --- a/trail/src/lib.rs +++ b/trail/src/lib.rs @@ -17,7 +17,12 @@ pub mod server; pub use db::{InitImportMode, Trail}; pub use error::{Error, Result}; -pub use ids::{AnchorId, ChangeId, FileId, LineId, MessageId, ObjectId, WorkspaceId}; +pub use ids::{ + AnchorId, ArtifactAttemptId, ArtifactAttestationId, ArtifactBlobId, ArtifactChunkId, + ArtifactChunkListId, ArtifactDesiredKeyV2, ArtifactEnvelopeId, ArtifactFileId, + ArtifactGenerationBindingId, ArtifactQuarantineId, ArtifactTreeId, ChangeId, FileId, LineId, + MessageId, ObjectId, WorkspaceId, +}; pub use model::*; #[cfg(debug_assertions)] diff --git a/trail/src/mcp/capabilities/resources.rs b/trail/src/mcp/capabilities/resources.rs index d9fa9b01..80630248 100644 --- a/trail/src/mcp/capabilities/resources.rs +++ b/trail/src/mcp/capabilities/resources.rs @@ -16,6 +16,20 @@ pub(crate) fn resources_templates_list_result() -> Value { pub(crate) fn resource_templates() -> Value { json!([ + { + "uriTemplate": RESOURCE_ARTIFACT_TEMPLATE, + "name": "artifact", + "title": "Artifact Inspection", + "description": "Read one artifact envelope, content identity, attestation, trust, verification, quarantine, and storage report.", + "mimeType": "application/json" + }, + { + "uriTemplate": RESOURCE_ARTIFACT_QUARANTINE_TEMPLATE, + "name": "artifact-quarantine", + "title": "Artifact Quarantine", + "description": "Read one durable artifact nondeterminism quarantine and its retained evidence.", + "mimeType": "application/json" + }, { "uriTemplate": RESOURCE_AGENT_SUMMARY_TEMPLATE, "name": "agent-task-summary", @@ -245,6 +259,20 @@ pub(crate) fn resources() -> Value { "description": "Current lane branches and lifecycle metadata.", "mimeType": "application/json" }, + { + "uri": RESOURCE_ARTIFACT_SPACE, + "name": "artifact-space", + "title": "Artifact Space", + "description": "CAS-aware logical, authoritative, shared, materialized, private, and reclaimable storage accounting.", + "mimeType": "application/json" + }, + { + "uri": RESOURCE_ARTIFACT_QUARANTINES, + "name": "artifact-quarantines", + "title": "Artifact Quarantines", + "description": "List durable artifact nondeterminism quarantines and their explicit recovery commands.", + "mimeType": "application/json" + }, { "uri": RESOURCE_LANE_MERGE_QUEUE, "name": "lane-merge-queue", diff --git a/trail/src/mcp/completion.rs b/trail/src/mcp/completion.rs index b0120a1e..5e91e23a 100644 --- a/trail/src/mcp/completion.rs +++ b/trail/src/mcp/completion.rs @@ -80,6 +80,12 @@ fn resource_completion_candidates( (RESOURCE_APPROVAL_TEMPLATE, "approval_id") => approval_completion_candidates(db), (RESOURCE_RUN_TEMPLATE, "run_id") => run_completion_candidates(db), (RESOURCE_SPAN_TEMPLATE, "span_id") => span_completion_candidates(db), + (RESOURCE_ARTIFACT_TEMPLATE, "artifact_id") => db.artifact_envelope_ids(), + (RESOURCE_ARTIFACT_QUARANTINE_TEMPLATE, "quarantine_id") => Ok(db + .list_artifact_quarantines()? + .into_iter() + .map(|record| record.quarantine_id.0) + .collect()), ( RESOURCE_LANE_TEMPLATE | RESOURCE_LANE_STATUS_TEMPLATE @@ -93,7 +99,9 @@ fn resource_completion_candidates( | RESOURCE_CONFLICT_TEMPLATE | RESOURCE_APPROVAL_TEMPLATE | RESOURCE_RUN_TEMPLATE - | RESOURCE_SPAN_TEMPLATE, + | RESOURCE_SPAN_TEMPLATE + | RESOURCE_ARTIFACT_TEMPLATE + | RESOURCE_ARTIFACT_QUARANTINE_TEMPLATE, _, ) => Ok(Vec::new()), (other, _) => Err(Error::InvalidInput(format!( diff --git a/trail/src/mcp/resource.rs b/trail/src/mcp/resource.rs index fb91464e..a6758683 100644 --- a/trail/src/mcp/resource.rs +++ b/trail/src/mcp/resource.rs @@ -1,6 +1,6 @@ use serde_json::{json, Value}; -use crate::{Error, Result, Trail}; +use crate::{ArtifactEnvelopeId, ArtifactQuarantineId, Error, Result, Trail}; use super::{response::pretty_json, types::*, utils::from_arguments}; @@ -18,6 +18,14 @@ fn resource_read_response(db: &mut Trail, args: ResourceReadArgs) -> Result ("application/json", pretty_json(&db.doctor()?)?), RESOURCE_LANES => ("application/json", pretty_json(&db.list_lanes()?)?), + RESOURCE_ARTIFACT_SPACE => ( + "application/json", + pretty_json(&db.workspace_artifact_space()?)?, + ), + RESOURCE_ARTIFACT_QUARANTINES => ( + "application/json", + pretty_json(&db.artifact_quarantine_list_report()?)?, + ), RESOURCE_LANE_MERGE_QUEUE => ( "application/json", pretty_json(&db.list_lane_merge_queue()?)?, @@ -108,6 +116,33 @@ fn resource_read_response(db: &mut Trail, args: ResourceReadArgs) -> Result Result<(&'static str, String)> { + if let Some(artifact_id) = template_uri_argument( + uri, + "trail://workspace/artifacts/", + "", + RESOURCE_ARTIFACT_TEMPLATE, + )? { + let artifact = ArtifactEnvelopeId::parse(&artifact_id).map_err(|error| { + Error::InvalidInput(format!("invalid artifact envelope ID: {error}")) + })?; + return Ok(( + "application/json", + pretty_json(&db.inspect_artifact(&artifact)?)?, + )); + } + if let Some(quarantine_id) = template_uri_argument( + uri, + "trail://workspace/artifact-quarantines/", + "", + RESOURCE_ARTIFACT_QUARANTINE_TEMPLATE, + )? { + let quarantine = ArtifactQuarantineId::parse(&quarantine_id) + .map_err(|error| Error::InvalidInput(format!("invalid quarantine ID: {error}")))?; + return Ok(( + "application/json", + pretty_json(&db.artifact_quarantine(&quarantine)?)?, + )); + } if let Some(selector) = template_uri_argument( uri, "trail://workspace/agent-tasks/", diff --git a/trail/src/mcp/tool_call/lane.rs b/trail/src/mcp/tool_call/lane.rs index 7143e0c2..915fc1ea 100644 --- a/trail/src/mcp/tool_call/lane.rs +++ b/trail/src/mcp/tool_call/lane.rs @@ -2,7 +2,10 @@ use std::path::PathBuf; use serde_json::Value; -use crate::{LaneGateOptions, Result, Trail}; +use crate::{ + ArtifactEnvelopeId, ArtifactQuarantineId, ArtifactSourceExportAuthorizationV1, LaneGateOptions, + Result, Trail, +}; use super::{super::response::tool_result, super::types::*, parse_args}; @@ -189,6 +192,68 @@ pub(super) fn handle(db: &mut Trail, name: &str, arguments: &Value) -> Result { + let args: EnvironmentResolveArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + tool_result(db.resolve_workspace_environment_component( + &lane, + &args.component, + args.path.as_deref(), + args.refresh, + )?) + } + "trail.env_resolve_all" => { + let args: EnvironmentResolveAllArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + tool_result(db.resolve_all_workspace_environment_components( + &lane, + args.path.as_deref(), + args.refresh, + )?) + } + "trail.artifact_space" => tool_result(db.workspace_artifact_space()?), + "trail.artifact_inspect" => { + let args: ArtifactIdArgs = parse_args(arguments)?; + let artifact = + ArtifactEnvelopeId::parse(args.artifact).map_err(crate::Error::InvalidInput)?; + tool_result(db.inspect_artifact(&artifact)?) + } + "trail.artifact_reachability" => { + let args: ArtifactIdArgs = parse_args(arguments)?; + let artifact = + ArtifactEnvelopeId::parse(args.artifact).map_err(crate::Error::InvalidInput)?; + tool_result(db.artifact_content_reachability(&artifact)?) + } + "trail.artifact_verify" => { + let args: ArtifactVerifyArgs = parse_args(arguments)?; + let artifact = + ArtifactEnvelopeId::parse(args.artifact).map_err(crate::Error::InvalidInput)?; + tool_result(db.verify_artifact(&artifact, args.level)?) + } + "trail.artifact_quarantine_list" => tool_result(db.artifact_quarantine_list_report()?), + "trail.artifact_quarantine_show" => { + let args: ArtifactQuarantineArgs = parse_args(arguments)?; + let quarantine = + ArtifactQuarantineId::parse(args.quarantine).map_err(crate::Error::InvalidInput)?; + tool_result(db.artifact_quarantine(&quarantine)?) + } + "trail.artifact_quarantine_resolve" => { + let args: ArtifactQuarantineResolveArgs = parse_args(arguments)?; + let quarantine = + ArtifactQuarantineId::parse(args.quarantine).map_err(crate::Error::InvalidInput)?; + tool_result(db.resolve_artifact_quarantine_report(&quarantine, args.resolution)?) + } + "trail.env_source_export" => { + let args: ArtifactSourceExportArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + let plan = db.plan_artifact_source_export( + &lane, + &args.component, + &args.export, + ArtifactSourceExportAuthorizationV1::ExplicitUser, + )?; + tool_result(db.execute_artifact_source_export(plan)?) + } "trail.env_sync" => { let args: EnvironmentSyncArgs = parse_args(arguments)?; let lane = db.resolve_lane_handle(&args.lane)?; diff --git a/trail/src/mcp/tools/annotations.rs b/trail/src/mcp/tools/annotations.rs index 24955e02..676ccdc9 100644 --- a/trail/src/mcp/tools/annotations.rs +++ b/trail/src/mcp/tools/annotations.rs @@ -153,6 +153,12 @@ fn classified_tool_risk_class(name: &str) -> Option { | "trail.env_runtime_status" | "trail.env_explain" | "trail.env_plan" + | "trail.artifact_space" + | "trail.artifact_inspect" + | "trail.artifact_reachability" + | "trail.artifact_verify" + | "trail.artifact_quarantine_list" + | "trail.artifact_quarantine_show" | "trail.cache_list" | "trail.cache_inspect" | "trail.cache_verify" @@ -231,6 +237,9 @@ fn classified_tool_risk_class(name: &str) -> Option { | "trail.read_file" | "trail.lane_hydrate" | "trail.sync_workdir" => Some(ToolRiskClass::DestructiveWrite), + "trail.artifact_quarantine_resolve" | "trail.env_source_export" => { + Some(ToolRiskClass::DestructiveWrite) + } "trail.cache_gc" => Some(ToolRiskClass::DestructiveWrite), "trail.agent_test" | "trail.agent_eval" @@ -240,6 +249,8 @@ fn classified_tool_risk_class(name: &str) -> Option { | "trail.deps_sync" | "trail.env_sync" | "trail.env_sync_all" + | "trail.env_resolve" + | "trail.env_resolve_all" | "trail.env_promote" | "trail.env_runtime_reconcile" => Some(ToolRiskClass::OpenWorldWrite), "trail.env_runtime_stop" => Some(ToolRiskClass::OpenWorldDestructiveWrite), diff --git a/trail/src/mcp/tools/lane.rs b/trail/src/mcp/tools/lane.rs index d8a3fb2b..621bde11 100644 --- a/trail/src/mcp/tools/lane.rs +++ b/trail/src/mcp/tools/lane.rs @@ -320,6 +320,91 @@ pub(super) fn tools() -> Value { "path": { "type": "string" } }), vec!["lane"]) }, + { + "name": "trail.env_resolve", + "title": "Resolve Workspace Environment", + "description": "Run one exact host-validated resolver plan against pinned source and publish or reuse its immutable snapshot.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" }, + "component": { "type": "string" }, + "path": { "type": "string" }, + "refresh": { "type": "boolean", "default": false } + }), vec!["lane", "component"]) + }, + { + "name": "trail.env_resolve_all", + "title": "Resolve All Workspace Environments", + "description": "Resolve or reuse every managed dependency snapshot in deterministic discovery order.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" }, + "path": { "type": "string" }, + "refresh": { "type": "boolean", "default": false } + }), vec!["lane"]) + }, + { + "name": "trail.artifact_space", + "title": "Workspace Artifact Space", + "description": "Report CAS-aware logical, unique, shared, materialized, private, and reclaimable workspace artifact bytes.", + "inputSchema": object_schema(json!({}), vec![]) + }, + { + "name": "trail.artifact_inspect", + "title": "Inspect Artifact", + "description": "Inspect one immutable artifact envelope with bindings, trust, quarantine, reachability, and storage evidence.", + "inputSchema": object_schema(json!({ + "artifact": { "type": "string" } + }), vec!["artifact"]) + }, + { + "name": "trail.artifact_reachability", + "title": "Artifact Reachability", + "description": "Return a bounded object-kind and byte summary for one artifact's durable CAS graph.", + "inputSchema": object_schema(json!({ + "artifact": { "type": "string" } + }), vec!["artifact"]) + }, + { + "name": "trail.artifact_verify", + "title": "Verify Artifact", + "description": "Verify one artifact at attach, sample, full, or reproducibility-evidence depth.", + "inputSchema": object_schema(json!({ + "artifact": { "type": "string" }, + "level": { "type": "string", "enum": ["attach", "sample", "full", "reproduce"] } + }), vec!["artifact", "level"]) + }, + { + "name": "trail.artifact_quarantine_list", + "title": "List Artifact Quarantines", + "description": "List active and resolved divergent-producer quarantine evidence.", + "inputSchema": object_schema(json!({}), vec![]) + }, + { + "name": "trail.artifact_quarantine_show", + "title": "Show Artifact Quarantine", + "description": "Show one durable artifact quarantine record.", + "inputSchema": object_schema(json!({ + "quarantine": { "type": "string" } + }), vec!["quarantine"]) + }, + { + "name": "trail.artifact_quarantine_resolve", + "title": "Resolve Artifact Quarantine", + "description": "Resolve divergent-producer evidence through one explicit retention or acceptance policy.", + "inputSchema": object_schema(json!({ + "quarantine": { "type": "string" }, + "resolution": { "type": "string", "enum": ["retain_private", "accept_incumbent", "accept_candidate", "retire_all"] } + }), vec!["quarantine", "resolution"]) + }, + { + "name": "trail.env_source_export", + "title": "Export Artifact Source", + "description": "Explicitly export one declared artifact subtree through ordinary guarded lane source writes.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" }, + "component": { "type": "string" }, + "export": { "type": "string" } + }), vec!["lane", "component", "export"]) + }, { "name": "trail.env_sync_all", "title": "Synchronize All Workspace Environments", diff --git a/trail/src/mcp/types/constants.rs b/trail/src/mcp/types/constants.rs index b75f7e04..00978146 100644 --- a/trail/src/mcp/types/constants.rs +++ b/trail/src/mcp/types/constants.rs @@ -4,6 +4,8 @@ pub(crate) const MCP_PROTOCOL_VERSION: &str = "2025-11-25"; pub(crate) const RESOURCE_STATUS: &str = "trail://workspace/status"; pub(crate) const RESOURCE_DOCTOR: &str = "trail://workspace/doctor"; pub(crate) const RESOURCE_LANES: &str = "trail://workspace/lanes"; +pub(crate) const RESOURCE_ARTIFACT_SPACE: &str = "trail://workspace/artifacts/space"; +pub(crate) const RESOURCE_ARTIFACT_QUARANTINES: &str = "trail://workspace/artifact-quarantines"; pub(crate) const RESOURCE_LANE_MERGE_QUEUE: &str = "trail://workspace/lane-merge-queue"; pub(crate) const RESOURCE_CONFLICTS: &str = "trail://workspace/conflicts"; pub(crate) const RESOURCE_OPENAPI: &str = "trail://workspace/openapi"; @@ -82,6 +84,9 @@ pub(crate) const RESOURCE_CONFLICT_TEMPLATE: &str = "trail://workspace/conflicts pub(crate) const RESOURCE_APPROVAL_TEMPLATE: &str = "trail://workspace/approvals/{approval_id}"; pub(crate) const RESOURCE_RUN_TEMPLATE: &str = "trail://workspace/runs/{run_id}"; pub(crate) const RESOURCE_SPAN_TEMPLATE: &str = "trail://workspace/spans/{span_id}"; +pub(crate) const RESOURCE_ARTIFACT_TEMPLATE: &str = "trail://workspace/artifacts/{artifact_id}"; +pub(crate) const RESOURCE_ARTIFACT_QUARANTINE_TEMPLATE: &str = + "trail://workspace/artifact-quarantines/{quarantine_id}"; pub(crate) const PROMPT_LANE_TASK: &str = "trail.lane_task"; pub(crate) const PROMPT_REVIEW_LANE: &str = "trail.review_lane"; diff --git a/trail/src/mcp/types/lane.rs b/trail/src/mcp/types/lane.rs index 1373e38f..b93d011f 100644 --- a/trail/src/mcp/types/lane.rs +++ b/trail/src/mcp/types/lane.rs @@ -1,5 +1,7 @@ use serde::Deserialize; +use crate::{ArtifactQuarantineResolutionV1, ArtifactVerificationLevelV1}; + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct LaneSpawnArgs { @@ -232,6 +234,71 @@ pub(crate) struct EnvironmentSyncArgs { pub(crate) path: Option, } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct EnvironmentResolveArgs { + #[serde(alias = "lane_or_id", alias = "name")] + pub(crate) lane: String, + #[serde(alias = "component_id")] + pub(crate) component: String, + #[serde(default, alias = "component_root")] + pub(crate) path: Option, + #[serde(default)] + pub(crate) refresh: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct EnvironmentResolveAllArgs { + #[serde(alias = "lane_or_id", alias = "name")] + pub(crate) lane: String, + #[serde(default, alias = "component_root")] + pub(crate) path: Option, + #[serde(default)] + pub(crate) refresh: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ArtifactIdArgs { + #[serde(alias = "artifact_id", alias = "envelope_id")] + pub(crate) artifact: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ArtifactVerifyArgs { + #[serde(alias = "artifact_id", alias = "envelope_id")] + pub(crate) artifact: String, + pub(crate) level: ArtifactVerificationLevelV1, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ArtifactQuarantineArgs { + #[serde(alias = "quarantine_id")] + pub(crate) quarantine: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ArtifactQuarantineResolveArgs { + #[serde(alias = "quarantine_id")] + pub(crate) quarantine: String, + pub(crate) resolution: ArtifactQuarantineResolutionV1, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ArtifactSourceExportArgs { + #[serde(alias = "lane_or_id", alias = "name")] + pub(crate) lane: String, + #[serde(alias = "component_id")] + pub(crate) component: String, + #[serde(alias = "export_name")] + pub(crate) export: String, +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct EnvironmentPromoteArgs { diff --git a/trail/src/model/domain.rs b/trail/src/model/domain.rs index 63d8c0b2..542634a1 100644 --- a/trail/src/model/domain.rs +++ b/trail/src/model/domain.rs @@ -1,8 +1,14 @@ use serde::{Deserialize, Serialize}; -use crate::ids::{AnchorId, ChangeId, FileId, LineId, MessageId, ObjectId, WorkspaceId}; +use crate::ids::{ + AnchorId, ArtifactAttemptId, ArtifactAttestationId, ArtifactBlobId, ArtifactChunkId, + ArtifactChunkListId, ArtifactDesiredKeyV2, ArtifactEnvelopeId, ArtifactFileId, + ArtifactQuarantineId, ArtifactTreeId, ChangeId, FileId, LineId, MessageId, ObjectId, + WorkspaceId, +}; include!("domain/config.rs"); +include!("domain/artifacts.rs"); include!("domain/memory.rs"); include!("domain/objects.rs"); include!("domain/operations.rs"); diff --git a/trail/src/model/domain/artifacts.rs b/trail/src/model/domain/artifacts.rs new file mode 100644 index 00000000..a8f5b2ea --- /dev/null +++ b/trail/src/model/domain/artifacts.rs @@ -0,0 +1,1102 @@ +use std::collections::BTreeMap; + +pub const ARTIFACT_RESOLUTION_PLAN_VERSION: u16 = 1; +pub const ARTIFACT_RESOLUTION_SNAPSHOT_VERSION: u16 = 1; +pub const ARTIFACT_RESOLUTION_SNAPSHOT_KIND: &str = "ArtifactResolutionSnapshot"; +pub const ARTIFACT_RESOLUTION_CONTENT_KIND: &str = "ArtifactResolutionContent"; +pub const ARTIFACT_RESOLUTION_PLAN_KIND: &str = "ArtifactResolutionPlan"; +pub const ARTIFACT_RESOLUTION_CAPTURE_KIND: &str = "ArtifactResolutionCapture"; +pub const ARTIFACT_RESOLUTION_FAILURE_KIND: &str = "ArtifactResolutionFailure"; +pub const ARTIFACT_DIRECTORY_NODE_VERSION: u16 = 1; +pub const ARTIFACT_FILE_NODE_VERSION: u16 = 1; +pub const ARTIFACT_BLOB_VERSION: u16 = 1; +pub const ARTIFACT_CHUNK_LIST_VERSION: u16 = 1; +pub const ARTIFACT_CHUNK_VERSION: u16 = 1; +pub const ARTIFACT_TREE_ROOT_VERSION: u16 = 1; +pub const ARTIFACT_ENVELOPE_VERSION: u16 = 1; +pub const ARTIFACT_DIVERGENCE_EVIDENCE_VERSION: u16 = 1; +pub const ARTIFACT_VALIDATION_RECEIPT_VERSION: u16 = 1; +pub const ARTIFACT_ATTESTATION_VERSION: u16 = 1; + +pub const ARTIFACT_DIRECTORY_NODE_KIND: &str = "ArtifactDirectoryNode"; +pub const ARTIFACT_FILE_NODE_KIND: &str = "ArtifactFileNode"; +pub const ARTIFACT_BLOB_KIND: &str = "ArtifactBlob"; +pub const ARTIFACT_CHUNK_LIST_KIND: &str = "ArtifactChunkList"; +pub const ARTIFACT_CHUNK_KIND: &str = "ArtifactChunk"; +pub const ARTIFACT_TREE_ROOT_KIND: &str = "ArtifactTreeRoot"; +pub const ARTIFACT_ENVELOPE_KIND: &str = "ArtifactEnvelope"; +pub const ARTIFACT_DIVERGENCE_EVIDENCE_KIND: &str = "ArtifactDivergenceEvidence"; +pub const ARTIFACT_VALIDATION_RECEIPT_KIND: &str = "ArtifactValidationReceipt"; +pub const ARTIFACT_ATTESTATION_KIND: &str = "ArtifactAttestation"; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ArtifactDirectoryEntryTargetV1 { + Directory { node_id: ArtifactTreeId }, + File { node_id: ArtifactFileId }, + Symlink { target: String }, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct ArtifactDirectoryEntryV1 { + pub name: String, + pub target: ArtifactDirectoryEntryTargetV1, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactDirectoryNodeV1 { + pub version: u16, + pub entries: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ArtifactFileContentV1 { + Blob { blob_id: ArtifactBlobId }, + Chunks { chunk_list_id: ArtifactChunkListId }, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactFileNodeV1 { + pub version: u16, + pub mode: u32, + pub executable: bool, + pub size_bytes: u64, + pub content_sha256: String, + pub content: ArtifactFileContentV1, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactBlobV1 { + pub version: u16, + pub content_sha256: String, + #[serde(with = "serde_bytes")] + pub bytes: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactChunkV1 { + pub version: u16, + pub content_sha256: String, + #[serde(with = "serde_bytes")] + pub bytes: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactChunkRefV1 { + pub chunk_id: ArtifactChunkId, + pub size_bytes: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactChunkListV1 { + pub version: u16, + pub algorithm: String, + pub file_size_bytes: u64, + pub file_sha256: String, + pub chunks: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactTreeRootV1 { + pub version: u16, + pub root_directory_id: ArtifactTreeId, + pub logical_bytes: u64, + pub entry_count: u64, + pub path_normalizer: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "identity_version", rename_all = "snake_case")] +pub enum ArtifactDesiredIdentityV1 { + WorkspaceLayerV1 { + cache_key: String, + canonical_key: WorkspaceLayerKeyV1, + }, + ArtifactDesiredV2 { + desired_key: ArtifactDesiredKeyV2, + }, +} + +impl ArtifactDesiredIdentityV1 { + pub fn desired_key_v2(&self) -> Option<&ArtifactDesiredKeyV2> { + match self { + Self::WorkspaceLayerV1 { .. } => None, + Self::ArtifactDesiredV2 { desired_key } => Some(desired_key), + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactEnvelopeV1 { + pub version: u16, + pub desired_identity: ArtifactDesiredIdentityV1, + pub tree_root_id: ArtifactTreeId, + pub component_id: String, + pub output_name: String, + pub output_policy: EnvironmentOutputPolicy, + pub portability_scope: String, + pub trust_scope: String, + #[serde(default, skip_serializing_if = "ArtifactSecretTaintV1::is_clear")] + pub secret_taint: ArtifactSecretTaintV1, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolution_snapshot_id: Option, + #[serde(default)] + pub validation_receipt_ids: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactAttestationSignatureV1 { + pub algorithm: String, + pub key_id: String, + pub public_key_hex: String, + pub signature_hex: String, +} + +/// Deterministic host statement about one sealed artifact envelope. +/// +/// Local storage paths and wall-clock observations are intentionally absent. +/// Publisher and package fields identify trust evidence but do not grant trust; +/// attachment rechecks their current local status. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactAttestationStatementV1 { + pub version: u16, + pub envelope_id: ArtifactEnvelopeId, + pub desired_identity: ArtifactDesiredIdentityV1, + pub tree_root_id: ArtifactTreeId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolution_snapshot_id: Option, + #[serde(default)] + pub upstream_identities: BTreeMap, + pub producer_identity: String, + pub producer_trust: ArtifactProducerTrustTierV1, + pub adapter_implementation_version: String, + pub adapter_distribution_digest: String, + pub adapter_protocol: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub publisher: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub publisher_key_id: Option, + #[serde(default)] + pub executable_identities: BTreeMap, + pub platform: String, + pub architecture: String, + pub abi: String, + pub capability_ceiling: ArtifactCapabilityCeilingV1, + pub sandbox_enforcement: String, + pub network_policy: String, + pub script_policy: ArtifactScriptPolicyV1, + pub output_name: String, + pub output_policy: EnvironmentOutputPolicy, + pub portability_scope: String, + pub trust_scope: String, + #[serde(default)] + pub validation_receipt_ids: Vec, + #[serde(default, skip_serializing_if = "ArtifactSecretTaintV1::is_clear")] + pub secret_taint: ArtifactSecretTaintV1, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactAttestationV1 { + pub statement: ArtifactAttestationStatementV1, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signature: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactAttestationReportV1 { + pub attestation_id: ArtifactAttestationId, + pub object_id: ObjectId, + pub state: String, + pub attestation: ArtifactAttestationV1, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactAttestationVerificationReportV1 { + pub attestation_id: ArtifactAttestationId, + pub envelope_id: ArtifactEnvelopeId, + pub state: String, + pub content_identity_valid: bool, + pub envelope_binding_valid: bool, + pub producer_trusted: bool, + pub signature_status: String, + pub valid: bool, + pub diagnostics: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactGenerationBindingReportV1 { + /// Durable database key. Legacy schema-v1 rows use the `binding_` prefix; + /// `binding_identity` carries the portable content identity. + pub binding_id: String, + pub generation_id: String, + pub component_id: String, + pub output_name: String, + pub desired_key: String, + pub envelope_id: ArtifactEnvelopeId, + pub tree_root_id: ArtifactTreeId, + pub binding_identity: String, + pub created_at: i64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactReachabilityKindReportV1 { + pub kind: String, + pub object_count: u64, + pub encoded_bytes: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactContentReachabilityReportV1 { + pub envelope_id: ArtifactEnvelopeId, + pub tree_root_id: ArtifactTreeId, + pub object_count: u64, + pub encoded_bytes: u64, + pub logical_bytes: u64, + pub by_kind: Vec, + pub complete: bool, + pub recovery_commands: Vec, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactVerificationLevelV1 { + Attach, + Sample, + Full, + Reproduce, +} + +impl ArtifactVerificationLevelV1 { + pub fn as_str(self) -> &'static str { + match self { + Self::Attach => "attach", + Self::Sample => "sample", + Self::Full => "full", + Self::Reproduce => "reproduce", + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactInspectionReportV1 { + pub envelope_id: ArtifactEnvelopeId, + pub object_id: ObjectId, + pub desired_key: String, + pub tree_root_id: ArtifactTreeId, + pub state: String, + pub verification_state: String, + pub trust_state: String, + pub quarantine_state: String, + pub envelope: ArtifactEnvelopeV1, + pub bindings: Vec, + pub attestations: Vec, + pub quarantines: Vec, + pub reachability: ArtifactContentReachabilityReportV1, + pub storage: ArtifactStorageAccountingReport, + pub recovery_commands: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactVerificationReportV1 { + pub envelope_id: ArtifactEnvelopeId, + pub level: ArtifactVerificationLevelV1, + pub desired_key: String, + pub tree_root_id: ArtifactTreeId, + pub envelope_state: String, + pub verification_state: String, + pub trust_state: String, + pub quarantine_state: String, + pub content_identity_valid: bool, + pub tree_integrity_valid: bool, + pub validation_receipts_valid: bool, + pub attestations_valid: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reproduction_evidence_valid: Option, + pub valid: bool, + pub diagnostics: Vec, + pub recovery_commands: Vec, + pub reachability: ArtifactContentReachabilityReportV1, + pub storage: ArtifactStorageAccountingReport, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactSpaceReportV1 { + pub scope: String, + pub envelope_count: u64, + pub active_quarantine_count: u64, + pub storage: ArtifactStorageAccountingReport, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactResolutionContentV1 { + pub version: u16, + pub content_sha256: String, + #[serde(with = "serde_bytes")] + pub bytes: Vec, +} + +/// One pinned source input that a resolver may read. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct ArtifactResolutionInputV1 { + pub source_path: String, + pub content_hash: String, + pub size_bytes: u64, +} + +/// The semantic role of an environment variable made visible to a resolver. +/// +/// Values are never part of this contract. Credential material is supplied by +/// an opaque handle at execution time and must not enter durable objects. +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactEnvironmentRoleV1 { + Identity, + Runtime, + CredentialHandle, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactScriptPolicyV1 { + Deny, + AllowDeclared, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactProducerTrustTierV1 { + ReviewedBuiltin, + CertifiedSignedPlugin, + LocallyTrustedPlugin, + RepositoryDeclaration, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactExecutionPhaseV1 { + DiscoveryPlanning, + Resolve, + Construct, + Validate, + MountedExecution, + SourceExport, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactNetworkCapabilityV1 { + Deny, + ExactAuthorities, + ReviewedBuiltinManaged, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactFilesystemReadCapabilityV1 { + None, + DeclaredInputs, + PinnedSourceClosure, + ArtifactCandidate, + LaneView, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactFilesystemWriteCapabilityV1 { + None, + IsolatedCandidate, + CandidateAndHostCache, + ValidationReceipt, + LaneBindings, + SourceExportDestination, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactProcessCapabilityV1 { + Deny, + DeclaredExecutable, + ReviewedBuiltinGraph, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactSecretCapabilityV1 { + Deny, + OpaqueHandles, + RuntimeInjection, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "state", rename_all = "snake_case")] +pub enum ArtifactSecretTaintV1 { + #[default] + Clear, + Tainted { channels: Vec }, +} + +impl ArtifactSecretTaintV1 { + pub fn is_clear(&self) -> bool { + matches!(self, Self::Clear) + } + + pub fn channels(&self) -> &[String] { + match self { + Self::Clear => &[], + Self::Tainted { channels } => channels, + } + } +} + +/// Maximum authority available to one producer tier in one execution phase. +/// +/// This is a host policy result, not an adapter request. Repository and plugin +/// declarations may narrow it but cannot widen it, and publication authority +/// is intentionally absent from every executable phase. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactCapabilityCeilingV1 { + pub producer_trust: ArtifactProducerTrustTierV1, + pub phase: ArtifactExecutionPhaseV1, + pub network: ArtifactNetworkCapabilityV1, + pub filesystem_read: ArtifactFilesystemReadCapabilityV1, + pub filesystem_write: ArtifactFilesystemWriteCapabilityV1, + pub processes: ArtifactProcessCapabilityV1, + pub secrets: ArtifactSecretCapabilityV1, + pub publication_authority: bool, +} + +impl ArtifactCapabilityCeilingV1 { + pub fn for_phase( + producer_trust: ArtifactProducerTrustTierV1, + phase: ArtifactExecutionPhaseV1, + ) -> Self { + use ArtifactExecutionPhaseV1 as Phase; + use ArtifactFilesystemReadCapabilityV1 as Read; + use ArtifactFilesystemWriteCapabilityV1 as Write; + use ArtifactNetworkCapabilityV1 as Network; + use ArtifactProcessCapabilityV1 as Process; + use ArtifactProducerTrustTierV1 as Trust; + use ArtifactSecretCapabilityV1 as Secrets; + + let (network, filesystem_read, filesystem_write, processes, secrets) = match phase { + Phase::DiscoveryPlanning => ( + Network::Deny, + Read::None, + Write::None, + Process::Deny, + Secrets::Deny, + ), + Phase::Resolve => ( + Network::ExactAuthorities, + if producer_trust == Trust::ReviewedBuiltin { + Read::PinnedSourceClosure + } else { + Read::DeclaredInputs + }, + if producer_trust == Trust::ReviewedBuiltin { + Write::CandidateAndHostCache + } else { + Write::IsolatedCandidate + }, + if producer_trust == Trust::ReviewedBuiltin { + Process::ReviewedBuiltinGraph + } else { + Process::DeclaredExecutable + }, + Secrets::OpaqueHandles, + ), + Phase::Construct => match producer_trust { + Trust::ReviewedBuiltin => ( + Network::ReviewedBuiltinManaged, + Read::PinnedSourceClosure, + Write::CandidateAndHostCache, + Process::ReviewedBuiltinGraph, + Secrets::Deny, + ), + Trust::CertifiedSignedPlugin | Trust::LocallyTrustedPlugin => ( + Network::Deny, + Read::DeclaredInputs, + Write::CandidateAndHostCache, + Process::DeclaredExecutable, + Secrets::Deny, + ), + Trust::RepositoryDeclaration => ( + Network::Deny, + Read::DeclaredInputs, + Write::IsolatedCandidate, + Process::DeclaredExecutable, + Secrets::Deny, + ), + }, + Phase::Validate => ( + Network::Deny, + Read::ArtifactCandidate, + Write::ValidationReceipt, + if producer_trust == Trust::ReviewedBuiltin { + Process::ReviewedBuiltinGraph + } else { + Process::DeclaredExecutable + }, + Secrets::Deny, + ), + Phase::MountedExecution => match producer_trust { + Trust::RepositoryDeclaration => ( + Network::Deny, + Read::None, + Write::None, + Process::Deny, + Secrets::Deny, + ), + Trust::ReviewedBuiltin => ( + Network::Deny, + Read::LaneView, + Write::LaneBindings, + Process::ReviewedBuiltinGraph, + Secrets::RuntimeInjection, + ), + Trust::CertifiedSignedPlugin | Trust::LocallyTrustedPlugin => ( + Network::Deny, + Read::LaneView, + Write::LaneBindings, + Process::DeclaredExecutable, + Secrets::RuntimeInjection, + ), + }, + Phase::SourceExport => ( + Network::Deny, + Read::ArtifactCandidate, + Write::SourceExportDestination, + Process::Deny, + Secrets::Deny, + ), + }; + Self { + producer_trust, + phase, + network, + filesystem_read, + filesystem_write, + processes, + secrets, + publication_authority: false, + } + } +} + +/// Finite host-enforced ceilings for a resolver attempt. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactActionLimitsV1 { + pub timeout_ms: u64, + pub stdout_bytes: u64, + pub stderr_bytes: u64, + pub candidate_bytes: u64, + pub candidate_entries: u64, + pub child_processes: u32, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactValidationKindV1 { + Structural, + Loadability, + Framework, + Policy, + Gate, + Reproducibility, + /// Legacy wire value retained for exact compatibility with existing plans. + Ecosystem, +} + +/// A deterministic validation declaration applied before snapshot publication. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct ArtifactValidationV1 { + pub name: String, + pub kind: ArtifactValidationKindV1, + pub required: bool, + #[serde(default)] + pub parameters: BTreeMap, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactValidationOutcomeV1 { + Passed, + Failed, +} + +/// Canonical, path-independent evidence for one host-run validation. +/// +/// Wall-clock time and local filesystem paths are intentionally absent so the +/// same declaration, inputs, validator, result, and bounded evidence produce +/// the same object identity. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactValidationReceiptV1 { + pub version: u16, + pub declaration: ArtifactValidationV1, + pub desired_identity: ArtifactDesiredIdentityV1, + pub tree_root_id: ArtifactTreeId, + pub validator_identity: String, + pub validated_input_digest: String, + pub outcome: ArtifactValidationOutcomeV1, + #[serde(default)] + pub evidence: BTreeMap, +} + +/// A complete, host-validated contract for an optional dependency resolver. +/// +/// The structure contains data only. It grants no authority until Trail checks +/// it against workspace policy and launches the exact executable itself. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactResolutionPlanV1 { + pub version: u16, + pub proposal_key: String, + pub source_root: ObjectId, + pub component_id: String, + pub adapter_identity: String, + pub policy_identity: String, + pub program: String, + pub resolved_program: String, + pub executable_identity: String, + pub argv: Vec, + pub working_directory: String, + pub readable_inputs: Vec, + pub candidate_output: String, + #[serde(default)] + pub allowed_authorities: Vec, + #[serde(default)] + pub credential_handles: Vec, + pub script_policy: ArtifactScriptPolicyV1, + #[serde(default)] + pub environment_roles: BTreeMap, + pub limits: ArtifactActionLimitsV1, + pub snapshot_format: String, + #[serde(default)] + pub validations: Vec, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactResolutionVerificationStateV1 { + Verified, + Rejected, +} + +/// Immutable provenance envelope for one resolver-produced snapshot. +/// +/// `content_object_id` owns the exact snapshot bytes while this object records +/// why those bytes are valid for a proposal. Wall-clock time deliberately does +/// not participate, so identical evidence has identical content identity. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactResolutionSnapshotV1 { + pub version: u16, + pub proposal_key: String, + pub source_root: ObjectId, + pub component_id: String, + pub adapter_identity: String, + pub snapshot_format: String, + pub content_object_id: ObjectId, + pub content_sha256: String, + #[serde(default)] + pub resolved_identities: BTreeMap, + #[serde(default)] + pub checksums: BTreeMap, + pub resolver_executable_identity: String, + pub policy_identity: String, + #[serde(default)] + pub contacted_authorities: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub predecessor_snapshot_id: Option, + #[serde(default, skip_serializing_if = "ArtifactSecretTaintV1::is_clear")] + pub secret_taint: ArtifactSecretTaintV1, + pub verification_state: ArtifactResolutionVerificationStateV1, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactResolutionAttemptStatusV1 { + Running, + Succeeded, + Failed, + Cancelled, + Abandoned, +} + +/// Durable non-secret evidence describing the authority boundary of one +/// resolver attempt. Credential handles are names only; their values are +/// supplied late and never enter this object or an attempt row. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactResolutionAuthorityEvidenceV1 { + pub allowed_authorities: Vec, + pub contacted_authorities: Vec, + pub credential_handles: Vec, + pub credential_values_redacted: bool, +} + +/// One bounded stream captured from a resolver. `original_bytes` allows the +/// failure receipt to explain a limit violation without retaining excess data. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactResolutionCaptureV1 { + pub version: u16, + pub original_bytes: u64, + pub truncated: bool, + #[serde(with = "serde_bytes")] + pub bytes: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactResolutionFailureReceiptV1 { + pub version: u16, + pub attempt_id: ArtifactAttemptId, + pub proposal_key: String, + pub source_root: ObjectId, + pub code: String, + pub message: String, + pub authority_evidence: ArtifactResolutionAuthorityEvidenceV1, + #[serde(default, skip_serializing_if = "ArtifactSecretTaintV1::is_clear")] + pub secret_taint: ArtifactSecretTaintV1, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stdout_object_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stderr_object_id: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactResolutionAttemptReportV1 { + pub attempt_id: ArtifactAttemptId, + pub proposal_key: String, + pub source_root: ObjectId, + pub plan_object_id: ObjectId, + pub owner_generation: u64, + pub owner_pid: u32, + pub status: ArtifactResolutionAttemptStatusV1, + pub cancel_requested: bool, + pub authority_evidence: ArtifactResolutionAuthorityEvidenceV1, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stdout_object_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stderr_object_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub snapshot_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failure_receipt_object_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failure_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failure_message: Option, + pub started_at: i64, + pub heartbeat_at: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finished_at: Option, + pub recovery_command: Vec, +} + +/// Ephemeral output returned by an authorized resolver executor. +/// +/// This value is intentionally not serializable. `redactions` can contain +/// credential bytes used only to scrub bounded diagnostics before Trail makes +/// attempt evidence durable. +#[derive(Clone, PartialEq, Eq)] +pub struct ArtifactResolutionCandidateV1 { + pub snapshot_bytes: Vec, + pub resolved_identities: BTreeMap, + pub checksums: BTreeMap, + pub contacted_authorities: Vec, + pub stdout: Vec, + pub stderr: Vec, + pub redactions: Vec>, +} + +impl std::fmt::Debug for ArtifactResolutionCandidateV1 { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ArtifactResolutionCandidateV1") + .field("snapshot_bytes", &self.snapshot_bytes.len()) + .field("resolved_identities", &self.resolved_identities) + .field("checksums", &self.checksums) + .field("contacted_authorities", &self.contacted_authorities) + .field("stdout_bytes", &self.stdout.len()) + .field("stderr_bytes", &self.stderr.len()) + .field("redactions", &"[REDACTED]") + .finish() + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ArtifactResolutionRequestV1 { + pub plan: ArtifactResolutionPlanV1, + pub candidate: ArtifactResolutionCandidateV1, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactResolutionDecisionV1 { + Resolved, + Reused, + Refreshed, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactResolutionComponentReportV1 { + pub component_id: String, + pub proposal_key: String, + pub source_root: ObjectId, + pub snapshot_id: ObjectId, + pub snapshot: ArtifactResolutionSnapshotV1, + pub decision: ArtifactResolutionDecisionV1, + pub refresh_requested: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub attempt: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactResolutionBatchReportV1 { + pub source_root: ObjectId, + pub refresh_requested: bool, + pub components: Vec, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactActionPhaseV2 { + Resolve, + Construct, + Validate, + Finalize, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct ArtifactActionIdentityV2 { + pub name: String, + pub phase: ArtifactActionPhaseV2, + pub executable_identity: String, + pub argv: Vec, + pub working_directory: String, + pub environment_names: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactSourceClosureV2 { + pub normalizer_version: String, + pub certified_complete: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub complete_source_root: Option, + pub declared_inputs: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct ArtifactOutputContractV2 { + pub name: String, + pub output_path: String, + pub mount_path: String, + pub policy: EnvironmentOutputPolicy, + pub reuse: EnvironmentReuseMode, + pub scope: EnvironmentSharingScope, + pub publish: EnvironmentPublicationTrigger, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gate: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct ArtifactSourceExportContractV2 { + pub name: String, + pub output_name: String, + pub artifact_subpath: String, + pub destination: String, + pub collision_policy: String, + pub required_validation: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub required_gate: Option, + pub authorization_mode: String, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactSourceExportCollisionModeV1 { + Fail, + Replace, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactSourceExportAuthorizationV1 { + ExplicitUser, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ArtifactSourceExportSubtreeV1 { + Directory { node_id: ArtifactTreeId }, + File { node_id: ArtifactFileId }, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactSourceExportDestinationPinV1 { + pub exists: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content_digest: Option, + pub entry_count: u64, + pub logical_bytes: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactSourceExportGatePinV1 { + pub event_id: String, + pub kind: String, + pub source_root: ObjectId, + pub view_id: String, + pub view_generation: u64, +} + +/// Immutable evidence required before generated artifact content may enter +/// normal lane source. Execution must revalidate every pin and write through +/// the ordinary source guardrail/checkpoint path. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactSourceExportPlanV1 { + pub version: u16, + pub lane_id: String, + pub lane: String, + pub component_id: String, + pub export_name: String, + pub output_name: String, + pub source_root: ObjectId, + pub generation_id: String, + pub generation_source_root: ObjectId, + pub desired_identity: ArtifactDesiredIdentityV1, + pub envelope_id: ArtifactEnvelopeId, + pub tree_root_id: ArtifactTreeId, + pub subtree: ArtifactSourceExportSubtreeV1, + pub artifact_subpath: String, + pub destination: String, + pub destination_pin: ArtifactSourceExportDestinationPinV1, + pub collision_mode: ArtifactSourceExportCollisionModeV1, + pub validation_receipt_id: ObjectId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gate: Option, + pub authorization: ArtifactSourceExportAuthorizationV1, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ArtifactSourceExportExecutionReportV1 { + pub plan: ArtifactSourceExportPlanV1, + pub operation: ChangeId, + pub root_id: ObjectId, + pub changed_paths: Vec, + pub checkpointed: bool, + pub git_handoff: Vec, +} + +/// Canonical identity inputs for the framework-neutral artifact pipeline. +/// +/// Secret values and mutable provider allocation IDs have no representation in +/// this structure. Callers must pass only non-secret build environment values. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactDesiredKeyMaterialV2 { + pub version: u16, + pub component_id: String, + pub adapter_identity: String, + pub adapter_implementation_version: String, + pub adapter_distribution_digest: String, + pub adapter_protocol: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolution_snapshot_id: Option, + pub source_closure: ArtifactSourceClosureV2, + pub upstream_identities: BTreeMap, + pub actions: Vec, + pub outputs: Vec, + pub validations: Vec, + pub source_exports: Vec, + pub build_environment: BTreeMap, + pub target: String, + pub platform: String, + pub architecture: String, + pub abi: String, + /// True only when validation evidence permits reuse outside the producing + /// lane. Missing fields decode conservatively as unproven portability. + #[serde(default)] + pub portability_certified: bool, + pub portability_scope: String, + pub trust_scope: String, + pub network_policy: String, + pub script_policy: ArtifactScriptPolicyV1, + pub sandbox_policy: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct ArtifactInvalidationEdgeV2 { + pub dimension: String, + pub name: String, + pub change: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactDesiredKeyDiffV2 { + pub previous_key: ArtifactDesiredKeyV2, + pub current_key: ArtifactDesiredKeyV2, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub first: Option, + pub edges: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactDivergenceEvidenceV1 { + pub version: u16, + pub trust_scope: String, + pub desired_key: String, + pub incumbent_envelope_id: ArtifactEnvelopeId, + pub incumbent_tree_root_id: ArtifactTreeId, + pub candidate_envelope_id: ArtifactEnvelopeId, + pub candidate_tree_root_id: ArtifactTreeId, + pub reason_code: String, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactQuarantineResolutionV1 { + RetainPrivate, + AcceptIncumbent, + AcceptCandidate, + RetireAll, +} + +impl ArtifactQuarantineResolutionV1 { + pub fn as_str(self) -> &'static str { + match self { + Self::RetainPrivate => "retain_private", + Self::AcceptIncumbent => "accept_incumbent", + Self::AcceptCandidate => "accept_candidate", + Self::RetireAll => "retire_all", + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactQuarantineRecordV1 { + pub quarantine_id: ArtifactQuarantineId, + pub trust_scope: String, + pub desired_key: String, + pub incumbent_envelope_id: Option, + pub candidate_envelope_id: ArtifactEnvelopeId, + pub reason_code: String, + pub evidence_object_id: ObjectId, + pub state: String, + pub resolution: Option, + pub created_at: i64, + pub resolved_at: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactQuarantineListReportV1 { + pub active_count: u64, + pub resolved_count: u64, + pub quarantines: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactQuarantineResolutionReportV1 { + pub quarantine: ArtifactQuarantineRecordV1, + pub affected_envelopes: Vec, + pub recovery_commands: Vec, +} diff --git a/trail/src/model/reports/lane.rs b/trail/src/model/reports/lane.rs index 3ab2154a..4eb9c5a3 100644 --- a/trail/src/model/reports/lane.rs +++ b/trail/src/model/reports/lane.rs @@ -299,6 +299,26 @@ pub struct EnvironmentAdapterIdentityReport { pub distribution_digest: Option, } +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentPluginProtocolCapabilitiesReport { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selected_protocol: Option, + #[serde(default)] + pub resolution_capable: bool, + #[serde(default)] + pub source_export_capable: bool, + #[serde(default)] + pub host_attestation_evidence_capable: bool, + #[serde(default)] + pub host_quarantine_evidence_capable: bool, + #[serde(default)] + pub certification_ceiling: String, + #[serde(default)] + pub content_policy: String, + #[serde(default)] + pub attestation_policy: String, +} + /// One adapter available to the environment host. /// /// Catalog entries describe discovery and compatibility only. They never grant @@ -315,6 +335,8 @@ pub struct EnvironmentAdapterCatalogEntryReport { /// Built-ins and repository recipes use the in-process host contract and /// therefore report an empty list. pub protocols: Vec, + #[serde(default)] + pub protocol_capabilities: EnvironmentPluginProtocolCapabilitiesReport, pub supported_operating_systems: Vec, pub supported_architectures: Vec, pub source: String, @@ -343,6 +365,8 @@ pub struct EnvironmentPluginInstallReport { pub publisher_key_id: Option, pub trust: String, pub certification_tier: String, + #[serde(default)] + pub protocol_capabilities: EnvironmentPluginProtocolCapabilitiesReport, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -354,12 +378,16 @@ pub struct EnvironmentPluginPackageInspectionReport { pub signature_present: bool, pub publisher: Option, pub publisher_key_id: Option, + #[serde(default)] + pub protocol_capabilities: EnvironmentPluginProtocolCapabilitiesReport, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct EnvironmentPluginRemoveReport { pub canonical_identity: String, pub removed_distribution_digest: Option, + #[serde(default)] + pub protocol_capabilities: EnvironmentPluginProtocolCapabilitiesReport, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -744,12 +772,55 @@ pub struct EnvironmentGenerationReport { pub retired_at: Option, } +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentComponentProposalStatus { + #[default] + Ready, + Resolvable, + Blocked, + Unsupported, + Ambiguous, +} + +impl EnvironmentComponentProposalStatus { + pub fn as_str(&self) -> &'static str { + match self { + Self::Ready => "ready", + Self::Resolvable => "resolvable", + Self::Blocked => "blocked", + Self::Unsupported => "unsupported", + Self::Ambiguous => "ambiguous", + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentProposalReasonReport { + pub code: String, + pub message: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentRecoveryActionReport { + pub code: String, + pub description: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub command: Option>, +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct EnvironmentDiscoveredComponentReport { pub component_id: String, pub component_root: String, pub kind: String, pub adapter_identity: String, + #[serde(default)] + pub status: EnvironmentComponentProposalStatus, + #[serde(default)] + pub reasons: Vec, + #[serde(default)] + pub recovery_actions: Vec, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -948,6 +1019,29 @@ pub enum PhysicalSharing { Unknown, } +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactStorageAccountingReport { + /// Sum of logical file bytes across distinct artifact tree roots in scope. + pub logical_bytes: u64, + /// Encoded authoritative object bytes referenced by exactly one artifact. + pub unique_authoritative_bytes: u64, + /// Encoded authoritative object bytes referenced by multiple artifacts. + pub cross_artifact_shared_bytes: u64, + /// Filesystem-allocated bytes in reconstructible artifact/layer materializations. + pub materialized_bytes: u64, + /// Filesystem-allocated bytes owned by lane-private source/generated/scratch state. + pub lane_private_bytes: u64, + /// Persisted bytes created specifically by prefetch. OS page-cache warming is excluded. + pub prefetched_bytes: u64, + /// Filesystem-allocated bytes in content projections created on demand. + pub demand_loaded_bytes: u64, + /// Independently reclaimable cache bytes in the report scope. + pub reclaimable_bytes: u64, + /// Measured bytes whose artifact/cache ownership cannot be classified safely. + pub unknown_bytes: u64, + /// Exact byte bases and attribution boundary used by this report. + pub accounting: String, +} #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct WorkspaceSpaceReport { @@ -975,6 +1069,8 @@ pub struct WorkspaceSpaceReport { pub physical_sharing: PhysicalSharing, #[serde(default)] pub physical_sharing_evidence: String, + #[serde(default)] + pub artifact_storage: ArtifactStorageAccountingReport, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -1001,6 +1097,191 @@ pub struct WorkspaceExecReport { pub lifecycle: ManagedExecutionLifecycleReport, } +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ManagedExecutionMissingResolutionPolicy { + #[default] + Explicit, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ManagedExecutionResolutionPin { + pub component_id: String, + pub adapter_identity: String, + pub status: EnvironmentComponentProposalStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub proposal_key: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub snapshot_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_command: Option>, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ManagedExecutionOutputPin { + pub component_id: String, + pub output_name: String, + pub component_key: String, + pub policy: EnvironmentOutputPolicy, + pub storage_identity: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub artifact_binding_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub artifact_envelope_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub artifact_tree_root_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub artifact_binding_identity: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ManagedExecutionPreparationReceipt { + pub source_root: ObjectId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub view_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub view_generation: Option, + pub missing_resolution_policy: ManagedExecutionMissingResolutionPolicy, + pub resolution_pins: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub environment_generation: Option, + pub output_pins: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ManagedExecutionSealingDecision { + pub component_id: String, + pub output_name: String, + pub policy: EnvironmentOutputPolicy, + pub publication: EnvironmentPublicationTrigger, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gate: Option, + pub decision: String, + pub reason: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ManagedExecutionFinalizationReceipt { + pub source_root_before: ObjectId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_root_after: Option, + pub source_changed: bool, + pub checkpoint_status: String, + pub disposal_status: String, + pub unmount_status: String, + pub complete: bool, + pub sealing_decisions: Vec, + pub errors: Vec, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactAdapterConformanceStatusV1 { + Passed, + Failed, + Skipped, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactAdapterConformanceCheckV1 { + pub stage: String, + pub applicable: bool, + pub status: ArtifactAdapterConformanceStatusV1, + pub evidence: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactAdapterCertificationReportV1 { + pub schema: String, + pub producer_family: String, + pub adapter_identity: String, + pub trust_tier: ArtifactProducerTrustTierV1, + pub status: ArtifactAdapterConformanceStatusV1, + pub authority_effect: String, + pub checks: Vec, +} + +impl ArtifactAdapterCertificationReportV1 { + pub fn validate(&self) -> std::result::Result<(), String> { + if self.schema != "trail.artifact-adapter-certification/v1" { + return Err("unsupported artifact adapter certification schema".to_string()); + } + if self.authority_effect != "evidence_only" { + return Err("artifact adapter certification cannot grant authority".to_string()); + } + if self.producer_family.is_empty() + || self.producer_family.len() > 64 + || self.adapter_identity.is_empty() + || self.adapter_identity.len() > 256 + { + return Err("artifact adapter certification identity is empty or oversized".to_string()); + } + let expected = [ + "discovery", + "resolution", + "identity", + "validation", + "sealing", + "cow", + "recovery", + "invalidation", + "export", + "retirement", + "collection", + ]; + if self.checks.len() != expected.len() + || self + .checks + .iter() + .zip(expected) + .any(|(check, expected)| check.stage != expected) + { + return Err("artifact adapter certification checks are not complete and canonical" + .to_string()); + } + for check in &self.checks { + if check.evidence.is_empty() + || check.evidence.len() > 32 + || check + .evidence + .iter() + .any(|item| item.is_empty() || item.len() > 512) + { + return Err(format!( + "artifact adapter certification stage `{}` has empty or oversized evidence", + check.stage + )); + } + if !check + .evidence + .windows(2) + .all(|pair| pair[0] < pair[1]) + { + return Err(format!( + "artifact adapter certification stage `{}` evidence is not canonical", + check.stage + )); + } + if check.applicable + && check.status != ArtifactAdapterConformanceStatusV1::Passed + && self.status == ArtifactAdapterConformanceStatusV1::Passed + { + return Err(format!( + "artifact adapter certification cannot pass while required stage `{}` is not passed", + check.stage + )); + } + if !check.applicable && check.status != ArtifactAdapterConformanceStatusV1::Skipped { + return Err(format!( + "non-applicable artifact adapter certification stage `{}` must be skipped", + check.stage + )); + } + } + Ok(()) + } +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct ManagedExecutionPhaseReceipt { pub phase: String, @@ -1035,6 +1316,8 @@ pub struct ManagedExecutionLifecycleReport { pub surface: String, pub command_fingerprint: String, #[serde(default, skip_serializing_if = "Option::is_none")] + pub preparation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub environment_generation: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub checkpoint: Option, @@ -1046,6 +1329,8 @@ pub struct ManagedExecutionLifecycleReport { pub disposal_error: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub recorded: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finalization: Option, pub phases: Vec, } @@ -1104,6 +1389,8 @@ pub struct WorkspaceCacheGcReport { pub reclaimed_bytes: u64, pub candidates: Vec, pub deleted: Vec, + #[serde(default)] + pub artifact_storage: ArtifactStorageAccountingReport, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -1456,6 +1743,153 @@ mod workdir_mode_tests { assert_eq!(report.backend(), WorkdirBackend::Clone); } + #[test] + fn legacy_discovered_component_defaults_to_ready_proposal_status() { + let report: EnvironmentDiscoveredComponentReport = serde_json::from_value( + serde_json::json!({ + "component_id": "node", + "component_root": "", + "kind": "dependency", + "adapter_identity": "trail/node@1" + }), + ) + .unwrap(); + assert_eq!(report.status, EnvironmentComponentProposalStatus::Ready); + assert!(report.reasons.is_empty()); + assert!(report.recovery_actions.is_empty()); + + let value = serde_json::to_value(report).unwrap(); + assert_eq!(value["status"], "ready"); + assert_eq!(value["reasons"], serde_json::json!([])); + assert_eq!(value["recovery_actions"], serde_json::json!([])); + } + + #[test] + fn managed_execution_receipts_are_additive_and_wire_stable() { + let legacy: ManagedExecutionLifecycleReport = serde_json::from_value( + serde_json::json!({ + "execution_id": "exec-legacy", + "surface": "lane_exec", + "command_fingerprint": "command", + "phases": [] + }), + ) + .unwrap(); + assert!(legacy.preparation.is_none()); + assert!(legacy.finalization.is_none()); + + let current: ManagedExecutionLifecycleReport = serde_json::from_value( + serde_json::json!({ + "execution_id": "exec-current", + "surface": "lane_test", + "command_fingerprint": "command", + "preparation": { + "source_root": "object_source", + "view_id": "view-1", + "view_generation": 7, + "missing_resolution_policy": "explicit", + "resolution_pins": [{ + "component_id": "node", + "adapter_identity": "trail/node@1", + "status": "ready", + "snapshot_id": "object_lock" + }], + "environment_generation": "generation-1", + "output_pins": [{ + "component_id": "node", + "output_name": "dependencies", + "component_key": "desired-key", + "policy": "immutable_seed_private", + "storage_identity": "storage-key" + }] + }, + "finalization": { + "source_root_before": "object_source", + "source_root_after": "object_after", + "source_changed": true, + "checkpoint_status": "succeeded", + "disposal_status": "succeeded", + "unmount_status": "succeeded", + "complete": true, + "sealing_decisions": [{ + "component_id": "node", + "output_name": "dependencies", + "policy": "immutable_seed_private", + "publication": "never", + "decision": "retain_private_delta", + "reason": "private copy-on-write output is never published" + }], + "errors": [] + }, + "phases": [] + }), + ) + .unwrap(); + let value = serde_json::to_value(current).unwrap(); + assert_eq!( + value["preparation"]["missing_resolution_policy"], + "explicit" + ); + assert_eq!( + value["preparation"]["resolution_pins"][0]["status"], + "ready" + ); + assert_eq!( + value["finalization"]["sealing_decisions"][0]["decision"], + "retain_private_delta" + ); + } + + #[test] + fn artifact_adapter_certification_is_complete_canonical_and_evidence_only() { + let stages = [ + "discovery", + "resolution", + "identity", + "validation", + "sealing", + "cow", + "recovery", + "invalidation", + "export", + "retirement", + "collection", + ]; + let report = ArtifactAdapterCertificationReportV1 { + schema: "trail.artifact-adapter-certification/v1".into(), + producer_family: "plugin_v3".into(), + adapter_identity: "example/fixture@1".into(), + trust_tier: ArtifactProducerTrustTierV1::LocallyTrustedPlugin, + status: ArtifactAdapterConformanceStatusV1::Passed, + authority_effect: "evidence_only".into(), + checks: stages + .into_iter() + .map(|stage| ArtifactAdapterConformanceCheckV1 { + stage: stage.into(), + applicable: true, + status: ArtifactAdapterConformanceStatusV1::Passed, + evidence: vec![format!("fixture:{stage}")], + }) + .collect(), + }; + report.validate().unwrap(); + let value = serde_json::to_value(&report).unwrap(); + assert_eq!(value["status"], "passed"); + assert_eq!(value["authority_effect"], "evidence_only"); + assert_eq!(value["checks"][0]["stage"], "discovery"); + assert_eq!(value["checks"][10]["stage"], "collection"); + + let mut skipped = report.clone(); + skipped.checks[5].status = ArtifactAdapterConformanceStatusV1::Skipped; + assert!(skipped.validate().unwrap_err().contains("required stage `cow`")); + let mut authority = report; + authority.authority_effect = "certified_signed_plugin".into(); + assert!(authority + .validate() + .unwrap_err() + .contains("cannot grant authority")); + } + #[test] fn legacy_patch_and_record_reports_default_path_index_metrics() { let patch: LanePatchReport = serde_json::from_value(serde_json::json!({ @@ -1500,10 +1934,133 @@ mod workdir_mode_tests { assert_eq!(report.clone_count, 0); assert_eq!(report.physical_sharing, PhysicalSharing::Unknown); assert_eq!(report.physical_sharing_evidence, ""); + assert_eq!( + report.artifact_storage, + ArtifactStorageAccountingReport::default() + ); + + let cache: WorkspaceCacheGcReport = serde_json::from_value(serde_json::json!({ + "dry_run": true, + "retention_secs": 60, + "cache_physical_bytes_before": 10, + "reclaimable_bytes": 5, + "reclaimed_bytes": 0, + "candidates": [], + "deleted": [] + })) + .unwrap(); + assert_eq!( + cache.artifact_storage, + ArtifactStorageAccountingReport::default() + ); + } + + #[test] + fn artifact_storage_accounting_wire_fields_are_stable() { + let report = ArtifactStorageAccountingReport { + logical_bytes: 1, + unique_authoritative_bytes: 2, + cross_artifact_shared_bytes: 3, + materialized_bytes: 4, + lane_private_bytes: 5, + prefetched_bytes: 6, + demand_loaded_bytes: 7, + reclaimable_bytes: 8, + unknown_bytes: 9, + accounting: "fixture".into(), + }; + assert_eq!( + serde_json::to_value(report).unwrap(), + serde_json::json!({ + "logical_bytes": 1, + "unique_authoritative_bytes": 2, + "cross_artifact_shared_bytes": 3, + "materialized_bytes": 4, + "lane_private_bytes": 5, + "prefetched_bytes": 6, + "demand_loaded_bytes": 7, + "reclaimable_bytes": 8, + "unknown_bytes": 9, + "accounting": "fixture" + }) + ); } #[test] fn environment_artifact_contract_enums_are_stable_and_reject_unknown_values() { + for (kind, wire) in [ + (ArtifactValidationKindV1::Structural, "structural"), + (ArtifactValidationKindV1::Loadability, "loadability"), + (ArtifactValidationKindV1::Framework, "framework"), + (ArtifactValidationKindV1::Policy, "policy"), + (ArtifactValidationKindV1::Gate, "gate"), + ( + ArtifactValidationKindV1::Reproducibility, + "reproducibility", + ), + (ArtifactValidationKindV1::Ecosystem, "ecosystem"), + ] { + assert_eq!(serde_json::to_value(kind).unwrap(), wire); + assert_eq!( + serde_json::from_value::(wire.into()).unwrap(), + kind + ); + } + assert!( + serde_json::from_value::("custom".into()).is_err() + ); + for (tier, wire) in [ + ( + ArtifactProducerTrustTierV1::ReviewedBuiltin, + "reviewed_builtin", + ), + ( + ArtifactProducerTrustTierV1::CertifiedSignedPlugin, + "certified_signed_plugin", + ), + ( + ArtifactProducerTrustTierV1::LocallyTrustedPlugin, + "locally_trusted_plugin", + ), + ( + ArtifactProducerTrustTierV1::RepositoryDeclaration, + "repository_declaration", + ), + ] { + assert_eq!(serde_json::to_value(tier).unwrap(), wire); + assert_eq!( + serde_json::from_value::(wire.into()).unwrap(), + tier + ); + } + assert!( + serde_json::from_value::("remote_trusted".into()) + .is_err() + ); + for (phase, wire) in [ + ( + ArtifactExecutionPhaseV1::DiscoveryPlanning, + "discovery_planning", + ), + (ArtifactExecutionPhaseV1::Resolve, "resolve"), + (ArtifactExecutionPhaseV1::Construct, "construct"), + (ArtifactExecutionPhaseV1::Validate, "validate"), + ( + ArtifactExecutionPhaseV1::MountedExecution, + "mounted_execution", + ), + (ArtifactExecutionPhaseV1::SourceExport, "source_export"), + ] { + assert_eq!(serde_json::to_value(phase).unwrap(), wire); + assert_eq!( + serde_json::from_value::(wire.into()).unwrap(), + phase + ); + } + assert!( + serde_json::from_value::("publish".into()).is_err() + ); + for (policy, wire) in [ (EnvironmentOutputPolicy::ImmutableShared, "immutable_shared"), ( diff --git a/trail/src/model/reports/maintenance.rs b/trail/src/model/reports/maintenance.rs index 413b69b0..9bfafa57 100644 --- a/trail/src/model/reports/maintenance.rs +++ b/trail/src/model/reports/maintenance.rs @@ -289,6 +289,16 @@ pub struct BackupCreateReport { pub sqlite_bytes: u64, pub sqlite_sha256: String, pub worktree_bytes: u64, + #[serde(default)] + pub retained_private_views: u64, + #[serde(default)] + pub retained_private_bytes: u64, + #[serde(default)] + pub rebuildable_materializations: u64, + #[serde(default)] + pub rebuildable_materialization_bytes: u64, + #[serde(default)] + pub rebuildable_performance_caches: u64, pub fsck_errors: Vec, } @@ -303,6 +313,16 @@ pub struct BackupVerifyReport { pub checked_texts: u64, pub sqlite_bytes: Option, pub sqlite_sha256: Option, + #[serde(default)] + pub retained_private_views: u64, + #[serde(default)] + pub retained_private_bytes: u64, + #[serde(default)] + pub rebuildable_materializations: u64, + #[serde(default)] + pub rebuildable_materialization_bytes: u64, + #[serde(default)] + pub rebuildable_performance_caches: u64, pub errors: Vec, } @@ -316,6 +336,16 @@ pub struct BackupRestoreReport { pub replaced_existing: bool, pub restored_trailignore: bool, pub rewritten_workdirs: u64, + #[serde(default)] + pub restored_private_views: u64, + #[serde(default)] + pub retained_private_bytes: u64, + #[serde(default)] + pub rebuildable_materializations: u64, + #[serde(default)] + pub rebuildable_materialization_bytes: u64, + #[serde(default)] + pub rebuildable_performance_caches: u64, pub checked_refs: u64, pub checked_roots: u64, pub checked_texts: u64, @@ -363,4 +393,28 @@ mod maintenance_tests { assert!(report.path_index_repaired_roots.is_empty()); assert!(report.path_index_repaired_refs.is_empty()); } + + #[test] + fn legacy_backup_restore_report_defaults_portable_artifact_fields() { + let report: BackupRestoreReport = serde_json::from_value(serde_json::json!({ + "workspace": "/workspace", + "db_dir": "/workspace/.trail", + "backup_path": "/backup", + "workspace_id": "workspace_00000000000000000000000000000000", + "branch": "main", + "replaced_existing": false, + "restored_trailignore": false, + "rewritten_workdirs": 0, + "checked_refs": 0, + "checked_roots": 0, + "checked_texts": 0 + })) + .unwrap(); + + assert_eq!(report.restored_private_views, 0); + assert_eq!(report.retained_private_bytes, 0); + assert_eq!(report.rebuildable_materializations, 0); + assert_eq!(report.rebuildable_materialization_bytes, 0); + assert_eq!(report.rebuildable_performance_caches, 0); + } } diff --git a/trail/src/server/openapi/paths/lanes.rs b/trail/src/server/openapi/paths/lanes.rs index 0885b9a3..7a611b98 100644 --- a/trail/src/server/openapi/paths/lanes.rs +++ b/trail/src/server/openapi/paths/lanes.rs @@ -10,6 +10,37 @@ pub(super) fn lane_paths() -> Value { "/v1/environment/adapters": { "get": openapi_operation_with_response_schema("environmentAdapterCatalog", "Workspace environment adapters", "List registered adapters and their side-effect-free discovery metadata, provenance, and stability.", vec![], None, "EnvironmentAdapterCatalogReport", true) }, + "/v1/artifacts/space": { + "get": openapi_operation_with_response_schema("artifactSpace", "Artifact storage accounting", "Report CAS-aware logical, authoritative, shared, materialized, private, and reclaimable storage.", vec![], None, "ArtifactSpaceReportV1", true) + }, + "/v1/artifacts/{artifact_id}": { + "get": openapi_operation_with_response_schema("artifactInspect", "Inspect artifact", "Inspect one artifact envelope, content identity, bindings, attestations, quarantine state, reachability, and storage.", vec![ + openapi_path_param("artifact_id", "string") + ], None, "ArtifactInspectionReportV1", true) + }, + "/v1/artifacts/{artifact_id}/reachability": { + "get": openapi_operation_with_response_schema("artifactReachability", "Inspect artifact reachability", "Trace the authoritative content objects reachable from one artifact envelope.", vec![ + openapi_path_param("artifact_id", "string") + ], None, "ArtifactContentReachabilityReportV1", true) + }, + "/v1/artifacts/{artifact_id}/verify": { + "post": openapi_operation_with_response_schema("artifactVerify", "Verify artifact", "Verify one artifact at attach, sample, full, or reproduce level.", vec![ + openapi_path_param("artifact_id", "string") + ], Some("ArtifactVerifyRequest"), "ArtifactVerificationReportV1", true) + }, + "/v1/artifact-quarantines": { + "get": openapi_operation_with_response_schema("artifactQuarantineList", "List artifact quarantines", "List durable artifact nondeterminism quarantines.", vec![], None, "ArtifactQuarantineListReportV1", true) + }, + "/v1/artifact-quarantines/{quarantine_id}": { + "get": openapi_operation_with_response_schema("artifactQuarantineShow", "Show artifact quarantine", "Show one quarantine and its retained divergence evidence.", vec![ + openapi_path_param("quarantine_id", "string") + ], None, "ArtifactQuarantineRecordV1", true) + }, + "/v1/artifact-quarantines/{quarantine_id}/resolve": { + "post": openapi_operation_with_response_schema("artifactQuarantineResolve", "Resolve artifact quarantine", "Apply one explicit audited quarantine resolution without relabeling artifact bytes.", vec![ + openapi_path_param("quarantine_id", "string") + ], Some("ArtifactQuarantineResolveRequest"), "ArtifactQuarantineResolutionReportV1", true) + }, "/v1/lanes": { "get": openapi_operation("laneList", "List lanes", "List lane branches with metadata and branch state.", vec![], None, true), "post": lane_spawn_operation() @@ -136,6 +167,21 @@ pub(super) fn lane_paths() -> Value { openapi_query("path", "string") ], None, "EnvironmentDiscoveryReport", true) }, + "/v1/lanes/{lane_or_id}/environment/resolve": { + "post": openapi_operation_with_response_schema("laneEnvironmentResolve", "Resolve workspace environment component", "Run the authorized host-owned resolver for one component or reuse its pinned immutable snapshot.", vec![ + openapi_path_param("lane_or_id", "string") + ], Some("EnvironmentResolveRequest"), "ArtifactResolutionComponentReportV1", true) + }, + "/v1/lanes/{lane_or_id}/environment/resolve-all": { + "post": openapi_operation_with_response_schema("laneEnvironmentResolveAll", "Resolve all workspace environment components", "Resolve every incomplete component in deterministic graph order or reuse pinned immutable snapshots.", vec![ + openapi_path_param("lane_or_id", "string") + ], Some("EnvironmentResolveAllRequest"), "ArtifactResolutionBatchReportV1", true) + }, + "/v1/lanes/{lane_or_id}/environment/source-export": { + "post": openapi_operation_with_response_schema("laneEnvironmentSourceExport", "Export generated source", "Revalidate and export a declared artifact subtree through normal lane source guardrails and checkpoint semantics.", vec![ + openapi_path_param("lane_or_id", "string") + ], Some("ArtifactSourceExportRequest"), "ArtifactSourceExportExecutionReportV1", true) + }, "/v1/lanes/{lane_or_id}/environment/graph": { "get": openapi_operation_with_response_schema("laneEnvironmentGraph", "Desired environment graph", "Return the validated component DAG, deterministic topological order, output ownership, component keys, and ordering/invalidation edges without executing tools or mutating state.", vec![ openapi_path_param("lane_or_id", "string"), diff --git a/trail/src/server/openapi/schemas/lane.rs b/trail/src/server/openapi/schemas/lane.rs index 361a0313..90d2d46b 100644 --- a/trail/src/server/openapi/schemas/lane.rs +++ b/trail/src/server/openapi/schemas/lane.rs @@ -2,6 +2,236 @@ use serde_json::{json, Value}; pub(super) fn lane_schemas() -> Value { json!({ + "EnvironmentResolveRequest": { + "type": "object", + "required": ["component"], + "properties": { + "component": { "type": "string" }, + "path": { "type": ["string", "null"] }, + "refresh": { "type": "boolean", "default": false } + } + }, + "EnvironmentResolveAllRequest": { + "type": "object", + "properties": { + "path": { "type": ["string", "null"] }, + "refresh": { "type": "boolean", "default": false } + } + }, + "ArtifactVerifyRequest": { + "type": "object", + "required": ["level"], + "properties": { + "level": { "type": "string", "enum": ["attach", "sample", "full", "reproduce"] } + } + }, + "ArtifactQuarantineResolveRequest": { + "type": "object", + "required": ["resolution"], + "properties": { + "resolution": { "type": "string", "enum": ["retain_private", "accept_incumbent", "accept_candidate", "retire_all"] } + } + }, + "ArtifactSourceExportRequest": { + "type": "object", + "required": ["component", "export"], + "properties": { + "component": { "type": "string" }, + "export": { "type": "string" } + } + }, + "ArtifactStorageAccountingReport": { + "type": "object", + "required": ["logical_bytes", "unique_authoritative_bytes", "cross_artifact_shared_bytes", "materialized_bytes", "lane_private_bytes", "prefetched_bytes", "demand_loaded_bytes", "reclaimable_bytes", "unknown_bytes", "accounting"], + "additionalProperties": false, + "properties": { + "logical_bytes": { "type": "integer", "minimum": 0 }, + "unique_authoritative_bytes": { "type": "integer", "minimum": 0 }, + "cross_artifact_shared_bytes": { "type": "integer", "minimum": 0 }, + "materialized_bytes": { "type": "integer", "minimum": 0 }, + "lane_private_bytes": { "type": "integer", "minimum": 0 }, + "prefetched_bytes": { "type": "integer", "minimum": 0 }, + "demand_loaded_bytes": { "type": "integer", "minimum": 0 }, + "reclaimable_bytes": { "type": "integer", "minimum": 0 }, + "unknown_bytes": { "type": "integer", "minimum": 0 }, + "accounting": { "type": "string" } + } + }, + "ArtifactReachabilityKindReportV1": { + "type": "object", + "required": ["kind", "object_count", "encoded_bytes"], + "additionalProperties": false, + "properties": { + "kind": { "type": "string" }, + "object_count": { "type": "integer", "minimum": 0 }, + "encoded_bytes": { "type": "integer", "minimum": 0 } + } + }, + "ArtifactContentReachabilityReportV1": { + "type": "object", + "required": ["envelope_id", "tree_root_id", "object_count", "encoded_bytes", "logical_bytes", "by_kind", "complete", "recovery_commands"], + "additionalProperties": false, + "properties": { + "envelope_id": { "type": "string" }, + "tree_root_id": { "type": "string" }, + "object_count": { "type": "integer", "minimum": 0 }, + "encoded_bytes": { "type": "integer", "minimum": 0 }, + "logical_bytes": { "type": "integer", "minimum": 0 }, + "by_kind": { "type": "array", "items": { "$ref": "#/components/schemas/ArtifactReachabilityKindReportV1" } }, + "complete": { "type": "boolean" }, + "recovery_commands": { "type": "array", "items": { "type": "string" } } + } + }, + "ArtifactGenerationBindingReportV1": { + "type": "object", + "required": ["binding_id", "generation_id", "component_id", "output_name", "desired_key", "envelope_id", "tree_root_id", "binding_identity", "created_at"], + "additionalProperties": false, + "properties": { + "binding_id": { "type": "string" }, + "generation_id": { "type": "string" }, + "component_id": { "type": "string" }, + "output_name": { "type": "string" }, + "desired_key": { "type": "string" }, + "envelope_id": { "type": "string" }, + "tree_root_id": { "type": "string" }, + "binding_identity": { "type": "string" }, + "created_at": { "type": "integer" } + } + }, + "ArtifactQuarantineRecordV1": { + "type": "object", + "required": ["quarantine_id", "trust_scope", "desired_key", "incumbent_envelope_id", "candidate_envelope_id", "reason_code", "evidence_object_id", "state", "resolution", "created_at", "resolved_at"], + "additionalProperties": false, + "properties": { + "quarantine_id": { "type": "string" }, + "trust_scope": { "type": "string" }, + "desired_key": { "type": "string" }, + "incumbent_envelope_id": { "type": ["string", "null"] }, + "candidate_envelope_id": { "type": "string" }, + "reason_code": { "type": "string" }, + "evidence_object_id": { "type": "string" }, + "state": { "type": "string" }, + "resolution": { "type": ["string", "null"] }, + "created_at": { "type": "integer" }, + "resolved_at": { "type": ["integer", "null"] } + } + }, + "ArtifactQuarantineListReportV1": { + "type": "object", + "required": ["active_count", "resolved_count", "quarantines"], + "additionalProperties": false, + "properties": { + "active_count": { "type": "integer", "minimum": 0 }, + "resolved_count": { "type": "integer", "minimum": 0 }, + "quarantines": { "type": "array", "items": { "$ref": "#/components/schemas/ArtifactQuarantineRecordV1" } } + } + }, + "ArtifactQuarantineResolutionReportV1": { + "type": "object", + "required": ["quarantine", "affected_envelopes", "recovery_commands"], + "additionalProperties": false, + "properties": { + "quarantine": { "$ref": "#/components/schemas/ArtifactQuarantineRecordV1" }, + "affected_envelopes": { "type": "array", "items": { "type": "string" } }, + "recovery_commands": { "type": "array", "items": { "type": "string" } } + } + }, + "ArtifactSpaceReportV1": { + "type": "object", + "required": ["scope", "envelope_count", "active_quarantine_count", "storage"], + "additionalProperties": false, + "properties": { + "scope": { "type": "string" }, + "envelope_count": { "type": "integer", "minimum": 0 }, + "active_quarantine_count": { "type": "integer", "minimum": 0 }, + "storage": { "$ref": "#/components/schemas/ArtifactStorageAccountingReport" } + } + }, + "ArtifactInspectionReportV1": { + "type": "object", + "required": ["envelope_id", "object_id", "desired_key", "tree_root_id", "state", "verification_state", "trust_state", "quarantine_state", "envelope", "bindings", "attestations", "quarantines", "reachability", "storage", "recovery_commands"], + "additionalProperties": false, + "properties": { + "envelope_id": { "type": "string" }, + "object_id": { "type": "string" }, + "desired_key": { "type": "string" }, + "tree_root_id": { "type": "string" }, + "state": { "type": "string" }, + "verification_state": { "type": "string" }, + "trust_state": { "type": "string" }, + "quarantine_state": { "type": "string" }, + "envelope": { "$ref": "#/components/schemas/JsonValue" }, + "bindings": { "type": "array", "items": { "$ref": "#/components/schemas/ArtifactGenerationBindingReportV1" } }, + "attestations": { "type": "array", "items": { "$ref": "#/components/schemas/JsonValue" } }, + "quarantines": { "type": "array", "items": { "$ref": "#/components/schemas/ArtifactQuarantineRecordV1" } }, + "reachability": { "$ref": "#/components/schemas/ArtifactContentReachabilityReportV1" }, + "storage": { "$ref": "#/components/schemas/ArtifactStorageAccountingReport" }, + "recovery_commands": { "type": "array", "items": { "type": "string" } } + } + }, + "ArtifactVerificationReportV1": { + "type": "object", + "required": ["envelope_id", "level", "desired_key", "tree_root_id", "envelope_state", "verification_state", "trust_state", "quarantine_state", "content_identity_valid", "tree_integrity_valid", "validation_receipts_valid", "attestations_valid", "valid", "diagnostics", "recovery_commands", "reachability", "storage"], + "additionalProperties": false, + "properties": { + "envelope_id": { "type": "string" }, + "level": { "type": "string", "enum": ["attach", "sample", "full", "reproduce"] }, + "desired_key": { "type": "string" }, + "tree_root_id": { "type": "string" }, + "envelope_state": { "type": "string" }, + "verification_state": { "type": "string" }, + "trust_state": { "type": "string" }, + "quarantine_state": { "type": "string" }, + "content_identity_valid": { "type": "boolean" }, + "tree_integrity_valid": { "type": "boolean" }, + "validation_receipts_valid": { "type": "boolean" }, + "attestations_valid": { "type": "boolean" }, + "reproduction_evidence_valid": { "type": ["boolean", "null"] }, + "valid": { "type": "boolean" }, + "diagnostics": { "type": "array", "items": { "type": "string" } }, + "recovery_commands": { "type": "array", "items": { "type": "string" } }, + "reachability": { "$ref": "#/components/schemas/ArtifactContentReachabilityReportV1" }, + "storage": { "$ref": "#/components/schemas/ArtifactStorageAccountingReport" } + } + }, + "ArtifactResolutionComponentReportV1": { + "type": "object", + "required": ["component_id", "proposal_key", "source_root", "snapshot_id", "snapshot", "decision", "refresh_requested"], + "additionalProperties": false, + "properties": { + "component_id": { "type": "string" }, + "proposal_key": { "type": "string" }, + "source_root": { "type": "string" }, + "snapshot_id": { "type": "string" }, + "snapshot": { "$ref": "#/components/schemas/JsonValue" }, + "decision": { "type": "string", "enum": ["resolved", "reused", "refreshed"] }, + "refresh_requested": { "type": "boolean" }, + "attempt": { "$ref": "#/components/schemas/JsonValue" } + } + }, + "ArtifactResolutionBatchReportV1": { + "type": "object", + "required": ["source_root", "refresh_requested", "components"], + "additionalProperties": false, + "properties": { + "source_root": { "type": "string" }, + "refresh_requested": { "type": "boolean" }, + "components": { "type": "array", "items": { "$ref": "#/components/schemas/ArtifactResolutionComponentReportV1" } } + } + }, + "ArtifactSourceExportExecutionReportV1": { + "type": "object", + "required": ["plan", "operation", "root_id", "changed_paths", "checkpointed", "git_handoff"], + "additionalProperties": false, + "properties": { + "plan": { "$ref": "#/components/schemas/JsonValue" }, + "operation": { "type": "string" }, + "root_id": { "type": "string" }, + "changed_paths": { "type": "array", "items": { "$ref": "#/components/schemas/FileDiffSummary" } }, + "checkpointed": { "type": "boolean" }, + "git_handoff": { "type": "array", "items": { "type": "string" } } + } + }, "LaneRemoveReport": { "type": "object", "required": ["lane_id", "ref_name", "removed_workdir", "forced"], @@ -78,6 +308,79 @@ pub(super) fn lane_schemas() -> Value { "details": { "$ref": "#/components/schemas/JsonValue" } } }, + "ManagedExecutionResolutionPin": { + "type": "object", + "required": ["component_id", "adapter_identity", "status"], + "additionalProperties": false, + "properties": { + "component_id": { "type": "string" }, + "adapter_identity": { "type": "string" }, + "status": { "type": "string", "enum": ["ready", "resolvable", "blocked", "unsupported", "ambiguous"] }, + "proposal_key": { "type": "string" }, + "snapshot_id": { "type": "string" }, + "recovery_command": { "type": "array", "items": { "type": "string" } } + } + }, + "ManagedExecutionOutputPin": { + "type": "object", + "required": ["component_id", "output_name", "component_key", "policy", "storage_identity"], + "additionalProperties": false, + "properties": { + "component_id": { "type": "string" }, + "output_name": { "type": "string" }, + "component_key": { "type": "string" }, + "policy": { "type": "string", "enum": ["immutable_shared", "immutable_seed_private", "writable_private", "disposable"] }, + "storage_identity": { "type": "string" }, + "artifact_binding_id": { "type": "string" }, + "artifact_envelope_id": { "type": "string" }, + "artifact_tree_root_id": { "type": "string" }, + "artifact_binding_identity": { "type": "string" } + } + }, + "ManagedExecutionPreparationReceipt": { + "type": "object", + "required": ["source_root", "missing_resolution_policy", "resolution_pins", "output_pins"], + "additionalProperties": false, + "properties": { + "source_root": { "type": "string" }, + "view_id": { "type": "string" }, + "view_generation": { "type": "integer", "minimum": 0 }, + "missing_resolution_policy": { "type": "string", "enum": ["explicit"] }, + "resolution_pins": { "type": "array", "items": { "$ref": "#/components/schemas/ManagedExecutionResolutionPin" } }, + "environment_generation": { "type": "string" }, + "output_pins": { "type": "array", "items": { "$ref": "#/components/schemas/ManagedExecutionOutputPin" } } + } + }, + "ManagedExecutionSealingDecision": { + "type": "object", + "required": ["component_id", "output_name", "policy", "publication", "decision", "reason"], + "additionalProperties": false, + "properties": { + "component_id": { "type": "string" }, + "output_name": { "type": "string" }, + "policy": { "type": "string", "enum": ["immutable_shared", "immutable_seed_private", "writable_private", "disposable"] }, + "publication": { "type": "string", "enum": ["never", "manual", "on_sync", "successful_gate"] }, + "gate": { "type": "string" }, + "decision": { "type": "string" }, + "reason": { "type": "string" } + } + }, + "ManagedExecutionFinalizationReceipt": { + "type": "object", + "required": ["source_root_before", "source_changed", "checkpoint_status", "disposal_status", "unmount_status", "complete", "sealing_decisions", "errors"], + "additionalProperties": false, + "properties": { + "source_root_before": { "type": "string" }, + "source_root_after": { "type": "string" }, + "source_changed": { "type": "boolean" }, + "checkpoint_status": { "type": "string", "enum": ["succeeded", "failed"] }, + "disposal_status": { "type": "string", "enum": ["succeeded", "failed", "skipped"] }, + "unmount_status": { "type": "string", "enum": ["succeeded", "failed", "skipped"] }, + "complete": { "type": "boolean" }, + "sealing_decisions": { "type": "array", "items": { "$ref": "#/components/schemas/ManagedExecutionSealingDecision" } }, + "errors": { "type": "array", "items": { "type": "string" } } + } + }, "ManagedExecutionLifecycleReport": { "type": "object", "required": ["execution_id", "surface", "command_fingerprint", "phases"], @@ -86,12 +389,14 @@ pub(super) fn lane_schemas() -> Value { "execution_id": { "type": "string" }, "surface": { "type": "string", "enum": ["lane_exec", "lane_test", "lane_eval", "terminal_agent", "acp_prompt"] }, "command_fingerprint": { "type": "string" }, + "preparation": { "$ref": "#/components/schemas/ManagedExecutionPreparationReceipt" }, "environment_generation": { "type": "string" }, "checkpoint": { "$ref": "#/components/schemas/WorkspaceCheckpointReport" }, "checkpoint_error": { "type": "string" }, "checkpoint_error_code": { "type": "string" }, "disposal_error": { "type": "string" }, "recorded": { "$ref": "#/components/schemas/LaneRecordReport" }, + "finalization": { "$ref": "#/components/schemas/ManagedExecutionFinalizationReceipt" }, "phases": { "type": "array", "items": { "$ref": "#/components/schemas/ManagedExecutionPhaseReceipt" } } } }, @@ -612,7 +917,7 @@ pub(super) fn lane_schemas() -> Value { }, "EnvironmentAdapterCatalogEntryReport": { "type": "object", - "required": ["identity", "canonical_identity", "selectors", "kind", "layer_adapter_name", "discovery_markers", "protocols", "supported_operating_systems", "supported_architectures", "source", "publisher", "publisher_key_id", "trust", "certification_tier", "stability", "description"], + "required": ["identity", "canonical_identity", "selectors", "kind", "layer_adapter_name", "discovery_markers", "protocols", "protocol_capabilities", "supported_operating_systems", "supported_architectures", "source", "publisher", "publisher_key_id", "trust", "certification_tier", "stability", "description"], "additionalProperties": false, "properties": { "identity": { "$ref": "#/components/schemas/EnvironmentAdapterIdentityReport" }, @@ -621,7 +926,8 @@ pub(super) fn lane_schemas() -> Value { "kind": { "type": "string" }, "layer_adapter_name": { "type": "string" }, "discovery_markers": { "type": "array", "items": { "type": "string" } }, - "protocols": { "type": "array", "items": { "type": "string", "enum": ["trail.environment-adapter/v1", "trail.environment-adapter/v2"] } }, + "protocols": { "type": "array", "items": { "type": "string", "enum": ["trail.environment-adapter/v1", "trail.environment-adapter/v2", "trail.environment-adapter/v3"] } }, + "protocol_capabilities": { "$ref": "#/components/schemas/EnvironmentPluginProtocolCapabilitiesReport" }, "supported_operating_systems": { "type": "array", "items": { "type": "string", "enum": ["linux", "macos", "windows"] } }, "supported_architectures": { "type": "array", "items": { "type": "string", "enum": ["aarch64", "x86_64"] } }, "source": { "type": "string", "enum": ["builtin", "recipe", "plugin"] }, @@ -633,6 +939,21 @@ pub(super) fn lane_schemas() -> Value { "description": { "type": "string" } } }, + "EnvironmentPluginProtocolCapabilitiesReport": { + "type": "object", + "required": ["resolution_capable", "source_export_capable", "host_attestation_evidence_capable", "host_quarantine_evidence_capable", "certification_ceiling", "content_policy", "attestation_policy"], + "additionalProperties": false, + "properties": { + "selected_protocol": { "type": ["string", "null"], "enum": ["trail.environment-adapter/v1", "trail.environment-adapter/v2", "trail.environment-adapter/v3", null] }, + "resolution_capable": { "type": "boolean" }, + "source_export_capable": { "type": "boolean" }, + "host_attestation_evidence_capable": { "type": "boolean" }, + "host_quarantine_evidence_capable": { "type": "boolean" }, + "certification_ceiling": { "type": "string", "enum": ["", "legacy_exact", "local_artifact", "portable_artifact"] }, + "content_policy": { "type": "string", "enum": ["", "legacy_exact_layer", "host_verified_local_cas"] }, + "attestation_policy": { "type": "string", "enum": ["", "legacy_host_evidence", "host_authored_required"] } + } + }, "EnvironmentAdapterCatalogReport": { "type": "object", "required": ["contract_major", "adapters"], @@ -725,13 +1046,42 @@ pub(super) fn lane_schemas() -> Value { }, "EnvironmentDiscoveredComponentReport": { "type": "object", - "required": ["component_id", "component_root", "kind", "adapter_identity"], + "required": ["component_id", "component_root", "kind", "adapter_identity", "status", "reasons", "recovery_actions"], "additionalProperties": false, "properties": { "component_id": { "type": "string" }, "component_root": { "type": "string" }, "kind": { "type": "string" }, - "adapter_identity": { "type": "string" } + "adapter_identity": { "type": "string" }, + "status": { "type": "string", "enum": ["ready", "resolvable", "blocked", "unsupported", "ambiguous"] }, + "reasons": { + "type": "array", + "items": { + "type": "object", + "required": ["code", "message"], + "additionalProperties": false, + "properties": { + "code": { "type": "string" }, + "message": { "type": "string" } + } + } + }, + "recovery_actions": { + "type": "array", + "items": { + "type": "object", + "required": ["code", "description"], + "additionalProperties": false, + "properties": { + "code": { "type": "string" }, + "description": { "type": "string" }, + "command": { + "type": ["array", "null"], + "items": { "type": "string" } + } + } + } + } } }, "EnvironmentDiscoveryConflictReport": { diff --git a/trail/src/server/request_types/lane.rs b/trail/src/server/request_types/lane.rs index d8402f7e..bb87de42 100644 --- a/trail/src/server/request_types/lane.rs +++ b/trail/src/server/request_types/lane.rs @@ -1,5 +1,7 @@ use serde::Deserialize; +use crate::{ArtifactQuarantineResolutionV1, ArtifactVerificationLevelV1}; + use super::default_completed_status; #[derive(Debug, Deserialize)] @@ -207,6 +209,47 @@ pub(crate) struct EnvironmentPromoteRequest { pub(crate) output: String, } +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct EnvironmentResolveAllRequest { + #[serde(default, alias = "component_root")] + pub(crate) path: Option, + #[serde(default)] + pub(crate) refresh: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct EnvironmentResolveRequest { + #[serde(alias = "component_id")] + pub(crate) component: String, + #[serde(default, alias = "component_root")] + pub(crate) path: Option, + #[serde(default)] + pub(crate) refresh: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ArtifactVerifyRequest { + pub(crate) level: ArtifactVerificationLevelV1, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ArtifactQuarantineResolveRequest { + pub(crate) resolution: ArtifactQuarantineResolutionV1, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ArtifactSourceExportRequest { + #[serde(alias = "component_id")] + pub(crate) component: String, + #[serde(alias = "export_name")] + pub(crate) export: String, +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct CacheGcRequest { diff --git a/trail/src/server/route/lane/lanes.rs b/trail/src/server/route/lane/lanes.rs index 249bbef2..45dedae2 100644 --- a/trail/src/server/route/lane/lanes.rs +++ b/trail/src/server/route/lane/lanes.rs @@ -2,16 +2,21 @@ use std::path::PathBuf; use crate::model::{LaneGateOptions, LaneInitializationPhase}; use crate::server::request_types::{ - DependencySyncRequest, EnvironmentPromoteRequest, EnvironmentSyncRequest, LaneClaimRequest, - LaneReadFileRequest, LaneRecordRequest, LaneRewindRequest, LaneTestRequest, LaneUpdateRequest, - SpawnLaneRequest, SyncWorkdirRequest, WorkspaceCheckpointRequest, WorkspaceExecRequest, + ArtifactQuarantineResolveRequest, ArtifactSourceExportRequest, ArtifactVerifyRequest, + DependencySyncRequest, EnvironmentPromoteRequest, EnvironmentResolveAllRequest, + EnvironmentResolveRequest, EnvironmentSyncRequest, LaneClaimRequest, LaneReadFileRequest, + LaneRecordRequest, LaneRewindRequest, LaneTestRequest, LaneUpdateRequest, SpawnLaneRequest, + SyncWorkdirRequest, WorkspaceCheckpointRequest, WorkspaceExecRequest, }; use crate::server::route::utils::{ json_response, parse_patch_request, query_flag, query_line_ids_flag, query_usize, query_value, reject_unexpected_body, }; use crate::server::transport::{HttpRequest, HttpResponse}; -use crate::{Error, Result, Trail}; +use crate::{ + ArtifactEnvelopeId, ArtifactQuarantineId, ArtifactSourceExportAuthorizationV1, Error, Result, + Trail, +}; pub(super) fn handle_lane_resources( db: &mut Trail, @@ -25,6 +30,98 @@ pub(super) fn handle_lane_resources( return Ok(Some(json_response(200, "OK", &report)?)); } + if request.method == "GET" && path == "/v1/artifacts/space" { + return Ok(Some(json_response( + 200, + "OK", + &db.workspace_artifact_space()?, + )?)); + } + + if request.method == "GET" && path == "/v1/artifact-quarantines" { + return Ok(Some(json_response( + 200, + "OK", + &db.artifact_quarantine_list_report()?, + )?)); + } + + if parts.len() == 3 && parts[0] == "v1" && parts[1] == "artifacts" { + let artifact = ArtifactEnvelopeId::parse(parts[2]).map_err(|error| { + Error::InvalidInput(format!("invalid artifact envelope ID: {error}")) + })?; + if request.method == "GET" { + return Ok(Some(json_response( + 200, + "OK", + &db.inspect_artifact(&artifact)?, + )?)); + } + } + + if parts.len() == 4 + && parts[0] == "v1" + && parts[1] == "artifacts" + && parts[3] == "reachability" + && request.method == "GET" + { + let artifact = ArtifactEnvelopeId::parse(parts[2]).map_err(|error| { + Error::InvalidInput(format!("invalid artifact envelope ID: {error}")) + })?; + return Ok(Some(json_response( + 200, + "OK", + &db.artifact_content_reachability(&artifact)?, + )?)); + } + + if parts.len() == 4 + && parts[0] == "v1" + && parts[1] == "artifacts" + && parts[3] == "verify" + && request.method == "POST" + { + let artifact = ArtifactEnvelopeId::parse(parts[2]).map_err(|error| { + Error::InvalidInput(format!("invalid artifact envelope ID: {error}")) + })?; + let body: ArtifactVerifyRequest = serde_json::from_slice(&request.body)?; + return Ok(Some(json_response( + 200, + "OK", + &db.verify_artifact(&artifact, body.level)?, + )?)); + } + + if parts.len() == 3 + && parts[0] == "v1" + && parts[1] == "artifact-quarantines" + && request.method == "GET" + { + let quarantine = ArtifactQuarantineId::parse(parts[2]) + .map_err(|error| Error::InvalidInput(format!("invalid quarantine ID: {error}")))?; + return Ok(Some(json_response( + 200, + "OK", + &db.artifact_quarantine(&quarantine)?, + )?)); + } + + if parts.len() == 4 + && parts[0] == "v1" + && parts[1] == "artifact-quarantines" + && parts[3] == "resolve" + && request.method == "POST" + { + let quarantine = ArtifactQuarantineId::parse(parts[2]) + .map_err(|error| Error::InvalidInput(format!("invalid quarantine ID: {error}")))?; + let body: ArtifactQuarantineResolveRequest = serde_json::from_slice(&request.body)?; + return Ok(Some(json_response( + 200, + "OK", + &db.resolve_artifact_quarantine_report(&quarantine, body.resolution)?, + )?)); + } + if request.method == "GET" && path == "/v1/lanes" { let lanes = db.list_lanes()?; return Ok(Some(json_response(200, "OK", &lanes)?)); @@ -234,6 +331,64 @@ pub(super) fn handle_lane_resources( return Ok(Some(json_response(200, "OK", &report)?)); } + if parts.len() == 5 + && parts[0] == "v1" + && parts[1] == "lanes" + && parts[3] == "environment" + && parts[4] == "resolve" + && request.method == "POST" + { + let lane = db.resolve_lane_handle(parts[2])?; + let body: EnvironmentResolveRequest = serde_json::from_slice(&request.body)?; + let report = db.resolve_workspace_environment_component( + &lane, + &body.component, + body.path.as_deref(), + body.refresh, + )?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + + if parts.len() == 5 + && parts[0] == "v1" + && parts[1] == "lanes" + && parts[3] == "environment" + && parts[4] == "resolve-all" + && request.method == "POST" + { + let lane = db.resolve_lane_handle(parts[2])?; + let body: EnvironmentResolveAllRequest = if request.body.is_empty() { + EnvironmentResolveAllRequest::default() + } else { + serde_json::from_slice(&request.body)? + }; + let report = db.resolve_all_workspace_environment_components( + &lane, + body.path.as_deref(), + body.refresh, + )?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + + if parts.len() == 5 + && parts[0] == "v1" + && parts[1] == "lanes" + && parts[3] == "environment" + && parts[4] == "source-export" + && request.method == "POST" + { + let lane = db.resolve_lane_handle(parts[2])?; + let body: ArtifactSourceExportRequest = serde_json::from_slice(&request.body)?; + let plan = db.plan_artifact_source_export( + &lane, + &body.component, + &body.export, + ArtifactSourceExportAuthorizationV1::ExplicitUser, + )?; + let report = db.execute_artifact_source_export(plan)?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + if parts.len() == 5 && parts[0] == "v1" && parts[1] == "lanes" diff --git a/trail/tests/artifact_public_api.rs b/trail/tests/artifact_public_api.rs new file mode 100644 index 00000000..8c9d5e91 --- /dev/null +++ b/trail/tests/artifact_public_api.rs @@ -0,0 +1,63 @@ +use std::fs; + +use trail::{ + ArtifactEnvelopeId, ArtifactQuarantineId, ArtifactQuarantineResolutionV1, + ArtifactResolutionBatchReportV1, ArtifactResolutionComponentReportV1, + ArtifactResolutionRequestV1, ArtifactVerificationLevelV1, Error, InitImportMode, Result, Trail, +}; + +#[test] +fn public_artifact_operations_share_serializable_reports_and_bounded_empty_state() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "artifact public API\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + + let space = db.workspace_artifact_space().unwrap(); + assert_eq!(space.scope, "workspace"); + assert_eq!(space.envelope_count, 0); + assert_eq!(space.active_quarantine_count, 0); + assert_eq!( + serde_json::to_value(&space).unwrap()["storage"]["logical_bytes"], + 0 + ); + + let quarantines = db.artifact_quarantine_list_report().unwrap(); + assert_eq!(quarantines.active_count, 0); + assert_eq!(quarantines.resolved_count, 0); + assert!(quarantines.quarantines.is_empty()); + assert_eq!( + serde_json::to_value(ArtifactVerificationLevelV1::Reproduce).unwrap(), + "reproduce" + ); + + let missing = ArtifactEnvelopeId::new(b"missing-public-artifact"); + for error in [ + db.inspect_artifact(&missing).unwrap_err(), + db.artifact_content_reachability(&missing).unwrap_err(), + db.verify_artifact(&missing, ArtifactVerificationLevelV1::Attach) + .unwrap_err(), + ] { + assert!(matches!(error, Error::ObjectNotFound { .. })); + } +} + +#[test] +fn public_artifact_mutation_signatures_remain_typed() { + let _resolve_component: fn( + &Trail, + ArtifactResolutionRequestV1, + bool, + ) -> Result = Trail::resolve_artifact_component; + let _resolve_all: fn( + &Trail, + Vec, + bool, + ) -> Result = Trail::resolve_all_artifact_components; + let _resolve_quarantine: fn( + &Trail, + &ArtifactQuarantineId, + ArtifactQuarantineResolutionV1, + ) -> Result = + Trail::resolve_artifact_quarantine_report; +} diff --git a/trail/tests/changed_path_ledger_activation.rs b/trail/tests/changed_path_ledger_activation.rs index a206dcab..bbebace2 100644 --- a/trail/tests/changed_path_ledger_activation.rs +++ b/trail/tests/changed_path_ledger_activation.rs @@ -90,11 +90,11 @@ fn authority_requires_every_checked_gate_and_supported_platform() { ); assert_eq!( complete["raw_mutation_inventory_sha256"], - "4fa1ed52b64a9d4a33f09a2915063a695890aaa80af5aa352a7f0d8815c2f1c4" + "a6708f5ae489be0126aef47d8ec4e9b6e687d8a3f2908bff657ea87b5988e5d1" ); assert_eq!( complete["activation_audit_sha256"], - "5bc402bb1f51cd7f9e2b48dc85cbd84f5ca5cee0deb33b4dab7b97b5432df74a" + "b09c7edc5e2c45d6a53826175553fa83701cf02c5fc69820e762108c7971b01e" ); assert!(!trail::test_support::changed_path_authority_enabled_for("windows").unwrap()); assert!(!trail::test_support::changed_path_authority_enabled_for("freebsd").unwrap()); diff --git a/trail/tests/e2e.rs b/trail/tests/e2e.rs index bbba866b..5bc37ae0 100644 --- a/trail/tests/e2e.rs +++ b/trail/tests/e2e.rs @@ -30,6 +30,49 @@ fn git_available() -> bool { .unwrap_or(false) } +fn native_cow_supported_for_test() -> bool { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + match db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "native-cow-probe", + Some("main"), + LaneWorkdirMode::NativeCow, + None, + None, + None, + &[], + false, + ) { + Ok(_) => true, + Err(Error::CloneUnsupported | Error::CloneCrossDevice) => false, + Err(error) => panic!("native COW capability probe failed unexpectedly: {error}"), + } +} + +fn owning_host_layered_workdir_mode() -> Option { + #[cfg(target_os = "linux")] + { + return (std::env::var_os("TRAIL_RUN_FUSE_COW_TESTS").as_deref() + == Some(std::ffi::OsStr::new("1"))) + .then_some(LaneWorkdirMode::FuseCow); + } + #[cfg(target_os = "macos")] + { + return (std::env::var_os("TRAIL_RUN_NFS_COW_TESTS").as_deref() + == Some(std::ffi::OsStr::new("1"))) + .then_some(LaneWorkdirMode::NfsCow); + } + #[cfg(windows)] + { + return (std::env::var_os("TRAIL_RUN_DOKAN_COW_TESTS").as_deref() + == Some(std::ffi::OsStr::new("1"))) + .then_some(LaneWorkdirMode::DokanCow); + } + #[allow(unreachable_code)] + None +} + fn run_git(cwd: &Path, args: &[&str]) { let output = Command::new("git") .arg("-C") @@ -303,6 +346,9 @@ fn test_zed_settings_path(home: &Path) -> PathBuf { #[cfg(unix)] #[test] fn terminal_agent_start_aligns_process_context_with_the_lane_workdir() { + if !native_cow_supported_for_test() { + return; + } let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); @@ -429,7 +475,7 @@ fn terminal_agent_start_loads_project_hook_settings_in_the_isolated_provider() { #[cfg(unix)] #[test] fn terminal_agent_native_cow_does_not_discover_or_write_the_parent_git_checkout() { - if !git_available() { + if !git_available() || !native_cow_supported_for_test() { return; } let temp = tempfile::tempdir().unwrap(); @@ -551,6 +597,9 @@ fn terminal_agent_native_cow_denies_explicit_writes_to_the_original_workspace() #[cfg(unix)] #[test] fn terminal_agent_native_hooks_enrich_the_existing_task_without_duplication() { + if !native_cow_supported_for_test() { + return; + } let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); @@ -2252,6 +2301,202 @@ fn ndjson_rejects_single_report_commands_with_a_structured_diagnostic() { .contains("streaming watch commands")); } +#[test] +fn environment_artifact_cli_supports_help_plain_json_ndjson_and_stable_errors() { + for args in [ + &["env", "resolve", "--help"][..], + &["env", "artifact", "--help"][..], + &["env", "source", "export", "--help"][..], + ] { + let output = Command::new(trail_bin()).args(args).output().unwrap(); + assert!( + output.status.success(), + "trail {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("Usage:")); + } + + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + let json = run_trail_json(temp.path(), &["env", "artifact", "quarantine", "list"]); + assert_eq!(json["active_count"], 0); + assert_eq!(json["resolved_count"], 0); + assert_eq!(json["quarantines"], serde_json::json!([])); + + let ndjson = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args([ + "--format", + "ndjson", + "env", + "artifact", + "quarantine", + "list", + ]) + .output() + .unwrap(); + assert!(ndjson.status.success()); + assert_eq!( + ndjson.stdout.iter().filter(|byte| **byte == b'\n').count(), + 1 + ); + let ndjson_value: serde_json::Value = serde_json::from_slice(&ndjson.stdout).unwrap(); + assert_eq!(ndjson_value, json); + + let plain = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["--format", "plain", "env", "artifact", "quarantine", "list"]) + .output() + .unwrap(); + assert!(plain.status.success()); + let plain = String::from_utf8(plain.stdout).unwrap(); + assert!(plain.contains("Artifact quarantines")); + assert!(plain.contains("Active Count : 0")); + + let missing = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .args([ + "env", + "artifact", + "inspect", + "artifact_envelope_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ]) + .output() + .unwrap(); + assert_eq!(missing.status.code(), Some(1)); + let error: serde_json::Value = serde_json::from_slice(&missing.stderr).unwrap(); + assert_eq!(error["error"]["code"], "OBJECT_NOT_FOUND"); +} + +#[test] +fn environment_resolve_cli_executes_and_reuses_a_real_cargo_snapshot() { + if !Command::new("cargo") + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) + { + return; + } + let temp = tempfile::tempdir().unwrap(); + fs::create_dir_all(temp.path().join("src")).unwrap(); + fs::write( + temp.path().join("Cargo.toml"), + "[package]\nname = \"trail-cli-resolver\"\nversion = \"0.1.0\"\nedition = \"2024\"\n", + ) + .unwrap(); + fs::write( + temp.path().join("src/lib.rs"), + "pub fn value() -> u8 { 1 }\n", + ) + .unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "cargo-resolve-cli", + Some("main"), + LaneWorkdirMode::Virtual, + None, + None, + None, + &[], + false, + ) + .unwrap(); + drop(db); + + let first = run_trail_json( + temp.path(), + &[ + "env", + "resolve", + "component", + "cargo-target-seed", + "--lane", + "cargo-resolve-cli", + ], + ); + assert_eq!(first["decision"], "resolved"); + assert!(first["attempt"].is_object()); + assert!(!temp.path().join("Cargo.lock").exists()); + + let second = run_trail_json( + temp.path(), + &[ + "env", + "resolve", + "component", + "cargo-target-seed", + "--lane", + "cargo-resolve-cli", + ], + ); + assert_eq!(second["decision"], "reused"); + assert_eq!(second["snapshot_id"], first["snapshot_id"]); + assert!(second.get("attempt").is_none()); + + let mut db = Trail::open(temp.path()).unwrap(); + let http = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/lanes/cargo-resolve-cli/environment/resolve", + serde_json::json!({"component": "cargo-target-seed"}), + ), + ); + assert_eq!(http.status, 200); + let http: serde_json::Value = http.body_json().unwrap(); + assert_eq!(http["decision"], "reused"); + assert_eq!(http["snapshot_id"], first["snapshot_id"]); + assert_eq!(http["snapshot"], first["snapshot"]); + + let mcp = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 57, + "method": "tools/call", + "params": { + "name": "trail.env_resolve", + "arguments": { + "lane": "cargo-resolve-cli", + "component": "cargo-target-seed" + } + } + }), + ) + .unwrap(); + assert_eq!(mcp["result"]["isError"], false); + assert_eq!(mcp["result"]["structuredContent"], http); + assert!(!temp.path().join("Cargo.lock").exists()); + + let missing = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .args([ + "env", + "resolve", + "component", + "missing-component", + "--lane", + "cargo-resolve-cli", + ]) + .output() + .unwrap(); + assert_eq!(missing.status.code(), Some(2)); + let error: serde_json::Value = serde_json::from_slice(&missing.stderr).unwrap(); + assert_eq!(error["error"]["code"], "INVALID_INPUT"); + assert!(error["error"]["message"] + .as_str() + .unwrap() + .contains("trail env discover cargo-resolve-cli")); +} + #[test] fn plain_redirected_and_quiet_output_follow_terminal_policy() { let temp = tempfile::tempdir().unwrap(); @@ -2479,7 +2724,7 @@ fn doctor_treats_removed_lane_tombstones_as_audit_history() { db.spawn_lane_with_workdir_mode_paths_and_neighbors( "retired-doctor-bot", Some("main"), - LaneWorkdirMode::NativeCow, + LaneWorkdirMode::PortableCopy, None, None, None, @@ -11510,38 +11755,22 @@ fn layered_workspace_reports_have_http_mcp_and_openapi_parity() { } #[test] -fn environment_graph_has_cli_http_mcp_and_openapi_parity() { +fn manifest_only_environment_discovery_has_cli_http_mcp_and_openapi_parity() { let temp = tempfile::tempdir().unwrap(); - fs::write(temp.path().join("input.txt"), "graph\n").unwrap(); fs::write( - temp.path().join("trail.environment.toml"), - r#"schema = "trail.environment/v1" - -[[component]] -id = "graph.a" -adapter = "trail/command@1" -kind = "generated" -inputs = [{ path = "input.txt" }] -outputs = [{ source = "generated-a", target = ".trail-generated/a" }] -[component.build] -command = ["git", "--version"] - -[[component]] -id = "graph.b" -adapter = "trail/command@1" -kind = "generated" -depends_on = ["graph.a"] -inputs = [{ path = "input.txt" }] -outputs = [{ source = "generated-b", target = ".trail-generated/b" }] -[component.build] -command = ["git", "--version"] -"#, + temp.path().join("Cargo.toml"), + "[package]\nname = \"manifest-only\"\nversion = \"0.1.0\"\nedition = \"2024\"\n", + ) + .unwrap(); + fs::write( + temp.path().join("package.json"), + r#"{"name":"manifest-only","version":"1.0.0","private":true}"#, ) .unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(temp.path()).unwrap(); db.spawn_lane_with_workdir_mode_paths_and_neighbors( - "graph", + "manifest-only", Some("main"), if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow @@ -11558,22 +11787,35 @@ command = ["git", "--version"] ) .unwrap(); - let rust = db.workspace_environment_graph("graph", None).unwrap(); + let rust = db + .discover_workspace_environment("manifest-only", None) + .unwrap(); let expected = serde_json::to_value(&rust).unwrap(); - assert_eq!(rust.nodes.len(), 2); - assert_eq!(rust.edges.len(), 1); - assert_eq!(rust.nodes[0].component_id, "graph.a"); - assert_eq!(rust.nodes[1].component_id, "graph.b"); - assert_eq!( - rust.edges[0].source_component_key, - rust.nodes[0].component_key - ); + assert_eq!(expected["components"].as_array().unwrap().len(), 2); + assert!(expected["components"] + .as_array() + .unwrap() + .iter() + .all(|component| { + component["status"] == "resolvable" + && component["reasons"][0]["code"] == "resolution_snapshot_missing" + && component["recovery_actions"][0]["command"] + == serde_json::json!([ + "trail", + "env", + "resolve", + "component", + component["component_id"].as_str().unwrap(), + "--lane", + "manifest-only" + ]) + })); let http = trail::server::handle_http_request( &mut db, &api_request( "GET", - "/v1/lanes/graph/environment/graph", + "/v1/lanes/manifest-only/environment/discover", serde_json::Value::Null, ), ); @@ -11584,11 +11826,11 @@ command = ["git", "--version"] &mut db, serde_json::json!({ "jsonrpc": "2.0", - "id": 41, + "id": 40, "method": "tools/call", "params": { - "name": "trail.env_graph", - "arguments": {"lane": "graph"} + "name": "trail.env_discover", + "arguments": {"lane": "manifest-only"} } }), ) @@ -11596,66 +11838,370 @@ command = ["git", "--version"] assert_eq!(mcp["result"]["isError"], false); assert_eq!(mcp["result"]["structuredContent"], expected); - let cli = run_trail_json(temp.path(), &["env", "graph", "graph"]); + let cli = run_trail_json(temp.path(), &["env", "discover", "manifest-only"]); assert_eq!(cli, expected); + let openapi = trail::server::openapi_spec(); + let component = &openapi["components"]["schemas"]["EnvironmentDiscoveredComponentReport"]; + assert!(component["required"] + .as_array() + .unwrap() + .iter() + .any(|field| field == "status")); assert_eq!( - openapi["paths"]["/v1/lanes/{lane_or_id}/environment/graph"]["get"]["responses"]["200"] - ["content"]["application/json"]["schema"]["$ref"], - "#/components/schemas/EnvironmentGraphReport" + component["properties"]["status"]["enum"], + serde_json::json!(["ready", "resolvable", "blocked", "unsupported", "ambiguous"]) ); - assert!(db.list_workspace_layers().unwrap().is_empty()); } #[test] -fn pinned_oci_metadata_has_cli_http_mcp_openapi_and_gc_parity() { +fn artifact_lifecycle_http_mcp_resources_and_openapi_are_aligned() { let temp = tempfile::tempdir().unwrap(); - let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; - fs::write( - temp.path().join("trail.oci.toml"), - format!( - "schema = \"trail.oci-images/v1\"\n\n[[image]]\nname = \"web\"\nreference = \"ghcr.io/example/web@{digest}\"\nplatform = \"linux/amd64\"\n" - ), - ) - .unwrap(); + fs::write(temp.path().join("README.md"), "artifact reports\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(temp.path()).unwrap(); - db.spawn_lane_with_workdir_mode_paths_and_neighbors( - "oci-surfaces", - Some("main"), - if cfg!(target_os = "macos") { - LaneWorkdirMode::NfsCow - } else if cfg!(target_os = "windows") { - LaneWorkdirMode::DokanCow - } else { - LaneWorkdirMode::FuseCow - }, - None, - None, - None, - &[], - false, - ) - .unwrap(); - let http_plan = trail::server::handle_http_request( + let expected_space = serde_json::to_value(db.workspace_artifact_space().unwrap()).unwrap(); + let http_space = trail::server::handle_http_request( &mut db, - &api_request( - "GET", - "/v1/lanes/oci-surfaces/environment/plan?adapter=oci-image", - serde_json::Value::Null, - ), + &api_request("GET", "/v1/artifacts/space", serde_json::Value::Null), ); - assert_eq!(http_plan.status, 200); - let plan: serde_json::Value = http_plan.body_json().unwrap(); - assert_eq!(plan["component_id"], "oci-images"); - assert_eq!(plan["kind"], "external"); - assert!(plan["outputs"].as_array().unwrap().is_empty()); - assert!(plan["commands"].as_array().unwrap().is_empty()); - assert_eq!(plan["external_artifacts"][0]["digest"], digest); + assert_eq!(http_space.status, 200); assert_eq!( - plan["capabilities"]["sandbox"], - "not-applicable-metadata-only" + http_space.body_json::().unwrap(), + expected_space + ); + + let mcp_space = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 50, + "method": "tools/call", + "params": {"name": "trail.artifact_space", "arguments": {}} + }), + ) + .unwrap(); + assert_eq!(mcp_space["result"]["isError"], false); + assert_eq!(mcp_space["result"]["structuredContent"], expected_space); + + let space_resource = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 51, + "method": "resources/read", + "params": {"uri": "trail://workspace/artifacts/space"} + }), + ) + .unwrap(); + let space_resource: serde_json::Value = serde_json::from_str( + space_resource["result"]["contents"][0]["text"] + .as_str() + .unwrap(), + ) + .unwrap(); + assert_eq!(space_resource, expected_space); + + let expected_quarantines = + serde_json::to_value(db.artifact_quarantine_list_report().unwrap()).unwrap(); + let http_quarantines = trail::server::handle_http_request( + &mut db, + &api_request("GET", "/v1/artifact-quarantines", serde_json::Value::Null), + ); + assert_eq!(http_quarantines.status, 200); + assert_eq!( + http_quarantines.body_json::().unwrap(), + expected_quarantines + ); + let mcp_quarantines = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 52, + "method": "tools/call", + "params": {"name": "trail.artifact_quarantine_list", "arguments": {}} + }), + ) + .unwrap(); + assert_eq!( + mcp_quarantines["result"]["structuredContent"], + expected_quarantines + ); + let quarantine_resource = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 53, + "method": "resources/read", + "params": {"uri": "trail://workspace/artifact-quarantines"} + }), + ) + .unwrap(); + let quarantine_resource: serde_json::Value = serde_json::from_str( + quarantine_resource["result"]["contents"][0]["text"] + .as_str() + .unwrap(), + ) + .unwrap(); + assert_eq!(quarantine_resource, expected_quarantines); + + let resources = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 54, + "method": "resources/list", + "params": {} + }), + ) + .unwrap(); + let resources = resources["result"]["resources"].as_array().unwrap(); + assert!(resources + .iter() + .any(|resource| resource["uri"] == "trail://workspace/artifacts/space")); + assert!(resources + .iter() + .any(|resource| resource["uri"] == "trail://workspace/artifact-quarantines")); + let templates = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 55, + "method": "resources/templates/list", + "params": {} + }), + ) + .unwrap(); + let templates = templates["result"]["resourceTemplates"].as_array().unwrap(); + assert!(templates.iter().any(|template| { + template["uriTemplate"] == "trail://workspace/artifacts/{artifact_id}" + })); + assert!(templates.iter().any(|template| { + template["uriTemplate"] == "trail://workspace/artifact-quarantines/{quarantine_id}" + })); + + let tools = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 56, + "method": "tools/list", + "params": {} + }), + ) + .unwrap(); + let tools = tools["result"]["tools"].as_array().unwrap(); + for (name, read_only, destructive, open_world) in [ + ("trail.env_resolve", false, false, true), + ("trail.env_resolve_all", false, false, true), + ("trail.artifact_space", true, false, false), + ("trail.artifact_inspect", true, false, false), + ("trail.artifact_reachability", true, false, false), + ("trail.artifact_verify", true, false, false), + ("trail.artifact_quarantine_list", true, false, false), + ("trail.artifact_quarantine_show", true, false, false), + ("trail.artifact_quarantine_resolve", false, true, false), + ("trail.env_source_export", false, true, false), + ] { + let tool = tools.iter().find(|tool| tool["name"] == name).unwrap(); + assert_eq!(tool["annotations"]["readOnlyHint"], read_only, "{name}"); + assert_eq!( + tool["annotations"]["destructiveHint"], destructive, + "{name}" + ); + assert_eq!(tool["annotations"]["openWorldHint"], open_world, "{name}"); + } + + let openapi = trail::server::openapi_spec(); + for path in [ + "/v1/artifacts/space", + "/v1/artifacts/{artifact_id}", + "/v1/artifacts/{artifact_id}/reachability", + "/v1/artifacts/{artifact_id}/verify", + "/v1/artifact-quarantines", + "/v1/artifact-quarantines/{quarantine_id}", + "/v1/artifact-quarantines/{quarantine_id}/resolve", + "/v1/lanes/{lane_or_id}/environment/resolve", + "/v1/lanes/{lane_or_id}/environment/resolve-all", + "/v1/lanes/{lane_or_id}/environment/source-export", + ] { + assert!( + openapi["paths"].get(path).is_some(), + "missing OpenAPI path {path}" + ); + } + for schema in [ + "ArtifactResolutionComponentReportV1", + "ArtifactResolutionBatchReportV1", + "ArtifactInspectionReportV1", + "ArtifactVerificationReportV1", + "ArtifactContentReachabilityReportV1", + "ArtifactSpaceReportV1", + "ArtifactQuarantineListReportV1", + "ArtifactQuarantineRecordV1", + "ArtifactQuarantineResolutionReportV1", + "ArtifactSourceExportExecutionReportV1", + "ManagedExecutionResolutionPin", + "ManagedExecutionOutputPin", + "ManagedExecutionPreparationReceipt", + "ManagedExecutionSealingDecision", + "ManagedExecutionFinalizationReceipt", + ] { + assert!( + openapi["components"]["schemas"].get(schema).is_some(), + "missing OpenAPI schema {schema}" + ); + } +} + +#[test] +fn environment_graph_has_cli_http_mcp_and_openapi_parity() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("input.txt"), "graph\n").unwrap(); + fs::write( + temp.path().join("trail.environment.toml"), + r#"schema = "trail.environment/v1" + +[[component]] +id = "graph.a" +adapter = "trail/command@1" +kind = "generated" +inputs = [{ path = "input.txt" }] +outputs = [{ source = "generated-a", target = ".trail-generated/a" }] +[component.build] +command = ["git", "--version"] + +[[component]] +id = "graph.b" +adapter = "trail/command@1" +kind = "generated" +depends_on = ["graph.a"] +inputs = [{ path = "input.txt" }] +outputs = [{ source = "generated-b", target = ".trail-generated/b" }] +[component.build] +command = ["git", "--version"] +"#, + ) + .unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "graph", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + + let rust = db.workspace_environment_graph("graph", None).unwrap(); + let expected = serde_json::to_value(&rust).unwrap(); + assert_eq!(rust.nodes.len(), 2); + assert_eq!(rust.edges.len(), 1); + assert_eq!(rust.nodes[0].component_id, "graph.a"); + assert_eq!(rust.nodes[1].component_id, "graph.b"); + assert_eq!( + rust.edges[0].source_component_key, + rust.nodes[0].component_key + ); + + let http = trail::server::handle_http_request( + &mut db, + &api_request( + "GET", + "/v1/lanes/graph/environment/graph", + serde_json::Value::Null, + ), + ); + assert_eq!(http.status, 200); + assert_eq!(http.body_json::().unwrap(), expected); + + let mcp = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 41, + "method": "tools/call", + "params": { + "name": "trail.env_graph", + "arguments": {"lane": "graph"} + } + }), + ) + .unwrap(); + assert_eq!(mcp["result"]["isError"], false); + assert_eq!(mcp["result"]["structuredContent"], expected); + + let cli = run_trail_json(temp.path(), &["env", "graph", "graph"]); + assert_eq!(cli, expected); + let openapi = trail::server::openapi_spec(); + assert_eq!( + openapi["paths"]["/v1/lanes/{lane_or_id}/environment/graph"]["get"]["responses"]["200"] + ["content"]["application/json"]["schema"]["$ref"], + "#/components/schemas/EnvironmentGraphReport" + ); + assert!(db.list_workspace_layers().unwrap().is_empty()); +} + +#[test] +fn pinned_oci_metadata_has_cli_http_mcp_openapi_and_gc_parity() { + let temp = tempfile::tempdir().unwrap(); + let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + fs::write( + temp.path().join("trail.oci.toml"), + format!( + "schema = \"trail.oci-images/v1\"\n\n[[image]]\nname = \"web\"\nreference = \"ghcr.io/example/web@{digest}\"\nplatform = \"linux/amd64\"\n" + ), + ) + .unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "oci-surfaces", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + + let http_plan = trail::server::handle_http_request( + &mut db, + &api_request( + "GET", + "/v1/lanes/oci-surfaces/environment/plan?adapter=oci-image", + serde_json::Value::Null, + ), + ); + assert_eq!(http_plan.status, 200); + let plan: serde_json::Value = http_plan.body_json().unwrap(); + assert_eq!(plan["component_id"], "oci-images"); + assert_eq!(plan["kind"], "external"); + assert!(plan["outputs"].as_array().unwrap().is_empty()); + assert!(plan["commands"].as_array().unwrap().is_empty()); + assert_eq!(plan["external_artifacts"][0]["digest"], digest); + assert_eq!( + plan["capabilities"]["sandbox"], + "not-applicable-metadata-only" ); assert_eq!( run_trail_json( @@ -11946,81 +12492,349 @@ fn environment_sync_reuses_one_node_layer_across_http_and_mcp_parity() { r#"{"name":"env-surface","version":"1.0.1","lockfileVersion":3,"requires":true,"packages":{"":{"name":"env-surface","version":"1.0.1"}}}"#, ) .unwrap(); - db.request_lane_workspace_unmount("env-http").unwrap(); - db.checkpoint_lane_workspace("env-http", Some("change lock".to_string())) + db.request_lane_workspace_unmount("env-http").unwrap(); + db.checkpoint_lane_workspace("env-http", Some("change lock".to_string())) + .unwrap(); + db.lane_readiness("env-http").unwrap(); + let explained = trail::server::handle_http_request( + &mut db, + &api_request( + "GET", + "/v1/lanes/env-http/environment/explain?component=node", + serde_json::Value::Null, + ), + ); + assert_eq!(explained.status, 200); + let explained: serde_json::Value = explained.body_json().unwrap(); + assert_eq!(explained["status"], "stale"); + assert_eq!(explained["complete"], true); + assert!(explained["changes"] + .as_array() + .unwrap() + .iter() + .any(|change| { + change["dimension"] == "input" + && change["name"] == "package-lock.json" + && change["change"] == "modified" + })); + let mcp_explained = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 21, + "method": "tools/call", + "params": { + "name": "trail.env_explain", + "arguments": {"lane": "env-http", "component": "node"} + } + }), + ) + .unwrap(); + assert_eq!(mcp_explained["result"]["isError"], false); + assert_eq!(mcp_explained["result"]["structuredContent"], explained); + let cli_explained = run_trail_json( + temp.path(), + &["env", "explain", "env-http", "--component", "node"], + ); + assert_eq!(cli_explained, explained); + + let all_http = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/lanes/env-all-http/environment/sync-all", + serde_json::json!({}), + ), + ); + assert_eq!(all_http.status, 200); + let all_http: serde_json::Value = all_http.body_json().unwrap(); + assert_eq!(all_http["generation"]["generation_sequence"], 1); + assert_eq!(all_http["layers"][0]["layer_id"], http_layer["layer_id"]); + let all_mcp = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": "trail.env_sync_all", + "arguments": {"lane": "env-all-mcp"} + } + }), + ) + .unwrap(); + assert_eq!(all_mcp["result"]["isError"], false); + assert_eq!( + all_mcp["result"]["structuredContent"]["layers"][0]["layer_id"], + http_layer["layer_id"] + ); +} + +#[test] +fn manual_private_output_promotion_uses_native_cli_sandbox_and_preserves_private_bytes() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("input.txt"), "promotion input\n").unwrap(); + let build_command = if cfg!(target_os = "windows") { + r#"command = ["cmd.exe", "/D", "/S", "/C", "copy /Y input.txt out\\value.txt >NUL"]"# + } else { + r#"command = ["cp", "input.txt", "out/value.txt"]"# + }; + fs::write( + temp.path().join("trail.environment.toml"), + r#"schema = "trail.environment/v1" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "generated.promotable" +adapter = "trail/command@1" +kind = "generated" +inputs = [{ path = "input.txt" }] +outputs = [{ name = "result", source = "out", target = ".trail-generated/promotable", policy = "writable_private", reuse = "none", scope = "lane", publish = "manual" }] + +[component.build] +__BUILD_COMMAND__ +network = "deny" +scripts = "deny" +"# + .replace("__BUILD_COMMAND__", build_command), + ) + .unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "promotion", + Some("main"), + mode, + None, + None, + None, + &[], + false, + ) + .unwrap(); + drop(db); + + let synchronized = run_trail_json(temp.path(), &["env", "sync", "all", "promotion"]); + let predecessor = synchronized["generation"]["generation_id"] + .as_str() + .unwrap() + .to_string(); + let db = Trail::open(temp.path()).unwrap(); + let view = db.lane_workspace_view("promotion").unwrap().unwrap(); + let private_file = + Path::new(&view.generated_upper).join(".trail-generated/promotable/value.txt"); + fs::write(&private_file, "lane-private promoted bytes\n").unwrap(); + drop(db); + + let promoted = run_trail_json( + temp.path(), + &[ + "env", + "promote", + "promotion", + "generated.promotable", + "result", + ], + ); + assert_eq!(promoted["phase"], "activated"); + assert_eq!(promoted["predecessor_generation_id"], predecessor); + assert_ne!( + promoted["successor_generation_id"], + promoted["predecessor_generation_id"] + ); + assert_eq!( + fs::read_to_string(&private_file).unwrap(), + "lane-private promoted bytes\n" + ); + let storage_path = promoted["layer"]["storage_path"].as_str().unwrap(); + assert_eq!( + fs::read_to_string(Path::new(storage_path).join("value.txt")).unwrap(), + "lane-private promoted bytes\n" + ); +} + +#[cfg(unix)] +#[test] +fn next_and_vite_v2_components_compose_through_native_cli_sandbox() { + if !Command::new("npm") + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) + || !Command::new("node") + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) + { + return; + } + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join("package.json"), + r#"{"name":"framework-composition","version":"1.0.0","private":true}"#, + ) + .unwrap(); + fs::write( + temp.path().join("package-lock.json"), + r#"{"name":"framework-composition","version":"1.0.0","lockfileVersion":3,"requires":true,"packages":{"":{"name":"framework-composition","version":"1.0.0"}}}"#, + ) + .unwrap(); + fs::write(temp.path().join("next-source.js"), "next fixture\n").unwrap(); + fs::write(temp.path().join("vite-source.js"), "vite fixture\n").unwrap(); + fs::write( + temp.path().join("trail.environment.toml"), + r#"schema = "trail.environment/v2" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "web.next-build" +adapter = "trail/command@1" +kind = "generated" +depends_on = ["node"] +inputs = [{ path = "next-source.js", role = "identity", format = "bytes" }] +outputs = [{ name = "next-state", source = "next-output", target = ".next", policy = "writable_private", reuse = "none", scope = "lane", publish = "manual", portability = "host" }] +[component.build] +command = ["cp", "next-source.js", "next-output/server.js"] +cwd = "." +network = "deny" +scripts = "deny" + +[[component]] +id = "web.vite-build" +adapter = "trail/command@1" +kind = "generated" +depends_on = ["node"] +inputs = [{ path = "vite-source.js", role = "identity", format = "bytes" }] +outputs = [{ name = "dist", source = "dist", target = "dist", policy = "immutable_shared", reuse = "exact", scope = "workspace", publish = "on_sync", portability = "host" }] +[component.build] +command = ["cp", "vite-source.js", "dist/app.js"] +cwd = "." +network = "deny" +scripts = "deny" + +[[component.validation]] +name = "dist-path-contract" +kind = "path_contract" +path = "dist" +required = true +parameters = { maximum_entries = "1000" } + +[[component]] +id = "web.vite-cache" +adapter = "trail/command@1" +kind = "generated" +depends_on = ["node"] +inputs = [{ path = "vite-source.js", role = "identity", format = "bytes" }] +outputs = [{ name = "vite-cache", source = "vite-cache", target = ".vite", policy = "writable_private", reuse = "none", scope = "lane", publish = "manual", portability = "host" }] +[component.build] +command = ["cp", "vite-source.js", "vite-cache/metadata.json"] +cwd = "." +network = "deny" +scripts = "deny" +"#, + ) + .unwrap(); + + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else { + LaneWorkdirMode::FuseCow + }; + for lane in ["framework-one", "framework-two"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + mode.clone(), + None, + None, + None, + &[], + false, + ) .unwrap(); - db.lane_readiness("env-http").unwrap(); - let explained = trail::server::handle_http_request( - &mut db, - &api_request( - "GET", - "/v1/lanes/env-http/environment/explain?component=node", - serde_json::Value::Null, - ), - ); - assert_eq!(explained.status, 200); - let explained: serde_json::Value = explained.body_json().unwrap(); - assert_eq!(explained["status"], "stale"); - assert_eq!(explained["complete"], true); - assert!(explained["changes"] - .as_array() - .unwrap() + } + let graph = db + .workspace_environment_graph("framework-one", None) + .unwrap(); + assert_eq!(graph.nodes.len(), 4); + assert_eq!(graph.edges.len(), 3); + assert!(graph + .edges .iter() - .any(|change| { - change["dimension"] == "input" - && change["name"] == "package-lock.json" - && change["change"] == "modified" - })); - let mcp_explained = trail::mcp::handle_json_rpc( - &mut db, - serde_json::json!({ - "jsonrpc": "2.0", - "id": 21, - "method": "tools/call", - "params": { - "name": "trail.env_explain", - "arguments": {"lane": "env-http", "component": "node"} - } - }), - ) - .unwrap(); - assert_eq!(mcp_explained["result"]["isError"], false); - assert_eq!(mcp_explained["result"]["structuredContent"], explained); - let cli_explained = run_trail_json( - temp.path(), - &["env", "explain", "env-http", "--component", "node"], + .all(|edge| { edge.source_component_id == "node" && edge.edge_type == "build_requires" })); + drop(db); + + let first = run_trail_json(temp.path(), &["env", "sync", "all", "framework-one"]); + let second = run_trail_json(temp.path(), &["env", "sync", "all", "framework-two"]); + for synchronized in [&first, &second] { + let components = synchronized["generation"]["components"].as_array().unwrap(); + assert_eq!(components.len(), 4); + let output = |component_id: &str| { + &components + .iter() + .find(|component| component["component_id"] == component_id) + .unwrap()["outputs"][0] + }; + assert_eq!(output("web.next-build")["policy"], "writable_private"); + assert!(output("web.next-build")["layer_id"].is_null()); + assert_eq!(output("web.vite-build")["policy"], "immutable_shared"); + assert!(output("web.vite-build")["layer_id"].is_string()); + assert_eq!(output("web.vite-cache")["policy"], "writable_private"); + assert!(output("web.vite-cache")["layer_id"].is_null()); + } + assert_eq!( + first["generation"]["components"] + .as_array() + .unwrap() + .iter() + .find(|component| component["component_id"] == "web.vite-build") + .unwrap()["outputs"][0]["layer_id"], + second["generation"]["components"] + .as_array() + .unwrap() + .iter() + .find(|component| component["component_id"] == "web.vite-build") + .unwrap()["outputs"][0]["layer_id"] ); - assert_eq!(cli_explained, explained); - let all_http = trail::server::handle_http_request( - &mut db, - &api_request( - "POST", - "/v1/lanes/env-all-http/environment/sync-all", - serde_json::json!({}), - ), + let db = Trail::open(temp.path()).unwrap(); + let first_view = db.lane_workspace_view("framework-one").unwrap().unwrap(); + let second_view = db.lane_workspace_view("framework-two").unwrap().unwrap(); + let first_generated = Path::new(&first_view.generated_upper); + let second_generated = Path::new(&second_view.generated_upper); + fs::write(first_generated.join(".next/lane.txt"), "one\n").unwrap(); + fs::write(second_generated.join(".next/lane.txt"), "two\n").unwrap(); + fs::write(first_generated.join(".vite/cache.txt"), "one\n").unwrap(); + fs::write(second_generated.join(".vite/cache.txt"), "two\n").unwrap(); + assert_eq!( + fs::read_to_string(first_generated.join(".next/lane.txt")).unwrap(), + "one\n" ); - assert_eq!(all_http.status, 200); - let all_http: serde_json::Value = all_http.body_json().unwrap(); - assert_eq!(all_http["generation"]["generation_sequence"], 1); - assert_eq!(all_http["layers"][0]["layer_id"], http_layer["layer_id"]); - let all_mcp = trail::mcp::handle_json_rpc( - &mut db, - serde_json::json!({ - "jsonrpc": "2.0", - "id": 3, - "method": "tools/call", - "params": { - "name": "trail.env_sync_all", - "arguments": {"lane": "env-all-mcp"} - } - }), - ) - .unwrap(); - assert_eq!(all_mcp["result"]["isError"], false); assert_eq!( - all_mcp["result"]["structuredContent"]["layers"][0]["layer_id"], - http_layer["layer_id"] + fs::read_to_string(second_generated.join(".next/lane.txt")).unwrap(), + "two\n" + ); + assert_eq!( + fs::read_to_string(first_generated.join(".vite/cache.txt")).unwrap(), + "one\n" + ); + assert_eq!( + fs::read_to_string(second_generated.join(".vite/cache.txt")).unwrap(), + "two\n" ); } @@ -12245,9 +13059,22 @@ publish = "manual" let restored = tempfile::tempdir().unwrap(); Trail::restore_backup(restored.path(), &backup, false).unwrap(); let restored_db = Trail::open(restored.path()).unwrap(); - assert!(restored_db + let restored_view = restored_db .lane_workspace_view("private-http") .unwrap() + .expect("backup must retain the lane's private source view"); + assert!(Path::new(&restored_view.source_upper) + .starts_with(restored.path().canonicalize().unwrap().join(".trail"))); + assert!(Path::new(&restored_view.generated_upper).is_dir()); + assert!( + !Path::new(&restored_view.generated_upper) + .join(".trail-generated/private") + .exists(), + "generated private output is rebuildable and must not enter the backup" + ); + assert!(restored_db + .active_environment_generation("private-http") + .unwrap() .is_none()); assert!(restored_db.fsck().unwrap().errors.is_empty()); } @@ -12255,6 +13082,9 @@ publish = "manual" #[cfg(unix)] #[test] fn successful_gate_promotes_only_exact_successful_environment_evidence() { + let Some(mode) = owning_host_layered_workdir_mode() else { + return; + }; let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("input.txt"), "gate input\n").unwrap(); fs::write( @@ -12293,11 +13123,6 @@ gate = "build" .unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(temp.path()).unwrap(); - let mode = if cfg!(target_os = "macos") { - LaneWorkdirMode::NfsCow - } else { - LaneWorkdirMode::FuseCow - }; for lane in ["gate-pass", "gate-fail"] { db.spawn_lane_with_workdir_mode_paths_and_neighbors( lane, @@ -14542,6 +15367,32 @@ fn mcp_stdio_tools_drive_lane_turn_workflow() { .iter() .any(|value| value.as_str() == Some("main"))); + for (uri, argument) in [ + ("trail://workspace/artifacts/{artifact_id}", "artifact_id"), + ( + "trail://workspace/artifact-quarantines/{quarantine_id}", + "quarantine_id", + ), + ] { + let completion = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 180, + "method": "completion/complete", + "params": { + "ref": { "type": "ref/resource", "uri": uri }, + "argument": { "name": argument, "value": "" } + } + }), + ) + .unwrap(); + assert_eq!( + completion["result"]["completion"]["values"], + serde_json::json!([]) + ); + } + let agent_selector_completion = trail::mcp::handle_json_rpc( &mut db, serde_json::json!({ @@ -22584,6 +23435,7 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let workdir_parent = tempfile::tempdir().unwrap(); + let native_cow_supported = native_cow_supported_for_test(); let default_spawn = run_trail_json( temp.path(), &["lane", "spawn", "default-bot", "--from", "main"], @@ -22604,61 +23456,63 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { assert_eq!(default_spawn["sparse_paths"].as_array().unwrap().len(), 0); assert_eq!(default_spawn["transparent_cow_available"], true); - let cli_workdir = workdir_parent.path().join("cli-bot"); - let cli_spawn = run_trail_json( - temp.path(), - &[ - "lane", - "spawn", - "cli-bot", - "--from", - "main", - "--workdir", - cli_workdir.to_str().unwrap(), - "--workdir-mode", - "native-cow", - ], - ); - assert_eq!(cli_spawn["requested_workdir_mode"], "native-cow"); - assert_eq!(cli_spawn["workdir_mode"], "native-cow"); - assert_eq!(cli_spawn["workdir_backend"], "clone"); - assert_eq!(cli_spawn["materialization"]["copied_files"], 0); - assert!( - cli_spawn["materialization"]["cloned_files"] - .as_u64() - .unwrap() - > 0 - ); - assert_eq!( - PathBuf::from(cli_spawn["workdir"].as_str().unwrap()) - .canonicalize() - .unwrap(), - cli_workdir.canonicalize().unwrap() - ); - assert_eq!( - fs::read_to_string(cli_workdir.join("README.md")).unwrap(), - "hello\n" - ); + if native_cow_supported { + let cli_workdir = workdir_parent.path().join("cli-bot"); + let cli_spawn = run_trail_json( + temp.path(), + &[ + "lane", + "spawn", + "cli-bot", + "--from", + "main", + "--workdir", + cli_workdir.to_str().unwrap(), + "--workdir-mode", + "native-cow", + ], + ); + assert_eq!(cli_spawn["requested_workdir_mode"], "native-cow"); + assert_eq!(cli_spawn["workdir_mode"], "native-cow"); + assert_eq!(cli_spawn["workdir_backend"], "clone"); + assert_eq!(cli_spawn["materialization"]["copied_files"], 0); + assert!( + cli_spawn["materialization"]["cloned_files"] + .as_u64() + .unwrap() + > 0 + ); + assert_eq!( + PathBuf::from(cli_spawn["workdir"].as_str().unwrap()) + .canonicalize() + .unwrap(), + cli_workdir.canonicalize().unwrap() + ); + assert_eq!( + fs::read_to_string(cli_workdir.join("README.md")).unwrap(), + "hello\n" + ); - let native_workdir = workdir_parent.path().join("native-bot"); - let native_spawn = run_trail_json( - temp.path(), - &[ - "lane", - "spawn", - "native-bot", - "--from", - "main", - "--workdir-mode", - "native-cow", - "--workdir", - native_workdir.to_str().unwrap(), - ], - ); - assert_eq!(native_spawn["requested_workdir_mode"], "native-cow"); - assert_eq!(native_spawn["workdir_mode"], "native-cow"); - assert_eq!(native_spawn["workdir_backend"], "clone"); - assert_eq!(native_spawn["materialization"]["copied_files"], 0); + let native_workdir = workdir_parent.path().join("native-bot"); + let native_spawn = run_trail_json( + temp.path(), + &[ + "lane", + "spawn", + "native-bot", + "--from", + "main", + "--workdir-mode", + "native-cow", + "--workdir", + native_workdir.to_str().unwrap(), + ], + ); + assert_eq!(native_spawn["requested_workdir_mode"], "native-cow"); + assert_eq!(native_spawn["workdir_mode"], "native-cow"); + assert_eq!(native_spawn["workdir_backend"], "clone"); + assert_eq!(native_spawn["materialization"]["copied_files"], 0); + } let portable_workdir = workdir_parent.path().join("portable-bot"); let portable_spawn = run_trail_json( @@ -22677,8 +23531,15 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { ); assert_eq!(portable_spawn["requested_workdir_mode"], "portable-copy"); assert_eq!(portable_spawn["workdir_mode"], "portable-copy"); - assert_eq!(portable_spawn["workdir_backend"], "clone"); - assert_eq!(portable_spawn["materialization"]["copied_files"], 0); + if native_cow_supported { + assert_eq!(portable_spawn["workdir_backend"], "clone"); + assert_eq!(portable_spawn["materialization"]["cloned_files"], 5); + assert_eq!(portable_spawn["materialization"]["copied_files"], 0); + } else { + assert_eq!(portable_spawn["workdir_backend"], "copy"); + assert_eq!(portable_spawn["materialization"]["cloned_files"], 0); + assert_eq!(portable_spawn["materialization"]["copied_files"], 5); + } let headless_spawn = run_trail_json( temp.path(), @@ -22779,21 +23640,23 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { } let db = Trail::open(temp.path()).unwrap(); - assert_eq!( - db.lane_status("cli-bot").unwrap().workdir_state, - Some(WorktreeState::Clean) - ); - let persisted_workdir = db.lane_workdir("cli-bot").unwrap(); - assert_eq!( - persisted_workdir.requested_workdir_mode, - LaneWorkdirMode::NativeCow - ); - assert_eq!(persisted_workdir.workdir_mode, LaneWorkdirMode::NativeCow); - assert_eq!( - persisted_workdir.workdir_backend, - Some(WorkdirBackend::Clone) - ); - assert!(persisted_workdir.materialization.is_some()); + if native_cow_supported { + assert_eq!( + db.lane_status("cli-bot").unwrap().workdir_state, + Some(WorktreeState::Clean) + ); + let persisted_workdir = db.lane_workdir("cli-bot").unwrap(); + assert_eq!( + persisted_workdir.requested_workdir_mode, + LaneWorkdirMode::NativeCow + ); + assert_eq!(persisted_workdir.workdir_mode, LaneWorkdirMode::NativeCow); + assert_eq!( + persisted_workdir.workdir_backend, + Some(WorkdirBackend::Clone) + ); + assert!(persisted_workdir.materialization.is_some()); + } let sparse_spawn = run_trail_json( temp.path(), @@ -22808,9 +23671,15 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { ], ); assert_eq!(sparse_spawn["workdir_mode"], "sparse"); - assert_eq!(sparse_spawn["workdir_backend"], "clone"); - assert_eq!(sparse_spawn["materialization"]["cloned_files"], 1); - assert_eq!(sparse_spawn["materialization"]["copied_files"], 0); + if native_cow_supported { + assert_eq!(sparse_spawn["workdir_backend"], "clone"); + assert_eq!(sparse_spawn["materialization"]["cloned_files"], 1); + assert_eq!(sparse_spawn["materialization"]["copied_files"], 0); + } else { + assert_eq!(sparse_spawn["workdir_backend"], "copy"); + assert_eq!(sparse_spawn["materialization"]["cloned_files"], 0); + assert_eq!(sparse_spawn["materialization"]["copied_files"], 1); + } assert_eq!( sparse_spawn["sparse_paths"] .as_array() @@ -23120,6 +23989,11 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { assert_eq!(cleanup.changed_paths[0].path, "README.md"); let api_workdir = workdir_parent.path().join("api-bot"); + let api_workdir_mode = if native_cow_supported { + "native-cow" + } else { + "portable-copy" + }; let api_response = trail::server::handle_http_request( &mut db, &api_request( @@ -23128,14 +24002,33 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { serde_json::json!({ "name": "api-bot", "from_ref": "main", - "workdir_mode": "native-cow", + "workdir_mode": api_workdir_mode, "workdir": api_workdir }), ), ); assert_eq!(api_response.status, 201); let api_spawn: serde_json::Value = api_response.body_json().unwrap(); - assert_eq!(api_spawn["workdir_mode"], "native-cow"); + assert_eq!(api_spawn["workdir_mode"], api_workdir_mode); + if native_cow_supported { + assert_eq!(api_spawn["workdir_backend"], "clone"); + assert_eq!(api_spawn["materialization"]["copied_files"], 0); + assert!( + api_spawn["materialization"]["cloned_files"] + .as_u64() + .unwrap() + > 0 + ); + } else { + assert_eq!(api_spawn["workdir_backend"], "copy"); + assert_eq!(api_spawn["materialization"]["cloned_files"], 0); + assert!( + api_spawn["materialization"]["copied_files"] + .as_u64() + .unwrap() + > 0 + ); + } assert_eq!( PathBuf::from(api_spawn["workdir"].as_str().unwrap()) .canonicalize() @@ -23410,6 +24303,13 @@ fn native_cow_lane_space_reports_allocated_and_changed_bytes_honestly() { report["physical_sharing_evidence"], "allocated_blocks_do_not_prove_apfs_extent_sharing" ); + assert_eq!(report["artifact_storage"]["lane_private_bytes"], 0); + assert_eq!(report["artifact_storage"]["reclaimable_bytes"], 0); + assert_eq!( + report["artifact_storage"]["unknown_bytes"], + report["filesystem_allocated_bytes"] + ); + assert_eq!(report["artifact_storage"]["prefetched_bytes"], 0); let output = Command::new(trail_bin()) .arg("--workspace") @@ -23433,6 +24333,15 @@ fn native_cow_lane_space_reports_allocated_and_changed_bytes_honestly() { "Physical sharing", "unknown", "allocated_blocks_do_not_prove_apfs_extent_sharing", + "Artifact logical", + "Artifact authoritative unique", + "Artifact authoritative shared", + "Artifact materialized", + "Artifact lane-private", + "Artifact prefetched", + "Artifact demand-loaded", + "Artifact reclaimable", + "Artifact unknown", ] { assert!( stdout.contains(expected), @@ -23658,7 +24567,7 @@ fn lane_spawn_materialization_ignores_dirty_workspace_for_recorded_root() { } #[test] -fn auto_reports_mixed_when_portable_restart_can_clone_only_clean_files() { +fn portable_restart_reports_clone_or_copy_evidence_for_clean_files() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("clean.txt"), "clean\n").unwrap(); fs::write(temp.path().join("dirty.txt"), "recorded\n").unwrap(); @@ -23671,9 +24580,17 @@ fn auto_reports_mixed_when_portable_restart_can_clone_only_clean_files() { .unwrap(); let report = spawned.materialization.unwrap(); assert_eq!(spawned.workdir_mode, LaneWorkdirMode::PortableCopy); - assert_eq!(spawned.workdir_backend, Some(WorkdirBackend::Mixed)); - assert_eq!(report.cloned_files, 1); - assert_eq!(report.copied_files, 1); + match spawned.workdir_backend { + Some(WorkdirBackend::Mixed) => { + assert_eq!(report.cloned_files, 1); + assert_eq!(report.copied_files, 1); + } + Some(WorkdirBackend::Copy) => { + assert_eq!(report.cloned_files, 0); + assert_eq!(report.copied_files, 2); + } + backend => panic!("unexpected portable-copy backend: {backend:?}"), + } assert_eq!(report.fallback_reason, None); let workdir = PathBuf::from(spawned.workdir.unwrap()); assert_eq!( @@ -23717,6 +24634,9 @@ fn strict_native_cow_refuses_an_unvalidated_source_without_copying() { #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn strict_native_cow_accepts_a_git_tracked_file_inside_an_ignored_directory() { + if !native_cow_supported_for_test() { + return; + } let temp = tempfile::tempdir().unwrap(); run_git(temp.path(), &["init", "--quiet"]); run_git(temp.path(), &["config", "user.name", "Trail COW"]); @@ -23788,6 +24708,9 @@ fn strict_native_cow_accepts_a_git_tracked_file_inside_an_ignored_directory() { #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn strict_native_cow_accepts_a_non_git_baseline_file_inside_an_ignored_directory() { + if !native_cow_supported_for_test() { + return; + } let temp = tempfile::tempdir().unwrap(); fs::create_dir(temp.path().join("generated")).unwrap(); fs::write(temp.path().join("generated/baseline.txt"), "baseline\n").unwrap(); @@ -23855,6 +24778,9 @@ fn non_git_ignore_cannot_hide_a_trail_baseline_file() { #[test] fn strict_native_cow_reuses_a_complete_clean_lane_source() { + if !native_cow_supported_for_test() { + return; + } let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); @@ -23891,6 +24817,9 @@ fn strict_native_cow_reuses_a_complete_clean_lane_source() { #[cfg(unix)] #[test] fn strict_native_cow_does_not_preserve_source_hardlink_aliasing() { + if !native_cow_supported_for_test() { + return; + } let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("a.txt"), "shared\n").unwrap(); fs::hard_link(temp.path().join("a.txt"), temp.path().join("b.txt")).unwrap(); @@ -23922,6 +24851,9 @@ fn strict_native_cow_does_not_preserve_source_hardlink_aliasing() { #[test] fn strict_native_cow_probes_an_empty_root() { + if !native_cow_supported_for_test() { + return; + } let temp = tempfile::tempdir().unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); @@ -25610,6 +26542,9 @@ fn lane_merge_queue_runs_lane_branch_into_main() { #[test] fn lane_merge_queue_preserves_environment_gate_qualification() { + let Some(mode) = owning_host_layered_workdir_mode() else { + return; + }; let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); fs::create_dir_all(temp.path().join("src")).unwrap(); @@ -25632,13 +26567,6 @@ fn lane_merge_queue_preserves_environment_gate_qualification() { Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(temp.path()).unwrap(); - let mode = if cfg!(target_os = "macos") { - LaneWorkdirMode::NfsCow - } else if cfg!(target_os = "windows") { - LaneWorkdirMode::DokanCow - } else { - LaneWorkdirMode::FuseCow - }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "environment-bot", Some("main"), diff --git a/trail/tests/fixtures/changed_path_raw_mutations.v1 b/trail/tests/fixtures/changed_path_raw_mutations.v1 index 1306b036..e810846a 100644 --- a/trail/tests/fixtures/changed_path_raw_mutations.v1 +++ b/trail/tests/fixtures/changed_path_raw_mutations.v1 @@ -114,17 +114,14 @@ reviewed|db/core/backup/create.rs|create_backup_inner|fs::copy|3 reviewed|db/core/backup/create.rs|create_backup_inner|fs::create_dir_all|3 reviewed|db/core/backup/create.rs|create_backup_inner|fs::write|3 reviewed|db/core/backup/create.rs|create_backup|fs::create_dir_all|1 -reviewed|db/core/backup/publication.rs|publish_staged_tree_with_exchange|fs::rename|1 -reviewed|db/core/backup/publication.rs|remove_any|fs::remove_dir_all|1 -reviewed|db/core/backup/publication.rs|remove_any|fs::remove_file|1 +reviewed|db/core/backup/publication.rs|remove_any_once|fs::remove_dir_all|1 +reviewed|db/core/backup/publication.rs|remove_any_once|fs::remove_file|1 +reviewed|db/core/backup/publication.rs|rename_publication_entry|fs::rename|2 reviewed|db/core/backup/publication.rs|sibling_stage|fs::create_dir|1 reviewed|db/core/backup/restore_transaction.rs|allocate_policy_stage|OpenOptions::create_new|1 reviewed|db/core/backup/restore_transaction.rs|allocate_policy_stage|OpenOptions::write|1 reviewed|db/core/backup/restore_transaction.rs|prepare|fs::copy|2 reviewed|db/core/backup/restore_transaction.rs|prepare|fs::write|1 -reviewed|db/core/backup/restore_transaction.rs|publish_policy_entry|fs::rename|1 -reviewed|db/core/backup/restore_transaction.rs|restore_old_entry|fs::rename|1 -reviewed|db/core/backup/restore_transaction.rs|write_marker|fs::rename|1 reviewed|db/core/backup/restore_transaction.rs|write_marker|fs::write|1 reviewed|db/core/backup/restore.rs|restore_backup|fs::copy|3 reviewed|db/core/backup/restore.rs|restore_backup|fs::create_dir_all|4 @@ -141,6 +138,9 @@ reviewed|db/core/init.rs|init_with_options|fs::write|4 reviewed|db/core/init.rs|open_at_without_recovery|fs::create_dir_all|1 reviewed|db/core/workspace/ignore.rs|ignore_add|fs::write|1 reviewed|db/core/workspace/ignore.rs|ignore_remove|fs::write|1 +reviewed|db/lane/identity.rs|rewrite_restored_lane_workdir_paths|fs::create_dir_all|2 +reviewed|db/lane/identity.rs|scrub_restored_view_metadata|fs::remove_dir_all|1 +reviewed|db/lane/identity.rs|scrub_restored_view_metadata|fs::remove_file|1 reviewed|db/lane/lifecycle.rs|lane_initialization_crash_cut|OpenOptions::create_new|1 reviewed|db/lane/lifecycle.rs|lane_initialization_crash_cut|OpenOptions::write|1 reviewed|db/lane/lifecycle.rs|write_sparse_workdir_manifest|fs::create_dir_all|1 @@ -196,6 +196,10 @@ reviewed|db/lane/workdir/sync.rs|sync_lane_workdir_with_paths_and_neighbors|fs:: reviewed|db/lane/workdir/view_barrier.rs|acquire|fs::create_dir_all|1 reviewed|db/lane/workdir/view_barrier.rs|open_barrier_no_follow|OpenOptions::create|1 reviewed|db/lane/workdir/view_barrier.rs|open_barrier_no_follow|OpenOptions::write|1 +reviewed|db/lane/workdir/view_conformance.rs|lazy_artifact_conformance_binding|fs::create_dir_all|1 +reviewed|db/lane/workdir/view_conformance.rs|lazy_artifact_conformance_binding|fs::write|2 +reviewed|db/lane/workdir/view_conformance.rs|run_mounted_lazy_artifact_conformance|fs::remove_file|1 +reviewed|db/lane/workdir/view_conformance.rs|run_mounted_lazy_artifact_conformance|OpenOptions::write|1 reviewed|db/lane/workdir/view_conformance.rs|run_mounted_view_conformance|fs::create_dir_all|1 reviewed|db/lane/workdir/view_conformance.rs|run_mounted_view_conformance|fs::remove_file|2 reviewed|db/lane/workdir/view_conformance.rs|run_mounted_view_conformance|fs::rename|1 @@ -214,6 +218,7 @@ reviewed|db/lane/workdir/view_core.rs|ensure_upper_file_under_barrier|OpenOption reviewed|db/lane/workdir/view_core.rs|ensure_upper_file_under_barrier|OpenOptions::write|1 reviewed|db/lane/workdir/view_core.rs|ensure_upper_parent|fs::create_dir_all|1 reviewed|db/lane/workdir/view_core.rs|merge_lower_subtree_into_upper|fs::create_dir_all|2 +reviewed|db/lane/workdir/view_core.rs|merge_artifact_subtree_into_upper|fs::create_dir_all|1 reviewed|db/lane/workdir/view_core.rs|mkdir|fs::create_dir_all|1 reviewed|db/lane/workdir/view_core.rs|move_upper_subtree|fs::create_dir_all|2 reviewed|db/lane/workdir/view_core.rs|move_upper_subtree|fs::remove_dir_all|2 @@ -258,6 +263,12 @@ reviewed|db/lane/workspace_environment.rs|environment_cache_namespace_has_live_l reviewed|db/lane/workspace_environment.rs|execute_workspace_environment_plan_in_directory|fs::create_dir_all|2 reviewed|db/lane/workspace_environment.rs|initialize_mounted_workspace_environment_plans|fs::create_dir_all|1 reviewed|db/lane/workspace_environment.rs|initialize_mounted_workspace_environment_plans|fs::remove_dir_all|1 +reviewed|db/lane/workspace_environment.rs|execute_reviewed_builtin_resolution_plan|fs::create_dir_all|4 +reviewed|db/lane/workspace_environment.rs|execute_reviewed_builtin_resolution_plan|fs::set_permissions|1 +reviewed|db/lane/workspace_environment.rs|materialize_workspace_environment_resolution_input|fs::create_dir_all|1 +reviewed|db/lane/workspace_environment.rs|materialize_workspace_environment_resolution_input|fs::set_permissions|1 +reviewed|db/lane/workspace_environment.rs|materialize_workspace_environment_resolution_input|OpenOptions::create_new|1 +reviewed|db/lane/workspace_environment.rs|materialize_workspace_environment_resolution_input|OpenOptions::write|1 reviewed|db/lane/workspace_environment.rs|materialize_workspace_environment_input|fs::copy|1 reviewed|db/lane/workspace_environment.rs|materialize_workspace_environment_input|fs::create_dir_all|1 reviewed|db/lane/workspace_environment.rs|materialize_workspace_environment_input|fs::set_permissions|1 @@ -273,27 +284,47 @@ reviewed|db/lane/workspace_layer.rs|acquire_environment_cache_maintenance|fs::re reviewed|db/lane/workspace_layer.rs|acquire_environment_cache_maintenance|OpenOptions::create_new|1 reviewed|db/lane/workspace_layer.rs|acquire_environment_cache_maintenance|OpenOptions::write|1 reviewed|db/lane/workspace_layer.rs|active_cache_builder_count|fs::remove_file|1 -reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight|fs::create_dir_all|2 -reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight|fs::remove_dir_all|1 -reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight|fs::rename|2 -reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight|OpenOptions::create_new|1 -reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight|OpenOptions::write|1 -reviewed|db/lane/workspace_layer.rs|copy_layer_tree|fs::copy|1 +reviewed|db/lane/workspace_artifact.rs|artifact_materialization_cache_parents|fs::create_dir|1 +reviewed|db/lane/workspace_artifact.rs|ensure_artifact_tree_materialization_under_write_lock|fs::remove_dir_all|6 +reviewed|db/lane/workspace_artifact.rs|ensure_artifact_tree_materialization_under_write_lock|fs::rename|1 +reviewed|db/lane/workspace_artifact.rs|materialize_artifact_directory|fs::create_dir|1 +reviewed|db/lane/workspace_artifact.rs|materialize_artifact_directory|symlink_file|2 +reviewed|db/lane/workspace_artifact.rs|materialize_artifact_directory|OpenOptions::create_new|1 +reviewed|db/lane/workspace_artifact.rs|materialize_artifact_directory|OpenOptions::write|1 +reviewed|db/lane/workspace_artifact.rs|materialize_artifact_file|fs::remove_file|1 +reviewed|db/lane/workspace_artifact.rs|materialize_artifact_file|OpenOptions::create_new|1 +reviewed|db/lane/workspace_artifact.rs|materialize_artifact_file|OpenOptions::write|1 +reviewed|db/lane/workspace_artifact.rs|materialize_artifact_tree_under_write_lock|fs::create_dir|1 +reviewed|db/lane/workspace_artifact.rs|materialize_artifact_tree_under_write_lock|fs::remove_dir_all|1 +reviewed|db/lane/workspace_artifact.rs|set_artifact_materialized_mode|fs::set_permissions|1 +reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight_with_cancel|fs::create_dir_all|2 +reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight_with_cancel|fs::remove_dir_all|1 +reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight_with_cancel|fs::rename|2 +reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight_with_cancel|OpenOptions::create_new|1 +reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight_with_cancel|OpenOptions::write|1 reviewed|db/lane/workspace_layer.rs|copy_layer_tree|fs::create_dir_all|4 reviewed|db/lane/workspace_layer.rs|create_layer_symlink|symlink_file|1 reviewed|db/lane/workspace_layer.rs|drop|fs::remove_file|1 +reviewed|db/lane/workspace_layer.rs|enforce_workspace_cache_build_quota|fs::create_dir_all|1 reviewed|db/lane/workspace_layer.rs|make_layer_root_writable|fs::set_permissions|1 reviewed|db/lane/workspace_layer.rs|make_tree_writable|fs::set_permissions|1 reviewed|db/lane/workspace_layer.rs|preserve_layer_mode|fs::set_permissions|1 -reviewed|db/lane/workspace_layer.rs|publish_workspace_layer_from_directory|fs::create_dir_all|2 -reviewed|db/lane/workspace_layer.rs|publish_workspace_layer_from_directory|fs::rename|2 +reviewed|db/lane/workspace_layer.rs|promote_workspace_environment_output_with_evidence|fs::remove_dir_all|1 +reviewed|db/lane/workspace_layer.rs|publish_workspace_layer_from_directory_under_write_lock|fs::create_dir_all|2 +reviewed|db/lane/workspace_layer.rs|publish_workspace_layer_from_directory_under_write_lock|fs::rename|2 +reviewed|db/lane/workspace_layer.rs|recover_artifact_construction_attempts_under_write_lock|fs::remove_file|1 +reviewed|db/lane/workspace_layer.rs|recover_workspace_layer_materializations|fs::create_dir_all|2 +reviewed|db/lane/workspace_layer.rs|recover_workspace_layer_materializations|fs::remove_dir_all|1 +reviewed|db/lane/workspace_layer.rs|recover_workspace_layer_materializations|fs::rename|1 +reviewed|db/lane/workspace_layer.rs|recover_workspace_layer_publications|fs::remove_dir_all|1 +reviewed|db/lane/workspace_layer.rs|remove_exact_recovery_directory|fs::remove_dir_all|1 reviewed|db/lane/workspace_layer.rs|remove_workspace_layer_trash_entries|fs::remove_dir_all|1 reviewed|db/lane/workspace_layer.rs|remove_workspace_layer_trash_entries|fs::remove_file|1 reviewed|db/lane/workspace_layer.rs|set_layer_read_only|fs::set_permissions|2 reviewed|db/lane/workspace_layer.rs|workspace_cache_gc|fs::create_dir_all|1 -reviewed|db/lane/workspace_layer.rs|workspace_cache_gc|fs::remove_dir_all|3 +reviewed|db/lane/workspace_layer.rs|workspace_cache_gc|fs::remove_dir_all|4 reviewed|db/lane/workspace_layer.rs|workspace_cache_gc|fs::remove_file|3 -reviewed|db/lane/workspace_layer.rs|workspace_cache_gc|fs::rename|5 +reviewed|db/lane/workspace_layer.rs|workspace_cache_gc|fs::rename|9 reviewed|db/lane/workspace_plugin.rs|append_environment_plugin_registry_record|fs::create_dir_all|1 reviewed|db/lane/workspace_plugin.rs|append_publisher_trust_record|fs::create_dir_all|1 reviewed|db/lane/workspace_plugin.rs|install_environment_adapter_plugin|fs::create_dir_all|2 diff --git a/trail/tests/lane_environment_inheritance.rs b/trail/tests/lane_environment_inheritance.rs index 2978b408..e02bf790 100644 --- a/trail/tests/lane_environment_inheritance.rs +++ b/trail/tests/lane_environment_inheritance.rs @@ -66,6 +66,19 @@ fn lane_fork_inherits_verified_immutable_layer_with_fresh_private_uppers() { params![&layer.cache_key, &layer.layer_id], ) .unwrap(); + conn.execute( + "INSERT INTO environment_component_states( + view_id,component_id,adapter_identity,adapter_version,implementation_version, + distribution_digest,kind,expected_key,attached_key,status,reason,updated_at) + VALUES(?1,'node','trail/node@1',1,?2,'builtin:node-plan-v1','dependency', + ?3,?3,'ready',NULL,1)", + params![ + &parent_view.view_id, + env!("CARGO_PKG_VERSION"), + &layer.cache_key + ], + ) + .unwrap(); conn.execute( "INSERT INTO environment_generation_outputs( generation_id,component_id,output_name,policy,reuse_mode,sharing_scope, @@ -77,6 +90,17 @@ fn lane_fork_inherits_verified_immutable_layer_with_fresh_private_uppers() { params![&layer.cache_key, &layer.layer_id], ) .unwrap(); + conn.execute( + "INSERT INTO environment_generation_outputs( + generation_id,component_id,output_name,policy,reuse_mode,sharing_scope, + publication_trigger,publication_gate,storage_identity,layer_id, + manifest_object_id,publication_id,mount_path,layer_subpath) + VALUES('env_parent','node','rejected-sibling','immutable_seed_private','exact', + 'workspace','on_sync',NULL,'rejected-key','layer_missing','object_missing',NULL, + 'vendor/rejected','')", + [], + ) + .unwrap(); conn.execute( "INSERT INTO environment_view_generations(view_id,generation_id,updated_at) VALUES(?1,'env_parent',1)", @@ -151,7 +175,7 @@ fn lane_fork_inherits_verified_immutable_layer_with_fresh_private_uppers() { Some(layer.layer_id.as_str()) ); - let conn = Connection::open(sqlite).unwrap(); + let conn = Connection::open(&sqlite).unwrap(); let child_binding: String = conn .query_row( "SELECT layer_id FROM workspace_view_layers @@ -170,6 +194,27 @@ fn lane_fork_inherits_verified_immutable_layer_with_fresh_private_uppers() { ) .unwrap(); assert_eq!(inherited_runtime_count, 0); + let child_output_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM environment_generation_outputs WHERE generation_id=?1", + [&generation.generation_id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(child_output_count, 1); + let child_artifact_binding: (String, String, String, String) = conn + .query_row( + "SELECT desired_key,envelope_id,tree_root_id,binding_identity + FROM artifact_generation_bindings + WHERE generation_id=?1 AND component_id='node' AND output_name='dependencies'", + [&generation.generation_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .unwrap(); + assert_eq!(child_artifact_binding.0, layer.cache_key); + assert!(child_artifact_binding.1.starts_with("artifact_envelope_")); + assert!(child_artifact_binding.2.starts_with("artifact_tree_")); + assert!(child_artifact_binding.3.starts_with("artifact_binding_")); drop(conn); let inheritance_event = db .list_lane_events( @@ -195,6 +240,12 @@ fn lane_fork_inherits_verified_immutable_layer_with_fresh_private_uppers() { && output["decision"] == "rejected" && output["reason"] == "layer_verification_failed" })); + assert!(outputs.iter().any(|output| { + output["component_id"] == "node" + && output["output_name"] == "rejected-sibling" + && output["decision"] == "rejected" + && output["reason"] == "layer_verification_failed" + })); let fork_names = (0..4) .map(|index| format!("concurrent-child-{index}")) @@ -239,6 +290,24 @@ fn lane_fork_inherits_verified_immutable_layer_with_fresh_private_uppers() { assert_ne!(fork_view.scratch_upper, child_view.scratch_upper); } + let conn = Connection::open(&sqlite).unwrap(); + let mut artifact_binding_identities = conn + .prepare( + "SELECT b.binding_identity + FROM artifact_generation_bindings b + JOIN environment_generations g ON g.generation_id=b.generation_id + WHERE b.component_id='node' AND b.output_name='dependencies' + ORDER BY b.binding_identity", + ) + .unwrap() + .query_map([], |row| row.get::<_, String>(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + artifact_binding_identities.dedup(); + assert_eq!(artifact_binding_identities.len(), fork_names.len() + 1); + drop(conn); + let conn = Connection::open(root.path().join(".trail/index/trail.sqlite")).unwrap(); conn.execute( "DELETE FROM environment_generation_runtime_resources diff --git a/trail/tests/managed_execution.rs b/trail/tests/managed_execution.rs index c1704467..f8346764 100644 --- a/trail/tests/managed_execution.rs +++ b/trail/tests/managed_execution.rs @@ -122,6 +122,13 @@ fn lane_test_uses_managed_lifecycle_and_checkpoints_after_command_failure() { assert!(!report.success); assert_eq!(report.exit_code, Some(9)); assert_eq!(report.lifecycle.surface, "lane_test"); + let preparation = report.lifecycle.preparation.as_ref().unwrap(); + assert_eq!( + serde_json::to_value(preparation.missing_resolution_policy).unwrap(), + "explicit" + ); + assert!(preparation.resolution_pins.is_empty()); + assert!(preparation.output_pins.is_empty()); assert!(report .lifecycle .checkpoint @@ -150,6 +157,12 @@ fn lane_test_uses_managed_lifecycle_and_checkpoints_after_command_failure() { "unmount", ] ); + let finalization = report.lifecycle.finalization.as_ref().unwrap(); + assert!(finalization.complete); + assert!(finalization.source_changed); + assert_eq!(finalization.checkpoint_status, "succeeded"); + assert_eq!(finalization.disposal_status, "skipped"); + assert_eq!(finalization.unmount_status, "skipped"); } #[test] @@ -315,6 +328,19 @@ fn terminal_agent_uses_managed_lifecycle_and_returns_its_receipt() { let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); assert_eq!(report["status"], "completed"); assert_eq!(report["lifecycle"]["surface"], "terminal_agent"); + assert_eq!( + report["lifecycle"]["preparation"]["missing_resolution_policy"], + "explicit" + ); + assert!(report["lifecycle"]["preparation"]["resolution_pins"] + .as_array() + .unwrap() + .is_empty()); + assert_eq!( + report["lifecycle"]["finalization"]["checkpoint_status"], + "succeeded" + ); + assert_eq!(report["lifecycle"]["finalization"]["complete"], true); assert!(report["lifecycle"]["checkpoint"]["source_paths"] .as_array() .unwrap()