From 4762f019068e48f70871fcb81d51f4b00cab024d Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 16:04:20 -0700 Subject: [PATCH 01/19] perf(storage): retain packs and collect Cellule state Require fresh storage prefixes for repository schema 3 and the current compiled Cellule release. Remove legacy Git-body storage readers and keep release, format, identity, and fencing validation. --- .github/dependabot.yml | 13 + .github/workflows/verify.yml | 13 +- .gitignore | 1 + AGENTS.md | 19 + Cargo.lock | 401 +++- Cargo.toml | 14 +- README.md | 16 +- ROADMAP.md | 24 +- docs/api-reference.md | 35 +- docs/benchmarks/2026-10-13-rustfs-browse.json | 1622 +++++++++++++++++ .../2026-10-14-storage-optimization.md | 175 ++ docs/contracts.md | 246 ++- docs/delivery-plan.md | 103 +- docs/git-compatibility.md | 7 +- docs/implementation.md | 113 +- docs/operations.md | 51 +- docs/performance-plan.md | 247 ++- docs/repository-cell-primitives.md | 10 +- scripts/benchmark_browse.py | 330 ++++ scripts/check_cellule.py | 54 + scripts/qualify_size.py | 21 +- scripts/smoke_s3_activation.py | 4 +- scripts/smoke_s3_backup.py | 5 +- scripts/smoke_s3_cache.py | 4 +- scripts/smoke_s3_peers.py | 7 +- scripts/smoke_s3_process.py | 28 +- scripts/smoke_s3_retention.py | 92 + scripts/test_benchmark_browse.py | 70 + scripts/test_check_cellule.py | 67 + src/deployment.rs | 16 + src/deployment/backup.rs | 4 +- src/deployment/backup/bodies.rs | 145 +- src/deployment/maintenance.rs | 165 ++ src/deployment/recovery.rs | 7 +- src/deployment/retention.rs | 108 ++ src/deployment/tests.rs | 181 +- src/external.rs | 24 +- src/git_cache.rs | 207 ++- src/git_cache/tests.rs | 105 +- src/git_gateway.rs | 97 +- src/git_gateway/fetch.rs | 144 +- src/git_gateway/hydration.rs | 86 +- src/git_gateway/push.rs | 3 + src/git_http.rs | 5 +- src/git_objects.rs | 102 +- src/git_objects/tests.rs | 20 +- src/git_pack.rs | 274 +++ src/git_pack/build.rs | 241 +++ src/git_pack/build/tests.rs | 115 ++ src/git_pack/store.rs | 433 +++++ src/git_pack/store/tests.rs | 211 +++ src/git_pack/tests.rs | 166 ++ src/graph.rs | 158 +- src/graph/preparation.rs | 118 +- src/graph/stream.rs | 305 ++++ src/graph/tests.rs | 50 + src/http.rs | 4 + src/large_blob.rs | 167 -- src/large_blob/read.rs | 82 - src/large_blob/tests.rs | 165 -- src/lib.rs | 149 +- src/main.rs | 33 +- src/native_git.rs | 110 +- src/object_batch.rs | 466 ++--- src/object_batch/tests.rs | 47 +- src/object_chunks.rs | 190 -- src/object_packs.rs | 338 ++++ src/object_reads.rs | 151 +- src/object_reads/tests.rs | 16 +- src/pulls/candidates.rs | 11 +- src/pulls/candidates/rebase.rs | 111 +- src/schema.sql | 35 +- src/server.rs | 11 +- src/server/residency.rs | 8 + tests/directory_cell.rs | 43 +- tests/multi_server/backup.rs | 3 +- tests/multi_server/bulk_refs.rs | 28 +- tests/multi_server/large_objects.rs | 2 +- tests/multi_server/lifecycle.rs | 2 +- tests/multi_server/residency/faults.rs | 15 + tests/multi_server/residency/git_discovery.rs | 18 +- tests/multi_server/size.rs | 17 +- tests/owner_restart.rs | 11 +- tests/repository_cell.rs | 55 +- tests/repository_cell/batches.rs | 45 +- tests/repository_cell/bulk_refs.rs | 2 +- tests/repository_cell/chunks.rs | 187 -- tests/repository_cell/graph.rs | 13 +- tests/repository_cell/packs.rs | 106 ++ tests/repository_cell/pages.rs | 312 +--- tests/smart_http.rs | 59 +- tests/smart_http/cache_admission.rs | 19 +- tests/smart_http/cache_reuse.rs | 4 +- tests/smart_http/publication.rs | 151 +- tests/smart_http/ref_snapshots.rs | 2 +- tests/support/objects.rs | 2 +- tests/support/packs.rs | 14 + tests/support/paused_blobs.rs | 4 +- 98 files changed, 8229 insertions(+), 2261 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 AGENTS.md create mode 100644 docs/benchmarks/2026-10-13-rustfs-browse.json create mode 100644 docs/benchmarks/2026-10-14-storage-optimization.md create mode 100644 scripts/benchmark_browse.py create mode 100644 scripts/check_cellule.py create mode 100644 scripts/smoke_s3_retention.py create mode 100644 scripts/test_benchmark_browse.py create mode 100644 scripts/test_check_cellule.py create mode 100644 src/deployment/maintenance.rs create mode 100644 src/deployment/retention.rs create mode 100644 src/git_pack.rs create mode 100644 src/git_pack/build.rs create mode 100644 src/git_pack/build/tests.rs create mode 100644 src/git_pack/store.rs create mode 100644 src/git_pack/store/tests.rs create mode 100644 src/git_pack/tests.rs create mode 100644 src/graph/stream.rs delete mode 100644 src/large_blob.rs delete mode 100644 src/large_blob/read.rs delete mode 100644 src/large_blob/tests.rs delete mode 100644 src/object_chunks.rs create mode 100644 src/object_packs.rs delete mode 100644 tests/repository_cell/chunks.rs create mode 100644 tests/repository_cell/packs.rs create mode 100644 tests/support/packs.rs diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..33fa04ed --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: cargo + directory: / + schedule: + interval: daily + allow: + - dependency-name: "cellule-*" + groups: + cellule: + patterns: + - "cellule-*" + open-pull-requests-limit: 1 diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index c7b4a3a0..fcbca1a2 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -9,19 +9,24 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Check Cellule release alignment + run: python3 scripts/check_cellule.py - name: Install tools run: | rustup component add clippy rustfmt sudo apt-get update - sudo apt-get install -y git-lfs openssh-client + sudo apt-get install -y git-lfs openssh-client openssl git lfs install - name: Check formatting run: cargo fmt --all -- --check - name: Check lints run: cargo clippy --all-targets --locked -- -D warnings - name: Test - run: cargo test --locked - - name: Qualify Git compatibility against RustFS - run: python3 scripts/qualify_size.py --provider-only + # Bound fixture load; individual tests still run concurrent nodes and workers. + run: cargo test --locked -- --test-threads=1 - name: Build server run: cargo build --locked --bin canopy + - name: Test qualification harnesses + run: python3 -B -m unittest discover -s scripts -p 'test_*.py' + - name: Qualify Git compatibility and process recovery against RustFS + run: python3 scripts/qualify_size.py --provider-only --process-binary target/debug/canopy diff --git a/.gitignore b/.gitignore index 55897546..45a42f3e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target/ +__pycache__/ .env .env.* !.env.example diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..62a2fed3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,19 @@ +# Cellule integration + +- Keep Canopy on the latest published Cellule release. Update all direct + `cellule-*` exact version pins and `Cargo.lock` together, then adapt callers + to the current APIs and run the verification gates. +- Support only the compiled release and its current storage formats. Breaking + Cellule or Canopy schema changes use a fresh storage prefix. Do not add old + readers, retained module code, migration chains, compatibility adapters, or + mixed-version operation to preserve an earlier deployment. +- Keep release, identity, format, and fencing validation. Rejection of older + state is intentional. Same-release crash recovery, retries, backup/restore, + and standard Git/Git LFS protocol support remain required. +- Cellule's single schema-initialization `MigrationDescriptor` is required for + fresh Cells; it is not an upgrade path. Keep command codec identities aligned + with the current handlers; do not reset version numbers to hide a change. +- Run `python3 scripts/check_cellule.py`, formatting, Clippy, ordinary tests, + and the isolated provider/process qualification for dependency changes. Record + any qualification that was not run; retain historical evidence with its + original dependency version. diff --git a/Cargo.lock b/Cargo.lock index 426af0ac..c718b134 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -268,6 +268,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.2" @@ -334,6 +340,17 @@ dependencies = [ "cipher", ] +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -369,6 +386,9 @@ dependencies = [ "ed25519-dalek 2.2.0", "flate2", "futures-core", + "gix-hash", + "gix-pack", + "gix-zlib", "hex", "http-body", "libc", @@ -418,8 +438,9 @@ dependencies = [ [[package]] name = "cellule-app" -version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule.git?rev=a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303#a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b211ee2e9719b8c75906f0218d5621827526a67a959dd99fc54ad0e1a969c52e" dependencies = [ "blake3", "cellule-runtime", @@ -427,9 +448,11 @@ dependencies = [ [[package]] name = "cellule-host" -version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule.git?rev=a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303#a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c372232c1e30332d7468003d9efed396fbb67d4c79f2bef8d1efecb12ae1debf" dependencies = [ + "blake3", "cellule-app", "cellule-runtime", "futures-util", @@ -441,8 +464,9 @@ dependencies = [ [[package]] name = "cellule-ltx" -version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule.git?rev=a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303#a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a5bcca2da813d30d048651212ab76bf678e7ad001a70df3b2997e0661179e7" dependencies = [ "async-trait", "blake3", @@ -463,8 +487,9 @@ dependencies = [ [[package]] name = "cellule-runtime" -version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule.git?rev=a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303#a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3bc9e5375b4668b01454a73e7ca5ec8ac7efd4d8fd23e1b7064418466fba4d" dependencies = [ "blake3", "bytes", @@ -489,8 +514,9 @@ dependencies = [ [[package]] name = "cellule-store" -version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule.git?rev=a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303#a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29273d95baf35553f8512ccc0b72b88ad8f0c447de5c142c1e83d2048688a07f" dependencies = [ "async-trait", "blake3", @@ -512,8 +538,9 @@ dependencies = [ [[package]] name = "cellule-types" -version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule.git?rev=a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303#a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff52f9c6b2e11fb9065d77dced00b9b680c7c35617051b9a6b3b91dca6c8a996" dependencies = [ "schemars", "serde", @@ -804,6 +831,46 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" +[[package]] +name = "defmt" +version = "0.3.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0963443817029b2024136fc4dd07a5107eb8f977eaf18fcd1fdeb11306b64ad" +dependencies = [ + "defmt 1.1.1", +] + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] + [[package]] name = "delegate" version = "0.13.5" @@ -1043,6 +1110,18 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "faster-hex" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04839bdf9d8c10f66806fad16b852fc72aab80873aebc3cb69d85b4fa41543ed" +dependencies = [ + "autocfg", + "defmt 0.3.100", + "heapless", + "serde", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -1279,6 +1358,169 @@ dependencies = [ "zeroize", ] +[[package]] +name = "gix-actor" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2127da41042965ed205568a52443465806944f71c8acd73f48ec94b8cd8e798e" +dependencies = [ + "bstr", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-chunk" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626335faf30ed235744642c2309aa370683b8bb2757f1c22a807e274dff60074" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-date" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f7eeeecc599082f32b5ea6a8c4c734e38be9730e2c09e3fe2851bb4692393e9" +dependencies = [ + "bstr", + "gix-error", + "itoa", + "jiff", +] + +[[package]] +name = "gix-error" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb3308a948a01b94ef13d9ee7a9b28c1303d8d9a9dc33b401ff40c2184d9e04a" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-hash" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ca156064e378d8c657084365846cedb8dc498c6ea4500965a64d74f12af2be" +dependencies = [ + "faster-hex", + "gix-error", + "gix-utils", + "sha1dc", + "sha2 0.11.0", +] + +[[package]] +name = "gix-hashtable" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916cb6b2ec42ffaca624e7867bb382a4673015cb4ffcd7330b48085d02e4c615" +dependencies = [ + "gix-hash", + "hashbrown 0.17.1", + "parking_lot", +] + +[[package]] +name = "gix-object" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0e38f50177e211642af154f9a7f710b7a3a637e635f9b6adb9d51973b0f9209" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-utils", + "gix-validate", + "itoa", + "smallvec", +] + +[[package]] +name = "gix-pack" +version = "0.76.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5024cc503cd29461a705ae90b9eae772fa8f0e6fafc1c48513877dc374de05" +dependencies = [ + "crc32fast", + "gix-chunk", + "gix-error", + "gix-hash", + "gix-object", + "gix-parallel", + "gix-path", + "gix-trace", + "gix-utils", + "gix-zlib", + "memmap2", + "smallvec", +] + +[[package]] +name = "gix-parallel" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994cf1eec247ba4be7f9e7c0d58cc73f83eecb5fcb9feb595df5ba3d2078546" +dependencies = [ + "gix-trace", +] + +[[package]] +name = "gix-path" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6d563d4d0c975ed0df4e853f9ec962a2ba46633b1d324e1a5582541fcae0c6b" +dependencies = [ + "bstr", + "gix-error", + "gix-trace", + "gix-validate", +] + +[[package]] +name = "gix-trace" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133f2de116104d637541831b6acb157898556d5eeec363ab2eb1c583adc6bb56" + +[[package]] +name = "gix-utils" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8e770b8c34311044d1addc7b18957db0ac27183707e1140dc90e6f674a6354e" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "gix-error", + "prodash", + "unicode-normalization", +] + +[[package]] +name = "gix-validate" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbcd59cc52742468bc5c8a0e7123390d154f45e6cbd37461b0dcecdc681f2179" +dependencies = [ + "bstr", + "gix-error", +] + +[[package]] +name = "gix-zlib" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c2edb206bf06f3c801f4f6789c696396171558ab9b4c7f1fe3ba4d39b00b0f4" +dependencies = [ + "gix-error", + "zlib-rs", +] + [[package]] name = "group" version = "0.14.0" @@ -1309,6 +1551,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -1333,6 +1584,16 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.5.0" @@ -1672,6 +1933,60 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt 1.1.1", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt 1.1.1", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.22.4" @@ -1864,6 +2179,15 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "mime" version = "0.3.17" @@ -1934,7 +2258,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -2301,6 +2625,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.6" @@ -2371,6 +2710,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prodash" +version = "31.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf52d87428247c5aa3760ac8e7d8b13ec7b3579b955a8c3f311668b6860acba5" +dependencies = [ + "parking_lot", +] + [[package]] name = "prost" version = "0.13.5" @@ -2659,7 +3007,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.13.2", ] [[package]] @@ -2817,7 +3165,7 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37e34486da88d8e051c7c0e23c3f15fd806ea8546260aa2fec247e97242ec143" dependencies = [ - "bitflags", + "bitflags 2.13.2", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -2832,7 +3180,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "036204edbd199552a5b3832f63c60dcdf395dc44c7f06b4af1c0e8139cc11bce" dependencies = [ "aes", - "bitflags", + "bitflags 2.13.2", "block-padding", "byteorder", "bytes", @@ -2949,7 +3297,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ - "bitflags", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys", @@ -3135,7 +3483,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.13.2", "core-foundation", "core-foundation-sys", "libc", @@ -3256,6 +3604,12 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sha1dc" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5b0ebf6537e3e8db8bb7d21ecb6d7399738c953c757945c6b95e46c4e95df7b" + [[package]] name = "sha2" version = "0.10.9" @@ -3711,7 +4065,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.13.2", "bytes", "futures-util", "http", @@ -3834,6 +4188,15 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "universal-hash" version = "0.6.1" diff --git a/Cargo.toml b/Cargo.toml index b9a0f77c..b53da44d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ name = "canopy-server" version = "0.1.0" edition = "2024" +rust-version = "1.97" license = "Apache-2.0" [[bin]] @@ -13,14 +14,17 @@ axum = "0.8.9" base64 = "0.22" blake3 = "1.8" bytes = "1.11" -cellule-app = { git = "https://github.com/crabbuild/cellule.git", rev = "a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303" } -cellule-host = { git = "https://github.com/crabbuild/cellule.git", rev = "a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303" } -cellule-ltx = { git = "https://github.com/crabbuild/cellule.git", rev = "a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303", features = ["replica"] } -cellule-runtime = { git = "https://github.com/crabbuild/cellule.git", rev = "a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303" } -cellule-store = { git = "https://github.com/crabbuild/cellule.git", rev = "a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303" } +cellule-app = { version = "=0.2.0" } +cellule-host = { version = "=0.2.0" } +cellule-ltx = { version = "=0.2.0", features = ["replica"] } +cellule-runtime = { version = "=0.2.0" } +cellule-store = { version = "=0.2.0" } ed25519-dalek = "2" flate2 = "1.1" futures-core = "0.3" +gix-hash = { version = "0.28.0", features = ["sha1", "sha256"] } +gix-pack = { version = "0.76.0", default-features = false, features = ["sha1", "sha256"] } +gix-zlib = "0.3.0" hex = "0.4" http-body = "1" object_store = "0.14.1" diff --git a/README.md b/README.md index a6fe357c..a19cb598 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ This page keeps the project overview, setup path, and first repository workflow. | Browser | Embedded repository, issue, pull-request and account views | [Repository browser](docs/user-guide.md#repository-browser) | These are implementation capabilities, not a production capacity guarantee. -The [roadmap](ROADMAP.md) lists remaining work such as migrations, full fault +The [roadmap](ROADMAP.md) lists remaining work such as full fault qualification, collection, provider qualification, observability and a repeatable deployment. The [delivery plan](docs/delivery-plan.md) records release gates; the [performance plan](docs/performance-plan.md) separates measured results from @@ -46,8 +46,8 @@ before relying on a feature in a new environment. The **Directory Cell** maps an owner and repository name to a stable repository UUID and owns accounts. Each UUID identifies a **Repository Cell** containing -authoritative refs, Git metadata, ACLs and collaboration records. Large Git -blobs and LFS bodies are immutable external objects referenced by that Cell. +authoritative refs, Git metadata, ACLs and collaboration records. Git +object bodies live in immutable native Git pack/index pairs referenced by that Cell. Git LFS retains its separate immutable body store. Native Git handles wire protocols using a **disposable cache**: after local disk loss, Canopy restores published Cell state from the object store and rebuilds the cache. See the [persisted contracts](docs/contracts.md) for publication and @@ -67,7 +67,7 @@ The existing [architecture SVG](docs/architecture.svg) provides the same model w ### A durable push in one view -The gateway reports a successful push after it has authenticated the request, verified incoming objects, published required external bodies, and committed the ref update with a replayable response. +The gateway reports a successful push after it has authenticated the request, verified incoming objects, published and verified native packs and required LFS bodies, and committed the ref update with a replayable response. ```mermaid sequenceDiagram @@ -89,7 +89,11 @@ sequenceDiagram This is a development path for one node with an existing S3-compatible object store. The [bounded Linux deployment](deploy/README.md) describes the container profile and its resource limits. Use a **new storage prefix** for this build; -there is no migration from earlier preview layouts yet. +Canopy follows the latest Cellule release without backward compatibility. +Cellule 0.2.0 changes the persisted bundle format, and there is no migration +from earlier preview layouts. Before retiring an older deployment, drain and +materialize its acknowledged history with the previous binary. See the +[storage compatibility notes](docs/implementation.md#storage-compatibility-and-graph-verification). 1. Install Rust 1.97 or newer and Git with `http-backend`, `merge-tree --write-tree` (`-z --name-only --no-messages`) and `commit-tree`. @@ -197,7 +201,7 @@ The long-form material moved from this README is preserved in those pages. The [ Keep behavior, tests, and documentation aligned when you change Canopy: -1. Update the relevant [persisted contract](docs/contracts.md). +1. Follow the [Cellule integration policy](AGENTS.md) and update the relevant [persisted contract](docs/contracts.md). 2. Add or update a stock-client, API, or fault test. 3. Record the revision, provider, workload, and hardware for measurements. 4. Update the matching compatibility row or [delivery gate](docs/delivery-plan.md). diff --git a/ROADMAP.md b/ROADMAP.md index 37b0b2c8..74f524cf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -13,14 +13,13 @@ Canopy's Git-hosting core works, but the service is not yet ready for an unatten This is the first intended production-use milestone. The existing bounded Compose profile is a starting point, not a completed install or capacity claim. -### R01. Safe upgrades and schema migration +### R01. Current Cellule release and fresh-prefix deployment -- [ ] Version persistent Canopy and runtime formats; add a fenced, repeatable upgrade command and migration path for a populated deployment. -- [ ] Require a verified backup and compatible release/configuration before migration; document failed-upgrade recovery and rollback limits. -- [ ] Upgrade a prior release containing Git objects, LFS, ACLs, issues, checks, and pull requests; interrupt and retry each migration phase, then clone and inspect the restored state. -- [ ] Define whether mixed-version nodes are supported. Until proven, enforce a fully drained, single-version upgrade. +- [ ] Keep all Cellule crates pinned to the latest published release together and qualify its current APIs, storage format, and recovery behavior. +- [ ] Verify that mismatched releases and unrecognized storage formats are rejected without adopting or rewriting them. +- [ ] Drill fresh-prefix deployment and same-release backup/restore with the intended production provider. -**Current state:** schema changes still require a fresh storage prefix; maintenance admission and drain exist, but there is no upgrade path. +**Policy:** incompatible runtime or Canopy schema changes require a fresh storage prefix. Old-format readers, retained code, in-place migrations, and mixed-version operation are out of scope. Same-release crash recovery and backup/restore remain required. See the [Cellule integration policy](AGENTS.md). ### R02. Backup, restore, and owner-recovery fault qualification @@ -37,7 +36,10 @@ This is the first intended production-use milestone. The existing bounded Compos - [ ] Add a fenced collector for abandoned uploads, old outcomes/candidates, and unreachable Git/LFS objects, with an inspectable dry run and bounded work. - [ ] Prove with concurrent reads, backup, publication, owner loss, and injected collector failures that required bytes are never removed; collect abandoned bytes only after their grace period. -**Current state:** no production collector; abandoned staging and outcomes can accumulate. Collection must remain conservative until its root proof passes. +**Current state:** the maintenance CLI runs Cellule's retention-aware collector for +obsolete immutable database objects, with a grace window, bounded deletes and a +persistent worker claim. External Git/LFS orphan collection, staging/outcome +retention and the full injected-failure matrix remain open. ### R04. Trustworthy CI and release artifacts @@ -46,7 +48,7 @@ This is the first intended production-use milestone. The existing bounded Compos - [ ] Add required failure-path and provider gates to branch protection, then publish versioned, digest-identified images and a repeatable release checklist. - [ ] Record toolchain, Git, Git LFS, Cellule revision, image digest, provider, and test outcomes for each release candidate. -**Current state:** `.github/workflows/verify.yml` invokes the full size gate on `ubuntu-latest`; a previous hosted run exhausted disk. +**Current state:** `.github/workflows/verify.yml` runs ordinary tests, eight smaller provider probes, and full process recovery. The separate >5 GiB gate still needs a runner with sufficient space. ### R05. Production object-store qualification @@ -60,13 +62,13 @@ This is the first intended production-use milestone. The existing bounded Compos - [ ] Expose low-cardinality request/transfer latency and error metrics, active and pending Cells, lease and renewal health, queues, object-store calls, local disk, and admission failures. - [ ] Set alert thresholds from measured limits and test that alerts fire during injected faults. -- [ ] Write and drill runbooks for 503/507, exhausted resources, lost ownership/fencing, failed upgrades, backup verification, and restore. +- [ ] Write and drill runbooks for 503/507, exhausted resources, lost ownership/fencing, rejected release admission, backup verification, and restore. **Current state:** health endpoints and tracing logs exist; metrics, alerting, and a complete operator contract do not. ### R07. Repeatable single-node deployment -- [ ] Turn [`deploy/README.md`](deploy/README.md) and the bounded Compose profile into a documented install with TLS ingress, SSH host-key handling, secret rotation, bootstrap, health checks, backup schedule, upgrade, and restore drill. +- [ ] Turn [`deploy/README.md`](deploy/README.md) and the bounded Compose profile into a documented install with TLS ingress, SSH host-key handling, secret rotation, bootstrap, health checks, backup schedule, fresh-prefix cutover, and restore drill. - [ ] Verify the actual cgroup, descriptor, scratch, and Git-worker limits in the install; publish a measured resource envelope and operator checklist. - [ ] Add a two-node recipe after R02 and R05 establish failover behavior on the selected provider. @@ -99,7 +101,7 @@ This is the first intended production-use milestone. The existing bounded Compos ### Private team release exit checks - [ ] R01–R10 acceptance proof is recorded for the exact release candidate and selected object store. -- [ ] A clean install, upgrade of a populated prior release, backup, source-prefix loss, and restore all pass the operator runbook. +- [ ] A fresh-prefix install, rejection of incompatible existing state, same-release backup, source-prefix loss, and restore all pass the operator runbook. - [ ] Stock Git and Git LFS work after recovery; permissions, issues, reviews, checks, and pull requests retain their state. - [ ] The release notes state measured limits for active repositories, concurrent transfers, local disk, memory, restore time, and object-store cost. Unmeasured capacity is not advertised. diff --git a/docs/api-reference.md b/docs/api-reference.md index 3b44c923..f781323e 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -17,6 +17,7 @@ for exact limits and failure semantics. | Surface | Reference section | | --- | --- | | Repository lifecycle and default branch | [Repository creation, discovery, and rename](#repository-creation-discovery-and-rename), [Default branch](#default-branch) | +| Refs, trees, blobs, and commit history | [Repository browsing](#repository-browsing) | | Accounts, grants, and tokens | [Accounts and repository access](#accounts-and-repository-access), [Token lifecycle](#token-lifecycle) | | Issues, pull requests, reviews, and line discussions | [Issues and comments](#issues-and-comments), [Pull requests and reviews](#pull-requests-and-reviews), [Compare revisions and discuss lines](#compare-revisions-and-discuss-lines) | | Merge, checks, and branch rules | [Merge and rebase](#merge-and-rebase), [Checks and branch rules](#checks-and-branch-rules) | @@ -60,6 +61,38 @@ there are no live branches. Stock Git discovery and clone use this durable HEAD, including after restart. Protocol v2 also reports an unborn target. Branch deletion preserves HEAD's name; recreating the branch makes it live again. +### Repository browsing + +`POST /api/repositories//browse` accepts a repository identity and one query: + +```json +{"repository_id":"","query":{"kind":"tree","commit":"","path_base64":""}} +``` + +The response contains `repository_id` and `view`, with a member named `refs`, +`resolved`, `tree`, `file`, or `history`. Authenticated readers need read access; +public repositories also permit anonymous reads. Paths and byte-valued metadata +use URL-safe Base64 without padding. Pass an empty path for the root tree. + +| Query kind | Fields | Result and bounds | +| --- | --- | --- | +| `refs` | Optional `after`, `generation` | Up to 256 refs within the response byte limit, default branch, generation, and `next_after`. Carry the generation into subsequent pages. | +| `resolve` | Optional fully qualified `reference` | Resolve a ref or the default branch to an immutable OID before browsing. | +| `tree` | `commit`, `path_base64`; optional `after` | Up to 32 directory entries, the selected commit, and `next_after`. | +| `file` | `commit`, `path_base64` | Blob identity, mode, size, and exact `content_base64` for files up to 256 KiB. Larger files return `too_large` metadata. | +| `history` | `commit` | Up to 32 commits following first parents, with `next_commit` for continuation. | + +Commit records include OID, tree, parents, raw `author_base64` and +`committer_base64` Git headers, and up to 4 KiB of message bytes. This is commit +authorship metadata. There is no line attribution/blame query or separate +attribution API; an unknown browse kind returns HTTP 422. + +`scripts/benchmark_browse.py` imports local bare repositories, verifies these +views against stock Git, and measures authenticated reads with persistent +connections. It requires `CANOPY_GIT_TOKEN`; use its `--help` for corpus and +output arguments. Reports distinguish successful throughput, failed attempts, +and successful versus all-attempt latency. + ### Accounts and repository access The configured token bootstraps a durable owner account in the Directory Cell. @@ -256,7 +289,7 @@ The thread author, pull author or repository writer may do so with a write-scope token. Stale versions return 409. Resolution is informational and does not replace required approvals or checks. See [line discussion contracts](contracts.md#line-discussions). This unreleased schema adds discussion tables and requires a fresh development -storage prefix; there is no upgrade migration yet. +storage prefix, as required by the current-release-only storage policy. ### Merge and rebase diff --git a/docs/benchmarks/2026-10-13-rustfs-browse.json b/docs/benchmarks/2026-10-13-rustfs-browse.json new file mode 100644 index 00000000..613d6f5e --- /dev/null +++ b/docs/benchmarks/2026-10-13-rustfs-browse.json @@ -0,0 +1,1622 @@ +{ + "date": "2026-10-13", + "timezone": "America/Vancouver", + "environment": { + "cpu": "Apple M2 Max", + "logical_cpus": 12, + "memory_bytes": 34359738368, + "os": "macOS Darwin 25.5.0 arm64", + "dedicated_host": false, + "rustc": "rustc 1.97.0 (2d8144b78 2026-07-07)", + "git": "git version 2.50.1 (Apple Git-155)", + "cellule_version": "0.2.0", + "head": "9c5f1d1bf837fdc2e38ee229aa409712b9579f69", + "dirty_worktree": true, + "build_profile": "release", + "binary_sha256": "7292067d1a34a796066f32f07e2cba84c69ac619cae64d16c8f7e670ac0095c1", + "cargo_lock_sha256": "1e400d63d86f6f8d364de72e669be1d4db7c183a4e079d419a859bb46e385366", + "transport": "HTTP/1.1 loopback, authenticated single account, no reverse proxy/TLS", + "provider": { + "container": "canopy-api-bench-20261013", + "volume": "canopy-api-bench-20261013-data", + "image": "rustfs/rustfs:1.0.0-beta.8-glibc", + "endpoint": "http://127.0.0.1:32793", + "bucket": "canopy-benchmark", + "cpu_limit": 4, + "memory_limit_gib": 8, + "image_id": "sha256:040304b66e029a5cde4bed140b41513e925909839a9b912a40a98340610d1f66" + }, + "admission": { + "node_active_transfers": 8, + "account_active_transfers": 4, + "account_waiters": 4, + "maximum_wait_seconds": 1 + }, + "max_active_repositories": 8 + }, + "corpus": { + "repositories": [ + { + "name": "bat", + "source_url": "https://github.com/sharkdp/bat.git", + "branch": "master", + "provider_pull_refs_excluded": 2577, + "source_stats": "count: 0\nsize: 0\nin-pack: 20952\npacks: 1\nsize-pack: 35959\nprune-packable: 0\ngarbage: 0\nsize-garbage: 0\n", + "commit_count": 4068, + "object_count": 20952, + "commit": "d9559c69f541873f8c47ac8ad81d985f4441edf6", + "pack_bytes": 36234681, + "object_format": "sha1", + "owner": "canopy", + "repository_id": "c26304a7-f3e4-433a-9332-77ffcbb26cbd", + "clone_url": "http://127.0.0.1:60523/canopy/bat.git", + "import": { + "seconds": 28.921057333005592, + "exit_code": 0, + "objects_per_second": 724.454841285797, + "source_pack_mib_per_second": 1.1948416766020888 + } + }, + { + "name": "starship", + "source_url": "https://github.com/starship/starship.git", + "branch": "main", + "provider_pull_refs_excluded": 4513, + "source_stats": "count: 0\nsize: 0\nin-pack: 38333\npacks: 1\nsize-pack: 48427\nprune-packable: 0\ngarbage: 0\nsize-garbage: 0\n", + "commit_count": 4556, + "object_count": 38333, + "commit": "df688000684138943610bbaa2a6c454db6987d48", + "pack_bytes": 48515539, + "object_format": "sha1", + "owner": "canopy", + "repository_id": "f82d4523-84d1-4ab4-8099-743457d303ae", + "clone_url": "http://127.0.0.1:60523/canopy/starship.git", + "import": { + "seconds": 79.99967504100641, + "exit_code": 0, + "objects_per_second": 479.16444635995316, + "source_pack_mib_per_second": 0.5783526428974817 + } + }, + { + "name": "libgit2", + "source_url": "https://github.com/libgit2/libgit2.git", + "branch": "main", + "provider_pull_refs_excluded": 6804, + "source_stats": "count: 0\nsize: 0\nin-pack: 134967\npacks: 1\nsize-pack: 77541\nprune-packable: 0\ngarbage: 0\nsize-garbage: 0\n", + "commit_count": 19565, + "object_count": 134967, + "commit": "4af47f040cb4d80c75f37f626f511843e0dd8d44", + "pack_bytes": 75622259, + "object_format": "sha1", + "owner": "canopy", + "repository_id": "beb16211-3d72-4397-94e6-09639ba05503", + "clone_url": "http://127.0.0.1:60523/canopy/libgit2.git", + "import": { + "seconds": 813.6219757079962, + "exit_code": 0, + "objects_per_second": 165.8841624607726, + "source_pack_mib_per_second": 0.08863945211677081 + } + } + ] + }, + "verification": { + "repositories": [ + { + "repository": "bat", + "refs_verified": 66, + "clone_seconds": 19.773580291017424, + "clone_fsck_passed": true, + "clone_objects_verified": 20952, + "commit_metadata_and_attribution_verified": 128, + "files": [ + { + "path": "tests/snapshots/output/grid.snapshot.txt", + "bytes": 1025, + "oid": "7061949760a95e1bdacff061c222e7600523c038" + }, + { + "path": "tests/examples/longline.json", + "bytes": 16450, + "oid": "0e11bf142d38256d157782dc928f31dbdc2d6c64" + }, + { + "path": "tests/integration_tests.rs", + "bytes": 153394, + "oid": "9463be375f3c0d2f2e9aebbb5adea07b0d350924" + } + ], + "attribution_query_status": 422 + }, + { + "repository": "starship", + "refs_verified": 164, + "clone_seconds": 48.21346141700633, + "clone_fsck_passed": true, + "clone_objects_verified": 38333, + "commit_metadata_and_attribution_verified": 128, + "files": [ + { + "path": "install/macos_packages/pkg_resources/English.lproj/license.html", + "bytes": 1024, + "oid": "f446f65e4956895ede297314924caf390d3f4f17" + }, + { + "path": ".github/workflows/release.yml", + "bytes": 16378, + "oid": "1b39bb0accbd8be9259e952df20dd059c21b2282" + }, + { + "path": "media/icon.png", + "bytes": 129048, + "oid": "aaf79dc967943414d67f9ff00f7b06775a86f320" + } + ], + "attribution_query_status": 422 + }, + { + "repository": "libgit2", + "refs_verified": 271, + "clone_seconds": 214.87667675002012, + "clone_fsck_passed": true, + "clone_objects_verified": 134967, + "commit_metadata_and_attribution_verified": 128, + "files": [ + { + "path": "tests/libgit2/object/raw/chars.c", + "bytes": 1025, + "oid": "24d630d38614cfa2c97937f7bf896aa478925b40" + }, + { + "path": "deps/pcre2/pcre2_compile.h", + "bytes": 16360, + "oid": "331ca3ad3965ea7d1c67222c736eaabe8830ccc0" + }, + { + "path": "tests/resources/gitgit.index", + "bytes": 134799, + "oid": "215da649e1c68079fb03f4f9bc0f196cca9855c8" + } + ], + "attribution_query_status": 422 + } + ], + "attribution_api": "not implemented; browse kind rejected with 422" + }, + "large_file_preview_limits": [ + { + "repository": "bat", + "path": "assets/syntaxes.bin", + "bytes": 1012454, + "content_status": "too_large", + "status": 200 + }, + { + "repository": "starship", + "path": "media/demo.gif", + "bytes": 2469588, + "content_status": "too_large", + "status": 200 + }, + { + "repository": "libgit2", + "path": "benchmarks/cli/_script/flamegraph/example-dtrace-stacks.txt", + "bytes": 1382457, + "content_status": "too_large", + "status": 200 + } + ], + "warm": { + "method": "closed-loop, persistent HTTP/1.1 connections, full response read, no retries", + "mixed_workload": "equal rotation over one query for each repository/API pair; medium sampled blob", + "duration_per_repeat_seconds": 8, + "repeats": 2, + "rows": [ + { + "repository": "all", + "operation": "mixed", + "concurrency": 1, + "seconds": 16.00467266701162, + "requests": 8787, + "outcomes": { + "ok": 8787 + }, + "latency_ms_all_attempts": { + "p50": 1.0325420298613608, + "p95": 4.82308299979195, + "p99": 6.397457967977971, + "min": 0.5297499592415988, + "max": 92.74037496652454, + "mean": 1.8120717856468307 + }, + "latency_ms_successful": { + "p50": 1.0325420298613608, + "p95": 4.82308299979195, + "p99": 6.397457967977971, + "min": 0.5297499592415988, + "max": 92.74037496652454, + "mean": 1.8120717856468307 + }, + "successful_requests_per_second": 549.0271611809667, + "successful_response_mib_per_second": 8.169318987425775, + "error_percent": 0.0 + }, + { + "repository": "all", + "operation": "mixed", + "concurrency": 4, + "seconds": 16.015075459028594, + "requests": 14714, + "outcomes": { + "ok": 14714 + }, + "latency_ms_all_attempts": { + "p50": 2.1343749831430614, + "p95": 11.538084014318883, + "p99": 15.04329196177423, + "min": 0.8282500202767551, + "max": 88.08912499807775, + "mean": 4.336179986815298 + }, + "latency_ms_successful": { + "p50": 2.1343749831430614, + "p95": 11.538084014318883, + "p99": 15.04329196177423, + "min": 0.8282500202767551, + "max": 88.08912499807775, + "mean": 4.336179986815298 + }, + "successful_requests_per_second": 918.7593300851352, + "successful_response_mib_per_second": 13.670334790148779, + "error_percent": 0.0 + }, + { + "repository": "all", + "operation": "mixed", + "concurrency": 16, + "seconds": 16.030485082999803, + "requests": 145990, + "outcomes": { + "http_503_or_invalid_body": 122815, + "ok": 10565, + "BrokenPipeError": 12458, + "ConnectionResetError": 66, + "RemoteDisconnected": 86 + }, + "latency_ms_all_attempts": { + "p50": 0.8549580234102905, + "p95": 8.482416975311935, + "p99": 21.860542008653283, + "min": 0.02929195761680603, + "max": 47.4909589975141, + "mean": 1.7490142179625363 + }, + "latency_ms_successful": { + "p50": 10.354708007071167, + "p95": 27.166916988790035, + "p99": 32.0184999727644, + "min": 2.8825420304201543, + "max": 47.4909589975141, + "mean": 13.002402145956355 + }, + "successful_requests_per_second": 659.0567874458206, + "successful_response_mib_per_second": 9.770577278489611, + "error_percent": 92.76320295910679 + }, + { + "repository": "bat", + "operation": "blob", + "concurrency": 1, + "seconds": 16.00199016695842, + "requests": 13848, + "outcomes": { + "ok": 13848 + }, + "latency_ms_all_attempts": { + "p50": 1.101000001654029, + "p95": 1.4443330001085997, + "p99": 1.646874996367842, + "min": 0.7540420047007501, + "max": 6.462041987106204, + "mean": 1.12150191992839 + }, + "latency_ms_successful": { + "p50": 1.101000001654029, + "p95": 1.4443330001085997, + "p99": 1.646874996367842, + "min": 0.7540420047007501, + "max": 6.462041987106204, + "mean": 1.12150191992839 + }, + "successful_requests_per_second": 865.3923577952154, + "successful_response_mib_per_second": 62.95434571926579, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "blob", + "concurrency": 4, + "seconds": 16.004783916985616, + "requests": 28449, + "outcomes": { + "ok": 28449 + }, + "latency_ms_all_attempts": { + "p50": 2.164374978747219, + "p95": 2.7350829914212227, + "p99": 3.00733296899125, + "min": 1.1347500258125365, + "max": 55.061583989299834, + "mean": 2.2109012953796157 + }, + "latency_ms_successful": { + "p50": 2.164374978747219, + "p95": 2.7350829914212227, + "p99": 3.00733296899125, + "min": 1.1347500258125365, + "max": 55.061583989299834, + "mean": 2.2109012953796157 + }, + "successful_requests_per_second": 1777.531027445334, + "successful_response_mib_per_second": 129.33296811357653, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "blob", + "concurrency": 16, + "seconds": 16.01165054203011, + "requests": 146284, + "outcomes": { + "ok": 19061, + "http_503_or_invalid_body": 87036, + "BrokenPipeError": 39775, + "RemoteDisconnected": 217, + "ConnectionResetError": 195 + }, + "latency_ms_all_attempts": { + "p50": 0.7972500170581043, + "p95": 7.779459003359079, + "p99": 9.129625046625733, + "min": 0.026124995201826096, + "max": 26.476458995603025, + "mean": 1.7295034789342059 + }, + "latency_ms_successful": { + "p50": 7.500582956708968, + "p95": 9.475333034060895, + "p99": 11.623708007391542, + "min": 3.350542043335736, + "max": 26.476458995603025, + "mean": 7.679052330335933 + }, + "successful_requests_per_second": 1190.4456664205502, + "successful_response_mib_per_second": 87.55830221538214, + "error_percent": 86.96986683437696 + }, + { + "repository": "bat", + "operation": "commits", + "concurrency": 1, + "seconds": 16.017477457993664, + "requests": 3501, + "outcomes": { + "ok": 3501 + }, + "latency_ms_all_attempts": { + "p50": 4.089916998054832, + "p95": 5.8344590361230075, + "p99": 9.358957991935313, + "min": 2.862958994228393, + "max": 28.306666994467378, + "mean": 4.558941061042278 + }, + "latency_ms_successful": { + "p50": 4.089916998054832, + "p95": 5.8344590361230075, + "p99": 9.358957991935313, + "min": 2.862958994228393, + "max": 28.306666994467378, + "mean": 4.558941061042278 + }, + "successful_requests_per_second": 218.57374291185874, + "successful_response_mib_per_second": 4.370237237001032, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "commits", + "concurrency": 4, + "seconds": 16.006958667945582, + "requests": 5430, + "outcomes": { + "ok": 5430 + }, + "latency_ms_all_attempts": { + "p50": 10.898125008679926, + "p95": 15.361499972641468, + "p99": 15.846750000491738, + "min": 8.433957991655916, + "max": 21.503625030163676, + "mean": 11.774078276888236 + }, + "latency_ms_successful": { + "p50": 10.898125008679926, + "p95": 15.361499972641468, + "p99": 15.846750000491738, + "min": 8.433957991655916, + "max": 21.503625030163676, + "mean": 11.774078276888236 + }, + "successful_requests_per_second": 339.2274642948719, + "successful_response_mib_per_second": 6.7831833481177455, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "commits", + "concurrency": 16, + "seconds": 16.05331412499072, + "requests": 151688, + "outcomes": { + "ok": 2867, + "http_503_or_invalid_body": 108631, + "BrokenPipeError": 39776, + "ConnectionResetError": 189, + "RemoteDisconnected": 225 + }, + "latency_ms_all_attempts": { + "p50": 0.7049579871818423, + "p95": 2.1658329642377794, + "p99": 42.14174998924136, + "min": 0.022416003048419952, + "max": 128.7855840055272, + "mean": 1.6847534488501597 + }, + "latency_ms_successful": { + "p50": 42.33108297921717, + "p95": 67.43995798751712, + "p99": 99.09195901127532, + "min": 20.263166981749237, + "max": 128.7855840055272, + "mean": 45.66223086149068 + }, + "successful_requests_per_second": 178.59240638273235, + "successful_response_mib_per_second": 3.5661834712808487, + "error_percent": 98.10993618480038 + }, + { + "repository": "bat", + "operation": "refs", + "concurrency": 1, + "seconds": 16.014794999966398, + "requests": 18622, + "outcomes": { + "ok": 18622 + }, + "latency_ms_all_attempts": { + "p50": 0.8425840060226619, + "p95": 0.9940830059349537, + "p99": 1.0678750113584101, + "min": 0.6624590023420751, + "max": 5.0185840227641165, + "mean": 0.8540398355497097 + }, + "latency_ms_successful": { + "p50": 0.8425840060226619, + "p95": 0.9940830059349537, + "p99": 1.0678750113584101, + "min": 0.6624590023420751, + "max": 5.0185840227641165, + "mean": 0.8540398355497097 + }, + "successful_requests_per_second": 1162.7997735868034, + "successful_response_mib_per_second": 7.57622533144478, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "refs", + "concurrency": 4, + "seconds": 16.01462700002594, + "requests": 35073, + "outcomes": { + "ok": 35073 + }, + "latency_ms_all_attempts": { + "p50": 1.7980000120587647, + "p95": 2.1177500020712614, + "p99": 2.2350000217556953, + "min": 1.0270840139128268, + "max": 10.337167012039572, + "mean": 1.8192831381431933 + }, + "latency_ms_successful": { + "p50": 1.7980000120587647, + "p95": 2.1177500020712614, + "p99": 2.2350000217556953, + "min": 1.0270840139128268, + "max": 10.337167012039572, + "mean": 1.8192831381431933 + }, + "successful_requests_per_second": 2190.0603741781306, + "successful_response_mib_per_second": 14.269344784150112, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "refs", + "concurrency": 16, + "seconds": 16.02138833294157, + "requests": 158165, + "outcomes": { + "ok": 20054, + "http_503_or_invalid_body": 124862, + "BrokenPipeError": 13093, + "RemoteDisconnected": 105, + "ConnectionResetError": 51 + }, + "latency_ms_all_attempts": { + "p50": 0.799959001597017, + "p95": 7.2485419805161655, + "p99": 7.970708014909178, + "min": 0.029250048100948334, + "max": 17.770208010915667, + "mean": 1.6139088356820033 + }, + "latency_ms_successful": { + "p50": 7.107457960955799, + "p95": 8.147875021677464, + "p99": 8.82079201983288, + "min": 2.2863749763928354, + "max": 17.770208010915667, + "mean": 7.126413922704734 + }, + "successful_requests_per_second": 1251.7017616236778, + "successful_response_mib_per_second": 8.155466494954076, + "error_percent": 87.32083583599406 + }, + { + "repository": "bat", + "operation": "tree", + "concurrency": 1, + "seconds": 16.015277875005268, + "requests": 18057, + "outcomes": { + "ok": 18057 + }, + "latency_ms_all_attempts": { + "p50": 0.8279170142486691, + "p95": 1.1947079910896719, + "p99": 1.6283750301226974, + "min": 0.5131660145707428, + "max": 34.24637496937066, + "mean": 0.8809236574188942 + }, + "latency_ms_successful": { + "p50": 0.8279170142486691, + "p95": 1.1947079910896719, + "p99": 1.6283750301226974, + "min": 0.5131660145707428, + "max": 34.24637496937066, + "mean": 0.8809236574188942 + }, + "successful_requests_per_second": 1127.4859007086732, + "successful_response_mib_per_second": 6.566964872894387, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "tree", + "concurrency": 4, + "seconds": 16.027722750033718, + "requests": 37382, + "outcomes": { + "ok": 37382 + }, + "latency_ms_all_attempts": { + "p50": 1.640458998735994, + "p95": 2.4475829559378326, + "p99": 3.062208998017013, + "min": 0.7026249659247696, + "max": 19.468375016003847, + "mean": 1.7061047552683481 + }, + "latency_ms_successful": { + "p50": 1.640458998735994, + "p95": 2.4475829559378326, + "p99": 3.062208998017013, + "min": 0.7026249659247696, + "max": 19.468375016003847, + "mean": 1.7061047552683481 + }, + "successful_requests_per_second": 2332.3338307634103, + "successful_response_mib_per_second": 13.585427441303326, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "tree", + "concurrency": 16, + "seconds": 16.00996450096136, + "requests": 150745, + "outcomes": { + "ok": 27129, + "http_503_or_invalid_body": 82495, + "BrokenPipeError": 40716, + "ConnectionResetError": 201, + "RemoteDisconnected": 204 + }, + "latency_ms_all_attempts": { + "p50": 0.8531250059604645, + "p95": 6.033874989952892, + "p99": 7.056166010443121, + "min": 0.028458016458898783, + "max": 12.190042005386204, + "mean": 1.6899882608471022 + }, + "latency_ms_successful": { + "p50": 5.48045797040686, + "p95": 7.101957977283746, + "p99": 7.9649160034023225, + "min": 1.79462501546368, + "max": 12.190042005386204, + "mean": 5.547666112374419 + }, + "successful_requests_per_second": 1694.50719259065, + "successful_response_mib_per_second": 9.90357989569809, + "error_percent": 82.00338319678929 + }, + { + "repository": "libgit2", + "operation": "blob", + "concurrency": 1, + "seconds": 16.00327133195242, + "requests": 15154, + "outcomes": { + "ok": 15154 + }, + "latency_ms_all_attempts": { + "p50": 1.0045829694718122, + "p95": 1.2737499782815576, + "p99": 1.4454579795710742, + "min": 0.7080409559421241, + "max": 4.052291973493993, + "mean": 1.0254417736195982 + }, + "latency_ms_successful": { + "p50": 1.0045829694718122, + "p95": 1.2737499782815576, + "p99": 1.4454579795710742, + "min": 0.7080409559421241, + "max": 4.052291973493993, + "mean": 1.0254417736195982 + }, + "successful_requests_per_second": 946.9313920675238, + "successful_response_mib_per_second": 61.389964203748015, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "blob", + "concurrency": 4, + "seconds": 16.005381707975175, + "requests": 27930, + "outcomes": { + "ok": 27930 + }, + "latency_ms_all_attempts": { + "p50": 2.2345830220729113, + "p95": 2.8366249753162265, + "p99": 3.283208003267646, + "min": 1.2243750388734043, + "max": 20.5275000189431, + "mean": 2.257089662635542 + }, + "latency_ms_successful": { + "p50": 2.2345830220729113, + "p95": 2.8366249753162265, + "p99": 3.283208003267646, + "min": 1.2243750388734043, + "max": 20.5275000189431, + "mean": 2.257089662635542 + }, + "successful_requests_per_second": 1745.0380446773734, + "successful_response_mib_per_second": 113.14824424794928, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "blob", + "concurrency": 16, + "seconds": 16.012136042059865, + "requests": 152697, + "outcomes": { + "ok": 15933, + "http_503_or_invalid_body": 93457, + "BrokenPipeError": 42830, + "RemoteDisconnected": 253, + "ConnectionResetError": 224 + }, + "latency_ms_all_attempts": { + "p50": 0.7473749574273825, + "p95": 8.710167021490633, + "p99": 10.828375001437962, + "min": 0.026999972760677338, + "max": 41.841500031296164, + "mean": 1.6610423994319796 + }, + "latency_ms_successful": { + "p50": 8.64333298522979, + "p95": 11.448832985479385, + "p99": 13.542667031288147, + "min": 3.043250006157905, + "max": 41.841500031296164, + "mean": 8.961353011451289 + }, + "successful_requests_per_second": 995.0577460838458, + "successful_response_mib_per_second": 64.75569542375841, + "error_percent": 89.56561032633255 + }, + { + "repository": "libgit2", + "operation": "commits", + "concurrency": 1, + "seconds": 16.02726750000147, + "requests": 3940, + "outcomes": { + "ok": 3940 + }, + "latency_ms_all_attempts": { + "p50": 3.8279169821180403, + "p95": 5.187375005334616, + "p99": 6.551624974235892, + "min": 2.923749969340861, + "max": 50.19733402878046, + "mean": 4.05026076800015 + }, + "latency_ms_successful": { + "p50": 3.8279169821180403, + "p95": 5.187375005334616, + "p99": 6.551624974235892, + "min": 2.923749969340861, + "max": 50.19733402878046, + "mean": 4.05026076800015 + }, + "successful_requests_per_second": 245.83105011503915, + "successful_response_mib_per_second": 4.322186698838102, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "commits", + "concurrency": 4, + "seconds": 16.184691207949072, + "requests": 4750, + "outcomes": { + "ok": 4750 + }, + "latency_ms_all_attempts": { + "p50": 11.091416992712766, + "p95": 20.288583997171372, + "p99": 55.95716700190678, + "min": 8.686790999490768, + "max": 278.26662501320243, + "mean": 13.5943113943856 + }, + "latency_ms_successful": { + "p50": 11.091416992712766, + "p95": 20.288583997171372, + "p99": 55.95716700190678, + "min": 8.686790999490768, + "max": 278.26662501320243, + "mean": 13.5943113943856 + }, + "successful_requests_per_second": 293.48721819709783, + "successful_response_mib_per_second": 5.160118765610484, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "commits", + "concurrency": 16, + "seconds": 16.069224500039127, + "requests": 146005, + "outcomes": { + "ok": 2324, + "http_503_or_invalid_body": 105713, + "BrokenPipeError": 37551, + "RemoteDisconnected": 200, + "ConnectionResetError": 217 + }, + "latency_ms_all_attempts": { + "p50": 0.7667079917155206, + "p95": 2.0916659850627184, + "p99": 43.76049997517839, + "min": 0.026250025257468224, + "max": 373.9662499865517, + "mean": 1.7513654750468928 + }, + "latency_ms_successful": { + "p50": 44.48579205200076, + "p95": 92.43291697930545, + "p99": 145.10820800205693, + "min": 18.669291981495917, + "max": 373.9662499865517, + "mean": 56.13284755673295 + }, + "successful_requests_per_second": 144.62427853903847, + "successful_response_mib_per_second": 2.5428051366668414, + "error_percent": 98.40827368925721 + }, + { + "repository": "libgit2", + "operation": "refs", + "concurrency": 1, + "seconds": 16.01616116700461, + "requests": 10256, + "outcomes": { + "ok": 10256 + }, + "latency_ms_all_attempts": { + "p50": 1.505708962213248, + "p95": 1.7992090433835983, + "p99": 1.9750000210478902, + "min": 1.3214590144343674, + "max": 7.476333004888147, + "mean": 1.5469342338044034 + }, + "latency_ms_successful": { + "p50": 1.505708962213248, + "p95": 1.7992090433835983, + "p99": 1.9750000210478902, + "min": 1.3214590144343674, + "max": 7.476333004888147, + "mean": 1.5469342338044034 + }, + "successful_requests_per_second": 640.353196565523, + "successful_response_mib_per_second": 15.304461917046138, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "refs", + "concurrency": 4, + "seconds": 16.005978625034913, + "requests": 18717, + "outcomes": { + "ok": 18717 + }, + "latency_ms_all_attempts": { + "p50": 3.3329169964417815, + "p95": 4.086041997652501, + "p99": 4.763916949741542, + "min": 1.8169160466641188, + "max": 11.757583008147776, + "mean": 3.4061987174350397 + }, + "latency_ms_successful": { + "p50": 3.3329169964417815, + "p95": 4.086041997652501, + "p99": 4.763916949741542, + "min": 1.8169160466641188, + "max": 11.757583008147776, + "mean": 3.4061987174350397 + }, + "successful_requests_per_second": 1169.3755463801997, + "successful_response_mib_per_second": 27.94811302932185, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "refs", + "concurrency": 16, + "seconds": 16.015708165999968, + "requests": 155085, + "outcomes": { + "http_503_or_invalid_body": 106425, + "BrokenPipeError": 36914, + "ok": 11365, + "RemoteDisconnected": 201, + "ConnectionResetError": 180 + }, + "latency_ms_all_attempts": { + "p50": 0.741208961699158, + "p95": 11.474333005025983, + "p99": 13.217750005424023, + "min": 0.025707995519042015, + "max": 22.79691700823605, + "mean": 1.6395113912237211 + }, + "latency_ms_successful": { + "p50": 11.879750003572553, + "p95": 14.615916996262968, + "p99": 19.109999993816018, + "min": 3.878708987031132, + "max": 22.79691700823605, + "mean": 12.176306611578461 + }, + "successful_requests_per_second": 709.6158272992861, + "successful_response_mib_per_second": 16.95984101099721, + "error_percent": 92.67176064738692 + }, + { + "repository": "libgit2", + "operation": "tree", + "concurrency": 1, + "seconds": 16.0273771249922, + "requests": 19854, + "outcomes": { + "ok": 19854 + }, + "latency_ms_all_attempts": { + "p50": 0.7934999885037541, + "p95": 0.9943749755620956, + "p99": 1.1671249521896243, + "min": 0.5053749773651361, + "max": 36.6911250166595, + "mean": 0.8013181317481717 + }, + "latency_ms_successful": { + "p50": 0.7934999885037541, + "p95": 0.9943749755620956, + "p99": 1.1671249521896243, + "min": 0.5053749773651361, + "max": 36.6911250166595, + "mean": 0.8013181317481717 + }, + "successful_requests_per_second": 1238.7554024070962, + "successful_response_mib_per_second": 6.556396755168436, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "tree", + "concurrency": 4, + "seconds": 16.003774209006224, + "requests": 35592, + "outcomes": { + "ok": 35592 + }, + "latency_ms_all_attempts": { + "p50": 1.7680840101093054, + "p95": 2.4791660252958536, + "p99": 2.884333021938801, + "min": 0.777917040977627, + "max": 46.537958027329296, + "mean": 1.7930966172506462 + }, + "latency_ms_successful": { + "p50": 1.7680840101093054, + "p95": 2.4791660252958536, + "p99": 2.884333021938801, + "min": 0.777917040977627, + "max": 46.537958027329296, + "mean": 1.7930966172506462 + }, + "successful_requests_per_second": 2223.9753907531626, + "successful_response_mib_per_second": 11.77081581752995, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "tree", + "concurrency": 16, + "seconds": 16.023573751037475, + "requests": 141247, + "outcomes": { + "ok": 20254, + "http_503_or_invalid_body": 83517, + "RemoteDisconnected": 173, + "BrokenPipeError": 37141, + "ConnectionResetError": 162 + }, + "latency_ms_all_attempts": { + "p50": 0.8657919825054705, + "p95": 7.516624988056719, + "p99": 9.28116700379178, + "min": 0.026124995201826096, + "max": 77.92679202975705, + "mean": 1.806408299506879 + }, + "latency_ms_successful": { + "p50": 7.014915987383574, + "p95": 9.628708998207003, + "p99": 11.940707976464182, + "min": 2.885083027649671, + "max": 77.92679202975705, + "mean": 7.218813678929034 + }, + "successful_requests_per_second": 1264.0126550226423, + "successful_response_mib_per_second": 6.711990129438018, + "error_percent": 85.66058040170765 + }, + { + "repository": "starship", + "operation": "blob", + "concurrency": 1, + "seconds": 16.00264966697432, + "requests": 16234, + "outcomes": { + "ok": 16234 + }, + "latency_ms_all_attempts": { + "p50": 0.9408749756403267, + "p95": 1.1576669639907777, + "p99": 1.299374969676137, + "min": 0.660583027638495, + "max": 3.7677089567296207, + "mean": 0.956252978667922 + }, + "latency_ms_successful": { + "p50": 0.9408749756403267, + "p95": 1.1576669639907777, + "p99": 1.299374969676137, + "min": 0.660583027638495, + "max": 3.7677089567296207, + "mean": 0.956252978667922 + }, + "successful_requests_per_second": 1014.4570016741122, + "successful_response_mib_per_second": 63.30858390772817, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "blob", + "concurrency": 4, + "seconds": 16.019204750016797, + "requests": 26541, + "outcomes": { + "ok": 26541 + }, + "latency_ms_all_attempts": { + "p50": 2.2444999776780605, + "p95": 3.361875016707927, + "p99": 4.2891670018434525, + "min": 1.1972079519182444, + "max": 14.385791961103678, + "mean": 2.3772514884462317 + }, + "latency_ms_successful": { + "p50": 2.2444999776780605, + "p95": 3.361875016707927, + "p99": 4.2891670018434525, + "min": 1.1972079519182444, + "max": 14.385791961103678, + "mean": 2.3772514884462317 + }, + "successful_requests_per_second": 1656.823819545235, + "successful_response_mib_per_second": 103.39808580733056, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "blob", + "concurrency": 16, + "seconds": 16.01056633295957, + "requests": 149181, + "outcomes": { + "ok": 17041, + "http_503_or_invalid_body": 88827, + "BrokenPipeError": 42854, + "RemoteDisconnected": 236, + "ConnectionResetError": 223 + }, + "latency_ms_all_attempts": { + "p50": 0.7764159818179905, + "p95": 8.426666958257556, + "p99": 10.070791991893202, + "min": 0.0272089964710176, + "max": 19.993667025119066, + "mean": 1.6990347348808084 + }, + "latency_ms_successful": { + "p50": 8.269332989584655, + "p95": 10.661875014193356, + "p99": 12.635833001695573, + "min": 3.636666981037706, + "max": 19.993667025119066, + "mean": 8.451818296576976 + }, + "successful_requests_per_second": 1064.3596013789447, + "successful_response_mib_per_second": 66.54636524212845, + "error_percent": 88.57696355434003 + }, + { + "repository": "starship", + "operation": "commits", + "concurrency": 1, + "seconds": 16.004689000023063, + "requests": 3663, + "outcomes": { + "ok": 3663 + }, + "latency_ms_all_attempts": { + "p50": 4.277792002540082, + "p95": 5.101667018607259, + "p99": 6.068875023629516, + "min": 3.3681250060908496, + "max": 11.682791984640062, + "mean": 4.358136191547304 + }, + "latency_ms_successful": { + "p50": 4.277792002540082, + "p95": 5.101667018607259, + "p99": 6.068875023629516, + "min": 3.3681250060908496, + "max": 11.682791984640062, + "mean": 4.358136191547304 + }, + "successful_requests_per_second": 228.87042666025698, + "successful_response_mib_per_second": 4.041330736553808, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "commits", + "concurrency": 4, + "seconds": 16.037280582997482, + "requests": 5826, + "outcomes": { + "ok": 5826 + }, + "latency_ms_all_attempts": { + "p50": 10.686875029932708, + "p95": 12.921207991894335, + "p99": 16.019333037547767, + "min": 8.48954200046137, + "max": 32.082749996334314, + "mean": 10.97801928644032 + }, + "latency_ms_successful": { + "p50": 10.686875029932708, + "p95": 12.921207991894335, + "p99": 16.019333037547767, + "min": 8.48954200046137, + "max": 32.082749996334314, + "mean": 10.97801928644032 + }, + "successful_requests_per_second": 363.27854774684494, + "successful_response_mib_per_second": 6.4148433799877305, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "commits", + "concurrency": 16, + "seconds": 16.044194416026585, + "requests": 155344, + "outcomes": { + "http_503_or_invalid_body": 102128, + "BrokenPipeError": 49763, + "ok": 2908, + "ConnectionResetError": 289, + "RemoteDisconnected": 256 + }, + "latency_ms_all_attempts": { + "p50": 0.6870829965919256, + "p95": 2.0948330056853592, + "p99": 44.14208303205669, + "min": 0.023916014470160007, + "max": 79.94050002889708, + "mean": 1.645229124241228 + }, + "latency_ms_successful": { + "p50": 44.31204200955108, + "p95": 49.60324999410659, + "p99": 68.43279098393396, + "min": 21.835583029314876, + "max": 79.94050002889708, + "mean": 44.99723243675155 + }, + "successful_requests_per_second": 181.24936189349538, + "successful_response_mib_per_second": 3.1978944968471947, + "error_percent": 98.12802554331033 + }, + { + "repository": "starship", + "operation": "refs", + "concurrency": 1, + "seconds": 16.017063624982256, + "requests": 12267, + "outcomes": { + "ok": 12267 + }, + "latency_ms_all_attempts": { + "p50": 1.2107499642297626, + "p95": 1.5799999819137156, + "p99": 3.1397080165334046, + "min": 0.9955419809557498, + "max": 20.694000006187707, + "mean": 1.2947523266364327 + }, + "latency_ms_successful": { + "p50": 1.2107499642297626, + "p95": 1.5799999819137156, + "p99": 3.1397080165334046, + "min": 0.9955419809557498, + "max": 20.694000006187707, + "mean": 1.2947523266364327 + }, + "successful_requests_per_second": 765.8707168314435, + "successful_response_mib_per_second": 11.012839001068597, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "refs", + "concurrency": 4, + "seconds": 16.027744332968723, + "requests": 26701, + "outcomes": { + "ok": 26701 + }, + "latency_ms_all_attempts": { + "p50": 2.362457977142185, + "p95": 2.665541018359363, + "p99": 2.864292007870972, + "min": 1.3547090347856283, + "max": 4.374417010694742, + "mean": 2.388159744118872 + }, + "latency_ms_successful": { + "p50": 2.362457977142185, + "p95": 2.665541018359363, + "p99": 2.864292007870972, + "min": 1.3547090347856283, + "max": 4.374417010694742, + "mean": 2.388159744118872 + }, + "successful_requests_per_second": 1665.9237535425757, + "successful_response_mib_per_second": 23.95515285102363, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "refs", + "concurrency": 16, + "seconds": 16.02671875001397, + "requests": 157651, + "outcomes": { + "ok": 14602, + "http_503_or_invalid_body": 111964, + "BrokenPipeError": 30744, + "RemoteDisconnected": 194, + "ConnectionResetError": 147 + }, + "latency_ms_all_attempts": { + "p50": 0.7643330027349293, + "p95": 9.344458987470716, + "p99": 10.822499985806644, + "min": 0.026958005037158728, + "max": 16.665208037011325, + "mean": 1.613133108635022 + }, + "latency_ms_successful": { + "p50": 9.434541978407651, + "p95": 11.390750005375594, + "p99": 12.63729203492403, + "min": 3.121916961390525, + "max": 16.665208037011325, + "mean": 9.575170548322289 + }, + "successful_requests_per_second": 911.1035282869285, + "successful_response_mib_per_second": 13.101214408407504, + "error_percent": 90.73776886921111 + }, + { + "repository": "starship", + "operation": "tree", + "concurrency": 1, + "seconds": 16.013972209009808, + "requests": 19365, + "outcomes": { + "ok": 19365 + }, + "latency_ms_all_attempts": { + "p50": 0.74954197043553, + "p95": 1.1352500296197832, + "p99": 2.40679201669991, + "min": 0.4905420355498791, + "max": 11.217541992664337, + "mean": 0.8226428006385161 + }, + "latency_ms_successful": { + "p50": 0.74954197043553, + "p95": 1.1352500296197832, + "p99": 2.40679201669991, + "min": 0.4905420355498791, + "max": 11.217541992664337, + "mean": 0.8226428006385161 + }, + "successful_requests_per_second": 1209.256500963879, + "successful_response_mib_per_second": 3.833752690815503, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "tree", + "concurrency": 4, + "seconds": 16.004292124009226, + "requests": 31545, + "outcomes": { + "ok": 31545 + }, + "latency_ms_all_attempts": { + "p50": 1.613333006389439, + "p95": 3.273292037192732, + "p99": 10.248832986690104, + "min": 0.6747079896740615, + "max": 74.0720410249196, + "mean": 2.024663219069402 + }, + "latency_ms_successful": { + "p50": 1.613333006389439, + "p95": 3.273292037192732, + "p99": 10.248832986690104, + "min": 0.6747079896740615, + "max": 74.0720410249196, + "mean": 2.024663219069402 + }, + "successful_requests_per_second": 1971.033754918595, + "successful_response_mib_per_second": 6.2487241553177215, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "tree", + "concurrency": 16, + "seconds": 16.009098416019697, + "requests": 152799, + "outcomes": { + "ok": 24050, + "http_503_or_invalid_body": 85628, + "BrokenPipeError": 42647, + "RemoteDisconnected": 236, + "ConnectionResetError": 238 + }, + "latency_ms_all_attempts": { + "p50": 0.8189580403268337, + "p95": 6.577374995686114, + "p99": 7.767125032842159, + "min": 0.02887495793402195, + "max": 14.533292036503553, + "mean": 1.6718909593842297 + }, + "latency_ms_successful": { + "p50": 6.068458023946732, + "p95": 7.914166955742985, + "p99": 8.801666961517185, + "min": 2.592250006273389, + "max": 14.533292036503553, + "mean": 6.141423780936748 + }, + "successful_requests_per_second": 1502.2707322439894, + "successful_response_mib_per_second": 4.78240448478861, + "error_percent": 84.26036819612693 + } + ], + "overload_outcomes": { + "concurrency": 16, + "attempts": 1962881, + "http_503": 1303498, + "transport_errors": 471230, + "successful": 188153, + "invalid_successful_bodies": 0, + "note": "Single account, closed-loop workload without Retry-After backoff; failed attempts retained separately from successful latency and throughput." + } + }, + "cold": { + "method": "one first authenticated read per repository/API after graceful restart with a new empty Canopy data directory; RustFS and OS caches remain warm", + "reads": [ + { + "repository": "bat", + "operation": "refs", + "startup_seconds": 0.6589105000020936, + "latency_ms": 474.42112502176315, + "status": 200, + "valid": true, + "response_bytes": 6832, + "query": { + "kind": "refs" + } + }, + { + "repository": "bat", + "operation": "tree", + "startup_seconds": 1.0333095410023816, + "latency_ms": 745.7135000149719, + "status": 200, + "valid": true, + "response_bytes": 4541, + "query": { + "kind": "tree", + "commit": "d9559c69f541873f8c47ac8ad81d985f4441edf6", + "path_base64": "" + } + }, + { + "repository": "bat", + "operation": "blob", + "startup_seconds": 1.0326245839823969, + "latency_ms": 381.3612499507144, + "status": 200, + "valid": true, + "response_bytes": 22279, + "query": { + "kind": "file", + "commit": "d9559c69f541873f8c47ac8ad81d985f4441edf6", + "path_base64": "dGVzdHMvZXhhbXBsZXMvbG9uZ2xpbmUuanNvbg" + } + }, + { + "repository": "bat", + "operation": "commits", + "startup_seconds": 1.0474249579710886, + "latency_ms": 236.28650000318885, + "status": 200, + "valid": true, + "response_bytes": 20535, + "query": { + "kind": "history", + "commit": "d9559c69f541873f8c47ac8ad81d985f4441edf6" + } + }, + { + "repository": "libgit2", + "operation": "refs", + "startup_seconds": 0.5767115830094554, + "latency_ms": 975.2154999878258, + "status": 200, + "valid": true, + "response_bytes": 25061, + "query": { + "kind": "refs" + } + }, + { + "repository": "libgit2", + "operation": "tree", + "startup_seconds": 1.0377434589900076, + "latency_ms": 991.5179999661632, + "status": 200, + "valid": true, + "response_bytes": 5398, + "query": { + "kind": "tree", + "commit": "4af47f040cb4d80c75f37f626f511843e0dd8d44", + "path_base64": "" + } + }, + { + "repository": "libgit2", + "operation": "blob", + "startup_seconds": 1.0659633749746718, + "latency_ms": 859.9925839807838, + "status": 200, + "valid": true, + "response_bytes": 22154, + "query": { + "kind": "file", + "commit": "4af47f040cb4d80c75f37f626f511843e0dd8d44", + "path_base64": "ZGVwcy9wY3JlMi9wY3JlMl9jb21waWxlLmg" + } + }, + { + "repository": "libgit2", + "operation": "commits", + "startup_seconds": 0.5351114589720964, + "latency_ms": 1275.7054580142722, + "status": 200, + "valid": true, + "response_bytes": 18498, + "query": { + "kind": "history", + "commit": "4af47f040cb4d80c75f37f626f511843e0dd8d44" + } + }, + { + "repository": "starship", + "operation": "refs", + "startup_seconds": 0.5408446250366978, + "latency_ms": 352.6365830330178, + "status": 200, + "valid": true, + "response_bytes": 15078, + "query": { + "kind": "refs" + } + }, + { + "repository": "starship", + "operation": "tree", + "startup_seconds": 0.581210583972279, + "latency_ms": 409.1240420239046, + "status": 200, + "valid": true, + "response_bytes": 5088, + "query": { + "kind": "tree", + "commit": "df688000684138943610bbaa2a6c454db6987d48", + "path_base64": "" + } + }, + { + "repository": "starship", + "operation": "blob", + "startup_seconds": 0.5271440829965286, + "latency_ms": 396.08150004642084, + "status": 200, + "valid": true, + "response_bytes": 22185, + "query": { + "kind": "file", + "commit": "df688000684138943610bbaa2a6c454db6987d48", + "path_base64": "LmdpdGh1Yi93b3JrZmxvd3MvcmVsZWFzZS55bWw" + } + }, + { + "repository": "starship", + "operation": "commits", + "startup_seconds": 1.040699000004679, + "latency_ms": 499.0982919698581, + "status": 200, + "valid": true, + "response_bytes": 19598, + "query": { + "kind": "history", + "commit": "df688000684138943610bbaa2a6c454db6987d48" + } + } + ] + }, + "resources": { + "server": { + "sampling_interval_seconds": 2, + "max_observed_rss_mib": 40.375, + "max_observed_ps_cpu_percent": 424.5, + "median_observed_ps_cpu_percent": 199.2 + }, + "client": { + "sampling_interval_seconds": 2, + "max_observed_rss_mib": 82.375, + "max_observed_ps_cpu_percent": 201.1, + "median_observed_ps_cpu_percent": 18.4 + }, + "scope": "Two-second ps samples of the Canopy parent and benchmark client only; excludes SQLite/Git children, RustFS, OS page cache and true between-sample peaks." + }, + "storage": { + "repositories": [ + { + "repository_id": "c26304a7f3e4433a933277ffcbb26cbd", + "sqlite_bytes": 190627840, + "objects": 20952, + "refs": 66 + }, + { + "repository_id": "beb162113d72439794e609639ba05503", + "sqlite_bytes": 2093539328, + "objects": 134967, + "refs": 271 + }, + { + "repository_id": "f82d452384d14ab48099743457d303ae", + "sqlite_bytes": 685969408, + "objects": 38333, + "refs": 164 + } + ], + "provider_df": "Filesystem 1B-blocks Used Available Use% Mounted on\n/dev/vdb1 83953606656 76902137856 2739830784 97% /data\n", + "rustfs_data_directory_apparent_bytes": 22525385112, + "provider_df_after_cold_reads": "Filesystem 1B-blocks Used Available Use% Mounted on\n/dev/vdb1 83953606656 76903243776 2738724864 97% /data\n" + }, + "harness_sha256": { + "benchmark_repositories.py": "17515a1a8471fe8a35433b9eeeb8bc809d377dea44252f34c06ccedad4767320", + "benchmark_browse.py": "fb2945e0ca503413d5d5fd81351cb3e7b4ca6ab76c441693459a6417ec24d97a" + }, + "sample_count": 2425079, + "normal_load_requests": 462198, + "normal_load_errors": 0, + "raw_evidence_directory": "~/.local/share/canopy-benchmark-20261013/results" +} diff --git a/docs/benchmarks/2026-10-14-storage-optimization.md b/docs/benchmarks/2026-10-14-storage-optimization.md new file mode 100644 index 00000000..3fd630c3 --- /dev/null +++ b/docs/benchmarks/2026-10-14-storage-optimization.md @@ -0,0 +1,175 @@ +# RustFS storage optimization qualification — 2026-10-14 + +The storage changes keep Git bodies in verified, immutable native pack/index +pairs, batch metadata publication, and expose Cellule's retention collector +through drained deployment maintenance. Repository schema 3 requires a fresh +storage prefix. Schema-2 measurements below remain historical evidence. The current product contains no reader or migration path for +the earlier Git-body representation. + +The full-history corpus is unchanged from the +[original benchmark](2026-10-13-rustfs-browse.json): bat, starship and libgit2, +194,252 reachable objects, 28,189 commits and 501 branch/tag refs. Its source Git +packs total 152.94 MiB. All builds use published Cellule 0.2.0, with exact pins +and the lockfile checked together. The five direct crates remained at 0.2.0 in +the crates.io check on 2026-10-14. + +## Implemented priorities + +| Priority | Implementation | Measured evidence | +| --- | --- | --- | +| Retention-aware GC | Verified live roots and backup pins; grace boundary; bounded deletes; durable claim; drained owners and node advertisements | The completed three-repository native trial deleted 276,723 obsolete Cellule objects and retained 810 reachable objects. | +| Immutable Git packs | Separate self-contained structural/blob pack pairs; native and BLAKE3 verification; conditional publication; metadata only in SQLite | Durable Git pack bodies total 194.87 MiB, versus 3.07 GiB of expanded bodies in the original format. | +| Less directory rewriting | Smaller body-free SQLite state and fewer bounded metadata transactions | Before GC, directory backing files fell from 7.36 GiB / 351,653 keys to 5.72 GiB / 270,152 keys. | +| Batched import metadata | Up to 512 object metadata records per transaction, blob certificates in that batch, and up to 8,192 dependency rows per SQL write | Current SQLite files total 776,110,080 bytes, approximately 740 MiB, versus 2.77 GiB originally. SHA-256 wire-size and 8,193-edge boundary tests passed. | + +Directory rewrite reduction is the combined effect of fewer modified database +pages and fewer metadata publications. The comparison is not an isolated +measurement of either change. The repository namespace's existing `shards: 1` +is unrelated to LTX page-index sharding and was not changed by this work. + +## Completed native storage trial + +The storage trial used binary +`81a8084224da322a73eb1c73e6af90499da9a5cedba56a072e6d58f867b8a07c`. +Imports completed in 48.75, 90.53 and 910.33 seconds respectively. These are +observations on a shared host, rather than an isolated throughput comparison. + +| RustFS apparent backing bytes | Before GC | After GC | +| --- | ---: | ---: | +| All prefix files | 16,313,052,025 | 1,062,134,213 | +| Directory revisions | 6,141,486,337 | 17,132,798 | +| LTX segments | 9,678,730,257 | 810,393,284 | +| Native Git packs | 204,345,936 | 204,345,936 | +| Native Git indexes | 5,448,648 | 5,448,648 | + +The final 0.989 GiB is approximately 95.3% below the original 20.98 GiB. +The comparison counts regular files under each exact RustFS prefix, including +provider framing and metadata; it is not a bucket-wide allocated-disk measure. +The collector ran three bounded passes and did not delete native Git packs. +Receipts and before/after inventories are retained under +`~/.local/share/canopy-storage-optimization/medium-widebatch-20261014/results/`. + +The original schema-1 fixture's collection also completed on its exact archived +release. Its remaining sweep removed 260,957 obsolete objects and retained +2,985 reachable objects. That original representation occupies 2,937,040,978 +apparent bytes (2.735 GiB) after GC. Thus GC accounts for most of the reduction +from 20.98 GiB; the native representation is approximately 64% smaller than the +collected original representation. No old reader was added to the current +product to collect the archived fixture. + +Native packs are larger than the source's 152.94 MiB because Canopy creates +self-contained structural/blob pairs under its bounded native Git memory +policy. They still avoid the original expansion to 3.07 GiB. Cold reads fetch +the complete referenced pair; sparse S3 range reads are not implemented. +Selected serving objects use disposable, disk-accounted loose projections. + +## Index-warming checkpoint qualification + +The index-warming checkpoint is +`08d4e52c17d89808251d2516d99f797c821f7b48fc56b50034cf353f7e736674`. +Before its first uncertified object, preparation scans both existing graph +indexes using read-only 512-pair pages and indexed tuple seeks. That materializes +cold pages outside the write transaction without producing directory revisions. +Certification retains the 128-object / 64 MiB batching policy and 512-row edge +inserts. The real SQLite regression checks duplicate cursor keys, both index +orders, exact completeness and absence of full scans. +Both complete existing indexes are traversed during graph preparation; page +and statement sizes are bounded, but total warmup read work grows with history. + +Ref-target reachability checks obey Cellule's 128-statement SQL limit. The +regression uses 129 distinct commit tips. The earlier native storage trial +exposed this limit when cloning Starship's 164 refs; its storage result does +not constitute a successful three-repository clone qualification. + +That source passed formatting, all-target Clippy with warnings denied, +the exact dependency policy, Python's 19 tests, doctests, all eight ignored +real-S3 probes and the paired SHA-1/SHA-256 native-pack probe. The library +suite passed serially: 120 passed, one ignored. Two earlier parallel +workspace-lock failures passed unchanged focused rechecks; those failure +logs remain alongside the successful serial run. Integration results are +Main one, Directory 10, Git HTTP one, multi-server 93 with nine ignored, owner restart +one, repository one and smart HTTP one, all passed. + +The complete isolated process profile passed with 80 MiB Git/LFS objects, +65 MiB structural objects, 256 additional objects, 300 refs, SIGKILL and +takeover, two TLS peers, backup with every source object deleted, retained-pin +GC and an exact cold stock-Git clone. Final source and gate evidence are frozen +in `~/.local/share/canopy-storage-optimization/format2-index-warm-final-20261014/`. + +At that checkpoint the full 5 GiB Git/LFS and 512 MiB incompressible-pack +qualification was not run: its required 40 GiB free scratch and provider +capacity was unavailable. +Historical evidence retains each original binary and dependency version. + +An exploratory build, +`e89d1caea09a29a2bce6a09161ec4e4c6355a3aa4425468b230c813c9be02798`, +capped each certificate batch at 1,024 dependencies to avoid the published +runtime's five-second SQL execution deadline on cold indexes. Repeated crash +trials increased publication history until the provider filled. The final +build instead warms existing indexes with read-only seeks and retains the +larger certificate batches. The failed trial was collected under its exact +compiled release; its logs and partial-import evidence remain separate. + +## Limits and retained failure evidence + +The shared Docker VM filled during exploratory imports. The first disposable +512-dependency-row fixture was discarded by its exact prefix, with its logs +retained. The final build's libgit2 import was interrupted after durable object +staging; its recovery and retry are recorded separately. These attempts must +not be reported as successful clean import timings. + +The subsequent `08d4` three-repository run completed imports and collection, +leaving 1,191,805,853 apparent backing bytes (1.110 GiB). Its bat and starship +clone/API checks passed, but libgit2 exposed two serving limits: a cold SQL read +deadline and a fixed 120-second native CGI lifetime. That run is storage +evidence, not a complete final serving qualification. + +The current transport-budget build is +`7b16ef3133c39ad84290459e0c256765bcd0acb0fd3e4ac04eb85e2ebd1a0ec3`. +Reachability queries now use 16 statements per batch within Cellule's +128-statement ceiling, and native Git CGI has a finite 30-minute lifetime. +Storage schema 2 and command codecs 4/3 are unchanged. This build passed the +entire serial ordinary test run, formatting, all-target Clippy, the dependency +policy and Python's 19 tests. The complete isolated size/provider/process qualification passed on this exact +source and frozen release binary in 2,960.21 seconds. It transferred a Git blob +and an LFS object each larger than 5 GiB, plus over 512 MiB of incompressible +pack data. The native Git pack measured 595,260,812 bytes while SQLite measured +675,840 bytes after push. A fresh-server stock clone and strict fsck passed; +restored Git/LFS sizes and hashes matched. All eight real-provider protocol +probes, the paired SHA-1/SHA-256 pack probe and the extended process profile +passed, including retained-pin GC, source-deleted backup/restore and SIGKILL +recovery. Library/provider tests used the debug profile; process qualification +used the frozen release binary. + +The current medium run imported bat in 56.05 seconds and starship in 124.66 +seconds. Libgit2's first attempt failed after 1,492.34 seconds when graph +certification exceeded Cellule's five-second SQL deadline. Exact-release +maintenance recovery and retry completed: retry took 903.49 seconds, and the +failed attempt plus retry totaled 2,395.83 seconds, excluding recovery. This is +a retained failure and recovered import, not successful clean-import throughput. +Collection completed in three bounded passes: 276,750 obsolete objects were +deleted and all 824 marked objects were retained. The passes took 3,373.37 +seconds (56.22 minutes), excluding maintenance setup and restart. The post-GC +prefix occupies 1,259,651,402 apparent bytes (1.173 GiB), 94.4% below the +original measurement. Bat and Starship passed fresh mirror/fsck/object/ref/API comparisons. Libgit2 +failed its fresh full mirror on a five-second Cellule read deadline immediately +after explicit wants were hydrated. This build's medium serving qualification +is incomplete; its storage result remains valid. + +The next source revision adds a covering `(oid, kind)` index and uses bounded +keyset graph pages, limiting edge rows before type lookup and blob filtering. +Dependency certification checks kind and closure through the covering index, +without fetching unused wide metadata rows. It retains the 128-object / 64 MiB +certificate publication batches and the 512-object metadata batch. Repository +schema 3 has one fresh-cell initialization descriptor; certificate command codec +is 4 and object metadata codec remains 4. Qualification requires a fresh prefix; +no old reader or in-place index upgrade is provided. Its results are pending. + +Cellule GC is operator-invoked under maintenance. It does not reclaim orphaned +native Git/LFS bodies or replace a product-level object retention policy. +The separate Cellule collector performance patch is unpublished and is not +consumed by Canopy's registry pins. + +Refs, tree, exact blob bytes and commit history are checked against source Git. +Author/committer metadata is checked separately. A line-attribution query still +returns HTTP 422; there is no measured blame endpoint. diff --git a/docs/contracts.md b/docs/contracts.md index 825c45d2..bd4d1e61 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -1,6 +1,6 @@ # Understand Canopy's persisted contracts -This reference defines the identities, storage boundaries, authorization checks and retry behavior that Canopy must preserve across node loss. It describes the current development format. [Versioned migrations](../ROADMAP.md#r01-safe-upgrades-and-schema-migration) are still required before persistent customer repositories can survive upgrades. +This reference defines the identities, storage boundaries, authorization checks and retry behavior that Canopy must preserve across node loss. It describes only the current compiled release. Canopy follows the latest Cellule release and uses a fresh storage prefix for incompatible changes; backward compatibility and in-place migrations are out of scope. See the [Cellule integration policy](../AGENTS.md). > **Document type:** Reference. **Goal:** preserve durable behavior and make conflicts, retries, and recovery observable. @@ -24,7 +24,7 @@ request ▼ Directory Cell ── repository UUID ──► Repository Cell │ refs, ACL, issues, PRs, checks - │ Git object metadata and small bodies + │ Git object metadata and graph proofs ├──► immutable large Git/LFS bodies │ in object storage └──► disposable native Git cache @@ -46,7 +46,7 @@ sequenceDiagram A->>D: authenticate and resolve owner/name D-->>A: repository UUID and access context A->>R: submit mutation with identity and preconditions - R->>O: publish immutable external bodies when needed + R->>O: publish and verify immutable native Git pack/index pairs R->>R: validate ACL, version, graph, and policy R-->>A: commit result or conflict Note over R: The mutation and replay record share one durable boundary. @@ -80,15 +80,15 @@ The Directory Cell binds names and accounts to durable identities; the Repositor ### Git and LFS storage -Small Git objects remain in SQLite. Large Git blobs and Git LFS bodies use immutable object-store bytes with SQLite references and verification metadata. +Repository schema 3 keeps object identities, pack references and verified typed dependencies in SQLite. All Git bodies remain in self-contained native pack/index pairs. Git LFS retains its separate immutable object-store protocol. | Surface | Current value | Owner | | --- | --- | --- | | Git object ID | SHA-1 or SHA-256, according to the repository's immutable object format, over canonical Git type, decimal length, NUL and body | `object_id` | -| Small Git objects | SQLite `objects.body`, maximum 768 KiB | Repository Cell | -| Large trees, commits and tags | SQLite chunks of at most 512 KiB; object size above 768 KiB without a fixed byte quota | `object_chunks`, verified before object publication | -| Object publication | at most 128 records, 3 MiB aggregate inline payload; SQLite verification targets 64 MiB, with larger objects verified individually | `PutObjects`, operation 5, codec 3 | -| Large Git blobs | immutable `repos//git-blobs/` body, SQLite digest/size/reference | `LargeBlobStore` | +| Git bodies | Immutable native pack/index pairs, SHA-1 or SHA-256; both files bound by BLAKE3, exact lengths and native checksums | `git_pack`, Repository Cell metadata | +| Structural object verification | Streaming graph projection; no body persisted in SQLite | native reader, verified dependency metadata | +| Object publication | at most 512 metadata records; transient in-memory ingress up to 768 KiB per object / 3 MiB per batch; dependency writes contain at most 8,192 rows and reads at most 512 | `PutObjects`, operation 5, codec 4 | +| Large Git blobs | streamed canonical-OID/BLAKE3 verification; bodies remain compressed in native packs | native Git reader, `PackStore` | | LFS objects | immutable `repos//lfs/` body, SQLite digest/size/reference | `LfsService` | | LFS locks | unique path per repository; UUID identity, owning account and UTC second-precision timestamp in SQLite | `LfsService` / lock API | | External bodies | No fixed logical byte quota; 8 MiB immutable parts, a 16-byte Git manifest or a part-digest LFS manifest | Each conditional S3 copy handles one bounded part | @@ -106,7 +106,7 @@ These limits bound individual requests and publication work. They do not cap tot | Symbolic HEAD | `ref_generation.default_branch`, initially `refs/heads/main`; owner-authorized compare-and-set with ref generation | `RepositoryCell::set_default_branch` | | HTTP push identity | repository-local UUID bound to account and BLAKE3 request digest; different IDs identify independent operations | `pushes` | | HTTP push outcome | status, headers and BLAKE3-verified body in SQLite chunks; publish response pointer, rejection decision and ordered push notes atomically with accepted refs | `CompletePush`, codec 5 | -| Graph certificates | at most 128 candidates; SQLite verification targets 64 MiB, with larger objects verified individually; typed dependencies must already be certified | `CertifyObjects`, operation 6, codec 1 | +| Graph certificates | at most 128 candidates; SQLite verification targets 64 MiB, with larger objects verified individually; typed dependencies must already be certified | `CertifyObjects`, operation 6, codec 3 | | Git connectivity at ref publication | at most 100,000 certified new tips, with commit-only branch tips; same transaction as ACL, ref CAS and outcome | `object_closure`, shared ref finalization | | LFS metadata publication | check actor's write role in the SQLite insert transaction | `record_lfs_object` | @@ -235,13 +235,15 @@ checks child status in pages of 128. Certified children need no separate Cell call; uncertified children carry their observed immutable metadata into traversal. It groups at most 128 candidate object IDs in dependency order. Its cache of objects ready for certification covers only the pending batch. `CertifyObjects` independently -reads each candidate from SQLite, recomputes hashes and graph edges, and requires -all typed children to be certified. It checks child metadata in groups of at most -128 rows. The same transaction records deduplicated parent/child edges in +reads each candidate's verified dependency metadata from SQLite, checks its complete count and BLAKE3 digest, and requires +all typed children to be certified. It checks child kind and closure in groups +of at most 128 rows through the covering `objects_by_oid_kind` index; unused wide +metadata rows are not fetched. The same transaction records deduplicated parent/child edges in `object_edges`, with a reverse child index for fetch authorization. Only certified typed edges enter this index; gitlinks do not create local edges. -Newly verified inline and chunked bodies target 64 MiB per command; a larger -chunked object is verified individually; references to verified external blob bodies do not consume that budget. +Structural objects target 64 MiB per certificate command. Verified blob certificates are published with their metadata batch; they +have no outgoing edges and do not consume the structural byte budget. Canonical +object IDs and raw-body digests are checked before their native pack is published. The final ref transaction checks up to 100,000 new tips and their certificates in groups of 128, together with current ACL, versions and namespace conflicts. An ordered update map detects duplicates and planned ancestor conflicts. Exact @@ -255,8 +257,8 @@ rejects its whole application savepoint. The client traversal cannot assert a certificate. Certificates and refs are separate publications. A failed or interrupted preparation may retain valid certificates from earlier batches; retries reuse them. Partially certified roots cannot publish refs, and certificate -commits do not advance `ref_generation`. An external blob certificate relies on -the verified immutable upload before its SQLite record; there is no external +commits do not advance `ref_generation`. Every object certificate relies on +its verified immutable native pack before the SQLite record; there is no external network I/O inside certificate or ref transactions. `RepositoryCell::finalize_push` and HTTP completion share this preparation path. @@ -615,9 +617,10 @@ collector removes those bodies yet. Account creation is idempotent only for the same account, active token digest and scope. Bootstrap accepts an issued active owner admin token and does not replace or revive credentials. -Git packs and the bare repository cache are transport and acceleration -artifacts. Neither is authoritative. The gateway can reconstruct cache objects -from the Cell and external store. Integration tests prove exact-root restore +The Repository Cell owns Git identity, graph and ref metadata; verified immutable +pack/index pairs in the object store hold the corresponding Git bodies. The bare +repository cache is disposable. The gateway reconstructs its selected objects +from Cell metadata and those pairs. Integration tests prove exact-root restore after clean node shutdown and local SQLite loss. The process smoke also proves Git and LFS fetch after an unclean owner exit, lease expiry and a third process claiming the Cell from an S3-compatible object store. A proxy also drops a @@ -635,10 +638,10 @@ required. See [SQLite's contract](https://www.sqlite.org/autoinc.html). Each refresh captures the maximum stored sequence after reading its ref snapshot. Header pages query `after < sequence <= maximum` in sequence order, so concurrent -new publications wait for a subsequent refresh. Each page contains at most 128 -records and 768 KiB of aggregate inline bodies. A second query reads the exact +new publications wait for a subsequent refresh. Each page contains at most 512 +metadata records and no body payload. A second query reads the exact missing IDs, sorted by OID, with the header receipt as its minimum observation. -A byte-limited page advances only through its selected prefix. The gateway advances +A page advances only through its selected metadata records. The gateway advances its in-memory cursor only after every selected body has verified and completed its cache write. Failure or cancellation leaves the failed page eligible for retry; completed files are reused. The cursor belongs to the same cache owner and resets @@ -649,15 +652,17 @@ Each gateway retains one disk-accounted cache of verified immutable objects. Ref snapshots, pushes and native merge candidates get separate bare directories that borrow those objects through Git's [alternates contract](https://git-scm.com/docs/gitrepository-layout). -The shared cache receives only verified durable Cell records and external bodies. +The shared cache receives only objects verified against durable Cell metadata +and native packs. Native writes stay in the private generation; failed or unpublished pushes never populate the shared cache. A successful push's new objects enter it through the next durable hydration read. Snapshot HEAD/refs remain private and unchanged for the lifetime of a fetch, even while new snapshots reuse the same object bytes. -An object is compressed into an accounted temporary file, then installed at its -OID only after verification and compression finish. Incomplete files never count -as cache hits. Existing OIDs skip body queries, external downloads and compression; +The repository pack cache verifies each complete pack/index pair before reuse. +The serving cache projects only selected objects into accounted loose files; +large objects stream through native Git and small objects use bounded decoding. +Incomplete writes never count as cache hits. Existing OIDs skip repeated writes; refresh seeks past the completed sequence and checks local paths only for new headers or a retried page. A ref-only refresh reads the high-water mark without scanning old headers. No cache serves as @@ -674,10 +679,9 @@ writer's reservation through a read-then-resize race. Records are immutable through product APIs, and no collector removes them. This makes the two observations safe; a future collector must fence active reads before deleting records or bodies. The reader rejects changed/missing IDs, -invalid descriptors, and inline size, Git OID or BLAKE3 mismatches. Inline bodies -move into one bounded blocking verification task without a payload clone. -Chunked and external records carry descriptors; their body readers verify the -actual bytes before the gateway writes them to the disposable cache. Refs become +invalid descriptors, format mismatches and corrupt native pack/index identities. +Native index membership and entry headers bind stored object metadata to its pack. +Direct body reads additionally check canonical OIDs, raw lengths and BLAKE3. Refs become visible there only after successful hydration for the selected fetch mode. This bounds each query, not total repository hydration time or native Git scratch usage. The previous @@ -706,7 +710,7 @@ Both commands disable replacement refs so stored bytes retain their canonical identity. The batch reader caps headers at 128 bytes, validates kind and size before allocation, reads the exact body and delimiter, and recomputes its Git OID. Missing, truncated, malformed or corrupt output fails the push. Enumeration, -headers, inline/non-blob body reads, delimiters and process completion have +headers, bounded structural body reads, delimiters and process completion have 120-second deadlines. External blobs stream in at most 8 MiB chunks, with 120-second input/part idle deadlines and no whole-upload duration ceiling. OID hashing uses a blocking worker so large bodies do not occupy an async @@ -717,73 +721,45 @@ successful exits before publishing refs or recording the successful report. Previously published graph closure makes exclusions safe; ref publication still requires every new tip to have a durable certificate. -External Git blobs use multipart staging for bounded parts under a unique -repository-scoped prefix. Canopy hashes the exact declared body as it streams, -checks the Git OID, conditionally copies the parts, and creates the immutable -SHA-256 manifest. It verifies the destination before returning metadata. Conflicting canonical bytes fail; they -are never overwritten. Staging deletion and incomplete multipart abort are -attempted on ordinary failure. Process loss can leave staging/multipart debris; -operator lifecycle cleanup and qualified collection remain required. - -Hydration and backup verification read one part of at most 8 MiB at a time, -pinning the manifest version/ETag. The final range is withheld until Git SHA-1, -SHA-256 and BLAKE3 all match the SQLite reference. A failed or canceled range -invalidates its reader. Hydration compresses into disk-accounted cache files on -blocking workers; each writer retains the cache reservation until it exits. -The ref generation is not published for Git service until hydration finishes. - -External Git blobs use an immutable `CANOPY01` manifest (eight magic bytes, -then an eight-byte little-endian logical size). LFS bodies use `CANOPY02` with -one BLAKE3 digest per part after the size; SQLite pins the manifest's BLAKE3 -digest. The body bytes live at `.parts/<16-digit hex index>`. Each part is -at most 8 MiB; empty bodies have one empty part. -Parts are conditionally copied before the manifest is created, and the whole -logical body is reverified before SQLite metadata is published. Backup copies -all referenced parts before copying the manifest. There is no whole-object S3 -copy and no fixed logical file-size quota. `CANOPY02` replaces the previous LFS -manifest and digest meaning. This hard cutover requires a fresh preview prefix -or re-uploaded LFS bodies; no old-body decoder or migration is provided. - -Trees, commits and tags have no fixed byte quota, but publication and graph -parsing currently materialize individual bodies. Available worker memory remains -a practical constraint; this is not production capacity qualification. - -Candidate existence queries group up to 128 IDs. Missing records accumulate in -an `ObjectBatch` with at most 128 records and 3 MiB of aggregate inline bodies, -leaving room for metadata beneath the 4 MiB operation input limit. External -blob records count toward the record limit; their bytes are verified and -uploaded before publication. Chunk references count toward the record limit and -a 64 MiB aggregate verification target shared with inline bytes. An oversized -individual object is verified alone. The -decoder enforces these bounds before publication. One typed Cell command publishes the batch and returns one receipt. -It recomputes inline Git OIDs and BLAKE3 digests, then compares every inserted -or existing row with the complete expected record. A mismatch rejects the -command and rolls back all inserts in that batch. Repeating the same mutation -identity and payload returns the recorded result through Cellule deduplication. - -Trees, commits and tags larger than 768 KiB are staged in `object_uploads` and -`object_chunks`; their bodies stay in SQLite. Each part is at most 512 KiB and -there is no fixed number of parts per object. Part command identities derive from -the caller's upload identity and part index, so retrying the same staging -operation replays committed parts. Staging does not create an `objects` row: -queries and ref validation cannot see incomplete uploads. - -`PutObjects` codec 3 accepts a chunk reference containing upload ID, size and -BLAKE3. In the publication transaction it requires the exact chunk count and -part lengths, reconstructs the body, verifies the canonical Git OID and BLAKE3, -and then inserts the immutable object record. A failure rolls back every object -in that batch. Inline and external-blob records use the same publication command. -Repeated identical objects converge on the existing record; unused duplicate -uploads remain staged until a collector can prove they are unreferenced. - -Hydration and direct SQLite object reads use the same chunk count, length and -hash checks. Every SQL result contains at most one chunk, fitting Cellule's -1 MiB result limit. Reads reconstruct a bounded whole object in memory. Graph -certification reconstructs chunked objects before applying the same typed edge -checks as inline objects; chunking does not certify missing edges. Part data is -immutable through product APIs, and no collector exists yet. Any future collector -must preserve chunks reachable from objects and invalidate graph certificates -before removing their dependencies. +Native Git produces separate self-contained structural and blob packs for each +import or push's new history, excluding previously published tips without +enabling thin packs. Each v2 index +is generated under disk admission. Both files are uploaded to staging keys, +conditionally copied to content-addressed `repos//git-packs/` keys, and +verified in full before metadata may name them. A conflicting immutable file +fails publication. No old Git-body reader is compiled. + +Object verification streams canonical Git hashing and BLAKE3 with bounded buffers. +Structural graph verification retains edges without retaining messages or path +names. Merge and rebase validation uses content digests, typed edges and immutable normalized +commit fingerprints; commit bodies remain in the native pack. + +Candidate existence queries and metadata publication group up to 512 objects. +Transient in-memory ingress is bounded to 768 KiB per object and 3 MiB per batch; +it is packed and verified before dispatching `PutObjects` codec 4. The command +accepts only prepared pack references. It compares immutable content and graph +metadata after bounded multi-row insertion. Any conflicting record rejects +and rolls back the entire application savepoint. Dependency rows are staged in +multi-row writes of at most 8,192 within Cellule's 1 MiB SQL command limit, then +their exact count and BLAKE3 digest are +checked during graph certification. Staging never publishes a ref. + +Certificate preparation groups at most 128 objects and 64 MiB of structural +bodies. One oversized structural object may occupy a batch by itself. Before +preparing its first uncertified object, it warms both existing graph indexes +through read-only 512-pair pages with indexed tuple seeks. This moves cold page +I/O outside Cellule's five-second write execution budget without creating extra +publications. Certified edges use multi-row writes of at most 512 rows; typed +closure verification and atomic rejection remain unchanged. + +Hydration verifies pack/index identity before projecting selected objects into +the disposable Git cache. Large +objects stream through native Git without whole-object allocation by Canopy; +small objects use bounded decoding. Direct object-body reads have a 64 MiB +allocation bound. A cold read currently downloads its complete referenced pack; +provider traffic can include bodies omitted from a filtered clone's wire pack. +Sparse range reads are not yet implemented. Git pack orphan reclamation remains +separate from the Cellule retention collector. Object batches remain separate from ref and push-outcome publication. A later failure can leave earlier batches unreferenced; a retry finds those records @@ -800,8 +776,9 @@ concurrent cache replacement cannot delete files beneath an admitted reader. Receive-pack collects its response with the existing 64 MiB limit, then persists objects, validates refs and records the outcome before sending HTTP success. -CGI headers and stderr are each bounded at 64 KiB. The existing 120-second -subprocess deadline includes output backpressure. Once headers are sent, a +CGI headers and stderr are each bounded at 64 KiB. The finite 30-minute +subprocess deadline permits native packing of large histories and includes +output backpressure. Once headers are sent, a process failure or timeout produces an HTTP body error rather than successful EOF; before headers it fails the request. Dropping the response cancels its worker. The spawn helper consumes and drops the parent command before waiting for @@ -825,8 +802,8 @@ annotated-tag chains. Native Git checks target existence, peels tags and emits all protocol bytes. Commit ancestors and tree children are not traversed for this exchange. Direct blob/tree refs still require their own verified bodies; a large blob tag can therefore consume substantial discovery cache space. -Selected SQLite object reads reuse the 128-record/768-KiB inline page limit; -chunked and external objects use the same verified hydration path as transfers. +Selected SQLite object reads reuse the 512-record metadata page limit and +the same verified native pack installation path as transfers. Missing or corrupt targets fail preparation instead of disappearing from refs. A current ready cache is reused when its mutex is immediately available. Discovery @@ -1180,10 +1157,10 @@ Schema version 1 is still changing in this unreleased repository. The chunk, HEAD, discovery, token-metadata, issue, check, branch-rule, pull/review, review-head, merge, candidate, object insertion-sequence and membership-version layouts, operations 7–10, and the operation-5/8/9 codec changes require a fresh development storage prefix; -there is no upgrade reader for older development databases. The module -descriptor and object paths will become compatibility boundaries at the first -persistent preview. The current build pins Cellule revision -`a28de7bc09ce36d87e642adc4f4b6be50d6fcb69`. The earlier entity-partition +there is no upgrade reader for older development databases. The module descriptor +and object paths identify the current format; older formats are rejected rather +than migrated. The current build pins the Cellule crates to +exact crates.io version `=0.2.0`. The earlier entity-partition cutover also made pre-cutover prefixes incompatible; no migration is available. ## Accounts and authentication @@ -1811,8 +1788,8 @@ or restoring write access therefore cannot revive an old approval. The immutable owner uses generation zero. Unauthorized ACL calls cannot change generations. Existing ACL access, roster and check-reporting policy stays the same; check results continue to use their configured context versions, as documented above. -A fresh development prefix is required; old memberships have no generation -backfill. A future migration must initialize those before enabling these APIs. +A fresh storage prefix is required; memberships are initialized with generations +by the current schema. Old memberships are not backfilled or accepted. All three mutations are bounded guarded SQL batches through the registered Cell SQL command. Their recorded pre-mutation domain decision, conditional write and @@ -2221,7 +2198,7 @@ native worker is trusted to compute the merge tree; the Cell validates exact canonical commit bytes and certified graph closure before recording readiness. It does not independently recompute the merge algorithm. -Generated objects use the same verified SQLite/chunk/external-blob ingestion as +Generated objects use the same verified native-pack ingestion as pushes. A ready result and `refs/canopy/merge-candidates/` commit in one transaction, advancing the ref generation for cache invalidation and coherent pagination. The entire `refs/canopy` namespace, including its root, is reserved. @@ -2253,10 +2230,11 @@ its cache lifetime. Completed scratch writes are charged before object ingestion native peak disk/memory/CPU bounds and crash-left cleanup remain release gates. Candidate input/result objects and their ancestor closure are retention roots. -No GC runs today. Abandoned pending rows, ready refs, external orphan objects and -historic candidates need quota/retention policy before persistent public use. -Schema 1 remains unreleased: new candidate tables, operation 10 and operation 9 -codec 4 require a fresh development prefix. No dependency or lockfile changes. +Maintenance GC reclaims obsolete Cellule database objects; it does not remove +candidate rows, ready refs or external Git/LFS bodies. Abandoned pending rows, +external orphan objects and historic candidates need quota/retention policy +before persistent public use. Repository schema 3 initializes only fresh Cells; +current command identities and codecs are checked with the compiled release. ## Repository browser @@ -2475,7 +2453,7 @@ Stock `git lfs pull` without credentials verifies this behavior. Both Directory and Repository initialization schemas changed in this unreleased build. Use a fresh development prefix; existing data is not migrated or removed. Source digest validation rejects old modules; mixed-build rolling upgrades are -not supported. Production upgrade migration remains a delivery gate. +not supported. These are intentional boundaries of the current-release policy. ## Deployment release enrollment and maintenance @@ -2521,9 +2499,9 @@ Session enumeration is bounded at 4,096 and fails closed on overflow. Maintenanc end requires the exact operation and drain proof, then uses the upstream release CAS. Retrying completed end returns its unchanged Ready record. -The upstream contracts are implemented in pinned Cellule revision -`a28de7bc09ce36d87e642adc4f4b6be50d6fcb69`, specifically runtime -`cell/application.rs`, `recovery/release.rs` and `node.rs::advertised_sessions`. `start_maintenance` closes release +The upstream contracts are implemented in Cellule 0.2.0, specifically runtime +`cell/application.rs`, `recovery/release/mod.rs` and +`node/directory/advertisement.rs::advertised_sessions`. `start_maintenance` closes release admission but does not itself drain writers; Canopy supplies that lifecycle. Heartbeat expiry is not writer-close evidence. Source store errors propagate; there is no fallback to an unregistered release. @@ -2609,16 +2587,15 @@ so the reservation is also checked on serving and maintenance admission. An olde binary without that check must not access these destinations. Create copies the runtime pin/graph, then restores each pinned repository SQLite -snapshot locally to enumerate external Git/LFS references. The pinned SQLite data -is the body manifest: no second mutable index determines backup contents. Rows -are paginated in batches of 256. Source and destination bytes must match the -recorded size and SHA-256; Git blobs also match their Git OID and whole-body -BLAKE3, while LFS parts match the SQLite-anchored manifest. Body parts and manifests are -conditionally copied before the completion CAS. Conflicting destination bodies -must verify or the operation fails. Snapshot disk reservations cover each restored -database and remain held until its scratch directory is removed. Capture admits -at most 100,000 Cells. Git blob and LFS body verification stream bounded ranges -without a fixed logical byte quota through their respective verified readers. +snapshot locally to enumerate native Git pack/index pairs and LFS references. +The pinned SQLite data determines backup contents; no second mutable index is +used. Rows are paginated in batches of 256. Each native file must match its +recorded BLAKE3 digest and length. LFS parts match the SQLite-anchored manifest. +Both native files and all LFS parts/manifests are conditionally copied before +completion CAS. Conflicting destination data must verify or the operation fails. +Snapshot disk reservations cover each restored database until its scratch is +removed. Capture admits at most 100,000 Cells. Verification streams bounded +buffers without a fixed logical pack or LFS byte quota. Verify and restore enroll at the completed backup prefix and read no original source prefix. Restore copies into a disjoint, reserved prefix and preserves the @@ -2759,8 +2736,10 @@ remain required. No new configuration surface was added for diagnostic output. ## Cellule integration Canopy directly uses `cellule-app`, `cellule-host`, `cellule-runtime`, -`cellule-ltx` and `cellule-store`, pinned to Cellule commit -`a28de7bc09ce36d87e642adc4f4b6be50d6fcb69`. `cellule-types` is transitive. +`cellule-ltx` and `cellule-store`, each pinned to exact crates.io version +`=0.2.0`, released from commit `aaacf311622a2fea009067d1f5f217751ee1a2cf`. +`cellule-types` is transitive at the same version; Cargo.lock records registry +checksums for all six crates. The lockfile contains no Crab Cell, Crab product/server or Xet packages. Historical qualification runs in the delivery/performance logs retain their original dependency revisions; they are not performance evidence for this build. @@ -2790,9 +2769,12 @@ its derived target against the catalog partition and control Cell ID before copying referenced bodies. A provisioned repository before owner initialization may have no identity only when it has no external bodies. -This changes the compiled release and runtime storage path. Use a fresh -development prefix. Existing selected-release validation rejects the old -compiled release; there is no Crab Cell reader, mixed-version rolling upgrade, +Cellule 0.2.0 changes the compiled release and runtime storage path to Node +Bundle v1, including authenticated catalog/history encodings and `.nodebundle` +paths. Older encodings and paths are rejected without a fallback reader. Use a +fresh development prefix; before retiring an older deployment, drain and +materialize its acknowledged history with the previous binary. Existing selected-release validation rejects +the old compiled release; there is no Crab Cell reader, mixed-version rolling upgrade, or backup conversion. `server::storage::probe` runs before serving-node or maintenance-worker enrollment. diff --git a/docs/delivery-plan.md b/docs/delivery-plan.md index ce7f66ec..0c1383fc 100644 --- a/docs/delivery-plan.md +++ b/docs/delivery-plan.md @@ -144,8 +144,8 @@ These are the next implementation slices. Each slice must update its contract, b retries must converge. Pinned SQLite roots provide the external body manifest. - Extend independent same-provider copy qualification to the production store and cross-provider export. Preserve identity/release validation, exact Git/LFS - bytes and collaboration/ACL state after source deletion. Add old-release - migration only with explicit format/version proof. + bytes and collaboration/ACL state after source deletion. Old-release + migration is out of scope under the current Cellule integration policy. - Add retention and collection only after roots include active transfers, pending operations, recovery roots and backup pins. Acceptance: concurrent fetch/merge/backup and injected collector failure never remove required @@ -159,6 +159,11 @@ should remove bytes while fetch, backup or a pending merge can still read them. The remaining sections record completed slices and open proof at the revision where each was tested. Read the gate table above for current release status. Start with a topic below if you need the detailed acceptance record: +Historical suggestions to add upgrade migrations or preserve old formats are +superseded by the [current Cellule integration policy](../AGENTS.md). Incompatible +changes intentionally require a fresh prefix; the recorded test evidence below +retains its original release and scope. + | Topic | Representative evidence | | --- | --- | | Git object and ref publication | [Atomic object batches](#atomic-object-batch-qualification), [bounded graph certification](#bounded-graph-certification-before-ref-publication), [durable push refusals](#durable-push-preparation-refusals) | @@ -3799,3 +3804,97 @@ discovery test expects 401 for an anonymous repository listing, while the existing public-listing route deliberately returns 200. That assertion also predates this cutover. Hosted CI and production-provider qualification remain open. + +### Cellule 0.2.0 release pin (2026-10-13) + +Canopy now pins all five direct Cellule crates to exact crates.io version +`=0.2.0`, published from `aaacf311622a2fea009067d1f5f217751ee1a2cf`. +`cellule-types` resolves to the same version, and the lockfile records registry +checksums for all six crates. Unrelated locked dependencies are unchanged. +The manifest now declares the already-documented Rust 1.97 minimum. + +This is a fresh-prefix cutover. Cellule 0.2.0 uses Node Bundle v1 and rejects +older bundle encodings and paths without migration. Before retiring an older +deployment, drain and materialize its acknowledged history with the previous +binary. Existing prefixes and backups are not qualified for in-place upgrade. + +The verification run exposed fixture assumptions that needed correction: + +- Process takeover checks still waited for a ten-second lease; the signed node + lease is thirty seconds. The smoke scripts now share a 31-second expiry wait. +- Disconnecting an admission request does not join its tracked release task. + The regression now waits for local cleanup and asserts durable `Idle` state + with no owner before testing subsequent admission and exact Git recovery. +- The combined Repository Cell debug scenario overflows libtest's default + stack. A larger-stack diagnostic run passed. Only this scenario's driver now + has an explicit 8 MiB stack; runtime workers retain their defaults, and all + original assertions remain. +- Independent fixtures run serially in CI to bound process load. Individual + scenarios still exercise concurrent requests, native workers and multiple nodes. + +`qualify_size.py --provider-only --process-binary ` now runs the existing +process smoke after the eight isolated RustFS protocol probes. It snapshots the +supplied executable into the temporary fixture so all restarts use one build, +sets isolated provider credentials, and removes its container and volume on +completion or failure. CI builds the server, runs the Python harness tests and +invokes this combined qualification. + +Local validation used Rust 1.97.0, Git 2.50.1 and RustFS +`1.0.0-beta.8-glibc` on macOS: + +- All 219 ordinary Rust tests pass across the library, binary and integration + targets, including the 93 full-server tests. Seven completed test binaries + were staged onto local disk for the final serial run to avoid shared build + volume I/O stalls; the corrected Repository Cell target passed through Cargo. +- All eight isolated RustFS protocol probes pass: SHA-256 repositories and merge + candidates, signed HTTP/SSH pushes, stock SSH, bulk refs, partial clones and + stock LFS over SSH. +- The eleven Python harness tests, normal server build, all-target Clippy with + warnings denied, formatting and diff checks pass. +- The complete process qualification passes with the locally staged executable: + Git/LFS and collaboration API state survive restart, local disk corruption, + dropped replies and lease takeover. Two live HTTPS nodes serve eight + repositories beyond resident capacity, including owner loss and interrupted + maintenance recovery. The backup CLI restores exact 80 MiB Git and LFS bodies, + empty LFS objects and issue state after every original source object is deleted. + The isolated provider container and volume were removed successfully. + +An initial parallel run hit a transient workspace-lock failure that passed in +isolation and in the serial run. The first process run from the shared build +volume timed out during restart; the successful repeat used an immutable local +executable. +These runs are functional qualification, not latency or throughput evidence. +The separate >5 GiB gate was not run: the Docker VM had about 26 GiB free, +below its 40 GiB preflight requirement. + +### Current-release-only integration audit (2026-10-13) + +Fresh-prefix cutovers are intentional product policy. Canopy follows the latest +published Cellule release without preserving older runtime or Canopy formats. +The upstream HEAD and `v0.2.0` tag still resolve to +`aaacf311622a2fea009067d1f5f217751ee1a2cf` at this audit. + +The audit covered module registration, command codecs, dependency sources, +deployment admission, root reservations, local workspace markers, external-body +readers, configuration parsing and their callers. No legacy runtime adapter, +old-format reader, retained module implementation, configuration alias, or +upgrade migration chain was found to remove. Each module has one schema and an +empty `retained_codes` list. Its single `MigrationDescriptor` initializes fresh +SQLite state through the current Cellule API. Chunked non-blob storage, optional +API fields, Git protocol support, and recovery of the same compiled release are +active behavior, not old-format compatibility. + +Removed stale migration promises from the roadmap and current operational +contracts, including a reference to a nonexistent maintenance upgrade path. +`AGENTS.md` records the policy for future changes. CI now checks that every direct +and transitive Cellule crate resolves to one exact stable crates.io release; +the grouped Dependabot configuration requests updates after reaching the default +branch. No runtime source or dependency version changed in this audit. + +Validation: all 18 Python tests pass, including seven dependency-policy cases +covering partial upgrades, version ranges, aliases, alternate sources, duplicate +versions and missing lock entries. All ten existing deployment admission/recovery +tests pass against Cellule 0.2.0. Both workflow YAML files parse; formatting and +`git diff --check` pass. The complete protocol/process qualification above remains +the evidence for the unchanged runtime; it was not repeated for this policy and +CI change. diff --git a/docs/git-compatibility.md b/docs/git-compatibility.md index 33ae303d..94e57f79 100644 --- a/docs/git-compatibility.md +++ b/docs/git-compatibility.md @@ -127,8 +127,11 @@ to the dedicated test volume. Docker must mount that host volume; the script verifies visibility before writes. Use `DOCKER_CONTEXT` to select an isolated Docker environment if needed. The fixture uses public test credentials and removes its own container and data. -The Verify workflow invokes this gate, but its current `ubuntu-latest` runner does not have the required temporary space; [release CI work](../ROADMAP.md#r04-trustworthy-ci-and-release-artifacts) remains open. Ordinary `cargo test` skips -the resource-intensive provider tests. `--provider-only` runs the smaller Git +The Verify workflow runs the smaller `--provider-only` compatibility probes +and `--process-binary target/debug/canopy` restart, takeover and backup checks. +The full size gate needs a dedicated runner with the required temporary space; +[release CI work](../ROADMAP.md#r04-trustworthy-ci-and-release-artifacts) remains open. +Ordinary `cargo test` skips the resource-intensive provider tests. `--provider-only` runs the smaller Git compatibility probes against an isolated Docker volume when the host's large test volume is unavailable. For the full gate on a Docker VM with sufficient dedicated disk but no host-volume sharing, use `--docker-volume`; it creates and diff --git a/docs/implementation.md b/docs/implementation.md index cdc143c6..9b0678a9 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -45,7 +45,8 @@ These are current admission limits, not measured production throughput: | Concurrent Git/LFS transfers | Eight per node; excess requests return `503` and `Retry-After: 1`. | | Fetch request body | Up to 64 MiB; clone/fetch response packs stream with backpressure. | | Push report | Buffered up to 64 MiB; push bodies have no fixed byte quota. | -| Git blob and LFS body | Immutable 8 MiB parts, with no fixed logical file-size quota. | +| Git content | Immutable self-contained native pack/index pairs; blob ingestion streams with no fixed logical blob-size quota. | +| LFS body | Immutable 8 MiB parts, with no fixed logical file-size quota. | | Active repository gateways | `max_active_repositories` per node (1–9,999); stored repositories may exceed this. | | Local admitted disk | Set by `local_disk_limit_bytes`; exhaustion returns `507`. | @@ -53,14 +54,16 @@ The current service supports one repository owner. Incoming Git requests stream to temporary files charged to the same disk budget as the node's SQLite files. Push bodies have no fixed byte quota; fetch requests remain bounded at 64 MiB. Push reports are buffered up to 64 MiB, while clone/fetch -packs stream with backpressure. Git blobs and LFS objects use immutable 8 MiB -parts without a fixed logical file-size quota. Trees, commits and tags above -768 KiB use SQLite chunks without a fixed individual object-size quota. +packs stream with backpressure. Git bodies remain compressed in native pack/index pairs. Blob verification streams +through a persistent native Git reader. Structural graph projection streams with bounded buffers. Direct whole-object +reads retain a 64 MiB allocation limit. LFS objects use +immutable 8 MiB parts without a fixed logical file-size quota. Git LFS basic downloads resume with a tail `Range` request and a verified `206` response. The server reads only the requested parts and verifies each against a manifest digest pinned in SQLite. -Publication and graph parsing still materialize non-blob bodies, so very large -structural objects depend on available worker memory. +Publication hashes structural bodies through bounded buffers and retains their +typed graph edges, rather than complete messages or tree path names. Memory for +the graph projection grows with the number of dependencies. Partial uploads stay invisible to Git. Each node admits eight Git/LFS transfers across all repositories. Overload returns 503 with `Retry-After: 1`; retry after capacity is available. Health, @@ -136,14 +139,32 @@ A terminal ownership-release failure leaves that repository unavailable until node restart; confirmed-release cleanup errors are retried on later admission. There is no account deletion API, organization model or production capacity evidence. -`Cargo.toml` pins Cellule's runtime, app, host, LTX and store crates to revision -`a28de7bc09ce36d87e642adc4f4b6be50d6fcb69`. Canopy remains a separate +`Cargo.toml` pins Cellule's runtime, app, host, LTX and store crates to the exact +crates.io version `=0.2.0` (release source +`aaacf311622a2fea009067d1f5f217751ee1a2cf`). Canopy remains a separate product crate and builds without a local Cellule checkout. There are no Crab product/server or Xet dependencies. ### Storage compatibility and graph verification -Use a **fresh storage prefix** for this build. Cellule derives a 33-byte entity +Canopy adopts current Cellule APIs directly and supports one compiled release. +There are no old-format readers, retained module implementations, or upgrade +migrations. The single `MigrationDescriptor` in each module creates the fresh +SQLite schema required by Cellule; it does not migrate previous deployments. +Format, identity, release and fencing checks remain mandatory. + +`python3 scripts/check_cellule.py` verifies that direct dependencies use one exact +stable crates.io version and that every locked Cellule crate uses that release. +CI runs this check before building. The checked-in Dependabot configuration +requests grouped Cellule updates on its daily schedule once it is on the default +branch; those updates must pass the same provider/process qualification as any +other dependency change. The check itself is offline and does not establish +whether a newer upstream release exists. + +Use a **fresh storage prefix** for this build. Cellule 0.2.0 uses Node Bundle v1 +and rejects older bundle encodings and paths; there is no automatic migration. +Before retiring an older deployment, drain and materialize its acknowledged +history with the previous binary. Cellule derives a 33-byte entity partition from the repository UUID; the UUID is also persisted in repository SQLite for backup recovery. This is a hard cutover: previous runtime prefixes/backups are unsupported. Startup rejects unmarked application roots, @@ -153,29 +174,33 @@ and release admission rejects a different compiled release. See the Before ref publication, bounded certificate batches verify the durable Git graph: commit trees and parents, tree entries and tag targets must exist with the correct object type. Each batch covers at most 128 objects, targeting 64 MiB of -SQLite object bytes. Ref publication checks certified tips atomically with +structural object bytes. One oversized structural object is handled alone. +Bounded read-only seeks warm both existing graph indexes before certification +so cold page retrieval does not consume the write transaction's SQL budget. Ref publication checks certified tips atomically with permissions and ref versions; branch tips must be commits. Submodule gitlinks may name commits in another repository. SQLite certificates let later pushes reuse validated history. Push ingestion streams candidates from accepted ref tips, excludes previously published history, and reads missing objects through one persistent Git batch process. Object sizes are checked before allocation and canonical OIDs before storage. SQLite lookups -group up to 128 candidate IDs; object publication groups up to 128 records and -3 MiB of inline bytes in one Cell transaction, targeting 64 MiB of SQLite -object bytes verified per batch. A conflicting record rejects +group up to 512 candidate IDs; object publication groups up to 512 metadata +records in one Cell transaction. Typed dependency rows are staged in bounded +multi-row pages before publication. Canonical OIDs and raw-body BLAKE3 hashes +are verified before recording a pack reference; no Git body enters SQLite. A conflicting record rejects the whole batch. Recovery tests include annotated tags, submodules and `git fsck` on the restored clone. ### Cache hydration and partial clone -Cold cache hydration reads insertion-ordered pages of at most 128 records and 768 KiB -of inline bodies. It verifies inline identities on a blocking worker; chunked -and external bodies retain their own verification before cache writes. Pages -reduce SQLite query overhead. Blobless fetch (`--filter=blob:none`) skips ordinary -blob bodies until explicitly requested. Other fetches enumerate missing blobs -reachable from requested tips, applying the native filter before hydration. -Tree and object-type constraints, including those in combined filters, skip -omitted bodies; size filters still need missing blob bodies. Advertised ref/tag targets are always prepared. -Non-blob history is still prepared across the repository. Supported filters and remaining gaps are listed in +Cold cache hydration reads insertion-ordered pages of at most 512 metadata records. +Each referenced pack/index pair verifies its exact size, BLAKE3 identity, native +checksums and pair binding before installation. Native Git reads disposable projections of selected objects. Small objects are +decoded under a memory bound; larger objects stream through native Git into +disk-accounted loose files. Durable Git content remains in native packs. Filtered clone and fetch retain +native Git's wire filtering and authorization rules. A cold read currently +fetches a complete referenced pack, so provider traffic can include objects +omitted from a filtered wire response. Sparse range loading remains open. +Advertised ref/tag targets are prepared from their immutable packs. +Supported filters and remaining gaps are listed in [Git compatibility](git-compatibility.md). While a gateway remains resident, verified object files are shared across private ref snapshots, pushes and merge candidates. An indexed insertion cursor limits @@ -192,11 +217,28 @@ tests. The repository's `target/` directory is ignored by Git: ```bash export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$PWD/target}" -cargo test --locked +python3 scripts/check_cellule.py +cargo test --locked -- --test-threads=1 cargo clippy --all-targets --locked -- -D warnings cargo fmt --all -- --check ``` +CI serializes independent test fixtures; individual tests still exercise +concurrent requests, Git workers and multi-node ownership. + +To run the same isolated provider and process checks as CI, install Docker, +the AWS CLI and OpenSSL, then run: + +```bash +cargo build --locked --bin canopy +python3 scripts/qualify_size.py --provider-only \ + --process-binary "$CARGO_TARGET_DIR/debug/canopy" +``` + +The harness creates its own RustFS volume and bucket with test credentials, +runs the eight Git/SSH/LFS provider probes and the process smoke below, then +removes the container and volume, including on failure. + The integration tests need `git` and `git-lfs` on `PATH`. See [delivery plan](delivery-plan.md) for the remaining release gates and [contracts](contracts.md) for persisted identities and storage rules. @@ -232,7 +274,7 @@ Optional flags expand the qualification: | --- | --- | | `--large-clone` | Push two 80 MiB random blobs, then clone with protocol v0 and v2 after takeover. Each pack exceeds 64 MiB; both file hashes and `git fsck` must match. This checks transfer size, not production capacity. | | `--many-objects 256` | Push 256 files, update one file with an annotated tag, then clone after takeover. Combined with `--large-clone`, exercise resident eviction across four repositories and Git/LFS recovery before restart. | -| `--sqlite-chunks` | Push a 32,000-entry tree and commit/tag messages above 1 MiB; verify raw bytes and OIDs after takeover with strict `git fsck`. | +| `--structural-objects` | Structural-object gate; push a 32,000-entry tree and commit/tag messages above 1 MiB; verify raw bytes and OIDs after takeover with strict `git fsck`. | | `--corpus-repository /path/to/existing/repository` | Read that checkout's HEAD history, bundle it into temporary fixtures under `--work-parent`, then compare every reachable object's type, size and bytes in v0/v2 clones after takeover, with strict `git fsck`. Other source branches and tags are outside this check. | For a local container store, verify that the intended data/log volume is shared @@ -247,20 +289,20 @@ To measure cold recovery, set `RUST_LOG=warn,canopy_server::git_gateway=debug,canopy_server::server::residency=debug`. The node reports Cell acquisition time and cache hydration time, with object count, raw/cache bytes and time spent in page reads, body retrieval and cache -writes. Page time includes inline integrity verification; cache time includes -worker scheduling, OID verification, compression and admitted disk writes. +writes. Page time includes metadata decoding; body time includes verified pair +retrieval, and cache time includes native pair installation and admitted writes. Use release builds for performance measurements. The chunk, default-branch, repository-discovery, token-metadata, collaboration, visibility and object insertion-sequence layouts change the unreleased schema; use a fresh development storage prefix when moving -from older builds. No upgrade migration or mixed-build rolling upgrade is supported yet. +from older builds. Upgrade migrations and mixed-build rolling upgrades are out of scope. Do not reuse an existing development prefix with this changed initialization schema. ## Contributing Keep behavior, tests, and documentation aligned when you change Canopy: -1. Update the relevant [persisted contract](contracts.md). +1. Follow the [Cellule integration policy](../AGENTS.md) and update the relevant [persisted contract](contracts.md). 2. Add or update a stock-client, API, or fault test. 3. Record the revision, provider, workload, and hardware for measurements. 4. Update the matching compatibility row or [delivery gate](delivery-plan.md). @@ -269,3 +311,18 @@ Keep behavior, tests, and documentation aligned when you change Canopy: ## License Canopy declares the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) in `Cargo.toml`. + +### Bounded graph reads and covering types + +Repository schema 3 adds `objects_by_oid_kind` for immutable kind lookups without +reading wide object rows. Certification checks dependency kind and closure through +that covering index, retaining its 128-object / 64 MiB publication batches. The +metadata write batch remains 512 objects and dependency writes remain 8,192 rows. + +Fetch graph traversal selects at most 64 parents and 128 edge rows per query. It +seeks the current parent's child key directly, excludes completed parents, and +limits edges before the kind lookup. Blob filtering happens after that bounded +read, so a tree containing mostly blobs does not require an unbounded SQL scan to +fill a structural page. Both Git hash formats have dense-parent pagination and +covering-query regression checks. This schema initializes a fresh prefix only; +there is no index upgrade or old-schema reader. diff --git a/docs/operations.md b/docs/operations.md index 7cb59efd..dc8c475a 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -63,8 +63,8 @@ takeover. Cross-node placement races and larger hot sets still need qualificatio Each storage prefix has one durable tenant/application identity and a selected compiled release. First startup initializes an empty deployment; later nodes must match its release and configured image identity. A nonempty catalog without -release metadata is rejected. Existing preview prefixes require explicit future -migration; do not point this build at them as an upgrade. +release metadata is rejected. Incompatible releases use fresh storage prefixes; old-format readers and +in-place migrations are intentionally unsupported. To close admission and drain the fleet, choose a fresh operation UUID and use the same binary and configuration as the deployment: @@ -113,8 +113,45 @@ failed root verification return an error. Retry the same operation after the reported condition is resolved; recovery never resumes serving automatically. Do not remove authority records or force `drained` to bypass an error. -Upgrade/migration and object collection remain pending. Maintenance and owner -recovery do not provide a separate backup copy. +Maintenance and owner recovery operate within the selected release; they do not +upgrade it or provide a separate backup copy. + +### Collect obsolete Cellule objects + +Begin maintenance, stop serving nodes cleanly, recover any interrupted owners, +and wait for `status` to report `drained: true`. Then run a bounded collection: + +```bash +canopy gc deploy/config.json "$CANOPY_OPERATION_UUID" 86400000 100000 +``` + +The final arguments are a grace window in milliseconds and the maximum number +of deletes in this pass (1–100,000). The example retains objects modified in the +last 24 hours. The JSON receipt reports live/pinned, grace-protected, eligible and +deleted counts. Repeat while `complete` is false. Finally end maintenance with +the same operation UUID. + +Collection uses Cellule's verified live-root and backup-pin traversal. It deletes +only old, unreachable immutable Cellule objects; native Git packs and Git LFS content have a +separate reference set. Missing or corrupt reachable data prevents the sweep. +Expired node advertisements do not count as a completed drain. + +Plan provider capacity for the history accumulated between collection windows, +including import and compaction publications. The three-repository native-pack +trial occupied 15.19 GiB before collection and 0.989 GiB afterward. A small live +database does not bound that temporary remote history. Retain an operational +grace window and all required backup pins; the benchmark's short grace applies +only to its isolated, fully drained fixture. See the +[storage qualification](benchmarks/2026-10-14-storage-optimization.md). + +The worker requires provider credentials, the node signing key, and an exclusive +local `data_dir`. A persistent claim prevents recovery or reopening admission +while deletion is in progress. After a killed worker, repeat `gc` with the same +operation, node identity, signing key and local directory: its workspace lock +proves the original local process has closed before adopting the claim. A grace +window or expired lease never clears that claim. Caller cancellation leaves the +worker responsible for finishing and releasing exclusion. Collection leaves +the deployment in maintenance until an explicit `end`. ### Backup and restore @@ -147,7 +184,7 @@ head and Cell control twice and rejects concurrent changes. On busy deployments, stop nodes cleanly without entering Maintenance, then capture. Retry an uncertain operation with the same UUID and destination; a new snapshot needs a new UUID. -The copy includes runtime roots, SQLite state, external Git blobs and LFS bodies. +The copy includes runtime roots, SQLite state, native Git pack/index pairs and LFS bodies. Verification reads only the backup prefix, so the original prefix may be lost. Restore preserves the pinned identity and release and needs matching configuration and binary. Point a fresh node's `storage_url` at the completed destination. @@ -157,8 +194,8 @@ failed operation after repairing its reported cause. Replaying a completed resto does not reset subsequently published service state. This is a same-provider copy, not protection from losing the entire bucket or -provider. Cross-provider export, old-release migration, automated retention and -the complete interruption fault matrix remain pending. Do not use older binaries +provider. Cross-provider export, automated retention and the complete interruption fault +matrix remain pending. Old-release migration is out of scope. Do not use older binaries that lack prefix reservations with these backup/restore destinations. ### Recover a lost push reply diff --git a/docs/performance-plan.md b/docs/performance-plan.md index 22472965..c59cfd94 100644 --- a/docs/performance-plan.md +++ b/docs/performance-plan.md @@ -1,6 +1,6 @@ # Measure repository density and latency -Use this plan to design capacity work and interpret Canopy benchmark results. It separates demonstrated behavior from proposed targets. The current Cellule dependency is `a3fbfb0115a1ae2519ee8f8e0cf6b8e72fdaa303`; earlier runs below retain their original pins and do not establish this build's density or latency. +Use this plan to design capacity work and interpret Canopy benchmark results. It separates demonstrated behavior from proposed targets. The current Cellule dependency is exact crates.io version `=0.2.0` (release source `aaacf311622a2fea009067d1f5f217751ee1a2cf`); earlier runs below retain their original pins and do not establish this build's density or latency. > **Document type:** How-to and evidence reference. **Goal:** design a repeatable workload, record its resource envelope, and avoid turning one measurement into a general capacity claim. @@ -34,8 +34,8 @@ update rate. The local SQL-only RustFS gate now covers 1,000 active Cells, but repeat it on the reference Linux node under foreground load before raising the active Cell limit or claiming 1,000- or 10,000-Cell production residency. This dependency and lockfile change also changes Canopy's compiled release digest; -test against a fresh store prefix or use the documented maintenance upgrade -path for an existing deployment. +test against a fresh store prefix. Maintenance does not upgrade an existing +deployment to a different compiled release. ## Recorded large-transfer qualification @@ -1943,3 +1943,244 @@ Focused smart-HTTP, default-branch recovery, visibility/revocation, paused-histo discovery and fetchable merge-candidate tests passed, along with all-target Clippy, formatting and the release build. Full-primitive density, renewal fairness and production latency gates remain open. + +## RustFS API benchmark — 2026-10-13 + +An optimized Canopy build using Cellule `=0.2.0` imported three public repositories into a fresh RustFS-backed deployment. Full clone-back, `git fsck --full`, equality of every reachable object ID, and API comparisons against stock Git passed. All 12 first reads from fresh local workspaces also returned exact expected responses. No runtime code was changed or tuned for this benchmark. + +[Machine-readable results](benchmarks/2026-10-13-rustfs-browse.json) retain revisions, binary and harness hashes, workload details, both latency populations, errors, and all per-repository/concurrency results. The local raw samples, executed harness snapshot, logs, and private operating instructions are under `~/.local/share/canopy-benchmark-20261013/`. + +### Environment and method + +- Apple M2 Max, 12 logical CPUs, 32 GiB RAM, macOS arm64; shared workstation rather than a dedicated capacity runner. Canopy and its local workspace ran on the internal disk. Rust 1.97.0, Git 2.50.1, release build. +- RustFS `1.0.0-beta.8-glibc` in a dedicated Docker volume, limited to 4 CPUs and 8 GiB RAM. The Docker VM had 8 CPUs and about 15.6 GiB RAM. +- Authenticated HTTP/1.1 over loopback, persistent per-client connections, one account, no TLS or reverse proxy. Canopy allowed eight resident repositories. +- Two eight-second passes per repository/API/concurrency combination, plus a mixed workload. Concurrency levels were 1, 4, and 16; 78 timed groups ran in a fixed shuffled order. The mixed workload rotates equally over each repository/API pair. +- Latency runs from request start through receipt of the complete response; response checksum validation follows that timer. Throughput counts only validated successful responses over actual elapsed time, including the final in-flight requests. Warmup requests are excluded. Combined percentiles are recalculated from individual samples, not averaged from per-pass percentiles. +- This is a closed-loop workload over a small, verified working set. It measures achieved throughput and latency under that workload, not open-loop arrival capacity, arbitrary-path cold-cache performance, or an internet-facing SLO. + +### Initial imports and clone-back + +All branch and tag histories were imported with one atomic stock-Git push per repository. GitHub pull-request refs were excluded. Downloading the source repositories from GitHub is outside these timings. Import time ends only when the push client receives successful durable publication; clone time excludes the subsequent fsck. + +| Repository | Packed source MiB | Commits | Objects | Import seconds | Objects/sec | First clone-back seconds | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| bat | 34.56 | 4,068 | 20,952 | 28.92 | 724.5 | 19.77 | +| starship | 46.27 | 4,556 | 38,333 | 80.00 | 479.2 | 48.21 | +| libgit2 | 72.12 | 19,565 | 134,967 | 813.62 | 165.9 | 214.88 | + +The packed-source-size/import-time ratios were 1.19, 0.58, and 0.089 MiB/sec respectively. These are effective source-pack rates, not measured network wire rates. During the slow libgit2 import, all 134,967 objects were already staged while graph certification continued and refs remained unpublished. Read-only progress probes recorded that phase; this does not isolate its CPU, SQLite, or storage costs. + +### Warm API reads + +All 462,198 requests at concurrency 1 and 4 succeeded and matched verified response hashes. The table reports four-client throughput and successful-response p50/p95/p99, combining both passes. Each row is a separate workload; the rates should not be added together. + +| Repository | API view | Successful req/sec | p50 ms | p95 ms | p99 ms | +| --- | --- | ---: | ---: | ---: | ---: | +| bat | refs | 2190.1 | 1.80 | 2.12 | 2.24 | +| bat | tree | 2332.3 | 1.64 | 2.45 | 3.06 | +| bat | blob | 1777.5 | 2.16 | 2.74 | 3.01 | +| bat | commits | 339.2 | 10.90 | 15.36 | 15.85 | +| starship | refs | 1665.9 | 2.36 | 2.67 | 2.86 | +| starship | tree | 1971.0 | 1.61 | 3.27 | 10.25 | +| starship | blob | 1656.8 | 2.24 | 3.36 | 4.29 | +| starship | commits | 363.3 | 10.69 | 12.92 | 16.02 | +| libgit2 | refs | 1169.4 | 3.33 | 4.09 | 4.76 | +| libgit2 | tree | 2224.0 | 1.77 | 2.48 | 2.88 | +| libgit2 | blob | 1745.0 | 2.23 | 2.84 | 3.28 | +| libgit2 | commits | 293.5 | 11.09 | 20.29 | 55.96 | + +At one client, p95 ranges across the repositories were 0.99–1.80 ms for refs, 0.99–1.19 ms for trees, 1.16–1.44 ms for blob previews, and 5.10–5.83 ms for commit-history pages. + +API views are `POST /api/repositories//browse` queries, with these measured units: + +- `refs`: the first page, containing 66, 164, or 256 refs. Correctness verification paginated every ref, including all 271 libgit2 refs. +- `tree`: a root or sampled parent directory, up to 32 entries per page. +- `blob`: `kind: file`, cycling over three exact-byte previews per repository, approximately 1 KiB, 16 KiB, and 126–150 KiB before Base64 encoding. This is not an unrestricted raw-blob endpoint. Separate probes confirmed that files above 256 KiB return `too_large` metadata without content. +- `commits`: `kind: history`, a page of 32 first-parent commits at offsets 0, 32, 128, or 512. Git author/committer headers, parents, trees, and message bytes were checked against the source. +- There is no attribution/blame API. `kind: attribution` was rejected with HTTP 422 for all three repositories. Commit author/committer metadata is available through history, but no line-attribution performance is claimed. + +### Mixed traffic and overload + +| Concurrent clients | Successful req/sec | Successful p95 ms | Successful p99 ms | Error rate | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 549.0 | 4.82 | 6.40 | 0.00% | +| 4 | 918.8 | 11.54 | 15.04 | 0.00% | +| 16 | 659.1 | 27.17 | 32.02 | 92.76% | + +Canopy admits eight active transfers per node and four per account, with four account waiters and a one-second wait limit. The 16-client single-account driver kept issuing work without honoring `Retry-After`; across those stress groups, 90.41% of attempts failed: 1,303,498 HTTP 503 responses (66.41%) and 471,230 transport failures (24.01%, broken pipes, resets, or disconnected connections). There were no mismatched successful bodies. These failures include connection instability during the overload run; its cause was not isolated. This is overload evidence; fast rejections are excluded from successful-read percentiles and successful throughput. Raw results also retain latency for all attempts. + +### First reads after recovery + +Each cell below is one observation after a graceful stop and restart with a new empty Canopy data directory. Timing begins after server readiness and includes on-demand repository activation plus the API read. RustFS and host/VM caches remained running; these are not provider-cold measurements or percentile estimates. Observed startup readiness was 0.53–1.07 seconds with 500 ms polling, separately from these read timings. + +| Repository | Refs ms | Tree ms | Blob preview ms | Commit history ms | +| --- | ---: | ---: | ---: | ---: | +| bat | 474.42 | 745.71 | 381.36 | 236.29 | +| starship | 352.64 | 409.12 | 396.08 | 499.10 | +| libgit2 | 975.22 | 991.52 | 859.99 | 1275.71 | + +### Storage and retained deployment + +The three packed sources total 152.94 MiB. Their SQLite files had apparent sizes of 181.80 MiB, 654.19 MiB, and 1,996.55 MiB after import. The RustFS `/data` tree occupied 20.98 GiB of apparent file bytes, including storage history and provider files; this is not a logical live-object size or a GC-compacted size. The Docker VM had only 2.55 GiB free afterward. No runtime history was manually removed. + +Two-second `ps` samples observed up to 40.4 MiB RSS for the Canopy parent during the warm tests. That excludes SQLite/Git child processes, RustFS, and OS page caches and is not a service-wide memory bound. Parent CPU reached 424.5% and the Python driver reached 201.1% (100% denotes one CPU); neither run was a dedicated-host capacity proof. + +At the time of this historical run, Canopy served `http://127.0.0.1:60523` with RustFS at `http://127.0.0.1:32793` and three ready private repositories. That original provider became unavailable after a later Docker runtime change; these addresses do not describe the current fixture. Credentials and start/stop instructions are stored locally in `~/.local/share/canopy-benchmark-20261013/OPERATING.txt` and `client.env`; credentials are not included in this report or checked into the repository. Inactive caches created by the recovery checks were removed, while the active workspace, source mirrors, verification clones, raw results, and durable RustFS volume were retained. + +For another read-only warm run against this exact imported corpus: + +```bash +CANOPY_BENCH_ROOT="$HOME/.local/share/canopy-benchmark-20261013" +source "$CANOPY_BENCH_ROOT/client.env" +mkdir "$CANOPY_BENCH_ROOT/results-repeat" +cp "$CANOPY_BENCH_ROOT/results/queries.json" "$CANOPY_BENCH_ROOT/results-repeat/" +python3 -B scripts/benchmark_browse.py measure \ + --base-url "$CANOPY_BASE_URL" \ + --corpus "$CANOPY_BENCH_ROOT/corpus.json" \ + --output "$CANOPY_BENCH_ROOT/results-repeat" +``` + +The release build, 19 Python harness tests, formatting, and diff checks passed. The harness regression exercises HTTP 503 and corrupt HTTP 200 responses and proves they are counted without retries. Rust runtime source and dependencies were unchanged by this benchmarking work; the preceding dependency qualification remains its functional test record. + +### Follow-up: why the provider retained 20.98 GiB + +A read-only scan of RustFS backing files measured this breakdown. Values include +RustFS metadata and part framing, use apparent bytes, and do not classify objects +as safe to delete. + +| Stored category | GiB | Immutable object keys | +| --- | ---: | ---: | +| Cellule LTX database page segments | 12.468 | 3,362 | +| Cellule page-directory objects | 7.357 | 351,653 | +| External Git blobs | 0.768 | — | +| Segment indexes | 0.320 | 3,362 | +| Published root records | 0.036 | 3,256 | +| Packed Cellule captures | 0.027 | 2,298 | +| Other bucket files | 0.002 | — | + +Two sources of amplification are visible: + +1. Canopy ingests expanded Git objects rather than preserving Git packfile delta + compression. The source packs total 152.94 MiB, but the object sizes recorded + in SQLite total 3.07 GiB. The current three SQLite databases plus external + blob payloads total 3.53 GiB. This includes object graph tables and indexes; + it is a current-state baseline, not a prediction of compacted remote size. +2. Cellule retains immutable page segments, directory revisions and root records + produced by publication and compaction. Compaction writes new representations; + its examined code does not delete replaced remote inputs. The deployed path + has no identified remote garbage-collection pass. The accumulated directory + objects alone occupy 7.36 GiB. Live-root reachability was not traversed, so the + exact reclaimable amount is not established. + +Each of the four Cells (three repositories and the directory) has only one +incarnation in the bucket. The fresh-local-workspace recovery checks reused that +remote data; the inventory is not twelve independent repository copies. The +RustFS system directory was below 1 MiB during the scan. + +This is approximately 140 times the packed source size and six times the current +SQLite-plus-blob baseline. It exposes a storage-efficiency gap in the combined +Canopy/Cellule implementation. The next engineering work should measure live-root +reachability, provide retention-aware reclamation in Cellule, reduce directory +rewrite amplification, and evaluate compressed Git-object storage in Canopy. +Recovery and fencing references must be accounted for before reclaiming objects. + +Raw evidence is retained in the local benchmark results directory as +`storage-files.txt` and `storage-breakdown.json`. This investigation changed no +runtime data or configuration. + +### Storage optimization implementation — 2026-10-13 + +The follow-up implementation found that the published Cellule 0.2.0 already +includes `recovery::retention::CellGarbageCollector`. The earlier investigation +had not identified that API. Canopy now exposes it through the maintenance CLI; +the original benchmark had never invoked it. Retained space therefore does not +establish that Cellule lacks a collector. + +The CLI refuses unsettled owners or node advertisements, retains backup pins, +honors a grace boundary, and bounds deletions per pass. A durable administrative +claim serializes collection, owner recovery, and reopening the release. A killed +collector can resume only through its original worker identity and locked local +workspace; there is no lease-timeout bypass for deletion in flight. The release +remains in maintenance after collection. + +Thirteen focused deployment tests pass, including grace preservation, collection +of an unreachable directory, rejection of expired advertisements, and recovery +of an interrupted collector claim. A dedicated RustFS process qualification +removed 177 obsolete immutable objects across bounded passes, then verified the +retained backup and an exact cold stock-Git clone with `git fsck --full`. +Raw receipts and logs are under `~/.local/share/canopy-storage-optimization/`. + +The original corpus required same-release recovery of four unsettled Cells +before collection. Its first bounded sweep removed obsolete objects before a +RustFS paginated listing failed with HTTP 500 (`Io error: timeout`). That first collection +was incomplete. After all collector work returned, maintenance was ended and the +exact original binary was restarted. The service-prefix backing files measured +17,349,826,791 apparent bytes (16.16 GiB), about 4.82 GiB below the original +provider measurement. All three repositories then passed new full clone/fsck, +object-ID equality and API comparisons. This verifies the partial collection; +it does not quantify the remaining reclaimable bytes. + +A separate Cellule worktree now completes disk-backed inventory before any +deletion and joins up to sixteen concurrent deletes within the existing bound. +Seven focused tests, the isolated RustFS retention test, Clippy and ownership/ +boundary checks passed. The serial baseline fails the new concurrency regression +with 1.7 seconds versus the required 200 milliseconds for seventeen delayed +deletes. This framework patch is not yet consumed by Canopy's published pins. + +The native-pack reader verified all 194,252 source object IDs directly from the +152.94 MiB corpus packs. Repository schema 2 now stores only metadata and pack +references; the old inline, chunked and external Git-body formats are removed. +Native packs are self-contained across incremental pushes. Structural and +blob pairs are disjoint, so discovery does not wait for blob downloads. +Selected objects use disposable, disk-accounted cache projections. Object and dependency metadata use bounded +multi-row writes; verified blob certificates are published in the same batch. +Merge/rebase checks use immutable typed edges and normalized commit fingerprints. +Backup/restore copies and verifies both native files. + +The repository scenario, owner-loss cold clone and Directory Cell tests passed +at intermediate checkpoints. The pack primitives pass SHA-1/SHA-256, delta +compression, immutable provider round-trip and cancellation ownership tests. +A new process-fence regression checks both private output and cache ancestors. +The completed clean native trial measured 16,313,052,025 apparent provider bytes +before GC and 1,062,134,213 bytes (0.989 GiB) afterward. Its original binary and +receipts are retained. The original schema-1 fixture's remaining collection also +completed on its archived release; it measured 2,937,040,978 bytes afterward, +and all three original repositories passed full clone/fsck, object-ID equality +and browse comparisons. GC accounts for most of the original 20.98 GiB; the +native representation is about 64% smaller than the collected original. + +The final build additionally bounds many-ref SQL checks and uses bounded +read-only seeks to warm both existing graph indexes before certification. +An exploratory smaller-write-batch fix prevented cold SQL timeouts but generated +too much immutable history during crash trials; it is not the final policy. +Current formatting, Clippy, dependency checks, ordinary tests across retained +broad/focused runs, all real-provider probes and the extended process profile +passed. Two parallel workspace-lock unit failures passed unchanged isolated +rechecks; the failure logs remain. The final medium crash/cold-retry and API +measurements are recorded in the [storage qualification](benchmarks/2026-10-14-storage-optimization.md). +The earlier index-warming checkpoint did not run the full 5 GiB qualification +because free capacity was unavailable. The subsequent transport-budget build +(`7b16ef3133c39ad84290459e0c256765bcd0acb0fd3e4ac04eb85e2ebd1a0ec3`) passed the +complete isolated size/provider/process qualification, including Git/LFS objects +larger than 5 GiB, over 512 MiB incompressible pack data, fresh-server clone/fsck, +source-deleted backup/restore and retained-pin GC. Its ordinary tests passed in +one clean serial run. Its fresh-prefix medium run retained a libgit2 certificate +SQL deadline failure; recovery/retry and final serving measurements remain +pending in the linked qualification. +A cold read currently downloads its referenced complete pack. Structural graph projection streams with bounded buffers. Direct whole-object +reads retain a 64 MiB allocation limit. The +historical measurements above retain their original storage representation. + +The transport-budget checkpoint's final collection deleted 276,750 obsolete +objects and retained 824 reachable objects. It measured 1,259,651,402 apparent +bytes (1.173 GiB) after 3,373.37 seconds of bounded collection. Bat and Starship +passed fresh full mirror/API checks; Libgit2 still hit the five-second Cellule +read deadline during graph hydration. Its serving qualification is incomplete. + +The subsequent schema-3 source adds a covering kind index for graph reads and +dependency certification, plus bounded keyset pages before blob filtering. It +retains the larger publication batches rather than multiplying immutable +publication history to avoid a read deadline. Dense-parent SHA-1/SHA-256 query +regressions, certificate-kind checks and Clippy passed. Ordinary tests and a +fresh-prefix size/provider/process and medium-corpus qualification are pending. diff --git a/docs/repository-cell-primitives.md b/docs/repository-cell-primitives.md index 167e8075..85e394c1 100644 --- a/docs/repository-cell-primitives.md +++ b/docs/repository-cell-primitives.md @@ -49,8 +49,8 @@ recovery lifecycle. Nodes must host thousands of these repository identities with bounded memory, disk, descriptors, workers and queued operations. All existing Git smart-HTTP, LFS, collaboration, ACL and recovery requirements -remain. Ordinary Git objects stay in SQLite; large bodies stay in immutable -object storage. A SQL-only density measurement does not qualify the complete +remain. Git object metadata stays in SQLite; all Git bodies stay in immutable +native pack/index pairs in object storage. A SQL-only density measurement does not qualify the complete primitive-enabled repository design. ## Current contract gap @@ -63,8 +63,10 @@ in `src/lib.rs`. `RepositoryCell` holds `SqlCell`. Its schema contains Git/collaboration state; no repository KV, queue or workflow capability is wired into the product. -The pinned Cellule revision is `a28de7bc09ce36d87e642adc4f4b6be50d6fcb69`. -Read-only inspection establishes the following constraints in that source: +The following source audit was recorded against Cellule revision +`a28de7bc09ce36d87e642adc4f4b6be50d6fcb69`. Canopy now pins exact crates.io +version `=0.2.0`; this historical audit does not qualify composed primitives +on the new release. The recorded constraints were: | Surface | Existing contract | Required change | | --- | --- | --- | diff --git a/scripts/benchmark_browse.py b/scripts/benchmark_browse.py new file mode 100644 index 00000000..7cf4256d --- /dev/null +++ b/scripts/benchmark_browse.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +"""Import real Git histories, verify browse APIs against Git, and measure reads. + +Uses CANOPY_GIT_TOKEN. Corpus entries need name, source (local bare repository), +branch and commit. Never mutates source repositories. Timed reads use closed-loop +clients; throughput is achieved throughput, not a scheduled-arrival capacity claim. +""" + +import argparse +import base64 +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import math +import os +from pathlib import Path +import random +import subprocess +import threading +import time + +from benchmark_repositories import Client, save + + +def encoded(value): + return base64.urlsafe_b64encode(value).decode().rstrip("=") + + +def decoded(value): + return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + + +def git(repo, *args): + return subprocess.check_output(["git", "-C", repo["source"], *args]) + + +def git_environment(token): + env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} + env.update(GIT_TERMINAL_PROMPT="0", GIT_CONFIG_NOSYSTEM="1", + GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_COUNT="2", + GIT_CONFIG_KEY_0="credential.helper", GIT_CONFIG_VALUE_0="", + GIT_CONFIG_KEY_1="http.extraHeader", GIT_CONFIG_VALUE_1=f"Authorization: Bearer {token}") + return env + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def post(client, repo, query): + payload = {"repository_id": repo["repository_id"], "query": query} + status, body = client.request(f"/api/repositories/{repo['name']}/browse", payload) + require(status == 200, f"{repo['name']} {query['kind']}: HTTP {status}: {body[:200]!r}") + return json.loads(body)["view"], body + + +def import_repositories(args, client, token): + corpus = json.loads(args.corpus.read_text()) + for repo in corpus["repositories"]: + require("repository_id" not in repo, "import requires repositories not previously imported") + status, body = client.request("/api/repositories", {"name": repo["name"]}) + require(status == 200, f"create {repo['name']}: HTTP {status}: {body[:200]!r}") + repo.update(json.loads(body)) + save(args.corpus, corpus) + env = git_environment(token) + started = time.monotonic() + result = subprocess.run(["git", "-C", repo["source"], "push", "--atomic", + repo["clone_url"], "refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"], + env=env, capture_output=True, timeout=3600) + seconds = time.monotonic() - started + (args.output / f"{repo['name']}-push.log").write_bytes(result.stdout + result.stderr) + repo["import"] = {"seconds": seconds, "exit_code": result.returncode, + "objects_per_second": repo["object_count"] / seconds, + "source_pack_mib_per_second": repo["pack_bytes"] / 1024**2 / seconds} + save(args.corpus, corpus) + require(result.returncode == 0, f"{repo['name']} import failed; see push log") + print(f"IMPORTED {repo['name']}: {seconds:.2f}s, {repo['object_count'] / seconds:.1f} objects/s", flush=True) + status, body = client.request(f"/api/repositories/{repo['name']}/default-branch") + require(status == 200, "default branch read failed") + current = json.loads(body) + # Client.request intentionally only supports GET/POST; use the same connection for PUT. + connection = client.connection() + connection.request("PUT", f"/api/repositories/{repo['name']}/default-branch", + json.dumps({"repository_id": repo["repository_id"], + "reference": f"refs/heads/{repo['branch']}", + "expected_generation": current["generation"]}), + {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}) + response = connection.getresponse() + response.read() + require(response.status == 200, "default branch update failed") + save(args.output / "imports.json", corpus) + + +def check_commit(repo, actual): + raw = git(repo, "cat-file", "commit", actual["oid"]) + headers, message = raw.split(b"\n\n", 1) + lines = headers.splitlines() + require(actual["tree_oid"] == lines[0].removeprefix(b"tree ").decode(), "commit tree mismatch") + parents = [] + for line in lines[1:]: + if not line.startswith(b"parent "): + break + parents.append(line[7:].decode()) + require(actual["parents"] == parents, "commit parents mismatch") + for field in ("author", "committer"): + original = next((line[len(field) + 1:] for line in lines if line.startswith(field.encode() + b" ")), None) + require(actual[f"{field}_base64"] == (encoded(original) if original is not None else None), + f"commit {field} attribution mismatch") + require(decoded(actual["message_base64"]) == message[:4096], "commit message mismatch") + require(actual["message_truncated"] == (len(message) > 4096), "commit truncation mismatch") + + +def tree_entries(repo, revision, path=b""): + target = revision + (":" + os.fsdecode(path) if path else "^{tree}") + result = [] + for record in git(repo, "ls-tree", "-z", target).split(b"\0"): + if record: + meta, name = record.split(b"\t", 1) + mode, kind, oid = meta.split() + result.append((name, mode.decode(), kind.decode(), oid.decode())) + return sorted(result) + + +def verify(args, client): + corpus = json.loads(args.corpus.read_text()) + queries, proofs = [], [] + for repo in corpus["repositories"]: + clone = args.output / "clones" / f"{repo['name']}.git" + clone.parent.mkdir(exist_ok=True) + started = time.monotonic() + cloned = subprocess.run(["git", "clone", "--mirror", repo["clone_url"], str(clone)], + env=git_environment(client.token), capture_output=True, timeout=1800) + clone_seconds = time.monotonic() - started + (args.output / f"{repo['name']}-clone.log").write_bytes(cloned.stdout + cloned.stderr) + require(cloned.returncode == 0, "clone-back failed") + checked = subprocess.run(["git", "-C", str(clone), "fsck", "--full"], capture_output=True, timeout=600) + (args.output / f"{repo['name']}-fsck.log").write_bytes(checked.stdout + checked.stderr) + require(checked.returncode == 0, "clone-back fsck failed") + original_objects = set(git(repo, "rev-list", "--objects", "--all", "--no-object-names").splitlines()) + restored_objects = set(subprocess.check_output(["git", "-C", str(clone), "rev-list", "--objects", "--all", "--no-object-names"]).splitlines()) + require(original_objects == restored_objects, "clone-back object graph differs") + expected_refs = dict(line.split() for line in git(repo, "for-each-ref", "--format=%(refname) %(objectname)", + "refs/heads", "refs/tags").decode().splitlines()) + seen, query = {}, {"kind": "refs"} + while True: + view, _ = post(client, repo, query) + page = view["refs"] + seen.update((entry["name"], entry["oid"]) for entry in page["entries"]) + if page["next_after"] is None: + break + query = {"kind": "refs", "after": page["next_after"], "generation": page["generation"]} + require(seen == expected_refs, f"{repo['name']}: imported refs mismatch") + variants = [("refs", {"kind": "refs"})] + files = [] + for record in git(repo, "ls-tree", "-rlz", repo["commit"]).split(b"\0"): + if not record: + continue + meta, path = record.split(b"\t", 1) + mode, kind, oid, size = meta.split() + if kind == b"blob" and mode == b"100644" and 0 < int(size) <= 256 * 1024: + files.append((int(size), path, oid.decode())) + selected = [] + for target in (1024, 16 * 1024, 128 * 1024): + candidate = min(files, key=lambda item: abs(item[0] - target)) + if candidate not in selected: + selected.append(candidate) + directories = {b"", *(path.rpartition(b"/")[0] for _, path, _ in selected)} + for path in sorted(directories): + query = {"kind": "tree", "commit": repo["commit"], "path_base64": encoded(path)} + view, _ = post(client, repo, query) + actual = view["tree"] + check_commit(repo, actual["commit"]) + expected = tree_entries(repo, repo["commit"], path) + require([(decoded(entry["name_base64"]), entry["mode"], entry["oid"]) for entry in actual["entries"]] + == [(name, mode, oid) for name, mode, _, oid in expected[:32]], "tree page mismatch") + require(actual["next_after"] == (encoded(expected[31][0]) if len(expected) > 32 else None), "tree cursor mismatch") + variants.append(("tree", query)) + for size, path, oid in selected: + query = {"kind": "file", "commit": repo["commit"], "path_base64": encoded(path)} + view, _ = post(client, repo, query) + actual = view["file"] + require(actual["content_status"] == "included" and actual["size"] == size and actual["oid"] == oid, + "file metadata mismatch") + require(decoded(actual["content_base64"]) == git(repo, "cat-file", "blob", oid), "blob bytes mismatch") + variants.append(("blob", query)) + history = git(repo, "rev-list", "--first-parent", "-n", "545", repo["commit"]).decode().splitlines() + for offset in (0, 32, 128, 512): + if offset >= len(history): + continue + query = {"kind": "history", "commit": history[offset]} + view, _ = post(client, repo, query) + actual = view["history"] + expected = history[offset:offset + 32] + require([commit["oid"] for commit in actual["commits"]] == expected, "first-parent history mismatch") + for commit in actual["commits"]: + check_commit(repo, commit) + require(actual["next_commit"] == (history[offset + 32] if len(history) > offset + 32 else None), "history cursor mismatch") + variants.append(("commits", query)) + status, _ = client.request(f"/api/repositories/{repo['name']}/browse", + {"repository_id": repo["repository_id"], "query": {"kind": "attribution"}}) + require(status == 422, "unexpected attribution support; update benchmark coverage") + for operation, query in variants: + _, body = post(client, repo, query) + queries.append({"repository": repo["name"], "operation": operation, + "path": f"/api/repositories/{repo['name']}/browse", + "payload": {"repository_id": repo["repository_id"], "query": query}, + "response_sha256": hashlib.sha256(body).hexdigest(), "response_bytes": len(body)}) + proofs.append({"repository": repo["name"], "refs_verified": len(seen), + "clone_seconds": clone_seconds, "clone_fsck_passed": True, + "clone_objects_verified": len(restored_objects), + "commit_metadata_and_attribution_verified": sum(min(32, len(history) - offset) for offset in (0, 32, 128, 512) if offset < len(history)), + "files": [{"path": os.fsdecode(path), "bytes": size, "oid": oid} for size, path, oid in selected], + "attribution_query_status": status}) + print(f"VERIFIED {repo['name']}: {len(seen)} refs, trees, exact blob bytes, history and author/committer metadata", flush=True) + save(args.output / "queries.json", {"queries": queries}) + save(args.output / "verification.json", {"repositories": proofs, "attribution_api": "not implemented; browse kind rejected with 422"}) + + +def distribution(values): + values = sorted(values) + return {**{f"p{p}": values[max(0, math.ceil(len(values) * p / 100) - 1)] for p in (50, 95, 99)}, + "min": values[0], "max": values[-1], "mean": sum(values) / len(values)} if values else None + + +def measure(args, token): + queries = json.loads((args.output / "queries.json").read_text())["queries"] + groups = {} + for query in queries: + groups.setdefault((query["repository"], query["operation"]), []).append(query) + if len(groups) > 1: + groups[("all", "mixed")] = [variants[min(1, len(variants) - 1)] if operation == "blob" else variants[0] + for (_, operation), variants in list(groups.items())] + plan = [(key, concurrency, repeat) for key in groups for concurrency in args.concurrency for repeat in range(args.repeats)] + random.Random(42).shuffle(plan) + report = {"method": "closed-loop, persistent HTTP/1.1 connections, full response read, no retries", + "mixed_workload": "equal rotation over one query for each repository/API pair; medium sampled blob", + "duration_per_group_seconds": args.duration, "concurrency": args.concurrency, + "repeats": args.repeats, "results": []} + samples_path = args.output / "samples.jsonl" + require(not samples_path.exists(), "measurement output already exists") + with samples_path.open("w") as samples: + for (repo, operation), concurrency, repeat in plan: + group_unix_seconds = time.time() + client = Client(args.base_url, token, 130) + variants = groups[(repo, operation)] + # Establish and validate the entire working set before timed requests. + for variant in variants: + status, body = client.request(variant["path"], variant["payload"]) + require(status == 200 and hashlib.sha256(body).hexdigest() == variant["response_sha256"], "warmup validation failed") + gate = threading.Barrier(concurrency + 1) + started = [0.0] + + def worker(index): + observations = [] + gate.wait() + sequence = index + while time.monotonic() < started[0] + args.duration: + variant_index = sequence % len(variants) + variant = variants[variant_index] + before = time.monotonic() + try: + status, body = client.request(variant["path"], variant["payload"]) + elapsed = (time.monotonic() - before) * 1000 + outcome = "ok" if status == 200 and hashlib.sha256(body).hexdigest() == variant["response_sha256"] else f"http_{status}_or_invalid_body" + size = len(body) + except Exception as error: + elapsed = (time.monotonic() - before) * 1000 + status, size, outcome = 0, 0, type(error).__name__ + observations.append({"latency_ms": elapsed, "status": status, "bytes": size, + "outcome": outcome, "variant": variant_index}) + sequence += 1 + return observations + + try: + with ThreadPoolExecutor(max_workers=concurrency) as pool: + tasks = [pool.submit(worker, index) for index in range(concurrency)] + started[0] = time.monotonic() + gate.wait() + observations = [sample for task in tasks for sample in task.result()] + elapsed = time.monotonic() - started[0] + finally: + client.close() + identity = {"repository": repo, "operation": operation, "concurrency": concurrency, "repeat": repeat + 1} + outcomes = Counter(sample["outcome"] for sample in observations) + result = {**identity, "unix_seconds": group_unix_seconds, + "requests": len(observations), "outcomes": dict(outcomes), "elapsed_seconds": elapsed, + "successful_requests_per_second": outcomes["ok"] / elapsed, + "response_mib_per_second": sum(sample["bytes"] for sample in observations if sample["outcome"] == "ok") / 1024**2 / elapsed, + "latency_ms_all_attempts": distribution([sample["latency_ms"] for sample in observations]), + "latency_ms_successful": distribution([sample["latency_ms"] for sample in observations if sample["outcome"] == "ok"])} + for sample in observations: + samples.write(json.dumps({**identity, **sample}) + "\n") + samples.flush() + report["results"].append(result) + save(args.output / "measurements.json", report) + successful = result["latency_ms_successful"] + successful_p95 = f"{successful['p95']:.1f}" if successful else "n/a" + print(f"MEASURED {repo}/{operation} c={concurrency} repeat={repeat + 1}: {result['successful_requests_per_second']:.1f} successful req/s, p95(success)={successful_p95} ms, p95(all)={result['latency_ms_all_attempts']['p95']:.1f} ms, errors={len(observations) - outcomes['ok']}", flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("import", "verify", "measure")) + parser.add_argument("--base-url", required=True) + parser.add_argument("--corpus", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--duration", type=float, default=8) + parser.add_argument("--concurrency", type=int, nargs="+", default=[1, 4, 16]) + parser.add_argument("--repeats", type=int, default=2) + args = parser.parse_args() + require(args.duration > 0 and args.repeats > 0 and all(1 <= n <= 64 for n in args.concurrency), "invalid workload") + args.output.mkdir(parents=True, exist_ok=True) + token = os.environ["CANOPY_GIT_TOKEN"] + if args.action == "measure": + measure(args, token) + return + client = Client(args.base_url, token, 130) + try: + if args.action == "import": + import_repositories(args, client, token) + else: + verify(args, client) + finally: + client.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/check_cellule.py b/scripts/check_cellule.py new file mode 100644 index 00000000..75a19111 --- /dev/null +++ b/scripts/check_cellule.py @@ -0,0 +1,54 @@ +"""Require one exact crates.io Cellule release, including transitive crates.""" + +from pathlib import Path +import re +import tomllib + + +def check(manifest, lock): + pins = {} + + def dependencies(table): + for key, value in table.items(): + if key in ("dependencies", "dev-dependencies", "build-dependencies"): + for alias, dependency in value.items(): + spec = {"version": dependency} if isinstance(dependency, str) else dependency + name = spec.get("package", alias) + if not name.startswith("cellule-"): + continue + version = spec.get("version", "") + if (not re.fullmatch(r"=\d+\.\d+\.\d+", version) + or any(field in spec for field in ("git", "path", "registry", "workspace"))): + raise ValueError(f"{name} must pin an exact stable crates.io release") + pins.setdefault(name, set()).add(version[1:]) + elif isinstance(value, dict): + dependencies(value) + + dependencies(manifest) + versions = {version for values in pins.values() for version in values} + if len(versions) != 1: + raise ValueError("all direct Cellule dependencies must pin the same release") + version = versions.pop() + locked = {} + for package in lock["package"]: + name = package["name"] + if not name.startswith("cellule-"): + continue + if (name in locked or package["version"] != version + or package.get("source") != "registry+https://github.com/rust-lang/crates.io-index" + or not package.get("checksum")): + raise ValueError(f"{name} must resolve once to crates.io {version}") + locked[name] = package + if missing := pins.keys() - locked.keys(): + raise ValueError(f"Cellule dependencies missing from Cargo.lock: {sorted(missing)}") + return version, len(locked) + + +if __name__ == "__main__": + root = Path(__file__).resolve().parents[1] + try: + with (root / "Cargo.toml").open("rb") as manifest, (root / "Cargo.lock").open("rb") as lock: + version, count = check(tomllib.load(manifest), tomllib.load(lock)) + except ValueError as error: + raise SystemExit(f"Cellule dependency policy: {error}") from error + print(f"Cellule {version}: {count} crates locked to one exact registry release") diff --git a/scripts/qualify_size.py b/scripts/qualify_size.py index 8f5b8113..fa7a900f 100644 --- a/scripts/qualify_size.py +++ b/scripts/qualify_size.py @@ -4,6 +4,7 @@ from pathlib import Path import shutil import subprocess +import sys import tempfile import time import uuid @@ -38,10 +39,19 @@ def main(): parser.add_argument("--provider-only", action="store_true") parser.add_argument("--docker-volume", action="store_true") parser.add_argument("--release", action="store_true") + parser.add_argument("--process-binary", type=Path, + help="Also exercise this Canopy binary through restart, takeover, peers and backup") args = parser.parse_args() + if args.process_binary is not None: + args.process_binary = args.process_binary.resolve(strict=True) name = f"canopy-size-{uuid.uuid4().hex[:12]}" with tempfile.TemporaryDirectory(prefix=name) as temporary: directory = Path(temporary).resolve() + process_binary = directory / "canopy" + if args.process_binary is not None: + # Every restart must execute the same build, even if Cargo replaces + # its output while the fixture is running. + shutil.copy2(args.process_binary, process_binary) image = "rustfs/rustfs:1.0.0-beta.8-glibc" owner = f"{os.getuid()}:{os.getgid()}" volume = None @@ -102,7 +112,7 @@ def main(): env["CANOPY_TEST_S3_ENDPOINT"] = endpoint env["CANOPY_TEST_S3_BUCKET"] = "canopy-size" tests = [] if args.provider_only else [ - "size::push_and_database_exceed_512_mib_and_lfs_exceeds_5_gib_after_restore" + "size::native_pack_exceeds_512_mib_and_git_lfs_exceed_5_gib_after_restore" ] tests.extend([ "sha256::sha256_real_provider_round_trip", @@ -126,6 +136,15 @@ def main(): subprocess.run([*cargo, test, "--", "--exact", "--ignored", "--nocapture"], env=env, check=True, cwd=Path(__file__).resolve().parents[1]) + if args.process_binary is not None: + process_env = {**env, "AWS_ENDPOINT": endpoint, "AWS_ALLOW_HTTP": "true", + "CANOPY_NODE_SIGNING_KEY_HEX": os.urandom(32).hex()} + subprocess.run([ + sys.executable, "-B", str(Path(__file__).with_name("smoke_s3_process.py")), + "--binary", str(process_binary), + "--storage-url", "s3://canopy-size/process-qualification", + "--work-parent", str(directory), + ], env=process_env, check=True) except Exception: if started: subprocess.run(["docker", "logs", "--tail", "60", name]) diff --git a/scripts/smoke_s3_activation.py b/scripts/smoke_s3_activation.py index ec14dc67..18560319 100644 --- a/scripts/smoke_s3_activation.py +++ b/scripts/smoke_s3_activation.py @@ -12,7 +12,7 @@ import uuid from benchmark_repositories import Client, corpus, percentiles, seed, verify -from smoke_s3_process import start +from smoke_s3_process import start, wait_for_lease_expiry def qualify(args): @@ -42,7 +42,7 @@ def qualify(args): client.close() process.kill() process.wait(timeout=10) - time.sleep(11) + wait_for_lease_expiry() report["phase"] = "recovery_startup" started = time.monotonic() process, base = start(args.binary, args.work_dir, settings, "restored") diff --git a/scripts/smoke_s3_backup.py b/scripts/smoke_s3_backup.py index c79cf9a9..07a475cd 100644 --- a/scripts/smoke_s3_backup.py +++ b/scripts/smoke_s3_backup.py @@ -70,7 +70,8 @@ def administer(action, *roots, fails=False): return json.loads(result.stdout) saved = administer("create", backup_prefix) - assert saved["cells"] == 2 and saved["external_objects"] == 3 + # Structural/blob pack pairs plus the nonempty and empty LFS bodies. + assert saved["cells"] == 2 and saved["external_objects"] == 4, saved assert administer("create", backup_prefix) == saved aws = ["aws"] if os.environ.get("AWS_ENDPOINT"): @@ -96,7 +97,7 @@ def administer(action, *roots, fails=False): assert json.loads(listing.stdout).get("KeyCount", 0) == 0 assert administer("verify", backup_prefix) == saved restored = administer("restore", backup_prefix, restore_prefix) - assert restored["cells"] == 2 and restored["external_objects"] == 3 + assert restored["cells"] == 2 and restored["external_objects"] == 4, restored node, restored_base = start(binary, directory, {**settings, "storage_url": restored_url, "node_id": str(uuid.uuid4())}, "backup-new-node") processes.append(node) diff --git a/scripts/smoke_s3_cache.py b/scripts/smoke_s3_cache.py index d9315d58..303e6102 100644 --- a/scripts/smoke_s3_cache.py +++ b/scripts/smoke_s3_cache.py @@ -11,7 +11,7 @@ import urllib.request import uuid -from smoke_s3_process import create_repository, git, start +from smoke_s3_process import create_repository, git, start, wait_for_lease_expiry AUTH = "http.extraHeader=Authorization: Bearer local-test-token" @@ -160,7 +160,7 @@ def clone(remote, name, protocol): print("PASS: stock pushes/clones reuse 260 object files and scan/hydrate only three new objects", flush=True) process.kill() process.wait(timeout=10) - time.sleep(11) + wait_for_lease_expiry() process, restored = start(args.binary, args.work_dir, settings, "restored") processes.append(process) restored_url = url.replace(base, restored, 1) diff --git a/scripts/smoke_s3_peers.py b/scripts/smoke_s3_peers.py index 7b1976f3..4d8fd020 100644 --- a/scripts/smoke_s3_peers.py +++ b/scripts/smoke_s3_peers.py @@ -9,7 +9,6 @@ import ssl import subprocess import threading -import time import uuid @@ -53,7 +52,7 @@ def do_POST(self): def qualify(binary, directory, settings, processes): - from smoke_s3_process import port, start, create_repository, git, clone_and_verify, api_status, api_get, verify_discovery + from smoke_s3_process import port, start, create_repository, git, clone_and_verify, api_status, api_get, verify_discovery, wait_for_lease_expiry ca, ca_key = directory / "peer-ca.pem", directory / "peer-ca-key.pem" certificate, key = directory / "peer-server.pem", directory / "peer-key.pem" @@ -119,7 +118,7 @@ def qualify(binary, directory, settings, processes): first.kill() first.wait(timeout=10) assert api_status(base_b, "/api/repositories", "local-test-token") == 503 - time.sleep(11) + wait_for_lease_expiry() for name, oid, readme, lfs in expected: clone_and_verify(f"{base_b}/canopy/{name}.git", directory / f"{name}-takeover", oid, readme, lfs) @@ -172,7 +171,7 @@ def maintenance(action, operation_id=None): pass else: raise AssertionError("maintenance resumed before crashed-owner recovery") - time.sleep(11) # Node lease is 10 seconds; expiry still requires a fencing CAS. + wait_for_lease_expiry() recovered = maintenance("recover", interrupted_operation) assert recovered["drained"] and recovered["release"]["state"] == "maintenance" assert maintenance("end", interrupted_operation)["release"]["state"] == "ready" diff --git a/scripts/smoke_s3_process.py b/scripts/smoke_s3_process.py index ae91e0cd..72790195 100644 --- a/scripts/smoke_s3_process.py +++ b/scripts/smoke_s3_process.py @@ -25,6 +25,7 @@ import smoke_s3_account_audit import smoke_s3_backup +import smoke_s3_retention import smoke_s3_branch_rules import smoke_s3_checks import smoke_s3_corpus @@ -37,6 +38,12 @@ import smoke_s3_peers +def wait_for_lease_expiry(): + # src/server.rs signs a 30-second node lease. Expiry permits fencing; it + # does not itself transfer ownership or prove recovery succeeded. + time.sleep(31) + + def port(): with socket.socket() as listener: listener.bind(("127.0.0.1", 0)) @@ -485,8 +492,8 @@ def verify_many_objects(base_url, directory, count, expected): print(f"PASS: {count}-file clone restored both commits, annotated tag and 300 refs after takeover in {time.monotonic() - started:.2f}s", flush=True) -def seed_sqlite_chunks(base_url, directory): - url, _ = create_repository(base_url, "sqlite-chunks") +def seed_structural_objects(base_url, directory): + url, _ = create_repository(base_url, "structural-objects") local = directory / "chunk-source" git("init", "--bare", str(local)) expected = {} @@ -517,15 +524,15 @@ def write(kind, body): started = time.monotonic() git("-c", "http.extraHeader=Authorization: Bearer local-test-token", "push", url, "refs/heads/main", "refs/tags/release", cwd=local) - print(f"PASS: large tree, commit and tag pushed into SQLite chunks in {time.monotonic() - started:.2f}s", flush=True) + print(f"PASS: large tree, commit and tag pushed into native packs in {time.monotonic() - started:.2f}s", flush=True) return expected -def verify_sqlite_chunks(base_url, directory, expected): +def verify_structural_objects(base_url, directory, expected): clone = directory / "chunk-restored" started = time.monotonic() git("-c", "http.extraHeader=Authorization: Bearer local-test-token", - "clone", "--bare", f"{base_url}/canopy/sqlite-chunks.git", str(clone)) + "clone", "--bare", f"{base_url}/canopy/structural-objects.git", str(clone)) for kind, (oid, path) in expected.items(): # run()/git() strip whitespace for command output. Compare raw object bytes here. actual = subprocess.run(["git", "cat-file", kind, oid], cwd=clone, @@ -534,7 +541,7 @@ def verify_sqlite_chunks(base_url, directory, expected): assert git("rev-parse", "refs/heads/main", cwd=clone).decode() == expected["commit"][0] assert git("rev-parse", "refs/tags/release", cwd=clone).decode() == expected["tag"][0] git("fsck", "--strict", "--full", cwd=clone) - print(f"PASS: SQLite chunks restored exact tree/commit/tag bytes and OIDs after takeover in {time.monotonic() - started:.2f}s", flush=True) + print(f"PASS: Native packs restored exact tree/commit/tag bytes and OIDs after takeover in {time.monotonic() - started:.2f}s", flush=True) def main(): @@ -544,7 +551,7 @@ def main(): parser.add_argument("--work-parent", type=Path, required=True) parser.add_argument("--large-clone", action="store_true", help="Qualify a push and v0/v2 clones above 64 MiB, including takeover") parser.add_argument("--many-objects", type=int, default=0, metavar="COUNT", help="Qualify many small objects, an incremental push, and takeover recovery") - parser.add_argument("--sqlite-chunks", action="store_true", help="Qualify large tree, commit and tag objects stored in SQLite and restored after takeover") + parser.add_argument("--structural-objects", action="store_true", help="Qualify large tree, commit and tag objects stored in native packs and restored after takeover") parser.add_argument("--corpus-repository", type=Path, help="Read HEAD history from an existing repository without changing it; qualify exact object bytes after takeover") args = parser.parse_args() if args.many_objects < 0: @@ -701,7 +708,7 @@ def main(): expiring_token = seed_expiring_token(base_url) disabled_tokens = seed_disabled_account(base_url) audit_state = smoke_s3_account_audit.seed(base_url) - chunks = seed_sqlite_chunks(base_url, directory) if args.sqlite_chunks else None + chunks = seed_structural_objects(base_url, directory) if args.structural_objects else None large = seed_large_repository(base_url, directory) if args.large_clone else None many = seed_many_objects(base_url, directory, args.many_objects) if args.many_objects else None corpus = None @@ -746,7 +753,7 @@ def main(): # SQLite is a recovered cache: corrupting the killed node's copy must # not prevent restoration of acknowledged data from object storage. (abandoned / "directory.sqlite").write_bytes(b"discard this stale database") - time.sleep(11) # Wait past the signed node advertisement's 10-second lease. + wait_for_lease_expiry() third, base_url = start(args.binary, directory, settings, "third", data_instance="second") processes.append(third) assert not sentinel.exists() @@ -834,7 +841,7 @@ def main(): if large is not None: verify_large_clone(base_url, directory, large) if chunks is not None: - verify_sqlite_chunks(base_url, directory, chunks) + verify_structural_objects(base_url, directory, chunks) if many is not None: verify_many_objects(base_url, directory, args.many_objects, many) if corpus is not None: @@ -845,6 +852,7 @@ def main(): raise RuntimeError("takeover owner did not shut down cleanly") smoke_s3_peers.qualify(args.binary, directory, settings, processes) smoke_s3_backup.qualify(args.binary, directory, settings, processes) + smoke_s3_retention.qualify(args.binary, directory, settings, processes) assert {path.name for path in (directory / "host-git").iterdir()} == {"config"} print("PASS: native Git ignores host config, object paths, exec path, protocol and trace settings", flush=True) print("PASS: Git/LFS, repository discovery, default branch, ACL, ref outcomes, and a dropped push reply survived restart, disk loss, and lease takeover") diff --git a/scripts/smoke_s3_retention.py b/scripts/smoke_s3_retention.py new file mode 100644 index 00000000..9244bc18 --- /dev/null +++ b/scripts/smoke_s3_retention.py @@ -0,0 +1,92 @@ +"""Qualify fenced Cellule collection with retained backup pins and cold clones.""" + +import json +import signal +import subprocess +import time +import urllib.parse +import uuid + + +def qualify(binary, directory, settings, processes): + from smoke_s3_process import start, create_repository, git + + settings = {**settings, "storage_url": settings["storage_url"] + "/retention"} + server, base = start(binary, directory, settings, "retention-source") + processes.append(server) + url, _ = create_repository(base, "retained") + source = directory / "retention-client" + git("init", "-b", "main", str(source)) + git("config", "user.name", "Retention qualification", cwd=source) + git("config", "user.email", "retention@example.invalid", cwd=source) + config = directory / "retention-source.json" + + def cli(*args, fails=False): + result = subprocess.run([str(binary), *args], capture_output=True, timeout=180) + if fails: + assert result.returncode != 0, "administration accepted an unsafe collection" + return + if result.returncode: + raise RuntimeError(result.stderr.decode(errors="replace")) + return json.loads(result.stdout) + + for revision in range(6): + for index in range(64): + (source / f"file-{index}").write_bytes( + (f"revision={revision}, file={index}\n".encode() * 1024)) + git("add", ".", cwd=source) + git("commit", "-m", f"Revision {revision}", cwd=source) + git("-c", "http.extraHeader=Authorization: Bearer local-test-token", + "push", url, "HEAD:refs/heads/main", cwd=source) + if revision == 0: + # Freeze a published snapshot before backup capture. A serving + # owner's asynchronous checkpoint may legitimately change controls. + server.send_signal(signal.SIGTERM) + server.wait(timeout=90) + assert server.returncode == 0 + pin = str(uuid.uuid4()) + backup_prefix = urllib.parse.urlsplit(settings["storage_url"]).path.lstrip("/") + "-backup" + # An enrolled backup needs a workspace independent of the serving node. + backup_config = directory / "retention-backup.json" + backup = json.loads(config.read_text()) + backup.update(node_id=str(uuid.uuid4()), data_dir=str(directory / "retention-backup-work")) + backup_config.write_text(json.dumps(backup)) + saved = cli("backup", str(backup_config), "create", pin, backup_prefix) + server, base = start(binary, directory, settings, "retention-next", data_instance="retention-source") + processes.append(server) + url = f"{base}/canopy/retained.git" + + operation = str(uuid.uuid4()) + cli("gc", str(config), operation, "1", "128", fails=True) + cli("maintenance", str(config), "begin", operation) + server.send_signal(signal.SIGTERM) + server.wait(timeout=90) + assert server.returncode == 0 + time.sleep(0.02) + partial = cli("gc", str(config), operation, "1", "1") + assert partial["deleted_objects"] == 1 and not partial["complete"] + assert partial["retained_pins"] == 1 + deleted = partial["deleted_objects"] + for _ in range(100): + report = cli("gc", str(config), operation, "1", "100000") + deleted += report["deleted_objects"] + if report["complete"]: + break + else: + raise AssertionError("bounded collection did not converge") + assert deleted > 1 + assert cli("backup", str(backup_config), "verify", pin, backup_prefix) == saved + cli("maintenance", str(config), "end", operation) + recovered, base = start(binary, directory, settings, "retention-cold") + processes.append(recovered) + clone = directory / "retention-clone" + git("-c", "http.extraHeader=Authorization: Bearer local-test-token", "clone", + f"{base}/canopy/retained.git", str(clone)) + git("fsck", "--full", cwd=clone) + assert git("rev-parse", "HEAD", cwd=clone) == git("rev-parse", "HEAD", cwd=source) + assert git("rev-list", "--objects", "--all", cwd=clone).splitlines() == git("rev-list", "--objects", "--all", cwd=source).splitlines() + result = {"deleted_objects": deleted, "last_report": report, + "backup_verified": True, "cold_clone_verified": True} + (directory / "retention-results.json").write_text(json.dumps(result, indent=2)) + print(f"PASS: collected {deleted} obsolete Cellule objects; backup pin and cold stock-Git clone remain exact", flush=True) + return result diff --git a/scripts/test_benchmark_browse.py b/scripts/test_benchmark_browse.py new file mode 100644 index 00000000..57d6f16c --- /dev/null +++ b/scripts/test_benchmark_browse.py @@ -0,0 +1,70 @@ +"""Ensure a real HTTP measurement counts both failures and corrupt responses.""" + +import contextlib +import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import io +import json +from pathlib import Path +import socket +import tempfile +import threading +from types import SimpleNamespace +import unittest + +from benchmark_browse import measure + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args): + pass + + def do_POST(self): + self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self.rfile.read(int(self.headers["Content-Length"])) + with self.server.guard: + self.server.requests += 1 + sequence = self.server.requests + body = b"verified" if sequence % 3 else b"corrupt" + self.send_response(503 if sequence % 5 == 0 else 200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class BrowseMeasurementTests(unittest.TestCase): + def test_errors_and_invalid_bodies_are_counted_without_retries(self): + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.guard, server.requests = threading.Lock(), 0 + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + (output / "queries.json").write_text(json.dumps({"queries": [{ + "repository": "fixture", "operation": "refs", "path": "/browse", + "payload": {}, "response_sha256": hashlib.sha256(b"verified").hexdigest(), + }]})) + with contextlib.redirect_stdout(io.StringIO()): + measure(SimpleNamespace(output=output, base_url=f"http://127.0.0.1:{server.server_port}", + duration=0.25, concurrency=[2], repeats=1), "fixture-token") + report = json.loads((output / "measurements.json").read_text())["results"][0] + samples = [json.loads(line) for line in (output / "samples.jsonl").read_text().splitlines()] + self.assertEqual(len(samples), server.requests - 1) # One untimed warmup. + self.assertEqual(report["requests"], len(samples)) + self.assertEqual(sum(report["outcomes"].values()), len(samples)) + self.assertGreater(report["outcomes"]["http_503_or_invalid_body"], 0) + self.assertGreater(report["outcomes"]["http_200_or_invalid_body"], 0) + self.assertAlmostEqual(report["successful_requests_per_second"], + report["outcomes"]["ok"] / report["elapsed_seconds"]) + self.assertIsNotNone(report["latency_ms_successful"]) + finally: + server.shutdown() + server.server_close() + thread.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_check_cellule.py b/scripts/test_check_cellule.py new file mode 100644 index 00000000..669ef87b --- /dev/null +++ b/scripts/test_check_cellule.py @@ -0,0 +1,67 @@ +"""Guard against partial Cellule upgrades and alternate dependency sources.""" + +import copy +import unittest + +from check_cellule import check + + +class CelluleReleaseTests(unittest.TestCase): + def setUp(self): + self.manifest = {"dependencies": { + "cellule-app": {"version": "=0.2.0"}, + "cellule-runtime": {"version": "=0.2.0"}, + }} + self.lock = {"package": [ + {"name": name, "version": "0.2.0", "checksum": "digest", + "source": "registry+https://github.com/rust-lang/crates.io-index"} + for name in ("cellule-app", "cellule-runtime", "cellule-types") + ]} + + def test_accepts_aligned_direct_and_transitive_crates(self): + self.assertEqual(check(self.manifest, self.lock), ("0.2.0", 3)) + + def test_rejects_ranges_and_alternate_sources(self): + for spec in ({"version": "0.2.0"}, {"version": "^0.2.0"}, + {"version": "=0.2.0", "git": "https://example.com/cellule"}, + {"version": "=0.2.0", "path": "../cellule"}, + {"version": "=0.2.0", "registry": "private"}): + with self.subTest(spec=spec): + self.manifest["dependencies"]["cellule-app"] = spec + with self.assertRaisesRegex(ValueError, "exact stable crates.io"): + check(self.manifest, self.lock) + + def test_rejects_partial_direct_upgrade(self): + self.manifest["dependencies"]["cellule-app"]["version"] = "=0.3.0" + with self.assertRaisesRegex(ValueError, "same release"): + check(self.manifest, self.lock) + + def test_rejects_hidden_target_alias_on_another_release(self): + self.manifest["target"] = {"cfg(unix)": {"dependencies": { + "runtime": {"package": "cellule-runtime", "version": "=0.1.0"}, + }}} + with self.assertRaisesRegex(ValueError, "same release"): + check(self.manifest, self.lock) + + def test_rejects_old_or_replaced_transitive_crate(self): + for field, value in (("version", "0.1.0"), ("source", "git+https://example.com/cellule"), + ("source", None), ("checksum", None)): + with self.subTest(field=field, value=value): + lock = copy.deepcopy(self.lock) + lock["package"][-1][field] = value + with self.assertRaisesRegex(ValueError, "cellule-types must resolve once"): + check(self.manifest, lock) + + def test_rejects_duplicate_locked_crate(self): + self.lock["package"].append(self.lock["package"][-1].copy()) + with self.assertRaisesRegex(ValueError, "resolve once"): + check(self.manifest, self.lock) + + def test_rejects_missing_locked_crate(self): + self.lock["package"].pop(0) + with self.assertRaisesRegex(ValueError, "missing from Cargo.lock"): + check(self.manifest, self.lock) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/deployment.rs b/src/deployment.rs index 285962dd..5ae08240 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -15,9 +15,12 @@ use serde::Serialize; mod backup; pub use backup::{BackupError, BackupReport}; +mod maintenance; mod recovery; +mod retention; mod root; pub use recovery::WorkerConfig; +pub use retention::GarbageCollectionReport; /// Application-wide admission shared by nodes and offline administration. #[derive(Clone)] @@ -146,6 +149,7 @@ impl Deployment { /// Rejects nodes when identity, selected release, image or phase differs. pub async fn require_ready(&self) -> Result<()> { + maintenance::require_idle(self).await?; self.require_service_root().await?; self.identities.layout(self.identity).await?; let record = self.record().await?; @@ -193,6 +197,7 @@ impl Deployment { /// Closes fleet admission under an explicit, retryable maintenance operation. pub async fn begin_maintenance(&self, operation: RequestId) -> Result { + maintenance::require_idle(self).await?; self.require_service_root().await?; self.identities.layout(self.identity).await?; let record = self.record().await?; @@ -263,6 +268,17 @@ impl Deployment { &self, operation: RequestId, now_ms: i64, + ) -> Result { + let claim = maintenance::acquire(self, operation, maintenance::Action::End, None).await?; + let result = self.end_maintenance_inner(operation, now_ms).await; + claim.finish(self).await?; + result + } + + async fn end_maintenance_inner( + &self, + operation: RequestId, + now_ms: i64, ) -> Result { self.require_service_root().await?; self.identities.layout(self.identity).await?; diff --git a/src/deployment/backup.rs b/src/deployment/backup.rs index 4b15f0bc..bc8a54d9 100644 --- a/src/deployment/backup.rs +++ b/src/deployment/backup.rs @@ -3,7 +3,7 @@ use super::*; use crate::{ REPOSITORIES, REPOSITORY_DATABASE_LIMIT_BYTES, - large_blob::LargeBlobError, + git_pack::PackError, lfs::LfsError, server::{ServerError, unix_now_ms}, }; @@ -33,7 +33,7 @@ pub enum BackupError { #[error("backup external part copy failed")] External(#[from] object_store::Error), #[error("backup Git blob verification failed")] - Blob(#[from] LargeBlobError), + Pack(#[from] PackError), #[error("backup LFS verification failed")] Lfs(#[from] LfsError), #[error("backup SQLite read failed")] diff --git a/src/deployment/backup/bodies.rs b/src/deployment/backup/bodies.rs index db7af14a..3041e639 100644 --- a/src/deployment/backup/bodies.rs +++ b/src/deployment/backup/bodies.rs @@ -1,6 +1,6 @@ use super::*; use crate::{ - large_blob::{LargeBlobReference, LargeBlobStore, blob_path}, + git_pack::{PackIdentity, PackStore}, lfs::{LfsObject, lfs_path, verify_lfs_object}, }; use cellule_ltx::rusqlite::{Connection, OpenFlags, OptionalExtension, params}; @@ -13,7 +13,7 @@ enum BodyKind { Lfs, } enum Reference { - Git(LargeBlobReference), + Git(PackIdentity), Lfs(LfsObject), } @@ -101,7 +101,7 @@ impl Deployment { break; } cursor = match page.last().ok_or(BackupError::Invalid("empty page"))? { - Reference::Git(value) => value.oid.to_vec(), + Reference::Git(value) => value.pack_digest.to_vec(), Reference::Lfs(value) => value.sha256.to_vec(), }; for reference in page { @@ -111,37 +111,85 @@ impl Deployment { "external body has no repository identity", ))?; guard.check()?; - let path = match &reference { - Reference::Git(value) => blob_path(repository_id, &value.sha256), - Reference::Lfs(value) => lfs_path(repository_id, &value.sha256), - }; - if let (Some(source), Some(source_store)) = (source, &source_store) { - verify(source_store.clone(), repository_id, &reference).await?; - let size = match &reference { - Reference::Git(value) => value.size, - Reference::Lfs(value) => value.size, - }; - crate::external::copy_parts( - self.layout.store().inner().as_ref(), - &source.parts().chain(path.parts()).collect(), - &self.prefix.parts().chain(path.parts()).collect(), - size, - ) - .await?; - match self - .layout - .store() - .copy_if_not_exists( - &source.parts().chain(path.parts()).collect(), - &self.prefix.parts().chain(path.parts()).collect(), + match &reference { + Reference::Git(value) => { + let packs = PackStore::new( + destination_store.clone(), + repository_id, + scratch.into(), + host.local_disk_budget().clone(), + 256 * 1024 * 1024, + 64 * 1024 * 1024, + )?; + if let (Some(source), Some(source_store)) = (source, &source_store) + { + PackStore::new( + source_store.clone(), + repository_id, + scratch.into(), + host.local_disk_budget().clone(), + 256 * 1024 * 1024, + 64 * 1024 * 1024, + )? + .verify(value) + .await?; + for path in packs.paths(value) { + match self + .layout + .store() + .copy_if_not_exists( + &source.parts().chain(path.parts()).collect(), + &self.prefix.parts().chain(path.parts()).collect(), + ) + .await + { + Ok(()) | Err(StorageError::StateConflict { .. }) => {} + Err(error) => return Err(error.into()), + } + } + } + packs.verify(value).await?; + } + Reference::Lfs(value) => { + let path = lfs_path(repository_id, &value.sha256); + if let (Some(source), Some(source_store)) = (source, &source_store) + { + verify_lfs_object( + source_store.clone(), + repository_id, + *value, + None, + ) + .await?; + crate::external::copy_parts( + self.layout.store().inner().as_ref(), + &source.parts().chain(path.parts()).collect(), + &self.prefix.parts().chain(path.parts()).collect(), + value.size, + ) + .await?; + match self + .layout + .store() + .copy_if_not_exists( + &source.parts().chain(path.parts()).collect(), + &self.prefix.parts().chain(path.parts()).collect(), + ) + .await + { + Ok(()) | Err(StorageError::StateConflict { .. }) => {} + Err(error) => return Err(error.into()), + } + } + verify_lfs_object( + destination_store.clone(), + repository_id, + *value, + None, ) - .await - { - Ok(()) | Err(StorageError::StateConflict { .. }) => {} - Err(error) => return Err(error.into()), + .await?; } } - verify(destination_store.clone(), repository_id, &reference).await?; count = count .checked_add(1) .ok_or(BackupError::Invalid("external object count overflow"))?; @@ -176,24 +224,6 @@ async fn read_identity(database: PathBuf) -> BackupResult> { .await? } -async fn verify( - store: Arc, - repository_id: [u8; 16], - reference: &Reference, -) -> BackupResult<()> { - match reference { - Reference::Git(value) => { - LargeBlobStore::new(store, repository_id) - .verify(value) - .await?; - } - Reference::Lfs(value) => { - verify_lfs_object(store, repository_id, *value, None).await?; - } - } - Ok(()) -} - async fn read_page( database: PathBuf, kind: BodyKind, @@ -202,7 +232,7 @@ async fn read_page( tokio::task::spawn_blocking(move || { let connection = Connection::open_with_flags(database, OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX)?; let sql = match kind { - BodyKind::Git => "SELECT oid, size, digest, external_sha256 FROM objects WHERE storage = 'external' AND oid > ?1 ORDER BY oid LIMIT 256", + BodyKind::Git => "SELECT digest, bytes, index_digest, index_bytes, format FROM git_packs WHERE digest > ?1 ORDER BY digest LIMIT 256", BodyKind::Lfs => "SELECT sha256, size, digest, sha256 FROM lfs_objects WHERE sha256 > ?1 ORDER BY sha256 LIMIT 256", }; let mut statement = connection.prepare(sql)?; @@ -212,13 +242,20 @@ async fn read_page( let oid: Vec = row.get(0)?; let size: i64 = row.get(1)?; let digest: Vec = row.get(2)?; - let sha256: Vec = row.get(3)?; let size = u64::try_from(size).map_err(|_| BackupError::Invalid("invalid body size"))?; let digest = digest.try_into().map_err(|_| BackupError::Invalid("invalid body digest"))?; - let sha256 = sha256.try_into().map_err(|_| BackupError::Invalid("invalid body SHA-256"))?; page.push(match kind { - BodyKind::Git => Reference::Git(LargeBlobReference { oid: oid.try_into().map_err(|_| BackupError::Invalid("invalid Git OID"))?, size, blake3: digest, sha256 }), - BodyKind::Lfs => Reference::Lfs(LfsObject { sha256, size, parts_digest: digest }), + BodyKind::Git => { + let format: String = row.get(4)?; + let index_bytes: i64 = row.get(3)?; + Reference::Git(PackIdentity { format: crate::ObjectFormat::parse(&format).ok_or(BackupError::Invalid("invalid pack format"))?, + pack_digest: oid.try_into().map_err(|_| BackupError::Invalid("invalid pack digest"))?, pack_bytes: size, + index_digest: digest, index_bytes: u64::try_from(index_bytes).map_err(|_| BackupError::Invalid("invalid pack index size"))? }) + }, + BodyKind::Lfs => { + let sha256: Vec = row.get(3)?; + Reference::Lfs(LfsObject { sha256: sha256.try_into().map_err(|_| BackupError::Invalid("invalid LFS SHA-256"))?, size, parts_digest: digest }) + }, }); } Ok(page) diff --git a/src/deployment/maintenance.rs b/src/deployment/maintenance.rs new file mode 100644 index 00000000..9b72c183 --- /dev/null +++ b/src/deployment/maintenance.rs @@ -0,0 +1,165 @@ +//! Persistent exclusion for offline administration, including interrupted sweeps. + +use super::*; +use bytes::Bytes; +use cellule_store::{ETag, StorageError}; +use serde::Deserialize; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(super) enum Action { + Collect, + Recover, + End, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Worker { + node: String, + key: String, + workspace: String, +} + +impl Worker { + pub(super) fn new(config: &WorkerConfig, workspace: &std::path::Path) -> Result { + Ok(Self { + node: hex::encode(config.node.as_bytes()), + key: hex::encode(config.signing_key.verifying_key().as_bytes()), + workspace: workspace + .to_str() + .ok_or(Error::Retention("maintenance workspace is not UTF-8"))? + .into(), + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +enum State { + Idle, + Active { + operation: String, + action: Action, + worker: Option, + run: String, + }, +} + +pub(super) struct Claim { + path: Path, + state: State, + token: ETag, +} + +fn path(deployment: &Deployment) -> Path { + deployment.prefix.clone().join("canopy-maintenance-v1.json") +} + +async fn load(deployment: &Deployment) -> Result> { + match deployment + .layout + .store() + .get_with_etag_bounded(&path(deployment), 4096) + .await + { + Ok((body, token)) => Ok(Some((serde_json::from_slice(&body)?, token))), + Err(StorageError::NotFound { .. }) => Ok(None), + Err(error) => Err(error.into()), + } +} + +pub(super) async fn require_idle(deployment: &Deployment) -> Result<()> { + if load(deployment) + .await? + .is_some_and(|(state, _)| state != State::Idle) + { + return Err(Error::Retention( + "an offline maintenance worker has not settled", + )); + } + Ok(()) +} + +/// Worker adoption is permitted only while holding the original local workspace +/// lock. There is deliberately no timeout takeover of a collector: a paused +/// delete must never overlap reopened admission, even after a node lease expires. +pub(super) async fn acquire( + deployment: &Deployment, + operation: RequestId, + action: Action, + worker: Option, +) -> Result { + let operation = hex::encode(operation.as_bytes()); + let previous = load(deployment).await?; + if let Some(( + State::Active { + operation: old_operation, + action: old_action, + worker: old_worker, + .. + }, + _, + )) = &previous + && (*old_operation != operation || *old_action != action || *old_worker != worker) + { + return Err(Error::Retention( + "another offline maintenance worker owns the deployment", + )); + } + // End is an idempotent release CAS. Collect/recover callers prove the + // old process closed by acquiring its same canonical workspace first. + let state = State::Active { + operation, + action, + worker, + run: uuid::Uuid::new_v4().to_string(), + }; + let bytes = serde_json::to_vec(&state)?; + if bytes.len() > 4096 { + return Err(Error::Retention("maintenance claim exceeds size limit")); + } + let path = path(deployment); + let token = match previous { + Some((_, token)) => { + deployment + .layout + .store() + .update(&path, Bytes::from(bytes), token) + .await? + } + None => { + deployment + .layout + .store() + .create_strict_with_etag(&path, Bytes::from(bytes)) + .await? + } + }; + Ok(Claim { path, state, token }) +} + +impl Claim { + pub(super) async fn require_current(&self, deployment: &Deployment) -> Result<()> { + if !load(deployment) + .await? + .is_some_and(|(state, token)| state == self.state && token == self.token) + { + return Err(Error::Retention("maintenance worker claim changed")); + } + Ok(()) + } + + pub(super) async fn finish(self, deployment: &Deployment) -> Result<()> { + deployment + .layout + .store() + .update( + &self.path, + Bytes::from(serde_json::to_vec(&State::Idle)?), + self.token, + ) + .await?; + Ok(()) + } +} diff --git a/src/deployment/recovery.rs b/src/deployment/recovery.rs index 65731a23..dd6331f0 100644 --- a/src/deployment/recovery.rs +++ b/src/deployment/recovery.rs @@ -57,9 +57,13 @@ impl Deployment { config: WorkerConfig, ) -> std::result::Result<(), ServerError> { self.require_maintenance(operation).await?; - let data_dir = config.data_dir; + let data_dir = config.data_dir.clone(); let local = tokio::task::spawn_blocking(move || workspace::Workspace::open(&data_dir)).await??; + let worker = maintenance::Worker::new(&config, local.path())?; + let claim = + maintenance::acquire(self, operation, maintenance::Action::Recover, Some(worker)) + .await?; crate::server::storage::probe( self.layout.store(), &self.layout.application_prefix().join("canopy-probe"), @@ -224,6 +228,7 @@ impl Deployment { self.nodes .withdraw(&*observed.lock().await, unix_now_ms()?) .await?; + claim.finish(self).await?; result } } diff --git a/src/deployment/retention.rs b/src/deployment/retention.rs new file mode 100644 index 00000000..9cdff7be --- /dev/null +++ b/src/deployment/retention.rs @@ -0,0 +1,108 @@ +//! Product maintenance drain around Cellule's authoritative object collector. + +use super::*; +use crate::{ + REPOSITORY_DATABASE_LIMIT_BYTES, + server::{ServerError, unix_now_ms, workspace}, +}; +use cellule_ltx::{DiskBudget, Host}; +use cellule_runtime::recovery::retention::{CellGarbageCollector, GarbageCollectionPolicy}; + +/// Counts from a verified live-root/backup-pin mark and bounded immutable sweep. +#[derive(Debug, Serialize)] +pub struct GarbageCollectionReport { + pub listed_objects: u64, + pub immutable_candidates: u64, + pub reachable_objects: u64, + pub retained_objects: u64, + pub grace_objects: u64, + pub eligible_objects: u64, + pub deleted_objects: u64, + pub current_controls: u64, + pub retained_pins: u64, + pub complete: bool, +} + +impl Deployment { + /// Reclaims old immutable Cell objects after every server, reader, backup + /// worker and recovery owner has drained. Leaves the release in maintenance. + /// Same-workspace retries resume a killed worker; dropping the caller cannot + /// release exclusion before the sweep finishes. Git packs/LFS are separate + /// product objects and are not deleted by the Cellule collector. + pub async fn collect_garbage( + &self, + operation: RequestId, + config: WorkerConfig, + grace_ms: u64, + max_deletes: u64, + ) -> std::result::Result { + let deployment = self.clone(); + tokio::spawn(async move { + deployment + .collect(operation, config, grace_ms, max_deletes) + .await + }) + .await? + } + + async fn collect( + &self, + operation: RequestId, + config: WorkerConfig, + grace_ms: u64, + max_deletes: u64, + ) -> std::result::Result { + self.require_maintenance(operation).await?; + // Validate policy before claiming any persistent exclusion. + GarbageCollectionPolicy::new(unix_now_ms()?, grace_ms, max_deletes)?; + let data_dir = config.data_dir.clone(); + let local = + tokio::task::spawn_blocking(move || workspace::Workspace::open(&data_dir)).await??; + let worker = maintenance::Worker::new(&config, local.path())?; + let claim = + maintenance::acquire(self, operation, maintenance::Action::Collect, Some(worker)) + .await?; + let result = async { + self.require_maintenance(operation).await?; + if !self.status(unix_now_ms()?).await?.drained { + return Err( + Error::Retention("deployment readers and writers have not drained").into(), + ); + } + claim.require_current(self).await?; + let host = Host::default() + .with_local_disk_budget(DiskBudget::new(config.local_disk_limit_bytes)); + let collector = CellGarbageCollector::new( + self.layout.clone(), + self.identity, + crate::replica_limits(REPOSITORY_DATABASE_LIMIT_BYTES, 64 * 1024 * 1024), + host, + )?; + let report = collector + .collect( + &self.record().await?, + local.path(), + GarbageCollectionPolicy::new(unix_now_ms()?, grace_ms, max_deletes)?, + ) + .await?; + claim.require_current(self).await?; + Ok::<_, ServerError>(GarbageCollectionReport { + listed_objects: report.listed_objects(), + immutable_candidates: report.immutable_candidates(), + reachable_objects: report.reachable_objects(), + retained_objects: report.retained_objects(), + grace_objects: report.grace_objects(), + eligible_objects: report.eligible_objects(), + deleted_objects: report.deleted_objects(), + current_controls: report.current_controls(), + retained_pins: report.retained_pins(), + complete: report.complete(), + }) + } + .await; + // The collector has joined all work on return. Errors retain the release + // fence but allow a fresh bounded pass, including partial provider failure. + claim.finish(self).await?; + result + } +} diff --git a/src/deployment/tests.rs b/src/deployment/tests.rs index d57ecb70..40e51233 100644 --- a/src/deployment/tests.rs +++ b/src/deployment/tests.rs @@ -157,7 +157,12 @@ async fn existing_catalog_without_release_is_not_silently_adopted() -> TestResul .module_code(RepositoryModule::NAME) .ok_or("missing code")?; CellCatalog::new(deployment.layout.clone(), deployment.identity.tenant()) - .provision(CatalogEntry::new(&target, CatalogRole::Sql, code, 1)?) + .provision(CatalogEntry::new( + &target, + CatalogRole::Sql, + code, + crate::REPOSITORY_SCHEMA_VERSION, + )?) .await?; assert!(deployment.initialize().await.is_err()); assert!(deployment.releases.load().await?.is_none()); @@ -233,6 +238,166 @@ fn recovery_config(data_dir: std::path::PathBuf) -> WorkerConfig { } } +#[tokio::test(flavor = "multi_thread")] +async fn collection_deletes_only_old_unreachable_cell_objects_and_stays_in_maintenance() +-> TestResult { + let deployment = fixture()?; + deployment.initialize().await?; + let files = tempfile::TempDir::new()?; + let operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); + let orphan = deployment.layout.incarnation_object_path( + &[11; 32], + &[12; 16], + &[13; 32], + cellule_ltx::CellObjectKind::Directory, + ); + let product = deployment + .prefix + .clone() + .join("repos/fixture/git-packs/unreferenced.pack"); + for path in [&orphan, &product] { + deployment + .layout + .store() + .create_strict(path, bytes::Bytes::from_static(b"unreferenced")) + .await?; + } + assert!( + deployment + .collect_garbage( + operation, + recovery_config(files.path().join("before")), + 1, + 1 + ) + .await + .is_err() + ); + deployment.begin_maintenance(operation).await?; + // A large grace window retains the candidate, and the compiled descriptor + // remains marked even when its modification time is beyond that boundary. + let report = deployment + .collect_garbage( + operation, + recovery_config(files.path().join("grace")), + 60_000, + 1, + ) + .await?; + assert_eq!(report.deleted_objects, 0); + assert!(report.grace_objects >= 1); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + let report = deployment + .collect_garbage( + operation, + recovery_config(files.path().join("collect")), + 1, + 1, + ) + .await?; + assert_eq!(report.deleted_objects, 1); + assert!(report.complete); + assert!(matches!( + deployment.layout.store().get_with_etag(&orphan).await, + Err(cellule_store::StorageError::NotFound { .. }) + )); + assert!( + deployment + .layout + .store() + .get_with_etag(&product) + .await + .is_ok() + ); + assert!( + deployment + .status(crate::server::unix_now_ms()?) + .await? + .drained + ); + assert!(deployment.require_ready().await.is_err()); + deployment + .end_maintenance(operation, crate::server::unix_now_ms()?) + .await?; + deployment.require_ready().await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn collection_rejects_even_expired_advertisements_before_deleting() -> TestResult { + let deployment = fixture()?; + deployment.initialize().await?; + advertise_owner(&deployment, true).await?; + let operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); + deployment.begin_maintenance(operation).await?; + let files = tempfile::TempDir::new()?; + assert!( + deployment + .collect_garbage(operation, recovery_config(files.path().to_path_buf()), 1, 1) + .await + .is_err() + ); + maintenance::require_idle(&deployment).await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn interrupted_collection_blocks_reopening_until_same_workspace_worker_resumes() -> TestResult +{ + let deployment = fixture()?; + deployment.initialize().await?; + let operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); + deployment.begin_maintenance(operation).await?; + let files = tempfile::TempDir::new()?; + let config = recovery_config(files.path().to_path_buf()); + let local = crate::server::workspace::Workspace::open(files.path())?; + let worker = maintenance::Worker::new(&config, local.path())?; + let claim = maintenance::acquire( + &deployment, + operation, + maintenance::Action::Collect, + Some(worker), + ) + .await?; + assert!( + deployment + .end_maintenance(operation, crate::server::unix_now_ms()?) + .await + .is_err() + ); + assert!( + deployment + .recover_maintenance(operation, recovery_config(files.path().join("other"))) + .await + .is_err() + ); + // The same worker cannot adopt its claim while its original workspace is held. + let identical = || WorkerConfig { + node: config.node, + signing_key: config.signing_key.clone(), + endpoint: config.endpoint.clone(), + data_dir: config.data_dir.clone(), + local_disk_limit_bytes: config.local_disk_limit_bytes, + }; + assert!( + deployment + .collect_garbage(operation, identical(), 1, 1) + .await + .is_err() + ); + drop(local); + let report = deployment + .collect_garbage(operation, identical(), 1, 1) + .await?; + assert_eq!(report.deleted_objects, 0); + assert!(claim.require_current(&deployment).await.is_err()); + maintenance::require_idle(&deployment).await?; + deployment + .end_maintenance(operation, crate::server::unix_now_ms()?) + .await?; + Ok(()) +} + async fn advertise_owner(deployment: &Deployment, expired: bool) -> TestResult { use cellule_runtime::{ SessionId, identity::NodeId, node::NodeAdvertisement, node::NodeCapacity, @@ -282,7 +447,12 @@ async fn maintenance_recovery_initializes_abandoned_catalog_and_unpublished_owne uuid::Uuid::new_v4().into_bytes(), )?; let proof = catalog - .provision(CatalogEntry::new(&target, CatalogRole::Sql, code, 1)?) + .provision(CatalogEntry::new( + &target, + CatalogRole::Sql, + code, + crate::REPOSITORY_SCHEMA_VERSION, + )?) .await?; if unpublished { authority @@ -392,7 +562,12 @@ async fn maintenance_recovery_rejects_missing_published_root() -> TestResult { .module_code(RepositoryModule::NAME) .ok_or("missing code")?; let proof = CellCatalog::new(deployment.layout.clone(), deployment.identity.tenant()) - .provision(CatalogEntry::new(&target, CatalogRole::Sql, code, 1)?) + .provision(CatalogEntry::new( + &target, + CatalogRole::Sql, + code, + crate::REPOSITORY_SCHEMA_VERSION, + )?) .await?; let authority = CellAuthority::new(deployment.layout.clone()); let initial = authority diff --git a/src/external.rs b/src/external.rs index 51cd9202..d4bcb38a 100644 --- a/src/external.rs +++ b/src/external.rs @@ -1,4 +1,4 @@ -//! Immutable, fixed-size parts for Git blob and LFS bodies. +//! Immutable, fixed-size parts for Git LFS bodies. use bytes::{Bytes, BytesMut}; use object_store::{ @@ -9,7 +9,6 @@ use std::future::poll_fn; use std::{future::Future, sync::Arc, time::Duration}; pub(crate) const PART_BYTES: usize = 8 * 1024 * 1024; -const MAGIC: &[u8; 8] = b"CANOPY01"; const LFS_MAGIC: &[u8; 8] = b"CANOPY02"; pub(crate) struct Manifest { @@ -92,12 +91,6 @@ impl Upload { Ok(()) } - pub(crate) async fn publish(&mut self, path: &Path, size: u64) -> object_store::Result<()> { - let mut manifest = MAGIC.to_vec(); - manifest.extend_from_slice(&size.to_le_bytes()); - self.publish_manifest(path, size, manifest).await - } - pub(crate) async fn publish_lfs( &mut self, path: &Path, @@ -206,21 +199,6 @@ async fn bounded( .await } -pub(crate) async fn open( - store: &dyn ObjectStore, - path: &Path, - size: u64, -) -> object_store::Result { - let (meta, bytes) = bounded(store, path, GetOptions::default(), 16).await?; - if &bytes[..8] != MAGIC || bytes[8..] != size.to_le_bytes() { - return Err(invalid()); - } - if size == 0 { - bounded(store, &part(path, 0), GetOptions::default(), 0).await?; - } - Ok(Manifest { meta, bytes }) -} - pub(crate) async fn open_lfs( store: &dyn ObjectStore, path: &Path, diff --git a/src/git_cache.rs b/src/git_cache.rs index 1a0ba85c..23f300aa 100644 --- a/src/git_cache.rs +++ b/src/git_cache.rs @@ -5,19 +5,14 @@ use std::{ fs::{self, File, OpenOptions}, io::{self, BufWriter, Write}, path::{Path, PathBuf}, - sync::{Arc, OnceLock}, + sync::{Arc, OnceLock, RwLock}, }; use cellule_ltx::{DiskBudget, DiskReservation, LtxError}; use flate2::{Compression, write::ZlibEncoder}; use tokio::sync::Mutex; -use crate::{ - ObjectKind, RefExpectation, - large_blob::{LargeBlobError, LargeBlobRead}, - object_id, - refs::valid_ref_name, -}; +use crate::{ObjectKind, RefExpectation, object_id, refs::valid_ref_name}; pub(crate) const CACHE_PREFIX: &str = "canopy-git-"; @@ -29,8 +24,6 @@ pub enum CacheError { Io(#[from] io::Error), #[error("Git cache disk admission failed")] Budget(#[from] LtxError), - #[error("Git blob hydration failed")] - Blob(#[from] LargeBlobError), #[error("Git cache worker failed")] Task(#[from] tokio::task::JoinError), } @@ -55,6 +48,8 @@ pub(crate) struct GitCache { // Only durable hydration writes this cache. Stripe by OID so concurrent // fetches share a completed loose object without serializing all objects. object_writes: OnceLock<[Arc>; 64]>, + packs: RwLock>>, + pack_writes: Arc>, } impl GitCache { @@ -87,6 +82,8 @@ impl GitCache { reservation: Some(budget.try_reserve(0)?), objects, object_writes: OnceLock::new(), + packs: RwLock::new(BTreeMap::new()), + pack_writes: Arc::new(Mutex::new(())), }); for directory in ["objects/info", "objects/pack", "refs/heads", "refs/tags", "hooks"] { fs::create_dir_all(cache.git_dir().join(directory))?; @@ -111,6 +108,10 @@ impl GitCache { }).await? } + pub(crate) fn object_format(&self) -> crate::ObjectFormat { + self.object_format + } + pub(crate) fn root(&self) -> &Path { self.directory.path() } @@ -170,6 +171,20 @@ impl GitCache { } fn object_present(&self, oid: crate::ObjectId) -> io::Result { + for pack in self + .packs + .read() + .map_err(|_| io::Error::other("Git pack cache lock poisoned"))? + .values() + { + if pack + .data() + .and_then(|data| data.contains(oid)) + .map_err(io::Error::other)? + { + return Ok(true); + } + } match fs::symlink_metadata(self.object_path(oid)) { Ok(metadata) if metadata.is_file() => Ok(true), Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), @@ -209,6 +224,53 @@ impl GitCache { )) } + /// Installs a native pair with the index renamed last. Git never observes + /// an index naming an incomplete pack. The cache retains the immutable + /// source owner and charges linked files conservatively to its own budget. + pub(crate) async fn install_pack( + self: &Arc, + pack: Arc, + ) -> Result<(), CacheError> { + let write = Arc::clone(&self.pack_writes).lock_owned().await; + let cache = Arc::clone(self); + tokio::task::spawn_blocking(move || { + let _write = write; + let identity = pack.data().map_err(io::Error::other)?.identity(); + if identity.format != cache.object_format { + return Err( + io::Error::new(io::ErrorKind::InvalidData, "Git pack format differs").into(), + ); + } + if cache + .packs + .read() + .map_err(|_| io::Error::other("Git pack cache lock poisoned"))? + .contains_key(&identity.pack_digest) + { + return Ok(()); + } + let source = pack.directory().map_err(io::Error::other)?; + let stem = format!("objects/pack/pack-{}", hex::encode(identity.pack_digest)); + for (extension, bytes) in [("pack", identity.pack_bytes), ("idx", identity.index_bytes)] + { + let destination = cache.git_dir().join(format!("{stem}.{extension}")); + let pending = cache.git_dir().join(format!("{stem}.{extension}.pending")); + cache.reservation()?.try_grow(bytes)?; + // Source and cache share the configured scratch filesystem. + // Count each link's bytes even though physical blocks are shared. + fs::hard_link(source.join(format!("objects.{extension}")), &pending)?; + fs::rename(pending, destination)?; + } + cache + .packs + .write() + .map_err(|_| io::Error::other("Git pack cache lock poisoned"))? + .insert(identity.pack_digest, pack); + Ok(()) + }) + .await? + } + pub(crate) async fn store_receive_hook( self: &Arc, hook: ReceiveHook, @@ -274,48 +336,103 @@ impl GitCache { .await? } - pub(crate) async fn store_blob( + /// Streams one selected object through native Git without making omitted + /// pack entries available to the serving cache. The immutable pair remains + /// the durable authority; this loose file is a disposable projection. + pub(crate) async fn store_native_object( self: &Arc, - mut reader: LargeBlobRead, + source: Arc, + reference: crate::PackedObject, + root: PathBuf, + budget: DiskBudget, ) -> Result<(), CacheError> { - let reference = reader.reference(); - let write = self.object_write_lock(reference.oid).lock_owned().await; + let oid = reference.oid; + let kind = reference.kind; + let size = reference.size; + let digest = reference.digest; + let write = self.object_write_lock(oid).lock_owned().await; let cache = Arc::clone(self); - let pending = tokio::task::spawn_blocking(move || { - if reference.oid.format() != cache.object_format { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Git object format mismatch", - ) - .into()); - } - if cache.object_present(reference.oid)? { - return Ok::<_, CacheError>(None); + tokio::spawn(async move { + let _write = write; + if cache.object_present(oid)? { + return Ok(()); } - let (mut encoder, temporary, destination) = cache.object_writer(reference.oid)?; - encoder.write_all(format!("blob {}\0", reference.size).as_bytes())?; - Ok::<_, CacheError>(Some((encoder, temporary, destination))) - }) - .await??; - let Some((mut encoder, temporary, destination)) = pending else { - return Ok(()); - }; - while let Some(bytes) = reader.next().await? { - // The writer owns the cache reservation until each compression worker - // exits, including when its async waiter is canceled. - encoder = tokio::task::spawn_blocking(move || { - encoder.write_all(&bytes)?; - Ok::<_, CacheError>(encoder) + let input = GitCache::create(root, budget, "refs/heads/main", oid.format()).await?; + input.install_pack(source).await?; + let mut child = crate::native_git::command(&input.git_dir())? + .args(["cat-file", kind.git_name(), &hex::encode(oid)]) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn()?; + let mut reader = child + .stdout + .take() + .ok_or_else(|| io::Error::other("missing native object stream"))?; + let owner = Arc::clone(&cache); + let mut pending = tokio::task::spawn_blocking(move || { + let (mut encoder, temporary, destination) = owner.object_writer(oid)?; + encoder.write_all(format!("{} {size}\0", kind.git_name()).as_bytes())?; + Ok::<_, CacheError>(( + encoder, + temporary, + destination, + crate::git_format::ObjectHasher::new(oid.format(), kind, size), + blake3::Hasher::new(), + )) }) .await??; - } - tokio::task::spawn_blocking(move || { - let _write = write; - drop(encoder.finish()?); - temporary - .persist_noclobber(destination) - .map_err(|error| error.error)?; - Ok::<_, CacheError>(()) + let transferred = async { + use tokio::io::AsyncReadExt; + let mut count = 0u64; + loop { + let mut buffer = vec![0; 1024 * 1024]; + let bytes = tokio::time::timeout( + std::time::Duration::from_secs(120), + reader.read(&mut buffer), + ) + .await + .map_err(io::Error::other)??; + if bytes == 0 { + break; + } + count = count + .checked_add(bytes as u64) + .filter(|count| *count <= size) + .ok_or_else(|| io::Error::other("native object length differs"))?; + buffer.truncate(bytes); + pending = tokio::task::spawn_blocking(move || { + pending.3.update(&buffer); + pending.4.update(&buffer); + pending.0.write_all(&buffer)?; + Ok::<_, CacheError>(pending) + }) + .await??; + } + Ok::<_, CacheError>((count, pending)) + } + .await; + if transferred.is_err() { + let _ = child.start_kill(); + } + let status = child.wait().await?; + let (count, (encoder, temporary, destination, hash, body_hash)) = transferred?; + if !status.success() + || count != size + || hash.finalize() != oid + || body_hash.finalize().as_bytes() != &digest + { + return Err(io::Error::other("corrupt native object stream").into()); + } + tokio::task::spawn_blocking(move || { + drop(encoder.finish()?); + temporary + .persist_noclobber(destination) + .map_err(|error| error.error)?; + Ok::<_, CacheError>(()) + }) + .await? }) .await? } diff --git a/src/git_cache/tests.rs b/src/git_cache/tests.rs index 09c775c1..0b6ae3f2 100644 --- a/src/git_cache/tests.rs +++ b/src/git_cache/tests.rs @@ -300,7 +300,7 @@ async fn failed_cleanup_does_not_release_disk_admission() -> Result<(), Box Result<(), Box> { let root = tempfile::TempDir::new()?; let budget = DiskBudget::new(1 << 20); @@ -311,32 +311,37 @@ async fn streaming_blob_hydration_preserves_bytes_and_disk_admission() crate::ObjectFormat::Sha1, ) .await?; - let blobs = crate::large_blob::LargeBlobStore::new( + let packs = crate::git_pack::PackStore::new( Arc::new(object_store::memory::InMemory::new()), - [7; 16], - ); + uuid::Uuid::new_v4().into_bytes(), + root.path().into(), + budget.clone(), + 1 << 20, + 16 << 20, + )?; let body = vec![17; 9 * 1024 * 1024]; - let reference = blobs - .put( - object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, &body), - body.len() as u64, - &mut body.as_slice(), + let oid = object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, &body); + let source = packs + .prepare( + crate::ObjectFormat::Sha1, + vec![(ObjectKind::Blob, body.clone())], ) .await?; - cache.store_blob(blobs.read(&reference).await?).await?; - let output = tokio::process::Command::new("git") - .arg("--git-dir") - .arg(cache.git_dir()) - .args(["cat-file", "blob", &hex::encode(reference.oid)]) + let identity = packs.publish(Arc::clone(&source)).await?; + drop(source); + let pack = packs.load(&identity).await?; + cache.install_pack(pack.clone()).await?; + assert!(cache.missing_objects(vec![oid]).await?.is_empty()); + let output = crate::native_git::command(&cache.git_dir())? + .args(["cat-file", "blob", &hex::encode(oid)]) .output() .await?; assert!(output.status.success()); assert_eq!(output.stdout, body); - assert_eq!(budget.used(), tree_bytes(cache.root())?); - drop(cache); - assert_eq!(budget.used(), 0); - - let cache = GitCache::create( + let before = budget.used(); + cache.install_pack(pack.clone()).await?; + assert_eq!(budget.used(), before); + let other = GitCache::create( root.path().into(), budget.clone(), "refs/heads/main", @@ -345,65 +350,17 @@ async fn streaming_blob_hydration_preserves_bytes_and_disk_admission() .await?; let occupied = budget.try_reserve(budget.capacity() - budget.used())?; assert!( - cache - .store_blob(blobs.read(&reference).await?) + other + .install_pack(pack.clone()) .await .is_err_and(|error| error.is_admission()) ); + assert_eq!(other.missing_objects(vec![oid]).await?, vec![oid]); + drop(occupied); + drop(other); drop(cache); - assert_eq!(budget.used(), occupied.bytes()); - Ok(()) -} - -#[tokio::test] -async fn corrupt_stream_never_installs_a_reusable_object() -> Result<(), Box> -{ - use object_store::ObjectStoreExt; - let root = tempfile::TempDir::new()?; - let budget = DiskBudget::new(1 << 20); - let cache = GitCache::create( - root.path().into(), - budget.clone(), - "refs/heads/main", - crate::ObjectFormat::Sha1, - ) - .await?; - let store = Arc::new(object_store::memory::InMemory::new()); - let blobs = crate::large_blob::LargeBlobStore::new(store.clone(), [9; 16]); - let mut body = vec![23; 9 * 1024 * 1024]; - let reference = blobs - .put( - object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, &body), - body.len() as u64, - &mut body.as_slice(), - ) - .await?; - // The first range can be written before the final range detects corruption. - // Only the test-owned blob key is changed, then restored before retry. - let path = crate::external::part(&crate::large_blob::blob_path([9; 16], &reference.sha256), 1); - let metadata = store.head(&path).await?; - let last = body.last_mut().ok_or("empty fixture")?; - *last ^= 1; - store - .put(&metadata.location, body[8 * 1024 * 1024..].to_vec().into()) - .await?; - assert!( - cache - .store_blob(blobs.read(&reference).await?) - .await - .is_err() - ); - assert_eq!( - cache.missing_objects(vec![reference.oid]).await?, - vec![reference.oid] - ); - *body.last_mut().ok_or("empty fixture")? ^= 1; - store - .put(&metadata.location, body[8 * 1024 * 1024..].to_vec().into()) - .await?; - cache.store_blob(blobs.read(&reference).await?).await?; - assert!(cache.missing_objects(vec![reference.oid]).await?.is_empty()); - drop(cache); + drop(pack); + drop(packs); assert_eq!(budget.used(), 0); Ok(()) } diff --git a/src/git_gateway.rs b/src/git_gateway.rs index 5c021e62..708763eb 100644 --- a/src/git_gateway.rs +++ b/src/git_gateway.rs @@ -18,14 +18,13 @@ use object_store::ObjectStore; use tokio::sync::{Mutex, OnceCell}; use crate::{ - INLINE_OBJECT_LIMIT, ObjectBatch, ObjectKind, ObjectStorage, PushPlan, RefExpectation, - RefUpdate, RepositoryCell, StoredObject, + ObjectBatch, ObjectKind, ObjectStorage, PackedObject, PushPlan, RefExpectation, RefUpdate, + RepositoryCell, StoredObject, directory::TokenScope, git_cache::{CacheError, GitCache}, git_http::{GitHttpBackend, GitHttpError, GitHttpRequest, GitHttpResponse}, git_input::{GitInput, InputError, MAX_FETCH_REQUEST_BYTES}, git_objects::GitObjects, - large_blob::{LargeBlobError, LargeBlobStore}, lfs::LfsService, object_batch::MAX_OBJECTS, push::{PushCompletion, PushError}, @@ -57,7 +56,7 @@ pub enum GatewayError { #[error("repository Cell operation failed")] Cell(#[source] CellError), #[error("large Git blob store failed")] - Blob(#[from] LargeBlobError), + Pack(#[from] crate::git_pack::PackError), #[error("cache I/O failed")] Io(#[from] std::io::Error), #[error("Git cache command failed: {0}")] @@ -102,7 +101,6 @@ pub struct GitGateway { repository: Arc, signer_directory: Option>, certificate_seed: OnceCell<[u8; 32]>, - large_blobs: LargeBlobStore, lfs: LfsService, scratch_root: PathBuf, disk_budget: DiskBudget, @@ -118,13 +116,11 @@ impl GitGateway { blob_store: Arc, disk_budget: DiskBudget, ) -> Self { - let large_blobs = LargeBlobStore::new(Arc::clone(&blob_store), repository.repository_id()); let lfs = LfsService::new(Arc::clone(&repository), blob_store); Self { repository, signer_directory: None, certificate_seed: OnceCell::new(), - large_blobs, lfs, scratch_root, disk_budget, @@ -438,7 +434,27 @@ impl GitGateway { } // Published refs already have durable graph closure. Excluding them avoids // re-reading old history; the final Cell transaction still verifies every new tip. - let excluded = before.values().filter_map(|state| state.oid).collect(); + let excluded: Vec<_> = before.values().filter_map(|state| state.oid).collect(); + let [structural, blobs] = + GitObjects::classify(&backend.git_dir(), included.clone(), excluded.clone()).await?; + if structural.is_empty() && blobs.is_empty() { + return Ok(()); + } + // Separate compact structural reads from potentially large blob data; + // both representations remain immutable native packs. + let mut pairs = Vec::new(); + for ids in [structural, blobs] { + if !ids.is_empty() { + let pair = self + .repository + .packs + .build(Arc::clone(&backend.cache), ids, Vec::new(), false) + .await?; + let identity = self.repository.packs.publish(Arc::clone(&pair)).await?; + pairs.push((pair, identity)); + } + } + let mut objects = GitObjects::start(&backend.git_dir(), included, excluded)?; let mut batch = ObjectBatch::default(); loop { @@ -459,38 +475,39 @@ impl GitGateway { .map_err(|error| GatewayError::Cell(Box::new(error)))? .output; for oid in candidates.into_iter().filter(|oid| !present.contains(oid)) { - let mut input = objects.read(oid).await?; - let object = - if input.kind == ObjectKind::Blob && input.size > INLINE_OBJECT_LIMIT as u64 { - let uploaded = self - .large_blobs - .put(oid, input.size, &mut input.reader) - .await?; - input.finish().await?; - StoredObject { - oid, - kind: ObjectKind::Blob, - storage: ObjectStorage::External { - size: uploaded.size, - blake3: uploaded.blake3, - sha256: uploaded.sha256, - }, - } - } else { - let (kind, body) = input.body().await?; - if body.len() > INLINE_OBJECT_LIMIT { - self.repository - .stage_object(new_identity()?, kind, &body) - .await - .map_err(|error| GatewayError::Cell(Box::new(error)))? - } else { - StoredObject { - oid, - kind, - storage: ObjectStorage::Inline(body), - } - } - }; + let (kind, size, digest, dependencies, rebase_metadata) = + objects.read(oid).await?.metadata().await?; + let mut selected = None; + for (pair, identity) in &pairs { + if pair.data()?.contains(oid)? { + selected = Some((pair, identity)); + break; + } + } + let (pair, pack) = selected.ok_or(GatewayError::MalformedCache)?; + let header = pair + .data()? + .header(oid)? + .ok_or(GatewayError::MalformedCache)?; + if header.kind != kind || header.size != size { + return Err(GatewayError::MalformedCache); + } + let mut object = StoredObject { + oid, + kind, + storage: ObjectStorage::Packed(Box::new(PackedObject::verified( + self.repository.id, + pack.clone(), + oid, + kind, + size, + digest, + dependencies, + ))), + }; + if let ObjectStorage::Packed(value) = &mut object.storage { + value.rebase_metadata = rebase_metadata; + } if let Err(object) = batch.try_push(object) { self.repository .put_objects(new_identity()?, std::mem::take(&mut batch)) diff --git a/src/git_gateway/fetch.rs b/src/git_gateway/fetch.rs index 13910f75..5285864f 100644 --- a/src/git_gateway/fetch.rs +++ b/src/git_gateway/fetch.rs @@ -4,6 +4,34 @@ use cellule_runtime::primitives::sql::{SqlBatch, SqlStatement, SqlValue}; // Correlate each ancestor lookup so SQLite can stop at the first live ref, // without materializing the entire reverse graph or scanning all refs. const REACHABLE_WANT: &str = "WITH RECURSIVE ancestors(oid) AS (VALUES (?1) UNION SELECT e.parent FROM object_edges e JOIN ancestors a ON e.child = a.oid) SELECT g.generation, EXISTS (SELECT 1 FROM ancestors a WHERE EXISTS (SELECT 1 FROM refs r WHERE r.oid = a.oid)) FROM ref_generation g WHERE g.singleton = 1"; +// The protocol ceiling is 128 statements, but scattered cold index reads also +// share Cellule's five-second execution budget. Leave headroom for each batch. +const MAX_REACHABILITY_STATEMENTS: usize = 16; +const GRAPH_READ_PARENTS: usize = 64; +const GRAPH_READ_ROWS: usize = 128; + +fn graph_page(ids: &[crate::ObjectId], parent: &[u8], child: &[u8]) -> SqlBatch { + let remaining: Vec<_> = ids.iter().filter(|oid| oid.as_ref() > parent).collect(); + let mut parameters = vec![ + SqlValue::Blob(parent.to_vec()), + SqlValue::Blob(child.to_vec()), + ]; + parameters.extend(remaining.iter().map(|oid| SqlValue::Blob(oid.to_vec()))); + parameters.push(SqlValue::Integer(GRAPH_READ_ROWS as i64)); + SqlBatch { + statements: vec![SqlStatement { + // Seek the current parent's child directly; never rescan its old + // children or completed parents. Limit edges before looking up type. + // Blob filtering happens after this bounded read, so a blob-heavy + // tree cannot force an unbounded scan to fill a structural page. + sql: format!( + "SELECT e.parent, e.child, o.kind FROM (SELECT parent, child FROM object_edges WHERE parent = ? AND child > ? UNION ALL SELECT parent, child FROM object_edges WHERE parent IN ({}) ORDER BY parent, child LIMIT ?) e JOIN objects o INDEXED BY objects_by_oid_kind ON o.oid = e.child ORDER BY e.parent, e.child", + vec!["?"; remaining.len()].join(",") + ), + parameters, + }], + } +} pub(super) struct FetchRequest { pub(super) wants: BTreeSet, @@ -195,7 +223,9 @@ impl GitGateway { // certified by the Cell also fence trees and blobs, including cached // objects retained after a ref deletion. Generation binds every batch. let ids: Vec<_> = wants.iter().collect(); - for ids in ids.chunks(MAX_OBJECTS) { + // Each wanted ID consumes one statement, independently of the object + // metadata page size. Use smaller batches within SQL's 128-statement cap. + for ids in ids.chunks(MAX_REACHABILITY_STATEMENTS) { let result = self .repository .sql @@ -287,34 +317,22 @@ impl GitGateway { let mut visited = BTreeSet::new(); let mut stats = Hydration::default(); while !pending.is_empty() { - let ids: Vec<_> = pending.iter().take(MAX_OBJECTS).copied().collect(); + let ids: Vec<_> = pending.iter().take(GRAPH_READ_PARENTS).copied().collect(); for oid in &ids { pending.remove(oid); visited.insert(*oid); } - let placeholders = vec!["?"; ids.len()].join(","); - let kind_filter = if include_blobs { - "" - } else { - "AND o.kind != 'blob'" - }; let mut after_parent = Vec::new(); let mut after_child = Vec::new(); loop { // Certified edges supply only reachable structure. Page both // parents and children so a large tree stays within SQL wire bounds. - let mut parameters: Vec<_> = - ids.iter().map(|oid| SqlValue::Blob(oid.to_vec())).collect(); - parameters.extend([ - SqlValue::Blob(after_parent.clone()), - SqlValue::Blob(after_parent.clone()), - SqlValue::Blob(after_child.clone()), - SqlValue::Integer(MAX_OBJECTS as i64), - ]); - let result = self.repository.sql.query(None, SqlBatch { statements: vec![SqlStatement { - sql: format!("SELECT e.parent, e.child, o.kind FROM object_edges e JOIN objects o ON o.oid = e.child WHERE e.parent IN ({placeholders}) {kind_filter} AND (e.parent > ? OR (e.parent = ? AND e.child > ?)) ORDER BY e.parent, e.child LIMIT ?"), - parameters, - }] }).await.map_err(|error| GatewayError::Cell(Box::new(error)))?; + let result = self + .repository + .sql + .query(None, graph_page(&ids, &after_parent, &after_child)) + .await + .map_err(|error| GatewayError::Cell(Box::new(error)))?; let rows = &result .output .first() @@ -347,7 +365,7 @@ impl GitGateway { if !blobs.is_empty() { self.hydrate_objects(cache, blobs, &mut stats).await?; } - if rows.len() < MAX_OBJECTS { + if rows.len() < GRAPH_READ_ROWS { break; } } @@ -393,12 +411,94 @@ impl GitGateway { mod tests { use super::*; + #[test] + fn graph_pages_seek_dense_parents_and_cover_kind_without_wide_rows() + -> Result<(), Box> { + use cellule_ltx::rusqlite::{ + Connection, StatementStatus, params, params_from_iter, types::Value, + }; + for format in [crate::ObjectFormat::Sha1, crate::ObjectFormat::Sha256] { + let db = Connection::open_in_memory()?; + db.execute_batch("CREATE TABLE objects(sequence INTEGER PRIMARY KEY, oid BLOB UNIQUE, kind TEXT, padding BLOB); CREATE INDEX objects_by_oid_kind ON objects(oid, kind); CREATE TABLE object_edges(parent BLOB, child BLOB, PRIMARY KEY(parent, child)) WITHOUT ROWID")?; + let oid = |n: u32| { + let mut bytes = vec![0; format.bytes()]; + bytes[..4].copy_from_slice(&n.to_be_bytes()); + crate::ObjectId::try_from(bytes).unwrap() + }; + let ids: Vec<_> = [10, 30, 40].into_iter().map(oid).collect(); + let mut expected = Vec::new(); + db.execute_batch("BEGIN")?; + for n in 0..2_071 { + let child = oid(100 + n); + let kind = if n % 43 == 0 { "tree" } else { "blob" }; + db.execute( + "INSERT INTO objects(oid, kind, padding) VALUES (?1, ?2, zeroblob(4096))", + params![child.as_ref(), kind], + )?; + for parent in ids.iter().copied().chain([oid(20)]) { + db.execute( + "INSERT INTO object_edges VALUES (?1, ?2)", + params![parent.as_ref(), child.as_ref()], + )?; + if ids.contains(&parent) { + expected.push((parent.to_vec(), child.to_vec(), kind.to_owned())); + } + } + } + db.execute_batch("COMMIT")?; + expected.sort(); + let mut cursor = (Vec::new(), Vec::new()); + let mut actual = Vec::new(); + loop { + let batch = graph_page(&ids, &cursor.0, &cursor.1); + let query = &batch.statements[0]; + let values: Vec<_> = query + .parameters + .iter() + .map(|value| match value { + SqlValue::Blob(value) => Value::Blob(value.clone()), + SqlValue::Integer(value) => Value::Integer(*value), + _ => unreachable!(), + }) + .collect(); + let plans: Vec = db + .prepare(&format!("EXPLAIN QUERY PLAN {}", query.sql))? + .query_map(params_from_iter(&values), |row| row.get(3))? + .collect::>()?; + assert!( + plans + .iter() + .any(|plan| plan.contains("COVERING INDEX objects_by_oid_kind")), + "{plans:?}" + ); + let mut statement = db.prepare(&query.sql)?; + let rows: Vec<(Vec, Vec, String)> = statement + .query_map(params_from_iter(&values), |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + })? + .collect::>()?; + assert!(rows.len() <= GRAPH_READ_ROWS); + assert!(statement.get_status(StatementStatus::VmStep) < 30_000); + if rows.is_empty() { + break; + } + assert!((&rows[0].0, &rows[0].1) > (&cursor.0, &cursor.1)); + let last = rows.last().unwrap(); + cursor = (last.0.clone(), last.1.clone()); + actual.extend(rows); + } + assert_eq!(actual, expected); + } + Ok(()) + } + #[test] fn reachability_stops_at_live_refs_without_scanning_other_history() -> Result<(), Box> { use cellule_ltx::rusqlite::{Connection, StatementStatus, params}; let db = Connection::open_in_memory()?; db.execute_batch(crate::SCHEMA)?; + db.execute("INSERT INTO git_packs (digest, format, bytes, index_digest, index_bytes) VALUES (zeroblob(32), 'sha1', 1, zeroblob(32), 1)", [])?; let oid = |n: u32| { let mut oid = [0; 20]; oid[..4].copy_from_slice(&n.to_be_bytes()); @@ -406,7 +506,7 @@ mod tests { }; db.execute_batch("BEGIN")?; for n in 0..10_000 { - db.execute("INSERT INTO objects (oid, kind, size, digest, storage, body) VALUES (?1, 'blob', 0, zeroblob(32), 'inline', X'')", [oid(n).as_slice()])?; + db.execute("INSERT INTO objects (oid, kind, size, digest, pack, graph_valid, graph_count, graph_digest) VALUES (?1, 'blob', 0, zeroblob(32), zeroblob(32), 1, 0, zeroblob(32))", [oid(n).as_slice()])?; db.execute( "INSERT INTO refs (name, oid, version) VALUES (?1, ?2, 1)", params![format!("refs/tags/{n}"), oid(n).as_slice()], diff --git a/src/git_gateway/hydration.rs b/src/git_gateway/hydration.rs index 86442141..c0cd6071 100644 --- a/src/git_gateway/hydration.rs +++ b/src/git_gateway/hydration.rs @@ -1,5 +1,4 @@ use super::*; -use crate::large_blob::LargeBlobReference; use std::time::{Duration, Instant}; #[derive(Default)] @@ -77,50 +76,61 @@ impl GitGateway { stats: &mut Hydration, ) -> Result, GatewayError> { let read = Instant::now(); - let body = match object.storage { - ObjectStorage::Inline(body) => body, - ObjectStorage::Chunked { - upload, - size, - blake3, - } => self + let ObjectStorage::Packed(reference) = object.storage else { + return Err(GatewayError::MalformedCache); + }; + let pack = self.repository.packs.load(&reference.pack).await?; + let header = pack + .data()? + .header(object.oid)? + .ok_or(GatewayError::MalformedCache)?; + if header.kind != object.kind || header.size != reference.size { + return Err(GatewayError::MalformedCache); + } + let target = if object.kind == ObjectKind::Tag { + let edges = self .repository - .chunked_body(object.oid, object.kind, upload, size, blake3) + .dependencies(object.oid) .await - .map_err(|error| GatewayError::Cell(Box::new(error)))?, - ObjectStorage::External { - size, - blake3, - sha256, - } => { - let reference = LargeBlobReference { - oid: object.oid, - size, - blake3, - sha256, - }; - let reader = self.large_blobs.read(&reference).await?; - stats.body_time += read.elapsed(); - let written = Instant::now(); - cache.store_blob(reader).await?; - stats.cache_time += written.elapsed(); - stats.objects += 1; - stats.bytes += reference.size; - return Ok(None); + .map_err(|error| GatewayError::Cell(Box::new(error)))? + .ok_or(GatewayError::MalformedCache)?; + match edges.as_slice() { + [(target, _)] => Some(*target), + _ => return Err(GatewayError::MalformedCache), } + } else { + None }; stats.body_time += read.elapsed(); stats.objects += 1; - stats.bytes += body.len() as u64; - let target = (object.kind == ObjectKind::Tag) - .then(|| { - crate::graph::tag_edge(&body) - .map(|(oid, _)| oid) - .filter(|target| target.format() == object.oid.format()) - }) - .flatten(); + stats.bytes += header.size; let written = Instant::now(); - cache.store_object(object.oid, object.kind, body).await?; + if header.size <= 4 * 1024 * 1024 { + let oid = object.oid; + let expected = reference.digest; + let kind = object.kind; + let body = tokio::task::spawn_blocking(move || { + let (found, body) = pack + .data()? + .object(oid, 4 * 1024 * 1024)? + .ok_or(crate::git_pack::PackError::Integrity)?; + if found != kind || blake3::hash(&body).as_bytes() != &expected { + return Err(crate::git_pack::PackError::Integrity); + } + Ok::<_, crate::git_pack::PackError>(body) + }) + .await??; + cache.store_object(object.oid, object.kind, body).await?; + } else { + cache + .store_native_object( + pack, + *reference, + self.scratch_root.clone(), + self.disk_budget.clone(), + ) + .await?; + } stats.cache_time += written.elapsed(); Ok(target) } diff --git a/src/git_gateway/push.rs b/src/git_gateway/push.rs index 81a7203c..3c1969e0 100644 --- a/src/git_gateway/push.rs +++ b/src/git_gateway/push.rs @@ -66,6 +66,9 @@ impl GitGateway { Ok(prepared) => prepared, Err(error) => { let reason = match &error { + GatewayError::Pack(error) if error.is_admission() => { + "Canopy push failed before publication: pack disk budget exhausted" + } GatewayError::Cache(error) | GatewayError::Http(GitHttpError::Cache(error)) if error.is_admission() => { diff --git a/src/git_http.rs b/src/git_http.rs index 4075c1df..86a42b1e 100644 --- a/src/git_http.rs +++ b/src/git_http.rs @@ -30,6 +30,9 @@ const MAX_CGI_OUTPUT_BYTES: usize = 64 * 1024 * 1024; const MAX_CGI_STDERR_BYTES: usize = 64 * 1024; const CHUNK_BYTES: usize = 64 * 1024; const MAX_CGI_HEADER_BYTES: usize = 64 * 1024; +// Native pack generation and large transfers can exceed two minutes. Keep a +// finite lifetime while allowing complete repository histories to be served. +const CGI_DEADLINE: Duration = Duration::from_secs(30 * 60); #[derive(Debug, thiserror::Error)] pub enum GitHttpError { @@ -243,7 +246,7 @@ impl GitHttpBackend { start_stream( process, (keep_alive, Arc::clone(&self.cache), request.body), - Duration::from_secs(120), + CGI_DEADLINE, ) .await } diff --git a/src/git_objects.rs b/src/git_objects.rs index e5e18714..7022dc3e 100644 --- a/src/git_objects.rs +++ b/src/git_objects.rs @@ -9,7 +9,9 @@ use tokio::{ }; use tokio_util::task::AbortOnDropHandle; -use crate::{INLINE_OBJECT_LIMIT, ObjectKind, object_id}; +use crate::ObjectKind; +#[cfg(test)] +use crate::{INLINE_OBJECT_LIMIT, object_id}; const IO_TIMEOUT: Duration = Duration::from_secs(120); const HEADER_LIMIT: usize = 128; @@ -173,6 +175,53 @@ pub(crate) struct GitObjects { } impl GitObjects { + pub(crate) async fn classify( + git_dir: &Path, + included: Vec, + excluded: Vec, + ) -> Result<[Vec; 2], ObjectReadError> { + let mut walk = GitObjectWalk::start(git_dir, included, excluded, false, None)?; + let (mut batch, mut requests) = Process::start(git_dir, &["cat-file", "--batch-check"])?; + let mut groups = [Vec::new(), Vec::new()]; + while let Some(oid) = walk.next().await? { + let line = timeout(IO_TIMEOUT, async { + requests + .write_all(format!("{}\n", hex::encode(oid)).as_bytes()) + .await?; + header(&mut batch.output) + .await? + .ok_or(ObjectReadError::Malformed) + }) + .await + .map_err(|_| ObjectReadError::Timeout)??; + let mut fields = line.split(|byte| *byte == b' '); + if parse_oid(fields.next().ok_or(ObjectReadError::Malformed)?)? != oid { + return Err(ObjectReadError::Malformed); + } + let kind = fields.next().ok_or(ObjectReadError::Malformed)?; + let group = match kind { + b"blob" => 1, + b"tree" | b"commit" | b"tag" => 0, + _ => return Err(ObjectReadError::Malformed), + }; + let size = fields.next().ok_or(ObjectReadError::Malformed)?; + if size.is_empty() || !size.iter().all(u8::is_ascii_digit) || fields.next().is_some() { + return Err(ObjectReadError::Malformed); + } + groups[group] + .try_reserve(1) + .map_err(ObjectReadError::Allocation)?; + groups[group].push(oid); + } + walk.finish().await?; + drop(requests); + if header(&mut batch.output).await?.is_some() { + return Err(ObjectReadError::Malformed); + } + batch.finish().await?; + Ok(groups) + } + pub(crate) fn start( git_dir: &Path, included: Vec, @@ -247,6 +296,7 @@ pub(crate) struct GitObject<'a, R> { } impl GitObject<'_, R> { + #[cfg(test)] pub(crate) async fn body(mut self) -> Result<(ObjectKind, Vec), ObjectReadError> { let limit = if self.kind == ObjectKind::Blob { INLINE_OBJECT_LIMIT @@ -277,6 +327,56 @@ impl GitObject<'_, R> { .map_err(|_| ObjectReadError::Timeout)? } + pub(crate) async fn metadata( + mut self, + ) -> Result< + ( + ObjectKind, + u64, + [u8; 32], + Option>, + Option>, + ), + ObjectReadError, + > { + let kind = self.kind; + let size = self.size; + let oid = self.oid; + let mut graph = crate::graph::stream::StreamGraph::new(oid.format(), kind); + let mut small_commit = (kind == ObjectKind::Commit && size <= 64 * 1024) + .then(|| Vec::with_capacity(size as usize)); + let mut git = crate::git_format::ObjectHasher::new(oid.format(), kind, size); + let mut hash = blake3::Hasher::new(); + let mut buffer = vec![0; 1024 * 1024]; + while self.reader.limit() != 0 { + let count = timeout(IO_TIMEOUT, self.reader.read(&mut buffer)) + .await + .map_err(|_| ObjectReadError::Timeout)??; + if count == 0 { + return Err(ObjectReadError::Malformed); + } + git.update(&buffer[..count]); + hash.update(&buffer[..count]); + graph + .feed(&buffer[..count]) + .map_err(ObjectReadError::Allocation)?; + if let Some(body) = &mut small_commit { + body.extend_from_slice(&buffer[..count]); + } + } + self.finish().await?; + if git.finalize() != oid { + return Err(ObjectReadError::Malformed); + } + Ok(( + kind, + size, + *hash.finalize().as_bytes(), + graph.finish(), + small_commit.and_then(|body| crate::pulls::candidates::rebase_metadata(kind, &body)), + )) + } + // cat-file separates bodies with a newline. The sized reader prevents an // external upload from consuming that separator or the following response. pub(crate) async fn finish(self) -> Result<(), ObjectReadError> { diff --git a/src/git_objects/tests.rs b/src/git_objects/tests.rs index f49285eb..24660e91 100644 --- a/src/git_objects/tests.rs +++ b/src/git_objects/tests.rs @@ -196,22 +196,14 @@ async fn batch_reads_large_blob_across_pipe_buffers() -> TestResult { let oid = parse_oid(output.trim_ascii())?; let mut objects = GitObjects::start(&directory.path().join(".git"), vec![oid], vec![])?; assert_eq!(objects.next().await?, Some(oid)); - let mut object = objects.read(oid).await?; - let store = crate::large_blob::LargeBlobStore::new( - std::sync::Arc::new(object_store::memory::InMemory::new()), - [1; 16], - ); - let reference = store.put(oid, object.size, &mut object.reader).await?; - object.finish().await?; + let (kind, size, digest, edges, projection) = objects.read(oid).await?.metadata().await?; + assert!(projection.is_none()); + assert_eq!(kind, ObjectKind::Blob); + assert_eq!(size, body.len() as u64); + assert_eq!(digest, *blake3::hash(&body).as_bytes()); + assert_eq!(edges, Some(Vec::new())); assert_eq!(objects.next().await?, None); objects.finish().await?; - let mut reader = store.read(&reference).await?; - let mut offset = 0; - while let Some(bytes) = reader.next().await? { - assert_eq!(bytes.as_ref(), &body[offset..offset + bytes.len()]); - offset += bytes.len(); - } - assert_eq!(offset, body.len()); Ok(()) } diff --git a/src/git_pack.rs b/src/git_pack.rs new file mode 100644 index 00000000..a3dad7cf --- /dev/null +++ b/src/git_pack.rs @@ -0,0 +1,274 @@ +//! Verified native Git packs with bounded delta decoding. + +use std::{fs::File, io::Read, path::Path}; + +use crate::{ObjectFormat, ObjectId, ObjectKind, object_id}; +use serde::{Deserialize, Serialize}; +use sha1::Digest as _; + +#[derive(Debug, thiserror::Error)] +pub enum PackError { + #[error("Git pack object-store operation failed")] + Store(#[from] object_store::Error), + #[error("Git pack disk admission failed")] + Budget(#[from] cellule_ltx::LtxError), + #[error("Git pack worker failed")] + Task(#[from] tokio::task::JoinError), + #[error("Git pack I/O failed")] + Io(#[from] std::io::Error), + #[error("Git pack decoding failed")] + Decode(#[source] Box), + #[error("Git pack integrity or object identity differs")] + Integrity, + #[error("Git pack read exceeds its allocation or output bound")] + Limit, + #[error("Git pack allocation failed")] + Allocation(#[from] std::collections::TryReserveError), +} + +impl PackError { + pub(crate) fn is_admission(&self) -> bool { + match self { + Self::Budget(_) => true, + Self::Io(error) => error + .get_ref() + .is_some_and(|source| source.is::()), + Self::Decode(error) => error + .downcast_ref::() + .is_some_and(crate::git_cache::CacheError::is_admission), + _ => false, + } + } +} + +mod build; +mod store; +pub use store::{OwnedPack, PackStore}; + +/// Exact immutable pack/index bytes selected by repository metadata. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PackIdentity { + pub format: ObjectFormat, + pub pack_digest: [u8; 32], + pub pack_bytes: u64, + pub index_digest: [u8; 32], + pub index_bytes: u64, +} + +/// Object metadata obtained without expanding its complete delta chain. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PackObjectHeader { + pub oid: ObjectId, + pub kind: ObjectKind, + pub size: u64, + pub offset: u64, + pub delta_depth: u32, +} + +/// Private, immutable local pack owned by the repository's disk cache. +/// +/// The owner must retain the files and their disk reservation until this value +/// and all readers close. Native Git never modifies these files in place. +pub struct GitPack { + identity: PackIdentity, + data: gix_pack::data::File, + index: gix_pack::index::File>, +} + +impl GitPack { + /// Opens an exact content-addressed pair. Index and delta allocations have + /// explicit caller budgets; the pack body itself is read-only memory mapped. + pub fn open( + path: &Path, + expected: PackIdentity, + max_index_bytes: usize, + max_decode_bytes: usize, + ) -> Result { + if expected.index_bytes > max_index_bytes as u64 || max_decode_bytes == 0 { + return Err(PackError::Limit); + } + let index_path = path.with_extension("idx"); + let mut file = File::open(&index_path)?; + if file.metadata()?.len() != expected.index_bytes { + return Err(PackError::Integrity); + } + let index_length = usize::try_from(expected.index_bytes).map_err(|_| PackError::Limit)?; + let mut index_bytes = Vec::new(); + index_bytes.try_reserve_exact(index_length)?; + index_bytes.resize(index_length, 0); + file.read_exact(&mut index_bytes)?; + if file.read(&mut [0; 1])? != 0 { + return Err(PackError::Integrity); + } + if index_bytes.len() as u64 != expected.index_bytes + || *blake3::hash(&index_bytes).as_bytes() != expected.index_digest + { + return Err(PackError::Integrity); + } + verify_trailer(&index_bytes, expected.format)?; + let index = gix_pack::index::File::from_data( + index_bytes.into_boxed_slice(), + index_path, + hash_kind(expected.format), + ) + .map_err(decode_error)?; + let data = gix_pack::data::File::at(path, hash_kind(expected.format)) + .map_err(decode_error)? + .with_alloc_limit_bytes(Some(max_decode_bytes)); + let bytes = data + .entry_slice(0..data.data_len() as u64) + .ok_or(PackError::Integrity)?; + if bytes.len() as u64 != expected.pack_bytes + || *blake3::hash(bytes).as_bytes() != expected.pack_digest + || data.checksum() != index.pack_checksum() + || data.num_objects() != index.num_objects() + { + return Err(PackError::Integrity); + } + verify_trailer(bytes, expected.format)?; + Ok(Self { + identity: expected, + data, + index, + }) + } + + pub fn identity(&self) -> &PackIdentity { + &self.identity + } + + pub fn object_ids(&self) -> impl Iterator> + '_ { + self.index.iter().map(|entry| { + entry + .oid + .as_bytes() + .try_into() + .map_err(|_| PackError::Integrity) + }) + } + + /// Tests index membership without allocating or expanding a delta chain. + pub fn contains(&self, oid: ObjectId) -> Result { + Ok(self.offset(oid)?.is_some()) + } + + pub fn header(&self, oid: ObjectId) -> Result, PackError> { + let Some(offset) = self.offset(oid)? else { + return Ok(None); + }; + let header = self + .data + .decode_header( + self.data.entry(offset).map_err(decode_error)?, + &mut gix_zlib::Inflate::default(), + &|oid| { + self.index + .lookup(oid) + .map(|index| self.data.entry(self.index.pack_offset_at_index(index))) + .transpose() + .map(|entry| { + entry.map(gix_pack::data::decode::header::ResolvedBase::InPack) + }) + }, + ) + .map_err(decode_error)?; + Ok(Some(PackObjectHeader { + oid, + kind: parse_kind(header.kind.as_bytes())?, + size: header.object_size, + offset, + delta_depth: header.num_deltas, + })) + } + + /// Decodes a self-contained native pack entry and checks its canonical Git + /// object ID. Ref deltas may resolve only within this same verified pack. + pub fn object( + &self, + oid: ObjectId, + max_output_bytes: usize, + ) -> Result)>, PackError> { + let Some(header) = self.header(oid)? else { + return Ok(None); + }; + if header.size > max_output_bytes as u64 { + return Err(PackError::Limit); + } + let mut body = Vec::new(); + let outcome = self + .data + .decode_entry( + self.data.entry(header.offset).map_err(decode_error)?, + &mut body, + &mut gix_zlib::Inflate::default(), + &|oid, _| { + self.index + .lookup(oid) + .map(|index| self.data.entry(self.index.pack_offset_at_index(index))) + .transpose() + .map(|entry| entry.map(gix_pack::data::decode::entry::ResolvedBase::InPack)) + }, + &mut gix_pack::cache::Never, + ) + .map_err(decode_error)?; + let kind = parse_kind(outcome.kind.as_bytes())?; + if kind != header.kind + || body.len() as u64 != header.size + || object_id(oid.format(), kind, &body) != oid + { + return Err(PackError::Integrity); + } + Ok(Some((kind, body))) + } + + fn offset(&self, oid: ObjectId) -> Result, PackError> { + if oid.format() != self.identity.format { + return Err(PackError::Integrity); + } + let oid = gix_hash::oid::try_from_bytes(&oid).map_err(decode_error)?; + Ok(self + .index + .lookup(oid) + .map(|index| self.index.pack_offset_at_index(index))) + } +} + +fn hash_kind(format: ObjectFormat) -> gix_hash::Kind { + match format { + ObjectFormat::Sha1 => gix_hash::Kind::Sha1, + ObjectFormat::Sha256 => gix_hash::Kind::Sha256, + } +} + +fn parse_kind(kind: &[u8]) -> Result { + match kind { + b"blob" => Ok(ObjectKind::Blob), + b"tree" => Ok(ObjectKind::Tree), + b"commit" => Ok(ObjectKind::Commit), + b"tag" => Ok(ObjectKind::Tag), + _ => Err(PackError::Integrity), + } +} + +fn verify_trailer(bytes: &[u8], format: ObjectFormat) -> Result<(), PackError> { + let length = bytes + .len() + .checked_sub(format.bytes()) + .ok_or(PackError::Integrity)?; + let digest = match format { + ObjectFormat::Sha1 => sha1::Sha1::digest(&bytes[..length]).to_vec(), + ObjectFormat::Sha256 => sha2::Sha256::digest(&bytes[..length]).to_vec(), + }; + if digest != bytes[length..] { + return Err(PackError::Integrity); + } + Ok(()) +} + +fn decode_error(error: impl std::error::Error + Send + Sync + 'static) -> PackError { + PackError::Decode(Box::new(error)) +} + +#[cfg(test)] +mod tests; diff --git a/src/git_pack/build.rs b/src/git_pack/build.rs new file mode 100644 index 00000000..641d0f86 --- /dev/null +++ b/src/git_pack/build.rs @@ -0,0 +1,241 @@ +//! Supervised native pack creation with disk admission before every write. + +use super::{OwnedPack, PackError, PackIdentity, PackStore}; +use crate::{ObjectId, git_cache::GitCache}; +use std::{io::Read, path::Path, process::Stdio, sync::Arc}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +const COPY_BYTES: usize = 1024 * 1024; + +impl PackStore { + /// Creates a verified native pair from bounded in-memory ingress. Bodies + /// are transient: callers publish the pair before recording its metadata. + pub async fn prepare( + &self, + format: crate::ObjectFormat, + objects: Vec<(crate::ObjectKind, Vec)>, + ) -> Result, PackError> { + if objects.is_empty() || objects.len() > 512 { + return Err(PackError::Limit); + } + let bytes = objects + .iter() + .try_fold(0usize, |sum, (_, body)| sum.checked_add(body.len())) + .ok_or(PackError::Limit)?; + if bytes > 64 * 1024 * 1024 { + return Err(PackError::Limit); + } + let store = self.clone(); + tokio::spawn(async move { + let cache = GitCache::create( + store.local_root().into(), + store.disk_budget().clone(), + "refs/heads/main", + format, + ) + .await + .map_err(|error| match error { + crate::git_cache::CacheError::Io(error) => PackError::Io(error), + crate::git_cache::CacheError::Budget(error) => PackError::Budget(error), + error => PackError::Decode(Box::new(error)), + })?; + let mut included = Vec::with_capacity(objects.len()); + for (kind, body) in objects { + let oid = crate::object_id(format, kind, &body); + cache + .store_object(oid, kind, body) + .await + .map_err(|error| PackError::Decode(Box::new(error)))?; + included.push(oid); + } + included.sort_unstable(); + included.dedup(); + store.build(cache, included, Vec::new(), false).await + }) + .await? + } + + /// Produces a self-contained pack containing the selected new history. + /// Native Git may reuse in-pack deltas but is never asked for a thin pack. + /// The supervised task retains the cache, native fence, files and admission + /// if the request which dispatched it is cancelled. + pub(crate) async fn build( + &self, + cache: Arc, + included: Vec, + excluded: Vec, + walk: bool, + ) -> Result, PackError> { + let store = self.clone(); + tokio::spawn(async move { store.build_inner(cache, included, excluded, walk).await }) + .await? + } + + async fn build_inner( + &self, + cache: Arc, + included: Vec, + excluded: Vec, + walk: bool, + ) -> Result, PackError> { + if included.is_empty() + || included + .iter() + .chain(&excluded) + .any(|oid| oid.format() != cache.object_format()) + || (!walk && !excluded.is_empty()) + { + return Err(PackError::Integrity); + } + let directory = tempfile::Builder::new() + .prefix("canopy-pack-build-") + .tempdir_in(cache.git_dir())?; + let owned = OwnedPack::pending(directory, self.disk_budget().try_reserve(0)?) + .with_native_owner(Arc::clone(&cache)); + let path = owned.directory()?.join("objects.pack"); + let mut output = tokio::fs::File::create(&path).await?; + let mut command = crate::native_git::command(&cache.git_dir())?; + command.args([ + "-c", + "core.bigFileThreshold=8m", + "pack-objects", + "--stdout", + "--delta-base-offset", + "--no-reuse-delta", + ]); + if walk { + command.arg("--revs"); + } + let mut child = command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .spawn()?; + let mut input = child.stdin.take().ok_or(PackError::Integrity)?; + let mut stream = child.stdout.take().ok_or(PackError::Integrity)?; + let writer = tokio::spawn(async move { + for (prefix, roots) in [("", included), ("^", excluded)] { + for oid in roots { + input + .write_all(format!("{prefix}{}\n", hex::encode(oid)).as_bytes()) + .await?; + } + } + input.shutdown().await + }); + let transfer = async { + let mut buffer = vec![0; COPY_BYTES]; + loop { + let count = stream.read(&mut buffer).await?; + if count == 0 { + break; + } + owned.reservation()?.try_grow(count as u64)?; + output.write_all(&buffer[..count]).await?; + } + output.sync_all().await?; + Ok::<_, PackError>(()) + } + .await; + if transfer.is_err() { + let _ = child.start_kill(); + } + // Join both pipes and the worker before returning even after an I/O or + // admission error. The owner also checks the descendant fence on drop. + let status = child.wait().await?; + let supplied = writer.await?; + transfer?; + supplied?; + if !status.success() { + return Err(PackError::Integrity); + } + drop(output); + // Native index-pack writes its temporary index itself. Reserve its + // worst-case v2 size first (including all large-offset entries). + let mut header = [0; 12]; + tokio::fs::File::open(&path) + .await? + .read_exact(&mut header) + .await?; + if &header[..4] != b"PACK" + || u32::from_be_bytes(header[4..8].try_into().map_err(|_| PackError::Integrity)?) != 2 + { + return Err(PackError::Integrity); + } + let objects = u64::from(u32::from_be_bytes( + header[8..12].try_into().map_err(|_| PackError::Integrity)?, + )); + let index_bound = objects + .checked_mul(cache.object_format().bytes() as u64 + 16) + .and_then(|bytes| bytes.checked_add(4096)) + .ok_or(PackError::Limit)?; + if index_bound > self.index_limit() as u64 { + return Err(PackError::Limit); + } + owned.reservation()?.try_grow(index_bound)?; + let index_path = path.with_extension("idx"); + let status = + crate::native_git::command_with_extra_fence(owned.directory()?, &cache.git_dir())? + .arg("--git-dir") + .arg(cache.git_dir()) + .args(["index-pack", "--index-version=2", "-o"]) + .arg(&index_path) + .arg(&path) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .status() + .await?; + if !status.success() { + return Err(PackError::Integrity); + } + let format = cache.object_format(); + let index_limit = self.index_limit(); + let decode_limit = self.decode_limit(); + tokio::task::spawn_blocking(move || { + let (pack_bytes, pack_digest) = file_identity(&path)?; + let (index_bytes, index_digest) = file_identity(&index_path)?; + let identity = PackIdentity { + format, + pack_digest, + pack_bytes, + index_digest, + index_bytes, + }; + owned.reservation()?.resize( + pack_bytes + .checked_add(index_bytes) + .ok_or(PackError::Limit)?, + )?; + let mut owned = owned; + owned.verify(identity, index_limit, decode_limit)?; + Ok(Arc::new(owned)) + }) + .await? + } +} + +fn file_identity(path: &Path) -> Result<(u64, [u8; 32]), PackError> { + let mut file = std::fs::File::open(path)?; + let bytes = file.metadata()?.len(); + let mut digest = blake3::Hasher::new(); + let mut buffer = vec![0; COPY_BYTES]; + let mut read = 0; + loop { + let count = file.read(&mut buffer)?; + if count == 0 { + break; + } + read += count as u64; + digest.update(&buffer[..count]); + } + if read != bytes { + return Err(PackError::Integrity); + } + Ok((bytes, *digest.finalize().as_bytes())) +} + +#[cfg(test)] +mod tests; diff --git a/src/git_pack/build/tests.rs b/src/git_pack/build/tests.rs new file mode 100644 index 00000000..7e768635 --- /dev/null +++ b/src/git_pack/build/tests.rs @@ -0,0 +1,115 @@ +use super::*; +use crate::{ObjectFormat, ObjectKind, object_id}; +use cellule_ltx::DiskBudget; +use object_store::memory::InMemory; + +type TestResult = Result>; + +async fn fixture( + format: ObjectFormat, +) -> TestResult<(tempfile::TempDir, Arc, PackStore, DiskBudget)> { + let local = tempfile::TempDir::new()?; + let budget = DiskBudget::new(8 * 1024 * 1024); + let cache = GitCache::create( + local.path().into(), + budget.clone(), + "refs/heads/main", + format, + ) + .await?; + let store = PackStore::new( + Arc::new(InMemory::new()), + uuid::Uuid::new_v4().into_bytes(), + local.path().into(), + budget.clone(), + 1024 * 1024, + 1024 * 1024, + )?; + Ok((local, cache, store, budget)) +} + +#[tokio::test] +async fn builder_preserves_native_delta_compression_and_exact_objects() -> TestResult { + for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + let (_local, cache, store, budget) = fixture(format).await?; + let mut expected = Vec::new(); + for revision in 0..12 { + let mut body = b"native immutable packs retain repeated content\n".repeat(2048); + body[revision * 12] = revision as u8; + let oid = object_id(format, ObjectKind::Blob, &body); + cache + .store_object(oid, ObjectKind::Blob, body.clone()) + .await?; + expected.push((oid, body)); + } + let source = store + .build( + Arc::clone(&cache), + expected.iter().map(|(oid, _)| *oid).collect(), + Vec::new(), + false, + ) + .await?; + let native = source.data()?; + assert_eq!(native.object_ids().count(), expected.len()); + assert!(native.identity().pack_bytes < 16 * 1024); + let mut delta_count = 0; + for (oid, body) in expected { + assert!(native.contains(oid)?); + delta_count += u64::from(native.header(oid)?.ok_or("missing header")?.delta_depth > 0); + assert_eq!( + native.object(oid, 1024 * 1024)?, + Some((ObjectKind::Blob, body)) + ); + } + assert!(delta_count > 0); + let before = budget.used(); + let bytes = native.identity().pack_bytes + native.identity().index_bytes; + assert!(before > bytes); + let identity = store.publish(Arc::clone(&source)).await?; + let read = store.load(&identity).await?; + assert_eq!(read.data()?.identity(), &identity); + assert_eq!(budget.used(), before + bytes); + drop(read); + drop(source); + drop(store); + drop(cache); + assert_eq!(budget.used(), 0); + } + Ok(()) +} + +#[tokio::test] +async fn incremental_history_pack_never_needs_an_excluded_delta_base() -> TestResult { + let (source, _, _) = crate::git_pack::tests::fixture(ObjectFormat::Sha1).await?; + let (_local, cache, store, _budget) = fixture(ObjectFormat::Sha1).await?; + // Install the source pack as native Git's input, not loose copies of bodies. + let output = crate::native_git::command(&cache.git_dir())? + .args(["-c", "protocol.file.allow=always", "--git-dir"]) + .arg(cache.git_dir()) + .args(["fetch", "--no-tags"]) + .arg(source.path()) + .arg("HEAD") + .output() + .await?; + // Protocol policy is an explicit test fixture choice before the command. + if !output.status.success() { + return Err(String::from_utf8_lossy(&output.stderr).into_owned().into()); + } + cache.reconcile().await?; + let tip = crate::git_pack::tests::git(source.path(), &["rev-parse", "HEAD"]).await?; + let old = crate::git_pack::tests::git(source.path(), &["rev-parse", "HEAD~1"]).await?; + let tip = ObjectId::from_hex(tip.strip_suffix(b"\n").ok_or("tip newline")?)?; + let old = ObjectId::from_hex(old.strip_suffix(b"\n").ok_or("old newline")?)?; + let pack = store.build(cache, vec![tip], vec![old], true).await?; + let ids: Vec<_> = pack.data()?.object_ids().collect::>()?; + assert_eq!(ids.len(), 3); + for oid in ids { + let (kind, body) = pack + .data()? + .object(oid, 1024 * 1024)? + .ok_or("missing object")?; + assert_eq!(object_id(ObjectFormat::Sha1, kind, &body), oid); + } + Ok(()) +} diff --git a/src/git_pack/store.rs b/src/git_pack/store.rs new file mode 100644 index 00000000..f85865a3 --- /dev/null +++ b/src/git_pack/store.rs @@ -0,0 +1,433 @@ +//! Immutable native pack publication and repository-scoped disposable copies. + +use super::*; +use bytes::Bytes; +use cellule_ltx::{DiskBudget, DiskReservation}; +use object_store::{ObjectStore, ObjectStoreExt, path::Path as ObjectPath}; +use std::future::poll_fn; +use std::{collections::BTreeMap, path::PathBuf, sync::Arc}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::Mutex; + +const TRANSFER_BYTES: usize = 8 * 1024 * 1024; + +/// Native files and their disk charge remain owned through background work. +pub struct OwnedPack { + data: Option, + directory: Option, + reservation: Option, + native_owner: Option>, +} + +#[cfg(test)] +mod tests; + +impl OwnedPack { + /// Takes exclusive ownership of `objects.pack` and `objects.idx`. + pub fn open( + directory: tempfile::TempDir, + identity: PackIdentity, + reservation: DiskReservation, + max_index_bytes: usize, + max_decode_bytes: usize, + ) -> Result { + let mut pack = Self::pending(directory, reservation); + pack.verify(identity, max_index_bytes, max_decode_bytes)?; + Ok(pack) + } + + pub(super) fn pending(directory: tempfile::TempDir, reservation: DiskReservation) -> Self { + Self { + data: None, + directory: Some(directory), + reservation: Some(reservation), + native_owner: None, + } + } + + pub(super) fn verify( + &mut self, + identity: PackIdentity, + max_index_bytes: usize, + max_decode_bytes: usize, + ) -> Result<(), PackError> { + let required = identity + .pack_bytes + .checked_add(identity.index_bytes) + .ok_or(PackError::Limit)?; + if self + .reservation + .as_ref() + .ok_or(PackError::Integrity)? + .bytes() + < required + { + return Err(PackError::Limit); + } + let data = GitPack::open( + &self.directory()?.join("objects.pack"), + identity, + max_index_bytes, + max_decode_bytes, + )?; + self.data = Some(data); + Ok(()) + } + + pub(super) fn with_native_owner(mut self, cache: Arc) -> Self { + self.native_owner = Some(cache); + self + } + + pub(super) fn reservation(&self) -> Result<&DiskReservation, PackError> { + self.reservation.as_ref().ok_or(PackError::Integrity) + } + + pub fn data(&self) -> Result<&GitPack, PackError> { + self.data.as_ref().ok_or(PackError::Integrity) + } + + pub fn directory(&self) -> Result<&Path, PackError> { + self.directory + .as_ref() + .map(|directory| directory.path()) + .ok_or(PackError::Integrity) + } +} + +impl Drop for OwnedPack { + fn drop(&mut self) { + if let Some(directory) = &self.directory + && crate::native_git::idle_fence(directory.path()).is_err() + { + tracing::error!( + "Git pack cleanup fenced by a live native worker; disk admission retained" + ); + if let Some(reservation) = self.reservation.take() { + std::mem::forget(reservation); + } + if let Some(cache) = self.native_owner.take() { + std::mem::forget(cache); + } + if let Some(directory) = self.directory.as_mut() { + directory.disable_cleanup(true); + } + return; + } + // Close mappings before unlinking files or releasing their disk charge. + drop(self.data.take()); + if let Some(directory) = self.directory.take() { + let path = directory.path().to_path_buf(); + if let Err(error) = directory.close() { + tracing::error!(path = %path.display(), error = %error, "Git pack cleanup failed; disk admission retained until process restart"); + if let Some(reservation) = self.reservation.take() { + std::mem::forget(reservation); + } + } + } + } +} + +/// Stores native compressed content separately from the transactional Cell. +#[derive(Clone)] +pub struct PackStore { + store: Arc, + repository: [u8; 16], + local: PathBuf, + budget: DiskBudget, + max_index_bytes: usize, + max_decode_bytes: usize, + cache: Arc>>>, +} + +impl PackStore { + pub fn new( + store: Arc, + repository: [u8; 16], + local: PathBuf, + budget: DiskBudget, + max_index_bytes: usize, + max_decode_bytes: usize, + ) -> cellule_runtime::Result { + crate::validate_repository_id(repository)?; + if max_index_bytes == 0 || max_decode_bytes == 0 { + return Err(cellule_runtime::Error::Command( + "Git pack reader budgets must be nonzero", + )); + } + Ok(Self { + store, + repository, + local, + budget, + max_index_bytes, + max_decode_bytes, + cache: Arc::new(Mutex::new(BTreeMap::new())), + }) + } + + pub fn repository_id(&self) -> [u8; 16] { + self.repository + } + pub async fn verify(&self, identity: &PackIdentity) -> Result<(), PackError> { + self.verify_file( + &self.pack_path(identity), + identity.pack_bytes, + identity.pack_digest, + ) + .await?; + self.verify_file( + &self.index_path(identity), + identity.index_bytes, + identity.index_digest, + ) + .await + } + pub fn paths(&self, identity: &PackIdentity) -> [ObjectPath; 2] { + [self.pack_path(identity), self.index_path(identity)] + } + pub(super) fn local_root(&self) -> &Path { + &self.local + } + pub(super) fn disk_budget(&self) -> &DiskBudget { + &self.budget + } + pub(super) fn index_limit(&self) -> usize { + self.max_index_bytes + } + pub(super) fn decode_limit(&self) -> usize { + self.max_decode_bytes + } + + /// Publication owns source files through immutable copying and cleanup even + /// when the request that started it disappears. Both files verify remotely + /// before metadata may name this pair. + pub async fn publish(&self, source: Arc) -> Result { + let store = self.clone(); + tokio::spawn(async move { + let identity = source.data()?.identity().clone(); + store + .publish_file( + source.directory()?.join("objects.pack"), + store.pack_path(&identity), + identity.pack_bytes, + identity.pack_digest, + ) + .await?; + store + .publish_file( + source.directory()?.join("objects.idx"), + store.index_path(&identity), + identity.index_bytes, + identity.index_digest, + ) + .await?; + Ok(identity) + }) + .await? + } + + /// Opens a verified local pair once per resident repository. The shared disk + /// budget is charged before download; repository eviction releases the pair. + pub async fn load(&self, identity: &PackIdentity) -> Result, PackError> { + if identity.index_bytes > self.max_index_bytes as u64 { + return Err(PackError::Limit); + } + // Waiters are cancellable before dispatch. One resident repository may + // dispatch only one download; cancellation then retains its disk charge + // and scratch until file I/O, verification and cache installation join. + let mut cache = Arc::clone(&self.cache).lock_owned().await; + if let Some(pack) = cache.get(&identity.pack_digest) { + if pack.data()?.identity() != identity { + return Err(PackError::Integrity); + } + return Ok(Arc::clone(pack)); + } + let store = self.clone(); + let identity = identity.clone(); + tokio::spawn(async move { store.load_inner(&identity, &mut cache).await }).await? + } + + async fn load_inner( + &self, + identity: &PackIdentity, + cache: &mut BTreeMap<[u8; 32], Arc>, + ) -> Result, PackError> { + let required = identity + .pack_bytes + .checked_add(identity.index_bytes) + .ok_or(PackError::Limit)?; + let reservation = self.budget.try_reserve(required)?; + let local = self.local.clone(); + let directory = tokio::task::spawn_blocking(move || { + tempfile::Builder::new() + .prefix("canopy-pack-") + .tempdir_in(local) + }) + .await??; + let mut owned = OwnedPack::pending(directory, reservation); + self.download_file( + self.pack_path(identity), + owned.directory()?.join("objects.pack"), + identity.pack_bytes, + identity.pack_digest, + ) + .await?; + self.download_file( + self.index_path(identity), + owned.directory()?.join("objects.idx"), + identity.index_bytes, + identity.index_digest, + ) + .await?; + let identity = identity.clone(); + let (max_index_bytes, max_decode_bytes) = (self.max_index_bytes, self.max_decode_bytes); + let pack = Arc::new( + tokio::task::spawn_blocking(move || { + owned.verify(identity, max_index_bytes, max_decode_bytes)?; + Ok::<_, PackError>(owned) + }) + .await??, + ); + cache.insert(pack.data()?.identity().pack_digest, Arc::clone(&pack)); + Ok(pack) + } + + fn pack_path(&self, identity: &PackIdentity) -> ObjectPath { + ObjectPath::from(format!( + "repos/{}/git-packs/{}.pack", + hex::encode(self.repository), + hex::encode(identity.pack_digest) + )) + } + + fn index_path(&self, identity: &PackIdentity) -> ObjectPath { + ObjectPath::from(format!( + "repos/{}/git-packs/{}.idx", + hex::encode(self.repository), + hex::encode(identity.index_digest) + )) + } + + async fn publish_file( + &self, + source: PathBuf, + destination: ObjectPath, + size: u64, + digest: [u8; 32], + ) -> Result<(), PackError> { + let staging = ObjectPath::from(format!( + "repos/{}/git-pack-staging/{}", + hex::encode(self.repository), + uuid::Uuid::new_v4() + )); + let mut upload = self.store.put_multipart(&staging).await?; + let result = async { + let mut file = tokio::fs::File::open(source).await?; + let mut buffer = vec![0; TRANSFER_BYTES]; + let mut hash = blake3::Hasher::new(); + let mut length = 0_u64; + loop { + let count = (size - length).min(TRANSFER_BYTES as u64) as usize; + if count == 0 { + break; + } + file.read_exact(&mut buffer[..count]).await?; + length = length.checked_add(count as u64).ok_or(PackError::Limit)?; + if length > size { + return Err(PackError::Integrity); + } + hash.update(&buffer[..count]); + upload + .put_part(Bytes::copy_from_slice(&buffer[..count]).into()) + .await?; + } + if length != size || hash.finalize().as_bytes() != &digest { + return Err(PackError::Integrity); + } + if file.read(&mut [0; 1]).await? != 0 { + return Err(PackError::Integrity); + } + upload.complete().await?; + match self.store.copy_if_not_exists(&staging, &destination).await { + Ok(()) | Err(object_store::Error::AlreadyExists { .. }) => {} + Err(error) => return Err(error.into()), + } + self.verify_file(&destination, size, digest).await + } + .await; + if result.is_err() + && let Err(error) = upload.abort().await + { + tracing::warn!(error = %error, "Git pack multipart cleanup failed"); + } + let cleanup = self.store.delete(&staging).await; + result?; + match cleanup { + Ok(()) | Err(object_store::Error::NotFound { .. }) => Ok(()), + Err(error) => Err(error.into()), + } + } + + async fn verify_file( + &self, + path: &ObjectPath, + size: u64, + digest: [u8; 32], + ) -> Result<(), PackError> { + let result = self.store.get(path).await?; + if result.meta.size != size || result.range != (0..size) { + return Err(PackError::Integrity); + } + let mut stream = result.into_stream(); + let mut length = 0_u64; + let mut hash = blake3::Hasher::new(); + while let Some(bytes) = poll_fn(|cx| stream.as_mut().poll_next(cx)).await { + let bytes = bytes?; + length = length + .checked_add(bytes.len() as u64) + .ok_or(PackError::Limit)?; + if length > size { + return Err(PackError::Integrity); + } + hash.update(&bytes); + } + if length != size || hash.finalize().as_bytes() != &digest { + return Err(PackError::Integrity); + } + Ok(()) + } + + async fn download_file( + &self, + source: ObjectPath, + destination: PathBuf, + size: u64, + digest: [u8; 32], + ) -> Result<(), PackError> { + let result = self.store.get(&source).await?; + if result.meta.size != size || result.range != (0..size) { + return Err(PackError::Integrity); + } + let mut stream = result.into_stream(); + let mut file = tokio::fs::File::create(destination).await?; + let mut length = 0_u64; + let mut hash = blake3::Hasher::new(); + while let Some(bytes) = poll_fn(|cx| stream.as_mut().poll_next(cx)).await { + let bytes = bytes?; + length = length + .checked_add(bytes.len() as u64) + .ok_or(PackError::Limit)?; + if length > size { + return Err(PackError::Integrity); + } + hash.update(&bytes); + file.write_all(&bytes).await?; + } + if length != size || hash.finalize().as_bytes() != &digest { + return Err(PackError::Integrity); + } + file.sync_all().await?; + Ok(()) + } +} diff --git a/src/git_pack/store/tests.rs b/src/git_pack/store/tests.rs new file mode 100644 index 00000000..d9ac525d --- /dev/null +++ b/src/git_pack/store/tests.rs @@ -0,0 +1,211 @@ +use super::*; +use object_store::memory::InMemory; + +type TestResult = Result>; + +async fn prepared(format: ObjectFormat, budget: &DiskBudget) -> TestResult> { + let (_source, path, identity) = crate::git_pack::tests::fixture(format).await?; + let directory = tempfile::TempDir::new()?; + let reservation = budget.try_reserve(identity.pack_bytes + identity.index_bytes)?; + std::fs::copy(&path, directory.path().join("objects.pack"))?; + std::fs::copy( + path.with_extension("idx"), + directory.path().join("objects.idx"), + )?; + Ok(Arc::new(OwnedPack::open( + directory, + identity, + reservation, + 1024 * 1024, + 1024 * 1024, + )?)) +} + +#[tokio::test] +async fn immutable_pairs_round_trip_once_and_share_disk_admission() -> TestResult { + for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + let provider = Arc::new(InMemory::new()); + let budget = DiskBudget::new(1024 * 1024); + let source = prepared(format, &budget).await?; + let local = tempfile::TempDir::new()?; + let store = PackStore::new( + provider.clone(), + uuid::Uuid::new_v4().into_bytes(), + local.path().into(), + budget.clone(), + 1024 * 1024, + 1024 * 1024, + )?; + let identity = store.publish(Arc::clone(&source)).await?; + assert_eq!(&identity, source.data()?.identity()); + assert_eq!(store.publish(Arc::clone(&source)).await?, identity); + let (first, second) = tokio::try_join!(store.load(&identity), store.load(&identity))?; + assert!(Arc::ptr_eq(&first, &second)); + assert_eq!( + budget.used(), + 2 * (identity.pack_bytes + identity.index_bytes) + ); + for oid in source.data()?.object_ids() { + let oid = oid?; + assert_eq!( + first.data()?.object(oid, 1024 * 1024)?, + source.data()?.object(oid, 1024 * 1024)? + ); + } + drop(first); + drop(second); + drop(store); + drop(source); + assert_eq!(budget.used(), 0); + assert_eq!(std::fs::read_dir(local.path())?.count(), 0); + let mut remote = provider.list(None); + let mut count = 0; + while let Some(object) = poll_fn(|cx| remote.as_mut().poll_next(cx)).await { + let object = object?; + assert!(!object.location.as_ref().contains("staging")); + count += 1; + } + assert_eq!(count, 2); + } + Ok(()) +} + +#[tokio::test] +async fn corruption_cannot_be_overwritten_or_installed_in_a_fresh_cache() -> TestResult { + let provider = Arc::new(InMemory::new()); + let budget = DiskBudget::new(1024 * 1024); + let source = prepared(ObjectFormat::Sha1, &budget).await?; + let local = tempfile::TempDir::new()?; + let store = PackStore::new( + provider.clone(), + uuid::Uuid::new_v4().into_bytes(), + local.path().into(), + budget.clone(), + 1024 * 1024, + 1024 * 1024, + )?; + let identity = store.publish(Arc::clone(&source)).await?; + let path = store.pack_path(&identity); + let mut bytes = std::fs::read(source.directory()?.join("objects.pack"))?; + bytes[12] ^= 1; + provider + .put(&path, Bytes::from(bytes.clone()).into()) + .await?; + assert!(matches!( + store.publish(Arc::clone(&source)).await, + Err(PackError::Integrity) + )); + assert!(matches!( + store.load(&identity).await, + Err(PackError::Integrity) + )); + assert_eq!( + provider.get(&path).await?.bytes().await?.as_ref(), + bytes.as_slice() + ); + assert_eq!(budget.used(), identity.pack_bytes + identity.index_bytes); + assert_eq!(std::fs::read_dir(local.path())?.count(), 0); + Ok(()) +} + +#[tokio::test] +async fn a_cancelled_publisher_retains_source_files_until_remote_work_settles() -> TestResult { + use object_store::throttle::{ThrottleConfig, ThrottledStore}; + let provider = Arc::new(ThrottledStore::new( + InMemory::new(), + ThrottleConfig { + wait_get_per_call: std::time::Duration::from_millis(150), + ..ThrottleConfig::default() + }, + )); + let budget = DiskBudget::new(1024 * 1024); + let source = prepared(ObjectFormat::Sha1, &budget).await?; + let local = tempfile::TempDir::new()?; + let store = PackStore::new( + provider.clone(), + uuid::Uuid::new_v4().into_bytes(), + local.path().into(), + budget.clone(), + 1024 * 1024, + 1024 * 1024, + )?; + let identity = source.data()?.identity().clone(); + let work = store.clone(); + let owned = Arc::clone(&source); + let caller = tokio::spawn(async move { work.publish(owned).await }); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + let prefix = + ObjectPath::from(format!("repos/{}/git-packs", hex::encode(store.repository))); + let mut objects = provider.list(Some(&prefix)); + if let Some(object) = poll_fn(|cx| objects.as_mut().poll_next(cx)).await { + object?; + break Ok::<_, object_store::Error>(()); + } + tokio::task::yield_now().await; + } + }) + .await??; + caller.abort(); + let _ = caller.await; + drop(source); + assert!( + budget.used() > 0, + "the detached publisher still owns its source files" + ); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + while budget.used() != 0 { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await?; + assert!(store.load(&identity).await.is_ok()); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires an isolated S3-compatible provider namespace and explicit credentials"] +async fn native_pair_round_trips_through_rustfs() -> TestResult { + use object_store::prefix::PrefixStore; + let bucket = std::env::var("CANOPY_TEST_S3_BUCKET")?; + let prefix = std::env::var("CANOPY_TEST_S3_PREFIX")?; + let url = url::Url::parse(&format!("s3://{bucket}/{prefix}"))?; + let options = std::env::vars() + .flat_map(|(key, value)| { + [ + (key.clone(), value.clone()), + (key.to_ascii_lowercase(), value), + ] + }) + .chain(std::iter::once(( + "aws_copy_if_not_exists".into(), + object_store::aws::S3CopyIfNotExists::Multipart.to_string(), + ))); + let (provider, root) = object_store::parse_url_opts(&url, options)?; + let provider: Arc = Arc::from(provider); + let provider: Arc = Arc::new(PrefixStore::new(provider, root)); + for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + let budget = DiskBudget::new(1024 * 1024); + let source = prepared(format, &budget).await?; + let local = tempfile::TempDir::new()?; + let store = PackStore::new( + provider.clone(), + uuid::Uuid::new_v4().into_bytes(), + local.path().into(), + budget.clone(), + 1024 * 1024, + 1024 * 1024, + )?; + let identity = store.publish(Arc::clone(&source)).await?; + assert_eq!(store.publish(Arc::clone(&source)).await?, identity); + let restored = store.load(&identity).await?; + for oid in source.data()?.object_ids() { + let oid = oid?; + assert_eq!( + restored.data()?.object(oid, 1024 * 1024)?, + source.data()?.object(oid, 1024 * 1024)? + ); + } + } + Ok(()) +} diff --git a/src/git_pack/tests.rs b/src/git_pack/tests.rs new file mode 100644 index 00000000..cf9cffd5 --- /dev/null +++ b/src/git_pack/tests.rs @@ -0,0 +1,166 @@ +use super::*; +use std::path::PathBuf; + +type TestResult = Result>; + +pub(super) async fn git(directory: &Path, args: &[&str]) -> TestResult> { + let output = crate::native_git::command(directory)? + .args(args) + .output() + .await?; + if !output.status.success() { + return Err(String::from_utf8_lossy(&output.stderr).into_owned().into()); + } + Ok(output.stdout) +} + +pub(super) async fn fixture( + format: ObjectFormat, +) -> TestResult<(tempfile::TempDir, PathBuf, PackIdentity)> { + let directory = tempfile::TempDir::new()?; + git( + directory.path(), + &[ + "init", + "-b", + "main", + &format!("--object-format={}", format.as_str()), + ], + ) + .await?; + git( + directory.path(), + &["config", "user.name", "Pack qualification"], + ) + .await?; + git( + directory.path(), + &["config", "user.email", "pack@example.invalid"], + ) + .await?; + for index in 0..12 { + let mut body = b"immutable native pack and bounded delta decoding\n".repeat(2048); + body[1000 + index] = index as u8; + std::fs::write(directory.path().join("file.txt"), body)?; + git(directory.path(), &["add", "file.txt"]).await?; + git( + directory.path(), + &["commit", "-m", &format!("Revision {index}")], + ) + .await?; + } + git( + directory.path(), + &["tag", "-a", "release", "-m", "Native annotated tag"], + ) + .await?; + let pack_path = directory.path().join("objects.pack"); + std::fs::write( + &pack_path, + git( + directory.path(), + &["pack-objects", "--all", "--stdout", "--depth=20"], + ) + .await?, + )?; + git( + directory.path(), + &[ + "index-pack", + "--strict", + pack_path.to_str().ok_or("invalid test path")?, + ], + ) + .await?; + let pack = std::fs::read(&pack_path)?; + let index = std::fs::read(pack_path.with_extension("idx"))?; + let identity = PackIdentity { + format, + pack_digest: *blake3::hash(&pack).as_bytes(), + pack_bytes: pack.len() as u64, + index_digest: *blake3::hash(&index).as_bytes(), + index_bytes: index.len() as u64, + }; + Ok((directory, pack_path, identity)) +} + +#[tokio::test] +async fn native_sha1_and_sha256_delta_packs_return_every_exact_git_object() -> TestResult { + for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + let (directory, path, identity) = fixture(format).await?; + let pack = GitPack::open(&path, identity.clone(), 1024 * 1024, 1024 * 1024)?; + assert_eq!(pack.identity(), &identity); + let mut deltas = 0; + let objects: Vec<_> = pack.object_ids().collect::>()?; + assert_eq!(objects.len(), 37); + for oid in objects { + let header = pack.header(oid)?.ok_or("missing pack header")?; + deltas += usize::from(header.delta_depth > 0); + let (kind, bytes) = pack + .object(oid, 1024 * 1024)? + .ok_or("missing pack object")?; + let expected = git( + directory.path(), + &["cat-file", kind.git_name(), &hex::encode(oid)], + ) + .await?; + assert_eq!(bytes, expected); + assert_eq!(header.size, bytes.len() as u64); + assert_eq!(header.kind, kind); + } + assert!(deltas > 0, "the fixture must exercise real delta chains"); + assert!(pack.object(format.zero(), 1024)?.is_none()); + if format == ObjectFormat::Sha256 { + assert!(matches!( + pack.header(ObjectId::Sha1([1; 20])), + Err(PackError::Integrity) + )); + } + } + Ok(()) +} + +#[tokio::test] +async fn identity_native_checksums_and_allocation_bounds_fail_closed() -> TestResult { + let (directory, path, identity) = fixture(ObjectFormat::Sha1).await?; + assert!(matches!( + GitPack::open(&path, identity.clone(), 1, 1024), + Err(PackError::Limit) + )); + let pack = GitPack::open(&path, identity.clone(), 1024 * 1024, 1024 * 1024)?; + let oid = ObjectId::from_hex( + git(directory.path(), &["rev-parse", "HEAD:file.txt"]) + .await? + .strip_suffix(b"\n") + .ok_or("missing newline")?, + )?; + assert!(matches!(pack.object(oid, 1), Err(PackError::Limit))); + let bounded = GitPack::open(&path, identity.clone(), 1024 * 1024, 1)?; + assert!(bounded.object(oid, 1024 * 1024).is_err()); + let mut wrong = identity.clone(); + wrong.pack_digest[0] ^= 1; + assert!(matches!( + GitPack::open(&path, wrong, 1024 * 1024, 1024), + Err(PackError::Integrity) + )); + // Even a matching outer digest cannot hide a broken native pack checksum. + let mut bytes = std::fs::read(&path)?; + bytes[12] ^= 1; + std::fs::write(directory.path().join("corrupt.pack"), &bytes)?; + std::fs::copy( + path.with_extension("idx"), + directory.path().join("corrupt.idx"), + )?; + let mut corrupt = identity; + corrupt.pack_digest = *blake3::hash(&bytes).as_bytes(); + assert!(matches!( + GitPack::open( + &directory.path().join("corrupt.pack"), + corrupt, + 1024 * 1024, + 1024 + ), + Err(PackError::Integrity) + )); + Ok(()) +} diff --git a/src/graph.rs b/src/graph.rs index 0afd89fa..9dd5f0f7 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; -use crate::{ObjectKind, PushPlan, RepositoryModule, object_batch::VERIFY_BATCH_BYTES, object_id}; +use crate::{ObjectKind, PushPlan, RepositoryModule, object_batch::VERIFY_BATCH_BYTES}; use cellule_runtime::{ CellModule, Command, Error, codec::BoundedDecoder, codec::BoundedEncoder, codec::CodecError, codec::WireValue, primitives::sql::SqlBatch, primitives::sql::SqlResultSet, @@ -11,10 +11,12 @@ use cellule_runtime::{ }; mod preparation; +pub(crate) mod stream; type Oid = crate::ObjectId; -type Edge = (Oid, Option); +pub(crate) type Edge = (Oid, Option); const MAX_CERTIFICATES: usize = 128; +const EDGE_WRITE_ROWS: usize = 512; #[derive(Default)] pub(crate) struct CertificateBatch(Vec); @@ -60,7 +62,7 @@ fn status_query(oids: &[Oid]) -> SqlBatch { SqlBatch { statements: vec![SqlStatement { sql: format!( - "SELECT o.oid, o.kind, CASE WHEN o.storage = 'external' THEN 0 ELSE o.size END, c.oid FROM objects o LEFT JOIN object_closure c ON c.oid = o.oid WHERE o.oid IN ({placeholders})" + "SELECT o.oid, o.kind, CASE WHEN o.kind = 'blob' THEN 0 ELSE o.size END, c.oid FROM objects o LEFT JOIN object_closure c ON c.oid = o.oid WHERE o.oid IN ({placeholders})" ), parameters: oids .iter() @@ -106,7 +108,7 @@ pub(crate) struct CertifyObjects; impl Command for CertifyObjects { const MODULE: &'static str = RepositoryModule::NAME; const ID: u32 = 6; - const CODEC_VERSION: u32 = 2; + const CODEC_VERSION: u32 = 4; type Input = CertificateBatch; type Output = bool; @@ -132,11 +134,12 @@ impl Command for CertifyObjects { }; for edges in edges.chunks(MAX_CERTIFICATES) { let oids: Vec<_> = edges.iter().map(|(oid, _)| *oid).collect(); - let states = statuses(&context.sql(&status_query(&oids))?)?; + let result = context.sql(&certified_kind_query(&oids))?; + let states = certified_kinds(&result)?; if edges.iter().any(|(oid, kind)| { - !states.get(oid).is_some_and(|state| { - state.certified && kind.is_none_or(|kind| kind == state.kind) - }) + !states + .get(oid) + .is_some_and(|actual| kind.is_none_or(|expected| expected == *actual)) }) { return Ok(CommandResult::Rejected(false)); } @@ -144,18 +147,20 @@ impl Command for CertifyObjects { // Only certified, typed edges enter the reachability index. Keeping // this with closure publication prevents lazy fetch from trusting // staged or malformed object graphs. - for edges in edges.chunks(MAX_CERTIFICATES) { + for edges in edges.chunks(EDGE_WRITE_ROWS) { context.sql(&SqlBatch { - statements: edges - .iter() - .map(|(child, _)| SqlStatement { - sql: "INSERT INTO object_edges (parent, child) VALUES (?1, ?2)".into(), - parameters: vec![ - SqlValue::Blob(oid.to_vec()), - SqlValue::Blob(child.to_vec()), - ], - }) - .collect(), + statements: vec![SqlStatement { + sql: format!( + "INSERT INTO object_edges (parent, child) VALUES {}", + vec!["(?, ?)"; edges.len()].join(",") + ), + parameters: edges + .iter() + .flat_map(|(child, _)| { + [SqlValue::Blob(oid.to_vec()), SqlValue::Blob(child.to_vec())] + }) + .collect(), + }], })?; } if state.kind == ObjectKind::Commit { @@ -184,6 +189,43 @@ impl Command for CertifyObjects { } } +fn certified_kind_query(oids: &[Oid]) -> SqlBatch { + SqlBatch { + statements: vec![SqlStatement { + // Dependencies need only immutable type and a closure proof. A + // covering lookup avoids fetching each wide object metadata row. + sql: format!( + "SELECT o.oid, o.kind FROM objects o INDEXED BY objects_by_oid_kind JOIN object_closure c ON c.oid = o.oid WHERE o.oid IN ({})", + vec!["?"; oids.len()].join(",") + ), + parameters: oids + .iter() + .map(|oid| SqlValue::Blob(oid.to_vec())) + .collect(), + }], + } +} + +fn certified_kinds(results: &[SqlResultSet]) -> cellule_runtime::Result> { + let mut kinds = BTreeMap::new(); + for row in &results + .first() + .ok_or(Error::Command("missing certified kinds"))? + .rows + { + let [SqlValue::Blob(oid), SqlValue::Text(kind)] = row.as_slice() else { + return Err(Error::Command("invalid certified kind")); + }; + kinds.insert( + oid.as_slice() + .try_into() + .map_err(|_| Error::Command("invalid certified OID"))?, + parse_kind(kind.as_bytes()).ok_or(Error::Command("invalid certified kind"))?, + ); + } + Ok(kinds) +} + pub(crate) fn certified_roots( context: &CommandContext<'_, '_>, plan: &PushPlan, @@ -216,62 +258,66 @@ fn object_edges( ) -> cellule_runtime::Result>> { let result = context.sql(&SqlBatch { statements: vec![SqlStatement { - sql: "SELECT storage, body, digest, size, chunk_id FROM objects WHERE oid = ?1".into(), + sql: "SELECT pack, graph_valid, graph_count, graph_digest FROM objects WHERE oid = ?1" + .into(), parameters: vec![SqlValue::Blob(oid.to_vec())], }], })?; let Some( [ - SqlValue::Text(storage), - body, + SqlValue::Blob(pack), + SqlValue::Integer(valid), + SqlValue::Integer(count), SqlValue::Blob(digest), - SqlValue::Integer(size), - upload, ], ) = result .first() .and_then(|set| set.rows.first()) .map(Vec::as_slice) else { - return Err(Error::Command("invalid stored graph object")); + return Err(Error::Command("invalid packed graph")); }; - let edges = match (storage.as_str(), body, upload) { - ("inline", SqlValue::Blob(body), SqlValue::Null) => { - if object_id(oid.format(), kind, body) != oid - || blake3::hash(body).as_bytes() != digest.as_slice() - { - return Err(Error::Command("corrupt stored graph object")); - } - let Some(edges) = edges(oid.format(), kind, body) else { - return Ok(None); - }; - edges + if *valid != 1 { + return Ok(None); + } + let mut cursor = Vec::new(); + let mut edges = Vec::new(); + let mut hash = blake3::Hasher::new(); + loop { + let result = context.sql(&crate::object_packs::dependency_query(pack, oid, &cursor))?; + let rows = &result + .first() + .ok_or(Error::Command("missing packed dependencies"))? + .rows; + if rows.is_empty() { + break; } - // The immutable upload is verified before its SQLite record is published. - // Blobs have no outgoing Git edges; no network I/O belongs in this transaction. - ("external", SqlValue::Null, SqlValue::Null) if kind == ObjectKind::Blob => Vec::new(), - ("chunked", SqlValue::Null, SqlValue::Blob(upload)) => { - let invalid = || Error::Command("invalid stored graph object chunks"); - let body = crate::object_chunks::body( - context, - oid, - kind, - upload.as_slice().try_into().map_err(|_| invalid())?, - u64::try_from(*size).map_err(|_| invalid())?, - digest.as_slice().try_into().map_err(|_| invalid())?, - )? - .ok_or_else(invalid)?; - let Some(edges) = edges(oid.format(), kind, &body) else { - return Ok(None); + for row in rows { + let [SqlValue::Blob(key)] = row.as_slice() else { + return Err(Error::Command("invalid packed dependency row")); }; - edges + if key <= &cursor || edges.len() as i64 >= *count { + return Err(Error::Command("invalid packed dependency count")); + } + hash.update(key); + edges.push(crate::object_packs::decode_dependency(oid.format(), key)?); + cursor.clone_from(key); } - _ => return Err(Error::Command("invalid stored graph object")), - }; + } + if edges.len() as i64 != *count + || hash.finalize().as_bytes().as_slice() != digest + || (kind == ObjectKind::Blob && !edges.is_empty()) + { + return Err(Error::Command("corrupt packed graph")); + } Ok(Some(edges)) } -fn edges(format: crate::ObjectFormat, kind: ObjectKind, mut body: &[u8]) -> Option> { +pub(crate) fn edges( + format: crate::ObjectFormat, + kind: ObjectKind, + mut body: &[u8], +) -> Option> { let mut edges = match kind { ObjectKind::Blob => Some(Vec::new()), ObjectKind::Tree => tree_edges(format, body), diff --git a/src/graph/preparation.rs b/src/graph/preparation.rs index eb40a838..f07b2211 100644 --- a/src/graph/preparation.rs +++ b/src/graph/preparation.rs @@ -80,6 +80,7 @@ impl RepositoryCell { let mut ready = HashMap::new(); let mut batch = CertificateBatch::default(); let mut bytes = 0; + let mut indexes_warm = false; while let Some(visit) = pending.pop() { let (oid, expected, observed) = match visit { Visit::Enter((oid, expected), observed) => (oid, expected, observed), @@ -147,15 +148,14 @@ impl RepositoryCell { if !visiting.insert(oid) { return Ok(()); } + if !indexes_warm { + self.warm_graph_indexes().await?; + indexes_warm = true; + } let edges = if state.kind == ObjectKind::Blob { Vec::new() } else { - let Some((kind, body)) = self.object(oid, None).await?.output else { - return Ok(()); - }; - let Some(edges) = - tokio::task::spawn_blocking(move || edges(oid.format(), kind, &body)).await? - else { + let Some(edges) = self.dependencies(oid).await? else { return Ok(()); }; edges @@ -169,6 +169,42 @@ impl RepositoryCell { Ok(()) } + // Certification inserts into both reachability indexes. After recovery, + // their old pages may still be remote. Pull them through bounded read-only + // queries before the write transaction; this preserves large publication + // batches without consuming its five-second SQL execution budget on I/O. + async fn warm_graph_indexes(&self) -> Result<(), PrepareError> { + for reverse in [false, true] { + let mut cursor = (Vec::new(), Vec::new()); + let mut observed = None; + loop { + let result = self + .sql + .query(observed, index_page(reverse, &cursor)) + .await?; + let rows = &result + .output + .first() + .ok_or(Error::Command("missing graph index page"))? + .rows; + if rows.is_empty() { + break; + } + for row in rows { + let [SqlValue::Blob(first), SqlValue::Blob(second)] = row.as_slice() else { + return Err(Error::Command("invalid graph index page").into()); + }; + if (first, second) <= (&cursor.0, &cursor.1) { + return Err(Error::Command("invalid graph index order").into()); + } + cursor = (first.clone(), second.clone()); + } + observed = Some(result.receipt); + } + } + Ok(()) + } + async fn certify_batch(&self, batch: CertificateBatch) -> Result<(), PrepareError> { let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; let identity = MutationIdentity { @@ -184,3 +220,73 @@ impl RepositoryCell { Ok(()) } } + +fn index_page(reverse: bool, cursor: &(Vec, Vec)) -> SqlBatch { + let sql = if reverse { + "SELECT child, parent FROM object_edges INDEXED BY object_edges_by_child WHERE (child, parent) > (?1, ?2) ORDER BY child, parent LIMIT 512" + } else { + "SELECT parent, child FROM object_edges WHERE (parent, child) > (?1, ?2) ORDER BY parent, child LIMIT 512" + }; + SqlBatch { + statements: vec![SqlStatement { + sql: sql.into(), + parameters: vec![ + SqlValue::Blob(cursor.0.clone()), + SqlValue::Blob(cursor.1.clone()), + ], + }], + } +} + +#[cfg(test)] +mod tests { + use super::*; + use cellule_ltx::rusqlite::{Connection, StatementStatus, params}; + + #[test] + fn index_warming_seeks_every_pair_across_duplicate_cursor_keys() + -> Result<(), Box> { + let db = Connection::open_in_memory()?; + db.execute_batch("CREATE TABLE object_edges (parent BLOB, child BLOB, PRIMARY KEY(parent, child)) WITHOUT ROWID; CREATE INDEX object_edges_by_child ON object_edges(child, parent)")?; + let mut expected = Vec::new(); + for n in 0_u32..2_049 { + let parent = (n / 7).to_be_bytes().to_vec(); + let child = (n % 7).to_be_bytes().to_vec(); + db.execute( + "INSERT INTO object_edges VALUES (?1, ?2)", + params![&parent, &child], + )?; + expected.push((parent, child)); + } + for reverse in [false, true] { + let mut cursor = (Vec::new(), Vec::new()); + let mut seen = Vec::new(); + loop { + let query = index_page(reverse, &cursor); + let mut statement = db.prepare(&query.statements[0].sql)?; + let rows: Vec<(Vec, Vec)> = statement + .query_map(params![&cursor.0, &cursor.1], |row| { + Ok((row.get(0)?, row.get(1)?)) + })? + .collect::>()?; + assert!(rows.len() <= 512); + assert_eq!(statement.get_status(StatementStatus::FullscanStep), 0); + if rows.is_empty() { + break; + } + assert!(rows[0] > cursor); + cursor = rows.last().unwrap().clone(); + seen.extend(rows); + } + let mut ordered = expected.clone(); + if reverse { + for pair in &mut ordered { + std::mem::swap(&mut pair.0, &mut pair.1); + } + } + ordered.sort(); + assert_eq!(seen, ordered); + } + Ok(()) + } +} diff --git a/src/graph/stream.rs b/src/graph/stream.rs new file mode 100644 index 00000000..84809d55 --- /dev/null +++ b/src/graph/stream.rs @@ -0,0 +1,305 @@ +//! Streaming graph projection for structural bodies; messages and names need no body buffer. +use super::{Edge, ObjectKind}; +use crate::{ObjectFormat, ObjectId}; +use std::collections::TryReserveError; + +pub(crate) struct StreamGraph { + format: ObjectFormat, + kind: ObjectKind, + edges: Vec, + invalid: bool, + done: bool, + line: Vec, + line_number: u8, + target: Option, + target_kind: Option, + mode: u32, + digits: usize, + tree_state: u8, + name_bytes: u64, + dot_name: bool, + oid: [u8; 32], + oid_bytes: usize, +} +impl StreamGraph { + pub(crate) fn new(format: ObjectFormat, kind: ObjectKind) -> Self { + Self { + format, + kind, + edges: Vec::new(), + invalid: false, + done: kind == ObjectKind::Blob, + line: Vec::new(), + line_number: 0, + target: None, + target_kind: None, + mode: 0, + digits: 0, + tree_state: 0, + name_bytes: 0, + dot_name: true, + oid: [0; 32], + oid_bytes: 0, + } + } + fn edge(&mut self, oid: ObjectId, kind: Option) -> Result<(), TryReserveError> { + self.edges.try_reserve(1)?; + self.edges.push((oid, kind)); + Ok(()) + } + pub(crate) fn feed(&mut self, bytes: &[u8]) -> Result<(), TryReserveError> { + if self.invalid || self.done { + return Ok(()); + } + for &byte in bytes { + if self.invalid || self.done { + break; + } + if self.kind == ObjectKind::Tree { + self.tree_byte(byte)?; + continue; + } + if byte == b'\n' { + self.end_line()?; + continue; + } + if self.kind == ObjectKind::Tag && self.line_number == 2 { + if self.line.len() < 4 { + self.line.push(byte); + } + if self.line.len() == 4 && self.line != b"tag " { + self.invalid = true; + } + } else if self.line.len() < 128 { + self.line.push(byte); + } else if self.kind == ObjectKind::Commit + && self.line_number != 0 + && !self.line.starts_with(b"parent ") + { + self.done = true; + } else { + self.invalid = true; + } + } + Ok(()) + } + fn parsed_oid(&self, bytes: &[u8]) -> Option { + ObjectId::from_hex(bytes) + .ok() + .filter(|oid| oid.format() == self.format && !oid.is_zero()) + } + fn end_line(&mut self) -> Result<(), TryReserveError> { + if self.kind == ObjectKind::Commit { + if self.line_number == 0 { + let oid = self + .line + .strip_prefix(b"tree ") + .and_then(|value| self.parsed_oid(value)); + if let Some(oid) = oid { + self.edge(oid, Some(ObjectKind::Tree))?; + self.line_number = 1; + } else { + self.invalid = true; + } + } else if let Some(value) = self.line.strip_prefix(b"parent ") { + if let Some(oid) = self.parsed_oid(value) { + self.edge(oid, Some(ObjectKind::Commit))?; + } else { + self.invalid = true; + } + } else { + self.done = true; + } + } else { + match self.line_number { + 0 => { + self.target = self + .line + .strip_prefix(b"object ") + .and_then(|value| self.parsed_oid(value)); + if self.target.is_none() { + self.invalid = true; + } + } + 1 => { + self.target_kind = self.line.strip_prefix(b"type ").and_then(super::parse_kind); + if self.target_kind.is_none() { + self.invalid = true; + } + } + 2 => { + if self.line == b"tag " { + if let Some(oid) = self.target { + self.edge(oid, self.target_kind)?; + self.done = true; + } else { + self.invalid = true; + } + } else { + self.invalid = true; + } + } + _ => self.invalid = true, + } + self.line_number += 1; + } + self.line.clear(); + Ok(()) + } + fn tree_byte(&mut self, byte: u8) -> Result<(), TryReserveError> { + match self.tree_state { + 0 => { + if byte == b' ' { + if self.digits == 0 || self.mode > 0o177777 { + self.invalid = true; + } else { + self.tree_state = 1; + } + } else if (b'0'..=b'7').contains(&byte) { + match self + .mode + .checked_mul(8) + .and_then(|mode| mode.checked_add(u32::from(byte - b'0'))) + { + Some(mode) => { + self.mode = mode; + self.digits += 1; + } + None => self.invalid = true, + } + } else { + self.invalid = true; + } + } + 1 => { + if byte == 0 { + if self.name_bytes == 0 || (self.dot_name && self.name_bytes <= 2) { + self.invalid = true; + } else { + self.tree_state = 2; + } + } else { + if byte == b'/' { + self.invalid = true; + } + self.dot_name &= byte == b'.'; + self.name_bytes += 1; + } + } + 2 => { + self.oid[self.oid_bytes] = byte; + self.oid_bytes += 1; + if self.oid_bytes == self.format.bytes() { + let Some(oid) = ObjectId::try_from(&self.oid[..self.oid_bytes]) + .ok() + .filter(|oid| !oid.is_zero()) + else { + self.invalid = true; + return Ok(()); + }; + match self.mode & 0o170000 { + 0o040000 => self.edge(oid, Some(ObjectKind::Tree))?, + 0o100000 | 0o120000 => self.edge(oid, Some(ObjectKind::Blob))?, + 0o160000 => {} + _ => self.invalid = true, + } + self.mode = 0; + self.digits = 0; + self.name_bytes = 0; + self.dot_name = true; + self.oid_bytes = 0; + self.tree_state = 0; + } + } + _ => self.invalid = true, + } + Ok(()) + } + pub(crate) fn finish(mut self) -> Option> { + if self.invalid { + return None; + } + match self.kind { + ObjectKind::Tree if self.tree_state != 0 || self.digits != 0 => return None, + ObjectKind::Commit + if self.line_number == 0 || (!self.done && self.line.starts_with(b"parent ")) => + { + return None; + } + ObjectKind::Tag if !self.done => return None, + _ => {} + } + self.edges.sort_unstable(); + self.edges.dedup(); + Some(self.edges) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn streaming_matches_whole_body_graph_parser_at_every_chunk_boundary() { + for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + let oid = crate::object_id(format, ObjectKind::Blob, b"graph leaf"); + let mut tree = b"100644 file\0".to_vec(); + tree.extend_from_slice(&oid); + let commit = format!( + "tree {}\nparent {}\nauthor Person\n\nmessage", + hex::encode(oid), + hex::encode(oid) + ) + .into_bytes(); + let tag = format!( + "object {}\ntype commit\ntag release\n\nmessage", + hex::encode(oid) + ) + .into_bytes(); + for (kind, body) in [ + (ObjectKind::Blob, b"content".to_vec()), + (ObjectKind::Tree, tree), + (ObjectKind::Commit, commit), + (ObjectKind::Tag, tag), + ] { + for size in 1..body.len() + 1 { + let mut parser = StreamGraph::new(format, kind); + for chunk in body.chunks(size) { + parser.feed(chunk).unwrap(); + } + assert_eq!(parser.finish(), super::super::edges(format, kind, &body)); + } + } + } + } + #[test] + fn malformed_graphs_fail_closed_across_fragmented_input() { + let oid = crate::object_id(ObjectFormat::Sha1, ObjectKind::Blob, b"leaf"); + let mut malformed_tree = b"100644 ../bad\0".to_vec(); + malformed_tree.extend_from_slice(&oid); + let cases = [ + (ObjectKind::Tree, malformed_tree), + (ObjectKind::Tree, b"100644 name\0short".to_vec()), + ( + ObjectKind::Commit, + format!("tree {}\nparent missing", hex::encode(oid)).into_bytes(), + ), + ( + ObjectKind::Tag, + format!("object {}\ntype wrong\ntag release\n", hex::encode(oid)).into_bytes(), + ), + ( + ObjectKind::Tag, + format!("object {}\ntype blob\nwrong release\n", hex::encode(oid)).into_bytes(), + ), + ]; + for (kind, body) in cases { + for size in [1, 7, 128] { + let mut parser = StreamGraph::new(ObjectFormat::Sha1, kind); + for chunk in body.chunks(size) { + parser.feed(chunk).unwrap(); + } + assert!(parser.finish().is_none()); + } + } + } +} diff --git a/src/graph/tests.rs b/src/graph/tests.rs index e4b054bb..64a360fe 100644 --- a/src/graph/tests.rs +++ b/src/graph/tests.rs @@ -1,5 +1,55 @@ use super::*; +#[test] +fn dependency_type_checks_cover_kind_and_require_a_committed_certificate() +-> Result<(), Box> { + use cellule_ltx::rusqlite::{Connection, params, params_from_iter, types::Value}; + let db = Connection::open_in_memory()?; + db.execute_batch(crate::SCHEMA)?; + db.execute("INSERT INTO git_packs(digest, format, bytes, index_digest, index_bytes) VALUES (zeroblob(32), 'sha1', 1, zeroblob(32), 1)", [])?; + let ids = [Oid::Sha1([1; 20]), Oid::Sha1([2; 20]), Oid::Sha1([3; 20])]; + for (oid, kind) in ids.iter().zip(["blob", "tree", "commit"]) { + db.execute("INSERT INTO objects(oid, kind, size, digest, pack, graph_valid, graph_count, graph_digest) VALUES (?1, ?2, 0, zeroblob(32), zeroblob(32), 1, 0, zeroblob(32))", params![oid.as_ref(), kind])?; + } + for oid in &ids[..2] { + db.execute("INSERT INTO object_closure VALUES (?1)", [oid.as_ref()])?; + } + let batch = certified_kind_query(&ids); + let query = &batch.statements[0]; + let values: Vec<_> = query + .parameters + .iter() + .map(|value| match value { + SqlValue::Blob(value) => Value::Blob(value.clone()), + _ => unreachable!(), + }) + .collect(); + let plans: Vec = db + .prepare(&format!("EXPLAIN QUERY PLAN {}", query.sql))? + .query_map(params_from_iter(&values), |row| row.get(3))? + .collect::>()?; + assert!( + plans + .iter() + .any(|plan| plan.contains("COVERING INDEX objects_by_oid_kind")), + "{plans:?}" + ); + let rows: Vec<(Vec, String)> = db + .prepare(&query.sql)? + .query_map(params_from_iter(&values), |row| { + Ok((row.get(0)?, row.get(1)?)) + })? + .collect::>()?; + assert_eq!( + rows, + vec![ + (ids[0].to_vec(), "blob".into()), + (ids[1].to_vec(), "tree".into()) + ] + ); + Ok(()) +} + #[test] fn certificate_codec_rejects_empty_oversized_and_malformed_batches() { for count in [0, MAX_CERTIFICATES + 1] { diff --git a/src/http.rs b/src/http.rs index 911ba4f4..201c54fa 100644 --- a/src/http.rs +++ b/src/http.rs @@ -684,6 +684,10 @@ async fn git_request(State(api): State>, request: Request) StatusCode::SERVICE_UNAVAILABLE, "Repository refs are changing; retry the request", ), + Err(GatewayError::Pack(error)) if error.is_admission() => plain( + StatusCode::INSUFFICIENT_STORAGE, + "Git cache disk budget exhausted", + ), Err( GatewayError::Cache(error) | GatewayError::Http(crate::git_http::GitHttpError::Cache(error)), diff --git a/src/large_blob.rs b/src/large_blob.rs deleted file mode 100644 index 14f7e19b..00000000 --- a/src/large_blob.rs +++ /dev/null @@ -1,167 +0,0 @@ -//! Immutable external bytes for large Git blobs. - -use std::{sync::Arc, time::Duration}; - -use crate::git_format::ObjectHasher; -use bytes::Bytes; -use object_store::{ObjectStore, path::Path}; -use sha2::{Digest, Sha256}; -use tokio::io::{AsyncRead, AsyncReadExt}; - -mod read; -pub use read::LargeBlobRead; - -const CHUNK_BYTES: usize = crate::external::PART_BYTES; -const IO_TIMEOUT: Duration = Duration::from_secs(120); - -#[derive(Clone, Copy, Debug)] -pub struct LargeBlobReference { - pub oid: crate::ObjectId, - pub size: u64, - pub blake3: [u8; 32], - pub sha256: [u8; 32], -} - -#[derive(Debug, thiserror::Error)] -pub enum LargeBlobError { - #[error("large blob store failed")] - Store(#[from] object_store::Error), - #[error("large blob input failed")] - Io(#[from] std::io::Error), - #[error("large blob worker failed")] - Task(#[from] tokio::task::JoinError), - #[error("large blob transfer timed out")] - Timeout, - #[error("large blob bytes disagree with their SQLite reference")] - Corrupt, -} - -struct Hashes { - oid: ObjectHasher, - sha256: Sha256, - blake3: blake3::Hasher, -} - -impl Hashes { - fn new(format: crate::ObjectFormat, size: u64) -> Self { - let oid = ObjectHasher::new(format, crate::ObjectKind::Blob, size); - Self { - oid, - sha256: Sha256::new(), - blake3: blake3::Hasher::new(), - } - } - - async fn update(mut self, bytes: Bytes) -> Result<(Self, Bytes), LargeBlobError> { - Ok(tokio::task::spawn_blocking(move || { - self.oid.update(&bytes); - self.sha256.update(&bytes); - self.blake3.update(&bytes); - (self, bytes) - }) - .await?) - } - - fn finish(self, size: u64) -> LargeBlobReference { - LargeBlobReference { - oid: self.oid.finalize(), - size, - sha256: self.sha256.finalize().into(), - blake3: *self.blake3.finalize().as_bytes(), - } - } -} - -/// Repository-scoped content-addressed body store. -pub struct LargeBlobStore { - store: Arc, - repository_id: [u8; 16], -} - -impl LargeBlobStore { - pub fn new(store: Arc, repository_id: [u8; 16]) -> Self { - Self { - store, - repository_id, - } - } - - /// Streams exactly `size` bytes, verifying the Git OID before immutable publication. - /// The caller retains this future through cleanup; abrupt process loss may leave staging. - pub async fn put( - &self, - oid: crate::ObjectId, - size: u64, - input: &mut (impl AsyncRead + Unpin), - ) -> Result { - let stage = Path::from(format!( - "repos/{}/git-blob-staging/{}", - hex::encode(self.repository_id), - uuid::Uuid::new_v4() - )); - let mut upload = crate::external::Upload::new(self.store.clone(), stage).await?; - let result = async { - let mut hashes = Hashes::new(oid.format(), size); - let mut remaining = size; - loop { - let length = remaining.min(CHUNK_BYTES as u64) as usize; - let mut bytes = vec![0; length]; - tokio::time::timeout(IO_TIMEOUT, input.read_exact(&mut bytes)) - .await - .map_err(|_| LargeBlobError::Timeout)??; - let (updated, bytes) = hashes.update(Bytes::from(bytes)).await?; - hashes = updated; - upload.write(bytes).await?; - remaining -= length as u64; - if remaining == 0 { - break; - } - } - let reference = hashes.finish(size); - if reference.oid != oid { - return Err(LargeBlobError::Corrupt); - } - upload - .publish(&blob_path(self.repository_id, &reference.sha256), size) - .await?; - self.verify(&reference).await?; - Ok(reference) - } - .await; - let cleanup = upload.cleanup().await; - if result.is_err() - && let Err(error) = &cleanup - { - tracing::warn!(error = %error, "Git blob staging cleanup failed"); - } - let reference = result?; - cleanup?; - Ok(reference) - } - - /// Opens a bounded reader that validates all three hashes before returning its final range. - pub async fn read( - &self, - reference: &LargeBlobReference, - ) -> Result { - LargeBlobRead::open(self.store.clone(), self.repository_id, *reference).await - } - - /// Checks an immutable object's bytes without materializing the full body. - pub async fn verify(&self, reference: &LargeBlobReference) -> Result<(), LargeBlobError> { - let mut read = self.read(reference).await?; - while read.next().await?.is_some() {} - Ok(()) - } -} - -pub(crate) fn blob_path(repository_id: [u8; 16], sha256: &[u8; 32]) -> Path { - Path::from(format!( - "repos/{}/git-blobs/{}", - hex::encode(repository_id), - hex::encode(sha256) - )) -} - -#[cfg(test)] -mod tests; diff --git a/src/large_blob/read.rs b/src/large_blob/read.rs deleted file mode 100644 index aa2d3084..00000000 --- a/src/large_blob/read.rs +++ /dev/null @@ -1,82 +0,0 @@ -use super::*; - -/// Sequential bounded ranges from one immutable Git blob. -pub struct LargeBlobRead { - store: Arc, - path: Path, - manifest: crate::external::Manifest, - reference: LargeBlobReference, - offset: u64, - hashes: Option, -} - -impl LargeBlobRead { - pub(super) async fn open( - store: Arc, - repository_id: [u8; 16], - reference: LargeBlobReference, - ) -> Result { - let path = blob_path(repository_id, &reference.sha256); - let manifest = crate::external::open(store.as_ref(), &path, reference.size).await?; - let read = Self { - store, - path, - manifest, - reference, - offset: 0, - hashes: Some(Hashes::new(reference.oid.format(), reference.size)), - }; - if reference.size == 0 { - read.check(Hashes::new(reference.oid.format(), 0))?; - } - Ok(read) - } - - pub(crate) fn reference(&self) -> LargeBlobReference { - self.reference - } - - /// Returns at most 8 MiB; an error or canceled read invalidates the reader. - pub async fn next(&mut self) -> Result, LargeBlobError> { - if self.offset == self.reference.size { - return Ok(None); - } - // Taking the verifier poisons cancellation/error paths. A partially read - // range cannot be silently retried with a truncated checksum history. - let hashes = self.hashes.take().ok_or(LargeBlobError::Corrupt)?; - let bytes = tokio::time::timeout(IO_TIMEOUT, self.range()) - .await - .map_err(|_| LargeBlobError::Timeout)??; - let (hashes, bytes) = hashes.update(bytes).await?; - let end = self.offset + bytes.len() as u64; - if end == self.reference.size { - self.check(hashes)?; - } else { - self.hashes = Some(hashes); - } - self.offset = end; - Ok(Some(bytes)) - } - - async fn range(&self) -> Result { - Ok(crate::external::read( - self.store.as_ref(), - &self.path, - &self.manifest, - self.reference.size, - self.offset, - ) - .await?) - } - - fn check(&self, hashes: Hashes) -> Result<(), LargeBlobError> { - let actual = hashes.finish(self.reference.size); - if actual.oid != self.reference.oid - || actual.sha256 != self.reference.sha256 - || actual.blake3 != self.reference.blake3 - { - return Err(LargeBlobError::Corrupt); - } - Ok(()) - } -} diff --git a/src/large_blob/tests.rs b/src/large_blob/tests.rs deleted file mode 100644 index 74fd4be0..00000000 --- a/src/large_blob/tests.rs +++ /dev/null @@ -1,165 +0,0 @@ -use super::*; -use crate::{ObjectKind, object_id}; -use object_store::{ObjectStoreExt, memory::InMemory}; -use std::future::poll_fn; - -type Result = std::result::Result>; - -#[tokio::test] -async fn streamed_objects_verify_all_hashes_without_consuming_the_next_frame() -> Result { - let store: Arc = Arc::new(InMemory::new()); - let blobs = LargeBlobStore::new(store.clone(), [1; 16]); - for (format, size) in [crate::ObjectFormat::Sha1, crate::ObjectFormat::Sha256] - .into_iter() - .flat_map(|format| [0, 1, CHUNK_BYTES, CHUNK_BYTES + 1].map(move |size| (format, size))) - { - let body = vec![21; size]; - let mut input = body.clone(); - input.extend_from_slice(b"\nnext frame"); - let mut input = input.as_slice(); - let oid = object_id(format, ObjectKind::Blob, &body); - let reference = blobs.put(oid, size as u64, &mut input).await?; - assert_eq!(input, b"\nnext frame"); - let replay = blobs.put(oid, size as u64, &mut body.as_slice()).await?; - assert_eq!(reference.sha256, replay.sha256); - let mut reader = blobs.read(&reference).await?; - let mut bytes_read = 0; - while let Some(bytes) = reader.next().await? { - assert!(bytes.len() <= CHUNK_BYTES); - assert_eq!(bytes.as_ref(), &body[bytes_read..bytes_read + bytes.len()]); - bytes_read += bytes.len(); - } - assert_eq!(bytes_read, size); - } - let mut objects = store.list(None); - while let Some(object) = poll_fn(|cx| objects.as_mut().poll_next(cx)).await { - assert!(!object?.location.as_ref().contains("staging")); - } - Ok(()) -} - -#[tokio::test] -async fn bad_input_does_not_publish_and_existing_corruption_is_not_overwritten() -> Result { - let store: Arc = Arc::new(InMemory::new()); - let blobs = LargeBlobStore::new(store.clone(), [2; 16]); - let body = b"immutable blob"; - let oid = object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, body); - assert!(matches!( - blobs - .put( - crate::ObjectId::Sha1([0; 20]), - body.len() as u64, - &mut body.as_slice() - ) - .await, - Err(LargeBlobError::Corrupt) - )); - assert!( - blobs - .put(oid, body.len() as u64 + 1, &mut body.as_slice()) - .await - .is_err() - ); - let listing = store.list_with_delimiter(None).await?; - assert!(listing.objects.is_empty() && listing.common_prefixes.is_empty()); - let reference = blobs - .put(oid, body.len() as u64, &mut body.as_slice()) - .await?; - let path = crate::external::part(&blob_path([2; 16], &reference.sha256), 0); - store - .put(&path, Bytes::from(vec![0; body.len()]).into()) - .await?; - let mut reader = blobs.read(&reference).await?; - assert!(matches!(reader.next().await, Err(LargeBlobError::Corrupt))); - assert!(matches!(reader.next().await, Err(LargeBlobError::Corrupt))); - assert!(matches!( - blobs - .put(oid, body.len() as u64, &mut body.as_slice()) - .await, - Err(LargeBlobError::Corrupt) - )); - assert_eq!( - store.get(&path).await?.bytes().await?.as_ref(), - vec![0; body.len()] - ); - Ok(()) -} - -#[tokio::test] -async fn metadata_hash_corruption_withholds_the_final_range() -> Result { - let store: Arc = Arc::new(InMemory::new()); - let blobs = LargeBlobStore::new(store.clone(), [3; 16]); - let body = vec![42; CHUNK_BYTES + 17]; - let reference = blobs - .put( - object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, &body), - body.len() as u64, - &mut body.as_slice(), - ) - .await?; - crate::external::copy_parts( - store.as_ref(), - &blob_path([3; 16], &reference.sha256), - &blob_path([3; 16], &[0; 32]), - reference.size, - ) - .await?; - store - .copy( - &blob_path([3; 16], &reference.sha256), - &blob_path([3; 16], &[0; 32]), - ) - .await?; - for bad in [ - LargeBlobReference { - sha256: [0; 32], - ..reference - }, - LargeBlobReference { - oid: crate::ObjectId::Sha1([0; 20]), - ..reference - }, - LargeBlobReference { - blake3: [0; 32], - ..reference - }, - ] { - let mut reader = blobs.read(&bad).await?; - assert_eq!( - reader.next().await?.ok_or("first range")?.len(), - CHUNK_BYTES - ); - assert!(matches!(reader.next().await, Err(LargeBlobError::Corrupt))); - } - Ok(()) -} - -#[tokio::test] -async fn replacement_between_ranges_invalidates_the_reader() -> Result { - let store: Arc = Arc::new(InMemory::new()); - let blobs = LargeBlobStore::new(store.clone(), [4; 16]); - let body = vec![42; CHUNK_BYTES + 17]; - let reference = blobs - .put( - object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, &body), - body.len() as u64, - &mut body.as_slice(), - ) - .await?; - let mut reader = blobs.read(&reference).await?; - reader.next().await?.ok_or("first range")?; - store - .put( - &blob_path([4; 16], &reference.sha256), - Bytes::from(vec![21; body.len()]).into(), - ) - .await?; - assert!(matches!( - reader.next().await, - Err(LargeBlobError::Store( - object_store::Error::Precondition { .. } - )) - )); - assert!(matches!(reader.next().await, Err(LargeBlobError::Corrupt))); - Ok(()) -} diff --git a/src/lib.rs b/src/lib.rs index 5a0dfc76..026c2f57 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,15 +30,15 @@ pub mod git_gateway; pub mod git_http; pub mod git_input; mod git_objects; +pub mod git_pack; mod git_read; mod graph; pub mod http; pub mod issues; -pub mod large_blob; pub mod lfs; mod native_git; mod object_batch; -mod object_chunks; +mod object_packs; mod object_reads; pub mod pulls; mod push; @@ -52,11 +52,12 @@ mod web; pub use access::{COLLABORATOR_PAGE_SIZE, Collaborator, ReadIdentity}; pub use default_branch::DefaultBranch; pub use object_batch::ObjectBatch; -pub use object_chunks::ObjectStageError; +pub use object_packs::{ObjectStageError, PackedObject}; pub use push::{PushCertificateReceipt, PushError, PushReceipt}; pub use refs::{FinalizePush, PushPlan, RefExpectation, RefPage, RefReadError, RefUpdate}; pub const REPOSITORIES: NamespaceId = NamespaceId::from_bytes([71; 16]); +pub const REPOSITORY_SCHEMA_VERSION: u32 = 3; pub const INLINE_OBJECT_LIMIT: usize = 768 * 1024; // SQLite's on-disk page count and maximum page size bound representable databases. // This is a format boundary, not a repository quota. @@ -81,9 +82,9 @@ const COMMANDS: [OperationDescriptor; 9] = [ operation_with_codec(4, 6), OperationDescriptor { input_limit: object_batch::INPUT_LIMIT, - ..operation_with_codec(5, 3) + ..operation_with_codec(5, 4) }, - operation_with_codec(6, 2), + operation_with_codec(6, 4), operation_with_codec(7, 2), operation_with_codec(8, 2), operation_with_codec(9, 4), @@ -99,8 +100,8 @@ const fn operation_with_codec(id: u32, codec_version: u32) -> OperationDescripto OperationDescriptor { id, codec_version, - schema_min: 1, - schema_max: 1, + schema_min: REPOSITORY_SCHEMA_VERSION, + schema_max: REPOSITORY_SCHEMA_VERSION, input_limit: 1 << 20, output_limit: 1 << 20, } @@ -148,19 +149,11 @@ impl ObjectKind { } } -/// Byte location for a verified Git object record. +/// In-memory ingress or a verified immutable native pack reference. +/// `Input` is never persisted; only `Packed` appears in stored object pages. pub enum ObjectStorage { - Inline(Vec), - Chunked { - upload: [u8; 16], - size: u64, - blake3: [u8; 32], - }, - External { - size: u64, - blake3: [u8; 32], - sha256: [u8; 32], - }, + Input(Vec), + Packed(Box), } /// Immutable Git object returned by one bounded Repository Cell read. @@ -190,6 +183,9 @@ impl CellModule for RepositoryModule { let mut source = blake3::Hasher::new(); source.update(include_bytes!("lib.rs")); source.update(include_bytes!("git_format.rs")); + source.update(include_bytes!("git_pack.rs")); + source.update(include_bytes!("git_pack/store.rs")); + source.update(include_bytes!("git_pack/build.rs")); source.update(include_bytes!("refs.rs")); source.update(include_bytes!("default_branch.rs")); source.update(include_bytes!("graph.rs")); @@ -197,10 +193,10 @@ impl CellModule for RepositoryModule { source.update(include_bytes!("branch_rules.rs")); source.update(include_bytes!("branch_rules/command.rs")); source.update(include_bytes!("graph/preparation.rs")); + source.update(include_bytes!("graph/stream.rs")); source.update(include_bytes!("object_batch.rs")); - source.update(include_bytes!("object_chunks.rs")); + source.update(include_bytes!("object_packs.rs")); source.update(include_bytes!("object_reads.rs")); - source.update(include_bytes!("large_blob.rs")); source.update(include_bytes!("external.rs")); source.update(include_bytes!("push.rs")); source.update(include_bytes!("push/plan.rs")); @@ -224,11 +220,11 @@ impl CellModule for RepositoryModule { Digest::from_bytes(*source.finalize().as_bytes()) }, retained_codes: &[], - schema_min: 1, - schema_max: 1, + schema_min: REPOSITORY_SCHEMA_VERSION, + schema_max: REPOSITORY_SCHEMA_VERSION, migrations: MIGRATIONS.get_or_init(|| { [MigrationDescriptor { - version: 1, + version: REPOSITORY_SCHEMA_VERSION, sql: SCHEMA, digest: Digest::from_bytes(*blake3::hash(SCHEMA.as_bytes()).as_bytes()), }] @@ -283,6 +279,7 @@ fn repository_cell_type() -> cellule_runtime::Result { 1, )? .with_entity_partitions()? + .with_schema_range(REPOSITORY_SCHEMA_VERSION, REPOSITORY_SCHEMA_VERSION)? .with_limits(REPOSITORY_DATABASE_LIMIT_BYTES, 64 * 1024 * 1024) } @@ -293,6 +290,7 @@ pub struct RepositoryCell { sql: SqlCell, application: ApplicationHandle, target: CellTarget, + packs: git_pack::PackStore, } impl RepositoryCell { @@ -309,16 +307,21 @@ impl RepositoryCell { target: CellTarget, id: [u8; 16], object_format: ObjectFormat, + packs: git_pack::PackStore, ) -> cellule_runtime::Result { if target != repository_target(target.tenant(), target.application(), id)? { return Err(Error::Identity("repository UUID differs from Cell target")); } + if packs.repository_id() != id { + return Err(Error::Identity("pack store UUID differs from repository")); + } Ok(Self { id, object_format, sql: application.sql::(target.clone())?, application: application.clone(), target, + packs, }) } @@ -343,86 +346,62 @@ impl RepositoryCell { Observed)>>, cellule_runtime::InvocationError>, > { - let result = self + let headers = self .sql .query( minimum, SqlBatch { statements: vec![SqlStatement { - sql: - "SELECT kind, body, digest, size, chunk_id FROM objects WHERE oid = ?1" - .into(), + sql: "SELECT oid FROM objects WHERE oid = ?1".into(), parameters: vec![SqlValue::Blob(oid.to_vec())], }], }, ) .await?; - let Some(row) = result.output.first().and_then(|set| set.rows.first()) else { + if headers.output.first().is_none_or(|set| set.rows.is_empty()) { return Ok(Observed { output: None, - receipt: result.receipt, + receipt: headers.receipt, }); - }; - let [ - SqlValue::Text(kind), - body, - SqlValue::Blob(digest), - SqlValue::Integer(size), - upload, - ] = row.as_slice() - else { + } + let objects = self + .object_records(Observed { + output: vec![oid], + receipt: headers.receipt, + }) + .await?; + let object = objects.output.into_iter().next().ok_or_else(|| { + cellule_runtime::InvocationError::NotStarted(Error::Command("missing packed object")) + })?; + let ObjectStorage::Packed(reference) = object.storage else { return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("invalid stored object row"), + Error::Command("invalid persisted storage"), )); }; - let kind = match kind.as_str() { - "blob" => ObjectKind::Blob, - "tree" => ObjectKind::Tree, - "commit" => ObjectKind::Commit, - "tag" => ObjectKind::Tag, - _ => { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("invalid stored object kind"), - )); - } - }; - let body = match (body, upload) { - (SqlValue::Blob(bytes), SqlValue::Null) - if usize::try_from(*size).ok() == Some(bytes.len()) => + let pack = self + .packs + .load(&reference.pack) + .await + .map_err(object_packs::pack_invocation)?; + let bytes = tokio::task::spawn_blocking(move || { + let (kind, body) = pack + .data()? + .object(oid, 64 * 1024 * 1024)? + .ok_or(git_pack::PackError::Integrity)?; + if kind != object.kind + || body.len() as u64 != reference.size + || blake3::hash(&body).as_bytes() != &reference.digest { - bytes.clone() - } - (SqlValue::Null, SqlValue::Blob(upload)) => { - let invalid = || { - cellule_runtime::InvocationError::NotStarted(Error::Command( - "invalid object chunk reference", - )) - }; - self.chunked_body( - oid, - kind, - upload.as_slice().try_into().map_err(|_| invalid())?, - u64::try_from(*size).map_err(|_| invalid())?, - digest.as_slice().try_into().map_err(|_| invalid())?, - ) - .await? + return Err(git_pack::PackError::Integrity); } - _ => { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("invalid stored object body"), - )); - } - }; - if object_id(oid.format(), kind, &body) != oid - || blake3::hash(&body).as_bytes() != digest.as_slice() - { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("corrupt stored object"), - )); - } + Ok((kind, body)) + }) + .await + .map_err(|error| object_packs::pack_invocation(git_pack::PackError::Task(error)))? + .map_err(object_packs::pack_invocation)?; Ok(Observed { - output: Some((kind, body)), - receipt: result.receipt, + output: Some(bytes), + receipt: objects.receipt, }) } } diff --git a/src/main.rs b/src/main.rs index 3e2a3c6e..b3f04777 100644 --- a/src/main.rs +++ b/src/main.rs @@ -46,7 +46,7 @@ struct FileSshConfig { #[derive(Debug, Error)] enum StartupError { #[error( - "usage: canopy | canopy maintenance status | canopy maintenance begin|recover|end | canopy backup create|verify | canopy backup restore " + "usage: canopy | canopy maintenance status | canopy maintenance begin|recover|end | canopy gc | canopy backup create|verify | canopy backup restore " )] Usage, #[error("SSH host key is invalid")] @@ -117,6 +117,37 @@ async fn run() -> Result<(), StartupError> { let Some(first) = args.next() else { return Err(StartupError::Usage); }; + if first == "gc" { + let path = args.next().ok_or(StartupError::Usage)?; + let operation = args.next().ok_or(StartupError::Usage)?; + let grace = args.next().ok_or(StartupError::Usage)?; + let limit = args.next().ok_or(StartupError::Usage)?; + if args.next().is_some() { + return Err(StartupError::Usage); + } + let file: FileConfig = serde_json::from_slice(&std::fs::read(path)?)?; + let operation = RequestId::from_bytes( + Uuid::parse_str(operation.to_str().ok_or(StartupError::Usage)?)?.into_bytes(), + ); + let number = |value: &std::ffi::OsStr| -> Result { + value + .to_str() + .ok_or(StartupError::Usage)? + .parse() + .map_err(|_| StartupError::Usage) + }; + let deployment = deployment(&file)?; + let report = deployment + .collect_garbage( + operation, + worker_config(file)?, + number(&grace)?, + number(&limit)?, + ) + .await?; + println!("{}", serde_json::to_string(&report)?); + return Ok(()); + } if first == "backup" { let path = args.next().ok_or(StartupError::Usage)?; let action = args.next().ok_or(StartupError::Usage)?; diff --git a/src/native_git.rs b/src/native_git.rs index 00ffd9ea..e3a9318b 100644 --- a/src/native_git.rs +++ b/src/native_git.rs @@ -35,33 +35,7 @@ pub(crate) fn idle_fence(git_dir: &Path) -> io::Result> { pub(crate) fn command(git_dir: &Path) -> io::Result { let mut command = Command::new("git"); - let fence = lock_file(&git_dir.join(WORKER_LOCK))?; - fence.try_lock_shared().map_err(io::Error::from)?; - #[cfg(unix)] - { - use std::os::fd::{AsRawFd, FromRawFd}; - // Keep the fence outside stdio slots that spawn replaces. CLOEXEC is - // cleared only in this child, never in the multithreaded parent. - // SAFETY: fcntl duplicates this live descriptor into a new owned slot. - let fd = unsafe { libc::fcntl(fence.as_raw_fd(), libc::F_DUPFD_CLOEXEC, 3) }; - if fd == -1 { - return Err(io::Error::last_os_error()); - } - // SAFETY: the successful duplication returned an unowned descriptor. - let fence = unsafe { File::from_raw_fd(fd) }; - // SAFETY: only async-signal-safe fcntl runs after fork. The closure owns - // the descriptor through spawn; exec and descendants retain its lock. - unsafe { - command.pre_exec(move || { - let fd = fence.as_raw_fd(); - let flags = libc::fcntl(fd, libc::F_GETFD); - if flags == -1 || libc::fcntl(fd, libc::F_SETFD, flags & !libc::FD_CLOEXEC) == -1 { - return Err(io::Error::last_os_error()); - } - Ok(()) - }); - } - } + inherit_fence(&mut command, git_dir)?; // Host configuration can redirect objects, execute helpers or emit traces // outside our accounting. Provider credentials must not reach Git or hooks. command.env_clear(); @@ -107,6 +81,45 @@ pub(crate) fn command(git_dir: &Path) -> io::Result { Ok(command) } +/// Retains both the private output fence and its managed cache ancestor through +/// exec and descendants. Startup fences cache ancestors before reclaiming them. +pub(crate) fn command_with_extra_fence(directory: &Path, ancestor: &Path) -> io::Result { + let mut command = command(directory)?; + inherit_fence(&mut command, ancestor)?; + Ok(command) +} + +fn inherit_fence(command: &mut Command, directory: &Path) -> io::Result<()> { + let fence = lock_file(&directory.join(WORKER_LOCK))?; + fence.try_lock_shared().map_err(io::Error::from)?; + #[cfg(unix)] + { + use std::os::fd::{AsRawFd, FromRawFd}; + // Keep the fence outside stdio slots that spawn replaces. CLOEXEC is + // cleared only in this child, never in the multithreaded parent. + // SAFETY: fcntl duplicates this live descriptor into a new owned slot. + let fd = unsafe { libc::fcntl(fence.as_raw_fd(), libc::F_DUPFD_CLOEXEC, 3) }; + if fd == -1 { + return Err(io::Error::last_os_error()); + } + // SAFETY: the successful duplication returned an unowned descriptor. + let fence = unsafe { File::from_raw_fd(fd) }; + // SAFETY: only async-signal-safe fcntl runs after fork. The closure owns + // the descriptor through spawn; exec and descendants retain its lock. + unsafe { + command.pre_exec(move || { + let fd = fence.as_raw_fd(); + let flags = libc::fcntl(fd, libc::F_GETFD); + if flags == -1 || libc::fcntl(fd, libc::F_SETFD, flags & !libc::FD_CLOEXEC) == -1 { + return Err(io::Error::last_os_error()); + } + Ok(()) + }); + } + } + Ok(()) +} + #[cfg(all(test, unix))] mod tests { use super::*; @@ -202,3 +215,46 @@ mod tests { Ok(()) } } + +#[cfg(all(test, unix))] +mod pack_fence_tests { + use super::*; + use std::process::Stdio; + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + + #[tokio::test] + async fn private_pack_worker_also_fences_its_managed_cache_ancestor() + -> Result<(), Box> { + let root = tempfile::TempDir::new()?; + let private = root.path().join("private"); + let ancestor = root.path().join("ancestor"); + std::fs::create_dir(&private)?; + std::fs::create_dir(&ancestor)?; + let mut child = command_with_extra_fence(&private, &ancestor)? + .args(["-c", "alias.hold=!printf 'ready\\n'; read release", "hold"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .spawn()?; + let mut input = child.stdin.take().ok_or("stdin")?; + let mut output = BufReader::new(child.stdout.take().ok_or("stdout")?); + let mut ready = String::new(); + tokio::time::timeout( + std::time::Duration::from_secs(5), + output.read_line(&mut ready), + ) + .await??; + assert_eq!(ready, "ready\n"); + for path in [&private, &ancestor] { + assert!(idle_fence(path).is_err_and(|error| error.kind() == io::ErrorKind::WouldBlock)); + } + input.write_all(b"release\n").await?; + drop(input); + assert!(child.wait().await?.success()); + for path in [&private, &ancestor] { + assert!(idle_fence(path)?.is_some()); + } + Ok(()) + } +} diff --git a/src/object_batch.rs b/src/object_batch.rs index 44661062..1c685fd4 100644 --- a/src/object_batch.rs +++ b/src/object_batch.rs @@ -1,320 +1,380 @@ -//! Bounded, atomic publication of verified repository object records. - -use std::collections::BTreeSet; +//! Bounded atomic metadata publication after immutable native-pack verification. +use crate::{ + INLINE_OBJECT_LIMIT, ObjectKind, ObjectStorage, PackedObject, RepositoryCell, RepositoryModule, + StoredObject, +}; use cellule_runtime::{ CellModule, Command, Committed, Error, InvocationError, MutationIdentity, Observed, - codec::BoundedDecoder, codec::BoundedEncoder, codec::CodecError, codec::WireValue, - primitives::sql::SqlBatch, primitives::sql::SqlResultSet, primitives::sql::SqlStatement, - primitives::sql::SqlValue, registry::CommandContext, registry::CommandResult, -}; - -use crate::{ - INLINE_OBJECT_LIMIT, ObjectKind, ObjectStorage, RepositoryCell, RepositoryModule, StoredObject, - object_id, + codec::{BoundedDecoder, BoundedEncoder, CodecError, WireValue}, + primitives::sql::{SqlBatch, SqlResultSet, SqlStatement, SqlValue}, + registry::{CommandContext, CommandResult}, }; +use std::collections::{BTreeMap, BTreeSet}; -pub(crate) const MAX_OBJECTS: usize = 128; +pub(crate) const MAX_OBJECTS: usize = 512; pub(crate) const VERIFY_BATCH_BYTES: u64 = 64 * 1024 * 1024; pub(crate) const INPUT_LIMIT: u32 = 4 * 1024 * 1024; -// Leave room for record metadata inside Cellule's bounded command wire format. -const INLINE_BATCH_BYTES: usize = 3 * 1024 * 1024; +const INPUT_BYTES: usize = 3 * 1024 * 1024; -/// Batches small objects together and verifies oversized SQLite objects individually. #[derive(Default)] pub struct ObjectBatch { objects: Vec, - inline_bytes: usize, - verified_bytes: u64, + input_bytes: usize, } - impl ObjectBatch { - /// Adds a record, returning it unchanged if the batch's count or byte budget is full. pub fn try_push(&mut self, object: StoredObject) -> Result<(), StoredObject> { let bytes = match &object.storage { - ObjectStorage::Inline(body) => body.len(), - ObjectStorage::External { .. } | ObjectStorage::Chunked { .. } => 0, - }; - let verified = match &object.storage { - ObjectStorage::Inline(body) => body.len() as u64, - ObjectStorage::Chunked { size, .. } => *size, - ObjectStorage::External { .. } => 0, + ObjectStorage::Input(body) => body.len(), + ObjectStorage::Packed(_) => 0, }; if self.objects.len() == MAX_OBJECTS || bytes > INLINE_OBJECT_LIMIT - || bytes > INLINE_BATCH_BYTES - self.inline_bytes - || i64::try_from(verified).is_err() - || (!self.objects.is_empty() - && verified > VERIFY_BATCH_BYTES.saturating_sub(self.verified_bytes)) + || bytes > INPUT_BYTES - self.input_bytes { return Err(object); } - self.inline_bytes += bytes; - self.verified_bytes += verified; + self.input_bytes += bytes; self.objects.push(object); Ok(()) } - - #[must_use] pub fn is_empty(&self) -> bool { self.objects.is_empty() } } +fn kind_tag(kind: ObjectKind) -> u8 { + match kind { + ObjectKind::Blob => 0, + ObjectKind::Tree => 1, + ObjectKind::Commit => 2, + ObjectKind::Tag => 3, + } +} +fn kind_decode(tag: u8) -> Result { + match tag { + 0 => Ok(ObjectKind::Blob), + 1 => Ok(ObjectKind::Tree), + 2 => Ok(ObjectKind::Commit), + 3 => Ok(ObjectKind::Tag), + _ => Err(CodecError::Invalid("invalid object kind")), + } +} +fn fixed(decoder: &mut BoundedDecoder<'_>) -> Result<[u8; N], CodecError> { + decoder + .read_bytes()? + .try_into() + .map_err(|_| CodecError::Invalid("invalid identity width")) +} + impl WireValue for ObjectBatch { fn encode(&self, encoder: &mut BoundedEncoder) -> Result<(), CodecError> { if self.is_empty() { - return Err(CodecError::Invalid("object batch is empty")); + return Err(CodecError::Invalid("empty object batch")); } encoder.write_count(self.objects.len())?; for object in &self.objects { encoder.write_bytes(&object.oid)?; - encoder.write_u8(match object.kind { - ObjectKind::Blob => 0, - ObjectKind::Tree => 1, - ObjectKind::Commit => 2, - ObjectKind::Tag => 3, - })?; + encoder.write_u8(kind_tag(object.kind))?; match &object.storage { - ObjectStorage::Inline(body) => { + ObjectStorage::Input(body) => { encoder.write_u8(0)?; encoder.write_bytes(body)?; } - ObjectStorage::Chunked { - upload, - size, - blake3, - } => { - encoder.write_u8(2)?; - encoder.write_bytes(upload)?; - encoder.write_u64(*size)?; - encoder.write_bytes(blake3)?; - } - ObjectStorage::External { - size, - blake3, - sha256, - } => { + ObjectStorage::Packed(value) => { encoder.write_u8(1)?; - encoder.write_u64(*size)?; - encoder.write_bytes(blake3)?; - encoder.write_bytes(sha256)?; + encoder.write_bytes(&value.repository)?; + encoder.write_bytes(&value.oid)?; + encoder.write_u8(kind_tag(value.kind))?; + encoder.write_bytes( + &serde_json::to_vec(&value.pack) + .map_err(|_| CodecError::Invalid("pack identity"))?, + )?; + encoder.write_u64(value.size)?; + encoder.write_bytes(&value.digest)?; + encoder.write_u8(u8::from(value.graph_valid))?; + encoder.write_u64(value.graph_count)?; + encoder.write_bytes(&value.graph_digest)?; + encoder.write_bytes(value.rebase_metadata.as_deref().unwrap_or_default())?; } } } Ok(()) } - fn decode(decoder: &mut BoundedDecoder<'_>) -> Result { let count = decoder.read_count()?; if !(1..=MAX_OBJECTS).contains(&count) { - return Err(CodecError::Invalid("object count is outside batch bounds")); + return Err(CodecError::Invalid("object count")); } let mut batch = Self::default(); for _ in 0..count { - let oid = crate::ObjectId::try_from(decoder.read_bytes()?) - .map_err(|_| CodecError::Invalid("invalid Git object ID"))?; - let kind = match decoder.read_u8()? { - 0 => ObjectKind::Blob, - 1 => ObjectKind::Tree, - 2 => ObjectKind::Commit, - 3 => ObjectKind::Tag, - _ => return Err(CodecError::Invalid("invalid Git object kind")), - }; + let oid = decoder + .read_bytes()? + .try_into() + .map_err(|_| CodecError::Invalid("object OID"))?; + let kind = kind_decode(decoder.read_u8()?)?; let storage = match decoder.read_u8()? { - 0 => { - let body = decoder.read_bytes()?; - if body.len() > INLINE_OBJECT_LIMIT - || body.len() > INLINE_BATCH_BYTES - batch.inline_bytes - { - return Err(CodecError::Invalid( - "object batch exceeds inline byte budget", - )); - } - ObjectStorage::Inline(body.to_vec()) + 0 => ObjectStorage::Input(decoder.read_bytes()?.to_vec()), + 1 => { + let repository = fixed(decoder)?; + let token_oid = decoder + .read_bytes()? + .try_into() + .map_err(|_| CodecError::Invalid("pack object OID"))?; + let token_kind = kind_decode(decoder.read_u8()?)?; + let pack = serde_json::from_slice(decoder.read_bytes()?) + .map_err(|_| CodecError::Invalid("pack identity"))?; + let size = decoder.read_u64()?; + let digest = fixed(decoder)?; + let graph_valid = match decoder.read_u8()? { + 0 => false, + 1 => true, + _ => return Err(CodecError::Invalid("graph validity")), + }; + ObjectStorage::Packed(Box::new(PackedObject { + repository, + oid: token_oid, + kind: token_kind, + pack, + size, + digest, + graph_valid, + graph_count: decoder.read_u64()?, + graph_digest: fixed(decoder)?, + dependencies: None, + rebase_metadata: match decoder.read_bytes()? { + [] => None, + bytes if bytes.len() == 65 => Some(bytes.to_vec()), + _ => return Err(CodecError::Invalid("rebase metadata width")), + }, + })) } - 1 => ObjectStorage::External { - size: decoder.read_u64()?, - blake3: fixed(decoder)?, - sha256: fixed(decoder)?, - }, - 2 => ObjectStorage::Chunked { - upload: fixed(decoder)?, - size: decoder.read_u64()?, - blake3: fixed(decoder)?, - }, - _ => return Err(CodecError::Invalid("invalid Git object storage")), + _ => return Err(CodecError::Invalid("object storage")), }; batch .try_push(StoredObject { oid, kind, storage }) - .map_err(|_| CodecError::Invalid("object batch exceeds bounds"))?; + .map_err(|_| CodecError::Invalid("object batch bounds"))?; } Ok(batch) } } -fn fixed(decoder: &mut BoundedDecoder<'_>) -> Result<[u8; N], CodecError> { - decoder - .read_bytes()? - .try_into() - .map_err(|_| CodecError::Invalid("invalid object identity width")) -} - pub(crate) struct PutObjects; - impl Command for PutObjects { const MODULE: &'static str = RepositoryModule::NAME; const ID: u32 = 5; - const CODEC_VERSION: u32 = 3; + const CODEC_VERSION: u32 = 4; type Input = ObjectBatch; type Output = (); - fn execute( context: &mut CommandContext<'_, '_>, - batch: Self::Input, + batch: ObjectBatch, ) -> cellule_runtime::Result> { let identity = context.sql(&SqlBatch { statements: vec![SqlStatement { - sql: "SELECT object_format FROM repository_identity WHERE singleton = 1".into(), - parameters: Vec::new(), + sql: "SELECT object_format, repository_id FROM repository_identity WHERE singleton = 1".into(), + parameters: vec![], }], })?; - let Some([SqlValue::Text(format)]) = identity + let Some([SqlValue::Text(format), SqlValue::Blob(repository)]) = identity .first() .and_then(|set| set.rows.first()) .map(Vec::as_slice) else { - return Err(Error::Command("repository identity is absent")); + return Err(Error::Command("repository identity absent")); }; let format = crate::ObjectFormat::parse(format) - .ok_or(Error::Command("invalid repository object format"))?; - if batch - .objects - .iter() - .any(|object| object.oid.format() != format) - { - return Ok(CommandResult::Rejected(())); - } + .ok_or(Error::Command("invalid repository format"))?; + let mut records = BTreeMap::new(); + let mut packs = BTreeMap::new(); for object in batch.objects { - let (size, digest, storage, body, sha256, chunk_id) = match object.storage { - ObjectStorage::Inline(body) => { - if object_id(object.oid.format(), object.kind, &body) != object.oid { - return Ok(CommandResult::Rejected(())); - } - ( - body.len() as i64, - blake3::hash(&body).as_bytes().to_vec(), - "inline", - SqlValue::Blob(body), - SqlValue::Null, - SqlValue::Null, - ) - } - ObjectStorage::External { - size, - blake3, - sha256, - } => { - if object.kind != ObjectKind::Blob || i64::try_from(size).is_err() { - return Ok(CommandResult::Rejected(())); - } - ( - size as i64, - blake3.to_vec(), - "external", - SqlValue::Null, - SqlValue::Blob(sha256.to_vec()), - SqlValue::Null, - ) - } - ObjectStorage::Chunked { - upload, - size, - blake3, - } => { - if crate::object_chunks::body( - context, - object.oid, - object.kind, - upload, - size, - blake3, - )? - .is_none() - { - return Ok(CommandResult::Rejected(())); - } - ( - size as i64, - blake3.to_vec(), - "chunked", - SqlValue::Null, - SqlValue::Null, - SqlValue::Blob(upload.to_vec()), - ) - } + let ObjectStorage::Packed(value) = object.storage else { + return Ok(CommandResult::Rejected(())); }; + if value.repository.as_slice() != repository.as_slice() + || object.oid != value.oid + || object.kind != value.kind + || object.oid.format() != format + || value.pack.format != format + || value.pack.pack_bytes == 0 + || value.pack.index_bytes == 0 + || (object.kind == ObjectKind::Blob + && (!value.graph_valid + || value.graph_count != 0 + || value.graph_digest != *blake3::hash(b"").as_bytes() + || value.rebase_metadata.is_some())) + || i64::try_from(value.size).is_err() + || i64::try_from(value.pack.pack_bytes).is_err() + || i64::try_from(value.pack.index_bytes).is_err() + || value.graph_count > VERIFY_BATCH_BYTES / (format.bytes() as u64 + 1) + { + return Ok(CommandResult::Rejected(())); + } let expected = vec![ SqlValue::Text(object.kind.git_name().into()), - SqlValue::Integer(size), - SqlValue::Blob(digest), - SqlValue::Text(storage.into()), - body, - sha256, + SqlValue::Integer(value.size as i64), + SqlValue::Blob(value.digest.to_vec()), + SqlValue::Integer(i64::from(value.graph_valid)), + SqlValue::Integer(value.graph_count as i64), + SqlValue::Blob(value.graph_digest.to_vec()), + value.rebase_metadata.map_or(SqlValue::Null, SqlValue::Blob), + ]; + if let Some((previous, _)) = + records.insert(object.oid, (expected.clone(), value.pack.pack_digest)) + && previous != expected + { + return Ok(CommandResult::Rejected(())); + } + let pack = &value.pack; + let expected = vec![ + SqlValue::Text(pack.format.as_str().into()), + SqlValue::Integer(pack.pack_bytes as i64), + SqlValue::Blob(pack.index_digest.to_vec()), + SqlValue::Integer(pack.index_bytes as i64), ]; - let mut parameters = vec![SqlValue::Blob(object.oid.to_vec())]; + if let Some(previous) = packs.insert(pack.pack_digest, expected.clone()) + && previous != expected + { + return Ok(CommandResult::Rejected(())); + } + } + for (digest, expected) in packs { + let mut parameters = vec![SqlValue::Blob(digest.to_vec())]; parameters.extend(expected.iter().cloned()); - parameters.push(chunk_id); let result = context.sql(&SqlBatch { statements: vec![ - SqlStatement { - sql: "INSERT INTO objects (oid, kind, size, digest, storage, body, external_sha256, chunk_id) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) ON CONFLICT(oid) DO NOTHING".into(), - parameters, - }, - SqlStatement { - sql: "SELECT kind, size, digest, storage, body, external_sha256 FROM objects WHERE oid = ?1".into(), - parameters: vec![SqlValue::Blob(object.oid.to_vec())], - }, + SqlStatement { sql: "INSERT INTO git_packs (digest, format, bytes, index_digest, index_bytes) VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING".into(), parameters }, + SqlStatement { sql: "SELECT format, bytes, index_digest, index_bytes FROM git_packs WHERE digest = ?".into(), parameters: vec![SqlValue::Blob(digest.to_vec())] }, ]})?; - // A collision or corrupt existing record rejects the entire application - // savepoint. No earlier record in this batch may survive that rejection. if result.get(1).and_then(|set| set.rows.first()) != Some(&expected) { return Ok(CommandResult::Rejected(())); } } + let mut parameters = Vec::with_capacity(records.len() * 9); + for (oid, (expected, pack)) in &records { + parameters.push(SqlValue::Blob(oid.to_vec())); + parameters.extend(expected.iter().cloned()); + parameters.push(SqlValue::Blob(pack.to_vec())); + } + let result = context.sql(&SqlBatch { statements: vec![ + SqlStatement { sql: format!("INSERT INTO objects (oid, kind, size, digest, graph_valid, graph_count, graph_digest, rebase_metadata, pack) VALUES {} ON CONFLICT(oid) DO NOTHING", vec!["(?, ?, ?, ?, ?, ?, ?, ?, ?)"; records.len()].join(",")), parameters }, + SqlStatement { sql: format!("SELECT oid, kind, size, digest, graph_valid, graph_count, graph_digest, rebase_metadata FROM objects WHERE oid IN ({}) ORDER BY oid", vec!["?"; records.len()].join(",")), parameters: records.keys().map(|oid| SqlValue::Blob(oid.to_vec())).collect() }, + ]})?; + let rows = &result + .get(1) + .ok_or(Error::Command("missing object collision check"))? + .rows; + if rows.len() != records.len() { + return Ok(CommandResult::Rejected(())); + } + for (row, (oid, (expected, _))) in rows.iter().zip(&records) { + let Some((SqlValue::Blob(stored), fields)) = row.split_first() else { + return Err(Error::Command("invalid object collision check")); + }; + if stored.as_slice() != oid.as_ref() || fields != expected.as_slice() { + return Ok(CommandResult::Rejected(())); + } + } + // Verified blobs have no dependencies. Publish their closure proofs + // with the metadata batch rather than scheduling per-blob graph work. + let blobs: Vec<_> = records + .iter() + .filter(|(_, (expected, _))| expected[0] == SqlValue::Text("blob".into())) + .map(|(oid, _)| *oid) + .collect(); + if !blobs.is_empty() { + context.sql(&SqlBatch { + statements: vec![SqlStatement { + sql: format!( + "INSERT INTO object_closure (oid) VALUES {} ON CONFLICT DO NOTHING", + vec!["(?)"; blobs.len()].join(",") + ), + parameters: blobs + .iter() + .map(|oid| SqlValue::Blob(oid.to_vec())) + .collect(), + }], + })?; + } Ok(CommandResult::Success(())) } } impl RepositoryCell { - /// Publishes a bounded object batch atomically; external bytes must already be verified and uploaded. pub async fn put_objects( &self, identity: MutationIdentity, - batch: ObjectBatch, + mut batch: ObjectBatch, ) -> Result, InvocationError<()>> { + let input: Vec<_> = batch + .objects + .iter() + .filter_map(|object| match &object.storage { + ObjectStorage::Input(body) + if object.oid.format() == self.object_format + && crate::object_id(self.object_format, object.kind, body) + == object.oid => + { + Some((object.kind, body.clone())) + } + _ => None, + }) + .collect(); + if !input.is_empty() { + let source = self + .packs + .prepare(self.object_format, input) + .await + .map_err(crate::object_packs::pack_invocation)?; + let pack = self + .packs + .publish(source.clone()) + .await + .map_err(crate::object_packs::pack_invocation)?; + for object in &mut batch.objects { + if let ObjectStorage::Input(body) = &object.storage + && object.oid.format() == self.object_format + && crate::object_id(self.object_format, object.kind, body) == object.oid + { + object.storage = self + .record_from_pack(source.clone(), pack.clone(), object.oid) + .await + .map_err(crate::object_packs::pack_invocation)? + .storage; + } + } + } + self.stage_dependencies(identity, &mut batch.objects) + .await + .map_err(|error| { + InvocationError::NotStarted(Error::Facility { + name: "pack dependency staging", + source: Box::new(error), + }) + })?; self.application .command::(&self.target, identity, batch) .await } - /// Returns recorded IDs from a nonempty candidate list of at most 128 objects. pub async fn existing_objects( &self, candidates: &[crate::ObjectId], ) -> Result>, InvocationError>> { if !(1..=MAX_OBJECTS).contains(&candidates.len()) { return Err(InvocationError::NotStarted(Error::Command( - "object lookup count is outside bounds", + "object lookup count outside bounds", ))); } - let placeholders = vec!["?"; candidates.len()].join(","); let result = self .sql .query( None, SqlBatch { statements: vec![SqlStatement { - sql: format!("SELECT oid FROM objects WHERE oid IN ({placeholders})"), + sql: format!( + "SELECT oid FROM objects WHERE oid IN ({})", + vec!["?"; candidates.len()].join(",") + ), parameters: candidates .iter() .map(|oid| SqlValue::Blob(oid.to_vec())) @@ -326,20 +386,20 @@ impl RepositoryCell { let rows = &result .output .first() - .ok_or(InvocationError::NotStarted(Error::Command( - "missing object lookup result", - )))? + .ok_or_else(|| InvocationError::NotStarted(Error::Command("missing object lookup")))? .rows; let mut objects = BTreeSet::new(); for row in rows { let [SqlValue::Blob(oid)] = row.as_slice() else { return Err(InvocationError::NotStarted(Error::Command( - "invalid object lookup row", + "invalid object lookup", ))); }; - objects.insert(oid.as_slice().try_into().map_err(|_| { - InvocationError::NotStarted(Error::Command("invalid stored Git OID")) - })?); + objects.insert( + oid.as_slice().try_into().map_err(|_| { + InvocationError::NotStarted(Error::Command("invalid stored OID")) + })?, + ); } Ok(Observed { output: objects, diff --git a/src/object_batch/tests.rs b/src/object_batch/tests.rs index 5cddd839..147fd102 100644 --- a/src/object_batch/tests.rs +++ b/src/object_batch/tests.rs @@ -4,7 +4,7 @@ fn inline(size: usize) -> StoredObject { StoredObject { oid: crate::ObjectId::Sha1([0; 20]), kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(vec![11; size]), + storage: ObjectStorage::Input(vec![11; size]), } } @@ -17,11 +17,7 @@ fn maximum_batch_round_trips_below_the_operation_wire_limit() { .try_push(StoredObject { oid: crate::ObjectId::Sha1([1; 20]), kind: ObjectKind::Blob, - storage: ObjectStorage::External { - size: 1_000_000, - blake3: [2; 32], - sha256: [3; 32] - }, + storage: ObjectStorage::Input(Vec::new()), }) .is_ok() ); @@ -43,7 +39,7 @@ fn maximum_batch_round_trips_below_the_operation_wire_limit() { fn a_full_batch_returns_the_unconsumed_record() { let mut batch = ObjectBatch::default(); assert!(batch.try_push(inline(INLINE_OBJECT_LIMIT + 1)).is_err()); - for _ in 0..INLINE_BATCH_BYTES / INLINE_OBJECT_LIMIT { + for _ in 0..INPUT_BYTES / INLINE_OBJECT_LIMIT { assert!(batch.try_push(inline(INLINE_OBJECT_LIMIT)).is_ok()); } let leftover = match batch.try_push(inline(1)) { @@ -101,40 +97,3 @@ fn decoding_enforces_aggregate_bytes_and_count_before_building_a_batch() { .is_err() ); } - -#[test] -fn chunk_references_bound_total_verification_bytes_and_round_trip() { - let chunked = |size| StoredObject { - oid: crate::ObjectId::Sha1([1; 20]), - kind: ObjectKind::Commit, - storage: ObjectStorage::Chunked { - upload: [2; 16], - size, - blake3: [3; 32], - }, - }; - let mut batch = ObjectBatch::default(); - assert!(batch.try_push(chunked(VERIFY_BATCH_BYTES)).is_ok()); - assert!(batch.try_push(inline(1)).is_err()); - let mut encoded = BoundedEncoder::new(1 << 20).unwrap(); - batch.encode(&mut encoded).unwrap(); - let encoded = encoded.finish(); - let mut decoder = BoundedDecoder::new(&encoded, 1 << 20).unwrap(); - let mut decoded = ObjectBatch::decode(&mut decoder).unwrap(); - decoder.finish().unwrap(); - assert!(decoded.try_push(chunked(1)).is_err()); - let mut malicious = BoundedEncoder::new(1 << 20).unwrap(); - malicious.write_count(2).unwrap(); - for size in [VERIFY_BATCH_BYTES, 1] { - malicious.write_bytes(&[1; 20]).unwrap(); - malicious.write_u8(2).unwrap(); - malicious.write_u8(2).unwrap(); - malicious.write_bytes(&[2; 16]).unwrap(); - malicious.write_u64(size).unwrap(); - malicious.write_bytes(&[3; 32]).unwrap(); - } - let malicious = malicious.finish(); - assert!(ObjectBatch::decode(&mut BoundedDecoder::new(&malicious, 1 << 20).unwrap()).is_err()); - let mut oversized = ObjectBatch::default(); - assert!(oversized.try_push(chunked(u64::MAX)).is_err()); -} diff --git a/src/object_chunks.rs b/src/object_chunks.rs deleted file mode 100644 index 62d4360e..00000000 --- a/src/object_chunks.rs +++ /dev/null @@ -1,190 +0,0 @@ -//! Staged SQLite chunks, verified before an immutable object record is published. - -use cellule_runtime::{ - Error, InvocationError, MutationIdentity, identity::RequestId, primitives::sql::SqlBatch, - primitives::sql::SqlResultSet, primitives::sql::SqlStatement, primitives::sql::SqlValue, - registry::CommandContext, -}; - -use crate::{ - INLINE_OBJECT_LIMIT, ObjectKind, ObjectStorage, RepositoryCell, StoredObject, object_id, -}; - -pub(crate) const CHUNK_BYTES: usize = 512 * 1024; - -#[derive(Debug, thiserror::Error)] -pub enum ObjectStageError { - #[error("only non-blob objects above the inline limit can be staged")] - Invalid, - #[error("object chunk staging failed")] - Cell(#[from] InvocationError>), -} - -impl RepositoryCell { - /// Stages a large non-blob object in SQLite; put_objects must verify and publish the returned record. - pub async fn stage_object( - &self, - identity: MutationIdentity, - kind: ObjectKind, - body: &[u8], - ) -> Result { - if kind == ObjectKind::Blob - || body.len() <= INLINE_OBJECT_LIMIT - || i64::try_from(body.len()).is_err() - { - return Err(ObjectStageError::Invalid); - } - let upload = *identity.request_id.as_bytes(); - let mut statements = vec![SqlStatement { - sql: "INSERT INTO object_uploads (id) VALUES (?1)".into(), - parameters: vec![SqlValue::Blob(upload.to_vec())], - }]; - for (part, bytes) in body.chunks(CHUNK_BYTES).enumerate() { - statements.push(SqlStatement { - sql: "INSERT INTO object_chunks (upload_id, part, body) VALUES (?1, ?2, ?3)".into(), - parameters: vec![ - SqlValue::Blob(upload.to_vec()), - SqlValue::Integer(part as i64), - SqlValue::Blob(bytes.to_vec()), - ], - }); - let mut hash = blake3::Hasher::new(); - hash.update(b"canopy-object-chunk-v1"); - hash.update(&upload); - hash.update(&(part as u64).to_le_bytes()); - let mut request_id = [0; 16]; - request_id.copy_from_slice(&hash.finalize().as_bytes()[..16]); - self.sql - .batch( - MutationIdentity { - request_id: RequestId::from_bytes(request_id), - issued_at_ms: identity.issued_at_ms, - expires_at_ms: identity.expires_at_ms, - }, - SqlBatch { - statements: std::mem::take(&mut statements), - }, - ) - .await?; - } - Ok(StoredObject { - oid: object_id(self.object_format(), kind, body), - kind, - storage: ObjectStorage::Chunked { - upload, - size: body.len() as u64, - blake3: *blake3::hash(body).as_bytes(), - }, - }) - } - - pub(crate) async fn chunked_body( - &self, - oid: crate::ObjectId, - kind: ObjectKind, - upload: [u8; 16], - size: u64, - digest: [u8; 32], - ) -> Result, InvocationError>> { - let invalid = - || InvocationError::NotStarted(Error::Command("incomplete or corrupt object chunks")); - let mut chunks = Chunks::new(oid, kind, upload, size, digest).ok_or_else(invalid)?; - while !chunks.complete() { - let result = self.sql.query(None, chunks.query()).await?; - if !chunks.append(&result.output) { - return Err(invalid()); - } - } - chunks.finish().ok_or_else(invalid) - } -} - -/// The caller's command transaction binds verification and publication atomically. -pub(crate) fn body( - context: &CommandContext<'_, '_>, - oid: crate::ObjectId, - kind: ObjectKind, - upload: [u8; 16], - size: u64, - digest: [u8; 32], -) -> cellule_runtime::Result>> { - let Some(mut chunks) = Chunks::new(oid, kind, upload, size, digest) else { - return Ok(None); - }; - while !chunks.complete() { - if !chunks.append(&context.sql(&chunks.query())?) { - return Ok(None); - } - } - Ok(chunks.finish()) -} - -struct Chunks { - oid: crate::ObjectId, - kind: ObjectKind, - upload: [u8; 16], - size: usize, - digest: [u8; 32], - body: Vec, - part: usize, -} - -impl Chunks { - fn new( - oid: crate::ObjectId, - kind: ObjectKind, - upload: [u8; 16], - size: u64, - digest: [u8; 32], - ) -> Option { - let size = usize::try_from(size).ok()?; - if kind == ObjectKind::Blob || size <= INLINE_OBJECT_LIMIT || i64::try_from(size).is_err() { - return None; - } - Some(Self { - oid, - kind, - upload, - size, - digest, - body: Vec::new(), - part: 0, - }) - } - - fn query(&self) -> SqlBatch { - SqlBatch { statements: vec![SqlStatement { - // Count also rejects extra chunks; each result carries at most one 512 KiB body. - sql: "SELECT body, (SELECT COUNT(*) FROM object_chunks WHERE upload_id = ?1) FROM object_chunks WHERE upload_id = ?1 AND part = ?2".into(), - parameters: vec![SqlValue::Blob(self.upload.to_vec()), SqlValue::Integer(self.part as i64)], - }] } - } - - fn append(&mut self, results: &[SqlResultSet]) -> bool { - let Some([SqlValue::Blob(bytes), SqlValue::Integer(count)]) = results - .first() - .and_then(|set| set.rows.first()) - .map(Vec::as_slice) - else { - return false; - }; - if *count != self.size.div_ceil(CHUNK_BYTES) as i64 - || bytes.len() != CHUNK_BYTES.min(self.size - self.body.len()) - { - return false; - } - self.body.extend_from_slice(bytes); - self.part += 1; - true - } - - fn complete(&self) -> bool { - self.body.len() == self.size - } - - fn finish(self) -> Option> { - (object_id(self.oid.format(), self.kind, &self.body) == self.oid - && blake3::hash(&self.body).as_bytes() == &self.digest) - .then_some(self.body) - } -} diff --git a/src/object_packs.rs b/src/object_packs.rs new file mode 100644 index 00000000..8f3f0b8c --- /dev/null +++ b/src/object_packs.rs @@ -0,0 +1,338 @@ +//! Verified pack metadata and transient graph preparation. No Git bodies enter SQLite. + +use crate::{ + ObjectId, ObjectKind, ObjectStorage, RepositoryCell, StoredObject, + git_pack::{OwnedPack, PackError, PackIdentity}, + graph::Edge, +}; +use cellule_runtime::{ + Error, InvocationError, MutationIdentity, + identity::RequestId, + primitives::sql::{SqlBatch, SqlResultSet, SqlStatement, SqlValue}, +}; +use std::sync::Arc; + +// SHA-256 rows encode at most 122 bytes each, including SQL placeholders. +// 8,192 rows stay below Cellule SQL's 1 MiB input and 32,766 parameter limits. +const DEPENDENCY_WRITE_ROWS: usize = 8_192; + +#[derive(Clone)] +pub struct PackedObject { + pub(crate) repository: [u8; 16], + pub(crate) oid: ObjectId, + pub(crate) kind: ObjectKind, + pub(crate) pack: PackIdentity, + pub(crate) size: u64, + pub(crate) digest: [u8; 32], + pub(crate) graph_valid: bool, + pub(crate) graph_count: u64, + pub(crate) graph_digest: [u8; 32], + pub(crate) dependencies: Option>, + pub(crate) rebase_metadata: Option>, +} + +impl PackedObject { + pub fn size(&self) -> u64 { + self.size + } + pub fn pack(&self) -> &PackIdentity { + &self.pack + } + + pub(crate) fn verified( + repository: [u8; 16], + pack: PackIdentity, + oid: ObjectId, + kind: ObjectKind, + size: u64, + digest: [u8; 32], + dependencies: Option>, + ) -> Self { + let mut hash = blake3::Hasher::new(); + if let Some(edges) = &dependencies { + for edge in edges { + hash.update(&dependency_key(*edge)); + } + } + Self { + repository, + oid, + kind, + pack, + size, + digest, + graph_valid: dependencies.is_some(), + graph_count: dependencies.as_ref().map_or(0, |edges| edges.len() as u64), + graph_digest: *hash.finalize().as_bytes(), + dependencies, + rebase_metadata: None, + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum ObjectStageError { + #[error("object pack preparation failed")] + Pack(#[from] PackError), + #[error("object metadata staging failed")] + Cell(#[from] InvocationError>), +} + +pub(crate) fn pack_invocation(error: PackError) -> InvocationError { + InvocationError::NotStarted(Error::Facility { + name: "Git native pack", + source: Box::new(error), + }) +} + +pub(crate) fn dependency_key((oid, kind): Edge) -> Vec { + let mut key = oid.to_vec(); + key.push(match kind { + None => 0, + Some(ObjectKind::Blob) => 1, + Some(ObjectKind::Tree) => 2, + Some(ObjectKind::Commit) => 3, + Some(ObjectKind::Tag) => 4, + }); + key +} + +pub(crate) fn decode_dependency( + format: crate::ObjectFormat, + key: &[u8], +) -> cellule_runtime::Result { + if key.len() != format.bytes() + 1 { + return Err(Error::Command("invalid packed dependency")); + } + let oid = key[..format.bytes()] + .try_into() + .map_err(|_| Error::Command("invalid packed dependency OID"))?; + let kind = match key[format.bytes()] { + 0 => None, + 1 => Some(ObjectKind::Blob), + 2 => Some(ObjectKind::Tree), + 3 => Some(ObjectKind::Commit), + 4 => Some(ObjectKind::Tag), + _ => return Err(Error::Command("invalid packed dependency kind")), + }; + Ok((oid, kind)) +} + +impl RepositoryCell { + /// Creates and publishes a native pack before exposing a prepared reference. + pub async fn stage_object( + &self, + kind: ObjectKind, + body: &[u8], + ) -> Result { + let source = self + .packs + .prepare(self.object_format, vec![(kind, body.to_vec())]) + .await?; + let pack = self.packs.publish(Arc::clone(&source)).await?; + let oid = crate::object_id(self.object_format, kind, body); + self.record_from_pack(source, pack, oid) + .await + .map_err(Into::into) + } + + pub(crate) async fn record_from_pack( + &self, + source: Arc, + pack: PackIdentity, + oid: ObjectId, + ) -> Result { + let repository = self.id; + tokio::task::spawn_blocking(move || { + let (kind, body) = source + .data()? + .object(oid, 64 * 1024 * 1024)? + .ok_or(PackError::Integrity)?; + let mut storage = PackedObject::verified( + repository, + pack, + oid, + kind, + body.len() as u64, + *blake3::hash(&body).as_bytes(), + crate::graph::edges(oid.format(), kind, &body), + ); + storage.rebase_metadata = crate::pulls::candidates::rebase_metadata(kind, &body); + Ok(StoredObject { + oid, + kind, + storage: ObjectStorage::Packed(Box::new(storage)), + }) + }) + .await? + } + + pub(crate) async fn stage_dependencies( + &self, + identity: MutationIdentity, + objects: &mut [StoredObject], + ) -> Result<(), InvocationError>> { + let mut parameters = Vec::new(); + let mut page = 0; + for object in objects { + let ObjectStorage::Packed(value) = &mut object.storage else { + continue; + }; + let Some(edges) = value.dependencies.take() else { + continue; + }; + for edge in edges { + parameters.push(SqlValue::Blob(value.pack.pack_digest.to_vec())); + parameters.push(SqlValue::Blob(value.oid.to_vec())); + parameters.push(SqlValue::Blob(dependency_key(edge))); + if parameters.len() == DEPENDENCY_WRITE_ROWS * 3 { + self.stage_dependency_page(identity, page, std::mem::take(&mut parameters)) + .await?; + page += 1; + } + } + } + if !parameters.is_empty() { + self.stage_dependency_page(identity, page, parameters) + .await?; + } + Ok(()) + } + + async fn stage_dependency_page( + &self, + identity: MutationIdentity, + page: u64, + parameters: Vec, + ) -> Result<(), InvocationError>> { + let mut hash = blake3::Hasher::new(); + hash.update(b"canopy-pack-dependencies-v3"); + hash.update(identity.request_id.as_bytes()); + hash.update(&page.to_le_bytes()); + let mut request = [0; 16]; + request.copy_from_slice(&hash.finalize().as_bytes()[..16]); + self.sql + .batch( + MutationIdentity { + request_id: RequestId::from_bytes(request), + ..identity + }, + dependency_batch(parameters), + ) + .await?; + Ok(()) + } + + pub(crate) async fn dependencies( + &self, + oid: ObjectId, + ) -> Result>, InvocationError>> { + let result = self.sql.query(None, SqlBatch { statements: vec![SqlStatement { + sql: "SELECT pack, graph_valid, graph_count, graph_digest FROM objects WHERE oid = ?1".into(), + parameters: vec![SqlValue::Blob(oid.to_vec())], + }]}).await?; + let Some( + [ + SqlValue::Blob(pack), + SqlValue::Integer(valid), + SqlValue::Integer(count), + SqlValue::Blob(digest), + ], + ) = result + .output + .first() + .and_then(|set| set.rows.first()) + .map(Vec::as_slice) + else { + return Err(InvocationError::NotStarted(Error::Command( + "invalid packed graph metadata", + ))); + }; + if *valid != 1 { + return Ok(None); + } + let mut cursor = Vec::new(); + let mut edges = Vec::new(); + let mut hash = blake3::Hasher::new(); + loop { + let page = self + .sql + .query(Some(result.receipt), dependency_query(pack, oid, &cursor)) + .await?; + let rows = &page + .output + .first() + .ok_or_else(|| { + InvocationError::NotStarted(Error::Command("missing packed dependencies")) + })? + .rows; + if rows.is_empty() { + break; + } + for row in rows { + let [SqlValue::Blob(key)] = row.as_slice() else { + return Err(InvocationError::NotStarted(Error::Command( + "invalid packed dependency row", + ))); + }; + if key <= &cursor || edges.len() as i64 >= *count { + return Err(InvocationError::NotStarted(Error::Command( + "corrupt packed dependency count", + ))); + } + hash.update(key); + edges.push( + decode_dependency(oid.format(), key).map_err(InvocationError::NotStarted)?, + ); + cursor.clone_from(key); + } + } + if edges.len() as i64 != *count || hash.finalize().as_bytes().as_slice() != digest { + return Err(InvocationError::NotStarted(Error::Command( + "corrupt packed dependencies", + ))); + } + Ok(Some(edges)) + } +} + +fn dependency_batch(parameters: Vec) -> SqlBatch { + SqlBatch { + statements: vec![SqlStatement { + sql: format!( + "INSERT INTO object_dependencies (pack, parent, dependency) VALUES {} ON CONFLICT DO NOTHING", + vec!["(?, ?, ?)"; parameters.len() / 3].join(",") + ), + parameters, + }], + } +} + +pub(crate) fn dependency_query(pack: &[u8], oid: ObjectId, after: &[u8]) -> SqlBatch { + SqlBatch { statements: vec![SqlStatement { + sql: "SELECT dependency FROM object_dependencies WHERE pack = ?1 AND parent = ?2 AND dependency > ?3 ORDER BY dependency LIMIT 512".into(), + parameters: vec![SqlValue::Blob(pack.to_vec()), SqlValue::Blob(oid.to_vec()), SqlValue::Blob(after.to_vec())], + }]} +} + +#[cfg(test)] +mod tests { + use super::*; + use cellule_runtime::codec::{BoundedEncoder, WireValue}; + + #[test] + fn sha256_dependency_write_fits_published_sql_limits() { + let mut parameters = Vec::new(); + for _ in 0..DEPENDENCY_WRITE_ROWS { + parameters.extend([ + SqlValue::Blob(vec![1; 32]), + SqlValue::Blob(vec![2; 32]), + SqlValue::Blob(vec![3; 33]), + ]); + } + assert!(parameters.len() < 32_766); + // Exercise the SDK's wire encoder at the actual SQL input limit. + let mut encoder = BoundedEncoder::new(1 << 20).unwrap(); + dependency_batch(parameters).encode(&mut encoder).unwrap(); + } +} diff --git a/src/object_reads.rs b/src/object_reads.rs index d455b477..09cfa6de 100644 --- a/src/object_reads.rs +++ b/src/object_reads.rs @@ -6,8 +6,8 @@ use cellule_runtime::{ }; use crate::{ - INLINE_OBJECT_LIMIT, ObjectKind, ObjectStorage, RepositoryCell, StoredObject, - object_batch::MAX_OBJECTS, object_id, + ObjectKind, ObjectStorage, PackedObject, RepositoryCell, StoredObject, + object_batch::MAX_OBJECTS, }; pub(crate) struct ObjectHeaders { @@ -15,10 +15,10 @@ pub(crate) struct ObjectHeaders { pub(crate) through: i64, } -const CHANGED_HEADERS: &str = "SELECT sequence, oid, CASE WHEN storage = 'inline' THEN size ELSE 0 END FROM objects WHERE sequence > ?1 AND sequence <= ?2 ORDER BY sequence LIMIT ?3"; +const CHANGED_HEADERS: &str = "SELECT sequence, oid, 0 FROM objects WHERE sequence > ?1 AND sequence <= ?2 ORDER BY sequence LIMIT ?3"; impl RepositoryCell { - /// Reads at most 128 objects and 768 KiB of inline bodies, in OID order. + /// Reads at most 512 immutable native-pack metadata records, in OID order. /// /// Continue after the last OID until an empty page, including after short /// pages. Objects are immutable; a future collector must fence this read. @@ -26,10 +26,20 @@ impl RepositoryCell { &self, after: Option, ) -> Result>, InvocationError>> { - let headers = self.read_object_headers(None, SqlStatement { - sql: "SELECT sequence, oid, CASE WHEN storage = 'inline' THEN size ELSE 0 END FROM objects WHERE oid > ?1 ORDER BY oid LIMIT ?2".into(), - parameters: vec![SqlValue::Blob(after.map_or_else(Vec::new, |oid| oid.to_vec())), SqlValue::Integer(MAX_OBJECTS as i64)], - }).await?; + let headers = self + .read_object_headers( + None, + SqlStatement { + sql: + "SELECT sequence, oid, 0 FROM objects WHERE oid > ?1 ORDER BY oid LIMIT ?2" + .into(), + parameters: vec![ + SqlValue::Blob(after.map_or_else(Vec::new, |oid| oid.to_vec())), + SqlValue::Integer(MAX_OBJECTS as i64), + ], + }, + ) + .await?; self.object_records(headers.objects).await } @@ -44,7 +54,7 @@ impl RepositoryCell { } let placeholders = vec!["?"; ids.len()].join(","); let headers = self.read_object_headers(None, SqlStatement { - sql: format!("SELECT sequence, oid, CASE WHEN storage = 'inline' THEN size ELSE 0 END FROM objects WHERE oid IN ({placeholders}) ORDER BY oid"), + sql: format!("SELECT sequence, oid, 0 FROM objects WHERE oid IN ({placeholders}) ORDER BY oid"), parameters: ids.iter().map(|oid| SqlValue::Blob(oid.to_vec())).collect(), }).await?; Ok(self.object_records(headers.objects).await?.output) @@ -129,7 +139,7 @@ impl RepositoryCell { headers: Observed>, ) -> Result>, InvocationError>> { // Callers may remove already cached IDs, but never add IDs: the header - // query's size bound and receipt protect this body read's wire ceiling. + // query's count bound and receipt protect this metadata read. let mut ids = headers.output; ids.sort_unstable(); if ids.is_empty() { @@ -139,11 +149,11 @@ impl RepositoryCell { }); } let placeholders = vec!["?"; ids.len()].join(","); - // Immutable records bind this second read to the selected headers. The - // payload bound leaves room for record metadata under Cellule's 1 MiB cap. + // Immutable records bind this second read to the selected headers. + // Fixed metadata widths fit comfortably under Cellule's 1 MiB cap. let result = self.sql.query(Some(headers.receipt), SqlBatch { statements: vec![SqlStatement { - sql: format!("SELECT oid, kind, size, digest, storage, body, external_sha256, chunk_id FROM objects WHERE oid IN ({placeholders}) ORDER BY oid"), + sql: format!("SELECT o.oid, o.kind, o.size, o.digest, o.graph_valid, o.graph_count, o.graph_digest, p.format, p.digest, p.bytes, p.index_digest, p.index_bytes, o.rebase_metadata FROM objects o JOIN git_packs p ON p.digest = o.pack WHERE o.oid IN ({placeholders}) ORDER BY o.oid"), parameters: ids.iter().map(|oid| SqlValue::Blob(oid.to_vec())).collect(), }], }).await?; @@ -153,12 +163,12 @@ impl RepositoryCell { .next() .ok_or_else(|| InvocationError::NotStarted(Error::Command("missing object page")))? .rows; - // Hash a whole bounded page off the async executor. Move SQL bodies into - // the result so page verification does not duplicate their payloads. + // Decode the bounded metadata page off the async executor. + let repository = self.id; let output = tokio::task::spawn_blocking(move || { let objects = rows .into_iter() - .map(decode_object) + .map(|row| decode_object(row, repository)) .collect::>>()?; if objects.iter().map(|object| object.oid).ne(ids) { return Err(Error::Command("objects changed during page read")); @@ -182,7 +192,6 @@ impl RepositoryCell { fn decode_headers(rows: &[Vec]) -> cellule_runtime::Result<(Vec, i64)> { let mut ids = Vec::new(); - let mut bytes = 0; let mut through = 0; for row in rows { let [ @@ -200,84 +209,78 @@ fn decode_headers(rows: &[Vec]) -> cellule_runtime::Result<(Vec INLINE_OBJECT_LIMIT - bytes { - break; + if *size != 0 { + return Err(Error::Command("unexpected metadata header payload")); } - bytes += size; ids.push(oid); through = *sequence; } Ok((ids, through)) } -fn decode_object(row: Vec) -> cellule_runtime::Result { - let row: [SqlValue; 8] = row - .try_into() - .map_err(|_| Error::Command("invalid stored object row"))?; +fn decode_object( + row: Vec, + repository: [u8; 16], +) -> cellule_runtime::Result { let [ SqlValue::Blob(oid), SqlValue::Text(kind), SqlValue::Integer(size), SqlValue::Blob(digest), - SqlValue::Text(storage), - body, - external_sha256, - chunk_id, - ] = row + SqlValue::Integer(valid), + SqlValue::Integer(count), + SqlValue::Blob(graph_digest), + SqlValue::Text(format), + SqlValue::Blob(pack_digest), + SqlValue::Integer(pack_bytes), + SqlValue::Blob(index_digest), + SqlValue::Integer(index_bytes), + rebase_metadata, + ] = row.as_slice() else { - return Err(Error::Command("invalid stored object row")); + return Err(Error::Command("invalid packed object row")); }; - let oid: crate::ObjectId = oid - .try_into() - .map_err(|_| Error::Command("invalid stored object ID"))?; - let digest: [u8; 32] = digest - .try_into() - .map_err(|_| Error::Command("invalid object digest"))?; + let invalid = || Error::Command("invalid packed object metadata"); + let oid: crate::ObjectId = oid.as_slice().try_into().map_err(|_| invalid())?; let kind = match kind.as_str() { "blob" => ObjectKind::Blob, "tree" => ObjectKind::Tree, "commit" => ObjectKind::Commit, "tag" => ObjectKind::Tag, - _ => return Err(Error::Command("invalid stored object kind")), + _ => return Err(invalid()), }; - let storage = match (storage.as_str(), body, external_sha256, chunk_id) { - ("inline", SqlValue::Blob(body), SqlValue::Null, SqlValue::Null) - if usize::try_from(size).ok() == Some(body.len()) - && body.len() <= INLINE_OBJECT_LIMIT - && object_id(oid.format(), kind, &body) == oid - && blake3::hash(&body).as_bytes() == &digest => - { - ObjectStorage::Inline(body) - } - ("external", SqlValue::Null, SqlValue::Blob(sha256), SqlValue::Null) - if kind == ObjectKind::Blob && size >= 0 => - { - ObjectStorage::External { - size: size as u64, - blake3: digest, - sha256: sha256 - .try_into() - .map_err(|_| Error::Command("invalid SHA-256 digest"))?, - } - } - ("chunked", SqlValue::Null, SqlValue::Null, SqlValue::Blob(upload)) - if kind != ObjectKind::Blob && size > INLINE_OBJECT_LIMIT as i64 => - { - ObjectStorage::Chunked { - upload: upload - .try_into() - .map_err(|_| Error::Command("invalid object chunk reference"))?, - size: size as u64, - blake3: digest, - } - } - _ => return Err(Error::Command("corrupt stored object")), + let format = crate::ObjectFormat::parse(format).ok_or_else(invalid)?; + if oid.format() != format || ![0, 1].contains(valid) || *pack_bytes <= 0 || *index_bytes <= 0 { + return Err(invalid()); + } + let reference = PackedObject { + repository, + oid, + kind, + size: u64::try_from(*size).map_err(|_| invalid())?, + digest: digest.as_slice().try_into().map_err(|_| invalid())?, + pack: crate::git_pack::PackIdentity { + format, + pack_digest: pack_digest.as_slice().try_into().map_err(|_| invalid())?, + pack_bytes: *pack_bytes as u64, + index_digest: index_digest.as_slice().try_into().map_err(|_| invalid())?, + index_bytes: *index_bytes as u64, + }, + graph_valid: *valid == 1, + graph_count: u64::try_from(*count).map_err(|_| invalid())?, + graph_digest: graph_digest.as_slice().try_into().map_err(|_| invalid())?, + dependencies: None, + rebase_metadata: match rebase_metadata { + SqlValue::Null => None, + SqlValue::Blob(bytes) if bytes.len() == 65 => Some(bytes.clone()), + _ => return Err(invalid()), + }, }; - Ok(StoredObject { oid, kind, storage }) + Ok(StoredObject { + oid, + kind, + storage: ObjectStorage::Packed(Box::new(reference)), + }) } #[cfg(test)] diff --git a/src/object_reads/tests.rs b/src/object_reads/tests.rs index 384d9352..98132698 100644 --- a/src/object_reads/tests.rs +++ b/src/object_reads/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::{INLINE_OBJECT_LIMIT, object_id}; use cellule_ltx::rusqlite::{Connection, StatementStatus, params}; type Result = std::result::Result>; @@ -11,10 +12,9 @@ fn database() -> Result { fn insert(db: &Connection, body: &[u8]) -> Result { let oid = object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, body); - db.execute( - "INSERT INTO objects (oid, kind, size, digest, storage, body) VALUES (?1, 'blob', ?2, ?3, 'inline', ?4) ON CONFLICT(oid) DO NOTHING", - params![oid.as_ref(), body.len() as i64, blake3::hash(body).as_bytes().as_slice(), body], - )?; + db.execute("INSERT INTO git_packs (digest, format, bytes, index_digest, index_bytes) VALUES (zeroblob(32), 'sha1', 1, zeroblob(32), 1) ON CONFLICT DO NOTHING", [])?; + db.execute("INSERT INTO objects (oid, kind, size, digest, pack, graph_valid, graph_count, graph_digest) VALUES (?1, 'blob', ?2, ?3, zeroblob(32), 1, 0, ?4) ON CONFLICT(oid) DO NOTHING", + params![oid.as_ref(), body.len() as i64, blake3::hash(body).as_bytes().as_slice(), blake3::hash(b"").as_bytes().as_slice()])?; Ok(oid) } @@ -75,17 +75,15 @@ fn insertion_cursor_finds_lower_oids_and_excludes_later_publications() -> Result } #[test] -fn byte_limited_page_advances_only_over_the_selected_prefix() -> Result { +fn metadata_page_is_independent_of_body_sizes() -> Result { let db = database()?; let first = insert(&db, &vec![1; INLINE_OBJECT_LIMIT / 2])?; let second = insert(&db, &vec![2; INLINE_OBJECT_LIMIT])?; let third = insert(&db, b"third")?; let high = high_water(&db)?; let (ids, after) = page(&db, 0, high)?; - assert_eq!(ids, vec![first]); - let (ids, after) = page(&db, after, high)?; - assert_eq!(ids, vec![second]); - assert_eq!(page(&db, after, high)?.0, vec![third]); + assert_eq!(ids, vec![first, second, third]); + assert_eq!(after, high); Ok(()) } diff --git a/src/pulls/candidates.rs b/src/pulls/candidates.rs index 8905c3f4..da0e867a 100644 --- a/src/pulls/candidates.rs +++ b/src/pulls/candidates.rs @@ -241,9 +241,16 @@ fn certified( return Ok(false); } let rows=context.sql(&SqlBatch {statements:vec![SqlStatement { - sql:"SELECT o.body FROM objects o JOIN object_closure c ON c.oid = o.oid WHERE o.oid = ?1 AND o.kind = 'commit' AND o.storage = 'inline'".into(),parameters:vec![SqlValue::Blob(commit.to_vec())], + sql:"SELECT o.digest, o.size FROM objects o JOIN object_closure c ON c.oid = o.oid WHERE o.oid = ?1 AND o.kind = 'commit'".into(),parameters:vec![SqlValue::Blob(commit.to_vec())], }]})?; Ok( - matches!(rows.first().and_then(|set|set.rows.first()).map(Vec::as_slice),Some([SqlValue::Blob(stored)]) if *stored == body), + matches!(rows.first().and_then(|set|set.rows.first()).map(Vec::as_slice),Some([SqlValue::Blob(digest), SqlValue::Integer(size)]) if *size == body.len() as i64 && digest.as_slice() == blake3::hash(&body).as_bytes()), ) } + +pub(crate) fn rebase_metadata(kind: ObjectKind, body: &[u8]) -> Option> { + if kind != ObjectKind::Commit { + return None; + } + rebase::Commit::parse(body).map(|commit| commit.metadata()) +} diff --git a/src/pulls/candidates/rebase.rs b/src/pulls/candidates/rebase.rs index 988aff89..172982e9 100644 --- a/src/pulls/candidates/rebase.rs +++ b/src/pulls/candidates/rebase.rs @@ -6,7 +6,6 @@ pub(crate) const MAX_COMMITS: usize = 128; pub(crate) const MAX_COMMIT_BYTES: usize = 64 * 1024; pub(crate) struct Commit<'a> { - pub(crate) tree: &'a str, pub(crate) parent: &'a str, headers: Vec<&'a [u8]>, message: &'a [u8], @@ -39,13 +38,44 @@ impl<'a> Commit<'a> { field(b"author ")?; field(b"committer ")?; Some(Self { - tree, parent, headers, message: &body[split + 2..], }) } + pub(super) fn metadata(&self) -> Vec { + let mut hash = blake3::Hasher::new(); + let mut committer = [0; 32]; + let mut signed = false; + for header in &self.headers { + let normalized: &[u8] = if header.starts_with(b"tree ") { + b"tree" + } else if header.starts_with(b"parent ") { + b"parent" + } else if header.starts_with(b"committer ") { + committer = *blake3::hash(header).as_bytes(); + b"committer" + } else if header.starts_with(b"gpgsig ") + || header.starts_with(b"gpgsig-sha256 ") + || header.starts_with(b"mergetag ") + { + signed = true; + continue; + } else { + header + }; + hash.update(&(normalized.len() as u64).to_le_bytes()); + hash.update(normalized); + } + hash.update(&(self.message.len() as u64).to_le_bytes()); + hash.update(self.message); + let mut metadata = hash.finalize().as_bytes().to_vec(); + metadata.extend_from_slice(&committer); + metadata.push(u8::from(signed)); + metadata + } + pub(crate) fn rewrite(&self, candidate: &MergeCandidate, tree: &str, parent: &str) -> Vec { let actor = &candidate.actor; let time = candidate.created_at_ms / 1000; @@ -91,29 +121,31 @@ pub(super) fn certified( let mut current = tip.to_owned(); let base = &candidate.request.revision.base_oid; for index in 0..MAX_COMMITS { - let Some(original) = body(context, &source)? else { - return Ok(false); - }; - let Some(rewritten) = body(context, ¤t)? else { - return Ok(false); - }; - let Some(original_commit) = Commit::parse(&original) else { + let Some(original_commit) = projection(context, &source)? else { return Ok(false); }; - let Some(rewritten_commit) = Commit::parse(&rewritten) else { + let Some(rewritten_commit) = projection(context, ¤t)? else { return Ok(false); }; + let expected_committer = format!( + "committer {} <{}@users.canopy.invalid> {} +0000", + candidate.actor, + candidate.actor, + candidate.created_at_ms / 1000 + ); if (index == 0 && rewritten_commit.tree != tree) - || original_commit.rewrite(candidate, rewritten_commit.tree, rewritten_commit.parent) - != rewritten + || original_commit.metadata[..32] != rewritten_commit.metadata[..32] + || rewritten_commit.metadata[32..64] + != blake3::hash(expected_committer.as_bytes()).as_bytes()[..] + || rewritten_commit.metadata[64] != 0 { return Ok(false); } // Native Git owns replay selection and merge semantics. The transaction // binds every rewritten commit to the contiguous original chain and a // certified common ancestor, so a worker cannot omit an intermediate commit. - source = original_commit.parent.to_owned(); - current = rewritten_commit.parent.to_owned(); + source = original_commit.parent; + current = rewritten_commit.parent; if ¤t == base { if &source == base { return Ok(true); @@ -134,25 +166,52 @@ pub(super) fn certified( Ok(false) } -fn body( +struct Projection { + tree: String, + parent: String, + metadata: Vec, +} +fn projection( context: &CommandContext<'_, '_>, commit: &str, -) -> cellule_runtime::Result>> { +) -> cellule_runtime::Result> { + let id = oid(commit)?; let rows = context.sql(&SqlBatch { statements: vec![SqlStatement { - sql: "SELECT o.body FROM objects o JOIN object_closure c ON c.oid = o.oid WHERE o.oid = ?1 AND o.kind = 'commit' AND o.storage = 'inline' AND o.size <= ?2".into(), - parameters: vec![SqlValue::Blob(oid(commit)?.to_vec()), SqlValue::Integer(MAX_COMMIT_BYTES as i64)], - }] })?; - match rows + sql: "SELECT o.rebase_metadata FROM objects o JOIN object_closure c ON c.oid = o.oid WHERE o.oid = ?1 AND o.kind = 'commit' AND o.size <= ?2".into(), parameters: vec![SqlValue::Blob(id.to_vec()), SqlValue::Integer(MAX_COMMIT_BYTES as i64)], + }]})?; + let Some([SqlValue::Blob(metadata)]) = rows .first() .and_then(|set| set.rows.first()) .map(Vec::as_slice) + else { + return Ok(None); + }; + if metadata.len() != 65 { + return Err(Error::Command("invalid rebase metadata")); + } + let rows = context.sql(&SqlBatch { statements: vec![SqlStatement { + sql: "SELECT child.oid, child.kind FROM object_edges e JOIN objects child ON child.oid = e.child WHERE e.parent = ?1 ORDER BY child.kind, child.oid".into(), parameters: vec![SqlValue::Blob(id.to_vec())], + }]})?; + let mut tree = None; + let mut parent = None; + for row in &rows + .first() + .ok_or(Error::Command("missing rebase edges"))? + .rows { - Some([SqlValue::Blob(bytes)]) - if crate::object_id(oid(commit)?.format(), ObjectKind::Commit, bytes) - == oid(commit)? => - { - Ok(Some(bytes.clone())) + let [SqlValue::Blob(oid), SqlValue::Text(kind)] = row.as_slice() else { + return Err(Error::Command("invalid rebase edge")); + }; + let value = hex::encode(oid); + match kind.as_str() { + "tree" if tree.is_none() => tree = Some(value), + "commit" if parent.is_none() => parent = Some(value), + _ => return Ok(None), } - _ => Ok(None), } + Ok(tree.zip(parent).map(|(tree, parent)| Projection { + tree, + parent, + metadata: metadata.clone(), + })) } diff --git a/src/schema.sql b/src/schema.sql index 7462f5d2..d5f257e5 100644 --- a/src/schema.sql +++ b/src/schema.sql @@ -1,12 +1,16 @@ -CREATE TABLE object_uploads ( - id BLOB PRIMARY KEY CHECK(length(id) = 16) +CREATE TABLE git_packs ( + digest BLOB PRIMARY KEY CHECK(length(digest) = 32), + format TEXT NOT NULL CHECK(format IN ('sha1', 'sha256')), + bytes INTEGER NOT NULL CHECK(bytes > 0), + index_digest BLOB NOT NULL CHECK(length(index_digest) = 32), + index_bytes INTEGER NOT NULL CHECK(index_bytes > 0) ) WITHOUT ROWID; -CREATE TABLE object_chunks ( - upload_id BLOB NOT NULL REFERENCES object_uploads(id), - part INTEGER NOT NULL CHECK(part >= 0), - body BLOB NOT NULL CHECK(length(body) BETWEEN 1 AND 524288), - PRIMARY KEY(upload_id, part) +CREATE TABLE object_dependencies ( + pack BLOB NOT NULL CHECK(length(pack) = 32), + parent BLOB NOT NULL CHECK(length(parent) IN (20, 32)), + dependency BLOB NOT NULL CHECK(length(dependency) IN (21, 33)), + PRIMARY KEY(pack, parent, dependency) ) WITHOUT ROWID; CREATE TABLE objects ( @@ -15,18 +19,13 @@ CREATE TABLE objects ( kind TEXT NOT NULL CHECK(kind IN ('blob', 'tree', 'commit', 'tag')), size INTEGER NOT NULL CHECK(size >= 0), digest BLOB NOT NULL CHECK(length(digest) = 32), - storage TEXT NOT NULL CHECK(storage IN ('inline', 'external', 'chunked')), - body BLOB, - external_sha256 BLOB, - chunk_id BLOB UNIQUE REFERENCES object_uploads(id) CHECK(chunk_id IS NULL OR length(chunk_id) = 16), - CHECK( - (storage = 'inline' AND body IS NOT NULL AND external_sha256 IS NULL AND chunk_id IS NULL AND size = length(body)) - OR - (storage = 'external' AND kind = 'blob' AND body IS NULL AND chunk_id IS NULL AND length(external_sha256) = 32) - OR - (storage = 'chunked' AND kind != 'blob' AND body IS NULL AND external_sha256 IS NULL AND chunk_id IS NOT NULL AND size > 786432) - ) + pack BLOB NOT NULL REFERENCES git_packs(digest), + graph_valid INTEGER NOT NULL CHECK(graph_valid IN (0, 1)), + graph_count INTEGER NOT NULL CHECK(graph_count >= 0), + graph_digest BLOB NOT NULL CHECK(length(graph_digest) = 32), + rebase_metadata BLOB CHECK(rebase_metadata IS NULL OR (kind = 'commit' AND length(rebase_metadata) = 65)) ); +CREATE INDEX objects_by_oid_kind ON objects(oid, kind); CREATE TABLE object_edges ( parent BLOB NOT NULL REFERENCES objects(oid), diff --git a/src/server.rs b/src/server.rs index 834f186d..4ca2516f 100644 --- a/src/server.rs +++ b/src/server.rs @@ -766,7 +766,16 @@ async fn acquire_sql_cell( .provision( &catalog, ®istry, - CatalogEntry::new(target, CatalogRole::Sql, code, 1)?, + CatalogEntry::new( + target, + CatalogRole::Sql, + code, + if spec.module == crate::RepositoryModule::NAME { + crate::REPOSITORY_SCHEMA_VERSION + } else { + 1 + }, + )?, ) .await?; acquire_provisioned_sql_cell(node, layout, directory, spec, session, endpoint, proof).await diff --git a/src/server/residency.rs b/src/server/residency.rs index 8a5f61d0..81e63774 100644 --- a/src/server/residency.rs +++ b/src/server/residency.rs @@ -486,6 +486,14 @@ impl RepositoryManager { target, entry.repository_id, entry.object_format, + crate::git_pack::PackStore::new( + Arc::clone(&self.external_store), + entry.repository_id, + self.local.path().to_path_buf(), + self.disk_budget.clone(), + 256 * 1024 * 1024, + 64 * 1024 * 1024, + )?, )?); let gateway = Arc::new( GitGateway::new( diff --git a/tests/directory_cell.rs b/tests/directory_cell.rs index 316898b8..6ca11aa0 100644 --- a/tests/directory_cell.rs +++ b/tests/directory_cell.rs @@ -4,6 +4,8 @@ mod accounts; mod expiry; #[path = "support/objects.rs"] mod objects; +#[path = "support/packs.rs"] +mod pack_store; #[path = "directory/ssh_keys.rs"] mod ssh_keys; @@ -166,6 +168,7 @@ async fn directory_reservations_recover_two_distinct_repository_cells() first_target.clone(), first_id, canopy_server::ObjectFormat::Sha1, + pack_store::new(&layout, files.path(), first_id)?, )?; let second_repository = RepositoryCell::new( &app_handle( @@ -186,6 +189,7 @@ async fn directory_reservations_recover_two_distinct_repository_cells() second_target.clone(), second_id, canopy_server::ObjectFormat::Sha1, + pack_store::new(&layout, files.path(), second_id)?, )?; first_repository .ensure_owner(random_identity()?, "alice") @@ -290,6 +294,21 @@ async fn directory_reservations_recover_two_distinct_repository_cells() oid, object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, body) ); + // A verified capability remains bound to the repository whose prefix + // contains its immutable pair, even when another Cell shares the store. + let prepared = first_repository + .object_page(None) + .await? + .output + .into_iter() + .find(|object| object.oid == oid) + .ok_or("missing prepared pack reference")?; + let mut batch = canopy_server::ObjectBatch::default(); + assert!(batch.try_push(prepared).is_ok()); + assert!(matches!( + second_repository.put_objects(identity(60)?, batch).await, + Err(cellule_runtime::InvocationError::Rejected(_)) + )); assert!( second_repository .existing_objects(&[oid]) @@ -362,6 +381,7 @@ async fn directory_reservations_recover_two_distinct_repository_cells() first_target, first_id, canopy_server::ObjectFormat::Sha1, + pack_store::new(&layout, files.path(), first_id)?, )?; let beta = RepositoryCell::new( &app_handle( @@ -382,6 +402,7 @@ async fn directory_reservations_recover_two_distinct_repository_cells() second_target, second_id, canopy_server::ObjectFormat::Sha1, + pack_store::new(&layout, files.path(), second_id)?, )?; assert_eq!( alpha.object(oid, None).await?.output, @@ -437,7 +458,16 @@ async fn bootstrap( let (module, schema) = module_schema; let code = registry.module_code(module).ok_or("module missing")?; let proof = CellCatalog::new(layout.clone(), target.tenant()) - .provision(CatalogEntry::new(target, CatalogRole::Sql, code, 1)?) + .provision(CatalogEntry::new( + target, + CatalogRole::Sql, + code, + if module == RepositoryModule::NAME { + canopy_server::REPOSITORY_SCHEMA_VERSION + } else { + 1 + }, + )?) .await?; let authority = CellAuthority::new(layout.clone()); let observed = authority @@ -482,7 +512,16 @@ async fn restore( ) -> Result> { let code = registry.module_code(module).ok_or("module missing")?; let proof = CellCatalog::new(layout.clone(), target.tenant()) - .provision(CatalogEntry::new(target, CatalogRole::Sql, code, 1)?) + .provision(CatalogEntry::new( + target, + CatalogRole::Sql, + code, + if module == RepositoryModule::NAME { + canopy_server::REPOSITORY_SCHEMA_VERSION + } else { + 1 + }, + )?) .await?; let authority = CellAuthority::new(layout.clone()); let observed = authority diff --git a/tests/multi_server/backup.rs b/tests/multi_server/backup.rs index 73eff689..e4799542 100644 --- a/tests/multi_server/backup.rs +++ b/tests/multi_server/backup.rs @@ -90,7 +90,8 @@ async fn backup_restores_git_lfs_and_collaboration_without_original_storage() -> let report = deployment .create_backup(id, backup.clone(), worker()) .await?; - assert_eq!(report.external_objects, 2); + // One structural pair, one blob pair and one LFS body. + assert_eq!(report.external_objects, 3); assert_eq!(report.cells, 2); deployment .create_backup(id, backup.clone(), worker()) diff --git a/tests/multi_server/bulk_refs.rs b/tests/multi_server/bulk_refs.rs index b8310921..ac027c4f 100644 --- a/tests/multi_server/bulk_refs.rs +++ b/tests/multi_server/bulk_refs.rs @@ -110,9 +110,35 @@ async fn bulk_mirror_round_trip(store: Arc) -> Result { .map(|index| format!("{prefix}-{index:04}")) .collect(); assert!(names[0].len() > 255); + // Distinct targets cross the SQL reachability statement limit during clone; + // thousands of aliases to one object would exercise only a single want. + let tree = run_git(Some(&source), &["rev-parse", "HEAD^{tree}"]).await?; + let tree = std::str::from_utf8(&tree)?.trim(); + let mut targets = Vec::new(); + for index in 0..129 { + let message = format!("Distinct mirror target {index}"); + let target = run_git( + Some(&source), + &[ + "-c", + "user.name=Canopy", + "-c", + "user.email=test@example.invalid", + "commit-tree", + tree, + "-p", + &oid, + "-m", + &message, + ], + ) + .await?; + targets.push(String::from_utf8(target)?.trim().to_owned()); + } let create: String = names .iter() - .map(|name| format!("create {name} {oid}\n")) + .enumerate() + .map(|(index, name)| format!("create {name} {}\n", targets[index % targets.len()])) .collect(); edit_refs(&source, &create).await?; let before = generation(&client, &api).await?; diff --git a/tests/multi_server/large_objects.rs b/tests/multi_server/large_objects.rs index b0274552..e54fb1c4 100644 --- a/tests/multi_server/large_objects.rs +++ b/tests/multi_server/large_objects.rs @@ -1,7 +1,7 @@ use super::*; #[tokio::test(flavor = "multi_thread")] -async fn large_tree_commit_and_tag_restore_from_sqlite_after_owner_restart() +async fn large_tree_commit_and_tag_restore_from_native_packs_after_owner_restart() -> Result<(), Box> { let store: Arc = Arc::new(InMemory::new()); let workspace = tempfile::TempDir::new()?; diff --git a/tests/multi_server/lifecycle.rs b/tests/multi_server/lifecycle.rs index 74b75686..14ce085b 100644 --- a/tests/multi_server/lifecycle.rs +++ b/tests/multi_server/lifecycle.rs @@ -450,7 +450,7 @@ async fn cancelled_maintenance_recovery_retains_enrollment_until_cell_cleanup() .registry() .module_code(RepositoryModule::NAME) .ok_or("missing module")?, - 1, + canopy_server::REPOSITORY_SCHEMA_VERSION, )?) .await?; let operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); diff --git a/tests/multi_server/residency/faults.rs b/tests/multi_server/residency/faults.rs index 0b2db138..e03863a7 100644 --- a/tests/multi_server/residency/faults.rs +++ b/tests/multi_server/residency/faults.rs @@ -363,6 +363,21 @@ async fn disconnected_admission_finishes_release_and_allows_a_later_restore() -> pending.abort(); assert!(pending.await.unwrap_err().is_cancelled()); fixture.store.proceed.notify_one(); + // Disconnecting the HTTP waiter does not join the tracked release task. + // Prove that it finishes without another request driving progress before + // asking for capacity again; an in-flight release may legitimately be busy. + timeout(Duration::from_secs(10), async { + while fixture.repository_dir.exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await?; + let released = CellAuthority::new(fixture.layout.clone()) + .load(fixture.target.cell_id()) + .await? + .ok_or("released Cell authority missing")?; + assert_eq!(released.value().state, ControlState::Idle); + assert!(released.value().owner.is_none()); create(&fixture.client, fixture.address, "fourth").await?; fixture.clone_original(fixture.address).await?; fixture.server.shutdown().await?; diff --git a/tests/multi_server/residency/git_discovery.rs b/tests/multi_server/residency/git_discovery.rs index 99e4b7f4..4b19675d 100644 --- a/tests/multi_server/residency/git_discovery.rs +++ b/tests/multi_server/residency/git_discovery.rs @@ -9,7 +9,7 @@ async fn ref_discovery_does_not_wait_for_a_full_history_restore() -> Result { blake3::Hasher::new().finalize_xof().fill(&mut body); tokio::fs::write(source.join("history.bin"), &body).await?; run_git(Some(&source), &["add", "history.bin"]).await?; - run_git(Some(&source), &["commit", "-m", "External history body"]).await?; + run_git(Some(&source), &["commit", "-m", "Packed history body"]).await?; run_git( Some(&source), &[ @@ -23,7 +23,19 @@ async fn ref_discovery_does_not_wait_for_a_full_history_restore() -> Result { .await?; let oid = run_git(Some(&source), &["rev-parse", "HEAD"]).await?; let advertised = format!("{} refs/heads/main", std::str::from_utf8(&oid)?.trim()); - let suffix = format!("/git-blobs/{}", hex::encode(Sha256::digest(&body))); + let database = cellule_ltx::rusqlite::Connection::open_with_flags( + fixture.repository_dir.join("repository.sqlite"), + cellule_ltx::rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + )?; + let blob = run_git(Some(&source), &["rev-parse", "HEAD:history.bin"]).await?; + let commit = canopy_server::ObjectId::from_hex(blob.trim_ascii())?; + let pack: Vec = database.query_row( + "SELECT pack FROM objects WHERE oid = ?1", + [commit.as_ref()], + |row| row.get(0), + )?; + drop(database); + let suffix = format!("/git-packs/{}.pack", hex::encode(pack)); let mut stored = fixture.store.inner.list(None); let mut blob = None; while let Some(meta) = std::future::poll_fn(|cx| stored.as_mut().poll_next(cx)).await { @@ -37,7 +49,7 @@ async fn ref_discovery_does_not_wait_for_a_full_history_restore() -> Result { create(&fixture.client, fixture.address, name).await?; } assert!(!fixture.repository_dir.exists()); - *fixture.store.paused_read.lock().unwrap() = Some(blob.ok_or("external Git body missing")?); + *fixture.store.paused_read.lock().unwrap() = Some(blob.ok_or("native Git pack missing")?); let destination = fixture.workspace.path().join("cold-clone"); let clone = async { run_git( diff --git a/tests/multi_server/size.rs b/tests/multi_server/size.rs index a3b9548b..4a0e5233 100644 --- a/tests/multi_server/size.rs +++ b/tests/multi_server/size.rs @@ -32,7 +32,7 @@ impl http_body::Body for Repeated { #[tokio::test(flavor = "multi_thread")] #[ignore = "large transfer qualification; run on a dedicated disk with at least 40 GiB free"] -async fn push_and_database_exceed_512_mib_and_lfs_exceeds_5_gib_after_restore() -> Result { +async fn native_pack_exceeds_512_mib_and_git_lfs_exceed_5_gib_after_restore() -> Result { let _ = tracing_subscriber::fmt() .with_max_level(tracing::Level::WARN) .try_init(); @@ -62,8 +62,8 @@ async fn push_and_database_exceed_512_mib_and_lfs_exceeds_5_gib_after_restore() ] { run_git(Some(&source), &["config", name, value]).await?; } - // Each incompressible blob stays in SQLite. The combined pack and database - // both exceed their former caps, so externalization cannot conceal a quota. + // Incompressible bodies make the native pack exceed the old transfer cap. + // SQLite must retain only metadata, independently of the logical Git bytes. for index in 0_u64..900 { let mut bytes = vec![0; 640 * 1024]; blake3::Hasher::new() @@ -81,7 +81,11 @@ async fn push_and_database_exceed_512_mib_and_lfs_exceeds_5_gib_after_restore() .set_len((PART * parts) as u64) .await?; run_git(Some(&source), &["add", "."]).await?; - run_git(Some(&source), &["commit", "-m", "Large SQLite repository"]).await?; + run_git( + Some(&source), + &["commit", "-m", "Large native-pack repository"], + ) + .await?; run_git(Some(&source), &["repack", "-ad"]).await?; let mut packs = tokio::fs::read_dir(source.join(".git/objects/pack")).await?; let mut pack_bytes = 0; @@ -121,7 +125,10 @@ async fn push_and_database_exceed_512_mib_and_lfs_exceeds_5_gib_after_restore() ) .await? .map_err(|error| error as Box)?; - assert!(database_bytes > 512 * 1024 * 1024); + assert!( + database_bytes < 16 * 1024 * 1024, + "Git bodies must not expand into SQLite" + ); eprintln!("large push published; SQLite database: {database_bytes} bytes"); // Independent SHA-256 reference (Python hashlib): 641 blocks of 8 MiB zeros. diff --git a/tests/owner_restart.rs b/tests/owner_restart.rs index 53360fee..4d8e13c9 100644 --- a/tests/owner_restart.rs +++ b/tests/owner_restart.rs @@ -1,3 +1,5 @@ +#[path = "support/packs.rs"] +mod pack_store; use std::{ path::Path, sync::Arc, @@ -53,7 +55,12 @@ async fn a_second_node_clones_from_the_published_root_after_local_disk_loss() .ok_or(Error::Registry("repository module missing"))?; let catalog = CellCatalog::new(layout.clone(), tenant); let proof = catalog - .provision(CatalogEntry::new(&target, CatalogRole::Sql, code, 1)?) + .provision(CatalogEntry::new( + &target, + CatalogRole::Sql, + code, + canopy_server::REPOSITORY_SCHEMA_VERSION, + )?) .await?; let authority = CellAuthority::new(layout.clone()); let first_session = SessionId::from_bytes([34; 16]); @@ -99,6 +106,7 @@ async fn a_second_node_clones_from_the_published_root_after_local_disk_loss() target.clone(), repository_id, canopy_server::ObjectFormat::Sha1, + pack_store::new(&layout, first_disk.path(), repository_id)?, )?); let owner_identity = support::identity()?; repository.ensure_owner(owner_identity, "canopy").await?; @@ -307,6 +315,7 @@ async fn a_second_node_clones_from_the_published_root_after_local_disk_loss() target, repository_id, canopy_server::ObjectFormat::Sha1, + pack_store::new(&layout, second_disk.path(), repository_id)?, )?); assert_eq!(push_cert_seed(&sql).await?, first_seed); assert_eq!( diff --git a/tests/repository_cell.rs b/tests/repository_cell.rs index 77f2470c..3b707f34 100644 --- a/tests/repository_cell.rs +++ b/tests/repository_cell.rs @@ -6,8 +6,6 @@ mod branch_rules; mod bulk_refs; #[path = "repository_cell/checks.rs"] mod checks; -#[path = "repository_cell/chunks.rs"] -mod chunks; #[path = "repository_cell/default_branch.rs"] mod default_branch; #[path = "repository_cell/graph.rs"] @@ -18,6 +16,10 @@ mod issues; mod merge; #[path = "support/objects.rs"] mod objects; +#[path = "support/packs.rs"] +mod pack_store; +#[path = "repository_cell/packs.rs"] +mod packs; #[path = "repository_cell/pages.rs"] mod pages; #[path = "repository_cell/pulls.rs"] @@ -48,9 +50,29 @@ use cellule_runtime::{ use cellule_store::Store; use object_store::{memory::InMemory, path::Path}; -#[tokio::test(flavor = "multi_thread")] -async fn repository_cell_publishes_objects_and_refs_atomically() --> Result<(), Box> { +#[test] +fn repository_cell_publishes_objects_and_refs_atomically() -> Result<(), Box> +{ + // This one scenario composes every repository primitive. Constructing and + // polling its combined future in debug builds exceeds libtest's 2 MiB stack. + // Give only the scenario driver extra space; runtime workers keep defaults. + std::thread::Builder::new() + .name("repository-cell-scenario".into()) + .stack_size(8 * 1024 * 1024) + .spawn(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("test runtime") + .block_on(repository_cell_scenario()) + .expect("repository Cell scenario"); + })? + .join() + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)); + Ok(()) +} + +async fn repository_cell_scenario() -> Result<(), Box> { let application = Arc::new(CanopyApplication::compile(build_descriptor( include_bytes!("../Cargo.toml"), "repository-cell-test", @@ -71,7 +93,12 @@ async fn repository_cell_publishes_objects_and_refs_atomically() .module_code(RepositoryModule::NAME) .ok_or(Error::Registry("repository module missing"))?; let proof = CellCatalog::new(layout.clone(), tenant) - .provision(CatalogEntry::new(&target, CatalogRole::Sql, code, 1)?) + .provision(CatalogEntry::new( + &target, + CatalogRole::Sql, + code, + canopy_server::REPOSITORY_SCHEMA_VERSION, + )?) .await?; let authority = CellAuthority::new(layout.clone()); let incarnation = IncarnationId::from_bytes([5; 16]); @@ -98,7 +125,7 @@ async fn repository_cell_publishes_objects_and_refs_atomically() .bootstrap( proof, CellReplica::new( - layout, + layout.clone(), *target.cell_id().as_bytes(), *incarnation.as_bytes(), Limits::default(), @@ -161,11 +188,19 @@ async fn repository_cell_publishes_objects_and_refs_atomically() let mut other_id = repository_id; other_id[0] ^= 1; assert!(matches!( - RepositoryCell::new(&application_handle, target.clone(), other_id, canopy_server::ObjectFormat::Sha1), + RepositoryCell::new(&application_handle, +target.clone(), +other_id, +canopy_server::ObjectFormat::Sha1, +pack_store::new(&layout, files.path(), other_id)?), Err(Error::Identity("repository UUID differs from Cell target")) )); let graph_sql = application_handle.sql::(target.clone())?; - let repository = RepositoryCell::new(&application_handle, target.clone(), repository_id, canopy_server::ObjectFormat::Sha1)?; + let repository = RepositoryCell::new(&application_handle, +target.clone(), +repository_id, +canopy_server::ObjectFormat::Sha1, +pack_store::new(&layout, files.path(), repository_id)?)?; let empty = repository.refs_page("", None).await?.output; assert_eq!(empty.generation, 0); assert!(empty.refs.is_empty()); @@ -485,7 +520,7 @@ async fn repository_cell_publishes_objects_and_refs_atomically() pulls::verify(&repository).await?; merge::verify(&repository, &graph_sql, &application_handle, &target).await?; rebase::verify(&repository, &graph_sql).await?; - chunks::exercise(&repository, &graph_sql).await?; + packs::exercise(&repository, &graph_sql).await?; bulk_refs::verify(&repository).await?; Ok(()) } diff --git a/tests/repository_cell/batches.rs b/tests/repository_cell/batches.rs index bc7be274..b2be477f 100644 --- a/tests/repository_cell/batches.rs +++ b/tests/repository_cell/batches.rs @@ -11,7 +11,7 @@ use cellule_runtime::{ type Result = std::result::Result>; -fn identity() -> Result { +pub(super) fn identity() -> Result { let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; Ok(MutationIdentity { request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), @@ -24,7 +24,7 @@ fn inline(body: Vec) -> StoredObject { StoredObject { oid: object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &body), kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body), + storage: ObjectStorage::Input(body), } } @@ -50,7 +50,7 @@ pub async fn exercise(repository: &RepositoryCell, sql: &SqlCell Result<(), Box = std::result::Result>; - -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -fn batch(objects: impl IntoIterator) -> ObjectBatch { - let mut batch = ObjectBatch::default(); - for object in objects { - assert!(batch.try_push(object).is_ok()); - } - batch -} - -pub async fn exercise(repository: &RepositoryCell, sql: &SqlCell) -> Result { - let body = vec![b't'; INLINE_OBJECT_LIMIT + 1]; - let staged_id = identity()?; - let object = repository - .stage_object(staged_id, ObjectKind::Tag, &body) - .await?; - let oid = object.oid; - assert!(repository.object(oid, None).await?.output.is_none()); - assert!(repository.existing_objects(&[oid]).await?.output.is_empty()); - let committed = repository.put_objects(identity()?, batch([object])).await?; - assert_eq!( - repository - .object(oid, Some(committed.receipt)) - .await? - .output, - Some((ObjectKind::Tag, body.clone())) - ); - // Stage retries replay their chunk commands; a second upload of identical bytes converges. - for request in [staged_id, identity()?] { - let duplicate = repository - .stage_object(request, ObjectKind::Tag, &body) - .await?; - repository - .put_objects(identity()?, batch([duplicate])) - .await?; - } - let layout = sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT storage, body, external_sha256, size FROM objects WHERE oid = ?1" - .into(), - parameters: vec![SqlValue::Blob(oid.to_vec())], - }], - }, - ) - .await?; - assert_eq!( - layout.output[0].rows[0], - vec![ - SqlValue::Text("chunked".into()), - SqlValue::Null, - SqlValue::Null, - SqlValue::Integer(body.len() as i64) - ] - ); - - for case in 0..7 { - let bytes = vec![case + 10; INLINE_OBJECT_LIMIT + 32]; - let mut object = repository - .stage_object(identity()?, ObjectKind::Commit, &bytes) - .await?; - let oid = object.oid; - let ObjectStorage::Chunked { - upload, - size, - blake3, - } = &mut object.storage - else { - return Err("chunk reference missing".into()); - }; - let upload = *upload; - let statement = match case { - 0 => Some(SqlStatement { sql: "DELETE FROM object_chunks WHERE upload_id = ?1 AND part = 1".into(), parameters: vec![SqlValue::Blob(upload.to_vec())] }), - 1 => Some(SqlStatement { sql: "UPDATE object_chunks SET body = zeroblob(length(body)) WHERE upload_id = ?1 AND part = 0".into(), parameters: vec![SqlValue::Blob(upload.to_vec())] }), - 2 => Some(SqlStatement { sql: "INSERT INTO object_chunks (upload_id, part, body) VALUES (?1, 2, ?2)".into(), parameters: vec![SqlValue::Blob(upload.to_vec()), SqlValue::Blob(vec![1])] }), - 3 => { blake3[0] ^= 1; None }, - 4 => { object.oid = match object.oid { canopy_server::ObjectId::Sha1(mut bytes) => { bytes[0] ^= 1; canopy_server::ObjectId::Sha1(bytes) }, canopy_server::ObjectId::Sha256(mut bytes) => { bytes[0] ^= 1; canopy_server::ObjectId::Sha256(bytes) } }; None }, - 5 => { object.kind = ObjectKind::Blob; None }, - _ => { *size -= 1; None }, - }; - if let Some(statement) = statement { - sql.batch( - identity()?, - SqlBatch { - statements: vec![statement], - }, - ) - .await?; - } - let first = format!("rolled back with invalid chunks {case}").into_bytes(); - let first_id = object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &first); - let first = StoredObject { - oid: first_id, - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(first), - }; - assert!( - matches!( - repository - .put_objects(identity()?, batch([first, object])) - .await, - Err(InvocationError::Rejected(_)) - ), - "case {case}" - ); - assert!( - repository - .existing_objects(&[first_id, oid]) - .await? - .output - .is_empty(), - "case {case}" - ); - } - // Chunk storage must preserve typed graph validation, including missing edges. - let mut tree = Vec::new(); - for index in 0..24_000 { - tree.extend_from_slice(format!("100644 file-{index:05}\0").as_bytes()); - tree.extend_from_slice(&[91; 20]); - } - let mut commit = format!("tree {}\nauthor Canopy 0 +0000\ncommitter Canopy 0 +0000\n\n", hex::encode([92; 20])).into_bytes(); - commit.extend(vec![b'c'; INLINE_OBJECT_LIMIT]); - let mut tag = format!( - "object {}\ntype commit\ntag missing\n\n", - hex::encode([93; 20]) - ) - .into_bytes(); - tag.extend(vec![b't'; INLINE_OBJECT_LIMIT]); - for (kind, bytes) in [ - (ObjectKind::Tree, tree), - (ObjectKind::Commit, commit), - (ObjectKind::Tag, tag), - ] { - let object = repository.stage_object(identity()?, kind, &bytes).await?; - let name = format!("refs/tags/chunked-{}", kind.git_name()); - let plan = PushPlan { - actor: "canopy".into(), - updates: vec![RefUpdate { - name: name.clone(), - expected: None, - new_oid: Some(object.oid), - }], - }; - repository.put_objects(identity()?, batch([object])).await?; - assert!(matches!( - repository.finalize_push(identity()?, plan).await, - Err(InvocationError::Rejected(_)) - )); - assert!(repository.ref_state(&name, None).await?.output.is_none()); - } - // Corruption after publication must also be detected during recovery reads. - sql.batch(identity()?, SqlBatch { statements: vec![SqlStatement { - sql: "DELETE FROM object_chunks WHERE upload_id = (SELECT chunk_id FROM objects WHERE oid = ?1) AND part = 1".into(), - parameters: vec![SqlValue::Blob(oid.to_vec())], - }] }).await?; - assert!(repository.object(oid, None).await.is_err()); - assert!( - repository - .stage_object(identity()?, ObjectKind::Blob, &body) - .await - .is_err() - ); - assert!( - repository - .stage_object(identity()?, ObjectKind::Tree, b"small") - .await - .is_err() - ); - Ok(()) -} diff --git a/tests/repository_cell/graph.rs b/tests/repository_cell/graph.rs index a8e1ce32..42bd2232 100644 --- a/tests/repository_cell/graph.rs +++ b/tests/repository_cell/graph.rs @@ -114,6 +114,7 @@ pub async fn verify( .contains(&SqlValue::Blob(root.to_vec())) ); put(repository, ObjectKind::Blob, blob_body).await?; + let before = certificates(sql).await?; for candidates in [ vec![root, tree, blob], vec![blob, root], @@ -125,11 +126,7 @@ pub async fn verify( .await, Err(InvocationError::Rejected(_)) )); - assert!( - !certificates(sql) - .await? - .contains(&SqlValue::Blob(blob.to_vec())) - ); + assert_eq!(certificates(sql).await?, before); } // A rejected batch must roll back edges as well as closure certificates; // otherwise a staged tree could authorize a later raw-OID fetch. @@ -307,7 +304,7 @@ async fn resumable(repository: &RepositoryCell, sql: &SqlCell) .try_push(StoredObject { oid: *oid, kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body.clone()) + storage: ObjectStorage::Input(body.clone()) }) .is_ok() ); @@ -327,7 +324,7 @@ async fn resumable(repository: &RepositoryCell, sql: &SqlCell) .iter() .filter(|(oid, _)| persisted.contains(&SqlValue::Blob(oid.to_vec()))) .count(), - 256 + leaves.len() ); assert!(!persisted.contains(&SqlValue::Blob(root.to_vec()))); assert_eq!( @@ -389,7 +386,7 @@ struct CertificateCommand; impl cellule_runtime::Command for CertificateCommand { const MODULE: &'static str = "repository"; const ID: u32 = 6; - const CODEC_VERSION: u32 = 2; + const CODEC_VERSION: u32 = 4; type Input = CertificateInput; type Output = bool; fn execute( diff --git a/tests/repository_cell/packs.rs b/tests/repository_cell/packs.rs new file mode 100644 index 00000000..bfdfed25 --- /dev/null +++ b/tests/repository_cell/packs.rs @@ -0,0 +1,106 @@ +use super::*; +use canopy_server::{INLINE_OBJECT_LIMIT, ObjectBatch, ObjectStorage}; +use cellule_runtime::{ + SqlCell, + primitives::sql::{SqlBatch, SqlStatement, SqlValue}, +}; +type Result = std::result::Result>; + +pub async fn exercise(repository: &RepositoryCell, sql: &SqlCell) -> Result { + for kind in [ + ObjectKind::Blob, + ObjectKind::Tree, + ObjectKind::Commit, + ObjectKind::Tag, + ] { + let body = vec![kind.git_name().as_bytes()[0]; INLINE_OBJECT_LIMIT + 1]; + let object = repository.stage_object(kind, &body).await?; + let oid = object.oid; + assert!(repository.object(oid, None).await?.output.is_none()); + let ObjectStorage::Packed(reference) = &object.storage else { + return Err("stage must return native pack".into()); + }; + assert_eq!(reference.size(), body.len() as u64); + assert!(reference.pack().pack_bytes < body.len() as u64 / 100); + let mut batch = ObjectBatch::default(); + assert!(batch.try_push(object).is_ok()); + let publication = repository + .put_objects(super::batches::identity()?, batch) + .await?; + assert_eq!( + repository + .object(oid, Some(publication.receipt)) + .await? + .output, + Some((kind, body.clone())) + ); + let object = repository.stage_object(kind, &body).await?; + let mut batch = ObjectBatch::default(); + assert!(batch.try_push(object).is_ok()); + repository + .put_objects(super::batches::identity()?, batch) + .await?; + // Metadata corruption cannot silently change content read from the pack. + sql.batch( + super::batches::identity()?, + SqlBatch { + statements: vec![SqlStatement { + sql: "UPDATE objects SET digest = zeroblob(32) WHERE oid = ?1".into(), + parameters: vec![SqlValue::Blob(oid.to_vec())], + }], + }, + ) + .await?; + assert!(repository.object(oid, None).await.is_err()); + sql.batch( + super::batches::identity()?, + SqlBatch { + statements: vec![SqlStatement { + sql: "UPDATE objects SET digest = ?2 WHERE oid = ?1".into(), + parameters: vec![ + SqlValue::Blob(oid.to_vec()), + SqlValue::Blob(blake3::hash(&body).as_bytes().to_vec()), + ], + }], + }, + ) + .await?; + } + // Cross the dependency-write boundary with distinct child IDs. These + // unreferenced objects may stage before their children are available. + let mut tree = Vec::new(); + for index in 0..8_193_u32 { + let child = object_id( + repository.object_format(), + ObjectKind::Blob, + &index.to_le_bytes(), + ); + tree.extend_from_slice(format!("100644 file-{index:05}\0").as_bytes()); + tree.extend_from_slice(&child); + } + let object = repository.stage_object(ObjectKind::Tree, &tree).await?; + let oid = object.oid; + let mut batch = ObjectBatch::default(); + assert!(batch.try_push(object).is_ok()); + repository + .put_objects(super::batches::identity()?, batch) + .await?; + let rows = sql + .query( + None, + SqlBatch { + statements: vec![SqlStatement { + sql: "SELECT count(*) FROM object_dependencies WHERE parent = ?".into(), + parameters: vec![SqlValue::Blob(oid.to_vec())], + }], + }, + ) + .await? + .output; + assert_eq!(rows[0].rows[0], vec![SqlValue::Integer(8_193)]); + assert_eq!( + repository.object(oid, None).await?.output, + Some((ObjectKind::Tree, tree)) + ); + Ok(()) +} diff --git a/tests/repository_cell/pages.rs b/tests/repository_cell/pages.rs index 00f21c89..500589e0 100644 --- a/tests/repository_cell/pages.rs +++ b/tests/repository_cell/pages.rs @@ -1,51 +1,78 @@ use super::*; -use canopy_server::{INLINE_OBJECT_LIMIT, ObjectBatch, ObjectStorage, StoredObject}; +use canopy_server::{ObjectBatch, ObjectStorage, StoredObject}; use cellule_runtime::{ - SqlCell, primitives::sql::SqlBatch, primitives::sql::SqlStatement, primitives::sql::SqlValue, + SqlCell, + primitives::sql::{SqlBatch, SqlStatement, SqlValue}, }; -use sha2::{Digest as _, Sha256}; use std::collections::BTreeMap; - type Result = std::result::Result>; -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -fn inline(body: Vec) -> StoredObject { - StoredObject { - oid: object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &body), - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body), +pub async fn exercise(repository: &RepositoryCell, sql: &SqlCell) -> Result { + let mut expected = BTreeMap::new(); + for group in 0..3 { + let mut batch = ObjectBatch::default(); + for n in 0..512 { + let body = format!("metadata-page-{group}-{n}").into_bytes(); + let oid = object_id(repository.object_format(), ObjectKind::Blob, &body); + expected.insert(oid, body.clone()); + assert!( + batch + .try_push(StoredObject { + oid, + kind: ObjectKind::Blob, + storage: ObjectStorage::Input(body) + }) + .is_ok() + ); + } + repository + .put_objects(super::batches::identity()?, batch) + .await?; } -} - -async fn put( - repository: &RepositoryCell, - objects: impl IntoIterator, -) -> Result { - let mut batch = ObjectBatch::default(); - for object in objects { - assert!(batch.try_push(object).is_ok()); + let mut after = None; + let mut actual = BTreeMap::new(); + let mut pages = 0; + loop { + let page = repository.object_page(after).await?.output; + if page.is_empty() { + break; + } + assert!(page.len() <= 512); + pages += 1; + for object in page { + assert!(after.is_none_or(|prior| object.oid > prior)); + assert!(matches!(object.storage, ObjectStorage::Packed(_))); + let (_, body) = repository + .object(object.oid, None) + .await? + .output + .ok_or("missing packed body")?; + after = Some(object.oid); + actual.insert(object.oid, body); + } } - repository.put_objects(identity()?, batch).await?; - Ok(()) -} - -async fn clear(sql: &SqlCell) -> Result { - // This exercise runs first in a fresh fixture and owns all these records. + assert_eq!(pages, 3); + assert_eq!(actual, expected); + let packs = sql + .query( + None, + SqlBatch { + statements: vec![SqlStatement { + sql: "SELECT COUNT(*) FROM git_packs".into(), + parameters: vec![], + }], + }, + ) + .await?; + assert_eq!(packs.output[0].rows, vec![vec![SqlValue::Integer(3)]]); sql.batch( - identity()?, + super::batches::identity()?, SqlBatch { statements: [ + "DELETE FROM object_dependencies", + "DELETE FROM object_closure", "DELETE FROM objects", - "DELETE FROM object_chunks", - "DELETE FROM object_uploads", + "DELETE FROM git_packs", ] .into_iter() .map(|sql| SqlStatement { @@ -58,216 +85,3 @@ async fn clear(sql: &SqlCell) -> Result { .await?; Ok(()) } - -async fn scan( - repository: &RepositoryCell, -) -> Result> { - let mut after = None; - let mut objects = BTreeMap::new(); - for _ in 0..10 { - let page = repository.object_page(after).await?.output; - if page.is_empty() { - return Ok(objects); - } - assert!(page.len() <= 128); - let bytes: usize = page - .iter() - .map(|object| match &object.storage { - ObjectStorage::Inline(body) => body.len(), - _ => 0, - }) - .sum(); - assert!(bytes <= INLINE_OBJECT_LIMIT); - for object in page { - assert!(after.is_none_or(|old| object.oid > old)); - after = Some(object.oid); - assert!(objects.insert(object.oid, object).is_none()); - } - } - Err("object pagination did not finish".into()) -} - -fn before(oid: canopy_server::ObjectId) -> Option { - let mut bytes = match oid { - canopy_server::ObjectId::Sha1(bytes) => bytes.to_vec(), - canopy_server::ObjectId::Sha256(bytes) => bytes.to_vec(), - }; - for byte in bytes.iter_mut().rev() { - if *byte != 0 { - *byte -= 1; - return canopy_server::ObjectId::try_from(bytes).ok(); - } - *byte = u8::MAX; - } - None -} - -pub async fn exercise(repository: &RepositoryCell, sql: &SqlCell) -> Result { - assert!(repository.object_page(None).await?.output.is_empty()); - let mut expected = BTreeMap::new(); - for start in [0, 128] { - let batch = (start..start + 128) - .map(|n| { - let body = format!("object-page-{n}").into_bytes(); - let object = inline(body.clone()); - expected.insert(object.oid, body); - object - }) - .collect::>(); - put(repository, batch).await?; - } - let first = repository.object_page(None).await?.output; - assert_eq!(first.len(), 128); - let second = repository - .object_page(Some(first.last().ok_or("missing first page")?.oid)) - .await? - .output; - assert_eq!(second.len(), 128); - assert!( - repository - .object_page(Some(second.last().ok_or("missing second page")?.oid)) - .await? - .output - .is_empty() - ); - let actual = scan(repository).await?; - assert_eq!( - actual.keys().copied().collect::>(), - expected.keys().copied().collect::>() - ); - for (oid, object) in actual { - let ObjectStorage::Inline(body) = object.storage else { - return Err("inline body missing".into()); - }; - assert_eq!(body, expected[&oid]); - } - clear(sql).await?; - expected.clear(); - - let halves: Vec<_> = [11, 12] - .map(|byte| inline(vec![byte; INLINE_OBJECT_LIMIT / 2])) - .into(); - for object in &halves { - let ObjectStorage::Inline(body) = &object.storage else { - return Err("inline body missing".into()); - }; - expected.insert(object.oid, body.clone()); - } - put(repository, halves).await?; - let full = repository.object_page(None).await?.output; - assert_eq!(full.len(), 2); - assert_eq!( - full.iter() - .map(|object| match &object.storage { - ObjectStorage::Inline(body) => body.len(), - _ => 0, - }) - .sum::(), - INLINE_OBJECT_LIMIT - ); - for body in [vec![13; INLINE_OBJECT_LIMIT], vec![14], Vec::new()] { - let object = inline(body.clone()); - expected.insert(object.oid, body); - put(repository, [object]).await?; - } - let chunked = vec![15; INLINE_OBJECT_LIMIT + 1]; - let staged = repository - .stage_object(identity()?, ObjectKind::Tag, &chunked) - .await?; - let chunked_oid = staged.oid; - let ObjectStorage::Chunked { upload, .. } = staged.storage else { - return Err("missing chunk reference".into()); - }; - put( - repository, - [StoredObject { - oid: chunked_oid, - kind: ObjectKind::Tag, - storage: ObjectStorage::Chunked { - upload, - size: chunked.len() as u64, - blake3: *blake3::hash(&chunked).as_bytes(), - }, - }], - ) - .await?; - let external = vec![16; INLINE_OBJECT_LIMIT + 1]; - let external_oid = object_id( - canopy_server::ObjectFormat::Sha1, - ObjectKind::Blob, - &external, - ); - let sha256: [u8; 32] = Sha256::digest(&external).into(); - put( - repository, - [StoredObject { - oid: external_oid, - kind: ObjectKind::Blob, - storage: ObjectStorage::External { - size: external.len() as u64, - blake3: *blake3::hash(&external).as_bytes(), - sha256, - }, - }], - ) - .await?; - let mut actual = scan(repository).await?; - assert_eq!(actual.len(), expected.len() + 2); - for (oid, expected_body) in &expected { - let object = actual.remove(oid).ok_or("missing inline object")?; - let ObjectStorage::Inline(body) = object.storage else { - return Err("inline body missing".into()); - }; - assert_eq!(&body, expected_body); - } - let descriptor = actual - .remove(&chunked_oid) - .ok_or("missing chunked object")?; - assert!( - matches!(descriptor.storage, ObjectStorage::Chunked { upload: stored, size, blake3: digest } if stored == upload && size == chunked.len() as u64 && digest == *blake3::hash(&chunked).as_bytes()) - ); - let descriptor = actual - .remove(&external_oid) - .ok_or("missing external object")?; - assert!( - matches!(descriptor.storage, ObjectStorage::External { size, blake3: digest, sha256: stored } if size == external.len() as u64 && digest == *blake3::hash(&external).as_bytes() && stored == sha256) - ); - - let damaged = object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &[14]); - for (statement, parameters) in [ - ( - "UPDATE objects SET digest = zeroblob(32) WHERE oid = ?1", - vec![SqlValue::Blob(damaged.to_vec())], - ), - ( - "UPDATE objects SET body = ?2, digest = ?3 WHERE oid = ?1", - vec![ - SqlValue::Blob(damaged.to_vec()), - SqlValue::Blob(vec![17]), - SqlValue::Blob(blake3::hash(&[17]).as_bytes().to_vec()), - ], - ), - ( - "UPDATE objects SET body = zeroblob(?2), size = ?2 WHERE oid = ?1", - vec![ - SqlValue::Blob(damaged.to_vec()), - SqlValue::Integer(INLINE_OBJECT_LIMIT as i64 + 1), - ], - ), - ] { - sql.batch( - identity()?, - SqlBatch { - statements: vec![SqlStatement { - sql: statement.into(), - parameters, - }], - }, - ) - .await?; - assert!(repository.object_page(before(damaged)).await.is_err()); - } - clear(sql).await?; - assert!(repository.object_page(None).await?.output.is_empty()); - Ok(()) -} diff --git a/tests/smart_http.rs b/tests/smart_http.rs index 665414ea..5202ac13 100644 --- a/tests/smart_http.rs +++ b/tests/smart_http.rs @@ -60,7 +60,12 @@ async fn stock_git_push_and_clone_are_backed_by_one_repository_cell() .module_code(RepositoryModule::NAME) .ok_or(Error::Registry("repository module missing"))?; let proof = CellCatalog::new(layout.clone(), tenant) - .provision(CatalogEntry::new(&target, CatalogRole::Sql, code, 1)?) + .provision(CatalogEntry::new( + &target, + CatalogRole::Sql, + code, + canopy_server::REPOSITORY_SCHEMA_VERSION, + )?) .await?; let authority = CellAuthority::new(layout.clone()); let incarnation = IncarnationId::from_bytes([24; 16]); @@ -107,18 +112,26 @@ async fn stock_git_push_and_clone_are_backed_by_one_repository_cell() tenant, application_id, )?; + let paused_blobs = Arc::new(paused_blobs::PausedBlobs::default()); + let blob_store: Arc = paused_blobs.clone(); + let disk_budget = DiskBudget::new(1 << 30); let repository = Arc::new(RepositoryCell::new( &application_handle, - target, + target.clone(), repository_id, canopy_server::ObjectFormat::Sha1, + canopy_server::git_pack::PackStore::new( + Arc::clone(&blob_store), + repository_id, + scratch.path().into(), + disk_budget.clone(), + 256 * 1024 * 1024, + 64 * 1024 * 1024, + )?, )?); repository .ensure_owner(support::identity()?, "canopy") .await?; - let paused_blobs = Arc::new(paused_blobs::PausedBlobs::default()); - let blob_store: Arc = paused_blobs.clone(); - let disk_budget = DiskBudget::new(1 << 30); let gateway = Arc::new(GitGateway::new( Arc::clone(&repository), scratch.path().to_path_buf(), @@ -379,7 +392,7 @@ async fn stock_git_push_and_clone_are_backed_by_one_repository_cell() } for object in page { cursor = Some(object.oid); - external_seen |= matches!(object.storage, ObjectStorage::External { .. }); + external_seen |= matches!(object.storage, ObjectStorage::Packed(_)); } } assert!(external_seen); @@ -447,10 +460,30 @@ async fn stock_git_push_and_clone_are_backed_by_one_repository_cell() }) .await?; drop(teardown_gateway); + let mut pack_bytes = 0; + for entry in std::fs::read_dir(scratch.path())? { + let entry = entry?; + if entry + .file_name() + .as_encoded_bytes() + .starts_with(b"canopy-pack-") + { + for file in std::fs::read_dir(entry.path())? { + pack_bytes += file?.metadata()?.len(); + } + } + assert!( + !entry + .file_name() + .as_encoded_bytes() + .starts_with(b"canopy-git-"), + "gateway teardown must remove native cache generations" + ); + } assert_eq!( disk_budget.used(), - 0, - "gateway teardown releases retained object and snapshot charges" + pack_bytes, + "only the Repository Cell's verified native pairs remain charged" ); let gateway = Arc::new(GitGateway::new( @@ -511,7 +544,15 @@ async fn stock_git_push_and_clone_are_backed_by_one_repository_cell() assert!(tokio::fs::read(clone.join("tracked.lfs")).await? == lfs_body); native_resources::verify(scratch.path(), &url).await?; cache_reuse::verify(scratch.path(), &local, &url).await?; - publication::verify(scratch.path(), &repository, &paused_blobs, &url).await?; + publication::verify( + scratch.path(), + &repository, + &paused_blobs, + &url, + &application_handle, + &target, + ) + .await?; let _ = stop_tx.send(()); server.await??; Ok(()) diff --git a/tests/smart_http/cache_admission.rs b/tests/smart_http/cache_admission.rs index acdb8a80..11e2e72e 100644 --- a/tests/smart_http/cache_admission.rs +++ b/tests/smart_http/cache_admission.rs @@ -195,9 +195,9 @@ pub async fn verify( assert!(budget.used() >= occupied.bytes()); assert!(budget.used() <= budget.capacity()); drop(occupied); - // Ref discovery fits even when expanding the repository history cannot. + // Compressed native pairs let discovery and fetch share verified history. // Nested tags and direct tree/blob refs must preserve native peeling rules. - let retained = budget.used(); + let mut retained = budget.used(); let occupied = budget.try_reserve(budget.capacity() - retained - 64 * 1024)?; for service in ["git-upload-pack", "git-receive-pack"] { let bytes = client @@ -224,7 +224,15 @@ pub async fn verify( .any(|part| part == peeled.as_bytes()) ); } - assert_eq!(budget.used(), occupied.bytes() + retained); + if service == "git-upload-pack" { + assert!(budget.used() >= occupied.bytes() + retained); + assert!(budget.used() <= budget.capacity()); + // A completed pair can remain resident after temporary discovery + // cache files are released; the next request must reuse that pair. + retained = budget.used() - occupied.bytes(); + } else { + assert_eq!(budget.used(), occupied.bytes() + retained); + } } let bytes = client .post(format!("{url}/git-upload-pack")) @@ -259,7 +267,10 @@ pub async fn verify( ); drop(occupied); fetch().send().await?.error_for_status()?.bytes().await?; - assert!(budget.used() > common.len() as u64 * 40); + assert!( + budget.used() > common.len() as u64 * 40, + "selected disposable projections remain fully charged" + ); let retained = budget.used(); // A current full cache can serve discovery without recopying its targets. let occupied = budget.try_reserve(budget.capacity() - retained - 512)?; diff --git a/tests/smart_http/cache_reuse.rs b/tests/smart_http/cache_reuse.rs index ec86baa4..4982c264 100644 --- a/tests/smart_http/cache_reuse.rs +++ b/tests/smart_http/cache_reuse.rs @@ -69,8 +69,8 @@ pub async fn verify( ) .await?; let after = cached_objects(root)?; - // A new blob, tree and commit are hydrated. Existing bodies keep their - // actual files across receive-pack and the next fetch of the published ref generation. + // Only selected new objects enter the disposable projection; existing + // files remain unchanged across push and fetch. assert_eq!(after.len(), before.len() + 3); for (path, metadata) in before { assert_eq!(after.get(&path), Some(&metadata)); diff --git a/tests/smart_http/publication.rs b/tests/smart_http/publication.rs index a2891dda..56012b45 100644 --- a/tests/smart_http/publication.rs +++ b/tests/smart_http/publication.rs @@ -3,6 +3,9 @@ use crate::paused_blobs::PausedBlobs; use canopy_server::{PushPlan, RefUpdate, branch_rules::BranchRuleEdit}; use std::{sync::atomic::Ordering, time::Duration}; +type Application = cellule_app::ApplicationHandle; +type Target = cellule_runtime::CellTarget; + type Result = std::result::Result>; const AUTH: &str = "http.extraHeader=Authorization: Bearer late-writer-token"; @@ -15,6 +18,8 @@ pub async fn verify( repository: &Arc, store: &Arc, url: &str, + application: &Application, + target: &Target, ) -> Result { let local = root.join("publication-race"); run_git(None, &["init", "-b", "main", local.to_str().ok_or("path")?]).await?; @@ -135,7 +140,9 @@ pub async fn verify( } else { None }; - tokio::time::timeout(Duration::from_secs(15), store.entered.notified()).await?; + tokio::time::timeout(Duration::from_secs(15), store.entered.notified()) + .await + .map_err(|_| format!("native pack fault hook not reached: {kind}"))?; if kind == "refs" || kind == "no-report" { repository .finalize_push( @@ -177,7 +184,18 @@ pub async fn verify( // A different gateway can commit the same ID while this attempt is paused. // The losing ingestion failure must return that success, never a local ng. let winner = if raced { - Some(gateway_request(root, repository, store.clone(), &body, &id).await?) + Some( + gateway_request( + root, + repository, + store.clone(), + &body, + &id, + application, + target, + ) + .await?, + ) } else { None }; @@ -246,8 +264,16 @@ pub async fn verify( } // Recreate the gateway with no native/object cache; the saved Cell // decision must survive permission/policy or storage recovery. - let (status, saved) = - gateway_request(root, repository, Arc::new(InMemory::new()), &body, &id).await?; + let (status, saved) = gateway_request( + root, + repository, + Arc::new(InMemory::new()), + &body, + &id, + application, + target, + ) + .await?; assert_eq!(status, if no_report { 409 } else { 200 }); assert_eq!(saved, report); } @@ -287,7 +313,16 @@ pub async fn verify( } } } - verify_preparation(root, repository, store, &local, current).await?; + verify_preparation( + root, + repository, + store, + &local, + current, + application, + target, + ) + .await?; Ok(()) } @@ -297,17 +332,28 @@ async fn verify_preparation( store: &Arc, source: &Path, current: canopy_server::ObjectId, + application: &Application, + target: &Target, ) -> Result { use sha1::{Digest as _, Sha1}; let mut pack = b"PACK\0\0\0\x02\0\0\0\0".to_vec(); pack.extend_from_slice(&Sha1::digest(&pack)); for kind in ["plain", "sideband", "no-report", "stock", "race"] { let scratch = tempfile::tempdir_in(root)?; + let budget = DiskBudget::new(1 << 30); + let cold = fresh_repository( + scratch.path(), + repository, + store.clone(), + budget.clone(), + application, + target, + )?; let gateway = Arc::new(GitGateway::new( - repository.clone(), + cold, scratch.path().into(), store.clone(), - DiskBudget::new(1 << 30), + budget, )); let listener = TcpListener::bind("127.0.0.1:0").await?; let address = listener.local_addr()?; @@ -357,7 +403,23 @@ async fn verify_preparation( .body(body.clone()) }; let generation = repository.default_branch(None).await?.output.generation; - store.read_armed.store(true, Ordering::SeqCst); + if kind == "stock" { + // Stock Git reads discovery first. A cold pack failure there is a + // transport failure, so exercise its durable ingestion refusal on + // a new commit after discovery has completed instead. + tokio::fs::write( + source.join("preparation-stock"), + b"new native publication +", + ) + .await?; + run_git(Some(source), &["add", "preparation-stock"]).await?; + run_git(Some(source), &["commit", "-m", "Native publication fault"]).await?; + store.armed.store(true, Ordering::SeqCst); + } else { + store.read_armed.store(true, Ordering::SeqCst); + } + let child = if kind == "stock" { Some( Command::new("git") @@ -387,9 +449,22 @@ async fn verify_preparation( } else { None }; - tokio::time::timeout(Duration::from_secs(15), store.entered.notified()).await?; + tokio::time::timeout(Duration::from_secs(15), store.entered.notified()) + .await + .map_err(|_| format!("native pack fault hook not reached: {kind}"))?; let winner = if kind == "race" { - Some(gateway_request(root, repository, store.clone(), &body, &id).await?) + Some( + gateway_request( + root, + repository, + store.clone(), + &body, + &id, + application, + target, + ) + .await?, + ) } else { None }; @@ -400,7 +475,7 @@ async fn verify_preparation( let stderr = String::from_utf8_lossy(&output.stderr); assert!(!output.status.success(), "{stderr}"); assert!( - stderr.contains("Canopy push failed before publication"), + stderr.contains("Canopy object ingestion failed"), "{stderr}" ); assert!(!stderr.contains("HTTP 500"), "{stderr}"); @@ -425,7 +500,16 @@ async fn verify_preparation( } } } - let replay = gateway_request(root, repository, store.clone(), &body, &id).await?; + let replay = gateway_request( + root, + repository, + store.clone(), + &body, + &id, + application, + target, + ) + .await?; assert_eq!(replay, (status, report.to_vec())); } if winner.is_none() { @@ -468,14 +552,20 @@ async fn gateway_request( store: Arc, body: &[u8], id: &str, + application: &Application, + target: &Target, ) -> Result<(u16, Vec)> { let scratch = tempfile::tempdir_in(root)?; - let gateway = GitGateway::new( - Arc::clone(repository), - scratch.path().into(), - store, - DiskBudget::new(1 << 30), - ); + let budget = DiskBudget::new(1 << 30); + let cold = fresh_repository( + scratch.path(), + repository, + store.clone(), + budget.clone(), + application, + target, + )?; + let gateway = GitGateway::new(cold, scratch.path().into(), store, budget); let response = gateway .handle( canopy_server::git_http::GitHttpRequest { @@ -498,3 +588,28 @@ async fn gateway_request( axum::body::to_bytes(response.body, 1 << 20).await?.to_vec(), )) } + +fn fresh_repository( + root: &Path, + repository: &RepositoryCell, + store: Arc, + budget: DiskBudget, + application: &Application, + target: &Target, +) -> Result> { + let packs = canopy_server::git_pack::PackStore::new( + store, + repository.repository_id(), + root.into(), + budget, + 256 * 1024 * 1024, + 64 * 1024 * 1024, + )?; + Ok(Arc::new(RepositoryCell::new( + application, + target.clone(), + repository.repository_id(), + repository.object_format(), + packs, + )?)) +} diff --git a/tests/smart_http/ref_snapshots.rs b/tests/smart_http/ref_snapshots.rs index cfc4f534..bdd67eb8 100644 --- a/tests/smart_http/ref_snapshots.rs +++ b/tests/smart_http/ref_snapshots.rs @@ -60,7 +60,7 @@ pub async fn verify( .try_push(StoredObject { oid, kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body.to_vec()), + storage: ObjectStorage::Input(body.to_vec()), }) .map_err(|_| "fixture object exceeds batch limit")?; } diff --git a/tests/support/objects.rs b/tests/support/objects.rs index aab80c65..69d55106 100644 --- a/tests/support/objects.rs +++ b/tests/support/objects.rs @@ -15,7 +15,7 @@ pub async fn put( .try_push(StoredObject { oid, kind, - storage: ObjectStorage::Inline(body.to_vec()), + storage: ObjectStorage::Input(body.to_vec()), }) .map_err(|_| "fixture object exceeds batch bounds")?; let result = repository.put_objects(identity, batch).await?; diff --git a/tests/support/packs.rs b/tests/support/packs.rs new file mode 100644 index 00000000..876f3507 --- /dev/null +++ b/tests/support/packs.rs @@ -0,0 +1,14 @@ +pub fn new( + layout: &cellule_ltx::CellStorageLayout, + root: &std::path::Path, + id: [u8; 16], +) -> cellule_runtime::Result { + canopy_server::git_pack::PackStore::new( + layout.store().inner().clone(), + id, + root.into(), + cellule_ltx::DiskBudget::new(1 << 30), + 256 * 1024 * 1024, + 64 * 1024 * 1024, + ) +} diff --git a/tests/support/paused_blobs.rs b/tests/support/paused_blobs.rs index 57329463..618c96e4 100644 --- a/tests/support/paused_blobs.rs +++ b/tests/support/paused_blobs.rs @@ -44,7 +44,7 @@ impl ObjectStore for PausedBlobs { path: &StorePath, options: PutMultipartOptions, ) -> object_store::Result> { - // Large-blob ingestion starts only after native receive-pack has + // Native-pack ingestion starts only after native receive-pack has // accepted the disposable refs, but before Cell ref publication. if self.armed.swap(false, Ordering::SeqCst) { self.entered.notify_one(); @@ -63,7 +63,7 @@ impl ObjectStore for PausedBlobs { path: &StorePath, options: GetOptions, ) -> object_store::Result { - if path.as_ref().contains("/git-blobs/") && self.read_armed.swap(false, Ordering::SeqCst) { + if path.as_ref().contains("/git-packs/") && self.read_armed.swap(false, Ordering::SeqCst) { self.entered.notify_one(); self.proceed.notified().await; if self.fail.swap(false, Ordering::SeqCst) { From 5d6e7c09156545d83a63afc8329692dc8aef25d5 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 16:36:11 -0700 Subject: [PATCH 02/19] refactor(storage): remove retired Git body formats Remove row-based body readers, chunk staging, old graph commands, and unhashed manifests. Keep native catalog verification and the current operation identities. Use the current repository schema in Directory fixtures and qualify foreign-object isolation and predecessor-manifest rejection. --- .../src/artifact/tests.rs | 26 + crates/canopy-object-storage/src/blob/mod.rs | 167 ------ crates/canopy-object-storage/src/blob/read.rs | 82 --- .../canopy-object-storage/src/blob/tests.rs | 165 ------ crates/canopy-object-storage/src/external.rs | 37 +- crates/canopy-object-storage/src/lib.rs | 5 +- crates/canopy-server/src/ancestry.rs | 201 ------- crates/canopy-server/src/branch_rules/mod.rs | 88 +-- .../src/deployment/backup/mod.rs | 3 - .../src/git_cache/maintenance.rs | 167 ------ crates/canopy-server/src/git_cache/mod.rs | 108 +--- crates/canopy-server/src/git_cache/tests.rs | 216 +------- crates/canopy-server/src/git_gateway/mod.rs | 23 - crates/canopy-server/src/git_objects/tests.rs | 20 +- crates/canopy-server/src/graph/mod.rs | 280 +--------- crates/canopy-server/src/graph/preparation.rs | 232 -------- crates/canopy-server/src/graph/tests.rs | 24 - crates/canopy-server/src/lib.rs | 237 +-------- crates/canopy-server/src/object_batch/mod.rs | 401 -------------- .../canopy-server/src/object_batch/tests.rs | 169 ------ crates/canopy-server/src/object_chunks/mod.rs | 230 -------- .../canopy-server/src/object_chunks/tests.rs | 98 ---- crates/canopy-server/src/object_reads/mod.rs | 242 --------- crates/canopy-server/src/pack_store.rs | 337 ------------ .../canopy-server/src/pulls/candidates/mod.rs | 54 -- .../src/pulls/candidates/rebase.rs | 78 --- .../canopy-server/src/pulls/merge/command.rs | 129 +---- crates/canopy-server/src/pulls/merge/mod.rs | 81 +-- crates/canopy-server/src/refs.rs | 205 +------ crates/canopy-server/src/schema.sql | 336 ------------ .../tests/directory_cell/main.rs | 4 +- .../tests/repository_cell/batches.rs | 217 -------- .../tests/repository_cell/branch_rules.rs | 351 ------------ .../tests/repository_cell/bulk_refs.rs | 183 ------- .../tests/repository_cell/checks.rs | 192 ------- .../tests/repository_cell/chunks.rs | 187 ------- .../tests/repository_cell/default_branch.rs | 173 ------ .../tests/repository_cell/graph.rs | 408 -------------- .../tests/repository_cell/issues.rs | 202 ------- .../tests/repository_cell/merge.rs | 190 ------- .../tests/repository_cell/pages.rs | 274 ---------- .../tests/repository_cell/pulls.rs | 271 ---------- .../tests/repository_cell/rebase.rs | 123 ----- .../tests/repository_cell/visibility.rs | 116 ---- .../tests/smart_http/cache_admission.rs | 291 ---------- .../tests/smart_http/cache_reuse.rs | 87 --- .../tests/smart_http/encoded_input.rs | 165 ------ .../tests/smart_http/native_resources.rs | 105 ---- .../tests/smart_http/publication.rs | 502 ------------------ .../tests/smart_http/push_uploads.rs | 71 --- .../tests/smart_http/ref_snapshots.rs | 299 ----------- crates/canopy-server/tests/support/mod.rs | 53 -- crates/canopy-server/tests/support/objects.rs | 26 - docs/contracts.md | 340 +++--------- docs/git-compatibility.md | 7 +- docs/implementation.md | 89 ++-- docs/repository-cell-primitives.md | 6 +- 57 files changed, 199 insertions(+), 9174 deletions(-) delete mode 100644 crates/canopy-object-storage/src/blob/mod.rs delete mode 100644 crates/canopy-object-storage/src/blob/read.rs delete mode 100644 crates/canopy-object-storage/src/blob/tests.rs delete mode 100644 crates/canopy-server/src/ancestry.rs delete mode 100644 crates/canopy-server/src/graph/preparation.rs delete mode 100644 crates/canopy-server/src/object_batch/mod.rs delete mode 100644 crates/canopy-server/src/object_batch/tests.rs delete mode 100644 crates/canopy-server/src/object_chunks/mod.rs delete mode 100644 crates/canopy-server/src/object_chunks/tests.rs delete mode 100644 crates/canopy-server/src/object_reads/mod.rs delete mode 100644 crates/canopy-server/src/pack_store.rs delete mode 100644 crates/canopy-server/src/schema.sql delete mode 100644 crates/canopy-server/tests/repository_cell/batches.rs delete mode 100644 crates/canopy-server/tests/repository_cell/branch_rules.rs delete mode 100644 crates/canopy-server/tests/repository_cell/bulk_refs.rs delete mode 100644 crates/canopy-server/tests/repository_cell/checks.rs delete mode 100644 crates/canopy-server/tests/repository_cell/chunks.rs delete mode 100644 crates/canopy-server/tests/repository_cell/default_branch.rs delete mode 100644 crates/canopy-server/tests/repository_cell/graph.rs delete mode 100644 crates/canopy-server/tests/repository_cell/issues.rs delete mode 100644 crates/canopy-server/tests/repository_cell/merge.rs delete mode 100644 crates/canopy-server/tests/repository_cell/pages.rs delete mode 100644 crates/canopy-server/tests/repository_cell/pulls.rs delete mode 100644 crates/canopy-server/tests/repository_cell/rebase.rs delete mode 100644 crates/canopy-server/tests/repository_cell/visibility.rs delete mode 100644 crates/canopy-server/tests/smart_http/cache_admission.rs delete mode 100644 crates/canopy-server/tests/smart_http/cache_reuse.rs delete mode 100644 crates/canopy-server/tests/smart_http/encoded_input.rs delete mode 100644 crates/canopy-server/tests/smart_http/native_resources.rs delete mode 100644 crates/canopy-server/tests/smart_http/publication.rs delete mode 100644 crates/canopy-server/tests/smart_http/push_uploads.rs delete mode 100644 crates/canopy-server/tests/smart_http/ref_snapshots.rs delete mode 100644 crates/canopy-server/tests/support/mod.rs delete mode 100644 crates/canopy-server/tests/support/objects.rs diff --git a/crates/canopy-object-storage/src/artifact/tests.rs b/crates/canopy-object-storage/src/artifact/tests.rs index 3fe8ba9f..8d4ef713 100644 --- a/crates/canopy-object-storage/src/artifact/tests.rs +++ b/crates/canopy-object-storage/src/artifact/tests.rs @@ -11,6 +11,32 @@ fn key(body: &[u8]) -> ArtifactKey { } } +#[tokio::test] +async fn predecessor_manifest_is_rejected_even_with_a_matching_descriptor_digest() -> Result { + let store: Arc = Arc::new(InMemory::new()); + let artifacts = ArtifactStore::new(Arc::clone(&store), [1; 16]); + let body = b"current immutable artifact"; + let key = key(body); + let mut descriptor = artifacts + .put( + key, + body.len() as u64, + key.binding_digest, + &mut body.as_slice(), + ) + .await?; + let path = artifacts.path(key, descriptor.digest)?; + let mut manifest = store.get(&path).await?.bytes().await?.to_vec(); + manifest[..8].copy_from_slice(b"CANOPY01"); + descriptor.manifest_digest = *blake3::hash(&manifest).as_bytes(); + store.put(&path, manifest.into()).await?; + assert!(matches!( + artifacts.ranges_owned(key, descriptor, Arc::new(())).await, + Err(ArtifactError::Corrupt) + )); + Ok(()) +} + #[tokio::test] async fn selected_parts_authenticate_without_fetching_unrequested_bytes() -> Result { let store: Arc = Arc::new(InMemory::new()); diff --git a/crates/canopy-object-storage/src/blob/mod.rs b/crates/canopy-object-storage/src/blob/mod.rs deleted file mode 100644 index 640520d8..00000000 --- a/crates/canopy-object-storage/src/blob/mod.rs +++ /dev/null @@ -1,167 +0,0 @@ -//! Immutable external bytes for large Git blobs. - -use std::{sync::Arc, time::Duration}; - -use bytes::Bytes; -use canopy_git_format::{ObjectFormat, ObjectHasher, ObjectId, ObjectKind}; -use object_store::{ObjectStore, path::Path}; -use sha2::{Digest, Sha256}; -use tokio::io::{AsyncRead, AsyncReadExt}; - -mod read; -pub use read::LargeBlobRead; - -const CHUNK_BYTES: usize = crate::external::PART_BYTES; -const IO_TIMEOUT: Duration = Duration::from_secs(120); - -#[derive(Clone, Copy, Debug)] -pub struct LargeBlobReference { - pub oid: ObjectId, - pub size: u64, - pub blake3: [u8; 32], - pub sha256: [u8; 32], -} - -#[derive(Debug, thiserror::Error)] -pub enum LargeBlobError { - #[error("large blob store failed")] - Store(#[from] object_store::Error), - #[error("large blob input failed")] - Io(#[from] std::io::Error), - #[error("large blob worker failed")] - Task(#[from] tokio::task::JoinError), - #[error("large blob transfer timed out")] - Timeout, - #[error("large blob bytes disagree with their SQLite reference")] - Corrupt, -} - -struct Hashes { - oid: ObjectHasher, - sha256: Sha256, - blake3: blake3::Hasher, -} - -impl Hashes { - fn new(format: ObjectFormat, size: u64) -> Self { - let oid = ObjectHasher::new(format, ObjectKind::Blob, size); - Self { - oid, - sha256: Sha256::new(), - blake3: blake3::Hasher::new(), - } - } - - async fn update(mut self, bytes: Bytes) -> Result<(Self, Bytes), LargeBlobError> { - Ok(tokio::task::spawn_blocking(move || { - self.oid.update(&bytes); - self.sha256.update(&bytes); - self.blake3.update(&bytes); - (self, bytes) - }) - .await?) - } - - fn finish(self, size: u64) -> LargeBlobReference { - LargeBlobReference { - oid: self.oid.finalize(), - size, - sha256: self.sha256.finalize().into(), - blake3: *self.blake3.finalize().as_bytes(), - } - } -} - -/// Repository-scoped content-addressed body store. -pub struct LargeBlobStore { - store: Arc, - repository_id: [u8; 16], -} - -impl LargeBlobStore { - pub fn new(store: Arc, repository_id: [u8; 16]) -> Self { - Self { - store, - repository_id, - } - } - - /// Streams exactly `size` bytes, verifying the Git OID before immutable publication. - /// The caller retains this future through cleanup; abrupt process loss may leave staging. - pub async fn put( - &self, - oid: ObjectId, - size: u64, - input: &mut (impl AsyncRead + Unpin), - ) -> Result { - let stage = Path::from(format!( - "repos/{}/git-blob-staging/{}", - hex::encode(self.repository_id), - uuid::Uuid::new_v4() - )); - let mut upload = crate::external::Upload::new(self.store.clone(), stage).await?; - let result = async { - let mut hashes = Hashes::new(oid.format(), size); - let mut remaining = size; - loop { - let length = remaining.min(CHUNK_BYTES as u64) as usize; - let mut bytes = vec![0; length]; - tokio::time::timeout(IO_TIMEOUT, input.read_exact(&mut bytes)) - .await - .map_err(|_| LargeBlobError::Timeout)??; - let (updated, bytes) = hashes.update(Bytes::from(bytes)).await?; - hashes = updated; - upload.write(bytes).await?; - remaining -= length as u64; - if remaining == 0 { - break; - } - } - let reference = hashes.finish(size); - if reference.oid != oid { - return Err(LargeBlobError::Corrupt); - } - upload - .publish(&blob_path(self.repository_id, &reference.sha256), size) - .await?; - self.verify(&reference).await?; - Ok(reference) - } - .await; - let cleanup = upload.cleanup().await; - if result.is_err() - && let Err(error) = &cleanup - { - tracing::warn!(error = %error, "Git blob staging cleanup failed"); - } - let reference = result?; - cleanup?; - Ok(reference) - } - - /// Opens a bounded reader that validates all three hashes before returning its final range. - pub async fn read( - &self, - reference: &LargeBlobReference, - ) -> Result { - LargeBlobRead::open(self.store.clone(), self.repository_id, *reference).await - } - - /// Checks an immutable object's bytes without materializing the full body. - pub async fn verify(&self, reference: &LargeBlobReference) -> Result<(), LargeBlobError> { - let mut read = self.read(reference).await?; - while read.next().await?.is_some() {} - Ok(()) - } -} - -pub fn blob_path(repository_id: [u8; 16], sha256: &[u8; 32]) -> Path { - Path::from(format!( - "repos/{}/git-blobs/{}", - hex::encode(repository_id), - hex::encode(sha256) - )) -} - -#[cfg(test)] -mod tests; diff --git a/crates/canopy-object-storage/src/blob/read.rs b/crates/canopy-object-storage/src/blob/read.rs deleted file mode 100644 index 4b1f2460..00000000 --- a/crates/canopy-object-storage/src/blob/read.rs +++ /dev/null @@ -1,82 +0,0 @@ -use super::*; - -/// Sequential bounded ranges from one immutable Git blob. -pub struct LargeBlobRead { - store: Arc, - path: Path, - manifest: crate::external::Manifest, - reference: LargeBlobReference, - offset: u64, - hashes: Option, -} - -impl LargeBlobRead { - pub(super) async fn open( - store: Arc, - repository_id: [u8; 16], - reference: LargeBlobReference, - ) -> Result { - let path = blob_path(repository_id, &reference.sha256); - let manifest = crate::external::open(store.as_ref(), &path, reference.size).await?; - let read = Self { - store, - path, - manifest, - reference, - offset: 0, - hashes: Some(Hashes::new(reference.oid.format(), reference.size)), - }; - if reference.size == 0 { - read.check(Hashes::new(reference.oid.format(), 0))?; - } - Ok(read) - } - - pub fn reference(&self) -> LargeBlobReference { - self.reference - } - - /// Returns at most 8 MiB; an error or canceled read invalidates the reader. - pub async fn next(&mut self) -> Result, LargeBlobError> { - if self.offset == self.reference.size { - return Ok(None); - } - // Taking the verifier poisons cancellation/error paths. A partially read - // range cannot be silently retried with a truncated checksum history. - let hashes = self.hashes.take().ok_or(LargeBlobError::Corrupt)?; - let bytes = tokio::time::timeout(IO_TIMEOUT, self.range()) - .await - .map_err(|_| LargeBlobError::Timeout)??; - let (hashes, bytes) = hashes.update(bytes).await?; - let end = self.offset + bytes.len() as u64; - if end == self.reference.size { - self.check(hashes)?; - } else { - self.hashes = Some(hashes); - } - self.offset = end; - Ok(Some(bytes)) - } - - async fn range(&self) -> Result { - Ok(crate::external::read( - self.store.as_ref(), - &self.path, - &self.manifest, - self.reference.size, - self.offset, - ) - .await?) - } - - fn check(&self, hashes: Hashes) -> Result<(), LargeBlobError> { - let actual = hashes.finish(self.reference.size); - if actual.oid != self.reference.oid - || actual.sha256 != self.reference.sha256 - || actual.blake3 != self.reference.blake3 - { - return Err(LargeBlobError::Corrupt); - } - Ok(()) - } -} diff --git a/crates/canopy-object-storage/src/blob/tests.rs b/crates/canopy-object-storage/src/blob/tests.rs deleted file mode 100644 index 1517e773..00000000 --- a/crates/canopy-object-storage/src/blob/tests.rs +++ /dev/null @@ -1,165 +0,0 @@ -use super::*; -use canopy_git_format::{ObjectFormat, ObjectId, ObjectKind, object_id}; -use object_store::{ObjectStoreExt, memory::InMemory}; -use std::future::poll_fn; - -type Result = std::result::Result>; - -#[tokio::test] -async fn streamed_objects_verify_all_hashes_without_consuming_the_next_frame() -> Result { - let store: Arc = Arc::new(InMemory::new()); - let blobs = LargeBlobStore::new(store.clone(), [1; 16]); - for (format, size) in [ObjectFormat::Sha1, ObjectFormat::Sha256] - .into_iter() - .flat_map(|format| [0, 1, CHUNK_BYTES, CHUNK_BYTES + 1].map(move |size| (format, size))) - { - let body = vec![21; size]; - let mut input = body.clone(); - input.extend_from_slice(b"\nnext frame"); - let mut input = input.as_slice(); - let oid = object_id(format, ObjectKind::Blob, &body); - let reference = blobs.put(oid, size as u64, &mut input).await?; - assert_eq!(input, b"\nnext frame"); - let replay = blobs.put(oid, size as u64, &mut body.as_slice()).await?; - assert_eq!(reference.sha256, replay.sha256); - let mut reader = blobs.read(&reference).await?; - let mut bytes_read = 0; - while let Some(bytes) = reader.next().await? { - assert!(bytes.len() <= CHUNK_BYTES); - assert_eq!(bytes.as_ref(), &body[bytes_read..bytes_read + bytes.len()]); - bytes_read += bytes.len(); - } - assert_eq!(bytes_read, size); - } - let mut objects = store.list(None); - while let Some(object) = poll_fn(|cx| objects.as_mut().poll_next(cx)).await { - assert!(!object?.location.as_ref().contains("staging")); - } - Ok(()) -} - -#[tokio::test] -async fn bad_input_does_not_publish_and_existing_corruption_is_not_overwritten() -> Result { - let store: Arc = Arc::new(InMemory::new()); - let blobs = LargeBlobStore::new(store.clone(), [2; 16]); - let body = b"immutable blob"; - let oid = object_id(ObjectFormat::Sha1, ObjectKind::Blob, body); - assert!(matches!( - blobs - .put( - ObjectId::Sha1([0; 20]), - body.len() as u64, - &mut body.as_slice() - ) - .await, - Err(LargeBlobError::Corrupt) - )); - assert!( - blobs - .put(oid, body.len() as u64 + 1, &mut body.as_slice()) - .await - .is_err() - ); - let listing = store.list_with_delimiter(None).await?; - assert!(listing.objects.is_empty() && listing.common_prefixes.is_empty()); - let reference = blobs - .put(oid, body.len() as u64, &mut body.as_slice()) - .await?; - let path = crate::external::part(&blob_path([2; 16], &reference.sha256), 0); - store - .put(&path, Bytes::from(vec![0; body.len()]).into()) - .await?; - let mut reader = blobs.read(&reference).await?; - assert!(matches!(reader.next().await, Err(LargeBlobError::Corrupt))); - assert!(matches!(reader.next().await, Err(LargeBlobError::Corrupt))); - assert!(matches!( - blobs - .put(oid, body.len() as u64, &mut body.as_slice()) - .await, - Err(LargeBlobError::Corrupt) - )); - assert_eq!( - store.get(&path).await?.bytes().await?.as_ref(), - vec![0; body.len()] - ); - Ok(()) -} - -#[tokio::test] -async fn metadata_hash_corruption_withholds_the_final_range() -> Result { - let store: Arc = Arc::new(InMemory::new()); - let blobs = LargeBlobStore::new(store.clone(), [3; 16]); - let body = vec![42; CHUNK_BYTES + 17]; - let reference = blobs - .put( - object_id(ObjectFormat::Sha1, ObjectKind::Blob, &body), - body.len() as u64, - &mut body.as_slice(), - ) - .await?; - crate::external::copy_parts( - store.as_ref(), - &blob_path([3; 16], &reference.sha256), - &blob_path([3; 16], &[0; 32]), - reference.size, - ) - .await?; - store - .copy( - &blob_path([3; 16], &reference.sha256), - &blob_path([3; 16], &[0; 32]), - ) - .await?; - for bad in [ - LargeBlobReference { - sha256: [0; 32], - ..reference - }, - LargeBlobReference { - oid: ObjectId::Sha1([0; 20]), - ..reference - }, - LargeBlobReference { - blake3: [0; 32], - ..reference - }, - ] { - let mut reader = blobs.read(&bad).await?; - assert_eq!( - reader.next().await?.ok_or("first range")?.len(), - CHUNK_BYTES - ); - assert!(matches!(reader.next().await, Err(LargeBlobError::Corrupt))); - } - Ok(()) -} - -#[tokio::test] -async fn replacement_between_ranges_invalidates_the_reader() -> Result { - let store: Arc = Arc::new(InMemory::new()); - let blobs = LargeBlobStore::new(store.clone(), [4; 16]); - let body = vec![42; CHUNK_BYTES + 17]; - let reference = blobs - .put( - object_id(ObjectFormat::Sha1, ObjectKind::Blob, &body), - body.len() as u64, - &mut body.as_slice(), - ) - .await?; - let mut reader = blobs.read(&reference).await?; - reader.next().await?.ok_or("first range")?; - store - .put( - &blob_path([4; 16], &reference.sha256), - Bytes::from(vec![21; body.len()]).into(), - ) - .await?; - assert!(matches!( - reader.next().await, - Err(LargeBlobError::Store( - object_store::Error::Precondition { .. } - )) - )); - assert!(matches!(reader.next().await, Err(LargeBlobError::Corrupt))); - Ok(()) -} diff --git a/crates/canopy-object-storage/src/external.rs b/crates/canopy-object-storage/src/external.rs index 07f06679..9523fc5c 100644 --- a/crates/canopy-object-storage/src/external.rs +++ b/crates/canopy-object-storage/src/external.rs @@ -1,4 +1,4 @@ -//! Immutable, fixed-size parts for Git blob and LFS bodies. +//! Immutable, verified parts for native artifacts and LFS bodies. use bytes::{Bytes, BytesMut}; use object_store::{ @@ -11,7 +11,6 @@ use std::{future::Future, sync::Arc, time::Duration}; pub const PART_BYTES: usize = 8 * 1024 * 1024; pub const MAX_PARTS: u64 = 65_536; pub const MAX_ARTIFACT_BYTES: u64 = PART_BYTES as u64 * MAX_PARTS; -const MAGIC: &[u8; 8] = b"CANOPY01"; const HASHED_MAGIC: &[u8; 8] = b"CANOPY02"; pub struct Manifest { @@ -107,12 +106,6 @@ impl Upload { Ok(()) } - pub async fn publish(&mut self, path: &Path, size: u64) -> object_store::Result<()> { - let mut manifest = MAGIC.to_vec(); - manifest.extend_from_slice(&size.to_le_bytes()); - self.publish_manifest(path, size, manifest).await - } - pub async fn publish_hashed( &mut self, path: &Path, @@ -161,19 +154,6 @@ impl Upload { Ok(root) } - async fn publish_manifest( - &mut self, - path: &Path, - size: u64, - manifest: Vec, - ) -> object_store::Result<()> { - if self.active.is_some() || self.parts != size.div_ceil(PART_BYTES as u64).max(1) { - return Err(invalid()); - } - copy_parts(self.store.as_ref(), &self.stage, path, size).await?; - self.create_manifest(path, manifest).await - } - async fn create_manifest( &mut self, path: &Path, @@ -271,21 +251,6 @@ async fn bounded( .await } -pub async fn open( - store: &dyn ObjectStore, - path: &Path, - size: u64, -) -> object_store::Result { - let (meta, bytes) = bounded(store, path, GetOptions::default(), 16).await?; - if &bytes[..8] != MAGIC || bytes[8..] != size.to_le_bytes() { - return Err(invalid()); - } - if size == 0 { - bounded(store, &part(path, 0), GetOptions::default(), 0).await?; - } - Ok(Manifest { meta, bytes }) -} - pub async fn open_hashed( store: &dyn ObjectStore, path: &Path, diff --git a/crates/canopy-object-storage/src/lib.rs b/crates/canopy-object-storage/src/lib.rs index 279aeb66..d8293de4 100644 --- a/crates/canopy-object-storage/src/lib.rs +++ b/crates/canopy-object-storage/src/lib.rs @@ -1,7 +1,4 @@ -//! Immutable object parts and content-addressed large Git blobs. +//! Immutable native artifacts and object parts for Git/LFS. pub mod artifact; -pub mod blob; pub mod external; - -pub use blob::{LargeBlobError, LargeBlobRead, LargeBlobReference, LargeBlobStore}; diff --git a/crates/canopy-server/src/ancestry.rs b/crates/canopy-server/src/ancestry.rs deleted file mode 100644 index 6166de44..00000000 --- a/crates/canopy-server/src/ancestry.rs +++ /dev/null @@ -1,201 +0,0 @@ -//! Bounded certificates over parent links extracted from verified commit objects. - -use crate::{RepositoryCell, RepositoryModule, server::mutation_identity}; -use cellule_runtime::{ - CellModule, Command, Error, InvocationError, codec::BoundedDecoder, codec::BoundedEncoder, - codec::CodecError, codec::WireValue, primitives::sql::SqlBatch, primitives::sql::SqlStatement, - primitives::sql::SqlValue, registry::CommandContext, registry::CommandResult, -}; -use std::collections::{HashMap, hash_map::Entry}; - -type Oid = crate::ObjectId; -const PAGE: usize = 128; - -pub(crate) struct AncestryProof { - ancestor: Oid, - steps: Vec<(Oid, Oid)>, -} - -impl WireValue for AncestryProof { - fn encode(&self, out: &mut BoundedEncoder) -> Result<(), CodecError> { - if self.steps.is_empty() || self.steps.len() > PAGE { - return Err(CodecError::Invalid("ancestry proof size")); - } - out.write_bytes(&self.ancestor)?; - out.write_count(self.steps.len())?; - for (child, parent) in &self.steps { - out.write_bytes(child)?; - out.write_bytes(parent)?; - } - Ok(()) - } - fn decode(input: &mut BoundedDecoder<'_>) -> Result { - let ancestor = oid(input)?; - let count = input.read_count()?; - if count == 0 || count > PAGE { - return Err(CodecError::Invalid("ancestry proof size")); - } - let mut steps = Vec::with_capacity(count); - for _ in 0..count { - steps.push((oid(input)?, oid(input)?)); - } - Ok(Self { ancestor, steps }) - } -} -fn oid(input: &mut BoundedDecoder<'_>) -> Result { - input - .read_bytes()? - .try_into() - .map_err(|_| CodecError::Invalid("ancestry OID width")) -} - -pub(crate) struct CertifyAncestry; -impl Command for CertifyAncestry { - const MODULE: &'static str = RepositoryModule::NAME; - const ID: u32 = 7; - const CODEC_VERSION: u32 = 2; - type Input = AncestryProof; - type Output = bool; - fn execute( - context: &mut CommandContext<'_, '_>, - proof: AncestryProof, - ) -> cellule_runtime::Result> { - for (child, parent) in proof.steps { - let result = context.sql(&SqlBatch { statements: vec![SqlStatement { - sql: "SELECT EXISTS (SELECT 1 FROM commit_parents WHERE child = ?1 AND parent = ?2) AND (?2 = ?3 OR EXISTS (SELECT 1 FROM commit_ancestry WHERE ancestor = ?3 AND descendant = ?2))".into(), - parameters: vec![SqlValue::Blob(child.to_vec()), SqlValue::Blob(parent.to_vec()), SqlValue::Blob(proof.ancestor.to_vec())], - }] })?; - if !matches!( - result - .first() - .and_then(|set| set.rows.first()) - .map(Vec::as_slice), - Some([SqlValue::Integer(1)]) - ) { - return Ok(CommandResult::Rejected(false)); - } - context.sql(&SqlBatch { statements: vec![SqlStatement { - sql: "INSERT INTO commit_ancestry (ancestor, descendant) VALUES (?1, ?2) ON CONFLICT DO NOTHING".into(), - parameters: vec![SqlValue::Blob(proof.ancestor.to_vec()), SqlValue::Blob(child.to_vec())], - }] })?; - } - Ok(CommandResult::Success(true)) - } -} - -impl RepositoryCell { - pub(crate) async fn prepare_ancestry( - &self, - ancestor: Oid, - descendant: Oid, - ) -> Result<(), InvocationError> { - self.ancestry_path(ancestor, descendant) - .await - .map_err(|source| { - InvocationError::NotStarted(Error::Facility { - name: "commit ancestry preparation", - source, - }) - }) - } - - async fn ancestry_path( - &self, - ancestor: Oid, - descendant: Oid, - ) -> Result<(), Box> { - if ancestor == descendant { - return Ok(()); - } - let known = self - .sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: - "SELECT 1 FROM commit_ancestry WHERE ancestor = ?1 AND descendant = ?2" - .into(), - parameters: vec![ - SqlValue::Blob(ancestor.to_vec()), - SqlValue::Blob(descendant.to_vec()), - ], - }], - }, - ) - .await?; - if known.output.first().is_some_and(|set| !set.rows.is_empty()) { - return Ok(()); - } - let mut previous: HashMap> = HashMap::from([(descendant, None)]); - let mut pending = vec![(descendant, None::)]; - let mut edges = 0; - while let Some((child, after)) = pending.pop() { - let result = self.sql.query(None, SqlBatch { statements: vec![SqlStatement { - sql: format!("SELECT parent, EXISTS (SELECT 1 FROM commit_ancestry WHERE ancestor = ?3 AND descendant = p.parent) FROM commit_parents p WHERE child = ?1 AND parent > ?2 ORDER BY parent LIMIT {PAGE}"), - parameters: vec![SqlValue::Blob(child.to_vec()), SqlValue::Blob(after.map_or_else(Vec::new, |oid| oid.to_vec())), SqlValue::Blob(ancestor.to_vec())], - }] }).await?; - let rows = &result - .output - .first() - .ok_or(Error::Command("missing commit parents"))? - .rows; - edges += rows.len(); - if edges > 250_000 { - return Err(Error::Command("commit ancestry traversal limit").into()); - } - let mut parents = Vec::with_capacity(rows.len()); - for row in rows { - let [SqlValue::Blob(parent), SqlValue::Integer(known)] = row.as_slice() else { - return Err(Error::Command("invalid commit parent").into()); - }; - let parent: Oid = parent - .as_slice() - .try_into() - .map_err(|_| Error::Command("invalid parent OID"))?; - parents.push(parent); - let Entry::Vacant(entry) = previous.entry(parent) else { - continue; - }; - entry.insert(Some(child)); - if previous.len() > 100_000 { - return Err(Error::Command("commit ancestry traversal limit").into()); - } - if parent == ancestor || *known == 1 { - let mut steps = Vec::new(); - let mut parent = parent; - while let Some(Some(child)) = previous.get(&parent).copied() { - steps.push((child, parent)); - parent = child; - } - // Each bounded command proves its chosen links itself. Partial - // preparation retains only immutable facts, never ref authority. - for steps in steps.chunks(PAGE) { - self.application - .command::( - &self.target, - mutation_identity()?, - AncestryProof { - ancestor, - steps: steps.to_vec(), - }, - ) - .await?; - } - return Ok(()); - } - } - if parents.len() == PAGE { - pending.push((child, parents.last().copied())); - } - for parent in parents.into_iter().rev() { - if previous.get(&parent) == Some(&Some(child)) { - pending.push((parent, None)); - } - } - } - // A non-ancestor has no positive certificate. Final publication decides - // against the current rule; this search never certifies a negative result. - Ok(()) - } -} diff --git a/crates/canopy-server/src/branch_rules/mod.rs b/crates/canopy-server/src/branch_rules/mod.rs index b4a21490..a6d8bd3c 100644 --- a/crates/canopy-server/src/branch_rules/mod.rs +++ b/crates/canopy-server/src/branch_rules/mod.rs @@ -5,7 +5,7 @@ use crate::ReadIdentity; pub(crate) mod command; use crate::{ - PushPlan, RefUpdate, RepositoryCell, + RefUpdate, RepositoryCell, default_branch::valid_default_branch, directory::{TokenScope, validate_component}, }; @@ -197,52 +197,6 @@ impl RepositoryCell { .collect::>>() .map_err(InvocationError::NotStarted) } - - pub(crate) async fn prepare_branch_proofs( - &self, - plan: &PushPlan, - ) -> Result<(), InvocationError> { - if plan.updates.is_empty() || plan.updates.len() > crate::refs::MAX_UPDATES { - return Ok(()); - } - let role = self - .access_level(&plan.actor, None) - .await - .map_err(|source| { - InvocationError::NotStarted(Error::Facility { - name: "branch proof authorization", - source: Box::new(source), - }) - })?; - if !role.output.is_some_and(|role| role >= TokenScope::Write) { - return Ok(()); - } - for updates in plan.updates.chunks(128) { - let policies = self.branch_policies(updates).await.map_err(|source| { - InvocationError::NotStarted(Error::Facility { - name: "branch proof policy", - source: Box::new(source), - }) - })?; - for (update, policy) in updates.iter().zip(policies) { - let (Some(old), Some(new)) = ( - update.expected.as_ref().and_then(|old| old.oid), - update.new_oid, - ) else { - continue; - }; - if policy.is_some_and(|policy| { - !policy.require_pull_request - && policy.fast_forward_only - && policy.checks_pass - && !policy.ancestry - }) { - self.prepare_ancestry(old, new).await?; - } - } - } - Ok(()) - } } pub(crate) struct Policy { @@ -271,53 +225,19 @@ impl Policy { } } -pub(crate) fn policies_allow( - context: &cellule_runtime::registry::CommandContext<'_, '_>, - plan: &PushPlan, - merge: Option<&crate::pulls::merge::ReviewedMerge>, -) -> cellule_runtime::Result { - for updates in plan.updates.chunks(128) { - let result = context.sql(&SqlBatch { - statements: updates.iter().map(policy_statement).collect(), - })?; - for (update, set) in updates.iter().zip(&result) { - if decode_policy(std::slice::from_ref(set))?.is_some_and(|policy| { - !policy.allows_ref(update, true) - || (policy.require_pull_request - && !merge.is_some_and(|merge| merge.authorizes(update))) - }) { - return Ok(false); - } - } - } - Ok(true) -} - -fn policy_statement(update: &RefUpdate) -> SqlStatement { - policy_statement_for(update, None) -} /// The final publisher supplies only MAC-verified catalog ancestry evidence. /// Current check-context versions/reporters and branch rules stay in this SQL. pub(crate) fn policy_statement_with_ancestry(update: &RefUpdate, ancestry: bool) -> SqlStatement { - policy_statement_for(update, Some(ancestry)) -} -fn policy_statement_for(update: &RefUpdate, ancestry: Option) -> SqlStatement { let old = update.expected.as_ref().and_then(|old| old.oid); - let ancestry_sql = if ancestry.is_some() { - "?4" - } else { - "(?2 IS NULL OR coalesce(?2 = ?3, 0) OR EXISTS (SELECT 1 FROM commit_ancestry WHERE ancestor = ?2 AND descendant = ?3))" - }; - let mut parameters = vec![ + let ancestry_sql = "?4"; + let parameters = vec![ SqlValue::Text(update.name.clone()), old.map_or(SqlValue::Null, |oid| SqlValue::Blob(oid.to_vec())), update .new_oid .map_or(SqlValue::Null, |oid| SqlValue::Blob(oid.to_vec())), + SqlValue::Integer(i64::from(ancestry)), ]; - if let Some(value) = ancestry { - parameters.push(SqlValue::Integer(i64::from(value))); - } SqlStatement { sql: format!( "SELECT b.deny_deletions, b.fast_forward, NOT EXISTS (SELECT 1 FROM branch_required_checks q LEFT JOIN check_contexts c ON c.name = q.context LEFT JOIN check_runs r ON r.number = (SELECT number FROM check_runs WHERE oid = ?3 AND context = q.context AND context_version = c.version ORDER BY number DESC LIMIT 1) WHERE q.reference = b.reference AND (c.enabled IS NOT 1 OR r.state IS NOT 'success' OR r.reporter IS NOT c.reporter)), {ancestry_sql}, b.require_pull_request FROM branch_rules b WHERE b.reference = ?1 AND b.enabled = 1" diff --git a/crates/canopy-server/src/deployment/backup/mod.rs b/crates/canopy-server/src/deployment/backup/mod.rs index 76dd6ace..689d7ca3 100644 --- a/crates/canopy-server/src/deployment/backup/mod.rs +++ b/crates/canopy-server/src/deployment/backup/mod.rs @@ -3,7 +3,6 @@ use super::*; use crate::{ REPOSITORIES, REPOSITORY_DATABASE_LIMIT_BYTES, - blob::LargeBlobError, lfs::LfsError, server::{ServerError, unix_now_ms}, }; @@ -33,8 +32,6 @@ pub enum BackupError { Store(#[from] StorageError), #[error("backup external part copy failed")] External(#[from] object_store::Error), - #[error("backup Git blob verification failed")] - Blob(#[from] LargeBlobError), #[error("backup LFS verification failed")] Lfs(#[from] LfsError), #[error("backup SQLite read failed")] diff --git a/crates/canopy-server/src/git_cache/maintenance.rs b/crates/canopy-server/src/git_cache/maintenance.rs index 4b2e3683..3345f2ce 100644 --- a/crates/canopy-server/src/git_cache/maintenance.rs +++ b/crates/canopy-server/src/git_cache/maintenance.rs @@ -71,18 +71,6 @@ fn index_order(path: &Path, format: crate::ObjectFormat) -> io::Result bool { - self.durable_packs - .read() - .expect("durable inventory poisoned") - .contains(&sha) - } - pub(crate) fn mark_durable_pack(&self, sha: [u8; 32]) { - self.durable_packs - .write() - .expect("durable inventory poisoned") - .insert(sha); - } #[cfg(test)] pub(crate) async fn pack_sources( self: &Arc, @@ -112,147 +100,6 @@ impl GitCache { }) .await? } - pub(crate) async fn install_pack( - self: &Arc, - hash: crate::ObjectId, - pack: LargeBlobRead, - index: LargeBlobRead, - ) -> Result<(), CacheError> { - if hash.format() != self.object_format { - return Err(io::Error::other("pack format mismatch").into()); - } - self.write_artifact( - format!("objects/pack/pack-{}.pack", hex::encode(hash)), - pack, - ) - .await?; - self.write_artifact( - format!("objects/pack/pack-{}.idx", hex::encode(hash)), - index, - ) - .await?; - let cache = Arc::clone(self); - tokio::task::spawn_blocking(move || { - let path = cache - .git_dir() - .join(format!("objects/pack/pack-{}.idx", hex::encode(hash))); - let index = crate::git_format::pack_index::PackIndex::open(&path, cache.object_format)?; - if index.pack_checksum() != hash { - return Err(io::Error::other("index/pack binding mismatch").into()); - } - cache.register_checked_index(index)?; - cache - .pack_files - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - cache - .write_generation - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - Ok(()) - }) - .await? - } - async fn write_artifact( - self: &Arc, - relative: String, - mut reader: LargeBlobRead, - ) -> Result<(), CacheError> { - let destination = self.git_dir().join(relative); - // Retry a canceled pair without downloading/charging a second full pack. - // File names alone (possibly SHA-1) are insufficient: verify all hashes. - let expected = reader.reference(); - let existing = destination.clone(); - if tokio::task::spawn_blocking(move || -> io::Result { - use sha2::Digest; - use std::io::Read; - let mut file = match File::open(existing) { - Ok(file) => file, - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), - Err(error) => return Err(error), - }; - if file.metadata()?.len() != expected.size { - return Err(io::Error::other("cached artifact size mismatch")); - } - let mut oid = crate::git_format::ObjectHasher::new( - expected.oid.format(), - ObjectKind::Blob, - expected.size, - ); - let mut sha = sha2::Sha256::new(); - let mut blake = blake3::Hasher::new(); - let mut buffer = vec![0; 8 << 20]; - loop { - let n = file.read(&mut buffer)?; - if n == 0 { - break; - } - oid.update(&buffer[..n]); - sha.update(&buffer[..n]); - blake.update(&buffer[..n]); - } - if oid.finalize() != expected.oid - || <[u8; 32]>::from(sha.finalize()) != expected.sha256 - || *blake.finalize().as_bytes() != expected.blake3 - { - return Err(io::Error::other("cached artifact digest mismatch")); - } - Ok(true) - }) - .await?? - { - return Ok(()); - } - let (file, temporary) = tempfile::NamedTempFile::new_in( - destination - .parent() - .ok_or_else(|| io::Error::other("missing pack directory"))?, - )? - .into_parts(); - let mut writer = CacheWriter { - file, - cache: Arc::clone(self), - }; - while let Some(bytes) = reader.next().await? { - writer = tokio::task::spawn_blocking(move || { - writer.write_all(&bytes)?; - Ok::<_, io::Error>(writer) - }) - .await??; - } - tokio::task::spawn_blocking(move || { - writer.flush()?; - writer.file.sync_all()?; - drop(writer); - if destination.exists() { - // Compare complete bytes, not merely Git's SHA-1 filename. - use std::io::Read; - let mut old = File::open(&destination)?; - let mut new = File::open(&temporary)?; - let mut a = vec![0; 1 << 20]; - let mut b = vec![0; 1 << 20]; - loop { - let n = old.read(&mut a)?; - let m = new.read(&mut b)?; - if n != m || a[..n] != b[..m] { - return Err(io::Error::other( - "cached pack differs from authoritative bytes", - )); - } - if n == 0 { - break; - } - } - return Ok(()); - } - temporary - .persist_noclobber(destination) - .map_err(|error| error.error)?; - Ok(()) - }) - .await??; - Ok(()) - } - /// Reuse only packs whose *every* object was verified and durably recorded - /// during this ingestion. Extra/unverified objects disable this optimization. #[cfg(test)] pub(crate) async fn retain_verified_packs( self: &Arc, @@ -306,9 +153,6 @@ impl GitCache { cache .write_generation .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - cache - .pack_files - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); cache.register_index( &cache.git_dir().join("objects/pack").join( path.file_name() @@ -354,11 +198,6 @@ impl GitCache { next.reservation()? .try_grow(reserve) .map_err(io::Error::other)?; - let durable = self - .durable_packs - .read() - .map_err(|_| io::Error::other("durable inventory poisoned"))? - .clone(); let maintenance = self .native .for_class(crate::native_resources::NativeClass::Maintenance); @@ -502,12 +341,6 @@ impl GitCache { .and_then(|v| v); next.reconcile().await?; result.map_err(worker_error)?; - next.pack_files - .store(1, std::sync::atomic::Ordering::Relaxed); - *next - .durable_packs - .write() - .map_err(|_| io::Error::other("durable inventory poisoned"))? = durable; Ok(next) } } diff --git a/crates/canopy-server/src/git_cache/mod.rs b/crates/canopy-server/src/git_cache/mod.rs index d4badbbe..1018bea4 100644 --- a/crates/canopy-server/src/git_cache/mod.rs +++ b/crates/canopy-server/src/git_cache/mod.rs @@ -1,7 +1,6 @@ //! Disposable Git files charged to the node's shared disk budget. use std::{ - collections::HashSet, fs::{self, File, OpenOptions}, io::{self, BufWriter, Write}, path::{Path, PathBuf}, @@ -14,14 +13,15 @@ use flate2::{Compression, write::ZlibEncoder}; use std::collections::BTreeMap; use tokio::sync::Mutex; -use crate::{ - blob::{LargeBlobError, LargeBlobRead}, - refs::valid_ref_name, -}; +use crate::refs::valid_ref_name; +#[cfg(test)] +use crate::ObjectKind; +use crate::RefExpectation; #[cfg(test)] use crate::object_id; -use crate::{ObjectKind, RefExpectation}; +#[cfg(test)] +use std::collections::HashSet; pub(crate) const CACHE_PREFIX: &str = "canopy-git-"; @@ -38,8 +38,6 @@ pub enum CacheError { Io(#[from] io::Error), #[error("Git cache disk admission failed")] Budget(#[from] LtxError), - #[error("Git blob hydration failed")] - Blob(#[from] LargeBlobError), #[error("Git cache worker failed")] Task(#[from] tokio::task::JoinError), } @@ -74,11 +72,9 @@ pub(crate) struct GitCache { // fetches share a completed loose object without serializing all objects. object_writes: OnceLock<[Arc>; 64]>, packed: RwLock>, - durable_packs: RwLock>, pub(crate) selection: Mutex<()>, #[cfg(test)] pub(crate) loose_objects: std::sync::atomic::AtomicU64, - pub(crate) pack_files: std::sync::atomic::AtomicU64, pub(crate) write_generation: std::sync::atomic::AtomicU64, } @@ -142,11 +138,9 @@ impl GitCache { objects, object_writes: OnceLock::new(), packed: RwLock::new(Vec::new()), - durable_packs: RwLock::new(HashSet::new()), selection: Mutex::new(()), #[cfg(test)] loose_objects: std::sync::atomic::AtomicU64::new(0), - pack_files: std::sync::atomic::AtomicU64::new(0), write_generation: std::sync::atomic::AtomicU64::new(0), }); for directory in ["objects/info", "objects/pack", "refs/heads", "refs/tags", "hooks"] { @@ -354,71 +348,6 @@ impl GitCache { .await? } - #[cfg(test)] - pub(crate) async fn store_blob( - self: &Arc, - reader: LargeBlobRead, - ) -> Result<(), CacheError> { - self.store_blob_reader(BlobReader::External(reader)).await - } - #[cfg(test)] - pub(crate) async fn store_native_blob( - self: &Arc, - reader: crate::pack_store::NativePackedRead, - ) -> Result<(), CacheError> { - self.store_blob_reader(BlobReader::Packed(reader)).await - } - #[cfg(test)] - async fn store_blob_reader(self: &Arc, mut reader: BlobReader) -> Result<(), CacheError> { - let (oid, size) = reader.metadata(); - let write = self.object_write_lock(oid).lock_owned().await; - let cache = Arc::clone(self); - let pending = tokio::task::spawn_blocking(move || { - if oid.format() != cache.object_format { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Git object format mismatch", - ) - .into()); - } - if cache.object_present(oid)? { - return Ok::<_, CacheError>(None); - } - let (mut encoder, temporary, destination) = cache.object_writer(oid)?; - encoder.write_all(format!("blob {}\0", size).as_bytes())?; - Ok::<_, CacheError>(Some((encoder, temporary, destination))) - }) - .await??; - let Some((mut encoder, temporary, destination)) = pending else { - return Ok(()); - }; - while let Some(bytes) = reader.next().await? { - // The writer owns the cache reservation until each compression worker - // exits, including when its async waiter is canceled. - encoder = tokio::task::spawn_blocking(move || { - encoder.write_all(&bytes)?; - Ok::<_, CacheError>(encoder) - }) - .await??; - } - let cache = Arc::clone(self); - tokio::task::spawn_blocking(move || { - let _write = write; - drop(encoder.finish()?); - temporary - .persist_noclobber(destination) - .map_err(|error| error.error)?; - cache - .loose_objects - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - cache - .write_generation - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - Ok::<_, CacheError>(()) - }) - .await? - } - #[cfg(test)] pub(crate) async fn store_refs( self: &Arc, @@ -495,6 +424,7 @@ impl GitCache { )?) } + #[cfg(test)] fn register_checked_index( &self, index: crate::git_format::pack_index::PackIndex, @@ -605,27 +535,3 @@ fn tree_bytes(path: &Path) -> io::Result { mod tests; mod maintenance; - -#[cfg(test)] -enum BlobReader { - External(LargeBlobRead), - Packed(crate::pack_store::NativePackedRead), -} -#[cfg(test)] -impl BlobReader { - fn metadata(&self) -> (crate::ObjectId, u64) { - match self { - Self::External(read) => (read.reference().oid, read.reference().size), - Self::Packed(read) => (read.oid, read.size), - } - } - async fn next(&mut self) -> Result, CacheError> { - match self { - Self::External(read) => Ok(read.next().await?), - Self::Packed(read) => read - .next() - .await - .map_err(|error| io::Error::other(error).into()), - } - } -} diff --git a/crates/canopy-server/src/git_cache/tests.rs b/crates/canopy-server/src/git_cache/tests.rs index 47118eb4..ba3a0b44 100644 --- a/crates/canopy-server/src/git_cache/tests.rs +++ b/crates/canopy-server/src/git_cache/tests.rs @@ -39,6 +39,11 @@ async fn streamed_native_extraction_publishes_only_a_verified_complete_object() source .store_object(expected.oid, expected.kind, body.clone()) .await?; + let foreign_body = b"an unrelated object must remain private"; + let foreign = object_id(format, ObjectKind::Blob, foreign_body); + source + .store_object(foreign, ObjectKind::Blob, foreign_body.to_vec()) + .await?; let mut wrong = expected; wrong.digest[0] ^= 1; assert!( @@ -52,6 +57,7 @@ async fn streamed_native_extraction_publishes_only_a_verified_complete_object() .copy_native_owned(source.git_dir(), expected, source.clone()) .await?; assert!(target.object_present(expected.oid)?); + assert!(!target.object_present(foreign)?); let mut objects = GitObjects::batch_owned(&target.git_dir(), &native, target.clone())?; assert_eq!(objects.read_verified(expected, body.len()).await?, body); objects.finish().await?; @@ -533,213 +539,3 @@ async fn failed_cleanup_does_not_release_disk_admission() -> Result<(), Box Result<(), Box> { - let root = tempfile::TempDir::new()?; - let budget = DiskBudget::new(1 << 20); - let cache = GitCache::create( - root.path().into(), - budget.clone(), - "refs/heads/main", - crate::ObjectFormat::Sha1, - crate::native_resources::NativeResources::default() - .scope(crate::native_resources::NativeClass::Foreground), - ) - .await?; - let blobs = - crate::blob::LargeBlobStore::new(Arc::new(object_store::memory::InMemory::new()), [7; 16]); - let body = vec![17; 9 * 1024 * 1024]; - let reference = blobs - .put( - object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, &body), - body.len() as u64, - &mut body.as_slice(), - ) - .await?; - cache.store_blob(blobs.read(&reference).await?).await?; - let output = tokio::process::Command::new("git") - .arg("--git-dir") - .arg(cache.git_dir()) - .args(["cat-file", "blob", &hex::encode(reference.oid)]) - .output() - .await?; - assert!(output.status.success()); - assert_eq!(output.stdout, body); - assert_eq!(budget.used(), tree_bytes(cache.root())?); - drop(cache); - assert_eq!(budget.used(), 0); - - let cache = GitCache::create( - root.path().into(), - budget.clone(), - "refs/heads/main", - crate::ObjectFormat::Sha1, - crate::native_resources::NativeResources::default() - .scope(crate::native_resources::NativeClass::Foreground), - ) - .await?; - let occupied = budget.try_reserve(budget.capacity() - budget.used())?; - assert!( - cache - .store_blob(blobs.read(&reference).await?) - .await - .is_err_and(|error| error.is_admission()) - ); - drop(cache); - assert_eq!(budget.used(), occupied.bytes()); - Ok(()) -} - -#[tokio::test] -async fn corrupt_stream_never_installs_a_reusable_object() -> Result<(), Box> -{ - use object_store::ObjectStoreExt; - let root = tempfile::TempDir::new()?; - let budget = DiskBudget::new(1 << 20); - let cache = GitCache::create( - root.path().into(), - budget.clone(), - "refs/heads/main", - crate::ObjectFormat::Sha1, - crate::native_resources::NativeResources::default() - .scope(crate::native_resources::NativeClass::Foreground), - ) - .await?; - let store = Arc::new(object_store::memory::InMemory::new()); - let blobs = crate::blob::LargeBlobStore::new(store.clone(), [9; 16]); - let mut body = vec![23; 9 * 1024 * 1024]; - let reference = blobs - .put( - object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, &body), - body.len() as u64, - &mut body.as_slice(), - ) - .await?; - // The first range can be written before the final range detects corruption. - // Only the test-owned blob key is changed, then restored before retry. - let path = crate::external::part(&crate::blob::blob_path([9; 16], &reference.sha256), 1); - let metadata = store.head(&path).await?; - let last = body.last_mut().ok_or("empty fixture")?; - *last ^= 1; - store - .put(&metadata.location, body[8 * 1024 * 1024..].to_vec().into()) - .await?; - assert!( - cache - .store_blob(blobs.read(&reference).await?) - .await - .is_err() - ); - assert_eq!( - cache.missing_objects(vec![reference.oid]).await?, - vec![reference.oid] - ); - *body.last_mut().ok_or("empty fixture")? ^= 1; - store - .put(&metadata.location, body[8 * 1024 * 1024..].to_vec().into()) - .await?; - cache.store_blob(blobs.read(&reference).await?).await?; - assert!(cache.missing_objects(vec![reference.oid]).await?.is_empty()); - drop(cache); - assert_eq!(budget.used(), 0); - Ok(()) -} - -#[tokio::test] -async fn incomplete_pack_extracts_verified_large_blobs_without_admitting_foreign_members() --> Result<(), Box> { - let root = tempfile::TempDir::new()?; - let budget = DiskBudget::new(256 << 20); - let source = GitCache::create( - root.path().into(), - budget.clone(), - "refs/heads/main", - crate::ObjectFormat::Sha1, - crate::native_resources::NativeResources::default() - .scope(crate::native_resources::NativeClass::Foreground), - ) - .await?; - let body = vec![b'x'; 2 << 20]; - let oid = object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, &body); - let digest = *blake3::hash(&body).as_bytes(); - source - .store_object(oid, ObjectKind::Blob, body.clone()) - .await?; - let foreign = object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, b"foreign"); - source - .store_object(foreign, ObjectKind::Blob, b"foreign".to_vec()) - .await?; - let packed = source.repacked(root.path().into(), budget.clone()).await?; - let (hash, pack, index, ids) = packed.pack_sources().await?.pop().unwrap(); - assert_eq!( - ids.iter().copied().collect::>(), - [oid, foreign].into_iter().collect() - ); - let store = Arc::new(object_store::memory::InMemory::new()); - let reader = crate::pack_store::PackReader::new( - store, - [4; 16], - root.path().into(), - budget.clone(), - crate::ObjectFormat::Sha1, - source.native.clone(), - ); - let record = crate::pack_store::PackRecord { - hash, - pack: reader.upload(pack).await?, - index: reader.upload(index).await?, - approved: false, - }; - let target = GitCache::create( - root.path().into(), - budget.clone(), - "refs/heads/main", - crate::ObjectFormat::Sha1, - crate::native_resources::NativeResources::default() - .scope(crate::native_resources::NativeClass::Foreground), - ) - .await?; - target - .store_native_blob( - reader - .native_reader(record.clone(), oid, body.len() as u64, digest) - .await?, - ) - .await?; - assert!(target.missing_objects(vec![oid]).await?.is_empty()); - assert_eq!(target.missing_objects(vec![foreign]).await?, vec![foreign]); - let output = crate::native_git::command(&target.git_dir())? - .args(["cat-file", "blob", &hex::encode(oid)]) - .output() - .await?; - assert!(output.status.success()); - assert_eq!(output.stdout, body); - assert!( - reader - .cache() - .await? - .missing_objects(vec![oid, foreign]) - .await? - .len() - == 2 - ); - let invalid = GitCache::create( - root.path().into(), - budget.clone(), - "refs/heads/main", - crate::ObjectFormat::Sha1, - crate::native_resources::NativeResources::default() - .scope(crate::native_resources::NativeClass::Foreground), - ) - .await?; - assert!( - invalid - .store_native_blob(reader.native_reader(record, oid, 2 << 20, [0; 32]).await?) - .await - .is_err() - ); - assert_eq!(invalid.missing_objects(vec![oid]).await?, vec![oid]); - Ok(()) -} diff --git a/crates/canopy-server/src/git_gateway/mod.rs b/crates/canopy-server/src/git_gateway/mod.rs index 5069b8ac..34d0e4e2 100644 --- a/crates/canopy-server/src/git_gateway/mod.rs +++ b/crates/canopy-server/src/git_gateway/mod.rs @@ -20,7 +20,6 @@ use tokio::sync::{Mutex, OnceCell}; use crate::{ PushPlan, RefExpectation, RefUpdate, RepositoryCell, - blob::LargeBlobError, directory::TokenScope, git_cache::CacheError, git_http::{GitHttpBackend, GitHttpError, GitHttpRequest, GitHttpResponse}, @@ -53,8 +52,6 @@ pub enum GatewayError { Input(#[from] InputError), #[error("repository Cell operation failed")] Cell(#[source] CellError), - #[error("large Git blob store failed")] - Blob(#[from] LargeBlobError), #[error("cache I/O failed")] Io(#[from] std::io::Error), #[error("Git cache command failed: {0}")] @@ -88,7 +85,6 @@ pub struct GitGateway { repository: Arc, signer_directory: Option>, certificate_seed: Arc>, - _pack_reader: Arc, lfs: Arc, artifacts: Arc, scratch_root: PathBuf, @@ -110,30 +106,11 @@ impl GitGateway { Arc::clone(&blob_store), repository.repository_id(), )); - // A reader belongs to this gateway's workspace and disk admission. - // Another gateway may use a different root/budget for the same Cell. - let pack_reader = Arc::new(crate::pack_store::PackReader::new( - Arc::clone(&blob_store), - repository.repository_id(), - scratch_root.clone(), - disk_budget.clone(), - repository.object_format(), - native.clone(), - )); - { - let mut readers = repository - .pack_readers - .lock() - .expect("packed reader registry poisoned"); - readers.retain(|reader| reader.strong_count() > 0); - readers.push(Arc::downgrade(&pack_reader)); - } let lfs = Arc::new(LfsService::new(Arc::clone(&repository), blob_store)); Self { repository, signer_directory: None, certificate_seed: Arc::new(OnceCell::new()), - _pack_reader: pack_reader, lfs, artifacts, scratch_root, diff --git a/crates/canopy-server/src/git_objects/tests.rs b/crates/canopy-server/src/git_objects/tests.rs index 52b85028..e1ea1515 100644 --- a/crates/canopy-server/src/git_objects/tests.rs +++ b/crates/canopy-server/src/git_objects/tests.rs @@ -225,21 +225,17 @@ async fn batch_reads_large_blob_across_pipe_buffers() -> TestResult { )?; assert_eq!(objects.next().await?, Some(oid)); let mut object = objects.read(oid).await?; - let store = crate::blob::LargeBlobStore::new( - std::sync::Arc::new(object_store::memory::InMemory::new()), - [1; 16], - ); - let reference = store.put(oid, object.size, &mut object.reader).await?; + let mut buffer = vec![0; 64 << 10]; + let mut offset = 0; + while offset < body.len() { + let length = buffer.len().min(body.len() - offset); + object.reader.read_exact(&mut buffer[..length]).await?; + assert_eq!(&buffer[..length], &body[offset..offset + length]); + offset += length; + } object.finish().await?; assert_eq!(objects.next().await?, None); objects.finish().await?; - let mut reader = store.read(&reference).await?; - let mut offset = 0; - while let Some(bytes) = reader.next().await? { - assert_eq!(bytes.as_ref(), &body[offset..offset + bytes.len()]); - offset += bytes.len(); - } - assert_eq!(offset, body.len()); struct BlobSink; impl EdgeSink for BlobSink { async fn append( diff --git a/crates/canopy-server/src/graph/mod.rs b/crates/canopy-server/src/graph/mod.rs index be80433e..56347f0d 100644 --- a/crates/canopy-server/src/graph/mod.rs +++ b/crates/canopy-server/src/graph/mod.rs @@ -1,279 +1,13 @@ -//! Bounded object certificates and constant-per-ref graph checks at publication. +//! Typed structural Git parsing for native catalog verification. +use crate::ObjectKind; -use std::collections::BTreeMap; - -use crate::{ObjectKind, PushPlan, RepositoryModule, object_batch::VERIFY_BATCH_BYTES, object_id}; -use cellule_runtime::{ - CellModule, Command, Error, codec::BoundedDecoder, codec::BoundedEncoder, codec::CodecError, - codec::WireValue, primitives::sql::SqlBatch, primitives::sql::SqlResultSet, - primitives::sql::SqlStatement, primitives::sql::SqlValue, registry::CommandContext, - registry::CommandResult, -}; - -mod preparation; pub(crate) mod stream; - +#[cfg(test)] type Oid = crate::ObjectId; +#[cfg(test)] type Edge = (Oid, Option); -const MAX_CERTIFICATES: usize = 512; - -#[derive(Default)] -pub(crate) struct CertificateBatch(Vec); - -impl WireValue for CertificateBatch { - fn encode(&self, encoder: &mut BoundedEncoder) -> Result<(), CodecError> { - if !(1..=MAX_CERTIFICATES).contains(&self.0.len()) { - return Err(CodecError::Invalid("certificate batch count")); - } - encoder.write_count(self.0.len())?; - for oid in &self.0 { - encoder.write_bytes(oid)?; - } - Ok(()) - } - fn decode(decoder: &mut BoundedDecoder<'_>) -> Result { - let count = decoder.read_count()?; - if !(1..=MAX_CERTIFICATES).contains(&count) { - return Err(CodecError::Invalid("certificate batch count")); - } - let mut batch = Self(Vec::with_capacity(count)); - for _ in 0..count { - batch.0.push( - decoder - .read_bytes()? - .try_into() - .map_err(|_| CodecError::Invalid("certificate OID width"))?, - ); - } - Ok(batch) - } -} - -#[derive(Clone)] -struct Status { - kind: ObjectKind, - bytes: u64, - certified: bool, -} - -fn status_query(oids: &[Oid]) -> SqlBatch { - let placeholders = vec!["?"; oids.len()].join(","); - SqlBatch { - statements: vec![SqlStatement { - sql: format!( - "SELECT o.oid, o.kind, CASE WHEN o.storage IN ('external', 'packed') THEN 0 ELSE o.size END, c.oid FROM objects o LEFT JOIN object_closure c ON c.oid = o.oid WHERE o.oid IN ({placeholders})" - ), - parameters: oids - .iter() - .map(|oid| SqlValue::Blob(oid.to_vec())) - .collect(), - }], - } -} - -fn statuses(results: &[SqlResultSet]) -> cellule_runtime::Result> { - let mut states = BTreeMap::new(); - for row in &results - .first() - .ok_or(Error::Command("missing graph result"))? - .rows - { - let [ - SqlValue::Blob(oid), - SqlValue::Text(kind), - SqlValue::Integer(bytes), - certificate, - ] = row.as_slice() - else { - return Err(Error::Command("invalid graph status")); - }; - let kind = parse_kind(kind.as_bytes()).ok_or(Error::Command("invalid graph kind"))?; - let bytes = u64::try_from(*bytes).map_err(|_| Error::Command("invalid graph size"))?; - states.insert( - oid.as_slice() - .try_into() - .map_err(|_| Error::Command("invalid graph OID"))?, - Status { - kind, - bytes, - certified: matches!(certificate, SqlValue::Blob(_)), - }, - ); - } - Ok(states) -} - -pub(crate) struct CertifyObjects; -impl Command for CertifyObjects { - const MODULE: &'static str = RepositoryModule::NAME; - const ID: u32 = 6; - const CODEC_VERSION: u32 = 3; - type Input = CertificateBatch; - type Output = bool; - - fn execute( - context: &mut CommandContext<'_, '_>, - batch: CertificateBatch, - ) -> cellule_runtime::Result> { - let mut verified = 0; - for oid in batch.0 { - let states = statuses(&context.sql(&status_query(&[oid]))?)?; - let Some(state) = states.get(&oid) else { - return Ok(CommandResult::Rejected(false)); - }; - if state.certified { - continue; - } - if verified != 0 && state.bytes > VERIFY_BATCH_BYTES.saturating_sub(verified) { - return Ok(CommandResult::Rejected(false)); - } - verified += state.bytes; - let Some(edges) = object_edges(context, oid, state.kind)? else { - return Ok(CommandResult::Rejected(false)); - }; - for edges in edges.chunks(MAX_CERTIFICATES) { - let oids: Vec<_> = edges.iter().map(|(oid, _)| *oid).collect(); - let states = statuses(&context.sql(&status_query(&oids))?)?; - if edges.iter().any(|(oid, kind)| { - !states.get(oid).is_some_and(|state| { - state.certified && kind.is_none_or(|kind| kind == state.kind) - }) - }) { - return Ok(CommandResult::Rejected(false)); - } - } - // Only certified, typed edges enter the reachability index. Keeping - // this with closure publication prevents lazy fetch from trusting - // staged or malformed object graphs. - for edges in edges.chunks(crate::object_batch::MAX_OBJECTS) { - context.sql(&SqlBatch { - statements: edges - .iter() - .map(|(child, _)| SqlStatement { - sql: "INSERT INTO object_edges (parent, child) VALUES (?1, ?2)".into(), - parameters: vec![ - SqlValue::Blob(oid.to_vec()), - SqlValue::Blob(child.to_vec()), - ], - }) - .collect(), - })?; - } - if state.kind == ObjectKind::Commit { - let parents: Vec<_> = edges - .iter() - .filter(|(_, kind)| *kind == Some(ObjectKind::Commit)) - .map(|(parent, _)| *parent) - .collect(); - for parents in parents.chunks(crate::object_batch::MAX_OBJECTS) { - context.sql(&SqlBatch { statements: parents.iter().map(|parent| SqlStatement { - sql: "INSERT INTO commit_parents (child, parent) VALUES (?1, ?2) ON CONFLICT DO NOTHING".into(), - parameters: vec![SqlValue::Blob(oid.to_vec()), SqlValue::Blob(parent.to_vec())], - }).collect() })?; - } - } - // These proofs depend only on immutable objects. A future collector must - // invalidate certificates before removing any reachable object or chunk. - context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "INSERT INTO object_closure (oid) VALUES (?1)".into(), - parameters: vec![SqlValue::Blob(oid.to_vec())], - }], - })?; - } - Ok(CommandResult::Success(true)) - } -} - -pub(crate) fn certified_roots( - context: &CommandContext<'_, '_>, - plan: &PushPlan, -) -> cellule_runtime::Result { - for updates in plan.updates.chunks(MAX_CERTIFICATES) { - let oids: Vec<_> = updates.iter().filter_map(|update| update.new_oid).collect(); - if oids.is_empty() { - continue; - } - let states = statuses(&context.sql(&status_query(&oids))?)?; - if !updates.iter().all(|update| { - update.new_oid.is_none_or(|oid| { - states.get(&oid).is_some_and(|state| { - state.certified - && (!update.name.starts_with("refs/heads/") - || state.kind == ObjectKind::Commit) - }) - }) - }) { - return Ok(false); - } - } - Ok(true) -} - -fn object_edges( - context: &CommandContext<'_, '_>, - oid: Oid, - kind: ObjectKind, -) -> cellule_runtime::Result>> { - let result = context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT storage, body, digest, size, chunk_id FROM objects WHERE oid = ?1".into(), - parameters: vec![SqlValue::Blob(oid.to_vec())], - }], - })?; - let Some( - [ - SqlValue::Text(storage), - body, - SqlValue::Blob(digest), - SqlValue::Integer(size), - upload, - ], - ) = result - .first() - .and_then(|set| set.rows.first()) - .map(Vec::as_slice) - else { - return Err(Error::Command("invalid stored graph object")); - }; - let edges = match (storage.as_str(), body, upload) { - ("inline", SqlValue::Blob(body), SqlValue::Null) => { - if object_id(oid.format(), kind, body) != oid - || blake3::hash(body).as_bytes() != digest.as_slice() - { - return Err(Error::Command("corrupt stored graph object")); - } - let Some(edges) = edges(oid.format(), kind, body) else { - return Ok(None); - }; - edges - } - // The immutable upload is verified before its SQLite record is published. - // Blobs have no outgoing Git edges; no network I/O belongs in this transaction. - ("external" | "packed", SqlValue::Null, SqlValue::Null) if kind == ObjectKind::Blob => { - Vec::new() - } - ("chunked", SqlValue::Null, SqlValue::Blob(upload)) => { - let invalid = || Error::Command("invalid stored graph object chunks"); - let body = crate::object_chunks::body( - context, - oid, - kind, - upload.as_slice().try_into().map_err(|_| invalid())?, - u64::try_from(*size).map_err(|_| invalid())?, - digest.as_slice().try_into().map_err(|_| invalid())?, - )? - .ok_or_else(invalid)?; - let Some(edges) = edges(oid.format(), kind, &body) else { - return Ok(None); - }; - edges - } - _ => return Err(Error::Command("invalid stored graph object")), - }; - Ok(Some(edges)) -} +#[cfg(test)] pub(crate) fn edges( format: crate::ObjectFormat, kind: ObjectKind, @@ -304,6 +38,7 @@ pub(crate) fn edges( Some(edges) } +#[cfg(test)] pub(crate) fn tag_edge(mut body: &[u8]) -> Option { let target = hex_oid(line(&mut body)?.strip_prefix(b"object ")?)?; let kind = parse_kind(line(&mut body)?.strip_prefix(b"type ")?)?; @@ -311,6 +46,7 @@ pub(crate) fn tag_edge(mut body: &[u8]) -> Option { Some((target, Some(kind))) } +#[cfg(test)] fn line<'a>(body: &mut &'a [u8]) -> Option<&'a [u8]> { let newline = body.iter().position(|byte| *byte == b'\n')?; let line = &body[..newline]; @@ -318,6 +54,7 @@ fn line<'a>(body: &mut &'a [u8]) -> Option<&'a [u8]> { Some(line) } +#[cfg(test)] fn tree_edges(format: crate::ObjectFormat, mut body: &[u8]) -> Option> { let mut edges = Vec::new(); while !body.is_empty() { @@ -363,6 +100,7 @@ fn tree_edges(format: crate::ObjectFormat, mut body: &[u8]) -> Option> Some(edges) } +#[cfg(test)] fn hex_oid(text: &[u8]) -> Option { let oid = Oid::from_hex(text).ok()?; (!oid.is_zero()).then_some(oid) diff --git a/crates/canopy-server/src/graph/preparation.rs b/crates/canopy-server/src/graph/preparation.rs deleted file mode 100644 index 0d1d02d0..00000000 --- a/crates/canopy-server/src/graph/preparation.rs +++ /dev/null @@ -1,232 +0,0 @@ -use super::*; -use crate::{ - RepositoryCell, - directory::{TokenScope, validate_component}, -}; -use cellule_runtime::{InvocationError, MutationIdentity, identity::RequestId}; -use std::{ - collections::{BTreeSet, HashMap, HashSet}, - time::{SystemTime, UNIX_EPOCH}, -}; - -#[derive(Debug, thiserror::Error)] -enum PrepareError { - #[error("graph query failed")] - Query(#[from] InvocationError>), - #[error("graph certificate publication failed")] - Certificate(#[from] InvocationError), - #[error("graph data is invalid")] - Invalid(#[from] Error), - #[error("graph worker failed")] - Worker(#[from] tokio::task::JoinError), - #[error("graph clock failed")] - Clock(#[from] std::time::SystemTimeError), - #[error("graph clock is outside the supported range")] - ClockRange(#[from] std::num::TryFromIntError), -} - -enum Visit { - Enter(Edge), - Leave(Oid, ObjectKind, u64), -} - -impl RepositoryCell { - pub(crate) async fn prepare_graph(&self, plan: &PushPlan) -> Result<(), InvocationError> { - self.prepare_graph_inner(plan).await.map_err(|error| { - InvocationError::NotStarted(Error::Facility { - name: "Git graph preparation", - source: Box::new(error), - }) - }) - } - - async fn prepare_graph_inner(&self, plan: &PushPlan) -> Result<(), PrepareError> { - // The authoritative ACL/version checks remain in finalization. Avoid doing - // graph work for invalid or unauthorized plans before that transaction. - if plan.updates.is_empty() - || plan.updates.len() > crate::refs::MAX_UPDATES - || validate_component(&plan.actor).is_err() - { - return Ok(()); - } - if !self - .access_level(&plan.actor, None) - .await? - .output - .is_some_and(|level| level >= TokenScope::Write) - { - return Ok(()); - } - let started = std::time::Instant::now(); - // One preparer per process bounds aggregate memory while retaining no - // arbitrary repository/object-count ceiling. Durable certificates still - // recheck typed dependencies inside each bounded Cell transaction. - static PREPARERS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); - let _preparer = PREPARERS - .acquire() - .await - .map_err(|_| Error::Command("graph admission closed"))?; - let mut nodes = HashMap::new(); - let mut ready = HashMap::new(); - let mut discovered = HashSet::new(); - let mut frontier: BTreeSet<_> = plan - .updates - .iter() - .filter_map(|update| update.new_oid) - .collect(); - while !frontier.is_empty() { - let page: Vec<_> = frontier.iter().take(MAX_CERTIFICATES).copied().collect(); - for oid in &page { - frontier.remove(oid); - discovered.insert(*oid); - } - let result = self.sql.query(None, status_query(&page)).await?; - let states = statuses(&result.output)?; - let mut structure = BTreeSet::new(); - for oid in &page { - let Some(state) = states.get(oid) else { - return Ok(()); - }; - if state.certified { - ready.insert(*oid, state.kind); - continue; - } - nodes.insert(*oid, (state.kind, state.bytes, Some(Vec::new()))); - if state.kind != ObjectKind::Blob { - structure.insert(*oid); - } - } - while !structure.is_empty() { - let ids: Vec<_> = structure - .iter() - .take(crate::object_batch::MAX_OBJECTS) - .copied() - .collect(); - let records = self.selected_objects(&ids).await?; - if records.is_empty() { - return Err(Error::Command("empty graph body page").into()); - } - for record in records { - structure.remove(&record.oid); - let oid = record.oid; - let kind = record.kind; - let body = match record.storage { - crate::ObjectStorage::Inline(body) => body, - crate::ObjectStorage::Chunked { .. } => { - self.object(oid, None) - .await? - .output - .ok_or(Error::Command("missing graph body"))? - .1 - } - _ => return Err(Error::Command("invalid structural storage").into()), - }; - let parsed = - tokio::task::spawn_blocking(move || edges(oid.format(), kind, &body)) - .await?; - if let Some(edges) = &parsed { - frontier.extend( - edges - .iter() - .map(|(oid, _)| *oid) - .filter(|oid| !discovered.contains(oid)), - ); - } - nodes - .get_mut(&oid) - .ok_or(Error::Command("missing graph node"))? - .2 = parsed; - } - } - } - drop(discovered); - let loaded = nodes.len(); - let mut pending: Vec<_> = plan - .updates - .iter() - .filter_map(|update| { - update.new_oid.map(|oid| { - Visit::Enter(( - oid, - update - .name - .starts_with("refs/heads/") - .then_some(ObjectKind::Commit), - )) - }) - }) - .collect(); - let mut visiting = HashSet::new(); - let mut batch = CertificateBatch::default(); - let mut bytes = 0; - while let Some(visit) = pending.pop() { - match visit { - Visit::Enter((oid, expected)) => { - if let Some(kind) = ready.get(&oid) { - if expected.is_some_and(|expected| expected != *kind) { - return Ok(()); - } - continue; - } - // Dependencies outside the captured uncertified inventory - // must already be certified. The command checks this, and - // final publication always checks the roots again. - let Some((kind, weight, children)) = nodes.remove(&oid) else { - if visiting.contains(&oid) { - return Ok(()); - } - continue; - }; - if expected.is_some_and(|expected| expected != kind) || !visiting.insert(oid) { - return Ok(()); - } - let Some(children) = children else { - return Ok(()); - }; - pending.push(Visit::Leave(oid, kind, weight)); - for edge in children.into_iter().rev() { - pending.push(Visit::Enter(edge)); - } - } - Visit::Leave(oid, kind, weight) => { - if !batch.0.is_empty() && weight > VERIFY_BATCH_BYTES.saturating_sub(bytes) { - self.certify_batch(std::mem::take(&mut batch)).await?; - bytes = 0; - } - batch.0.push(oid); - bytes += weight; - ready.insert(oid, kind); - visiting.remove(&oid); - if batch.0.len() == MAX_CERTIFICATES || bytes >= VERIFY_BATCH_BYTES { - self.certify_batch(std::mem::take(&mut batch)).await?; - bytes = 0; - } - } - } - } - if !batch.0.is_empty() { - self.certify_batch(batch).await?; - } - tracing::info!( - objects = loaded, - elapsed_seconds = started.elapsed().as_secs_f64(), - "prepared Git graph certificates" - ); - Ok(()) - } - - async fn certify_batch(&self, batch: CertificateBatch) -> Result<(), PrepareError> { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - let identity = MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now - .checked_add(60_000) - .ok_or(Error::Command("graph clock overflow"))?, - }; - self.application - .command::(&self.target, identity, batch) - .await?; - Ok(()) - } -} diff --git a/crates/canopy-server/src/graph/tests.rs b/crates/canopy-server/src/graph/tests.rs index e4b054bb..4dc948b1 100644 --- a/crates/canopy-server/src/graph/tests.rs +++ b/crates/canopy-server/src/graph/tests.rs @@ -1,29 +1,5 @@ use super::*; -#[test] -fn certificate_codec_rejects_empty_oversized_and_malformed_batches() { - for count in [0, MAX_CERTIFICATES + 1] { - let mut encoder = BoundedEncoder::new(1 << 20).unwrap(); - encoder.write_count(count).unwrap(); - let input = encoder.finish(); - assert!( - CertificateBatch::decode(&mut BoundedDecoder::new(&input, 1 << 20).unwrap()).is_err() - ); - } - let mut encoder = BoundedEncoder::new(1 << 20).unwrap(); - encoder.write_count(1).unwrap(); - encoder.write_bytes(&[0; 19]).unwrap(); - let input = encoder.finish(); - assert!(CertificateBatch::decode(&mut BoundedDecoder::new(&input, 1 << 20).unwrap()).is_err()); - let batch = CertificateBatch(vec![crate::ObjectId::Sha1([1; 20]); MAX_CERTIFICATES]); - let mut encoder = BoundedEncoder::new(1 << 20).unwrap(); - batch.encode(&mut encoder).unwrap(); - let input = encoder.finish(); - let mut decoder = BoundedDecoder::new(&input, 1 << 20).unwrap(); - assert_eq!(CertificateBatch::decode(&mut decoder).unwrap().0, batch.0); - decoder.finish().unwrap(); -} - #[test] fn repeated_edges_share_one_proof_but_different_types_do_not() { let mut body = Vec::new(); diff --git a/crates/canopy-server/src/lib.rs b/crates/canopy-server/src/lib.rs index 3ff336ec..1fb2a8ff 100644 --- a/crates/canopy-server/src/lib.rs +++ b/crates/canopy-server/src/lib.rs @@ -4,11 +4,10 @@ use std::sync::OnceLock; use cellule_app::{ApplicationBuilder, ApplicationHandle, CellApplication, CellType}; use cellule_runtime::{ - ApplicationId, BuildDescriptor, CellModule, CellTarget, Committed, Digest, Error, - MigrationDescriptor, ModuleDescriptor, NamespaceDescriptor, NamespaceId, Observed, Receipt, - RegistryBuilder, SqlCell, SqlModule, TenantId, cell::catalog::CatalogRole, - primitives::sql::SqlBatch, primitives::sql::SqlStatement, primitives::sql::SqlValue, - primitives::sql::register_sql, registry::OperationDescriptor, + ApplicationId, BuildDescriptor, CellModule, CellTarget, Digest, Error, MigrationDescriptor, + ModuleDescriptor, NamespaceDescriptor, NamespaceId, RegistryBuilder, SqlCell, SqlModule, + TenantId, cell::catalog::CatalogRole, primitives::sql::register_sql, + registry::OperationDescriptor, }; mod access; @@ -16,7 +15,6 @@ mod admission; pub use admission::AdmissionPermit; mod visibility; pub use visibility::{RepositoryVisibility, Visibility}; -mod ancestry; pub mod branch_rules; pub mod checks; mod default_branch; @@ -34,22 +32,9 @@ mod git_read; mod graph; pub mod http; pub mod issues; -pub mod blob { - //! Verified, immutable Git blob bodies stored outside the Repository Cell. - - #[cfg(test)] - pub(crate) use canopy_object_storage::blob::blob_path; - pub use canopy_object_storage::blob::{ - LargeBlobError, LargeBlobRead, LargeBlobReference, LargeBlobStore, - }; -} pub mod lfs; mod native_git; pub mod native_resources; -mod object_batch; -mod object_chunks; -mod object_reads; -mod pack_store; pub mod packs; pub mod pulls; mod push; @@ -62,10 +47,8 @@ mod web; pub use access::{COLLABORATOR_PAGE_SIZE, Collaborator, ReadIdentity}; pub use default_branch::DefaultBranch; -pub use object_batch::ObjectBatch; -pub use object_chunks::ObjectStageError; pub use push::{PushCertificateReceipt, PushError, PushReceipt, VerifiedPushCertificate}; -pub use refs::{FinalizePush, PushPlan, RefExpectation, RefPage, RefReadError, RefUpdate}; +pub use refs::{PushPlan, RefExpectation, RefPage, RefUpdate}; pub const REPOSITORIES: NamespaceId = NamespaceId::from_bytes([71; 16]); pub const REPOSITORY_SCHEMA_VERSION: u32 = 4; @@ -128,33 +111,6 @@ pub(crate) fn validate_repository_id(repository: [u8; 16]) -> cellule_runtime::R Ok(()) } -/// Byte location for a verified Git object record. -pub enum ObjectStorage { - Inline(Vec), - Chunked { - upload: [u8; 16], - size: u64, - blake3: [u8; 32], - }, - Packed { - size: u64, - blake3: [u8; 32], - pack: [u8; 32], - }, - External { - size: u64, - blake3: [u8; 32], - sha256: [u8; 32], - }, -} - -/// Immutable Git object returned by one bounded Repository Cell read. -pub struct StoredObject { - pub oid: crate::ObjectId, - pub kind: ObjectKind, - pub storage: ObjectStorage, -} - pub struct RepositoryModule; impl SqlModule for RepositoryModule { @@ -197,14 +153,8 @@ impl CellModule for RepositoryModule { source.update(include_bytes!("refs.rs")); source.update(include_bytes!("default_branch.rs")); source.update(include_bytes!("graph/mod.rs")); - source.update(include_bytes!("ancestry.rs")); source.update(include_bytes!("branch_rules/mod.rs")); source.update(include_bytes!("branch_rules/command.rs")); - source.update(include_bytes!("graph/preparation.rs")); - source.update(include_bytes!("object_batch/mod.rs")); - source.update(include_bytes!("object_chunks/mod.rs")); - source.update(include_bytes!("object_reads/mod.rs")); - source.update(include_bytes!("pack_store.rs")); source.update(include_bytes!("git_objects/mod.rs")); source.update(include_bytes!("git_cache/artifacts.rs")); source.update(include_bytes!("git_cache/verified.rs")); @@ -421,9 +371,6 @@ impl CellModule for RepositoryModule { source.update(include_bytes!( "../../canopy-object-storage/src/artifact.rs" )); - source.update(include_bytes!( - "../../canopy-object-storage/src/blob/mod.rs" - )); source.update(include_bytes!( "../../canopy-object-storage/src/external.rs" )); @@ -517,9 +464,6 @@ pub struct RepositoryCell { sql: SqlCell, application: ApplicationHandle, target: CellTarget, - // Gateways own the cache lifetime. Sharing the reader through a weak - // reference must not retain its original disk budget after gateway eviction. - pack_readers: std::sync::Mutex>>, serving: std::sync::Mutex>>, staging: std::sync::Mutex>>, } @@ -548,7 +492,6 @@ impl RepositoryCell { sql: application.sql::(target.clone())?, application: application.clone(), target, - pack_readers: std::sync::Mutex::new(Vec::new()), serving: std::sync::Mutex::new(None), staging: std::sync::Mutex::new(None), }) @@ -610,176 +553,6 @@ impl RepositoryCell { }; pool.snapshot(actor).await } - - /// Prepares bounded graph certificates, then publishes one all-or-none ref plan. - pub async fn finalize_push( - &self, - identity: cellule_runtime::MutationIdentity, - plan: PushPlan, - ) -> std::result::Result, cellule_runtime::InvocationError> { - // Keep each transport-heavy phase in its own allocation. Embedding all - // three futures multiplies stack copies when debug callers poll a push. - Box::pin(self.prepare_graph(&plan)).await?; - Box::pin(self.prepare_branch_proofs(&plan)).await?; - Box::pin( - self.application - .command::(&self.target, identity, plan), - ) - .await - } - - pub async fn object( - &self, - oid: crate::ObjectId, - minimum: Option, - ) -> std::result::Result< - Observed)>>, - cellule_runtime::InvocationError>, - > { - let result = self - .sql - .query( - minimum, - SqlBatch { - statements: vec![SqlStatement { - sql: - "SELECT kind, body, digest, size, chunk_id, storage, external_sha256 FROM objects WHERE oid = ?1" - .into(), - parameters: vec![SqlValue::Blob(oid.to_vec())], - }], - }, - ) - .await?; - let Some(row) = result.output.first().and_then(|set| set.rows.first()) else { - return Ok(Observed { - output: None, - receipt: result.receipt, - }); - }; - let [ - SqlValue::Text(kind), - body, - SqlValue::Blob(digest), - SqlValue::Integer(size), - upload, - SqlValue::Text(storage), - locator, - ] = row.as_slice() - else { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("invalid stored object row"), - )); - }; - let kind = match kind.as_str() { - "blob" => ObjectKind::Blob, - "tree" => ObjectKind::Tree, - "commit" => ObjectKind::Commit, - "tag" => ObjectKind::Tag, - _ => { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("invalid stored object kind"), - )); - } - }; - if storage == "packed" { - let SqlValue::Blob(pack) = locator else { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("invalid pack locator"), - )); - }; - let pack: [u8; 32] = pack.as_slice().try_into().map_err(|_| { - cellule_runtime::InvocationError::NotStarted(Error::Command("invalid pack locator")) - })?; - if *size < 0 || *size > INLINE_OBJECT_LIMIT as i64 { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("object exceeds bounded body reader"), - )); - } - let record = self.pack_record(pack).await?; - let reader = self - .pack_readers - .lock() - .map_err(|_| { - cellule_runtime::InvocationError::NotStarted(Error::Command( - "packed reader registry poisoned", - )) - })? - .iter() - .find_map(std::sync::Weak::upgrade) - .ok_or(cellule_runtime::InvocationError::NotStarted( - Error::Command("packed reader unavailable"), - ))?; - let body = reader - .read_blob( - record, - oid, - *size as u64, - digest.as_slice().try_into().map_err(|_| { - cellule_runtime::InvocationError::NotStarted(Error::Command( - "invalid packed digest", - )) - })?, - ) - .await - .map_err(|error| { - cellule_runtime::InvocationError::NotStarted(Error::Facility { - name: "packed Git body", - source: Box::new(error), - }) - })?; - if kind != ObjectKind::Blob - || body.len() as i64 != *size - || object_id(oid.format(), kind, &body) != oid - || blake3::hash(&body).as_bytes() != digest.as_slice() - { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("corrupt packed Git body"), - )); - } - return Ok(Observed { - output: Some((kind, body)), - receipt: result.receipt, - }); - } - let body = match (body, upload) { - (SqlValue::Blob(bytes), SqlValue::Null) - if usize::try_from(*size).ok() == Some(bytes.len()) => - { - bytes.clone() - } - (SqlValue::Null, SqlValue::Blob(upload)) => { - let invalid = || { - cellule_runtime::InvocationError::NotStarted(Error::Command( - "invalid object chunk reference", - )) - }; - self.chunked_body( - oid, - kind, - upload.as_slice().try_into().map_err(|_| invalid())?, - u64::try_from(*size).map_err(|_| invalid())?, - digest.as_slice().try_into().map_err(|_| invalid())?, - ) - .await? - } - _ => { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("invalid stored object body"), - )); - } - }; - if object_id(oid.format(), kind, &body) != oid - || blake3::hash(&body).as_bytes() != digest.as_slice() - { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("corrupt stored object"), - )); - } - Ok(Observed { - output: Some((kind, body)), - receipt: result.receipt, - }) - } } /// Build evidence for the statically linked Canopy Cell application. diff --git a/crates/canopy-server/src/object_batch/mod.rs b/crates/canopy-server/src/object_batch/mod.rs deleted file mode 100644 index f698ad30..00000000 --- a/crates/canopy-server/src/object_batch/mod.rs +++ /dev/null @@ -1,401 +0,0 @@ -//! Bounded, atomic publication of verified repository object records. - -use std::collections::BTreeSet; - -use cellule_runtime::{ - CellModule, Command, Committed, Error, InvocationError, MutationIdentity, Observed, - codec::BoundedDecoder, codec::BoundedEncoder, codec::CodecError, codec::WireValue, - primitives::sql::SqlBatch, primitives::sql::SqlResultSet, primitives::sql::SqlStatement, - primitives::sql::SqlValue, registry::CommandContext, registry::CommandResult, -}; - -use crate::{ - INLINE_OBJECT_LIMIT, ObjectKind, ObjectStorage, RepositoryCell, RepositoryModule, StoredObject, - object_id, -}; - -pub(crate) const MAX_OBJECTS: usize = 128; -// Publication amortizes durable commits independently of bounded read pages. -pub(crate) const MAX_BATCH_OBJECTS: usize = 2048; -pub(crate) const VERIFY_BATCH_BYTES: u64 = 64 * 1024 * 1024; -#[cfg(test)] -pub(crate) const INPUT_LIMIT: u32 = 4 * 1024 * 1024; -// Leave room for record metadata inside Cellule's bounded command wire format. -const INLINE_BATCH_BYTES: usize = 3 * 1024 * 1024; - -/// Batches small objects together and verifies oversized SQLite objects individually. -#[derive(Default)] -pub struct ObjectBatch { - objects: Vec, - inline_bytes: usize, - verified_bytes: u64, -} - -impl ObjectBatch { - /// Adds a record, returning it unchanged if the batch's count or byte budget is full. - pub fn try_push(&mut self, object: StoredObject) -> Result<(), StoredObject> { - let bytes = match &object.storage { - ObjectStorage::Inline(body) => body.len(), - ObjectStorage::Packed { .. } - | ObjectStorage::External { .. } - | ObjectStorage::Chunked { .. } => 0, - }; - let verified = match &object.storage { - ObjectStorage::Inline(body) => body.len() as u64, - ObjectStorage::Chunked { size, .. } => *size, - ObjectStorage::External { .. } | ObjectStorage::Packed { .. } => 0, - }; - if self.objects.len() == MAX_BATCH_OBJECTS - || bytes > INLINE_OBJECT_LIMIT - || bytes > INLINE_BATCH_BYTES - self.inline_bytes - || i64::try_from(verified).is_err() - || (!self.objects.is_empty() - && verified > VERIFY_BATCH_BYTES.saturating_sub(self.verified_bytes)) - { - return Err(object); - } - self.inline_bytes += bytes; - self.verified_bytes += verified; - self.objects.push(object); - Ok(()) - } - - #[must_use] - pub fn is_empty(&self) -> bool { - self.objects.is_empty() - } -} - -impl WireValue for ObjectBatch { - fn encode(&self, encoder: &mut BoundedEncoder) -> Result<(), CodecError> { - if self.is_empty() { - return Err(CodecError::Invalid("object batch is empty")); - } - encoder.write_count(self.objects.len())?; - for object in &self.objects { - encoder.write_bytes(&object.oid)?; - encoder.write_u8(match object.kind { - ObjectKind::Blob => 0, - ObjectKind::Tree => 1, - ObjectKind::Commit => 2, - ObjectKind::Tag => 3, - })?; - match &object.storage { - ObjectStorage::Inline(body) => { - encoder.write_u8(0)?; - encoder.write_bytes(body)?; - } - ObjectStorage::Chunked { - upload, - size, - blake3, - } => { - encoder.write_u8(2)?; - encoder.write_bytes(upload)?; - encoder.write_u64(*size)?; - encoder.write_bytes(blake3)?; - } - ObjectStorage::Packed { size, blake3, pack } => { - encoder.write_u8(3)?; - encoder.write_u64(*size)?; - encoder.write_bytes(blake3)?; - encoder.write_bytes(pack)?; - } - ObjectStorage::External { - size, - blake3, - sha256, - } => { - encoder.write_u8(1)?; - encoder.write_u64(*size)?; - encoder.write_bytes(blake3)?; - encoder.write_bytes(sha256)?; - } - } - } - Ok(()) - } - - fn decode(decoder: &mut BoundedDecoder<'_>) -> Result { - let count = decoder.read_count()?; - if !(1..=MAX_BATCH_OBJECTS).contains(&count) { - return Err(CodecError::Invalid("object count is outside batch bounds")); - } - let mut batch = Self::default(); - for _ in 0..count { - let oid = crate::ObjectId::try_from(decoder.read_bytes()?) - .map_err(|_| CodecError::Invalid("invalid Git object ID"))?; - let kind = match decoder.read_u8()? { - 0 => ObjectKind::Blob, - 1 => ObjectKind::Tree, - 2 => ObjectKind::Commit, - 3 => ObjectKind::Tag, - _ => return Err(CodecError::Invalid("invalid Git object kind")), - }; - let storage = match decoder.read_u8()? { - 0 => { - let body = decoder.read_bytes()?; - if body.len() > INLINE_OBJECT_LIMIT - || body.len() > INLINE_BATCH_BYTES - batch.inline_bytes - { - return Err(CodecError::Invalid( - "object batch exceeds inline byte budget", - )); - } - ObjectStorage::Inline(body.to_vec()) - } - 1 => ObjectStorage::External { - size: decoder.read_u64()?, - blake3: fixed(decoder)?, - sha256: fixed(decoder)?, - }, - 3 => ObjectStorage::Packed { - size: decoder.read_u64()?, - blake3: fixed(decoder)?, - pack: fixed(decoder)?, - }, - 2 => ObjectStorage::Chunked { - upload: fixed(decoder)?, - size: decoder.read_u64()?, - blake3: fixed(decoder)?, - }, - _ => return Err(CodecError::Invalid("invalid Git object storage")), - }; - batch - .try_push(StoredObject { oid, kind, storage }) - .map_err(|_| CodecError::Invalid("object batch exceeds bounds"))?; - } - Ok(batch) - } -} - -fn fixed(decoder: &mut BoundedDecoder<'_>) -> Result<[u8; N], CodecError> { - decoder - .read_bytes()? - .try_into() - .map_err(|_| CodecError::Invalid("invalid object identity width")) -} - -pub(crate) struct PutObjects; - -impl Command for PutObjects { - const MODULE: &'static str = RepositoryModule::NAME; - const ID: u32 = 5; - const CODEC_VERSION: u32 = 5; - type Input = ObjectBatch; - type Output = (); - - fn execute( - context: &mut CommandContext<'_, '_>, - batch: Self::Input, - ) -> cellule_runtime::Result> { - let identity = context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT object_format FROM repository_identity WHERE singleton = 1".into(), - parameters: Vec::new(), - }], - })?; - let Some([SqlValue::Text(format)]) = identity - .first() - .and_then(|set| set.rows.first()) - .map(Vec::as_slice) - else { - return Err(Error::Command("repository identity is absent")); - }; - let format = crate::ObjectFormat::parse(format) - .ok_or(Error::Command("invalid repository object format"))?; - if batch - .objects - .iter() - .any(|object| object.oid.format() != format) - { - return Ok(CommandResult::Rejected(())); - } - for object in batch.objects { - let (size, digest, storage, body, sha256, chunk_id) = match object.storage { - ObjectStorage::Inline(body) => { - if object_id(object.oid.format(), object.kind, &body) != object.oid { - return Ok(CommandResult::Rejected(())); - } - ( - body.len() as i64, - blake3::hash(&body).as_bytes().to_vec(), - "inline", - SqlValue::Blob(body), - SqlValue::Null, - SqlValue::Null, - ) - } - ObjectStorage::Packed { size, blake3, pack } => { - if object.kind != ObjectKind::Blob || i64::try_from(size).is_err() { - return Ok(CommandResult::Rejected(())); - } - let result = context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT 1 FROM git_packs WHERE sha256 = ?1".into(), - parameters: vec![SqlValue::Blob(pack.to_vec())], - }], - })?; - if result.first().is_none_or(|set| set.rows.is_empty()) { - return Ok(CommandResult::Rejected(())); - } - ( - size as i64, - blake3.to_vec(), - "packed", - SqlValue::Null, - SqlValue::Blob(pack.to_vec()), - SqlValue::Null, - ) - } - ObjectStorage::External { - size, - blake3, - sha256, - } => { - if object.kind != ObjectKind::Blob || i64::try_from(size).is_err() { - return Ok(CommandResult::Rejected(())); - } - ( - size as i64, - blake3.to_vec(), - "external", - SqlValue::Null, - SqlValue::Blob(sha256.to_vec()), - SqlValue::Null, - ) - } - ObjectStorage::Chunked { - upload, - size, - blake3, - } => { - if crate::object_chunks::body( - context, - object.oid, - object.kind, - upload, - size, - blake3, - )? - .is_none() - { - return Ok(CommandResult::Rejected(())); - } - ( - size as i64, - blake3.to_vec(), - "chunked", - SqlValue::Null, - SqlValue::Null, - SqlValue::Blob(upload.to_vec()), - ) - } - }; - let expected = vec![ - SqlValue::Text(object.kind.git_name().into()), - SqlValue::Integer(size), - SqlValue::Blob(digest), - SqlValue::Text(storage.into()), - body, - sha256, - ]; - let mut parameters = vec![SqlValue::Blob(object.oid.to_vec())]; - parameters.extend(expected.iter().cloned()); - parameters.push(chunk_id); - let result = context.sql(&SqlBatch { statements: vec![ - SqlStatement { - sql: "INSERT INTO objects (oid, kind, size, digest, storage, body, external_sha256, chunk_id) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) ON CONFLICT(oid) DO NOTHING".into(), - parameters, - }, - SqlStatement { - sql: "SELECT kind, size, digest, storage, body, external_sha256 FROM objects WHERE oid = ?1".into(), - parameters: vec![SqlValue::Blob(object.oid.to_vec())], - }, - ]})?; - // A collision or corrupt existing record rejects the entire application - // savepoint. No earlier record in this batch may survive that rejection. - if result.get(1).and_then(|set| set.rows.first()) != Some(&expected) { - return Ok(CommandResult::Rejected(())); - } - // Verified blobs are leaves. Certify them in this same savepoint, - // avoiding another full read/hash of every historical blob later. - if object.kind == ObjectKind::Blob { - context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "INSERT INTO object_closure (oid) VALUES (?1) ON CONFLICT DO NOTHING" - .into(), - parameters: vec![SqlValue::Blob(object.oid.to_vec())], - }], - })?; - } - } - Ok(CommandResult::Success(())) - } -} - -impl RepositoryCell { - /// Publishes a bounded object batch atomically; external bytes must already be verified and uploaded. - pub async fn put_objects( - &self, - identity: MutationIdentity, - batch: ObjectBatch, - ) -> Result, InvocationError<()>> { - self.application - .command::(&self.target, identity, batch) - .await - } - - /// Returns recorded IDs from a nonempty candidate list of at most 128 objects. - pub async fn existing_objects( - &self, - candidates: &[crate::ObjectId], - ) -> Result>, InvocationError>> { - if !(1..=MAX_OBJECTS).contains(&candidates.len()) { - return Err(InvocationError::NotStarted(Error::Command( - "object lookup count is outside bounds", - ))); - } - let placeholders = vec!["?"; candidates.len()].join(","); - let result = self - .sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: format!("SELECT oid FROM objects WHERE oid IN ({placeholders})"), - parameters: candidates - .iter() - .map(|oid| SqlValue::Blob(oid.to_vec())) - .collect(), - }], - }, - ) - .await?; - let rows = &result - .output - .first() - .ok_or(InvocationError::NotStarted(Error::Command( - "missing object lookup result", - )))? - .rows; - let mut objects = BTreeSet::new(); - for row in rows { - let [SqlValue::Blob(oid)] = row.as_slice() else { - return Err(InvocationError::NotStarted(Error::Command( - "invalid object lookup row", - ))); - }; - objects.insert(oid.as_slice().try_into().map_err(|_| { - InvocationError::NotStarted(Error::Command("invalid stored Git OID")) - })?); - } - Ok(Observed { - output: objects, - receipt: result.receipt, - }) - } -} - -#[cfg(test)] -mod tests; diff --git a/crates/canopy-server/src/object_batch/tests.rs b/crates/canopy-server/src/object_batch/tests.rs deleted file mode 100644 index 6b5bf95e..00000000 --- a/crates/canopy-server/src/object_batch/tests.rs +++ /dev/null @@ -1,169 +0,0 @@ -use super::*; - -fn inline(size: usize) -> StoredObject { - StoredObject { - oid: crate::ObjectId::Sha1([0; 20]), - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(vec![11; size]), - } -} - -#[test] -fn maximum_batch_round_trips_below_the_operation_wire_limit() { - let mut batch = ObjectBatch::default(); - for _ in 0..MAX_BATCH_OBJECTS - 1 { - assert!( - batch - .try_push(StoredObject { - oid: crate::ObjectId::Sha1([1; 20]), - kind: ObjectKind::Blob, - storage: ObjectStorage::External { - size: 1_000_000, - blake3: [2; 32], - sha256: [3; 32] - }, - }) - .is_ok() - ); - } - assert!(batch.try_push(inline(INLINE_OBJECT_LIMIT)).is_ok()); - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - batch.encode(&mut encoder).unwrap(); - let encoded = encoder.finish(); - assert!(encoded.len() < INPUT_LIMIT as usize); - let mut decoder = BoundedDecoder::new(&encoded, INPUT_LIMIT).unwrap(); - let decoded = ObjectBatch::decode(&mut decoder).unwrap(); - decoder.finish().unwrap(); - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - decoded.encode(&mut encoder).unwrap(); - assert_eq!(encoder.finish(), encoded); -} - -#[test] -fn a_full_batch_returns_the_unconsumed_record() { - let mut batch = ObjectBatch::default(); - assert!(batch.try_push(inline(INLINE_OBJECT_LIMIT + 1)).is_err()); - for _ in 0..INLINE_BATCH_BYTES / INLINE_OBJECT_LIMIT { - assert!(batch.try_push(inline(INLINE_OBJECT_LIMIT)).is_ok()); - } - let leftover = match batch.try_push(inline(1)) { - Err(object) => object, - Ok(()) => panic!("byte budget must reject the record"), - }; - let mut next = ObjectBatch::default(); - assert!(next.try_push(leftover).is_ok()); - for _ in 1..MAX_BATCH_OBJECTS { - assert!(next.try_push(inline(0)).is_ok()); - } - assert!(next.try_push(inline(0)).is_err()); -} - -#[test] -fn decoding_enforces_aggregate_bytes_and_count_before_building_a_batch() { - for count in [0, MAX_BATCH_OBJECTS + 1] { - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - encoder.write_count(count).unwrap(); - let bytes = encoder.finish(); - let mut decoder = BoundedDecoder::new(&bytes, INPUT_LIMIT).unwrap(); - assert!(ObjectBatch::decode(&mut decoder).is_err()); - } - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - encoder.write_count(5).unwrap(); - for size in [ - INLINE_OBJECT_LIMIT, - INLINE_OBJECT_LIMIT, - INLINE_OBJECT_LIMIT, - INLINE_OBJECT_LIMIT, - 1, - ] { - encoder.write_bytes(&[0; 20]).unwrap(); - encoder.write_u8(0).unwrap(); // Blob. - encoder.write_u8(0).unwrap(); // Inline. - encoder.write_bytes(&vec![5; size]).unwrap(); - } - let bytes = encoder.finish(); - let mut decoder = BoundedDecoder::new(&bytes, INPUT_LIMIT).unwrap(); - assert!(ObjectBatch::decode(&mut decoder).is_err()); - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - encoder.write_count(1).unwrap(); - encoder.write_bytes(&[0; 20]).unwrap(); - encoder.write_u8(0).unwrap(); - encoder.write_u8(0).unwrap(); - encoder - .write_bytes(&vec![5; INLINE_OBJECT_LIMIT + 1]) - .unwrap(); - let bytes = encoder.finish(); - let mut decoder = BoundedDecoder::new(&bytes, INPUT_LIMIT).unwrap(); - assert!(ObjectBatch::decode(&mut decoder).is_err()); - assert!( - ObjectBatch::default() - .encode(&mut BoundedEncoder::new(1 << 20).unwrap()) - .is_err() - ); -} - -#[test] -fn chunk_references_bound_total_verification_bytes_and_round_trip() { - let chunked = |size| StoredObject { - oid: crate::ObjectId::Sha1([1; 20]), - kind: ObjectKind::Commit, - storage: ObjectStorage::Chunked { - upload: [2; 16], - size, - blake3: [3; 32], - }, - }; - let mut batch = ObjectBatch::default(); - assert!(batch.try_push(chunked(VERIFY_BATCH_BYTES)).is_ok()); - assert!(batch.try_push(inline(1)).is_err()); - let mut encoded = BoundedEncoder::new(1 << 20).unwrap(); - batch.encode(&mut encoded).unwrap(); - let encoded = encoded.finish(); - let mut decoder = BoundedDecoder::new(&encoded, 1 << 20).unwrap(); - let mut decoded = ObjectBatch::decode(&mut decoder).unwrap(); - decoder.finish().unwrap(); - assert!(decoded.try_push(chunked(1)).is_err()); - let mut malicious = BoundedEncoder::new(1 << 20).unwrap(); - malicious.write_count(2).unwrap(); - for size in [VERIFY_BATCH_BYTES, 1] { - malicious.write_bytes(&[1; 20]).unwrap(); - malicious.write_u8(2).unwrap(); - malicious.write_u8(2).unwrap(); - malicious.write_bytes(&[2; 16]).unwrap(); - malicious.write_u64(size).unwrap(); - malicious.write_bytes(&[3; 32]).unwrap(); - } - let malicious = malicious.finish(); - assert!(ObjectBatch::decode(&mut BoundedDecoder::new(&malicious, 1 << 20).unwrap()).is_err()); - let mut oversized = ObjectBatch::default(); - assert!(oversized.try_push(chunked(u64::MAX)).is_err()); -} - -#[test] -fn packed_metadata_batch_preserves_immutable_locator_without_blob_expansion() { - let mut batch = ObjectBatch::default(); - for _ in 0..MAX_BATCH_OBJECTS { - batch - .try_push(StoredObject { - oid: crate::ObjectId::Sha1([1; 20]), - kind: ObjectKind::Blob, - storage: ObjectStorage::Packed { - size: INLINE_OBJECT_LIMIT as u64, - blake3: [2; 32], - pack: [3; 32], - }, - }) - .ok() - .unwrap(); - } - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - batch.encode(&mut encoder).unwrap(); - let bytes = encoder.finish(); - assert!(bytes.len() < 256 * 1024); - let mut decoder = BoundedDecoder::new(&bytes, INPUT_LIMIT).unwrap(); - let decoded = ObjectBatch::decode(&mut decoder).unwrap(); - decoder.finish().unwrap(); - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - decoded.encode(&mut encoder).unwrap(); - assert_eq!(encoder.finish(), bytes); -} diff --git a/crates/canopy-server/src/object_chunks/mod.rs b/crates/canopy-server/src/object_chunks/mod.rs deleted file mode 100644 index aacd4fb9..00000000 --- a/crates/canopy-server/src/object_chunks/mod.rs +++ /dev/null @@ -1,230 +0,0 @@ -//! Staged SQLite chunks, verified before an immutable object record is published. - -use cellule_runtime::{ - Error, InvocationError, MutationIdentity, identity::RequestId, primitives::sql::SqlBatch, - primitives::sql::SqlResultSet, primitives::sql::SqlStatement, primitives::sql::SqlValue, - registry::CommandContext, -}; - -use crate::{ - INLINE_OBJECT_LIMIT, ObjectKind, ObjectStorage, RepositoryCell, StoredObject, object_id, -}; - -pub(crate) const CHUNK_BYTES: usize = 512 * 1024; - -#[derive(Debug, thiserror::Error)] -pub enum ObjectStageError { - #[error("only non-blob objects above the inline limit can be staged")] - Invalid, - #[error("object chunk staging failed")] - Cell(#[from] InvocationError>), -} - -impl RepositoryCell { - /// Stages a large non-blob object in SQLite; put_objects must verify and publish the returned record. - pub async fn stage_object( - &self, - identity: MutationIdentity, - kind: ObjectKind, - body: &[u8], - ) -> Result { - if kind == ObjectKind::Blob - || body.len() <= INLINE_OBJECT_LIMIT - || i64::try_from(body.len()).is_err() - { - return Err(ObjectStageError::Invalid); - } - let upload = *identity.request_id.as_bytes(); - let mut statements = vec![SqlStatement { - sql: "INSERT INTO object_uploads (id) VALUES (?1)".into(), - parameters: vec![SqlValue::Blob(upload.to_vec())], - }]; - for (part, bytes) in body.chunks(CHUNK_BYTES).enumerate() { - statements.push(SqlStatement { - sql: "INSERT INTO object_chunks (upload_id, part, body) VALUES (?1, ?2, ?3)".into(), - parameters: vec![ - SqlValue::Blob(upload.to_vec()), - SqlValue::Integer(part as i64), - SqlValue::Blob(bytes.to_vec()), - ], - }); - let mut hash = blake3::Hasher::new(); - hash.update(b"canopy-object-chunk-v1"); - hash.update(&upload); - hash.update(&(part as u64).to_le_bytes()); - let mut request_id = [0; 16]; - request_id.copy_from_slice(&hash.finalize().as_bytes()[..16]); - self.sql - .batch( - MutationIdentity { - request_id: RequestId::from_bytes(request_id), - issued_at_ms: identity.issued_at_ms, - expires_at_ms: identity.expires_at_ms, - }, - SqlBatch { - statements: std::mem::take(&mut statements), - }, - ) - .await?; - } - Ok(StoredObject { - oid: object_id(self.object_format(), kind, body), - kind, - storage: ObjectStorage::Chunked { - upload, - size: body.len() as u64, - blake3: *blake3::hash(body).as_bytes(), - }, - }) - } - - pub(crate) async fn chunked_body( - &self, - oid: crate::ObjectId, - kind: ObjectKind, - upload: [u8; 16], - size: u64, - digest: [u8; 32], - ) -> Result, InvocationError>> { - let invalid = - || InvocationError::NotStarted(Error::Command("incomplete or corrupt object chunks")); - let mut chunks = Chunks::new(oid, kind, upload, size, digest).ok_or_else(invalid)?; - while !chunks.complete() { - let result = self.sql.query(None, chunks.query()).await?; - if !chunks.append(&result.output) { - return Err(invalid()); - } - } - chunks.finish().ok_or_else(invalid) - } -} - -/// The caller's command transaction binds verification and publication atomically. -pub(crate) fn body( - context: &CommandContext<'_, '_>, - oid: crate::ObjectId, - kind: ObjectKind, - upload: [u8; 16], - size: u64, - digest: [u8; 32], -) -> cellule_runtime::Result>> { - let Some(mut chunks) = Chunks::new(oid, kind, upload, size, digest) else { - return Ok(None); - }; - while !chunks.complete() { - let result = context.sql(&chunks.snapshot_query())?; - if !chunks.append_snapshot(&result) { - return Ok(None); - } - } - Ok(chunks.finish()) -} - -struct Chunks { - oid: crate::ObjectId, - kind: ObjectKind, - upload: [u8; 16], - size: usize, - digest: [u8; 32], - body: Vec, - part: usize, -} - -impl Chunks { - fn new( - oid: crate::ObjectId, - kind: ObjectKind, - upload: [u8; 16], - size: u64, - digest: [u8; 32], - ) -> Option { - let size = usize::try_from(size).ok()?; - if kind == ObjectKind::Blob || size <= INLINE_OBJECT_LIMIT || i64::try_from(size).is_err() { - return None; - } - Some(Self { - oid, - kind, - upload, - size, - digest, - body: Vec::new(), - part: 0, - }) - } - - fn query(&self) -> SqlBatch { - SqlBatch { statements: vec![SqlStatement { - // Count also rejects extra chunks; each result carries at most one 512 KiB body. - sql: "SELECT body, (SELECT COUNT(*) FROM object_chunks WHERE upload_id = ?1) FROM object_chunks WHERE upload_id = ?1 AND part = ?2".into(), - parameters: vec![SqlValue::Blob(self.upload.to_vec()), SqlValue::Integer(self.part as i64)], - }] } - } - - fn snapshot_query(&self) -> SqlBatch { - if self.part == 0 { - return self.query(); - } - // The command transaction keeps the count checked by the first read - // stable. Recounting every part scans this upload quadratically. - SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT body FROM object_chunks WHERE upload_id = ?1 AND part = ?2".into(), - parameters: vec![ - SqlValue::Blob(self.upload.to_vec()), - SqlValue::Integer(self.part as i64), - ], - }], - } - } - - fn append_snapshot(&mut self, results: &[SqlResultSet]) -> bool { - if self.part == 0 { - return self.append(results); - } - let Some([SqlValue::Blob(bytes)]) = results - .first() - .and_then(|set| set.rows.first()) - .map(Vec::as_slice) - else { - return false; - }; - self.append_bytes(bytes) - } - - fn append(&mut self, results: &[SqlResultSet]) -> bool { - let Some([SqlValue::Blob(bytes), SqlValue::Integer(count)]) = results - .first() - .and_then(|set| set.rows.first()) - .map(Vec::as_slice) - else { - return false; - }; - if *count != self.size.div_ceil(CHUNK_BYTES) as i64 { - return false; - } - self.append_bytes(bytes) - } - - fn append_bytes(&mut self, bytes: &[u8]) -> bool { - if bytes.len() != CHUNK_BYTES.min(self.size - self.body.len()) { - return false; - } - self.body.extend_from_slice(bytes); - self.part += 1; - true - } - - fn complete(&self) -> bool { - self.body.len() == self.size - } - - fn finish(self) -> Option> { - (object_id(self.oid.format(), self.kind, &self.body) == self.oid - && blake3::hash(&self.body).as_bytes() == &self.digest) - .then_some(self.body) - } -} - -#[cfg(test)] -mod tests; diff --git a/crates/canopy-server/src/object_chunks/tests.rs b/crates/canopy-server/src/object_chunks/tests.rs deleted file mode 100644 index f5a449b1..00000000 --- a/crates/canopy-server/src/object_chunks/tests.rs +++ /dev/null @@ -1,98 +0,0 @@ -use super::*; - -fn results(row: Vec) -> [SqlResultSet; 1] { - [SqlResultSet { - columns: Vec::new(), - rows: vec![row], - rows_affected: 0, - }] -} - -#[test] -fn snapshot_verification_counts_the_upload_only_once() { - let bytes = vec![17; CHUNK_BYTES * 3 + 1]; - let oid = object_id(crate::ObjectFormat::Sha1, ObjectKind::Commit, &bytes); - let mut chunks = Chunks::new( - oid, - ObjectKind::Commit, - [7; 16], - bytes.len() as u64, - *blake3::hash(&bytes).as_bytes(), - ) - .unwrap(); - for (part, body) in bytes.chunks(CHUNK_BYTES).enumerate() { - let query = chunks.snapshot_query(); - assert_eq!(query.statements[0].sql.contains("COUNT(*)"), part == 0); - // Separate async reads have no shared snapshot: keep their count check. - assert!(chunks.query().statements[0].sql.contains("COUNT(*)")); - let mut row = vec![SqlValue::Blob(body.to_vec())]; - if part == 0 { - row.push(SqlValue::Integer(bytes.len().div_ceil(CHUNK_BYTES) as i64)); - } - assert!(chunks.append_snapshot(&results(row))); - } - assert!(chunks.complete()); - assert_eq!(chunks.finish(), Some(bytes)); -} - -#[test] -fn snapshot_verification_rejects_extra_missing_and_malformed_chunks() { - let bytes = vec![17; INLINE_OBJECT_LIMIT + 1]; - let oid = object_id(crate::ObjectFormat::Sha1, ObjectKind::Commit, &bytes); - let mut chunks = Chunks::new( - oid, - ObjectKind::Commit, - [7; 16], - bytes.len() as u64, - *blake3::hash(&bytes).as_bytes(), - ) - .unwrap(); - let first = bytes[..CHUNK_BYTES].to_vec(); - assert!(!chunks.append_snapshot(&results(vec![ - SqlValue::Blob(first.clone()), - SqlValue::Integer(3) - ]))); - assert_eq!(chunks.part, 0); - assert!(chunks.append_snapshot(&results(vec![SqlValue::Blob(first), SqlValue::Integer(2)]))); - assert!(!chunks.append_snapshot(&[])); - assert!(!chunks.append_snapshot(&results(vec![SqlValue::Blob(vec![17, 17])]))); - assert!(!chunks.append_snapshot(&results(vec![SqlValue::Integer(17)]))); - assert_eq!(chunks.part, 1); - assert!(chunks.append_snapshot(&results(vec![SqlValue::Blob( - bytes[CHUNK_BYTES..].to_vec() - )]))); - assert_eq!(chunks.finish(), Some(bytes)); -} - -#[test] -fn snapshot_verification_keeps_both_digest_checks() { - let bytes = vec![17; INLINE_OBJECT_LIMIT + 1]; - for format in [crate::ObjectFormat::Sha1, crate::ObjectFormat::Sha256] { - for corrupt_oid in [false, true] { - let oid = object_id( - format, - ObjectKind::Commit, - if corrupt_oid { - b"wrong identity" - } else { - &bytes - }, - ); - let mut digest = *blake3::hash(&bytes).as_bytes(); - if !corrupt_oid { - digest[0] ^= 1; - } - let mut chunks = - Chunks::new(oid, ObjectKind::Commit, [7; 16], bytes.len() as u64, digest).unwrap(); - assert!(chunks.append_snapshot(&results(vec![ - SqlValue::Blob(bytes[..CHUNK_BYTES].to_vec()), - SqlValue::Integer(2) - ]))); - assert!(chunks.append_snapshot(&results(vec![SqlValue::Blob( - bytes[CHUNK_BYTES..].to_vec() - )]))); - assert!(chunks.complete()); - assert!(chunks.finish().is_none()); - } - } -} diff --git a/crates/canopy-server/src/object_reads/mod.rs b/crates/canopy-server/src/object_reads/mod.rs deleted file mode 100644 index 5562563d..00000000 --- a/crates/canopy-server/src/object_reads/mod.rs +++ /dev/null @@ -1,242 +0,0 @@ -//! Bounded immutable object pages for cold cache hydration. - -use cellule_runtime::{ - Error, InvocationError, Observed, primitives::sql::SqlBatch, primitives::sql::SqlResultSet, - primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; - -use crate::{ - INLINE_OBJECT_LIMIT, ObjectKind, ObjectStorage, RepositoryCell, StoredObject, - object_batch::MAX_OBJECTS, object_id, -}; - -pub(crate) struct ObjectHeaders { - pub(crate) objects: Observed>, -} - -impl RepositoryCell { - /// Reads at most 128 objects and 768 KiB of inline bodies, in OID order. - /// - /// Continue after the last OID until an empty page, including after short - /// pages. Objects are immutable; a future collector must fence this read. - pub async fn object_page( - &self, - after: Option, - ) -> Result>, InvocationError>> { - let headers = self.read_object_headers(None, SqlStatement { - sql: "SELECT sequence, oid, CASE WHEN storage = 'inline' THEN size ELSE 0 END FROM objects WHERE oid > ?1 ORDER BY oid LIMIT ?2".into(), - parameters: vec![SqlValue::Blob(after.map_or_else(Vec::new, |oid| oid.to_vec())), SqlValue::Integer(MAX_OBJECTS as i64)], - }).await?; - self.object_records(headers.objects).await - } - - pub(crate) async fn selected_objects( - &self, - ids: &[crate::ObjectId], - ) -> Result, InvocationError>> { - if ids.is_empty() || ids.len() > MAX_OBJECTS { - return Err(InvocationError::NotStarted(Error::Command( - "invalid object selection", - ))); - } - let placeholders = vec!["?"; ids.len()].join(","); - let headers = self.read_object_headers(None, SqlStatement { - sql: format!("SELECT sequence, oid, CASE WHEN storage = 'inline' THEN size ELSE 0 END FROM objects WHERE oid IN ({placeholders}) ORDER BY oid"), - parameters: ids.iter().map(|oid| SqlValue::Blob(oid.to_vec())).collect(), - }).await?; - Ok(self.object_records(headers.objects).await?.output) - } - - async fn read_object_headers( - &self, - minimum: Option, - statement: SqlStatement, - ) -> Result>> { - let headers = self - .sql - .query( - minimum, - SqlBatch { - statements: vec![statement], - }, - ) - .await?; - let rows = headers - .output - .first() - .ok_or_else(|| InvocationError::NotStarted(Error::Command("missing object headers")))?; - let (ids, _) = decode_headers(&rows.rows).map_err(InvocationError::NotStarted)?; - Ok(ObjectHeaders { - objects: Observed { - output: ids, - receipt: headers.receipt, - }, - }) - } - - pub(crate) async fn object_records( - &self, - headers: Observed>, - ) -> Result>, InvocationError>> { - // Callers may remove already cached IDs, but never add IDs: the header - // query's size bound and receipt protect this body read's wire ceiling. - let mut ids = headers.output; - ids.sort_unstable(); - if ids.is_empty() { - return Ok(Observed { - output: Vec::new(), - receipt: headers.receipt, - }); - } - let placeholders = vec!["?"; ids.len()].join(","); - // Immutable records bind this second read to the selected headers. The - // payload bound leaves room for record metadata under Cellule's 1 MiB cap. - let result = self.sql.query(Some(headers.receipt), SqlBatch { - statements: vec![SqlStatement { - sql: format!("SELECT oid, kind, size, digest, storage, body, external_sha256, chunk_id FROM objects WHERE oid IN ({placeholders}) ORDER BY oid"), - parameters: ids.iter().map(|oid| SqlValue::Blob(oid.to_vec())).collect(), - }], - }).await?; - let rows = result - .output - .into_iter() - .next() - .ok_or_else(|| InvocationError::NotStarted(Error::Command("missing object page")))? - .rows; - // Hash a whole bounded page off the async executor. Move SQL bodies into - // the result so page verification does not duplicate their payloads. - let output = tokio::task::spawn_blocking(move || { - let objects = rows - .into_iter() - .map(decode_object) - .collect::>>()?; - if objects.iter().map(|object| object.oid).ne(ids) { - return Err(Error::Command("objects changed during page read")); - } - Ok(objects) - }) - .await - .map_err(|error| { - InvocationError::NotStarted(Error::Facility { - name: "object page verification", - source: Box::new(error), - }) - })? - .map_err(InvocationError::NotStarted)?; - Ok(Observed { - output, - receipt: result.receipt, - }) - } -} - -fn decode_headers(rows: &[Vec]) -> cellule_runtime::Result<(Vec, i64)> { - let mut ids = Vec::new(); - let mut bytes = 0; - let mut through = 0; - for row in rows { - let [ - SqlValue::Integer(sequence), - SqlValue::Blob(oid), - SqlValue::Integer(size), - ] = row.as_slice() - else { - return Err(Error::Command("invalid object header")); - }; - if *sequence <= 0 { - return Err(Error::Command("invalid object sequence")); - } - let oid = oid - .as_slice() - .try_into() - .map_err(|_| Error::Command("invalid stored object ID"))?; - let size = usize::try_from(*size) - .ok() - .filter(|size| *size <= INLINE_OBJECT_LIMIT) - .ok_or(Error::Command("invalid inline object size"))?; - if size > INLINE_OBJECT_LIMIT - bytes { - break; - } - bytes += size; - ids.push(oid); - through = *sequence; - } - Ok((ids, through)) -} - -fn decode_object(row: Vec) -> cellule_runtime::Result { - let row: [SqlValue; 8] = row - .try_into() - .map_err(|_| Error::Command("invalid stored object row"))?; - let [ - SqlValue::Blob(oid), - SqlValue::Text(kind), - SqlValue::Integer(size), - SqlValue::Blob(digest), - SqlValue::Text(storage), - body, - external_sha256, - chunk_id, - ] = row - else { - return Err(Error::Command("invalid stored object row")); - }; - let oid: crate::ObjectId = oid - .try_into() - .map_err(|_| Error::Command("invalid stored object ID"))?; - let digest: [u8; 32] = digest - .try_into() - .map_err(|_| Error::Command("invalid object digest"))?; - let kind = match kind.as_str() { - "blob" => ObjectKind::Blob, - "tree" => ObjectKind::Tree, - "commit" => ObjectKind::Commit, - "tag" => ObjectKind::Tag, - _ => return Err(Error::Command("invalid stored object kind")), - }; - let storage = match (storage.as_str(), body, external_sha256, chunk_id) { - ("inline", SqlValue::Blob(body), SqlValue::Null, SqlValue::Null) - if usize::try_from(size).ok() == Some(body.len()) - && body.len() <= INLINE_OBJECT_LIMIT - && object_id(oid.format(), kind, &body) == oid - && blake3::hash(&body).as_bytes() == &digest => - { - ObjectStorage::Inline(body) - } - ("packed", SqlValue::Null, SqlValue::Blob(pack), SqlValue::Null) - if kind == ObjectKind::Blob && size >= 0 => - { - ObjectStorage::Packed { - size: size as u64, - blake3: digest, - pack: pack - .try_into() - .map_err(|_| Error::Command("invalid pack locator"))?, - } - } - ("external", SqlValue::Null, SqlValue::Blob(sha256), SqlValue::Null) - if kind == ObjectKind::Blob && size >= 0 => - { - ObjectStorage::External { - size: size as u64, - blake3: digest, - sha256: sha256 - .try_into() - .map_err(|_| Error::Command("invalid SHA-256 digest"))?, - } - } - ("chunked", SqlValue::Null, SqlValue::Null, SqlValue::Blob(upload)) - if kind != ObjectKind::Blob && size > INLINE_OBJECT_LIMIT as i64 => - { - ObjectStorage::Chunked { - upload: upload - .try_into() - .map_err(|_| Error::Command("invalid object chunk reference"))?, - size: size as u64, - blake3: digest, - } - } - _ => return Err(Error::Command("corrupt stored object")), - }; - Ok(StoredObject { oid, kind, storage }) -} diff --git a/crates/canopy-server/src/pack_store.rs b/crates/canopy-server/src/pack_store.rs deleted file mode 100644 index 1cab3aa1..00000000 --- a/crates/canopy-server/src/pack_store.rs +++ /dev/null @@ -1,337 +0,0 @@ -//! Immutable pack artifacts; SQLite retains canonical identities and closure. -use crate::{ - ObjectId, ObjectKind, RepositoryCell, - blob::{LargeBlobReference, LargeBlobStore}, - git_cache::GitCache, - git_gateway::GatewayError, - git_http::GitProcess, -}; -use cellule_ltx::DiskBudget; -use cellule_runtime::{ - Error, InvocationError, - primitives::sql::{SqlBatch, SqlResultSet, SqlStatement, SqlValue}, -}; -use object_store::ObjectStore; -use std::{path::PathBuf, sync::Arc}; -use tokio::sync::Mutex; - -#[derive(Clone)] -pub(crate) struct PackRecord { - pub hash: ObjectId, - pub pack: LargeBlobReference, - pub index: LargeBlobReference, - pub approved: bool, -} - -pub(crate) struct PackReader { - store: LargeBlobStore, - root: PathBuf, - budget: DiskBudget, - format: crate::ObjectFormat, - native: crate::native_resources::NativeScope, - cache: Mutex>>, - installation: Mutex<()>, - private: Mutex)>>, -} -impl PackReader { - pub(crate) fn new( - store: Arc, - repository: [u8; 16], - root: PathBuf, - budget: DiskBudget, - format: crate::ObjectFormat, - native: crate::native_resources::NativeScope, - ) -> Self { - Self { - store: LargeBlobStore::new(store, repository), - root, - budget, - format, - native, - cache: Mutex::new(None), - installation: Mutex::new(()), - private: Mutex::new(None), - } - } - pub(crate) async fn cache(&self) -> Result, GatewayError> { - let mut cache = self.cache.lock().await; - if cache.is_none() { - *cache = Some( - GitCache::create( - self.root.clone(), - self.budget.clone(), - "refs/heads/main", - self.format, - self.native.clone(), - ) - .await?, - ); - } - Ok(Arc::clone( - cache.as_ref().ok_or(GatewayError::MalformedCache)?, - )) - } - pub(crate) async fn install( - &self, - cache: &Arc, - record: &PackRecord, - ) -> Result<(), GatewayError> { - let _installation = self.installation.lock().await; - if cache.has_durable_pack(record.pack.sha256) { - return Ok(()); - } - cache - .install_pack( - record.hash, - self.store.read(&record.pack).await?, - self.store.read(&record.index).await?, - ) - .await?; - if record.approved { - cache.mark_durable_pack(record.pack.sha256); - } - Ok(()) - } - pub(crate) async fn native_reader( - &self, - record: PackRecord, - oid: ObjectId, - size: u64, - digest: [u8; 32], - ) -> Result { - // Incomplete archives stay in one bounded private cache. Their foreign - // members cannot enter a transport cache; each extracted body is checked. - let cache = if record.approved { - self.cache().await? - } else { - let mut private = self.private.lock().await; - if private - .as_ref() - .is_none_or(|(sha, _)| *sha != record.pack.sha256) - { - *private = Some(( - record.pack.sha256, - GitCache::create( - self.root.clone(), - self.budget.clone(), - "refs/heads/main", - self.format, - self.native.clone(), - ) - .await?, - )); - } - Arc::clone(&private.as_ref().ok_or(GatewayError::MalformedCache)?.1) - }; - self.install(&cache, &record).await?; - if !record.approved { - cache.mark_durable_pack(record.pack.sha256); - } - let mut command = crate::native_git::command(&cache.git_dir())?; - command - .args(["cat-file", "blob", &hex::encode(oid)]) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::null()); - let mut process = GitProcess::spawn( - command, - Arc::clone(&cache), - cache - .native - .try_admit(crate::native_resources::NativeWork::Read)?, - )?; - let output = process - .child - .stdout - .take() - .ok_or(GatewayError::MalformedCache)?; - Ok(NativePackedRead { - process, - output, - oid, - #[cfg(test)] - size, - remaining: size, - expected: digest, - canonical: Some(crate::git_format::ObjectHasher::new( - oid.format(), - ObjectKind::Blob, - size, - )), - digest: blake3::Hasher::new(), - finished: false, - }) - } - pub(crate) async fn read_blob( - &self, - record: PackRecord, - oid: ObjectId, - size: u64, - digest: [u8; 32], - ) -> Result, GatewayError> { - if size > crate::INLINE_OBJECT_LIMIT as u64 { - return Err(GatewayError::MalformedCache); - } - let mut reader = self.native_reader(record, oid, size, digest).await?; - let mut body = Vec::with_capacity(size as usize); - while let Some(bytes) = reader.next().await? { - body.extend_from_slice(&bytes); - } - Ok(body) - } - #[cfg(test)] - pub(crate) async fn upload(&self, path: PathBuf) -> Result { - let hash_path = path.clone(); - let (oid, size) = tokio::task::spawn_blocking(move || { - use std::io::Read; - let mut file = std::fs::File::open(hash_path)?; - let size = file.metadata()?.len(); - let mut hash = crate::git_format::ObjectHasher::new( - crate::ObjectFormat::Sha256, - ObjectKind::Blob, - size, - ); - let mut buffer = vec![0; 8 << 20]; - loop { - let count = file.read(&mut buffer)?; - if count == 0 { - break; - } - hash.update(&buffer[..count]); - } - Ok::<_, std::io::Error>((hash.finalize(), size)) - }) - .await??; - Ok(self - .store - .put(oid, size, &mut tokio::fs::File::open(path).await?) - .await?) - } -} - -/// Constant-memory extraction used only when an incomplete pack cannot be -/// admitted as a whole. Verification precedes the final returned body range. -pub(crate) struct NativePackedRead { - process: GitProcess>, - output: tokio::process::ChildStdout, - pub(crate) oid: ObjectId, - #[cfg(test)] - pub(crate) size: u64, - remaining: u64, - expected: [u8; 32], - canonical: Option, - digest: blake3::Hasher, - finished: bool, -} -impl NativePackedRead { - pub(crate) async fn next(&mut self) -> Result, GatewayError> { - use tokio::io::AsyncReadExt; - if self.finished { - return Ok(None); - } - let mut bytes = vec![0; self.remaining.min(8 << 20) as usize]; - tokio::time::timeout( - std::time::Duration::from_secs(120), - self.output.read_exact(&mut bytes), - ) - .await - .map_err(|_| crate::git_http::GitHttpError::Timeout)??; - self.remaining -= bytes.len() as u64; - self.canonical - .as_mut() - .ok_or(GatewayError::MalformedCache)? - .update(&bytes); - self.digest.update(&bytes); - if self.remaining == 0 { - let mut trailing = [0]; - let eof = tokio::time::timeout( - std::time::Duration::from_secs(120), - self.output.read(&mut trailing), - ) - .await - .map_err(|_| crate::git_http::GitHttpError::Timeout)??; - if eof != 0 - || self - .canonical - .take() - .ok_or(GatewayError::MalformedCache)? - .finalize() - != self.oid - || self.digest.finalize().as_bytes() != &self.expected - { - return Err(GatewayError::MalformedCache); - } - let status = - tokio::time::timeout(std::time::Duration::from_secs(120), self.process.wait()) - .await - .map_err(|_| crate::git_http::GitHttpError::Timeout)??; - if !status.success() { - return Err(GatewayError::MalformedCache); - } - self.finished = true; - } - Ok(Some(bytes.into())) - } -} - -const COLUMNS: &str = "sha256, pack_hash, pack_oid, pack_size, pack_digest, index_oid, index_size, index_digest, index_sha256, approved, covered_through"; -type ReadError = InvocationError>; -fn invalid() -> ReadError { - InvocationError::NotStarted(Error::Command("invalid durable pack record")) -} -fn decode(row: &[SqlValue]) -> Result { - let [ - SqlValue::Blob(sha), - SqlValue::Blob(hash), - SqlValue::Blob(pack_oid), - SqlValue::Integer(pack_size), - SqlValue::Blob(pack_digest), - SqlValue::Blob(index_oid), - SqlValue::Integer(index_size), - SqlValue::Blob(index_digest), - SqlValue::Blob(index_sha), - SqlValue::Integer(approved), - SqlValue::Integer(_covered_through), - ] = row - else { - return Err(invalid()); - }; - Ok(PackRecord { - hash: hash.as_slice().try_into().map_err(|_| invalid())?, - pack: LargeBlobReference { - oid: pack_oid.as_slice().try_into().map_err(|_| invalid())?, - size: (*pack_size).try_into().map_err(|_| invalid())?, - blake3: pack_digest.as_slice().try_into().map_err(|_| invalid())?, - sha256: sha.as_slice().try_into().map_err(|_| invalid())?, - }, - index: LargeBlobReference { - oid: index_oid.as_slice().try_into().map_err(|_| invalid())?, - size: (*index_size).try_into().map_err(|_| invalid())?, - blake3: index_digest.as_slice().try_into().map_err(|_| invalid())?, - sha256: index_sha.as_slice().try_into().map_err(|_| invalid())?, - }, - approved: *approved == 1, - }) -} -impl RepositoryCell { - pub(crate) async fn pack_record(&self, sha: [u8; 32]) -> Result { - let result = self - .sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: format!("SELECT {COLUMNS} FROM git_packs WHERE sha256 = ?1"), - parameters: vec![SqlValue::Blob(sha.to_vec())], - }], - }, - ) - .await?; - decode( - result - .output - .first() - .and_then(|set| set.rows.first()) - .ok_or_else(invalid)?, - ) - } -} diff --git a/crates/canopy-server/src/pulls/candidates/mod.rs b/crates/canopy-server/src/pulls/candidates/mod.rs index a43167d3..99c2d631 100644 --- a/crates/canopy-server/src/pulls/candidates/mod.rs +++ b/crates/canopy-server/src/pulls/candidates/mod.rs @@ -6,7 +6,6 @@ pub(crate) mod command; pub(crate) mod rebase; use super::merge::{MergeStrategy, oid, policy_state, policy_statement}; use super::*; -use crate::ObjectKind; use cellule_runtime::{ codec::BoundedDecoder, codec::BoundedEncoder, codec::CodecError, codec::WireValue, registry::CommandContext, @@ -276,59 +275,6 @@ pub(crate) fn commit_body(candidate: &MergeCandidate, tree_oid: &str) -> Vec format!("tree {tree_oid}\n{parents}author {actor} <{actor}@users.canopy.invalid> {time} +0000\ncommitter {actor} <{actor}@users.canopy.invalid> {time} +0000\n\n{message}{}",if message.ends_with('\n') {""} else {"\n"}).into_bytes() } -pub(crate) fn publication_oid( - context: &CommandContext<'_, '_>, - number: i64, - request: &merge::MergeRequest, -) -> cellule_runtime::Result> { - let Some(id) = request.candidate_id.as_deref() else { - return Ok(None); - }; - let Some((_, candidate)) = decode(&context.sql(&query(id)?)?)? else { - return Ok(None); - }; - if candidate.number != number - || candidate.request.revision != request.revision - || candidate.request.strategy != request.strategy - { - return Ok(None); - } - let CandidateResult::Ready { - oid: commit, - tree_oid, - } = &candidate.result - else { - return Ok(None); - }; - if !certified(context, &candidate, commit, tree_oid)? { - return Ok(None); - } - Ok(Some(oid(commit)?)) -} -fn certified( - context: &CommandContext<'_, '_>, - candidate: &MergeCandidate, - commit: &str, - tree: &str, -) -> cellule_runtime::Result { - if candidate.request.strategy == MergeStrategy::Rebase { - return rebase::certified(context, candidate, commit, tree); - } - // The trusted native worker owns tree-merging semantics. The transaction - // verifies exact parent order, author, timestamp, message and graph closure. - let body = commit_body(candidate, tree); - let commit = oid(commit)?; - if crate::object_id(commit.format(), ObjectKind::Commit, &body) != commit { - return Ok(false); - } - let rows=context.sql(&SqlBatch {statements:vec![SqlStatement { - sql:"SELECT o.body FROM objects o JOIN object_closure c ON c.oid = o.oid WHERE o.oid = ?1 AND o.kind = 'commit' AND o.storage = 'inline'".into(),parameters:vec![SqlValue::Blob(commit.to_vec())], - }]})?; - Ok( - matches!(rows.first().and_then(|set|set.rows.first()).map(Vec::as_slice),Some([SqlValue::Blob(stored)]) if *stored == body), - ) -} - pub(crate) fn intent_binding(candidate: &MergeCandidate) -> cellule_runtime::Result> { let request = serde_json::to_string(&candidate.request) .map_err(|_| Error::Command("candidate intent encoding"))?; diff --git a/crates/canopy-server/src/pulls/candidates/rebase.rs b/crates/canopy-server/src/pulls/candidates/rebase.rs index 988aff89..cecd653a 100644 --- a/crates/canopy-server/src/pulls/candidates/rebase.rs +++ b/crates/canopy-server/src/pulls/candidates/rebase.rs @@ -6,7 +6,6 @@ pub(crate) const MAX_COMMITS: usize = 128; pub(crate) const MAX_COMMIT_BYTES: usize = 64 * 1024; pub(crate) struct Commit<'a> { - pub(crate) tree: &'a str, pub(crate) parent: &'a str, headers: Vec<&'a [u8]>, message: &'a [u8], @@ -39,7 +38,6 @@ impl<'a> Commit<'a> { field(b"author ")?; field(b"committer ")?; Some(Self { - tree, parent, headers, message: &body[split + 2..], @@ -80,79 +78,3 @@ impl<'a> Commit<'a> { body } } - -pub(super) fn certified( - context: &CommandContext<'_, '_>, - candidate: &MergeCandidate, - tip: &str, - tree: &str, -) -> cellule_runtime::Result { - let mut source = candidate.request.revision.source_oid.clone(); - let mut current = tip.to_owned(); - let base = &candidate.request.revision.base_oid; - for index in 0..MAX_COMMITS { - let Some(original) = body(context, &source)? else { - return Ok(false); - }; - let Some(rewritten) = body(context, ¤t)? else { - return Ok(false); - }; - let Some(original_commit) = Commit::parse(&original) else { - return Ok(false); - }; - let Some(rewritten_commit) = Commit::parse(&rewritten) else { - return Ok(false); - }; - if (index == 0 && rewritten_commit.tree != tree) - || original_commit.rewrite(candidate, rewritten_commit.tree, rewritten_commit.parent) - != rewritten - { - return Ok(false); - } - // Native Git owns replay selection and merge semantics. The transaction - // binds every rewritten commit to the contiguous original chain and a - // certified common ancestor, so a worker cannot omit an intermediate commit. - source = original_commit.parent.to_owned(); - current = rewritten_commit.parent.to_owned(); - if ¤t == base { - if &source == base { - return Ok(true); - } - let rows = context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT 1 FROM commit_ancestry WHERE ancestor = ?1 AND descendant = ?2" - .into(), - parameters: vec![ - SqlValue::Blob(oid(&source)?.to_vec()), - SqlValue::Blob(oid(base)?.to_vec()), - ], - }], - })?; - return Ok(rows.first().is_some_and(|set| !set.rows.is_empty())); - } - } - Ok(false) -} - -fn body( - context: &CommandContext<'_, '_>, - commit: &str, -) -> cellule_runtime::Result>> { - let rows = context.sql(&SqlBatch { statements: vec![SqlStatement { - sql: "SELECT o.body FROM objects o JOIN object_closure c ON c.oid = o.oid WHERE o.oid = ?1 AND o.kind = 'commit' AND o.storage = 'inline' AND o.size <= ?2".into(), - parameters: vec![SqlValue::Blob(oid(commit)?.to_vec()), SqlValue::Integer(MAX_COMMIT_BYTES as i64)], - }] })?; - match rows - .first() - .and_then(|set| set.rows.first()) - .map(Vec::as_slice) - { - Some([SqlValue::Blob(bytes)]) - if crate::object_id(oid(commit)?.format(), ObjectKind::Commit, bytes) - == oid(commit)? => - { - Ok(Some(bytes.clone())) - } - _ => Ok(None), - } -} diff --git a/crates/canopy-server/src/pulls/merge/command.rs b/crates/canopy-server/src/pulls/merge/command.rs index 706ff12b..5074fe1c 100644 --- a/crates/canopy-server/src/pulls/merge/command.rs +++ b/crates/canopy-server/src/pulls/merge/command.rs @@ -1,11 +1,6 @@ use super::*; -use crate::{ - PushPlan, RepositoryModule, - access::{access_statement, decode_access}, -}; use cellule_runtime::{ - CellModule, Command, codec::BoundedDecoder, codec::BoundedEncoder, codec::CodecError, - codec::WireValue, registry::CommandContext, registry::CommandResult, + codec::BoundedDecoder, codec::BoundedEncoder, codec::CodecError, codec::WireValue, }; #[derive(Clone, Debug, Deserialize, Serialize)] @@ -57,128 +52,6 @@ impl WireValue for MergeOutcome { .map_err(|_| CodecError::Invalid("merge output JSON")) } } -pub(crate) struct MergePull; -impl Command for MergePull { - const MODULE: &'static str = RepositoryModule::NAME; - const ID: u32 = 9; - const CODEC_VERSION: u32 = 4; - type Input = MergeInput; - type Output = MergeOutcome; - fn execute( - context: &mut CommandContext<'_, '_>, - input: MergeInput, - ) -> cellule_runtime::Result> { - let role = decode_access(&context.sql(&SqlBatch { - statements: vec![access_statement(&input.actor)], - })?)?; - let rejected = |value| Ok(CommandResult::Rejected(value)); - let Some(role) = role else { - return rejected(MergeOutcome::NotFound); - }; - if role < TokenScope::Write { - return rejected(MergeOutcome::Forbidden); - } - let id = uuid::Uuid::parse_str(&input.request.id) - .map_err(|_| Error::Command("invalid merge UUID"))?; - // Bind actor, parent and full intent. Command timestamps are not part of - // application retry identity, so a lost reply can use a fresh command. - let binding = request_binding(&input); - let previous = context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT binding, id, pull_number, oid, merged_ms, pull_version, source_oid, source_version, base_oid, base_version FROM pull_merges WHERE id = ?1" - .into(), - parameters: vec![SqlValue::Blob(id.as_bytes().to_vec())], - }], - })?; - if let Some(row) = previous.first().and_then(|set| set.rows.first()) { - let Some(SqlValue::Blob(stored)) = row.first() else { - return Err(Error::Command("invalid merge binding")); - }; - if *stored != binding { - return rejected(MergeOutcome::Conflict); - } - return Ok(CommandResult::Success(MergeOutcome::Applied { - merge: record(&row[1..])?, - })); - } - let Some(state) = policy_state(&context.sql(&SqlBatch { - statements: vec![policy_statement(&input.actor, input.number)], - })?)? - else { - return rejected(MergeOutcome::NotFound); - }; - if !state.policy.ready || state.policy.revision.as_ref() != Some(&input.request.revision) { - return rejected(MergeOutcome::Conflict); - } - if !state.policy.reviews_satisfied { - return rejected(MergeOutcome::ReviewsRequired); - } - let base = oid(&input.request.revision.base_oid)?; - let source = match input.request.strategy { - MergeStrategy::FastForward => oid(&input.request.revision.source_oid)?, - MergeStrategy::MergeCommit | MergeStrategy::Squash | MergeStrategy::Rebase => { - match super::super::candidates::publication_oid( - context, - input.number, - &input.request, - )? { - Some(oid) => oid, - None => return rejected(MergeOutcome::Conflict), - } - } - }; - let ancestry = context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT 1 FROM commit_ancestry WHERE ancestor = ?1 AND descendant = ?2".into(), - parameters: vec![ - SqlValue::Blob(base.to_vec()), - SqlValue::Blob(source.to_vec()), - ], - }], - })?; - if ancestry.first().is_none_or(|set| set.rows.is_empty()) { - return rejected(MergeOutcome::NotFastForward); - } - let update = RefUpdate { - name: state.base, - expected: Some(RefExpectation { - oid: Some(base), - version: input.request.revision.base_version, - }), - new_oid: Some(source), - }; - let reviewed = ReviewedMerge { - update: update.clone(), - }; - let plan = PushPlan { - actor: input.actor, - updates: vec![update], - }; - // All writes below share this command transaction. Current checks and - // every normal ref invariant still pass through the canonical publisher. - if !crate::refs::apply_refs(context, &plan, Some(&reviewed))? { - return rejected(MergeOutcome::BranchPolicy); - } - let result=context.sql(&SqlBatch {statements:vec![SqlStatement { - sql:"UPDATE pull_requests SET state = 'merged', version = version + 1, updated_ms = max(updated_ms, ?2) WHERE number = ?1 AND state = 'open'".into(),parameters:vec![SqlValue::Integer(input.number),SqlValue::Integer(input.issued_at_ms)], - },SqlStatement { - sql:"INSERT INTO pull_merges (id, binding, pull_number, oid, merged_ms, pull_version, source_oid, source_version, base_oid, base_version) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)".into(),parameters:vec![SqlValue::Blob(id.as_bytes().to_vec()),SqlValue::Blob(binding),SqlValue::Integer(input.number),SqlValue::Blob(source.to_vec()),SqlValue::Integer(input.issued_at_ms),SqlValue::Integer(input.request.revision.pull_version),SqlValue::Blob(oid(&input.request.revision.source_oid)?.to_vec()),SqlValue::Integer(input.request.revision.source_version),SqlValue::Blob(base.to_vec()),SqlValue::Integer(input.request.revision.base_version)], - }]})?; - if result.first().is_none_or(|set| set.rows_affected != 1) { - return Err(Error::Command("merged pull was not updated")); - } - Ok(CommandResult::Success(MergeOutcome::Applied { - merge: MergeRecord { - id: input.request.id, - number: input.number, - oid: hex::encode(source), - merged_at_ms: input.issued_at_ms, - revision: input.request.revision, - }, - })) - } -} - pub(crate) fn request_binding(input: &MergeInput) -> Vec { super::super::mutations::binding(&[ &input.actor, diff --git a/crates/canopy-server/src/pulls/merge/mod.rs b/crates/canopy-server/src/pulls/merge/mod.rs index daf982a9..cab4c015 100644 --- a/crates/canopy-server/src/pulls/merge/mod.rs +++ b/crates/canopy-server/src/pulls/merge/mod.rs @@ -4,7 +4,7 @@ use crate::ReadIdentity; pub(crate) mod command; use super::*; -use crate::{RefExpectation, RefUpdate, directory::TokenScope}; +use crate::{RefExpectation, RefUpdate}; #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] @@ -129,7 +129,6 @@ pub(crate) fn reviewed_native_update( pub(super) struct ReviewState { pub(super) policy: ReviewPolicy, base: String, - writable: bool, } pub(crate) fn valid_request(request: &MergeRequest) -> bool { @@ -187,81 +186,6 @@ impl RepositoryCell { receipt: result.receipt, }) } - /// Prepares ancestry facts and atomically publishes an exact reviewed merge. - /// - /// Current write authority, reviews, checks and ref versions are rechecked at - /// publication. Exact request retries preserve the original merge record. - pub async fn merge_pull( - &self, - identity: MutationIdentity, - actor: &str, - number: i64, - request: MergeRequest, - ) -> Result, InvocationError> { - if !valid_request(&request) || number < 1 || validate_component(actor).is_err() { - return Err(InvocationError::NotStarted(Error::Command( - "invalid merge request", - ))); - } - let state = self - .pull_review_state(ReadIdentity::Account(actor), number) - .await - .map_err(preparation)? - .output; - if state.is_some_and(|state| { - state.writable - && state.policy.ready - && state.policy.reviews_satisfied - && state.policy.revision.as_ref() == Some(&request.revision) - }) { - let base = oid(&request.revision.base_oid).map_err(InvocationError::NotStarted)?; - let source = match &request.candidate_id { - None => { - Some(oid(&request.revision.source_oid).map_err(InvocationError::NotStarted)?) - } - Some(id) => self - .merge_candidate(actor, number, id) - .await - .map_err(preparation)? - .output - .filter(|candidate| { - candidate.request.revision == request.revision - && candidate.request.strategy == request.strategy - }) - .and_then(|candidate| match candidate.result { - super::candidates::CandidateResult::Ready { oid: commit, .. } => { - parse_oid(&commit).and_then(|bytes| bytes.try_into().ok()) - } - _ => None, - }), - }; - if let Some(source) = source { - self.prepare_ancestry(base, source) - .await - .map_err(preparation)?; - } - } - self.application - .command::( - &self.target, - identity, - command::MergeInput { - actor: actor.into(), - number, - request, - issued_at_ms: identity.issued_at_ms, - }, - ) - .await - } -} -fn preparation( - error: impl std::error::Error + Send + Sync + 'static, -) -> InvocationError { - InvocationError::NotStarted(Error::Facility { - name: "merge preparation", - source: Box::new(error), - }) } pub(crate) fn oid(text: &str) -> cellule_runtime::Result { parse_oid(text) @@ -304,7 +228,7 @@ pub(super) fn policy_state(sets: &[SqlResultSet]) -> cellule_runtime::Result cellule_runtime::Result>), -} - /// Expected ref version and optional tip; a missing tip is a retained deletion. #[derive(Clone, Debug, PartialEq, Eq)] pub struct RefExpectation { @@ -56,152 +46,6 @@ pub struct PushPlan { pub updates: Vec, } -impl RepositoryCell { - /// Reads a live or deleted ref and its version; None means the name has never existed. - pub async fn ref_state( - &self, - name: &str, - minimum: Option, - ) -> std::result::Result>, InvocationError>> - { - let result = self - .sql - .query( - minimum, - SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT oid, version FROM refs WHERE name = ?1".into(), - parameters: vec![SqlValue::Text(name.into())], - }], - }, - ) - .await?; - let state = result - .output - .first() - .and_then(|set| set.rows.first()) - .map(|row| decode_ref_row(row)) - .transpose() - .map_err(InvocationError::NotStarted)?; - Ok(Observed { - output: state, - receipt: result.receipt, - }) - } - - /// Reads a byte-bounded page of at most 256 refs; continuations require the first page's generation. - /// - /// A changed generation rejects the page, requiring a new scan from the start. - pub async fn refs_page( - &self, - after: &str, - generation: Option, - ) -> Result, RefReadError> { - if !after.is_empty() && generation.is_none() { - return Err(InvocationError::NotStarted(Error::Command( - "ref cursor requires a generation", - )) - .into()); - } - let result = self - .sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - // One SQLite statement binds the generation even to an empty - // final page. Separate observations could miss a concurrent push. - sql: r#" - WITH candidates AS MATERIALIZED ( - SELECT name, oid, version FROM refs - WHERE name > ?1 ORDER BY name LIMIT ?2 - ), ranked AS ( - SELECT name, oid, version, - ROW_NUMBER() OVER (ORDER BY name) AS position, - SUM(LENGTH(CAST(name AS BLOB)) + 64) - OVER (ORDER BY name) AS bytes - FROM candidates - ), page AS MATERIALIZED ( - SELECT name, oid, version FROM ranked - WHERE bytes <= ?3 OR position = 1 - ) - SELECT g.generation, g.default_branch, NULL, NULL, NULL, - EXISTS(SELECT 1 FROM refs WHERE name > - COALESCE((SELECT MAX(name) FROM page), ?1)) - FROM ref_generation g WHERE g.singleton = 1 - UNION ALL - SELECT NULL, NULL, name, oid, version, NULL FROM page - ORDER BY name - "# - .into(), - parameters: vec![ - SqlValue::Text(after.into()), - SqlValue::Integer(REF_PAGE_SIZE as i64), - SqlValue::Integer(REF_PAGE_NAME_BYTES), - ], - }], - }, - ) - .await?; - let rows = result - .output - .first() - .ok_or_else(|| InvocationError::NotStarted(Error::Command("missing refs page")))?; - let row = rows - .rows - .first() - .ok_or_else(|| InvocationError::NotStarted(Error::Command("missing ref generation")))?; - let head = crate::default_branch::decode_head(row).map_err(InvocationError::NotStarted)?; - let has_more = match row.as_slice() { - [ - _, - _, - SqlValue::Null, - SqlValue::Null, - SqlValue::Null, - SqlValue::Integer(more), - ] => *more != 0, - _ => { - return Err( - InvocationError::NotStarted(Error::Command("invalid ref page head")).into(), - ); - } - }; - let current = head.generation; - if generation.is_some_and(|expected| expected != current) { - return Err(RefReadError::Changed); - } - let mut refs = Vec::with_capacity(rows.rows.len().saturating_sub(1)); - for row in rows.rows.iter().skip(1) { - let [ - SqlValue::Null, - SqlValue::Null, - SqlValue::Text(name), - oid, - version, - SqlValue::Null, - ] = row.as_slice() - else { - return Err( - InvocationError::NotStarted(Error::Command("invalid stored ref row")).into(), - ); - }; - let state = decode_ref_row(&[oid.clone(), version.clone()]) - .map_err(InvocationError::NotStarted)?; - refs.push((name.clone(), state)); - } - Ok(Observed { - output: RefPage { - generation: current, - default_branch: head.reference, - refs, - has_more, - }, - receipt: result.receipt, - }) - } -} - impl WireValue for PushPlan { fn encode(&self, encoder: &mut BoundedEncoder) -> Result<(), CodecError> { self.encode_prefix(encoder)?; @@ -318,47 +162,6 @@ fn read_oid(decoder: &mut BoundedDecoder<'_>) -> Result, - plan: Self::Input, - ) -> cellule_runtime::Result> { - Ok(if apply_refs(context, &plan, None)? { - CommandResult::Success(true) - } else { - CommandResult::Rejected(false) - }) - } -} - -// A false decision must precede every write: HTTP completion records its -// rejection report in the same transaction without relying on rollback. -pub(crate) fn apply_refs( - context: &mut CommandContext<'_, '_>, - plan: &PushPlan, - merge: Option<&crate::pulls::merge::ReviewedMerge>, -) -> cellule_runtime::Result { - let Some(validated) = validate_refs(context, plan)? else { - return Ok(false); - }; - if !crate::graph::certified_roots(context, plan)? - || !crate::branch_rules::policies_allow(context, plan, merge)? - { - return Ok(false); - } - validated.apply(context)?; - Ok(true) -} - /// Only ref/ACL validation constructs this result. Catalog membership and /// current branch/merge policy must also pass before consuming it. It borrows /// the immutable plan and cannot be reused by another admitted command. diff --git a/crates/canopy-server/src/schema.sql b/crates/canopy-server/src/schema.sql deleted file mode 100644 index 0948f0ca..00000000 --- a/crates/canopy-server/src/schema.sql +++ /dev/null @@ -1,336 +0,0 @@ -CREATE TABLE object_uploads ( - id BLOB PRIMARY KEY CHECK(length(id) = 16) -) WITHOUT ROWID; - -CREATE TABLE object_chunks ( - upload_id BLOB NOT NULL REFERENCES object_uploads(id), - part INTEGER NOT NULL CHECK(part >= 0), - body BLOB NOT NULL CHECK(length(body) BETWEEN 1 AND 524288), - PRIMARY KEY(upload_id, part) -) WITHOUT ROWID; - -CREATE TABLE objects ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - oid BLOB NOT NULL UNIQUE CHECK(length(oid) IN (20, 32)), - kind TEXT NOT NULL CHECK(kind IN ('blob', 'tree', 'commit', 'tag')), - size INTEGER NOT NULL CHECK(size >= 0), - digest BLOB NOT NULL CHECK(length(digest) = 32), - storage TEXT NOT NULL CHECK(storage IN ('inline', 'external', 'chunked', 'packed')), - body BLOB, - external_sha256 BLOB, - chunk_id BLOB UNIQUE REFERENCES object_uploads(id) CHECK(chunk_id IS NULL OR length(chunk_id) = 16), - CHECK( - (storage = 'inline' AND body IS NOT NULL AND external_sha256 IS NULL AND chunk_id IS NULL AND size = length(body)) - OR - (storage = 'external' AND kind = 'blob' AND body IS NULL AND chunk_id IS NULL AND length(external_sha256) = 32) - OR - (storage = 'packed' AND kind = 'blob' AND body IS NULL AND chunk_id IS NULL AND length(external_sha256) = 32 ) - OR - (storage = 'chunked' AND kind != 'blob' AND body IS NULL AND external_sha256 IS NULL AND chunk_id IS NOT NULL AND size > 786432) - ) -); - -CREATE TABLE object_edges ( - parent BLOB NOT NULL REFERENCES objects(oid), - child BLOB NOT NULL REFERENCES objects(oid), - PRIMARY KEY(parent, child) -) WITHOUT ROWID; -CREATE INDEX object_edges_by_child ON object_edges(child, parent); - -CREATE TABLE object_closure ( - oid BLOB PRIMARY KEY REFERENCES objects(oid) CHECK(length(oid) IN (20, 32)) -) WITHOUT ROWID; - -CREATE TABLE refs ( - name TEXT PRIMARY KEY, - oid BLOB CHECK(oid IS NULL OR length(oid) IN (20, 32)), - version INTEGER NOT NULL CHECK(version > 0) -) WITHOUT ROWID; - -CREATE INDEX refs_by_oid ON refs(oid) WHERE oid IS NOT NULL; - -CREATE TABLE ref_generation ( - singleton INTEGER PRIMARY KEY CHECK(singleton = 1), - generation INTEGER NOT NULL CHECK(typeof(generation) = 'integer' AND generation >= 0), - default_branch TEXT NOT NULL, - visibility TEXT NOT NULL DEFAULT 'private' CHECK(visibility IN ('private', 'public')) -) WITHOUT ROWID; -INSERT INTO ref_generation (singleton, generation, default_branch) VALUES (1, 0, 'refs/heads/main'); - -CREATE TABLE lfs_objects ( - sha256 BLOB PRIMARY KEY CHECK(length(sha256) = 32), - size INTEGER NOT NULL CHECK(size >= 0), - digest BLOB NOT NULL CHECK(length(digest) = 32) -) WITHOUT ROWID; - -CREATE TABLE lfs_locks ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - id TEXT NOT NULL UNIQUE, - path TEXT NOT NULL UNIQUE, - locked_at TEXT NOT NULL, - owner TEXT NOT NULL -); - -CREATE TABLE repository_identity ( - object_format TEXT NOT NULL CHECK(object_format IN ('sha1', 'sha256')), - singleton INTEGER PRIMARY KEY CHECK(singleton = 1), - repository_id BLOB NOT NULL CHECK(length(repository_id) = 16), - owner TEXT NOT NULL, - push_cert_seed BLOB NOT NULL CHECK(length(push_cert_seed) = 32) -) WITHOUT ROWID; - -CREATE TABLE repository_members ( - account TEXT PRIMARY KEY, - role TEXT NOT NULL CHECK(role IN ('read', 'write')) -) WITHOUT ROWID; - -CREATE TABLE membership_versions ( - account TEXT PRIMARY KEY, - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0) -) WITHOUT ROWID; - -CREATE TABLE pushes ( - id BLOB PRIMARY KEY CHECK(length(id) = 16), - actor TEXT NOT NULL, - request_digest BLOB NOT NULL CHECK(length(request_digest) = 32), - options TEXT NOT NULL DEFAULT '[]' CHECK(length(CAST(options AS BLOB)) <= 32768), - response_id BLOB CHECK(response_id IS NULL OR length(response_id) = 16), - rejected INTEGER CHECK(rejected IN (0, 1)), - rejection_reason TEXT, - CHECK((response_id IS NULL) = (rejected IS NULL)) -) WITHOUT ROWID; - -CREATE TABLE push_certificates ( - digest BLOB PRIMARY KEY CHECK(length(digest) = 32), - push_id BLOB NOT NULL UNIQUE REFERENCES pushes(id), - actor TEXT NOT NULL, - signer TEXT NOT NULL, - key TEXT NOT NULL, - size INTEGER NOT NULL CHECK(size > 0), - recorded_at_ms INTEGER NOT NULL CHECK(recorded_at_ms >= 0) -) WITHOUT ROWID; - -CREATE TABLE push_certificate_chunks ( - push_id BLOB NOT NULL REFERENCES pushes(id), - part INTEGER NOT NULL CHECK(part >= 0), - body BLOB NOT NULL CHECK(length(body) BETWEEN 1 AND 524288), - PRIMARY KEY(push_id, part) -) WITHOUT ROWID; - -CREATE TABLE push_responses ( - id BLOB PRIMARY KEY CHECK(length(id) = 16), - push_id BLOB NOT NULL REFERENCES pushes(id), - status INTEGER NOT NULL CHECK(status BETWEEN 100 AND 599), - headers TEXT NOT NULL, - size INTEGER NOT NULL CHECK(size BETWEEN 0 AND 67108864), - digest BLOB NOT NULL CHECK(length(digest) = 32) -) WITHOUT ROWID; - -CREATE TABLE push_response_chunks ( - response_id BLOB NOT NULL REFERENCES push_responses(id), - part INTEGER NOT NULL CHECK(part >= 0), - body BLOB NOT NULL CHECK(length(body) BETWEEN 1 AND 524288), - PRIMARY KEY(response_id, part) -) WITHOUT ROWID; - -CREATE TABLE push_plan_chunks ( - response_id BLOB NOT NULL REFERENCES push_responses(id), - part INTEGER NOT NULL CHECK(part >= 0), - body BLOB NOT NULL CHECK(length(body) BETWEEN 1 AND 65536), - PRIMARY KEY(response_id, part) -) WITHOUT ROWID; - -CREATE TABLE issues ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - creation_digest BLOB NOT NULL CHECK(length(creation_digest) = 32), - author TEXT NOT NULL, - title TEXT NOT NULL CHECK(length(CAST(title AS BLOB)) BETWEEN 1 AND 256), - body TEXT NOT NULL CHECK(length(CAST(body AS BLOB)) <= 16384), - state TEXT NOT NULL CHECK(state IN ('open', 'closed')), - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0), - updated_ms INTEGER NOT NULL CHECK(updated_ms >= created_ms) -); -CREATE INDEX issues_by_state ON issues(state, number); - -CREATE TABLE issue_comments ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - issue_number INTEGER NOT NULL REFERENCES issues(number), - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - creation_digest BLOB NOT NULL CHECK(length(creation_digest) = 32), - author TEXT NOT NULL, - body TEXT NOT NULL CHECK(length(CAST(body AS BLOB)) BETWEEN 1 AND 16384), - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0), - updated_ms INTEGER NOT NULL CHECK(updated_ms >= created_ms) -); -CREATE INDEX comments_by_issue ON issue_comments(issue_number, number); - -CREATE TABLE check_contexts ( - name TEXT PRIMARY KEY, - reporter TEXT NOT NULL, - enabled INTEGER NOT NULL CHECK(enabled IN (0, 1)), - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0) -) WITHOUT ROWID; - -CREATE INDEX check_contexts_by_enabled ON check_contexts(enabled, name); - -CREATE TABLE check_runs ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(oid) IN (20, 32)), - context TEXT NOT NULL REFERENCES check_contexts(name), - context_version INTEGER NOT NULL CHECK(context_version > 0), - reporter TEXT NOT NULL, - state TEXT NOT NULL CHECK(state IN ('queued', 'in_progress', 'success', 'failure', 'cancelled')), - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0), - summary TEXT NOT NULL CHECK(length(CAST(summary AS BLOB)) <= 4096), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0), - updated_ms INTEGER NOT NULL CHECK(updated_ms >= created_ms) -); -CREATE INDEX check_runs_by_commit ON check_runs(oid, context, context_version, number); - -CREATE TABLE commit_parents ( - child BLOB NOT NULL REFERENCES objects(oid) CHECK(length(child) IN (20, 32)), - parent BLOB NOT NULL REFERENCES objects(oid) CHECK(length(parent) IN (20, 32)), - PRIMARY KEY(child, parent) -) WITHOUT ROWID; - -CREATE TABLE commit_ancestry ( - ancestor BLOB NOT NULL REFERENCES objects(oid) CHECK(length(ancestor) IN (20, 32)), - descendant BLOB NOT NULL REFERENCES objects(oid) CHECK(length(descendant) IN (20, 32)), - PRIMARY KEY(ancestor, descendant) -) WITHOUT ROWID; - -CREATE TABLE branch_rules ( - reference TEXT PRIMARY KEY, - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0), - enabled INTEGER NOT NULL CHECK(enabled IN (0, 1)), - deny_deletions INTEGER NOT NULL CHECK(deny_deletions IN (0, 1)), - fast_forward INTEGER NOT NULL CHECK(fast_forward IN (0, 1)), - require_pull_request INTEGER NOT NULL CHECK(require_pull_request IN (0, 1)), - required_approvals INTEGER NOT NULL CHECK(required_approvals BETWEEN 0 AND 16 AND (require_pull_request = 1 OR required_approvals = 0)) -) WITHOUT ROWID; -CREATE TABLE branch_required_checks ( - reference TEXT NOT NULL REFERENCES branch_rules(reference), - context TEXT NOT NULL, - PRIMARY KEY(reference, context) -) WITHOUT ROWID; -CREATE INDEX branch_rules_by_enabled ON branch_rules(enabled, reference); - -CREATE TABLE pull_requests ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - creation_digest BLOB NOT NULL CHECK(length(creation_digest) = 32), - author TEXT NOT NULL, - title TEXT NOT NULL CHECK(length(CAST(title AS BLOB)) BETWEEN 1 AND 256), - body TEXT NOT NULL CHECK(length(CAST(body AS BLOB)) <= 16384), - state TEXT NOT NULL CHECK(state IN ('open', 'closed', 'merged')), - draft INTEGER NOT NULL CHECK(draft IN (0, 1)), - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0), - source_ref TEXT NOT NULL REFERENCES refs(name), - base_ref TEXT NOT NULL REFERENCES refs(name) CHECK(source_ref != base_ref), - initial_source_oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(initial_source_oid) IN (20, 32)), - initial_base_oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(initial_base_oid) IN (20, 32)), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0), - updated_ms INTEGER NOT NULL CHECK(updated_ms >= created_ms) -); -CREATE INDEX pulls_by_state ON pull_requests(state, number); - -CREATE TABLE pull_reviews ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - creation_digest BLOB NOT NULL CHECK(length(creation_digest) = 32), - pull_number INTEGER NOT NULL REFERENCES pull_requests(number), - reviewer TEXT NOT NULL, - membership_version INTEGER NOT NULL CHECK(membership_version >= 0), - kind TEXT NOT NULL CHECK(kind IN ('comment', 'approve', 'request_changes')), - body TEXT NOT NULL CHECK(length(CAST(body AS BLOB)) <= 16384), - pull_version INTEGER NOT NULL CHECK(pull_version > 0), - source_oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(source_oid) IN (20, 32)), - source_version INTEGER NOT NULL CHECK(source_version > 0), - base_oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(base_oid) IN (20, 32)), - base_version INTEGER NOT NULL CHECK(base_version > 0), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0) -); -CREATE INDEX reviews_by_pull ON pull_reviews(pull_number, number); - -CREATE TABLE pull_review_heads ( - pull_number INTEGER NOT NULL REFERENCES pull_requests(number), - reviewer TEXT NOT NULL, - review_number INTEGER NOT NULL REFERENCES pull_reviews(number), - PRIMARY KEY(pull_number, reviewer) -) WITHOUT ROWID; - -CREATE TABLE pull_merges ( - id BLOB PRIMARY KEY CHECK(length(id) = 16), - binding BLOB NOT NULL CHECK(length(binding) = 32), - pull_number INTEGER NOT NULL UNIQUE REFERENCES pull_requests(number), - oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(oid) IN (20, 32)), - merged_ms INTEGER NOT NULL CHECK(merged_ms >= 0), - pull_version INTEGER NOT NULL CHECK(pull_version > 0), - source_oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(source_oid) IN (20, 32)), - source_version INTEGER NOT NULL CHECK(source_version > 0), - base_oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(base_oid) IN (20, 32)), - base_version INTEGER NOT NULL CHECK(base_version > 0) -) WITHOUT ROWID; - -CREATE TABLE merge_candidates ( - id BLOB PRIMARY KEY CHECK(length(id) = 16), - binding BLOB NOT NULL CHECK(length(binding) = 32), - pull_number INTEGER NOT NULL REFERENCES pull_requests(number), - actor TEXT NOT NULL, - request TEXT NOT NULL CHECK(length(CAST(request AS BLOB)) <= 131072), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0), - result TEXT NOT NULL CHECK(length(CAST(result AS BLOB)) <= 262144), - source_oid BLOB NOT NULL REFERENCES objects(oid), - base_oid BLOB NOT NULL REFERENCES objects(oid), - oid BLOB REFERENCES objects(oid) -) WITHOUT ROWID; - -CREATE TABLE pull_threads ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - creation_digest BLOB NOT NULL CHECK(length(creation_digest) = 32), - pull_number INTEGER NOT NULL REFERENCES pull_requests(number), - author TEXT NOT NULL, - body TEXT NOT NULL CHECK(length(CAST(body AS BLOB)) BETWEEN 1 AND 16384), - resolved INTEGER NOT NULL CHECK(resolved IN (0, 1)), - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0), - pull_version INTEGER NOT NULL CHECK(pull_version > 0), - source_oid BLOB NOT NULL REFERENCES objects(oid), - source_version INTEGER NOT NULL CHECK(source_version > 0), - base_oid BLOB NOT NULL REFERENCES objects(oid), - base_version INTEGER NOT NULL CHECK(base_version > 0), - merge_base BLOB NOT NULL REFERENCES objects(oid), - path BLOB NOT NULL CHECK(length(path) BETWEEN 1 AND 4096), - side TEXT NOT NULL CHECK(side IN ('before', 'after')), - line INTEGER NOT NULL CHECK(line BETWEEN 1 AND 20000), - blob_oid BLOB NOT NULL REFERENCES objects(oid), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0), - updated_ms INTEGER NOT NULL CHECK(updated_ms >= created_ms) -); -CREATE INDEX threads_by_pull ON pull_threads(pull_number, number); - -CREATE TABLE pull_thread_comments ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - creation_digest BLOB NOT NULL CHECK(length(creation_digest) = 32), - thread_number INTEGER NOT NULL REFERENCES pull_threads(number), - author TEXT NOT NULL, - body TEXT NOT NULL CHECK(length(CAST(body AS BLOB)) BETWEEN 1 AND 16384), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0) -); -CREATE INDEX comments_by_thread ON pull_thread_comments(thread_number, number); - --- Immutable uploaded pack/index pairs. Approval follows canonical verification --- of every native index member, including pre-existing objects and thin bases. -CREATE TABLE git_packs ( - sha256 BLOB PRIMARY KEY CHECK(length(sha256) = 32), - pack_hash BLOB NOT NULL UNIQUE CHECK(length(pack_hash) IN (20, 32)), - pack_oid BLOB NOT NULL, pack_size INTEGER NOT NULL CHECK(pack_size > 0), pack_digest BLOB NOT NULL CHECK(length(pack_digest) = 32), - index_oid BLOB NOT NULL, index_size INTEGER NOT NULL CHECK(index_size > 0), index_digest BLOB NOT NULL CHECK(length(index_digest) = 32), index_sha256 BLOB NOT NULL CHECK(length(index_sha256) = 32), - approved INTEGER NOT NULL DEFAULT 0 CHECK(approved IN (0, 1)), - covered_through INTEGER NOT NULL DEFAULT 0 CHECK(covered_through >= 0) -) WITHOUT ROWID; diff --git a/crates/canopy-server/tests/directory_cell/main.rs b/crates/canopy-server/tests/directory_cell/main.rs index a6174c5e..d0a97f84 100644 --- a/crates/canopy-server/tests/directory_cell/main.rs +++ b/crates/canopy-server/tests/directory_cell/main.rs @@ -154,7 +154,7 @@ async fn directory_reservations_recover_two_distinct_repository_cells() ®istry, &layout, &first_target, - (RepositoryModule::NAME, include_str!("../../src/schema.sql")), + (RepositoryModule::NAME, canopy_server::REPOSITORY_SCHEMA), first_session, &files.path().join("first-alpha.sqlite"), ) @@ -174,7 +174,7 @@ async fn directory_reservations_recover_two_distinct_repository_cells() ®istry, &layout, &second_target, - (RepositoryModule::NAME, include_str!("../../src/schema.sql")), + (RepositoryModule::NAME, canopy_server::REPOSITORY_SCHEMA), first_session, &files.path().join("first-beta.sqlite"), ) diff --git a/crates/canopy-server/tests/repository_cell/batches.rs b/crates/canopy-server/tests/repository_cell/batches.rs deleted file mode 100644 index bc7be274..00000000 --- a/crates/canopy-server/tests/repository_cell/batches.rs +++ /dev/null @@ -1,217 +0,0 @@ -use std::time::{SystemTime, UNIX_EPOCH}; - -use canopy_server::{ - INLINE_OBJECT_LIMIT, ObjectBatch, ObjectKind, ObjectStorage, RepositoryCell, RepositoryModule, - StoredObject, object_id, -}; -use cellule_runtime::{ - InvocationError, MutationIdentity, SqlCell, identity::RequestId, primitives::sql::SqlBatch, - primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; - -type Result = std::result::Result>; - -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -fn inline(body: Vec) -> StoredObject { - StoredObject { - oid: object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &body), - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body), - } -} - -fn batch(objects: impl IntoIterator) -> ObjectBatch { - let mut batch = ObjectBatch::default(); - for object in objects { - assert!(batch.try_push(object).is_ok()); - } - batch -} - -pub async fn exercise(repository: &RepositoryCell, sql: &SqlCell) -> Result { - let foreign = b"foreign hash format"; - let foreign_id = object_id( - canopy_server::ObjectFormat::Sha256, - ObjectKind::Blob, - foreign, - ); - assert!(matches!( - repository - .put_objects( - identity()?, - batch([StoredObject { - oid: foreign_id, - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(foreign.to_vec()), - }]), - ) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!(repository.object(foreign_id, None).await?.output.is_none()); - - // Every inserted object becomes visible at the same publication receipt. - let objects: Vec<_> = (0..128) - .map(|n| inline(format!("batch-{n}").into_bytes())) - .collect(); - let ids: Vec<_> = objects.iter().map(|object| object.oid).collect(); - let request_id = identity()?; - let committed = repository.put_objects(request_id, batch(objects)).await?; - for (n, oid) in ids.iter().enumerate() { - assert_eq!( - repository - .object(*oid, Some(committed.receipt)) - .await? - .output, - Some((ObjectKind::Blob, format!("batch-{n}").into_bytes())) - ); - } - assert_eq!( - repository.existing_objects(&ids).await?.output, - ids.iter().copied().collect() - ); - let replay = repository - .put_objects( - request_id, - batch((0..128).map(|n| inline(format!("batch-{n}").into_bytes()))), - ) - .await?; - assert_eq!(replay.receipt, committed.receipt); - - // Two bodies exactly fill the shared payload budget without exceeding the wire limit. - let first = vec![17; INLINE_OBJECT_LIMIT / 2]; - let second = vec![18; INLINE_OBJECT_LIMIT / 2]; - let first_id = object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &first); - let second_id = object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &second); - let stored = repository - .put_objects( - identity()?, - batch([inline(first.clone()), inline(second.clone())]), - ) - .await?; - assert_eq!( - repository - .object(first_id, Some(stored.receipt)) - .await? - .output, - Some((ObjectKind::Blob, first)) - ); - assert_eq!( - repository - .object(second_id, Some(stored.receipt)) - .await? - .output, - Some((ObjectKind::Blob, second)) - ); - - // A later invalid identity rolls back earlier valid records in the same command. - let fresh = inline(b"must roll back with bad OID".to_vec()); - let fresh_id = fresh.oid; - let mut wrong = inline(b"wrong object identity".to_vec()); - wrong.oid = canopy_server::ObjectId::Sha1([13; 20]); - assert!(matches!( - repository - .put_objects(identity()?, batch([fresh, wrong])) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!( - repository - .existing_objects(&[fresh_id]) - .await? - .output - .is_empty() - ); - - // Existing corrupt rows cannot be accepted just because their OID is present. - sql.batch( - identity()?, - SqlBatch { - statements: vec![SqlStatement { - sql: "UPDATE objects SET digest = ?1 WHERE oid = ?2".into(), - parameters: vec![ - SqlValue::Blob(vec![91; 32]), - SqlValue::Blob(ids[0].to_vec()), - ], - }], - }, - ) - .await?; - let fresh = inline(b"must roll back with corrupt existing object".to_vec()); - let fresh_id = fresh.oid; - assert!(matches!( - repository - .put_objects(identity()?, batch([fresh, inline(b"batch-0".to_vec())])) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!( - repository - .existing_objects(&[fresh_id]) - .await? - .output - .is_empty() - ); - - // External bytes are uploaded by the caller; their identity record is immutable here. - let external = |size| StoredObject { - oid: canopy_server::ObjectId::Sha1([27; 20]), - kind: ObjectKind::Blob, - storage: ObjectStorage::External { - size, - blake3: [28; 32], - sha256: [29; 32], - }, - }; - repository - .put_objects(identity()?, batch([external(1_000_000)])) - .await?; - let fresh = inline(b"must roll back with external identity conflict".to_vec()); - let fresh_id = fresh.oid; - assert!(matches!( - repository - .put_objects(identity()?, batch([fresh, external(1_000_001)])) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!( - repository - .existing_objects(&[fresh_id]) - .await? - .output - .is_empty() - ); - for object in [ - StoredObject { - oid: canopy_server::ObjectId::Sha1([31; 20]), - kind: ObjectKind::Tree, - storage: ObjectStorage::External { - size: 1, - blake3: [32; 32], - sha256: [33; 32], - }, - }, - external(u64::MAX), - ] { - assert!(matches!( - repository.put_objects(identity()?, batch([object])).await, - Err(InvocationError::Rejected(_)) - )); - } - assert!(repository.existing_objects(&[]).await.is_err()); - assert!( - repository - .existing_objects(&vec![canopy_server::ObjectId::Sha1([0; 20]); 129]) - .await - .is_err() - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/branch_rules.rs b/crates/canopy-server/tests/repository_cell/branch_rules.rs deleted file mode 100644 index f0cdf167..00000000 --- a/crates/canopy-server/tests/repository_cell/branch_rules.rs +++ /dev/null @@ -1,351 +0,0 @@ -use super::*; -use canopy_server::{ - FinalizePush, - branch_rules::BranchRuleEdit, - checks::{CheckContextEdit, CheckEdit, CheckState, NewCheck}, -}; -use cellule_runtime::{ - SqlCell, primitives::sql::SqlBatch, primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; - -type Result = std::result::Result>; -type Oid = canopy_server::ObjectId; -pub(super) fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} -fn rule(version: i64, checks: &[&str]) -> BranchRuleEdit { - BranchRuleEdit { - reference: "refs/heads/protected".into(), - expected_version: version, - enabled: true, - deny_deletions: true, - fast_forward_only: true, - require_pull_request: false, - required_approvals: 0, - required_checks: checks.iter().map(|s| (*s).into()).collect(), - } -} -pub(super) async fn plan(repo: &RepositoryCell, name: &str, oid: Option) -> Result { - Ok(PushPlan { - actor: "canopy".into(), - updates: vec![RefUpdate { - name: name.into(), - expected: repo.ref_state(name, None).await?.output, - new_oid: oid, - }], - }) -} -pub(super) async fn commit( - repo: &RepositoryCell, - tree: Oid, - parents: &[Oid], - message: &str, -) -> Result { - let parents: String = parents - .iter() - .map(|oid| format!("parent {}\n", hex::encode(oid))) - .collect(); - let body = format!( - "tree {}\n{parents}author Test 0 +0000\ncommitter Test 0 +0000\n\n{message}\n", - hex::encode(tree) - ); - Ok( - objects::put(repo, identity()?, ObjectKind::Commit, body.as_bytes()) - .await? - .output, - ) -} -async fn certificate( - sql: &SqlCell, - ancestor: Oid, - descendant: Oid, -) -> Result { - let result = sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT 1 FROM commit_ancestry WHERE ancestor = ?1 AND descendant = ?2" - .into(), - parameters: vec![ - SqlValue::Blob(ancestor.to_vec()), - SqlValue::Blob(descendant.to_vec()), - ], - }], - }, - ) - .await?; - Ok(!result.output[0].rows.is_empty()) -} - -pub async fn verify( - repo: &RepositoryCell, - sql: &SqlCell, - app: &ApplicationHandle, - target: &CellTarget, -) -> Result { - let name = "refs/heads/protected"; - let tree = objects::put(repo, identity()?, ObjectKind::Tree, b"") - .await? - .output; - let root = commit(repo, tree, &[], "Root").await?; - let child = commit(repo, tree, &[root], "Child").await?; - let unrelated = commit(repo, tree, &[], "Unrelated").await?; - let mut tail = child; - for number in 0..130 { - tail = commit(repo, tree, &[tail], &format!("Chain {number}")).await?; - } - let merge = commit(repo, tree, &[unrelated, tail], "Merge").await?; - repo.finalize_push( - identity()?, - plan(repo, "refs/heads/branch-fixture", Some(merge)).await?, - ) - .await?; - repo.finalize_push(identity()?, plan(repo, name, Some(root)).await?) - .await?; - assert!(matches!( - repo.set_branch_rule(identity()?, "outsider", rule(0, &[])) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - let policy_id = identity()?; - let first = repo - .set_branch_rule(policy_id, "canopy", rule(0, &[])) - .await?; - assert_eq!( - repo.set_branch_rule(policy_id, "canopy", rule(0, &[])) - .await? - .receipt, - first.receipt - ); - assert!(matches!( - repo.set_branch_rule(identity()?, "canopy", rule(0, &[])) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - let advance = plan(repo, name, Some(merge)).await?; - let generation = repo.refs_page("", None).await?.output.generation; - // Even valid Git ancestry needs a server-verified certificate at publication. - assert!(matches!( - app.command::(target, identity()?, advance.clone()) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - // A malicious certificate cannot substitute a caller-asserted link, including - // a valid first step followed by a false one in the same transaction. - assert!(matches!( - app.command::( - target, - identity()?, - Proof { - ancestor: root, - steps: vec![(child, root), (unrelated, child)] - } - ) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - assert!(!certificate(sql, root, child).await?); - assert!(!certificate(sql, root, unrelated).await?); - assert_eq!( - repo.refs_page("", None).await?.output.generation, - generation - ); - // More than one proof page, including a merge's alternate ancestry path. - let mutation = identity()?; - let committed = repo.finalize_push(mutation, advance.clone()).await?; - assert!(certificate(sql, root, merge).await?); - let delete = plan(repo, name, None).await?; - let force = plan(repo, name, Some(unrelated)).await?; - for rejected in [delete, force] { - assert!(matches!( - app.command::(target, identity()?, rejected) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - } - repo.set_check_context( - identity()?, - "canopy", - "branch-ci", - CheckContextEdit { - expected_version: 0, - reporter: "canopy", - enabled: true, - }, - ) - .await?; - repo.set_branch_rule(identity()?, "canopy", rule(1, &["branch-ci"])) - .await?; - // Completed receipt replay returns the prior result without applying new refs. - assert_eq!( - app.command::(target, mutation, advance) - .await? - .receipt, - committed.receipt - ); - assert_eq!( - repo.ref_state(name, None).await?.output.unwrap().oid, - Some(merge) - ); - let candidate = commit(repo, tree, &[merge], "Candidate").await?; - repo.finalize_push( - identity()?, - plan(repo, "refs/heads/branch-fixture", Some(candidate)).await?, - ) - .await?; - app.command::( - target, - identity()?, - Proof { - ancestor: merge, - steps: vec![(candidate, merge)], - }, - ) - .await?; - let proposed = plan(repo, name, Some(candidate)).await?; - let old = uuid::Uuid::new_v4().into_bytes(); - repo.start_check( - identity()?, - "canopy", - NewCheck { - id: old, - oid: candidate, - context: "branch-ci", - context_version: 1, - }, - ) - .await?; - repo.update_check( - identity()?, - "canopy", - old, - CheckEdit { - expected_version: 1, - state: CheckState::Success, - summary: "passed", - }, - ) - .await?; - // Captured push intent remains the same while a new queued attempt arrives. - let newer = uuid::Uuid::new_v4().into_bytes(); - repo.start_check( - identity()?, - "canopy", - NewCheck { - id: newer, - oid: candidate, - context: "branch-ci", - context_version: 1, - }, - ) - .await?; - assert!(matches!( - app.command::(target, identity()?, proposed.clone()) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - repo.update_check( - identity()?, - "canopy", - newer, - CheckEdit { - expected_version: 1, - state: CheckState::Success, - summary: "passed again", - }, - ) - .await?; - // Current context enablement is checked in the same transaction as refs. - repo.set_check_context( - identity()?, - "canopy", - "branch-ci", - CheckContextEdit { - expected_version: 1, - reporter: "canopy", - enabled: false, - }, - ) - .await?; - assert!(matches!( - app.command::(target, identity()?, proposed.clone()) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - repo.set_branch_rule(identity()?, "canopy", rule(2, &[])) - .await?; - app.command::(target, identity()?, proposed) - .await?; - assert_eq!( - repo.ref_state(name, None).await?.output.unwrap().oid, - Some(candidate) - ); - // Rule paging uses names; disabled records retain versions and requirements. - for number in 0..33 { - let mut edit = rule(0, &["branch-ci"]); - edit.reference = format!("refs/heads/page-{number:02}"); - edit.enabled = false; - repo.set_branch_rule(identity()?, "canopy", edit).await?; - } - let first = repo.branch_rules("canopy", None).await?.output.unwrap(); - assert_eq!(first.len(), 32); - let after = &first.last().unwrap().reference; - let remaining = repo - .branch_rules("canopy", Some(after)) - .await? - .output - .unwrap(); - assert_eq!(remaining.len(), 2); - assert!(repo.branch_rules("outsider", None).await?.output.is_none()); - Ok(()) -} - -// Invoke the registered command with independently encoded input, as an untrusted -// client can; the local fixture handler must never be selected by the server. -struct Proof { - ancestor: Oid, - steps: Vec<(Oid, Oid)>, -} -impl cellule_runtime::codec::WireValue for Proof { - fn encode( - &self, - out: &mut cellule_runtime::codec::BoundedEncoder, - ) -> std::result::Result<(), cellule_runtime::codec::CodecError> { - out.write_bytes(&self.ancestor)?; - out.write_count(self.steps.len())?; - for (child, parent) in &self.steps { - out.write_bytes(child)?; - out.write_bytes(parent)?; - } - Ok(()) - } - fn decode( - _: &mut cellule_runtime::codec::BoundedDecoder<'_>, - ) -> std::result::Result { - Err(cellule_runtime::codec::CodecError::Invalid( - "encode-only test client", - )) - } -} -struct CertificateCommand; -impl cellule_runtime::Command for CertificateCommand { - const MODULE: &'static str = "repository"; - const ID: u32 = 7; - const CODEC_VERSION: u32 = 2; - type Input = Proof; - type Output = bool; - fn execute( - _: &mut cellule_runtime::registry::CommandContext<'_, '_>, - _: Proof, - ) -> cellule_runtime::Result> { - Err(cellule_runtime::Error::Command( - "client handler must not execute", - )) - } -} diff --git a/crates/canopy-server/tests/repository_cell/bulk_refs.rs b/crates/canopy-server/tests/repository_cell/bulk_refs.rs deleted file mode 100644 index b1d15592..00000000 --- a/crates/canopy-server/tests/repository_cell/bulk_refs.rs +++ /dev/null @@ -1,183 +0,0 @@ -use super::*; -use canopy_server::{ObjectBatch, ObjectStorage, StoredObject}; - -pub async fn verify(repository: &RepositoryCell) -> Result<(), Box> { - let mut updates = Vec::new(); - for start in (0..1001).step_by(128) { - let mut objects = ObjectBatch::default(); - for index in start..(start + 128).min(1001) { - let body = format!("bulk ref object {index}").into_bytes(); - let oid = object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &body); - objects - .try_push(StoredObject { - oid, - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body), - }) - .map_err(|_| "fixture exceeds object batch")?; - updates.push(RefUpdate { - name: format!("refs/tags/bulk/{index:04}"), - expected: None, - new_oid: Some(oid), - }); - } - repository - .put_objects(super::graph::identity()?, objects) - .await?; - } - let generation = repository.default_branch(None).await?.output.generation; - repository - .finalize_push( - super::graph::identity()?, - PushPlan { - actor: "canopy".into(), - updates: updates.clone(), - }, - ) - .await?; - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + 1 - ); - - let oid = updates[0].new_oid; - let mut replacement: Vec<_> = updates - .into_iter() - .map(|update| RefUpdate { - name: update.name, - expected: Some(RefExpectation { - oid: update.new_oid, - version: 1, - }), - new_oid: None, - }) - .collect(); - let last = replacement.pop().ok_or("missing final descendant")?; - replacement.push(RefUpdate { - name: "refs/tags/bulk".into(), - expected: None, - new_oid: oid, - }); - // The remaining descendant is beyond several SQL pages. Reject the whole - // replacement without deleting any earlier siblings or advancing generation. - assert!(matches!( - repository - .finalize_push( - super::graph::identity()?, - PushPlan { - actor: "canopy".into(), - updates: replacement.clone() - } - ) - .await, - Err(InvocationError::Rejected(_)) - )); - assert_eq!( - repository - .ref_state("refs/tags/bulk/0000", None) - .await? - .output - .map(|state| state.oid), - Some(oid) - ); - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + 1 - ); - - replacement.push(last.clone()); - repository - .finalize_push( - super::graph::identity()?, - PushPlan { - actor: "canopy".into(), - updates: replacement, - }, - ) - .await?; - assert_eq!( - repository - .ref_state("refs/tags/bulk", None) - .await? - .output - .map(|state| state.oid), - Some(oid) - ); - assert_eq!( - repository.ref_state(&last.name, None).await?.output, - Some(RefExpectation { - oid: None, - version: 2 - }) - ); - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + 2 - ); - let long_prefix = format!( - "refs/tags/long-page/{}", - vec!["r".repeat(200); 21].join("/") - ); - let long_names: Vec<_> = (0..270) - .map(|index| format!("{long_prefix}/{index:04}")) - .collect(); - assert!(long_names[0].len() * 256 > 1 << 20); - for chunk in long_names.chunks(16) { - repository - .finalize_push( - super::graph::identity()?, - PushPlan { - actor: "canopy".into(), - updates: chunk - .iter() - .map(|name| RefUpdate { - name: name.clone(), - expected: None, - new_oid: oid, - }) - .collect(), - }, - ) - .await?; - } - assert!(matches!( - repository - .finalize_push( - super::graph::identity()?, - PushPlan { - actor: "canopy".into(), - updates: vec![RefUpdate { - name: "refs/tags/long-page".into(), - expected: None, - new_oid: oid, - }], - }, - ) - .await, - Err(InvocationError::Rejected(_)) - )); - let mut after = String::new(); - let mut page_generation = None; - let mut observed = Vec::new(); - let mut byte_bounded = false; - loop { - let page = repository.refs_page(&after, page_generation).await?.output; - page_generation = Some(page.generation); - byte_bounded |= page.has_more && page.refs.len() < 256; - if let Some((name, _)) = page.refs.last() { - after.clone_from(name); - } - observed.extend( - page.refs - .into_iter() - .filter(|(name, _)| name.starts_with("refs/tags/long-page/")) - .map(|(name, _)| name), - ); - if !page.has_more { - break; - } - } - assert!(byte_bounded); - assert_eq!(observed, long_names); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/checks.rs b/crates/canopy-server/tests/repository_cell/checks.rs deleted file mode 100644 index f43021a2..00000000 --- a/crates/canopy-server/tests/repository_cell/checks.rs +++ /dev/null @@ -1,192 +0,0 @@ -use super::*; -use canopy_server::checks::{CheckChange, CheckContextEdit, CheckEdit, CheckState, NewCheck}; - -type Result = std::result::Result>; -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -pub async fn exercise(repository: &RepositoryCell, oid: canopy_server::ObjectId) -> Result { - let policy = || CheckContextEdit { - expected_version: 0, - reporter: "ci-agent", - enabled: true, - }; - assert_eq!( - repository - .set_check_context(identity()?, "outsider", "unit", policy()) - .await? - .output, - CheckChange::Forbidden - ); - assert_eq!( - repository - .set_check_context(identity()?, "canopy", "unit", policy()) - .await? - .output, - CheckChange::NotFound - ); - repository - .grant_member(identity()?, "canopy", "ci-agent", TokenScope::Read) - .await?; - let config_identity = identity()?; - let configured = repository - .set_check_context(config_identity, "canopy", "unit", policy()) - .await?; - assert_eq!(configured.output, CheckChange::Applied); - assert_eq!( - repository - .set_check_context(config_identity, "canopy", "unit", policy()) - .await? - .receipt, - configured.receipt - ); - let id = uuid::Uuid::new_v4().into_bytes(); - let new_check = || NewCheck { - id, - oid, - context: "unit", - context_version: 1, - }; - assert_eq!( - repository - .start_check(identity()?, "canopy", new_check()) - .await? - .output, - CheckChange::Forbidden - ); - let creation = identity()?; - let created = repository - .start_check(creation, "ci-agent", new_check()) - .await?; - assert_eq!(created.output, CheckChange::Applied); - assert_eq!( - repository - .start_check(creation, "ci-agent", new_check()) - .await? - .receipt, - created.receipt - ); - assert!( - repository - .check_contexts("outsider", None) - .await? - .output - .is_none() - ); - assert!(repository.check_run("outsider", id).await?.output.is_none()); - assert!( - repository - .commit_checks("outsider", oid, None) - .await? - .output - .is_none() - ); - let summary = "\n".repeat(4096); - let edit = || CheckEdit { - expected_version: 1, - state: CheckState::Success, - summary: &summary, - }; - assert_eq!( - repository - .update_check(identity()?, "canopy", id, edit()) - .await? - .output, - CheckChange::Forbidden - ); - repository - .revoke_member(identity()?, "canopy", "ci-agent") - .await?; - assert_eq!( - repository - .start_check(identity()?, "ci-agent", new_check()) - .await? - .output, - CheckChange::NotFound - ); - assert_eq!( - repository - .update_check(identity()?, "ci-agent", id, edit()) - .await? - .output, - CheckChange::NotFound - ); - repository - .grant_member(identity()?, "canopy", "ci-agent", TokenScope::Read) - .await?; - let completion = identity()?; - let completed = repository - .update_check(completion, "ci-agent", id, edit()) - .await?; - assert_eq!(completed.output, CheckChange::Applied); - assert_eq!( - repository - .update_check(completion, "ci-agent", id, edit()) - .await? - .receipt, - completed.receipt - ); - assert_eq!( - repository - .update_check( - identity()?, - "ci-agent", - id, - CheckEdit { - expected_version: 2, - state: CheckState::Failure, - summary: "Must not overwrite" - } - ) - .await? - .output, - CheckChange::Conflict - ); - assert_eq!( - repository - .check_run("canopy", id) - .await? - .output - .ok_or("missing check")? - .summary, - summary - ); - repository - .set_check_context( - identity()?, - "canopy", - "unit", - CheckContextEdit { - expected_version: 1, - reporter: "ci-agent", - enabled: false, - }, - ) - .await?; - assert_eq!( - repository - .commit_checks("canopy", oid, None) - .await? - .output - .ok_or("missing checks")? - .len(), - 0 - ); - assert_eq!( - repository - .start_check(identity()?, "ci-agent", new_check()) - .await? - .output, - CheckChange::Conflict - ); - repository - .revoke_member(identity()?, "canopy", "ci-agent") - .await?; - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/chunks.rs b/crates/canopy-server/tests/repository_cell/chunks.rs deleted file mode 100644 index 18d17b10..00000000 --- a/crates/canopy-server/tests/repository_cell/chunks.rs +++ /dev/null @@ -1,187 +0,0 @@ -use super::*; -use canopy_server::{INLINE_OBJECT_LIMIT, ObjectBatch, ObjectStorage, StoredObject}; -use cellule_runtime::{ - SqlCell, primitives::sql::SqlBatch, primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; - -type Result = std::result::Result>; - -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -fn batch(objects: impl IntoIterator) -> ObjectBatch { - let mut batch = ObjectBatch::default(); - for object in objects { - assert!(batch.try_push(object).is_ok()); - } - batch -} - -pub async fn exercise(repository: &RepositoryCell, sql: &SqlCell) -> Result { - let body = vec![b't'; INLINE_OBJECT_LIMIT + 1]; - let staged_id = identity()?; - let object = repository - .stage_object(staged_id, ObjectKind::Tag, &body) - .await?; - let oid = object.oid; - assert!(repository.object(oid, None).await?.output.is_none()); - assert!(repository.existing_objects(&[oid]).await?.output.is_empty()); - let committed = repository.put_objects(identity()?, batch([object])).await?; - assert_eq!( - repository - .object(oid, Some(committed.receipt)) - .await? - .output, - Some((ObjectKind::Tag, body.clone())) - ); - // Stage retries replay their chunk commands; a second upload of identical bytes converges. - for request in [staged_id, identity()?] { - let duplicate = repository - .stage_object(request, ObjectKind::Tag, &body) - .await?; - repository - .put_objects(identity()?, batch([duplicate])) - .await?; - } - let layout = sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT storage, body, external_sha256, size FROM objects WHERE oid = ?1" - .into(), - parameters: vec![SqlValue::Blob(oid.to_vec())], - }], - }, - ) - .await?; - assert_eq!( - layout.output[0].rows[0], - vec![ - SqlValue::Text("chunked".into()), - SqlValue::Null, - SqlValue::Null, - SqlValue::Integer(body.len() as i64) - ] - ); - - for case in 0..7 { - let bytes = vec![case + 10; INLINE_OBJECT_LIMIT + 32]; - let mut object = repository - .stage_object(identity()?, ObjectKind::Commit, &bytes) - .await?; - let oid = object.oid; - let ObjectStorage::Chunked { - upload, - size, - blake3, - } = &mut object.storage - else { - return Err("chunk reference missing".into()); - }; - let upload = *upload; - let statement = match case { - 0 => Some(SqlStatement { sql: "DELETE FROM object_chunks WHERE upload_id = ?1 AND part = 1".into(), parameters: vec![SqlValue::Blob(upload.to_vec())] }), - 1 => Some(SqlStatement { sql: "UPDATE object_chunks SET body = zeroblob(length(body)) WHERE upload_id = ?1 AND part = 0".into(), parameters: vec![SqlValue::Blob(upload.to_vec())] }), - 2 => Some(SqlStatement { sql: "INSERT INTO object_chunks (upload_id, part, body) VALUES (?1, 2, ?2)".into(), parameters: vec![SqlValue::Blob(upload.to_vec()), SqlValue::Blob(vec![1])] }), - 3 => { blake3[0] ^= 1; None }, - 4 => { object.oid = match object.oid { canopy_server::ObjectId::Sha1(mut bytes) => { bytes[0] ^= 1; canopy_server::ObjectId::Sha1(bytes) }, canopy_server::ObjectId::Sha256(mut bytes) => { bytes[0] ^= 1; canopy_server::ObjectId::Sha256(bytes) } }; None }, - 5 => { object.kind = ObjectKind::Blob; None }, - _ => { *size -= 1; None }, - }; - if let Some(statement) = statement { - sql.batch( - identity()?, - SqlBatch { - statements: vec![statement], - }, - ) - .await?; - } - let first = format!("rolled back with invalid chunks {case}").into_bytes(); - let first_id = object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &first); - let first = StoredObject { - oid: first_id, - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(first), - }; - assert!( - matches!( - repository - .put_objects(identity()?, batch([first, object])) - .await, - Err(InvocationError::Rejected(_)) - ), - "case {case}" - ); - assert!( - repository - .existing_objects(&[first_id, oid]) - .await? - .output - .is_empty(), - "case {case}" - ); - } - // Chunk storage must preserve typed graph validation, including missing edges. - let mut tree = Vec::new(); - for index in 0..24_000 { - tree.extend_from_slice(format!("100644 file-{index:05}\0").as_bytes()); - tree.extend_from_slice(&[91; 20]); - } - let mut commit = format!("tree {}\nauthor Canopy 0 +0000\ncommitter Canopy 0 +0000\n\n", hex::encode([92; 20])).into_bytes(); - commit.extend(vec![b'c'; INLINE_OBJECT_LIMIT]); - let mut tag = format!( - "object {}\ntype commit\ntag missing\n\n", - hex::encode([93; 20]) - ) - .into_bytes(); - tag.extend(vec![b't'; INLINE_OBJECT_LIMIT]); - for (kind, bytes) in [ - (ObjectKind::Tree, tree), - (ObjectKind::Commit, commit), - (ObjectKind::Tag, tag), - ] { - let object = repository.stage_object(identity()?, kind, &bytes).await?; - let name = format!("refs/tags/chunked-{}", kind.git_name()); - let plan = PushPlan { - actor: "canopy".into(), - updates: vec![RefUpdate { - name: name.clone(), - expected: None, - new_oid: Some(object.oid), - }], - }; - repository.put_objects(identity()?, batch([object])).await?; - assert!(matches!( - repository.finalize_push(identity()?, plan).await, - Err(InvocationError::Rejected(_)) - )); - assert!(repository.ref_state(&name, None).await?.output.is_none()); - } - // Corruption after publication must also be detected during recovery reads. - sql.batch(identity()?, SqlBatch { statements: vec![SqlStatement { - sql: "DELETE FROM object_chunks WHERE upload_id = (SELECT chunk_id FROM objects WHERE oid = ?1) AND part = 1".into(), - parameters: vec![SqlValue::Blob(oid.to_vec())], - }] }).await?; - assert!(repository.object(oid, None).await.is_err()); - assert!( - repository - .stage_object(identity()?, ObjectKind::Blob, &body) - .await - .is_err() - ); - assert!( - repository - .stage_object(identity()?, ObjectKind::Tree, b"small") - .await - .is_err() - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/default_branch.rs b/crates/canopy-server/tests/repository_cell/default_branch.rs deleted file mode 100644 index 07555396..00000000 --- a/crates/canopy-server/tests/repository_cell/default_branch.rs +++ /dev/null @@ -1,173 +0,0 @@ -use super::*; -use canopy_server::RefReadError; - -fn identity() -> Result> { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -pub async fn empty(repository: &RepositoryCell) -> Result<(), Box> { - let initial = repository.default_branch(None).await?.output; - assert_eq!(initial.reference, "refs/heads/main"); - let changed = repository - .set_default_branch( - identity()?, - "canopy", - initial.generation, - "refs/heads/trunk", - ) - .await?; - assert!(changed.output); - let head = repository - .default_branch(Some(changed.receipt)) - .await? - .output; - assert_eq!(head.reference, "refs/heads/trunk"); - assert_eq!(head.generation, initial.generation + 1); - let page = repository.refs_page("", None).await?.output; - assert!(page.refs.is_empty()); - assert_eq!(page.default_branch, head.reference); - assert!(matches!( - repository.refs_page("zzzz", Some(initial.generation)).await, - Err(RefReadError::Changed) - )); - assert!( - repository - .set_default_branch(identity()?, "canopy", head.generation, "refs/heads/main") - .await? - .output - ); - Ok(()) -} - -pub async fn verify(repository: &RepositoryCell) -> Result<(), Box> { - let initial = repository.default_branch(None).await?.output; - for name in [ - "HEAD", - "refs/tags/release", - "refs/heads/", - "refs/heads/a\nref: refs/heads/b", - "refs/heads/a.lock", - "refs/heads/a..b", - ] { - assert!( - repository - .set_default_branch(identity()?, "canopy", initial.generation, name) - .await - .is_err(), - "{name}" - ); - } - for generation in [-1, i64::MAX] { - assert!( - repository - .set_default_branch(identity()?, "canopy", generation, "refs/heads/main") - .await - .is_err() - ); - } - assert!( - !repository - .set_default_branch( - identity()?, - "canopy", - initial.generation, - "refs/heads/absent" - ) - .await? - .output - ); - repository - .grant_member(identity()?, "canopy", "writer", TokenScope::Write) - .await?; - for actor in ["writer", "outsider"] { - assert!( - !repository - .set_default_branch(identity()?, actor, initial.generation, "refs/heads/main") - .await? - .output - ); - } - assert_eq!(repository.default_branch(None).await?.output, initial); - let oid = repository - .ref_state("refs/heads/main", None) - .await? - .output - .ok_or("missing main")? - .oid; - repository - .finalize_push( - identity()?, - PushPlan { - actor: "canopy".into(), - updates: vec![RefUpdate { - name: "refs/heads/trunk".into(), - expected: None, - new_oid: oid, - }], - }, - ) - .await?; - // A push also fences an administrative update derived from an older ref view. - assert!( - !repository - .set_default_branch( - identity()?, - "canopy", - initial.generation, - "refs/heads/trunk" - ) - .await? - .output - ); - let current = repository.default_branch(None).await?.output; - let replay_id = identity()?; - for _ in 0..2 { - assert!( - repository - .set_default_branch(replay_id, "canopy", current.generation, "refs/heads/trunk") - .await? - .output - ); - } - let changed = repository.default_branch(None).await?.output; - assert_eq!(changed.generation, current.generation + 1); - assert_eq!(changed.reference, "refs/heads/trunk"); - assert!(matches!( - repository.refs_page("zzzz", Some(current.generation)).await, - Err(RefReadError::Changed) - )); - assert!( - repository - .set_default_branch(identity()?, "canopy", changed.generation, "refs/heads/main") - .await? - .output - ); - // Returning HEAD to its former name must not make a stale request valid again. - assert!( - !repository - .set_default_branch( - identity()?, - "canopy", - current.generation, - "refs/heads/trunk" - ) - .await? - .output - ); - let generation = repository.default_branch(None).await?.output.generation; - let (left, right) = tokio::join!( - repository.set_default_branch(identity()?, "canopy", generation, "refs/heads/trunk"), - repository.set_default_branch(identity()?, "canopy", generation, "refs/heads/main"), - ); - assert_ne!(left?.output, right?.output); - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + 1 - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/graph.rs b/crates/canopy-server/tests/repository_cell/graph.rs deleted file mode 100644 index abe58a58..00000000 --- a/crates/canopy-server/tests/repository_cell/graph.rs +++ /dev/null @@ -1,408 +0,0 @@ -use std::time::{SystemTime, UNIX_EPOCH}; - -use canopy_server::{ObjectKind, PushPlan, RefUpdate, RepositoryCell, RepositoryModule, object_id}; -use cellule_runtime::{ - InvocationError, MutationIdentity, SqlCell, identity::RequestId, primitives::sql::SqlBatch, - primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; - -type Result = std::result::Result>; - -pub(super) fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -fn plan(name: &str, oid: canopy_server::ObjectId) -> PushPlan { - PushPlan { - actor: "canopy".into(), - updates: vec![RefUpdate { - name: name.into(), - expected: None, - new_oid: Some(oid), - }], - } -} - -async fn put( - repository: &RepositoryCell, - kind: ObjectKind, - body: &[u8], -) -> Result { - Ok(super::objects::put(repository, identity()?, kind, body) - .await? - .output) -} - -fn commit(tree: canopy_server::ObjectId, parent: Option) -> Vec { - let parent = parent.map_or_else(String::new, |oid| format!("parent {}\n", hex::encode(oid))); - format!("tree {}\n{parent}author Canopy 0 +0000\ncommitter Canopy 0 +0000\n\nGraph test\n", hex::encode(tree)).into_bytes() -} - -fn entry(mode: &str, name: &[u8], oid: canopy_server::ObjectId) -> Vec { - let mut body = format!("{mode} ").into_bytes(); - body.extend_from_slice(name); - body.push(0); - body.extend_from_slice(&oid); - body -} - -async fn certificates(sql: &SqlCell) -> Result> { - let result = sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT oid FROM object_closure ORDER BY oid".into(), - parameters: vec![], - }], - }, - ) - .await?; - Ok(result - .output - .into_iter() - .flat_map(|set| set.rows) - .flatten() - .collect()) -} - -pub async fn verify( - repository: &RepositoryCell, - sql: &SqlCell, - application: &cellule_app::ApplicationHandle, - target: &cellule_runtime::CellTarget, -) -> Result<()> { - let blob_body = b"graph closure leaf"; - let blob = object_id( - canopy_server::ObjectFormat::Sha1, - ObjectKind::Blob, - blob_body, - ); - let tree_body = entry("100644", b"leaf", blob); - let tree = object_id( - canopy_server::ObjectFormat::Sha1, - ObjectKind::Tree, - &tree_body, - ); - let root = put(repository, ObjectKind::Commit, &commit(tree, None)).await?; - - let push = plan("refs/heads/graph", root); - assert!(matches!( - repository.finalize_push(identity()?, push.clone()).await, - Err(InvocationError::Rejected(_)) - )); - assert!( - repository - .ref_state("refs/heads/graph", None) - .await? - .output - .is_none() - ); - put(repository, ObjectKind::Tree, &tree_body).await?; - assert!(matches!( - repository.finalize_push(identity()?, push.clone()).await, - Err(InvocationError::Rejected(_)) - )); - assert!( - !certificates(sql) - .await? - .contains(&SqlValue::Blob(root.to_vec())) - ); - put(repository, ObjectKind::Blob, blob_body).await?; - for candidates in [ - vec![root, tree, blob], - vec![blob, root], - vec![blob, tree, canopy_server::ObjectId::Sha1([71; 20])], - ] { - assert!(matches!( - application - .command::(target, identity()?, CertificateInput(candidates)) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!( - certificates(sql) - .await? - .contains(&SqlValue::Blob(blob.to_vec())) - ); - assert!( - !certificates(sql) - .await? - .contains(&SqlValue::Blob(tree.to_vec())) - ); - } - // A rejected batch must roll back edges as well as closure certificates; - // otherwise a staged tree could authorize a later raw-OID fetch. - let edges = sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT child FROM object_edges WHERE parent = ?1".into(), - parameters: vec![SqlValue::Blob(tree.to_vec())], - }], - }, - ) - .await?; - assert!(edges.output[0].rows.is_empty()); - assert!( - application - .command::(target, identity()?, CertificateInput(vec![blob, tree])) - .await? - .output - ); - // Calling the low-level command cannot bypass certificate preparation. - assert!(matches!( - application - .command::(target, identity()?, push.clone()) - .await, - Err(InvocationError::Rejected(_)) - )); - repository.finalize_push(identity()?, push).await?; - let after = certificates(sql).await?; - for oid in [root, tree, blob] { - assert!(after.contains(&SqlValue::Blob(oid.to_vec()))); - } - repository - .finalize_push(identity()?, plan("refs/heads/graph-again", root)) - .await?; - assert_eq!(certificates(sql).await?, after); - - let wrong_tree = entry("40000", b"directory", blob); - let malformed_tree = b"100644 truncated\0short"; - let wrong_parent = commit(tree, Some(blob)); - let missing_parent = commit(tree, Some(canopy_server::ObjectId::Sha1([71; 20]))); - let wrong_tag = format!( - "object {}\ntype commit\ntag wrong-kind\n\n", - hex::encode(blob) - ); - let missing_tag = format!( - "object {}\ntype tag\ntag missing\n\n", - hex::encode([72; 20]) - ); - for (name, kind, body) in [ - ("wrong-tree", ObjectKind::Tree, wrong_tree.as_slice()), - ("short-tree", ObjectKind::Tree, malformed_tree.as_slice()), - ("wrong-parent", ObjectKind::Commit, wrong_parent.as_slice()), - ( - "missing-parent", - ObjectKind::Commit, - missing_parent.as_slice(), - ), - ("wrong-tag", ObjectKind::Tag, wrong_tag.as_bytes()), - ("missing-tag", ObjectKind::Tag, missing_tag.as_bytes()), - ( - "bad-commit", - ObjectKind::Commit, - b"tree not-an-object-id\n".as_slice(), - ), - ] { - let oid = put(repository, kind, body).await?; - let name = format!("refs/tags/{name}"); - assert!(matches!( - application - .command::(target, identity()?, CertificateInput(vec![oid])) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!( - matches!( - repository - .finalize_push(identity()?, plan(&name, oid)) - .await, - Err(InvocationError::Rejected(_)) - ), - "{name}" - ); - assert!(repository.ref_state(&name, None).await?.output.is_none()); - } - assert!(matches!( - repository - .finalize_push(identity()?, plan("refs/heads/blob", blob)) - .await, - Err(InvocationError::Rejected(_)) - )); - assert_eq!(certificates(sql).await?, after); - - // A missing root must reject every ref even when another root is valid. - // Certificate preparation is independent of ref publication. - let independent = put(repository, ObjectKind::Blob, b"independent graph leaf").await?; - let mut mixed = plan( - "refs/tags/missing-root", - canopy_server::ObjectId::Sha1([73; 20]), - ); - mixed - .updates - .extend(plan("refs/tags/independent", independent).updates); - assert!(matches!( - repository.finalize_push(identity()?, mixed).await, - Err(InvocationError::Rejected(_)) - )); - assert!( - repository - .ref_state("refs/tags/independent", None) - .await? - .output - .is_none() - ); - - // Gitlinks point outside this repository. Symlinks and non-UTF8 names - // still point to local blobs; repeated edges can share a certified leaf. - let mut linked_tree = entry("120000", b"link", blob); - linked_tree.extend(entry( - "160000", - b"submodule", - canopy_server::ObjectId::Sha1([74; 20]), - )); - linked_tree.extend(entry("100755", b"\xff", blob)); - let linked_tree = put(repository, ObjectKind::Tree, &linked_tree).await?; - let linked_commit = put( - repository, - ObjectKind::Commit, - &commit(linked_tree, Some(root)), - ) - .await?; - let tag = format!( - "object {}\ntype commit\ntag v1\n\n", - hex::encode(linked_commit) - ); - let tag = put(repository, ObjectKind::Tag, tag.as_bytes()).await?; - let nested_tag = format!("object {}\ntype tag\ntag nested\n\n", hex::encode(tag)); - let nested_tag = put(repository, ObjectKind::Tag, nested_tag.as_bytes()).await?; - repository - .finalize_push(identity()?, plan("refs/tags/nested", nested_tag)) - .await?; - assert!(matches!( - repository - .finalize_push(identity()?, plan("refs/heads/tag", tag)) - .await, - Err(InvocationError::Rejected(_)) - )); - resumable(repository, sql).await?; - Ok(()) -} - -async fn resumable(repository: &RepositoryCell, sql: &SqlCell) -> Result<()> { - use canopy_server::{ObjectBatch, ObjectStorage, StoredObject}; - let mut leaves: Vec<_> = (0..270) - .map(|index| { - let body = format!("resumable leaf {index}").into_bytes(); - ( - object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &body), - body, - ) - }) - .collect(); - leaves.sort_by_key(|(oid, _)| *oid); - let mut tree = Vec::new(); - for (index, (oid, _)) in leaves.iter().enumerate() { - tree.extend(entry("100644", format!("leaf-{index:03}").as_bytes(), *oid)); - } - let (missing, body) = leaves.pop().ok_or("missing fixture leaf")?; - for group in leaves.chunks(128) { - let mut batch = ObjectBatch::default(); - for (oid, body) in group { - assert!( - batch - .try_push(StoredObject { - oid: *oid, - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body.clone()) - }) - .is_ok() - ); - } - repository.put_objects(identity()?, batch).await?; - } - let root = put(repository, ObjectKind::Tree, &tree).await?; - let generation = repository.refs_page("", None).await?.output.generation; - let plan = plan("refs/tags/resumable-graph", root); - assert!(matches!( - repository.finalize_push(identity()?, plan.clone()).await, - Err(InvocationError::Rejected(_)) - )); - let persisted = certificates(sql).await?; - assert_eq!( - leaves - .iter() - .filter(|(oid, _)| persisted.contains(&SqlValue::Blob(oid.to_vec()))) - .count(), - leaves.len() - ); - assert!(!persisted.contains(&SqlValue::Blob(root.to_vec()))); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - generation - ); - assert!( - repository - .ref_state("refs/tags/resumable-graph", None) - .await? - .output - .is_none() - ); - assert_eq!(put(repository, ObjectKind::Blob, &body).await?, missing); - repository.finalize_push(identity()?, plan).await?; - assert_eq!( - repository.refs_page("", None).await?.output.generation, - generation + 1 - ); - assert_eq!( - repository - .ref_state("refs/tags/resumable-graph", None) - .await? - .output - .and_then(|state| state.oid), - Some(root) - ); - let persisted = certificates(sql).await?; - assert!(persisted.contains(&SqlValue::Blob(root.to_vec()))); - for (oid, _) in leaves { - assert!(persisted.contains(&SqlValue::Blob(oid.to_vec()))); - } - Ok(()) -} - -// A separately encoded client invokes the registered server command. Its local -// handler cannot run, so these checks exercise the actual wire trust boundary. -struct CertificateInput(Vec); -impl cellule_runtime::codec::WireValue for CertificateInput { - fn encode( - &self, - encoder: &mut cellule_runtime::codec::BoundedEncoder, - ) -> std::result::Result<(), cellule_runtime::codec::CodecError> { - encoder.write_count(self.0.len())?; - for oid in &self.0 { - encoder.write_bytes(oid)?; - } - Ok(()) - } - fn decode( - _: &mut cellule_runtime::codec::BoundedDecoder<'_>, - ) -> std::result::Result { - Err(cellule_runtime::codec::CodecError::Invalid( - "client fixture is encode-only", - )) - } -} -struct CertificateCommand; -impl cellule_runtime::Command for CertificateCommand { - const MODULE: &'static str = "repository"; - const ID: u32 = 6; - const CODEC_VERSION: u32 = 3; - type Input = CertificateInput; - type Output = bool; - fn execute( - _: &mut cellule_runtime::registry::CommandContext<'_, '_>, - _: Self::Input, - ) -> cellule_runtime::Result> { - Err(cellule_runtime::Error::Command( - "client fixture handler must not execute", - )) - } -} diff --git a/crates/canopy-server/tests/repository_cell/issues.rs b/crates/canopy-server/tests/repository_cell/issues.rs deleted file mode 100644 index 4c4803ec..00000000 --- a/crates/canopy-server/tests/repository_cell/issues.rs +++ /dev/null @@ -1,202 +0,0 @@ -use super::*; -use canopy_server::issues::{ - CommentEdit, IssueChange, IssueEdit, IssueState, NewComment, NewIssue, -}; - -type Result = std::result::Result>; - -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -pub async fn exercise(repository: &RepositoryCell) -> Result { - let id = uuid::Uuid::new_v4().into_bytes(); - let title = "界".repeat(85); - let body = "\n".repeat(canopy_server::issues::ISSUE_BODY_LIMIT); - let new_issue = || NewIssue { - id, - title: &title, - body: &body, - }; - assert_eq!( - repository - .create_issue(identity()?, "outsider", new_issue()) - .await? - .output, - IssueChange::NotFound - ); - assert!( - repository - .issues("outsider", 0, None) - .await? - .output - .is_none() - ); - repository - .grant_member(identity()?, "canopy", "reporter", TokenScope::Read) - .await?; - let creation = identity()?; - let created = repository - .create_issue(creation, "reporter", new_issue()) - .await?; - assert_eq!(created.output, IssueChange::Applied(1)); - let replayed = repository - .create_issue(creation, "reporter", new_issue()) - .await?; - assert_eq!(created.receipt, replayed.receipt); - assert_eq!( - repository - .issue("reporter", 1) - .await? - .output - .ok_or("missing issue")? - .body, - body - ); - assert!(repository.issue("outsider", 1).await?.output.is_none()); - assert!( - repository - .issue_comments("outsider", 1, 0) - .await? - .output - .is_none() - ); - assert!( - repository - .issue_comments("canopy", 2, 0) - .await? - .output - .is_none() - ); - let new_comment = || NewComment { - id: uuid::Uuid::new_v4().into_bytes(), - body: "comment", - }; - assert_eq!( - repository - .create_issue_comment(identity()?, "reporter", 1, new_comment()) - .await? - .output, - IssueChange::Applied(1) - ); - let edit = || IssueEdit { - expected_version: 1, - title: "Closed", - body: "Finished", - state: IssueState::Closed, - }; - let comment_edit = || CommentEdit { - expected_version: 1, - body: "Edited", - }; - repository - .grant_member(identity()?, "canopy", "discussion-reader", TokenScope::Read) - .await?; - assert_eq!( - repository - .edit_issue(identity()?, "discussion-reader", 1, edit()) - .await? - .output, - IssueChange::Forbidden - ); - assert_eq!( - repository - .edit_issue_comment(identity()?, "discussion-reader", 1, 1, comment_edit()) - .await? - .output, - IssueChange::Forbidden - ); - repository - .grant_member( - identity()?, - "canopy", - "discussion-reader", - TokenScope::Write, - ) - .await?; - let update = identity()?; - let closed = repository - .edit_issue(update, "discussion-reader", 1, edit()) - .await?; - assert_eq!(closed.output, IssueChange::Applied(1)); - assert_eq!( - repository - .edit_issue(update, "discussion-reader", 1, edit()) - .await? - .receipt, - closed.receipt - ); - assert_eq!( - repository - .edit_issue(identity()?, "discussion-reader", 1, edit()) - .await? - .output, - IssueChange::Conflict - ); - assert_eq!( - repository - .edit_issue_comment(identity()?, "discussion-reader", 1, 1, comment_edit()) - .await? - .output, - IssueChange::Applied(1) - ); - repository - .revoke_member(identity()?, "canopy", "reporter") - .await?; - assert_eq!( - repository - .create_issue(identity()?, "reporter", new_issue()) - .await? - .output, - IssueChange::NotFound - ); - assert_eq!( - repository - .create_issue_comment(identity()?, "reporter", 1, new_comment()) - .await? - .output, - IssueChange::NotFound - ); - assert_eq!( - repository - .edit_issue(identity()?, "reporter", 1, edit()) - .await? - .output, - IssueChange::NotFound - ); - assert_eq!( - repository - .edit_issue_comment(identity()?, "reporter", 1, 1, comment_edit()) - .await? - .output, - IssueChange::NotFound - ); - repository - .revoke_member(identity()?, "canopy", "discussion-reader") - .await?; - assert_eq!( - repository - .issues("canopy", 0, Some(IssueState::Closed)) - .await? - .output - .ok_or("missing page")? - .len(), - 1 - ); - assert_eq!( - repository - .issue("canopy", 1) - .await? - .output - .ok_or("missing issue")? - .summary - .version, - 2 - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/merge.rs b/crates/canopy-server/tests/repository_cell/merge.rs deleted file mode 100644 index 183d5c32..00000000 --- a/crates/canopy-server/tests/repository_cell/merge.rs +++ /dev/null @@ -1,190 +0,0 @@ -use super::branch_rules::{commit, identity, plan}; -use super::*; -use canopy_server::{ - FinalizePush, - branch_rules::BranchRuleEdit, - pulls::{ - NewPull, NewReview, PullChange, PullRevision, PullState, ReviewKind, - merge::{MergeOutcome, MergeRequest, MergeStrategy}, - }, -}; -use cellule_runtime::{ - SqlCell, primitives::sql::SqlBatch, primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; -type Result = std::result::Result>; - -pub async fn verify( - repo: &RepositoryCell, - sql: &SqlCell, - app: &ApplicationHandle, - target: &CellTarget, -) -> Result { - let tree = objects::put(repo, identity()?, ObjectKind::Tree, b"") - .await? - .output; - let base = commit(repo, tree, &[], "Merge base").await?; - let source = commit(repo, tree, &[base], "Merge source").await?; - for name in ["refs/canopy", "refs/canopy/merge-candidates/forged"] { - let plan = plan(repo, name, Some(source)).await?; - assert!(matches!( - app.command::(target, identity()?, plan).await, - Err(InvocationError::Rejected(_)) - )); - assert!(repo.ref_state(name, None).await?.output.is_none()); - } - let base_ref = "refs/heads/merge-base"; - let source_ref = "refs/heads/merge-source"; - repo.finalize_push(identity()?, plan(repo, base_ref, Some(base)).await?) - .await?; - repo.finalize_push(identity()?, plan(repo, source_ref, Some(source)).await?) - .await?; - repo.grant_member(identity()?, "canopy", "merge-author", TokenScope::Read) - .await?; - let PullChange::Applied(number) = repo - .create_pull( - identity()?, - "merge-author", - NewPull { - id: uuid::Uuid::new_v4().into_bytes(), - title: "Merge", - body: "", - draft: false, - source_ref, - source_oid: &hex::encode(source), - base_ref, - base_oid: &hex::encode(base), - }, - ) - .await? - .output - else { - return Err("missing pull".into()); - }; - let edit = |version| BranchRuleEdit { - reference: base_ref.into(), - expected_version: version, - enabled: true, - deny_deletions: false, - fast_forward_only: false, - required_checks: vec![], - require_pull_request: true, - required_approvals: 1, - }; - repo.set_branch_rule(identity()?, "canopy", edit(0)).await?; - let revision = PullRevision { - pull_version: 1, - source_oid: hex::encode(source), - source_version: 1, - base_oid: hex::encode(base), - base_version: 1, - }; - let request = MergeRequest { - id: uuid::Uuid::new_v4().to_string(), - revision: revision.clone(), - strategy: MergeStrategy::FastForward, - candidate_id: None, - }; - assert!( - matches!(repo.merge_pull(identity()?,"outsider",number,request.clone()).await,Err(InvocationError::Rejected(rejected)) if rejected.output == MergeOutcome::NotFound) - ); - assert!( - matches!(repo.merge_pull(identity()?,"merge-author",number,request.clone()).await,Err(InvocationError::Rejected(rejected)) if rejected.output == MergeOutcome::Forbidden) - ); - let denied_identity = identity()?; - assert!( - matches!(repo.merge_pull(denied_identity,"canopy",number,request.clone()).await,Err(InvocationError::Rejected(rejected)) if rejected.output == MergeOutcome::ReviewsRequired) - ); - repo.review_pull( - identity()?, - "canopy", - number, - NewReview { - id: uuid::Uuid::new_v4().into_bytes(), - revision: &revision, - kind: ReviewKind::Approve, - body: "Approved", - }, - ) - .await?; - assert!( - repo.pull_review_policy("canopy", number) - .await? - .output - .unwrap() - .reviews_satisfied - ); - // The raw publication command has no merge capability, even for the owner - // and a tip with an eligible approved proposal. - let direct = plan(repo, base_ref, Some(source)).await?; - assert!( - matches!(app.command::(target,identity()?,direct).await,Err(InvocationError::Rejected(rejected)) if !rejected.output) - ); - assert!( - matches!(repo.merge_pull(denied_identity,"canopy",number,request.clone()).await,Err(InvocationError::Rejected(rejected)) if rejected.output == MergeOutcome::ReviewsRequired) - ); - // Force a constraint failure after the ref update inside the merge command. - // The injected duplicate parent record is removed after proving rollback. - let injected = uuid::Uuid::new_v4().into_bytes(); - sql.batch(identity()?,SqlBatch {statements:vec![SqlStatement { - sql:"INSERT INTO pull_merges (id,binding,pull_number,oid,merged_ms,pull_version,source_oid,source_version,base_oid,base_version) VALUES (?1,?2,?3,?4,0,1,?5,1,?4,1)".into(),parameters:vec![SqlValue::Blob(injected.to_vec()),SqlValue::Blob(vec![0;32]),SqlValue::Integer(number),SqlValue::Blob(base.to_vec()),SqlValue::Blob(source.to_vec())], - }]}).await?; - let before = repo.ref_state(base_ref, None).await?.output; - let generation = repo.refs_page("", None).await?.output.generation; - assert!( - repo.merge_pull(identity()?, "canopy", number, request.clone()) - .await - .is_err() - ); - assert_eq!(repo.ref_state(base_ref, None).await?.output, before); - assert_eq!( - repo.refs_page("", None).await?.output.generation, - generation - ); - assert_eq!( - repo.pull("canopy", number) - .await? - .output - .unwrap() - .summary - .state, - PullState::Open - ); - sql.batch( - identity()?, - SqlBatch { - statements: vec![SqlStatement { - sql: "DELETE FROM pull_merges WHERE id = ?1".into(), - parameters: vec![SqlValue::Blob(injected.to_vec())], - }], - }, - ) - .await?; - let command_id = identity()?; - let merged = repo - .merge_pull(command_id, "canopy", number, request.clone()) - .await?; - let MergeOutcome::Applied { merge: record } = &merged.output else { - return Err("merge not applied".into()); - }; - assert_eq!(record.oid, hex::encode(source)); - assert_eq!( - repo.merge_pull(command_id, "canopy", number, request.clone()) - .await? - .receipt, - merged.receipt - ); - let mut stricter = edit(1); - stricter.required_approvals = 2; - repo.set_branch_rule(identity()?, "canopy", stricter) - .await?; - assert_eq!( - repo.merge_pull(identity()?, "canopy", number, request) - .await? - .output, - merged.output - ); - let published = repo.ref_state(base_ref, None).await?.output.unwrap(); - assert_eq!(published.oid, Some(source)); - assert_eq!(published.version, 2); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/pages.rs b/crates/canopy-server/tests/repository_cell/pages.rs deleted file mode 100644 index 2b38b285..00000000 --- a/crates/canopy-server/tests/repository_cell/pages.rs +++ /dev/null @@ -1,274 +0,0 @@ -use super::*; -use canopy_server::{INLINE_OBJECT_LIMIT, ObjectBatch, ObjectStorage, StoredObject}; -use cellule_runtime::{ - SqlCell, primitives::sql::SqlBatch, primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; -use sha2::{Digest as _, Sha256}; -use std::collections::BTreeMap; - -type Result = std::result::Result>; - -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -fn inline(body: Vec) -> StoredObject { - StoredObject { - oid: object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &body), - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body), - } -} - -async fn put( - repository: &RepositoryCell, - objects: impl IntoIterator, -) -> Result { - let mut batch = ObjectBatch::default(); - for object in objects { - assert!(batch.try_push(object).is_ok()); - } - repository.put_objects(identity()?, batch).await?; - Ok(()) -} - -async fn clear(sql: &SqlCell) -> Result { - // This exercise runs first in a fresh fixture and owns all these records. - sql.batch( - identity()?, - SqlBatch { - statements: [ - "DELETE FROM object_closure", - "DELETE FROM objects", - "DELETE FROM object_chunks", - "DELETE FROM object_uploads", - ] - .into_iter() - .map(|sql| SqlStatement { - sql: sql.into(), - parameters: vec![], - }) - .collect(), - }, - ) - .await?; - Ok(()) -} - -async fn scan( - repository: &RepositoryCell, -) -> Result> { - let mut after = None; - let mut objects = BTreeMap::new(); - for _ in 0..10 { - let page = repository.object_page(after).await?.output; - if page.is_empty() { - return Ok(objects); - } - assert!(page.len() <= 128); - let bytes: usize = page - .iter() - .map(|object| match &object.storage { - ObjectStorage::Inline(body) => body.len(), - _ => 0, - }) - .sum(); - assert!(bytes <= INLINE_OBJECT_LIMIT); - for object in page { - assert!(after.is_none_or(|old| object.oid > old)); - after = Some(object.oid); - assert!(objects.insert(object.oid, object).is_none()); - } - } - Err("object pagination did not finish".into()) -} - -fn before(oid: canopy_server::ObjectId) -> Option { - let mut bytes = match oid { - canopy_server::ObjectId::Sha1(bytes) => bytes.to_vec(), - canopy_server::ObjectId::Sha256(bytes) => bytes.to_vec(), - }; - for byte in bytes.iter_mut().rev() { - if *byte != 0 { - *byte -= 1; - return canopy_server::ObjectId::try_from(bytes).ok(); - } - *byte = u8::MAX; - } - None -} - -pub async fn exercise(repository: &RepositoryCell, sql: &SqlCell) -> Result { - assert!(repository.object_page(None).await?.output.is_empty()); - let mut expected = BTreeMap::new(); - for start in [0, 128] { - let batch = (start..start + 128) - .map(|n| { - let body = format!("object-page-{n}").into_bytes(); - let object = inline(body.clone()); - expected.insert(object.oid, body); - object - }) - .collect::>(); - put(repository, batch).await?; - } - let first = repository.object_page(None).await?.output; - assert_eq!(first.len(), 128); - let second = repository - .object_page(Some(first.last().ok_or("missing first page")?.oid)) - .await? - .output; - assert_eq!(second.len(), 128); - assert!( - repository - .object_page(Some(second.last().ok_or("missing second page")?.oid)) - .await? - .output - .is_empty() - ); - let actual = scan(repository).await?; - assert_eq!( - actual.keys().copied().collect::>(), - expected.keys().copied().collect::>() - ); - for (oid, object) in actual { - let ObjectStorage::Inline(body) = object.storage else { - return Err("inline body missing".into()); - }; - assert_eq!(body, expected[&oid]); - } - clear(sql).await?; - expected.clear(); - - let halves: Vec<_> = [11, 12] - .map(|byte| inline(vec![byte; INLINE_OBJECT_LIMIT / 2])) - .into(); - for object in &halves { - let ObjectStorage::Inline(body) = &object.storage else { - return Err("inline body missing".into()); - }; - expected.insert(object.oid, body.clone()); - } - put(repository, halves).await?; - let full = repository.object_page(None).await?.output; - assert_eq!(full.len(), 2); - assert_eq!( - full.iter() - .map(|object| match &object.storage { - ObjectStorage::Inline(body) => body.len(), - _ => 0, - }) - .sum::(), - INLINE_OBJECT_LIMIT - ); - for body in [vec![13; INLINE_OBJECT_LIMIT], vec![14], Vec::new()] { - let object = inline(body.clone()); - expected.insert(object.oid, body); - put(repository, [object]).await?; - } - let chunked = vec![15; INLINE_OBJECT_LIMIT + 1]; - let staged = repository - .stage_object(identity()?, ObjectKind::Tag, &chunked) - .await?; - let chunked_oid = staged.oid; - let ObjectStorage::Chunked { upload, .. } = staged.storage else { - return Err("missing chunk reference".into()); - }; - put( - repository, - [StoredObject { - oid: chunked_oid, - kind: ObjectKind::Tag, - storage: ObjectStorage::Chunked { - upload, - size: chunked.len() as u64, - blake3: *blake3::hash(&chunked).as_bytes(), - }, - }], - ) - .await?; - let external = vec![16; INLINE_OBJECT_LIMIT + 1]; - let external_oid = object_id( - canopy_server::ObjectFormat::Sha1, - ObjectKind::Blob, - &external, - ); - let sha256: [u8; 32] = Sha256::digest(&external).into(); - put( - repository, - [StoredObject { - oid: external_oid, - kind: ObjectKind::Blob, - storage: ObjectStorage::External { - size: external.len() as u64, - blake3: *blake3::hash(&external).as_bytes(), - sha256, - }, - }], - ) - .await?; - let mut actual = scan(repository).await?; - assert_eq!(actual.len(), expected.len() + 2); - for (oid, expected_body) in &expected { - let object = actual.remove(oid).ok_or("missing inline object")?; - let ObjectStorage::Inline(body) = object.storage else { - return Err("inline body missing".into()); - }; - assert_eq!(&body, expected_body); - } - let descriptor = actual - .remove(&chunked_oid) - .ok_or("missing chunked object")?; - assert!( - matches!(descriptor.storage, ObjectStorage::Chunked { upload: stored, size, blake3: digest } if stored == upload && size == chunked.len() as u64 && digest == *blake3::hash(&chunked).as_bytes()) - ); - let descriptor = actual - .remove(&external_oid) - .ok_or("missing external object")?; - assert!( - matches!(descriptor.storage, ObjectStorage::External { size, blake3: digest, sha256: stored } if size == external.len() as u64 && digest == *blake3::hash(&external).as_bytes() && stored == sha256) - ); - - let damaged = object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &[14]); - for (statement, parameters) in [ - ( - "UPDATE objects SET digest = zeroblob(32) WHERE oid = ?1", - vec![SqlValue::Blob(damaged.to_vec())], - ), - ( - "UPDATE objects SET body = ?2, digest = ?3 WHERE oid = ?1", - vec![ - SqlValue::Blob(damaged.to_vec()), - SqlValue::Blob(vec![17]), - SqlValue::Blob(blake3::hash(&[17]).as_bytes().to_vec()), - ], - ), - ( - "UPDATE objects SET body = zeroblob(?2), size = ?2 WHERE oid = ?1", - vec![ - SqlValue::Blob(damaged.to_vec()), - SqlValue::Integer(INLINE_OBJECT_LIMIT as i64 + 1), - ], - ), - ] { - sql.batch( - identity()?, - SqlBatch { - statements: vec![SqlStatement { - sql: statement.into(), - parameters, - }], - }, - ) - .await?; - assert!(repository.object_page(before(damaged)).await.is_err()); - } - clear(sql).await?; - assert!(repository.object_page(None).await?.output.is_empty()); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/pulls.rs b/crates/canopy-server/tests/repository_cell/pulls.rs deleted file mode 100644 index 68d461c6..00000000 --- a/crates/canopy-server/tests/repository_cell/pulls.rs +++ /dev/null @@ -1,271 +0,0 @@ -use super::*; -use canopy_server::pulls::{ - NewPull, NewReview, PullChange, PullEdit, PullRevision, PullState, ReviewKind, -}; -type Result = std::result::Result>; -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} -async fn number(repo: &RepositoryCell, actor: &str, input: NewPull<'_>) -> Result { - let PullChange::Applied(number) = repo.create_pull(identity()?, actor, input).await?.output - else { - return Err("pull creation failed".into()); - }; - Ok(number) -} -pub async fn verify(repo: &RepositoryCell) -> Result { - let tree = objects::put(repo, identity()?, ObjectKind::Tree, b"") - .await? - .output; - let body = format!( - "tree {}\nauthor Test 0 +0000\ncommitter Test 0 +0000\n\nPull base\n", - hex::encode(tree) - ); - let base = objects::put(repo, identity()?, ObjectKind::Commit, body.as_bytes()) - .await? - .output; - let body = format!( - "tree {}\nparent {}\nauthor Test 0 +0000\ncommitter Test 0 +0000\n\nPull source\n", - hex::encode(tree), - hex::encode(base) - ); - let source = objects::put(repo, identity()?, ObjectKind::Commit, body.as_bytes()) - .await? - .output; - repo.finalize_push( - identity()?, - PushPlan { - actor: "canopy".into(), - updates: vec![ - RefUpdate { - name: "refs/heads/pull-base".into(), - expected: None, - new_oid: Some(base), - }, - RefUpdate { - name: "refs/heads/pull-source".into(), - expected: None, - new_oid: Some(source), - }, - ], - }, - ) - .await?; - repo.grant_member(identity()?, "canopy", "pull-author", TokenScope::Read) - .await?; - repo.grant_member(identity()?, "canopy", "pull-reviewer", TokenScope::Write) - .await?; - let base_oid = hex::encode(base); - let source_oid = hex::encode(source); - let id = uuid::Uuid::new_v4().into_bytes(); - let title = "é".repeat(128); - let body = "x".repeat(16384); - let new = || NewPull { - id, - title: &title, - body: &body, - draft: false, - source_ref: "refs/heads/pull-source", - source_oid: &source_oid, - base_ref: "refs/heads/pull-base", - base_oid: &base_oid, - }; - assert_eq!( - repo.create_pull(identity()?, "outsider", new()) - .await? - .output, - PullChange::NotFound - ); - let create_id = identity()?; - let created = repo.create_pull(create_id, "pull-author", new()).await?; - assert_eq!( - repo.create_pull(create_id, "pull-author", new()) - .await? - .receipt, - created.receipt - ); - let PullChange::Applied(pull) = created.output else { - return Err("missing pull".into()); - }; - let other = number( - repo, - "pull-author", - NewPull { - id: uuid::Uuid::new_v4().into_bytes(), - ..new() - }, - ) - .await?; - assert!(repo.pull("outsider", pull).await?.output.is_none()); - assert!(repo.pulls("outsider", 0, None).await?.output.is_none()); - assert!( - repo.pull_reviews("outsider", pull, 0) - .await? - .output - .is_none() - ); - let revision = PullRevision { - pull_version: 1, - source_oid: source_oid.clone(), - source_version: 1, - base_oid: base_oid.clone(), - base_version: 1, - }; - let review_id = uuid::Uuid::new_v4().into_bytes(); - let review = || NewReview { - id: review_id, - revision: &revision, - kind: ReviewKind::Approve, - body: &body, - }; - assert_eq!( - repo.review_pull(identity()?, "pull-author", pull, review()) - .await? - .output, - PullChange::Forbidden - ); - assert_eq!( - repo.review_pull(identity()?, "outsider", pull, review()) - .await? - .output, - PullChange::NotFound - ); - let review_mutation = identity()?; - let result = repo - .review_pull(review_mutation, "pull-reviewer", pull, review()) - .await?; - assert_eq!( - repo.review_pull(review_mutation, "pull-reviewer", pull, review()) - .await? - .receipt, - result.receipt - ); - assert_eq!( - repo.review_pull(identity()?, "pull-reviewer", other, review()) - .await? - .output, - PullChange::Conflict - ); - assert!(repo.pull_reviews("canopy", pull, 0).await?.output.unwrap()[0].applicable); - let stale = PullRevision { - source_version: 2, - ..revision.clone() - }; - assert_eq!( - repo.review_pull( - identity()?, - "pull-reviewer", - pull, - NewReview { - id: uuid::Uuid::new_v4().into_bytes(), - revision: &stale, - ..review() - } - ) - .await? - .output, - PullChange::Conflict - ); - // Only authorized ACL changes can advance the reviewer's grant version. - assert!( - !repo - .revoke_member(identity()?, "pull-author", "pull-reviewer") - .await? - .output - ); - assert!(repo.pull_reviews("canopy", pull, 0).await?.output.unwrap()[0].applicable); - repo.revoke_member(identity()?, "canopy", "pull-reviewer") - .await?; - assert_eq!( - repo.review_pull(identity()?, "pull-reviewer", pull, review()) - .await? - .output, - PullChange::NotFound - ); - repo.grant_member(identity()?, "canopy", "pull-reviewer", TokenScope::Write) - .await?; - repo.review_pull(identity()?, "pull-reviewer", pull, review()) - .await?; - assert!(!repo.pull_reviews("canopy", pull, 0).await?.output.unwrap()[0].applicable); - let owner_review = NewReview { - id: uuid::Uuid::new_v4().into_bytes(), - ..review() - }; - assert!(matches!( - repo.review_pull(identity()?, "canopy", pull, owner_review) - .await? - .output, - PullChange::Applied(_) - )); - let edit = || PullEdit { - expected_version: 1, - title: "Edited", - body: "Changed", - state: PullState::Closed, - draft: false, - }; - assert_eq!( - repo.edit_pull(identity()?, "outsider", pull, edit()) - .await? - .output, - PullChange::NotFound - ); - assert_eq!( - repo.edit_pull(identity()?, "pull-author", pull, edit()) - .await? - .output, - PullChange::Applied(pull) - ); - assert_eq!( - repo.edit_pull(identity()?, "pull-author", pull, edit()) - .await? - .output, - PullChange::Conflict - ); - repo.create_pull(identity()?, "pull-author", new()).await?; - assert_eq!( - repo.pull("canopy", pull) - .await? - .output - .unwrap() - .summary - .title, - "Edited" - ); - assert!( - repo.pull_reviews("canopy", pull, 0) - .await? - .output - .unwrap() - .iter() - .all(|r| !r.applicable) - ); - repo.revoke_member(identity()?, "canopy", "pull-author") - .await?; - assert_eq!( - repo.create_pull(identity()?, "pull-author", new()) - .await? - .output, - PullChange::NotFound - ); - assert_eq!( - repo.edit_pull( - identity()?, - "pull-author", - pull, - PullEdit { - expected_version: 2, - ..edit() - } - ) - .await? - .output, - PullChange::NotFound - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/rebase.rs b/crates/canopy-server/tests/repository_cell/rebase.rs deleted file mode 100644 index 94a5330f..00000000 --- a/crates/canopy-server/tests/repository_cell/rebase.rs +++ /dev/null @@ -1,123 +0,0 @@ -use super::branch_rules::{commit, identity, plan}; -use super::*; -use canopy_server::pulls::{ - NewPull, PullChange, PullRevision, - candidates::{CandidateRequest, CandidateResult}, - merge::{MergeOutcome, MergeRequest, MergeStrategy}, -}; -use cellule_runtime::{ - SqlCell, primitives::sql::SqlBatch, primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; -type Result = std::result::Result>; - -pub async fn verify(repo: &RepositoryCell, sql: &SqlCell) -> Result { - let tree = objects::put(repo, identity()?, ObjectKind::Tree, b"") - .await? - .output; - let base = commit(repo, tree, &[], "Rebase base").await?; - let first = commit(repo, tree, &[base], "First").await?; - let source = commit(repo, tree, &[first], "Second").await?; - let base_ref = "refs/heads/rebase-base"; - let source_ref = "refs/heads/rebase-source"; - for (reference, commit) in [(base_ref, base), (source_ref, source)] { - repo.finalize_push(identity()?, plan(repo, reference, Some(commit)).await?) - .await?; - } - let PullChange::Applied(number) = repo - .create_pull( - identity()?, - "canopy", - NewPull { - id: uuid::Uuid::new_v4().into_bytes(), - title: "Certified rebase", - body: "", - draft: false, - source_ref, - source_oid: &hex::encode(source), - base_ref, - base_oid: &hex::encode(base), - }, - ) - .await? - .output - else { - return Err("pull not created".into()); - }; - let revision = PullRevision { - pull_version: 1, - source_oid: hex::encode(source), - source_version: 1, - base_oid: hex::encode(base), - base_version: 1, - }; - let rewritten = |parent, author, message| { - format!( - "tree {}\nparent {}\nauthor {author} 0 +0000\ncommitter canopy 1 +0000\n\n{message}\n", - hex::encode(tree), - hex::encode(parent) - ) - }; - let body = rewritten(base, "Test", "First"); - let correct_first = objects::put(repo, identity()?, ObjectKind::Commit, body.as_bytes()) - .await? - .output; - for (name, parent, author, message, accepted) in [ - ("skip", base, "Test", "Second", false), - ("author", correct_first, "Forged", "Second", false), - ("message", correct_first, "Test", "Replacement", false), - ("valid", correct_first, "Test", "Second", true), - ] { - let body = rewritten(parent, author, message); - let tip = objects::put(repo, identity()?, ObjectKind::Commit, body.as_bytes()) - .await? - .output; - // Certify genuine Git objects first. A valid graph alone must never - // certify a chain with a dropped commit or changed author/message. - repo.finalize_push( - identity()?, - plan(repo, &format!("refs/heads/rebase-proof-{name}"), Some(tip)).await?, - ) - .await?; - let id = uuid::Uuid::new_v4(); - let request = CandidateRequest { - id: id.to_string(), - revision: revision.clone(), - strategy: MergeStrategy::Rebase, - message: String::new(), - }; - let result = CandidateResult::Ready { - oid: hex::encode(tip), - tree_oid: hex::encode(tree), - }; - sql.batch(identity()?, SqlBatch { statements: vec![SqlStatement { - sql: "INSERT INTO merge_candidates (id,binding,pull_number,actor,request,created_ms,result,source_oid,base_oid,oid) VALUES (?1,?2,?3,'canopy',?4,1000,?5,?6,?7,?8)".into(), - parameters: vec![SqlValue::Blob(id.as_bytes().to_vec()), SqlValue::Blob(vec![0;32]), SqlValue::Integer(number), SqlValue::Text(serde_json::to_string(&request)?), SqlValue::Text(serde_json::to_string(&result)?), SqlValue::Blob(source.to_vec()), SqlValue::Blob(base.to_vec()), SqlValue::Blob(tip.to_vec())], - }] }).await?; - let publish = MergeRequest { - id: uuid::Uuid::new_v4().to_string(), - revision: revision.clone(), - strategy: MergeStrategy::Rebase, - candidate_id: Some(id.to_string()), - }; - let outcome = repo - .merge_pull(identity()?, "canopy", number, publish) - .await; - if accepted { - assert!(matches!(outcome?.output, MergeOutcome::Applied { .. })); - assert_eq!( - repo.ref_state(base_ref, None).await?.output.unwrap().oid, - Some(tip) - ); - } else { - assert!( - matches!(outcome, Err(InvocationError::Rejected(result)) if result.output == MergeOutcome::Conflict), - "{name}" - ); - assert_eq!( - repo.ref_state(base_ref, None).await?.output.unwrap().oid, - Some(base) - ); - } - } - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/visibility.rs b/crates/canopy-server/tests/repository_cell/visibility.rs deleted file mode 100644 index fdbe50e7..00000000 --- a/crates/canopy-server/tests/repository_cell/visibility.rs +++ /dev/null @@ -1,116 +0,0 @@ -use super::*; -use canopy_server::{ReadIdentity, Visibility}; - -fn identity() -> Result> { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -pub async fn verify(repository: &RepositoryCell) -> Result<(), Box> { - let initial = repository.visibility().await?.output; - assert_eq!(initial.visibility, Visibility::Private); - assert_eq!( - repository - .access_level(ReadIdentity::Anonymous, None) - .await? - .output, - None - ); - repository - .grant_member(identity()?, "canopy", "writer", TokenScope::Write) - .await?; - for actor in ["writer", "outsider"] { - assert!( - !repository - .set_visibility(identity()?, actor, initial.generation, Visibility::Public) - .await? - .output - ); - } - for generation in [-1, i64::MAX] { - assert!( - repository - .set_visibility(identity()?, "canopy", generation, Visibility::Public) - .await - .is_err() - ); - } - let request = identity()?; - let published = repository - .set_visibility(request, "canopy", initial.generation, Visibility::Public) - .await?; - assert!(published.output); - let replayed = repository - .set_visibility(request, "canopy", initial.generation, Visibility::Public) - .await?; - assert_eq!(replayed.receipt, published.receipt); - assert!(replayed.output); - for (reader, role) in [ - (ReadIdentity::Anonymous, TokenScope::Read), - (ReadIdentity::Account("outsider"), TokenScope::Read), - (ReadIdentity::Account("writer"), TokenScope::Write), - (ReadIdentity::Account("canopy"), TokenScope::Admin), - ] { - assert_eq!( - repository - .access_level(reader, Some(published.receipt)) - .await? - .output, - Some(role) - ); - } - let public = repository.visibility().await?.output; - assert_eq!(public.generation, initial.generation + 1); - let head = repository.ref_state("refs/heads/main", None).await?.output; - assert!(matches!( - repository - .finalize_push( - identity()?, - PushPlan { - actor: "outsider".into(), - updates: vec![RefUpdate { - name: "refs/heads/main".into(), - expected: head, - new_oid: None - }], - } - ) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!( - repository - .set_visibility( - identity()?, - "canopy", - public.generation, - Visibility::Private - ) - .await? - .output - ); - assert_eq!( - repository - .access_level(ReadIdentity::Anonymous, None) - .await? - .output, - None - ); - // Returning to private must not make an old public request valid again. - assert!( - !repository - .set_visibility( - identity()?, - "canopy", - initial.generation, - Visibility::Public - ) - .await? - .output - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/smart_http/cache_admission.rs b/crates/canopy-server/tests/smart_http/cache_admission.rs deleted file mode 100644 index acdb8a80..00000000 --- a/crates/canopy-server/tests/smart_http/cache_admission.rs +++ /dev/null @@ -1,291 +0,0 @@ -use super::*; - -pub async fn verify( - root: &Path, - repository: &RepositoryCell, - budget: &DiskBudget, - client: &reqwest::Client, - url: &str, -) -> Result<(), Box> { - let source = root.join("quota-source"); - run_git( - None, - &["init", "-b", "main", source.to_str().ok_or("invalid path")?], - ) - .await?; - run_git(Some(&source), &["config", "user.name", "Canopy Test"]).await?; - run_git( - Some(&source), - &["config", "user.email", "canopy@example.invalid"], - ) - .await?; - // A small delta pack expands into many loose objects below Git's unpack limit. - // This distinguishes native cache admission from request-spool admission. - let mut common = vec![0; 8192]; - blake3::Hasher::new().finalize_xof().fill(&mut common); - for index in 0..40 { - let mut body = common.clone(); - body.extend_from_slice(format!("file {index}\n").as_bytes()); - tokio::fs::write(source.join(format!("file-{index:02}")), body).await?; - } - run_git(Some(&source), &["add", "."]).await?; - run_git(Some(&source), &["commit", "-m", "Cache admission fixture"]).await?; - let commit = run_git(Some(&source), &["rev-parse", "HEAD"]).await?; - let commit = std::str::from_utf8(&commit)?.trim(); - run_git(Some(&source), &["tag", "-a", "inner", "-m", "Inner tag"]).await?; - run_git( - Some(&source), - &["tag", "-a", "outer", "inner", "-m", "Nested tag"], - ) - .await?; - let mut advertised = vec![("refs/heads/quota", commit.to_owned())]; - for (name, revision) in [ - ("refs/tags/inner", "inner"), - ("refs/tags/outer", "outer"), - ("refs/tags/tree", "HEAD^{tree}"), - ("refs/tags/blob", "HEAD:file-00"), - ] { - let oid = run_git(Some(&source), &["rev-parse", revision]).await?; - advertised.push((name, std::str::from_utf8(&oid)?.trim().to_owned())); - } - let pack = run_git( - Some(&source), - &["pack-objects", "--stdout", "--all", "--window=50"], - ) - .await?; - assert!( - pack.len() < common.len() * 4, - "fixture requires delta compression" - ); - let mut body = Vec::new(); - for (index, (name, oid)) in advertised.iter().enumerate() { - let capabilities = if index == 0 { "\0report-status" } else { "" }; - let command = format!("{} {oid} {name}{capabilities}\n", "0".repeat(40)); - body.extend_from_slice(format!("{:04x}{command}", command.len() + 4).as_bytes()); - } - body.extend_from_slice(b"0000"); - body.extend_from_slice(&pack); - encoded_input::reject_corruption_and_expansion(repository, budget, client, url, &body).await?; - let id = uuid::Uuid::new_v4().to_string(); - let request = |id: &str| { - client - .post(format!("{url}/git-receive-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-receive-pack-request") - .header("Idempotency-Key", id) - .body(body.clone()) - }; - let before = repository.refs_page("", None).await?.output.generation; - let retained = budget.used(); - let occupied = budget.try_reserve(budget.capacity() - retained - body.len() as u64 - 4096)?; - let response = request(&id).send().await?; - assert_eq!(response.status(), reqwest::StatusCode::OK); - let refused = response.bytes().await?; - let report = String::from_utf8_lossy(&refused); - for (name, _) in &advertised { - assert!( - report.contains(&format!( - "ng {name} Canopy push failed before publication: cache disk budget exhausted" - )), - "{report}" - ); - } - assert!( - repository - .ref_state("refs/heads/quota", None) - .await? - .output - .is_none() - ); - assert!(repository.object_page(None).await?.output.is_empty()); - assert_eq!(budget.used(), occupied.bytes() + retained); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - before - ); - drop(occupied); - - // Resource recovery cannot change the recorded outcome of the same push ID. - let replay = request(&id) - .send() - .await? - .error_for_status()? - .bytes() - .await?; - assert_eq!(replay, refused); - let id = uuid::Uuid::new_v4().to_string(); - let response = request(&id) - .send() - .await? - .error_for_status()? - .bytes() - .await?; - assert!( - response - .windows(b"ok refs/heads/quota".len()) - .any(|part| part == b"ok refs/heads/quota") - ); - let expected: canopy_server::ObjectId = - hex::decode(commit)?.try_into().map_err(|_| "invalid OID")?; - assert_eq!( - repository - .ref_state("refs/heads/quota", None) - .await? - .output - .and_then(|state| state.oid), - Some(expected) - ); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - before + 1 - ); - let replay = request(&id) - .header("Content-Encoding", "identity") - .send() - .await? - .error_for_status()? - .bytes() - .await?; - assert_eq!(replay, response); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - before + 1 - ); - assert_eq!( - request(&id) - .header("Content-Encoding", "gzip") - .send() - .await? - .status(), - reqwest::StatusCode::CONFLICT - ); - - // Failed hydration may retain completed immutable objects and conservative - // write charges. It must not publish a partial ref generation; retry resumes - // from the verified cache after external pressure is removed. - let retained = budget.used(); - let occupied = budget.try_reserve(budget.capacity() - retained - 512)?; - let advertisement = || { - client - .get(format!("{url}/info/refs?service=git-upload-pack")) - .bearer_auth("local-test-token") - }; - // Protocol v2 initially advertises capabilities only. Published history - // exceeds the available cache budget, but this exchange needs no objects. - let mut capabilities = None; - for query in ["service=git-upload-pack", "service=git%2Dupload%2Dpack"] { - let response = client - .get(format!("{url}/info/refs?{query}")) - .bearer_auth("local-test-token") - .header("Git-Protocol", "version=2") - .send() - .await?; - assert_eq!(response.status(), reqwest::StatusCode::OK); - let body = response.bytes().await?; - assert!(body.starts_with(b"000eversion 2\n")); - if let Some(expected) = &capabilities { - assert_eq!(&body, expected); - } - capabilities = Some(body); - assert_eq!(budget.used(), occupied.bytes() + retained); - } - let response = advertisement().send().await?; - assert_eq!(response.status(), reqwest::StatusCode::INSUFFICIENT_STORAGE); - assert_eq!(response.text().await?, "Git cache disk budget exhausted"); - assert!(budget.used() >= occupied.bytes()); - assert!(budget.used() <= budget.capacity()); - drop(occupied); - // Ref discovery fits even when expanding the repository history cannot. - // Nested tags and direct tree/blob refs must preserve native peeling rules. - let retained = budget.used(); - let occupied = budget.try_reserve(budget.capacity() - retained - 64 * 1024)?; - for service in ["git-upload-pack", "git-receive-pack"] { - let bytes = client - .get(format!("{url}/info/refs?service={service}")) - .bearer_auth("local-test-token") - .send() - .await? - .error_for_status()? - .bytes() - .await?; - for (name, oid) in &advertised { - let entry = format!("{oid} {name}"); - assert!( - bytes - .windows(entry.len()) - .any(|part| part == entry.as_bytes()) - ); - } - if service == "git-upload-pack" { - let peeled = format!("{commit} refs/tags/outer^{{}}"); - assert!( - bytes - .windows(peeled.len()) - .any(|part| part == peeled.as_bytes()) - ); - } - assert_eq!(budget.used(), occupied.bytes() + retained); - } - let bytes = client - .post(format!("{url}/git-upload-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-upload-pack-request") - .header("Git-Protocol", "version=2") - .body("0014command=ls-refs\n00010009peel\n000csymrefs\n0000") - .send() - .await? - .error_for_status()? - .bytes() - .await?; - let outer = &advertised[2].1; - let peeled = format!("{outer} refs/tags/outer peeled:{commit}"); - assert!( - bytes - .windows(peeled.len()) - .any(|part| part == peeled.as_bytes()) - ); - assert_eq!(budget.used(), occupied.bytes() + retained); - let want = format!("want {commit}\n"); - let fetch = || { - client - .post(format!("{url}/git-upload-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-upload-pack-request") - .body(format!("{:04x}{want}00000009done\n", want.len() + 4)) - }; - assert_eq!( - fetch().send().await?.status(), - reqwest::StatusCode::INSUFFICIENT_STORAGE - ); - drop(occupied); - fetch().send().await?.error_for_status()?.bytes().await?; - assert!(budget.used() > common.len() as u64 * 40); - let retained = budget.used(); - // A current full cache can serve discovery without recopying its targets. - let occupied = budget.try_reserve(budget.capacity() - retained - 512)?; - advertisement() - .send() - .await? - .error_for_status()? - .bytes() - .await?; - assert_eq!(budget.used(), occupied.bytes() + retained); - drop(occupied); - encoded_input::delete_with_admission_retry(repository, budget, client, url, commit).await?; - assert_eq!(budget.used(), retained); - run_git( - Some(&source), - &[ - "-c", - "http.extraHeader=Authorization: Bearer local-test-token", - "push", - url, - ":refs/tags/inner", - ":refs/tags/outer", - ":refs/tags/tree", - ":refs/tags/blob", - ], - ) - .await?; - Ok(()) -} diff --git a/crates/canopy-server/tests/smart_http/cache_reuse.rs b/crates/canopy-server/tests/smart_http/cache_reuse.rs deleted file mode 100644 index ec86baa4..00000000 --- a/crates/canopy-server/tests/smart_http/cache_reuse.rs +++ /dev/null @@ -1,87 +0,0 @@ -use super::*; -use std::{collections::BTreeMap, path::PathBuf, time::SystemTime}; - -fn cached_objects(root: &Path) -> std::io::Result> { - let mut objects = BTreeMap::new(); - for entry in std::fs::read_dir(root)? { - let entry = entry?; - if !entry - .file_name() - .to_string_lossy() - .starts_with("canopy-git-") - { - continue; - } - let git = entry.path().join("repo.git"); - if git.join("objects/info/alternates").exists() { - continue; - } - for shard in std::fs::read_dir(git.join("objects"))? { - let shard = shard?; - let name = shard.file_name(); - if name.len() != 2 || !name.as_encoded_bytes().iter().all(u8::is_ascii_hexdigit) { - continue; - } - for object in std::fs::read_dir(shard.path())? { - let object = object?; - if object.file_name().len() == 38 { - let metadata = object.metadata()?; - objects.insert(object.path(), (metadata.len(), metadata.modified()?)); - } - } - } - } - Ok(objects) -} - -pub async fn verify( - root: &Path, - local: &Path, - url: &str, -) -> Result<(), Box> { - let before = cached_objects(root)?; - assert!(!before.is_empty()); - tokio::fs::write(local.join("cache-reuse.txt"), b"only this blob is new\n").await?; - run_git(Some(local), &["add", "cache-reuse.txt"]).await?; - run_git(Some(local), &["commit", "-m", "Incremental object cache"]).await?; - let commit = run_git(Some(local), &["rev-parse", "HEAD"]).await?; - run_git( - Some(local), - &["-c", AUTH, "push", url, "HEAD:refs/heads/cache-reuse"], - ) - .await?; - run_git(None, &["-c", AUTH, "ls-remote", url]).await?; - assert_eq!(cached_objects(root)?, before); - let clone = root.join("cache-reuse-clone"); - run_git( - None, - &[ - "-c", - AUTH, - "clone", - "--single-branch", - "--branch", - "cache-reuse", - "--no-checkout", - url, - clone.to_str().ok_or("clone path")?, - ], - ) - .await?; - let after = cached_objects(root)?; - // A new blob, tree and commit are hydrated. Existing bodies keep their - // actual files across receive-pack and the next fetch of the published ref generation. - assert_eq!(after.len(), before.len() + 3); - for (path, metadata) in before { - assert_eq!(after.get(&path), Some(&metadata)); - } - assert_eq!(run_git(Some(&clone), &["rev-parse", "HEAD"]).await?, commit); - assert_eq!( - run_git(Some(&clone), &["show", "HEAD:cache-reuse.txt"]).await?, - b"only this blob is new\n" - ); - run_git(Some(&clone), &["fsck", "--strict", "--full"]).await?; - Ok(()) -} - -const AUTH: &str = "http.extraHeader=Authorization: Bearer local-test-token"; diff --git a/crates/canopy-server/tests/smart_http/encoded_input.rs b/crates/canopy-server/tests/smart_http/encoded_input.rs deleted file mode 100644 index 5d486de7..00000000 --- a/crates/canopy-server/tests/smart_http/encoded_input.rs +++ /dev/null @@ -1,165 +0,0 @@ -use super::*; -use std::io::Write; - -fn gzip(bytes: &[u8]) -> Result, Box> { - let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); - encoder.write_all(bytes)?; - Ok(encoder.finish()?) -} - -pub async fn reject_corruption_and_expansion( - repository: &RepositoryCell, - budget: &DiskBudget, - client: &reqwest::Client, - url: &str, - push: &[u8], -) -> Result<(), Box> { - let retained = budget.used(); - let valid = gzip(push)?; - let mut checksum = valid.clone(); - let crc = checksum.len() - 8; - checksum[crc] ^= 1; - let mut trailing = valid.clone(); - trailing.extend(b"invalid trailing bytes"); - for wire in [checksum, valid[..valid.len() - 1].to_vec(), trailing] { - let id = uuid::Uuid::new_v4().to_string(); - for _ in 0..2 { - let response = client - .post(format!("{url}/git-receive-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-receive-pack-request") - .header("Content-Encoding", "gzip") - .header("Idempotency-Key", &id) - .body(wire.clone()) - .send() - .await?; - assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST); - assert_eq!(response.text().await?, "Invalid Git gzip stream"); - assert_eq!(budget.used(), retained); - } - assert!( - repository - .ref_state("refs/heads/quota", None) - .await? - .output - .is_none() - ); - assert!(repository.object_page(None).await?.output.is_empty()); - } - let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - for _ in 0..1024 { - encoder.write_all(&[b'x'; 64 * 1024])?; - } - encoder.write_all(b"x")?; - let response = client - .post(format!("{url}/git-upload-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-upload-pack-request") - .header("Content-Encoding", "gzip") - .body(encoder.finish()?) - .send() - .await?; - assert_eq!(response.status(), reqwest::StatusCode::PAYLOAD_TOO_LARGE); - assert_eq!(budget.used(), retained); - - // A valid decoded request above Git CGI's default 10 MiB buffer remains - // below Canopy's 64 MiB limit. No ref matches these long prefixes. - let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - encoder.write_all(b"0014command=ls-refs\n0001")?; - let prefix = format!("ref-prefix {}\n", "z".repeat(60_000)); - let packet = format!("{:04x}{prefix}", prefix.len() + 4); - for _ in 0..190 { - encoder.write_all(packet.as_bytes())?; - } - encoder.write_all(b"0000")?; - assert!(packet.len() * 190 > 10 * 1024 * 1024); - let response = client - .post(format!("{url}/git-upload-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-upload-pack-request") - .header("Content-Encoding", "gzip") - .header("Git-Protocol", "version=2") - .body(encoder.finish()?) - .send() - .await? - .error_for_status()? - .bytes() - .await?; - assert_eq!(response, b"0000".as_slice()); - Ok(()) -} - -pub async fn delete_with_admission_retry( - repository: &RepositoryCell, - budget: &DiskBudget, - client: &reqwest::Client, - url: &str, - commit: &str, -) -> Result<(), Box> { - let command = format!( - "{commit} {} refs/heads/quota\0report-status delete-refs\n", - "0".repeat(40) - ); - let wire = gzip(format!("{:04x}{command}0000", command.len() + 4).as_bytes())?; - let id = uuid::Uuid::new_v4().to_string(); - let request = || { - client - .post(format!("{url}/git-receive-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-receive-pack-request") - .header("Content-Encoding", "gzip") - .header("Idempotency-Key", &id) - .body(wire.clone()) - }; - let warm = budget.used(); - let generation = repository.refs_page("", None).await?.output.generation; - let occupied = budget.try_reserve(budget.capacity() - warm - wire.len() as u64 - 1)?; - let response = request().send().await?; - assert_eq!(response.status(), reqwest::StatusCode::INSUFFICIENT_STORAGE); - assert_eq!(budget.used(), occupied.bytes() + warm); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - generation - ); - drop(occupied); - let response = request().send().await?.error_for_status()?.bytes().await?; - assert!( - response - .windows(b"ok refs/heads/quota".len()) - .any(|part| part == b"ok refs/heads/quota") - ); - assert!( - repository - .ref_state("refs/heads/quota", None) - .await? - .output - .is_some_and(|state| state.oid.is_none()) - ); - // Admit only the encoded spool: any repeated gzip expansion must fail. - // Completed replay must resolve the original response before decoding. - let occupied = budget.try_reserve(budget.capacity() - warm - wire.len() as u64)?; - let replay = request().send().await?.error_for_status()?.bytes().await?; - assert_eq!(replay, response); - assert_eq!(budget.used(), occupied.bytes() + warm); - drop(occupied); - // Changing only gzip metadata preserves decoded commands, but is a - // different encoded request and cannot reuse the completed operation ID. - let mut changed = wire.clone(); - changed[4..8].copy_from_slice(&1u32.to_le_bytes()); - let conflict = client - .post(format!("{url}/git-receive-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-receive-pack-request") - .header("Content-Encoding", "gzip") - .header("Idempotency-Key", &id) - .body(changed) - .send() - .await?; - assert_eq!(conflict.status(), reqwest::StatusCode::CONFLICT); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - generation + 1 - ); - assert_eq!(budget.used(), warm); - Ok(()) -} diff --git a/crates/canopy-server/tests/smart_http/native_resources.rs b/crates/canopy-server/tests/smart_http/native_resources.rs deleted file mode 100644 index 88b72dab..00000000 --- a/crates/canopy-server/tests/smart_http/native_resources.rs +++ /dev/null @@ -1,105 +0,0 @@ -use super::*; - -type Result = std::result::Result>; - -pub async fn verify(root: &Path, url: &str) -> Result { - let fixture = tempfile::TempDir::new_in(root)?; - let root = fixture.path(); - let source = root.join("native-source"); - let clone = root.join("native-clone"); - run_git( - None, - &["init", "-b", "pack-policy", source.to_str().ok_or("path")?], - ) - .await?; - run_git(Some(&source), &["config", "user.name", "Canopy Test"]).await?; - run_git( - Some(&source), - &["config", "user.email", "canopy@example.invalid"], - ) - .await?; - let mut small = b"small text line\n".repeat(32768); - let mut large = b"large text line\n".repeat(640000); - let mut small_oids = Vec::new(); - let mut large_oids = Vec::new(); - for generation in 0..2 { - small[0] = b'A' + generation; - large[0] = b'A' + generation; - tokio::fs::write(source.join("small.txt"), &small).await?; - tokio::fs::write(source.join("large.txt"), &large).await?; - run_git(Some(&source), &["add", "."]).await?; - run_git(Some(&source), &["commit", "-m", "Related objects"]).await?; - for (name, ids) in [ - ("HEAD:small.txt", &mut small_oids), - ("HEAD:large.txt", &mut large_oids), - ] { - ids.push( - String::from_utf8(run_git(Some(&source), &["rev-parse", name]).await?)? - .trim() - .to_owned(), - ); - } - } - run_git( - Some(&source), - &[ - "-c", - "http.extraHeader=Authorization: Bearer local-test-token", - "push", - url, - "pack-policy", - ], - ) - .await?; - run_git( - None, - &[ - "-c", - "fetch.unpackLimit=1", - "-c", - "http.extraHeader=Authorization: Bearer local-test-token", - "clone", - "--single-branch", - "--branch", - "pack-policy", - url, - clone.to_str().ok_or("path")?, - ], - ) - .await?; - assert_eq!(tokio::fs::read(clone.join("large.txt")).await?, large); - assert_eq!(tokio::fs::read(clone.join("small.txt")).await?, small); - run_git(Some(&clone), &["fsck", "--strict", "--full"]).await?; - let index = std::fs::read_dir(clone.join(".git/objects/pack"))? - .collect::>>()? - .into_iter() - .map(|entry| entry.path()) - .find(|path| path.extension().is_some_and(|extension| extension == "idx")) - .ok_or("clone pack index")?; - let listing = String::from_utf8( - run_git( - Some(&clone), - &["verify-pack", "-v", index.to_str().ok_or("path")?], - ) - .await?, - )?; - // Inspect the actual served pack: large similar blobs are whole objects, - // while ordinary source-size blobs retain delta compression. - for oid in large_oids { - let fields: Vec<_> = listing - .lines() - .find(|line| line.starts_with(&oid)) - .ok_or("large object in pack")? - .split_whitespace() - .collect(); - assert_eq!(fields.len(), 5, "large object must not be a delta"); - } - assert!( - listing.lines().any(|line| { - let fields: Vec<_> = line.split_whitespace().collect(); - fields.len() == 7 && small_oids.iter().any(|oid| fields[0] == oid) - }), - "ordinary blobs still use deltas" - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/smart_http/publication.rs b/crates/canopy-server/tests/smart_http/publication.rs deleted file mode 100644 index d58bb8ea..00000000 --- a/crates/canopy-server/tests/smart_http/publication.rs +++ /dev/null @@ -1,502 +0,0 @@ -use super::*; -use crate::paused_blobs::PausedBlobs; -use canopy_server::{PushPlan, RefUpdate, branch_rules::BranchRuleEdit}; -use std::{sync::atomic::Ordering, time::Duration}; - -type Result = std::result::Result>; -const AUTH: &str = "http.extraHeader=Authorization: Bearer late-writer-token"; - -fn packet(line: &str) -> Vec { - format!("{:04x}{line}", line.len() + 4).into_bytes() -} - -pub async fn verify( - root: &Path, - repository: &Arc, - store: &Arc, - url: &str, -) -> Result { - let local = root.join("publication-race"); - run_git(None, &["init", "-b", "main", local.to_str().ok_or("path")?]).await?; - let current = repository - .ref_state("refs/heads/main", None) - .await? - .output - .ok_or("main")? - .oid - .ok_or("main oid")?; - let client = reqwest::Client::new(); - repository - .grant_member( - support::identity()?, - "canopy", - "late-writer", - TokenScope::Write, - ) - .await?; - for kind in [ - "refs", - "policy", - "access", - "no-report", - "storage", - "storage-sideband", - "storage-no-report", - "storage-stock", - "storage-race", - ] { - let storage = kind.starts_with("storage"); - let no_report = kind.ends_with("no-report"); - let raced = kind == "storage-race"; - let reason = if storage { - "Canopy object ingestion failed" - } else { - "Canopy publication rejected" - }; - tokio::fs::write( - local.join("large"), - kind.as_bytes() - .iter() - .copied() - .cycle() - .take(2 * 1024 * 1024) - .collect::>(), - ) - .await?; - run_git(Some(&local), &["add", "."]).await?; - run_git( - Some(&local), - &[ - "-c", - "user.name=Test", - "-c", - "user.email=test@example.invalid", - "-c", - "commit.gpgsign=false", - "commit", - "-m", - kind, - ], - ) - .await?; - let oid = String::from_utf8(run_git(Some(&local), &["rev-parse", "HEAD"]).await?)? - .trim() - .to_owned(); - let reference = format!("refs/heads/late-{kind}"); - // Sort the valid sibling first so a premature per-ref write would leak. - let sibling = format!("refs/heads/early-{kind}"); - let id = uuid::Uuid::new_v4().to_string(); - let capabilities = match kind { - "policy" | "storage-sideband" => "report-status-v2 side-band-64k", - "access" | "storage" | "storage-race" => "report-status", - _ => "", - }; - let mut body = packet(&format!( - "{} {oid} {reference}\0{capabilities}\n", - "0".repeat(40) - )); - body.extend(packet(&format!("{} {oid} {sibling}\n", "0".repeat(40)))); - body.extend_from_slice(b"0000"); - body.extend(run_git(Some(&local), &["pack-objects", "--all", "--stdout"]).await?); - let post = |id: &str| { - client - .post(format!("{url}/git-receive-pack")) - .bearer_auth("late-writer-token") - .header("Content-Type", "application/x-git-receive-pack-request") - .header("Idempotency-Key", id) - .body(body.clone()) - }; - store.armed.store(true, Ordering::SeqCst); - let mut child = if kind == "refs" || kind == "storage-stock" { - Some( - Command::new("git") - .current_dir(&local) - .args([ - "-c", - "credential.helper=", - "-c", - AUTH, - "push", - "--atomic", - url, - &format!("HEAD:{reference}"), - &format!("HEAD:{sibling}"), - ]) - .env("GIT_TERMINAL_PROMPT", "0") - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn()?, - ) - } else { - None - }; - let request = if child.is_none() { - Some(tokio::spawn(post(&id).send())) - } else { - None - }; - tokio::time::timeout(Duration::from_secs(15), store.entered.notified()).await?; - if kind == "refs" || kind == "no-report" { - repository - .finalize_push( - support::identity()?, - PushPlan { - actor: "canopy".into(), - updates: vec![RefUpdate { - name: reference.clone(), - expected: None, - new_oid: Some(current), - }], - }, - ) - .await?; - } else if kind == "policy" { - repository - .set_branch_rule( - support::identity()?, - "canopy", - BranchRuleEdit { - reference: reference.clone(), - expected_version: 0, - enabled: true, - deny_deletions: false, - fast_forward_only: false, - require_pull_request: true, - required_approvals: 0, - required_checks: vec![], - }, - ) - .await?; - } else if kind == "access" { - repository - .revoke_member(support::identity()?, "canopy", "late-writer") - .await?; - } - let generation = repository.default_branch(None).await?.output.generation; - store.fail.store(storage, Ordering::SeqCst); - // A different gateway can commit the same ID while this attempt is paused. - // The losing ingestion failure must return that success, never a local ng. - let winner = if raced { - Some(gateway_request(root, repository, store.clone(), &body, &id).await?) - } else { - None - }; - store.proceed.notify_one(); - if let Some(child) = child.take() { - let output = child.wait_with_output().await?; - let stderr = String::from_utf8_lossy(&output.stderr); - assert!(!output.status.success(), "{stderr}"); - assert!( - stderr.contains("[remote rejected]") && stderr.contains(reason), - "{stderr}" - ); - assert!( - !stderr.contains("HTTP 409") && !stderr.contains("HTTP 500"), - "{stderr}" - ); - } - if let Some(request) = request { - let response = request.await??; - assert_eq!( - response.status().as_u16(), - if no_report { 409 } else { 200 } - ); - let report = response.bytes().await?; - let text = String::from_utf8_lossy(&report); - if let Some((status, saved)) = winner { - assert_eq!(status, 200); - assert_eq!(report.as_ref(), saved); - assert!(text.contains(&format!("ok {reference}")), "{text}"); - } else { - assert!(text.contains(reason), "{text}"); - } - if !no_report && !raced { - assert!( - text.contains(&format!("ng {reference} ")) - && text.contains(&format!("ng {sibling} ")), - "{text}" - ); - assert!(!text.contains(&format!("ok {reference}")), "{text}"); - } - repository - .grant_member( - support::identity()?, - "canopy", - "late-writer", - TokenScope::Write, - ) - .await?; - if kind == "policy" { - repository - .set_branch_rule( - support::identity()?, - "canopy", - BranchRuleEdit { - reference: reference.clone(), - expected_version: 1, - enabled: false, - deny_deletions: false, - fast_forward_only: false, - require_pull_request: false, - required_approvals: 0, - required_checks: vec![], - }, - ) - .await?; - } - // Recreate the gateway with no native/object cache; the saved Cell - // decision must survive permission/policy or storage recovery. - let (status, saved) = - gateway_request(root, repository, Arc::new(InMemory::new()), &body, &id).await?; - assert_eq!(status, if no_report { 409 } else { 200 }); - assert_eq!(saved, report); - } - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + i64::from(raced) - ); - if !raced { - assert!(repository.ref_state(&sibling, None).await?.output.is_none()); - } - if storage && !raced { - let response = post(&uuid::Uuid::new_v4().to_string()).send().await?; - assert_eq!(response.status().as_u16(), 200); - let report = response.bytes().await?; - if !no_report && kind != "storage-stock" { - assert!(String::from_utf8_lossy(&report).contains(&format!("ok {reference}"))); - } - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + 1 - ); - } - if storage { - for name in [&reference, &sibling] { - assert_eq!( - hex::encode( - repository - .ref_state(name, None) - .await? - .output - .ok_or("missing ref")? - .oid - .ok_or("missing oid")? - ), - oid - ); - } - } - } - verify_preparation(root, repository, store, &local, current).await?; - Ok(()) -} - -async fn verify_preparation( - root: &Path, - repository: &Arc, - store: &Arc, - source: &Path, - current: canopy_server::ObjectId, -) -> Result { - use sha1::{Digest as _, Sha1}; - let mut pack = b"PACK\0\0\0\x02\0\0\0\0".to_vec(); - pack.extend_from_slice(&Sha1::digest(&pack)); - for kind in ["plain", "sideband", "no-report", "stock", "race"] { - let scratch = tempfile::tempdir_in(root)?; - let gateway = Arc::new(GitGateway::new( - repository.clone(), - scratch.path().into(), - store.clone(), - DiskBudget::new(1 << 30), - canopy_server::native_resources::NativeResources::default(), - )); - let listener = TcpListener::bind("127.0.0.1:0").await?; - let address = listener.local_addr()?; - let api = Arc::new(GitHttpApi::new( - gateway, - "canopy".into(), - "preparation", - &format!("http://{address}"), - Arc::new(|| true), - )?); - let (stop, stopped) = oneshot::channel::<()>(); - let serving = tokio::spawn(async move { - axum::serve(listener, support::git_router(api)) - .with_graceful_shutdown(async move { - let _ = stopped.await; - }) - .await - }); - let url = format!("http://{address}/canopy/preparation.git"); - let reference = format!("refs/heads/準備-{kind}"); - let sibling = format!("refs/heads/preparation-sibling-{kind}"); - let capabilities = match kind { - "sideband" => "report-status-v2 side-band-64k", - "no-report" => "", - _ => "report-status", - }; - let mut body = packet(&format!( - "{} {} {reference}\0{capabilities}\n", - "0".repeat(40), - hex::encode(current) - )); - body.extend(packet(&format!( - "{} {} {sibling}\n", - "0".repeat(40), - hex::encode(current) - ))); - body.extend_from_slice(b"0000"); - body.extend_from_slice(&pack); - let id = uuid::Uuid::new_v4().to_string(); - let client = reqwest::Client::new(); - let post = |id: &str| { - client - .post(format!("{url}/git-receive-pack")) - .bearer_auth("late-writer-token") - .header("Content-Type", "application/x-git-receive-pack-request") - .header("Idempotency-Key", id) - .body(body.clone()) - }; - let generation = repository.default_branch(None).await?.output.generation; - store.read_armed.store(true, Ordering::SeqCst); - let child = if kind == "stock" { - Some( - Command::new("git") - .current_dir(source) - .args([ - "-c", - "credential.helper=", - "-c", - AUTH, - "push", - "--atomic", - &url, - &format!("HEAD:{reference}"), - &format!("HEAD:{sibling}"), - ]) - .env("GIT_TERMINAL_PROMPT", "0") - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .kill_on_drop(true) - .spawn()?, - ) - } else { - None - }; - let request = if child.is_none() { - Some(tokio::spawn(post(&id).send())) - } else { - None - }; - tokio::time::timeout(Duration::from_secs(15), store.entered.notified()).await?; - let winner = if kind == "race" { - Some(gateway_request(root, repository, store.clone(), &body, &id).await?) - } else { - None - }; - store.fail.store(true, Ordering::SeqCst); - store.proceed.notify_one(); - if let Some(child) = child { - let output = child.wait_with_output().await?; - let stderr = String::from_utf8_lossy(&output.stderr); - assert!(!output.status.success(), "{stderr}"); - assert!( - stderr.contains("Canopy push failed before publication"), - "{stderr}" - ); - assert!(!stderr.contains("HTTP 500"), "{stderr}"); - } - if let Some(request) = request { - let response = request.await??; - let status = response.status().as_u16(); - assert_eq!(status, if kind == "no-report" { 409 } else { 200 }); - let report = response.bytes().await?; - if let Some((status, saved)) = &winner { - assert_eq!(*status, 200); - assert_eq!(report.as_ref(), saved); - } else { - let text = String::from_utf8_lossy(&report); - assert!( - text.contains("Canopy push failed before publication"), - "{text}" - ); - if kind != "no-report" { - for name in [&reference, &sibling] { - assert!(text.contains(&format!("ng {name} ")), "{text}"); - } - } - } - let replay = gateway_request(root, repository, store.clone(), &body, &id).await?; - assert_eq!(replay, (status, report.to_vec())); - } - if winner.is_none() { - for name in [&reference, &sibling] { - assert!(repository.ref_state(name, None).await?.output.is_none()); - } - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation - ); - post(&uuid::Uuid::new_v4().to_string()) - .send() - .await? - .error_for_status()?; - } - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + 1 - ); - for name in [&reference, &sibling] { - assert_eq!( - repository - .ref_state(name, None) - .await? - .output - .ok_or("ref")? - .oid, - Some(current) - ); - } - let _ = stop.send(()); - serving.await??; - } - Ok(()) -} - -async fn gateway_request( - root: &Path, - repository: &Arc, - store: Arc, - body: &[u8], - id: &str, -) -> Result<(u16, Vec)> { - let scratch = tempfile::tempdir_in(root)?; - let gateway = GitGateway::new( - Arc::clone(repository), - scratch.path().into(), - store, - DiskBudget::new(1 << 30), - canopy_server::native_resources::NativeResources::default(), - ); - let response = gateway - .handle( - canopy_server::git_http::GitHttpRequest { - method: "POST".into(), - path_info: "/repo.git/git-receive-pack".into(), - query: String::new(), - content_type: Some("application/x-git-receive-pack-request".into()), - gzip: false, - protocol_v2: false, - authenticated: true, - body: Body::from(body.to_vec()), - }, - "late-writer", - Some(uuid::Uuid::parse_str(id)?.into_bytes()), - None, - ) - .await?; - Ok(( - response.status, - axum::body::to_bytes(response.body, 1 << 20).await?.to_vec(), - )) -} diff --git a/crates/canopy-server/tests/smart_http/push_uploads.rs b/crates/canopy-server/tests/smart_http/push_uploads.rs deleted file mode 100644 index a3cc36d8..00000000 --- a/crates/canopy-server/tests/smart_http/push_uploads.rs +++ /dev/null @@ -1,71 +0,0 @@ -use super::*; -use canopy_server::git_http::GitHttpRequest; -use futures_core::Stream; -use std::{ - future::Future, - io, - pin::Pin, - task::{Context, Poll}, - time::Duration, -}; - -struct PausedUpload { - entered: Option>, - release: oneshot::Receiver<()>, -} - -impl Stream for PausedUpload { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if let Some(entered) = self.entered.take() { - let _ = entered.send(()); - } - match Pin::new(&mut self.release).poll(cx) { - Poll::Pending => Poll::Pending, - Poll::Ready(_) => Poll::Ready(None), - } - } -} - -fn push(body: Body) -> GitHttpRequest { - GitHttpRequest { - method: "POST".into(), - path_info: "/repo.git/git-receive-pack".into(), - query: String::new(), - content_type: Some("application/x-git-receive-pack-request".into()), - gzip: false, - protocol_v2: false, - authenticated: true, - body, - } -} - -pub async fn verify(gateway: &Arc) -> Result<(), Box> { - let (entered_tx, entered_rx) = oneshot::channel(); - let (release_tx, release_rx) = oneshot::channel(); - let first_gateway = Arc::clone(gateway); - let stalled = tokio::spawn(async move { - first_gateway - .handle( - push(Body::from_stream(PausedUpload { - entered: Some(entered_tx), - release: release_rx, - })), - "canopy", - None, - None, - ) - .await - }); - tokio::time::timeout(Duration::from_secs(5), entered_rx).await??; - let second = tokio::time::timeout( - Duration::from_secs(5), - gateway.handle(push(Body::from("invalid")), "canopy", None, None), - ) - .await; - let _ = release_tx.send(()); - assert!(stalled.await?.is_err()); - assert!(second?.is_err()); - Ok(()) -} diff --git a/crates/canopy-server/tests/smart_http/ref_snapshots.rs b/crates/canopy-server/tests/smart_http/ref_snapshots.rs deleted file mode 100644 index cfc4f534..00000000 --- a/crates/canopy-server/tests/smart_http/ref_snapshots.rs +++ /dev/null @@ -1,299 +0,0 @@ -use super::*; -use canopy_server::{ - ObjectBatch, ObjectKind, PushPlan, RefExpectation, RefReadError, RefUpdate, StoredObject, - object_id, -}; -use std::collections::BTreeMap; - -const FIRST: &str = "refs/tags/snapshot-000"; -const LAST: &str = "refs/tags/snapshot-299"; - -async fn pair( - repository: &RepositoryCell, - version: i64, - old: Option, - new: Option, -) -> Result<(), Box> { - repository - .finalize_push(support::identity()?, pair_plan(version, old, new)) - .await?; - Ok(()) -} - -fn pair_plan( - version: i64, - old: Option, - new: Option, -) -> PushPlan { - PushPlan { - actor: "canopy".into(), - updates: [FIRST, LAST] - .into_iter() - .map(|name| RefUpdate { - name: name.into(), - expected: Some(RefExpectation { oid: old, version }), - new_oid: new, - }) - .collect(), - } -} - -pub async fn verify( - repository: &RepositoryCell, - client: &reqwest::Client, - url: &str, -) -> Result<(), Box> { - let initial = repository.refs_page("", None).await?.output.generation; - let mut batch = ObjectBatch::default(); - let a = object_id( - canopy_server::ObjectFormat::Sha1, - ObjectKind::Blob, - b"snapshot a", - ); - let b = object_id( - canopy_server::ObjectFormat::Sha1, - ObjectKind::Blob, - b"snapshot b", - ); - for (oid, body) in [(a, b"snapshot a"), (b, b"snapshot b")] { - batch - .try_push(StoredObject { - oid, - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body.to_vec()), - }) - .map_err(|_| "fixture object exceeds batch limit")?; - } - repository.put_objects(support::identity()?, batch).await?; - assert_eq!( - repository.refs_page("", None).await?.output.generation, - initial - ); - for start in (0..300).step_by(64) { - repository - .finalize_push( - support::identity()?, - PushPlan { - actor: "canopy".into(), - updates: (start..(start + 64).min(300)) - .map(|index| RefUpdate { - name: format!("refs/tags/snapshot-{index:03}"), - expected: None, - new_oid: Some(a), - }) - .collect(), - }, - ) - .await?; - } - let first = repository.refs_page("", None).await?.output; - assert_eq!(first.generation, initial + 5); - assert_eq!(first.refs.len(), 256); - let cursor = first.refs.last().ok_or("missing first page")?.0.clone(); - assert!(repository.refs_page(&cursor, None).await.is_err()); - pair(repository, 1, Some(a), Some(b)).await?; - for after in [&cursor, "zzzz"] { - assert!(matches!( - repository.refs_page(after, Some(first.generation)).await, - Err(RefReadError::Changed) - )); - } - let current = repository.refs_page("", None).await?.output; - assert_eq!(current.generation, first.generation + 1); - let mut refs: BTreeMap<_, _> = current.refs.into_iter().collect(); - let second = repository - .refs_page(&cursor, Some(current.generation)) - .await? - .output; - assert_eq!(second.generation, current.generation); - refs.extend(second.refs); - assert_eq!(refs[FIRST].oid, Some(b)); - assert_eq!(refs[LAST].oid, Some(b)); - assert_eq!( - refs.keys() - .filter(|name| name.starts_with("refs/tags/snapshot-")) - .count(), - 300 - ); - assert!( - repository - .refs_page("zzzz", Some(current.generation)) - .await? - .output - .refs - .is_empty() - ); - - // Returning to the same OIDs, and deleting/recreating names, must both - // invalidate old pages even when their ref tips appear unchanged. - pair(repository, 2, Some(b), Some(a)).await?; - pair(repository, 3, Some(a), Some(b)).await?; - assert!(matches!( - repository - .refs_page(&cursor, Some(current.generation)) - .await, - Err(RefReadError::Changed) - )); - pair(repository, 4, Some(b), None).await?; - let deleted = repository.refs_page("", None).await?.output; - assert!( - deleted - .refs - .iter() - .any(|(name, state)| name == FIRST && state.oid.is_none()) - ); - pair(repository, 5, None, Some(b)).await?; - assert!(matches!( - repository - .refs_page(&cursor, Some(deleted.generation)) - .await, - Err(RefReadError::Changed) - )); - let generation = repository.refs_page("", None).await?.output.generation; - assert!( - repository - .finalize_push(support::identity()?, pair_plan(5, Some(b), Some(a))) - .await - .is_err() - ); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - generation - ); - let identity = support::identity()?; - let plan = pair_plan(6, Some(b), Some(a)); - repository.finalize_push(identity, plan.clone()).await?; - repository.finalize_push(identity, plan).await?; - assert_eq!( - repository.refs_page("", None).await?.output.generation, - generation + 1 - ); - - // Both ends of the ordered scan move in one transaction while real HTTP - // advertisements repeatedly acquire, hydrate and replace cache generations. - let writer = async { - let mut old = a; - for version in 7..39 { - let new = if old == a { b } else { a }; - pair(repository, version, Some(old), Some(new)).await?; - old = new; - } - Ok::<_, Box>(()) - }; - let reader = async { - let mut successes = 0; - let mut busy = 0; - for _ in 0..32 { - let response = client - .get(format!("{url}/info/refs?service=git-upload-pack")) - .bearer_auth("local-test-token") - .send() - .await?; - if response.status() == reqwest::StatusCode::SERVICE_UNAVAILABLE { - assert_eq!( - response.text().await?, - "Repository refs are changing; retry the request" - ); - busy += 1; - continue; - } - let bytes = response.error_for_status()?.bytes().await?; - let advertised = advertisement_refs(&bytes)?; - assert_eq!(advertised[FIRST], advertised[LAST]); - assert_eq!( - advertised - .keys() - .filter(|name| name.starts_with("refs/tags/snapshot-")) - .count(), - 300 - ); - successes += 1; - } - assert!(successes > 0); - eprintln!("snapshot advertisements: {successes} consistent, {busy} retriable"); - Ok::<_, Box>(()) - }; - tokio::try_join!(writer, reader)?; - let listing = run_git( - None, - &[ - "-c", - "http.extraHeader=Authorization: Bearer local-test-token", - "ls-remote", - url, - "refs/tags/snapshot-*", - ], - ) - .await?; - assert_eq!(std::str::from_utf8(&listing)?.lines().count(), 300); - - // Git compresses sufficiently large fetch requests. The CGI contract must - // preserve their encoding while retaining the encoded bytes in the spool. - use std::io::Write; - let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); - encoder.write_all(b"0014command=ls-refs\n00010009peel\n000csymrefs\n0000")?; - let body = encoder.finish()?; - for encoding in ["gzip", "x-gzip", "GZIP"] { - let response = client - .post(format!("{url}/git-upload-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-upload-pack-request") - .header("Content-Encoding", encoding) - .header("Git-Protocol", "version=2") - .body(body.clone()) - .send() - .await? - .error_for_status()? - .bytes() - .await?; - assert_eq!( - advertisement_refs(&response)? - .keys() - .filter(|name| name.starts_with("refs/tags/snapshot-")) - .count(), - 300 - ); - } - for encoding in ["br", "gzip, identity"] { - let response = client - .post(format!("{url}/git-upload-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-upload-pack-request") - .header("Content-Encoding", encoding) - .body(body.clone()) - .send() - .await?; - assert_eq!( - response.status(), - reqwest::StatusCode::UNSUPPORTED_MEDIA_TYPE - ); - } - Ok(()) -} - -fn advertisement_refs( - mut bytes: &[u8], -) -> Result, Box> { - let mut refs = BTreeMap::new(); - while !bytes.is_empty() { - let header = bytes.get(..4).ok_or("short packet header")?; - let length = usize::from_str_radix(std::str::from_utf8(header)?, 16)?; - if length == 0 { - bytes = &bytes[4..]; - continue; - } - let packet = std::str::from_utf8(bytes.get(4..length).ok_or("short packet")?)?; - bytes = &bytes[length..]; - if let Some((oid, name)) = packet.split_once(' ') - && oid.len() == 40 - { - let name = name - .split('\0') - .next() - .ok_or("missing ref name")? - .trim_end(); - refs.insert(name.into(), oid.into()); - } - } - Ok(refs) -} diff --git a/crates/canopy-server/tests/support/mod.rs b/crates/canopy-server/tests/support/mod.rs deleted file mode 100644 index 7f8c170a..00000000 --- a/crates/canopy-server/tests/support/mod.rs +++ /dev/null @@ -1,53 +0,0 @@ -use std::{ - sync::Arc, - time::{SystemTime, UNIX_EPOCH}, -}; - -use axum::{ - Router, - body::Body, - http::{Request, StatusCode, header}, - middleware::{self, Next}, - response::{IntoResponse, Response}, -}; -use canopy_server::{ - directory::{Principal, TokenScope}, - http::{GitHttpApi, Viewer}, -}; -use cellule_runtime::{MutationIdentity, identity::RequestId}; - -pub fn git_router(api: Arc) -> Router { - api.router().layer(middleware::from_fn(test_identity)) -} - -async fn test_identity(mut request: Request, next: Next) -> Response { - let account = request - .headers() - .get(header::AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .and_then(|value| match value { - "Bearer local-test-token" => Some("canopy"), - "Bearer late-writer-token" => Some("late-writer"), - _ => None, - }); - let Some(account) = account else { - return StatusCode::UNAUTHORIZED.into_response(); - }; - request - .extensions_mut() - .insert(Viewer::Authenticated(Principal { - token_id: [1; 16], - account: account.into(), - scope: TokenScope::Admin, - })); - next.run(request).await -} - -pub fn identity() -> Result> { - let now_ms = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now_ms, - expires_at_ms: now_ms + 60_000, - }) -} diff --git a/crates/canopy-server/tests/support/objects.rs b/crates/canopy-server/tests/support/objects.rs deleted file mode 100644 index aab80c65..00000000 --- a/crates/canopy-server/tests/support/objects.rs +++ /dev/null @@ -1,26 +0,0 @@ -use canopy_server::{ - ObjectBatch, ObjectKind, ObjectStorage, RepositoryCell, StoredObject, object_id, -}; -use cellule_runtime::{Committed, MutationIdentity}; - -pub async fn put( - repository: &RepositoryCell, - identity: MutationIdentity, - kind: ObjectKind, - body: &[u8], -) -> Result, Box> { - let oid = object_id(repository.object_format(), kind, body); - let mut batch = ObjectBatch::default(); - batch - .try_push(StoredObject { - oid, - kind, - storage: ObjectStorage::Inline(body.to_vec()), - }) - .map_err(|_| "fixture object exceeds batch bounds")?; - let result = repository.put_objects(identity, batch).await?; - Ok(Committed { - output: oid, - receipt: result.receipt, - }) -} diff --git a/docs/contracts.md b/docs/contracts.md index b2a3d24a..280c62e7 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -1,6 +1,6 @@ # Understand Canopy's persisted contracts -This reference defines the identities, storage boundaries, authorization checks and retry behavior that Canopy must preserve across node loss. It describes the current development format. [Versioned migrations](../ROADMAP.md#r01-safe-upgrades-and-schema-migration) are still required before persistent customer repositories can survive upgrades. +This reference defines the identities, storage boundaries, authorization checks and retry behavior that Canopy must preserve across node loss. It describes the current compiled format. Breaking changes require a fresh storage prefix; older deployments and their readers are not retained. > **Document type:** Reference. **Goal:** preserve durable behavior and make conflicts, retries, and recovery observable. @@ -24,8 +24,8 @@ request ▼ Directory Cell ── repository UUID ──► Repository Cell │ refs, ACL, issues, PRs, checks - │ Git object metadata and small bodies - ├──► immutable large Git/LFS bodies + │ accepted catalog and ref roots + ├──► immutable native Git/LFS artifacts │ in object storage └──► disposable native Git cache for wire protocol work @@ -80,18 +80,19 @@ The Directory Cell binds names and accounts to durable identities; the Repositor ### Git and LFS storage -Small Git objects remain in SQLite. Large Git blobs and Git LFS bodies use immutable object-store bytes with SQLite references and verification metadata. +All Git bodies use verified immutable native pack/index artifacts. Repository +SQLite publishes accepted catalog/ref roots and business metadata; it has no +per-object body, chunk, placement or graph-certificate tables. -| Surface | Current value | Owner | +| Surface | Current representation | Owner | | --- | --- | --- | -| Git object ID | SHA-1 or SHA-256, according to the repository's immutable object format, over canonical Git type, decimal length, NUL and body | `object_id` | -| Small Git objects | SQLite `objects.body`, maximum 768 KiB | Repository Cell | -| Large trees, commits and tags | SQLite chunks of at most 512 KiB; object size above 768 KiB without a fixed byte quota | `object_chunks`, verified before object publication | -| Object publication | at most 128 records, 3 MiB aggregate inline payload; SQLite verification targets 64 MiB, with larger objects verified individually | `PutObjects`, operation 5, codec 3 | -| Large Git blobs | immutable `repos//git-blobs/` body, SQLite digest/size/reference | `LargeBlobStore` | -| LFS objects | immutable `repos//lfs/` body, SQLite digest/size/reference | `LfsService` | -| LFS locks | unique path per repository; UUID identity, owning account and UTC second-precision timestamp in SQLite | `LfsService` / lock API | -| External bodies | No fixed logical byte quota; 8 MiB immutable parts, a 16-byte Git manifest or a part-digest LFS manifest | Each conditional S3 copy handles one bounded part | +| Git identity | SHA-1 or SHA-256 canonical Git type, size and body; repository format is immutable | Native verifier and catalog | +| Git bodies | Immutable native pack/index pairs, scoped to repository and creating operation | `ArtifactStore` | +| Canonical metadata and typed graph | Immutable checked metadata shards and directory runs; pages of up to 512 objects | `packs::metadata`, `packs::directory`, closure verifier | +| Accepted refs and object membership | Joint accepted immutable roots, checked against current authority and policy | Repository Cell | +| LFS objects | Immutable `repos//lfs/` parts; SQLite pins size and manifest digest | `LfsService` | +| LFS locks | Unique repository path, UUID, account owner and timestamp in SQLite | LFS locking API | +| Artifact parts | At most 8 MiB per part, authenticated `CANOPY02` manifest; at most 65,536 parts (512 GiB) per artifact | Conditional part publication and verified readers | ### Admission and publication @@ -102,12 +103,12 @@ These limits bound individual requests and publication work. They do not cap tot | Node transfer admission | eight active Git/LFS requests across repositories; immediate 503 with `Retry-After: 1` when full | repository HTTP router | | LFS request deadline | Batch: 120 seconds; object PUT: 120-second input idle timeout with no whole-transfer deadline; timeout returns 408 | Git HTTP router / LFS service | | Git request admission | No receive-pack byte quota; 64 MiB for other requests; 120-second input idle deadline | anonymous request spool | -| Ref mutation | check actor's write role, compare expected optional OID and monotonic version; retain deletion records and apply all updates in one Cell transaction | `FinalizePush` | +| Ref mutation | Recheck current actor, expected OID/version and policy; atomically select accepted catalog/ref roots and exact outcome | `CompleteRootPush`, operation 36, codec 2 | | Symbolic HEAD | Accepted immutable `RefStateSnapshot.default_branch`, initially `refs/heads/main`; owner-authorized joint-root compare-and-set | `PublishNativeHead`, operation 54, codec 1 | | HTTP push identity | repository-local UUID bound to account and BLAKE3 request digest; different IDs identify independent operations | `pushes` | -| HTTP push outcome | status, headers and BLAKE3-verified body in SQLite chunks; publish response pointer, rejection decision and ordered push notes atomically with accepted refs | `CompletePush`, codec 5 | -| Graph certificates | at most 128 candidates; SQLite verification targets 64 MiB, with larger objects verified individually; typed dependencies must already be certified | `CertifyObjects`, operation 6, codec 1 | -| Git connectivity at ref publication | at most 100,000 certified new tips, with commit-only branch tips; same transaction as ACL, ref CAS and outcome | `object_closure`, shared ref finalization | +| HTTP push outcome | Immutable recorded native outcome; publish its authenticated pointer and decision atomically with accepted roots | Root completion and outcome commands | +| Graph certification | Native bytes, canonical identities and typed dependencies verified before catalog attestation; bounded metadata pages and disk-backed traversal | Canonical/closure verifier and owner-bound attestation | +| Git connectivity at ref publication | Prepared certified catalog/ref transition; commit-only branch tips; final ACL, policy, observation and fencing checks | Root publication | | LFS metadata publication | check actor's write role in the SQLite insert transaction | `record_lfs_object` | ## Git publication and retry @@ -242,65 +243,26 @@ report unchanged. A rejected atomic push changes no refs. Malformed packs retain Git's unpack failure report and publish no refs. The gateway does not infer transaction success by searching diagnostic text for status fragments. -Both ref commands require certified reachable Git graphs in the ref transaction. -Commit trees and parents, tree entries, and annotated tag targets must exist -with the required object kind. Branch tips must be commits. Tree symlinks and -regular files require blobs; gitlinks refer to another repository and do not -require a local object. The traversal follows Git's -[commit headers](https://github.com/git/git/blob/v2.50.1/commit.c#L435), -[tag targets](https://github.com/git/git/blob/v2.50.1/tag.c#L142), and -[tree connectivity rules](https://github.com/git/git/blob/v2.50.1/fsck.c#L334). -It checks graph structure and content hashes; it is not a replacement for all -of `git fsck`'s metadata, filename and portability checks. - -An async postorder traversal prepares certificates before either ref command. -It reads only uncertified history, deduplicates repeated typed roots/edges and -checks child status in pages of 128. Certified children need no separate Cell -call; uncertified children carry their observed immutable metadata into traversal. -It groups at most 128 candidate object IDs in dependency order. Its cache of objects ready -for certification covers only the pending batch. `CertifyObjects` independently -reads each candidate from SQLite, recomputes hashes and graph edges, and requires -all typed children to be certified. It checks child metadata in groups of at most -128 rows. The same transaction records deduplicated parent/child edges in -`object_edges`, with a reverse child index for fetch authorization. Only -certified typed edges enter this index; gitlinks do not create local edges. -Newly verified inline and chunked bodies target 64 MiB per command; a larger -chunked object is verified individually; references to verified external blob bodies do not consume that budget. -The final ref transaction checks up to 100,000 new tips and their certificates -in groups of 128, together with current ACL, versions and namespace conflicts. -An ordered update map detects duplicates and planned ancestor conflicts. Exact -ancestor lookups and indexed descendant pages check existing namespaces; every -conflict must be removed by a deletion in the same plan, without truncating at a -fixed number of matches. - -The certificate command is the trust boundary: incorrect candidate order, -missing or mismatched children, corrupt data or an invalid verification batch -rejects its whole application savepoint. The client traversal cannot assert a -certificate. Certificates and refs are separate publications. A failed or -interrupted preparation may retain valid certificates from earlier batches; -retries reuse them. Partially certified roots cannot publish refs, and certificate -commits do not advance `ref_generation`. An external blob certificate relies on -the verified immutable upload before its SQLite record; there is no external -network I/O inside certificate or ref transactions. - -`RepositoryCell::finalize_push` and HTTP completion share this preparation path. -The low-level `FinalizePush` command requires certificates already present. -A preparation failure is reported as ref invocation `NotStarted`, retaining its -underlying error, since the final ref command has not been dispatched. Certificate -work may itself have a pending outcome; retry rechecks committed state. Direct -callers must keep their supplied finalization identity valid through preparation; -HTTP completion creates its final command identity after preparation. - -Certificates rely on immutable object records and retained object bytes. There -is currently no object mutation or collector through the product API. Any -future collector or repair that removes or changes objects must invalidate all -affected ancestor certificates before ref publication resumes; clearing the -entire certificate table and its object-edge index is the conservative -implementation. Backup and restore -must preserve the database and its referenced external bodies together. Each -certificate transaction has explicit object/byte limits, but large individual -objects, traversal frontier memory and production-scale latency still need -capacity qualification. +Publication requires a certified native catalog. Commit trees and parents, tree +entries and tag targets must exist with the expected kind; gitlinks can refer +to a different repository. Native verification checks canonical Git identities, +sizes and BLAKE3 digests. Structural parsing streams bounded chunks rather than +reconstructing bodies in a Cell SQL transaction. Metadata shards and disk-backed +closure traversal bound working buffers; complete traversal still grows with +new history and needs capacity qualification. + +Only trusted verification can issue the catalog attestation. The final receiver +checks its exact repository, operation, base, lease/owner and publication facts, +then rechecks current actor access, ref observations and branch/review/check +policy before selecting the new roots and outcome. A client, cached object or +partially prepared artifact cannot assert certified membership. The native +publisher retains exact request identity and same-release recovery registration. + +Git bodies are immutable after publication. Backup/restore must preserve all +artifacts reachable from the accepted roots, retained outcomes and recovery +records. Cellule collection and product-artifact retention are separate: +operator-invoked Cellule GC reclaims superseded Cellule objects after a verified +drain, but does not reclaim abandoned product pack/catalog/LFS artifacts. ### HTTP push identity and exact replay @@ -646,186 +608,46 @@ collector removes those bodies yet. Account creation is idempotent only for the same account, active token digest and scope. Bootstrap accepts an issued active owner admin token and does not replace or revive credentials. -Git packs and the bare repository cache are transport and acceleration -artifacts. Neither is authoritative. The gateway can reconstruct cache objects -from the Cell and external store. Integration tests prove exact-root restore -after clean node shutdown and local SQLite loss. The process smoke also proves -Git and LFS fetch after an unclean owner exit, lease expiry and a third process -claiming the Cell from an S3-compatible object store. A proxy also drops a -successful push reply; exact replay after takeover returns its recorded report -and preserves a later ref deletion. Crashes during -individual staging/publication boundaries, the full multi-node fault matrix and -backup restore still need proof before service readiness. - -Cache hydration reads objects by an indexed, repository-local insertion sequence. -`objects.sequence` is an `INTEGER PRIMARY KEY AUTOINCREMENT`; `oid` remains unique -and non-null. Only committed insertions matter, duplicates retain their existing -sequence, and gaps are valid. SQLite never reuses a committed automatic sequence, -including after deletion; this is why the extra AUTOINCREMENT bookkeeping is -required. See [SQLite's contract](https://www.sqlite.org/autoinc.html). - -Each refresh captures the maximum stored sequence after reading its ref snapshot. -Header pages query `after < sequence <= maximum` in sequence order, so concurrent -new publications wait for a subsequent refresh. Each page contains at most 128 -records and 768 KiB of aggregate inline bodies. A second query reads the exact -missing IDs, sorted by OID, with the header receipt as its minimum observation. -A byte-limited page advances only through its selected prefix. The gateway advances -its in-memory cursor only after every selected body has verified and completed its -cache write. Failure or cancellation leaves the failed page eligible for retry; -completed files are reused. The cursor belongs to the same cache owner and resets -when that owner is discarded. `RepositoryCell::object_page(after)` retains its -OID-ordered full enumeration and uses the same header decoder and body reader. - -Each gateway retains one disk-accounted cache of verified immutable objects. -Ref snapshots, pushes and native merge candidates get separate bare directories -that borrow those objects through Git's -[alternates contract](https://git-scm.com/docs/gitrepository-layout). -The shared cache receives only verified durable Cell records and external bodies. -Native writes stay in the private generation; failed or unpublished pushes never -populate the shared cache. A successful push's new objects enter it through the -next durable hydration read. Snapshot HEAD/refs remain private and unchanged for -the lifetime of a fetch, even while new snapshots reuse the same object bytes. - -An object is compressed into an accounted temporary file, then installed at its -OID only after verification and compression finish. Incomplete files never count -as cache hits. Existing OIDs skip body queries, external downloads and compression; -refresh seeks past the completed sequence and checks local paths only for new -headers or a retried page. A ref-only refresh reads the high-water mark without -scanning old headers. No cache serves as -durable authority, and authorization still precedes cache access. - -Snapshot owners retain the shared cache through native work and streaming. If a -worker fence prevents generation cleanup, both its disk reservation and its -borrowed-object owner remain retained until process restart. Startup acquires all -generation fences before deleting any cache. This preserves borrowed objects even -when an orphan worker outlives the gateway. Failed/short writes retain conservative -charges until cache destruction; concurrent cancelled writers cannot refund another -writer's reservation through a read-then-resize race. - -Records are immutable through product APIs, and no collector removes them. -This makes the two observations safe; a future collector must fence active -reads before deleting records or bodies. The reader rejects changed/missing IDs, -invalid descriptors, and inline size, Git OID or BLAKE3 mismatches. Inline bodies -move into one bounded blocking verification task without a payload clone. -Chunked and external records carry descriptors; their body readers verify the -actual bytes before the gateway writes them to the disposable cache. Refs become -visible there only after successful hydration for the selected fetch mode. -This bounds each query, -not total repository hydration time or native Git scratch usage. The previous -single-record `next_object` API was removed from this unreleased crate. - -Debug events separate Repository Cell acquisition from successful cache -hydration. Hydration reports object count, raw bytes, admitted cache bytes, total -time, starting/completed insertion sequence, scanned/reused object counts, page-read/verification time, external/chunk body-read time and cache-write -time. Cache writes include worker scheduling, OID verification and compression; for -external Git blobs this interval also includes streamed object-store reads. -These elapsed wall times include waiting; they are not CPU profiles or a claim -that all page time belongs to SQLite execution. Acquisition includes authority -and root validation, sparse activation and publication. Failed phases remain -errors and do not emit a successful-completion event. - -Push object ingestion runs two Git processes regardless of object count: -[`rev-list --objects --no-object-names --stdin`](https://git-scm.com/docs/git-rev-list) -enumerates objects reachable from accepted new ref tips, excluding the previous -live tips; [`cat-file --batch`](https://git-scm.com/docs/git-cat-file) reads only -candidates missing from SQLite. Deletion-only pushes skip both. Enumeration is -consumed incrementally; ref roots travel over stdin rather than an unbounded -argument list. Incoming objects reachable only from rejected ref tips are not -newly persisted. - -Both commands disable replacement refs so stored bytes retain their canonical -identity. The batch reader caps headers at 128 bytes, validates kind and size -before allocation, reads the exact body and delimiter, and recomputes its Git -OID. Missing, truncated, malformed or corrupt output fails the push. Enumeration, -headers, inline/non-blob body reads, delimiters and process completion have -120-second deadlines. External blobs stream in at most 8 MiB chunks, with -120-second input/part idle deadlines and no whole-upload duration ceiling. -OID hashing uses a blocking worker so large bodies do not occupy an async -executor thread. Stderr is drained -concurrently, retaining at most 64 KiB per process. Dropping the reader kills -both direct children and aborts pipe tasks; normal completion requires both -successful exits before publishing refs or recording the successful report. -Previously published graph closure makes exclusions safe; ref publication -still requires every new tip to have a durable certificate. - -External Git blobs use multipart staging for bounded parts under a unique -repository-scoped prefix. Canopy hashes the exact declared body as it streams, -checks the Git OID, conditionally copies the parts, and creates the immutable -SHA-256 manifest. It verifies the destination before returning metadata. Conflicting canonical bytes fail; they -are never overwritten. Staging deletion and incomplete multipart abort are -attempted on ordinary failure. Process loss can leave staging/multipart debris; -operator lifecycle cleanup and qualified collection remain required. - -Hydration and backup verification read one part of at most 8 MiB at a time, -pinning the manifest version/ETag. The final range is withheld until Git SHA-1, -SHA-256 and BLAKE3 all match the SQLite reference. A failed or canceled range -invalidates its reader. Hydration compresses into disk-accounted cache files on -blocking workers; each writer retains the cache reservation until it exits. -The ref generation is not published for Git service until hydration finishes. - -External Git blobs use an immutable `CANOPY01` manifest (eight magic bytes, -then an eight-byte little-endian logical size). LFS bodies use `CANOPY02` with -one BLAKE3 digest per part after the size; SQLite pins the manifest's BLAKE3 -digest. The body bytes live at `.parts/<16-digit hex index>`. Each part is -at most 8 MiB; empty bodies have one empty part. -Parts are conditionally copied before the manifest is created, and the whole -logical body is reverified before SQLite metadata is published. Backup copies -all referenced parts before copying the manifest. There is no whole-object S3 -copy and no fixed logical file-size quota. `CANOPY02` replaces the previous LFS -manifest and digest meaning. This hard cutover requires a fresh preview prefix -or re-uploaded LFS bodies; no old-body decoder or migration is provided. - -Trees, commits and tags have no fixed byte quota, but publication and graph -parsing currently materialize individual bodies. Available worker memory remains -a practical constraint; this is not production capacity qualification. - -Candidate existence queries group up to 128 IDs. Missing records accumulate in -an `ObjectBatch` with at most 128 records and 3 MiB of aggregate inline bodies, -leaving room for metadata beneath the 4 MiB operation input limit. External -blob records count toward the record limit; their bytes are verified and -uploaded before publication. Chunk references count toward the record limit and -a 64 MiB aggregate verification target shared with inline bytes. An oversized -individual object is verified alone. The -decoder enforces these bounds before publication. One typed Cell command publishes the batch and returns one receipt. -It recomputes inline Git OIDs and BLAKE3 digests, then compares every inserted -or existing row with the complete expected record. A mismatch rejects the -command and rolls back all inserts in that batch. Repeating the same mutation -identity and payload returns the recorded result through Cellule deduplication. - -Trees, commits and tags larger than 768 KiB are staged in `object_uploads` and -`object_chunks`; their bodies stay in SQLite. Each part is at most 512 KiB and -there is no fixed number of parts per object. Part command identities derive from -the caller's upload identity and part index, so retrying the same staging -operation replays committed parts. Staging does not create an `objects` row: -queries and ref validation cannot see incomplete uploads. - -`PutObjects` codec 3 accepts a chunk reference containing upload ID, size and -BLAKE3. In the publication transaction it requires the exact chunk count and -part lengths, reconstructs the body, verifies the canonical Git OID and BLAKE3, -and then inserts the immutable object record. A failure rolls back every object -in that batch. Inline and external-blob records use the same publication command. -Command verification checks the total chunk count on the first part only: its -transaction holds a stable snapshot for the remaining indexed part reads. -Graph certification uses the same rule. Separate asynchronous object reads -retain a count check with every part because they do not share that snapshot. -Repeated identical objects converge on the existing record; unused duplicate -uploads remain staged until a collector can prove they are unreferenced. - -Hydration and direct SQLite object reads use the same chunk count, length and -hash checks. Every SQL result contains at most one chunk, fitting Cellule's -1 MiB result limit. Reads reconstruct a bounded whole object in memory. Graph -certification reconstructs chunked objects before applying the same typed edge -checks as inline objects; chunking does not certify missing edges. Part data is -immutable through product APIs, and no collector exists yet. Any future collector -must preserve chunks reachable from objects and invalidate graph certificates -before removing their dependencies. - -Object batches remain separate from ref and push-outcome publication. A later -failure can leave earlier batches unreferenced; a retry finds those records -through the bounded existence query. The final transaction still validates -graph closure before exposing new refs. Retention and collection of abandoned -objects remain open. Per-object buffers, complete cold-cache hydration and -large initial graph traversals still need capacity qualification. +Native Git packs are durable product artifacts only when the accepted catalog +certifies them. Local bare repositories remain disposable serving workspaces. +SQLite stores accepted roots and business state, not expanded Git object bodies. +The old `ObjectBatch`, chunk staging, object-page readers and mutable graph +certificate commands are removed from the library. + +Artifacts bind repository UUID, creating operation, content digest and kind. +Native packs, indexes, metadata shards, directory runs, input bodies and roots +use separate authenticated artifact paths. Reusing identical content after +retirement uses a new operation path, so a delayed delete cannot remove a later +incarnation. Publication copies bounded parts conditionally, verifies collisions +and writes the manifest only after the parts are complete. Ordinary failure +attempts staging cleanup; process loss can leave artifacts requiring a separate +product retention policy. + +Metadata builders collect canonical object/typed-edge records into immutable +files with pages of at most 512 objects. Directory runs target 64 MiB files. +The prepared catalog and joint ref snapshot replace per-object Repository Cell +writes; their final publication carries bounded roots and proofs. Building +metadata still consumes local I/O and native work. This design reduces modified +Cellule pages and publications; its actual storage reduction and throughput must +be measured on the exact executable and corpus. + +A serving session pins a certified catalog and coherent ref snapshot and checks +current actor access. Explicit fetch wants must be reachable in that snapshot; +physical pack presence alone grants no object authority. Native indexes and +selected ranges are authenticated before decoding. Selected objects are verified +against canonical metadata before entering private native workspaces. Snapshot +owners retain disk reservations and worker fences until all work closes. +Corruption or missing bytes fail the request; there is no old-body fallback. + +The current external-part format is `CANOPY02`: size and per-part BLAKE3 hashes +are pinned by the authenticated manifest digest. Readers verify requested parts +and withhold success until their complete identity checks pass. Unhashed +`CANOPY01` manifest readers and writers are removed. The separate `CANOPY01` +operation-ID namespace remains the current command identity format. +LFS uses the same verified part primitive and its own repository-scoped keys. +Backup copies referenced parts before the manifest and verifies the independent +copy. Logical artifact size is bounded by the current manifest format, not by +an S3 single-part copy limit. Git CGI responses use one subprocess/stream implementation. The HTTP gateway streams advertisements and fetch replies through four queued chunks of at most @@ -860,8 +682,8 @@ annotated-tag chains. Native Git checks target existence, peels tags and emits all protocol bytes. Commit ancestors and tree children are not traversed for this exchange. Direct blob/tree refs still require their own verified bodies; a large blob tag can therefore consume substantial discovery cache space. -Selected SQLite object reads reuse the 128-record/768-KiB inline page limit; -chunked and external objects use the same verified hydration path as transfers. +Selected objects come from the certified catalog and authenticated native +artifacts; no per-object Repository SQL body reader is used. Missing or corrupt targets fail preparation instead of disappearing from refs. A current ready cache is reused when its mutex is immediately available. Discovery @@ -1158,7 +980,7 @@ complete OOM/CPU/process/crash fault matrix require separate qualification. `data_dir/.canopy-owner.lock` prevents concurrent nodes from using one local directory. The server and repository manager retain the lock throughout their -lifetime, including detached request work. `data_dir/canopy-pack-v1/` is private to +lifetime, including detached request work. `data_dir/canopy-pack-v2/` is private to the node (created with mode 0700 on Unix) and identified by a version marker. Startup validates that marker and reclaims local SQLite files, Git caches and spools before opening Cells or advertising the node. These are disposable copies; diff --git a/docs/git-compatibility.md b/docs/git-compatibility.md index 3d65676e..31893270 100644 --- a/docs/git-compatibility.md +++ b/docs/git-compatibility.md @@ -135,14 +135,13 @@ dedicated disk but no host-volume sharing, use `--docker-volume`; it creates and removes an isolated named volume while retaining the full size assertions. Use `--release` to run the same gate against optimized server/test code. -The gate pushes an incompressible pack above 512 MiB, checks the SQLite database -exceeds 512 MiB, stores a Git blob and LFS object above 5 GiB, restarts on fresh +The gate pushes an incompressible pack above 512 MiB, checks Repository SQLite +stays below 16 MiB, stores a Git blob and LFS object above 5 GiB, restarts on fresh local storage, then checks clone bytes, strict fsck, and the LFS download hash. The ordinary large-object integration test covers a 65 MiB commit and recovery. These are qualification points, not configured maxima or performance targets. -The pinned runtime retains its five-second SQL command deadline. Very large -non-blob verification still materializes a complete object inside a transaction; +The pinned runtime retains its five-second SQL command deadline. Structural Git verification uses bounded native streams and immutable metadata; progressive verification is needed before claiming arbitrary structural-object capacity. LTX artifact promotion and backup copies still use the configured provider's conditional-copy implementation; very large LTX artifacts need diff --git a/docs/implementation.md b/docs/implementation.md index 9b3a5eaa..b309c177 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -45,7 +45,7 @@ These are current admission limits, not measured production throughput: | Concurrent Git/LFS transfers | Eight per node; excess requests return `503` and `Retry-After: 1`. | | Fetch request body | Up to 64 MiB; clone/fetch response packs stream with backpressure. | | Push report | Buffered up to 64 MiB; push bodies have no fixed byte quota. | -| Git blob and LFS body | Immutable 8 MiB parts, with no fixed logical file-size quota. | +| Native Git artifacts and LFS bodies | Verified immutable parts up to 8 MiB; current manifest format supports up to 512 GiB per artifact. | | Active repository gateways | `max_active_repositories` per node (1–9,999); stored repositories may exceed this. | | Local admitted disk | Set by `local_disk_limit_bytes`; exhaustion returns `507`. | @@ -53,14 +53,13 @@ The current service supports one repository owner. Incoming Git requests stream to temporary files charged to the same disk budget as the node's SQLite files. Push bodies have no fixed byte quota; fetch requests remain bounded at 64 MiB. Push reports are buffered up to 64 MiB, while clone/fetch -packs stream with backpressure. Git blobs and LFS objects use immutable 8 MiB -parts without a fixed logical file-size quota. Trees, commits and tags above -768 KiB use SQLite chunks without a fixed individual object-size quota. -Git LFS basic downloads resume with a tail `Range` request and a verified -`206` response. The server reads only the requested parts and verifies each -against a manifest digest pinned in SQLite. -Publication and graph parsing still materialize non-blob bodies, so very large -structural objects depend on available worker memory. +packs stream with backpressure. Git bodies remain in verified native pack/index +artifacts, with canonical metadata and typed graph facts in immutable shards and +directory runs. Repository SQLite stores accepted roots and business metadata. +Structural verification streams bounded chunks. LFS downloads can resume with +verified tail ranges. Each artifact uses authenticated parts up to 8 MiB and the +current manifest's 512 GiB format bound. These bounds are not measured production +capacity. Partial uploads stay invisible to Git. Each node admits eight Git/LFS transfers across all repositories. Overload returns 503 with `Retry-After: 1`; retry after capacity is available. Health, @@ -153,40 +152,27 @@ prefixes/backups are unsupported. Startup rejects unmarked application roots, and release admission rejects a different compiled release. See the [runtime integration contract](contracts.md#cellule-integration). -Before ref publication, bounded certificate batches verify the durable Git -graph: commit trees and parents, tree entries and tag targets must exist with -the correct object type. Each batch covers at most 128 objects, targeting 64 MiB of -SQLite object bytes. Ref publication checks certified tips atomically with -permissions and ref versions; branch tips must be commits. Submodule gitlinks may name commits in another repository. -SQLite certificates let later pushes reuse validated history. Push ingestion streams -candidates from accepted ref tips, excludes previously published history, and -reads missing objects through one persistent Git batch process. Object sizes -are checked before allocation and canonical OIDs before storage. SQLite lookups -group up to 128 candidate IDs; object publication groups up to 128 records and -3 MiB of inline bytes in one Cell transaction, targeting 64 MiB of SQLite -object bytes verified per batch. A conflicting record rejects -the whole batch. Recovery tests include annotated tags, submodules and -`git fsck` on the restored clone. +Before publication, trusted native and closure verification checks canonical Git +identities and typed dependencies. Accepted roots bind the exact repository, +operation, source catalog and owner/lease. Final publication rechecks actor +access, branch policy, ref observations and exact retry identity atomically with +the recorded native outcome. There are no Git-body rows, chunk staging or mutable +object-certificate readers in the Repository Cell. ### Cache hydration and partial clone -Cold cache hydration reads insertion-ordered pages of at most 128 records and 768 KiB -of inline bodies. It verifies inline identities on a blocking worker; chunked -and external bodies retain their own verification before cache writes. Pages -reduce SQLite query overhead. Blobless fetch (`--filter=blob:none`) skips ordinary -blob bodies until explicitly requested. Other fetches enumerate missing blobs -reachable from requested tips, applying the native filter before hydration. -Tree and object-type constraints, including those in combined filters, skip -omitted bodies; size filters still need missing blob bodies. Advertised ref/tag targets are always prepared. -Non-blob history is still prepared across the repository. Supported filters and remaining gaps are listed in -[Git compatibility](git-compatibility.md). -While a gateway remains resident, verified object files are shared across private -ref snapshots, pushes and merge candidates. An indexed insertion cursor limits -refresh to newly published object headers and missing bodies. Each refresh captures -a fixed upper bound; each fully verified page advances the cursor. Failed pages -retry without skipping bytes. Native output stays private until it is -published to the Cell and subsequently verified into the reusable cache. See the -[process proof and remaining limits](performance-plan.md#indexed-object-refresh). +Serving snapshots select a certified catalog and coherent refs and recheck live +access. Native workspaces authenticate indexes and required artifact ranges; +verified selected objects enter private disk-accounted caches. A cached OID alone +cannot authorize a fetch want. Blobless and supported native filters follow the +same certified reachability rules. See [Git compatibility](git-compatibility.md) +for supported filters and qualification limits. + +Publication batches canonical metadata into immutable files instead of mutating +a Cell for each object. Metadata pages contain up to 512 objects; directory runs +target 64 MiB. Sparse reads and decoded native working sets still consume I/O, +CPU and disk. Final qualification must measure the complete corpus on one frozen +executable, including cold clones and APIs after retention collection. ## Verify a build @@ -235,7 +221,7 @@ Optional flags expand the qualification: | --- | --- | | `--large-clone` | Push two 80 MiB random blobs, then clone with protocol v0 and v2 after takeover. Each pack exceeds 64 MiB; both file hashes and `git fsck` must match. This checks transfer size, not production capacity. | | `--many-objects 256` | Push 256 files, update one file with an annotated tag, then clone after takeover. Combined with `--large-clone`, exercise resident eviction across four repositories and Git/LFS recovery before restart. | -| `--sqlite-chunks` | Push a 32,000-entry tree and commit/tag messages above 1 MiB; verify raw bytes and OIDs after takeover with strict `git fsck`. | +| `--structural-objects` | Push a 32,000-entry tree and commit/tag messages above 1 MiB; verify raw bytes and OIDs after takeover with strict `git fsck`. | | `--corpus-repository /path/to/existing/repository` | Read that checkout's HEAD history, bundle it into temporary fixtures under `--work-parent`, then compare every reachable object's type, size and bytes in v0/v2 clones after takeover, with strict `git fsck`. Other source branches and tags are outside this check. | For a local container store, verify that the intended data/log volume is shared @@ -246,18 +232,15 @@ completed on a dedicated Docker volume, ending at 5.4 GiB with 5.1 GiB under its internal temporary directory. Size the test store accordingly; this is an observation from that run, not a production storage bound. -To measure cold recovery, set -`RUST_LOG=warn,canopy_server::git_gateway=debug,canopy_server::server::residency=debug`. -The node reports Cell acquisition time and cache hydration time, with object -count, raw/cache bytes and time spent in page reads, body retrieval and cache -writes. Page time includes inline integrity verification; cache time includes -worker scheduling, OID verification, compression and admitted disk writes. -Use release builds for performance measurements. - -The chunk, default-branch, repository-discovery, token-metadata, collaboration, -visibility and object insertion-sequence layouts change the unreleased schema; use a fresh development storage prefix when moving -from older builds. No upgrade migration or mixed-build rolling upgrade is supported yet. -Do not reuse an existing development prefix with this changed initialization schema. +Use release builds for performance measurements and retain the exact source, +lockfile, binary identity, configuration and raw timings. Measure complete cold +recovery, imports, clones, APIs and pre/post-GC storage independently. Historical +row/chunk hydration logs do not qualify the current native catalog path. + +The current schema and compiled release require a fresh storage prefix and data +directory. Older-state readers, upgrade chains and mixed-release operation are +not supported. Same-release recovery, backup/restore, retries and fencing remain +required. ## Contributing diff --git a/docs/repository-cell-primitives.md b/docs/repository-cell-primitives.md index 47170fcf..9f8af9b7 100644 --- a/docs/repository-cell-primitives.md +++ b/docs/repository-cell-primitives.md @@ -49,8 +49,8 @@ recovery lifecycle. Nodes must host thousands of these repository identities with bounded memory, disk, descriptors, workers and queued operations. All existing Git smart-HTTP, LFS, collaboration, ACL and recovery requirements -remain. Ordinary Git objects stay in SQLite; large bodies stay in immutable -object storage. A SQL-only density measurement does not qualify the complete +remain. Git bodies stay in immutable native pack artifacts; accepted roots and business +metadata stay in the Repository Cell. A SQL-only density measurement does not qualify the complete primitive-enabled repository design. ## Current contract gap @@ -63,7 +63,7 @@ in `crates/canopy-server/src/lib.rs`. `RepositoryCell` holds `SqlCell Date: Wed, 14 Oct 2026 16:44:09 -0700 Subject: [PATCH 03/19] test(storage): bind structural qualification size Make the process fixture size explicit and retain the 1.1 MiB default. Correct historical coverage descriptions and document the current native root publication path. --- .../2026-10-14-storage-optimization.md | 6 ++ docs/contracts.md | 83 +++++++------------ docs/implementation.md | 2 +- scripts/smoke_s3_process.py | 12 ++- 4 files changed, 46 insertions(+), 57 deletions(-) diff --git a/docs/benchmarks/2026-10-14-storage-optimization.md b/docs/benchmarks/2026-10-14-storage-optimization.md index 70a21756..f51f599f 100644 --- a/docs/benchmarks/2026-10-14-storage-optimization.md +++ b/docs/benchmarks/2026-10-14-storage-optimization.md @@ -70,6 +70,12 @@ policy. They still avoid the original expansion to 3.07 GiB. Cold reads fetch the complete referenced pair; sparse S3 range reads are not implemented. Selected serving objects use disposable, disk-accounted loose projections. +The older process profile's `--structural-objects` flag used 1,100,000-byte +commit/tag messages and a 32,000-entry tree. The historical references below to +65 MiB structural coverage refer to the ordinary large-object integration test, +not that isolated process profile. Current qualification specifies the structural +message size explicitly; retained raw logs and source snapshots are unchanged. + ## Index-warming checkpoint qualification The index-warming checkpoint is diff --git a/docs/contracts.md b/docs/contracts.md index 280c62e7..162ab6a7 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -128,31 +128,19 @@ verified object records ──► graph certificates ──► staged ref plan Git report to client ``` -The `objects` table stores one verified kind, size and independent BLAKE3 -digest per Git OID. External objects also store SHA-256. Readers verify the -bytes against the SQLite record and recompute the Git OID or LFS SHA-256. -The `refs` table stores name, optional OID and version. Deletion sets the OID -to NULL and advances the version; recreation advances it again. Never-seen -names and deleted names have different expectations, so stale plans fail even -after a ref returns to the same OID or to a deleted state. Deleted rows remain -in SQLite and are excluded from Git advertisements and namespace conflicts. -They must not be collected without a replacement mechanism for fencing stale -ref plans. `FinalizePush` uses codec version 3 for this expectation shape. -Before reporting accepted refs, a push persists all new objects and publishes -the accepted ref changes and replayable response through `CompletePush`. Both -`CompletePush` and the direct `FinalizePush` command share the same ref checks. -HTTP plans use `push_plan_chunks`, bound to one response UUID, with 128 updates -and at most 64 KiB per chunk. Completion carries the count and BLAKE3 digest, -loads every ordered chunk inside its transaction, verifies actor/count/digest, -and applies the whole plan. Ref generation advances once; the saved response -and deletion of consumed plan chunks commit atomically with refs. Interrupted -staging never publishes refs. Unconsumed chunks from failed attempts remain -until a future retention policy collects them. Direct typed `FinalizePush` -inputs retain their 1 MiB wire bound; HTTP bulk plans are not limited by it. -This changes the development schema and completion codec; use a fresh deployment -prefix, with no compatibility reader for the old inline completion shape. -Rejected or interrupted pushes may leave unreferenced objects; collection is not -implemented yet. +Canonical object metadata and typed closure facts live in authenticated native +catalog artifacts. Accepted immutable ref snapshots retain optional OIDs and +monotonic versions, so deletion/recreation and return-to-the-same-OID cannot +revive stale plans. Physical pack presence is not object membership authority. + +Before acknowledging accepted refs, the owner publishes the certified catalog, +ref roots and exact recorded native outcome through `CompleteRootPush`. +Bounded immutable request/outcome artifacts carry large plans and replies; +the final Cell command checks their authenticated roots, current authority and +preconditions. Interrupted preparation cannot publish a partial ref snapshot. +Unreferenced product artifacts remain until a separate retention policy can +prove them safe to remove. Older Git-body formats and completion APIs are not +exposed through a compatibility reader. ### Ref snapshots and default branch @@ -277,15 +265,15 @@ Authorization secrets and host addresses are not part of the digest. Explicit identity encoding and absent encoding have the same interpretation; changing to gzip under a completed request UUID produces a conflict. -A reservation binds the ID before running Git. Response metadata and 512 KiB -body chunks are durable before finalization. `CompletePush` checks the binding, -chunk completeness, then applies the ref plan under current ACL, policy and CAS -checks. It publishes the response pointer and a rejection decision in one Cell -transaction. `apply_refs` returns false only before any ref writes; SQL errors -roll back completion. A completed ID returns its existing outcome. Only a -published pointer can expose a staged response; replay verifies the stored body -length and BLAKE3 digest before applying the saved decision. The HTTP boundary -checks current token scope and repository access even for completed requests. +The current staging lifecycle binds the original request ID and exact commands +before native execution. Immutable request and result roots retain the replay +material. Root completion authenticates those roots and atomically records the +accepted catalog/ref transition or terminal refusal. Its recovery registration +retains the original command and input facts across owner loss. +Only an accepted published pointer can expose a completed outcome. Replay checks +its bytes and current token/repository access without reapplying an old plan to +newer refs. Preparation, policy and publication failures remain distinguishable +from a completed native result. No-op, rejected and non-200 Git backend responses are also terminal outcomes. Late ACL, policy and ref-CAS refusals are terminal rejected outcomes. Native @@ -1587,20 +1575,12 @@ ref mutation, including deletion and recreation. Otherwise deletion depends on `deny_deletions`; checks and fast-forward policy govern non-deletions. Branch creation needs checks but has no old ancestry to prove. -In the retired SQL graph contract, verified commit objects provided -`commit_parents(child, parent)` when graph closure was certified. Native -preparation reads verified catalog commit headers and uses MAC-bound ancestry -evidence; it does not populate these retired tables. The historical contract -below does not describe native production authority. Only commit-parent edges enter that table. Immutable -`commit_ancestry(ancestor, descendant)` certificates avoid graph traversal in the -ref transaction. Operation 7, codec 1 accepts at most 128 child/parent steps. -Every step must exist in verified parent links and lead either to the claimed -ancestor or to an existing certificate. Any false step rejects and rolls back -the complete command. Preparation reads parents in pages of 128, follows all -merge parents and publishes proof chunks from the known ancestor outward. -Interrupted preparation may retain positive immutable facts without changing -refs. A collector must invalidate graph and ancestry certificates before deleting -objects or their required chunks/edges. No collector is implemented yet. +Ancestry preparation reads verified catalog commit headers and produces +owner-bound native evidence. It follows merge parents through the certified +native graph and checks that evidence at final publication. It neither populates +mutable `commit_parents` / `commit_ancestry` tables nor publishes retired graph +commands. An interrupted preparation can retain immutable artifacts but cannot +move refs without passing the current final authority and policy checks. Native Git receives an executable, disposable `update` hook. Git's documented [update-hook contract](https://git-scm.com/docs/githooks#_update) supplies exact @@ -1645,13 +1625,12 @@ completion. A different push ID cannot reuse the same signed bytes: the final ref transaction records the certificate SHA-256 digest or rejects the update. The certificate body remains in SQLite chunks with its signer, key and Cell recording time. Exact request-ID retries replay the original push response. -Completed notes are stored in the same `CompletePush` transaction as the Git -response and refs. `GET /api/repositories//pushes/` returns the push +Completed notes and the native outcome are bound in the same root-completion +transaction as the accepted refs. `GET /api/repositories//pushes/` returns the push author, notes and optional certificate SHA-256, signer, key and recording time to the current repository owner or the author while they retain read access; other readers receive 404. A missing or unfinished push also returns -404. Notes are audit metadata; they do not run hooks or alter checks. This -unreleased schema and operation 4 codec 6 require a fresh development prefix. +404. Notes are audit metadata; they do not run hooks or alter checks. The current compiled schema requires a fresh storage prefix. Policy reads use batches of at most 128 statements. An `(enabled, reference)` index bounds the presence probe; rule and requirement primary keys and the check-attempt index bound final policy lookups. diff --git a/docs/implementation.md b/docs/implementation.md index b309c177..2ee94b85 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -221,7 +221,7 @@ Optional flags expand the qualification: | --- | --- | | `--large-clone` | Push two 80 MiB random blobs, then clone with protocol v0 and v2 after takeover. Each pack exceeds 64 MiB; both file hashes and `git fsck` must match. This checks transfer size, not production capacity. | | `--many-objects 256` | Push 256 files, update one file with an annotated tag, then clone after takeover. Combined with `--large-clone`, exercise resident eviction across four repositories and Git/LFS recovery before restart. | -| `--structural-objects` | Push a 32,000-entry tree and commit/tag messages above 1 MiB; verify raw bytes and OIDs after takeover with strict `git fsck`. | +| `--structural-objects` | Push a 32,000-entry tree and commit/tag messages above 1 MiB; verify exact bytes after takeover and strict fsck. Add `--structural-object-bytes 68157440` to exercise 65 MiB messages. | | `--corpus-repository /path/to/existing/repository` | Read that checkout's HEAD history, bundle it into temporary fixtures under `--work-parent`, then compare every reachable object's type, size and bytes in v0/v2 clones after takeover, with strict `git fsck`. Other source branches and tags are outside this check. | For a local container store, verify that the intended data/log volume is shared diff --git a/scripts/smoke_s3_process.py b/scripts/smoke_s3_process.py index bb2199c9..daff1bc6 100644 --- a/scripts/smoke_s3_process.py +++ b/scripts/smoke_s3_process.py @@ -493,7 +493,7 @@ def verify_many_objects(base_url, directory, count, expected): print(f"PASS: {count}-file clone restored both commits, annotated tag and 300 refs after takeover in {time.monotonic() - started:.2f}s", flush=True) -def seed_structural_objects(base_url, directory): +def seed_structural_objects(base_url, directory, message_bytes): url, _ = create_repository(base_url, "structural-objects") local = directory / "chunk-source" git("init", "--bare", str(local)) @@ -514,11 +514,11 @@ def write(kind, body): commit = write("commit", ( f"tree {tree}\nauthor Canopy 0 +0000\n" "committer Canopy 0 +0000\n\n" - ).encode() + b"c" * 1_100_000 + b"\n") + ).encode() + b"c" * message_bytes + b"\n") tag = write("tag", ( f"object {commit}\ntype commit\ntag release\n" "tagger Canopy 0 +0000\n\n" - ).encode() + b"t" * 1_100_000 + b"\n") + ).encode() + b"t" * message_bytes + b"\n") git("update-ref", "refs/heads/main", commit, cwd=local) git("symbolic-ref", "HEAD", "refs/heads/main", cwd=local) git("update-ref", "refs/tags/release", tag, cwd=local) @@ -553,10 +553,14 @@ def main(): parser.add_argument("--large-clone", action="store_true", help="Qualify a push and v0/v2 clones above 64 MiB, including takeover") parser.add_argument("--many-objects", type=int, default=0, metavar="COUNT", help="Qualify many small objects, an incremental push, and takeover recovery") parser.add_argument("--structural-objects", action="store_true", help="Qualify large tree, commit and tag objects stored in native packs and restored after takeover") + parser.add_argument("--structural-object-bytes", type=int, default=1_100_000, + help="Commit/tag message bytes for --structural-objects") parser.add_argument("--corpus-repository", type=Path, help="Read HEAD history from an existing repository without changing it; qualify exact object bytes after takeover") parser.add_argument("--peers-only", action="store_true", help="Run the two-node HTTPS Git/LFS and takeover gate without the other process suites") parser.add_argument("--retain-work-dir", action="store_true", help="Retain the temporary workspace and server logs for diagnosis") args = parser.parse_args() + if args.structural_object_bytes < 1: + parser.error("--structural-object-bytes must be positive") if args.many_objects < 0: parser.error("--many-objects must be nonnegative") for name in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "CANOPY_NODE_SIGNING_KEY_HEX"): @@ -719,7 +723,7 @@ def main(): expiring_token = seed_expiring_token(base_url) disabled_tokens = seed_disabled_account(base_url) audit_state = smoke_s3_account_audit.seed(base_url) - chunks = seed_structural_objects(base_url, directory) if args.structural_objects else None + chunks = seed_structural_objects(base_url, directory, args.structural_object_bytes) if args.structural_objects else None large = seed_large_repository(base_url, directory) if args.large_clone else None many = seed_many_objects(base_url, directory, args.many_objects) if args.many_objects else None corpus = None From e502097f7b039f82686e762a687f5c5e34da5c3d Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 17:06:41 -0700 Subject: [PATCH 04/19] test(publication): bound recovery fixture stack --- .../src/packs/publication/tests/candidate_publication.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/canopy-server/src/packs/publication/tests/candidate_publication.rs b/crates/canopy-server/src/packs/publication/tests/candidate_publication.rs index 77134e4e..775cff64 100644 --- a/crates/canopy-server/src/packs/publication/tests/candidate_publication.rs +++ b/crates/canopy-server/src/packs/publication/tests/candidate_publication.rs @@ -78,7 +78,9 @@ async fn command( async fn native_candidate_ready_is_atomic_retained_and_recovers_original_receipt_after_body_retirement() -> Result { for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { - let (f, graph, merge) = native_merge::initial(format, false).await?; + // Keep the stock-Git fixture construction off this recovery test's + // future frame; it independently exercises preparation and publication. + let (f, graph, merge) = Box::pin(native_merge::initial(format, false)).await?; let (base, _, _) = opened(&f, *uuid::Uuid::new_v4().as_bytes(), graph.store.clone()).await?; let mut native = prepared_for_store( From b09030af9ff2483779766dffcf4f926de2b43785 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 17:06:41 -0700 Subject: [PATCH 05/19] docs(storage): align release readiness policy --- README.md | 7 ++++--- ROADMAP.md | 19 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2ab98bf7..b6b64706 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ This page keeps the project overview, setup path, and first repository workflow. | Browser | Embedded repository, issue, pull-request and account views | [Repository browser](docs/user-guide.md#repository-browser) | These are implementation capabilities, not a production capacity guarantee. -The [roadmap](ROADMAP.md) lists remaining work such as migrations, full fault +The [roadmap](ROADMAP.md) lists remaining work such as release validation, full fault qualification, collection, provider qualification, observability and a repeatable deployment. The [delivery plan](docs/delivery-plan.md) records release gates; the [performance plan](docs/performance-plan.md) separates measured results from @@ -49,8 +49,9 @@ before relying on a feature in a new environment. The **Directory Cell** maps an owner and repository name to a stable repository UUID and owns accounts. Each UUID identifies a **Repository Cell** containing -authoritative refs, Git metadata, ACLs and collaboration records. Large Git -blobs and LFS bodies are immutable external objects referenced by that Cell. +authoritative catalog/ref roots, ACLs and collaboration records. Native Git +packs, indexes, catalog metadata and LFS bodies are immutable external objects +referenced by the committed state. Native Git handles wire protocols using a **disposable cache**: after local disk loss, Canopy restores published Cell state from the object store and rebuilds the cache. See the [persisted contracts](docs/contracts.md) for publication and diff --git a/ROADMAP.md b/ROADMAP.md index 37b0b2c8..51ee5bbc 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,6 +1,6 @@ # Canopy roadmap -Canopy's Git-hosting core works, but the service is not yet ready for an unattended team deployment. This roadmap tracks the remaining work from the current `main` baseline (`ab43e71`, 2026-09-27). It is a planning checklist, not a claim that every capability mentioned below is absent. See [delivery gates](docs/delivery-plan.md) for the detailed acceptance record, [Git compatibility](docs/git-compatibility.md) for protocol coverage, and [performance evidence](docs/performance-plan.md) for measured limits. +Canopy implements the Git-hosting core, but the current native-storage build still needs end-to-end qualification before an unattended team deployment. This is a planning checklist, not a claim that every capability mentioned below is absent. See the [current storage qualification status](docs/benchmarks/2026-10-14-storage-optimization.md), [delivery gates](docs/delivery-plan.md), [Git compatibility](docs/git-compatibility.md), and [performance evidence](docs/performance-plan.md). ## How to use this roadmap @@ -13,14 +13,13 @@ Canopy's Git-hosting core works, but the service is not yet ready for an unatten This is the first intended production-use milestone. The existing bounded Compose profile is a starting point, not a completed install or capacity claim. -### R01. Safe upgrades and schema migration +### R01. Current-release deployment and format validation -- [ ] Version persistent Canopy and runtime formats; add a fenced, repeatable upgrade command and migration path for a populated deployment. -- [ ] Require a verified backup and compatible release/configuration before migration; document failed-upgrade recovery and rollback limits. -- [ ] Upgrade a prior release containing Git objects, LFS, ACLs, issues, checks, and pull requests; interrupt and retry each migration phase, then clone and inspect the restored state. -- [ ] Define whether mixed-version nodes are supported. Until proven, enforce a fully drained, single-version upgrade. +- [ ] Qualify release, identity, storage-format, schema and fencing rejection against the exact release candidate. +- [ ] Qualify fresh initialization and same-release crash recovery, retries, backup and restore with populated Git, LFS and collaboration state. +- [ ] Document a fully drained deployment using the latest published Cellule release and a fresh storage prefix for breaking Cellule or Canopy format changes. -**Current state:** schema changes still require a fresh storage prefix; maintenance admission and drain exist, but there is no upgrade path. +**Current state:** only the compiled release and its current formats are supported. Fresh-prefix deployment for breaking changes is intentional. Old readers, retained module code, migration chains, compatibility adapters and mixed-version operation are outside the supported contract. Cellule's single schema-initialization descriptor remains required for fresh Cells. ### R02. Backup, restore, and owner-recovery fault qualification @@ -37,7 +36,7 @@ This is the first intended production-use milestone. The existing bounded Compos - [ ] Add a fenced collector for abandoned uploads, old outcomes/candidates, and unreachable Git/LFS objects, with an inspectable dry run and bounded work. - [ ] Prove with concurrent reads, backup, publication, owner loss, and injected collector failures that required bytes are never removed; collect abandoned bytes only after their grace period. -**Current state:** no production collector; abandoned staging and outcomes can accumulate. Collection must remain conservative until its root proof passes. +**Current state:** `canopy gc` uses Cellule's retention-aware collector under a verified maintenance drain, backup pins, grace/deletion bounds and a durable worker claim. It collects superseded Cellule state. Product-owned pack/catalog/LFS artifacts and abandoned staging need their own root and retention policy; complete collection qualification remains open. ### R04. Trustworthy CI and release artifacts @@ -46,7 +45,7 @@ This is the first intended production-use milestone. The existing bounded Compos - [ ] Add required failure-path and provider gates to branch protection, then publish versioned, digest-identified images and a repeatable release checklist. - [ ] Record toolchain, Git, Git LFS, Cellule revision, image digest, provider, and test outcomes for each release candidate. -**Current state:** `.github/workflows/verify.yml` invokes the full size gate on `ubuntu-latest`; a previous hosted run exhausted disk. +**Current state:** standard CI runs formatting, Clippy, ordinary tests, Python harness checks and provider/process probes. The separate release workflow repeats correctness checks in release mode with provider/process probes. Both use `--provider-only`; the full size gate still needs a runner with sufficient capacity and recorded evidence for the exact release candidate. ### R05. Production object-store qualification @@ -99,7 +98,7 @@ This is the first intended production-use milestone. The existing bounded Compos ### Private team release exit checks - [ ] R01–R10 acceptance proof is recorded for the exact release candidate and selected object store. -- [ ] A clean install, upgrade of a populated prior release, backup, source-prefix loss, and restore all pass the operator runbook. +- [ ] A clean install, current-format rejection checks, same-release backup, source-prefix loss and restore all pass the operator runbook. Breaking formats use a fresh prefix. - [ ] Stock Git and Git LFS work after recovery; permissions, issues, reviews, checks, and pull requests retain their state. - [ ] The release notes state measured limits for active repositories, concurrent transfers, local disk, memory, restore time, and object-store cost. Unmeasured capacity is not advertised. From 6706fbbe7e1a691bf8caee594943f7180e1ecebb Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 17:56:28 -0700 Subject: [PATCH 06/19] test(native): separate setup from recovery polling Keep the large native setup poll frame off the stack while durable policy and recovery qualifiers run. Preserve fixture ownership, custody, receipts, refusal and cleanup assertions on the default test-thread stack. --- .../packs/publication/tests/native_capture.rs | 93 +++++++++++-------- 1 file changed, 52 insertions(+), 41 deletions(-) diff --git a/crates/canopy-server/src/packs/publication/tests/native_capture.rs b/crates/canopy-server/src/packs/publication/tests/native_capture.rs index 39b41a83..9abfacdb 100644 --- a/crates/canopy-server/src/packs/publication/tests/native_capture.rs +++ b/crates/canopy-server/src/packs/publication/tests/native_capture.rs @@ -453,13 +453,14 @@ async fn native_receive_metadata_refuses_capacity_corrupt_replay_and_missing_sha MetadataFault::TruncatedReplay, MetadataFault::MissingArtifact, ] { - Box::pin(native_receive_metadata_case( + let qualification = Box::pin(native_receive_metadata_case( format, true, CompletionMode::Success, Some(fault), )) .await?; + assert!(qualification.is_none()); } } Ok(()) @@ -472,15 +473,21 @@ async fn publication_state(fixture: &Fixture) -> Result> { Ok([state.0.to_be_bytes(), state.1.to_be_bytes(), state.2.to_be_bytes()].concat()) }).await?) } +type RootedQualification = std::pin::Pin>>; + async fn native_receive_case(format: ObjectFormat, rooted: bool, mode: CompletionMode) -> Result { - Box::pin(native_receive_metadata_case(format, rooted, mode, None)).await + let qualification = Box::pin(native_receive_metadata_case(format, rooted, mode, None)).await?; + if let Some(qualification) = qualification { + qualification.await?; + } + Ok(()) } async fn native_receive_metadata_case( format: ObjectFormat, rooted: bool, mode: CompletionMode, metadata_fault: Option, -) -> Result { +) -> Result> { let ref_free = match mode { CompletionMode::RefFree { kind, .. } => Some(kind), CompletionMode::Durable { .. } @@ -846,7 +853,7 @@ async fn native_receive_metadata_case( .await?; } cleaned(work_root.path(), &disk).await?; - return Ok(()); + return Ok(None); } if let CompletionMode::RefFree { kind, @@ -876,7 +883,7 @@ async fn native_receive_metadata_case( .await?; cleaned(work_root.path(), &disk).await?; fixture.runtime.shutdown().await?; - return Ok(()); + return Ok(None); } let before_publication = publication_state(&fixture).await?; let physical_root = Arc::new(tempfile::TempDir::new()?); @@ -931,7 +938,7 @@ async fn native_receive_metadata_case( assert_eq!(publication_state(&fixture).await?, before_publication); cleaned(physical_root.path(), &physical_disk).await?; fixture.runtime.shutdown().await?; - return Ok(()); + return Ok(None); } let staged = staged.map_err(|error| error.to_string())?; assert_eq!(staged.shard_count(), input.object_count); @@ -1010,7 +1017,7 @@ async fn native_receive_metadata_case( assert_eq!(publication_state(&fixture).await?, before_publication); cleaned(physical_root.path(), &physical_disk).await?; fixture.runtime.shutdown().await?; - return Ok(()); + return Ok(None); } let prepared = prepared.map_err(|error| error.to_string())?; if rooted { @@ -1026,38 +1033,42 @@ async fn native_receive_metadata_case( | CompletionMode::PolicyDispatch { .. } | CompletionMode::Dispatch { .. } ) { - let result = qualify_rooted( - super::root_dispatch::Context { - fixture: &fixture, - prepared: Arc::new(prepared), - store: &store, - staging: &coordinator, - ticket: &ticket, - root: physical_root.path(), - budget: physical_disk.clone(), - request: recovered, - }, - mode, - ) - .await; - if mode == CompletionMode::MandatoryRegistration { - assert!(coordinator.close_and_drain().await.is_empty()); - fixture.runtime.shutdown().await?; - } - result?; - if matches!( - mode, - CompletionMode::PolicyRefusalDispatch { .. } - | CompletionMode::PolicyDispatch { .. } - | CompletionMode::Dispatch { .. } - ) { - cleaned(physical_root.path(), &physical_disk).await?; - fixture.runtime.shutdown().await?; - } else { - cleaned(work_root.path(), &disk).await?; - cleaned(physical_root.path(), &physical_disk).await?; - } - return Ok(()); + // Finish polling the large native setup before polling recovery. + // The continuation owns the same fixture, custody and scratch pins. + return Ok(Some(Box::pin(async move { + let result = qualify_rooted( + super::root_dispatch::Context { + fixture: &fixture, + prepared: Arc::new(prepared), + store: &store, + staging: &coordinator, + ticket: &ticket, + root: physical_root.path(), + budget: physical_disk.clone(), + request: recovered, + }, + mode, + ) + .await; + if mode == CompletionMode::MandatoryRegistration { + assert!(coordinator.close_and_drain().await.is_empty()); + fixture.runtime.shutdown().await?; + } + result?; + if matches!( + mode, + CompletionMode::PolicyRefusalDispatch { .. } + | CompletionMode::PolicyDispatch { .. } + | CompletionMode::Dispatch { .. } + ) { + cleaned(physical_root.path(), &physical_disk).await?; + fixture.runtime.shutdown().await?; + } else { + cleaned(work_root.path(), &disk).await?; + cleaned(physical_root.path(), &physical_disk).await?; + } + Ok(()) + }))); } let (fixture, replay) = super::root_completion::spawn_qualify( fixture, @@ -1073,7 +1084,7 @@ async fn native_receive_metadata_case( cleaned(physical_root.path(), &physical_disk).await?; assert!(coordinator.close_and_drain().await.is_empty()); super::root_completion::restored(&fixture, &store, replay).await?; - return Ok(()); + return Ok(None); } let expected = recovered.response.clone(); let producer_root = Arc::clone(&physical_root); @@ -1202,7 +1213,7 @@ async fn native_receive_metadata_case( Ok(Vec::new()) }).await?; fixture.runtime.shutdown().await?; - Ok(()) + Ok(None) } #[tokio::test] From cb2c585072b8c5b8a103d66ed134fe2cfb6362ca Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 19:04:53 -0700 Subject: [PATCH 07/19] fix: preserve retry and lease error contracts --- .../canopy-server/src/repository_http/mod.rs | 34 +++++++---- .../tests/multi_server/lifecycle/lease.rs | 24 ++++++-- .../tests/multi_server/lifecycle/mod.rs | 61 ++++++++++++++++++- 3 files changed, 98 insertions(+), 21 deletions(-) diff --git a/crates/canopy-server/src/repository_http/mod.rs b/crates/canopy-server/src/repository_http/mod.rs index 6c7acdc2..77fb257e 100644 --- a/crates/canopy-server/src/repository_http/mod.rs +++ b/crates/canopy-server/src/repository_http/mod.rs @@ -458,6 +458,23 @@ async fn list_repositories( State(state): State>, Query(query): Query, headers: axum::http::HeaderMap, +) -> Response { + let mut response = repository_list_page(state, query, headers).await; + if response.status() == StatusCode::SERVICE_UNAVAILABLE { + // Every unavailable page follows the same bounded GET retry contract, + // including readiness loss and errors reached while moving cold Cells. + response.headers_mut().insert( + header::RETRY_AFTER, + axum::http::HeaderValue::from_static("1"), + ); + } + response +} + +async fn repository_list_page( + state: Arc, + query: ListRepositoriesQuery, + headers: axum::http::HeaderMap, ) -> Response { if !(state.manager.ready)() { return plain(StatusCode::SERVICE_UNAVAILABLE, "Canopy node is not ready"); @@ -491,19 +508,12 @@ async fn list_repositories( }), ), Ok(_) => plain(StatusCode::SERVICE_UNAVAILABLE, "Canopy node is not ready"), - Err(ServerError::Runtime(cellule_runtime::Error::Capacity(_))) => { - let mut response = plain( - StatusCode::SERVICE_UNAVAILABLE, - "Repository listing capacity is full; retry the request", - ); - response.headers_mut().insert( - header::RETRY_AFTER, - axum::http::HeaderValue::from_static("1"), - ); - response - } + Err(ServerError::Runtime(cellule_runtime::Error::Capacity(_))) => plain( + StatusCode::SERVICE_UNAVAILABLE, + "Repository listing capacity is full; retry the request", + ), Err(error) => { - tracing::error!(error = %error, "repository listing failed"); + tracing::error!(error = ?error, "repository listing failed"); plain(StatusCode::SERVICE_UNAVAILABLE, "Repository listing failed") } } diff --git a/crates/canopy-server/tests/multi_server/lifecycle/lease.rs b/crates/canopy-server/tests/multi_server/lifecycle/lease.rs index a2bf4983..be32af23 100644 --- a/crates/canopy-server/tests/multi_server/lifecycle/lease.rs +++ b/crates/canopy-server/tests/multi_server/lifecycle/lease.rs @@ -249,13 +249,25 @@ async fn expired_successful_renewal_reply_cannot_reopen_ingress() -> Result { .send() .await; assert!(expired_at_reply > expires); + // Cellule retains the original drain failure through its host/facility + // wrappers. Require the actual typed fencing cause, not its outer shape. + let mut cause = shutdown + .as_ref() + .err() + .map(|error| error as &(dyn std::error::Error + 'static)); + let mut fenced = false; + while let Some(error) = cause { + if matches!( + error.downcast_ref::(), + Some(cellule_runtime::Error::Fenced) + ) { + fenced = true; + break; + } + cause = error.source(); + } assert!( - matches!( - shutdown, - Err(canopy_server::server::ServerError::Runtime( - cellule_runtime::Error::Fenced - )) - ), + fenced, "expired successful refresh did not fence: {shutdown:?}" ); assert!( diff --git a/crates/canopy-server/tests/multi_server/lifecycle/mod.rs b/crates/canopy-server/tests/multi_server/lifecycle/mod.rs index eef0eeaf..a4dcc571 100644 --- a/crates/canopy-server/tests/multi_server/lifecycle/mod.rs +++ b/crates/canopy-server/tests/multi_server/lifecycle/mod.rs @@ -48,6 +48,60 @@ impl PausedStore { } } +#[tokio::test] +async fn delayed_renewal_pause_uses_first_post_arm_progress() -> Result { + let store = Arc::new(PausedStore::default()); + let path = StorePath::from("renewal-probe"); + let body = |progress: u64, expires: i64| { + serde_json::to_vec(&serde_json::json!({ + "lease": {"progress": progress.to_string(), "expires_at_ms": expires.to_string()} + })) + .unwrap() + }; + // Startup may already have renewed several times before a test arms it. + store + .put_opts(&path, body(8, 100).into(), PutOptions::default()) + .await?; + store.delay_renewals.store(true, Ordering::SeqCst); + let first_body = body(9, 200); + let writer = store.clone(); + let key = path.clone(); + let payload = first_body.clone(); + let first = tokio::spawn(async move { + writer + .put_opts(&key, payload.into(), PutOptions::default()) + .await + }); + store.wait().await?; + let captured = *store.renewal_expiry.lock().unwrap(); + let published = store.inner.get(&path).await?.bytes().await?; + let held = !first.is_finished(); + store.proceed.notify_one(); + let result = timeout(Duration::from_secs(1), first).await??; + assert_eq!(captured, Some(200)); + assert_eq!(published.as_ref(), first_body); + assert!(held, "first successful renewal reply was not held"); + result?; + + let writer = store.clone(); + let key = path.clone(); + let later = tokio::spawn(async move { + writer + .put_opts(&key, body(10, 300).into(), PutOptions::default()) + .await + }); + store.wait().await?; + let published = store.inner.get(&path).await?.bytes().await?; + store.proceed.notify_one(); + let result = timeout(Duration::from_secs(1), later).await??; + assert_eq!(published.as_ref(), first_body); + assert!(matches!( + result, + Err(object_store::Error::PermissionDenied { .. }) + )); + Ok(()) +} + impl fmt::Display for PausedStore { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("paused-lifecycle-store") @@ -78,8 +132,9 @@ impl ObjectStore for PausedStore { *self.initial_advertisement.lock().unwrap() = advertisement.clone(); } let delayed = self.delay_renewals.load(Ordering::SeqCst); - if delayed && progress.is_some_and(|value| value > 2) { - // Keep later renewals unpublished until the confirmed lease expires. + let renewal = progress.is_some_and(|value| value > 1); + if delayed && renewal && self.renewal_expiry.lock().unwrap().is_some() { + // Keep renewals after the first held reply unpublished until expiry. self.entered.notify_one(); self.proceed.notified().await; return Err(object_store::Error::PermissionDenied { @@ -111,7 +166,7 @@ impl ObjectStore for PausedStore { } } let result = self.inner.put_opts(path, payload, options).await; - if delayed && progress == Some(2) && result.is_ok() { + if delayed && renewal && result.is_ok() { let expires = advertisement.as_ref().unwrap()["lease"]["expires_at_ms"] .as_str() .unwrap() From 703de9bafa889d664517d6d396446dec00737e56 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 20:01:00 -0700 Subject: [PATCH 08/19] fix: renew replay and reuse packs for fetches Keep multi-shard replay under renewed custody instead of its first SQL deadline. Full fetches reuse authenticated native packs after validating wants against the certified ref closure, avoiding per-object loose-file reconstruction. Preserve residence ceilings, owner fencing and rejection of unrelated wants. --- .../src/packs/publication/prepare.rs | 15 ++-- .../serving/session/workspace/prepare.rs | 34 +++++++++ .../src/packs/publication/staging_service.rs | 15 +++- .../packs/publication/tests/native_capture.rs | 71 ++++++++++++++++++- .../publication/tests/serving/workspace.rs | 24 +++++++ 5 files changed, 150 insertions(+), 9 deletions(-) diff --git a/crates/canopy-server/src/packs/publication/prepare.rs b/crates/canopy-server/src/packs/publication/prepare.rs index 52ad8951..9880b5a1 100644 --- a/crates/canopy-server/src/packs/publication/prepare.rs +++ b/crates/canopy-server/src/packs/publication/prepare.rs @@ -455,14 +455,14 @@ impl CatalogPreparation { &mut self, input: StagedNativeMetadata, ) -> Result<(), CatalogPreparationError> { - let deadline = self.start()?; + self.start()?; let context = self .staging .clone() .ok_or(CatalogPreparationError::Integrity)?; // The complete operation is poisoned on cancellation, including a // failed/absent provider artifact or an incomplete descriptor replay. - let result = timeout_at(deadline, async { + let work = async { let native = input.witness.native(); self.failed = false; self.begin_retained_pack(input.witness).await?; @@ -495,9 +495,14 @@ impl CatalogPreparation { context.ensure_live().map_err(PhysicalError::from)?; self.failed = false; self.finish_pack().await - }) - .await - .map_err(|_| PreparationBaseError::Inactive)?; + }; + // A full pack can span several renewed leases. Observe the shared + // custody clock and permanent fence instead of the first shard's + // deadline; the fixed bound residence ceiling still limits this work. + let result = tokio::select! { + result = work => result, + _ = context.fenced() => Err(PreparationBaseError::Inactive.into()), + }; if result.is_err() { self.failed = true; } diff --git a/crates/canopy-server/src/packs/publication/serving/session/workspace/prepare.rs b/crates/canopy-server/src/packs/publication/serving/session/workspace/prepare.rs index 89473ba8..992a28e2 100644 --- a/crates/canopy-server/src/packs/publication/serving/session/workspace/prepare.rs +++ b/crates/canopy-server/src/packs/publication/serving/session/workspace/prepare.rs @@ -218,6 +218,40 @@ impl NativeWorkspace { } } } + if filter.is_none() && !needs_blob_sizes { + // Wants were checked against the complete certified ref + // closure above. Reuse authenticated immutable packs for + // an unfiltered fetch; physical presence grants no wants. + let mut sources = reader.source_changes(None, None)?; + loop { + observation.refresh(&inner, &actor).await?; + let page = sources.page(128, 64 << 10).await?; + if page.is_empty() { + break; + } + for record in page { + observation.refresh(&inner, &actor).await?; + let native = record.native(); + native.validate(inner.context.repository(), inner.lease.format)?; + if !job(&spool, owner.clone(), move |s| s.pack_seen(native)).await? + { + inner + .context + .files + .install_pack_workspace( + core.cache.clone(), + native, + owner.clone(), + ) + .await?; + observation.refresh(&inner, &actor).await?; + job(&spool, owner.clone(), move |s| s.imported(native)).await?; + } + } + } + inner.observe(actor).await?; + return Ok(()); + } loop { observation.refresh(&inner, &actor).await?; let pending = job(&spool, owner.clone(), |s| s.pending()).await?; diff --git a/crates/canopy-server/src/packs/publication/staging_service.rs b/crates/canopy-server/src/packs/publication/staging_service.rs index 041e09a5..a5ca65e4 100644 --- a/crates/canopy-server/src/packs/publication/staging_service.rs +++ b/crates/canopy-server/src/packs/publication/staging_service.rs @@ -1518,6 +1518,19 @@ impl StagingTicket { Ok(ceiling) } #[cfg(test)] + pub(super) fn shorten_bound_lease_for_test( + &self, + remaining: Duration, + ) -> Result { + let mut local = self.job.local.lock().expect("staging local"); + let session = local.bound.as_ref().ok_or(StagingError::NotReady)?; + let deadline = (Instant::now() + remaining).min(session.live_lease()?.1); + *session.deadline.lock().expect("bound deadline") = deadline; + local.deadline = deadline; + self.job.changed.notify_one(); + Ok(deadline) + } + #[cfg(test)] pub(super) fn expire_bound_for_test(&self) -> Result { let mut local = self.job.local.lock().expect("staging local"); let session = local.bound.as_ref().ok_or(StagingError::NotReady)?; @@ -1599,7 +1612,7 @@ impl StagingContext { Ok(()) } } - async fn fenced(&self) { + pub(super) async fn fenced(&self) { let mut status = self.job.status.subscribe(); loop { let (deadline, session) = { diff --git a/crates/canopy-server/src/packs/publication/tests/native_capture.rs b/crates/canopy-server/src/packs/publication/tests/native_capture.rs index 9abfacdb..6c372255 100644 --- a/crates/canopy-server/src/packs/publication/tests/native_capture.rs +++ b/crates/canopy-server/src/packs/publication/tests/native_capture.rs @@ -444,6 +444,23 @@ enum MetadataFault { MissingArtifact, } +#[tokio::test] +async fn native_receive_metadata_replay_survives_original_deadline_after_custody_renewal() -> Result +{ + for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + let qualification = Box::pin(native_receive_metadata_case( + format, + false, + CompletionMode::Success, + None, + true, + )) + .await?; + assert!(qualification.is_none()); + } + Ok(()) +} + #[tokio::test] async fn native_receive_metadata_refuses_capacity_corrupt_replay_and_missing_shards_without_publication() -> Result { @@ -458,6 +475,7 @@ async fn native_receive_metadata_refuses_capacity_corrupt_replay_and_missing_sha true, CompletionMode::Success, Some(fault), + false, )) .await?; assert!(qualification.is_none()); @@ -476,7 +494,10 @@ async fn publication_state(fixture: &Fixture) -> Result> { type RootedQualification = std::pin::Pin>>; async fn native_receive_case(format: ObjectFormat, rooted: bool, mode: CompletionMode) -> Result { - let qualification = Box::pin(native_receive_metadata_case(format, rooted, mode, None)).await?; + let qualification = Box::pin(native_receive_metadata_case( + format, rooted, mode, None, false, + )) + .await?; if let Some(qualification) = qualification { qualification.await?; } @@ -487,6 +508,7 @@ async fn native_receive_metadata_case( rooted: bool, mode: CompletionMode, metadata_fault: Option, + renew_replay: bool, ) -> Result> { let ref_free = match mode { CompletionMode::RefFree { kind, .. } => Some(kind), @@ -497,7 +519,16 @@ async fn native_receive_metadata_case( }; let fixture = Fixture::new(format).await?; let provider = Arc::new(InMemory::new()); - let store = Arc::new(ArtifactStore::new(provider.clone(), fixture.repository)); + let throttled = Arc::new(object_store::throttle::ThrottledStore::new( + provider.clone(), + object_store::throttle::ThrottleConfig::default(), + )); + let object_store: Arc = if renew_replay { + throttled.clone() + } else { + provider.clone() + }; + let store = Arc::new(ArtifactStore::new(object_store, fixture.repository)); if rooted { let (base, _, _) = opened(&fixture, [159; 16], store.clone()).await?; let root = tempfile::TempDir::new()?; @@ -984,6 +1015,9 @@ async fn native_receive_metadata_case( } let producer_root = Arc::clone(&physical_root); let producer_disk = physical_disk.clone(); + let renewal_ticket = ticket.clone(); + let slow_store = throttled.clone(); + let (started, entered) = tokio::sync::oneshot::channel(); let work = ticket.spawn_bound(move |_, context| async move { let result = async { let mut builder = CatalogPreparation::new_staged( @@ -994,6 +1028,17 @@ async fn native_receive_metadata_case( limits(), ) .await?; + if renew_replay { + // Force the complete multi-shard replay across an old local + // deadline while a real registered renewal refreshes custody. + slow_store.config_mut(|config| { + config.wait_get_per_byte = Duration::from_micros(10); + }); + let deadline = renewal_ticket + .shorten_bound_lease_for_test(Duration::from_secs(1)) + .map_err(crate::packs::verification::PhysicalError::from)?; + let _ = started.send(deadline); + } let added = builder.add_staged_pack(staged).await; if metadata_fault.is_some() { let error = added.expect_err("invalid staged metadata accepted"); @@ -1009,7 +1054,23 @@ async fn native_receive_metadata_case( .await; result.map_err(|error| StagingError::Input(Box::new(error))) })?; + let original_deadline = if renew_replay { + let deadline = entered.await?; + // Let add_staged_pack capture the original deadline before renewal. + tokio::time::sleep(Duration::from_millis(50)).await; + ticket.renew_for_test(); + timeout(Duration::from_secs(1), async { + while ticket.bound_renewal().is_none() { + tokio::task::yield_now().await; + } + }) + .await?; + Some(deadline) + } else { + None + }; let prepared = work.wait().await; + throttled.config_mut(|config| *config = object_store::throttle::ThrottleConfig::default()); if metadata_fault.is_some() { assert!(prepared.is_err()); assert!(coordinator.close_and_drain().await.is_empty()); @@ -1019,7 +1080,11 @@ async fn native_receive_metadata_case( fixture.runtime.shutdown().await?; return Ok(None); } - let prepared = prepared.map_err(|error| error.to_string())?; + let prepared = prepared.map_err(|error| format!("{error:?}"))?; + if let Some(deadline) = original_deadline { + assert!(tokio::time::Instant::now() > deadline); + assert!(ticket.bound_session().is_ok()); + } if rooted { if matches!( mode, diff --git a/crates/canopy-server/src/packs/publication/tests/serving/workspace.rs b/crates/canopy-server/src/packs/publication/tests/serving/workspace.rs index 4103fbe0..8fb867e8 100644 --- a/crates/canopy-server/src/packs/publication/tests/serving/workspace.rs +++ b/crates/canopy-server/src/packs/publication/tests/serving/workspace.rs @@ -546,6 +546,30 @@ async fn reachability_stops_at_live_refs_without_scanning_other_history() -> Res String::from_utf8(refs)?, "refs/heads/main\nrefs/heads/side\n" ); + workspace + .prepare_fetch(vec![fixture.main], None, false) + .await?; + // The pack also physically contains this unrelated tag. Installing it + // must neither expand want authority nor create per-object loose files. + crate::packs::metadata::tests::git( + &workspace.git_dir(), + &["cat-file", "-e", &hex::encode(fixture.tag)], + None, + ) + .await?; + let main = hex::encode(fixture.main); + assert!( + !workspace + .git_dir() + .join("objects") + .join(&main[..2]) + .join(&main[2..]) + .exists() + ); + assert!(matches!( + GitGateway::validate_wants(&workspace, &BTreeSet::from([fixture.tag])).await, + Err(GatewayError::UnreachableWant) + )); let empty = RefStateSnapshotRoot::upload( &store, operation(122), From fdb8034c5a07a07a15e034acf4a90e3ee5d6b8f2 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 20:03:28 -0700 Subject: [PATCH 09/19] fix: bound production metadata replay to five minutes --- crates/canopy-server/src/server/residency/recovery.rs | 8 +++++++- docs/design/bound-preparation-lifecycle.md | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/canopy-server/src/server/residency/recovery.rs b/crates/canopy-server/src/server/residency/recovery.rs index 036e972b..a7728071 100644 --- a/crates/canopy-server/src/server/residency/recovery.rs +++ b/crates/canopy-server/src/server/residency/recovery.rs @@ -49,7 +49,13 @@ impl RecoveryServices { StagingCoordinator::new_resident( client.clone(), target.clone(), - StagingLimits::default(), + StagingLimits { + // Real histories can need more than one minute to replay + // verified metadata. Keep the existing five-minute ceiling; + // custody still renews independently every lease interval. + bound_lifetime_ms: crate::packs::publication::MAX_LEASE_MS, + ..StagingLimits::default() + }, authority.clone(), manager.staging_budget.clone(), coordinator.clone(), diff --git a/docs/design/bound-preparation-lifecycle.md b/docs/design/bound-preparation-lifecycle.md index 667aac42..34478442 100644 --- a/docs/design/bound-preparation-lifecycle.md +++ b/docs/design/bound-preparation-lifecycle.md @@ -14,7 +14,7 @@ A public Bound result remains recoverable after graceful stop and failed fresh c After binding or bound Claim, the same exact slot owns RenewPreparation. Its identity is created once, retained through absent/lost/panicked replies, and resolved before another renewal or checkpoint begins. A known renewal is saved in bound_renewal before fresh CheckPreparation. Renewing preserves the original attempt, namespace and generation floor; a current authorization/expiry/superseding-Claim failure fences the session without erasing the original renewal result. Unknown evidence remains admitted until exact recovery, including after close. -The existing overall staging lifetime still applies. bound_lifetime_ms adds a separate local ceiling from before preparation of Bind, or admission of bound Claim, rather than resetting it after a delayed/replayed reply. Default is 60 seconds; valid profiles use a nonzero ceiling up to MAX_LEASE_MS. The ceiling is fixed in the shared session and all derived bases. A direct fresh lease query or renewal cannot extend local use past it. The supervisor uses the raw fresh SQL deadline to schedule renewal and the residence ceiling to stop work; clipping the renewal schedule to the ceiling would cause a tight renewal loop near the limit. +The existing overall staging lifetime still applies. bound_lifetime_ms adds a separate local ceiling from before preparation of Bind, or admission of bound Claim, rather than resetting it after a delayed/replayed reply. The library default is 60 seconds; production repository residency selects the existing MAX_LEASE_MS ceiling of five minutes to admit metadata replay for real histories. SQL custody leases still renew independently at the normal interval. Valid profiles use a nonzero ceiling up to MAX_LEASE_MS. The ceiling is fixed in the shared session and all derived bases. A direct fresh lease query or renewal cannot extend local use past it. The supervisor uses the raw fresh SQL deadline to schedule renewal and the residence ceiling to stop work; clipping the renewal schedule to the ceiling would cause a tight renewal loop near the limit. The local ceiling does not shorten or remove the independent SQL pin. Already admitted or queued renewals can leave remaining SQL retention beyond local shutdown. Remaining-floor/fact capacity, command queue delay, server clock bounds and lease configuration still need qualification against the [large-team budgets](../large-team-scalability.md). Long physical imports belong in unbound staging; indefinitely renewing a bound floor is not permitted by this lifecycle. An expired local session cannot revive through exact replay or a successful later query. From 42ef51151ca3b0bbce9716d9061f56a9a154b973 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 21:09:04 -0700 Subject: [PATCH 10/19] fix: separate aggregate closure spool limit --- .../src/git_gateway/candidates/produce.rs | 1 + crates/canopy-server/src/git_gateway/head.rs | 1 + crates/canopy-server/src/git_gateway/merge.rs | 1 + .../src/git_gateway/push/native.rs | 1 + crates/canopy-server/src/packs/closure/mod.rs | 10 +++++ .../src/packs/publication/prepare.rs | 42 +++++++++++++++---- .../packs/publication/tests/native_capture.rs | 1 + docs/design/admitted-sqlite-growth.md | 2 + 8 files changed, 51 insertions(+), 8 deletions(-) diff --git a/crates/canopy-server/src/git_gateway/candidates/produce.rs b/crates/canopy-server/src/git_gateway/candidates/produce.rs index b2d85902..1c234406 100644 --- a/crates/canopy-server/src/git_gateway/candidates/produce.rs +++ b/crates/canopy-server/src/git_gateway/candidates/produce.rs @@ -231,6 +231,7 @@ impl GitGateway { gateway.disk_budget.clone(), base, MetadataLimits::default(), + crate::packs::closure::default_limits(), ) .await .map_err(work)?; diff --git a/crates/canopy-server/src/git_gateway/head.rs b/crates/canopy-server/src/git_gateway/head.rs index 30af755d..0653e7de 100644 --- a/crates/canopy-server/src/git_gateway/head.rs +++ b/crates/canopy-server/src/git_gateway/head.rs @@ -119,6 +119,7 @@ impl GitGateway { gateway.disk_budget.clone(), base, MetadataLimits::default(), + crate::packs::closure::default_limits(), ) .await .map_err(work)? diff --git a/crates/canopy-server/src/git_gateway/merge.rs b/crates/canopy-server/src/git_gateway/merge.rs index 8b642680..2d77b0ec 100644 --- a/crates/canopy-server/src/git_gateway/merge.rs +++ b/crates/canopy-server/src/git_gateway/merge.rs @@ -144,6 +144,7 @@ impl GitGateway { gateway.disk_budget.clone(), base, MetadataLimits::default(), + crate::packs::closure::default_limits(), ) .await .map_err(work)? diff --git a/crates/canopy-server/src/git_gateway/push/native.rs b/crates/canopy-server/src/git_gateway/push/native.rs index 9e4095f1..7febece7 100644 --- a/crates/canopy-server/src/git_gateway/push/native.rs +++ b/crates/canopy-server/src/git_gateway/push/native.rs @@ -244,6 +244,7 @@ impl GitGateway { gateway.disk_budget.clone(), base, MetadataLimits::default(), + crate::packs::closure::default_limits(), ) .await .map_err(input)?; diff --git a/crates/canopy-server/src/packs/closure/mod.rs b/crates/canopy-server/src/packs/closure/mod.rs index 1399199b..edae4f1e 100644 --- a/crates/canopy-server/src/packs/closure/mod.rs +++ b/crates/canopy-server/src/packs/closure/mod.rs @@ -34,6 +34,16 @@ use spool::Spool; pub use verifier::ClosureVerifier; pub use witness::ClosureWitness; +/// Aggregate incoming graphs contain the edges of every metadata shard. Keep +/// their disposable spool ceiling separate from immutable shard/read limits. +/// Growth still requires admission for database, journal and SQLite overhead. +pub fn default_limits() -> MetadataLimits { + MetadataLimits { + max_file_bytes: 1 << 30, + ..MetadataLimits::default() + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ClosureBase { pub catalog: StoredCatalog, diff --git a/crates/canopy-server/src/packs/publication/prepare.rs b/crates/canopy-server/src/packs/publication/prepare.rs index 9880b5a1..6331fe72 100644 --- a/crates/canopy-server/src/packs/publication/prepare.rs +++ b/crates/canopy-server/src/packs/publication/prepare.rs @@ -178,6 +178,11 @@ pub struct CatalogPreparation { input_checkpoint_digest: Option<[u8; 32]>, failed: bool, } +struct ConstructionLimits { + input: MetadataLimits, + closure: MetadataLimits, + output: MetadataLimits, +} impl CatalogPreparation { pub async fn new( root: &Path, @@ -206,16 +211,30 @@ impl CatalogPreparation { limits: MetadataLimits, output_limits: MetadataLimits, ) -> Result { - Self::new_inner(root, budget, base, limits, output_limits, None).await + Self::new_inner( + root, + budget, + base, + ConstructionLimits { + input: limits, + closure: limits, + output: output_limits, + }, + None, + ) + .await } /// Production construction retains the admitted worker through every /// detached assembler job, while the finished private proof owns no worker. + /// The aggregate graph has an explicit ceiling independent of input shards + /// and output runs; all three still share the caller's admitted disk budget. pub async fn new_staged( context: &StagingContext, root: &Path, budget: DiskBudget, base: Arc, limits: MetadataLimits, + closure_limits: MetadataLimits, ) -> Result { context.ensure_live().map_err(PhysicalError::from)?; if context.token().map_err(PhysicalError::from)? != base.context_token() @@ -227,10 +246,13 @@ impl CatalogPreparation { root, budget, base, - limits, - MetadataLimits { - max_file_bytes: limits.max_file_bytes.min(RUN_TARGET_BYTES), - ..limits + ConstructionLimits { + input: limits, + closure: closure_limits, + output: MetadataLimits { + max_file_bytes: limits.max_file_bytes.min(RUN_TARGET_BYTES), + ..limits + }, }, Some(context.clone()), ) @@ -240,10 +262,14 @@ impl CatalogPreparation { root: &Path, budget: DiskBudget, base: Arc, - limits: MetadataLimits, - output_limits: MetadataLimits, + limits: ConstructionLimits, staging: Option, ) -> Result { + let ConstructionLimits { + input: limits, + closure: closure_limits, + output: output_limits, + } = limits; DirectoryPartitioner::validate_limits(output_limits)?; let (lease, deadline) = base.live_lease()?; let indexes = base.indexes(); @@ -269,7 +295,7 @@ impl CatalogPreparation { Arc::clone(&workspace), budget.clone(), context, - limits, + closure_limits, activity.clone(), ) .await?; diff --git a/crates/canopy-server/src/packs/publication/tests/native_capture.rs b/crates/canopy-server/src/packs/publication/tests/native_capture.rs index 6c372255..e036ed63 100644 --- a/crates/canopy-server/src/packs/publication/tests/native_capture.rs +++ b/crates/canopy-server/src/packs/publication/tests/native_capture.rs @@ -1026,6 +1026,7 @@ async fn native_receive_metadata_case( producer_disk, base, limits(), + limits(), ) .await?; if renew_replay { diff --git a/docs/design/admitted-sqlite-growth.md b/docs/design/admitted-sqlite-growth.md index a91a9ddf..775d8ba5 100644 --- a/docs/design/admitted-sqlite-growth.md +++ b/docs/design/admitted-sqlite-growth.md @@ -32,6 +32,8 @@ Sealing closes SQLite, removes its journal, synchronizes and hashes the file, th At the 256 MiB main-file default, one empty construction spool initially charges 192 KiB instead of 768 MiB. Two such spools initially charge 384 KiB. Growing to the full ceiling can still charge 768 MiB per spool, and old immutable files, cached downloads, native workspaces and edge files retain their independent reservations. Geometric growth can deny a larger quantum before all shared free space is consumed; this is bounded backpressure, not permission to oversubscribe disk. Do not equate the initial charge with peak operation capacity. +Production staged catalog preparation gives the aggregate closure spool a separate 1 GiB main-file ceiling. It initially reserves the same 192 KiB and can grow only under the shared disk budget, reserving up to 3 GiB at the ceiling. Metadata shards retain their 256 MiB ceiling and directory output runs retain their 64 MiB bound. A full-history graph can exceed one shard’s size: the libgit2 verification corpus contains 134,967 objects and 3,158,723 typed dependency edges; a diagnostic reconstruction with the current closure schema measured 377,081,856 bytes after lookup creation. This is scratch sizing evidence, not a successful import or resource qualification. Explicit caller ceilings, denied-growth behavior, cancellation, lease fencing and the finite residence budget remain enforced. + Ancestry now uses admitted growth too. Native Git descendant RSS, CPU and I/O admission, large-file profiles, full-history throughput, durable authenticated input adoption, producer integration and continuous maintenance remain separate release gates. These construction changes do not select the production hard-cutover registry or prove capacity for 10,000 engineers. ## Evidence From 58b4b52a783a08c7d28f6a320f1e689ad5a87492 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 22:27:15 -0700 Subject: [PATCH 11/19] fix: renew assembly and reuse fetched packs --- .../src/packs/catalog/graph_spool.rs | 37 ++++++++++++- .../src/packs/catalog/graph_spool/tests.rs | 10 ++++ .../src/packs/publication/prepare.rs | 16 ++++-- .../publication/serving/session/workspace.rs | 4 ++ .../serving/session/workspace/prepare.rs | 15 +++++- .../packs/publication/tests/native_capture.rs | 53 +++++++++++++++---- .../publication/tests/serving/workspace.rs | 9 ++++ .../tests/multi_server/partial_clone.rs | 33 ++++++++++-- .../tests/multi_server/ssh/fetch.rs | 29 ++++++++-- 9 files changed, 181 insertions(+), 25 deletions(-) diff --git a/crates/canopy-server/src/packs/catalog/graph_spool.rs b/crates/canopy-server/src/packs/catalog/graph_spool.rs index 4e91a3fd..9dd1269b 100644 --- a/crates/canopy-server/src/packs/catalog/graph_spool.rs +++ b/crates/canopy-server/src/packs/catalog/graph_spool.rs @@ -40,7 +40,7 @@ impl GraphSpool { db.pragma_update(None, "cache_size", -(i64::from(cache_kib)))?; growth::configure(&db, &mut file)?; growth::transaction(&mut db, &mut file, maximum, |tx| { - tx.execute_batch("CREATE TABLE nodes(oid BLOB PRIMARY KEY,kind TEXT,done INTEGER NOT NULL DEFAULT 0 CHECK(done IN (0,1))) WITHOUT ROWID; CREATE INDEX pending ON nodes(done,oid); CREATE TABLE packs(checksum BLOB PRIMARY KEY,pack BLOB NOT NULL,idx BLOB NOT NULL,pack_bytes INTEGER NOT NULL,idx_bytes INTEGER NOT NULL,objects INTEGER NOT NULL) WITHOUT ROWID;")?; + tx.execute_batch("CREATE TABLE nodes(oid BLOB PRIMARY KEY,kind TEXT,done INTEGER NOT NULL DEFAULT 0 CHECK(done IN (0,1))) WITHOUT ROWID; CREATE INDEX pending ON nodes(done,oid); CREATE TABLE packs(checksum BLOB PRIMARY KEY,pack BLOB NOT NULL,idx BLOB NOT NULL,pack_bytes INTEGER NOT NULL,idx_bytes INTEGER NOT NULL,objects INTEGER NOT NULL,complete INTEGER NOT NULL DEFAULT 0 CHECK(complete IN (0,1))) WITHOUT ROWID;")?; Ok::<_, MetadataError>(()) })?; Ok(Self { @@ -172,7 +172,7 @@ impl GraphSpool { ) -> Result<(), MetadataError> { growth::transaction(&mut self.db, &mut self.file, self.maximum, |tx| { tx.execute( - "INSERT INTO packs VALUES(?1,?2,?3,?4,?5,?6)", + "INSERT INTO packs VALUES(?1,?2,?3,?4,?5,?6,0)", params![ p.git_checksum.as_ref(), p.pack.digest.as_slice(), @@ -185,6 +185,39 @@ impl GraphSpool { Ok::<_, MetadataError>(()) }) } + /// Logical source discovery does not mean its bytes were installed. Bind + /// physical completion to the exact verified pack/index identity separately. + pub(in crate::packs) fn pack_complete( + &self, + p: NativePackDescriptor, + ) -> Result { + if !self.pack_seen(p)? { + return Ok(false); + } + Ok(self.db.query_row( + "SELECT complete FROM packs WHERE checksum=?1", + [p.git_checksum.as_ref()], + |r| r.get(0), + )?) + } + pub(in crate::packs) fn mark_pack_complete( + &mut self, + p: NativePackDescriptor, + ) -> Result<(), MetadataError> { + if !self.pack_seen(p)? { + self.imported(p)?; + } + growth::transaction(&mut self.db, &mut self.file, self.maximum, |tx| { + if tx.execute( + "UPDATE packs SET complete=1 WHERE checksum=?1", + [p.git_checksum.as_ref()], + )? != 1 + { + return Err(MetadataError::Integrity); + } + Ok(()) + }) + } pub(in crate::packs) fn contains(&self, ids: &[ObjectId]) -> Result, MetadataError> { let mut q = self .db diff --git a/crates/canopy-server/src/packs/catalog/graph_spool/tests.rs b/crates/canopy-server/src/packs/catalog/graph_spool/tests.rs index bbe5e6e4..43c9adca 100644 --- a/crates/canopy-server/src/packs/catalog/graph_spool/tests.rs +++ b/crates/canopy-server/src/packs/catalog/graph_spool/tests.rs @@ -167,11 +167,16 @@ fn physical_input_dedup_ignores_namespace_but_rejects_conflicting_bindings() -> }, }; assert!(!s.pack_seen(p)?); + assert!(!s.pack_complete(p)?); s.imported(p)?; + assert!(!s.pack_complete(p)?); + s.mark_pack_complete(p)?; + assert!(s.pack_complete(p)?); let mut other = p; other.operation = [7; 16]; other.pack.manifest_digest = [8; 32]; assert!(s.pack_seen(other)?); + assert!(s.pack_complete(other)?); for mutation in 0..5 { let mut conflict = other; match mutation { @@ -185,7 +190,12 @@ fn physical_input_dedup_ignores_namespace_but_rejects_conflicting_bindings() -> s.pack_seen(conflict), Err(MetadataError::IdentityConflict) )); + assert!(matches!( + s.pack_complete(conflict), + Err(MetadataError::IdentityConflict) + )); } assert_eq!(s.counts()?, (0, 1)); + assert_eq!(s.contains(&[id(ObjectFormat::Sha1, 1)])?, [false]); Ok(()) } diff --git a/crates/canopy-server/src/packs/publication/prepare.rs b/crates/canopy-server/src/packs/publication/prepare.rs index 6331fe72..798ef97e 100644 --- a/crates/canopy-server/src/packs/publication/prepare.rs +++ b/crates/canopy-server/src/packs/publication/prepare.rs @@ -558,9 +558,19 @@ impl CatalogPreparation { if self.active.is_some() { return Err(CatalogPreparationError::Integrity); } - timeout_at(deadline, self.finish_inner()) - .await - .map_err(|_| PreparationBaseError::Inactive)? + if let Some(context) = self.staging.clone() { + // Closure certification and immutable output construction can span + // renewed leases. Observe shared custody and its residence ceiling + // rather than the deadline captured at the start of this phase. + tokio::select! { + result = self.finish_inner() => result, + _ = context.fenced() => Err(PreparationBaseError::Inactive.into()), + } + } else { + timeout_at(deadline, self.finish_inner()) + .await + .map_err(|_| PreparationBaseError::Inactive)? + } } async fn finish_inner(mut self) -> Result { let context = self.base.context(); diff --git a/crates/canopy-server/src/packs/publication/serving/session/workspace.rs b/crates/canopy-server/src/packs/publication/serving/session/workspace.rs index a1f45ea9..a379385f 100644 --- a/crates/canopy-server/src/packs/publication/serving/session/workspace.rs +++ b/crates/canopy-server/src/packs/publication/serving/session/workspace.rs @@ -301,6 +301,10 @@ impl ServingPin { } observation.refresh(&inner, &actor).await?; job(&spool, owner.clone(), move |s| s.imported(source)).await?; + if materialize { + job(&spool, owner.clone(), move |s| s.mark_pack_complete(source)) + .await?; + } stats.packs = stats .packs .checked_add(1) diff --git a/crates/canopy-server/src/packs/publication/serving/session/workspace/prepare.rs b/crates/canopy-server/src/packs/publication/serving/session/workspace/prepare.rs index 992a28e2..ae363d0c 100644 --- a/crates/canopy-server/src/packs/publication/serving/session/workspace/prepare.rs +++ b/crates/canopy-server/src/packs/publication/serving/session/workspace/prepare.rs @@ -222,6 +222,11 @@ impl NativeWorkspace { // Wants were checked against the complete certified ref // closure above. Reuse authenticated immutable packs for // an unfiltered fetch; physical presence grants no wants. + // SSH v2 can prepare several groups on the same cache. + // Keep completion on that cache's shared spool and + // serialize installation through its existing lock. + let _selection = core.cache.selection.lock().await; + observation.refresh(&inner, &actor).await?; let mut sources = reader.source_changes(None, None)?; loop { observation.refresh(&inner, &actor).await?; @@ -233,7 +238,10 @@ impl NativeWorkspace { observation.refresh(&inner, &actor).await?; let native = record.native(); native.validate(inner.context.repository(), inner.lease.format)?; - if !job(&spool, owner.clone(), move |s| s.pack_seen(native)).await? + if !job(&core.spool, owner.clone(), move |s| { + s.pack_complete(native) + }) + .await? { inner .context @@ -245,7 +253,10 @@ impl NativeWorkspace { ) .await?; observation.refresh(&inner, &actor).await?; - job(&spool, owner.clone(), move |s| s.imported(native)).await?; + job(&core.spool, owner.clone(), move |s| { + s.mark_pack_complete(native) + }) + .await?; } } } diff --git a/crates/canopy-server/src/packs/publication/tests/native_capture.rs b/crates/canopy-server/src/packs/publication/tests/native_capture.rs index e036ed63..79888cef 100644 --- a/crates/canopy-server/src/packs/publication/tests/native_capture.rs +++ b/crates/canopy-server/src/packs/publication/tests/native_capture.rs @@ -444,6 +444,29 @@ enum MetadataFault { MissingArtifact, } +#[derive(Clone, Copy, PartialEq, Eq)] +enum RenewalPhase { + Replay, + Finish, +} + +#[tokio::test] +async fn native_receive_metadata_finish_survives_original_deadline_after_custody_renewal() -> Result +{ + for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + let qualification = Box::pin(native_receive_metadata_case( + format, + false, + CompletionMode::Success, + None, + Some(RenewalPhase::Finish), + )) + .await?; + assert!(qualification.is_none()); + } + Ok(()) +} + #[tokio::test] async fn native_receive_metadata_replay_survives_original_deadline_after_custody_renewal() -> Result { @@ -453,7 +476,7 @@ async fn native_receive_metadata_replay_survives_original_deadline_after_custody false, CompletionMode::Success, None, - true, + Some(RenewalPhase::Replay), )) .await?; assert!(qualification.is_none()); @@ -475,7 +498,7 @@ async fn native_receive_metadata_refuses_capacity_corrupt_replay_and_missing_sha true, CompletionMode::Success, Some(fault), - false, + None, )) .await?; assert!(qualification.is_none()); @@ -495,7 +518,7 @@ type RootedQualification = std::pin::Pin Result { let qualification = Box::pin(native_receive_metadata_case( - format, rooted, mode, None, false, + format, rooted, mode, None, None, )) .await?; if let Some(qualification) = qualification { @@ -508,7 +531,7 @@ async fn native_receive_metadata_case( rooted: bool, mode: CompletionMode, metadata_fault: Option, - renew_replay: bool, + renewal: Option, ) -> Result> { let ref_free = match mode { CompletionMode::RefFree { kind, .. } => Some(kind), @@ -523,7 +546,7 @@ async fn native_receive_metadata_case( provider.clone(), object_store::throttle::ThrottleConfig::default(), )); - let object_store: Arc = if renew_replay { + let object_store: Arc = if renewal.is_some() { throttled.clone() } else { provider.clone() @@ -1018,6 +1041,7 @@ async fn native_receive_metadata_case( let renewal_ticket = ticket.clone(); let slow_store = throttled.clone(); let (started, entered) = tokio::sync::oneshot::channel(); + let mut started = Some(started); let work = ticket.spawn_bound(move |_, context| async move { let result = async { let mut builder = CatalogPreparation::new_staged( @@ -1029,7 +1053,7 @@ async fn native_receive_metadata_case( limits(), ) .await?; - if renew_replay { + if renewal == Some(RenewalPhase::Replay) { // Force the complete multi-shard replay across an old local // deadline while a real registered renewal refreshes custody. slow_store.config_mut(|config| { @@ -1038,7 +1062,7 @@ async fn native_receive_metadata_case( let deadline = renewal_ticket .shorten_bound_lease_for_test(Duration::from_secs(1)) .map_err(crate::packs::verification::PhysicalError::from)?; - let _ = started.send(deadline); + let _ = started.take().expect("one renewal phase").send(deadline); } let added = builder.add_staged_pack(staged).await; if metadata_fault.is_some() { @@ -1050,14 +1074,25 @@ async fn native_receive_metadata_case( return Err(error); } added?; + if renewal == Some(RenewalPhase::Finish) { + // Delay the first immutable output upload beyond the old + // deadline while the same bound session receives a renewal. + slow_store.config_mut(|config| { + config.wait_put_per_call = Duration::from_millis(1100); + }); + let deadline = renewal_ticket + .shorten_bound_lease_for_test(Duration::from_secs(1)) + .map_err(crate::packs::verification::PhysicalError::from)?; + let _ = started.take().expect("one renewal phase").send(deadline); + } builder.finish().await } .await; result.map_err(|error| StagingError::Input(Box::new(error))) })?; - let original_deadline = if renew_replay { + let original_deadline = if renewal.is_some() { let deadline = entered.await?; - // Let add_staged_pack capture the original deadline before renewal. + // Let the selected assembly phase capture its deadline before renewal. tokio::time::sleep(Duration::from_millis(50)).await; ticket.renew_for_test(); timeout(Duration::from_secs(1), async { diff --git a/crates/canopy-server/src/packs/publication/tests/serving/workspace.rs b/crates/canopy-server/src/packs/publication/tests/serving/workspace.rs index 8fb867e8..4ab5c2c0 100644 --- a/crates/canopy-server/src/packs/publication/tests/serving/workspace.rs +++ b/crates/canopy-server/src/packs/publication/tests/serving/workspace.rs @@ -549,6 +549,15 @@ async fn reachability_stops_at_live_refs_without_scanning_other_history() -> Res workspace .prepare_fetch(vec![fixture.main], None, false) .await?; + workspace + .prepare_fetch(vec![fixture.side], None, false) + .await?; + let (first, second) = tokio::join!( + workspace.prepare_fetch(vec![fixture.main], None, false), + workspace.prepare_fetch(vec![fixture.side], None, false), + ); + first?; + second?; // The pack also physically contains this unrelated tag. Installing it // must neither expand want authority nor create per-object loose files. crate::packs::metadata::tests::git( diff --git a/crates/canopy-server/tests/multi_server/partial_clone.rs b/crates/canopy-server/tests/multi_server/partial_clone.rs index 04ea2485..2193b566 100644 --- a/crates/canopy-server/tests/multi_server/partial_clone.rs +++ b/crates/canopy-server/tests/multi_server/partial_clone.rs @@ -133,7 +133,10 @@ async fn cold_single_branch_fetch_skips_unrelated_commit_and_tree_history() -> R ], ) .await?; - assert!(stored_objects(&clone).await?.contains(&last_tree)); + let objects = stored_objects(&clone).await?; + assert!(objects.contains(&last_tree)); + assert!(!objects.contains(&other_commit)); + assert!(!objects.contains(&other_tree)); assert!(!cache_contains(&disk, &other_commit)?); assert!(!cache_contains(&disk, &other_tree)?); run_git( @@ -147,8 +150,17 @@ async fn cold_single_branch_fetch_skips_unrelated_commit_and_tree_history() -> R ], ) .await?; - assert!(cache_contains(&disk, &other_commit)?); - assert!(cache_contains(&disk, &other_tree)?); + let objects = stored_objects(&clone).await?; + assert!(objects.contains(&other_commit)); + assert!(objects.contains(&other_tree)); + for (kind, object) in [("commit", &other_commit), ("tree", &other_tree)] { + assert_eq!( + run_git(Some(&clone), &["cat-file", kind, object]).await?, + run_git(Some(&source), &["cat-file", kind, object]).await? + ); + } + assert!(!cache_contains(&disk, &other_commit)?); + assert!(!cache_contains(&disk, &other_tree)?); restored.shutdown().await?; } Ok(()) @@ -313,8 +325,8 @@ async fn filtered_clones_round_trip(store: Arc) -> Result { assert_eq!(tokio::fs::read(clone.join("two")).await?, bodies[1]); run_git(Some(&clone), &["fsck", "--strict"]).await?; } - // Explicitly cache the branch while referenced, then delete it again. Fetch - // no longer loads orphan blobs, but cached orphans must still be denied. + // Cache an explicit blob want while its branch is referenced, then delete + // the branch. Physically cached orphans must fail reachability validation. run_git( Some(&source), &["-c", AUTH, "push", &url, "refs/heads/discarded"], @@ -327,6 +339,8 @@ async fn filtered_clones_round_trip(store: Arc) -> Result { "-c", AUTH, "clone", + "--filter=blob:none", + "--no-checkout", "--single-branch", "--branch", "discarded", @@ -335,6 +349,15 @@ async fn filtered_clones_round_trip(store: Arc) -> Result { ], ) .await?; + assert_eq!( + run_git( + Some(&discarded_clone), + &["-c", AUTH, "cat-file", "blob", &discarded] + ) + .await?, + run_git(Some(&source), &["cat-file", "blob", &discarded]).await? + ); + assert!(cache_contains(&restored_disk, &discarded)?); run_git( Some(&source), &["-c", AUTH, "push", &url, ":refs/heads/discarded"], diff --git a/crates/canopy-server/tests/multi_server/ssh/fetch.rs b/crates/canopy-server/tests/multi_server/ssh/fetch.rs index f827cc74..8d7a11a3 100644 --- a/crates/canopy-server/tests/multi_server/ssh/fetch.rs +++ b/crates/canopy-server/tests/multi_server/ssh/fetch.rs @@ -138,7 +138,7 @@ async fn cold_ssh_blobless_clones_hydrate_only_explicit_blob_wants() -> Result { } #[tokio::test(flavor = "multi_thread")] -async fn cold_full_fetches_only_hydrate_blobs_reachable_from_requested_tips() -> Result { +async fn cold_full_fetches_reuse_packs_without_expanding_blobs_into_loose_files() -> Result { let workspace = tempfile::TempDir::new()?; let store: Arc = Arc::new(InMemory::new()); let host = ssh_key::PrivateKey::new( @@ -228,7 +228,28 @@ async fn cold_full_fetches_only_hydrate_blobs_reachable_from_requested_tips() -> vec![0x61; 2 * 1024 * 1024] ); git(Some(&clone), &ssh, &["fsck", "--strict", "--full"]).await?; - assert!(cache_contains(&cold, &blobs[0])?); + let objects = String::from_utf8( + git( + Some(&clone), + &ssh, + &[ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname)", + ], + ) + .await?, + )?; + assert!(objects.contains(&blobs[0])); + assert!( + !objects.contains(&blobs[1]), + "{name}: unrelated branch sent to client" + ); + assert!( + !objects.contains(&blobs[2]), + "{name}: deleted branch sent to client" + ); + assert!(!cache_contains(&cold, &blobs[0])?); assert!( !cache_contains(&cold, &blobs[1])?, "{name}: unrelated branch blob hydrated" @@ -268,7 +289,7 @@ async fn cold_full_fetches_only_hydrate_blobs_reachable_from_requested_tips() -> git(Some(&clone), &ssh, &["show", "origin/other:asset"]).await?, vec![0x62; 2 * 1024 * 1024] ); - assert!(cache_contains(&cold, &blobs[1])?); + assert!(!cache_contains(&cold, &blobs[1])?); assert!(!cache_contains(&cold, &blobs[2])?); git(Some(&clone), &ssh, &["fsck", "--strict", "--full"]).await?; restored.shutdown().await?; @@ -311,7 +332,7 @@ async fn cold_full_fetches_only_hydrate_blobs_reachable_from_requested_tips() -> git(Some(&existing), &ssh, &["show", "origin/other:asset"]).await?, vec![0x62; 2 * 1024 * 1024] ); - assert!(cache_contains(&cold, &blobs[1])?); + assert!(!cache_contains(&cold, &blobs[1])?); assert!( !cache_contains(&cold, &blobs[0])?, "{name}: negotiation hydrated an unrequested blob" From 12dc9a19881ca4c87353a7643a374d20bc800d58 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 22:33:29 -0700 Subject: [PATCH 12/19] perf: retain medium repository metadata cache --- .../canopy-server/src/packs/catalog/files.rs | 4 +- .../src/packs/catalog/tests/files.rs | 53 +++++++++++++++++++ docs/design/admitted-sqlite-growth.md | 2 + 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/crates/canopy-server/src/packs/catalog/files.rs b/crates/canopy-server/src/packs/catalog/files.rs index 47f464fc..bfd42195 100644 --- a/crates/canopy-server/src/packs/catalog/files.rs +++ b/crates/canopy-server/src/packs/catalog/files.rs @@ -30,7 +30,9 @@ impl Default for CatalogFileLimits { fn default() -> Self { Self { open_files: 32, - cached_files: 16, + // A medium full-history import can have 17 metadata shards plus + // directory files. Leave eight of the 32 slots for borrowed work. + cached_files: 24, metadata: MetadataLimits::default(), } } diff --git a/crates/canopy-server/src/packs/catalog/tests/files.rs b/crates/canopy-server/src/packs/catalog/tests/files.rs index b5df889a..d6df291c 100644 --- a/crates/canopy-server/src/packs/catalog/tests/files.rs +++ b/crates/canopy-server/src/packs/catalog/tests/files.rs @@ -50,6 +50,59 @@ async fn source(prepared: &Prepared) -> Result { ) } +#[tokio::test] +async fn default_cache_reuses_medium_repository_metadata_working_set() -> Result { + for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + let prepared = prepared(format).await?; + let stored_run = run(&prepared).await?; + let objects: Vec<_> = prepared.fixture.objects.values().cloned().collect(); + let (&oid, (expected, _)) = prepared.fixture.objects.iter().next().ok_or("object")?; + let mut segments = Vec::new(); + // Seventeen distinct authenticated metadata files and one directory + // match the working-set shape that exceeded the former cache capacity. + for number in 80..97 { + let mut identity = prepared.fixture.identity; + identity.operation = [number; 16]; + let mut writer = builder(&prepared.fixture, DiskBudget::new(128 << 20), identity)?; + fill(&mut writer, &objects)?; + segments.push( + Arc::new(writer.seal(&prepared.fixture.index)?) + .upload(&prepared.store) + .await?, + ); + } + for (profile, expected_downloads) in [ + ( + CatalogFileLimits { + cached_files: 16, + ..CatalogFileLimits::default() + }, + 36, + ), + (CatalogFileLimits::default(), 18), + ] { + let budget = DiskBudget::new(128 << 20); + let loader = files(&prepared, budget.clone(), profile)?; + for pass in 0..2 { + drop(RunLoader::load(&loader, stored_run).await?); + for &stored in &segments { + let metadata = SourceLoader::load(&loader, stored).await?; + assert_eq!(metadata.header(oid)?.ok_or("header")?.object, *expected); + } + assert_eq!( + loader.stats()?.downloaded_files, + if pass == 0 { 18 } else { expected_downloads } + ); + assert!(loader.stats()?.open_files <= profile.open_files); + assert!(loader.stats()?.cached_files <= profile.cached_files); + } + drop(loader); + assert_eq!(budget.used(), 0); + } + } + Ok(()) +} + #[tokio::test] async fn worker_files_coalesce_native_lookups_and_reuse_cache_across_roots() -> Result { for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { diff --git a/docs/design/admitted-sqlite-growth.md b/docs/design/admitted-sqlite-growth.md index 775d8ba5..43a21e1f 100644 --- a/docs/design/admitted-sqlite-growth.md +++ b/docs/design/admitted-sqlite-growth.md @@ -34,6 +34,8 @@ At the 256 MiB main-file default, one empty construction spool initially charges Production staged catalog preparation gives the aggregate closure spool a separate 1 GiB main-file ceiling. It initially reserves the same 192 KiB and can grow only under the shared disk budget, reserving up to 3 GiB at the ceiling. Metadata shards retain their 256 MiB ceiling and directory output runs retain their 64 MiB bound. A full-history graph can exceed one shard’s size: the libgit2 verification corpus contains 134,967 objects and 3,158,723 typed dependency edges; a diagnostic reconstruction with the current closure schema measured 377,081,856 bytes after lookup creation. This is scratch sizing evidence, not a successful import or resource qualification. Explicit caller ceilings, denied-growth behavior, cancellation, lease fencing and the finite residence budget remain enforced. +The default authenticated file cache retains at most 24 files under the unchanged 32 open-file limit and shared disk budget. This leaves eight slots for borrowed or queued work while accommodating a directory and the 17 metadata files produced by the libgit2 corpus. The cache regression cycles 18 authenticated files twice: a 16-file control downloads 36 files, while the default cache downloads each file once. This is a bounded working-set regression; cold-clone and provider resource qualification remain separate evidence. + Ancestry now uses admitted growth too. Native Git descendant RSS, CPU and I/O admission, large-file profiles, full-history throughput, durable authenticated input adoption, producer integration and continuous maintenance remain separate release gates. These construction changes do not select the production hard-cutover registry or prove capacity for 10,000 engineers. ## Evidence From b197b49212a13e01bcdeac803aa262b4068c9ce7 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 23:40:44 -0700 Subject: [PATCH 13/19] test: bound merge retirement stack use --- .../tests/native_merge/retirement.rs | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs b/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs index 6a3a3b87..88641ba8 100644 --- a/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs +++ b/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs @@ -46,18 +46,27 @@ async fn native_merge_applied_attempt_releases_pin_without_losing_original_uuid_ let (command, registered) = prepared_command(&f, &prepared, request.clone(), root.path(), budget.clone()).await?; let admin = terminal_retention::maintenance(&f.handle, f.repository).await?; + // Keep these nested recovery futures off the default 2 MiB test stack. assert!( - registered - .ready_terminal_release(f.client(), &graph.store, admin.clone(), identity()?) - .await - .is_err() + Box::pin(registered.ready_terminal_release( + f.client(), + &graph.store, + admin.clone(), + identity()? + )) + .await + .is_err() ); let original = command.execute().await?; - let released = registered - .ready_terminal_release(f.client(), &graph.store, admin, identity()?) - .await? - .complete() - .await?; + let released = Box::pin(registered.ready_terminal_release( + f.client(), + &graph.store, + admin, + identity()?, + )) + .await? + .complete() + .await?; assert_eq!(released.output, TerminalReleaseReply::Released); let token = check.token; f.handle.query(0,128,move |db| { @@ -71,14 +80,13 @@ async fn native_merge_applied_attempt_releases_pin_without_losing_original_uuid_ let restored = RegisteredRootRecovery::load(&f.client(), &f.target, &graph.store, &check) .await? .ok_or("merge archive absent")?; - let PublicationOutcome::Merge(recovered) = restored - .dispatch_any( - &f.client(), - &graph.store, - &f.authority(), - &std::sync::atomic::AtomicBool::new(false), - ) - .await? + let PublicationOutcome::Merge(recovered) = Box::pin(restored.dispatch_any( + &f.client(), + &graph.store, + &f.authority(), + &std::sync::atomic::AtomicBool::new(false), + )) + .await? else { return Err("merge archive purpose".into()); }; @@ -94,12 +102,16 @@ async fn native_merge_applied_attempt_releases_pin_without_losing_original_uuid_ // transaction. Its independent pin can then select the original audit. let admin = terminal_retention::maintenance(&f.handle, f.repository).await?; assert_eq!( - retry_recovery - .ready_terminal_release(f.client(), &graph.store, admin, identity()?) - .await? - .complete() - .await? - .output, + Box::pin(retry_recovery.ready_terminal_release( + f.client(), + &graph.store, + admin, + identity()? + )) + .await? + .complete() + .await? + .output, TerminalReleaseReply::Released ); merged_roots(&f, &graph).await?; From 7362d076748d851db3d9e82a5e832e7090c34bbe Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 23:53:39 -0700 Subject: [PATCH 14/19] test: heap-allocate merge retirement scenario --- .../tests/native_merge/retirement.rs | 73 +++++++++---------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs b/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs index 88641ba8..74a81a5f 100644 --- a/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs +++ b/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs @@ -40,33 +40,35 @@ async fn retained_pin(f: &Fixture, check: &LeaseCheck, expected: u64) -> Result async fn native_merge_applied_attempt_releases_pin_without_losing_original_uuid_or_receipt() -> Result { for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + applied_attempt(format).await?; + } + Ok(()) +} + +// Construct the scenario in a separate call so the harness and polling frame +// hold its heap pointer instead of the large recovery future on a 2 MiB stack. +fn applied_attempt( + format: ObjectFormat, +) -> std::pin::Pin>> { + Box::pin(async move { let (f, graph, request) = initial(format, false).await?; let (prepared, root, budget) = preparation(&f, &graph).await?; let check = prepared.base.capability().2.clone(); let (command, registered) = prepared_command(&f, &prepared, request.clone(), root.path(), budget.clone()).await?; let admin = terminal_retention::maintenance(&f.handle, f.repository).await?; - // Keep these nested recovery futures off the default 2 MiB test stack. assert!( - Box::pin(registered.ready_terminal_release( - f.client(), - &graph.store, - admin.clone(), - identity()? - )) - .await - .is_err() + registered + .ready_terminal_release(f.client(), &graph.store, admin.clone(), identity()?) + .await + .is_err() ); let original = command.execute().await?; - let released = Box::pin(registered.ready_terminal_release( - f.client(), - &graph.store, - admin, - identity()?, - )) - .await? - .complete() - .await?; + let released = registered + .ready_terminal_release(f.client(), &graph.store, admin, identity()?) + .await? + .complete() + .await?; assert_eq!(released.output, TerminalReleaseReply::Released); let token = check.token; f.handle.query(0,128,move |db| { @@ -80,13 +82,14 @@ async fn native_merge_applied_attempt_releases_pin_without_losing_original_uuid_ let restored = RegisteredRootRecovery::load(&f.client(), &f.target, &graph.store, &check) .await? .ok_or("merge archive absent")?; - let PublicationOutcome::Merge(recovered) = Box::pin(restored.dispatch_any( - &f.client(), - &graph.store, - &f.authority(), - &std::sync::atomic::AtomicBool::new(false), - )) - .await? + let PublicationOutcome::Merge(recovered) = restored + .dispatch_any( + &f.client(), + &graph.store, + &f.authority(), + &std::sync::atomic::AtomicBool::new(false), + ) + .await? else { return Err("merge archive purpose".into()); }; @@ -102,16 +105,12 @@ async fn native_merge_applied_attempt_releases_pin_without_losing_original_uuid_ // transaction. Its independent pin can then select the original audit. let admin = terminal_retention::maintenance(&f.handle, f.repository).await?; assert_eq!( - Box::pin(retry_recovery.ready_terminal_release( - f.client(), - &graph.store, - admin, - identity()? - )) - .await? - .complete() - .await? - .output, + retry_recovery + .ready_terminal_release(f.client(), &graph.store, admin, identity()?) + .await? + .complete() + .await? + .output, TerminalReleaseReply::Released ); merged_roots(&f, &graph).await?; @@ -120,8 +119,8 @@ async fn native_merge_applied_attempt_releases_pin_without_losing_original_uuid_ drop(graph.prepared); cleaned(graph.root.path(), &graph.budget).await?; f.runtime.shutdown().await?; - } - Ok(()) + Ok(()) + }) } #[tokio::test] From 930e6c7a4018b02081ac1421d1f1d5ce9e63691e Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 00:38:59 -0700 Subject: [PATCH 15/19] test: heap-allocate retirement scenarios --- .../src/packs/publication/tests/native_head.rs | 14 ++++++++++++-- .../publication/tests/native_merge/retirement.rs | 13 +++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/crates/canopy-server/src/packs/publication/tests/native_head.rs b/crates/canopy-server/src/packs/publication/tests/native_head.rs index 042bdeb5..e8d5ddc8 100644 --- a/crates/canopy-server/src/packs/publication/tests/native_head.rs +++ b/crates/canopy-server/src/packs/publication/tests/native_head.rs @@ -73,6 +73,16 @@ async fn recover( async fn native_head_reuses_ref_tree_and_preserves_original_receipt_after_typed_retirement() -> Result { for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + retired_head(format).await?; + } + Ok(()) +} + +// Construct the complete scenario outside the harness's polling stack frame. +fn retired_head( + format: ObjectFormat, +) -> std::pin::Pin>> { + Box::pin(async move { let (f, graph, _) = initial(format, false).await?; let (prepared, root, budget) = preparation(&f, &graph).await?; let check = prepared.base.capability().2.clone(); @@ -219,8 +229,8 @@ async fn native_head_reuses_ref_tree_and_preserves_original_receipt_after_typed_ runtime.shutdown().await?; drop(graph.prepared); cleaned(graph.root.path(), &graph.budget).await?; - } - Ok(()) + Ok(()) + }) } #[tokio::test] async fn native_head_denials_recheck_current_authority_generation_and_native_branch_existence() diff --git a/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs b/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs index 74a81a5f..8b1c9049 100644 --- a/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs +++ b/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs @@ -126,6 +126,15 @@ fn applied_attempt( #[tokio::test] async fn negative_merge_archive_keeps_its_denial_after_same_uuid_succeeds() -> Result { for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + negative_archive(format).await?; + } + Ok(()) +} + +fn negative_archive( + format: ObjectFormat, +) -> std::pin::Pin>> { + Box::pin(async move { let (f, graph, request) = initial(format, false).await?; let (prepared, root, budget) = preparation(&f, &graph).await?; let check = prepared.base.capability().2.clone(); @@ -184,8 +193,8 @@ async fn negative_merge_archive_keeps_its_denial_after_same_uuid_succeeds() -> R drop(graph.prepared); cleaned(graph.root.path(), &graph.budget).await?; f.runtime.shutdown().await?; - } - Ok(()) + Ok(()) + }) } #[tokio::test] From 624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 10:42:53 -0700 Subject: [PATCH 16/19] test: hydrate cache before process takeover --- scripts/smoke_s3_process.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/smoke_s3_process.py b/scripts/smoke_s3_process.py index daff1bc6..2dba2894 100644 --- a/scripts/smoke_s3_process.py +++ b/scripts/smoke_s3_process.py @@ -336,6 +336,20 @@ def clone_and_verify(url, directory, expected_oid, expected_readme, expected_lfs assert (directory / "asset.lfs").read_bytes() == expected_lfs +def hydrate_fetch_cache(url, directory, expected_oid, expected_readme, branch="main"): + authorization = "http.extraHeader=Authorization: Bearer local-test-token" + git("-c", authorization, "clone", "--filter=blob:none", "--no-checkout", + "--single-branch", "--branch", branch, url, str(directory)) + readme = git("rev-parse", f"{expected_oid.decode()}:README.md", cwd=directory).decode() + actual = subprocess.run( + ["git", "-c", "credential.helper=", "-c", authorization, "cat-file", "blob", readme], + cwd=directory, capture_output=True, check=True, + env={**os.environ, "GIT_TERMINAL_PROMPT": "0"}, + ).stdout + assert actual == expected_readme + return readme + + def push_with_lost_reply(base_url, local): captured = {} push_id = str(uuid.uuid4()) @@ -758,11 +772,15 @@ def main(): verify_discovery(base_url, reader_token, ["renamed"]) clone_and_verify(url, directory / "clean-clone", oid, b"Canopy process smoke\n", lfs_body, branch="trunk") clone_and_verify(f"{base_url}/canopy/other.git", directory / "clean-other", other_oid, other_readme) + # Materialize real lazy-fetch hydration scratch for owner-death cleanup. + readme = hydrate_fetch_cache(url, directory / "partial-before-owner-death", + oid, b"Canopy process smoke\n", branch="trunk") second.kill() second.wait(timeout=10) abandoned = directory / "second" / "canopy-pack-v2" - stale_caches = list(abandoned.glob("canopy-git-*")) - assert stale_caches, "expected a retained fetch cache at owner death" + stale_caches = list(abandoned.rglob("canopy-git-*")) + assert any((cache / "repo.git" / "objects" / readme[:2] / readme[2:]).is_file() + for cache in stale_caches), "expected the hydrated blob cache at owner death" sentinel = abandoned / "abandoned-upload" sentinel.write_bytes(b"disposable scratch") # SQLite is a recovered cache: corrupting the killed node's copy must From 1a0fc93d526d205979e03523435ba0c6a9a027c1 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 12:02:59 -0700 Subject: [PATCH 17/19] test: preserve peer HTTP streaming --- scripts/smoke_s3_peers.py | 40 ++++----------------------------------- 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/scripts/smoke_s3_peers.py b/scripts/smoke_s3_peers.py index 351af300..b6d2838b 100644 --- a/scripts/smoke_s3_peers.py +++ b/scripts/smoke_s3_peers.py @@ -1,57 +1,25 @@ """Two live HTTPS peers, remote Git/LFS, and takeover without gateway restart.""" from contextlib import contextmanager -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -import http.client import json import os import signal import ssl import subprocess -import threading import time import uuid from lease_contract import NODE_LEASE_WAIT_SECONDS +from local_tcp_proxy import LocalProxy @contextmanager def proxy(upstream, certificate, key): - class Handler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def log_message(self, *_args): - pass - - def do_POST(self): - body = self.rfile.read(int(self.headers["Content-Length"])) - connection = http.client.HTTPConnection(upstream, timeout=30) - try: - connection.request("POST", self.path, body, - {"Content-Type": "application/octet-stream"}) - response = connection.getresponse() - reply = response.read() - self.send_response(response.status) - self.send_header("Content-Length", str(len(reply))) - self.end_headers() - self.wfile.write(reply) - except (OSError, http.client.HTTPException): - self.send_error(502) - finally: - connection.close() - context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) context.load_cert_chain(certificate, key) - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - server.socket = context.wrap_socket(server.socket, server_side=True) - worker = threading.Thread(target=server.serve_forever, daemon=True) - worker.start() - try: - yield f"https://127.0.0.1:{server.server_port}" - finally: - server.shutdown() - server.server_close() - worker.join() + # Preserve Git/LFS methods, authenticated headers, and streaming bodies. + with LocalProxy([upstream], tls_context=context) as relay: + yield relay.url def peer_certificate(directory): From 81925e121c2fad14c25ab993001835b1bc5dc3d3 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 12:13:21 -0700 Subject: [PATCH 18/19] test: verify native backup inventory --- scripts/smoke_s3_backup.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/scripts/smoke_s3_backup.py b/scripts/smoke_s3_backup.py index c79cf9a9..5197244e 100644 --- a/scripts/smoke_s3_backup.py +++ b/scripts/smoke_s3_backup.py @@ -69,12 +69,28 @@ def administer(action, *roots, fails=False): raise RuntimeError(f"backup {action} failed: {error}") return json.loads(result.stdout) - saved = administer("create", backup_prefix) - assert saved["cells"] == 2 and saved["external_objects"] == 3 - assert administer("create", backup_prefix) == saved aws = ["aws"] if os.environ.get("AWS_ENDPOINT"): aws.extend(["--endpoint-url", os.environ["AWS_ENDPOINT"]]) + + def external_roots(prefix): + root = f"{prefix}/repos/{uuid.UUID(repository).hex}/" + listing = subprocess.run([ + *aws, "s3api", "list-objects-v2", "--bucket", + urllib.parse.urlsplit(backup_url).netloc, "--prefix", root, + ], check=True, capture_output=True) + return sorted(item["Key"][len(root):] + for item in json.loads(listing.stdout).get("Contents", []) + if ".parts/" not in item["Key"]) + + saved = administer("create", backup_prefix) + roots = external_roots(backup_prefix) + assert {root.split("/")[0] for root in roots} == { + "git-packs", "git-catalogs", "git-inputs", "lfs"} + assert [root for root in roots if root.startswith("lfs/")] == sorted([ + f"lfs/{empty_oid}", f"lfs/{hashlib.sha256(lfs).hexdigest()}"]) + assert saved["cells"] == 2 and saved["external_objects"] == len(roots) + assert administer("create", backup_prefix) == saved lfs_url = f"{backup_url}/repos/{uuid.UUID(repository).hex}/lfs/{hashlib.sha256(lfs).hexdigest()}.parts/0000000000000000" corrupt = directory / "corrupt-backup-lfs" corrupt.write_bytes(b"corrupt fixture bytes") @@ -96,7 +112,9 @@ def administer(action, *roots, fails=False): assert json.loads(listing.stdout).get("KeyCount", 0) == 0 assert administer("verify", backup_prefix) == saved restored = administer("restore", backup_prefix, restore_prefix) - assert restored["cells"] == 2 and restored["external_objects"] == 3 + assert restored["cells"] == saved["cells"] + assert restored["external_objects"] == saved["external_objects"] + assert external_roots(restore_prefix) == roots node, restored_base = start(binary, directory, {**settings, "storage_url": restored_url, "node_id": str(uuid.uuid4())}, "backup-new-node") processes.append(node) @@ -107,5 +125,12 @@ def administer(action, *roots, fails=False): node.send_signal(signal.SIGTERM) node.wait(timeout=30) assert node.returncode == 0 + (directory / "backup-results.json").write_text(json.dumps({ + "saved": saved, "restored": restored, + "external_manifest_count": len(roots), + "external_inventory_sha256": hashlib.sha256("\n".join(roots).encode()).hexdigest(), + "source_deleted": True, "corrupt_copy_rejected": True, + "cold_clone_verified": True, "issue_verified": True, + }, indent=2)) print(f"LFS qualification: bytes={len(lfs)}, stock push={push_seconds:.2f}s, restored clone+verification={clone_seconds:.2f}s", flush=True) - print("PASS: stock 80 MiB Git blob plus 80 MiB and empty LFS transfers, then real backup CLI copies SQLite and external bodies; deleting every original source object still permits verification, isolated restore, exact stock clone/LFS and issue recovery", flush=True) + print("PASS: stock 80 MiB Git blob plus 80 MiB and empty LFS transfers, then real backup CLI copies SQLite and native artifacts; deleting every original source object still permits verification, isolated restore, exact stock clone/LFS and issue recovery", flush=True) From 96917d13193baf5819aa0525ce05e4fc12fe2002 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 12:22:26 -0700 Subject: [PATCH 19/19] docs: record complete storage qualification --- README.md | 3 +- ROADMAP.md | 2 +- .../2026-10-14-storage-optimization.md | 7 +- .../benchmarks/2026-10-15-native-storage.json | 1980 +++++++++++++++++ docs/benchmarks/2026-10-15-native-storage.md | 136 ++ docs/implementation.md | 8 +- .../large-repository-implementation-status.md | 12 + docs/operations.md | 15 +- 8 files changed, 2148 insertions(+), 15 deletions(-) create mode 100644 docs/benchmarks/2026-10-15-native-storage.json create mode 100644 docs/benchmarks/2026-10-15-native-storage.md diff --git a/README.md b/README.md index b6b64706..00fdd4c3 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,8 @@ sequenceDiagram This is a development path for one node with an existing S3-compatible object store. The [bounded Linux deployment](deploy/README.md) describes the container profile and its resource limits. Use a **new storage prefix** for this build; -there is no migration from earlier preview layouts yet. +breaking schema or format changes require a fresh prefix. This build supports +only its compiled release and current storage formats. 1. Install Rust 1.97 or newer and Git with `http-backend`, `merge-tree --write-tree` (`-z --name-only --no-messages`) and `commit-tree`. diff --git a/ROADMAP.md b/ROADMAP.md index 51ee5bbc..aa740a0a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -36,7 +36,7 @@ This is the first intended production-use milestone. The existing bounded Compos - [ ] Add a fenced collector for abandoned uploads, old outcomes/candidates, and unreachable Git/LFS objects, with an inspectable dry run and bounded work. - [ ] Prove with concurrent reads, backup, publication, owner loss, and injected collector failures that required bytes are never removed; collect abandoned bytes only after their grace period. -**Current state:** `canopy gc` uses Cellule's retention-aware collector under a verified maintenance drain, backup pins, grace/deletion bounds and a durable worker claim. It collects superseded Cellule state. Product-owned pack/catalog/LFS artifacts and abandoned staging need their own root and retention policy; complete collection qualification remains open. +**Current state:** `canopy gc` uses Cellule's retention-aware collector under a verified maintenance drain, backup pins, grace/deletion bounds and a durable worker claim. It collects superseded Cellule state. The [October 15 qualification](docs/benchmarks/2026-10-15-native-storage.md) records three full repositories before and after collection and fresh restore. Product-owned pack/catalog/LFS artifacts and abandoned staging still need their own root and retention policy; R03 remains open. ### R04. Trustworthy CI and release artifacts diff --git a/docs/benchmarks/2026-10-14-storage-optimization.md b/docs/benchmarks/2026-10-14-storage-optimization.md index f51f599f..c28cdf5f 100644 --- a/docs/benchmarks/2026-10-14-storage-optimization.md +++ b/docs/benchmarks/2026-10-14-storage-optimization.md @@ -3,9 +3,10 @@ The current integration uses the native catalog/publication implementation from `main` at `aa18585`, published Cellule 0.2.0, repository schema 4 and storage format `canopy-pack-v2`. It retains same-release recovery and requires fresh -storage prefixes. Current integrated gates and RustFS measurements are pending. -All measurements below belong to the earlier, separate implementation and its -recorded binary; they do not qualify this integration. +storage prefixes. The [October 15 qualification](2026-10-15-native-storage.md) +records the current integration and its separate source, binary and measurements. +All measurements below belong to the earlier implementation and its recorded +binary; they remain historical evidence. The earlier storage changes keep Git bodies in verified, immutable native pack/index pairs, batch metadata publication, and expose Cellule's retention collector diff --git a/docs/benchmarks/2026-10-15-native-storage.json b/docs/benchmarks/2026-10-15-native-storage.json new file mode 100644 index 00000000..a9cfb1ed --- /dev/null +++ b/docs/benchmarks/2026-10-15-native-storage.json @@ -0,0 +1,1980 @@ +{ + "captured_utc": "2026-10-15T19:22:15.998649+00:00", + "source_commit": "12dc9a19881ca4c87353a7643a374d20bc800d58", + "qualification_source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "test_stack_source_equivalence": { + "state": "passed", + "source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "runtime_base_commit": "12dc9a19881ca4c87353a7643a374d20bc800d58", + "files_compared": 727, + "changed_files": [ + "crates/canopy-server/src/packs/publication/tests/native_head.rs", + "crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs" + ], + "compiled_runtime_change": false, + "cfg_test_boundary": "crates/canopy-server/src/packs/publication/mod.rs:304", + "original_source_archive": "source.tar", + "qualification_source_archive": "source-test-retirement-heaps.tar", + "captured_utc": "2026-10-15T07:46:02.402713+00:00" + }, + "original_stack_reproduction": { + "binary": "canopy_server-4c94d44cbce73e9e", + "stack_bytes": 2097152, + "exit_code": -6, + "seconds": 73.89485187502578 + }, + "fixed_stack_reproduction": { + "state": "passed", + "stack_bytes": 2097152, + "source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "started_utc": "2026-10-15T07:46:02.507893+00:00", + "exit_code": 0, + "seconds": 196.71240079205018 + }, + "additional_stack_diagnosis": { + "state": "diagnosis-complete", + "source_commit": "7362d076748d851db3d9e82a5e832e7090c34bbe", + "binary_sha256": "40b38d0c10dbfc08798fe80252c8a4d0d4d95ddda3f75a68d2a6307cb5c44ba0", + "stack_bytes": 2097152, + "previously_passed": 432, + "known_failed": [ + "packs::publication::tests::native_head::native_head_reuses_ref_tree_and_preserves_original_receipt_after_typed_retirement" + ], + "runs": [ + { + "exit_code": -6, + "seconds": 15.571115583996288, + "finished": 12, + "stack_overflows": [ + "packs::publication::tests::native_merge::retirement::negative_merge_archive_keeps_its_denial_after_same_uuid_succeeds" + ], + "log": "remaining-unit-stacks-1.log" + }, + { + "exit_code": 0, + "seconds": 803.8268396250205, + "finished": 303, + "stack_overflows": [], + "log": "remaining-unit-stacks-2.log" + } + ], + "started_utc": "2026-10-15T07:31:31.349814+00:00", + "passed": 747, + "failed": [ + "packs::publication::tests::native_head::native_head_reuses_ref_tree_and_preserves_original_receipt_after_typed_retirement", + "packs::publication::tests::native_merge::retirement::negative_merge_archive_keeps_its_denial_after_same_uuid_succeeds" + ], + "remaining": 0, + "finished_utc": "2026-10-15T07:45:11.495453+00:00" + }, + "final_harness_source_equivalence": { + "state": "passed", + "source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "rust_qualified_source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "changed_paths": [ + "scripts/smoke_s3_backup.py", + "scripts/smoke_s3_peers.py", + "scripts/smoke_s3_process.py" + ], + "all_other_724_source_files_identical": true, + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8" + }, + "backup_harness_source_equivalence": { + "state": "passed", + "source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "previous_source_commit": "1a0fc93d526d205979e03523435ba0c6a9a027c1", + "changed_paths": [ + "scripts/smoke_s3_backup.py" + ], + "rust_source_unchanged": true, + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "python_tests": 105, + "python_exit_code": 0, + "python_seconds": 45.085, + "python_log": "python-backup-harness.log", + "backup_script_sha256": "ffb5a2ca1a7b5ab20bc735a980628f28c0217ee56aac8ca948fe6adeb783af0a" + }, + "backup_count_diagnosis": { + "state": "diagnosed", + "expected_legacy_count": 3, + "actual_external_manifest_count": 26, + "physical_key_count": 61, + "families": { + "git-catalogs": 9, + "git-inputs": 12, + "git-packs": 3, + "lfs": 2 + }, + "inventory_sha256": "7a1feb76fc5896bd0443c447522a7f19bbc69bccb5123b31de686f19cebbc9df" + }, + "backup_retention_probe": { + "state": "passed", + "started_utc": "2026-10-15T19:12:39.473607+00:00", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "script_sha256": { + "smoke_s3_backup.py": "ffb5a2ca1a7b5ab20bc735a980628f28c0217ee56aac8ca948fe6adeb783af0a", + "smoke_s3_retention.py": "551afec5c8e4cff728bbb48f0605967e838a28d67b3e45322be58ef7938558b1" + }, + "backup": { + "saved": { + "pin": "4d4de244-3545-4f5e-91ee-ed1126244eb4", + "root": "qualification/backup-native-inventory/process-smoke/d3c8abc2-9d5b-4da6-8ef6-bc504cb508bc/backup-copy", + "cells": 2, + "external_objects": 26 + }, + "restored": { + "pin": "4d4de244-3545-4f5e-91ee-ed1126244eb4", + "root": "qualification/backup-native-inventory/process-smoke/d3c8abc2-9d5b-4da6-8ef6-bc504cb508bc/backup-restored", + "cells": 2, + "external_objects": 26 + }, + "external_manifest_count": 26, + "external_inventory_sha256": "73183146886bfb8c69fdde0b93e6a0abb0c170a402ed1407fe427ddf9ffe06fd", + "source_deleted": true, + "corrupt_copy_rejected": true, + "cold_clone_verified": true, + "issue_verified": true + }, + "retention": { + "deleted_objects": 248, + "last_report": { + "listed_objects": 410, + "immutable_candidates": 299, + "reachable_objects": 52, + "retained_objects": 52, + "grace_objects": 0, + "eligible_objects": 247, + "deleted_objects": 247, + "current_controls": 2, + "retained_pins": 1, + "complete": true + }, + "backup_verified": true, + "cold_clone_verified": true + }, + "seconds": 120.83030666701961, + "finished_utc": "2026-10-15T19:14:40.315911+00:00" + }, + "superseded_extended_process": { + "state": "failed", + "started_utc": "2026-10-15T19:03:33.782137+00:00", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "peer_script_sha256": "62566d1eb9bb7910053af81040e070b3d583e270869f0e4ed0473f2d26b62b77", + "source_commit": "1a0fc93d526d205979e03523435ba0c6a9a027c1", + "profile": [ + "--large-clone", + "--many-objects", + "256", + "--structural-objects", + "--structural-object-bytes", + "68157440" + ], + "error": "", + "seconds": 416.4211825419916, + "finished_utc": "2026-10-15T19:10:30.257295+00:00" + }, + "peer_harness_source_equivalence": { + "state": "passed", + "source_commit": "1a0fc93d526d205979e03523435ba0c6a9a027c1", + "previous_source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "changed_paths": [ + "scripts/smoke_s3_peers.py" + ], + "rust_source_unchanged": true, + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "python_tests": 105, + "python_exit_code": 0, + "python_seconds": 47.655, + "python_log": "python-peer-harness.log", + "peer_script_sha256": "62566d1eb9bb7910053af81040e070b3d583e270869f0e4ed0473f2d26b62b77" + }, + "peer_tls_probe": { + "state": "passed", + "started_utc": "2026-10-15T19:02:05.942183+00:00", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "peer_script_sha256": "62566d1eb9bb7910053af81040e070b3d583e270869f0e4ed0473f2d26b62b77", + "seconds": 157.06976595905144, + "finished_utc": "2026-10-15T19:04:43.108462+00:00" + }, + "extended_process": { + "state": "passed", + "started_utc": "2026-10-15T19:13:22.285310+00:00", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "backup_script_sha256": "ffb5a2ca1a7b5ab20bc735a980628f28c0217ee56aac8ca948fe6adeb783af0a", + "source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "profile": [ + "--large-clone", + "--many-objects", + "256", + "--structural-objects", + "--structural-object-bytes", + "68157440" + ], + "seconds": 493.9588900420349, + "finished_utc": "2026-10-15T19:21:36.316138+00:00" + }, + "process_harness_python": { + "state": "passed", + "source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "started_utc": "2026-10-15T17:44:11.910904+00:00", + "exit_code": 0, + "seconds": 42.9214055420598, + "finished_utc": "2026-10-15T17:44:54.832541+00:00" + }, + "process_harness_activation": { + "state": "passed", + "started_utc": "2026-10-15T17:44:12.823561+00:00", + "source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "rust_qualified_source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "changed_files": [ + "scripts/smoke_s3_process.py" + ], + "all_other_files_unchanged": true, + "binary_sha256": "0be8986fe9d9a9c12780dead22f9c8dc8c63a3adf88cdc7f682778e5bfc9f254", + "files_verified": 727, + "finished_utc": "2026-10-15T18:34:34.242039+00:00" + }, + "process_cache_recovery_probe": { + "state": "passed", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "harness_sha256": "02a170d0925ae6f05e4c559001b7c4926331dfa83048587c45f061f9664f97ef", + "storage_url": "s3://canopy-benchmark/qualification/cache-hydration/cf6fead6-2877-4c18-8018-02ceaa875581", + "started_utc": "2026-10-15T17:40:35.009927+00:00", + "blob": "12735a460af03df3b27e7a8dd63e6e1d1c2188f4", + "stale_cache_count": 1, + "hydrated_cache_count": 1, + "stale_cache_reclaimed": true, + "corrupt_sqlite_discarded": true, + "acknowledged_git_and_head_restored": true, + "clean_shutdown": true, + "seconds": 35.77593683404848, + "finished_utc": "2026-10-15T17:41:10.786269+00:00" + }, + "binary": { + "source_commit": "12dc9a19881ca4c87353a7643a374d20bc800d58", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "binary_bytes": 70621392, + "profile": "release", + "cellule": "0.2.0", + "schema": 4, + "storage_format": "canopy-pack-v2", + "qualification_status": "repository verification running; full workspace/provider/process gates pending", + "runtime_equivalent_source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "runtime_image_equivalence": "runtime-image-equivalence.json", + "immutable_qualified_artifact": true + }, + "dependency_check": { + "captured_utc": "2026-10-15T05:35:35.029928+00:00", + "source": "Official crates.io API, live requests", + "releases": [ + { + "name": "cellule-app", + "source": "https://crates.io/api/v1/crates/cellule-app", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_utc": "2026-10-13T01:16:20.482357Z" + }, + { + "name": "cellule-host", + "source": "https://crates.io/api/v1/crates/cellule-host", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_utc": "2026-10-13T01:16:36.982608Z" + }, + { + "name": "cellule-ltx", + "source": "https://crates.io/api/v1/crates/cellule-ltx", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_utc": "2026-10-13T01:14:56.127567Z" + }, + { + "name": "cellule-runtime", + "source": "https://crates.io/api/v1/crates/cellule-runtime", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_utc": "2026-10-13T01:15:54.928938Z" + }, + { + "name": "cellule-store", + "source": "https://crates.io/api/v1/crates/cellule-store", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_utc": "2026-10-13T01:14:45.791071Z" + }, + { + "name": "cellule-types", + "source": "https://crates.io/api/v1/crates/cellule-types", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_utc": "2026-10-13T01:12:54.510864Z" + } + ], + "aligned_with_compiled_release": true + }, + "final_dependency_check": { + "checked_utc": "2026-10-15T07:08:45.122451+00:00", + "source": "Official crates.io API", + "crates": [ + { + "crate": "cellule-app", + "url": "https://crates.io/api/v1/crates/cellule-app", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_at": "2026-10-13T01:16:20.482357Z" + }, + { + "crate": "cellule-host", + "url": "https://crates.io/api/v1/crates/cellule-host", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_at": "2026-10-13T01:16:36.982608Z" + }, + { + "crate": "cellule-ltx", + "url": "https://crates.io/api/v1/crates/cellule-ltx", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_at": "2026-10-13T01:14:56.127567Z" + }, + { + "crate": "cellule-runtime", + "url": "https://crates.io/api/v1/crates/cellule-runtime", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_at": "2026-10-13T01:15:54.928938Z" + }, + { + "crate": "cellule-store", + "url": "https://crates.io/api/v1/crates/cellule-store", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_at": "2026-10-13T01:14:45.791071Z" + }, + { + "crate": "cellule-types", + "url": "https://crates.io/api/v1/crates/cellule-types", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_at": "2026-10-13T01:12:54.510864Z" + } + ] + }, + "evening_dependency_check": { + "checked_utc": "2026-10-15T18:41:35.144852+00:00", + "source": "Official crates.io API", + "crates": [ + { + "crate": "cellule-app", + "url": "https://crates.io/api/v1/crates/cellule-app", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0" + }, + { + "crate": "cellule-host", + "url": "https://crates.io/api/v1/crates/cellule-host", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0" + }, + { + "crate": "cellule-ltx", + "url": "https://crates.io/api/v1/crates/cellule-ltx", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0" + }, + { + "crate": "cellule-runtime", + "url": "https://crates.io/api/v1/crates/cellule-runtime", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0" + }, + { + "crate": "cellule-store", + "url": "https://crates.io/api/v1/crates/cellule-store", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0" + }, + { + "crate": "cellule-types", + "url": "https://crates.io/api/v1/crates/cellule-types", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0" + } + ] + }, + "runtime_image_equivalence": { + "state": "passed", + "source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "measured_runtime_source_commit": "12dc9a19881ca4c87353a7643a374d20bc800d58", + "measured_binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "current_build_binary_sha256": "cc08a2361f792147047b7c1ff1ca30ef2ca6ab0a4445db93adfda45510634190", + "binary_bytes": 70621392, + "differing_bytes": 47, + "excluded_metadata_ranges": [ + { + "kind": "Mach-O LC_UUID", + "start": 2024, + "end": 2040 + }, + { + "kind": "derived SHA-256 code-sign first-page hash", + "start": 70073936, + "end": 70073968 + } + ], + "all_other_file_bytes_equal": true, + "first_signature_page_hashes_verified": true, + "normalized_file_sha256": "7f3703573f7c4ff20ca1d302011d3462df08c4c7d4e47dd0b48651225b831956", + "method": "Compare every byte; only the explicit linker UUID and its verified derived ad-hoc signature page hash differ. Neither executable is modified.", + "captured_utc": "2026-10-15T18:47:57.518869+00:00" + }, + "corpus": { + "state": "passed", + "captured_utc": "2026-10-15T05:49:21.123694+00:00", + "source_commit": "58b4b52a783a08c7d28f6a320f1e689ad5a87492+working-91f9507aafeec4ba", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "cellule": "0.2.0", + "repository_schema": 4, + "storage_format": "canopy-pack-v2", + "repositories": [ + { + "repository": "bat", + "objects": 20952, + "refs": 66, + "all_object_bytes_equal": true, + "clone_fsck_passed": true, + "import_seconds": 21.917903000023216, + "clone_seconds": 30.795469499891624, + "api_blob_samples": 3, + "api_commit_records_checked": 128, + "head_reference": "refs/heads/master", + "head_oid": "d9559c69f541873f8c47ac8ad81d985f4441edf6", + "all_object_bytes_sha256": "ef957f77437a906af80f6381aaa0a78998ea8ce7f700876427f0624cb0c87100" + }, + { + "repository": "starship", + "objects": 38333, + "refs": 164, + "all_object_bytes_equal": true, + "clone_fsck_passed": true, + "import_seconds": 38.66624029201921, + "clone_seconds": 47.59193941601552, + "api_blob_samples": 3, + "api_commit_records_checked": 128, + "head_reference": "refs/heads/main", + "head_oid": "df688000684138943610bbaa2a6c454db6987d48", + "all_object_bytes_sha256": "b4a625af0cc416e04af88e614d0dcd2f8a2f4b4d82bb41805b980c90b7304b88" + }, + { + "repository": "libgit2", + "objects": 134967, + "refs": 271, + "all_object_bytes_equal": true, + "clone_fsck_passed": true, + "import_seconds": 363.5411464580102, + "clone_seconds": 240.1334371669218, + "api_blob_samples": 3, + "api_commit_records_checked": 128, + "head_reference": "refs/heads/main", + "head_oid": "4af47f040cb4d80c75f37f626f511843e0dd8d44", + "all_object_bytes_sha256": "9a2d91500b281267cf1e5f92493a8b76a3a5a3c626a7f9b3f6f7e770714a6821" + } + ], + "tree_pagination_paths": [ + { + "repository": "bat", + "path_base64": "", + "pages": 1, + "entries": 24, + "all_equal": true + }, + { + "repository": "bat", + "path_base64": "dGVzdHM", + "pages": 1, + "entries": 16, + "all_equal": true + }, + { + "repository": "bat", + "path_base64": "dGVzdHMvZXhhbXBsZXM", + "pages": 2, + "entries": 39, + "all_equal": true + }, + { + "repository": "bat", + "path_base64": "dGVzdHMvc25hcHNob3RzL291dHB1dA", + "pages": 2, + "entries": 33, + "all_equal": true + }, + { + "repository": "starship", + "path_base64": "", + "pages": 1, + "entries": 26, + "all_equal": true + }, + { + "repository": "starship", + "path_base64": "LmdpdGh1Yi93b3JrZmxvd3M", + "pages": 1, + "entries": 8, + "all_equal": true + }, + { + "repository": "starship", + "path_base64": "aW5zdGFsbC9tYWNvc19wYWNrYWdlcy9wa2dfcmVzb3VyY2VzL0VuZ2xpc2gubHByb2o", + "pages": 1, + "entries": 3, + "all_equal": true + }, + { + "repository": "starship", + "path_base64": "bWVkaWE", + "pages": 1, + "entries": 20, + "all_equal": true + }, + { + "repository": "libgit2", + "path_base64": "", + "pages": 1, + "entries": 29, + "all_equal": true + }, + { + "repository": "libgit2", + "path_base64": "ZGVwcy9wY3JlMg", + "pages": 2, + "entries": 42, + "all_equal": true + }, + { + "repository": "libgit2", + "path_base64": "dGVzdHMvbGliZ2l0Mi9vYmplY3QvcmF3", + "pages": 1, + "entries": 10, + "all_equal": true + }, + { + "repository": "libgit2", + "path_base64": "dGVzdHMvcmVzb3VyY2Vz", + "pages": 3, + "entries": 89, + "all_equal": true + } + ], + "api_scope": "Every ref; complete pages for selected tree paths; three exact blob previews and four history pages (128 commit records) per repository. Every stored Git object body compared separately through stock Git.", + "dedicated_attribution_endpoint": "not implemented (422), outside the four requested API kinds", + "wider_workspace_provider_process_qualification": "tracked separately in build root", + "equivalent_commit": "12dc9a19881ca4c87353a7643a374d20bc800d58", + "source_equivalence": "All 727 tracked files match the committed source; original build snapshot identity is retained.", + "post_gc_verification": { + "state": "passed", + "captured_utc": "2026-10-15T05:59:50.756120+00:00", + "objects": 194252, + "refs": 501, + "all_object_bytes_heads_fsck_and_apis_equal": true, + "full_selected_tree_pages": true, + "fresh_local_workspace": true, + "deleted_cellule_objects": 1664, + "proof_directory": "after-gc-verification" + }, + "fresh_workspace_api_reads": { + "state": "passed", + "verified": 12, + "evidence": "results/cold-reads.json" + } + }, + "before_gc": { + "captured_utc": "2026-10-15T05:49:28.440391+00:00", + "storage_url": "s3://canopy-benchmark/qualification/native-main/88272c1d-7d05-41a4-a287-e31c17cf50f9", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "physical_method": "Apparent bytes of every regular backing file under the exact RustFS fixture prefix. Longest listed-key prefix maps files to S3 objects; unmatched files are counted separately. Includes provider framing and metadata; excludes filesystem allocation overhead.", + "physical": { + "directory": { + "bytes": 7834274, + "files": 574, + "object_keys": 574, + "artifacts": 574 + }, + "input_body": { + "bytes": 160676473, + "files": 177, + "object_keys": 114, + "artifacts": 48 + }, + "input_root": { + "bytes": 94093, + "files": 189, + "object_keys": 126, + "artifacts": 63 + }, + "native_catalog_node": { + "bytes": 167004, + "files": 171, + "object_keys": 114, + "artifacts": 57 + }, + "native_directory": { + "bytes": 33846366, + "files": 13, + "object_keys": 8, + "artifacts": 3 + }, + "native_index": { + "bytes": 5445596, + "files": 9, + "object_keys": 6, + "artifacts": 3 + }, + "native_metadata": { + "bytes": 230728186, + "files": 107, + "object_keys": 66, + "artifacts": 25 + }, + "native_pack": { + "bytes": 160389768, + "files": 45, + "object_keys": 24, + "artifacts": 3 + }, + "other": { + "bytes": 436977, + "files": 591, + "object_keys": 591, + "artifacts": 591 + }, + "provider_unlisted": { + "bytes": 92898, + "files": 8, + "object_keys": 0, + "artifacts": 0 + }, + "root": { + "bytes": 4233657, + "files": 574, + "object_keys": 574, + "artifacts": 574 + }, + "cellule_pack": { + "bytes": 7908265, + "files": 574, + "object_keys": 574, + "artifacts": 574 + } + }, + "physical_bytes": 611853557, + "logical": { + "directory": { + "bytes": 7540960, + "files": 574, + "object_keys": 574, + "artifacts": 574 + }, + "input_body": { + "bytes": 160614876, + "files": 114, + "object_keys": 114, + "artifacts": 48 + }, + "input_root": { + "bytes": 30859, + "files": 126, + "object_keys": 126, + "artifacts": 63 + }, + "native_catalog_node": { + "bytes": 109777, + "files": 114, + "object_keys": 114, + "artifacts": 57 + }, + "native_directory": { + "bytes": 33841360, + "files": 8, + "object_keys": 8, + "artifacts": 3 + }, + "native_index": { + "bytes": 5442416, + "files": 6, + "object_keys": 6, + "artifacts": 3 + }, + "native_metadata": { + "bytes": 230688432, + "files": 66, + "object_keys": 66, + "artifacts": 25 + }, + "native_pack": { + "bytes": 160373199, + "files": 24, + "object_keys": 24, + "artifacts": 3 + }, + "other": { + "bytes": 136256, + "files": 591, + "object_keys": 591, + "artifacts": 591 + }, + "root": { + "bytes": 3939195, + "files": 574, + "object_keys": 574, + "artifacts": 574 + }, + "cellule_pack": { + "bytes": 7613803, + "files": 574, + "object_keys": 574, + "artifacts": 574 + } + }, + "logical_bytes": 610331133, + "logical_object_keys": 2771, + "unlisted_physical_files": [ + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/root-lineage/v1/e982bc491da741cda687cb9bbd3a2698/21d07a037cad956e727ffefbf523a5f8e7cca33beab11a9f13420533c948a093.bin/xl.meta", + "bytes": 727 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/root-lineage/v1/e982bc491da741cda687cb9bbd3a2698/2e0cfced117cd2f51fecc967dfc7a46dfc5da73b7c8762cded14c2eadfc94ad1.bin/xl.meta", + "bytes": 727 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/079ee476323918658a7c32475cb9226f1cedec4d09c90f29c37f2f01c424b121.pack/xl.meta", + "bytes": 13516 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/2e0cfced117cd2f51fecc967dfc7a46dfc5da73b7c8762cded14c2eadfc94ad1.root/xl.meta", + "bytes": 11838 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/c94376062284b1605fb4152f080c568493654da5479bd63ce48bc8a1df4bdbf1.dir/xl.meta", + "bytes": 18583 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/de780288e09ee843e9119ffcbb06c956a0d56c34dda7f0960d52c0c06aea0b39.pack/xl.meta", + "bytes": 17758 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/21d07a037cad956e727ffefbf523a5f8e7cca33beab11a9f13420533c948a093.root/xl.meta", + "bytes": 11342 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/56b34a68a53d518d4221f7c896b4e6f2f4a3348d77859057d899688fe6cdcab2.dir/xl.meta", + "bytes": 18407 + } + ], + "classification_correction": { + "source": "storage-before-gc.json", + "reason": "Every extension-only captures entry is a cells/v1/.../objects/*.pack Cellule packed data object. Corrected family name only; original reports, exact keys, counts and byte totals preserved.", + "verified_logical_keys": 574 + } + }, + "after_gc": { + "captured_utc": "2026-10-15T05:49:45.322035+00:00", + "storage_url": "s3://canopy-benchmark/qualification/native-main/88272c1d-7d05-41a4-a287-e31c17cf50f9", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "physical_method": "Apparent bytes of every regular backing file under the exact RustFS fixture prefix. Longest listed-key prefix maps files to S3 objects; unmatched files are counted separately. Includes provider framing and metadata; excludes filesystem allocation overhead.", + "physical": { + "directory": { + "bytes": 64051, + "files": 5, + "object_keys": 5, + "artifacts": 5 + }, + "input_body": { + "bytes": 160676473, + "files": 177, + "object_keys": 114, + "artifacts": 48 + }, + "input_root": { + "bytes": 94093, + "files": 189, + "object_keys": 126, + "artifacts": 63 + }, + "native_catalog_node": { + "bytes": 167004, + "files": 171, + "object_keys": 114, + "artifacts": 57 + }, + "native_directory": { + "bytes": 33846366, + "files": 13, + "object_keys": 8, + "artifacts": 3 + }, + "native_index": { + "bytes": 5445596, + "files": 9, + "object_keys": 6, + "artifacts": 3 + }, + "native_metadata": { + "bytes": 230728186, + "files": 107, + "object_keys": 66, + "artifacts": 25 + }, + "native_pack": { + "bytes": 160389768, + "files": 45, + "object_keys": 24, + "artifacts": 3 + }, + "other": { + "bytes": 452348, + "files": 611, + "object_keys": 611, + "artifacts": 611 + }, + "root": { + "bytes": 42293, + "files": 5, + "object_keys": 5, + "artifacts": 5 + }, + "cellule_pack": { + "bytes": 2118052, + "files": 75, + "object_keys": 75, + "artifacts": 75 + } + }, + "physical_bytes": 594024230, + "logical": { + "directory": { + "bytes": 61496, + "files": 5, + "object_keys": 5, + "artifacts": 5 + }, + "input_body": { + "bytes": 160614876, + "files": 114, + "object_keys": 114, + "artifacts": 48 + }, + "input_root": { + "bytes": 30859, + "files": 126, + "object_keys": 126, + "artifacts": 63 + }, + "native_catalog_node": { + "bytes": 109777, + "files": 114, + "object_keys": 114, + "artifacts": 57 + }, + "native_directory": { + "bytes": 33841360, + "files": 8, + "object_keys": 8, + "artifacts": 3 + }, + "native_index": { + "bytes": 5442416, + "files": 6, + "object_keys": 6, + "artifacts": 3 + }, + "native_metadata": { + "bytes": 230688432, + "files": 66, + "object_keys": 66, + "artifacts": 25 + }, + "native_pack": { + "bytes": 160373199, + "files": 24, + "object_keys": 24, + "artifacts": 3 + }, + "other": { + "bytes": 141649, + "files": 611, + "object_keys": 611, + "artifacts": 611 + }, + "root": { + "bytes": 39728, + "files": 5, + "object_keys": 5, + "artifacts": 5 + }, + "cellule_pack": { + "bytes": 2079577, + "files": 75, + "object_keys": 75, + "artifacts": 75 + } + }, + "logical_bytes": 593423369, + "logical_object_keys": 1154, + "unlisted_physical_files": [], + "classification_correction": { + "source": "storage-after-gc.json", + "reason": "Every extension-only captures entry is a cells/v1/.../objects/*.pack Cellule packed data object. Corrected family name only; original reports, exact keys, counts and byte totals preserved.", + "verified_logical_keys": 75 + } + }, + "post_browse": { + "captured_utc": "2026-10-15T06:23:52.158347+00:00", + "storage_url": "s3://canopy-benchmark/qualification/native-main/88272c1d-7d05-41a4-a287-e31c17cf50f9", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "physical_method": "Apparent bytes of every regular backing file under the exact RustFS fixture prefix. Longest listed-key prefix maps files to S3 objects; unmatched files are counted separately. Includes provider framing and metadata; excludes filesystem allocation overhead.", + "physical": { + "directory": { + "bytes": 17264316, + "files": 1535, + "object_keys": 1535, + "artifacts": 1535 + }, + "input_body": { + "bytes": 160676473, + "files": 177, + "object_keys": 114, + "artifacts": 48 + }, + "input_root": { + "bytes": 94093, + "files": 189, + "object_keys": 126, + "artifacts": 63 + }, + "native_catalog_node": { + "bytes": 167004, + "files": 171, + "object_keys": 114, + "artifacts": 57 + }, + "native_directory": { + "bytes": 33846366, + "files": 13, + "object_keys": 8, + "artifacts": 3 + }, + "native_index": { + "bytes": 5445596, + "files": 9, + "object_keys": 6, + "artifacts": 3 + }, + "native_metadata": { + "bytes": 230728186, + "files": 107, + "object_keys": 66, + "artifacts": 25 + }, + "native_pack": { + "bytes": 160389768, + "files": 45, + "object_keys": 24, + "artifacts": 3 + }, + "other": { + "bytes": 1036636, + "files": 1365, + "object_keys": 1365, + "artifacts": 1365 + }, + "provider_unlisted": { + "bytes": 247164, + "files": 22, + "object_keys": 0, + "artifacts": 0 + }, + "root": { + "bytes": 6417438, + "files": 690, + "object_keys": 690, + "artifacts": 690 + }, + "cellule_pack": { + "bytes": 15350335, + "files": 770, + "object_keys": 767, + "artifacts": 767 + } + }, + "physical_bytes": 631663375, + "logical": { + "directory": { + "bytes": 16481696, + "files": 1535, + "object_keys": 1535, + "artifacts": 1535 + }, + "input_body": { + "bytes": 160614876, + "files": 114, + "object_keys": 114, + "artifacts": 48 + }, + "input_root": { + "bytes": 30859, + "files": 126, + "object_keys": 126, + "artifacts": 63 + }, + "native_catalog_node": { + "bytes": 109777, + "files": 114, + "object_keys": 114, + "artifacts": 57 + }, + "native_directory": { + "bytes": 33841360, + "files": 8, + "object_keys": 8, + "artifacts": 3 + }, + "native_index": { + "bytes": 5442416, + "files": 6, + "object_keys": 6, + "artifacts": 3 + }, + "native_metadata": { + "bytes": 230688432, + "files": 66, + "object_keys": 66, + "artifacts": 25 + }, + "native_pack": { + "bytes": 160373199, + "files": 24, + "object_keys": 24, + "artifacts": 3 + }, + "other": { + "bytes": 342223, + "files": 1365, + "object_keys": 1365, + "artifacts": 1365 + }, + "root": { + "bytes": 6063468, + "files": 690, + "object_keys": 690, + "artifacts": 690 + }, + "cellule_pack": { + "bytes": 14957033, + "files": 767, + "object_keys": 767, + "artifacts": 767 + } + }, + "logical_bytes": 628945339, + "logical_object_keys": 4815, + "unlisted_physical_files": [ + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/root-lineage/v1/256aaa7da0ca448db66d775282cc434a/4833f053d41d17f1c5a14ecffef451dfc8f1e30a1e0ef69dda6ac0960b984171.bin/xl.meta", + "bytes": 727 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/root-lineage/v1/256aaa7da0ca448db66d775282cc434a/6aa76b970848254ef30a6b15dcd548c094edc24b38bee64c4b2481b56ba20a4f.bin/xl.meta", + "bytes": 727 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/fe3caf82c835e7844a053aa2c929f0c203259073c3ccaa5e3da15f87dbfddff0.dir/xl.meta", + "bytes": 8463 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/7bc461f204d00f376cd0bb8fb08d8931dd0b3cf0d266497ca3dcda11e0282e55.dir/xl.meta", + "bytes": 651 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/acc7bd0e463400ac8addd34c941f1d8fe1b660349aa9e078e6c3e740bdeea2d3.pack/xl.meta", + "bytes": 16511 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/6aa76b970848254ef30a6b15dcd548c094edc24b38bee64c4b2481b56ba20a4f.root/xl.meta", + "bytes": 13348 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/5963374840b39631342788675bd3d59c9792042ef472226a17cf734dff89e9b8.dir/xl.meta", + "bytes": 23071 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/c16091c1c1cb9807f789dbcc042c91effd9282f8f52b40e7c978b9eecd1e8311.dir/xl.meta", + "bytes": 8551 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/36992fcc90fe62f7e35fcbdb573c25a019b6bb3c5f8c4e196a06eb4e8713fe6c.dir/xl.meta", + "bytes": 23071 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/a07dcd20e6e2684282c51e9acfd6ddbb76274ad258f93b10f20a85db484c49e1.dir/xl.meta", + "bytes": 651 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/b4d48365fc62b95cd04c6d09492b6d548efee80f5b7dde8c8af89292756d5add.pack/xl.meta", + "bytes": 17488 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/4833f053d41d17f1c5a14ecffef451dfc8f1e30a1e0ef69dda6ac0960b984171.root/xl.meta", + "bytes": 12852 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/68b8a5074e4e282249b967b72692f0505b6d6a7863cbe3e3a346df04ffb16f03.dir/xl.meta", + "bytes": 651 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/2814d8e2a8a860c8fe62e4264ba2c1e9b334280c634eaa6d07eb4b5e85288019.dir/xl.meta", + "bytes": 23071 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/f00ac31b926bf87b201998e0db07233837467434524c94d1cd6dd402045fba01.root/xl.meta", + "bytes": 12356 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/79ad7899319af1b309a8b1faf507bafc6d5141f5baf714a64bc8f4fd8ba100dc.dir/xl.meta", + "bytes": 7847 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/585cca6e199e6c5dfe2f148495afbea8499140303e500a8d43c82abeb7a6b5a2.dir/xl.meta", + "bytes": 651 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/792a52fa8873e3896d21881e983e9720b6ac4c6e270944d2fca4ca4c25c918d7.pack/xl.meta", + "bytes": 16595 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/5273a9d9f28483a3af93ded3b823c1a77267ffe6c08fadafe1350d9d7833c574.dir/xl.meta", + "bytes": 7759 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/03a662847b49221ae89494148eccce6f3b7b109262541a0772739f2feecd59bf.dir/xl.meta", + "bytes": 23071 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/0284cf37e7ea55e6d6ad7c76cf236c33262dd9100bd308b0a402116ce434e556.root/xl.meta", + "bytes": 11860 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/d2cc64649b97603cd605d9a836a2a2a3541822aee60b769ce2dc1a3d4660287d.pack/xl.meta", + "bytes": 17192 + } + ], + "classification_correction": { + "source": "storage-post-browse.json", + "reason": "All extension-only captures entries are Cellule objects/*.pack keys. Renamed category without changing totals.", + "verified_logical_keys": 767 + }, + "scope_note": "Online observation after full post-GC verification, 12 cold-workspace reads and 11,474 warm benchmark requests; serving heartbeat/publication may continue during this non-atomic inventory. Includes unlisted backing files. No second GC performed." + }, + "baseline": { + "captured_utc": "2026-10-15T01:55:50.286170+00:00", + "baseline_cellule": "0.2.0", + "reported_rustfs_data_tree_apparent_bytes": 22525385112, + "bucket_regular_file_bytes": 22525370449, + "outside_bucket_difference_bytes": 14663, + "directory_regular_file_bytes": 7899358413, + "directory_immutable_keys": 351653, + "method": "Read-only scan of regular files under the RustFS bucket directory; apparent bytes include provider metadata and data-part framing. Object counts deduplicate backing files by immutable key. No deletion or reachability claim.", + "baseline_json_sha256": "57b401005da4eb5682e2c290d1b044aa41ff798e7d39d561b85267055b39032b", + "bucket_breakdown_sha256": "0eed4d56a8f79d580e3653f40b5e594cab1bea9984aaf4550c4d1780c87dc03f", + "scope_note": "Original report counts the whole RustFS data tree; dedicated corpus bucket inventory differs by 14663 bytes. Fresh native measurements scan the exact deployment prefix and report unlisted provider files separately. Compare apparent regular-file bytes, not allocated VM filesystem usage." + }, + "garbage_collection": { + "operation": "99b49e85-976e-4406-9930-de70098ad3a7", + "passes": [ + { + "seconds": 9.297116999980062, + "report": { + "listed_objects": 2350, + "immutable_candidates": 1751, + "reachable_objects": 87, + "retained_objects": 87, + "grace_objects": 0, + "eligible_objects": 1664, + "deleted_objects": 1664, + "current_controls": 4, + "retained_pins": 0, + "complete": true + } + } + ], + "state": "complete" + }, + "cold_reads": { + "method": "One first full authenticated HTTP read after graceful shutdown and fresh local workspace per repository/API. Excludes healthz startup; RustFS and host filesystem caches may be warm. Not a percentile or fully cold provider benchmark.", + "results": [ + { + "repository": "bat", + "operation": "blob", + "latency_ms": 1717.557291034609, + "response_bytes": 22279, + "startup_seconds": 1.0512839580187574, + "status": 200, + "response_verified": true + }, + { + "repository": "bat", + "operation": "commits", + "latency_ms": 3115.7047499436885, + "response_bytes": 20535, + "startup_seconds": 0.5792585840681568, + "status": 200, + "response_verified": true + }, + { + "repository": "bat", + "operation": "refs", + "latency_ms": 297.9790000244975, + "response_bytes": 6832, + "startup_seconds": 0.5556554580107331, + "status": 200, + "response_verified": true + }, + { + "repository": "bat", + "operation": "tree", + "latency_ms": 2241.450082976371, + "response_bytes": 4541, + "startup_seconds": 1.0334134169388562, + "status": 200, + "response_verified": true + }, + { + "repository": "libgit2", + "operation": "blob", + "latency_ms": 3872.913834056817, + "response_bytes": 22154, + "startup_seconds": 1.0310157909989357, + "status": 200, + "response_verified": true + }, + { + "repository": "libgit2", + "operation": "commits", + "latency_ms": 5007.826166925952, + "response_bytes": 18498, + "startup_seconds": 0.6664082080824301, + "status": 200, + "response_verified": true + }, + { + "repository": "libgit2", + "operation": "refs", + "latency_ms": 208.63866701256484, + "response_bytes": 25061, + "startup_seconds": 0.6468817499699071, + "status": 200, + "response_verified": true + }, + { + "repository": "libgit2", + "operation": "tree", + "latency_ms": 2760.773833026178, + "response_bytes": 5398, + "startup_seconds": 1.0328317920211703, + "status": 200, + "response_verified": true + }, + { + "repository": "starship", + "operation": "blob", + "latency_ms": 1210.0983749842271, + "response_bytes": 22185, + "startup_seconds": 1.049035458941944, + "status": 200, + "response_verified": true + }, + { + "repository": "starship", + "operation": "commits", + "latency_ms": 2843.330917065032, + "response_bytes": 19598, + "startup_seconds": 1.0459265420213342, + "status": 200, + "response_verified": true + }, + { + "repository": "starship", + "operation": "refs", + "latency_ms": 490.4339590575546, + "response_bytes": 15078, + "startup_seconds": 1.0539884169120342, + "status": 200, + "response_verified": true + }, + { + "repository": "starship", + "operation": "tree", + "latency_ms": 1907.0911249145865, + "response_bytes": 5088, + "startup_seconds": 1.0474654169520363, + "status": 200, + "response_verified": true + } + ] + }, + "warm_measurement_method": { + "method": "closed-loop, persistent HTTP/1.1 connections, full response read, no retries", + "mixed_workload": "equal rotation over one query for each repository/API pair; medium sampled blob", + "duration_per_group_seconds": 8.0, + "concurrency": [ + 1, + 4 + ], + "repeats": 2 + }, + "warm_aggregation": "Pool raw latency samples across both trials; nearest-rank percentiles. Successful requests and response bytes divided by the sum of actual trial durations; no summed rates or averaged percentiles.", + "warm_results": [ + { + "repository": "all", + "operation": "mixed", + "concurrency": 1, + "attempts": 32, + "successes": 32, + "errors": 0, + "elapsed_seconds": 19.138353791902773, + "requests_per_second": 1.6720351367701671, + "response_mib_per_second": 0.02444835302691302, + "latency_ms": { + "p50": 150.08925006259233, + "p95": 2850.6643750006333, + "p99": 2937.793624936603 + } + }, + { + "repository": "all", + "operation": "mixed", + "concurrency": 4, + "attempts": 97, + "successes": 97, + "errors": 0, + "elapsed_seconds": 21.218805790995248, + "requests_per_second": 4.571416551687583, + "response_mib_per_second": 0.06963764920448519, + "latency_ms": { + "p50": 208.67841690778732, + "p95": 3701.124874991365, + "p99": 4382.5643750606105 + } + }, + { + "repository": "bat", + "operation": "blob", + "concurrency": 1, + "attempts": 51, + "successes": 51, + "errors": 0, + "elapsed_seconds": 16.540011209086515, + "requests_per_second": 3.0834320095250205, + "response_mib_per_second": 0.21264022614481842, + "latency_ms": { + "p50": 292.68170800060034, + "p95": 647.6248749531806, + "p99": 1145.3940420178697 + } + }, + { + "repository": "bat", + "operation": "blob", + "concurrency": 4, + "attempts": 148, + "successes": 148, + "errors": 0, + "elapsed_seconds": 16.584007749101147, + "requests_per_second": 8.92426018119906, + "response_mib_per_second": 0.6450469724266509, + "latency_ms": { + "p50": 443.28795897308737, + "p95": 834.6678329398856, + "p99": 1487.2602919349447 + } + }, + { + "repository": "bat", + "operation": "commits", + "concurrency": 1, + "attempts": 6, + "successes": 6, + "errors": 0, + "elapsed_seconds": 19.923467208980583, + "requests_per_second": 0.30115240169118135, + "response_mib_per_second": 0.005701520422337849, + "latency_ms": { + "p50": 3142.1082080341876, + "p95": 4125.456375069916, + "p99": 4125.456375069916 + } + }, + { + "repository": "bat", + "operation": "commits", + "concurrency": 4, + "attempts": 28, + "successes": 28, + "errors": 0, + "elapsed_seconds": 18.9588872079039, + "requests_per_second": 1.476879929341364, + "response_mib_per_second": 0.02953123233652151, + "latency_ms": { + "p50": 3110.6809999328107, + "p95": 4257.9138750443235, + "p99": 4293.876167037524 + } + }, + { + "repository": "bat", + "operation": "refs", + "concurrency": 1, + "attempts": 713, + "successes": 713, + "errors": 0, + "elapsed_seconds": 16.042291291989386, + "requests_per_second": 44.445022660574175, + "response_mib_per_second": 0.28958167535499835, + "latency_ms": { + "p50": 24.8757500667125, + "p95": 44.15645799599588, + "p99": 57.99899995326996 + } + }, + { + "repository": "bat", + "operation": "refs", + "concurrency": 4, + "attempts": 2400, + "successes": 2400, + "errors": 0, + "elapsed_seconds": 16.042705750092864, + "requests_per_second": 149.60069936993688, + "response_mib_per_second": 0.9747237950281226, + "latency_ms": { + "p50": 16.39533299021423, + "p95": 61.51379202492535, + "p99": 78.73658300377429 + } + }, + { + "repository": "bat", + "operation": "tree", + "concurrency": 1, + "attempts": 72, + "successes": 72, + "errors": 0, + "elapsed_seconds": 16.54015316697769, + "requests_per_second": 4.353043123188699, + "response_mib_per_second": 0.025355624328285, + "latency_ms": { + "p50": 158.60500000417233, + "p95": 610.6297079240903, + "p99": 784.1266669565812 + } + }, + { + "repository": "bat", + "operation": "tree", + "concurrency": 4, + "attempts": 250, + "successes": 250, + "errors": 0, + "elapsed_seconds": 16.611242791172117, + "requests_per_second": 15.050047918922722, + "response_mib_per_second": 0.08757715109656573, + "latency_ms": { + "p50": 183.34029102697968, + "p95": 649.8822499997914, + "p99": 1036.7236660094932 + } + }, + { + "repository": "libgit2", + "operation": "blob", + "concurrency": 1, + "attempts": 41, + "successes": 41, + "errors": 0, + "elapsed_seconds": 16.558681790949777, + "requests_per_second": 2.4760425085533524, + "response_mib_per_second": 0.152900446179729, + "latency_ms": { + "p50": 359.654291998595, + "p95": 785.2027079788968, + "p99": 1080.859125009738 + } + }, + { + "repository": "libgit2", + "operation": "blob", + "concurrency": 4, + "attempts": 146, + "successes": 146, + "errors": 0, + "elapsed_seconds": 16.568943708087318, + "requests_per_second": 8.811666125025052, + "response_mib_per_second": 0.5636727630207855, + "latency_ms": { + "p50": 504.0497080190107, + "p95": 792.1191659988835, + "p99": 885.5659590335563 + } + }, + { + "repository": "libgit2", + "operation": "commits", + "concurrency": 1, + "attempts": 7, + "successes": 7, + "errors": 0, + "elapsed_seconds": 18.889531416003592, + "requests_per_second": 0.37057562974110936, + "response_mib_per_second": 0.006530836886871976, + "latency_ms": { + "p50": 2596.7865829588845, + "p95": 4647.928624995984, + "p99": 4647.928624995984 + } + }, + { + "repository": "libgit2", + "operation": "commits", + "concurrency": 4, + "attempts": 28, + "successes": 28, + "errors": 0, + "elapsed_seconds": 16.994784374022856, + "requests_per_second": 1.6475642987738646, + "response_mib_per_second": 0.028967376148409815, + "latency_ms": { + "p50": 2054.943374940194, + "p95": 3526.8444169778377, + "p99": 3527.486833045259 + } + }, + { + "repository": "libgit2", + "operation": "refs", + "concurrency": 1, + "attempts": 1039, + "successes": 1039, + "errors": 0, + "elapsed_seconds": 16.035073166014627, + "requests_per_second": 64.79546362171256, + "response_mib_per_second": 1.5486136568295845, + "latency_ms": { + "p50": 10.329957935027778, + "p95": 35.10695800650865, + "p99": 56.80791602935642 + } + }, + { + "repository": "libgit2", + "operation": "refs", + "concurrency": 4, + "attempts": 1904, + "successes": 1904, + "errors": 0, + "elapsed_seconds": 16.02469187497627, + "requests_per_second": 118.81663715314461, + "response_mib_per_second": 2.839721435255963, + "latency_ms": { + "p50": 18.749042064882815, + "p95": 89.0887079294771, + "p99": 129.4307919451967 + } + }, + { + "repository": "libgit2", + "operation": "tree", + "concurrency": 1, + "attempts": 59, + "successes": 59, + "errors": 0, + "elapsed_seconds": 16.37868070800323, + "requests_per_second": 3.602243736955591, + "response_mib_per_second": 0.018992690322077416, + "latency_ms": { + "p50": 203.3157499972731, + "p95": 666.7511670384556, + "p99": 841.6428329655901 + } + }, + { + "repository": "libgit2", + "operation": "tree", + "concurrency": 4, + "attempts": 222, + "successes": 222, + "errors": 0, + "elapsed_seconds": 16.703519250033423, + "requests_per_second": 13.290612395921045, + "response_mib_per_second": 0.0702799524570234, + "latency_ms": { + "p50": 208.73466599732637, + "p95": 688.8832499971613, + "p99": 874.1668340517208 + } + }, + { + "repository": "starship", + "operation": "blob", + "concurrency": 1, + "attempts": 61, + "successes": 61, + "errors": 0, + "elapsed_seconds": 16.687753124977462, + "requests_per_second": 3.6553752649119673, + "response_mib_per_second": 0.22452658926736563, + "latency_ms": { + "p50": 207.47679204214364, + "p95": 564.7345830220729, + "p99": 667.3221660312265 + } + }, + { + "repository": "starship", + "operation": "blob", + "concurrency": 4, + "attempts": 141, + "successes": 141, + "errors": 0, + "elapsed_seconds": 16.785071666003205, + "requests_per_second": 8.400321595622616, + "response_mib_per_second": 0.5134882748119307, + "latency_ms": { + "p50": 434.05975005589426, + "p95": 885.4027499910444, + "p99": 1100.3043330274522 + } + }, + { + "repository": "starship", + "operation": "commits", + "concurrency": 1, + "attempts": 9, + "successes": 9, + "errors": 0, + "elapsed_seconds": 18.718768041115254, + "requests_per_second": 0.4808008721637957, + "response_mib_per_second": 0.008545219324381596, + "latency_ms": { + "p50": 2330.3796669933945, + "p95": 3049.1317079868168, + "p99": 3049.1317079868168 + } + }, + { + "repository": "starship", + "operation": "commits", + "concurrency": 4, + "attempts": 32, + "successes": 32, + "errors": 0, + "elapsed_seconds": 18.660066998912953, + "requests_per_second": 1.7148920205840723, + "response_mib_per_second": 0.030281963971266444, + "latency_ms": { + "p50": 2094.548290944658, + "p95": 3081.987417070195, + "p99": 3101.8335840199143 + } + }, + { + "repository": "starship", + "operation": "refs", + "concurrency": 1, + "attempts": 834, + "successes": 834, + "errors": 0, + "elapsed_seconds": 16.03657645895146, + "requests_per_second": 52.00611253497747, + "response_mib_per_second": 0.747821965029135, + "latency_ms": { + "p50": 11.184707982465625, + "p95": 40.05862504709512, + "p99": 53.62862499896437 + } + }, + { + "repository": "starship", + "operation": "refs", + "concurrency": 4, + "attempts": 2812, + "successes": 2812, + "errors": 0, + "elapsed_seconds": 16.121262750122696, + "requests_per_second": 174.42802363472416, + "response_mib_per_second": 2.508188000072833, + "latency_ms": { + "p50": 12.591624981723726, + "p95": 54.89604198373854, + "p99": 85.55033407174051 + } + }, + { + "repository": "starship", + "operation": "tree", + "concurrency": 1, + "attempts": 63, + "successes": 63, + "errors": 0, + "elapsed_seconds": 16.077648540958762, + "requests_per_second": 3.9184834672498137, + "response_mib_per_second": 0.012581444608665482, + "latency_ms": { + "p50": 231.43070796504617, + "p95": 533.7357500102371, + "p99": 596.9165000133216 + } + }, + { + "repository": "starship", + "operation": "tree", + "concurrency": 4, + "attempts": 279, + "successes": 279, + "errors": 0, + "elapsed_seconds": 16.647743915906176, + "requests_per_second": 16.759027614151847, + "response_mib_per_second": 0.05278188486752141, + "latency_ms": { + "p50": 183.54470806661993, + "p95": 648.0151249561459, + "p99": 757.4171250453219 + } + } + ], + "workspace_gates": { + "state": "passed", + "source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "cellule": "0.2.0", + "schema": 4, + "storage_format": "canopy-pack-v2", + "started_utc": "2026-10-15T17:28:34.280422+00:00", + "gates": [ + { + "name": "dependency-policy", + "exit_code": 0, + "seconds": 0.0703032499877736, + "log": "dependency-policy.log" + }, + { + "name": "format", + "exit_code": 0, + "seconds": 1.4924878330202773, + "log": "format.log" + }, + { + "name": "all-target-clippy", + "exit_code": 0, + "seconds": 49.77081887505483, + "log": "all-target-clippy.log" + }, + { + "name": "ordinary-tests", + "exit_code": 0, + "seconds": 3332.7650082920445, + "log": "ordinary-tests.log" + }, + { + "name": "python", + "exit_code": 0, + "seconds": 44.6627316250233, + "log": "python.log" + }, + { + "name": "release-build", + "exit_code": 0, + "seconds": 530.0212494999869, + "log": "release-build.log" + } + ], + "stage": "release-build", + "binary_sha256": "0be8986fe9d9a9c12780dead22f9c8dc8c63a3adf88cdc7f682778e5bfc9f254", + "finished_utc": "2026-10-15T18:34:33.859565+00:00" + }, + "final_process_backup": { + "saved": { + "pin": "6005db5e-94ca-476e-a5bd-70cbf8f20e84", + "root": "qualification/extended-process-native-backup-harness/process-smoke/f81dade2-bb2a-4612-ace2-ce3ea5efd04b/backup-copy", + "cells": 2, + "external_objects": 26 + }, + "restored": { + "pin": "6005db5e-94ca-476e-a5bd-70cbf8f20e84", + "root": "qualification/extended-process-native-backup-harness/process-smoke/f81dade2-bb2a-4612-ace2-ce3ea5efd04b/backup-restored", + "cells": 2, + "external_objects": 26 + }, + "external_manifest_count": 26, + "external_inventory_sha256": "93df98819341b2ebec553973f270fe72f84674e270be51d1095edc9f7e1f44ae", + "source_deleted": true, + "corrupt_copy_rejected": true, + "cold_clone_verified": true, + "issue_verified": true + }, + "final_process_retention": { + "deleted_objects": 256, + "last_report": { + "listed_objects": 418, + "immutable_candidates": 305, + "reachable_objects": 50, + "retained_objects": 50, + "grace_objects": 0, + "eligible_objects": 255, + "deleted_objects": 255, + "current_controls": 2, + "retained_pins": 1, + "complete": true + }, + "backup_verified": true, + "cold_clone_verified": true + }, + "prior_provider_process_attempt": { + "state": "failed", + "cellule": "0.2.0", + "schema": 4, + "storage_format": "canopy-pack-v2", + "profile": "release", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "rust_qualified_source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "alternate_target_compile_check_sha256": "0be8986fe9d9a9c12780dead22f9c8dc8c63a3adf88cdc7f682778e5bfc9f254", + "runtime_image_equivalence_gate": "runtime-image-equivalence.json", + "process_profile": [ + "--large-clone", + "--many-objects", + "256", + "--structural-objects", + "--structural-object-bytes", + "68157440", + "--retain-work-dir" + ], + "started_utc": "2026-10-15T18:47:58.035643+00:00", + "error": "Command '['/opt/homebrew/opt/python@3.14/bin/python3.14', '-B', 'native-medium-cache-final-20261015/source/scripts/smoke_s3_process.py', '--binary', '/private/var/folders/rn/k2b7mvbs2dgb__6ys6_h94pm0000gn/T/canopy-size-95af668fccdf98z1k51f/canopy', '--storage-url', 's3://canopy-size/process-qualification', '--work-parent', 'native-medium-cache-final-20261015/process-work', '--large-clone', '--many-objects', '256', '--structural-objects', '--structural-object-bytes', '68157440', '--retain-work-dir']' died with .", + "exit_code": 1, + "seconds": 1779.200018582982, + "finished_utc": "2026-10-15T19:17:37.221470+00:00" + }, + "isolated_provider_process": { + "state": "passed", + "captured_utc": "2026-10-15T19:21:36.522060+00:00", + "source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "provider_source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "rust_qualified_source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "provider_cases_expected": [ + "size::native_pack_exceeds_512_mib_and_git_lfs_exceed_5_gib_after_restore", + "sha256::sha256_real_provider_round_trip", + "sha256::sha256_real_provider_native_merge_candidates", + "push_options::signed_push_real_provider_round_trip", + "ssh::signed_sha256_ssh_real_provider_round_trip", + "ssh::stock_ssh_real_provider_round_trip", + "bulk_refs::bulk_mirror_real_provider_round_trip", + "partial_clone::filtered_clones_real_provider_round_trip", + "ssh::lfs::stock_lfs_ssh_real_provider_round_trip" + ], + "provider_cases_passed": [ + "size::native_pack_exceeds_512_mib_and_git_lfs_exceed_5_gib_after_restore", + "sha256::sha256_real_provider_round_trip", + "sha256::sha256_real_provider_native_merge_candidates", + "push_options::signed_push_real_provider_round_trip", + "ssh::signed_sha256_ssh_real_provider_round_trip", + "ssh::stock_ssh_real_provider_round_trip", + "bulk_refs::bulk_mirror_real_provider_round_trip", + "partial_clone::filtered_clones_real_provider_round_trip", + "ssh::lfs::stock_lfs_ssh_real_provider_round_trip" + ], + "extended_process": { + "state": "passed", + "started_utc": "2026-10-15T19:13:22.285310+00:00", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "backup_script_sha256": "ffb5a2ca1a7b5ab20bc735a980628f28c0217ee56aac8ca948fe6adeb783af0a", + "source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "profile": [ + "--large-clone", + "--many-objects", + "256", + "--structural-objects", + "--structural-object-bytes", + "68157440" + ], + "seconds": 493.9588900420349, + "finished_utc": "2026-10-15T19:21:36.316138+00:00" + }, + "current_harness_source_equivalence": { + "state": "passed", + "source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "previous_source_commit": "1a0fc93d526d205979e03523435ba0c6a9a027c1", + "changed_paths": [ + "scripts/smoke_s3_backup.py" + ], + "rust_source_unchanged": true, + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "python_tests": 105, + "python_exit_code": 0, + "python_seconds": 45.085, + "python_log": "python-backup-harness.log", + "backup_script_sha256": "ffb5a2ca1a7b5ab20bc735a980628f28c0217ee56aac8ca948fe6adeb783af0a" + }, + "provider_log_sha256": "6101a15acdf53255d008d1950898fcee5a33d980b76b21bf7a9548be2279e175", + "provider_log": "isolated-provider-process-full-final.log", + "scope_note": "The nine Rust provider cases use identical Rust source. The complete extended process profile uses the corrected peer TLS proxy and native backup inventory at the current Python harness commit. Original attempts and their failures are preserved separately." + }, + "state": "passed", + "ordinary_rust_test_summary": { + "unique_passed": 896, + "failed": 0, + "provider_cases_deferred": 9, + "nested_child_result_duplicates_excluded": 2 + }, + "evidence_sha256": { + "medium-metadata-cache-final-20261015/verify_mirrors.py": "b1038426fd92c44b16d052e78b642b618007c6896563438a80d5d7e8648dfe9a", + "medium-metadata-cache-final-20261015/verify_tree_pagination.py": "75e01b0109515c6ff9f54e8ca9045d27eb7a8e9da360c2161fab9e29a0d94b4c", + "medium-metadata-cache-final-20261015/complete-verification-report.json": "745721a6b19594d78e4d09d5cdb6d067659cdc5a6cdf4b7543dccdf545890622", + "medium-metadata-cache-final-20261015/complete-repository-verification/mirror-ref-proof.json": "3c8b4f603399ca33490ba47c25792780accf6f69699ef79b365f2ba0d7769011", + "medium-metadata-cache-final-20261015/after-gc-verification/mirror-ref-proof.json": "a22be4ec4486b541336afba70d92823428459c3ebe8e1ed725996947391e3edd", + "medium-metadata-cache-final-20261015/results/samples.jsonl": "246f723c9afc33818deb179d3dc326848f696e4b456d624056e2ade9af58d90e", + "medium-metadata-cache-final-20261015/results/measurements.json": "f8873ef943819f5fa7a61a54e65151ca2efcaaec05efc7b6fa309214d742f685", + "medium-metadata-cache-final-20261015/results/inventory-before-gc.json": "6c813af07877231e2c6163dd357c7d9431dabbfad48982d136f0927b5ea63610", + "medium-metadata-cache-final-20261015/results/inventory-after-gc.json": "8a76e17cc846dab4c7e6b696aefdd09a78a50f322919abd2584f9f0bd18184a8", + "medium-metadata-cache-final-20261015/results/inventory-post-browse.json": "01d1c46c35c9c892e0d0fb9be780560969423bb57c4f3045f5b2edc93cf7faeb", + "native-medium-cache-final-20261015/verification-results.json": "03c2d081914c4201a10edf0775fe9db943b3c01a9f5f5fc4f716f15c3ed2ae13", + "native-medium-cache-final-20261015/source-manifest-rust-qualified-930.json": "048eca61b95b6a5c75c7d6b4e7cd18155f8414509139cc895f93fd55de5e9402", + "native-medium-cache-final-20261015/source-manifest-native-backup-harness.json": "632c2a1f075b7df7a33fc56d274092228759881d7513ff91ec83a74f449c4121", + "native-medium-cache-final-20261015/final-harness-source-equivalence.json": "315144c9b9feaf479abc9967a90431ba192c0add3d292076e12aa588483b774d", + "native-medium-cache-final-20261015/current-provider-process-results.json": "1da5190587d0db9e07645efae915879cae4feed33dd67d8c9248baa6c43b9ad5", + "native-medium-cache-final-20261015/runtime-image-equivalence.json": "6099d2744c54d7853874548d19df149185cc1e9f0915db4249a0f2b9ad16fedb", + "native-medium-cache-final-20261015/python-backup-harness.log": "ba2da238849eaf5a8f9ece2ef8b91639e89430039f9d4676dc39c87686371ac8", + "native-medium-cache-final-20261015/isolated-provider-process-full-final.log": "6101a15acdf53255d008d1950898fcee5a33d980b76b21bf7a9548be2279e175", + "native-medium-cache-final-20261015/extended-process-native-backup-harness.log": "f2ee4b194f362e7001663827f57928e0197c3dce672191bb9d83187394bd40db" + }, + "superseded_process_interruption": { + "state": "interrupted_superseded_process_fixture", + "signal": "SIGINT", + "pid": 48734, + "provider_cases_passed": [ + "bulk_refs::bulk_mirror_real_provider_round_trip", + "partial_clone::filtered_clones_real_provider_round_trip", + "push_options::signed_push_real_provider_round_trip", + "sha256::sha256_real_provider_native_merge_candidates", + "sha256::sha256_real_provider_round_trip", + "size::native_pack_exceeds_512_mib_and_git_lfs_exceed_5_gib_after_restore", + "ssh::lfs::stock_lfs_ssh_real_provider_round_trip", + "ssh::signed_sha256_ssh_real_provider_round_trip", + "ssh::stock_ssh_real_provider_round_trip" + ], + "replacement_source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "replacement_profile": "extended-process-native-backup-harness-20261015/results.json", + "reason": "Nine required Rust provider cases passed. The old POST-only TLS proxy and fixed external-object count are superseded by the independently running complete current harness. Interrupt only the old Python process child; its finally blocks reclaim its servers and isolated RustFS volume.", + "utc": "2026-10-15T19:17:35.274969+00:00" + } +} diff --git a/docs/benchmarks/2026-10-15-native-storage.md b/docs/benchmarks/2026-10-15-native-storage.md new file mode 100644 index 00000000..0c614a68 --- /dev/null +++ b/docs/benchmarks/2026-10-15-native-storage.md @@ -0,0 +1,136 @@ +# Native storage and full repository qualification — 2026-10-15 + +The three-repository integrity checks passed before and after Cellule retention GC on RustFS. This run uses committed source `12dc9a19881ca4c87353a7643a374d20bc800d58`, published Cellule **0.2.0**, Repository schema **4**, Directory schema **1** and **`canopy-pack-v2`**. The release executable SHA-256 is `ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8`. + +All 727 tracked source files in the frozen build snapshot match that commit. The earlier snapshot identifier is retained in the machine-readable evidence. All five direct Cellule pins are exact `=0.2.0`; all six lockfile entries use the published registry release. The official crates.io release check was recorded on 2026-10-15. Breaking format changes require a fresh storage prefix. Only the compiled release and its current formats are supported. + +Workspace qualification source `930e6c7a4018b02081ac1421d1f1d5ce9e63691e` differs from the corpus-tested commit only in two `cfg(test)` files. Three large retirement scenarios are constructed on the heap before polling so they fit the default 2 MiB stack. Original cases fail at that limit; all nine HEAD/merge-retirement regressions now pass, covering both Git formats and retaining every receipt, pin, rejection and restore assertion. Production source is unchanged; the corpus and process fixtures execute the selected immutable artifact described below. Interrupted/failed full runs and the first ineffective candidate remain separate evidence. + +The cache process harness is qualified separately at `624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05`. Its only change from the Rust-qualified source is `scripts/smoke_s3_process.py`: a real blobless clone/lazy read creates hydration scratch, and the fixture checks the current nested `repo.git` location before owner death. All 105 Python tests and a real RustFS crash/corrupt-SQLite/recovery probe pass with those exact script bytes. + +The peer proxy harness commit is `1a0fc93d526d205979e03523435ba0c6a9a027c1`. Its only change from that cache harness is `scripts/smoke_s3_peers.py`: the fixture uses the existing bounded stream proxy for TLS termination. The old POST-only proxy returned HTTP 501 for cross-owner Git discovery GETs. All 105 Python tests pass, and the complete two-node scenario passes with eight Git/LFS repositories, opposite-owner routing, public/private reads, owner death and interrupted-maintenance recovery. Rust source and the selected executable are unchanged. + +The final Python harness commit is `81925e121c2fad14c25ab993001835b1bc5dc3d3`. Its only additional change is `scripts/smoke_s3_backup.py`: the backup receipt must match the actual external-manifest inventory, require the current native pack/catalog/input families and both LFS objects, and retain that exact inventory on restore. The earlier assertion of three external objects predates native storage; the current fixture produces 26 manifests. All corruption, conditional-copy, source-deletion and exact restored Git/LFS/issue checks remain. All 105 Python tests pass with the final script bytes. + +The selected immutable deployment artifact remains `ffa22ca2…919f8`, so corpus, storage, browse and provider measurements refer to one executable. A current-source rebuild in the original target directory matches every file byte except the 16-byte Mach-O linker UUID and its independently verified 32-byte ad-hoc signature page hash (47 bytes differ). Code, data, loader commands and all remaining bytes match exactly. Neither executable was patched. The rebuild and the alternate-target build, whose generated protobuf panic path differs, are preserved separately; the original raw-hash reproduction checks remain recorded as unsuccessful. + +## Implemented storage changes + +| Priority | Implementation | Evidence | +| --- | --- | --- | +| Cellule retention GC | Verified live roots and backup pins, bounded grace/deletes, confirmed deployment drain and durable worker exclusion. | One complete pass deleted 1,664 eligible Cellule objects; all repository bytes and APIs passed again after restore. | +| Immutable Git packs | Authenticated native pack/index pairs, reusable metadata shards and catalog roots; full fetch reuses verified packs. | Native Git packs occupy 160,389,768 provider bytes for 194,252 objects; no persistent loose-body expansion is required. | +| Less directory rewriting | Native directory runs and catalog subtrees are immutable and reused; Cellule holds bounded publication state. | Cellule directory state is 7,834,274 bytes / 574 keys before GC and 64,051 bytes / 5 keys after GC. Native directory runs are reported separately below. | +| Batched import metadata | Bounded 512-object metadata pages, up to 8,192 objects per shard, and fenced publication of the completed root. | All three full histories imported successfully; 25 metadata artifacts replace per-object authoritative body rows. | + +The qualification also fixed renewal during staged metadata replay and final assembly, separated the aggregate closure-spool limit from per-shard limits, retained physical pack completion across repeated fetches, and raised the metadata cache from 16 to 24 files under the existing shared disk budget. The last change keeps libgit2’s 17 shards plus its directory resident; the deterministic control downloads 36 files with 16 slots versus 18 with the current default. Same-release recovery, authorization, fencing and bounded admission remain enforced. + +## Complete corpus checks + +| Repository | Stored objects compared byte for byte | Refs | Import seconds | Cold mirror seconds | +| --- | ---: | ---: | ---: | ---: | +| bat | 20,952 | 66 | 21.92 | 30.80 | +| starship | 38,333 | 164 | 38.67 | 47.59 | +| libgit2 | 134,967 | 271 | 363.54 | 240.13 | + +The pinned source repositories contain 28,189 commits. Their Git packs total 160,372,479 bytes (152.94 MiB); canonical uncompressed bodies total 3,297,080,585 bytes (3.07 GiB). Source and mirror repositories pass `git fsck --full`. Every stored object header and body is compared through paired stock-Git `cat-file --batch-all-objects --batch --buffer` streams. All 501 ref names/targets, symbolic HEAD names and HEAD object IDs match. This is a full body comparison, in addition to object-ID checks. + +API checks compare every paginated ref, all pages of 12 selected tree paths (17 pages total), nine exact blob previews and 384 commit records against stock Git. They do not exhaustively enumerate every tree path or every blob through the API; all underlying stored object bodies are checked independently. The complete object/ref/HEAD/fsck/API checks pass again after GC and restoration from S3 into a fresh local workspace. Twelve additional first API reads after separate fresh-workspace restarts return verified responses. + +The dedicated attribution endpoint returns 422 and remains unimplemented. It is outside the four API kinds qualified here. + +## Storage accounting + +Counts below are apparent bytes of regular RustFS backing files under the exact fresh deployment prefix, including provider framing and metadata. They exclude filesystem allocation overhead. Object keys, backing files and logical artifacts are distinct counts. The raw inventories and original classification are retained; `.pack` objects under Cellule paths are correctly classified as Cellule packed objects in this report. + +| Category | Before GC bytes | After GC bytes | After GC logical artifacts | +| --- | ---: | ---: | ---: | +| native_pack | 160,389,768 | 160,389,768 | 3 | +| native_index | 5,445,596 | 5,445,596 | 3 | +| native_metadata | 230,728,186 | 230,728,186 | 25 | +| native_directory | 33,846,366 | 33,846,366 | 3 | +| native_catalog_node | 167,004 | 167,004 | 57 | +| input_body | 160,676,473 | 160,676,473 | 48 | +| input_root | 94,093 | 94,093 | 63 | +| cellule_pack | 7,908,265 | 2,118,052 | 75 | +| directory | 7,834,274 | 64,051 | 5 | +| root | 4,233,657 | 42,293 | 5 | +| other | 436,977 | 452,348 | 611 | +| **Total** | **611,853,557** | **594,024,230** | | + +Before GC there are 2,771 listed S3 keys plus 92,898 bytes of unlisted provider files; after GC there are 1,154 keys and no unlisted files. The totals include those files. Logical S3 body bytes after GC total 593,423,369. One 9.30-second collector pass deleted 1,664 eligible objects and retained 87 reachable objects; new control objects explain why the net key-count difference is smaller than the deletion count. The complete maintenance/drain/GC sequence took 14.27 seconds. + +The original corpus bucket used 22,525,370,449 apparent bytes, close to the previously reported whole-RustFS total of 22,525,385,112 bytes (20.98 GiB). The fresh prefix uses 594,024,230 bytes (566.5 MiB), about **97.36% less**. This compares the historical bucket with a fresh current-format deployment and reflects the combined storage changes; it is not an isolated measurement of GC alone. Historical Cellule directory state used 7,899,358,413 bytes / 351,653 keys. Current native directory runs occupy 33,846,366 bytes separately from the 64,051 bytes of Cellule directory state. + +The retained compressed input bodies occupy another 160,676,473 provider bytes and support durable input/replay state. GC intentionally retains product-owned Git packs, catalogs and LFS bodies. Reclaiming unreferenced product artifacts requires its own root/retention policy; this collector does not implement that policy. + +After the complete post-GC verification, 12 fresh-workspace reads and 11,474 warm benchmark requests, an additional online inventory observed **631,663,375 bytes (602.4 MiB)** / 4,815 listed keys. Native packs, indexes, metadata, directory runs, catalogs and retained input families were unchanged; Cellule serving/recovery state accounted for the growth. Cellule directory objects then occupied 17,264,316 bytes / 1,535 keys. This was a non-atomic online observation with the server still running, including 247,164 bytes of unlisted provider files; it is not another post-GC bound. No second GC was performed. + +## Browse throughput and latency + +The client and Canopy run on the shared host over loopback HTTP without TLS; S3 requests reach RustFS in the Docker VM. The warm benchmark uses persistent HTTP/1.1 connections, closed-loop concurrency 1 and 4, two eight-second trials per repository/API group, randomized with seed 42. It reads and validates full response bodies, performs no retries and includes all attempts in latency percentiles. Rates divide successful requests by the sum of actual elapsed trial times; percentiles pool raw samples across both trials using nearest rank. The mixed workload rotates equally across the 12 repository/API pairs. These are short trials on a shared host, not a saturation or capacity guarantee. + +| Repository | API | Concurrency | Requests/s | p50 ms | p95 ms | p99 ms | Errors / attempts | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| all | mixed | 1 | 1.67 | 150.1 | 2850.7 | 2937.8 | 0 / 32 | +| all | mixed | 4 | 4.57 | 208.7 | 3701.1 | 4382.6 | 0 / 97 | +| bat | blob | 1 | 3.08 | 292.7 | 647.6 | 1145.4 | 0 / 51 | +| bat | blob | 4 | 8.92 | 443.3 | 834.7 | 1487.3 | 0 / 148 | +| bat | commits | 1 | 0.30 | 3142.1 | 4125.5 | 4125.5 | 0 / 6 | +| bat | commits | 4 | 1.48 | 3110.7 | 4257.9 | 4293.9 | 0 / 28 | +| bat | refs | 1 | 44.45 | 24.9 | 44.2 | 58.0 | 0 / 713 | +| bat | refs | 4 | 149.60 | 16.4 | 61.5 | 78.7 | 0 / 2400 | +| bat | tree | 1 | 4.35 | 158.6 | 610.6 | 784.1 | 0 / 72 | +| bat | tree | 4 | 15.05 | 183.3 | 649.9 | 1036.7 | 0 / 250 | +| libgit2 | blob | 1 | 2.48 | 359.7 | 785.2 | 1080.9 | 0 / 41 | +| libgit2 | blob | 4 | 8.81 | 504.0 | 792.1 | 885.6 | 0 / 146 | +| libgit2 | commits | 1 | 0.37 | 2596.8 | 4647.9 | 4647.9 | 0 / 7 | +| libgit2 | commits | 4 | 1.65 | 2054.9 | 3526.8 | 3527.5 | 0 / 28 | +| libgit2 | refs | 1 | 64.80 | 10.3 | 35.1 | 56.8 | 0 / 1039 | +| libgit2 | refs | 4 | 118.82 | 18.7 | 89.1 | 129.4 | 0 / 1904 | +| libgit2 | tree | 1 | 3.60 | 203.3 | 666.8 | 841.6 | 0 / 59 | +| libgit2 | tree | 4 | 13.29 | 208.7 | 688.9 | 874.2 | 0 / 222 | +| starship | blob | 1 | 3.66 | 207.5 | 564.7 | 667.3 | 0 / 61 | +| starship | blob | 4 | 8.40 | 434.1 | 885.4 | 1100.3 | 0 / 141 | +| starship | commits | 1 | 0.48 | 2330.4 | 3049.1 | 3049.1 | 0 / 9 | +| starship | commits | 4 | 1.71 | 2094.5 | 3082.0 | 3101.8 | 0 / 32 | +| starship | refs | 1 | 52.01 | 11.2 | 40.1 | 53.6 | 0 / 834 | +| starship | refs | 4 | 174.43 | 12.6 | 54.9 | 85.6 | 0 / 2812 | +| starship | tree | 1 | 3.92 | 231.4 | 533.7 | 596.9 | 0 / 63 | +| starship | tree | 4 | 16.76 | 183.5 | 648.0 | 757.4 | 0 / 279 | + +Commit history is the slowest measured API: concurrency-4 p95 is 3.1–4.3 seconds. No service-level latency target or large-team capacity claim is established by this run. + +Each cold result below is one first verified API request after a graceful shutdown and fresh local workspace. Health/startup time is excluded. RustFS and host filesystem caches may remain warm, so these are individual cold-local-workspace observations, not percentile estimates or a fully cold provider benchmark. + +| Repository | Refs ms | Tree ms | Blob ms | Commits ms | +| --- | ---: | ---: | ---: | ---: | +| bat | 298.0 | 2241.5 | 1717.6 | 3115.7 | +| starship | 490.4 | 1907.1 | 1210.1 | 2843.3 | +| libgit2 | 208.6 | 2760.8 | 3872.9 | 5007.8 | + +## Qualification and environment + +The dependency-policy check, workspace formatting, all-target Clippy with warnings denied, ordinary workspace tests and release compilation pass. The ordinary Rust suites contain 896 unique passing tests, zero failures and nine provider-only cases deferred to the isolated runner. All 105 Python harness tests also pass at the current harness commit. + +All nine isolated RustFS provider cases pass: the 595,260,811-byte native pack and 5,377,097,728-byte Git/LFS bodies after restore, SHA-256 round trips and native merge candidates, signed pushes, signed SHA-256 SSH, stock SSH, bulk refs, filtered clones and SSH LFS. The full current extended process profile passes in 493.96 seconds, including large v0/v2 clones, structural objects, repeated restart/takeover, cache corruption recovery, eight repositories through opposite HTTPS peers, interrupted maintenance, backup and retention. + +The final backup copies 26 external manifests, rejects corrupt retained bytes, verifies after deleting the disposable source prefix, restores the same inventory, and returns exact Git/LFS content and issue state. The process retention fixture rejects collection outside maintenance, enforces a one-delete partial pass, then completes collection of 256 obsolete Cellule objects while preserving its backup pin and cold stock-Git clone. An independent focused fixture also passes backup/retention and collects 248 objects; these are separate from the 1,664-object corpus collection above. + +The nine Rust provider cases use source `624bd26`, whose Rust bytes match the final harness source. After those cases passed, its superseded Python process child was deliberately interrupted; that original whole-run receipt remains failed/interrupted. The complete successful process profile uses `81925e1`. All 724 other source files match the Rust-qualified `930e6c7`; only the three Python fixtures described above differ. The combined qualification receipt records both scopes and their source identities. Earlier failures are retained with their original source versions, rather than relabeled as passes. + +The standard workspace gates use: + +```sh +python3 scripts/check_cellule.py +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --locked -- -D warnings +cargo test --workspace --locked -- --test-threads=1 +python3 -B -m unittest discover -s scripts -p 'test_*.py' +``` + +The nine isolated provider cases use `python3 scripts/qualify_size.py --release --docker-volume`. The extended process profile uses `scripts/smoke_s3_process.py` with a frozen release executable, disposable S3 prefix/work directory, `--large-clone --many-objects 256 --structural-objects --structural-object-bytes 68157440 --retain-work-dir`, and the fixture credentials in the environment. The 256-object option creates small files; it does not create 256 large structural objects. + +Hardware: Apple M2 Max, 12 CPU cores, 32 GiB RAM; macOS 26.5.2; Rust/Cargo 1.97; Apple Git 2.50.1; Git LFS 3.7.1; Python 3.14.6. The shared Docker VM has 6 CPUs and about 7.67 GiB RAM. The corpus RustFS `1.0.0-beta.8-glibc` container is limited to 4 CPUs / 6 GiB RAM. The corpus uses an isolated fresh prefix in the dedicated RustFS fixture; measurements are from the same release executable throughout. + +Historical results remain in [the October 14 report](2026-10-14-storage-optimization.md), with their original revisions and format versions. [Machine-readable results](2026-10-15-native-storage.json) retain the source/binary identities, corpus proofs, storage accounting, pooled metrics, qualification receipts and hashes of local raw evidence. The full local evidence lives under `~/.local/share/canopy-storage-optimization/{native-medium-cache-final-20261015,medium-metadata-cache-final-20261015}`. Failed earlier trials remain separate and are not counted as passes. diff --git a/docs/implementation.md b/docs/implementation.md index 2ee94b85..3bc15eff 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -264,6 +264,8 @@ verified live roots and backup pins, enforces its grace and deletion bounds, and joins work before releasing the claim. Product pack/catalog/LFS retention is a separate policy. The operational sequence is in [operations](operations.md#cellule-retention-collection). -The integrated source has preliminary type-check and Python harness results; -full Rust, isolated provider/process and fresh medium-corpus qualification remain -pending. Historical measurements retain their original binaries and formats. +The [October 15 qualification](benchmarks/2026-10-15-native-storage.md) records +full object-body/ref/HEAD/fsck/API checks for bat, Starship and libgit2 before and +after Cellule GC and fresh-workspace restoration, plus browse measurements. Full +workspace and isolated provider/process qualification are recorded separately in +that report. Historical evidence retains its original binaries and formats. diff --git a/docs/large-repository-implementation-status.md b/docs/large-repository-implementation-status.md index 8c3adb30..922b2a6b 100644 --- a/docs/large-repository-implementation-status.md +++ b/docs/large-repository-implementation-status.md @@ -1,5 +1,17 @@ # Large-repository implementation status +Current storage integration (2026-10-15): source `12dc9a1` uses published +Cellule 0.2.0, repository schema 4 and `canopy-pack-v2`, with fresh-prefix format +changes. Immutable native packs, bounded metadata publication and Cellule +retention collection are integrated. Full repository verification for bat, +Starship and libgit2 passes before and after GC/fresh-workspace restore; see the +[current qualification](benchmarks/2026-10-15-native-storage.md) for scope, +measurements and gate results. Product Git/LFS artifact collection and the +[large-team capacity amendment](large-team-scalability.md) remain open. + +The following implementation checkpoints are historical, including their +original dependency revisions, incomplete gates and source layouts. + Updated during implementation on 2026-10-05. **The full implementation and capacity goal remains open.** The [large-team amendment](large-team-scalability.md) is mandatory scope alongside the original storage design. Passing primitive tests is not completion of the hard cutover or proof of capacity. Packed cutover [PR #34](https://github.com/crabbuild/canopy/pull/34) was merged into diff --git a/docs/operations.md b/docs/operations.md index 15f00a7f..d103559c 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -63,8 +63,8 @@ takeover. Cross-node placement races and larger hot sets still need qualificatio Each storage prefix has one durable tenant/application identity and a selected compiled release. First startup initializes an empty deployment; later nodes must match its release and configured image identity. A nonempty catalog without -release metadata is rejected. Existing preview prefixes require explicit future -migration; do not point this build at them as an upgrade. +release metadata is rejected. Breaking schema or format changes require a fresh +storage prefix; this build has no compatibility reader or in-place upgrade path. To close admission and drain the fleet, choose a fresh operation UUID and use the same binary and configuration as the deployment: @@ -113,8 +113,9 @@ failed root verification return an error. Retry the same operation after the reported condition is resolved; recovery never resumes serving automatically. Do not remove authority records or force `drained` to bypass an error. -Upgrade/migration and object collection remain pending. Maintenance and owner -recovery do not provide a separate backup copy. +Cellule object collection uses the [retention maintenance procedure](#cellule-retention-collection). +Product-owned pack/catalog/LFS collection remains a separate policy. Maintenance +and owner recovery do not provide a separate backup copy. ### Backup and restore @@ -157,9 +158,9 @@ failed operation after repairing its reported cause. Replaying a completed resto does not reset subsequently published service state. This is a same-provider copy, not protection from losing the entire bucket or -provider. Cross-provider export, old-release migration, automated retention and -the complete interruption fault matrix remain pending. Do not use older binaries -that lack prefix reservations with these backup/restore destinations. +provider. Cross-provider export, automated retention and the complete interruption +fault matrix remain pending. Restore requires the same compiled release and +current formats; breaking changes use a fresh storage prefix. ### Recover a lost push reply